1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 #include <rte_common.h> 14 #include <rte_ether.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rxtx.h" 32 33 /** Device flow drivers. */ 34 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 35 36 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 37 38 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 39 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 40 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 41 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 42 #endif 43 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 44 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 45 }; 46 47 enum mlx5_expansion { 48 MLX5_EXPANSION_ROOT, 49 MLX5_EXPANSION_ROOT_OUTER, 50 MLX5_EXPANSION_ROOT_ETH_VLAN, 51 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 52 MLX5_EXPANSION_OUTER_ETH, 53 MLX5_EXPANSION_OUTER_ETH_VLAN, 54 MLX5_EXPANSION_OUTER_VLAN, 55 MLX5_EXPANSION_OUTER_IPV4, 56 MLX5_EXPANSION_OUTER_IPV4_UDP, 57 MLX5_EXPANSION_OUTER_IPV4_TCP, 58 MLX5_EXPANSION_OUTER_IPV6, 59 MLX5_EXPANSION_OUTER_IPV6_UDP, 60 MLX5_EXPANSION_OUTER_IPV6_TCP, 61 MLX5_EXPANSION_VXLAN, 62 MLX5_EXPANSION_VXLAN_GPE, 63 MLX5_EXPANSION_GRE, 64 MLX5_EXPANSION_MPLS, 65 MLX5_EXPANSION_ETH, 66 MLX5_EXPANSION_ETH_VLAN, 67 MLX5_EXPANSION_VLAN, 68 MLX5_EXPANSION_IPV4, 69 MLX5_EXPANSION_IPV4_UDP, 70 MLX5_EXPANSION_IPV4_TCP, 71 MLX5_EXPANSION_IPV6, 72 MLX5_EXPANSION_IPV6_UDP, 73 MLX5_EXPANSION_IPV6_TCP, 74 }; 75 76 /** Supported expansion of items. */ 77 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 78 [MLX5_EXPANSION_ROOT] = { 79 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 80 MLX5_EXPANSION_IPV4, 81 MLX5_EXPANSION_IPV6), 82 .type = RTE_FLOW_ITEM_TYPE_END, 83 }, 84 [MLX5_EXPANSION_ROOT_OUTER] = { 85 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 86 MLX5_EXPANSION_OUTER_IPV4, 87 MLX5_EXPANSION_OUTER_IPV6), 88 .type = RTE_FLOW_ITEM_TYPE_END, 89 }, 90 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 91 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 92 .type = RTE_FLOW_ITEM_TYPE_END, 93 }, 94 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 95 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 96 .type = RTE_FLOW_ITEM_TYPE_END, 97 }, 98 [MLX5_EXPANSION_OUTER_ETH] = { 99 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 100 MLX5_EXPANSION_OUTER_IPV6, 101 MLX5_EXPANSION_MPLS), 102 .type = RTE_FLOW_ITEM_TYPE_ETH, 103 .rss_types = 0, 104 }, 105 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_ETH, 108 .rss_types = 0, 109 }, 110 [MLX5_EXPANSION_OUTER_VLAN] = { 111 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 112 MLX5_EXPANSION_OUTER_IPV6), 113 .type = RTE_FLOW_ITEM_TYPE_VLAN, 114 }, 115 [MLX5_EXPANSION_OUTER_IPV4] = { 116 .next = RTE_FLOW_EXPAND_RSS_NEXT 117 (MLX5_EXPANSION_OUTER_IPV4_UDP, 118 MLX5_EXPANSION_OUTER_IPV4_TCP, 119 MLX5_EXPANSION_GRE, 120 MLX5_EXPANSION_IPV4, 121 MLX5_EXPANSION_IPV6), 122 .type = RTE_FLOW_ITEM_TYPE_IPV4, 123 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 124 ETH_RSS_NONFRAG_IPV4_OTHER, 125 }, 126 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 127 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 128 MLX5_EXPANSION_VXLAN_GPE), 129 .type = RTE_FLOW_ITEM_TYPE_UDP, 130 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 131 }, 132 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 133 .type = RTE_FLOW_ITEM_TYPE_TCP, 134 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 135 }, 136 [MLX5_EXPANSION_OUTER_IPV6] = { 137 .next = RTE_FLOW_EXPAND_RSS_NEXT 138 (MLX5_EXPANSION_OUTER_IPV6_UDP, 139 MLX5_EXPANSION_OUTER_IPV6_TCP, 140 MLX5_EXPANSION_IPV4, 141 MLX5_EXPANSION_IPV6), 142 .type = RTE_FLOW_ITEM_TYPE_IPV6, 143 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 144 ETH_RSS_NONFRAG_IPV6_OTHER, 145 }, 146 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 147 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 148 MLX5_EXPANSION_VXLAN_GPE), 149 .type = RTE_FLOW_ITEM_TYPE_UDP, 150 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 151 }, 152 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 153 .type = RTE_FLOW_ITEM_TYPE_TCP, 154 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 155 }, 156 [MLX5_EXPANSION_VXLAN] = { 157 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 158 MLX5_EXPANSION_IPV4, 159 MLX5_EXPANSION_IPV6), 160 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 161 }, 162 [MLX5_EXPANSION_VXLAN_GPE] = { 163 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 164 MLX5_EXPANSION_IPV4, 165 MLX5_EXPANSION_IPV6), 166 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 167 }, 168 [MLX5_EXPANSION_GRE] = { 169 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 170 .type = RTE_FLOW_ITEM_TYPE_GRE, 171 }, 172 [MLX5_EXPANSION_MPLS] = { 173 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 174 MLX5_EXPANSION_IPV6), 175 .type = RTE_FLOW_ITEM_TYPE_MPLS, 176 }, 177 [MLX5_EXPANSION_ETH] = { 178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 179 MLX5_EXPANSION_IPV6), 180 .type = RTE_FLOW_ITEM_TYPE_ETH, 181 }, 182 [MLX5_EXPANSION_ETH_VLAN] = { 183 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 184 .type = RTE_FLOW_ITEM_TYPE_ETH, 185 }, 186 [MLX5_EXPANSION_VLAN] = { 187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 188 MLX5_EXPANSION_IPV6), 189 .type = RTE_FLOW_ITEM_TYPE_VLAN, 190 }, 191 [MLX5_EXPANSION_IPV4] = { 192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 193 MLX5_EXPANSION_IPV4_TCP), 194 .type = RTE_FLOW_ITEM_TYPE_IPV4, 195 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 196 ETH_RSS_NONFRAG_IPV4_OTHER, 197 }, 198 [MLX5_EXPANSION_IPV4_UDP] = { 199 .type = RTE_FLOW_ITEM_TYPE_UDP, 200 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 201 }, 202 [MLX5_EXPANSION_IPV4_TCP] = { 203 .type = RTE_FLOW_ITEM_TYPE_TCP, 204 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 205 }, 206 [MLX5_EXPANSION_IPV6] = { 207 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 208 MLX5_EXPANSION_IPV6_TCP), 209 .type = RTE_FLOW_ITEM_TYPE_IPV6, 210 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 211 ETH_RSS_NONFRAG_IPV6_OTHER, 212 }, 213 [MLX5_EXPANSION_IPV6_UDP] = { 214 .type = RTE_FLOW_ITEM_TYPE_UDP, 215 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 216 }, 217 [MLX5_EXPANSION_IPV6_TCP] = { 218 .type = RTE_FLOW_ITEM_TYPE_TCP, 219 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 220 }, 221 }; 222 223 static const struct rte_flow_ops mlx5_flow_ops = { 224 .validate = mlx5_flow_validate, 225 .create = mlx5_flow_create, 226 .destroy = mlx5_flow_destroy, 227 .flush = mlx5_flow_flush, 228 .isolate = mlx5_flow_isolate, 229 .query = mlx5_flow_query, 230 .dev_dump = mlx5_flow_dev_dump, 231 .get_aged_flows = mlx5_flow_get_aged_flows, 232 }; 233 234 /* Convert FDIR request to Generic flow. */ 235 struct mlx5_fdir { 236 struct rte_flow_attr attr; 237 struct rte_flow_item items[4]; 238 struct rte_flow_item_eth l2; 239 struct rte_flow_item_eth l2_mask; 240 union { 241 struct rte_flow_item_ipv4 ipv4; 242 struct rte_flow_item_ipv6 ipv6; 243 } l3; 244 union { 245 struct rte_flow_item_ipv4 ipv4; 246 struct rte_flow_item_ipv6 ipv6; 247 } l3_mask; 248 union { 249 struct rte_flow_item_udp udp; 250 struct rte_flow_item_tcp tcp; 251 } l4; 252 union { 253 struct rte_flow_item_udp udp; 254 struct rte_flow_item_tcp tcp; 255 } l4_mask; 256 struct rte_flow_action actions[2]; 257 struct rte_flow_action_queue queue; 258 }; 259 260 /* Tunnel information. */ 261 struct mlx5_flow_tunnel_info { 262 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 263 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 264 }; 265 266 static struct mlx5_flow_tunnel_info tunnels_info[] = { 267 { 268 .tunnel = MLX5_FLOW_LAYER_VXLAN, 269 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 270 }, 271 { 272 .tunnel = MLX5_FLOW_LAYER_GENEVE, 273 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 274 }, 275 { 276 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 277 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 278 }, 279 { 280 .tunnel = MLX5_FLOW_LAYER_GRE, 281 .ptype = RTE_PTYPE_TUNNEL_GRE, 282 }, 283 { 284 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 285 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 286 }, 287 { 288 .tunnel = MLX5_FLOW_LAYER_MPLS, 289 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 290 }, 291 { 292 .tunnel = MLX5_FLOW_LAYER_NVGRE, 293 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 294 }, 295 { 296 .tunnel = MLX5_FLOW_LAYER_IPIP, 297 .ptype = RTE_PTYPE_TUNNEL_IP, 298 }, 299 { 300 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 301 .ptype = RTE_PTYPE_TUNNEL_IP, 302 }, 303 { 304 .tunnel = MLX5_FLOW_LAYER_GTP, 305 .ptype = RTE_PTYPE_TUNNEL_GTPU, 306 }, 307 }; 308 309 /** 310 * Translate tag ID to register. 311 * 312 * @param[in] dev 313 * Pointer to the Ethernet device structure. 314 * @param[in] feature 315 * The feature that request the register. 316 * @param[in] id 317 * The request register ID. 318 * @param[out] error 319 * Error description in case of any. 320 * 321 * @return 322 * The request register on success, a negative errno 323 * value otherwise and rte_errno is set. 324 */ 325 int 326 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 327 enum mlx5_feature_name feature, 328 uint32_t id, 329 struct rte_flow_error *error) 330 { 331 struct mlx5_priv *priv = dev->data->dev_private; 332 struct mlx5_dev_config *config = &priv->config; 333 enum modify_reg start_reg; 334 bool skip_mtr_reg = false; 335 336 switch (feature) { 337 case MLX5_HAIRPIN_RX: 338 return REG_B; 339 case MLX5_HAIRPIN_TX: 340 return REG_A; 341 case MLX5_METADATA_RX: 342 switch (config->dv_xmeta_en) { 343 case MLX5_XMETA_MODE_LEGACY: 344 return REG_B; 345 case MLX5_XMETA_MODE_META16: 346 return REG_C_0; 347 case MLX5_XMETA_MODE_META32: 348 return REG_C_1; 349 } 350 break; 351 case MLX5_METADATA_TX: 352 return REG_A; 353 case MLX5_METADATA_FDB: 354 switch (config->dv_xmeta_en) { 355 case MLX5_XMETA_MODE_LEGACY: 356 return REG_NON; 357 case MLX5_XMETA_MODE_META16: 358 return REG_C_0; 359 case MLX5_XMETA_MODE_META32: 360 return REG_C_1; 361 } 362 break; 363 case MLX5_FLOW_MARK: 364 switch (config->dv_xmeta_en) { 365 case MLX5_XMETA_MODE_LEGACY: 366 return REG_NON; 367 case MLX5_XMETA_MODE_META16: 368 return REG_C_1; 369 case MLX5_XMETA_MODE_META32: 370 return REG_C_0; 371 } 372 break; 373 case MLX5_MTR_SFX: 374 /* 375 * If meter color and flow match share one register, flow match 376 * should use the meter color register for match. 377 */ 378 if (priv->mtr_reg_share) 379 return priv->mtr_color_reg; 380 else 381 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 382 REG_C_3; 383 case MLX5_MTR_COLOR: 384 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 385 return priv->mtr_color_reg; 386 case MLX5_COPY_MARK: 387 /* 388 * Metadata COPY_MARK register using is in meter suffix sub 389 * flow while with meter. It's safe to share the same register. 390 */ 391 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 392 case MLX5_APP_TAG: 393 /* 394 * If meter is enable, it will engage the register for color 395 * match and flow match. If meter color match is not using the 396 * REG_C_2, need to skip the REG_C_x be used by meter color 397 * match. 398 * If meter is disable, free to use all available registers. 399 */ 400 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 401 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 402 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 403 if (id > (REG_C_7 - start_reg)) 404 return rte_flow_error_set(error, EINVAL, 405 RTE_FLOW_ERROR_TYPE_ITEM, 406 NULL, "invalid tag id"); 407 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 408 return rte_flow_error_set(error, ENOTSUP, 409 RTE_FLOW_ERROR_TYPE_ITEM, 410 NULL, "unsupported tag id"); 411 /* 412 * This case means meter is using the REG_C_x great than 2. 413 * Take care not to conflict with meter color REG_C_x. 414 * If the available index REG_C_y >= REG_C_x, skip the 415 * color register. 416 */ 417 if (skip_mtr_reg && config->flow_mreg_c 418 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 419 if (id >= (REG_C_7 - start_reg)) 420 return rte_flow_error_set(error, EINVAL, 421 RTE_FLOW_ERROR_TYPE_ITEM, 422 NULL, "invalid tag id"); 423 if (config->flow_mreg_c 424 [id + 1 + start_reg - REG_C_0] != REG_NON) 425 return config->flow_mreg_c 426 [id + 1 + start_reg - REG_C_0]; 427 return rte_flow_error_set(error, ENOTSUP, 428 RTE_FLOW_ERROR_TYPE_ITEM, 429 NULL, "unsupported tag id"); 430 } 431 return config->flow_mreg_c[id + start_reg - REG_C_0]; 432 } 433 MLX5_ASSERT(false); 434 return rte_flow_error_set(error, EINVAL, 435 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 436 NULL, "invalid feature name"); 437 } 438 439 /** 440 * Check extensive flow metadata register support. 441 * 442 * @param dev 443 * Pointer to rte_eth_dev structure. 444 * 445 * @return 446 * True if device supports extensive flow metadata register, otherwise false. 447 */ 448 bool 449 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 450 { 451 struct mlx5_priv *priv = dev->data->dev_private; 452 struct mlx5_dev_config *config = &priv->config; 453 454 /* 455 * Having available reg_c can be regarded inclusively as supporting 456 * extensive flow metadata register, which could mean, 457 * - metadata register copy action by modify header. 458 * - 16 modify header actions is supported. 459 * - reg_c's are preserved across different domain (FDB and NIC) on 460 * packet loopback by flow lookup miss. 461 */ 462 return config->flow_mreg_c[2] != REG_NON; 463 } 464 465 /** 466 * Verify the @p item specifications (spec, last, mask) are compatible with the 467 * NIC capabilities. 468 * 469 * @param[in] item 470 * Item specification. 471 * @param[in] mask 472 * @p item->mask or flow default bit-masks. 473 * @param[in] nic_mask 474 * Bit-masks covering supported fields by the NIC to compare with user mask. 475 * @param[in] size 476 * Bit-masks size in bytes. 477 * @param[out] error 478 * Pointer to error structure. 479 * 480 * @return 481 * 0 on success, a negative errno value otherwise and rte_errno is set. 482 */ 483 int 484 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 485 const uint8_t *mask, 486 const uint8_t *nic_mask, 487 unsigned int size, 488 struct rte_flow_error *error) 489 { 490 unsigned int i; 491 492 MLX5_ASSERT(nic_mask); 493 for (i = 0; i < size; ++i) 494 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 495 return rte_flow_error_set(error, ENOTSUP, 496 RTE_FLOW_ERROR_TYPE_ITEM, 497 item, 498 "mask enables non supported" 499 " bits"); 500 if (!item->spec && (item->mask || item->last)) 501 return rte_flow_error_set(error, EINVAL, 502 RTE_FLOW_ERROR_TYPE_ITEM, item, 503 "mask/last without a spec is not" 504 " supported"); 505 if (item->spec && item->last) { 506 uint8_t spec[size]; 507 uint8_t last[size]; 508 unsigned int i; 509 int ret; 510 511 for (i = 0; i < size; ++i) { 512 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 513 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 514 } 515 ret = memcmp(spec, last, size); 516 if (ret != 0) 517 return rte_flow_error_set(error, EINVAL, 518 RTE_FLOW_ERROR_TYPE_ITEM, 519 item, 520 "range is not valid"); 521 } 522 return 0; 523 } 524 525 /** 526 * Adjust the hash fields according to the @p flow information. 527 * 528 * @param[in] dev_flow. 529 * Pointer to the mlx5_flow. 530 * @param[in] tunnel 531 * 1 when the hash field is for a tunnel item. 532 * @param[in] layer_types 533 * ETH_RSS_* types. 534 * @param[in] hash_fields 535 * Item hash fields. 536 * 537 * @return 538 * The hash fields that should be used. 539 */ 540 uint64_t 541 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 542 int tunnel __rte_unused, uint64_t layer_types, 543 uint64_t hash_fields) 544 { 545 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 546 int rss_request_inner = rss_desc->level >= 2; 547 548 /* Check RSS hash level for tunnel. */ 549 if (tunnel && rss_request_inner) 550 hash_fields |= IBV_RX_HASH_INNER; 551 else if (tunnel || rss_request_inner) 552 return 0; 553 #endif 554 /* Check if requested layer matches RSS hash fields. */ 555 if (!(rss_desc->types & layer_types)) 556 return 0; 557 return hash_fields; 558 } 559 560 /** 561 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 562 * if several tunnel rules are used on this queue, the tunnel ptype will be 563 * cleared. 564 * 565 * @param rxq_ctrl 566 * Rx queue to update. 567 */ 568 static void 569 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 570 { 571 unsigned int i; 572 uint32_t tunnel_ptype = 0; 573 574 /* Look up for the ptype to use. */ 575 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 576 if (!rxq_ctrl->flow_tunnels_n[i]) 577 continue; 578 if (!tunnel_ptype) { 579 tunnel_ptype = tunnels_info[i].ptype; 580 } else { 581 tunnel_ptype = 0; 582 break; 583 } 584 } 585 rxq_ctrl->rxq.tunnel = tunnel_ptype; 586 } 587 588 /** 589 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 590 * flow. 591 * 592 * @param[in] dev 593 * Pointer to the Ethernet device structure. 594 * @param[in] dev_handle 595 * Pointer to device flow handle structure. 596 */ 597 static void 598 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 599 struct mlx5_flow_handle *dev_handle) 600 { 601 struct mlx5_priv *priv = dev->data->dev_private; 602 const int mark = dev_handle->mark; 603 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 604 struct mlx5_hrxq *hrxq; 605 unsigned int i; 606 607 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 608 return; 609 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 610 dev_handle->rix_hrxq); 611 if (!hrxq) 612 return; 613 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 614 int idx = hrxq->ind_table->queues[i]; 615 struct mlx5_rxq_ctrl *rxq_ctrl = 616 container_of((*priv->rxqs)[idx], 617 struct mlx5_rxq_ctrl, rxq); 618 619 /* 620 * To support metadata register copy on Tx loopback, 621 * this must be always enabled (metadata may arive 622 * from other port - not from local flows only. 623 */ 624 if (priv->config.dv_flow_en && 625 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 626 mlx5_flow_ext_mreg_supported(dev)) { 627 rxq_ctrl->rxq.mark = 1; 628 rxq_ctrl->flow_mark_n = 1; 629 } else if (mark) { 630 rxq_ctrl->rxq.mark = 1; 631 rxq_ctrl->flow_mark_n++; 632 } 633 if (tunnel) { 634 unsigned int j; 635 636 /* Increase the counter matching the flow. */ 637 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 638 if ((tunnels_info[j].tunnel & 639 dev_handle->layers) == 640 tunnels_info[j].tunnel) { 641 rxq_ctrl->flow_tunnels_n[j]++; 642 break; 643 } 644 } 645 flow_rxq_tunnel_ptype_update(rxq_ctrl); 646 } 647 } 648 } 649 650 /** 651 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 652 * 653 * @param[in] dev 654 * Pointer to the Ethernet device structure. 655 * @param[in] flow 656 * Pointer to flow structure. 657 */ 658 static void 659 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 660 { 661 struct mlx5_priv *priv = dev->data->dev_private; 662 uint32_t handle_idx; 663 struct mlx5_flow_handle *dev_handle; 664 665 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 666 handle_idx, dev_handle, next) 667 flow_drv_rxq_flags_set(dev, dev_handle); 668 } 669 670 /** 671 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 672 * device flow if no other flow uses it with the same kind of request. 673 * 674 * @param dev 675 * Pointer to Ethernet device. 676 * @param[in] dev_handle 677 * Pointer to the device flow handle structure. 678 */ 679 static void 680 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 681 struct mlx5_flow_handle *dev_handle) 682 { 683 struct mlx5_priv *priv = dev->data->dev_private; 684 const int mark = dev_handle->mark; 685 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 686 struct mlx5_hrxq *hrxq; 687 unsigned int i; 688 689 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 690 return; 691 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 692 dev_handle->rix_hrxq); 693 if (!hrxq) 694 return; 695 MLX5_ASSERT(dev->data->dev_started); 696 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 697 int idx = hrxq->ind_table->queues[i]; 698 struct mlx5_rxq_ctrl *rxq_ctrl = 699 container_of((*priv->rxqs)[idx], 700 struct mlx5_rxq_ctrl, rxq); 701 702 if (priv->config.dv_flow_en && 703 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 704 mlx5_flow_ext_mreg_supported(dev)) { 705 rxq_ctrl->rxq.mark = 1; 706 rxq_ctrl->flow_mark_n = 1; 707 } else if (mark) { 708 rxq_ctrl->flow_mark_n--; 709 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 710 } 711 if (tunnel) { 712 unsigned int j; 713 714 /* Decrease the counter matching the flow. */ 715 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 716 if ((tunnels_info[j].tunnel & 717 dev_handle->layers) == 718 tunnels_info[j].tunnel) { 719 rxq_ctrl->flow_tunnels_n[j]--; 720 break; 721 } 722 } 723 flow_rxq_tunnel_ptype_update(rxq_ctrl); 724 } 725 } 726 } 727 728 /** 729 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 730 * @p flow if no other flow uses it with the same kind of request. 731 * 732 * @param dev 733 * Pointer to Ethernet device. 734 * @param[in] flow 735 * Pointer to the flow. 736 */ 737 static void 738 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 739 { 740 struct mlx5_priv *priv = dev->data->dev_private; 741 uint32_t handle_idx; 742 struct mlx5_flow_handle *dev_handle; 743 744 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 745 handle_idx, dev_handle, next) 746 flow_drv_rxq_flags_trim(dev, dev_handle); 747 } 748 749 /** 750 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 751 * 752 * @param dev 753 * Pointer to Ethernet device. 754 */ 755 static void 756 flow_rxq_flags_clear(struct rte_eth_dev *dev) 757 { 758 struct mlx5_priv *priv = dev->data->dev_private; 759 unsigned int i; 760 761 for (i = 0; i != priv->rxqs_n; ++i) { 762 struct mlx5_rxq_ctrl *rxq_ctrl; 763 unsigned int j; 764 765 if (!(*priv->rxqs)[i]) 766 continue; 767 rxq_ctrl = container_of((*priv->rxqs)[i], 768 struct mlx5_rxq_ctrl, rxq); 769 rxq_ctrl->flow_mark_n = 0; 770 rxq_ctrl->rxq.mark = 0; 771 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 772 rxq_ctrl->flow_tunnels_n[j] = 0; 773 rxq_ctrl->rxq.tunnel = 0; 774 } 775 } 776 777 /** 778 * Set the Rx queue dynamic metadata (mask and offset) for a flow 779 * 780 * @param[in] dev 781 * Pointer to the Ethernet device structure. 782 */ 783 void 784 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 785 { 786 struct mlx5_priv *priv = dev->data->dev_private; 787 struct mlx5_rxq_data *data; 788 unsigned int i; 789 790 for (i = 0; i != priv->rxqs_n; ++i) { 791 if (!(*priv->rxqs)[i]) 792 continue; 793 data = (*priv->rxqs)[i]; 794 if (!rte_flow_dynf_metadata_avail()) { 795 data->dynf_meta = 0; 796 data->flow_meta_mask = 0; 797 data->flow_meta_offset = -1; 798 } else { 799 data->dynf_meta = 1; 800 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 801 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 802 } 803 } 804 } 805 806 /* 807 * return a pointer to the desired action in the list of actions. 808 * 809 * @param[in] actions 810 * The list of actions to search the action in. 811 * @param[in] action 812 * The action to find. 813 * 814 * @return 815 * Pointer to the action in the list, if found. NULL otherwise. 816 */ 817 const struct rte_flow_action * 818 mlx5_flow_find_action(const struct rte_flow_action *actions, 819 enum rte_flow_action_type action) 820 { 821 if (actions == NULL) 822 return NULL; 823 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 824 if (actions->type == action) 825 return actions; 826 return NULL; 827 } 828 829 /* 830 * Validate the flag action. 831 * 832 * @param[in] action_flags 833 * Bit-fields that holds the actions detected until now. 834 * @param[in] attr 835 * Attributes of flow that includes this action. 836 * @param[out] error 837 * Pointer to error structure. 838 * 839 * @return 840 * 0 on success, a negative errno value otherwise and rte_errno is set. 841 */ 842 int 843 mlx5_flow_validate_action_flag(uint64_t action_flags, 844 const struct rte_flow_attr *attr, 845 struct rte_flow_error *error) 846 { 847 if (action_flags & MLX5_FLOW_ACTION_MARK) 848 return rte_flow_error_set(error, EINVAL, 849 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 850 "can't mark and flag in same flow"); 851 if (action_flags & MLX5_FLOW_ACTION_FLAG) 852 return rte_flow_error_set(error, EINVAL, 853 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 854 "can't have 2 flag" 855 " actions in same flow"); 856 if (attr->egress) 857 return rte_flow_error_set(error, ENOTSUP, 858 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 859 "flag action not supported for " 860 "egress"); 861 return 0; 862 } 863 864 /* 865 * Validate the mark action. 866 * 867 * @param[in] action 868 * Pointer to the queue action. 869 * @param[in] action_flags 870 * Bit-fields that holds the actions detected until now. 871 * @param[in] attr 872 * Attributes of flow that includes this action. 873 * @param[out] error 874 * Pointer to error structure. 875 * 876 * @return 877 * 0 on success, a negative errno value otherwise and rte_errno is set. 878 */ 879 int 880 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 881 uint64_t action_flags, 882 const struct rte_flow_attr *attr, 883 struct rte_flow_error *error) 884 { 885 const struct rte_flow_action_mark *mark = action->conf; 886 887 if (!mark) 888 return rte_flow_error_set(error, EINVAL, 889 RTE_FLOW_ERROR_TYPE_ACTION, 890 action, 891 "configuration cannot be null"); 892 if (mark->id >= MLX5_FLOW_MARK_MAX) 893 return rte_flow_error_set(error, EINVAL, 894 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 895 &mark->id, 896 "mark id must in 0 <= id < " 897 RTE_STR(MLX5_FLOW_MARK_MAX)); 898 if (action_flags & MLX5_FLOW_ACTION_FLAG) 899 return rte_flow_error_set(error, EINVAL, 900 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 901 "can't flag and mark in same flow"); 902 if (action_flags & MLX5_FLOW_ACTION_MARK) 903 return rte_flow_error_set(error, EINVAL, 904 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 905 "can't have 2 mark actions in same" 906 " flow"); 907 if (attr->egress) 908 return rte_flow_error_set(error, ENOTSUP, 909 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 910 "mark action not supported for " 911 "egress"); 912 return 0; 913 } 914 915 /* 916 * Validate the drop action. 917 * 918 * @param[in] action_flags 919 * Bit-fields that holds the actions detected until now. 920 * @param[in] attr 921 * Attributes of flow that includes this action. 922 * @param[out] error 923 * Pointer to error structure. 924 * 925 * @return 926 * 0 on success, a negative errno value otherwise and rte_errno is set. 927 */ 928 int 929 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 930 const struct rte_flow_attr *attr, 931 struct rte_flow_error *error) 932 { 933 if (attr->egress) 934 return rte_flow_error_set(error, ENOTSUP, 935 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 936 "drop action not supported for " 937 "egress"); 938 return 0; 939 } 940 941 /* 942 * Validate the queue action. 943 * 944 * @param[in] action 945 * Pointer to the queue action. 946 * @param[in] action_flags 947 * Bit-fields that holds the actions detected until now. 948 * @param[in] dev 949 * Pointer to the Ethernet device structure. 950 * @param[in] attr 951 * Attributes of flow that includes this action. 952 * @param[out] error 953 * Pointer to error structure. 954 * 955 * @return 956 * 0 on success, a negative errno value otherwise and rte_errno is set. 957 */ 958 int 959 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 960 uint64_t action_flags, 961 struct rte_eth_dev *dev, 962 const struct rte_flow_attr *attr, 963 struct rte_flow_error *error) 964 { 965 struct mlx5_priv *priv = dev->data->dev_private; 966 const struct rte_flow_action_queue *queue = action->conf; 967 968 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 969 return rte_flow_error_set(error, EINVAL, 970 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 971 "can't have 2 fate actions in" 972 " same flow"); 973 if (!priv->rxqs_n) 974 return rte_flow_error_set(error, EINVAL, 975 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 976 NULL, "No Rx queues configured"); 977 if (queue->index >= priv->rxqs_n) 978 return rte_flow_error_set(error, EINVAL, 979 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 980 &queue->index, 981 "queue index out of range"); 982 if (!(*priv->rxqs)[queue->index]) 983 return rte_flow_error_set(error, EINVAL, 984 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 985 &queue->index, 986 "queue is not configured"); 987 if (attr->egress) 988 return rte_flow_error_set(error, ENOTSUP, 989 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 990 "queue action not supported for " 991 "egress"); 992 return 0; 993 } 994 995 /* 996 * Validate the rss action. 997 * 998 * @param[in] action 999 * Pointer to the queue action. 1000 * @param[in] action_flags 1001 * Bit-fields that holds the actions detected until now. 1002 * @param[in] dev 1003 * Pointer to the Ethernet device structure. 1004 * @param[in] attr 1005 * Attributes of flow that includes this action. 1006 * @param[in] item_flags 1007 * Items that were detected. 1008 * @param[out] error 1009 * Pointer to error structure. 1010 * 1011 * @return 1012 * 0 on success, a negative errno value otherwise and rte_errno is set. 1013 */ 1014 int 1015 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1016 uint64_t action_flags, 1017 struct rte_eth_dev *dev, 1018 const struct rte_flow_attr *attr, 1019 uint64_t item_flags, 1020 struct rte_flow_error *error) 1021 { 1022 struct mlx5_priv *priv = dev->data->dev_private; 1023 const struct rte_flow_action_rss *rss = action->conf; 1024 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1025 unsigned int i; 1026 1027 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1028 return rte_flow_error_set(error, EINVAL, 1029 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1030 "can't have 2 fate actions" 1031 " in same flow"); 1032 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1033 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1034 return rte_flow_error_set(error, ENOTSUP, 1035 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1036 &rss->func, 1037 "RSS hash function not supported"); 1038 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1039 if (rss->level > 2) 1040 #else 1041 if (rss->level > 1) 1042 #endif 1043 return rte_flow_error_set(error, ENOTSUP, 1044 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1045 &rss->level, 1046 "tunnel RSS is not supported"); 1047 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1048 if (rss->key_len == 0 && rss->key != NULL) 1049 return rte_flow_error_set(error, ENOTSUP, 1050 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1051 &rss->key_len, 1052 "RSS hash key length 0"); 1053 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1054 return rte_flow_error_set(error, ENOTSUP, 1055 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1056 &rss->key_len, 1057 "RSS hash key too small"); 1058 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1059 return rte_flow_error_set(error, ENOTSUP, 1060 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1061 &rss->key_len, 1062 "RSS hash key too large"); 1063 if (rss->queue_num > priv->config.ind_table_max_size) 1064 return rte_flow_error_set(error, ENOTSUP, 1065 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1066 &rss->queue_num, 1067 "number of queues too large"); 1068 if (rss->types & MLX5_RSS_HF_MASK) 1069 return rte_flow_error_set(error, ENOTSUP, 1070 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1071 &rss->types, 1072 "some RSS protocols are not" 1073 " supported"); 1074 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1075 !(rss->types & ETH_RSS_IP)) 1076 return rte_flow_error_set(error, EINVAL, 1077 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1078 "L3 partial RSS requested but L3 RSS" 1079 " type not specified"); 1080 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1081 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1082 return rte_flow_error_set(error, EINVAL, 1083 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1084 "L4 partial RSS requested but L4 RSS" 1085 " type not specified"); 1086 if (!priv->rxqs_n) 1087 return rte_flow_error_set(error, EINVAL, 1088 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1089 NULL, "No Rx queues configured"); 1090 if (!rss->queue_num) 1091 return rte_flow_error_set(error, EINVAL, 1092 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1093 NULL, "No queues configured"); 1094 for (i = 0; i != rss->queue_num; ++i) { 1095 if (rss->queue[i] >= priv->rxqs_n) 1096 return rte_flow_error_set 1097 (error, EINVAL, 1098 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1099 &rss->queue[i], "queue index out of range"); 1100 if (!(*priv->rxqs)[rss->queue[i]]) 1101 return rte_flow_error_set 1102 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1103 &rss->queue[i], "queue is not configured"); 1104 } 1105 if (attr->egress) 1106 return rte_flow_error_set(error, ENOTSUP, 1107 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1108 "rss action not supported for " 1109 "egress"); 1110 if (rss->level > 1 && !tunnel) 1111 return rte_flow_error_set(error, EINVAL, 1112 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1113 "inner RSS is not supported for " 1114 "non-tunnel flows"); 1115 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1116 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1117 return rte_flow_error_set(error, EINVAL, 1118 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1119 "RSS on eCPRI is not supported now"); 1120 } 1121 return 0; 1122 } 1123 1124 /* 1125 * Validate the default miss action. 1126 * 1127 * @param[in] action_flags 1128 * Bit-fields that holds the actions detected until now. 1129 * @param[out] error 1130 * Pointer to error structure. 1131 * 1132 * @return 1133 * 0 on success, a negative errno value otherwise and rte_errno is set. 1134 */ 1135 int 1136 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1137 const struct rte_flow_attr *attr, 1138 struct rte_flow_error *error) 1139 { 1140 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1141 return rte_flow_error_set(error, EINVAL, 1142 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1143 "can't have 2 fate actions in" 1144 " same flow"); 1145 if (attr->egress) 1146 return rte_flow_error_set(error, ENOTSUP, 1147 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1148 "default miss action not supported " 1149 "for egress"); 1150 if (attr->group) 1151 return rte_flow_error_set(error, ENOTSUP, 1152 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1153 "only group 0 is supported"); 1154 if (attr->transfer) 1155 return rte_flow_error_set(error, ENOTSUP, 1156 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1157 NULL, "transfer is not supported"); 1158 return 0; 1159 } 1160 1161 /* 1162 * Validate the count action. 1163 * 1164 * @param[in] dev 1165 * Pointer to the Ethernet device structure. 1166 * @param[in] attr 1167 * Attributes of flow that includes this action. 1168 * @param[out] error 1169 * Pointer to error structure. 1170 * 1171 * @return 1172 * 0 on success, a negative errno value otherwise and rte_errno is set. 1173 */ 1174 int 1175 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1176 const struct rte_flow_attr *attr, 1177 struct rte_flow_error *error) 1178 { 1179 if (attr->egress) 1180 return rte_flow_error_set(error, ENOTSUP, 1181 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1182 "count action not supported for " 1183 "egress"); 1184 return 0; 1185 } 1186 1187 /** 1188 * Verify the @p attributes will be correctly understood by the NIC and store 1189 * them in the @p flow if everything is correct. 1190 * 1191 * @param[in] dev 1192 * Pointer to the Ethernet device structure. 1193 * @param[in] attributes 1194 * Pointer to flow attributes 1195 * @param[out] error 1196 * Pointer to error structure. 1197 * 1198 * @return 1199 * 0 on success, a negative errno value otherwise and rte_errno is set. 1200 */ 1201 int 1202 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1203 const struct rte_flow_attr *attributes, 1204 struct rte_flow_error *error) 1205 { 1206 struct mlx5_priv *priv = dev->data->dev_private; 1207 uint32_t priority_max = priv->config.flow_prio - 1; 1208 1209 if (attributes->group) 1210 return rte_flow_error_set(error, ENOTSUP, 1211 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1212 NULL, "groups is not supported"); 1213 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1214 attributes->priority >= priority_max) 1215 return rte_flow_error_set(error, ENOTSUP, 1216 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1217 NULL, "priority out of range"); 1218 if (attributes->egress) 1219 return rte_flow_error_set(error, ENOTSUP, 1220 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1221 "egress is not supported"); 1222 if (attributes->transfer && !priv->config.dv_esw_en) 1223 return rte_flow_error_set(error, ENOTSUP, 1224 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1225 NULL, "transfer is not supported"); 1226 if (!attributes->ingress) 1227 return rte_flow_error_set(error, EINVAL, 1228 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1229 NULL, 1230 "ingress attribute is mandatory"); 1231 return 0; 1232 } 1233 1234 /** 1235 * Validate ICMP6 item. 1236 * 1237 * @param[in] item 1238 * Item specification. 1239 * @param[in] item_flags 1240 * Bit-fields that holds the items detected until now. 1241 * @param[out] error 1242 * Pointer to error structure. 1243 * 1244 * @return 1245 * 0 on success, a negative errno value otherwise and rte_errno is set. 1246 */ 1247 int 1248 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1249 uint64_t item_flags, 1250 uint8_t target_protocol, 1251 struct rte_flow_error *error) 1252 { 1253 const struct rte_flow_item_icmp6 *mask = item->mask; 1254 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1255 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1256 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1257 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1258 MLX5_FLOW_LAYER_OUTER_L4; 1259 int ret; 1260 1261 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1262 return rte_flow_error_set(error, EINVAL, 1263 RTE_FLOW_ERROR_TYPE_ITEM, item, 1264 "protocol filtering not compatible" 1265 " with ICMP6 layer"); 1266 if (!(item_flags & l3m)) 1267 return rte_flow_error_set(error, EINVAL, 1268 RTE_FLOW_ERROR_TYPE_ITEM, item, 1269 "IPv6 is mandatory to filter on" 1270 " ICMP6"); 1271 if (item_flags & l4m) 1272 return rte_flow_error_set(error, EINVAL, 1273 RTE_FLOW_ERROR_TYPE_ITEM, item, 1274 "multiple L4 layers not supported"); 1275 if (!mask) 1276 mask = &rte_flow_item_icmp6_mask; 1277 ret = mlx5_flow_item_acceptable 1278 (item, (const uint8_t *)mask, 1279 (const uint8_t *)&rte_flow_item_icmp6_mask, 1280 sizeof(struct rte_flow_item_icmp6), error); 1281 if (ret < 0) 1282 return ret; 1283 return 0; 1284 } 1285 1286 /** 1287 * Validate ICMP item. 1288 * 1289 * @param[in] item 1290 * Item specification. 1291 * @param[in] item_flags 1292 * Bit-fields that holds the items detected until now. 1293 * @param[out] error 1294 * Pointer to error structure. 1295 * 1296 * @return 1297 * 0 on success, a negative errno value otherwise and rte_errno is set. 1298 */ 1299 int 1300 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1301 uint64_t item_flags, 1302 uint8_t target_protocol, 1303 struct rte_flow_error *error) 1304 { 1305 const struct rte_flow_item_icmp *mask = item->mask; 1306 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1307 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1308 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1309 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1310 MLX5_FLOW_LAYER_OUTER_L4; 1311 int ret; 1312 1313 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1314 return rte_flow_error_set(error, EINVAL, 1315 RTE_FLOW_ERROR_TYPE_ITEM, item, 1316 "protocol filtering not compatible" 1317 " with ICMP layer"); 1318 if (!(item_flags & l3m)) 1319 return rte_flow_error_set(error, EINVAL, 1320 RTE_FLOW_ERROR_TYPE_ITEM, item, 1321 "IPv4 is mandatory to filter" 1322 " on ICMP"); 1323 if (item_flags & l4m) 1324 return rte_flow_error_set(error, EINVAL, 1325 RTE_FLOW_ERROR_TYPE_ITEM, item, 1326 "multiple L4 layers not supported"); 1327 if (!mask) 1328 mask = &rte_flow_item_icmp_mask; 1329 ret = mlx5_flow_item_acceptable 1330 (item, (const uint8_t *)mask, 1331 (const uint8_t *)&rte_flow_item_icmp_mask, 1332 sizeof(struct rte_flow_item_icmp), error); 1333 if (ret < 0) 1334 return ret; 1335 return 0; 1336 } 1337 1338 /** 1339 * Validate Ethernet item. 1340 * 1341 * @param[in] item 1342 * Item specification. 1343 * @param[in] item_flags 1344 * Bit-fields that holds the items detected until now. 1345 * @param[out] error 1346 * Pointer to error structure. 1347 * 1348 * @return 1349 * 0 on success, a negative errno value otherwise and rte_errno is set. 1350 */ 1351 int 1352 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1353 uint64_t item_flags, 1354 struct rte_flow_error *error) 1355 { 1356 const struct rte_flow_item_eth *mask = item->mask; 1357 const struct rte_flow_item_eth nic_mask = { 1358 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1359 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1360 .type = RTE_BE16(0xffff), 1361 }; 1362 int ret; 1363 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1364 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1365 MLX5_FLOW_LAYER_OUTER_L2; 1366 1367 if (item_flags & ethm) 1368 return rte_flow_error_set(error, ENOTSUP, 1369 RTE_FLOW_ERROR_TYPE_ITEM, item, 1370 "multiple L2 layers not supported"); 1371 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1372 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1373 return rte_flow_error_set(error, EINVAL, 1374 RTE_FLOW_ERROR_TYPE_ITEM, item, 1375 "L2 layer should not follow " 1376 "L3 layers"); 1377 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1378 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1379 return rte_flow_error_set(error, EINVAL, 1380 RTE_FLOW_ERROR_TYPE_ITEM, item, 1381 "L2 layer should not follow VLAN"); 1382 if (!mask) 1383 mask = &rte_flow_item_eth_mask; 1384 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1385 (const uint8_t *)&nic_mask, 1386 sizeof(struct rte_flow_item_eth), 1387 error); 1388 return ret; 1389 } 1390 1391 /** 1392 * Validate VLAN item. 1393 * 1394 * @param[in] item 1395 * Item specification. 1396 * @param[in] item_flags 1397 * Bit-fields that holds the items detected until now. 1398 * @param[in] dev 1399 * Ethernet device flow is being created on. 1400 * @param[out] error 1401 * Pointer to error structure. 1402 * 1403 * @return 1404 * 0 on success, a negative errno value otherwise and rte_errno is set. 1405 */ 1406 int 1407 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1408 uint64_t item_flags, 1409 struct rte_eth_dev *dev, 1410 struct rte_flow_error *error) 1411 { 1412 const struct rte_flow_item_vlan *spec = item->spec; 1413 const struct rte_flow_item_vlan *mask = item->mask; 1414 const struct rte_flow_item_vlan nic_mask = { 1415 .tci = RTE_BE16(UINT16_MAX), 1416 .inner_type = RTE_BE16(UINT16_MAX), 1417 }; 1418 uint16_t vlan_tag = 0; 1419 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1420 int ret; 1421 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1422 MLX5_FLOW_LAYER_INNER_L4) : 1423 (MLX5_FLOW_LAYER_OUTER_L3 | 1424 MLX5_FLOW_LAYER_OUTER_L4); 1425 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1426 MLX5_FLOW_LAYER_OUTER_VLAN; 1427 1428 if (item_flags & vlanm) 1429 return rte_flow_error_set(error, EINVAL, 1430 RTE_FLOW_ERROR_TYPE_ITEM, item, 1431 "multiple VLAN layers not supported"); 1432 else if ((item_flags & l34m) != 0) 1433 return rte_flow_error_set(error, EINVAL, 1434 RTE_FLOW_ERROR_TYPE_ITEM, item, 1435 "VLAN cannot follow L3/L4 layer"); 1436 if (!mask) 1437 mask = &rte_flow_item_vlan_mask; 1438 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1439 (const uint8_t *)&nic_mask, 1440 sizeof(struct rte_flow_item_vlan), 1441 error); 1442 if (ret) 1443 return ret; 1444 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1445 struct mlx5_priv *priv = dev->data->dev_private; 1446 1447 if (priv->vmwa_context) { 1448 /* 1449 * Non-NULL context means we have a virtual machine 1450 * and SR-IOV enabled, we have to create VLAN interface 1451 * to make hypervisor to setup E-Switch vport 1452 * context correctly. We avoid creating the multiple 1453 * VLAN interfaces, so we cannot support VLAN tag mask. 1454 */ 1455 return rte_flow_error_set(error, EINVAL, 1456 RTE_FLOW_ERROR_TYPE_ITEM, 1457 item, 1458 "VLAN tag mask is not" 1459 " supported in virtual" 1460 " environment"); 1461 } 1462 } 1463 if (spec) { 1464 vlan_tag = spec->tci; 1465 vlan_tag &= mask->tci; 1466 } 1467 /* 1468 * From verbs perspective an empty VLAN is equivalent 1469 * to a packet without VLAN layer. 1470 */ 1471 if (!vlan_tag) 1472 return rte_flow_error_set(error, EINVAL, 1473 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1474 item->spec, 1475 "VLAN cannot be empty"); 1476 return 0; 1477 } 1478 1479 /** 1480 * Validate IPV4 item. 1481 * 1482 * @param[in] item 1483 * Item specification. 1484 * @param[in] item_flags 1485 * Bit-fields that holds the items detected until now. 1486 * @param[in] last_item 1487 * Previous validated item in the pattern items. 1488 * @param[in] ether_type 1489 * Type in the ethernet layer header (including dot1q). 1490 * @param[in] acc_mask 1491 * Acceptable mask, if NULL default internal default mask 1492 * will be used to check whether item fields are supported. 1493 * @param[out] error 1494 * Pointer to error structure. 1495 * 1496 * @return 1497 * 0 on success, a negative errno value otherwise and rte_errno is set. 1498 */ 1499 int 1500 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1501 uint64_t item_flags, 1502 uint64_t last_item, 1503 uint16_t ether_type, 1504 const struct rte_flow_item_ipv4 *acc_mask, 1505 struct rte_flow_error *error) 1506 { 1507 const struct rte_flow_item_ipv4 *mask = item->mask; 1508 const struct rte_flow_item_ipv4 *spec = item->spec; 1509 const struct rte_flow_item_ipv4 nic_mask = { 1510 .hdr = { 1511 .src_addr = RTE_BE32(0xffffffff), 1512 .dst_addr = RTE_BE32(0xffffffff), 1513 .type_of_service = 0xff, 1514 .next_proto_id = 0xff, 1515 }, 1516 }; 1517 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1518 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1519 MLX5_FLOW_LAYER_OUTER_L3; 1520 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1521 MLX5_FLOW_LAYER_OUTER_L4; 1522 int ret; 1523 uint8_t next_proto = 0xFF; 1524 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1525 MLX5_FLOW_LAYER_OUTER_VLAN | 1526 MLX5_FLOW_LAYER_INNER_VLAN); 1527 1528 if ((last_item & l2_vlan) && ether_type && 1529 ether_type != RTE_ETHER_TYPE_IPV4) 1530 return rte_flow_error_set(error, EINVAL, 1531 RTE_FLOW_ERROR_TYPE_ITEM, item, 1532 "IPv4 cannot follow L2/VLAN layer " 1533 "which ether type is not IPv4"); 1534 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1535 if (mask && spec) 1536 next_proto = mask->hdr.next_proto_id & 1537 spec->hdr.next_proto_id; 1538 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1539 return rte_flow_error_set(error, EINVAL, 1540 RTE_FLOW_ERROR_TYPE_ITEM, 1541 item, 1542 "multiple tunnel " 1543 "not supported"); 1544 } 1545 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1546 return rte_flow_error_set(error, EINVAL, 1547 RTE_FLOW_ERROR_TYPE_ITEM, item, 1548 "wrong tunnel type - IPv6 specified " 1549 "but IPv4 item provided"); 1550 if (item_flags & l3m) 1551 return rte_flow_error_set(error, ENOTSUP, 1552 RTE_FLOW_ERROR_TYPE_ITEM, item, 1553 "multiple L3 layers not supported"); 1554 else if (item_flags & l4m) 1555 return rte_flow_error_set(error, EINVAL, 1556 RTE_FLOW_ERROR_TYPE_ITEM, item, 1557 "L3 cannot follow an L4 layer."); 1558 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1559 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1560 return rte_flow_error_set(error, EINVAL, 1561 RTE_FLOW_ERROR_TYPE_ITEM, item, 1562 "L3 cannot follow an NVGRE layer."); 1563 if (!mask) 1564 mask = &rte_flow_item_ipv4_mask; 1565 else if (mask->hdr.next_proto_id != 0 && 1566 mask->hdr.next_proto_id != 0xff) 1567 return rte_flow_error_set(error, EINVAL, 1568 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1569 "partial mask is not supported" 1570 " for protocol"); 1571 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1572 acc_mask ? (const uint8_t *)acc_mask 1573 : (const uint8_t *)&nic_mask, 1574 sizeof(struct rte_flow_item_ipv4), 1575 error); 1576 if (ret < 0) 1577 return ret; 1578 return 0; 1579 } 1580 1581 /** 1582 * Validate IPV6 item. 1583 * 1584 * @param[in] item 1585 * Item specification. 1586 * @param[in] item_flags 1587 * Bit-fields that holds the items detected until now. 1588 * @param[in] last_item 1589 * Previous validated item in the pattern items. 1590 * @param[in] ether_type 1591 * Type in the ethernet layer header (including dot1q). 1592 * @param[in] acc_mask 1593 * Acceptable mask, if NULL default internal default mask 1594 * will be used to check whether item fields are supported. 1595 * @param[out] error 1596 * Pointer to error structure. 1597 * 1598 * @return 1599 * 0 on success, a negative errno value otherwise and rte_errno is set. 1600 */ 1601 int 1602 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1603 uint64_t item_flags, 1604 uint64_t last_item, 1605 uint16_t ether_type, 1606 const struct rte_flow_item_ipv6 *acc_mask, 1607 struct rte_flow_error *error) 1608 { 1609 const struct rte_flow_item_ipv6 *mask = item->mask; 1610 const struct rte_flow_item_ipv6 *spec = item->spec; 1611 const struct rte_flow_item_ipv6 nic_mask = { 1612 .hdr = { 1613 .src_addr = 1614 "\xff\xff\xff\xff\xff\xff\xff\xff" 1615 "\xff\xff\xff\xff\xff\xff\xff\xff", 1616 .dst_addr = 1617 "\xff\xff\xff\xff\xff\xff\xff\xff" 1618 "\xff\xff\xff\xff\xff\xff\xff\xff", 1619 .vtc_flow = RTE_BE32(0xffffffff), 1620 .proto = 0xff, 1621 }, 1622 }; 1623 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1624 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1625 MLX5_FLOW_LAYER_OUTER_L3; 1626 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1627 MLX5_FLOW_LAYER_OUTER_L4; 1628 int ret; 1629 uint8_t next_proto = 0xFF; 1630 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1631 MLX5_FLOW_LAYER_OUTER_VLAN | 1632 MLX5_FLOW_LAYER_INNER_VLAN); 1633 1634 if ((last_item & l2_vlan) && ether_type && 1635 ether_type != RTE_ETHER_TYPE_IPV6) 1636 return rte_flow_error_set(error, EINVAL, 1637 RTE_FLOW_ERROR_TYPE_ITEM, item, 1638 "IPv6 cannot follow L2/VLAN layer " 1639 "which ether type is not IPv6"); 1640 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1641 if (mask && spec) 1642 next_proto = mask->hdr.proto & spec->hdr.proto; 1643 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1644 return rte_flow_error_set(error, EINVAL, 1645 RTE_FLOW_ERROR_TYPE_ITEM, 1646 item, 1647 "multiple tunnel " 1648 "not supported"); 1649 } 1650 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1651 return rte_flow_error_set(error, EINVAL, 1652 RTE_FLOW_ERROR_TYPE_ITEM, item, 1653 "wrong tunnel type - IPv4 specified " 1654 "but IPv6 item provided"); 1655 if (item_flags & l3m) 1656 return rte_flow_error_set(error, ENOTSUP, 1657 RTE_FLOW_ERROR_TYPE_ITEM, item, 1658 "multiple L3 layers not supported"); 1659 else if (item_flags & l4m) 1660 return rte_flow_error_set(error, EINVAL, 1661 RTE_FLOW_ERROR_TYPE_ITEM, item, 1662 "L3 cannot follow an L4 layer."); 1663 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1664 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1665 return rte_flow_error_set(error, EINVAL, 1666 RTE_FLOW_ERROR_TYPE_ITEM, item, 1667 "L3 cannot follow an NVGRE layer."); 1668 if (!mask) 1669 mask = &rte_flow_item_ipv6_mask; 1670 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1671 acc_mask ? (const uint8_t *)acc_mask 1672 : (const uint8_t *)&nic_mask, 1673 sizeof(struct rte_flow_item_ipv6), 1674 error); 1675 if (ret < 0) 1676 return ret; 1677 return 0; 1678 } 1679 1680 /** 1681 * Validate UDP item. 1682 * 1683 * @param[in] item 1684 * Item specification. 1685 * @param[in] item_flags 1686 * Bit-fields that holds the items detected until now. 1687 * @param[in] target_protocol 1688 * The next protocol in the previous item. 1689 * @param[in] flow_mask 1690 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1691 * @param[out] error 1692 * Pointer to error structure. 1693 * 1694 * @return 1695 * 0 on success, a negative errno value otherwise and rte_errno is set. 1696 */ 1697 int 1698 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1699 uint64_t item_flags, 1700 uint8_t target_protocol, 1701 struct rte_flow_error *error) 1702 { 1703 const struct rte_flow_item_udp *mask = item->mask; 1704 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1705 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1706 MLX5_FLOW_LAYER_OUTER_L3; 1707 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1708 MLX5_FLOW_LAYER_OUTER_L4; 1709 int ret; 1710 1711 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1712 return rte_flow_error_set(error, EINVAL, 1713 RTE_FLOW_ERROR_TYPE_ITEM, item, 1714 "protocol filtering not compatible" 1715 " with UDP layer"); 1716 if (!(item_flags & l3m)) 1717 return rte_flow_error_set(error, EINVAL, 1718 RTE_FLOW_ERROR_TYPE_ITEM, item, 1719 "L3 is mandatory to filter on L4"); 1720 if (item_flags & l4m) 1721 return rte_flow_error_set(error, EINVAL, 1722 RTE_FLOW_ERROR_TYPE_ITEM, item, 1723 "multiple L4 layers not supported"); 1724 if (!mask) 1725 mask = &rte_flow_item_udp_mask; 1726 ret = mlx5_flow_item_acceptable 1727 (item, (const uint8_t *)mask, 1728 (const uint8_t *)&rte_flow_item_udp_mask, 1729 sizeof(struct rte_flow_item_udp), error); 1730 if (ret < 0) 1731 return ret; 1732 return 0; 1733 } 1734 1735 /** 1736 * Validate TCP item. 1737 * 1738 * @param[in] item 1739 * Item specification. 1740 * @param[in] item_flags 1741 * Bit-fields that holds the items detected until now. 1742 * @param[in] target_protocol 1743 * The next protocol in the previous item. 1744 * @param[out] error 1745 * Pointer to error structure. 1746 * 1747 * @return 1748 * 0 on success, a negative errno value otherwise and rte_errno is set. 1749 */ 1750 int 1751 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1752 uint64_t item_flags, 1753 uint8_t target_protocol, 1754 const struct rte_flow_item_tcp *flow_mask, 1755 struct rte_flow_error *error) 1756 { 1757 const struct rte_flow_item_tcp *mask = item->mask; 1758 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1759 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1760 MLX5_FLOW_LAYER_OUTER_L3; 1761 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1762 MLX5_FLOW_LAYER_OUTER_L4; 1763 int ret; 1764 1765 MLX5_ASSERT(flow_mask); 1766 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1767 return rte_flow_error_set(error, EINVAL, 1768 RTE_FLOW_ERROR_TYPE_ITEM, item, 1769 "protocol filtering not compatible" 1770 " with TCP layer"); 1771 if (!(item_flags & l3m)) 1772 return rte_flow_error_set(error, EINVAL, 1773 RTE_FLOW_ERROR_TYPE_ITEM, item, 1774 "L3 is mandatory to filter on L4"); 1775 if (item_flags & l4m) 1776 return rte_flow_error_set(error, EINVAL, 1777 RTE_FLOW_ERROR_TYPE_ITEM, item, 1778 "multiple L4 layers not supported"); 1779 if (!mask) 1780 mask = &rte_flow_item_tcp_mask; 1781 ret = mlx5_flow_item_acceptable 1782 (item, (const uint8_t *)mask, 1783 (const uint8_t *)flow_mask, 1784 sizeof(struct rte_flow_item_tcp), error); 1785 if (ret < 0) 1786 return ret; 1787 return 0; 1788 } 1789 1790 /** 1791 * Validate VXLAN item. 1792 * 1793 * @param[in] item 1794 * Item specification. 1795 * @param[in] item_flags 1796 * Bit-fields that holds the items detected until now. 1797 * @param[in] target_protocol 1798 * The next protocol in the previous item. 1799 * @param[out] error 1800 * Pointer to error structure. 1801 * 1802 * @return 1803 * 0 on success, a negative errno value otherwise and rte_errno is set. 1804 */ 1805 int 1806 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1807 uint64_t item_flags, 1808 struct rte_flow_error *error) 1809 { 1810 const struct rte_flow_item_vxlan *spec = item->spec; 1811 const struct rte_flow_item_vxlan *mask = item->mask; 1812 int ret; 1813 union vni { 1814 uint32_t vlan_id; 1815 uint8_t vni[4]; 1816 } id = { .vlan_id = 0, }; 1817 1818 1819 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1820 return rte_flow_error_set(error, ENOTSUP, 1821 RTE_FLOW_ERROR_TYPE_ITEM, item, 1822 "multiple tunnel layers not" 1823 " supported"); 1824 /* 1825 * Verify only UDPv4 is present as defined in 1826 * https://tools.ietf.org/html/rfc7348 1827 */ 1828 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1829 return rte_flow_error_set(error, EINVAL, 1830 RTE_FLOW_ERROR_TYPE_ITEM, item, 1831 "no outer UDP layer found"); 1832 if (!mask) 1833 mask = &rte_flow_item_vxlan_mask; 1834 ret = mlx5_flow_item_acceptable 1835 (item, (const uint8_t *)mask, 1836 (const uint8_t *)&rte_flow_item_vxlan_mask, 1837 sizeof(struct rte_flow_item_vxlan), 1838 error); 1839 if (ret < 0) 1840 return ret; 1841 if (spec) { 1842 memcpy(&id.vni[1], spec->vni, 3); 1843 memcpy(&id.vni[1], mask->vni, 3); 1844 } 1845 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1846 return rte_flow_error_set(error, ENOTSUP, 1847 RTE_FLOW_ERROR_TYPE_ITEM, item, 1848 "VXLAN tunnel must be fully defined"); 1849 return 0; 1850 } 1851 1852 /** 1853 * Validate VXLAN_GPE item. 1854 * 1855 * @param[in] item 1856 * Item specification. 1857 * @param[in] item_flags 1858 * Bit-fields that holds the items detected until now. 1859 * @param[in] priv 1860 * Pointer to the private data structure. 1861 * @param[in] target_protocol 1862 * The next protocol in the previous item. 1863 * @param[out] error 1864 * Pointer to error structure. 1865 * 1866 * @return 1867 * 0 on success, a negative errno value otherwise and rte_errno is set. 1868 */ 1869 int 1870 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1871 uint64_t item_flags, 1872 struct rte_eth_dev *dev, 1873 struct rte_flow_error *error) 1874 { 1875 struct mlx5_priv *priv = dev->data->dev_private; 1876 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1877 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1878 int ret; 1879 union vni { 1880 uint32_t vlan_id; 1881 uint8_t vni[4]; 1882 } id = { .vlan_id = 0, }; 1883 1884 if (!priv->config.l3_vxlan_en) 1885 return rte_flow_error_set(error, ENOTSUP, 1886 RTE_FLOW_ERROR_TYPE_ITEM, item, 1887 "L3 VXLAN is not enabled by device" 1888 " parameter and/or not configured in" 1889 " firmware"); 1890 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1891 return rte_flow_error_set(error, ENOTSUP, 1892 RTE_FLOW_ERROR_TYPE_ITEM, item, 1893 "multiple tunnel layers not" 1894 " supported"); 1895 /* 1896 * Verify only UDPv4 is present as defined in 1897 * https://tools.ietf.org/html/rfc7348 1898 */ 1899 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1900 return rte_flow_error_set(error, EINVAL, 1901 RTE_FLOW_ERROR_TYPE_ITEM, item, 1902 "no outer UDP layer found"); 1903 if (!mask) 1904 mask = &rte_flow_item_vxlan_gpe_mask; 1905 ret = mlx5_flow_item_acceptable 1906 (item, (const uint8_t *)mask, 1907 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1908 sizeof(struct rte_flow_item_vxlan_gpe), 1909 error); 1910 if (ret < 0) 1911 return ret; 1912 if (spec) { 1913 if (spec->protocol) 1914 return rte_flow_error_set(error, ENOTSUP, 1915 RTE_FLOW_ERROR_TYPE_ITEM, 1916 item, 1917 "VxLAN-GPE protocol" 1918 " not supported"); 1919 memcpy(&id.vni[1], spec->vni, 3); 1920 memcpy(&id.vni[1], mask->vni, 3); 1921 } 1922 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1923 return rte_flow_error_set(error, ENOTSUP, 1924 RTE_FLOW_ERROR_TYPE_ITEM, item, 1925 "VXLAN-GPE tunnel must be fully" 1926 " defined"); 1927 return 0; 1928 } 1929 /** 1930 * Validate GRE Key item. 1931 * 1932 * @param[in] item 1933 * Item specification. 1934 * @param[in] item_flags 1935 * Bit flags to mark detected items. 1936 * @param[in] gre_item 1937 * Pointer to gre_item 1938 * @param[out] error 1939 * Pointer to error structure. 1940 * 1941 * @return 1942 * 0 on success, a negative errno value otherwise and rte_errno is set. 1943 */ 1944 int 1945 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 1946 uint64_t item_flags, 1947 const struct rte_flow_item *gre_item, 1948 struct rte_flow_error *error) 1949 { 1950 const rte_be32_t *mask = item->mask; 1951 int ret = 0; 1952 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 1953 const struct rte_flow_item_gre *gre_spec; 1954 const struct rte_flow_item_gre *gre_mask; 1955 1956 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 1957 return rte_flow_error_set(error, ENOTSUP, 1958 RTE_FLOW_ERROR_TYPE_ITEM, item, 1959 "Multiple GRE key not support"); 1960 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 1961 return rte_flow_error_set(error, ENOTSUP, 1962 RTE_FLOW_ERROR_TYPE_ITEM, item, 1963 "No preceding GRE header"); 1964 if (item_flags & MLX5_FLOW_LAYER_INNER) 1965 return rte_flow_error_set(error, ENOTSUP, 1966 RTE_FLOW_ERROR_TYPE_ITEM, item, 1967 "GRE key following a wrong item"); 1968 gre_mask = gre_item->mask; 1969 if (!gre_mask) 1970 gre_mask = &rte_flow_item_gre_mask; 1971 gre_spec = gre_item->spec; 1972 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 1973 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 1974 return rte_flow_error_set(error, EINVAL, 1975 RTE_FLOW_ERROR_TYPE_ITEM, item, 1976 "Key bit must be on"); 1977 1978 if (!mask) 1979 mask = &gre_key_default_mask; 1980 ret = mlx5_flow_item_acceptable 1981 (item, (const uint8_t *)mask, 1982 (const uint8_t *)&gre_key_default_mask, 1983 sizeof(rte_be32_t), error); 1984 return ret; 1985 } 1986 1987 /** 1988 * Validate GRE item. 1989 * 1990 * @param[in] item 1991 * Item specification. 1992 * @param[in] item_flags 1993 * Bit flags to mark detected items. 1994 * @param[in] target_protocol 1995 * The next protocol in the previous item. 1996 * @param[out] error 1997 * Pointer to error structure. 1998 * 1999 * @return 2000 * 0 on success, a negative errno value otherwise and rte_errno is set. 2001 */ 2002 int 2003 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2004 uint64_t item_flags, 2005 uint8_t target_protocol, 2006 struct rte_flow_error *error) 2007 { 2008 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2009 const struct rte_flow_item_gre *mask = item->mask; 2010 int ret; 2011 const struct rte_flow_item_gre nic_mask = { 2012 .c_rsvd0_ver = RTE_BE16(0xB000), 2013 .protocol = RTE_BE16(UINT16_MAX), 2014 }; 2015 2016 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2017 return rte_flow_error_set(error, EINVAL, 2018 RTE_FLOW_ERROR_TYPE_ITEM, item, 2019 "protocol filtering not compatible" 2020 " with this GRE layer"); 2021 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2022 return rte_flow_error_set(error, ENOTSUP, 2023 RTE_FLOW_ERROR_TYPE_ITEM, item, 2024 "multiple tunnel layers not" 2025 " supported"); 2026 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2027 return rte_flow_error_set(error, ENOTSUP, 2028 RTE_FLOW_ERROR_TYPE_ITEM, item, 2029 "L3 Layer is missing"); 2030 if (!mask) 2031 mask = &rte_flow_item_gre_mask; 2032 ret = mlx5_flow_item_acceptable 2033 (item, (const uint8_t *)mask, 2034 (const uint8_t *)&nic_mask, 2035 sizeof(struct rte_flow_item_gre), error); 2036 if (ret < 0) 2037 return ret; 2038 #ifndef HAVE_MLX5DV_DR 2039 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2040 if (spec && (spec->protocol & mask->protocol)) 2041 return rte_flow_error_set(error, ENOTSUP, 2042 RTE_FLOW_ERROR_TYPE_ITEM, item, 2043 "without MPLS support the" 2044 " specification cannot be used for" 2045 " filtering"); 2046 #endif 2047 #endif 2048 return 0; 2049 } 2050 2051 /** 2052 * Validate Geneve item. 2053 * 2054 * @param[in] item 2055 * Item specification. 2056 * @param[in] itemFlags 2057 * Bit-fields that holds the items detected until now. 2058 * @param[in] enPriv 2059 * Pointer to the private data structure. 2060 * @param[out] error 2061 * Pointer to error structure. 2062 * 2063 * @return 2064 * 0 on success, a negative errno value otherwise and rte_errno is set. 2065 */ 2066 2067 int 2068 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2069 uint64_t item_flags, 2070 struct rte_eth_dev *dev, 2071 struct rte_flow_error *error) 2072 { 2073 struct mlx5_priv *priv = dev->data->dev_private; 2074 const struct rte_flow_item_geneve *spec = item->spec; 2075 const struct rte_flow_item_geneve *mask = item->mask; 2076 int ret; 2077 uint16_t gbhdr; 2078 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2079 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2080 const struct rte_flow_item_geneve nic_mask = { 2081 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2082 .vni = "\xff\xff\xff", 2083 .protocol = RTE_BE16(UINT16_MAX), 2084 }; 2085 2086 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2087 return rte_flow_error_set(error, ENOTSUP, 2088 RTE_FLOW_ERROR_TYPE_ITEM, item, 2089 "L3 Geneve is not enabled by device" 2090 " parameter and/or not configured in" 2091 " firmware"); 2092 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2093 return rte_flow_error_set(error, ENOTSUP, 2094 RTE_FLOW_ERROR_TYPE_ITEM, item, 2095 "multiple tunnel layers not" 2096 " supported"); 2097 /* 2098 * Verify only UDPv4 is present as defined in 2099 * https://tools.ietf.org/html/rfc7348 2100 */ 2101 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2102 return rte_flow_error_set(error, EINVAL, 2103 RTE_FLOW_ERROR_TYPE_ITEM, item, 2104 "no outer UDP layer found"); 2105 if (!mask) 2106 mask = &rte_flow_item_geneve_mask; 2107 ret = mlx5_flow_item_acceptable 2108 (item, (const uint8_t *)mask, 2109 (const uint8_t *)&nic_mask, 2110 sizeof(struct rte_flow_item_geneve), error); 2111 if (ret) 2112 return ret; 2113 if (spec) { 2114 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2115 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2116 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2117 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2118 return rte_flow_error_set(error, ENOTSUP, 2119 RTE_FLOW_ERROR_TYPE_ITEM, 2120 item, 2121 "Geneve protocol unsupported" 2122 " fields are being used"); 2123 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2124 return rte_flow_error_set 2125 (error, ENOTSUP, 2126 RTE_FLOW_ERROR_TYPE_ITEM, 2127 item, 2128 "Unsupported Geneve options length"); 2129 } 2130 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2131 return rte_flow_error_set 2132 (error, ENOTSUP, 2133 RTE_FLOW_ERROR_TYPE_ITEM, item, 2134 "Geneve tunnel must be fully defined"); 2135 return 0; 2136 } 2137 2138 /** 2139 * Validate MPLS item. 2140 * 2141 * @param[in] dev 2142 * Pointer to the rte_eth_dev structure. 2143 * @param[in] item 2144 * Item specification. 2145 * @param[in] item_flags 2146 * Bit-fields that holds the items detected until now. 2147 * @param[in] prev_layer 2148 * The protocol layer indicated in previous item. 2149 * @param[out] error 2150 * Pointer to error structure. 2151 * 2152 * @return 2153 * 0 on success, a negative errno value otherwise and rte_errno is set. 2154 */ 2155 int 2156 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2157 const struct rte_flow_item *item __rte_unused, 2158 uint64_t item_flags __rte_unused, 2159 uint64_t prev_layer __rte_unused, 2160 struct rte_flow_error *error) 2161 { 2162 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2163 const struct rte_flow_item_mpls *mask = item->mask; 2164 struct mlx5_priv *priv = dev->data->dev_private; 2165 int ret; 2166 2167 if (!priv->config.mpls_en) 2168 return rte_flow_error_set(error, ENOTSUP, 2169 RTE_FLOW_ERROR_TYPE_ITEM, item, 2170 "MPLS not supported or" 2171 " disabled in firmware" 2172 " configuration."); 2173 /* MPLS over IP, UDP, GRE is allowed */ 2174 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2175 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2176 MLX5_FLOW_LAYER_GRE))) 2177 return rte_flow_error_set(error, EINVAL, 2178 RTE_FLOW_ERROR_TYPE_ITEM, item, 2179 "protocol filtering not compatible" 2180 " with MPLS layer"); 2181 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2182 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2183 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2184 return rte_flow_error_set(error, ENOTSUP, 2185 RTE_FLOW_ERROR_TYPE_ITEM, item, 2186 "multiple tunnel layers not" 2187 " supported"); 2188 if (!mask) 2189 mask = &rte_flow_item_mpls_mask; 2190 ret = mlx5_flow_item_acceptable 2191 (item, (const uint8_t *)mask, 2192 (const uint8_t *)&rte_flow_item_mpls_mask, 2193 sizeof(struct rte_flow_item_mpls), error); 2194 if (ret < 0) 2195 return ret; 2196 return 0; 2197 #else 2198 return rte_flow_error_set(error, ENOTSUP, 2199 RTE_FLOW_ERROR_TYPE_ITEM, item, 2200 "MPLS is not supported by Verbs, please" 2201 " update."); 2202 #endif 2203 } 2204 2205 /** 2206 * Validate NVGRE item. 2207 * 2208 * @param[in] item 2209 * Item specification. 2210 * @param[in] item_flags 2211 * Bit flags to mark detected items. 2212 * @param[in] target_protocol 2213 * The next protocol in the previous item. 2214 * @param[out] error 2215 * Pointer to error structure. 2216 * 2217 * @return 2218 * 0 on success, a negative errno value otherwise and rte_errno is set. 2219 */ 2220 int 2221 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2222 uint64_t item_flags, 2223 uint8_t target_protocol, 2224 struct rte_flow_error *error) 2225 { 2226 const struct rte_flow_item_nvgre *mask = item->mask; 2227 int ret; 2228 2229 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2230 return rte_flow_error_set(error, EINVAL, 2231 RTE_FLOW_ERROR_TYPE_ITEM, item, 2232 "protocol filtering not compatible" 2233 " with this GRE layer"); 2234 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2235 return rte_flow_error_set(error, ENOTSUP, 2236 RTE_FLOW_ERROR_TYPE_ITEM, item, 2237 "multiple tunnel layers not" 2238 " supported"); 2239 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2240 return rte_flow_error_set(error, ENOTSUP, 2241 RTE_FLOW_ERROR_TYPE_ITEM, item, 2242 "L3 Layer is missing"); 2243 if (!mask) 2244 mask = &rte_flow_item_nvgre_mask; 2245 ret = mlx5_flow_item_acceptable 2246 (item, (const uint8_t *)mask, 2247 (const uint8_t *)&rte_flow_item_nvgre_mask, 2248 sizeof(struct rte_flow_item_nvgre), error); 2249 if (ret < 0) 2250 return ret; 2251 return 0; 2252 } 2253 2254 /** 2255 * Validate eCPRI item. 2256 * 2257 * @param[in] item 2258 * Item specification. 2259 * @param[in] item_flags 2260 * Bit-fields that holds the items detected until now. 2261 * @param[in] last_item 2262 * Previous validated item in the pattern items. 2263 * @param[in] ether_type 2264 * Type in the ethernet layer header (including dot1q). 2265 * @param[in] acc_mask 2266 * Acceptable mask, if NULL default internal default mask 2267 * will be used to check whether item fields are supported. 2268 * @param[out] error 2269 * Pointer to error structure. 2270 * 2271 * @return 2272 * 0 on success, a negative errno value otherwise and rte_errno is set. 2273 */ 2274 int 2275 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2276 uint64_t item_flags, 2277 uint64_t last_item, 2278 uint16_t ether_type, 2279 const struct rte_flow_item_ecpri *acc_mask, 2280 struct rte_flow_error *error) 2281 { 2282 const struct rte_flow_item_ecpri *mask = item->mask; 2283 const struct rte_flow_item_ecpri nic_mask = { 2284 .hdr = { 2285 .common = { 2286 .u32 = 2287 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2288 .type = 0xFF, 2289 }).u32), 2290 }, 2291 .dummy[0] = 0xFFFFFFFF, 2292 }, 2293 }; 2294 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2295 MLX5_FLOW_LAYER_OUTER_VLAN); 2296 struct rte_flow_item_ecpri mask_lo; 2297 2298 if ((last_item & outer_l2_vlan) && ether_type && 2299 ether_type != RTE_ETHER_TYPE_ECPRI) 2300 return rte_flow_error_set(error, EINVAL, 2301 RTE_FLOW_ERROR_TYPE_ITEM, item, 2302 "eCPRI cannot follow L2/VLAN layer " 2303 "which ether type is not 0xAEFE."); 2304 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2305 return rte_flow_error_set(error, EINVAL, 2306 RTE_FLOW_ERROR_TYPE_ITEM, item, 2307 "eCPRI with tunnel is not supported " 2308 "right now."); 2309 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2310 return rte_flow_error_set(error, ENOTSUP, 2311 RTE_FLOW_ERROR_TYPE_ITEM, item, 2312 "multiple L3 layers not supported"); 2313 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2314 return rte_flow_error_set(error, EINVAL, 2315 RTE_FLOW_ERROR_TYPE_ITEM, item, 2316 "eCPRI cannot follow a TCP layer."); 2317 /* In specification, eCPRI could be over UDP layer. */ 2318 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2319 return rte_flow_error_set(error, EINVAL, 2320 RTE_FLOW_ERROR_TYPE_ITEM, item, 2321 "eCPRI over UDP layer is not yet " 2322 "supported right now."); 2323 /* Mask for type field in common header could be zero. */ 2324 if (!mask) 2325 mask = &rte_flow_item_ecpri_mask; 2326 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 2327 /* Input mask is in big-endian format. */ 2328 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 2329 return rte_flow_error_set(error, EINVAL, 2330 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2331 "partial mask is not supported " 2332 "for protocol"); 2333 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 2334 return rte_flow_error_set(error, EINVAL, 2335 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2336 "message header mask must be after " 2337 "a type mask"); 2338 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2339 acc_mask ? (const uint8_t *)acc_mask 2340 : (const uint8_t *)&nic_mask, 2341 sizeof(struct rte_flow_item_ecpri), 2342 error); 2343 } 2344 2345 /* Allocate unique ID for the split Q/RSS subflows. */ 2346 static uint32_t 2347 flow_qrss_get_id(struct rte_eth_dev *dev) 2348 { 2349 struct mlx5_priv *priv = dev->data->dev_private; 2350 uint32_t qrss_id, ret; 2351 2352 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2353 if (ret) 2354 return 0; 2355 MLX5_ASSERT(qrss_id); 2356 return qrss_id; 2357 } 2358 2359 /* Free unique ID for the split Q/RSS subflows. */ 2360 static void 2361 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2362 { 2363 struct mlx5_priv *priv = dev->data->dev_private; 2364 2365 if (qrss_id) 2366 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2367 } 2368 2369 /** 2370 * Release resource related QUEUE/RSS action split. 2371 * 2372 * @param dev 2373 * Pointer to Ethernet device. 2374 * @param flow 2375 * Flow to release id's from. 2376 */ 2377 static void 2378 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2379 struct rte_flow *flow) 2380 { 2381 struct mlx5_priv *priv = dev->data->dev_private; 2382 uint32_t handle_idx; 2383 struct mlx5_flow_handle *dev_handle; 2384 2385 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2386 handle_idx, dev_handle, next) 2387 if (dev_handle->split_flow_id) 2388 flow_qrss_free_id(dev, dev_handle->split_flow_id); 2389 } 2390 2391 static int 2392 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2393 const struct rte_flow_attr *attr __rte_unused, 2394 const struct rte_flow_item items[] __rte_unused, 2395 const struct rte_flow_action actions[] __rte_unused, 2396 bool external __rte_unused, 2397 int hairpin __rte_unused, 2398 struct rte_flow_error *error) 2399 { 2400 return rte_flow_error_set(error, ENOTSUP, 2401 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2402 } 2403 2404 static struct mlx5_flow * 2405 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2406 const struct rte_flow_attr *attr __rte_unused, 2407 const struct rte_flow_item items[] __rte_unused, 2408 const struct rte_flow_action actions[] __rte_unused, 2409 struct rte_flow_error *error) 2410 { 2411 rte_flow_error_set(error, ENOTSUP, 2412 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2413 return NULL; 2414 } 2415 2416 static int 2417 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2418 struct mlx5_flow *dev_flow __rte_unused, 2419 const struct rte_flow_attr *attr __rte_unused, 2420 const struct rte_flow_item items[] __rte_unused, 2421 const struct rte_flow_action actions[] __rte_unused, 2422 struct rte_flow_error *error) 2423 { 2424 return rte_flow_error_set(error, ENOTSUP, 2425 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2426 } 2427 2428 static int 2429 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2430 struct rte_flow *flow __rte_unused, 2431 struct rte_flow_error *error) 2432 { 2433 return rte_flow_error_set(error, ENOTSUP, 2434 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2435 } 2436 2437 static void 2438 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2439 struct rte_flow *flow __rte_unused) 2440 { 2441 } 2442 2443 static void 2444 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2445 struct rte_flow *flow __rte_unused) 2446 { 2447 } 2448 2449 static int 2450 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2451 struct rte_flow *flow __rte_unused, 2452 const struct rte_flow_action *actions __rte_unused, 2453 void *data __rte_unused, 2454 struct rte_flow_error *error) 2455 { 2456 return rte_flow_error_set(error, ENOTSUP, 2457 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2458 } 2459 2460 /* Void driver to protect from null pointer reference. */ 2461 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2462 .validate = flow_null_validate, 2463 .prepare = flow_null_prepare, 2464 .translate = flow_null_translate, 2465 .apply = flow_null_apply, 2466 .remove = flow_null_remove, 2467 .destroy = flow_null_destroy, 2468 .query = flow_null_query, 2469 }; 2470 2471 /** 2472 * Select flow driver type according to flow attributes and device 2473 * configuration. 2474 * 2475 * @param[in] dev 2476 * Pointer to the dev structure. 2477 * @param[in] attr 2478 * Pointer to the flow attributes. 2479 * 2480 * @return 2481 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2482 */ 2483 static enum mlx5_flow_drv_type 2484 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2485 { 2486 struct mlx5_priv *priv = dev->data->dev_private; 2487 /* The OS can determine first a specific flow type (DV, VERBS) */ 2488 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 2489 2490 if (type != MLX5_FLOW_TYPE_MAX) 2491 return type; 2492 /* If no OS specific type - continue with DV/VERBS selection */ 2493 if (attr->transfer && priv->config.dv_esw_en) 2494 type = MLX5_FLOW_TYPE_DV; 2495 if (!attr->transfer) 2496 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2497 MLX5_FLOW_TYPE_VERBS; 2498 return type; 2499 } 2500 2501 #define flow_get_drv_ops(type) flow_drv_ops[type] 2502 2503 /** 2504 * Flow driver validation API. This abstracts calling driver specific functions. 2505 * The type of flow driver is determined according to flow attributes. 2506 * 2507 * @param[in] dev 2508 * Pointer to the dev structure. 2509 * @param[in] attr 2510 * Pointer to the flow attributes. 2511 * @param[in] items 2512 * Pointer to the list of items. 2513 * @param[in] actions 2514 * Pointer to the list of actions. 2515 * @param[in] external 2516 * This flow rule is created by request external to PMD. 2517 * @param[in] hairpin 2518 * Number of hairpin TX actions, 0 means classic flow. 2519 * @param[out] error 2520 * Pointer to the error structure. 2521 * 2522 * @return 2523 * 0 on success, a negative errno value otherwise and rte_errno is set. 2524 */ 2525 static inline int 2526 flow_drv_validate(struct rte_eth_dev *dev, 2527 const struct rte_flow_attr *attr, 2528 const struct rte_flow_item items[], 2529 const struct rte_flow_action actions[], 2530 bool external, int hairpin, struct rte_flow_error *error) 2531 { 2532 const struct mlx5_flow_driver_ops *fops; 2533 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2534 2535 fops = flow_get_drv_ops(type); 2536 return fops->validate(dev, attr, items, actions, external, 2537 hairpin, error); 2538 } 2539 2540 /** 2541 * Flow driver preparation API. This abstracts calling driver specific 2542 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2543 * calculates the size of memory required for device flow, allocates the memory, 2544 * initializes the device flow and returns the pointer. 2545 * 2546 * @note 2547 * This function initializes device flow structure such as dv or verbs in 2548 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2549 * rest. For example, adding returning device flow to flow->dev_flow list and 2550 * setting backward reference to the flow should be done out of this function. 2551 * layers field is not filled either. 2552 * 2553 * @param[in] dev 2554 * Pointer to the dev structure. 2555 * @param[in] attr 2556 * Pointer to the flow attributes. 2557 * @param[in] items 2558 * Pointer to the list of items. 2559 * @param[in] actions 2560 * Pointer to the list of actions. 2561 * @param[in] flow_idx 2562 * This memory pool index to the flow. 2563 * @param[out] error 2564 * Pointer to the error structure. 2565 * 2566 * @return 2567 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2568 */ 2569 static inline struct mlx5_flow * 2570 flow_drv_prepare(struct rte_eth_dev *dev, 2571 const struct rte_flow *flow, 2572 const struct rte_flow_attr *attr, 2573 const struct rte_flow_item items[], 2574 const struct rte_flow_action actions[], 2575 uint32_t flow_idx, 2576 struct rte_flow_error *error) 2577 { 2578 const struct mlx5_flow_driver_ops *fops; 2579 enum mlx5_flow_drv_type type = flow->drv_type; 2580 struct mlx5_flow *mlx5_flow = NULL; 2581 2582 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2583 fops = flow_get_drv_ops(type); 2584 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 2585 if (mlx5_flow) 2586 mlx5_flow->flow_idx = flow_idx; 2587 return mlx5_flow; 2588 } 2589 2590 /** 2591 * Flow driver translation API. This abstracts calling driver specific 2592 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2593 * translates a generic flow into a driver flow. flow_drv_prepare() must 2594 * precede. 2595 * 2596 * @note 2597 * dev_flow->layers could be filled as a result of parsing during translation 2598 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2599 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2600 * flow->actions could be overwritten even though all the expanded dev_flows 2601 * have the same actions. 2602 * 2603 * @param[in] dev 2604 * Pointer to the rte dev structure. 2605 * @param[in, out] dev_flow 2606 * Pointer to the mlx5 flow. 2607 * @param[in] attr 2608 * Pointer to the flow attributes. 2609 * @param[in] items 2610 * Pointer to the list of items. 2611 * @param[in] actions 2612 * Pointer to the list of actions. 2613 * @param[out] error 2614 * Pointer to the error structure. 2615 * 2616 * @return 2617 * 0 on success, a negative errno value otherwise and rte_errno is set. 2618 */ 2619 static inline int 2620 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2621 const struct rte_flow_attr *attr, 2622 const struct rte_flow_item items[], 2623 const struct rte_flow_action actions[], 2624 struct rte_flow_error *error) 2625 { 2626 const struct mlx5_flow_driver_ops *fops; 2627 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2628 2629 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2630 fops = flow_get_drv_ops(type); 2631 return fops->translate(dev, dev_flow, attr, items, actions, error); 2632 } 2633 2634 /** 2635 * Flow driver apply API. This abstracts calling driver specific functions. 2636 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2637 * translated driver flows on to device. flow_drv_translate() must precede. 2638 * 2639 * @param[in] dev 2640 * Pointer to Ethernet device structure. 2641 * @param[in, out] flow 2642 * Pointer to flow structure. 2643 * @param[out] error 2644 * Pointer to error structure. 2645 * 2646 * @return 2647 * 0 on success, a negative errno value otherwise and rte_errno is set. 2648 */ 2649 static inline int 2650 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2651 struct rte_flow_error *error) 2652 { 2653 const struct mlx5_flow_driver_ops *fops; 2654 enum mlx5_flow_drv_type type = flow->drv_type; 2655 2656 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2657 fops = flow_get_drv_ops(type); 2658 return fops->apply(dev, flow, error); 2659 } 2660 2661 /** 2662 * Flow driver remove API. This abstracts calling driver specific functions. 2663 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2664 * on device. All the resources of the flow should be freed by calling 2665 * flow_drv_destroy(). 2666 * 2667 * @param[in] dev 2668 * Pointer to Ethernet device. 2669 * @param[in, out] flow 2670 * Pointer to flow structure. 2671 */ 2672 static inline void 2673 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2674 { 2675 const struct mlx5_flow_driver_ops *fops; 2676 enum mlx5_flow_drv_type type = flow->drv_type; 2677 2678 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2679 fops = flow_get_drv_ops(type); 2680 fops->remove(dev, flow); 2681 } 2682 2683 /** 2684 * Flow driver destroy API. This abstracts calling driver specific functions. 2685 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2686 * on device and releases resources of the flow. 2687 * 2688 * @param[in] dev 2689 * Pointer to Ethernet device. 2690 * @param[in, out] flow 2691 * Pointer to flow structure. 2692 */ 2693 static inline void 2694 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2695 { 2696 const struct mlx5_flow_driver_ops *fops; 2697 enum mlx5_flow_drv_type type = flow->drv_type; 2698 2699 flow_mreg_split_qrss_release(dev, flow); 2700 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2701 fops = flow_get_drv_ops(type); 2702 fops->destroy(dev, flow); 2703 } 2704 2705 /** 2706 * Get RSS action from the action list. 2707 * 2708 * @param[in] actions 2709 * Pointer to the list of actions. 2710 * 2711 * @return 2712 * Pointer to the RSS action if exist, else return NULL. 2713 */ 2714 static const struct rte_flow_action_rss* 2715 flow_get_rss_action(const struct rte_flow_action actions[]) 2716 { 2717 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2718 switch (actions->type) { 2719 case RTE_FLOW_ACTION_TYPE_RSS: 2720 return (const struct rte_flow_action_rss *) 2721 actions->conf; 2722 default: 2723 break; 2724 } 2725 } 2726 return NULL; 2727 } 2728 2729 static unsigned int 2730 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2731 { 2732 const struct rte_flow_item *item; 2733 unsigned int has_vlan = 0; 2734 2735 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2736 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2737 has_vlan = 1; 2738 break; 2739 } 2740 } 2741 if (has_vlan) 2742 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2743 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2744 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2745 MLX5_EXPANSION_ROOT_OUTER; 2746 } 2747 2748 /** 2749 * Get layer flags from the prefix flow. 2750 * 2751 * Some flows may be split to several subflows, the prefix subflow gets the 2752 * match items and the suffix sub flow gets the actions. 2753 * Some actions need the user defined match item flags to get the detail for 2754 * the action. 2755 * This function helps the suffix flow to get the item layer flags from prefix 2756 * subflow. 2757 * 2758 * @param[in] dev_flow 2759 * Pointer the created preifx subflow. 2760 * 2761 * @return 2762 * The layers get from prefix subflow. 2763 */ 2764 static inline uint64_t 2765 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 2766 { 2767 uint64_t layers = 0; 2768 2769 /* 2770 * Layers bits could be localization, but usually the compiler will 2771 * help to do the optimization work for source code. 2772 * If no decap actions, use the layers directly. 2773 */ 2774 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 2775 return dev_flow->handle->layers; 2776 /* Convert L3 layers with decap action. */ 2777 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 2778 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2779 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 2780 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2781 /* Convert L4 layers with decap action. */ 2782 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 2783 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 2784 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 2785 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 2786 return layers; 2787 } 2788 2789 /** 2790 * Get metadata split action information. 2791 * 2792 * @param[in] actions 2793 * Pointer to the list of actions. 2794 * @param[out] qrss 2795 * Pointer to the return pointer. 2796 * @param[out] qrss_type 2797 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2798 * if no QUEUE/RSS is found. 2799 * @param[out] encap_idx 2800 * Pointer to the index of the encap action if exists, otherwise the last 2801 * action index. 2802 * 2803 * @return 2804 * Total number of actions. 2805 */ 2806 static int 2807 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 2808 const struct rte_flow_action **qrss, 2809 int *encap_idx) 2810 { 2811 const struct rte_flow_action_raw_encap *raw_encap; 2812 int actions_n = 0; 2813 int raw_decap_idx = -1; 2814 2815 *encap_idx = -1; 2816 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2817 switch (actions->type) { 2818 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2819 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2820 *encap_idx = actions_n; 2821 break; 2822 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 2823 raw_decap_idx = actions_n; 2824 break; 2825 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2826 raw_encap = actions->conf; 2827 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2828 *encap_idx = raw_decap_idx != -1 ? 2829 raw_decap_idx : actions_n; 2830 break; 2831 case RTE_FLOW_ACTION_TYPE_QUEUE: 2832 case RTE_FLOW_ACTION_TYPE_RSS: 2833 *qrss = actions; 2834 break; 2835 default: 2836 break; 2837 } 2838 actions_n++; 2839 } 2840 if (*encap_idx == -1) 2841 *encap_idx = actions_n; 2842 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2843 return actions_n + 1; 2844 } 2845 2846 /** 2847 * Check meter action from the action list. 2848 * 2849 * @param[in] actions 2850 * Pointer to the list of actions. 2851 * @param[out] mtr 2852 * Pointer to the meter exist flag. 2853 * 2854 * @return 2855 * Total number of actions. 2856 */ 2857 static int 2858 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2859 { 2860 int actions_n = 0; 2861 2862 MLX5_ASSERT(mtr); 2863 *mtr = 0; 2864 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2865 switch (actions->type) { 2866 case RTE_FLOW_ACTION_TYPE_METER: 2867 *mtr = 1; 2868 break; 2869 default: 2870 break; 2871 } 2872 actions_n++; 2873 } 2874 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2875 return actions_n + 1; 2876 } 2877 2878 /** 2879 * Check if the flow should be split due to hairpin. 2880 * The reason for the split is that in current HW we can't 2881 * support encap and push-vlan on Rx, so if a flow contains 2882 * these actions we move it to Tx. 2883 * 2884 * @param dev 2885 * Pointer to Ethernet device. 2886 * @param[in] attr 2887 * Flow rule attributes. 2888 * @param[in] actions 2889 * Associated actions (list terminated by the END action). 2890 * 2891 * @return 2892 * > 0 the number of actions and the flow should be split, 2893 * 0 when no split required. 2894 */ 2895 static int 2896 flow_check_hairpin_split(struct rte_eth_dev *dev, 2897 const struct rte_flow_attr *attr, 2898 const struct rte_flow_action actions[]) 2899 { 2900 int queue_action = 0; 2901 int action_n = 0; 2902 int split = 0; 2903 const struct rte_flow_action_queue *queue; 2904 const struct rte_flow_action_rss *rss; 2905 const struct rte_flow_action_raw_encap *raw_encap; 2906 2907 if (!attr->ingress) 2908 return 0; 2909 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2910 switch (actions->type) { 2911 case RTE_FLOW_ACTION_TYPE_QUEUE: 2912 queue = actions->conf; 2913 if (queue == NULL) 2914 return 0; 2915 if (mlx5_rxq_get_type(dev, queue->index) != 2916 MLX5_RXQ_TYPE_HAIRPIN) 2917 return 0; 2918 queue_action = 1; 2919 action_n++; 2920 break; 2921 case RTE_FLOW_ACTION_TYPE_RSS: 2922 rss = actions->conf; 2923 if (rss == NULL || rss->queue_num == 0) 2924 return 0; 2925 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2926 MLX5_RXQ_TYPE_HAIRPIN) 2927 return 0; 2928 queue_action = 1; 2929 action_n++; 2930 break; 2931 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2932 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2933 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 2934 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 2935 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 2936 split++; 2937 action_n++; 2938 break; 2939 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2940 raw_encap = actions->conf; 2941 if (raw_encap->size > 2942 (sizeof(struct rte_flow_item_eth) + 2943 sizeof(struct rte_flow_item_ipv4))) 2944 split++; 2945 action_n++; 2946 break; 2947 default: 2948 action_n++; 2949 break; 2950 } 2951 } 2952 if (split && queue_action) 2953 return action_n; 2954 return 0; 2955 } 2956 2957 /* Declare flow create/destroy prototype in advance. */ 2958 static uint32_t 2959 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 2960 const struct rte_flow_attr *attr, 2961 const struct rte_flow_item items[], 2962 const struct rte_flow_action actions[], 2963 bool external, struct rte_flow_error *error); 2964 2965 static void 2966 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 2967 uint32_t flow_idx); 2968 2969 /** 2970 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2971 * 2972 * As mark_id is unique, if there's already a registered flow for the mark_id, 2973 * return by increasing the reference counter of the resource. Otherwise, create 2974 * the resource (mcp_res) and flow. 2975 * 2976 * Flow looks like, 2977 * - If ingress port is ANY and reg_c[1] is mark_id, 2978 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2979 * 2980 * For default flow (zero mark_id), flow is like, 2981 * - If ingress port is ANY, 2982 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2983 * 2984 * @param dev 2985 * Pointer to Ethernet device. 2986 * @param mark_id 2987 * ID of MARK action, zero means default flow for META. 2988 * @param[out] error 2989 * Perform verbose error reporting if not NULL. 2990 * 2991 * @return 2992 * Associated resource on success, NULL otherwise and rte_errno is set. 2993 */ 2994 static struct mlx5_flow_mreg_copy_resource * 2995 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2996 struct rte_flow_error *error) 2997 { 2998 struct mlx5_priv *priv = dev->data->dev_private; 2999 struct rte_flow_attr attr = { 3000 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3001 .ingress = 1, 3002 }; 3003 struct mlx5_rte_flow_item_tag tag_spec = { 3004 .data = mark_id, 3005 }; 3006 struct rte_flow_item items[] = { 3007 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3008 }; 3009 struct rte_flow_action_mark ftag = { 3010 .id = mark_id, 3011 }; 3012 struct mlx5_flow_action_copy_mreg cp_mreg = { 3013 .dst = REG_B, 3014 .src = REG_NON, 3015 }; 3016 struct rte_flow_action_jump jump = { 3017 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3018 }; 3019 struct rte_flow_action actions[] = { 3020 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3021 }; 3022 struct mlx5_flow_mreg_copy_resource *mcp_res; 3023 uint32_t idx = 0; 3024 int ret; 3025 3026 /* Fill the register fileds in the flow. */ 3027 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3028 if (ret < 0) 3029 return NULL; 3030 tag_spec.id = ret; 3031 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3032 if (ret < 0) 3033 return NULL; 3034 cp_mreg.src = ret; 3035 /* Check if already registered. */ 3036 MLX5_ASSERT(priv->mreg_cp_tbl); 3037 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 3038 if (mcp_res) { 3039 /* For non-default rule. */ 3040 if (mark_id != MLX5_DEFAULT_COPY_ID) 3041 mcp_res->refcnt++; 3042 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 3043 mcp_res->refcnt == 1); 3044 return mcp_res; 3045 } 3046 /* Provide the full width of FLAG specific value. */ 3047 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3048 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3049 /* Build a new flow. */ 3050 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3051 items[0] = (struct rte_flow_item){ 3052 .type = (enum rte_flow_item_type) 3053 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3054 .spec = &tag_spec, 3055 }; 3056 items[1] = (struct rte_flow_item){ 3057 .type = RTE_FLOW_ITEM_TYPE_END, 3058 }; 3059 actions[0] = (struct rte_flow_action){ 3060 .type = (enum rte_flow_action_type) 3061 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3062 .conf = &ftag, 3063 }; 3064 actions[1] = (struct rte_flow_action){ 3065 .type = (enum rte_flow_action_type) 3066 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3067 .conf = &cp_mreg, 3068 }; 3069 actions[2] = (struct rte_flow_action){ 3070 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3071 .conf = &jump, 3072 }; 3073 actions[3] = (struct rte_flow_action){ 3074 .type = RTE_FLOW_ACTION_TYPE_END, 3075 }; 3076 } else { 3077 /* Default rule, wildcard match. */ 3078 attr.priority = MLX5_FLOW_PRIO_RSVD; 3079 items[0] = (struct rte_flow_item){ 3080 .type = RTE_FLOW_ITEM_TYPE_END, 3081 }; 3082 actions[0] = (struct rte_flow_action){ 3083 .type = (enum rte_flow_action_type) 3084 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3085 .conf = &cp_mreg, 3086 }; 3087 actions[1] = (struct rte_flow_action){ 3088 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3089 .conf = &jump, 3090 }; 3091 actions[2] = (struct rte_flow_action){ 3092 .type = RTE_FLOW_ACTION_TYPE_END, 3093 }; 3094 } 3095 /* Build a new entry. */ 3096 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3097 if (!mcp_res) { 3098 rte_errno = ENOMEM; 3099 return NULL; 3100 } 3101 mcp_res->idx = idx; 3102 /* 3103 * The copy Flows are not included in any list. There 3104 * ones are referenced from other Flows and can not 3105 * be applied, removed, deleted in ardbitrary order 3106 * by list traversing. 3107 */ 3108 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3109 actions, false, error); 3110 if (!mcp_res->rix_flow) 3111 goto error; 3112 mcp_res->refcnt++; 3113 mcp_res->hlist_ent.key = mark_id; 3114 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3115 &mcp_res->hlist_ent); 3116 MLX5_ASSERT(!ret); 3117 if (ret) 3118 goto error; 3119 return mcp_res; 3120 error: 3121 if (mcp_res->rix_flow) 3122 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3123 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3124 return NULL; 3125 } 3126 3127 /** 3128 * Release flow in RX_CP_TBL. 3129 * 3130 * @param dev 3131 * Pointer to Ethernet device. 3132 * @flow 3133 * Parent flow for wich copying is provided. 3134 */ 3135 static void 3136 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3137 struct rte_flow *flow) 3138 { 3139 struct mlx5_flow_mreg_copy_resource *mcp_res; 3140 struct mlx5_priv *priv = dev->data->dev_private; 3141 3142 if (!flow->rix_mreg_copy) 3143 return; 3144 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3145 flow->rix_mreg_copy); 3146 if (!mcp_res || !priv->mreg_cp_tbl) 3147 return; 3148 if (flow->copy_applied) { 3149 MLX5_ASSERT(mcp_res->appcnt); 3150 flow->copy_applied = 0; 3151 --mcp_res->appcnt; 3152 if (!mcp_res->appcnt) { 3153 struct rte_flow *mcp_flow = mlx5_ipool_get 3154 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3155 mcp_res->rix_flow); 3156 3157 if (mcp_flow) 3158 flow_drv_remove(dev, mcp_flow); 3159 } 3160 } 3161 /* 3162 * We do not check availability of metadata registers here, 3163 * because copy resources are not allocated in this case. 3164 */ 3165 if (--mcp_res->refcnt) 3166 return; 3167 MLX5_ASSERT(mcp_res->rix_flow); 3168 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3169 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3170 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3171 flow->rix_mreg_copy = 0; 3172 } 3173 3174 /** 3175 * Start flow in RX_CP_TBL. 3176 * 3177 * @param dev 3178 * Pointer to Ethernet device. 3179 * @flow 3180 * Parent flow for wich copying is provided. 3181 * 3182 * @return 3183 * 0 on success, a negative errno value otherwise and rte_errno is set. 3184 */ 3185 static int 3186 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3187 struct rte_flow *flow) 3188 { 3189 struct mlx5_flow_mreg_copy_resource *mcp_res; 3190 struct mlx5_priv *priv = dev->data->dev_private; 3191 int ret; 3192 3193 if (!flow->rix_mreg_copy || flow->copy_applied) 3194 return 0; 3195 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3196 flow->rix_mreg_copy); 3197 if (!mcp_res) 3198 return 0; 3199 if (!mcp_res->appcnt) { 3200 struct rte_flow *mcp_flow = mlx5_ipool_get 3201 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3202 mcp_res->rix_flow); 3203 3204 if (mcp_flow) { 3205 ret = flow_drv_apply(dev, mcp_flow, NULL); 3206 if (ret) 3207 return ret; 3208 } 3209 } 3210 ++mcp_res->appcnt; 3211 flow->copy_applied = 1; 3212 return 0; 3213 } 3214 3215 /** 3216 * Stop flow in RX_CP_TBL. 3217 * 3218 * @param dev 3219 * Pointer to Ethernet device. 3220 * @flow 3221 * Parent flow for wich copying is provided. 3222 */ 3223 static void 3224 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3225 struct rte_flow *flow) 3226 { 3227 struct mlx5_flow_mreg_copy_resource *mcp_res; 3228 struct mlx5_priv *priv = dev->data->dev_private; 3229 3230 if (!flow->rix_mreg_copy || !flow->copy_applied) 3231 return; 3232 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3233 flow->rix_mreg_copy); 3234 if (!mcp_res) 3235 return; 3236 MLX5_ASSERT(mcp_res->appcnt); 3237 --mcp_res->appcnt; 3238 flow->copy_applied = 0; 3239 if (!mcp_res->appcnt) { 3240 struct rte_flow *mcp_flow = mlx5_ipool_get 3241 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3242 mcp_res->rix_flow); 3243 3244 if (mcp_flow) 3245 flow_drv_remove(dev, mcp_flow); 3246 } 3247 } 3248 3249 /** 3250 * Remove the default copy action from RX_CP_TBL. 3251 * 3252 * @param dev 3253 * Pointer to Ethernet device. 3254 */ 3255 static void 3256 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3257 { 3258 struct mlx5_flow_mreg_copy_resource *mcp_res; 3259 struct mlx5_priv *priv = dev->data->dev_private; 3260 3261 /* Check if default flow is registered. */ 3262 if (!priv->mreg_cp_tbl) 3263 return; 3264 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3265 MLX5_DEFAULT_COPY_ID); 3266 if (!mcp_res) 3267 return; 3268 MLX5_ASSERT(mcp_res->rix_flow); 3269 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3270 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3271 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3272 } 3273 3274 /** 3275 * Add the default copy action in in RX_CP_TBL. 3276 * 3277 * @param dev 3278 * Pointer to Ethernet device. 3279 * @param[out] error 3280 * Perform verbose error reporting if not NULL. 3281 * 3282 * @return 3283 * 0 for success, negative value otherwise and rte_errno is set. 3284 */ 3285 static int 3286 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3287 struct rte_flow_error *error) 3288 { 3289 struct mlx5_priv *priv = dev->data->dev_private; 3290 struct mlx5_flow_mreg_copy_resource *mcp_res; 3291 3292 /* Check whether extensive metadata feature is engaged. */ 3293 if (!priv->config.dv_flow_en || 3294 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3295 !mlx5_flow_ext_mreg_supported(dev) || 3296 !priv->sh->dv_regc0_mask) 3297 return 0; 3298 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3299 if (!mcp_res) 3300 return -rte_errno; 3301 return 0; 3302 } 3303 3304 /** 3305 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3306 * 3307 * All the flow having Q/RSS action should be split by 3308 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3309 * performs the following, 3310 * - CQE->flow_tag := reg_c[1] (MARK) 3311 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3312 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3313 * but there should be a flow per each MARK ID set by MARK action. 3314 * 3315 * For the aforementioned reason, if there's a MARK action in flow's action 3316 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3317 * the MARK ID to CQE's flow_tag like, 3318 * - If reg_c[1] is mark_id, 3319 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3320 * 3321 * For SET_META action which stores value in reg_c[0], as the destination is 3322 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3323 * MARK ID means the default flow. The default flow looks like, 3324 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3325 * 3326 * @param dev 3327 * Pointer to Ethernet device. 3328 * @param flow 3329 * Pointer to flow structure. 3330 * @param[in] actions 3331 * Pointer to the list of actions. 3332 * @param[out] error 3333 * Perform verbose error reporting if not NULL. 3334 * 3335 * @return 3336 * 0 on success, negative value otherwise and rte_errno is set. 3337 */ 3338 static int 3339 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3340 struct rte_flow *flow, 3341 const struct rte_flow_action *actions, 3342 struct rte_flow_error *error) 3343 { 3344 struct mlx5_priv *priv = dev->data->dev_private; 3345 struct mlx5_dev_config *config = &priv->config; 3346 struct mlx5_flow_mreg_copy_resource *mcp_res; 3347 const struct rte_flow_action_mark *mark; 3348 3349 /* Check whether extensive metadata feature is engaged. */ 3350 if (!config->dv_flow_en || 3351 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3352 !mlx5_flow_ext_mreg_supported(dev) || 3353 !priv->sh->dv_regc0_mask) 3354 return 0; 3355 /* Find MARK action. */ 3356 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3357 switch (actions->type) { 3358 case RTE_FLOW_ACTION_TYPE_FLAG: 3359 mcp_res = flow_mreg_add_copy_action 3360 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3361 if (!mcp_res) 3362 return -rte_errno; 3363 flow->rix_mreg_copy = mcp_res->idx; 3364 if (dev->data->dev_started) { 3365 mcp_res->appcnt++; 3366 flow->copy_applied = 1; 3367 } 3368 return 0; 3369 case RTE_FLOW_ACTION_TYPE_MARK: 3370 mark = (const struct rte_flow_action_mark *) 3371 actions->conf; 3372 mcp_res = 3373 flow_mreg_add_copy_action(dev, mark->id, error); 3374 if (!mcp_res) 3375 return -rte_errno; 3376 flow->rix_mreg_copy = mcp_res->idx; 3377 if (dev->data->dev_started) { 3378 mcp_res->appcnt++; 3379 flow->copy_applied = 1; 3380 } 3381 return 0; 3382 default: 3383 break; 3384 } 3385 } 3386 return 0; 3387 } 3388 3389 #define MLX5_MAX_SPLIT_ACTIONS 24 3390 #define MLX5_MAX_SPLIT_ITEMS 24 3391 3392 /** 3393 * Split the hairpin flow. 3394 * Since HW can't support encap and push-vlan on Rx, we move these 3395 * actions to Tx. 3396 * If the count action is after the encap then we also 3397 * move the count action. in this case the count will also measure 3398 * the outer bytes. 3399 * 3400 * @param dev 3401 * Pointer to Ethernet device. 3402 * @param[in] actions 3403 * Associated actions (list terminated by the END action). 3404 * @param[out] actions_rx 3405 * Rx flow actions. 3406 * @param[out] actions_tx 3407 * Tx flow actions.. 3408 * @param[out] pattern_tx 3409 * The pattern items for the Tx flow. 3410 * @param[out] flow_id 3411 * The flow ID connected to this flow. 3412 * 3413 * @return 3414 * 0 on success. 3415 */ 3416 static int 3417 flow_hairpin_split(struct rte_eth_dev *dev, 3418 const struct rte_flow_action actions[], 3419 struct rte_flow_action actions_rx[], 3420 struct rte_flow_action actions_tx[], 3421 struct rte_flow_item pattern_tx[], 3422 uint32_t *flow_id) 3423 { 3424 struct mlx5_priv *priv = dev->data->dev_private; 3425 const struct rte_flow_action_raw_encap *raw_encap; 3426 const struct rte_flow_action_raw_decap *raw_decap; 3427 struct mlx5_rte_flow_action_set_tag *set_tag; 3428 struct rte_flow_action *tag_action; 3429 struct mlx5_rte_flow_item_tag *tag_item; 3430 struct rte_flow_item *item; 3431 char *addr; 3432 int encap = 0; 3433 3434 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3435 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3436 switch (actions->type) { 3437 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3438 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3439 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3440 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3441 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3442 rte_memcpy(actions_tx, actions, 3443 sizeof(struct rte_flow_action)); 3444 actions_tx++; 3445 break; 3446 case RTE_FLOW_ACTION_TYPE_COUNT: 3447 if (encap) { 3448 rte_memcpy(actions_tx, actions, 3449 sizeof(struct rte_flow_action)); 3450 actions_tx++; 3451 } else { 3452 rte_memcpy(actions_rx, actions, 3453 sizeof(struct rte_flow_action)); 3454 actions_rx++; 3455 } 3456 break; 3457 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3458 raw_encap = actions->conf; 3459 if (raw_encap->size > 3460 (sizeof(struct rte_flow_item_eth) + 3461 sizeof(struct rte_flow_item_ipv4))) { 3462 memcpy(actions_tx, actions, 3463 sizeof(struct rte_flow_action)); 3464 actions_tx++; 3465 encap = 1; 3466 } else { 3467 rte_memcpy(actions_rx, actions, 3468 sizeof(struct rte_flow_action)); 3469 actions_rx++; 3470 } 3471 break; 3472 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3473 raw_decap = actions->conf; 3474 if (raw_decap->size < 3475 (sizeof(struct rte_flow_item_eth) + 3476 sizeof(struct rte_flow_item_ipv4))) { 3477 memcpy(actions_tx, actions, 3478 sizeof(struct rte_flow_action)); 3479 actions_tx++; 3480 } else { 3481 rte_memcpy(actions_rx, actions, 3482 sizeof(struct rte_flow_action)); 3483 actions_rx++; 3484 } 3485 break; 3486 default: 3487 rte_memcpy(actions_rx, actions, 3488 sizeof(struct rte_flow_action)); 3489 actions_rx++; 3490 break; 3491 } 3492 } 3493 /* Add set meta action and end action for the Rx flow. */ 3494 tag_action = actions_rx; 3495 tag_action->type = (enum rte_flow_action_type) 3496 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3497 actions_rx++; 3498 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3499 actions_rx++; 3500 set_tag = (void *)actions_rx; 3501 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3502 MLX5_ASSERT(set_tag->id > REG_NON); 3503 set_tag->data = *flow_id; 3504 tag_action->conf = set_tag; 3505 /* Create Tx item list. */ 3506 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3507 addr = (void *)&pattern_tx[2]; 3508 item = pattern_tx; 3509 item->type = (enum rte_flow_item_type) 3510 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3511 tag_item = (void *)addr; 3512 tag_item->data = *flow_id; 3513 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3514 MLX5_ASSERT(set_tag->id > REG_NON); 3515 item->spec = tag_item; 3516 addr += sizeof(struct mlx5_rte_flow_item_tag); 3517 tag_item = (void *)addr; 3518 tag_item->data = UINT32_MAX; 3519 tag_item->id = UINT16_MAX; 3520 item->mask = tag_item; 3521 item->last = NULL; 3522 item++; 3523 item->type = RTE_FLOW_ITEM_TYPE_END; 3524 return 0; 3525 } 3526 3527 /** 3528 * The last stage of splitting chain, just creates the subflow 3529 * without any modification. 3530 * 3531 * @param[in] dev 3532 * Pointer to Ethernet device. 3533 * @param[in] flow 3534 * Parent flow structure pointer. 3535 * @param[in, out] sub_flow 3536 * Pointer to return the created subflow, may be NULL. 3537 * @param[in] prefix_layers 3538 * Prefix subflow layers, may be 0. 3539 * @param[in] attr 3540 * Flow rule attributes. 3541 * @param[in] items 3542 * Pattern specification (list terminated by the END pattern item). 3543 * @param[in] actions 3544 * Associated actions (list terminated by the END action). 3545 * @param[in] external 3546 * This flow rule is created by request external to PMD. 3547 * @param[in] flow_idx 3548 * This memory pool index to the flow. 3549 * @param[out] error 3550 * Perform verbose error reporting if not NULL. 3551 * @return 3552 * 0 on success, negative value otherwise 3553 */ 3554 static int 3555 flow_create_split_inner(struct rte_eth_dev *dev, 3556 struct rte_flow *flow, 3557 struct mlx5_flow **sub_flow, 3558 uint64_t prefix_layers, 3559 const struct rte_flow_attr *attr, 3560 const struct rte_flow_item items[], 3561 const struct rte_flow_action actions[], 3562 bool external, uint32_t flow_idx, 3563 struct rte_flow_error *error) 3564 { 3565 struct mlx5_flow *dev_flow; 3566 3567 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 3568 flow_idx, error); 3569 if (!dev_flow) 3570 return -rte_errno; 3571 dev_flow->flow = flow; 3572 dev_flow->external = external; 3573 /* Subflow object was created, we must include one in the list. */ 3574 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 3575 dev_flow->handle, next); 3576 /* 3577 * If dev_flow is as one of the suffix flow, some actions in suffix 3578 * flow may need some user defined item layer flags. 3579 */ 3580 if (prefix_layers) 3581 dev_flow->handle->layers = prefix_layers; 3582 if (sub_flow) 3583 *sub_flow = dev_flow; 3584 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3585 } 3586 3587 /** 3588 * Split the meter flow. 3589 * 3590 * As meter flow will split to three sub flow, other than meter 3591 * action, the other actions make sense to only meter accepts 3592 * the packet. If it need to be dropped, no other additional 3593 * actions should be take. 3594 * 3595 * One kind of special action which decapsulates the L3 tunnel 3596 * header will be in the prefix sub flow, as not to take the 3597 * L3 tunnel header into account. 3598 * 3599 * @param dev 3600 * Pointer to Ethernet device. 3601 * @param[in] items 3602 * Pattern specification (list terminated by the END pattern item). 3603 * @param[out] sfx_items 3604 * Suffix flow match items (list terminated by the END pattern item). 3605 * @param[in] actions 3606 * Associated actions (list terminated by the END action). 3607 * @param[out] actions_sfx 3608 * Suffix flow actions. 3609 * @param[out] actions_pre 3610 * Prefix flow actions. 3611 * @param[out] pattern_sfx 3612 * The pattern items for the suffix flow. 3613 * @param[out] tag_sfx 3614 * Pointer to suffix flow tag. 3615 * 3616 * @return 3617 * 0 on success. 3618 */ 3619 static int 3620 flow_meter_split_prep(struct rte_eth_dev *dev, 3621 const struct rte_flow_item items[], 3622 struct rte_flow_item sfx_items[], 3623 const struct rte_flow_action actions[], 3624 struct rte_flow_action actions_sfx[], 3625 struct rte_flow_action actions_pre[]) 3626 { 3627 struct rte_flow_action *tag_action = NULL; 3628 struct rte_flow_item *tag_item; 3629 struct mlx5_rte_flow_action_set_tag *set_tag; 3630 struct rte_flow_error error; 3631 const struct rte_flow_action_raw_encap *raw_encap; 3632 const struct rte_flow_action_raw_decap *raw_decap; 3633 struct mlx5_rte_flow_item_tag *tag_spec; 3634 struct mlx5_rte_flow_item_tag *tag_mask; 3635 uint32_t tag_id; 3636 bool copy_vlan = false; 3637 3638 /* Prepare the actions for prefix and suffix flow. */ 3639 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3640 struct rte_flow_action **action_cur = NULL; 3641 3642 switch (actions->type) { 3643 case RTE_FLOW_ACTION_TYPE_METER: 3644 /* Add the extra tag action first. */ 3645 tag_action = actions_pre; 3646 tag_action->type = (enum rte_flow_action_type) 3647 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3648 actions_pre++; 3649 action_cur = &actions_pre; 3650 break; 3651 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3652 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3653 action_cur = &actions_pre; 3654 break; 3655 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3656 raw_encap = actions->conf; 3657 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 3658 action_cur = &actions_pre; 3659 break; 3660 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3661 raw_decap = actions->conf; 3662 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3663 action_cur = &actions_pre; 3664 break; 3665 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3666 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3667 copy_vlan = true; 3668 break; 3669 default: 3670 break; 3671 } 3672 if (!action_cur) 3673 action_cur = &actions_sfx; 3674 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 3675 (*action_cur)++; 3676 } 3677 /* Add end action to the actions. */ 3678 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3679 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3680 actions_pre++; 3681 /* Set the tag. */ 3682 set_tag = (void *)actions_pre; 3683 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3684 /* 3685 * Get the id from the qrss_pool to make qrss share the id with meter. 3686 */ 3687 tag_id = flow_qrss_get_id(dev); 3688 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 3689 assert(tag_action); 3690 tag_action->conf = set_tag; 3691 /* Prepare the suffix subflow items. */ 3692 tag_item = sfx_items++; 3693 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 3694 int item_type = items->type; 3695 3696 switch (item_type) { 3697 case RTE_FLOW_ITEM_TYPE_PORT_ID: 3698 memcpy(sfx_items, items, sizeof(*sfx_items)); 3699 sfx_items++; 3700 break; 3701 case RTE_FLOW_ITEM_TYPE_VLAN: 3702 if (copy_vlan) { 3703 memcpy(sfx_items, items, sizeof(*sfx_items)); 3704 /* 3705 * Convert to internal match item, it is used 3706 * for vlan push and set vid. 3707 */ 3708 sfx_items->type = (enum rte_flow_item_type) 3709 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 3710 sfx_items++; 3711 } 3712 break; 3713 default: 3714 break; 3715 } 3716 } 3717 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 3718 sfx_items++; 3719 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 3720 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 3721 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3722 tag_mask = tag_spec + 1; 3723 tag_mask->data = 0xffffff00; 3724 tag_item->type = (enum rte_flow_item_type) 3725 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3726 tag_item->spec = tag_spec; 3727 tag_item->last = NULL; 3728 tag_item->mask = tag_mask; 3729 return tag_id; 3730 } 3731 3732 /** 3733 * Split action list having QUEUE/RSS for metadata register copy. 3734 * 3735 * Once Q/RSS action is detected in user's action list, the flow action 3736 * should be split in order to copy metadata registers, which will happen in 3737 * RX_CP_TBL like, 3738 * - CQE->flow_tag := reg_c[1] (MARK) 3739 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3740 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3741 * This is because the last action of each flow must be a terminal action 3742 * (QUEUE, RSS or DROP). 3743 * 3744 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3745 * stored and kept in the mlx5_flow structure per each sub_flow. 3746 * 3747 * The Q/RSS action is replaced with, 3748 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3749 * And the following JUMP action is added at the end, 3750 * - JUMP, to RX_CP_TBL. 3751 * 3752 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3753 * flow_create_split_metadata() routine. The flow will look like, 3754 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3755 * 3756 * @param dev 3757 * Pointer to Ethernet device. 3758 * @param[out] split_actions 3759 * Pointer to store split actions to jump to CP_TBL. 3760 * @param[in] actions 3761 * Pointer to the list of original flow actions. 3762 * @param[in] qrss 3763 * Pointer to the Q/RSS action. 3764 * @param[in] actions_n 3765 * Number of original actions. 3766 * @param[out] error 3767 * Perform verbose error reporting if not NULL. 3768 * 3769 * @return 3770 * non-zero unique flow_id on success, otherwise 0 and 3771 * error/rte_error are set. 3772 */ 3773 static uint32_t 3774 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3775 struct rte_flow_action *split_actions, 3776 const struct rte_flow_action *actions, 3777 const struct rte_flow_action *qrss, 3778 int actions_n, struct rte_flow_error *error) 3779 { 3780 struct mlx5_rte_flow_action_set_tag *set_tag; 3781 struct rte_flow_action_jump *jump; 3782 const int qrss_idx = qrss - actions; 3783 uint32_t flow_id = 0; 3784 int ret = 0; 3785 3786 /* 3787 * Given actions will be split 3788 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3789 * - Add jump to mreg CP_TBL. 3790 * As a result, there will be one more action. 3791 */ 3792 ++actions_n; 3793 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3794 set_tag = (void *)(split_actions + actions_n); 3795 /* 3796 * If tag action is not set to void(it means we are not the meter 3797 * suffix flow), add the tag action. Since meter suffix flow already 3798 * has the tag added. 3799 */ 3800 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3801 /* 3802 * Allocate the new subflow ID. This one is unique within 3803 * device and not shared with representors. Otherwise, 3804 * we would have to resolve multi-thread access synch 3805 * issue. Each flow on the shared device is appended 3806 * with source vport identifier, so the resulting 3807 * flows will be unique in the shared (by master and 3808 * representors) domain even if they have coinciding 3809 * IDs. 3810 */ 3811 flow_id = flow_qrss_get_id(dev); 3812 if (!flow_id) 3813 return rte_flow_error_set(error, ENOMEM, 3814 RTE_FLOW_ERROR_TYPE_ACTION, 3815 NULL, "can't allocate id " 3816 "for split Q/RSS subflow"); 3817 /* Internal SET_TAG action to set flow ID. */ 3818 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3819 .data = flow_id, 3820 }; 3821 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3822 if (ret < 0) 3823 return ret; 3824 set_tag->id = ret; 3825 /* Construct new actions array. */ 3826 /* Replace QUEUE/RSS action. */ 3827 split_actions[qrss_idx] = (struct rte_flow_action){ 3828 .type = (enum rte_flow_action_type) 3829 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3830 .conf = set_tag, 3831 }; 3832 } 3833 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3834 jump = (void *)(set_tag + 1); 3835 *jump = (struct rte_flow_action_jump){ 3836 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3837 }; 3838 split_actions[actions_n - 2] = (struct rte_flow_action){ 3839 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3840 .conf = jump, 3841 }; 3842 split_actions[actions_n - 1] = (struct rte_flow_action){ 3843 .type = RTE_FLOW_ACTION_TYPE_END, 3844 }; 3845 return flow_id; 3846 } 3847 3848 /** 3849 * Extend the given action list for Tx metadata copy. 3850 * 3851 * Copy the given action list to the ext_actions and add flow metadata register 3852 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3853 * 3854 * @param[out] ext_actions 3855 * Pointer to the extended action list. 3856 * @param[in] actions 3857 * Pointer to the list of actions. 3858 * @param[in] actions_n 3859 * Number of actions in the list. 3860 * @param[out] error 3861 * Perform verbose error reporting if not NULL. 3862 * @param[in] encap_idx 3863 * The encap action inndex. 3864 * 3865 * @return 3866 * 0 on success, negative value otherwise 3867 */ 3868 static int 3869 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3870 struct rte_flow_action *ext_actions, 3871 const struct rte_flow_action *actions, 3872 int actions_n, struct rte_flow_error *error, 3873 int encap_idx) 3874 { 3875 struct mlx5_flow_action_copy_mreg *cp_mreg = 3876 (struct mlx5_flow_action_copy_mreg *) 3877 (ext_actions + actions_n + 1); 3878 int ret; 3879 3880 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3881 if (ret < 0) 3882 return ret; 3883 cp_mreg->dst = ret; 3884 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3885 if (ret < 0) 3886 return ret; 3887 cp_mreg->src = ret; 3888 if (encap_idx != 0) 3889 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 3890 if (encap_idx == actions_n - 1) { 3891 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3892 .type = (enum rte_flow_action_type) 3893 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3894 .conf = cp_mreg, 3895 }; 3896 ext_actions[actions_n] = (struct rte_flow_action){ 3897 .type = RTE_FLOW_ACTION_TYPE_END, 3898 }; 3899 } else { 3900 ext_actions[encap_idx] = (struct rte_flow_action){ 3901 .type = (enum rte_flow_action_type) 3902 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3903 .conf = cp_mreg, 3904 }; 3905 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 3906 sizeof(*ext_actions) * (actions_n - encap_idx)); 3907 } 3908 return 0; 3909 } 3910 3911 /** 3912 * The splitting for metadata feature. 3913 * 3914 * - Q/RSS action on NIC Rx should be split in order to pass by 3915 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3916 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3917 * 3918 * - All the actions on NIC Tx should have a mreg copy action to 3919 * copy reg_a from WQE to reg_c[0]. 3920 * 3921 * @param dev 3922 * Pointer to Ethernet device. 3923 * @param[in] flow 3924 * Parent flow structure pointer. 3925 * @param[in] prefix_layers 3926 * Prefix flow layer flags. 3927 * @param[in] attr 3928 * Flow rule attributes. 3929 * @param[in] items 3930 * Pattern specification (list terminated by the END pattern item). 3931 * @param[in] actions 3932 * Associated actions (list terminated by the END action). 3933 * @param[in] external 3934 * This flow rule is created by request external to PMD. 3935 * @param[in] flow_idx 3936 * This memory pool index to the flow. 3937 * @param[out] error 3938 * Perform verbose error reporting if not NULL. 3939 * @return 3940 * 0 on success, negative value otherwise 3941 */ 3942 static int 3943 flow_create_split_metadata(struct rte_eth_dev *dev, 3944 struct rte_flow *flow, 3945 uint64_t prefix_layers, 3946 const struct rte_flow_attr *attr, 3947 const struct rte_flow_item items[], 3948 const struct rte_flow_action actions[], 3949 bool external, uint32_t flow_idx, 3950 struct rte_flow_error *error) 3951 { 3952 struct mlx5_priv *priv = dev->data->dev_private; 3953 struct mlx5_dev_config *config = &priv->config; 3954 const struct rte_flow_action *qrss = NULL; 3955 struct rte_flow_action *ext_actions = NULL; 3956 struct mlx5_flow *dev_flow = NULL; 3957 uint32_t qrss_id = 0; 3958 int mtr_sfx = 0; 3959 size_t act_size; 3960 int actions_n; 3961 int encap_idx; 3962 int ret; 3963 3964 /* Check whether extensive metadata feature is engaged. */ 3965 if (!config->dv_flow_en || 3966 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3967 !mlx5_flow_ext_mreg_supported(dev)) 3968 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 3969 attr, items, actions, external, 3970 flow_idx, error); 3971 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 3972 &encap_idx); 3973 if (qrss) { 3974 /* Exclude hairpin flows from splitting. */ 3975 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3976 const struct rte_flow_action_queue *queue; 3977 3978 queue = qrss->conf; 3979 if (mlx5_rxq_get_type(dev, queue->index) == 3980 MLX5_RXQ_TYPE_HAIRPIN) 3981 qrss = NULL; 3982 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3983 const struct rte_flow_action_rss *rss; 3984 3985 rss = qrss->conf; 3986 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3987 MLX5_RXQ_TYPE_HAIRPIN) 3988 qrss = NULL; 3989 } 3990 } 3991 if (qrss) { 3992 /* Check if it is in meter suffix table. */ 3993 mtr_sfx = attr->group == (attr->transfer ? 3994 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3995 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3996 /* 3997 * Q/RSS action on NIC Rx should be split in order to pass by 3998 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3999 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4000 */ 4001 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4002 sizeof(struct rte_flow_action_set_tag) + 4003 sizeof(struct rte_flow_action_jump); 4004 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4005 SOCKET_ID_ANY); 4006 if (!ext_actions) 4007 return rte_flow_error_set(error, ENOMEM, 4008 RTE_FLOW_ERROR_TYPE_ACTION, 4009 NULL, "no memory to split " 4010 "metadata flow"); 4011 /* 4012 * If we are the suffix flow of meter, tag already exist. 4013 * Set the tag action to void. 4014 */ 4015 if (mtr_sfx) 4016 ext_actions[qrss - actions].type = 4017 RTE_FLOW_ACTION_TYPE_VOID; 4018 else 4019 ext_actions[qrss - actions].type = 4020 (enum rte_flow_action_type) 4021 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4022 /* 4023 * Create the new actions list with removed Q/RSS action 4024 * and appended set tag and jump to register copy table 4025 * (RX_CP_TBL). We should preallocate unique tag ID here 4026 * in advance, because it is needed for set tag action. 4027 */ 4028 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 4029 qrss, actions_n, error); 4030 if (!mtr_sfx && !qrss_id) { 4031 ret = -rte_errno; 4032 goto exit; 4033 } 4034 } else if (attr->egress && !attr->transfer) { 4035 /* 4036 * All the actions on NIC Tx should have a metadata register 4037 * copy action to copy reg_a from WQE to reg_c[meta] 4038 */ 4039 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4040 sizeof(struct mlx5_flow_action_copy_mreg); 4041 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4042 SOCKET_ID_ANY); 4043 if (!ext_actions) 4044 return rte_flow_error_set(error, ENOMEM, 4045 RTE_FLOW_ERROR_TYPE_ACTION, 4046 NULL, "no memory to split " 4047 "metadata flow"); 4048 /* Create the action list appended with copy register. */ 4049 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4050 actions_n, error, encap_idx); 4051 if (ret < 0) 4052 goto exit; 4053 } 4054 /* Add the unmodified original or prefix subflow. */ 4055 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, 4056 items, ext_actions ? ext_actions : 4057 actions, external, flow_idx, error); 4058 if (ret < 0) 4059 goto exit; 4060 MLX5_ASSERT(dev_flow); 4061 if (qrss) { 4062 const struct rte_flow_attr q_attr = { 4063 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4064 .ingress = 1, 4065 }; 4066 /* Internal PMD action to set register. */ 4067 struct mlx5_rte_flow_item_tag q_tag_spec = { 4068 .data = qrss_id, 4069 .id = REG_NON, 4070 }; 4071 struct rte_flow_item q_items[] = { 4072 { 4073 .type = (enum rte_flow_item_type) 4074 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4075 .spec = &q_tag_spec, 4076 .last = NULL, 4077 .mask = NULL, 4078 }, 4079 { 4080 .type = RTE_FLOW_ITEM_TYPE_END, 4081 }, 4082 }; 4083 struct rte_flow_action q_actions[] = { 4084 { 4085 .type = qrss->type, 4086 .conf = qrss->conf, 4087 }, 4088 { 4089 .type = RTE_FLOW_ACTION_TYPE_END, 4090 }, 4091 }; 4092 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4093 4094 /* 4095 * Configure the tag item only if there is no meter subflow. 4096 * Since tag is already marked in the meter suffix subflow 4097 * we can just use the meter suffix items as is. 4098 */ 4099 if (qrss_id) { 4100 /* Not meter subflow. */ 4101 MLX5_ASSERT(!mtr_sfx); 4102 /* 4103 * Put unique id in prefix flow due to it is destroyed 4104 * after suffix flow and id will be freed after there 4105 * is no actual flows with this id and identifier 4106 * reallocation becomes possible (for example, for 4107 * other flows in other threads). 4108 */ 4109 dev_flow->handle->split_flow_id = qrss_id; 4110 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4111 error); 4112 if (ret < 0) 4113 goto exit; 4114 q_tag_spec.id = ret; 4115 } 4116 dev_flow = NULL; 4117 /* Add suffix subflow to execute Q/RSS. */ 4118 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 4119 &q_attr, mtr_sfx ? items : 4120 q_items, q_actions, 4121 external, flow_idx, error); 4122 if (ret < 0) 4123 goto exit; 4124 /* qrss ID should be freed if failed. */ 4125 qrss_id = 0; 4126 MLX5_ASSERT(dev_flow); 4127 } 4128 4129 exit: 4130 /* 4131 * We do not destroy the partially created sub_flows in case of error. 4132 * These ones are included into parent flow list and will be destroyed 4133 * by flow_drv_destroy. 4134 */ 4135 flow_qrss_free_id(dev, qrss_id); 4136 mlx5_free(ext_actions); 4137 return ret; 4138 } 4139 4140 /** 4141 * The splitting for meter feature. 4142 * 4143 * - The meter flow will be split to two flows as prefix and 4144 * suffix flow. The packets make sense only it pass the prefix 4145 * meter action. 4146 * 4147 * - Reg_C_5 is used for the packet to match betweend prefix and 4148 * suffix flow. 4149 * 4150 * @param dev 4151 * Pointer to Ethernet device. 4152 * @param[in] flow 4153 * Parent flow structure pointer. 4154 * @param[in] attr 4155 * Flow rule attributes. 4156 * @param[in] items 4157 * Pattern specification (list terminated by the END pattern item). 4158 * @param[in] actions 4159 * Associated actions (list terminated by the END action). 4160 * @param[in] external 4161 * This flow rule is created by request external to PMD. 4162 * @param[in] flow_idx 4163 * This memory pool index to the flow. 4164 * @param[out] error 4165 * Perform verbose error reporting if not NULL. 4166 * @return 4167 * 0 on success, negative value otherwise 4168 */ 4169 static int 4170 flow_create_split_meter(struct rte_eth_dev *dev, 4171 struct rte_flow *flow, 4172 const struct rte_flow_attr *attr, 4173 const struct rte_flow_item items[], 4174 const struct rte_flow_action actions[], 4175 bool external, uint32_t flow_idx, 4176 struct rte_flow_error *error) 4177 { 4178 struct mlx5_priv *priv = dev->data->dev_private; 4179 struct rte_flow_action *sfx_actions = NULL; 4180 struct rte_flow_action *pre_actions = NULL; 4181 struct rte_flow_item *sfx_items = NULL; 4182 struct mlx5_flow *dev_flow = NULL; 4183 struct rte_flow_attr sfx_attr = *attr; 4184 uint32_t mtr = 0; 4185 uint32_t mtr_tag_id = 0; 4186 size_t act_size; 4187 size_t item_size; 4188 int actions_n = 0; 4189 int ret; 4190 4191 if (priv->mtr_en) 4192 actions_n = flow_check_meter_action(actions, &mtr); 4193 if (mtr) { 4194 /* The five prefix actions: meter, decap, encap, tag, end. */ 4195 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4196 sizeof(struct mlx5_rte_flow_action_set_tag); 4197 /* tag, vlan, port id, end. */ 4198 #define METER_SUFFIX_ITEM 4 4199 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4200 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4201 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 4202 0, SOCKET_ID_ANY); 4203 if (!sfx_actions) 4204 return rte_flow_error_set(error, ENOMEM, 4205 RTE_FLOW_ERROR_TYPE_ACTION, 4206 NULL, "no memory to split " 4207 "meter flow"); 4208 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4209 act_size); 4210 pre_actions = sfx_actions + actions_n; 4211 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4212 actions, sfx_actions, 4213 pre_actions); 4214 if (!mtr_tag_id) { 4215 ret = -rte_errno; 4216 goto exit; 4217 } 4218 /* Add the prefix subflow. */ 4219 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, 4220 items, pre_actions, external, 4221 flow_idx, error); 4222 if (ret) { 4223 ret = -rte_errno; 4224 goto exit; 4225 } 4226 dev_flow->handle->split_flow_id = mtr_tag_id; 4227 /* Setting the sfx group atrr. */ 4228 sfx_attr.group = sfx_attr.transfer ? 4229 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4230 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4231 } 4232 /* Add the prefix subflow. */ 4233 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4234 flow_get_prefix_layer_flags(dev_flow) : 4235 0, &sfx_attr, 4236 sfx_items ? sfx_items : items, 4237 sfx_actions ? sfx_actions : actions, 4238 external, flow_idx, error); 4239 exit: 4240 if (sfx_actions) 4241 mlx5_free(sfx_actions); 4242 return ret; 4243 } 4244 4245 /** 4246 * Split the flow to subflow set. The splitters might be linked 4247 * in the chain, like this: 4248 * flow_create_split_outer() calls: 4249 * flow_create_split_meter() calls: 4250 * flow_create_split_metadata(meter_subflow_0) calls: 4251 * flow_create_split_inner(metadata_subflow_0) 4252 * flow_create_split_inner(metadata_subflow_1) 4253 * flow_create_split_inner(metadata_subflow_2) 4254 * flow_create_split_metadata(meter_subflow_1) calls: 4255 * flow_create_split_inner(metadata_subflow_0) 4256 * flow_create_split_inner(metadata_subflow_1) 4257 * flow_create_split_inner(metadata_subflow_2) 4258 * 4259 * This provide flexible way to add new levels of flow splitting. 4260 * The all of successfully created subflows are included to the 4261 * parent flow dev_flow list. 4262 * 4263 * @param dev 4264 * Pointer to Ethernet device. 4265 * @param[in] flow 4266 * Parent flow structure pointer. 4267 * @param[in] attr 4268 * Flow rule attributes. 4269 * @param[in] items 4270 * Pattern specification (list terminated by the END pattern item). 4271 * @param[in] actions 4272 * Associated actions (list terminated by the END action). 4273 * @param[in] external 4274 * This flow rule is created by request external to PMD. 4275 * @param[in] flow_idx 4276 * This memory pool index to the flow. 4277 * @param[out] error 4278 * Perform verbose error reporting if not NULL. 4279 * @return 4280 * 0 on success, negative value otherwise 4281 */ 4282 static int 4283 flow_create_split_outer(struct rte_eth_dev *dev, 4284 struct rte_flow *flow, 4285 const struct rte_flow_attr *attr, 4286 const struct rte_flow_item items[], 4287 const struct rte_flow_action actions[], 4288 bool external, uint32_t flow_idx, 4289 struct rte_flow_error *error) 4290 { 4291 int ret; 4292 4293 ret = flow_create_split_meter(dev, flow, attr, items, 4294 actions, external, flow_idx, error); 4295 MLX5_ASSERT(ret <= 0); 4296 return ret; 4297 } 4298 4299 /** 4300 * Create a flow and add it to @p list. 4301 * 4302 * @param dev 4303 * Pointer to Ethernet device. 4304 * @param list 4305 * Pointer to a TAILQ flow list. If this parameter NULL, 4306 * no list insertion occurred, flow is just created, 4307 * this is caller's responsibility to track the 4308 * created flow. 4309 * @param[in] attr 4310 * Flow rule attributes. 4311 * @param[in] items 4312 * Pattern specification (list terminated by the END pattern item). 4313 * @param[in] actions 4314 * Associated actions (list terminated by the END action). 4315 * @param[in] external 4316 * This flow rule is created by request external to PMD. 4317 * @param[out] error 4318 * Perform verbose error reporting if not NULL. 4319 * 4320 * @return 4321 * A flow index on success, 0 otherwise and rte_errno is set. 4322 */ 4323 static uint32_t 4324 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4325 const struct rte_flow_attr *attr, 4326 const struct rte_flow_item items[], 4327 const struct rte_flow_action actions[], 4328 bool external, struct rte_flow_error *error) 4329 { 4330 struct mlx5_priv *priv = dev->data->dev_private; 4331 struct rte_flow *flow = NULL; 4332 struct mlx5_flow *dev_flow; 4333 const struct rte_flow_action_rss *rss; 4334 union { 4335 struct rte_flow_expand_rss buf; 4336 uint8_t buffer[2048]; 4337 } expand_buffer; 4338 union { 4339 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4340 uint8_t buffer[2048]; 4341 } actions_rx; 4342 union { 4343 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4344 uint8_t buffer[2048]; 4345 } actions_hairpin_tx; 4346 union { 4347 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4348 uint8_t buffer[2048]; 4349 } items_tx; 4350 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4351 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 4352 priv->rss_desc)[!!priv->flow_idx]; 4353 const struct rte_flow_action *p_actions_rx = actions; 4354 uint32_t i; 4355 uint32_t idx = 0; 4356 int hairpin_flow; 4357 uint32_t hairpin_id = 0; 4358 struct rte_flow_attr attr_tx = { .priority = 0 }; 4359 int ret; 4360 4361 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4362 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 4363 external, hairpin_flow, error); 4364 if (ret < 0) 4365 return 0; 4366 if (hairpin_flow > 0) { 4367 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4368 rte_errno = EINVAL; 4369 return 0; 4370 } 4371 flow_hairpin_split(dev, actions, actions_rx.actions, 4372 actions_hairpin_tx.actions, items_tx.items, 4373 &hairpin_id); 4374 p_actions_rx = actions_rx.actions; 4375 } 4376 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 4377 if (!flow) { 4378 rte_errno = ENOMEM; 4379 goto error_before_flow; 4380 } 4381 flow->drv_type = flow_get_drv_type(dev, attr); 4382 if (hairpin_id != 0) 4383 flow->hairpin_flow_id = hairpin_id; 4384 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4385 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4386 memset(rss_desc, 0, sizeof(*rss_desc)); 4387 rss = flow_get_rss_action(p_actions_rx); 4388 if (rss) { 4389 /* 4390 * The following information is required by 4391 * mlx5_flow_hashfields_adjust() in advance. 4392 */ 4393 rss_desc->level = rss->level; 4394 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4395 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 4396 } 4397 flow->dev_handles = 0; 4398 if (rss && rss->types) { 4399 unsigned int graph_root; 4400 4401 graph_root = find_graph_root(items, rss->level); 4402 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4403 items, rss->types, 4404 mlx5_support_expansion, 4405 graph_root); 4406 MLX5_ASSERT(ret > 0 && 4407 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4408 } else { 4409 buf->entries = 1; 4410 buf->entry[0].pattern = (void *)(uintptr_t)items; 4411 } 4412 /* 4413 * Record the start index when there is a nested call. All sub-flows 4414 * need to be translated before another calling. 4415 * No need to use ping-pong buffer to save memory here. 4416 */ 4417 if (priv->flow_idx) { 4418 MLX5_ASSERT(!priv->flow_nested_idx); 4419 priv->flow_nested_idx = priv->flow_idx; 4420 } 4421 for (i = 0; i < buf->entries; ++i) { 4422 /* 4423 * The splitter may create multiple dev_flows, 4424 * depending on configuration. In the simplest 4425 * case it just creates unmodified original flow. 4426 */ 4427 ret = flow_create_split_outer(dev, flow, attr, 4428 buf->entry[i].pattern, 4429 p_actions_rx, external, idx, 4430 error); 4431 if (ret < 0) 4432 goto error; 4433 } 4434 /* Create the tx flow. */ 4435 if (hairpin_flow) { 4436 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4437 attr_tx.ingress = 0; 4438 attr_tx.egress = 1; 4439 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 4440 actions_hairpin_tx.actions, 4441 idx, error); 4442 if (!dev_flow) 4443 goto error; 4444 dev_flow->flow = flow; 4445 dev_flow->external = 0; 4446 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4447 dev_flow->handle, next); 4448 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4449 items_tx.items, 4450 actions_hairpin_tx.actions, error); 4451 if (ret < 0) 4452 goto error; 4453 } 4454 /* 4455 * Update the metadata register copy table. If extensive 4456 * metadata feature is enabled and registers are supported 4457 * we might create the extra rte_flow for each unique 4458 * MARK/FLAG action ID. 4459 * 4460 * The table is updated for ingress Flows only, because 4461 * the egress Flows belong to the different device and 4462 * copy table should be updated in peer NIC Rx domain. 4463 */ 4464 if (attr->ingress && 4465 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4466 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4467 if (ret) 4468 goto error; 4469 } 4470 /* 4471 * If the flow is external (from application) OR device is started, then 4472 * the flow will be applied immediately. 4473 */ 4474 if (external || dev->data->dev_started) { 4475 ret = flow_drv_apply(dev, flow, error); 4476 if (ret < 0) 4477 goto error; 4478 } 4479 if (list) 4480 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 4481 flow, next); 4482 flow_rxq_flags_set(dev, flow); 4483 /* Nested flow creation index recovery. */ 4484 priv->flow_idx = priv->flow_nested_idx; 4485 if (priv->flow_nested_idx) 4486 priv->flow_nested_idx = 0; 4487 return idx; 4488 error: 4489 MLX5_ASSERT(flow); 4490 ret = rte_errno; /* Save rte_errno before cleanup. */ 4491 flow_mreg_del_copy_action(dev, flow); 4492 flow_drv_destroy(dev, flow); 4493 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 4494 rte_errno = ret; /* Restore rte_errno. */ 4495 error_before_flow: 4496 ret = rte_errno; 4497 if (hairpin_id) 4498 mlx5_flow_id_release(priv->sh->flow_id_pool, 4499 hairpin_id); 4500 rte_errno = ret; 4501 priv->flow_idx = priv->flow_nested_idx; 4502 if (priv->flow_nested_idx) 4503 priv->flow_nested_idx = 0; 4504 return 0; 4505 } 4506 4507 /** 4508 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4509 * incoming packets to table 1. 4510 * 4511 * Other flow rules, requested for group n, will be created in 4512 * e-switch table n+1. 4513 * Jump action to e-switch group n will be created to group n+1. 4514 * 4515 * Used when working in switchdev mode, to utilise advantages of table 1 4516 * and above. 4517 * 4518 * @param dev 4519 * Pointer to Ethernet device. 4520 * 4521 * @return 4522 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4523 */ 4524 struct rte_flow * 4525 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4526 { 4527 const struct rte_flow_attr attr = { 4528 .group = 0, 4529 .priority = 0, 4530 .ingress = 1, 4531 .egress = 0, 4532 .transfer = 1, 4533 }; 4534 const struct rte_flow_item pattern = { 4535 .type = RTE_FLOW_ITEM_TYPE_END, 4536 }; 4537 struct rte_flow_action_jump jump = { 4538 .group = 1, 4539 }; 4540 const struct rte_flow_action actions[] = { 4541 { 4542 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4543 .conf = &jump, 4544 }, 4545 { 4546 .type = RTE_FLOW_ACTION_TYPE_END, 4547 }, 4548 }; 4549 struct mlx5_priv *priv = dev->data->dev_private; 4550 struct rte_flow_error error; 4551 4552 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 4553 &attr, &pattern, 4554 actions, false, &error); 4555 } 4556 4557 /** 4558 * Validate a flow supported by the NIC. 4559 * 4560 * @see rte_flow_validate() 4561 * @see rte_flow_ops 4562 */ 4563 int 4564 mlx5_flow_validate(struct rte_eth_dev *dev, 4565 const struct rte_flow_attr *attr, 4566 const struct rte_flow_item items[], 4567 const struct rte_flow_action actions[], 4568 struct rte_flow_error *error) 4569 { 4570 int hairpin_flow; 4571 4572 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4573 return flow_drv_validate(dev, attr, items, actions, 4574 true, hairpin_flow, error); 4575 } 4576 4577 /** 4578 * Create a flow. 4579 * 4580 * @see rte_flow_create() 4581 * @see rte_flow_ops 4582 */ 4583 struct rte_flow * 4584 mlx5_flow_create(struct rte_eth_dev *dev, 4585 const struct rte_flow_attr *attr, 4586 const struct rte_flow_item items[], 4587 const struct rte_flow_action actions[], 4588 struct rte_flow_error *error) 4589 { 4590 struct mlx5_priv *priv = dev->data->dev_private; 4591 4592 /* 4593 * If the device is not started yet, it is not allowed to created a 4594 * flow from application. PMD default flows and traffic control flows 4595 * are not affected. 4596 */ 4597 if (unlikely(!dev->data->dev_started)) { 4598 DRV_LOG(DEBUG, "port %u is not started when " 4599 "inserting a flow", dev->data->port_id); 4600 rte_flow_error_set(error, ENODEV, 4601 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4602 NULL, 4603 "port not started"); 4604 return NULL; 4605 } 4606 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 4607 attr, items, actions, true, error); 4608 } 4609 4610 /** 4611 * Destroy a flow in a list. 4612 * 4613 * @param dev 4614 * Pointer to Ethernet device. 4615 * @param list 4616 * Pointer to the Indexed flow list. If this parameter NULL, 4617 * there is no flow removal from the list. Be noted that as 4618 * flow is add to the indexed list, memory of the indexed 4619 * list points to maybe changed as flow destroyed. 4620 * @param[in] flow_idx 4621 * Index of flow to destroy. 4622 */ 4623 static void 4624 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4625 uint32_t flow_idx) 4626 { 4627 struct mlx5_priv *priv = dev->data->dev_private; 4628 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 4629 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 4630 [MLX5_IPOOL_RTE_FLOW], flow_idx); 4631 4632 if (!flow) 4633 return; 4634 /* 4635 * Update RX queue flags only if port is started, otherwise it is 4636 * already clean. 4637 */ 4638 if (dev->data->dev_started) 4639 flow_rxq_flags_trim(dev, flow); 4640 if (flow->hairpin_flow_id) 4641 mlx5_flow_id_release(priv->sh->flow_id_pool, 4642 flow->hairpin_flow_id); 4643 flow_drv_destroy(dev, flow); 4644 if (list) 4645 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 4646 flow_idx, flow, next); 4647 flow_mreg_del_copy_action(dev, flow); 4648 if (flow->fdir) { 4649 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 4650 if (priv_fdir_flow->rix_flow == flow_idx) 4651 break; 4652 } 4653 if (priv_fdir_flow) { 4654 LIST_REMOVE(priv_fdir_flow, next); 4655 mlx5_free(priv_fdir_flow->fdir); 4656 mlx5_free(priv_fdir_flow); 4657 } 4658 } 4659 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 4660 } 4661 4662 /** 4663 * Destroy all flows. 4664 * 4665 * @param dev 4666 * Pointer to Ethernet device. 4667 * @param list 4668 * Pointer to the Indexed flow list. 4669 * @param active 4670 * If flushing is called avtively. 4671 */ 4672 void 4673 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 4674 { 4675 uint32_t num_flushed = 0; 4676 4677 while (*list) { 4678 flow_list_destroy(dev, list, *list); 4679 num_flushed++; 4680 } 4681 if (active) { 4682 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 4683 dev->data->port_id, num_flushed); 4684 } 4685 } 4686 4687 /** 4688 * Remove all flows. 4689 * 4690 * @param dev 4691 * Pointer to Ethernet device. 4692 * @param list 4693 * Pointer to the Indexed flow list. 4694 */ 4695 void 4696 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) 4697 { 4698 struct mlx5_priv *priv = dev->data->dev_private; 4699 struct rte_flow *flow = NULL; 4700 uint32_t idx; 4701 4702 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4703 flow, next) { 4704 flow_drv_remove(dev, flow); 4705 flow_mreg_stop_copy_action(dev, flow); 4706 } 4707 flow_mreg_del_default_copy_action(dev); 4708 flow_rxq_flags_clear(dev); 4709 } 4710 4711 /** 4712 * Add all flows. 4713 * 4714 * @param dev 4715 * Pointer to Ethernet device. 4716 * @param list 4717 * Pointer to the Indexed flow list. 4718 * 4719 * @return 4720 * 0 on success, a negative errno value otherwise and rte_errno is set. 4721 */ 4722 int 4723 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) 4724 { 4725 struct mlx5_priv *priv = dev->data->dev_private; 4726 struct rte_flow *flow = NULL; 4727 struct rte_flow_error error; 4728 uint32_t idx; 4729 int ret = 0; 4730 4731 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4732 ret = flow_mreg_add_default_copy_action(dev, &error); 4733 if (ret < 0) 4734 return -rte_errno; 4735 /* Apply Flows created by application. */ 4736 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4737 flow, next) { 4738 ret = flow_mreg_start_copy_action(dev, flow); 4739 if (ret < 0) 4740 goto error; 4741 ret = flow_drv_apply(dev, flow, &error); 4742 if (ret < 0) 4743 goto error; 4744 flow_rxq_flags_set(dev, flow); 4745 } 4746 return 0; 4747 error: 4748 ret = rte_errno; /* Save rte_errno before cleanup. */ 4749 mlx5_flow_stop(dev, list); 4750 rte_errno = ret; /* Restore rte_errno. */ 4751 return -rte_errno; 4752 } 4753 4754 /** 4755 * Stop all default actions for flows. 4756 * 4757 * @param dev 4758 * Pointer to Ethernet device. 4759 */ 4760 void 4761 mlx5_flow_stop_default(struct rte_eth_dev *dev) 4762 { 4763 flow_mreg_del_default_copy_action(dev); 4764 flow_rxq_flags_clear(dev); 4765 } 4766 4767 /** 4768 * Start all default actions for flows. 4769 * 4770 * @param dev 4771 * Pointer to Ethernet device. 4772 * @return 4773 * 0 on success, a negative errno value otherwise and rte_errno is set. 4774 */ 4775 int 4776 mlx5_flow_start_default(struct rte_eth_dev *dev) 4777 { 4778 struct rte_flow_error error; 4779 4780 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4781 return flow_mreg_add_default_copy_action(dev, &error); 4782 } 4783 4784 /** 4785 * Allocate intermediate resources for flow creation. 4786 * 4787 * @param dev 4788 * Pointer to Ethernet device. 4789 */ 4790 void 4791 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 4792 { 4793 struct mlx5_priv *priv = dev->data->dev_private; 4794 4795 if (!priv->inter_flows) { 4796 priv->inter_flows = mlx5_malloc(MLX5_MEM_ZERO, 4797 MLX5_NUM_MAX_DEV_FLOWS * 4798 sizeof(struct mlx5_flow) + 4799 (sizeof(struct mlx5_flow_rss_desc) + 4800 sizeof(uint16_t) * UINT16_MAX) * 2, 0, 4801 SOCKET_ID_ANY); 4802 if (!priv->inter_flows) { 4803 DRV_LOG(ERR, "can't allocate intermediate memory."); 4804 return; 4805 } 4806 } 4807 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) 4808 [MLX5_NUM_MAX_DEV_FLOWS]; 4809 /* Reset the index. */ 4810 priv->flow_idx = 0; 4811 priv->flow_nested_idx = 0; 4812 } 4813 4814 /** 4815 * Free intermediate resources for flows. 4816 * 4817 * @param dev 4818 * Pointer to Ethernet device. 4819 */ 4820 void 4821 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 4822 { 4823 struct mlx5_priv *priv = dev->data->dev_private; 4824 4825 mlx5_free(priv->inter_flows); 4826 priv->inter_flows = NULL; 4827 } 4828 4829 /** 4830 * Verify the flow list is empty 4831 * 4832 * @param dev 4833 * Pointer to Ethernet device. 4834 * 4835 * @return the number of flows not released. 4836 */ 4837 int 4838 mlx5_flow_verify(struct rte_eth_dev *dev) 4839 { 4840 struct mlx5_priv *priv = dev->data->dev_private; 4841 struct rte_flow *flow; 4842 uint32_t idx; 4843 int ret = 0; 4844 4845 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 4846 flow, next) { 4847 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4848 dev->data->port_id, (void *)flow); 4849 ++ret; 4850 } 4851 return ret; 4852 } 4853 4854 /** 4855 * Enable default hairpin egress flow. 4856 * 4857 * @param dev 4858 * Pointer to Ethernet device. 4859 * @param queue 4860 * The queue index. 4861 * 4862 * @return 4863 * 0 on success, a negative errno value otherwise and rte_errno is set. 4864 */ 4865 int 4866 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4867 uint32_t queue) 4868 { 4869 struct mlx5_priv *priv = dev->data->dev_private; 4870 const struct rte_flow_attr attr = { 4871 .egress = 1, 4872 .priority = 0, 4873 }; 4874 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4875 .queue = queue, 4876 }; 4877 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4878 .queue = UINT32_MAX, 4879 }; 4880 struct rte_flow_item items[] = { 4881 { 4882 .type = (enum rte_flow_item_type) 4883 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4884 .spec = &queue_spec, 4885 .last = NULL, 4886 .mask = &queue_mask, 4887 }, 4888 { 4889 .type = RTE_FLOW_ITEM_TYPE_END, 4890 }, 4891 }; 4892 struct rte_flow_action_jump jump = { 4893 .group = MLX5_HAIRPIN_TX_TABLE, 4894 }; 4895 struct rte_flow_action actions[2]; 4896 uint32_t flow_idx; 4897 struct rte_flow_error error; 4898 4899 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4900 actions[0].conf = &jump; 4901 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4902 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4903 &attr, items, actions, false, &error); 4904 if (!flow_idx) { 4905 DRV_LOG(DEBUG, 4906 "Failed to create ctrl flow: rte_errno(%d)," 4907 " type(%d), message(%s)", 4908 rte_errno, error.type, 4909 error.message ? error.message : " (no stated reason)"); 4910 return -rte_errno; 4911 } 4912 return 0; 4913 } 4914 4915 /** 4916 * Enable a control flow configured from the control plane. 4917 * 4918 * @param dev 4919 * Pointer to Ethernet device. 4920 * @param eth_spec 4921 * An Ethernet flow spec to apply. 4922 * @param eth_mask 4923 * An Ethernet flow mask to apply. 4924 * @param vlan_spec 4925 * A VLAN flow spec to apply. 4926 * @param vlan_mask 4927 * A VLAN flow mask to apply. 4928 * 4929 * @return 4930 * 0 on success, a negative errno value otherwise and rte_errno is set. 4931 */ 4932 int 4933 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4934 struct rte_flow_item_eth *eth_spec, 4935 struct rte_flow_item_eth *eth_mask, 4936 struct rte_flow_item_vlan *vlan_spec, 4937 struct rte_flow_item_vlan *vlan_mask) 4938 { 4939 struct mlx5_priv *priv = dev->data->dev_private; 4940 const struct rte_flow_attr attr = { 4941 .ingress = 1, 4942 .priority = MLX5_FLOW_PRIO_RSVD, 4943 }; 4944 struct rte_flow_item items[] = { 4945 { 4946 .type = RTE_FLOW_ITEM_TYPE_ETH, 4947 .spec = eth_spec, 4948 .last = NULL, 4949 .mask = eth_mask, 4950 }, 4951 { 4952 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4953 RTE_FLOW_ITEM_TYPE_END, 4954 .spec = vlan_spec, 4955 .last = NULL, 4956 .mask = vlan_mask, 4957 }, 4958 { 4959 .type = RTE_FLOW_ITEM_TYPE_END, 4960 }, 4961 }; 4962 uint16_t queue[priv->reta_idx_n]; 4963 struct rte_flow_action_rss action_rss = { 4964 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4965 .level = 0, 4966 .types = priv->rss_conf.rss_hf, 4967 .key_len = priv->rss_conf.rss_key_len, 4968 .queue_num = priv->reta_idx_n, 4969 .key = priv->rss_conf.rss_key, 4970 .queue = queue, 4971 }; 4972 struct rte_flow_action actions[] = { 4973 { 4974 .type = RTE_FLOW_ACTION_TYPE_RSS, 4975 .conf = &action_rss, 4976 }, 4977 { 4978 .type = RTE_FLOW_ACTION_TYPE_END, 4979 }, 4980 }; 4981 uint32_t flow_idx; 4982 struct rte_flow_error error; 4983 unsigned int i; 4984 4985 if (!priv->reta_idx_n || !priv->rxqs_n) { 4986 return 0; 4987 } 4988 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 4989 action_rss.types = 0; 4990 for (i = 0; i != priv->reta_idx_n; ++i) 4991 queue[i] = (*priv->reta_idx)[i]; 4992 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4993 &attr, items, actions, false, &error); 4994 if (!flow_idx) 4995 return -rte_errno; 4996 return 0; 4997 } 4998 4999 /** 5000 * Enable a flow control configured from the control plane. 5001 * 5002 * @param dev 5003 * Pointer to Ethernet device. 5004 * @param eth_spec 5005 * An Ethernet flow spec to apply. 5006 * @param eth_mask 5007 * An Ethernet flow mask to apply. 5008 * 5009 * @return 5010 * 0 on success, a negative errno value otherwise and rte_errno is set. 5011 */ 5012 int 5013 mlx5_ctrl_flow(struct rte_eth_dev *dev, 5014 struct rte_flow_item_eth *eth_spec, 5015 struct rte_flow_item_eth *eth_mask) 5016 { 5017 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 5018 } 5019 5020 /** 5021 * Create default miss flow rule matching lacp traffic 5022 * 5023 * @param dev 5024 * Pointer to Ethernet device. 5025 * @param eth_spec 5026 * An Ethernet flow spec to apply. 5027 * 5028 * @return 5029 * 0 on success, a negative errno value otherwise and rte_errno is set. 5030 */ 5031 int 5032 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 5033 { 5034 struct mlx5_priv *priv = dev->data->dev_private; 5035 /* 5036 * The LACP matching is done by only using ether type since using 5037 * a multicast dst mac causes kernel to give low priority to this flow. 5038 */ 5039 static const struct rte_flow_item_eth lacp_spec = { 5040 .type = RTE_BE16(0x8809), 5041 }; 5042 static const struct rte_flow_item_eth lacp_mask = { 5043 .type = 0xffff, 5044 }; 5045 const struct rte_flow_attr attr = { 5046 .ingress = 1, 5047 }; 5048 struct rte_flow_item items[] = { 5049 { 5050 .type = RTE_FLOW_ITEM_TYPE_ETH, 5051 .spec = &lacp_spec, 5052 .mask = &lacp_mask, 5053 }, 5054 { 5055 .type = RTE_FLOW_ITEM_TYPE_END, 5056 }, 5057 }; 5058 struct rte_flow_action actions[] = { 5059 { 5060 .type = (enum rte_flow_action_type) 5061 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 5062 }, 5063 { 5064 .type = RTE_FLOW_ACTION_TYPE_END, 5065 }, 5066 }; 5067 struct rte_flow_error error; 5068 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5069 &attr, items, actions, false, &error); 5070 5071 if (!flow_idx) 5072 return -rte_errno; 5073 return 0; 5074 } 5075 5076 /** 5077 * Destroy a flow. 5078 * 5079 * @see rte_flow_destroy() 5080 * @see rte_flow_ops 5081 */ 5082 int 5083 mlx5_flow_destroy(struct rte_eth_dev *dev, 5084 struct rte_flow *flow, 5085 struct rte_flow_error *error __rte_unused) 5086 { 5087 struct mlx5_priv *priv = dev->data->dev_private; 5088 5089 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 5090 return 0; 5091 } 5092 5093 /** 5094 * Destroy all flows. 5095 * 5096 * @see rte_flow_flush() 5097 * @see rte_flow_ops 5098 */ 5099 int 5100 mlx5_flow_flush(struct rte_eth_dev *dev, 5101 struct rte_flow_error *error __rte_unused) 5102 { 5103 struct mlx5_priv *priv = dev->data->dev_private; 5104 5105 mlx5_flow_list_flush(dev, &priv->flows, false); 5106 return 0; 5107 } 5108 5109 /** 5110 * Isolated mode. 5111 * 5112 * @see rte_flow_isolate() 5113 * @see rte_flow_ops 5114 */ 5115 int 5116 mlx5_flow_isolate(struct rte_eth_dev *dev, 5117 int enable, 5118 struct rte_flow_error *error) 5119 { 5120 struct mlx5_priv *priv = dev->data->dev_private; 5121 5122 if (dev->data->dev_started) { 5123 rte_flow_error_set(error, EBUSY, 5124 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5125 NULL, 5126 "port must be stopped first"); 5127 return -rte_errno; 5128 } 5129 priv->isolated = !!enable; 5130 if (enable) 5131 dev->dev_ops = &mlx5_os_dev_ops_isolate; 5132 else 5133 dev->dev_ops = &mlx5_os_dev_ops; 5134 5135 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 5136 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 5137 5138 return 0; 5139 } 5140 5141 /** 5142 * Query a flow. 5143 * 5144 * @see rte_flow_query() 5145 * @see rte_flow_ops 5146 */ 5147 static int 5148 flow_drv_query(struct rte_eth_dev *dev, 5149 uint32_t flow_idx, 5150 const struct rte_flow_action *actions, 5151 void *data, 5152 struct rte_flow_error *error) 5153 { 5154 struct mlx5_priv *priv = dev->data->dev_private; 5155 const struct mlx5_flow_driver_ops *fops; 5156 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5157 [MLX5_IPOOL_RTE_FLOW], 5158 flow_idx); 5159 enum mlx5_flow_drv_type ftype; 5160 5161 if (!flow) { 5162 return rte_flow_error_set(error, ENOENT, 5163 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5164 NULL, 5165 "invalid flow handle"); 5166 } 5167 ftype = flow->drv_type; 5168 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 5169 fops = flow_get_drv_ops(ftype); 5170 5171 return fops->query(dev, flow, actions, data, error); 5172 } 5173 5174 /** 5175 * Query a flow. 5176 * 5177 * @see rte_flow_query() 5178 * @see rte_flow_ops 5179 */ 5180 int 5181 mlx5_flow_query(struct rte_eth_dev *dev, 5182 struct rte_flow *flow, 5183 const struct rte_flow_action *actions, 5184 void *data, 5185 struct rte_flow_error *error) 5186 { 5187 int ret; 5188 5189 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 5190 error); 5191 if (ret < 0) 5192 return ret; 5193 return 0; 5194 } 5195 5196 /** 5197 * Convert a flow director filter to a generic flow. 5198 * 5199 * @param dev 5200 * Pointer to Ethernet device. 5201 * @param fdir_filter 5202 * Flow director filter to add. 5203 * @param attributes 5204 * Generic flow parameters structure. 5205 * 5206 * @return 5207 * 0 on success, a negative errno value otherwise and rte_errno is set. 5208 */ 5209 static int 5210 flow_fdir_filter_convert(struct rte_eth_dev *dev, 5211 const struct rte_eth_fdir_filter *fdir_filter, 5212 struct mlx5_fdir *attributes) 5213 { 5214 struct mlx5_priv *priv = dev->data->dev_private; 5215 const struct rte_eth_fdir_input *input = &fdir_filter->input; 5216 const struct rte_eth_fdir_masks *mask = 5217 &dev->data->dev_conf.fdir_conf.mask; 5218 5219 /* Validate queue number. */ 5220 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 5221 DRV_LOG(ERR, "port %u invalid queue number %d", 5222 dev->data->port_id, fdir_filter->action.rx_queue); 5223 rte_errno = EINVAL; 5224 return -rte_errno; 5225 } 5226 attributes->attr.ingress = 1; 5227 attributes->items[0] = (struct rte_flow_item) { 5228 .type = RTE_FLOW_ITEM_TYPE_ETH, 5229 .spec = &attributes->l2, 5230 .mask = &attributes->l2_mask, 5231 }; 5232 switch (fdir_filter->action.behavior) { 5233 case RTE_ETH_FDIR_ACCEPT: 5234 attributes->actions[0] = (struct rte_flow_action){ 5235 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 5236 .conf = &attributes->queue, 5237 }; 5238 break; 5239 case RTE_ETH_FDIR_REJECT: 5240 attributes->actions[0] = (struct rte_flow_action){ 5241 .type = RTE_FLOW_ACTION_TYPE_DROP, 5242 }; 5243 break; 5244 default: 5245 DRV_LOG(ERR, "port %u invalid behavior %d", 5246 dev->data->port_id, 5247 fdir_filter->action.behavior); 5248 rte_errno = ENOTSUP; 5249 return -rte_errno; 5250 } 5251 attributes->queue.index = fdir_filter->action.rx_queue; 5252 /* Handle L3. */ 5253 switch (fdir_filter->input.flow_type) { 5254 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5255 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5256 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5257 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 5258 .src_addr = input->flow.ip4_flow.src_ip, 5259 .dst_addr = input->flow.ip4_flow.dst_ip, 5260 .time_to_live = input->flow.ip4_flow.ttl, 5261 .type_of_service = input->flow.ip4_flow.tos, 5262 }; 5263 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 5264 .src_addr = mask->ipv4_mask.src_ip, 5265 .dst_addr = mask->ipv4_mask.dst_ip, 5266 .time_to_live = mask->ipv4_mask.ttl, 5267 .type_of_service = mask->ipv4_mask.tos, 5268 .next_proto_id = mask->ipv4_mask.proto, 5269 }; 5270 attributes->items[1] = (struct rte_flow_item){ 5271 .type = RTE_FLOW_ITEM_TYPE_IPV4, 5272 .spec = &attributes->l3, 5273 .mask = &attributes->l3_mask, 5274 }; 5275 break; 5276 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5277 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5278 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5279 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 5280 .hop_limits = input->flow.ipv6_flow.hop_limits, 5281 .proto = input->flow.ipv6_flow.proto, 5282 }; 5283 5284 memcpy(attributes->l3.ipv6.hdr.src_addr, 5285 input->flow.ipv6_flow.src_ip, 5286 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5287 memcpy(attributes->l3.ipv6.hdr.dst_addr, 5288 input->flow.ipv6_flow.dst_ip, 5289 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5290 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 5291 mask->ipv6_mask.src_ip, 5292 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5293 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 5294 mask->ipv6_mask.dst_ip, 5295 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5296 attributes->items[1] = (struct rte_flow_item){ 5297 .type = RTE_FLOW_ITEM_TYPE_IPV6, 5298 .spec = &attributes->l3, 5299 .mask = &attributes->l3_mask, 5300 }; 5301 break; 5302 default: 5303 DRV_LOG(ERR, "port %u invalid flow type%d", 5304 dev->data->port_id, fdir_filter->input.flow_type); 5305 rte_errno = ENOTSUP; 5306 return -rte_errno; 5307 } 5308 /* Handle L4. */ 5309 switch (fdir_filter->input.flow_type) { 5310 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5311 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5312 .src_port = input->flow.udp4_flow.src_port, 5313 .dst_port = input->flow.udp4_flow.dst_port, 5314 }; 5315 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5316 .src_port = mask->src_port_mask, 5317 .dst_port = mask->dst_port_mask, 5318 }; 5319 attributes->items[2] = (struct rte_flow_item){ 5320 .type = RTE_FLOW_ITEM_TYPE_UDP, 5321 .spec = &attributes->l4, 5322 .mask = &attributes->l4_mask, 5323 }; 5324 break; 5325 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5326 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5327 .src_port = input->flow.tcp4_flow.src_port, 5328 .dst_port = input->flow.tcp4_flow.dst_port, 5329 }; 5330 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5331 .src_port = mask->src_port_mask, 5332 .dst_port = mask->dst_port_mask, 5333 }; 5334 attributes->items[2] = (struct rte_flow_item){ 5335 .type = RTE_FLOW_ITEM_TYPE_TCP, 5336 .spec = &attributes->l4, 5337 .mask = &attributes->l4_mask, 5338 }; 5339 break; 5340 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5341 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5342 .src_port = input->flow.udp6_flow.src_port, 5343 .dst_port = input->flow.udp6_flow.dst_port, 5344 }; 5345 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5346 .src_port = mask->src_port_mask, 5347 .dst_port = mask->dst_port_mask, 5348 }; 5349 attributes->items[2] = (struct rte_flow_item){ 5350 .type = RTE_FLOW_ITEM_TYPE_UDP, 5351 .spec = &attributes->l4, 5352 .mask = &attributes->l4_mask, 5353 }; 5354 break; 5355 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5356 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5357 .src_port = input->flow.tcp6_flow.src_port, 5358 .dst_port = input->flow.tcp6_flow.dst_port, 5359 }; 5360 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5361 .src_port = mask->src_port_mask, 5362 .dst_port = mask->dst_port_mask, 5363 }; 5364 attributes->items[2] = (struct rte_flow_item){ 5365 .type = RTE_FLOW_ITEM_TYPE_TCP, 5366 .spec = &attributes->l4, 5367 .mask = &attributes->l4_mask, 5368 }; 5369 break; 5370 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5371 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5372 break; 5373 default: 5374 DRV_LOG(ERR, "port %u invalid flow type%d", 5375 dev->data->port_id, fdir_filter->input.flow_type); 5376 rte_errno = ENOTSUP; 5377 return -rte_errno; 5378 } 5379 return 0; 5380 } 5381 5382 #define FLOW_FDIR_CMP(f1, f2, fld) \ 5383 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 5384 5385 /** 5386 * Compare two FDIR flows. If items and actions are identical, the two flows are 5387 * regarded as same. 5388 * 5389 * @param dev 5390 * Pointer to Ethernet device. 5391 * @param f1 5392 * FDIR flow to compare. 5393 * @param f2 5394 * FDIR flow to compare. 5395 * 5396 * @return 5397 * Zero on match, 1 otherwise. 5398 */ 5399 static int 5400 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 5401 { 5402 if (FLOW_FDIR_CMP(f1, f2, attr) || 5403 FLOW_FDIR_CMP(f1, f2, l2) || 5404 FLOW_FDIR_CMP(f1, f2, l2_mask) || 5405 FLOW_FDIR_CMP(f1, f2, l3) || 5406 FLOW_FDIR_CMP(f1, f2, l3_mask) || 5407 FLOW_FDIR_CMP(f1, f2, l4) || 5408 FLOW_FDIR_CMP(f1, f2, l4_mask) || 5409 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 5410 return 1; 5411 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 5412 FLOW_FDIR_CMP(f1, f2, queue)) 5413 return 1; 5414 return 0; 5415 } 5416 5417 /** 5418 * Search device flow list to find out a matched FDIR flow. 5419 * 5420 * @param dev 5421 * Pointer to Ethernet device. 5422 * @param fdir_flow 5423 * FDIR flow to lookup. 5424 * 5425 * @return 5426 * Index of flow if found, 0 otherwise. 5427 */ 5428 static uint32_t 5429 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5430 { 5431 struct mlx5_priv *priv = dev->data->dev_private; 5432 uint32_t flow_idx = 0; 5433 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5434 5435 MLX5_ASSERT(fdir_flow); 5436 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5437 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 5438 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 5439 dev->data->port_id, flow_idx); 5440 flow_idx = priv_fdir_flow->rix_flow; 5441 break; 5442 } 5443 } 5444 return flow_idx; 5445 } 5446 5447 /** 5448 * Add new flow director filter and store it in list. 5449 * 5450 * @param dev 5451 * Pointer to Ethernet device. 5452 * @param fdir_filter 5453 * Flow director filter to add. 5454 * 5455 * @return 5456 * 0 on success, a negative errno value otherwise and rte_errno is set. 5457 */ 5458 static int 5459 flow_fdir_filter_add(struct rte_eth_dev *dev, 5460 const struct rte_eth_fdir_filter *fdir_filter) 5461 { 5462 struct mlx5_priv *priv = dev->data->dev_private; 5463 struct mlx5_fdir *fdir_flow; 5464 struct rte_flow *flow; 5465 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5466 uint32_t flow_idx; 5467 int ret; 5468 5469 fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0, 5470 SOCKET_ID_ANY); 5471 if (!fdir_flow) { 5472 rte_errno = ENOMEM; 5473 return -rte_errno; 5474 } 5475 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5476 if (ret) 5477 goto error; 5478 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 5479 if (flow_idx) { 5480 rte_errno = EEXIST; 5481 goto error; 5482 } 5483 priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, 5484 sizeof(struct mlx5_fdir_flow), 5485 0, SOCKET_ID_ANY); 5486 if (!priv_fdir_flow) { 5487 rte_errno = ENOMEM; 5488 goto error; 5489 } 5490 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5491 fdir_flow->items, fdir_flow->actions, true, 5492 NULL); 5493 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5494 if (!flow) 5495 goto error; 5496 flow->fdir = 1; 5497 priv_fdir_flow->fdir = fdir_flow; 5498 priv_fdir_flow->rix_flow = flow_idx; 5499 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 5500 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5501 dev->data->port_id, (void *)flow); 5502 return 0; 5503 error: 5504 mlx5_free(priv_fdir_flow); 5505 mlx5_free(fdir_flow); 5506 return -rte_errno; 5507 } 5508 5509 /** 5510 * Delete specific filter. 5511 * 5512 * @param dev 5513 * Pointer to Ethernet device. 5514 * @param fdir_filter 5515 * Filter to be deleted. 5516 * 5517 * @return 5518 * 0 on success, a negative errno value otherwise and rte_errno is set. 5519 */ 5520 static int 5521 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5522 const struct rte_eth_fdir_filter *fdir_filter) 5523 { 5524 struct mlx5_priv *priv = dev->data->dev_private; 5525 uint32_t flow_idx; 5526 struct mlx5_fdir fdir_flow = { 5527 .attr.group = 0, 5528 }; 5529 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5530 int ret; 5531 5532 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5533 if (ret) 5534 return -rte_errno; 5535 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5536 /* Find the fdir in priv list */ 5537 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 5538 break; 5539 } 5540 if (!priv_fdir_flow) 5541 return 0; 5542 LIST_REMOVE(priv_fdir_flow, next); 5543 flow_idx = priv_fdir_flow->rix_flow; 5544 flow_list_destroy(dev, &priv->flows, flow_idx); 5545 mlx5_free(priv_fdir_flow->fdir); 5546 mlx5_free(priv_fdir_flow); 5547 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 5548 dev->data->port_id, flow_idx); 5549 return 0; 5550 } 5551 5552 /** 5553 * Update queue for specific filter. 5554 * 5555 * @param dev 5556 * Pointer to Ethernet device. 5557 * @param fdir_filter 5558 * Filter to be updated. 5559 * 5560 * @return 5561 * 0 on success, a negative errno value otherwise and rte_errno is set. 5562 */ 5563 static int 5564 flow_fdir_filter_update(struct rte_eth_dev *dev, 5565 const struct rte_eth_fdir_filter *fdir_filter) 5566 { 5567 int ret; 5568 5569 ret = flow_fdir_filter_delete(dev, fdir_filter); 5570 if (ret) 5571 return ret; 5572 return flow_fdir_filter_add(dev, fdir_filter); 5573 } 5574 5575 /** 5576 * Flush all filters. 5577 * 5578 * @param dev 5579 * Pointer to Ethernet device. 5580 */ 5581 static void 5582 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5583 { 5584 struct mlx5_priv *priv = dev->data->dev_private; 5585 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5586 5587 while (!LIST_EMPTY(&priv->fdir_flows)) { 5588 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 5589 LIST_REMOVE(priv_fdir_flow, next); 5590 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 5591 mlx5_free(priv_fdir_flow->fdir); 5592 mlx5_free(priv_fdir_flow); 5593 } 5594 } 5595 5596 /** 5597 * Get flow director information. 5598 * 5599 * @param dev 5600 * Pointer to Ethernet device. 5601 * @param[out] fdir_info 5602 * Resulting flow director information. 5603 */ 5604 static void 5605 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5606 { 5607 struct rte_eth_fdir_masks *mask = 5608 &dev->data->dev_conf.fdir_conf.mask; 5609 5610 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5611 fdir_info->guarant_spc = 0; 5612 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5613 fdir_info->max_flexpayload = 0; 5614 fdir_info->flow_types_mask[0] = 0; 5615 fdir_info->flex_payload_unit = 0; 5616 fdir_info->max_flex_payload_segment_num = 0; 5617 fdir_info->flex_payload_limit = 0; 5618 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5619 } 5620 5621 /** 5622 * Deal with flow director operations. 5623 * 5624 * @param dev 5625 * Pointer to Ethernet device. 5626 * @param filter_op 5627 * Operation to perform. 5628 * @param arg 5629 * Pointer to operation-specific structure. 5630 * 5631 * @return 5632 * 0 on success, a negative errno value otherwise and rte_errno is set. 5633 */ 5634 static int 5635 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5636 void *arg) 5637 { 5638 enum rte_fdir_mode fdir_mode = 5639 dev->data->dev_conf.fdir_conf.mode; 5640 5641 if (filter_op == RTE_ETH_FILTER_NOP) 5642 return 0; 5643 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5644 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5645 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5646 dev->data->port_id, fdir_mode); 5647 rte_errno = EINVAL; 5648 return -rte_errno; 5649 } 5650 switch (filter_op) { 5651 case RTE_ETH_FILTER_ADD: 5652 return flow_fdir_filter_add(dev, arg); 5653 case RTE_ETH_FILTER_UPDATE: 5654 return flow_fdir_filter_update(dev, arg); 5655 case RTE_ETH_FILTER_DELETE: 5656 return flow_fdir_filter_delete(dev, arg); 5657 case RTE_ETH_FILTER_FLUSH: 5658 flow_fdir_filter_flush(dev); 5659 break; 5660 case RTE_ETH_FILTER_INFO: 5661 flow_fdir_info_get(dev, arg); 5662 break; 5663 default: 5664 DRV_LOG(DEBUG, "port %u unknown operation %u", 5665 dev->data->port_id, filter_op); 5666 rte_errno = EINVAL; 5667 return -rte_errno; 5668 } 5669 return 0; 5670 } 5671 5672 /** 5673 * Manage filter operations. 5674 * 5675 * @param dev 5676 * Pointer to Ethernet device structure. 5677 * @param filter_type 5678 * Filter type. 5679 * @param filter_op 5680 * Operation to perform. 5681 * @param arg 5682 * Pointer to operation-specific structure. 5683 * 5684 * @return 5685 * 0 on success, a negative errno value otherwise and rte_errno is set. 5686 */ 5687 int 5688 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5689 enum rte_filter_type filter_type, 5690 enum rte_filter_op filter_op, 5691 void *arg) 5692 { 5693 switch (filter_type) { 5694 case RTE_ETH_FILTER_GENERIC: 5695 if (filter_op != RTE_ETH_FILTER_GET) { 5696 rte_errno = EINVAL; 5697 return -rte_errno; 5698 } 5699 *(const void **)arg = &mlx5_flow_ops; 5700 return 0; 5701 case RTE_ETH_FILTER_FDIR: 5702 return flow_fdir_ctrl_func(dev, filter_op, arg); 5703 default: 5704 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5705 dev->data->port_id, filter_type); 5706 rte_errno = ENOTSUP; 5707 return -rte_errno; 5708 } 5709 return 0; 5710 } 5711 5712 /** 5713 * Create the needed meter and suffix tables. 5714 * 5715 * @param[in] dev 5716 * Pointer to Ethernet device. 5717 * @param[in] fm 5718 * Pointer to the flow meter. 5719 * 5720 * @return 5721 * Pointer to table set on success, NULL otherwise. 5722 */ 5723 struct mlx5_meter_domains_infos * 5724 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5725 const struct mlx5_flow_meter *fm) 5726 { 5727 const struct mlx5_flow_driver_ops *fops; 5728 5729 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5730 return fops->create_mtr_tbls(dev, fm); 5731 } 5732 5733 /** 5734 * Destroy the meter table set. 5735 * 5736 * @param[in] dev 5737 * Pointer to Ethernet device. 5738 * @param[in] tbl 5739 * Pointer to the meter table set. 5740 * 5741 * @return 5742 * 0 on success. 5743 */ 5744 int 5745 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5746 struct mlx5_meter_domains_infos *tbls) 5747 { 5748 const struct mlx5_flow_driver_ops *fops; 5749 5750 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5751 return fops->destroy_mtr_tbls(dev, tbls); 5752 } 5753 5754 /** 5755 * Create policer rules. 5756 * 5757 * @param[in] dev 5758 * Pointer to Ethernet device. 5759 * @param[in] fm 5760 * Pointer to flow meter structure. 5761 * @param[in] attr 5762 * Pointer to flow attributes. 5763 * 5764 * @return 5765 * 0 on success, -1 otherwise. 5766 */ 5767 int 5768 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5769 struct mlx5_flow_meter *fm, 5770 const struct rte_flow_attr *attr) 5771 { 5772 const struct mlx5_flow_driver_ops *fops; 5773 5774 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5775 return fops->create_policer_rules(dev, fm, attr); 5776 } 5777 5778 /** 5779 * Destroy policer rules. 5780 * 5781 * @param[in] fm 5782 * Pointer to flow meter structure. 5783 * @param[in] attr 5784 * Pointer to flow attributes. 5785 * 5786 * @return 5787 * 0 on success, -1 otherwise. 5788 */ 5789 int 5790 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5791 struct mlx5_flow_meter *fm, 5792 const struct rte_flow_attr *attr) 5793 { 5794 const struct mlx5_flow_driver_ops *fops; 5795 5796 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5797 return fops->destroy_policer_rules(dev, fm, attr); 5798 } 5799 5800 /** 5801 * Allocate a counter. 5802 * 5803 * @param[in] dev 5804 * Pointer to Ethernet device structure. 5805 * 5806 * @return 5807 * Index to allocated counter on success, 0 otherwise. 5808 */ 5809 uint32_t 5810 mlx5_counter_alloc(struct rte_eth_dev *dev) 5811 { 5812 const struct mlx5_flow_driver_ops *fops; 5813 struct rte_flow_attr attr = { .transfer = 0 }; 5814 5815 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5816 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5817 return fops->counter_alloc(dev); 5818 } 5819 DRV_LOG(ERR, 5820 "port %u counter allocate is not supported.", 5821 dev->data->port_id); 5822 return 0; 5823 } 5824 5825 /** 5826 * Free a counter. 5827 * 5828 * @param[in] dev 5829 * Pointer to Ethernet device structure. 5830 * @param[in] cnt 5831 * Index to counter to be free. 5832 */ 5833 void 5834 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 5835 { 5836 const struct mlx5_flow_driver_ops *fops; 5837 struct rte_flow_attr attr = { .transfer = 0 }; 5838 5839 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5840 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5841 fops->counter_free(dev, cnt); 5842 return; 5843 } 5844 DRV_LOG(ERR, 5845 "port %u counter free is not supported.", 5846 dev->data->port_id); 5847 } 5848 5849 /** 5850 * Query counter statistics. 5851 * 5852 * @param[in] dev 5853 * Pointer to Ethernet device structure. 5854 * @param[in] cnt 5855 * Index to counter to query. 5856 * @param[in] clear 5857 * Set to clear counter statistics. 5858 * @param[out] pkts 5859 * The counter hits packets number to save. 5860 * @param[out] bytes 5861 * The counter hits bytes number to save. 5862 * 5863 * @return 5864 * 0 on success, a negative errno value otherwise. 5865 */ 5866 int 5867 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 5868 bool clear, uint64_t *pkts, uint64_t *bytes) 5869 { 5870 const struct mlx5_flow_driver_ops *fops; 5871 struct rte_flow_attr attr = { .transfer = 0 }; 5872 5873 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5874 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5875 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5876 } 5877 DRV_LOG(ERR, 5878 "port %u counter query is not supported.", 5879 dev->data->port_id); 5880 return -ENOTSUP; 5881 } 5882 5883 #define MLX5_POOL_QUERY_FREQ_US 1000000 5884 5885 /** 5886 * Get number of all validate pools. 5887 * 5888 * @param[in] sh 5889 * Pointer to mlx5_dev_ctx_shared object. 5890 * 5891 * @return 5892 * The number of all validate pools. 5893 */ 5894 static uint32_t 5895 mlx5_get_all_valid_pool_count(struct mlx5_dev_ctx_shared *sh) 5896 { 5897 int i; 5898 uint32_t pools_n = 0; 5899 5900 for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i) 5901 pools_n += rte_atomic16_read(&sh->cmng.ccont[i].n_valid); 5902 return pools_n; 5903 } 5904 5905 /** 5906 * Set the periodic procedure for triggering asynchronous batch queries for all 5907 * the counter pools. 5908 * 5909 * @param[in] sh 5910 * Pointer to mlx5_dev_ctx_shared object. 5911 */ 5912 void 5913 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 5914 { 5915 uint32_t pools_n, us; 5916 5917 pools_n = mlx5_get_all_valid_pool_count(sh); 5918 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5919 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5920 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5921 sh->cmng.query_thread_on = 0; 5922 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5923 } else { 5924 sh->cmng.query_thread_on = 1; 5925 } 5926 } 5927 5928 /** 5929 * The periodic procedure for triggering asynchronous batch queries for all the 5930 * counter pools. This function is probably called by the host thread. 5931 * 5932 * @param[in] arg 5933 * The parameter for the alarm process. 5934 */ 5935 void 5936 mlx5_flow_query_alarm(void *arg) 5937 { 5938 struct mlx5_dev_ctx_shared *sh = arg; 5939 struct mlx5_devx_obj *dcs; 5940 uint16_t offset; 5941 int ret; 5942 uint8_t batch = sh->cmng.batch; 5943 uint8_t age = sh->cmng.age; 5944 uint16_t pool_index = sh->cmng.pool_index; 5945 struct mlx5_pools_container *cont; 5946 struct mlx5_flow_counter_pool *pool; 5947 int cont_loop = MLX5_CCONT_TYPE_MAX; 5948 5949 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5950 goto set_alarm; 5951 next_container: 5952 cont = MLX5_CNT_CONTAINER(sh, batch, age); 5953 rte_spinlock_lock(&cont->resize_sl); 5954 if (!cont->pools) { 5955 rte_spinlock_unlock(&cont->resize_sl); 5956 /* Check if all the containers are empty. */ 5957 if (unlikely(--cont_loop == 0)) 5958 goto set_alarm; 5959 batch ^= 0x1; 5960 pool_index = 0; 5961 if (batch == 0 && pool_index == 0) { 5962 age ^= 0x1; 5963 sh->cmng.batch = batch; 5964 sh->cmng.age = age; 5965 } 5966 goto next_container; 5967 } 5968 pool = cont->pools[pool_index]; 5969 rte_spinlock_unlock(&cont->resize_sl); 5970 if (pool->raw_hw) 5971 /* There is a pool query in progress. */ 5972 goto set_alarm; 5973 pool->raw_hw = 5974 LIST_FIRST(&sh->cmng.free_stat_raws); 5975 if (!pool->raw_hw) 5976 /* No free counter statistics raw memory. */ 5977 goto set_alarm; 5978 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5979 (&pool->a64_dcs); 5980 if (dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) { 5981 /* Pool without valid counter. */ 5982 pool->raw_hw = NULL; 5983 goto next_pool; 5984 } 5985 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5986 /* 5987 * Identify the counters released between query trigger and query 5988 * handle more effiecntly. The counter released in this gap period 5989 * should wait for a new round of query as the new arrived packets 5990 * will not be taken into account. 5991 */ 5992 pool->query_gen++; 5993 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5994 offset, NULL, NULL, 5995 pool->raw_hw->mem_mng->dm->id, 5996 (void *)(uintptr_t) 5997 (pool->raw_hw->data + offset), 5998 sh->devx_comp, 5999 (uint64_t)(uintptr_t)pool); 6000 if (ret) { 6001 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 6002 " %d", pool->min_dcs->id); 6003 pool->raw_hw = NULL; 6004 goto set_alarm; 6005 } 6006 pool->raw_hw->min_dcs_id = dcs->id; 6007 LIST_REMOVE(pool->raw_hw, next); 6008 sh->cmng.pending_queries++; 6009 next_pool: 6010 pool_index++; 6011 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 6012 batch ^= 0x1; 6013 pool_index = 0; 6014 if (batch == 0 && pool_index == 0) 6015 age ^= 0x1; 6016 } 6017 set_alarm: 6018 sh->cmng.batch = batch; 6019 sh->cmng.pool_index = pool_index; 6020 sh->cmng.age = age; 6021 mlx5_set_query_alarm(sh); 6022 } 6023 6024 /** 6025 * Check and callback event for new aged flow in the counter pool 6026 * 6027 * @param[in] sh 6028 * Pointer to mlx5_dev_ctx_shared object. 6029 * @param[in] pool 6030 * Pointer to Current counter pool. 6031 */ 6032 static void 6033 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 6034 struct mlx5_flow_counter_pool *pool) 6035 { 6036 struct mlx5_priv *priv; 6037 struct mlx5_flow_counter *cnt; 6038 struct mlx5_age_info *age_info; 6039 struct mlx5_age_param *age_param; 6040 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 6041 struct mlx5_counter_stats_raw *prev = pool->raw; 6042 uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10); 6043 uint32_t i; 6044 6045 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 6046 cnt = MLX5_POOL_GET_CNT(pool, i); 6047 age_param = MLX5_CNT_TO_AGE(cnt); 6048 if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE) 6049 continue; 6050 if (cur->data[i].hits != prev->data[i].hits) { 6051 age_param->expire = curr + age_param->timeout; 6052 continue; 6053 } 6054 if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2)) 6055 continue; 6056 /** 6057 * Hold the lock first, or if between the 6058 * state AGE_TMOUT and tailq operation the 6059 * release happened, the release procedure 6060 * may delete a non-existent tailq node. 6061 */ 6062 priv = rte_eth_devices[age_param->port_id].data->dev_private; 6063 age_info = GET_PORT_AGE_INFO(priv); 6064 rte_spinlock_lock(&age_info->aged_sl); 6065 /* If the cpmset fails, release happens. */ 6066 if (rte_atomic16_cmpset((volatile uint16_t *) 6067 &age_param->state, 6068 AGE_CANDIDATE, 6069 AGE_TMOUT) == 6070 AGE_CANDIDATE) { 6071 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 6072 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 6073 } 6074 rte_spinlock_unlock(&age_info->aged_sl); 6075 } 6076 for (i = 0; i < sh->max_port; i++) { 6077 age_info = &sh->port[i].age_info; 6078 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 6079 continue; 6080 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 6081 rte_eth_dev_callback_process 6082 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 6083 RTE_ETH_EVENT_FLOW_AGED, NULL); 6084 age_info->flags = 0; 6085 } 6086 } 6087 6088 /** 6089 * Handler for the HW respond about ready values from an asynchronous batch 6090 * query. This function is probably called by the host thread. 6091 * 6092 * @param[in] sh 6093 * The pointer to the shared device context. 6094 * @param[in] async_id 6095 * The Devx async ID. 6096 * @param[in] status 6097 * The status of the completion. 6098 */ 6099 void 6100 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 6101 uint64_t async_id, int status) 6102 { 6103 struct mlx5_flow_counter_pool *pool = 6104 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6105 struct mlx5_counter_stats_raw *raw_to_free; 6106 uint8_t age = !!IS_AGE_POOL(pool); 6107 uint8_t query_gen = pool->query_gen ^ 1; 6108 struct mlx5_pools_container *cont = 6109 MLX5_CNT_CONTAINER(sh, !IS_EXT_POOL(pool), age); 6110 6111 if (unlikely(status)) { 6112 raw_to_free = pool->raw_hw; 6113 } else { 6114 raw_to_free = pool->raw; 6115 if (IS_AGE_POOL(pool)) 6116 mlx5_flow_aging_check(sh, pool); 6117 rte_spinlock_lock(&pool->sl); 6118 pool->raw = pool->raw_hw; 6119 rte_spinlock_unlock(&pool->sl); 6120 /* Be sure the new raw counters data is updated in memory. */ 6121 rte_cio_wmb(); 6122 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 6123 rte_spinlock_lock(&cont->csl); 6124 TAILQ_CONCAT(&cont->counters, 6125 &pool->counters[query_gen], next); 6126 rte_spinlock_unlock(&cont->csl); 6127 } 6128 } 6129 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6130 pool->raw_hw = NULL; 6131 sh->cmng.pending_queries--; 6132 } 6133 6134 /** 6135 * Translate the rte_flow group index to HW table value. 6136 * 6137 * @param[in] attributes 6138 * Pointer to flow attributes 6139 * @param[in] external 6140 * Value is part of flow rule created by request external to PMD. 6141 * @param[in] group 6142 * rte_flow group index value. 6143 * @param[out] fdb_def_rule 6144 * Whether fdb jump to table 1 is configured. 6145 * @param[out] table 6146 * HW table value. 6147 * @param[out] error 6148 * Pointer to error structure. 6149 * 6150 * @return 6151 * 0 on success, a negative errno value otherwise and rte_errno is set. 6152 */ 6153 int 6154 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 6155 uint32_t group, bool fdb_def_rule, uint32_t *table, 6156 struct rte_flow_error *error) 6157 { 6158 if (attributes->transfer && external && fdb_def_rule) { 6159 if (group == UINT32_MAX) 6160 return rte_flow_error_set 6161 (error, EINVAL, 6162 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6163 NULL, 6164 "group index not supported"); 6165 *table = group + 1; 6166 } else { 6167 *table = group; 6168 } 6169 return 0; 6170 } 6171 6172 /** 6173 * Discover availability of metadata reg_c's. 6174 * 6175 * Iteratively use test flows to check availability. 6176 * 6177 * @param[in] dev 6178 * Pointer to the Ethernet device structure. 6179 * 6180 * @return 6181 * 0 on success, a negative errno value otherwise and rte_errno is set. 6182 */ 6183 int 6184 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 6185 { 6186 struct mlx5_priv *priv = dev->data->dev_private; 6187 struct mlx5_dev_config *config = &priv->config; 6188 enum modify_reg idx; 6189 int n = 0; 6190 6191 /* reg_c[0] and reg_c[1] are reserved. */ 6192 config->flow_mreg_c[n++] = REG_C_0; 6193 config->flow_mreg_c[n++] = REG_C_1; 6194 /* Discover availability of other reg_c's. */ 6195 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 6196 struct rte_flow_attr attr = { 6197 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6198 .priority = MLX5_FLOW_PRIO_RSVD, 6199 .ingress = 1, 6200 }; 6201 struct rte_flow_item items[] = { 6202 [0] = { 6203 .type = RTE_FLOW_ITEM_TYPE_END, 6204 }, 6205 }; 6206 struct rte_flow_action actions[] = { 6207 [0] = { 6208 .type = (enum rte_flow_action_type) 6209 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6210 .conf = &(struct mlx5_flow_action_copy_mreg){ 6211 .src = REG_C_1, 6212 .dst = idx, 6213 }, 6214 }, 6215 [1] = { 6216 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6217 .conf = &(struct rte_flow_action_jump){ 6218 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6219 }, 6220 }, 6221 [2] = { 6222 .type = RTE_FLOW_ACTION_TYPE_END, 6223 }, 6224 }; 6225 uint32_t flow_idx; 6226 struct rte_flow *flow; 6227 struct rte_flow_error error; 6228 6229 if (!config->dv_flow_en) 6230 break; 6231 /* Create internal flow, validation skips copy action. */ 6232 flow_idx = flow_list_create(dev, NULL, &attr, items, 6233 actions, false, &error); 6234 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6235 flow_idx); 6236 if (!flow) 6237 continue; 6238 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 6239 config->flow_mreg_c[n++] = idx; 6240 flow_list_destroy(dev, NULL, flow_idx); 6241 } 6242 for (; n < MLX5_MREG_C_NUM; ++n) 6243 config->flow_mreg_c[n] = REG_NON; 6244 return 0; 6245 } 6246 6247 /** 6248 * Dump flow raw hw data to file 6249 * 6250 * @param[in] dev 6251 * The pointer to Ethernet device. 6252 * @param[in] file 6253 * A pointer to a file for output. 6254 * @param[out] error 6255 * Perform verbose error reporting if not NULL. PMDs initialize this 6256 * structure in case of error only. 6257 * @return 6258 * 0 on success, a nagative value otherwise. 6259 */ 6260 int 6261 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6262 FILE *file, 6263 struct rte_flow_error *error __rte_unused) 6264 { 6265 struct mlx5_priv *priv = dev->data->dev_private; 6266 struct mlx5_dev_ctx_shared *sh = priv->sh; 6267 6268 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6269 sh->tx_domain, file); 6270 } 6271 6272 /** 6273 * Get aged-out flows. 6274 * 6275 * @param[in] dev 6276 * Pointer to the Ethernet device structure. 6277 * @param[in] context 6278 * The address of an array of pointers to the aged-out flows contexts. 6279 * @param[in] nb_countexts 6280 * The length of context array pointers. 6281 * @param[out] error 6282 * Perform verbose error reporting if not NULL. Initialized in case of 6283 * error only. 6284 * 6285 * @return 6286 * how many contexts get in success, otherwise negative errno value. 6287 * if nb_contexts is 0, return the amount of all aged contexts. 6288 * if nb_contexts is not 0 , return the amount of aged flows reported 6289 * in the context array. 6290 */ 6291 int 6292 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 6293 uint32_t nb_contexts, struct rte_flow_error *error) 6294 { 6295 const struct mlx5_flow_driver_ops *fops; 6296 struct rte_flow_attr attr = { .transfer = 0 }; 6297 6298 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6299 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6300 return fops->get_aged_flows(dev, contexts, nb_contexts, 6301 error); 6302 } 6303 DRV_LOG(ERR, 6304 "port %u get aged flows is not supported.", 6305 dev->data->port_id); 6306 return -ENOTSUP; 6307 } 6308