1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 12 /* Verbs header. */ 13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 14 #ifdef PEDANTIC 15 #pragma GCC diagnostic ignored "-Wpedantic" 16 #endif 17 #include <infiniband/verbs.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_common.h> 23 #include <rte_ether.h> 24 #include <rte_ethdev_driver.h> 25 #include <rte_flow.h> 26 #include <rte_flow_driver.h> 27 #include <rte_malloc.h> 28 #include <rte_ip.h> 29 30 #include "mlx5.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_flow.h" 33 #include "mlx5_glue.h" 34 #include "mlx5_devx_cmds.h" 35 #include "mlx5_prm.h" 36 #include "mlx5_rxtx.h" 37 38 /* Dev ops structure defined in mlx5.c */ 39 extern const struct eth_dev_ops mlx5_dev_ops; 40 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 41 42 /** Device flow drivers. */ 43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 44 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 45 #endif 46 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 47 48 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 49 50 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 51 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 52 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 53 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 54 #endif 55 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 56 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 57 }; 58 59 enum mlx5_expansion { 60 MLX5_EXPANSION_ROOT, 61 MLX5_EXPANSION_ROOT_OUTER, 62 MLX5_EXPANSION_ROOT_ETH_VLAN, 63 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 64 MLX5_EXPANSION_OUTER_ETH, 65 MLX5_EXPANSION_OUTER_ETH_VLAN, 66 MLX5_EXPANSION_OUTER_VLAN, 67 MLX5_EXPANSION_OUTER_IPV4, 68 MLX5_EXPANSION_OUTER_IPV4_UDP, 69 MLX5_EXPANSION_OUTER_IPV4_TCP, 70 MLX5_EXPANSION_OUTER_IPV6, 71 MLX5_EXPANSION_OUTER_IPV6_UDP, 72 MLX5_EXPANSION_OUTER_IPV6_TCP, 73 MLX5_EXPANSION_VXLAN, 74 MLX5_EXPANSION_VXLAN_GPE, 75 MLX5_EXPANSION_GRE, 76 MLX5_EXPANSION_MPLS, 77 MLX5_EXPANSION_ETH, 78 MLX5_EXPANSION_ETH_VLAN, 79 MLX5_EXPANSION_VLAN, 80 MLX5_EXPANSION_IPV4, 81 MLX5_EXPANSION_IPV4_UDP, 82 MLX5_EXPANSION_IPV4_TCP, 83 MLX5_EXPANSION_IPV6, 84 MLX5_EXPANSION_IPV6_UDP, 85 MLX5_EXPANSION_IPV6_TCP, 86 }; 87 88 /** Supported expansion of items. */ 89 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 90 [MLX5_EXPANSION_ROOT] = { 91 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 92 MLX5_EXPANSION_IPV4, 93 MLX5_EXPANSION_IPV6), 94 .type = RTE_FLOW_ITEM_TYPE_END, 95 }, 96 [MLX5_EXPANSION_ROOT_OUTER] = { 97 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 98 MLX5_EXPANSION_OUTER_IPV4, 99 MLX5_EXPANSION_OUTER_IPV6), 100 .type = RTE_FLOW_ITEM_TYPE_END, 101 }, 102 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 103 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 104 .type = RTE_FLOW_ITEM_TYPE_END, 105 }, 106 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 107 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 108 .type = RTE_FLOW_ITEM_TYPE_END, 109 }, 110 [MLX5_EXPANSION_OUTER_ETH] = { 111 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 112 MLX5_EXPANSION_OUTER_IPV6, 113 MLX5_EXPANSION_MPLS), 114 .type = RTE_FLOW_ITEM_TYPE_ETH, 115 .rss_types = 0, 116 }, 117 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 118 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 119 .type = RTE_FLOW_ITEM_TYPE_ETH, 120 .rss_types = 0, 121 }, 122 [MLX5_EXPANSION_OUTER_VLAN] = { 123 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 124 MLX5_EXPANSION_OUTER_IPV6), 125 .type = RTE_FLOW_ITEM_TYPE_VLAN, 126 }, 127 [MLX5_EXPANSION_OUTER_IPV4] = { 128 .next = RTE_FLOW_EXPAND_RSS_NEXT 129 (MLX5_EXPANSION_OUTER_IPV4_UDP, 130 MLX5_EXPANSION_OUTER_IPV4_TCP, 131 MLX5_EXPANSION_GRE, 132 MLX5_EXPANSION_IPV4, 133 MLX5_EXPANSION_IPV6), 134 .type = RTE_FLOW_ITEM_TYPE_IPV4, 135 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 136 ETH_RSS_NONFRAG_IPV4_OTHER, 137 }, 138 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 139 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 140 MLX5_EXPANSION_VXLAN_GPE), 141 .type = RTE_FLOW_ITEM_TYPE_UDP, 142 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 143 }, 144 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 145 .type = RTE_FLOW_ITEM_TYPE_TCP, 146 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 147 }, 148 [MLX5_EXPANSION_OUTER_IPV6] = { 149 .next = RTE_FLOW_EXPAND_RSS_NEXT 150 (MLX5_EXPANSION_OUTER_IPV6_UDP, 151 MLX5_EXPANSION_OUTER_IPV6_TCP, 152 MLX5_EXPANSION_IPV4, 153 MLX5_EXPANSION_IPV6), 154 .type = RTE_FLOW_ITEM_TYPE_IPV6, 155 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 156 ETH_RSS_NONFRAG_IPV6_OTHER, 157 }, 158 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 159 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 160 MLX5_EXPANSION_VXLAN_GPE), 161 .type = RTE_FLOW_ITEM_TYPE_UDP, 162 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 163 }, 164 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 165 .type = RTE_FLOW_ITEM_TYPE_TCP, 166 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 167 }, 168 [MLX5_EXPANSION_VXLAN] = { 169 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 170 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 171 }, 172 [MLX5_EXPANSION_VXLAN_GPE] = { 173 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 174 MLX5_EXPANSION_IPV4, 175 MLX5_EXPANSION_IPV6), 176 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 177 }, 178 [MLX5_EXPANSION_GRE] = { 179 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 180 .type = RTE_FLOW_ITEM_TYPE_GRE, 181 }, 182 [MLX5_EXPANSION_MPLS] = { 183 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 184 MLX5_EXPANSION_IPV6), 185 .type = RTE_FLOW_ITEM_TYPE_MPLS, 186 }, 187 [MLX5_EXPANSION_ETH] = { 188 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 189 MLX5_EXPANSION_IPV6), 190 .type = RTE_FLOW_ITEM_TYPE_ETH, 191 }, 192 [MLX5_EXPANSION_ETH_VLAN] = { 193 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 194 .type = RTE_FLOW_ITEM_TYPE_ETH, 195 }, 196 [MLX5_EXPANSION_VLAN] = { 197 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 198 MLX5_EXPANSION_IPV6), 199 .type = RTE_FLOW_ITEM_TYPE_VLAN, 200 }, 201 [MLX5_EXPANSION_IPV4] = { 202 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 203 MLX5_EXPANSION_IPV4_TCP), 204 .type = RTE_FLOW_ITEM_TYPE_IPV4, 205 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 206 ETH_RSS_NONFRAG_IPV4_OTHER, 207 }, 208 [MLX5_EXPANSION_IPV4_UDP] = { 209 .type = RTE_FLOW_ITEM_TYPE_UDP, 210 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 211 }, 212 [MLX5_EXPANSION_IPV4_TCP] = { 213 .type = RTE_FLOW_ITEM_TYPE_TCP, 214 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 215 }, 216 [MLX5_EXPANSION_IPV6] = { 217 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 218 MLX5_EXPANSION_IPV6_TCP), 219 .type = RTE_FLOW_ITEM_TYPE_IPV6, 220 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 221 ETH_RSS_NONFRAG_IPV6_OTHER, 222 }, 223 [MLX5_EXPANSION_IPV6_UDP] = { 224 .type = RTE_FLOW_ITEM_TYPE_UDP, 225 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 226 }, 227 [MLX5_EXPANSION_IPV6_TCP] = { 228 .type = RTE_FLOW_ITEM_TYPE_TCP, 229 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 230 }, 231 }; 232 233 static const struct rte_flow_ops mlx5_flow_ops = { 234 .validate = mlx5_flow_validate, 235 .create = mlx5_flow_create, 236 .destroy = mlx5_flow_destroy, 237 .flush = mlx5_flow_flush, 238 .isolate = mlx5_flow_isolate, 239 .query = mlx5_flow_query, 240 .dev_dump = mlx5_flow_dev_dump, 241 }; 242 243 /* Convert FDIR request to Generic flow. */ 244 struct mlx5_fdir { 245 struct rte_flow_attr attr; 246 struct rte_flow_item items[4]; 247 struct rte_flow_item_eth l2; 248 struct rte_flow_item_eth l2_mask; 249 union { 250 struct rte_flow_item_ipv4 ipv4; 251 struct rte_flow_item_ipv6 ipv6; 252 } l3; 253 union { 254 struct rte_flow_item_ipv4 ipv4; 255 struct rte_flow_item_ipv6 ipv6; 256 } l3_mask; 257 union { 258 struct rte_flow_item_udp udp; 259 struct rte_flow_item_tcp tcp; 260 } l4; 261 union { 262 struct rte_flow_item_udp udp; 263 struct rte_flow_item_tcp tcp; 264 } l4_mask; 265 struct rte_flow_action actions[2]; 266 struct rte_flow_action_queue queue; 267 }; 268 269 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 270 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 271 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 272 }; 273 274 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 275 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 276 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 277 { 9, 10, 11 }, { 12, 13, 14 }, 278 }; 279 280 /* Tunnel information. */ 281 struct mlx5_flow_tunnel_info { 282 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 283 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 284 }; 285 286 static struct mlx5_flow_tunnel_info tunnels_info[] = { 287 { 288 .tunnel = MLX5_FLOW_LAYER_VXLAN, 289 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 290 }, 291 { 292 .tunnel = MLX5_FLOW_LAYER_GENEVE, 293 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 294 }, 295 { 296 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 297 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 298 }, 299 { 300 .tunnel = MLX5_FLOW_LAYER_GRE, 301 .ptype = RTE_PTYPE_TUNNEL_GRE, 302 }, 303 { 304 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 305 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 306 }, 307 { 308 .tunnel = MLX5_FLOW_LAYER_MPLS, 309 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 310 }, 311 { 312 .tunnel = MLX5_FLOW_LAYER_NVGRE, 313 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 314 }, 315 { 316 .tunnel = MLX5_FLOW_LAYER_IPIP, 317 .ptype = RTE_PTYPE_TUNNEL_IP, 318 }, 319 { 320 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 321 .ptype = RTE_PTYPE_TUNNEL_IP, 322 }, 323 { 324 .tunnel = MLX5_FLOW_LAYER_GTP, 325 .ptype = RTE_PTYPE_TUNNEL_GTPU, 326 }, 327 }; 328 329 /** 330 * Translate tag ID to register. 331 * 332 * @param[in] dev 333 * Pointer to the Ethernet device structure. 334 * @param[in] feature 335 * The feature that request the register. 336 * @param[in] id 337 * The request register ID. 338 * @param[out] error 339 * Error description in case of any. 340 * 341 * @return 342 * The request register on success, a negative errno 343 * value otherwise and rte_errno is set. 344 */ 345 int 346 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 347 enum mlx5_feature_name feature, 348 uint32_t id, 349 struct rte_flow_error *error) 350 { 351 struct mlx5_priv *priv = dev->data->dev_private; 352 struct mlx5_dev_config *config = &priv->config; 353 enum modify_reg start_reg; 354 bool skip_mtr_reg = false; 355 356 switch (feature) { 357 case MLX5_HAIRPIN_RX: 358 return REG_B; 359 case MLX5_HAIRPIN_TX: 360 return REG_A; 361 case MLX5_METADATA_RX: 362 switch (config->dv_xmeta_en) { 363 case MLX5_XMETA_MODE_LEGACY: 364 return REG_B; 365 case MLX5_XMETA_MODE_META16: 366 return REG_C_0; 367 case MLX5_XMETA_MODE_META32: 368 return REG_C_1; 369 } 370 break; 371 case MLX5_METADATA_TX: 372 return REG_A; 373 case MLX5_METADATA_FDB: 374 switch (config->dv_xmeta_en) { 375 case MLX5_XMETA_MODE_LEGACY: 376 return REG_NONE; 377 case MLX5_XMETA_MODE_META16: 378 return REG_C_0; 379 case MLX5_XMETA_MODE_META32: 380 return REG_C_1; 381 } 382 break; 383 case MLX5_FLOW_MARK: 384 switch (config->dv_xmeta_en) { 385 case MLX5_XMETA_MODE_LEGACY: 386 return REG_NONE; 387 case MLX5_XMETA_MODE_META16: 388 return REG_C_1; 389 case MLX5_XMETA_MODE_META32: 390 return REG_C_0; 391 } 392 break; 393 case MLX5_MTR_SFX: 394 /* 395 * If meter color and flow match share one register, flow match 396 * should use the meter color register for match. 397 */ 398 if (priv->mtr_reg_share) 399 return priv->mtr_color_reg; 400 else 401 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 402 REG_C_3; 403 case MLX5_MTR_COLOR: 404 RTE_ASSERT(priv->mtr_color_reg != REG_NONE); 405 return priv->mtr_color_reg; 406 case MLX5_COPY_MARK: 407 /* 408 * Metadata COPY_MARK register using is in meter suffix sub 409 * flow while with meter. It's safe to share the same register. 410 */ 411 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 412 case MLX5_APP_TAG: 413 /* 414 * If meter is enable, it will engage the register for color 415 * match and flow match. If meter color match is not using the 416 * REG_C_2, need to skip the REG_C_x be used by meter color 417 * match. 418 * If meter is disable, free to use all available registers. 419 */ 420 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 421 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 422 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 423 if (id > (REG_C_7 - start_reg)) 424 return rte_flow_error_set(error, EINVAL, 425 RTE_FLOW_ERROR_TYPE_ITEM, 426 NULL, "invalid tag id"); 427 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 428 return rte_flow_error_set(error, ENOTSUP, 429 RTE_FLOW_ERROR_TYPE_ITEM, 430 NULL, "unsupported tag id"); 431 /* 432 * This case means meter is using the REG_C_x great than 2. 433 * Take care not to conflict with meter color REG_C_x. 434 * If the available index REG_C_y >= REG_C_x, skip the 435 * color register. 436 */ 437 if (skip_mtr_reg && config->flow_mreg_c 438 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 439 if (config->flow_mreg_c 440 [id + 1 + start_reg - REG_C_0] != REG_NONE) 441 return config->flow_mreg_c 442 [id + 1 + start_reg - REG_C_0]; 443 return rte_flow_error_set(error, ENOTSUP, 444 RTE_FLOW_ERROR_TYPE_ITEM, 445 NULL, "unsupported tag id"); 446 } 447 return config->flow_mreg_c[id + start_reg - REG_C_0]; 448 } 449 assert(false); 450 return rte_flow_error_set(error, EINVAL, 451 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 452 NULL, "invalid feature name"); 453 } 454 455 /** 456 * Check extensive flow metadata register support. 457 * 458 * @param dev 459 * Pointer to rte_eth_dev structure. 460 * 461 * @return 462 * True if device supports extensive flow metadata register, otherwise false. 463 */ 464 bool 465 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 466 { 467 struct mlx5_priv *priv = dev->data->dev_private; 468 struct mlx5_dev_config *config = &priv->config; 469 470 /* 471 * Having available reg_c can be regarded inclusively as supporting 472 * extensive flow metadata register, which could mean, 473 * - metadata register copy action by modify header. 474 * - 16 modify header actions is supported. 475 * - reg_c's are preserved across different domain (FDB and NIC) on 476 * packet loopback by flow lookup miss. 477 */ 478 return config->flow_mreg_c[2] != REG_NONE; 479 } 480 481 /** 482 * Discover the maximum number of priority available. 483 * 484 * @param[in] dev 485 * Pointer to the Ethernet device structure. 486 * 487 * @return 488 * number of supported flow priority on success, a negative errno 489 * value otherwise and rte_errno is set. 490 */ 491 int 492 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 493 { 494 struct mlx5_priv *priv = dev->data->dev_private; 495 struct { 496 struct ibv_flow_attr attr; 497 struct ibv_flow_spec_eth eth; 498 struct ibv_flow_spec_action_drop drop; 499 } flow_attr = { 500 .attr = { 501 .num_of_specs = 2, 502 .port = (uint8_t)priv->ibv_port, 503 }, 504 .eth = { 505 .type = IBV_FLOW_SPEC_ETH, 506 .size = sizeof(struct ibv_flow_spec_eth), 507 }, 508 .drop = { 509 .size = sizeof(struct ibv_flow_spec_action_drop), 510 .type = IBV_FLOW_SPEC_ACTION_DROP, 511 }, 512 }; 513 struct ibv_flow *flow; 514 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 515 uint16_t vprio[] = { 8, 16 }; 516 int i; 517 int priority = 0; 518 519 if (!drop) { 520 rte_errno = ENOTSUP; 521 return -rte_errno; 522 } 523 for (i = 0; i != RTE_DIM(vprio); i++) { 524 flow_attr.attr.priority = vprio[i] - 1; 525 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 526 if (!flow) 527 break; 528 claim_zero(mlx5_glue->destroy_flow(flow)); 529 priority = vprio[i]; 530 } 531 mlx5_hrxq_drop_release(dev); 532 switch (priority) { 533 case 8: 534 priority = RTE_DIM(priority_map_3); 535 break; 536 case 16: 537 priority = RTE_DIM(priority_map_5); 538 break; 539 default: 540 rte_errno = ENOTSUP; 541 DRV_LOG(ERR, 542 "port %u verbs maximum priority: %d expected 8/16", 543 dev->data->port_id, priority); 544 return -rte_errno; 545 } 546 DRV_LOG(INFO, "port %u flow maximum priority: %d", 547 dev->data->port_id, priority); 548 return priority; 549 } 550 551 /** 552 * Adjust flow priority based on the highest layer and the request priority. 553 * 554 * @param[in] dev 555 * Pointer to the Ethernet device structure. 556 * @param[in] priority 557 * The rule base priority. 558 * @param[in] subpriority 559 * The priority based on the items. 560 * 561 * @return 562 * The new priority. 563 */ 564 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 565 uint32_t subpriority) 566 { 567 uint32_t res = 0; 568 struct mlx5_priv *priv = dev->data->dev_private; 569 570 switch (priv->config.flow_prio) { 571 case RTE_DIM(priority_map_3): 572 res = priority_map_3[priority][subpriority]; 573 break; 574 case RTE_DIM(priority_map_5): 575 res = priority_map_5[priority][subpriority]; 576 break; 577 } 578 return res; 579 } 580 581 /** 582 * Verify the @p item specifications (spec, last, mask) are compatible with the 583 * NIC capabilities. 584 * 585 * @param[in] item 586 * Item specification. 587 * @param[in] mask 588 * @p item->mask or flow default bit-masks. 589 * @param[in] nic_mask 590 * Bit-masks covering supported fields by the NIC to compare with user mask. 591 * @param[in] size 592 * Bit-masks size in bytes. 593 * @param[out] error 594 * Pointer to error structure. 595 * 596 * @return 597 * 0 on success, a negative errno value otherwise and rte_errno is set. 598 */ 599 int 600 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 601 const uint8_t *mask, 602 const uint8_t *nic_mask, 603 unsigned int size, 604 struct rte_flow_error *error) 605 { 606 unsigned int i; 607 608 assert(nic_mask); 609 for (i = 0; i < size; ++i) 610 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 611 return rte_flow_error_set(error, ENOTSUP, 612 RTE_FLOW_ERROR_TYPE_ITEM, 613 item, 614 "mask enables non supported" 615 " bits"); 616 if (!item->spec && (item->mask || item->last)) 617 return rte_flow_error_set(error, EINVAL, 618 RTE_FLOW_ERROR_TYPE_ITEM, item, 619 "mask/last without a spec is not" 620 " supported"); 621 if (item->spec && item->last) { 622 uint8_t spec[size]; 623 uint8_t last[size]; 624 unsigned int i; 625 int ret; 626 627 for (i = 0; i < size; ++i) { 628 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 629 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 630 } 631 ret = memcmp(spec, last, size); 632 if (ret != 0) 633 return rte_flow_error_set(error, EINVAL, 634 RTE_FLOW_ERROR_TYPE_ITEM, 635 item, 636 "range is not valid"); 637 } 638 return 0; 639 } 640 641 /** 642 * Adjust the hash fields according to the @p flow information. 643 * 644 * @param[in] dev_flow. 645 * Pointer to the mlx5_flow. 646 * @param[in] tunnel 647 * 1 when the hash field is for a tunnel item. 648 * @param[in] layer_types 649 * ETH_RSS_* types. 650 * @param[in] hash_fields 651 * Item hash fields. 652 * 653 * @return 654 * The hash fields that should be used. 655 */ 656 uint64_t 657 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 658 int tunnel __rte_unused, uint64_t layer_types, 659 uint64_t hash_fields) 660 { 661 struct rte_flow *flow = dev_flow->flow; 662 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 663 int rss_request_inner = flow->rss.level >= 2; 664 665 /* Check RSS hash level for tunnel. */ 666 if (tunnel && rss_request_inner) 667 hash_fields |= IBV_RX_HASH_INNER; 668 else if (tunnel || rss_request_inner) 669 return 0; 670 #endif 671 /* Check if requested layer matches RSS hash fields. */ 672 if (!(flow->rss.types & layer_types)) 673 return 0; 674 return hash_fields; 675 } 676 677 /** 678 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 679 * if several tunnel rules are used on this queue, the tunnel ptype will be 680 * cleared. 681 * 682 * @param rxq_ctrl 683 * Rx queue to update. 684 */ 685 static void 686 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 687 { 688 unsigned int i; 689 uint32_t tunnel_ptype = 0; 690 691 /* Look up for the ptype to use. */ 692 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 693 if (!rxq_ctrl->flow_tunnels_n[i]) 694 continue; 695 if (!tunnel_ptype) { 696 tunnel_ptype = tunnels_info[i].ptype; 697 } else { 698 tunnel_ptype = 0; 699 break; 700 } 701 } 702 rxq_ctrl->rxq.tunnel = tunnel_ptype; 703 } 704 705 /** 706 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 707 * flow. 708 * 709 * @param[in] dev 710 * Pointer to the Ethernet device structure. 711 * @param[in] dev_flow 712 * Pointer to device flow structure. 713 */ 714 static void 715 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 716 { 717 struct mlx5_priv *priv = dev->data->dev_private; 718 struct rte_flow *flow = dev_flow->flow; 719 const int mark = !!(dev_flow->actions & 720 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 721 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 722 unsigned int i; 723 724 for (i = 0; i != flow->rss.queue_num; ++i) { 725 int idx = (*flow->rss.queue)[i]; 726 struct mlx5_rxq_ctrl *rxq_ctrl = 727 container_of((*priv->rxqs)[idx], 728 struct mlx5_rxq_ctrl, rxq); 729 730 /* 731 * To support metadata register copy on Tx loopback, 732 * this must be always enabled (metadata may arive 733 * from other port - not from local flows only. 734 */ 735 if (priv->config.dv_flow_en && 736 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 737 mlx5_flow_ext_mreg_supported(dev)) { 738 rxq_ctrl->rxq.mark = 1; 739 rxq_ctrl->flow_mark_n = 1; 740 } else if (mark) { 741 rxq_ctrl->rxq.mark = 1; 742 rxq_ctrl->flow_mark_n++; 743 } 744 if (tunnel) { 745 unsigned int j; 746 747 /* Increase the counter matching the flow. */ 748 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 749 if ((tunnels_info[j].tunnel & 750 dev_flow->layers) == 751 tunnels_info[j].tunnel) { 752 rxq_ctrl->flow_tunnels_n[j]++; 753 break; 754 } 755 } 756 flow_rxq_tunnel_ptype_update(rxq_ctrl); 757 } 758 } 759 } 760 761 /** 762 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 763 * 764 * @param[in] dev 765 * Pointer to the Ethernet device structure. 766 * @param[in] flow 767 * Pointer to flow structure. 768 */ 769 static void 770 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 771 { 772 struct mlx5_flow *dev_flow; 773 774 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 775 flow_drv_rxq_flags_set(dev, dev_flow); 776 } 777 778 /** 779 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 780 * device flow if no other flow uses it with the same kind of request. 781 * 782 * @param dev 783 * Pointer to Ethernet device. 784 * @param[in] dev_flow 785 * Pointer to the device flow. 786 */ 787 static void 788 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 789 { 790 struct mlx5_priv *priv = dev->data->dev_private; 791 struct rte_flow *flow = dev_flow->flow; 792 const int mark = !!(dev_flow->actions & 793 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 794 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 795 unsigned int i; 796 797 assert(dev->data->dev_started); 798 for (i = 0; i != flow->rss.queue_num; ++i) { 799 int idx = (*flow->rss.queue)[i]; 800 struct mlx5_rxq_ctrl *rxq_ctrl = 801 container_of((*priv->rxqs)[idx], 802 struct mlx5_rxq_ctrl, rxq); 803 804 if (priv->config.dv_flow_en && 805 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 806 mlx5_flow_ext_mreg_supported(dev)) { 807 rxq_ctrl->rxq.mark = 1; 808 rxq_ctrl->flow_mark_n = 1; 809 } else if (mark) { 810 rxq_ctrl->flow_mark_n--; 811 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 812 } 813 if (tunnel) { 814 unsigned int j; 815 816 /* Decrease the counter matching the flow. */ 817 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 818 if ((tunnels_info[j].tunnel & 819 dev_flow->layers) == 820 tunnels_info[j].tunnel) { 821 rxq_ctrl->flow_tunnels_n[j]--; 822 break; 823 } 824 } 825 flow_rxq_tunnel_ptype_update(rxq_ctrl); 826 } 827 } 828 } 829 830 /** 831 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 832 * @p flow if no other flow uses it with the same kind of request. 833 * 834 * @param dev 835 * Pointer to Ethernet device. 836 * @param[in] flow 837 * Pointer to the flow. 838 */ 839 static void 840 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 841 { 842 struct mlx5_flow *dev_flow; 843 844 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 845 flow_drv_rxq_flags_trim(dev, dev_flow); 846 } 847 848 /** 849 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 850 * 851 * @param dev 852 * Pointer to Ethernet device. 853 */ 854 static void 855 flow_rxq_flags_clear(struct rte_eth_dev *dev) 856 { 857 struct mlx5_priv *priv = dev->data->dev_private; 858 unsigned int i; 859 860 for (i = 0; i != priv->rxqs_n; ++i) { 861 struct mlx5_rxq_ctrl *rxq_ctrl; 862 unsigned int j; 863 864 if (!(*priv->rxqs)[i]) 865 continue; 866 rxq_ctrl = container_of((*priv->rxqs)[i], 867 struct mlx5_rxq_ctrl, rxq); 868 rxq_ctrl->flow_mark_n = 0; 869 rxq_ctrl->rxq.mark = 0; 870 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 871 rxq_ctrl->flow_tunnels_n[j] = 0; 872 rxq_ctrl->rxq.tunnel = 0; 873 } 874 } 875 876 /* 877 * return a pointer to the desired action in the list of actions. 878 * 879 * @param[in] actions 880 * The list of actions to search the action in. 881 * @param[in] action 882 * The action to find. 883 * 884 * @return 885 * Pointer to the action in the list, if found. NULL otherwise. 886 */ 887 const struct rte_flow_action * 888 mlx5_flow_find_action(const struct rte_flow_action *actions, 889 enum rte_flow_action_type action) 890 { 891 if (actions == NULL) 892 return NULL; 893 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 894 if (actions->type == action) 895 return actions; 896 return NULL; 897 } 898 899 /* 900 * Validate the flag action. 901 * 902 * @param[in] action_flags 903 * Bit-fields that holds the actions detected until now. 904 * @param[in] attr 905 * Attributes of flow that includes this action. 906 * @param[out] error 907 * Pointer to error structure. 908 * 909 * @return 910 * 0 on success, a negative errno value otherwise and rte_errno is set. 911 */ 912 int 913 mlx5_flow_validate_action_flag(uint64_t action_flags, 914 const struct rte_flow_attr *attr, 915 struct rte_flow_error *error) 916 { 917 if (action_flags & MLX5_FLOW_ACTION_MARK) 918 return rte_flow_error_set(error, EINVAL, 919 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 920 "can't mark and flag in same flow"); 921 if (action_flags & MLX5_FLOW_ACTION_FLAG) 922 return rte_flow_error_set(error, EINVAL, 923 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 924 "can't have 2 flag" 925 " actions in same flow"); 926 if (attr->egress) 927 return rte_flow_error_set(error, ENOTSUP, 928 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 929 "flag action not supported for " 930 "egress"); 931 return 0; 932 } 933 934 /* 935 * Validate the mark action. 936 * 937 * @param[in] action 938 * Pointer to the queue action. 939 * @param[in] action_flags 940 * Bit-fields that holds the actions detected until now. 941 * @param[in] attr 942 * Attributes of flow that includes this action. 943 * @param[out] error 944 * Pointer to error structure. 945 * 946 * @return 947 * 0 on success, a negative errno value otherwise and rte_errno is set. 948 */ 949 int 950 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 951 uint64_t action_flags, 952 const struct rte_flow_attr *attr, 953 struct rte_flow_error *error) 954 { 955 const struct rte_flow_action_mark *mark = action->conf; 956 957 if (!mark) 958 return rte_flow_error_set(error, EINVAL, 959 RTE_FLOW_ERROR_TYPE_ACTION, 960 action, 961 "configuration cannot be null"); 962 if (mark->id >= MLX5_FLOW_MARK_MAX) 963 return rte_flow_error_set(error, EINVAL, 964 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 965 &mark->id, 966 "mark id must in 0 <= id < " 967 RTE_STR(MLX5_FLOW_MARK_MAX)); 968 if (action_flags & MLX5_FLOW_ACTION_FLAG) 969 return rte_flow_error_set(error, EINVAL, 970 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 971 "can't flag and mark in same flow"); 972 if (action_flags & MLX5_FLOW_ACTION_MARK) 973 return rte_flow_error_set(error, EINVAL, 974 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 975 "can't have 2 mark actions in same" 976 " flow"); 977 if (attr->egress) 978 return rte_flow_error_set(error, ENOTSUP, 979 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 980 "mark action not supported for " 981 "egress"); 982 return 0; 983 } 984 985 /* 986 * Validate the drop action. 987 * 988 * @param[in] action_flags 989 * Bit-fields that holds the actions detected until now. 990 * @param[in] attr 991 * Attributes of flow that includes this action. 992 * @param[out] error 993 * Pointer to error structure. 994 * 995 * @return 996 * 0 on success, a negative errno value otherwise and rte_errno is set. 997 */ 998 int 999 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1000 const struct rte_flow_attr *attr, 1001 struct rte_flow_error *error) 1002 { 1003 if (attr->egress) 1004 return rte_flow_error_set(error, ENOTSUP, 1005 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1006 "drop action not supported for " 1007 "egress"); 1008 return 0; 1009 } 1010 1011 /* 1012 * Validate the queue action. 1013 * 1014 * @param[in] action 1015 * Pointer to the queue action. 1016 * @param[in] action_flags 1017 * Bit-fields that holds the actions detected until now. 1018 * @param[in] dev 1019 * Pointer to the Ethernet device structure. 1020 * @param[in] attr 1021 * Attributes of flow that includes this action. 1022 * @param[out] error 1023 * Pointer to error structure. 1024 * 1025 * @return 1026 * 0 on success, a negative errno value otherwise and rte_errno is set. 1027 */ 1028 int 1029 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1030 uint64_t action_flags, 1031 struct rte_eth_dev *dev, 1032 const struct rte_flow_attr *attr, 1033 struct rte_flow_error *error) 1034 { 1035 struct mlx5_priv *priv = dev->data->dev_private; 1036 const struct rte_flow_action_queue *queue = action->conf; 1037 1038 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1039 return rte_flow_error_set(error, EINVAL, 1040 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1041 "can't have 2 fate actions in" 1042 " same flow"); 1043 if (!priv->rxqs_n) 1044 return rte_flow_error_set(error, EINVAL, 1045 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1046 NULL, "No Rx queues configured"); 1047 if (queue->index >= priv->rxqs_n) 1048 return rte_flow_error_set(error, EINVAL, 1049 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1050 &queue->index, 1051 "queue index out of range"); 1052 if (!(*priv->rxqs)[queue->index]) 1053 return rte_flow_error_set(error, EINVAL, 1054 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1055 &queue->index, 1056 "queue is not configured"); 1057 if (attr->egress) 1058 return rte_flow_error_set(error, ENOTSUP, 1059 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1060 "queue action not supported for " 1061 "egress"); 1062 return 0; 1063 } 1064 1065 /* 1066 * Validate the rss action. 1067 * 1068 * @param[in] action 1069 * Pointer to the queue action. 1070 * @param[in] action_flags 1071 * Bit-fields that holds the actions detected until now. 1072 * @param[in] dev 1073 * Pointer to the Ethernet device structure. 1074 * @param[in] attr 1075 * Attributes of flow that includes this action. 1076 * @param[in] item_flags 1077 * Items that were detected. 1078 * @param[out] error 1079 * Pointer to error structure. 1080 * 1081 * @return 1082 * 0 on success, a negative errno value otherwise and rte_errno is set. 1083 */ 1084 int 1085 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1086 uint64_t action_flags, 1087 struct rte_eth_dev *dev, 1088 const struct rte_flow_attr *attr, 1089 uint64_t item_flags, 1090 struct rte_flow_error *error) 1091 { 1092 struct mlx5_priv *priv = dev->data->dev_private; 1093 const struct rte_flow_action_rss *rss = action->conf; 1094 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1095 unsigned int i; 1096 1097 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1098 return rte_flow_error_set(error, EINVAL, 1099 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1100 "can't have 2 fate actions" 1101 " in same flow"); 1102 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1103 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1104 return rte_flow_error_set(error, ENOTSUP, 1105 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1106 &rss->func, 1107 "RSS hash function not supported"); 1108 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1109 if (rss->level > 2) 1110 #else 1111 if (rss->level > 1) 1112 #endif 1113 return rte_flow_error_set(error, ENOTSUP, 1114 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1115 &rss->level, 1116 "tunnel RSS is not supported"); 1117 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1118 if (rss->key_len == 0 && rss->key != NULL) 1119 return rte_flow_error_set(error, ENOTSUP, 1120 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1121 &rss->key_len, 1122 "RSS hash key length 0"); 1123 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1124 return rte_flow_error_set(error, ENOTSUP, 1125 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1126 &rss->key_len, 1127 "RSS hash key too small"); 1128 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1129 return rte_flow_error_set(error, ENOTSUP, 1130 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1131 &rss->key_len, 1132 "RSS hash key too large"); 1133 if (rss->queue_num > priv->config.ind_table_max_size) 1134 return rte_flow_error_set(error, ENOTSUP, 1135 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1136 &rss->queue_num, 1137 "number of queues too large"); 1138 if (rss->types & MLX5_RSS_HF_MASK) 1139 return rte_flow_error_set(error, ENOTSUP, 1140 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1141 &rss->types, 1142 "some RSS protocols are not" 1143 " supported"); 1144 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1145 !(rss->types & ETH_RSS_IP)) 1146 return rte_flow_error_set(error, EINVAL, 1147 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1148 "L3 partial RSS requested but L3 RSS" 1149 " type not specified"); 1150 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1151 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1152 return rte_flow_error_set(error, EINVAL, 1153 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1154 "L4 partial RSS requested but L4 RSS" 1155 " type not specified"); 1156 if (!priv->rxqs_n) 1157 return rte_flow_error_set(error, EINVAL, 1158 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1159 NULL, "No Rx queues configured"); 1160 if (!rss->queue_num) 1161 return rte_flow_error_set(error, EINVAL, 1162 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1163 NULL, "No queues configured"); 1164 for (i = 0; i != rss->queue_num; ++i) { 1165 if (rss->queue[i] >= priv->rxqs_n) 1166 return rte_flow_error_set 1167 (error, EINVAL, 1168 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1169 &rss->queue[i], "queue index out of range"); 1170 if (!(*priv->rxqs)[rss->queue[i]]) 1171 return rte_flow_error_set 1172 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1173 &rss->queue[i], "queue is not configured"); 1174 } 1175 if (attr->egress) 1176 return rte_flow_error_set(error, ENOTSUP, 1177 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1178 "rss action not supported for " 1179 "egress"); 1180 if (rss->level > 1 && !tunnel) 1181 return rte_flow_error_set(error, EINVAL, 1182 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1183 "inner RSS is not supported for " 1184 "non-tunnel flows"); 1185 return 0; 1186 } 1187 1188 /* 1189 * Validate the count action. 1190 * 1191 * @param[in] dev 1192 * Pointer to the Ethernet device structure. 1193 * @param[in] attr 1194 * Attributes of flow that includes this action. 1195 * @param[out] error 1196 * Pointer to error structure. 1197 * 1198 * @return 1199 * 0 on success, a negative errno value otherwise and rte_errno is set. 1200 */ 1201 int 1202 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1203 const struct rte_flow_attr *attr, 1204 struct rte_flow_error *error) 1205 { 1206 if (attr->egress) 1207 return rte_flow_error_set(error, ENOTSUP, 1208 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1209 "count action not supported for " 1210 "egress"); 1211 return 0; 1212 } 1213 1214 /** 1215 * Verify the @p attributes will be correctly understood by the NIC and store 1216 * them in the @p flow if everything is correct. 1217 * 1218 * @param[in] dev 1219 * Pointer to the Ethernet device structure. 1220 * @param[in] attributes 1221 * Pointer to flow attributes 1222 * @param[out] error 1223 * Pointer to error structure. 1224 * 1225 * @return 1226 * 0 on success, a negative errno value otherwise and rte_errno is set. 1227 */ 1228 int 1229 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1230 const struct rte_flow_attr *attributes, 1231 struct rte_flow_error *error) 1232 { 1233 struct mlx5_priv *priv = dev->data->dev_private; 1234 uint32_t priority_max = priv->config.flow_prio - 1; 1235 1236 if (attributes->group) 1237 return rte_flow_error_set(error, ENOTSUP, 1238 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1239 NULL, "groups is not supported"); 1240 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1241 attributes->priority >= priority_max) 1242 return rte_flow_error_set(error, ENOTSUP, 1243 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1244 NULL, "priority out of range"); 1245 if (attributes->egress) 1246 return rte_flow_error_set(error, ENOTSUP, 1247 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1248 "egress is not supported"); 1249 if (attributes->transfer && !priv->config.dv_esw_en) 1250 return rte_flow_error_set(error, ENOTSUP, 1251 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1252 NULL, "transfer is not supported"); 1253 if (!attributes->ingress) 1254 return rte_flow_error_set(error, EINVAL, 1255 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1256 NULL, 1257 "ingress attribute is mandatory"); 1258 return 0; 1259 } 1260 1261 /** 1262 * Validate ICMP6 item. 1263 * 1264 * @param[in] item 1265 * Item specification. 1266 * @param[in] item_flags 1267 * Bit-fields that holds the items detected until now. 1268 * @param[out] error 1269 * Pointer to error structure. 1270 * 1271 * @return 1272 * 0 on success, a negative errno value otherwise and rte_errno is set. 1273 */ 1274 int 1275 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1276 uint64_t item_flags, 1277 uint8_t target_protocol, 1278 struct rte_flow_error *error) 1279 { 1280 const struct rte_flow_item_icmp6 *mask = item->mask; 1281 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1282 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1283 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1284 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1285 MLX5_FLOW_LAYER_OUTER_L4; 1286 int ret; 1287 1288 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1289 return rte_flow_error_set(error, EINVAL, 1290 RTE_FLOW_ERROR_TYPE_ITEM, item, 1291 "protocol filtering not compatible" 1292 " with ICMP6 layer"); 1293 if (!(item_flags & l3m)) 1294 return rte_flow_error_set(error, EINVAL, 1295 RTE_FLOW_ERROR_TYPE_ITEM, item, 1296 "IPv6 is mandatory to filter on" 1297 " ICMP6"); 1298 if (item_flags & l4m) 1299 return rte_flow_error_set(error, EINVAL, 1300 RTE_FLOW_ERROR_TYPE_ITEM, item, 1301 "multiple L4 layers not supported"); 1302 if (!mask) 1303 mask = &rte_flow_item_icmp6_mask; 1304 ret = mlx5_flow_item_acceptable 1305 (item, (const uint8_t *)mask, 1306 (const uint8_t *)&rte_flow_item_icmp6_mask, 1307 sizeof(struct rte_flow_item_icmp6), error); 1308 if (ret < 0) 1309 return ret; 1310 return 0; 1311 } 1312 1313 /** 1314 * Validate ICMP item. 1315 * 1316 * @param[in] item 1317 * Item specification. 1318 * @param[in] item_flags 1319 * Bit-fields that holds the items detected until now. 1320 * @param[out] error 1321 * Pointer to error structure. 1322 * 1323 * @return 1324 * 0 on success, a negative errno value otherwise and rte_errno is set. 1325 */ 1326 int 1327 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1328 uint64_t item_flags, 1329 uint8_t target_protocol, 1330 struct rte_flow_error *error) 1331 { 1332 const struct rte_flow_item_icmp *mask = item->mask; 1333 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1334 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1335 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1336 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1337 MLX5_FLOW_LAYER_OUTER_L4; 1338 int ret; 1339 1340 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1341 return rte_flow_error_set(error, EINVAL, 1342 RTE_FLOW_ERROR_TYPE_ITEM, item, 1343 "protocol filtering not compatible" 1344 " with ICMP layer"); 1345 if (!(item_flags & l3m)) 1346 return rte_flow_error_set(error, EINVAL, 1347 RTE_FLOW_ERROR_TYPE_ITEM, item, 1348 "IPv4 is mandatory to filter" 1349 " on ICMP"); 1350 if (item_flags & l4m) 1351 return rte_flow_error_set(error, EINVAL, 1352 RTE_FLOW_ERROR_TYPE_ITEM, item, 1353 "multiple L4 layers not supported"); 1354 if (!mask) 1355 mask = &rte_flow_item_icmp_mask; 1356 ret = mlx5_flow_item_acceptable 1357 (item, (const uint8_t *)mask, 1358 (const uint8_t *)&rte_flow_item_icmp_mask, 1359 sizeof(struct rte_flow_item_icmp), error); 1360 if (ret < 0) 1361 return ret; 1362 return 0; 1363 } 1364 1365 /** 1366 * Validate Ethernet item. 1367 * 1368 * @param[in] item 1369 * Item specification. 1370 * @param[in] item_flags 1371 * Bit-fields that holds the items detected until now. 1372 * @param[out] error 1373 * Pointer to error structure. 1374 * 1375 * @return 1376 * 0 on success, a negative errno value otherwise and rte_errno is set. 1377 */ 1378 int 1379 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1380 uint64_t item_flags, 1381 struct rte_flow_error *error) 1382 { 1383 const struct rte_flow_item_eth *mask = item->mask; 1384 const struct rte_flow_item_eth nic_mask = { 1385 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1386 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1387 .type = RTE_BE16(0xffff), 1388 }; 1389 int ret; 1390 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1391 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1392 MLX5_FLOW_LAYER_OUTER_L2; 1393 1394 if (item_flags & ethm) 1395 return rte_flow_error_set(error, ENOTSUP, 1396 RTE_FLOW_ERROR_TYPE_ITEM, item, 1397 "multiple L2 layers not supported"); 1398 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1399 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1400 return rte_flow_error_set(error, EINVAL, 1401 RTE_FLOW_ERROR_TYPE_ITEM, item, 1402 "L2 layer should not follow " 1403 "L3 layers"); 1404 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1405 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1406 return rte_flow_error_set(error, EINVAL, 1407 RTE_FLOW_ERROR_TYPE_ITEM, item, 1408 "L2 layer should not follow VLAN"); 1409 if (!mask) 1410 mask = &rte_flow_item_eth_mask; 1411 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1412 (const uint8_t *)&nic_mask, 1413 sizeof(struct rte_flow_item_eth), 1414 error); 1415 return ret; 1416 } 1417 1418 /** 1419 * Validate VLAN item. 1420 * 1421 * @param[in] item 1422 * Item specification. 1423 * @param[in] item_flags 1424 * Bit-fields that holds the items detected until now. 1425 * @param[in] dev 1426 * Ethernet device flow is being created on. 1427 * @param[out] error 1428 * Pointer to error structure. 1429 * 1430 * @return 1431 * 0 on success, a negative errno value otherwise and rte_errno is set. 1432 */ 1433 int 1434 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1435 uint64_t item_flags, 1436 struct rte_eth_dev *dev, 1437 struct rte_flow_error *error) 1438 { 1439 const struct rte_flow_item_vlan *spec = item->spec; 1440 const struct rte_flow_item_vlan *mask = item->mask; 1441 const struct rte_flow_item_vlan nic_mask = { 1442 .tci = RTE_BE16(UINT16_MAX), 1443 .inner_type = RTE_BE16(UINT16_MAX), 1444 }; 1445 uint16_t vlan_tag = 0; 1446 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1447 int ret; 1448 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1449 MLX5_FLOW_LAYER_INNER_L4) : 1450 (MLX5_FLOW_LAYER_OUTER_L3 | 1451 MLX5_FLOW_LAYER_OUTER_L4); 1452 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1453 MLX5_FLOW_LAYER_OUTER_VLAN; 1454 1455 if (item_flags & vlanm) 1456 return rte_flow_error_set(error, EINVAL, 1457 RTE_FLOW_ERROR_TYPE_ITEM, item, 1458 "multiple VLAN layers not supported"); 1459 else if ((item_flags & l34m) != 0) 1460 return rte_flow_error_set(error, EINVAL, 1461 RTE_FLOW_ERROR_TYPE_ITEM, item, 1462 "VLAN cannot follow L3/L4 layer"); 1463 if (!mask) 1464 mask = &rte_flow_item_vlan_mask; 1465 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1466 (const uint8_t *)&nic_mask, 1467 sizeof(struct rte_flow_item_vlan), 1468 error); 1469 if (ret) 1470 return ret; 1471 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1472 struct mlx5_priv *priv = dev->data->dev_private; 1473 1474 if (priv->vmwa_context) { 1475 /* 1476 * Non-NULL context means we have a virtual machine 1477 * and SR-IOV enabled, we have to create VLAN interface 1478 * to make hypervisor to setup E-Switch vport 1479 * context correctly. We avoid creating the multiple 1480 * VLAN interfaces, so we cannot support VLAN tag mask. 1481 */ 1482 return rte_flow_error_set(error, EINVAL, 1483 RTE_FLOW_ERROR_TYPE_ITEM, 1484 item, 1485 "VLAN tag mask is not" 1486 " supported in virtual" 1487 " environment"); 1488 } 1489 } 1490 if (spec) { 1491 vlan_tag = spec->tci; 1492 vlan_tag &= mask->tci; 1493 } 1494 /* 1495 * From verbs perspective an empty VLAN is equivalent 1496 * to a packet without VLAN layer. 1497 */ 1498 if (!vlan_tag) 1499 return rte_flow_error_set(error, EINVAL, 1500 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1501 item->spec, 1502 "VLAN cannot be empty"); 1503 return 0; 1504 } 1505 1506 /** 1507 * Validate IPV4 item. 1508 * 1509 * @param[in] item 1510 * Item specification. 1511 * @param[in] item_flags 1512 * Bit-fields that holds the items detected until now. 1513 * @param[in] acc_mask 1514 * Acceptable mask, if NULL default internal default mask 1515 * will be used to check whether item fields are supported. 1516 * @param[out] error 1517 * Pointer to error structure. 1518 * 1519 * @return 1520 * 0 on success, a negative errno value otherwise and rte_errno is set. 1521 */ 1522 int 1523 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1524 uint64_t item_flags, 1525 uint64_t last_item, 1526 uint16_t ether_type, 1527 const struct rte_flow_item_ipv4 *acc_mask, 1528 struct rte_flow_error *error) 1529 { 1530 const struct rte_flow_item_ipv4 *mask = item->mask; 1531 const struct rte_flow_item_ipv4 *spec = item->spec; 1532 const struct rte_flow_item_ipv4 nic_mask = { 1533 .hdr = { 1534 .src_addr = RTE_BE32(0xffffffff), 1535 .dst_addr = RTE_BE32(0xffffffff), 1536 .type_of_service = 0xff, 1537 .next_proto_id = 0xff, 1538 }, 1539 }; 1540 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1541 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1542 MLX5_FLOW_LAYER_OUTER_L3; 1543 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1544 MLX5_FLOW_LAYER_OUTER_L4; 1545 int ret; 1546 uint8_t next_proto = 0xFF; 1547 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1548 MLX5_FLOW_LAYER_OUTER_VLAN | 1549 MLX5_FLOW_LAYER_INNER_VLAN); 1550 1551 if ((last_item & l2_vlan) && ether_type && 1552 ether_type != RTE_ETHER_TYPE_IPV4) 1553 return rte_flow_error_set(error, EINVAL, 1554 RTE_FLOW_ERROR_TYPE_ITEM, item, 1555 "IPv4 cannot follow L2/VLAN layer " 1556 "which ether type is not IPv4"); 1557 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1558 if (mask && spec) 1559 next_proto = mask->hdr.next_proto_id & 1560 spec->hdr.next_proto_id; 1561 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1562 return rte_flow_error_set(error, EINVAL, 1563 RTE_FLOW_ERROR_TYPE_ITEM, 1564 item, 1565 "multiple tunnel " 1566 "not supported"); 1567 } 1568 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1569 return rte_flow_error_set(error, EINVAL, 1570 RTE_FLOW_ERROR_TYPE_ITEM, item, 1571 "wrong tunnel type - IPv6 specified " 1572 "but IPv4 item provided"); 1573 if (item_flags & l3m) 1574 return rte_flow_error_set(error, ENOTSUP, 1575 RTE_FLOW_ERROR_TYPE_ITEM, item, 1576 "multiple L3 layers not supported"); 1577 else if (item_flags & l4m) 1578 return rte_flow_error_set(error, EINVAL, 1579 RTE_FLOW_ERROR_TYPE_ITEM, item, 1580 "L3 cannot follow an L4 layer."); 1581 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1582 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1583 return rte_flow_error_set(error, EINVAL, 1584 RTE_FLOW_ERROR_TYPE_ITEM, item, 1585 "L3 cannot follow an NVGRE layer."); 1586 if (!mask) 1587 mask = &rte_flow_item_ipv4_mask; 1588 else if (mask->hdr.next_proto_id != 0 && 1589 mask->hdr.next_proto_id != 0xff) 1590 return rte_flow_error_set(error, EINVAL, 1591 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1592 "partial mask is not supported" 1593 " for protocol"); 1594 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1595 acc_mask ? (const uint8_t *)acc_mask 1596 : (const uint8_t *)&nic_mask, 1597 sizeof(struct rte_flow_item_ipv4), 1598 error); 1599 if (ret < 0) 1600 return ret; 1601 return 0; 1602 } 1603 1604 /** 1605 * Validate IPV6 item. 1606 * 1607 * @param[in] item 1608 * Item specification. 1609 * @param[in] item_flags 1610 * Bit-fields that holds the items detected until now. 1611 * @param[in] acc_mask 1612 * Acceptable mask, if NULL default internal default mask 1613 * will be used to check whether item fields are supported. 1614 * @param[out] error 1615 * Pointer to error structure. 1616 * 1617 * @return 1618 * 0 on success, a negative errno value otherwise and rte_errno is set. 1619 */ 1620 int 1621 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1622 uint64_t item_flags, 1623 uint64_t last_item, 1624 uint16_t ether_type, 1625 const struct rte_flow_item_ipv6 *acc_mask, 1626 struct rte_flow_error *error) 1627 { 1628 const struct rte_flow_item_ipv6 *mask = item->mask; 1629 const struct rte_flow_item_ipv6 *spec = item->spec; 1630 const struct rte_flow_item_ipv6 nic_mask = { 1631 .hdr = { 1632 .src_addr = 1633 "\xff\xff\xff\xff\xff\xff\xff\xff" 1634 "\xff\xff\xff\xff\xff\xff\xff\xff", 1635 .dst_addr = 1636 "\xff\xff\xff\xff\xff\xff\xff\xff" 1637 "\xff\xff\xff\xff\xff\xff\xff\xff", 1638 .vtc_flow = RTE_BE32(0xffffffff), 1639 .proto = 0xff, 1640 .hop_limits = 0xff, 1641 }, 1642 }; 1643 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1644 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1645 MLX5_FLOW_LAYER_OUTER_L3; 1646 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1647 MLX5_FLOW_LAYER_OUTER_L4; 1648 int ret; 1649 uint8_t next_proto = 0xFF; 1650 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1651 MLX5_FLOW_LAYER_OUTER_VLAN | 1652 MLX5_FLOW_LAYER_INNER_VLAN); 1653 1654 if ((last_item & l2_vlan) && ether_type && 1655 ether_type != RTE_ETHER_TYPE_IPV6) 1656 return rte_flow_error_set(error, EINVAL, 1657 RTE_FLOW_ERROR_TYPE_ITEM, item, 1658 "IPv6 cannot follow L2/VLAN layer " 1659 "which ether type is not IPv6"); 1660 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1661 if (mask && spec) 1662 next_proto = mask->hdr.proto & spec->hdr.proto; 1663 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1664 return rte_flow_error_set(error, EINVAL, 1665 RTE_FLOW_ERROR_TYPE_ITEM, 1666 item, 1667 "multiple tunnel " 1668 "not supported"); 1669 } 1670 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1671 return rte_flow_error_set(error, EINVAL, 1672 RTE_FLOW_ERROR_TYPE_ITEM, item, 1673 "wrong tunnel type - IPv4 specified " 1674 "but IPv6 item provided"); 1675 if (item_flags & l3m) 1676 return rte_flow_error_set(error, ENOTSUP, 1677 RTE_FLOW_ERROR_TYPE_ITEM, item, 1678 "multiple L3 layers not supported"); 1679 else if (item_flags & l4m) 1680 return rte_flow_error_set(error, EINVAL, 1681 RTE_FLOW_ERROR_TYPE_ITEM, item, 1682 "L3 cannot follow an L4 layer."); 1683 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1684 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1685 return rte_flow_error_set(error, EINVAL, 1686 RTE_FLOW_ERROR_TYPE_ITEM, item, 1687 "L3 cannot follow an NVGRE layer."); 1688 if (!mask) 1689 mask = &rte_flow_item_ipv6_mask; 1690 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1691 acc_mask ? (const uint8_t *)acc_mask 1692 : (const uint8_t *)&nic_mask, 1693 sizeof(struct rte_flow_item_ipv6), 1694 error); 1695 if (ret < 0) 1696 return ret; 1697 return 0; 1698 } 1699 1700 /** 1701 * Validate UDP item. 1702 * 1703 * @param[in] item 1704 * Item specification. 1705 * @param[in] item_flags 1706 * Bit-fields that holds the items detected until now. 1707 * @param[in] target_protocol 1708 * The next protocol in the previous item. 1709 * @param[in] flow_mask 1710 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1711 * @param[out] error 1712 * Pointer to error structure. 1713 * 1714 * @return 1715 * 0 on success, a negative errno value otherwise and rte_errno is set. 1716 */ 1717 int 1718 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1719 uint64_t item_flags, 1720 uint8_t target_protocol, 1721 struct rte_flow_error *error) 1722 { 1723 const struct rte_flow_item_udp *mask = item->mask; 1724 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1725 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1726 MLX5_FLOW_LAYER_OUTER_L3; 1727 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1728 MLX5_FLOW_LAYER_OUTER_L4; 1729 int ret; 1730 1731 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1732 return rte_flow_error_set(error, EINVAL, 1733 RTE_FLOW_ERROR_TYPE_ITEM, item, 1734 "protocol filtering not compatible" 1735 " with UDP layer"); 1736 if (!(item_flags & l3m)) 1737 return rte_flow_error_set(error, EINVAL, 1738 RTE_FLOW_ERROR_TYPE_ITEM, item, 1739 "L3 is mandatory to filter on L4"); 1740 if (item_flags & l4m) 1741 return rte_flow_error_set(error, EINVAL, 1742 RTE_FLOW_ERROR_TYPE_ITEM, item, 1743 "multiple L4 layers not supported"); 1744 if (!mask) 1745 mask = &rte_flow_item_udp_mask; 1746 ret = mlx5_flow_item_acceptable 1747 (item, (const uint8_t *)mask, 1748 (const uint8_t *)&rte_flow_item_udp_mask, 1749 sizeof(struct rte_flow_item_udp), error); 1750 if (ret < 0) 1751 return ret; 1752 return 0; 1753 } 1754 1755 /** 1756 * Validate TCP item. 1757 * 1758 * @param[in] item 1759 * Item specification. 1760 * @param[in] item_flags 1761 * Bit-fields that holds the items detected until now. 1762 * @param[in] target_protocol 1763 * The next protocol in the previous item. 1764 * @param[out] error 1765 * Pointer to error structure. 1766 * 1767 * @return 1768 * 0 on success, a negative errno value otherwise and rte_errno is set. 1769 */ 1770 int 1771 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1772 uint64_t item_flags, 1773 uint8_t target_protocol, 1774 const struct rte_flow_item_tcp *flow_mask, 1775 struct rte_flow_error *error) 1776 { 1777 const struct rte_flow_item_tcp *mask = item->mask; 1778 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1779 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1780 MLX5_FLOW_LAYER_OUTER_L3; 1781 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1782 MLX5_FLOW_LAYER_OUTER_L4; 1783 int ret; 1784 1785 assert(flow_mask); 1786 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1787 return rte_flow_error_set(error, EINVAL, 1788 RTE_FLOW_ERROR_TYPE_ITEM, item, 1789 "protocol filtering not compatible" 1790 " with TCP layer"); 1791 if (!(item_flags & l3m)) 1792 return rte_flow_error_set(error, EINVAL, 1793 RTE_FLOW_ERROR_TYPE_ITEM, item, 1794 "L3 is mandatory to filter on L4"); 1795 if (item_flags & l4m) 1796 return rte_flow_error_set(error, EINVAL, 1797 RTE_FLOW_ERROR_TYPE_ITEM, item, 1798 "multiple L4 layers not supported"); 1799 if (!mask) 1800 mask = &rte_flow_item_tcp_mask; 1801 ret = mlx5_flow_item_acceptable 1802 (item, (const uint8_t *)mask, 1803 (const uint8_t *)flow_mask, 1804 sizeof(struct rte_flow_item_tcp), error); 1805 if (ret < 0) 1806 return ret; 1807 return 0; 1808 } 1809 1810 /** 1811 * Validate VXLAN item. 1812 * 1813 * @param[in] item 1814 * Item specification. 1815 * @param[in] item_flags 1816 * Bit-fields that holds the items detected until now. 1817 * @param[in] target_protocol 1818 * The next protocol in the previous item. 1819 * @param[out] error 1820 * Pointer to error structure. 1821 * 1822 * @return 1823 * 0 on success, a negative errno value otherwise and rte_errno is set. 1824 */ 1825 int 1826 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1827 uint64_t item_flags, 1828 struct rte_flow_error *error) 1829 { 1830 const struct rte_flow_item_vxlan *spec = item->spec; 1831 const struct rte_flow_item_vxlan *mask = item->mask; 1832 int ret; 1833 union vni { 1834 uint32_t vlan_id; 1835 uint8_t vni[4]; 1836 } id = { .vlan_id = 0, }; 1837 uint32_t vlan_id = 0; 1838 1839 1840 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1841 return rte_flow_error_set(error, ENOTSUP, 1842 RTE_FLOW_ERROR_TYPE_ITEM, item, 1843 "multiple tunnel layers not" 1844 " supported"); 1845 /* 1846 * Verify only UDPv4 is present as defined in 1847 * https://tools.ietf.org/html/rfc7348 1848 */ 1849 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1850 return rte_flow_error_set(error, EINVAL, 1851 RTE_FLOW_ERROR_TYPE_ITEM, item, 1852 "no outer UDP layer found"); 1853 if (!mask) 1854 mask = &rte_flow_item_vxlan_mask; 1855 ret = mlx5_flow_item_acceptable 1856 (item, (const uint8_t *)mask, 1857 (const uint8_t *)&rte_flow_item_vxlan_mask, 1858 sizeof(struct rte_flow_item_vxlan), 1859 error); 1860 if (ret < 0) 1861 return ret; 1862 if (spec) { 1863 memcpy(&id.vni[1], spec->vni, 3); 1864 vlan_id = id.vlan_id; 1865 memcpy(&id.vni[1], mask->vni, 3); 1866 vlan_id &= id.vlan_id; 1867 } 1868 /* 1869 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if 1870 * only this layer is defined in the Verbs specification it is 1871 * interpreted as wildcard and all packets will match this 1872 * rule, if it follows a full stack layer (ex: eth / ipv4 / 1873 * udp), all packets matching the layers before will also 1874 * match this rule. To avoid such situation, VNI 0 is 1875 * currently refused. 1876 */ 1877 if (!vlan_id) 1878 return rte_flow_error_set(error, ENOTSUP, 1879 RTE_FLOW_ERROR_TYPE_ITEM, item, 1880 "VXLAN vni cannot be 0"); 1881 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1882 return rte_flow_error_set(error, ENOTSUP, 1883 RTE_FLOW_ERROR_TYPE_ITEM, item, 1884 "VXLAN tunnel must be fully defined"); 1885 return 0; 1886 } 1887 1888 /** 1889 * Validate VXLAN_GPE item. 1890 * 1891 * @param[in] item 1892 * Item specification. 1893 * @param[in] item_flags 1894 * Bit-fields that holds the items detected until now. 1895 * @param[in] priv 1896 * Pointer to the private data structure. 1897 * @param[in] target_protocol 1898 * The next protocol in the previous item. 1899 * @param[out] error 1900 * Pointer to error structure. 1901 * 1902 * @return 1903 * 0 on success, a negative errno value otherwise and rte_errno is set. 1904 */ 1905 int 1906 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1907 uint64_t item_flags, 1908 struct rte_eth_dev *dev, 1909 struct rte_flow_error *error) 1910 { 1911 struct mlx5_priv *priv = dev->data->dev_private; 1912 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1913 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1914 int ret; 1915 union vni { 1916 uint32_t vlan_id; 1917 uint8_t vni[4]; 1918 } id = { .vlan_id = 0, }; 1919 uint32_t vlan_id = 0; 1920 1921 if (!priv->config.l3_vxlan_en) 1922 return rte_flow_error_set(error, ENOTSUP, 1923 RTE_FLOW_ERROR_TYPE_ITEM, item, 1924 "L3 VXLAN is not enabled by device" 1925 " parameter and/or not configured in" 1926 " firmware"); 1927 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1928 return rte_flow_error_set(error, ENOTSUP, 1929 RTE_FLOW_ERROR_TYPE_ITEM, item, 1930 "multiple tunnel layers not" 1931 " supported"); 1932 /* 1933 * Verify only UDPv4 is present as defined in 1934 * https://tools.ietf.org/html/rfc7348 1935 */ 1936 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1937 return rte_flow_error_set(error, EINVAL, 1938 RTE_FLOW_ERROR_TYPE_ITEM, item, 1939 "no outer UDP layer found"); 1940 if (!mask) 1941 mask = &rte_flow_item_vxlan_gpe_mask; 1942 ret = mlx5_flow_item_acceptable 1943 (item, (const uint8_t *)mask, 1944 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1945 sizeof(struct rte_flow_item_vxlan_gpe), 1946 error); 1947 if (ret < 0) 1948 return ret; 1949 if (spec) { 1950 if (spec->protocol) 1951 return rte_flow_error_set(error, ENOTSUP, 1952 RTE_FLOW_ERROR_TYPE_ITEM, 1953 item, 1954 "VxLAN-GPE protocol" 1955 " not supported"); 1956 memcpy(&id.vni[1], spec->vni, 3); 1957 vlan_id = id.vlan_id; 1958 memcpy(&id.vni[1], mask->vni, 3); 1959 vlan_id &= id.vlan_id; 1960 } 1961 /* 1962 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1963 * layer is defined in the Verbs specification it is interpreted as 1964 * wildcard and all packets will match this rule, if it follows a full 1965 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1966 * before will also match this rule. To avoid such situation, VNI 0 1967 * is currently refused. 1968 */ 1969 if (!vlan_id) 1970 return rte_flow_error_set(error, ENOTSUP, 1971 RTE_FLOW_ERROR_TYPE_ITEM, item, 1972 "VXLAN-GPE vni cannot be 0"); 1973 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1974 return rte_flow_error_set(error, ENOTSUP, 1975 RTE_FLOW_ERROR_TYPE_ITEM, item, 1976 "VXLAN-GPE tunnel must be fully" 1977 " defined"); 1978 return 0; 1979 } 1980 /** 1981 * Validate GRE Key item. 1982 * 1983 * @param[in] item 1984 * Item specification. 1985 * @param[in] item_flags 1986 * Bit flags to mark detected items. 1987 * @param[in] gre_item 1988 * Pointer to gre_item 1989 * @param[out] error 1990 * Pointer to error structure. 1991 * 1992 * @return 1993 * 0 on success, a negative errno value otherwise and rte_errno is set. 1994 */ 1995 int 1996 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 1997 uint64_t item_flags, 1998 const struct rte_flow_item *gre_item, 1999 struct rte_flow_error *error) 2000 { 2001 const rte_be32_t *mask = item->mask; 2002 int ret = 0; 2003 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2004 const struct rte_flow_item_gre *gre_spec; 2005 const struct rte_flow_item_gre *gre_mask; 2006 2007 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2008 return rte_flow_error_set(error, ENOTSUP, 2009 RTE_FLOW_ERROR_TYPE_ITEM, item, 2010 "Multiple GRE key not support"); 2011 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2012 return rte_flow_error_set(error, ENOTSUP, 2013 RTE_FLOW_ERROR_TYPE_ITEM, item, 2014 "No preceding GRE header"); 2015 if (item_flags & MLX5_FLOW_LAYER_INNER) 2016 return rte_flow_error_set(error, ENOTSUP, 2017 RTE_FLOW_ERROR_TYPE_ITEM, item, 2018 "GRE key following a wrong item"); 2019 gre_mask = gre_item->mask; 2020 if (!gre_mask) 2021 gre_mask = &rte_flow_item_gre_mask; 2022 gre_spec = gre_item->spec; 2023 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2024 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2025 return rte_flow_error_set(error, EINVAL, 2026 RTE_FLOW_ERROR_TYPE_ITEM, item, 2027 "Key bit must be on"); 2028 2029 if (!mask) 2030 mask = &gre_key_default_mask; 2031 ret = mlx5_flow_item_acceptable 2032 (item, (const uint8_t *)mask, 2033 (const uint8_t *)&gre_key_default_mask, 2034 sizeof(rte_be32_t), error); 2035 return ret; 2036 } 2037 2038 /** 2039 * Validate GRE item. 2040 * 2041 * @param[in] item 2042 * Item specification. 2043 * @param[in] item_flags 2044 * Bit flags to mark detected items. 2045 * @param[in] target_protocol 2046 * The next protocol in the previous item. 2047 * @param[out] error 2048 * Pointer to error structure. 2049 * 2050 * @return 2051 * 0 on success, a negative errno value otherwise and rte_errno is set. 2052 */ 2053 int 2054 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2055 uint64_t item_flags, 2056 uint8_t target_protocol, 2057 struct rte_flow_error *error) 2058 { 2059 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2060 const struct rte_flow_item_gre *mask = item->mask; 2061 int ret; 2062 const struct rte_flow_item_gre nic_mask = { 2063 .c_rsvd0_ver = RTE_BE16(0xB000), 2064 .protocol = RTE_BE16(UINT16_MAX), 2065 }; 2066 2067 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2068 return rte_flow_error_set(error, EINVAL, 2069 RTE_FLOW_ERROR_TYPE_ITEM, item, 2070 "protocol filtering not compatible" 2071 " with this GRE layer"); 2072 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2073 return rte_flow_error_set(error, ENOTSUP, 2074 RTE_FLOW_ERROR_TYPE_ITEM, item, 2075 "multiple tunnel layers not" 2076 " supported"); 2077 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2078 return rte_flow_error_set(error, ENOTSUP, 2079 RTE_FLOW_ERROR_TYPE_ITEM, item, 2080 "L3 Layer is missing"); 2081 if (!mask) 2082 mask = &rte_flow_item_gre_mask; 2083 ret = mlx5_flow_item_acceptable 2084 (item, (const uint8_t *)mask, 2085 (const uint8_t *)&nic_mask, 2086 sizeof(struct rte_flow_item_gre), error); 2087 if (ret < 0) 2088 return ret; 2089 #ifndef HAVE_MLX5DV_DR 2090 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2091 if (spec && (spec->protocol & mask->protocol)) 2092 return rte_flow_error_set(error, ENOTSUP, 2093 RTE_FLOW_ERROR_TYPE_ITEM, item, 2094 "without MPLS support the" 2095 " specification cannot be used for" 2096 " filtering"); 2097 #endif 2098 #endif 2099 return 0; 2100 } 2101 2102 /** 2103 * Validate Geneve item. 2104 * 2105 * @param[in] item 2106 * Item specification. 2107 * @param[in] itemFlags 2108 * Bit-fields that holds the items detected until now. 2109 * @param[in] enPriv 2110 * Pointer to the private data structure. 2111 * @param[out] error 2112 * Pointer to error structure. 2113 * 2114 * @return 2115 * 0 on success, a negative errno value otherwise and rte_errno is set. 2116 */ 2117 2118 int 2119 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2120 uint64_t item_flags, 2121 struct rte_eth_dev *dev, 2122 struct rte_flow_error *error) 2123 { 2124 struct mlx5_priv *priv = dev->data->dev_private; 2125 const struct rte_flow_item_geneve *spec = item->spec; 2126 const struct rte_flow_item_geneve *mask = item->mask; 2127 int ret; 2128 uint16_t gbhdr; 2129 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2130 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2131 const struct rte_flow_item_geneve nic_mask = { 2132 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2133 .vni = "\xff\xff\xff", 2134 .protocol = RTE_BE16(UINT16_MAX), 2135 }; 2136 2137 if (!(priv->config.hca_attr.flex_parser_protocols & 2138 MLX5_HCA_FLEX_GENEVE_ENABLED) || 2139 !priv->config.hca_attr.tunnel_stateless_geneve_rx) 2140 return rte_flow_error_set(error, ENOTSUP, 2141 RTE_FLOW_ERROR_TYPE_ITEM, item, 2142 "L3 Geneve is not enabled by device" 2143 " parameter and/or not configured in" 2144 " firmware"); 2145 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2146 return rte_flow_error_set(error, ENOTSUP, 2147 RTE_FLOW_ERROR_TYPE_ITEM, item, 2148 "multiple tunnel layers not" 2149 " supported"); 2150 /* 2151 * Verify only UDPv4 is present as defined in 2152 * https://tools.ietf.org/html/rfc7348 2153 */ 2154 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2155 return rte_flow_error_set(error, EINVAL, 2156 RTE_FLOW_ERROR_TYPE_ITEM, item, 2157 "no outer UDP layer found"); 2158 if (!mask) 2159 mask = &rte_flow_item_geneve_mask; 2160 ret = mlx5_flow_item_acceptable 2161 (item, (const uint8_t *)mask, 2162 (const uint8_t *)&nic_mask, 2163 sizeof(struct rte_flow_item_geneve), error); 2164 if (ret) 2165 return ret; 2166 if (spec) { 2167 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2168 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2169 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2170 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2171 return rte_flow_error_set(error, ENOTSUP, 2172 RTE_FLOW_ERROR_TYPE_ITEM, 2173 item, 2174 "Geneve protocol unsupported" 2175 " fields are being used"); 2176 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2177 return rte_flow_error_set 2178 (error, ENOTSUP, 2179 RTE_FLOW_ERROR_TYPE_ITEM, 2180 item, 2181 "Unsupported Geneve options length"); 2182 } 2183 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2184 return rte_flow_error_set 2185 (error, ENOTSUP, 2186 RTE_FLOW_ERROR_TYPE_ITEM, item, 2187 "Geneve tunnel must be fully defined"); 2188 return 0; 2189 } 2190 2191 /** 2192 * Validate MPLS item. 2193 * 2194 * @param[in] dev 2195 * Pointer to the rte_eth_dev structure. 2196 * @param[in] item 2197 * Item specification. 2198 * @param[in] item_flags 2199 * Bit-fields that holds the items detected until now. 2200 * @param[in] prev_layer 2201 * The protocol layer indicated in previous item. 2202 * @param[out] error 2203 * Pointer to error structure. 2204 * 2205 * @return 2206 * 0 on success, a negative errno value otherwise and rte_errno is set. 2207 */ 2208 int 2209 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2210 const struct rte_flow_item *item __rte_unused, 2211 uint64_t item_flags __rte_unused, 2212 uint64_t prev_layer __rte_unused, 2213 struct rte_flow_error *error) 2214 { 2215 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2216 const struct rte_flow_item_mpls *mask = item->mask; 2217 struct mlx5_priv *priv = dev->data->dev_private; 2218 int ret; 2219 2220 if (!priv->config.mpls_en) 2221 return rte_flow_error_set(error, ENOTSUP, 2222 RTE_FLOW_ERROR_TYPE_ITEM, item, 2223 "MPLS not supported or" 2224 " disabled in firmware" 2225 " configuration."); 2226 /* MPLS over IP, UDP, GRE is allowed */ 2227 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2228 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2229 MLX5_FLOW_LAYER_GRE))) 2230 return rte_flow_error_set(error, EINVAL, 2231 RTE_FLOW_ERROR_TYPE_ITEM, item, 2232 "protocol filtering not compatible" 2233 " with MPLS layer"); 2234 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2235 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2236 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2237 return rte_flow_error_set(error, ENOTSUP, 2238 RTE_FLOW_ERROR_TYPE_ITEM, item, 2239 "multiple tunnel layers not" 2240 " supported"); 2241 if (!mask) 2242 mask = &rte_flow_item_mpls_mask; 2243 ret = mlx5_flow_item_acceptable 2244 (item, (const uint8_t *)mask, 2245 (const uint8_t *)&rte_flow_item_mpls_mask, 2246 sizeof(struct rte_flow_item_mpls), error); 2247 if (ret < 0) 2248 return ret; 2249 return 0; 2250 #endif 2251 return rte_flow_error_set(error, ENOTSUP, 2252 RTE_FLOW_ERROR_TYPE_ITEM, item, 2253 "MPLS is not supported by Verbs, please" 2254 " update."); 2255 } 2256 2257 /** 2258 * Validate NVGRE item. 2259 * 2260 * @param[in] item 2261 * Item specification. 2262 * @param[in] item_flags 2263 * Bit flags to mark detected items. 2264 * @param[in] target_protocol 2265 * The next protocol in the previous item. 2266 * @param[out] error 2267 * Pointer to error structure. 2268 * 2269 * @return 2270 * 0 on success, a negative errno value otherwise and rte_errno is set. 2271 */ 2272 int 2273 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2274 uint64_t item_flags, 2275 uint8_t target_protocol, 2276 struct rte_flow_error *error) 2277 { 2278 const struct rte_flow_item_nvgre *mask = item->mask; 2279 int ret; 2280 2281 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2282 return rte_flow_error_set(error, EINVAL, 2283 RTE_FLOW_ERROR_TYPE_ITEM, item, 2284 "protocol filtering not compatible" 2285 " with this GRE layer"); 2286 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2287 return rte_flow_error_set(error, ENOTSUP, 2288 RTE_FLOW_ERROR_TYPE_ITEM, item, 2289 "multiple tunnel layers not" 2290 " supported"); 2291 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2292 return rte_flow_error_set(error, ENOTSUP, 2293 RTE_FLOW_ERROR_TYPE_ITEM, item, 2294 "L3 Layer is missing"); 2295 if (!mask) 2296 mask = &rte_flow_item_nvgre_mask; 2297 ret = mlx5_flow_item_acceptable 2298 (item, (const uint8_t *)mask, 2299 (const uint8_t *)&rte_flow_item_nvgre_mask, 2300 sizeof(struct rte_flow_item_nvgre), error); 2301 if (ret < 0) 2302 return ret; 2303 return 0; 2304 } 2305 2306 /* Allocate unique ID for the split Q/RSS subflows. */ 2307 static uint32_t 2308 flow_qrss_get_id(struct rte_eth_dev *dev) 2309 { 2310 struct mlx5_priv *priv = dev->data->dev_private; 2311 uint32_t qrss_id, ret; 2312 2313 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2314 if (ret) 2315 return 0; 2316 assert(qrss_id); 2317 return qrss_id; 2318 } 2319 2320 /* Free unique ID for the split Q/RSS subflows. */ 2321 static void 2322 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2323 { 2324 struct mlx5_priv *priv = dev->data->dev_private; 2325 2326 if (qrss_id) 2327 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2328 } 2329 2330 /** 2331 * Release resource related QUEUE/RSS action split. 2332 * 2333 * @param dev 2334 * Pointer to Ethernet device. 2335 * @param flow 2336 * Flow to release id's from. 2337 */ 2338 static void 2339 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2340 struct rte_flow *flow) 2341 { 2342 struct mlx5_flow *dev_flow; 2343 2344 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 2345 if (dev_flow->qrss_id) 2346 flow_qrss_free_id(dev, dev_flow->qrss_id); 2347 } 2348 2349 static int 2350 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2351 const struct rte_flow_attr *attr __rte_unused, 2352 const struct rte_flow_item items[] __rte_unused, 2353 const struct rte_flow_action actions[] __rte_unused, 2354 bool external __rte_unused, 2355 struct rte_flow_error *error) 2356 { 2357 return rte_flow_error_set(error, ENOTSUP, 2358 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2359 } 2360 2361 static struct mlx5_flow * 2362 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, 2363 const struct rte_flow_item items[] __rte_unused, 2364 const struct rte_flow_action actions[] __rte_unused, 2365 struct rte_flow_error *error) 2366 { 2367 rte_flow_error_set(error, ENOTSUP, 2368 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2369 return NULL; 2370 } 2371 2372 static int 2373 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2374 struct mlx5_flow *dev_flow __rte_unused, 2375 const struct rte_flow_attr *attr __rte_unused, 2376 const struct rte_flow_item items[] __rte_unused, 2377 const struct rte_flow_action actions[] __rte_unused, 2378 struct rte_flow_error *error) 2379 { 2380 return rte_flow_error_set(error, ENOTSUP, 2381 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2382 } 2383 2384 static int 2385 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2386 struct rte_flow *flow __rte_unused, 2387 struct rte_flow_error *error) 2388 { 2389 return rte_flow_error_set(error, ENOTSUP, 2390 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2391 } 2392 2393 static void 2394 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2395 struct rte_flow *flow __rte_unused) 2396 { 2397 } 2398 2399 static void 2400 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2401 struct rte_flow *flow __rte_unused) 2402 { 2403 } 2404 2405 static int 2406 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2407 struct rte_flow *flow __rte_unused, 2408 const struct rte_flow_action *actions __rte_unused, 2409 void *data __rte_unused, 2410 struct rte_flow_error *error) 2411 { 2412 return rte_flow_error_set(error, ENOTSUP, 2413 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2414 } 2415 2416 /* Void driver to protect from null pointer reference. */ 2417 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2418 .validate = flow_null_validate, 2419 .prepare = flow_null_prepare, 2420 .translate = flow_null_translate, 2421 .apply = flow_null_apply, 2422 .remove = flow_null_remove, 2423 .destroy = flow_null_destroy, 2424 .query = flow_null_query, 2425 }; 2426 2427 /** 2428 * Select flow driver type according to flow attributes and device 2429 * configuration. 2430 * 2431 * @param[in] dev 2432 * Pointer to the dev structure. 2433 * @param[in] attr 2434 * Pointer to the flow attributes. 2435 * 2436 * @return 2437 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2438 */ 2439 static enum mlx5_flow_drv_type 2440 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2441 { 2442 struct mlx5_priv *priv = dev->data->dev_private; 2443 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2444 2445 if (attr->transfer && priv->config.dv_esw_en) 2446 type = MLX5_FLOW_TYPE_DV; 2447 if (!attr->transfer) 2448 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2449 MLX5_FLOW_TYPE_VERBS; 2450 return type; 2451 } 2452 2453 #define flow_get_drv_ops(type) flow_drv_ops[type] 2454 2455 /** 2456 * Flow driver validation API. This abstracts calling driver specific functions. 2457 * The type of flow driver is determined according to flow attributes. 2458 * 2459 * @param[in] dev 2460 * Pointer to the dev structure. 2461 * @param[in] attr 2462 * Pointer to the flow attributes. 2463 * @param[in] items 2464 * Pointer to the list of items. 2465 * @param[in] actions 2466 * Pointer to the list of actions. 2467 * @param[in] external 2468 * This flow rule is created by request external to PMD. 2469 * @param[out] error 2470 * Pointer to the error structure. 2471 * 2472 * @return 2473 * 0 on success, a negative errno value otherwise and rte_errno is set. 2474 */ 2475 static inline int 2476 flow_drv_validate(struct rte_eth_dev *dev, 2477 const struct rte_flow_attr *attr, 2478 const struct rte_flow_item items[], 2479 const struct rte_flow_action actions[], 2480 bool external, struct rte_flow_error *error) 2481 { 2482 const struct mlx5_flow_driver_ops *fops; 2483 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2484 2485 fops = flow_get_drv_ops(type); 2486 return fops->validate(dev, attr, items, actions, external, error); 2487 } 2488 2489 /** 2490 * Flow driver preparation API. This abstracts calling driver specific 2491 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2492 * calculates the size of memory required for device flow, allocates the memory, 2493 * initializes the device flow and returns the pointer. 2494 * 2495 * @note 2496 * This function initializes device flow structure such as dv or verbs in 2497 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2498 * rest. For example, adding returning device flow to flow->dev_flow list and 2499 * setting backward reference to the flow should be done out of this function. 2500 * layers field is not filled either. 2501 * 2502 * @param[in] attr 2503 * Pointer to the flow attributes. 2504 * @param[in] items 2505 * Pointer to the list of items. 2506 * @param[in] actions 2507 * Pointer to the list of actions. 2508 * @param[out] error 2509 * Pointer to the error structure. 2510 * 2511 * @return 2512 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2513 */ 2514 static inline struct mlx5_flow * 2515 flow_drv_prepare(const struct rte_flow *flow, 2516 const struct rte_flow_attr *attr, 2517 const struct rte_flow_item items[], 2518 const struct rte_flow_action actions[], 2519 struct rte_flow_error *error) 2520 { 2521 const struct mlx5_flow_driver_ops *fops; 2522 enum mlx5_flow_drv_type type = flow->drv_type; 2523 2524 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2525 fops = flow_get_drv_ops(type); 2526 return fops->prepare(attr, items, actions, error); 2527 } 2528 2529 /** 2530 * Flow driver translation API. This abstracts calling driver specific 2531 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2532 * translates a generic flow into a driver flow. flow_drv_prepare() must 2533 * precede. 2534 * 2535 * @note 2536 * dev_flow->layers could be filled as a result of parsing during translation 2537 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2538 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2539 * flow->actions could be overwritten even though all the expanded dev_flows 2540 * have the same actions. 2541 * 2542 * @param[in] dev 2543 * Pointer to the rte dev structure. 2544 * @param[in, out] dev_flow 2545 * Pointer to the mlx5 flow. 2546 * @param[in] attr 2547 * Pointer to the flow attributes. 2548 * @param[in] items 2549 * Pointer to the list of items. 2550 * @param[in] actions 2551 * Pointer to the list of actions. 2552 * @param[out] error 2553 * Pointer to the error structure. 2554 * 2555 * @return 2556 * 0 on success, a negative errno value otherwise and rte_errno is set. 2557 */ 2558 static inline int 2559 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2560 const struct rte_flow_attr *attr, 2561 const struct rte_flow_item items[], 2562 const struct rte_flow_action actions[], 2563 struct rte_flow_error *error) 2564 { 2565 const struct mlx5_flow_driver_ops *fops; 2566 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2567 2568 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2569 fops = flow_get_drv_ops(type); 2570 return fops->translate(dev, dev_flow, attr, items, actions, error); 2571 } 2572 2573 /** 2574 * Flow driver apply API. This abstracts calling driver specific functions. 2575 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2576 * translated driver flows on to device. flow_drv_translate() must precede. 2577 * 2578 * @param[in] dev 2579 * Pointer to Ethernet device structure. 2580 * @param[in, out] flow 2581 * Pointer to flow structure. 2582 * @param[out] error 2583 * Pointer to error structure. 2584 * 2585 * @return 2586 * 0 on success, a negative errno value otherwise and rte_errno is set. 2587 */ 2588 static inline int 2589 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2590 struct rte_flow_error *error) 2591 { 2592 const struct mlx5_flow_driver_ops *fops; 2593 enum mlx5_flow_drv_type type = flow->drv_type; 2594 2595 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2596 fops = flow_get_drv_ops(type); 2597 return fops->apply(dev, flow, error); 2598 } 2599 2600 /** 2601 * Flow driver remove API. This abstracts calling driver specific functions. 2602 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2603 * on device. All the resources of the flow should be freed by calling 2604 * flow_drv_destroy(). 2605 * 2606 * @param[in] dev 2607 * Pointer to Ethernet device. 2608 * @param[in, out] flow 2609 * Pointer to flow structure. 2610 */ 2611 static inline void 2612 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2613 { 2614 const struct mlx5_flow_driver_ops *fops; 2615 enum mlx5_flow_drv_type type = flow->drv_type; 2616 2617 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2618 fops = flow_get_drv_ops(type); 2619 fops->remove(dev, flow); 2620 } 2621 2622 /** 2623 * Flow driver destroy API. This abstracts calling driver specific functions. 2624 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2625 * on device and releases resources of the flow. 2626 * 2627 * @param[in] dev 2628 * Pointer to Ethernet device. 2629 * @param[in, out] flow 2630 * Pointer to flow structure. 2631 */ 2632 static inline void 2633 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2634 { 2635 const struct mlx5_flow_driver_ops *fops; 2636 enum mlx5_flow_drv_type type = flow->drv_type; 2637 2638 flow_mreg_split_qrss_release(dev, flow); 2639 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2640 fops = flow_get_drv_ops(type); 2641 fops->destroy(dev, flow); 2642 } 2643 2644 /** 2645 * Validate a flow supported by the NIC. 2646 * 2647 * @see rte_flow_validate() 2648 * @see rte_flow_ops 2649 */ 2650 int 2651 mlx5_flow_validate(struct rte_eth_dev *dev, 2652 const struct rte_flow_attr *attr, 2653 const struct rte_flow_item items[], 2654 const struct rte_flow_action actions[], 2655 struct rte_flow_error *error) 2656 { 2657 int ret; 2658 2659 ret = flow_drv_validate(dev, attr, items, actions, true, error); 2660 if (ret < 0) 2661 return ret; 2662 return 0; 2663 } 2664 2665 /** 2666 * Get port id item from the item list. 2667 * 2668 * @param[in] item 2669 * Pointer to the list of items. 2670 * 2671 * @return 2672 * Pointer to the port id item if exist, else return NULL. 2673 */ 2674 static const struct rte_flow_item * 2675 find_port_id_item(const struct rte_flow_item *item) 2676 { 2677 assert(item); 2678 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2679 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) 2680 return item; 2681 } 2682 return NULL; 2683 } 2684 2685 /** 2686 * Get RSS action from the action list. 2687 * 2688 * @param[in] actions 2689 * Pointer to the list of actions. 2690 * 2691 * @return 2692 * Pointer to the RSS action if exist, else return NULL. 2693 */ 2694 static const struct rte_flow_action_rss* 2695 flow_get_rss_action(const struct rte_flow_action actions[]) 2696 { 2697 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2698 switch (actions->type) { 2699 case RTE_FLOW_ACTION_TYPE_RSS: 2700 return (const struct rte_flow_action_rss *) 2701 actions->conf; 2702 default: 2703 break; 2704 } 2705 } 2706 return NULL; 2707 } 2708 2709 static unsigned int 2710 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2711 { 2712 const struct rte_flow_item *item; 2713 unsigned int has_vlan = 0; 2714 2715 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2716 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2717 has_vlan = 1; 2718 break; 2719 } 2720 } 2721 if (has_vlan) 2722 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2723 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2724 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2725 MLX5_EXPANSION_ROOT_OUTER; 2726 } 2727 2728 /** 2729 * Get QUEUE/RSS action from the action list. 2730 * 2731 * @param[in] actions 2732 * Pointer to the list of actions. 2733 * @param[out] qrss 2734 * Pointer to the return pointer. 2735 * @param[out] qrss_type 2736 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2737 * if no QUEUE/RSS is found. 2738 * 2739 * @return 2740 * Total number of actions. 2741 */ 2742 static int 2743 flow_parse_qrss_action(const struct rte_flow_action actions[], 2744 const struct rte_flow_action **qrss) 2745 { 2746 int actions_n = 0; 2747 2748 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2749 switch (actions->type) { 2750 case RTE_FLOW_ACTION_TYPE_QUEUE: 2751 case RTE_FLOW_ACTION_TYPE_RSS: 2752 *qrss = actions; 2753 break; 2754 default: 2755 break; 2756 } 2757 actions_n++; 2758 } 2759 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2760 return actions_n + 1; 2761 } 2762 2763 /** 2764 * Check meter action from the action list. 2765 * 2766 * @param[in] actions 2767 * Pointer to the list of actions. 2768 * @param[out] mtr 2769 * Pointer to the meter exist flag. 2770 * 2771 * @return 2772 * Total number of actions. 2773 */ 2774 static int 2775 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2776 { 2777 int actions_n = 0; 2778 2779 assert(mtr); 2780 *mtr = 0; 2781 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2782 switch (actions->type) { 2783 case RTE_FLOW_ACTION_TYPE_METER: 2784 *mtr = 1; 2785 break; 2786 default: 2787 break; 2788 } 2789 actions_n++; 2790 } 2791 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2792 return actions_n + 1; 2793 } 2794 2795 /** 2796 * Check if the flow should be splited due to hairpin. 2797 * The reason for the split is that in current HW we can't 2798 * support encap on Rx, so if a flow have encap we move it 2799 * to Tx. 2800 * 2801 * @param dev 2802 * Pointer to Ethernet device. 2803 * @param[in] attr 2804 * Flow rule attributes. 2805 * @param[in] actions 2806 * Associated actions (list terminated by the END action). 2807 * 2808 * @return 2809 * > 0 the number of actions and the flow should be split, 2810 * 0 when no split required. 2811 */ 2812 static int 2813 flow_check_hairpin_split(struct rte_eth_dev *dev, 2814 const struct rte_flow_attr *attr, 2815 const struct rte_flow_action actions[]) 2816 { 2817 int queue_action = 0; 2818 int action_n = 0; 2819 int encap = 0; 2820 const struct rte_flow_action_queue *queue; 2821 const struct rte_flow_action_rss *rss; 2822 const struct rte_flow_action_raw_encap *raw_encap; 2823 2824 if (!attr->ingress) 2825 return 0; 2826 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2827 switch (actions->type) { 2828 case RTE_FLOW_ACTION_TYPE_QUEUE: 2829 queue = actions->conf; 2830 if (queue == NULL) 2831 return 0; 2832 if (mlx5_rxq_get_type(dev, queue->index) != 2833 MLX5_RXQ_TYPE_HAIRPIN) 2834 return 0; 2835 queue_action = 1; 2836 action_n++; 2837 break; 2838 case RTE_FLOW_ACTION_TYPE_RSS: 2839 rss = actions->conf; 2840 if (rss == NULL || rss->queue_num == 0) 2841 return 0; 2842 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2843 MLX5_RXQ_TYPE_HAIRPIN) 2844 return 0; 2845 queue_action = 1; 2846 action_n++; 2847 break; 2848 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2849 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2850 encap = 1; 2851 action_n++; 2852 break; 2853 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2854 raw_encap = actions->conf; 2855 if (raw_encap->size > 2856 (sizeof(struct rte_flow_item_eth) + 2857 sizeof(struct rte_flow_item_ipv4))) 2858 encap = 1; 2859 action_n++; 2860 break; 2861 default: 2862 action_n++; 2863 break; 2864 } 2865 } 2866 if (encap == 1 && queue_action) 2867 return action_n; 2868 return 0; 2869 } 2870 2871 /* Declare flow create/destroy prototype in advance. */ 2872 static struct rte_flow * 2873 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2874 const struct rte_flow_attr *attr, 2875 const struct rte_flow_item items[], 2876 const struct rte_flow_action actions[], 2877 bool external, struct rte_flow_error *error); 2878 2879 static void 2880 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2881 struct rte_flow *flow); 2882 2883 /** 2884 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2885 * 2886 * As mark_id is unique, if there's already a registered flow for the mark_id, 2887 * return by increasing the reference counter of the resource. Otherwise, create 2888 * the resource (mcp_res) and flow. 2889 * 2890 * Flow looks like, 2891 * - If ingress port is ANY and reg_c[1] is mark_id, 2892 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2893 * 2894 * For default flow (zero mark_id), flow is like, 2895 * - If ingress port is ANY, 2896 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2897 * 2898 * @param dev 2899 * Pointer to Ethernet device. 2900 * @param mark_id 2901 * ID of MARK action, zero means default flow for META. 2902 * @param[out] error 2903 * Perform verbose error reporting if not NULL. 2904 * 2905 * @return 2906 * Associated resource on success, NULL otherwise and rte_errno is set. 2907 */ 2908 static struct mlx5_flow_mreg_copy_resource * 2909 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2910 struct rte_flow_error *error) 2911 { 2912 struct mlx5_priv *priv = dev->data->dev_private; 2913 struct rte_flow_attr attr = { 2914 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2915 .ingress = 1, 2916 }; 2917 struct mlx5_rte_flow_item_tag tag_spec = { 2918 .data = mark_id, 2919 }; 2920 struct rte_flow_item items[] = { 2921 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2922 }; 2923 struct rte_flow_action_mark ftag = { 2924 .id = mark_id, 2925 }; 2926 struct mlx5_flow_action_copy_mreg cp_mreg = { 2927 .dst = REG_B, 2928 .src = 0, 2929 }; 2930 struct rte_flow_action_jump jump = { 2931 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2932 }; 2933 struct rte_flow_action actions[] = { 2934 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2935 }; 2936 struct mlx5_flow_mreg_copy_resource *mcp_res; 2937 int ret; 2938 2939 /* Fill the register fileds in the flow. */ 2940 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2941 if (ret < 0) 2942 return NULL; 2943 tag_spec.id = ret; 2944 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 2945 if (ret < 0) 2946 return NULL; 2947 cp_mreg.src = ret; 2948 /* Check if already registered. */ 2949 assert(priv->mreg_cp_tbl); 2950 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 2951 if (mcp_res) { 2952 /* For non-default rule. */ 2953 if (mark_id != MLX5_DEFAULT_COPY_ID) 2954 mcp_res->refcnt++; 2955 assert(mark_id != MLX5_DEFAULT_COPY_ID || mcp_res->refcnt == 1); 2956 return mcp_res; 2957 } 2958 /* Provide the full width of FLAG specific value. */ 2959 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 2960 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 2961 /* Build a new flow. */ 2962 if (mark_id != MLX5_DEFAULT_COPY_ID) { 2963 items[0] = (struct rte_flow_item){ 2964 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 2965 .spec = &tag_spec, 2966 }; 2967 items[1] = (struct rte_flow_item){ 2968 .type = RTE_FLOW_ITEM_TYPE_END, 2969 }; 2970 actions[0] = (struct rte_flow_action){ 2971 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 2972 .conf = &ftag, 2973 }; 2974 actions[1] = (struct rte_flow_action){ 2975 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2976 .conf = &cp_mreg, 2977 }; 2978 actions[2] = (struct rte_flow_action){ 2979 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2980 .conf = &jump, 2981 }; 2982 actions[3] = (struct rte_flow_action){ 2983 .type = RTE_FLOW_ACTION_TYPE_END, 2984 }; 2985 } else { 2986 /* Default rule, wildcard match. */ 2987 attr.priority = MLX5_FLOW_PRIO_RSVD; 2988 items[0] = (struct rte_flow_item){ 2989 .type = RTE_FLOW_ITEM_TYPE_END, 2990 }; 2991 actions[0] = (struct rte_flow_action){ 2992 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2993 .conf = &cp_mreg, 2994 }; 2995 actions[1] = (struct rte_flow_action){ 2996 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2997 .conf = &jump, 2998 }; 2999 actions[2] = (struct rte_flow_action){ 3000 .type = RTE_FLOW_ACTION_TYPE_END, 3001 }; 3002 } 3003 /* Build a new entry. */ 3004 mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0); 3005 if (!mcp_res) { 3006 rte_errno = ENOMEM; 3007 return NULL; 3008 } 3009 /* 3010 * The copy Flows are not included in any list. There 3011 * ones are referenced from other Flows and can not 3012 * be applied, removed, deleted in ardbitrary order 3013 * by list traversing. 3014 */ 3015 mcp_res->flow = flow_list_create(dev, NULL, &attr, items, 3016 actions, false, error); 3017 if (!mcp_res->flow) 3018 goto error; 3019 mcp_res->refcnt++; 3020 mcp_res->hlist_ent.key = mark_id; 3021 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3022 &mcp_res->hlist_ent); 3023 assert(!ret); 3024 if (ret) 3025 goto error; 3026 return mcp_res; 3027 error: 3028 if (mcp_res->flow) 3029 flow_list_destroy(dev, NULL, mcp_res->flow); 3030 rte_free(mcp_res); 3031 return NULL; 3032 } 3033 3034 /** 3035 * Release flow in RX_CP_TBL. 3036 * 3037 * @param dev 3038 * Pointer to Ethernet device. 3039 * @flow 3040 * Parent flow for wich copying is provided. 3041 */ 3042 static void 3043 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3044 struct rte_flow *flow) 3045 { 3046 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3047 struct mlx5_priv *priv = dev->data->dev_private; 3048 3049 if (!mcp_res || !priv->mreg_cp_tbl) 3050 return; 3051 if (flow->copy_applied) { 3052 assert(mcp_res->appcnt); 3053 flow->copy_applied = 0; 3054 --mcp_res->appcnt; 3055 if (!mcp_res->appcnt) 3056 flow_drv_remove(dev, mcp_res->flow); 3057 } 3058 /* 3059 * We do not check availability of metadata registers here, 3060 * because copy resources are not allocated in this case. 3061 */ 3062 if (--mcp_res->refcnt) 3063 return; 3064 assert(mcp_res->flow); 3065 flow_list_destroy(dev, NULL, mcp_res->flow); 3066 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3067 rte_free(mcp_res); 3068 flow->mreg_copy = NULL; 3069 } 3070 3071 /** 3072 * Start flow in RX_CP_TBL. 3073 * 3074 * @param dev 3075 * Pointer to Ethernet device. 3076 * @flow 3077 * Parent flow for wich copying is provided. 3078 * 3079 * @return 3080 * 0 on success, a negative errno value otherwise and rte_errno is set. 3081 */ 3082 static int 3083 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3084 struct rte_flow *flow) 3085 { 3086 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3087 int ret; 3088 3089 if (!mcp_res || flow->copy_applied) 3090 return 0; 3091 if (!mcp_res->appcnt) { 3092 ret = flow_drv_apply(dev, mcp_res->flow, NULL); 3093 if (ret) 3094 return ret; 3095 } 3096 ++mcp_res->appcnt; 3097 flow->copy_applied = 1; 3098 return 0; 3099 } 3100 3101 /** 3102 * Stop flow in RX_CP_TBL. 3103 * 3104 * @param dev 3105 * Pointer to Ethernet device. 3106 * @flow 3107 * Parent flow for wich copying is provided. 3108 */ 3109 static void 3110 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3111 struct rte_flow *flow) 3112 { 3113 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3114 3115 if (!mcp_res || !flow->copy_applied) 3116 return; 3117 assert(mcp_res->appcnt); 3118 --mcp_res->appcnt; 3119 flow->copy_applied = 0; 3120 if (!mcp_res->appcnt) 3121 flow_drv_remove(dev, mcp_res->flow); 3122 } 3123 3124 /** 3125 * Remove the default copy action from RX_CP_TBL. 3126 * 3127 * @param dev 3128 * Pointer to Ethernet device. 3129 */ 3130 static void 3131 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3132 { 3133 struct mlx5_flow_mreg_copy_resource *mcp_res; 3134 struct mlx5_priv *priv = dev->data->dev_private; 3135 3136 /* Check if default flow is registered. */ 3137 if (!priv->mreg_cp_tbl) 3138 return; 3139 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3140 MLX5_DEFAULT_COPY_ID); 3141 if (!mcp_res) 3142 return; 3143 assert(mcp_res->flow); 3144 flow_list_destroy(dev, NULL, mcp_res->flow); 3145 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3146 rte_free(mcp_res); 3147 } 3148 3149 /** 3150 * Add the default copy action in in RX_CP_TBL. 3151 * 3152 * @param dev 3153 * Pointer to Ethernet device. 3154 * @param[out] error 3155 * Perform verbose error reporting if not NULL. 3156 * 3157 * @return 3158 * 0 for success, negative value otherwise and rte_errno is set. 3159 */ 3160 static int 3161 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3162 struct rte_flow_error *error) 3163 { 3164 struct mlx5_priv *priv = dev->data->dev_private; 3165 struct mlx5_flow_mreg_copy_resource *mcp_res; 3166 3167 /* Check whether extensive metadata feature is engaged. */ 3168 if (!priv->config.dv_flow_en || 3169 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3170 !mlx5_flow_ext_mreg_supported(dev) || 3171 !priv->sh->dv_regc0_mask) 3172 return 0; 3173 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3174 if (!mcp_res) 3175 return -rte_errno; 3176 return 0; 3177 } 3178 3179 /** 3180 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3181 * 3182 * All the flow having Q/RSS action should be split by 3183 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3184 * performs the following, 3185 * - CQE->flow_tag := reg_c[1] (MARK) 3186 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3187 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3188 * but there should be a flow per each MARK ID set by MARK action. 3189 * 3190 * For the aforementioned reason, if there's a MARK action in flow's action 3191 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3192 * the MARK ID to CQE's flow_tag like, 3193 * - If reg_c[1] is mark_id, 3194 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3195 * 3196 * For SET_META action which stores value in reg_c[0], as the destination is 3197 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3198 * MARK ID means the default flow. The default flow looks like, 3199 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3200 * 3201 * @param dev 3202 * Pointer to Ethernet device. 3203 * @param flow 3204 * Pointer to flow structure. 3205 * @param[in] actions 3206 * Pointer to the list of actions. 3207 * @param[out] error 3208 * Perform verbose error reporting if not NULL. 3209 * 3210 * @return 3211 * 0 on success, negative value otherwise and rte_errno is set. 3212 */ 3213 static int 3214 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3215 struct rte_flow *flow, 3216 const struct rte_flow_action *actions, 3217 struct rte_flow_error *error) 3218 { 3219 struct mlx5_priv *priv = dev->data->dev_private; 3220 struct mlx5_dev_config *config = &priv->config; 3221 struct mlx5_flow_mreg_copy_resource *mcp_res; 3222 const struct rte_flow_action_mark *mark; 3223 3224 /* Check whether extensive metadata feature is engaged. */ 3225 if (!config->dv_flow_en || 3226 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3227 !mlx5_flow_ext_mreg_supported(dev) || 3228 !priv->sh->dv_regc0_mask) 3229 return 0; 3230 /* Find MARK action. */ 3231 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3232 switch (actions->type) { 3233 case RTE_FLOW_ACTION_TYPE_FLAG: 3234 mcp_res = flow_mreg_add_copy_action 3235 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3236 if (!mcp_res) 3237 return -rte_errno; 3238 flow->mreg_copy = mcp_res; 3239 if (dev->data->dev_started) { 3240 mcp_res->appcnt++; 3241 flow->copy_applied = 1; 3242 } 3243 return 0; 3244 case RTE_FLOW_ACTION_TYPE_MARK: 3245 mark = (const struct rte_flow_action_mark *) 3246 actions->conf; 3247 mcp_res = 3248 flow_mreg_add_copy_action(dev, mark->id, error); 3249 if (!mcp_res) 3250 return -rte_errno; 3251 flow->mreg_copy = mcp_res; 3252 if (dev->data->dev_started) { 3253 mcp_res->appcnt++; 3254 flow->copy_applied = 1; 3255 } 3256 return 0; 3257 default: 3258 break; 3259 } 3260 } 3261 return 0; 3262 } 3263 3264 #define MLX5_MAX_SPLIT_ACTIONS 24 3265 #define MLX5_MAX_SPLIT_ITEMS 24 3266 3267 /** 3268 * Split the hairpin flow. 3269 * Since HW can't support encap on Rx we move the encap to Tx. 3270 * If the count action is after the encap then we also 3271 * move the count action. in this case the count will also measure 3272 * the outer bytes. 3273 * 3274 * @param dev 3275 * Pointer to Ethernet device. 3276 * @param[in] actions 3277 * Associated actions (list terminated by the END action). 3278 * @param[out] actions_rx 3279 * Rx flow actions. 3280 * @param[out] actions_tx 3281 * Tx flow actions.. 3282 * @param[out] pattern_tx 3283 * The pattern items for the Tx flow. 3284 * @param[out] flow_id 3285 * The flow ID connected to this flow. 3286 * 3287 * @return 3288 * 0 on success. 3289 */ 3290 static int 3291 flow_hairpin_split(struct rte_eth_dev *dev, 3292 const struct rte_flow_action actions[], 3293 struct rte_flow_action actions_rx[], 3294 struct rte_flow_action actions_tx[], 3295 struct rte_flow_item pattern_tx[], 3296 uint32_t *flow_id) 3297 { 3298 struct mlx5_priv *priv = dev->data->dev_private; 3299 const struct rte_flow_action_raw_encap *raw_encap; 3300 const struct rte_flow_action_raw_decap *raw_decap; 3301 struct mlx5_rte_flow_action_set_tag *set_tag; 3302 struct rte_flow_action *tag_action; 3303 struct mlx5_rte_flow_item_tag *tag_item; 3304 struct rte_flow_item *item; 3305 char *addr; 3306 int encap = 0; 3307 3308 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3309 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3310 switch (actions->type) { 3311 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3312 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3313 rte_memcpy(actions_tx, actions, 3314 sizeof(struct rte_flow_action)); 3315 actions_tx++; 3316 break; 3317 case RTE_FLOW_ACTION_TYPE_COUNT: 3318 if (encap) { 3319 rte_memcpy(actions_tx, actions, 3320 sizeof(struct rte_flow_action)); 3321 actions_tx++; 3322 } else { 3323 rte_memcpy(actions_rx, actions, 3324 sizeof(struct rte_flow_action)); 3325 actions_rx++; 3326 } 3327 break; 3328 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3329 raw_encap = actions->conf; 3330 if (raw_encap->size > 3331 (sizeof(struct rte_flow_item_eth) + 3332 sizeof(struct rte_flow_item_ipv4))) { 3333 memcpy(actions_tx, actions, 3334 sizeof(struct rte_flow_action)); 3335 actions_tx++; 3336 encap = 1; 3337 } else { 3338 rte_memcpy(actions_rx, actions, 3339 sizeof(struct rte_flow_action)); 3340 actions_rx++; 3341 } 3342 break; 3343 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3344 raw_decap = actions->conf; 3345 if (raw_decap->size < 3346 (sizeof(struct rte_flow_item_eth) + 3347 sizeof(struct rte_flow_item_ipv4))) { 3348 memcpy(actions_tx, actions, 3349 sizeof(struct rte_flow_action)); 3350 actions_tx++; 3351 } else { 3352 rte_memcpy(actions_rx, actions, 3353 sizeof(struct rte_flow_action)); 3354 actions_rx++; 3355 } 3356 break; 3357 default: 3358 rte_memcpy(actions_rx, actions, 3359 sizeof(struct rte_flow_action)); 3360 actions_rx++; 3361 break; 3362 } 3363 } 3364 /* Add set meta action and end action for the Rx flow. */ 3365 tag_action = actions_rx; 3366 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3367 actions_rx++; 3368 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3369 actions_rx++; 3370 set_tag = (void *)actions_rx; 3371 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3372 assert(set_tag->id > REG_NONE); 3373 set_tag->data = *flow_id; 3374 tag_action->conf = set_tag; 3375 /* Create Tx item list. */ 3376 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3377 addr = (void *)&pattern_tx[2]; 3378 item = pattern_tx; 3379 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3380 tag_item = (void *)addr; 3381 tag_item->data = *flow_id; 3382 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3383 assert(set_tag->id > REG_NONE); 3384 item->spec = tag_item; 3385 addr += sizeof(struct mlx5_rte_flow_item_tag); 3386 tag_item = (void *)addr; 3387 tag_item->data = UINT32_MAX; 3388 tag_item->id = UINT16_MAX; 3389 item->mask = tag_item; 3390 addr += sizeof(struct mlx5_rte_flow_item_tag); 3391 item->last = NULL; 3392 item++; 3393 item->type = RTE_FLOW_ITEM_TYPE_END; 3394 return 0; 3395 } 3396 3397 /** 3398 * The last stage of splitting chain, just creates the subflow 3399 * without any modification. 3400 * 3401 * @param dev 3402 * Pointer to Ethernet device. 3403 * @param[in] flow 3404 * Parent flow structure pointer. 3405 * @param[in, out] sub_flow 3406 * Pointer to return the created subflow, may be NULL. 3407 * @param[in] attr 3408 * Flow rule attributes. 3409 * @param[in] items 3410 * Pattern specification (list terminated by the END pattern item). 3411 * @param[in] actions 3412 * Associated actions (list terminated by the END action). 3413 * @param[in] external 3414 * This flow rule is created by request external to PMD. 3415 * @param[out] error 3416 * Perform verbose error reporting if not NULL. 3417 * @return 3418 * 0 on success, negative value otherwise 3419 */ 3420 static int 3421 flow_create_split_inner(struct rte_eth_dev *dev, 3422 struct rte_flow *flow, 3423 struct mlx5_flow **sub_flow, 3424 const struct rte_flow_attr *attr, 3425 const struct rte_flow_item items[], 3426 const struct rte_flow_action actions[], 3427 bool external, struct rte_flow_error *error) 3428 { 3429 struct mlx5_flow *dev_flow; 3430 3431 dev_flow = flow_drv_prepare(flow, attr, items, actions, error); 3432 if (!dev_flow) 3433 return -rte_errno; 3434 dev_flow->flow = flow; 3435 dev_flow->external = external; 3436 /* Subflow object was created, we must include one in the list. */ 3437 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 3438 if (sub_flow) 3439 *sub_flow = dev_flow; 3440 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3441 } 3442 3443 /** 3444 * Split the meter flow. 3445 * 3446 * As meter flow will split to three sub flow, other than meter 3447 * action, the other actions make sense to only meter accepts 3448 * the packet. If it need to be dropped, no other additional 3449 * actions should be take. 3450 * 3451 * One kind of special action which decapsulates the L3 tunnel 3452 * header will be in the prefix sub flow, as not to take the 3453 * L3 tunnel header into account. 3454 * 3455 * @param dev 3456 * Pointer to Ethernet device. 3457 * @param[in] actions 3458 * Associated actions (list terminated by the END action). 3459 * @param[out] actions_sfx 3460 * Suffix flow actions. 3461 * @param[out] actions_pre 3462 * Prefix flow actions. 3463 * @param[out] pattern_sfx 3464 * The pattern items for the suffix flow. 3465 * @param[out] tag_sfx 3466 * Pointer to suffix flow tag. 3467 * 3468 * @return 3469 * 0 on success. 3470 */ 3471 static int 3472 flow_meter_split_prep(struct rte_eth_dev *dev, 3473 const struct rte_flow_action actions[], 3474 struct rte_flow_action actions_sfx[], 3475 struct rte_flow_action actions_pre[]) 3476 { 3477 struct rte_flow_action *tag_action; 3478 struct mlx5_rte_flow_action_set_tag *set_tag; 3479 struct rte_flow_error error; 3480 const struct rte_flow_action_raw_encap *raw_encap; 3481 const struct rte_flow_action_raw_decap *raw_decap; 3482 uint32_t tag_id; 3483 3484 /* Add the extra tag action first. */ 3485 tag_action = actions_pre; 3486 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3487 actions_pre++; 3488 /* Prepare the actions for prefix and suffix flow. */ 3489 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3490 switch (actions->type) { 3491 case RTE_FLOW_ACTION_TYPE_METER: 3492 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3493 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3494 memcpy(actions_pre, actions, 3495 sizeof(struct rte_flow_action)); 3496 actions_pre++; 3497 break; 3498 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3499 raw_encap = actions->conf; 3500 if (raw_encap->size > 3501 (sizeof(struct rte_flow_item_eth) + 3502 sizeof(struct rte_flow_item_ipv4))) { 3503 memcpy(actions_sfx, actions, 3504 sizeof(struct rte_flow_action)); 3505 actions_sfx++; 3506 } else { 3507 rte_memcpy(actions_pre, actions, 3508 sizeof(struct rte_flow_action)); 3509 actions_pre++; 3510 } 3511 break; 3512 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3513 raw_decap = actions->conf; 3514 /* Size 0 decap means 50 bytes as vxlan decap. */ 3515 if (raw_decap->size && (raw_decap->size < 3516 (sizeof(struct rte_flow_item_eth) + 3517 sizeof(struct rte_flow_item_ipv4)))) { 3518 memcpy(actions_sfx, actions, 3519 sizeof(struct rte_flow_action)); 3520 actions_sfx++; 3521 } else { 3522 rte_memcpy(actions_pre, actions, 3523 sizeof(struct rte_flow_action)); 3524 actions_pre++; 3525 } 3526 break; 3527 default: 3528 memcpy(actions_sfx, actions, 3529 sizeof(struct rte_flow_action)); 3530 actions_sfx++; 3531 break; 3532 } 3533 } 3534 /* Add end action to the actions. */ 3535 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3536 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3537 actions_pre++; 3538 /* Set the tag. */ 3539 set_tag = (void *)actions_pre; 3540 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3541 /* 3542 * Get the id from the qrss_pool to make qrss share the id with meter. 3543 */ 3544 tag_id = flow_qrss_get_id(dev); 3545 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 3546 tag_action->conf = set_tag; 3547 return tag_id; 3548 } 3549 3550 /** 3551 * Split action list having QUEUE/RSS for metadata register copy. 3552 * 3553 * Once Q/RSS action is detected in user's action list, the flow action 3554 * should be split in order to copy metadata registers, which will happen in 3555 * RX_CP_TBL like, 3556 * - CQE->flow_tag := reg_c[1] (MARK) 3557 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3558 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3559 * This is because the last action of each flow must be a terminal action 3560 * (QUEUE, RSS or DROP). 3561 * 3562 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3563 * stored and kept in the mlx5_flow structure per each sub_flow. 3564 * 3565 * The Q/RSS action is replaced with, 3566 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3567 * And the following JUMP action is added at the end, 3568 * - JUMP, to RX_CP_TBL. 3569 * 3570 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3571 * flow_create_split_metadata() routine. The flow will look like, 3572 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3573 * 3574 * @param dev 3575 * Pointer to Ethernet device. 3576 * @param[out] split_actions 3577 * Pointer to store split actions to jump to CP_TBL. 3578 * @param[in] actions 3579 * Pointer to the list of original flow actions. 3580 * @param[in] qrss 3581 * Pointer to the Q/RSS action. 3582 * @param[in] actions_n 3583 * Number of original actions. 3584 * @param[out] error 3585 * Perform verbose error reporting if not NULL. 3586 * 3587 * @return 3588 * non-zero unique flow_id on success, otherwise 0 and 3589 * error/rte_error are set. 3590 */ 3591 static uint32_t 3592 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3593 struct rte_flow_action *split_actions, 3594 const struct rte_flow_action *actions, 3595 const struct rte_flow_action *qrss, 3596 int actions_n, struct rte_flow_error *error) 3597 { 3598 struct mlx5_rte_flow_action_set_tag *set_tag; 3599 struct rte_flow_action_jump *jump; 3600 const int qrss_idx = qrss - actions; 3601 uint32_t flow_id = 0; 3602 int ret = 0; 3603 3604 /* 3605 * Given actions will be split 3606 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3607 * - Add jump to mreg CP_TBL. 3608 * As a result, there will be one more action. 3609 */ 3610 ++actions_n; 3611 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3612 set_tag = (void *)(split_actions + actions_n); 3613 /* 3614 * If tag action is not set to void(it means we are not the meter 3615 * suffix flow), add the tag action. Since meter suffix flow already 3616 * has the tag added. 3617 */ 3618 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3619 /* 3620 * Allocate the new subflow ID. This one is unique within 3621 * device and not shared with representors. Otherwise, 3622 * we would have to resolve multi-thread access synch 3623 * issue. Each flow on the shared device is appended 3624 * with source vport identifier, so the resulting 3625 * flows will be unique in the shared (by master and 3626 * representors) domain even if they have coinciding 3627 * IDs. 3628 */ 3629 flow_id = flow_qrss_get_id(dev); 3630 if (!flow_id) 3631 return rte_flow_error_set(error, ENOMEM, 3632 RTE_FLOW_ERROR_TYPE_ACTION, 3633 NULL, "can't allocate id " 3634 "for split Q/RSS subflow"); 3635 /* Internal SET_TAG action to set flow ID. */ 3636 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3637 .data = flow_id, 3638 }; 3639 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3640 if (ret < 0) 3641 return ret; 3642 set_tag->id = ret; 3643 /* Construct new actions array. */ 3644 /* Replace QUEUE/RSS action. */ 3645 split_actions[qrss_idx] = (struct rte_flow_action){ 3646 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3647 .conf = set_tag, 3648 }; 3649 } 3650 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3651 jump = (void *)(set_tag + 1); 3652 *jump = (struct rte_flow_action_jump){ 3653 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3654 }; 3655 split_actions[actions_n - 2] = (struct rte_flow_action){ 3656 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3657 .conf = jump, 3658 }; 3659 split_actions[actions_n - 1] = (struct rte_flow_action){ 3660 .type = RTE_FLOW_ACTION_TYPE_END, 3661 }; 3662 return flow_id; 3663 } 3664 3665 /** 3666 * Extend the given action list for Tx metadata copy. 3667 * 3668 * Copy the given action list to the ext_actions and add flow metadata register 3669 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3670 * 3671 * @param[out] ext_actions 3672 * Pointer to the extended action list. 3673 * @param[in] actions 3674 * Pointer to the list of actions. 3675 * @param[in] actions_n 3676 * Number of actions in the list. 3677 * @param[out] error 3678 * Perform verbose error reporting if not NULL. 3679 * 3680 * @return 3681 * 0 on success, negative value otherwise 3682 */ 3683 static int 3684 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3685 struct rte_flow_action *ext_actions, 3686 const struct rte_flow_action *actions, 3687 int actions_n, struct rte_flow_error *error) 3688 { 3689 struct mlx5_flow_action_copy_mreg *cp_mreg = 3690 (struct mlx5_flow_action_copy_mreg *) 3691 (ext_actions + actions_n + 1); 3692 int ret; 3693 3694 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3695 if (ret < 0) 3696 return ret; 3697 cp_mreg->dst = ret; 3698 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3699 if (ret < 0) 3700 return ret; 3701 cp_mreg->src = ret; 3702 memcpy(ext_actions, actions, 3703 sizeof(*ext_actions) * actions_n); 3704 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3705 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3706 .conf = cp_mreg, 3707 }; 3708 ext_actions[actions_n] = (struct rte_flow_action){ 3709 .type = RTE_FLOW_ACTION_TYPE_END, 3710 }; 3711 return 0; 3712 } 3713 3714 /** 3715 * The splitting for metadata feature. 3716 * 3717 * - Q/RSS action on NIC Rx should be split in order to pass by 3718 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3719 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3720 * 3721 * - All the actions on NIC Tx should have a mreg copy action to 3722 * copy reg_a from WQE to reg_c[0]. 3723 * 3724 * @param dev 3725 * Pointer to Ethernet device. 3726 * @param[in] flow 3727 * Parent flow structure pointer. 3728 * @param[in] attr 3729 * Flow rule attributes. 3730 * @param[in] items 3731 * Pattern specification (list terminated by the END pattern item). 3732 * @param[in] actions 3733 * Associated actions (list terminated by the END action). 3734 * @param[in] external 3735 * This flow rule is created by request external to PMD. 3736 * @param[out] error 3737 * Perform verbose error reporting if not NULL. 3738 * @return 3739 * 0 on success, negative value otherwise 3740 */ 3741 static int 3742 flow_create_split_metadata(struct rte_eth_dev *dev, 3743 struct rte_flow *flow, 3744 const struct rte_flow_attr *attr, 3745 const struct rte_flow_item items[], 3746 const struct rte_flow_action actions[], 3747 bool external, struct rte_flow_error *error) 3748 { 3749 struct mlx5_priv *priv = dev->data->dev_private; 3750 struct mlx5_dev_config *config = &priv->config; 3751 const struct rte_flow_action *qrss = NULL; 3752 struct rte_flow_action *ext_actions = NULL; 3753 struct mlx5_flow *dev_flow = NULL; 3754 uint32_t qrss_id = 0; 3755 int mtr_sfx = 0; 3756 size_t act_size; 3757 int actions_n; 3758 int ret; 3759 3760 /* Check whether extensive metadata feature is engaged. */ 3761 if (!config->dv_flow_en || 3762 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3763 !mlx5_flow_ext_mreg_supported(dev)) 3764 return flow_create_split_inner(dev, flow, NULL, attr, items, 3765 actions, external, error); 3766 actions_n = flow_parse_qrss_action(actions, &qrss); 3767 if (qrss) { 3768 /* Exclude hairpin flows from splitting. */ 3769 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3770 const struct rte_flow_action_queue *queue; 3771 3772 queue = qrss->conf; 3773 if (mlx5_rxq_get_type(dev, queue->index) == 3774 MLX5_RXQ_TYPE_HAIRPIN) 3775 qrss = NULL; 3776 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3777 const struct rte_flow_action_rss *rss; 3778 3779 rss = qrss->conf; 3780 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3781 MLX5_RXQ_TYPE_HAIRPIN) 3782 qrss = NULL; 3783 } 3784 } 3785 if (qrss) { 3786 /* Check if it is in meter suffix table. */ 3787 mtr_sfx = attr->group == (attr->transfer ? 3788 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3789 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3790 /* 3791 * Q/RSS action on NIC Rx should be split in order to pass by 3792 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3793 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3794 */ 3795 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3796 sizeof(struct rte_flow_action_set_tag) + 3797 sizeof(struct rte_flow_action_jump); 3798 ext_actions = rte_zmalloc(__func__, act_size, 0); 3799 if (!ext_actions) 3800 return rte_flow_error_set(error, ENOMEM, 3801 RTE_FLOW_ERROR_TYPE_ACTION, 3802 NULL, "no memory to split " 3803 "metadata flow"); 3804 /* 3805 * If we are the suffix flow of meter, tag already exist. 3806 * Set the tag action to void. 3807 */ 3808 if (mtr_sfx) 3809 ext_actions[qrss - actions].type = 3810 RTE_FLOW_ACTION_TYPE_VOID; 3811 else 3812 ext_actions[qrss - actions].type = 3813 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3814 /* 3815 * Create the new actions list with removed Q/RSS action 3816 * and appended set tag and jump to register copy table 3817 * (RX_CP_TBL). We should preallocate unique tag ID here 3818 * in advance, because it is needed for set tag action. 3819 */ 3820 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3821 qrss, actions_n, error); 3822 if (!mtr_sfx && !qrss_id) { 3823 ret = -rte_errno; 3824 goto exit; 3825 } 3826 } else if (attr->egress && !attr->transfer) { 3827 /* 3828 * All the actions on NIC Tx should have a metadata register 3829 * copy action to copy reg_a from WQE to reg_c[meta] 3830 */ 3831 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3832 sizeof(struct mlx5_flow_action_copy_mreg); 3833 ext_actions = rte_zmalloc(__func__, act_size, 0); 3834 if (!ext_actions) 3835 return rte_flow_error_set(error, ENOMEM, 3836 RTE_FLOW_ERROR_TYPE_ACTION, 3837 NULL, "no memory to split " 3838 "metadata flow"); 3839 /* Create the action list appended with copy register. */ 3840 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 3841 actions_n, error); 3842 if (ret < 0) 3843 goto exit; 3844 } 3845 /* Add the unmodified original or prefix subflow. */ 3846 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 3847 ext_actions ? ext_actions : actions, 3848 external, error); 3849 if (ret < 0) 3850 goto exit; 3851 assert(dev_flow); 3852 if (qrss) { 3853 const struct rte_flow_attr q_attr = { 3854 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3855 .ingress = 1, 3856 }; 3857 /* Internal PMD action to set register. */ 3858 struct mlx5_rte_flow_item_tag q_tag_spec = { 3859 .data = qrss_id, 3860 .id = 0, 3861 }; 3862 struct rte_flow_item q_items[] = { 3863 { 3864 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3865 .spec = &q_tag_spec, 3866 .last = NULL, 3867 .mask = NULL, 3868 }, 3869 { 3870 .type = RTE_FLOW_ITEM_TYPE_END, 3871 }, 3872 }; 3873 struct rte_flow_action q_actions[] = { 3874 { 3875 .type = qrss->type, 3876 .conf = qrss->conf, 3877 }, 3878 { 3879 .type = RTE_FLOW_ACTION_TYPE_END, 3880 }, 3881 }; 3882 uint64_t hash_fields = dev_flow->hash_fields; 3883 3884 /* 3885 * Configure the tag item only if there is no meter subflow. 3886 * Since tag is already marked in the meter suffix subflow 3887 * we can just use the meter suffix items as is. 3888 */ 3889 if (qrss_id) { 3890 /* Not meter subflow. */ 3891 assert(!mtr_sfx); 3892 /* 3893 * Put unique id in prefix flow due to it is destroyed 3894 * after suffix flow and id will be freed after there 3895 * is no actual flows with this id and identifier 3896 * reallocation becomes possible (for example, for 3897 * other flows in other threads). 3898 */ 3899 dev_flow->qrss_id = qrss_id; 3900 qrss_id = 0; 3901 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 3902 error); 3903 if (ret < 0) 3904 goto exit; 3905 q_tag_spec.id = ret; 3906 } 3907 dev_flow = NULL; 3908 /* Add suffix subflow to execute Q/RSS. */ 3909 ret = flow_create_split_inner(dev, flow, &dev_flow, 3910 &q_attr, mtr_sfx ? items : 3911 q_items, q_actions, 3912 external, error); 3913 if (ret < 0) 3914 goto exit; 3915 assert(dev_flow); 3916 dev_flow->hash_fields = hash_fields; 3917 } 3918 3919 exit: 3920 /* 3921 * We do not destroy the partially created sub_flows in case of error. 3922 * These ones are included into parent flow list and will be destroyed 3923 * by flow_drv_destroy. 3924 */ 3925 flow_qrss_free_id(dev, qrss_id); 3926 rte_free(ext_actions); 3927 return ret; 3928 } 3929 3930 /** 3931 * The splitting for meter feature. 3932 * 3933 * - The meter flow will be split to two flows as prefix and 3934 * suffix flow. The packets make sense only it pass the prefix 3935 * meter action. 3936 * 3937 * - Reg_C_5 is used for the packet to match betweend prefix and 3938 * suffix flow. 3939 * 3940 * @param dev 3941 * Pointer to Ethernet device. 3942 * @param[in] flow 3943 * Parent flow structure pointer. 3944 * @param[in] attr 3945 * Flow rule attributes. 3946 * @param[in] items 3947 * Pattern specification (list terminated by the END pattern item). 3948 * @param[in] actions 3949 * Associated actions (list terminated by the END action). 3950 * @param[in] external 3951 * This flow rule is created by request external to PMD. 3952 * @param[out] error 3953 * Perform verbose error reporting if not NULL. 3954 * @return 3955 * 0 on success, negative value otherwise 3956 */ 3957 static int 3958 flow_create_split_meter(struct rte_eth_dev *dev, 3959 struct rte_flow *flow, 3960 const struct rte_flow_attr *attr, 3961 const struct rte_flow_item items[], 3962 const struct rte_flow_action actions[], 3963 bool external, struct rte_flow_error *error) 3964 { 3965 struct mlx5_priv *priv = dev->data->dev_private; 3966 struct rte_flow_action *sfx_actions = NULL; 3967 struct rte_flow_action *pre_actions = NULL; 3968 struct rte_flow_item *sfx_items = NULL; 3969 const struct rte_flow_item *sfx_port_id_item; 3970 struct mlx5_flow *dev_flow = NULL; 3971 struct rte_flow_attr sfx_attr = *attr; 3972 uint32_t mtr = 0; 3973 uint32_t mtr_tag_id = 0; 3974 size_t act_size; 3975 size_t item_size; 3976 int actions_n = 0; 3977 int ret; 3978 3979 if (priv->mtr_en) 3980 actions_n = flow_check_meter_action(actions, &mtr); 3981 if (mtr) { 3982 struct mlx5_rte_flow_item_tag *tag_spec; 3983 struct mlx5_rte_flow_item_tag *tag_mask; 3984 /* The five prefix actions: meter, decap, encap, tag, end. */ 3985 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 3986 sizeof(struct rte_flow_action_set_tag); 3987 /* tag, end. */ 3988 #define METER_SUFFIX_ITEM 3 3989 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 3990 sizeof(struct mlx5_rte_flow_item_tag) * 2; 3991 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 3992 if (!sfx_actions) 3993 return rte_flow_error_set(error, ENOMEM, 3994 RTE_FLOW_ERROR_TYPE_ACTION, 3995 NULL, "no memory to split " 3996 "meter flow"); 3997 pre_actions = sfx_actions + actions_n; 3998 mtr_tag_id = flow_meter_split_prep(dev, actions, sfx_actions, 3999 pre_actions); 4000 if (!mtr_tag_id) { 4001 ret = -rte_errno; 4002 goto exit; 4003 } 4004 /* Add the prefix subflow. */ 4005 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 4006 pre_actions, external, error); 4007 if (ret) { 4008 ret = -rte_errno; 4009 goto exit; 4010 } 4011 dev_flow->mtr_flow_id = mtr_tag_id; 4012 /* Prepare the suffix flow match pattern. */ 4013 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4014 act_size); 4015 tag_spec = (struct mlx5_rte_flow_item_tag *)(sfx_items + 4016 METER_SUFFIX_ITEM); 4017 tag_spec->data = dev_flow->mtr_flow_id << MLX5_MTR_COLOR_BITS; 4018 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, 4019 error); 4020 tag_mask = tag_spec + 1; 4021 tag_mask->data = 0xffffff00; 4022 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4023 sfx_items->spec = tag_spec; 4024 sfx_items->last = NULL; 4025 sfx_items->mask = tag_mask; 4026 sfx_items++; 4027 sfx_port_id_item = find_port_id_item(items); 4028 if (sfx_port_id_item) { 4029 memcpy(sfx_items, sfx_port_id_item, 4030 sizeof(*sfx_items)); 4031 sfx_items++; 4032 } 4033 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4034 sfx_items -= sfx_port_id_item ? 2 : 1; 4035 /* Setting the sfx group atrr. */ 4036 sfx_attr.group = sfx_attr.transfer ? 4037 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4038 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4039 } 4040 /* Add the prefix subflow. */ 4041 ret = flow_create_split_metadata(dev, flow, &sfx_attr, 4042 sfx_items ? sfx_items : items, 4043 sfx_actions ? sfx_actions : actions, 4044 external, error); 4045 exit: 4046 if (sfx_actions) 4047 rte_free(sfx_actions); 4048 return ret; 4049 } 4050 4051 /** 4052 * Split the flow to subflow set. The splitters might be linked 4053 * in the chain, like this: 4054 * flow_create_split_outer() calls: 4055 * flow_create_split_meter() calls: 4056 * flow_create_split_metadata(meter_subflow_0) calls: 4057 * flow_create_split_inner(metadata_subflow_0) 4058 * flow_create_split_inner(metadata_subflow_1) 4059 * flow_create_split_inner(metadata_subflow_2) 4060 * flow_create_split_metadata(meter_subflow_1) calls: 4061 * flow_create_split_inner(metadata_subflow_0) 4062 * flow_create_split_inner(metadata_subflow_1) 4063 * flow_create_split_inner(metadata_subflow_2) 4064 * 4065 * This provide flexible way to add new levels of flow splitting. 4066 * The all of successfully created subflows are included to the 4067 * parent flow dev_flow list. 4068 * 4069 * @param dev 4070 * Pointer to Ethernet device. 4071 * @param[in] flow 4072 * Parent flow structure pointer. 4073 * @param[in] attr 4074 * Flow rule attributes. 4075 * @param[in] items 4076 * Pattern specification (list terminated by the END pattern item). 4077 * @param[in] actions 4078 * Associated actions (list terminated by the END action). 4079 * @param[in] external 4080 * This flow rule is created by request external to PMD. 4081 * @param[out] error 4082 * Perform verbose error reporting if not NULL. 4083 * @return 4084 * 0 on success, negative value otherwise 4085 */ 4086 static int 4087 flow_create_split_outer(struct rte_eth_dev *dev, 4088 struct rte_flow *flow, 4089 const struct rte_flow_attr *attr, 4090 const struct rte_flow_item items[], 4091 const struct rte_flow_action actions[], 4092 bool external, struct rte_flow_error *error) 4093 { 4094 int ret; 4095 4096 ret = flow_create_split_meter(dev, flow, attr, items, 4097 actions, external, error); 4098 assert(ret <= 0); 4099 return ret; 4100 } 4101 4102 /** 4103 * Create a flow and add it to @p list. 4104 * 4105 * @param dev 4106 * Pointer to Ethernet device. 4107 * @param list 4108 * Pointer to a TAILQ flow list. If this parameter NULL, 4109 * no list insertion occurred, flow is just created, 4110 * this is caller's responsibility to track the 4111 * created flow. 4112 * @param[in] attr 4113 * Flow rule attributes. 4114 * @param[in] items 4115 * Pattern specification (list terminated by the END pattern item). 4116 * @param[in] actions 4117 * Associated actions (list terminated by the END action). 4118 * @param[in] external 4119 * This flow rule is created by request external to PMD. 4120 * @param[out] error 4121 * Perform verbose error reporting if not NULL. 4122 * 4123 * @return 4124 * A flow on success, NULL otherwise and rte_errno is set. 4125 */ 4126 static struct rte_flow * 4127 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 4128 const struct rte_flow_attr *attr, 4129 const struct rte_flow_item items[], 4130 const struct rte_flow_action actions[], 4131 bool external, struct rte_flow_error *error) 4132 { 4133 struct mlx5_priv *priv = dev->data->dev_private; 4134 struct rte_flow *flow = NULL; 4135 struct mlx5_flow *dev_flow; 4136 const struct rte_flow_action_rss *rss; 4137 union { 4138 struct rte_flow_expand_rss buf; 4139 uint8_t buffer[2048]; 4140 } expand_buffer; 4141 union { 4142 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4143 uint8_t buffer[2048]; 4144 } actions_rx; 4145 union { 4146 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4147 uint8_t buffer[2048]; 4148 } actions_hairpin_tx; 4149 union { 4150 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4151 uint8_t buffer[2048]; 4152 } items_tx; 4153 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4154 const struct rte_flow_action *p_actions_rx = actions; 4155 int ret; 4156 uint32_t i; 4157 uint32_t flow_size; 4158 int hairpin_flow = 0; 4159 uint32_t hairpin_id = 0; 4160 struct rte_flow_attr attr_tx = { .priority = 0 }; 4161 4162 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4163 if (hairpin_flow > 0) { 4164 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4165 rte_errno = EINVAL; 4166 return NULL; 4167 } 4168 flow_hairpin_split(dev, actions, actions_rx.actions, 4169 actions_hairpin_tx.actions, items_tx.items, 4170 &hairpin_id); 4171 p_actions_rx = actions_rx.actions; 4172 } 4173 ret = flow_drv_validate(dev, attr, items, p_actions_rx, external, 4174 error); 4175 if (ret < 0) 4176 goto error_before_flow; 4177 flow_size = sizeof(struct rte_flow); 4178 rss = flow_get_rss_action(p_actions_rx); 4179 if (rss) 4180 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 4181 sizeof(void *)); 4182 else 4183 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 4184 flow = rte_calloc(__func__, 1, flow_size, 0); 4185 if (!flow) { 4186 rte_errno = ENOMEM; 4187 goto error_before_flow; 4188 } 4189 flow->drv_type = flow_get_drv_type(dev, attr); 4190 if (hairpin_id != 0) 4191 flow->hairpin_flow_id = hairpin_id; 4192 assert(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4193 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4194 flow->rss.queue = (void *)(flow + 1); 4195 if (rss) { 4196 /* 4197 * The following information is required by 4198 * mlx5_flow_hashfields_adjust() in advance. 4199 */ 4200 flow->rss.level = rss->level; 4201 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4202 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 4203 } 4204 LIST_INIT(&flow->dev_flows); 4205 if (rss && rss->types) { 4206 unsigned int graph_root; 4207 4208 graph_root = find_graph_root(items, rss->level); 4209 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4210 items, rss->types, 4211 mlx5_support_expansion, 4212 graph_root); 4213 assert(ret > 0 && 4214 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4215 } else { 4216 buf->entries = 1; 4217 buf->entry[0].pattern = (void *)(uintptr_t)items; 4218 } 4219 for (i = 0; i < buf->entries; ++i) { 4220 /* 4221 * The splitter may create multiple dev_flows, 4222 * depending on configuration. In the simplest 4223 * case it just creates unmodified original flow. 4224 */ 4225 ret = flow_create_split_outer(dev, flow, attr, 4226 buf->entry[i].pattern, 4227 p_actions_rx, external, 4228 error); 4229 if (ret < 0) 4230 goto error; 4231 } 4232 /* Create the tx flow. */ 4233 if (hairpin_flow) { 4234 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4235 attr_tx.ingress = 0; 4236 attr_tx.egress = 1; 4237 dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items, 4238 actions_hairpin_tx.actions, error); 4239 if (!dev_flow) 4240 goto error; 4241 dev_flow->flow = flow; 4242 dev_flow->external = 0; 4243 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 4244 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4245 items_tx.items, 4246 actions_hairpin_tx.actions, error); 4247 if (ret < 0) 4248 goto error; 4249 } 4250 /* 4251 * Update the metadata register copy table. If extensive 4252 * metadata feature is enabled and registers are supported 4253 * we might create the extra rte_flow for each unique 4254 * MARK/FLAG action ID. 4255 * 4256 * The table is updated for ingress Flows only, because 4257 * the egress Flows belong to the different device and 4258 * copy table should be updated in peer NIC Rx domain. 4259 */ 4260 if (attr->ingress && 4261 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4262 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4263 if (ret) 4264 goto error; 4265 } 4266 if (dev->data->dev_started) { 4267 ret = flow_drv_apply(dev, flow, error); 4268 if (ret < 0) 4269 goto error; 4270 } 4271 if (list) 4272 TAILQ_INSERT_TAIL(list, flow, next); 4273 flow_rxq_flags_set(dev, flow); 4274 return flow; 4275 error_before_flow: 4276 if (hairpin_id) 4277 mlx5_flow_id_release(priv->sh->flow_id_pool, 4278 hairpin_id); 4279 return NULL; 4280 error: 4281 assert(flow); 4282 flow_mreg_del_copy_action(dev, flow); 4283 ret = rte_errno; /* Save rte_errno before cleanup. */ 4284 if (flow->hairpin_flow_id) 4285 mlx5_flow_id_release(priv->sh->flow_id_pool, 4286 flow->hairpin_flow_id); 4287 assert(flow); 4288 flow_drv_destroy(dev, flow); 4289 rte_free(flow); 4290 rte_errno = ret; /* Restore rte_errno. */ 4291 return NULL; 4292 } 4293 4294 /** 4295 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4296 * incoming packets to table 1. 4297 * 4298 * Other flow rules, requested for group n, will be created in 4299 * e-switch table n+1. 4300 * Jump action to e-switch group n will be created to group n+1. 4301 * 4302 * Used when working in switchdev mode, to utilise advantages of table 1 4303 * and above. 4304 * 4305 * @param dev 4306 * Pointer to Ethernet device. 4307 * 4308 * @return 4309 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4310 */ 4311 struct rte_flow * 4312 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4313 { 4314 const struct rte_flow_attr attr = { 4315 .group = 0, 4316 .priority = 0, 4317 .ingress = 1, 4318 .egress = 0, 4319 .transfer = 1, 4320 }; 4321 const struct rte_flow_item pattern = { 4322 .type = RTE_FLOW_ITEM_TYPE_END, 4323 }; 4324 struct rte_flow_action_jump jump = { 4325 .group = 1, 4326 }; 4327 const struct rte_flow_action actions[] = { 4328 { 4329 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4330 .conf = &jump, 4331 }, 4332 { 4333 .type = RTE_FLOW_ACTION_TYPE_END, 4334 }, 4335 }; 4336 struct mlx5_priv *priv = dev->data->dev_private; 4337 struct rte_flow_error error; 4338 4339 return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern, 4340 actions, false, &error); 4341 } 4342 4343 /** 4344 * Create a flow. 4345 * 4346 * @see rte_flow_create() 4347 * @see rte_flow_ops 4348 */ 4349 struct rte_flow * 4350 mlx5_flow_create(struct rte_eth_dev *dev, 4351 const struct rte_flow_attr *attr, 4352 const struct rte_flow_item items[], 4353 const struct rte_flow_action actions[], 4354 struct rte_flow_error *error) 4355 { 4356 struct mlx5_priv *priv = dev->data->dev_private; 4357 4358 return flow_list_create(dev, &priv->flows, 4359 attr, items, actions, true, error); 4360 } 4361 4362 /** 4363 * Destroy a flow in a list. 4364 * 4365 * @param dev 4366 * Pointer to Ethernet device. 4367 * @param list 4368 * Pointer to a TAILQ flow list. If this parameter NULL, 4369 * there is no flow removal from the list. 4370 * @param[in] flow 4371 * Flow to destroy. 4372 */ 4373 static void 4374 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 4375 struct rte_flow *flow) 4376 { 4377 struct mlx5_priv *priv = dev->data->dev_private; 4378 4379 /* 4380 * Update RX queue flags only if port is started, otherwise it is 4381 * already clean. 4382 */ 4383 if (dev->data->dev_started) 4384 flow_rxq_flags_trim(dev, flow); 4385 if (flow->hairpin_flow_id) 4386 mlx5_flow_id_release(priv->sh->flow_id_pool, 4387 flow->hairpin_flow_id); 4388 flow_drv_destroy(dev, flow); 4389 if (list) 4390 TAILQ_REMOVE(list, flow, next); 4391 flow_mreg_del_copy_action(dev, flow); 4392 rte_free(flow->fdir); 4393 rte_free(flow); 4394 } 4395 4396 /** 4397 * Destroy all flows. 4398 * 4399 * @param dev 4400 * Pointer to Ethernet device. 4401 * @param list 4402 * Pointer to a TAILQ flow list. 4403 */ 4404 void 4405 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 4406 { 4407 while (!TAILQ_EMPTY(list)) { 4408 struct rte_flow *flow; 4409 4410 flow = TAILQ_FIRST(list); 4411 flow_list_destroy(dev, list, flow); 4412 } 4413 } 4414 4415 /** 4416 * Remove all flows. 4417 * 4418 * @param dev 4419 * Pointer to Ethernet device. 4420 * @param list 4421 * Pointer to a TAILQ flow list. 4422 */ 4423 void 4424 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 4425 { 4426 struct rte_flow *flow; 4427 4428 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 4429 flow_drv_remove(dev, flow); 4430 flow_mreg_stop_copy_action(dev, flow); 4431 } 4432 flow_mreg_del_default_copy_action(dev); 4433 flow_rxq_flags_clear(dev); 4434 } 4435 4436 /** 4437 * Add all flows. 4438 * 4439 * @param dev 4440 * Pointer to Ethernet device. 4441 * @param list 4442 * Pointer to a TAILQ flow list. 4443 * 4444 * @return 4445 * 0 on success, a negative errno value otherwise and rte_errno is set. 4446 */ 4447 int 4448 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 4449 { 4450 struct rte_flow *flow; 4451 struct rte_flow_error error; 4452 int ret = 0; 4453 4454 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4455 ret = flow_mreg_add_default_copy_action(dev, &error); 4456 if (ret < 0) 4457 return -rte_errno; 4458 /* Apply Flows created by application. */ 4459 TAILQ_FOREACH(flow, list, next) { 4460 ret = flow_mreg_start_copy_action(dev, flow); 4461 if (ret < 0) 4462 goto error; 4463 ret = flow_drv_apply(dev, flow, &error); 4464 if (ret < 0) 4465 goto error; 4466 flow_rxq_flags_set(dev, flow); 4467 } 4468 return 0; 4469 error: 4470 ret = rte_errno; /* Save rte_errno before cleanup. */ 4471 mlx5_flow_stop(dev, list); 4472 rte_errno = ret; /* Restore rte_errno. */ 4473 return -rte_errno; 4474 } 4475 4476 /** 4477 * Verify the flow list is empty 4478 * 4479 * @param dev 4480 * Pointer to Ethernet device. 4481 * 4482 * @return the number of flows not released. 4483 */ 4484 int 4485 mlx5_flow_verify(struct rte_eth_dev *dev) 4486 { 4487 struct mlx5_priv *priv = dev->data->dev_private; 4488 struct rte_flow *flow; 4489 int ret = 0; 4490 4491 TAILQ_FOREACH(flow, &priv->flows, next) { 4492 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4493 dev->data->port_id, (void *)flow); 4494 ++ret; 4495 } 4496 return ret; 4497 } 4498 4499 /** 4500 * Enable default hairpin egress flow. 4501 * 4502 * @param dev 4503 * Pointer to Ethernet device. 4504 * @param queue 4505 * The queue index. 4506 * 4507 * @return 4508 * 0 on success, a negative errno value otherwise and rte_errno is set. 4509 */ 4510 int 4511 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4512 uint32_t queue) 4513 { 4514 struct mlx5_priv *priv = dev->data->dev_private; 4515 const struct rte_flow_attr attr = { 4516 .egress = 1, 4517 .priority = 0, 4518 }; 4519 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4520 .queue = queue, 4521 }; 4522 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4523 .queue = UINT32_MAX, 4524 }; 4525 struct rte_flow_item items[] = { 4526 { 4527 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4528 .spec = &queue_spec, 4529 .last = NULL, 4530 .mask = &queue_mask, 4531 }, 4532 { 4533 .type = RTE_FLOW_ITEM_TYPE_END, 4534 }, 4535 }; 4536 struct rte_flow_action_jump jump = { 4537 .group = MLX5_HAIRPIN_TX_TABLE, 4538 }; 4539 struct rte_flow_action actions[2]; 4540 struct rte_flow *flow; 4541 struct rte_flow_error error; 4542 4543 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4544 actions[0].conf = &jump; 4545 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4546 flow = flow_list_create(dev, &priv->ctrl_flows, 4547 &attr, items, actions, false, &error); 4548 if (!flow) { 4549 DRV_LOG(DEBUG, 4550 "Failed to create ctrl flow: rte_errno(%d)," 4551 " type(%d), message(%s)", 4552 rte_errno, error.type, 4553 error.message ? error.message : " (no stated reason)"); 4554 return -rte_errno; 4555 } 4556 return 0; 4557 } 4558 4559 /** 4560 * Enable a control flow configured from the control plane. 4561 * 4562 * @param dev 4563 * Pointer to Ethernet device. 4564 * @param eth_spec 4565 * An Ethernet flow spec to apply. 4566 * @param eth_mask 4567 * An Ethernet flow mask to apply. 4568 * @param vlan_spec 4569 * A VLAN flow spec to apply. 4570 * @param vlan_mask 4571 * A VLAN flow mask to apply. 4572 * 4573 * @return 4574 * 0 on success, a negative errno value otherwise and rte_errno is set. 4575 */ 4576 int 4577 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4578 struct rte_flow_item_eth *eth_spec, 4579 struct rte_flow_item_eth *eth_mask, 4580 struct rte_flow_item_vlan *vlan_spec, 4581 struct rte_flow_item_vlan *vlan_mask) 4582 { 4583 struct mlx5_priv *priv = dev->data->dev_private; 4584 const struct rte_flow_attr attr = { 4585 .ingress = 1, 4586 .priority = MLX5_FLOW_PRIO_RSVD, 4587 }; 4588 struct rte_flow_item items[] = { 4589 { 4590 .type = RTE_FLOW_ITEM_TYPE_ETH, 4591 .spec = eth_spec, 4592 .last = NULL, 4593 .mask = eth_mask, 4594 }, 4595 { 4596 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4597 RTE_FLOW_ITEM_TYPE_END, 4598 .spec = vlan_spec, 4599 .last = NULL, 4600 .mask = vlan_mask, 4601 }, 4602 { 4603 .type = RTE_FLOW_ITEM_TYPE_END, 4604 }, 4605 }; 4606 uint16_t queue[priv->reta_idx_n]; 4607 struct rte_flow_action_rss action_rss = { 4608 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4609 .level = 0, 4610 .types = priv->rss_conf.rss_hf, 4611 .key_len = priv->rss_conf.rss_key_len, 4612 .queue_num = priv->reta_idx_n, 4613 .key = priv->rss_conf.rss_key, 4614 .queue = queue, 4615 }; 4616 struct rte_flow_action actions[] = { 4617 { 4618 .type = RTE_FLOW_ACTION_TYPE_RSS, 4619 .conf = &action_rss, 4620 }, 4621 { 4622 .type = RTE_FLOW_ACTION_TYPE_END, 4623 }, 4624 }; 4625 struct rte_flow *flow; 4626 struct rte_flow_error error; 4627 unsigned int i; 4628 4629 if (!priv->reta_idx_n || !priv->rxqs_n) { 4630 return 0; 4631 } 4632 for (i = 0; i != priv->reta_idx_n; ++i) 4633 queue[i] = (*priv->reta_idx)[i]; 4634 flow = flow_list_create(dev, &priv->ctrl_flows, 4635 &attr, items, actions, false, &error); 4636 if (!flow) 4637 return -rte_errno; 4638 return 0; 4639 } 4640 4641 /** 4642 * Enable a flow control configured from the control plane. 4643 * 4644 * @param dev 4645 * Pointer to Ethernet device. 4646 * @param eth_spec 4647 * An Ethernet flow spec to apply. 4648 * @param eth_mask 4649 * An Ethernet flow mask to apply. 4650 * 4651 * @return 4652 * 0 on success, a negative errno value otherwise and rte_errno is set. 4653 */ 4654 int 4655 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4656 struct rte_flow_item_eth *eth_spec, 4657 struct rte_flow_item_eth *eth_mask) 4658 { 4659 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4660 } 4661 4662 /** 4663 * Destroy a flow. 4664 * 4665 * @see rte_flow_destroy() 4666 * @see rte_flow_ops 4667 */ 4668 int 4669 mlx5_flow_destroy(struct rte_eth_dev *dev, 4670 struct rte_flow *flow, 4671 struct rte_flow_error *error __rte_unused) 4672 { 4673 struct mlx5_priv *priv = dev->data->dev_private; 4674 4675 flow_list_destroy(dev, &priv->flows, flow); 4676 return 0; 4677 } 4678 4679 /** 4680 * Destroy all flows. 4681 * 4682 * @see rte_flow_flush() 4683 * @see rte_flow_ops 4684 */ 4685 int 4686 mlx5_flow_flush(struct rte_eth_dev *dev, 4687 struct rte_flow_error *error __rte_unused) 4688 { 4689 struct mlx5_priv *priv = dev->data->dev_private; 4690 4691 mlx5_flow_list_flush(dev, &priv->flows); 4692 return 0; 4693 } 4694 4695 /** 4696 * Isolated mode. 4697 * 4698 * @see rte_flow_isolate() 4699 * @see rte_flow_ops 4700 */ 4701 int 4702 mlx5_flow_isolate(struct rte_eth_dev *dev, 4703 int enable, 4704 struct rte_flow_error *error) 4705 { 4706 struct mlx5_priv *priv = dev->data->dev_private; 4707 4708 if (dev->data->dev_started) { 4709 rte_flow_error_set(error, EBUSY, 4710 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4711 NULL, 4712 "port must be stopped first"); 4713 return -rte_errno; 4714 } 4715 priv->isolated = !!enable; 4716 if (enable) 4717 dev->dev_ops = &mlx5_dev_ops_isolate; 4718 else 4719 dev->dev_ops = &mlx5_dev_ops; 4720 return 0; 4721 } 4722 4723 /** 4724 * Query a flow. 4725 * 4726 * @see rte_flow_query() 4727 * @see rte_flow_ops 4728 */ 4729 static int 4730 flow_drv_query(struct rte_eth_dev *dev, 4731 struct rte_flow *flow, 4732 const struct rte_flow_action *actions, 4733 void *data, 4734 struct rte_flow_error *error) 4735 { 4736 const struct mlx5_flow_driver_ops *fops; 4737 enum mlx5_flow_drv_type ftype = flow->drv_type; 4738 4739 assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 4740 fops = flow_get_drv_ops(ftype); 4741 4742 return fops->query(dev, flow, actions, data, error); 4743 } 4744 4745 /** 4746 * Query a flow. 4747 * 4748 * @see rte_flow_query() 4749 * @see rte_flow_ops 4750 */ 4751 int 4752 mlx5_flow_query(struct rte_eth_dev *dev, 4753 struct rte_flow *flow, 4754 const struct rte_flow_action *actions, 4755 void *data, 4756 struct rte_flow_error *error) 4757 { 4758 int ret; 4759 4760 ret = flow_drv_query(dev, flow, actions, data, error); 4761 if (ret < 0) 4762 return ret; 4763 return 0; 4764 } 4765 4766 /** 4767 * Convert a flow director filter to a generic flow. 4768 * 4769 * @param dev 4770 * Pointer to Ethernet device. 4771 * @param fdir_filter 4772 * Flow director filter to add. 4773 * @param attributes 4774 * Generic flow parameters structure. 4775 * 4776 * @return 4777 * 0 on success, a negative errno value otherwise and rte_errno is set. 4778 */ 4779 static int 4780 flow_fdir_filter_convert(struct rte_eth_dev *dev, 4781 const struct rte_eth_fdir_filter *fdir_filter, 4782 struct mlx5_fdir *attributes) 4783 { 4784 struct mlx5_priv *priv = dev->data->dev_private; 4785 const struct rte_eth_fdir_input *input = &fdir_filter->input; 4786 const struct rte_eth_fdir_masks *mask = 4787 &dev->data->dev_conf.fdir_conf.mask; 4788 4789 /* Validate queue number. */ 4790 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 4791 DRV_LOG(ERR, "port %u invalid queue number %d", 4792 dev->data->port_id, fdir_filter->action.rx_queue); 4793 rte_errno = EINVAL; 4794 return -rte_errno; 4795 } 4796 attributes->attr.ingress = 1; 4797 attributes->items[0] = (struct rte_flow_item) { 4798 .type = RTE_FLOW_ITEM_TYPE_ETH, 4799 .spec = &attributes->l2, 4800 .mask = &attributes->l2_mask, 4801 }; 4802 switch (fdir_filter->action.behavior) { 4803 case RTE_ETH_FDIR_ACCEPT: 4804 attributes->actions[0] = (struct rte_flow_action){ 4805 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 4806 .conf = &attributes->queue, 4807 }; 4808 break; 4809 case RTE_ETH_FDIR_REJECT: 4810 attributes->actions[0] = (struct rte_flow_action){ 4811 .type = RTE_FLOW_ACTION_TYPE_DROP, 4812 }; 4813 break; 4814 default: 4815 DRV_LOG(ERR, "port %u invalid behavior %d", 4816 dev->data->port_id, 4817 fdir_filter->action.behavior); 4818 rte_errno = ENOTSUP; 4819 return -rte_errno; 4820 } 4821 attributes->queue.index = fdir_filter->action.rx_queue; 4822 /* Handle L3. */ 4823 switch (fdir_filter->input.flow_type) { 4824 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4825 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4826 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4827 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 4828 .src_addr = input->flow.ip4_flow.src_ip, 4829 .dst_addr = input->flow.ip4_flow.dst_ip, 4830 .time_to_live = input->flow.ip4_flow.ttl, 4831 .type_of_service = input->flow.ip4_flow.tos, 4832 }; 4833 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 4834 .src_addr = mask->ipv4_mask.src_ip, 4835 .dst_addr = mask->ipv4_mask.dst_ip, 4836 .time_to_live = mask->ipv4_mask.ttl, 4837 .type_of_service = mask->ipv4_mask.tos, 4838 .next_proto_id = mask->ipv4_mask.proto, 4839 }; 4840 attributes->items[1] = (struct rte_flow_item){ 4841 .type = RTE_FLOW_ITEM_TYPE_IPV4, 4842 .spec = &attributes->l3, 4843 .mask = &attributes->l3_mask, 4844 }; 4845 break; 4846 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4847 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4848 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4849 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 4850 .hop_limits = input->flow.ipv6_flow.hop_limits, 4851 .proto = input->flow.ipv6_flow.proto, 4852 }; 4853 4854 memcpy(attributes->l3.ipv6.hdr.src_addr, 4855 input->flow.ipv6_flow.src_ip, 4856 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4857 memcpy(attributes->l3.ipv6.hdr.dst_addr, 4858 input->flow.ipv6_flow.dst_ip, 4859 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4860 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 4861 mask->ipv6_mask.src_ip, 4862 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4863 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 4864 mask->ipv6_mask.dst_ip, 4865 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4866 attributes->items[1] = (struct rte_flow_item){ 4867 .type = RTE_FLOW_ITEM_TYPE_IPV6, 4868 .spec = &attributes->l3, 4869 .mask = &attributes->l3_mask, 4870 }; 4871 break; 4872 default: 4873 DRV_LOG(ERR, "port %u invalid flow type%d", 4874 dev->data->port_id, fdir_filter->input.flow_type); 4875 rte_errno = ENOTSUP; 4876 return -rte_errno; 4877 } 4878 /* Handle L4. */ 4879 switch (fdir_filter->input.flow_type) { 4880 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4881 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4882 .src_port = input->flow.udp4_flow.src_port, 4883 .dst_port = input->flow.udp4_flow.dst_port, 4884 }; 4885 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4886 .src_port = mask->src_port_mask, 4887 .dst_port = mask->dst_port_mask, 4888 }; 4889 attributes->items[2] = (struct rte_flow_item){ 4890 .type = RTE_FLOW_ITEM_TYPE_UDP, 4891 .spec = &attributes->l4, 4892 .mask = &attributes->l4_mask, 4893 }; 4894 break; 4895 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4896 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4897 .src_port = input->flow.tcp4_flow.src_port, 4898 .dst_port = input->flow.tcp4_flow.dst_port, 4899 }; 4900 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4901 .src_port = mask->src_port_mask, 4902 .dst_port = mask->dst_port_mask, 4903 }; 4904 attributes->items[2] = (struct rte_flow_item){ 4905 .type = RTE_FLOW_ITEM_TYPE_TCP, 4906 .spec = &attributes->l4, 4907 .mask = &attributes->l4_mask, 4908 }; 4909 break; 4910 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4911 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4912 .src_port = input->flow.udp6_flow.src_port, 4913 .dst_port = input->flow.udp6_flow.dst_port, 4914 }; 4915 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4916 .src_port = mask->src_port_mask, 4917 .dst_port = mask->dst_port_mask, 4918 }; 4919 attributes->items[2] = (struct rte_flow_item){ 4920 .type = RTE_FLOW_ITEM_TYPE_UDP, 4921 .spec = &attributes->l4, 4922 .mask = &attributes->l4_mask, 4923 }; 4924 break; 4925 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4926 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4927 .src_port = input->flow.tcp6_flow.src_port, 4928 .dst_port = input->flow.tcp6_flow.dst_port, 4929 }; 4930 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4931 .src_port = mask->src_port_mask, 4932 .dst_port = mask->dst_port_mask, 4933 }; 4934 attributes->items[2] = (struct rte_flow_item){ 4935 .type = RTE_FLOW_ITEM_TYPE_TCP, 4936 .spec = &attributes->l4, 4937 .mask = &attributes->l4_mask, 4938 }; 4939 break; 4940 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4941 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4942 break; 4943 default: 4944 DRV_LOG(ERR, "port %u invalid flow type%d", 4945 dev->data->port_id, fdir_filter->input.flow_type); 4946 rte_errno = ENOTSUP; 4947 return -rte_errno; 4948 } 4949 return 0; 4950 } 4951 4952 #define FLOW_FDIR_CMP(f1, f2, fld) \ 4953 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 4954 4955 /** 4956 * Compare two FDIR flows. If items and actions are identical, the two flows are 4957 * regarded as same. 4958 * 4959 * @param dev 4960 * Pointer to Ethernet device. 4961 * @param f1 4962 * FDIR flow to compare. 4963 * @param f2 4964 * FDIR flow to compare. 4965 * 4966 * @return 4967 * Zero on match, 1 otherwise. 4968 */ 4969 static int 4970 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 4971 { 4972 if (FLOW_FDIR_CMP(f1, f2, attr) || 4973 FLOW_FDIR_CMP(f1, f2, l2) || 4974 FLOW_FDIR_CMP(f1, f2, l2_mask) || 4975 FLOW_FDIR_CMP(f1, f2, l3) || 4976 FLOW_FDIR_CMP(f1, f2, l3_mask) || 4977 FLOW_FDIR_CMP(f1, f2, l4) || 4978 FLOW_FDIR_CMP(f1, f2, l4_mask) || 4979 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 4980 return 1; 4981 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 4982 FLOW_FDIR_CMP(f1, f2, queue)) 4983 return 1; 4984 return 0; 4985 } 4986 4987 /** 4988 * Search device flow list to find out a matched FDIR flow. 4989 * 4990 * @param dev 4991 * Pointer to Ethernet device. 4992 * @param fdir_flow 4993 * FDIR flow to lookup. 4994 * 4995 * @return 4996 * Pointer of flow if found, NULL otherwise. 4997 */ 4998 static struct rte_flow * 4999 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5000 { 5001 struct mlx5_priv *priv = dev->data->dev_private; 5002 struct rte_flow *flow = NULL; 5003 5004 assert(fdir_flow); 5005 TAILQ_FOREACH(flow, &priv->flows, next) { 5006 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 5007 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 5008 dev->data->port_id, (void *)flow); 5009 break; 5010 } 5011 } 5012 return flow; 5013 } 5014 5015 /** 5016 * Add new flow director filter and store it in list. 5017 * 5018 * @param dev 5019 * Pointer to Ethernet device. 5020 * @param fdir_filter 5021 * Flow director filter to add. 5022 * 5023 * @return 5024 * 0 on success, a negative errno value otherwise and rte_errno is set. 5025 */ 5026 static int 5027 flow_fdir_filter_add(struct rte_eth_dev *dev, 5028 const struct rte_eth_fdir_filter *fdir_filter) 5029 { 5030 struct mlx5_priv *priv = dev->data->dev_private; 5031 struct mlx5_fdir *fdir_flow; 5032 struct rte_flow *flow; 5033 int ret; 5034 5035 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5036 if (!fdir_flow) { 5037 rte_errno = ENOMEM; 5038 return -rte_errno; 5039 } 5040 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5041 if (ret) 5042 goto error; 5043 flow = flow_fdir_filter_lookup(dev, fdir_flow); 5044 if (flow) { 5045 rte_errno = EEXIST; 5046 goto error; 5047 } 5048 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5049 fdir_flow->items, fdir_flow->actions, true, 5050 NULL); 5051 if (!flow) 5052 goto error; 5053 assert(!flow->fdir); 5054 flow->fdir = fdir_flow; 5055 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5056 dev->data->port_id, (void *)flow); 5057 return 0; 5058 error: 5059 rte_free(fdir_flow); 5060 return -rte_errno; 5061 } 5062 5063 /** 5064 * Delete specific filter. 5065 * 5066 * @param dev 5067 * Pointer to Ethernet device. 5068 * @param fdir_filter 5069 * Filter to be deleted. 5070 * 5071 * @return 5072 * 0 on success, a negative errno value otherwise and rte_errno is set. 5073 */ 5074 static int 5075 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5076 const struct rte_eth_fdir_filter *fdir_filter) 5077 { 5078 struct mlx5_priv *priv = dev->data->dev_private; 5079 struct rte_flow *flow; 5080 struct mlx5_fdir fdir_flow = { 5081 .attr.group = 0, 5082 }; 5083 int ret; 5084 5085 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5086 if (ret) 5087 return -rte_errno; 5088 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 5089 if (!flow) { 5090 rte_errno = ENOENT; 5091 return -rte_errno; 5092 } 5093 flow_list_destroy(dev, &priv->flows, flow); 5094 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 5095 dev->data->port_id, (void *)flow); 5096 return 0; 5097 } 5098 5099 /** 5100 * Update queue for specific filter. 5101 * 5102 * @param dev 5103 * Pointer to Ethernet device. 5104 * @param fdir_filter 5105 * Filter to be updated. 5106 * 5107 * @return 5108 * 0 on success, a negative errno value otherwise and rte_errno is set. 5109 */ 5110 static int 5111 flow_fdir_filter_update(struct rte_eth_dev *dev, 5112 const struct rte_eth_fdir_filter *fdir_filter) 5113 { 5114 int ret; 5115 5116 ret = flow_fdir_filter_delete(dev, fdir_filter); 5117 if (ret) 5118 return ret; 5119 return flow_fdir_filter_add(dev, fdir_filter); 5120 } 5121 5122 /** 5123 * Flush all filters. 5124 * 5125 * @param dev 5126 * Pointer to Ethernet device. 5127 */ 5128 static void 5129 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5130 { 5131 struct mlx5_priv *priv = dev->data->dev_private; 5132 5133 mlx5_flow_list_flush(dev, &priv->flows); 5134 } 5135 5136 /** 5137 * Get flow director information. 5138 * 5139 * @param dev 5140 * Pointer to Ethernet device. 5141 * @param[out] fdir_info 5142 * Resulting flow director information. 5143 */ 5144 static void 5145 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5146 { 5147 struct rte_eth_fdir_masks *mask = 5148 &dev->data->dev_conf.fdir_conf.mask; 5149 5150 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5151 fdir_info->guarant_spc = 0; 5152 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5153 fdir_info->max_flexpayload = 0; 5154 fdir_info->flow_types_mask[0] = 0; 5155 fdir_info->flex_payload_unit = 0; 5156 fdir_info->max_flex_payload_segment_num = 0; 5157 fdir_info->flex_payload_limit = 0; 5158 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5159 } 5160 5161 /** 5162 * Deal with flow director operations. 5163 * 5164 * @param dev 5165 * Pointer to Ethernet device. 5166 * @param filter_op 5167 * Operation to perform. 5168 * @param arg 5169 * Pointer to operation-specific structure. 5170 * 5171 * @return 5172 * 0 on success, a negative errno value otherwise and rte_errno is set. 5173 */ 5174 static int 5175 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5176 void *arg) 5177 { 5178 enum rte_fdir_mode fdir_mode = 5179 dev->data->dev_conf.fdir_conf.mode; 5180 5181 if (filter_op == RTE_ETH_FILTER_NOP) 5182 return 0; 5183 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5184 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5185 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5186 dev->data->port_id, fdir_mode); 5187 rte_errno = EINVAL; 5188 return -rte_errno; 5189 } 5190 switch (filter_op) { 5191 case RTE_ETH_FILTER_ADD: 5192 return flow_fdir_filter_add(dev, arg); 5193 case RTE_ETH_FILTER_UPDATE: 5194 return flow_fdir_filter_update(dev, arg); 5195 case RTE_ETH_FILTER_DELETE: 5196 return flow_fdir_filter_delete(dev, arg); 5197 case RTE_ETH_FILTER_FLUSH: 5198 flow_fdir_filter_flush(dev); 5199 break; 5200 case RTE_ETH_FILTER_INFO: 5201 flow_fdir_info_get(dev, arg); 5202 break; 5203 default: 5204 DRV_LOG(DEBUG, "port %u unknown operation %u", 5205 dev->data->port_id, filter_op); 5206 rte_errno = EINVAL; 5207 return -rte_errno; 5208 } 5209 return 0; 5210 } 5211 5212 /** 5213 * Manage filter operations. 5214 * 5215 * @param dev 5216 * Pointer to Ethernet device structure. 5217 * @param filter_type 5218 * Filter type. 5219 * @param filter_op 5220 * Operation to perform. 5221 * @param arg 5222 * Pointer to operation-specific structure. 5223 * 5224 * @return 5225 * 0 on success, a negative errno value otherwise and rte_errno is set. 5226 */ 5227 int 5228 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5229 enum rte_filter_type filter_type, 5230 enum rte_filter_op filter_op, 5231 void *arg) 5232 { 5233 switch (filter_type) { 5234 case RTE_ETH_FILTER_GENERIC: 5235 if (filter_op != RTE_ETH_FILTER_GET) { 5236 rte_errno = EINVAL; 5237 return -rte_errno; 5238 } 5239 *(const void **)arg = &mlx5_flow_ops; 5240 return 0; 5241 case RTE_ETH_FILTER_FDIR: 5242 return flow_fdir_ctrl_func(dev, filter_op, arg); 5243 default: 5244 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5245 dev->data->port_id, filter_type); 5246 rte_errno = ENOTSUP; 5247 return -rte_errno; 5248 } 5249 return 0; 5250 } 5251 5252 /** 5253 * Create the needed meter and suffix tables. 5254 * 5255 * @param[in] dev 5256 * Pointer to Ethernet device. 5257 * @param[in] fm 5258 * Pointer to the flow meter. 5259 * 5260 * @return 5261 * Pointer to table set on success, NULL otherwise. 5262 */ 5263 struct mlx5_meter_domains_infos * 5264 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5265 const struct mlx5_flow_meter *fm) 5266 { 5267 const struct mlx5_flow_driver_ops *fops; 5268 5269 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5270 return fops->create_mtr_tbls(dev, fm); 5271 } 5272 5273 /** 5274 * Destroy the meter table set. 5275 * 5276 * @param[in] dev 5277 * Pointer to Ethernet device. 5278 * @param[in] tbl 5279 * Pointer to the meter table set. 5280 * 5281 * @return 5282 * 0 on success. 5283 */ 5284 int 5285 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5286 struct mlx5_meter_domains_infos *tbls) 5287 { 5288 const struct mlx5_flow_driver_ops *fops; 5289 5290 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5291 return fops->destroy_mtr_tbls(dev, tbls); 5292 } 5293 5294 /** 5295 * Create policer rules. 5296 * 5297 * @param[in] dev 5298 * Pointer to Ethernet device. 5299 * @param[in] fm 5300 * Pointer to flow meter structure. 5301 * @param[in] attr 5302 * Pointer to flow attributes. 5303 * 5304 * @return 5305 * 0 on success, -1 otherwise. 5306 */ 5307 int 5308 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5309 struct mlx5_flow_meter *fm, 5310 const struct rte_flow_attr *attr) 5311 { 5312 const struct mlx5_flow_driver_ops *fops; 5313 5314 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5315 return fops->create_policer_rules(dev, fm, attr); 5316 } 5317 5318 /** 5319 * Destroy policer rules. 5320 * 5321 * @param[in] fm 5322 * Pointer to flow meter structure. 5323 * @param[in] attr 5324 * Pointer to flow attributes. 5325 * 5326 * @return 5327 * 0 on success, -1 otherwise. 5328 */ 5329 int 5330 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5331 struct mlx5_flow_meter *fm, 5332 const struct rte_flow_attr *attr) 5333 { 5334 const struct mlx5_flow_driver_ops *fops; 5335 5336 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5337 return fops->destroy_policer_rules(dev, fm, attr); 5338 } 5339 5340 /** 5341 * Allocate a counter. 5342 * 5343 * @param[in] dev 5344 * Pointer to Ethernet device structure. 5345 * 5346 * @return 5347 * Pointer to allocated counter on success, NULL otherwise. 5348 */ 5349 struct mlx5_flow_counter * 5350 mlx5_counter_alloc(struct rte_eth_dev *dev) 5351 { 5352 const struct mlx5_flow_driver_ops *fops; 5353 struct rte_flow_attr attr = { .transfer = 0 }; 5354 5355 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5356 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5357 return fops->counter_alloc(dev); 5358 } 5359 DRV_LOG(ERR, 5360 "port %u counter allocate is not supported.", 5361 dev->data->port_id); 5362 return NULL; 5363 } 5364 5365 /** 5366 * Free a counter. 5367 * 5368 * @param[in] dev 5369 * Pointer to Ethernet device structure. 5370 * @param[in] cnt 5371 * Pointer to counter to be free. 5372 */ 5373 void 5374 mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) 5375 { 5376 const struct mlx5_flow_driver_ops *fops; 5377 struct rte_flow_attr attr = { .transfer = 0 }; 5378 5379 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5380 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5381 fops->counter_free(dev, cnt); 5382 return; 5383 } 5384 DRV_LOG(ERR, 5385 "port %u counter free is not supported.", 5386 dev->data->port_id); 5387 } 5388 5389 /** 5390 * Query counter statistics. 5391 * 5392 * @param[in] dev 5393 * Pointer to Ethernet device structure. 5394 * @param[in] cnt 5395 * Pointer to counter to query. 5396 * @param[in] clear 5397 * Set to clear counter statistics. 5398 * @param[out] pkts 5399 * The counter hits packets number to save. 5400 * @param[out] bytes 5401 * The counter hits bytes number to save. 5402 * 5403 * @return 5404 * 0 on success, a negative errno value otherwise. 5405 */ 5406 int 5407 mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt, 5408 bool clear, uint64_t *pkts, uint64_t *bytes) 5409 { 5410 const struct mlx5_flow_driver_ops *fops; 5411 struct rte_flow_attr attr = { .transfer = 0 }; 5412 5413 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5414 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5415 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5416 } 5417 DRV_LOG(ERR, 5418 "port %u counter query is not supported.", 5419 dev->data->port_id); 5420 return -ENOTSUP; 5421 } 5422 5423 #define MLX5_POOL_QUERY_FREQ_US 1000000 5424 5425 /** 5426 * Set the periodic procedure for triggering asynchronous batch queries for all 5427 * the counter pools. 5428 * 5429 * @param[in] sh 5430 * Pointer to mlx5_ibv_shared object. 5431 */ 5432 void 5433 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5434 { 5435 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 5436 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 5437 uint32_t us; 5438 5439 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 5440 pools_n += rte_atomic16_read(&cont->n_valid); 5441 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5442 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5443 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5444 sh->cmng.query_thread_on = 0; 5445 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5446 } else { 5447 sh->cmng.query_thread_on = 1; 5448 } 5449 } 5450 5451 /** 5452 * The periodic procedure for triggering asynchronous batch queries for all the 5453 * counter pools. This function is probably called by the host thread. 5454 * 5455 * @param[in] arg 5456 * The parameter for the alarm process. 5457 */ 5458 void 5459 mlx5_flow_query_alarm(void *arg) 5460 { 5461 struct mlx5_ibv_shared *sh = arg; 5462 struct mlx5_devx_obj *dcs; 5463 uint16_t offset; 5464 int ret; 5465 uint8_t batch = sh->cmng.batch; 5466 uint16_t pool_index = sh->cmng.pool_index; 5467 struct mlx5_pools_container *cont; 5468 struct mlx5_pools_container *mcont; 5469 struct mlx5_flow_counter_pool *pool; 5470 5471 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5472 goto set_alarm; 5473 next_container: 5474 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 5475 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 5476 /* Check if resize was done and need to flip a container. */ 5477 if (cont != mcont) { 5478 if (cont->pools) { 5479 /* Clean the old container. */ 5480 rte_free(cont->pools); 5481 memset(cont, 0, sizeof(*cont)); 5482 } 5483 rte_cio_wmb(); 5484 /* Flip the host container. */ 5485 sh->cmng.mhi[batch] ^= (uint8_t)2; 5486 cont = mcont; 5487 } 5488 if (!cont->pools) { 5489 /* 2 empty containers case is unexpected. */ 5490 if (unlikely(batch != sh->cmng.batch)) 5491 goto set_alarm; 5492 batch ^= 0x1; 5493 pool_index = 0; 5494 goto next_container; 5495 } 5496 pool = cont->pools[pool_index]; 5497 if (pool->raw_hw) 5498 /* There is a pool query in progress. */ 5499 goto set_alarm; 5500 pool->raw_hw = 5501 LIST_FIRST(&sh->cmng.free_stat_raws); 5502 if (!pool->raw_hw) 5503 /* No free counter statistics raw memory. */ 5504 goto set_alarm; 5505 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5506 (&pool->a64_dcs); 5507 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5508 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5509 offset, NULL, NULL, 5510 pool->raw_hw->mem_mng->dm->id, 5511 (void *)(uintptr_t) 5512 (pool->raw_hw->data + offset), 5513 sh->devx_comp, 5514 (uint64_t)(uintptr_t)pool); 5515 if (ret) { 5516 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5517 " %d", pool->min_dcs->id); 5518 pool->raw_hw = NULL; 5519 goto set_alarm; 5520 } 5521 pool->raw_hw->min_dcs_id = dcs->id; 5522 LIST_REMOVE(pool->raw_hw, next); 5523 sh->cmng.pending_queries++; 5524 pool_index++; 5525 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5526 batch ^= 0x1; 5527 pool_index = 0; 5528 } 5529 set_alarm: 5530 sh->cmng.batch = batch; 5531 sh->cmng.pool_index = pool_index; 5532 mlx5_set_query_alarm(sh); 5533 } 5534 5535 /** 5536 * Handler for the HW respond about ready values from an asynchronous batch 5537 * query. This function is probably called by the host thread. 5538 * 5539 * @param[in] sh 5540 * The pointer to the shared IB device context. 5541 * @param[in] async_id 5542 * The Devx async ID. 5543 * @param[in] status 5544 * The status of the completion. 5545 */ 5546 void 5547 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 5548 uint64_t async_id, int status) 5549 { 5550 struct mlx5_flow_counter_pool *pool = 5551 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 5552 struct mlx5_counter_stats_raw *raw_to_free; 5553 5554 if (unlikely(status)) { 5555 raw_to_free = pool->raw_hw; 5556 } else { 5557 raw_to_free = pool->raw; 5558 rte_spinlock_lock(&pool->sl); 5559 pool->raw = pool->raw_hw; 5560 rte_spinlock_unlock(&pool->sl); 5561 rte_atomic64_add(&pool->query_gen, 1); 5562 /* Be sure the new raw counters data is updated in memory. */ 5563 rte_cio_wmb(); 5564 } 5565 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 5566 pool->raw_hw = NULL; 5567 sh->cmng.pending_queries--; 5568 } 5569 5570 /** 5571 * Translate the rte_flow group index to HW table value. 5572 * 5573 * @param[in] attributes 5574 * Pointer to flow attributes 5575 * @param[in] external 5576 * Value is part of flow rule created by request external to PMD. 5577 * @param[in] group 5578 * rte_flow group index value. 5579 * @param[out] table 5580 * HW table value. 5581 * @param[out] error 5582 * Pointer to error structure. 5583 * 5584 * @return 5585 * 0 on success, a negative errno value otherwise and rte_errno is set. 5586 */ 5587 int 5588 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 5589 uint32_t group, uint32_t *table, 5590 struct rte_flow_error *error) 5591 { 5592 if (attributes->transfer && external) { 5593 if (group == UINT32_MAX) 5594 return rte_flow_error_set 5595 (error, EINVAL, 5596 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 5597 NULL, 5598 "group index not supported"); 5599 *table = group + 1; 5600 } else { 5601 *table = group; 5602 } 5603 return 0; 5604 } 5605 5606 /** 5607 * Discover availability of metadata reg_c's. 5608 * 5609 * Iteratively use test flows to check availability. 5610 * 5611 * @param[in] dev 5612 * Pointer to the Ethernet device structure. 5613 * 5614 * @return 5615 * 0 on success, a negative errno value otherwise and rte_errno is set. 5616 */ 5617 int 5618 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 5619 { 5620 struct mlx5_priv *priv = dev->data->dev_private; 5621 struct mlx5_dev_config *config = &priv->config; 5622 enum modify_reg idx; 5623 int n = 0; 5624 5625 /* reg_c[0] and reg_c[1] are reserved. */ 5626 config->flow_mreg_c[n++] = REG_C_0; 5627 config->flow_mreg_c[n++] = REG_C_1; 5628 /* Discover availability of other reg_c's. */ 5629 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 5630 struct rte_flow_attr attr = { 5631 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5632 .priority = MLX5_FLOW_PRIO_RSVD, 5633 .ingress = 1, 5634 }; 5635 struct rte_flow_item items[] = { 5636 [0] = { 5637 .type = RTE_FLOW_ITEM_TYPE_END, 5638 }, 5639 }; 5640 struct rte_flow_action actions[] = { 5641 [0] = { 5642 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5643 .conf = &(struct mlx5_flow_action_copy_mreg){ 5644 .src = REG_C_1, 5645 .dst = idx, 5646 }, 5647 }, 5648 [1] = { 5649 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5650 .conf = &(struct rte_flow_action_jump){ 5651 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5652 }, 5653 }, 5654 [2] = { 5655 .type = RTE_FLOW_ACTION_TYPE_END, 5656 }, 5657 }; 5658 struct rte_flow *flow; 5659 struct rte_flow_error error; 5660 5661 if (!config->dv_flow_en) 5662 break; 5663 /* Create internal flow, validation skips copy action. */ 5664 flow = flow_list_create(dev, NULL, &attr, items, 5665 actions, false, &error); 5666 if (!flow) 5667 continue; 5668 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 5669 config->flow_mreg_c[n++] = idx; 5670 flow_list_destroy(dev, NULL, flow); 5671 } 5672 for (; n < MLX5_MREG_C_NUM; ++n) 5673 config->flow_mreg_c[n] = REG_NONE; 5674 return 0; 5675 } 5676 5677 /** 5678 * Dump flow raw hw data to file 5679 * 5680 * @param[in] dev 5681 * The pointer to Ethernet device. 5682 * @param[in] file 5683 * A pointer to a file for output. 5684 * @param[out] error 5685 * Perform verbose error reporting if not NULL. PMDs initialize this 5686 * structure in case of error only. 5687 * @return 5688 * 0 on success, a nagative value otherwise. 5689 */ 5690 int 5691 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 5692 FILE *file, 5693 struct rte_flow_error *error __rte_unused) 5694 { 5695 struct mlx5_priv *priv = dev->data->dev_private; 5696 struct mlx5_ibv_shared *sh = priv->sh; 5697 5698 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 5699 sh->tx_domain, file); 5700 } 5701