1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 12 /* Verbs header. */ 13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 14 #ifdef PEDANTIC 15 #pragma GCC diagnostic ignored "-Wpedantic" 16 #endif 17 #include <infiniband/verbs.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_common.h> 23 #include <rte_ether.h> 24 #include <rte_ethdev_driver.h> 25 #include <rte_flow.h> 26 #include <rte_flow_driver.h> 27 #include <rte_malloc.h> 28 #include <rte_ip.h> 29 30 #include "mlx5.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_flow.h" 33 #include "mlx5_glue.h" 34 #include "mlx5_prm.h" 35 #include "mlx5_rxtx.h" 36 37 /* Dev ops structure defined in mlx5.c */ 38 extern const struct eth_dev_ops mlx5_dev_ops; 39 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 40 41 /** Device flow drivers. */ 42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 43 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 44 #endif 45 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 46 47 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 48 49 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 50 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 51 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 52 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 53 #endif 54 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 55 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 56 }; 57 58 enum mlx5_expansion { 59 MLX5_EXPANSION_ROOT, 60 MLX5_EXPANSION_ROOT_OUTER, 61 MLX5_EXPANSION_ROOT_ETH_VLAN, 62 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 63 MLX5_EXPANSION_OUTER_ETH, 64 MLX5_EXPANSION_OUTER_ETH_VLAN, 65 MLX5_EXPANSION_OUTER_VLAN, 66 MLX5_EXPANSION_OUTER_IPV4, 67 MLX5_EXPANSION_OUTER_IPV4_UDP, 68 MLX5_EXPANSION_OUTER_IPV4_TCP, 69 MLX5_EXPANSION_OUTER_IPV6, 70 MLX5_EXPANSION_OUTER_IPV6_UDP, 71 MLX5_EXPANSION_OUTER_IPV6_TCP, 72 MLX5_EXPANSION_VXLAN, 73 MLX5_EXPANSION_VXLAN_GPE, 74 MLX5_EXPANSION_GRE, 75 MLX5_EXPANSION_MPLS, 76 MLX5_EXPANSION_ETH, 77 MLX5_EXPANSION_ETH_VLAN, 78 MLX5_EXPANSION_VLAN, 79 MLX5_EXPANSION_IPV4, 80 MLX5_EXPANSION_IPV4_UDP, 81 MLX5_EXPANSION_IPV4_TCP, 82 MLX5_EXPANSION_IPV6, 83 MLX5_EXPANSION_IPV6_UDP, 84 MLX5_EXPANSION_IPV6_TCP, 85 }; 86 87 /** Supported expansion of items. */ 88 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 89 [MLX5_EXPANSION_ROOT] = { 90 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 91 MLX5_EXPANSION_IPV4, 92 MLX5_EXPANSION_IPV6), 93 .type = RTE_FLOW_ITEM_TYPE_END, 94 }, 95 [MLX5_EXPANSION_ROOT_OUTER] = { 96 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 97 MLX5_EXPANSION_OUTER_IPV4, 98 MLX5_EXPANSION_OUTER_IPV6), 99 .type = RTE_FLOW_ITEM_TYPE_END, 100 }, 101 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 102 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 103 .type = RTE_FLOW_ITEM_TYPE_END, 104 }, 105 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_END, 108 }, 109 [MLX5_EXPANSION_OUTER_ETH] = { 110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 111 MLX5_EXPANSION_OUTER_IPV6, 112 MLX5_EXPANSION_MPLS), 113 .type = RTE_FLOW_ITEM_TYPE_ETH, 114 .rss_types = 0, 115 }, 116 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 117 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 118 .type = RTE_FLOW_ITEM_TYPE_ETH, 119 .rss_types = 0, 120 }, 121 [MLX5_EXPANSION_OUTER_VLAN] = { 122 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 123 MLX5_EXPANSION_OUTER_IPV6), 124 .type = RTE_FLOW_ITEM_TYPE_VLAN, 125 }, 126 [MLX5_EXPANSION_OUTER_IPV4] = { 127 .next = RTE_FLOW_EXPAND_RSS_NEXT 128 (MLX5_EXPANSION_OUTER_IPV4_UDP, 129 MLX5_EXPANSION_OUTER_IPV4_TCP, 130 MLX5_EXPANSION_GRE, 131 MLX5_EXPANSION_IPV4, 132 MLX5_EXPANSION_IPV6), 133 .type = RTE_FLOW_ITEM_TYPE_IPV4, 134 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 135 ETH_RSS_NONFRAG_IPV4_OTHER, 136 }, 137 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 138 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 139 MLX5_EXPANSION_VXLAN_GPE), 140 .type = RTE_FLOW_ITEM_TYPE_UDP, 141 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 142 }, 143 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 144 .type = RTE_FLOW_ITEM_TYPE_TCP, 145 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 146 }, 147 [MLX5_EXPANSION_OUTER_IPV6] = { 148 .next = RTE_FLOW_EXPAND_RSS_NEXT 149 (MLX5_EXPANSION_OUTER_IPV6_UDP, 150 MLX5_EXPANSION_OUTER_IPV6_TCP, 151 MLX5_EXPANSION_IPV4, 152 MLX5_EXPANSION_IPV6), 153 .type = RTE_FLOW_ITEM_TYPE_IPV6, 154 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 155 ETH_RSS_NONFRAG_IPV6_OTHER, 156 }, 157 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 158 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 159 MLX5_EXPANSION_VXLAN_GPE), 160 .type = RTE_FLOW_ITEM_TYPE_UDP, 161 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 162 }, 163 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 164 .type = RTE_FLOW_ITEM_TYPE_TCP, 165 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 166 }, 167 [MLX5_EXPANSION_VXLAN] = { 168 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 169 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 170 }, 171 [MLX5_EXPANSION_VXLAN_GPE] = { 172 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 173 MLX5_EXPANSION_IPV4, 174 MLX5_EXPANSION_IPV6), 175 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 176 }, 177 [MLX5_EXPANSION_GRE] = { 178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 179 .type = RTE_FLOW_ITEM_TYPE_GRE, 180 }, 181 [MLX5_EXPANSION_MPLS] = { 182 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 183 MLX5_EXPANSION_IPV6), 184 .type = RTE_FLOW_ITEM_TYPE_MPLS, 185 }, 186 [MLX5_EXPANSION_ETH] = { 187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 188 MLX5_EXPANSION_IPV6), 189 .type = RTE_FLOW_ITEM_TYPE_ETH, 190 }, 191 [MLX5_EXPANSION_ETH_VLAN] = { 192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 193 .type = RTE_FLOW_ITEM_TYPE_ETH, 194 }, 195 [MLX5_EXPANSION_VLAN] = { 196 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 197 MLX5_EXPANSION_IPV6), 198 .type = RTE_FLOW_ITEM_TYPE_VLAN, 199 }, 200 [MLX5_EXPANSION_IPV4] = { 201 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 202 MLX5_EXPANSION_IPV4_TCP), 203 .type = RTE_FLOW_ITEM_TYPE_IPV4, 204 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 205 ETH_RSS_NONFRAG_IPV4_OTHER, 206 }, 207 [MLX5_EXPANSION_IPV4_UDP] = { 208 .type = RTE_FLOW_ITEM_TYPE_UDP, 209 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 210 }, 211 [MLX5_EXPANSION_IPV4_TCP] = { 212 .type = RTE_FLOW_ITEM_TYPE_TCP, 213 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 214 }, 215 [MLX5_EXPANSION_IPV6] = { 216 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 217 MLX5_EXPANSION_IPV6_TCP), 218 .type = RTE_FLOW_ITEM_TYPE_IPV6, 219 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 220 ETH_RSS_NONFRAG_IPV6_OTHER, 221 }, 222 [MLX5_EXPANSION_IPV6_UDP] = { 223 .type = RTE_FLOW_ITEM_TYPE_UDP, 224 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 225 }, 226 [MLX5_EXPANSION_IPV6_TCP] = { 227 .type = RTE_FLOW_ITEM_TYPE_TCP, 228 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 229 }, 230 }; 231 232 static const struct rte_flow_ops mlx5_flow_ops = { 233 .validate = mlx5_flow_validate, 234 .create = mlx5_flow_create, 235 .destroy = mlx5_flow_destroy, 236 .flush = mlx5_flow_flush, 237 .isolate = mlx5_flow_isolate, 238 .query = mlx5_flow_query, 239 }; 240 241 /* Convert FDIR request to Generic flow. */ 242 struct mlx5_fdir { 243 struct rte_flow_attr attr; 244 struct rte_flow_item items[4]; 245 struct rte_flow_item_eth l2; 246 struct rte_flow_item_eth l2_mask; 247 union { 248 struct rte_flow_item_ipv4 ipv4; 249 struct rte_flow_item_ipv6 ipv6; 250 } l3; 251 union { 252 struct rte_flow_item_ipv4 ipv4; 253 struct rte_flow_item_ipv6 ipv6; 254 } l3_mask; 255 union { 256 struct rte_flow_item_udp udp; 257 struct rte_flow_item_tcp tcp; 258 } l4; 259 union { 260 struct rte_flow_item_udp udp; 261 struct rte_flow_item_tcp tcp; 262 } l4_mask; 263 struct rte_flow_action actions[2]; 264 struct rte_flow_action_queue queue; 265 }; 266 267 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 268 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 269 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 270 }; 271 272 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 273 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 274 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 275 { 9, 10, 11 }, { 12, 13, 14 }, 276 }; 277 278 /* Tunnel information. */ 279 struct mlx5_flow_tunnel_info { 280 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 281 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 282 }; 283 284 static struct mlx5_flow_tunnel_info tunnels_info[] = { 285 { 286 .tunnel = MLX5_FLOW_LAYER_VXLAN, 287 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 288 }, 289 { 290 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 291 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 292 }, 293 { 294 .tunnel = MLX5_FLOW_LAYER_GRE, 295 .ptype = RTE_PTYPE_TUNNEL_GRE, 296 }, 297 { 298 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 299 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 300 }, 301 { 302 .tunnel = MLX5_FLOW_LAYER_MPLS, 303 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 304 }, 305 { 306 .tunnel = MLX5_FLOW_LAYER_NVGRE, 307 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 308 }, 309 { 310 .tunnel = MLX5_FLOW_LAYER_IPIP, 311 .ptype = RTE_PTYPE_TUNNEL_IP, 312 }, 313 { 314 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 315 .ptype = RTE_PTYPE_TUNNEL_IP, 316 }, 317 }; 318 319 /** 320 * Discover the maximum number of priority available. 321 * 322 * @param[in] dev 323 * Pointer to the Ethernet device structure. 324 * 325 * @return 326 * number of supported flow priority on success, a negative errno 327 * value otherwise and rte_errno is set. 328 */ 329 int 330 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 331 { 332 struct mlx5_priv *priv = dev->data->dev_private; 333 struct { 334 struct ibv_flow_attr attr; 335 struct ibv_flow_spec_eth eth; 336 struct ibv_flow_spec_action_drop drop; 337 } flow_attr = { 338 .attr = { 339 .num_of_specs = 2, 340 .port = (uint8_t)priv->ibv_port, 341 }, 342 .eth = { 343 .type = IBV_FLOW_SPEC_ETH, 344 .size = sizeof(struct ibv_flow_spec_eth), 345 }, 346 .drop = { 347 .size = sizeof(struct ibv_flow_spec_action_drop), 348 .type = IBV_FLOW_SPEC_ACTION_DROP, 349 }, 350 }; 351 struct ibv_flow *flow; 352 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 353 uint16_t vprio[] = { 8, 16 }; 354 int i; 355 int priority = 0; 356 357 if (!drop) { 358 rte_errno = ENOTSUP; 359 return -rte_errno; 360 } 361 for (i = 0; i != RTE_DIM(vprio); i++) { 362 flow_attr.attr.priority = vprio[i] - 1; 363 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 364 if (!flow) 365 break; 366 claim_zero(mlx5_glue->destroy_flow(flow)); 367 priority = vprio[i]; 368 } 369 mlx5_hrxq_drop_release(dev); 370 switch (priority) { 371 case 8: 372 priority = RTE_DIM(priority_map_3); 373 break; 374 case 16: 375 priority = RTE_DIM(priority_map_5); 376 break; 377 default: 378 rte_errno = ENOTSUP; 379 DRV_LOG(ERR, 380 "port %u verbs maximum priority: %d expected 8/16", 381 dev->data->port_id, priority); 382 return -rte_errno; 383 } 384 DRV_LOG(INFO, "port %u flow maximum priority: %d", 385 dev->data->port_id, priority); 386 return priority; 387 } 388 389 /** 390 * Adjust flow priority based on the highest layer and the request priority. 391 * 392 * @param[in] dev 393 * Pointer to the Ethernet device structure. 394 * @param[in] priority 395 * The rule base priority. 396 * @param[in] subpriority 397 * The priority based on the items. 398 * 399 * @return 400 * The new priority. 401 */ 402 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 403 uint32_t subpriority) 404 { 405 uint32_t res = 0; 406 struct mlx5_priv *priv = dev->data->dev_private; 407 408 switch (priv->config.flow_prio) { 409 case RTE_DIM(priority_map_3): 410 res = priority_map_3[priority][subpriority]; 411 break; 412 case RTE_DIM(priority_map_5): 413 res = priority_map_5[priority][subpriority]; 414 break; 415 } 416 return res; 417 } 418 419 /** 420 * Verify the @p item specifications (spec, last, mask) are compatible with the 421 * NIC capabilities. 422 * 423 * @param[in] item 424 * Item specification. 425 * @param[in] mask 426 * @p item->mask or flow default bit-masks. 427 * @param[in] nic_mask 428 * Bit-masks covering supported fields by the NIC to compare with user mask. 429 * @param[in] size 430 * Bit-masks size in bytes. 431 * @param[out] error 432 * Pointer to error structure. 433 * 434 * @return 435 * 0 on success, a negative errno value otherwise and rte_errno is set. 436 */ 437 int 438 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 439 const uint8_t *mask, 440 const uint8_t *nic_mask, 441 unsigned int size, 442 struct rte_flow_error *error) 443 { 444 unsigned int i; 445 446 assert(nic_mask); 447 for (i = 0; i < size; ++i) 448 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 449 return rte_flow_error_set(error, ENOTSUP, 450 RTE_FLOW_ERROR_TYPE_ITEM, 451 item, 452 "mask enables non supported" 453 " bits"); 454 if (!item->spec && (item->mask || item->last)) 455 return rte_flow_error_set(error, EINVAL, 456 RTE_FLOW_ERROR_TYPE_ITEM, item, 457 "mask/last without a spec is not" 458 " supported"); 459 if (item->spec && item->last) { 460 uint8_t spec[size]; 461 uint8_t last[size]; 462 unsigned int i; 463 int ret; 464 465 for (i = 0; i < size; ++i) { 466 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 467 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 468 } 469 ret = memcmp(spec, last, size); 470 if (ret != 0) 471 return rte_flow_error_set(error, EINVAL, 472 RTE_FLOW_ERROR_TYPE_ITEM, 473 item, 474 "range is not valid"); 475 } 476 return 0; 477 } 478 479 /** 480 * Adjust the hash fields according to the @p flow information. 481 * 482 * @param[in] dev_flow. 483 * Pointer to the mlx5_flow. 484 * @param[in] tunnel 485 * 1 when the hash field is for a tunnel item. 486 * @param[in] layer_types 487 * ETH_RSS_* types. 488 * @param[in] hash_fields 489 * Item hash fields. 490 * 491 * @return 492 * The hash fields that should be used. 493 */ 494 uint64_t 495 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 496 int tunnel __rte_unused, uint64_t layer_types, 497 uint64_t hash_fields) 498 { 499 struct rte_flow *flow = dev_flow->flow; 500 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 501 int rss_request_inner = flow->rss.level >= 2; 502 503 /* Check RSS hash level for tunnel. */ 504 if (tunnel && rss_request_inner) 505 hash_fields |= IBV_RX_HASH_INNER; 506 else if (tunnel || rss_request_inner) 507 return 0; 508 #endif 509 /* Check if requested layer matches RSS hash fields. */ 510 if (!(flow->rss.types & layer_types)) 511 return 0; 512 return hash_fields; 513 } 514 515 /** 516 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 517 * if several tunnel rules are used on this queue, the tunnel ptype will be 518 * cleared. 519 * 520 * @param rxq_ctrl 521 * Rx queue to update. 522 */ 523 static void 524 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 525 { 526 unsigned int i; 527 uint32_t tunnel_ptype = 0; 528 529 /* Look up for the ptype to use. */ 530 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 531 if (!rxq_ctrl->flow_tunnels_n[i]) 532 continue; 533 if (!tunnel_ptype) { 534 tunnel_ptype = tunnels_info[i].ptype; 535 } else { 536 tunnel_ptype = 0; 537 break; 538 } 539 } 540 rxq_ctrl->rxq.tunnel = tunnel_ptype; 541 } 542 543 /** 544 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 545 * flow. 546 * 547 * @param[in] dev 548 * Pointer to the Ethernet device structure. 549 * @param[in] dev_flow 550 * Pointer to device flow structure. 551 */ 552 static void 553 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 554 { 555 struct mlx5_priv *priv = dev->data->dev_private; 556 struct rte_flow *flow = dev_flow->flow; 557 const int mark = !!(flow->actions & 558 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 559 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 560 unsigned int i; 561 562 for (i = 0; i != flow->rss.queue_num; ++i) { 563 int idx = (*flow->queue)[i]; 564 struct mlx5_rxq_ctrl *rxq_ctrl = 565 container_of((*priv->rxqs)[idx], 566 struct mlx5_rxq_ctrl, rxq); 567 568 if (mark) { 569 rxq_ctrl->rxq.mark = 1; 570 rxq_ctrl->flow_mark_n++; 571 } 572 if (tunnel) { 573 unsigned int j; 574 575 /* Increase the counter matching the flow. */ 576 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 577 if ((tunnels_info[j].tunnel & 578 dev_flow->layers) == 579 tunnels_info[j].tunnel) { 580 rxq_ctrl->flow_tunnels_n[j]++; 581 break; 582 } 583 } 584 flow_rxq_tunnel_ptype_update(rxq_ctrl); 585 } 586 } 587 } 588 589 /** 590 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 591 * 592 * @param[in] dev 593 * Pointer to the Ethernet device structure. 594 * @param[in] flow 595 * Pointer to flow structure. 596 */ 597 static void 598 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 599 { 600 struct mlx5_flow *dev_flow; 601 602 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 603 flow_drv_rxq_flags_set(dev, dev_flow); 604 } 605 606 /** 607 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 608 * device flow if no other flow uses it with the same kind of request. 609 * 610 * @param dev 611 * Pointer to Ethernet device. 612 * @param[in] dev_flow 613 * Pointer to the device flow. 614 */ 615 static void 616 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 617 { 618 struct mlx5_priv *priv = dev->data->dev_private; 619 struct rte_flow *flow = dev_flow->flow; 620 const int mark = !!(flow->actions & 621 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 622 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 623 unsigned int i; 624 625 assert(dev->data->dev_started); 626 for (i = 0; i != flow->rss.queue_num; ++i) { 627 int idx = (*flow->queue)[i]; 628 struct mlx5_rxq_ctrl *rxq_ctrl = 629 container_of((*priv->rxqs)[idx], 630 struct mlx5_rxq_ctrl, rxq); 631 632 if (mark) { 633 rxq_ctrl->flow_mark_n--; 634 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 635 } 636 if (tunnel) { 637 unsigned int j; 638 639 /* Decrease the counter matching the flow. */ 640 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 641 if ((tunnels_info[j].tunnel & 642 dev_flow->layers) == 643 tunnels_info[j].tunnel) { 644 rxq_ctrl->flow_tunnels_n[j]--; 645 break; 646 } 647 } 648 flow_rxq_tunnel_ptype_update(rxq_ctrl); 649 } 650 } 651 } 652 653 /** 654 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 655 * @p flow if no other flow uses it with the same kind of request. 656 * 657 * @param dev 658 * Pointer to Ethernet device. 659 * @param[in] flow 660 * Pointer to the flow. 661 */ 662 static void 663 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 664 { 665 struct mlx5_flow *dev_flow; 666 667 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 668 flow_drv_rxq_flags_trim(dev, dev_flow); 669 } 670 671 /** 672 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 673 * 674 * @param dev 675 * Pointer to Ethernet device. 676 */ 677 static void 678 flow_rxq_flags_clear(struct rte_eth_dev *dev) 679 { 680 struct mlx5_priv *priv = dev->data->dev_private; 681 unsigned int i; 682 683 for (i = 0; i != priv->rxqs_n; ++i) { 684 struct mlx5_rxq_ctrl *rxq_ctrl; 685 unsigned int j; 686 687 if (!(*priv->rxqs)[i]) 688 continue; 689 rxq_ctrl = container_of((*priv->rxqs)[i], 690 struct mlx5_rxq_ctrl, rxq); 691 rxq_ctrl->flow_mark_n = 0; 692 rxq_ctrl->rxq.mark = 0; 693 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 694 rxq_ctrl->flow_tunnels_n[j] = 0; 695 rxq_ctrl->rxq.tunnel = 0; 696 } 697 } 698 699 /* 700 * Validate the flag action. 701 * 702 * @param[in] action_flags 703 * Bit-fields that holds the actions detected until now. 704 * @param[in] attr 705 * Attributes of flow that includes this action. 706 * @param[out] error 707 * Pointer to error structure. 708 * 709 * @return 710 * 0 on success, a negative errno value otherwise and rte_errno is set. 711 */ 712 int 713 mlx5_flow_validate_action_flag(uint64_t action_flags, 714 const struct rte_flow_attr *attr, 715 struct rte_flow_error *error) 716 { 717 718 if (action_flags & MLX5_FLOW_ACTION_DROP) 719 return rte_flow_error_set(error, EINVAL, 720 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 721 "can't drop and flag in same flow"); 722 if (action_flags & MLX5_FLOW_ACTION_MARK) 723 return rte_flow_error_set(error, EINVAL, 724 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 725 "can't mark and flag in same flow"); 726 if (action_flags & MLX5_FLOW_ACTION_FLAG) 727 return rte_flow_error_set(error, EINVAL, 728 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 729 "can't have 2 flag" 730 " actions in same flow"); 731 if (attr->egress) 732 return rte_flow_error_set(error, ENOTSUP, 733 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 734 "flag action not supported for " 735 "egress"); 736 return 0; 737 } 738 739 /* 740 * Validate the mark action. 741 * 742 * @param[in] action 743 * Pointer to the queue action. 744 * @param[in] action_flags 745 * Bit-fields that holds the actions detected until now. 746 * @param[in] attr 747 * Attributes of flow that includes this action. 748 * @param[out] error 749 * Pointer to error structure. 750 * 751 * @return 752 * 0 on success, a negative errno value otherwise and rte_errno is set. 753 */ 754 int 755 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 756 uint64_t action_flags, 757 const struct rte_flow_attr *attr, 758 struct rte_flow_error *error) 759 { 760 const struct rte_flow_action_mark *mark = action->conf; 761 762 if (!mark) 763 return rte_flow_error_set(error, EINVAL, 764 RTE_FLOW_ERROR_TYPE_ACTION, 765 action, 766 "configuration cannot be null"); 767 if (mark->id >= MLX5_FLOW_MARK_MAX) 768 return rte_flow_error_set(error, EINVAL, 769 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 770 &mark->id, 771 "mark id must in 0 <= id < " 772 RTE_STR(MLX5_FLOW_MARK_MAX)); 773 if (action_flags & MLX5_FLOW_ACTION_DROP) 774 return rte_flow_error_set(error, EINVAL, 775 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 776 "can't drop and mark in same flow"); 777 if (action_flags & MLX5_FLOW_ACTION_FLAG) 778 return rte_flow_error_set(error, EINVAL, 779 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 780 "can't flag and mark in same flow"); 781 if (action_flags & MLX5_FLOW_ACTION_MARK) 782 return rte_flow_error_set(error, EINVAL, 783 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 784 "can't have 2 mark actions in same" 785 " flow"); 786 if (attr->egress) 787 return rte_flow_error_set(error, ENOTSUP, 788 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 789 "mark action not supported for " 790 "egress"); 791 return 0; 792 } 793 794 /* 795 * Validate the drop action. 796 * 797 * @param[in] action_flags 798 * Bit-fields that holds the actions detected until now. 799 * @param[in] attr 800 * Attributes of flow that includes this action. 801 * @param[out] error 802 * Pointer to error structure. 803 * 804 * @return 805 * 0 on success, a negative errno value otherwise and rte_errno is set. 806 */ 807 int 808 mlx5_flow_validate_action_drop(uint64_t action_flags, 809 const struct rte_flow_attr *attr, 810 struct rte_flow_error *error) 811 { 812 if (action_flags & MLX5_FLOW_ACTION_FLAG) 813 return rte_flow_error_set(error, EINVAL, 814 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 815 "can't drop and flag in same flow"); 816 if (action_flags & MLX5_FLOW_ACTION_MARK) 817 return rte_flow_error_set(error, EINVAL, 818 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 819 "can't drop and mark in same flow"); 820 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 821 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 822 return rte_flow_error_set(error, EINVAL, 823 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 824 "can't have 2 fate actions in" 825 " same flow"); 826 if (attr->egress) 827 return rte_flow_error_set(error, ENOTSUP, 828 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 829 "drop action not supported for " 830 "egress"); 831 return 0; 832 } 833 834 /* 835 * Validate the queue action. 836 * 837 * @param[in] action 838 * Pointer to the queue action. 839 * @param[in] action_flags 840 * Bit-fields that holds the actions detected until now. 841 * @param[in] dev 842 * Pointer to the Ethernet device structure. 843 * @param[in] attr 844 * Attributes of flow that includes this action. 845 * @param[out] error 846 * Pointer to error structure. 847 * 848 * @return 849 * 0 on success, a negative errno value otherwise and rte_errno is set. 850 */ 851 int 852 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 853 uint64_t action_flags, 854 struct rte_eth_dev *dev, 855 const struct rte_flow_attr *attr, 856 struct rte_flow_error *error) 857 { 858 struct mlx5_priv *priv = dev->data->dev_private; 859 const struct rte_flow_action_queue *queue = action->conf; 860 861 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 862 return rte_flow_error_set(error, EINVAL, 863 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 864 "can't have 2 fate actions in" 865 " same flow"); 866 if (!priv->rxqs_n) 867 return rte_flow_error_set(error, EINVAL, 868 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 869 NULL, "No Rx queues configured"); 870 if (queue->index >= priv->rxqs_n) 871 return rte_flow_error_set(error, EINVAL, 872 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 873 &queue->index, 874 "queue index out of range"); 875 if (!(*priv->rxqs)[queue->index]) 876 return rte_flow_error_set(error, EINVAL, 877 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 878 &queue->index, 879 "queue is not configured"); 880 if (attr->egress) 881 return rte_flow_error_set(error, ENOTSUP, 882 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 883 "queue action not supported for " 884 "egress"); 885 return 0; 886 } 887 888 /* 889 * Validate the rss action. 890 * 891 * @param[in] action 892 * Pointer to the queue action. 893 * @param[in] action_flags 894 * Bit-fields that holds the actions detected until now. 895 * @param[in] dev 896 * Pointer to the Ethernet device structure. 897 * @param[in] attr 898 * Attributes of flow that includes this action. 899 * @param[in] item_flags 900 * Items that were detected. 901 * @param[out] error 902 * Pointer to error structure. 903 * 904 * @return 905 * 0 on success, a negative errno value otherwise and rte_errno is set. 906 */ 907 int 908 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 909 uint64_t action_flags, 910 struct rte_eth_dev *dev, 911 const struct rte_flow_attr *attr, 912 uint64_t item_flags, 913 struct rte_flow_error *error) 914 { 915 struct mlx5_priv *priv = dev->data->dev_private; 916 const struct rte_flow_action_rss *rss = action->conf; 917 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 918 unsigned int i; 919 920 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 921 return rte_flow_error_set(error, EINVAL, 922 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 923 "can't have 2 fate actions" 924 " in same flow"); 925 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 926 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 927 return rte_flow_error_set(error, ENOTSUP, 928 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 929 &rss->func, 930 "RSS hash function not supported"); 931 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 932 if (rss->level > 2) 933 #else 934 if (rss->level > 1) 935 #endif 936 return rte_flow_error_set(error, ENOTSUP, 937 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 938 &rss->level, 939 "tunnel RSS is not supported"); 940 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 941 if (rss->key_len == 0 && rss->key != NULL) 942 return rte_flow_error_set(error, ENOTSUP, 943 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 944 &rss->key_len, 945 "RSS hash key length 0"); 946 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 947 return rte_flow_error_set(error, ENOTSUP, 948 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 949 &rss->key_len, 950 "RSS hash key too small"); 951 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 952 return rte_flow_error_set(error, ENOTSUP, 953 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 954 &rss->key_len, 955 "RSS hash key too large"); 956 if (rss->queue_num > priv->config.ind_table_max_size) 957 return rte_flow_error_set(error, ENOTSUP, 958 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 959 &rss->queue_num, 960 "number of queues too large"); 961 if (rss->types & MLX5_RSS_HF_MASK) 962 return rte_flow_error_set(error, ENOTSUP, 963 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 964 &rss->types, 965 "some RSS protocols are not" 966 " supported"); 967 if (!priv->rxqs_n) 968 return rte_flow_error_set(error, EINVAL, 969 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 970 NULL, "No Rx queues configured"); 971 if (!rss->queue_num) 972 return rte_flow_error_set(error, EINVAL, 973 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 974 NULL, "No queues configured"); 975 for (i = 0; i != rss->queue_num; ++i) { 976 if (!(*priv->rxqs)[rss->queue[i]]) 977 return rte_flow_error_set 978 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 979 &rss->queue[i], "queue is not configured"); 980 } 981 if (attr->egress) 982 return rte_flow_error_set(error, ENOTSUP, 983 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 984 "rss action not supported for " 985 "egress"); 986 if (rss->level > 1 && !tunnel) 987 return rte_flow_error_set(error, EINVAL, 988 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 989 "inner RSS is not supported for " 990 "non-tunnel flows"); 991 return 0; 992 } 993 994 /* 995 * Validate the count action. 996 * 997 * @param[in] dev 998 * Pointer to the Ethernet device structure. 999 * @param[in] attr 1000 * Attributes of flow that includes this action. 1001 * @param[out] error 1002 * Pointer to error structure. 1003 * 1004 * @return 1005 * 0 on success, a negative errno value otherwise and rte_errno is set. 1006 */ 1007 int 1008 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1009 const struct rte_flow_attr *attr, 1010 struct rte_flow_error *error) 1011 { 1012 if (attr->egress) 1013 return rte_flow_error_set(error, ENOTSUP, 1014 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1015 "count action not supported for " 1016 "egress"); 1017 return 0; 1018 } 1019 1020 /** 1021 * Verify the @p attributes will be correctly understood by the NIC and store 1022 * them in the @p flow if everything is correct. 1023 * 1024 * @param[in] dev 1025 * Pointer to the Ethernet device structure. 1026 * @param[in] attributes 1027 * Pointer to flow attributes 1028 * @param[out] error 1029 * Pointer to error structure. 1030 * 1031 * @return 1032 * 0 on success, a negative errno value otherwise and rte_errno is set. 1033 */ 1034 int 1035 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1036 const struct rte_flow_attr *attributes, 1037 struct rte_flow_error *error) 1038 { 1039 struct mlx5_priv *priv = dev->data->dev_private; 1040 uint32_t priority_max = priv->config.flow_prio - 1; 1041 1042 if (attributes->group) 1043 return rte_flow_error_set(error, ENOTSUP, 1044 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1045 NULL, "groups is not supported"); 1046 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1047 attributes->priority >= priority_max) 1048 return rte_flow_error_set(error, ENOTSUP, 1049 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1050 NULL, "priority out of range"); 1051 if (attributes->egress) 1052 return rte_flow_error_set(error, ENOTSUP, 1053 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1054 "egress is not supported"); 1055 if (attributes->transfer && !priv->config.dv_esw_en) 1056 return rte_flow_error_set(error, ENOTSUP, 1057 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1058 NULL, "transfer is not supported"); 1059 if (!attributes->ingress) 1060 return rte_flow_error_set(error, EINVAL, 1061 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1062 NULL, 1063 "ingress attribute is mandatory"); 1064 return 0; 1065 } 1066 1067 /** 1068 * Validate ICMP6 item. 1069 * 1070 * @param[in] item 1071 * Item specification. 1072 * @param[in] item_flags 1073 * Bit-fields that holds the items detected until now. 1074 * @param[out] error 1075 * Pointer to error structure. 1076 * 1077 * @return 1078 * 0 on success, a negative errno value otherwise and rte_errno is set. 1079 */ 1080 int 1081 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1082 uint64_t item_flags, 1083 uint8_t target_protocol, 1084 struct rte_flow_error *error) 1085 { 1086 const struct rte_flow_item_icmp6 *mask = item->mask; 1087 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1088 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1089 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1090 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1091 MLX5_FLOW_LAYER_OUTER_L4; 1092 int ret; 1093 1094 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1095 return rte_flow_error_set(error, EINVAL, 1096 RTE_FLOW_ERROR_TYPE_ITEM, item, 1097 "protocol filtering not compatible" 1098 " with ICMP6 layer"); 1099 if (!(item_flags & l3m)) 1100 return rte_flow_error_set(error, EINVAL, 1101 RTE_FLOW_ERROR_TYPE_ITEM, item, 1102 "IPv6 is mandatory to filter on" 1103 " ICMP6"); 1104 if (item_flags & l4m) 1105 return rte_flow_error_set(error, EINVAL, 1106 RTE_FLOW_ERROR_TYPE_ITEM, item, 1107 "multiple L4 layers not supported"); 1108 if (!mask) 1109 mask = &rte_flow_item_icmp6_mask; 1110 ret = mlx5_flow_item_acceptable 1111 (item, (const uint8_t *)mask, 1112 (const uint8_t *)&rte_flow_item_icmp6_mask, 1113 sizeof(struct rte_flow_item_icmp6), error); 1114 if (ret < 0) 1115 return ret; 1116 return 0; 1117 } 1118 1119 /** 1120 * Validate ICMP item. 1121 * 1122 * @param[in] item 1123 * Item specification. 1124 * @param[in] item_flags 1125 * Bit-fields that holds the items detected until now. 1126 * @param[out] error 1127 * Pointer to error structure. 1128 * 1129 * @return 1130 * 0 on success, a negative errno value otherwise and rte_errno is set. 1131 */ 1132 int 1133 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1134 uint64_t item_flags, 1135 uint8_t target_protocol, 1136 struct rte_flow_error *error) 1137 { 1138 const struct rte_flow_item_icmp *mask = item->mask; 1139 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1140 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1141 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1142 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1143 MLX5_FLOW_LAYER_OUTER_L4; 1144 int ret; 1145 1146 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1147 return rte_flow_error_set(error, EINVAL, 1148 RTE_FLOW_ERROR_TYPE_ITEM, item, 1149 "protocol filtering not compatible" 1150 " with ICMP layer"); 1151 if (!(item_flags & l3m)) 1152 return rte_flow_error_set(error, EINVAL, 1153 RTE_FLOW_ERROR_TYPE_ITEM, item, 1154 "IPv4 is mandatory to filter" 1155 " on ICMP"); 1156 if (item_flags & l4m) 1157 return rte_flow_error_set(error, EINVAL, 1158 RTE_FLOW_ERROR_TYPE_ITEM, item, 1159 "multiple L4 layers not supported"); 1160 if (!mask) 1161 mask = &rte_flow_item_icmp_mask; 1162 ret = mlx5_flow_item_acceptable 1163 (item, (const uint8_t *)mask, 1164 (const uint8_t *)&rte_flow_item_icmp_mask, 1165 sizeof(struct rte_flow_item_icmp), error); 1166 if (ret < 0) 1167 return ret; 1168 return 0; 1169 } 1170 1171 /** 1172 * Validate Ethernet item. 1173 * 1174 * @param[in] item 1175 * Item specification. 1176 * @param[in] item_flags 1177 * Bit-fields that holds the items detected until now. 1178 * @param[out] error 1179 * Pointer to error structure. 1180 * 1181 * @return 1182 * 0 on success, a negative errno value otherwise and rte_errno is set. 1183 */ 1184 int 1185 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1186 uint64_t item_flags, 1187 struct rte_flow_error *error) 1188 { 1189 const struct rte_flow_item_eth *mask = item->mask; 1190 const struct rte_flow_item_eth nic_mask = { 1191 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1192 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1193 .type = RTE_BE16(0xffff), 1194 }; 1195 int ret; 1196 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1197 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1198 MLX5_FLOW_LAYER_OUTER_L2; 1199 1200 if (item_flags & ethm) 1201 return rte_flow_error_set(error, ENOTSUP, 1202 RTE_FLOW_ERROR_TYPE_ITEM, item, 1203 "multiple L2 layers not supported"); 1204 if (!mask) 1205 mask = &rte_flow_item_eth_mask; 1206 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1207 (const uint8_t *)&nic_mask, 1208 sizeof(struct rte_flow_item_eth), 1209 error); 1210 return ret; 1211 } 1212 1213 /** 1214 * Validate VLAN item. 1215 * 1216 * @param[in] item 1217 * Item specification. 1218 * @param[in] item_flags 1219 * Bit-fields that holds the items detected until now. 1220 * @param[in] dev 1221 * Ethernet device flow is being created on. 1222 * @param[out] error 1223 * Pointer to error structure. 1224 * 1225 * @return 1226 * 0 on success, a negative errno value otherwise and rte_errno is set. 1227 */ 1228 int 1229 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1230 uint64_t item_flags, 1231 struct rte_eth_dev *dev, 1232 struct rte_flow_error *error) 1233 { 1234 const struct rte_flow_item_vlan *spec = item->spec; 1235 const struct rte_flow_item_vlan *mask = item->mask; 1236 const struct rte_flow_item_vlan nic_mask = { 1237 .tci = RTE_BE16(UINT16_MAX), 1238 .inner_type = RTE_BE16(UINT16_MAX), 1239 }; 1240 uint16_t vlan_tag = 0; 1241 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1242 int ret; 1243 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1244 MLX5_FLOW_LAYER_INNER_L4) : 1245 (MLX5_FLOW_LAYER_OUTER_L3 | 1246 MLX5_FLOW_LAYER_OUTER_L4); 1247 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1248 MLX5_FLOW_LAYER_OUTER_VLAN; 1249 1250 if (item_flags & vlanm) 1251 return rte_flow_error_set(error, EINVAL, 1252 RTE_FLOW_ERROR_TYPE_ITEM, item, 1253 "multiple VLAN layers not supported"); 1254 else if ((item_flags & l34m) != 0) 1255 return rte_flow_error_set(error, EINVAL, 1256 RTE_FLOW_ERROR_TYPE_ITEM, item, 1257 "L2 layer cannot follow L3/L4 layer"); 1258 if (!mask) 1259 mask = &rte_flow_item_vlan_mask; 1260 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1261 (const uint8_t *)&nic_mask, 1262 sizeof(struct rte_flow_item_vlan), 1263 error); 1264 if (ret) 1265 return ret; 1266 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1267 struct mlx5_priv *priv = dev->data->dev_private; 1268 1269 if (priv->vmwa_context) { 1270 /* 1271 * Non-NULL context means we have a virtual machine 1272 * and SR-IOV enabled, we have to create VLAN interface 1273 * to make hypervisor to setup E-Switch vport 1274 * context correctly. We avoid creating the multiple 1275 * VLAN interfaces, so we cannot support VLAN tag mask. 1276 */ 1277 return rte_flow_error_set(error, EINVAL, 1278 RTE_FLOW_ERROR_TYPE_ITEM, 1279 item, 1280 "VLAN tag mask is not" 1281 " supported in virtual" 1282 " environment"); 1283 } 1284 } 1285 if (spec) { 1286 vlan_tag = spec->tci; 1287 vlan_tag &= mask->tci; 1288 } 1289 /* 1290 * From verbs perspective an empty VLAN is equivalent 1291 * to a packet without VLAN layer. 1292 */ 1293 if (!vlan_tag) 1294 return rte_flow_error_set(error, EINVAL, 1295 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1296 item->spec, 1297 "VLAN cannot be empty"); 1298 return 0; 1299 } 1300 1301 /** 1302 * Validate IPV4 item. 1303 * 1304 * @param[in] item 1305 * Item specification. 1306 * @param[in] item_flags 1307 * Bit-fields that holds the items detected until now. 1308 * @param[in] acc_mask 1309 * Acceptable mask, if NULL default internal default mask 1310 * will be used to check whether item fields are supported. 1311 * @param[out] error 1312 * Pointer to error structure. 1313 * 1314 * @return 1315 * 0 on success, a negative errno value otherwise and rte_errno is set. 1316 */ 1317 int 1318 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1319 uint64_t item_flags, 1320 const struct rte_flow_item_ipv4 *acc_mask, 1321 struct rte_flow_error *error) 1322 { 1323 const struct rte_flow_item_ipv4 *mask = item->mask; 1324 const struct rte_flow_item_ipv4 *spec = item->spec; 1325 const struct rte_flow_item_ipv4 nic_mask = { 1326 .hdr = { 1327 .src_addr = RTE_BE32(0xffffffff), 1328 .dst_addr = RTE_BE32(0xffffffff), 1329 .type_of_service = 0xff, 1330 .next_proto_id = 0xff, 1331 }, 1332 }; 1333 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1334 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1335 MLX5_FLOW_LAYER_OUTER_L3; 1336 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1337 MLX5_FLOW_LAYER_OUTER_L4; 1338 int ret; 1339 uint8_t next_proto = 0xFF; 1340 1341 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1342 if (mask && spec) 1343 next_proto = mask->hdr.next_proto_id & 1344 spec->hdr.next_proto_id; 1345 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1346 return rte_flow_error_set(error, EINVAL, 1347 RTE_FLOW_ERROR_TYPE_ITEM, 1348 item, 1349 "multiple tunnel " 1350 "not supported"); 1351 } 1352 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1353 return rte_flow_error_set(error, EINVAL, 1354 RTE_FLOW_ERROR_TYPE_ITEM, item, 1355 "wrong tunnel type - IPv6 specified " 1356 "but IPv4 item provided"); 1357 if (item_flags & l3m) 1358 return rte_flow_error_set(error, ENOTSUP, 1359 RTE_FLOW_ERROR_TYPE_ITEM, item, 1360 "multiple L3 layers not supported"); 1361 else if (item_flags & l4m) 1362 return rte_flow_error_set(error, EINVAL, 1363 RTE_FLOW_ERROR_TYPE_ITEM, item, 1364 "L3 cannot follow an L4 layer."); 1365 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1366 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1367 return rte_flow_error_set(error, EINVAL, 1368 RTE_FLOW_ERROR_TYPE_ITEM, item, 1369 "L3 cannot follow an NVGRE layer."); 1370 if (!mask) 1371 mask = &rte_flow_item_ipv4_mask; 1372 else if (mask->hdr.next_proto_id != 0 && 1373 mask->hdr.next_proto_id != 0xff) 1374 return rte_flow_error_set(error, EINVAL, 1375 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1376 "partial mask is not supported" 1377 " for protocol"); 1378 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1379 acc_mask ? (const uint8_t *)acc_mask 1380 : (const uint8_t *)&nic_mask, 1381 sizeof(struct rte_flow_item_ipv4), 1382 error); 1383 if (ret < 0) 1384 return ret; 1385 return 0; 1386 } 1387 1388 /** 1389 * Validate IPV6 item. 1390 * 1391 * @param[in] item 1392 * Item specification. 1393 * @param[in] item_flags 1394 * Bit-fields that holds the items detected until now. 1395 * @param[in] acc_mask 1396 * Acceptable mask, if NULL default internal default mask 1397 * will be used to check whether item fields are supported. 1398 * @param[out] error 1399 * Pointer to error structure. 1400 * 1401 * @return 1402 * 0 on success, a negative errno value otherwise and rte_errno is set. 1403 */ 1404 int 1405 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1406 uint64_t item_flags, 1407 const struct rte_flow_item_ipv6 *acc_mask, 1408 struct rte_flow_error *error) 1409 { 1410 const struct rte_flow_item_ipv6 *mask = item->mask; 1411 const struct rte_flow_item_ipv6 *spec = item->spec; 1412 const struct rte_flow_item_ipv6 nic_mask = { 1413 .hdr = { 1414 .src_addr = 1415 "\xff\xff\xff\xff\xff\xff\xff\xff" 1416 "\xff\xff\xff\xff\xff\xff\xff\xff", 1417 .dst_addr = 1418 "\xff\xff\xff\xff\xff\xff\xff\xff" 1419 "\xff\xff\xff\xff\xff\xff\xff\xff", 1420 .vtc_flow = RTE_BE32(0xffffffff), 1421 .proto = 0xff, 1422 .hop_limits = 0xff, 1423 }, 1424 }; 1425 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1426 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1427 MLX5_FLOW_LAYER_OUTER_L3; 1428 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1429 MLX5_FLOW_LAYER_OUTER_L4; 1430 int ret; 1431 uint8_t next_proto = 0xFF; 1432 1433 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1434 if (mask && spec) 1435 next_proto = mask->hdr.proto & spec->hdr.proto; 1436 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1437 return rte_flow_error_set(error, EINVAL, 1438 RTE_FLOW_ERROR_TYPE_ITEM, 1439 item, 1440 "multiple tunnel " 1441 "not supported"); 1442 } 1443 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1444 return rte_flow_error_set(error, EINVAL, 1445 RTE_FLOW_ERROR_TYPE_ITEM, item, 1446 "wrong tunnel type - IPv4 specified " 1447 "but IPv6 item provided"); 1448 if (item_flags & l3m) 1449 return rte_flow_error_set(error, ENOTSUP, 1450 RTE_FLOW_ERROR_TYPE_ITEM, item, 1451 "multiple L3 layers not supported"); 1452 else if (item_flags & l4m) 1453 return rte_flow_error_set(error, EINVAL, 1454 RTE_FLOW_ERROR_TYPE_ITEM, item, 1455 "L3 cannot follow an L4 layer."); 1456 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1457 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1458 return rte_flow_error_set(error, EINVAL, 1459 RTE_FLOW_ERROR_TYPE_ITEM, item, 1460 "L3 cannot follow an NVGRE layer."); 1461 if (!mask) 1462 mask = &rte_flow_item_ipv6_mask; 1463 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1464 acc_mask ? (const uint8_t *)acc_mask 1465 : (const uint8_t *)&nic_mask, 1466 sizeof(struct rte_flow_item_ipv6), 1467 error); 1468 if (ret < 0) 1469 return ret; 1470 return 0; 1471 } 1472 1473 /** 1474 * Validate UDP item. 1475 * 1476 * @param[in] item 1477 * Item specification. 1478 * @param[in] item_flags 1479 * Bit-fields that holds the items detected until now. 1480 * @param[in] target_protocol 1481 * The next protocol in the previous item. 1482 * @param[in] flow_mask 1483 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1484 * @param[out] error 1485 * Pointer to error structure. 1486 * 1487 * @return 1488 * 0 on success, a negative errno value otherwise and rte_errno is set. 1489 */ 1490 int 1491 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1492 uint64_t item_flags, 1493 uint8_t target_protocol, 1494 struct rte_flow_error *error) 1495 { 1496 const struct rte_flow_item_udp *mask = item->mask; 1497 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1498 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1499 MLX5_FLOW_LAYER_OUTER_L3; 1500 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1501 MLX5_FLOW_LAYER_OUTER_L4; 1502 int ret; 1503 1504 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1505 return rte_flow_error_set(error, EINVAL, 1506 RTE_FLOW_ERROR_TYPE_ITEM, item, 1507 "protocol filtering not compatible" 1508 " with UDP layer"); 1509 if (!(item_flags & l3m)) 1510 return rte_flow_error_set(error, EINVAL, 1511 RTE_FLOW_ERROR_TYPE_ITEM, item, 1512 "L3 is mandatory to filter on L4"); 1513 if (item_flags & l4m) 1514 return rte_flow_error_set(error, EINVAL, 1515 RTE_FLOW_ERROR_TYPE_ITEM, item, 1516 "multiple L4 layers not supported"); 1517 if (!mask) 1518 mask = &rte_flow_item_udp_mask; 1519 ret = mlx5_flow_item_acceptable 1520 (item, (const uint8_t *)mask, 1521 (const uint8_t *)&rte_flow_item_udp_mask, 1522 sizeof(struct rte_flow_item_udp), error); 1523 if (ret < 0) 1524 return ret; 1525 return 0; 1526 } 1527 1528 /** 1529 * Validate TCP item. 1530 * 1531 * @param[in] item 1532 * Item specification. 1533 * @param[in] item_flags 1534 * Bit-fields that holds the items detected until now. 1535 * @param[in] target_protocol 1536 * The next protocol in the previous item. 1537 * @param[out] error 1538 * Pointer to error structure. 1539 * 1540 * @return 1541 * 0 on success, a negative errno value otherwise and rte_errno is set. 1542 */ 1543 int 1544 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1545 uint64_t item_flags, 1546 uint8_t target_protocol, 1547 const struct rte_flow_item_tcp *flow_mask, 1548 struct rte_flow_error *error) 1549 { 1550 const struct rte_flow_item_tcp *mask = item->mask; 1551 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1552 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1553 MLX5_FLOW_LAYER_OUTER_L3; 1554 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1555 MLX5_FLOW_LAYER_OUTER_L4; 1556 int ret; 1557 1558 assert(flow_mask); 1559 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1560 return rte_flow_error_set(error, EINVAL, 1561 RTE_FLOW_ERROR_TYPE_ITEM, item, 1562 "protocol filtering not compatible" 1563 " with TCP layer"); 1564 if (!(item_flags & l3m)) 1565 return rte_flow_error_set(error, EINVAL, 1566 RTE_FLOW_ERROR_TYPE_ITEM, item, 1567 "L3 is mandatory to filter on L4"); 1568 if (item_flags & l4m) 1569 return rte_flow_error_set(error, EINVAL, 1570 RTE_FLOW_ERROR_TYPE_ITEM, item, 1571 "multiple L4 layers not supported"); 1572 if (!mask) 1573 mask = &rte_flow_item_tcp_mask; 1574 ret = mlx5_flow_item_acceptable 1575 (item, (const uint8_t *)mask, 1576 (const uint8_t *)flow_mask, 1577 sizeof(struct rte_flow_item_tcp), error); 1578 if (ret < 0) 1579 return ret; 1580 return 0; 1581 } 1582 1583 /** 1584 * Validate VXLAN item. 1585 * 1586 * @param[in] item 1587 * Item specification. 1588 * @param[in] item_flags 1589 * Bit-fields that holds the items detected until now. 1590 * @param[in] target_protocol 1591 * The next protocol in the previous item. 1592 * @param[out] error 1593 * Pointer to error structure. 1594 * 1595 * @return 1596 * 0 on success, a negative errno value otherwise and rte_errno is set. 1597 */ 1598 int 1599 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1600 uint64_t item_flags, 1601 struct rte_flow_error *error) 1602 { 1603 const struct rte_flow_item_vxlan *spec = item->spec; 1604 const struct rte_flow_item_vxlan *mask = item->mask; 1605 int ret; 1606 union vni { 1607 uint32_t vlan_id; 1608 uint8_t vni[4]; 1609 } id = { .vlan_id = 0, }; 1610 uint32_t vlan_id = 0; 1611 1612 1613 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1614 return rte_flow_error_set(error, ENOTSUP, 1615 RTE_FLOW_ERROR_TYPE_ITEM, item, 1616 "multiple tunnel layers not" 1617 " supported"); 1618 /* 1619 * Verify only UDPv4 is present as defined in 1620 * https://tools.ietf.org/html/rfc7348 1621 */ 1622 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1623 return rte_flow_error_set(error, EINVAL, 1624 RTE_FLOW_ERROR_TYPE_ITEM, item, 1625 "no outer UDP layer found"); 1626 if (!mask) 1627 mask = &rte_flow_item_vxlan_mask; 1628 ret = mlx5_flow_item_acceptable 1629 (item, (const uint8_t *)mask, 1630 (const uint8_t *)&rte_flow_item_vxlan_mask, 1631 sizeof(struct rte_flow_item_vxlan), 1632 error); 1633 if (ret < 0) 1634 return ret; 1635 if (spec) { 1636 memcpy(&id.vni[1], spec->vni, 3); 1637 vlan_id = id.vlan_id; 1638 memcpy(&id.vni[1], mask->vni, 3); 1639 vlan_id &= id.vlan_id; 1640 } 1641 /* 1642 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if 1643 * only this layer is defined in the Verbs specification it is 1644 * interpreted as wildcard and all packets will match this 1645 * rule, if it follows a full stack layer (ex: eth / ipv4 / 1646 * udp), all packets matching the layers before will also 1647 * match this rule. To avoid such situation, VNI 0 is 1648 * currently refused. 1649 */ 1650 if (!vlan_id) 1651 return rte_flow_error_set(error, ENOTSUP, 1652 RTE_FLOW_ERROR_TYPE_ITEM, item, 1653 "VXLAN vni cannot be 0"); 1654 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1655 return rte_flow_error_set(error, ENOTSUP, 1656 RTE_FLOW_ERROR_TYPE_ITEM, item, 1657 "VXLAN tunnel must be fully defined"); 1658 return 0; 1659 } 1660 1661 /** 1662 * Validate VXLAN_GPE item. 1663 * 1664 * @param[in] item 1665 * Item specification. 1666 * @param[in] item_flags 1667 * Bit-fields that holds the items detected until now. 1668 * @param[in] priv 1669 * Pointer to the private data structure. 1670 * @param[in] target_protocol 1671 * The next protocol in the previous item. 1672 * @param[out] error 1673 * Pointer to error structure. 1674 * 1675 * @return 1676 * 0 on success, a negative errno value otherwise and rte_errno is set. 1677 */ 1678 int 1679 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1680 uint64_t item_flags, 1681 struct rte_eth_dev *dev, 1682 struct rte_flow_error *error) 1683 { 1684 struct mlx5_priv *priv = dev->data->dev_private; 1685 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1686 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1687 int ret; 1688 union vni { 1689 uint32_t vlan_id; 1690 uint8_t vni[4]; 1691 } id = { .vlan_id = 0, }; 1692 uint32_t vlan_id = 0; 1693 1694 if (!priv->config.l3_vxlan_en) 1695 return rte_flow_error_set(error, ENOTSUP, 1696 RTE_FLOW_ERROR_TYPE_ITEM, item, 1697 "L3 VXLAN is not enabled by device" 1698 " parameter and/or not configured in" 1699 " firmware"); 1700 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1701 return rte_flow_error_set(error, ENOTSUP, 1702 RTE_FLOW_ERROR_TYPE_ITEM, item, 1703 "multiple tunnel layers not" 1704 " supported"); 1705 /* 1706 * Verify only UDPv4 is present as defined in 1707 * https://tools.ietf.org/html/rfc7348 1708 */ 1709 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1710 return rte_flow_error_set(error, EINVAL, 1711 RTE_FLOW_ERROR_TYPE_ITEM, item, 1712 "no outer UDP layer found"); 1713 if (!mask) 1714 mask = &rte_flow_item_vxlan_gpe_mask; 1715 ret = mlx5_flow_item_acceptable 1716 (item, (const uint8_t *)mask, 1717 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1718 sizeof(struct rte_flow_item_vxlan_gpe), 1719 error); 1720 if (ret < 0) 1721 return ret; 1722 if (spec) { 1723 if (spec->protocol) 1724 return rte_flow_error_set(error, ENOTSUP, 1725 RTE_FLOW_ERROR_TYPE_ITEM, 1726 item, 1727 "VxLAN-GPE protocol" 1728 " not supported"); 1729 memcpy(&id.vni[1], spec->vni, 3); 1730 vlan_id = id.vlan_id; 1731 memcpy(&id.vni[1], mask->vni, 3); 1732 vlan_id &= id.vlan_id; 1733 } 1734 /* 1735 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1736 * layer is defined in the Verbs specification it is interpreted as 1737 * wildcard and all packets will match this rule, if it follows a full 1738 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1739 * before will also match this rule. To avoid such situation, VNI 0 1740 * is currently refused. 1741 */ 1742 if (!vlan_id) 1743 return rte_flow_error_set(error, ENOTSUP, 1744 RTE_FLOW_ERROR_TYPE_ITEM, item, 1745 "VXLAN-GPE vni cannot be 0"); 1746 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1747 return rte_flow_error_set(error, ENOTSUP, 1748 RTE_FLOW_ERROR_TYPE_ITEM, item, 1749 "VXLAN-GPE tunnel must be fully" 1750 " defined"); 1751 return 0; 1752 } 1753 /** 1754 * Validate GRE Key item. 1755 * 1756 * @param[in] item 1757 * Item specification. 1758 * @param[in] item_flags 1759 * Bit flags to mark detected items. 1760 * @param[in] gre_item 1761 * Pointer to gre_item 1762 * @param[out] error 1763 * Pointer to error structure. 1764 * 1765 * @return 1766 * 0 on success, a negative errno value otherwise and rte_errno is set. 1767 */ 1768 int 1769 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 1770 uint64_t item_flags, 1771 const struct rte_flow_item *gre_item, 1772 struct rte_flow_error *error) 1773 { 1774 const rte_be32_t *mask = item->mask; 1775 int ret = 0; 1776 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 1777 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 1778 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 1779 1780 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 1781 return rte_flow_error_set(error, ENOTSUP, 1782 RTE_FLOW_ERROR_TYPE_ITEM, item, 1783 "Multiple GRE key not support"); 1784 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 1785 return rte_flow_error_set(error, ENOTSUP, 1786 RTE_FLOW_ERROR_TYPE_ITEM, item, 1787 "No preceding GRE header"); 1788 if (item_flags & MLX5_FLOW_LAYER_INNER) 1789 return rte_flow_error_set(error, ENOTSUP, 1790 RTE_FLOW_ERROR_TYPE_ITEM, item, 1791 "GRE key following a wrong item"); 1792 if (!gre_mask) 1793 gre_mask = &rte_flow_item_gre_mask; 1794 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 1795 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 1796 return rte_flow_error_set(error, EINVAL, 1797 RTE_FLOW_ERROR_TYPE_ITEM, item, 1798 "Key bit must be on"); 1799 1800 if (!mask) 1801 mask = &gre_key_default_mask; 1802 ret = mlx5_flow_item_acceptable 1803 (item, (const uint8_t *)mask, 1804 (const uint8_t *)&gre_key_default_mask, 1805 sizeof(rte_be32_t), error); 1806 return ret; 1807 } 1808 1809 /** 1810 * Validate GRE item. 1811 * 1812 * @param[in] item 1813 * Item specification. 1814 * @param[in] item_flags 1815 * Bit flags to mark detected items. 1816 * @param[in] target_protocol 1817 * The next protocol in the previous item. 1818 * @param[out] error 1819 * Pointer to error structure. 1820 * 1821 * @return 1822 * 0 on success, a negative errno value otherwise and rte_errno is set. 1823 */ 1824 int 1825 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 1826 uint64_t item_flags, 1827 uint8_t target_protocol, 1828 struct rte_flow_error *error) 1829 { 1830 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 1831 const struct rte_flow_item_gre *mask = item->mask; 1832 int ret; 1833 const struct rte_flow_item_gre nic_mask = { 1834 .c_rsvd0_ver = RTE_BE16(0xB000), 1835 .protocol = RTE_BE16(UINT16_MAX), 1836 }; 1837 1838 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 1839 return rte_flow_error_set(error, EINVAL, 1840 RTE_FLOW_ERROR_TYPE_ITEM, item, 1841 "protocol filtering not compatible" 1842 " with this GRE layer"); 1843 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1844 return rte_flow_error_set(error, ENOTSUP, 1845 RTE_FLOW_ERROR_TYPE_ITEM, item, 1846 "multiple tunnel layers not" 1847 " supported"); 1848 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 1849 return rte_flow_error_set(error, ENOTSUP, 1850 RTE_FLOW_ERROR_TYPE_ITEM, item, 1851 "L3 Layer is missing"); 1852 if (!mask) 1853 mask = &rte_flow_item_gre_mask; 1854 ret = mlx5_flow_item_acceptable 1855 (item, (const uint8_t *)mask, 1856 (const uint8_t *)&nic_mask, 1857 sizeof(struct rte_flow_item_gre), error); 1858 if (ret < 0) 1859 return ret; 1860 #ifndef HAVE_MLX5DV_DR 1861 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 1862 if (spec && (spec->protocol & mask->protocol)) 1863 return rte_flow_error_set(error, ENOTSUP, 1864 RTE_FLOW_ERROR_TYPE_ITEM, item, 1865 "without MPLS support the" 1866 " specification cannot be used for" 1867 " filtering"); 1868 #endif 1869 #endif 1870 return 0; 1871 } 1872 1873 /** 1874 * Validate MPLS item. 1875 * 1876 * @param[in] dev 1877 * Pointer to the rte_eth_dev structure. 1878 * @param[in] item 1879 * Item specification. 1880 * @param[in] item_flags 1881 * Bit-fields that holds the items detected until now. 1882 * @param[in] prev_layer 1883 * The protocol layer indicated in previous item. 1884 * @param[out] error 1885 * Pointer to error structure. 1886 * 1887 * @return 1888 * 0 on success, a negative errno value otherwise and rte_errno is set. 1889 */ 1890 int 1891 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 1892 const struct rte_flow_item *item __rte_unused, 1893 uint64_t item_flags __rte_unused, 1894 uint64_t prev_layer __rte_unused, 1895 struct rte_flow_error *error) 1896 { 1897 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1898 const struct rte_flow_item_mpls *mask = item->mask; 1899 struct mlx5_priv *priv = dev->data->dev_private; 1900 int ret; 1901 1902 if (!priv->config.mpls_en) 1903 return rte_flow_error_set(error, ENOTSUP, 1904 RTE_FLOW_ERROR_TYPE_ITEM, item, 1905 "MPLS not supported or" 1906 " disabled in firmware" 1907 " configuration."); 1908 /* MPLS over IP, UDP, GRE is allowed */ 1909 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 1910 MLX5_FLOW_LAYER_OUTER_L4_UDP | 1911 MLX5_FLOW_LAYER_GRE))) 1912 return rte_flow_error_set(error, EINVAL, 1913 RTE_FLOW_ERROR_TYPE_ITEM, item, 1914 "protocol filtering not compatible" 1915 " with MPLS layer"); 1916 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 1917 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 1918 !(item_flags & MLX5_FLOW_LAYER_GRE)) 1919 return rte_flow_error_set(error, ENOTSUP, 1920 RTE_FLOW_ERROR_TYPE_ITEM, item, 1921 "multiple tunnel layers not" 1922 " supported"); 1923 if (!mask) 1924 mask = &rte_flow_item_mpls_mask; 1925 ret = mlx5_flow_item_acceptable 1926 (item, (const uint8_t *)mask, 1927 (const uint8_t *)&rte_flow_item_mpls_mask, 1928 sizeof(struct rte_flow_item_mpls), error); 1929 if (ret < 0) 1930 return ret; 1931 return 0; 1932 #endif 1933 return rte_flow_error_set(error, ENOTSUP, 1934 RTE_FLOW_ERROR_TYPE_ITEM, item, 1935 "MPLS is not supported by Verbs, please" 1936 " update."); 1937 } 1938 1939 /** 1940 * Validate NVGRE item. 1941 * 1942 * @param[in] item 1943 * Item specification. 1944 * @param[in] item_flags 1945 * Bit flags to mark detected items. 1946 * @param[in] target_protocol 1947 * The next protocol in the previous item. 1948 * @param[out] error 1949 * Pointer to error structure. 1950 * 1951 * @return 1952 * 0 on success, a negative errno value otherwise and rte_errno is set. 1953 */ 1954 int 1955 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 1956 uint64_t item_flags, 1957 uint8_t target_protocol, 1958 struct rte_flow_error *error) 1959 { 1960 const struct rte_flow_item_nvgre *mask = item->mask; 1961 int ret; 1962 1963 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 1964 return rte_flow_error_set(error, EINVAL, 1965 RTE_FLOW_ERROR_TYPE_ITEM, item, 1966 "protocol filtering not compatible" 1967 " with this GRE layer"); 1968 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1969 return rte_flow_error_set(error, ENOTSUP, 1970 RTE_FLOW_ERROR_TYPE_ITEM, item, 1971 "multiple tunnel layers not" 1972 " supported"); 1973 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 1974 return rte_flow_error_set(error, ENOTSUP, 1975 RTE_FLOW_ERROR_TYPE_ITEM, item, 1976 "L3 Layer is missing"); 1977 if (!mask) 1978 mask = &rte_flow_item_nvgre_mask; 1979 ret = mlx5_flow_item_acceptable 1980 (item, (const uint8_t *)mask, 1981 (const uint8_t *)&rte_flow_item_nvgre_mask, 1982 sizeof(struct rte_flow_item_nvgre), error); 1983 if (ret < 0) 1984 return ret; 1985 return 0; 1986 } 1987 1988 static int 1989 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 1990 const struct rte_flow_attr *attr __rte_unused, 1991 const struct rte_flow_item items[] __rte_unused, 1992 const struct rte_flow_action actions[] __rte_unused, 1993 struct rte_flow_error *error) 1994 { 1995 return rte_flow_error_set(error, ENOTSUP, 1996 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 1997 } 1998 1999 static struct mlx5_flow * 2000 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, 2001 const struct rte_flow_item items[] __rte_unused, 2002 const struct rte_flow_action actions[] __rte_unused, 2003 struct rte_flow_error *error) 2004 { 2005 rte_flow_error_set(error, ENOTSUP, 2006 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2007 return NULL; 2008 } 2009 2010 static int 2011 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2012 struct mlx5_flow *dev_flow __rte_unused, 2013 const struct rte_flow_attr *attr __rte_unused, 2014 const struct rte_flow_item items[] __rte_unused, 2015 const struct rte_flow_action actions[] __rte_unused, 2016 struct rte_flow_error *error) 2017 { 2018 return rte_flow_error_set(error, ENOTSUP, 2019 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2020 } 2021 2022 static int 2023 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2024 struct rte_flow *flow __rte_unused, 2025 struct rte_flow_error *error) 2026 { 2027 return rte_flow_error_set(error, ENOTSUP, 2028 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2029 } 2030 2031 static void 2032 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2033 struct rte_flow *flow __rte_unused) 2034 { 2035 } 2036 2037 static void 2038 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2039 struct rte_flow *flow __rte_unused) 2040 { 2041 } 2042 2043 static int 2044 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2045 struct rte_flow *flow __rte_unused, 2046 const struct rte_flow_action *actions __rte_unused, 2047 void *data __rte_unused, 2048 struct rte_flow_error *error) 2049 { 2050 return rte_flow_error_set(error, ENOTSUP, 2051 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2052 } 2053 2054 /* Void driver to protect from null pointer reference. */ 2055 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2056 .validate = flow_null_validate, 2057 .prepare = flow_null_prepare, 2058 .translate = flow_null_translate, 2059 .apply = flow_null_apply, 2060 .remove = flow_null_remove, 2061 .destroy = flow_null_destroy, 2062 .query = flow_null_query, 2063 }; 2064 2065 /** 2066 * Select flow driver type according to flow attributes and device 2067 * configuration. 2068 * 2069 * @param[in] dev 2070 * Pointer to the dev structure. 2071 * @param[in] attr 2072 * Pointer to the flow attributes. 2073 * 2074 * @return 2075 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2076 */ 2077 static enum mlx5_flow_drv_type 2078 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2079 { 2080 struct mlx5_priv *priv = dev->data->dev_private; 2081 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2082 2083 if (attr->transfer && priv->config.dv_esw_en) 2084 type = MLX5_FLOW_TYPE_DV; 2085 if (!attr->transfer) 2086 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2087 MLX5_FLOW_TYPE_VERBS; 2088 return type; 2089 } 2090 2091 #define flow_get_drv_ops(type) flow_drv_ops[type] 2092 2093 /** 2094 * Flow driver validation API. This abstracts calling driver specific functions. 2095 * The type of flow driver is determined according to flow attributes. 2096 * 2097 * @param[in] dev 2098 * Pointer to the dev structure. 2099 * @param[in] attr 2100 * Pointer to the flow attributes. 2101 * @param[in] items 2102 * Pointer to the list of items. 2103 * @param[in] actions 2104 * Pointer to the list of actions. 2105 * @param[out] error 2106 * Pointer to the error structure. 2107 * 2108 * @return 2109 * 0 on success, a negative errno value otherwise and rte_errno is set. 2110 */ 2111 static inline int 2112 flow_drv_validate(struct rte_eth_dev *dev, 2113 const struct rte_flow_attr *attr, 2114 const struct rte_flow_item items[], 2115 const struct rte_flow_action actions[], 2116 struct rte_flow_error *error) 2117 { 2118 const struct mlx5_flow_driver_ops *fops; 2119 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2120 2121 fops = flow_get_drv_ops(type); 2122 return fops->validate(dev, attr, items, actions, error); 2123 } 2124 2125 /** 2126 * Flow driver preparation API. This abstracts calling driver specific 2127 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2128 * calculates the size of memory required for device flow, allocates the memory, 2129 * initializes the device flow and returns the pointer. 2130 * 2131 * @note 2132 * This function initializes device flow structure such as dv or verbs in 2133 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2134 * rest. For example, adding returning device flow to flow->dev_flow list and 2135 * setting backward reference to the flow should be done out of this function. 2136 * layers field is not filled either. 2137 * 2138 * @param[in] attr 2139 * Pointer to the flow attributes. 2140 * @param[in] items 2141 * Pointer to the list of items. 2142 * @param[in] actions 2143 * Pointer to the list of actions. 2144 * @param[out] error 2145 * Pointer to the error structure. 2146 * 2147 * @return 2148 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2149 */ 2150 static inline struct mlx5_flow * 2151 flow_drv_prepare(const struct rte_flow *flow, 2152 const struct rte_flow_attr *attr, 2153 const struct rte_flow_item items[], 2154 const struct rte_flow_action actions[], 2155 struct rte_flow_error *error) 2156 { 2157 const struct mlx5_flow_driver_ops *fops; 2158 enum mlx5_flow_drv_type type = flow->drv_type; 2159 2160 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2161 fops = flow_get_drv_ops(type); 2162 return fops->prepare(attr, items, actions, error); 2163 } 2164 2165 /** 2166 * Flow driver translation API. This abstracts calling driver specific 2167 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2168 * translates a generic flow into a driver flow. flow_drv_prepare() must 2169 * precede. 2170 * 2171 * @note 2172 * dev_flow->layers could be filled as a result of parsing during translation 2173 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2174 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2175 * flow->actions could be overwritten even though all the expanded dev_flows 2176 * have the same actions. 2177 * 2178 * @param[in] dev 2179 * Pointer to the rte dev structure. 2180 * @param[in, out] dev_flow 2181 * Pointer to the mlx5 flow. 2182 * @param[in] attr 2183 * Pointer to the flow attributes. 2184 * @param[in] items 2185 * Pointer to the list of items. 2186 * @param[in] actions 2187 * Pointer to the list of actions. 2188 * @param[out] error 2189 * Pointer to the error structure. 2190 * 2191 * @return 2192 * 0 on success, a negative errno value otherwise and rte_errno is set. 2193 */ 2194 static inline int 2195 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2196 const struct rte_flow_attr *attr, 2197 const struct rte_flow_item items[], 2198 const struct rte_flow_action actions[], 2199 struct rte_flow_error *error) 2200 { 2201 const struct mlx5_flow_driver_ops *fops; 2202 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2203 2204 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2205 fops = flow_get_drv_ops(type); 2206 return fops->translate(dev, dev_flow, attr, items, actions, error); 2207 } 2208 2209 /** 2210 * Flow driver apply API. This abstracts calling driver specific functions. 2211 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2212 * translated driver flows on to device. flow_drv_translate() must precede. 2213 * 2214 * @param[in] dev 2215 * Pointer to Ethernet device structure. 2216 * @param[in, out] flow 2217 * Pointer to flow structure. 2218 * @param[out] error 2219 * Pointer to error structure. 2220 * 2221 * @return 2222 * 0 on success, a negative errno value otherwise and rte_errno is set. 2223 */ 2224 static inline int 2225 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2226 struct rte_flow_error *error) 2227 { 2228 const struct mlx5_flow_driver_ops *fops; 2229 enum mlx5_flow_drv_type type = flow->drv_type; 2230 2231 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2232 fops = flow_get_drv_ops(type); 2233 return fops->apply(dev, flow, error); 2234 } 2235 2236 /** 2237 * Flow driver remove API. This abstracts calling driver specific functions. 2238 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2239 * on device. All the resources of the flow should be freed by calling 2240 * flow_drv_destroy(). 2241 * 2242 * @param[in] dev 2243 * Pointer to Ethernet device. 2244 * @param[in, out] flow 2245 * Pointer to flow structure. 2246 */ 2247 static inline void 2248 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2249 { 2250 const struct mlx5_flow_driver_ops *fops; 2251 enum mlx5_flow_drv_type type = flow->drv_type; 2252 2253 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2254 fops = flow_get_drv_ops(type); 2255 fops->remove(dev, flow); 2256 } 2257 2258 /** 2259 * Flow driver destroy API. This abstracts calling driver specific functions. 2260 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2261 * on device and releases resources of the flow. 2262 * 2263 * @param[in] dev 2264 * Pointer to Ethernet device. 2265 * @param[in, out] flow 2266 * Pointer to flow structure. 2267 */ 2268 static inline void 2269 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2270 { 2271 const struct mlx5_flow_driver_ops *fops; 2272 enum mlx5_flow_drv_type type = flow->drv_type; 2273 2274 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2275 fops = flow_get_drv_ops(type); 2276 fops->destroy(dev, flow); 2277 } 2278 2279 /** 2280 * Validate a flow supported by the NIC. 2281 * 2282 * @see rte_flow_validate() 2283 * @see rte_flow_ops 2284 */ 2285 int 2286 mlx5_flow_validate(struct rte_eth_dev *dev, 2287 const struct rte_flow_attr *attr, 2288 const struct rte_flow_item items[], 2289 const struct rte_flow_action actions[], 2290 struct rte_flow_error *error) 2291 { 2292 int ret; 2293 2294 ret = flow_drv_validate(dev, attr, items, actions, error); 2295 if (ret < 0) 2296 return ret; 2297 return 0; 2298 } 2299 2300 /** 2301 * Get RSS action from the action list. 2302 * 2303 * @param[in] actions 2304 * Pointer to the list of actions. 2305 * 2306 * @return 2307 * Pointer to the RSS action if exist, else return NULL. 2308 */ 2309 static const struct rte_flow_action_rss* 2310 flow_get_rss_action(const struct rte_flow_action actions[]) 2311 { 2312 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2313 switch (actions->type) { 2314 case RTE_FLOW_ACTION_TYPE_RSS: 2315 return (const struct rte_flow_action_rss *) 2316 actions->conf; 2317 default: 2318 break; 2319 } 2320 } 2321 return NULL; 2322 } 2323 2324 static unsigned int 2325 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2326 { 2327 const struct rte_flow_item *item; 2328 unsigned int has_vlan = 0; 2329 2330 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2331 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2332 has_vlan = 1; 2333 break; 2334 } 2335 } 2336 if (has_vlan) 2337 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2338 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2339 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2340 MLX5_EXPANSION_ROOT_OUTER; 2341 } 2342 2343 /** 2344 * Create a flow and add it to @p list. 2345 * 2346 * @param dev 2347 * Pointer to Ethernet device. 2348 * @param list 2349 * Pointer to a TAILQ flow list. 2350 * @param[in] attr 2351 * Flow rule attributes. 2352 * @param[in] items 2353 * Pattern specification (list terminated by the END pattern item). 2354 * @param[in] actions 2355 * Associated actions (list terminated by the END action). 2356 * @param[out] error 2357 * Perform verbose error reporting if not NULL. 2358 * 2359 * @return 2360 * A flow on success, NULL otherwise and rte_errno is set. 2361 */ 2362 static struct rte_flow * 2363 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2364 const struct rte_flow_attr *attr, 2365 const struct rte_flow_item items[], 2366 const struct rte_flow_action actions[], 2367 struct rte_flow_error *error) 2368 { 2369 struct rte_flow *flow = NULL; 2370 struct mlx5_flow *dev_flow; 2371 const struct rte_flow_action_rss *rss; 2372 union { 2373 struct rte_flow_expand_rss buf; 2374 uint8_t buffer[2048]; 2375 } expand_buffer; 2376 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 2377 int ret; 2378 uint32_t i; 2379 uint32_t flow_size; 2380 2381 ret = flow_drv_validate(dev, attr, items, actions, error); 2382 if (ret < 0) 2383 return NULL; 2384 flow_size = sizeof(struct rte_flow); 2385 rss = flow_get_rss_action(actions); 2386 if (rss) 2387 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 2388 sizeof(void *)); 2389 else 2390 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 2391 flow = rte_calloc(__func__, 1, flow_size, 0); 2392 if (!flow) { 2393 rte_errno = ENOMEM; 2394 return NULL; 2395 } 2396 flow->drv_type = flow_get_drv_type(dev, attr); 2397 flow->ingress = attr->ingress; 2398 flow->transfer = attr->transfer; 2399 assert(flow->drv_type > MLX5_FLOW_TYPE_MIN && 2400 flow->drv_type < MLX5_FLOW_TYPE_MAX); 2401 flow->queue = (void *)(flow + 1); 2402 LIST_INIT(&flow->dev_flows); 2403 if (rss && rss->types) { 2404 unsigned int graph_root; 2405 2406 graph_root = find_graph_root(items, rss->level); 2407 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 2408 items, rss->types, 2409 mlx5_support_expansion, 2410 graph_root); 2411 assert(ret > 0 && 2412 (unsigned int)ret < sizeof(expand_buffer.buffer)); 2413 } else { 2414 buf->entries = 1; 2415 buf->entry[0].pattern = (void *)(uintptr_t)items; 2416 } 2417 for (i = 0; i < buf->entries; ++i) { 2418 dev_flow = flow_drv_prepare(flow, attr, buf->entry[i].pattern, 2419 actions, error); 2420 if (!dev_flow) 2421 goto error; 2422 dev_flow->flow = flow; 2423 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 2424 ret = flow_drv_translate(dev, dev_flow, attr, 2425 buf->entry[i].pattern, 2426 actions, error); 2427 if (ret < 0) 2428 goto error; 2429 } 2430 if (dev->data->dev_started) { 2431 ret = flow_drv_apply(dev, flow, error); 2432 if (ret < 0) 2433 goto error; 2434 } 2435 TAILQ_INSERT_TAIL(list, flow, next); 2436 flow_rxq_flags_set(dev, flow); 2437 return flow; 2438 error: 2439 ret = rte_errno; /* Save rte_errno before cleanup. */ 2440 assert(flow); 2441 flow_drv_destroy(dev, flow); 2442 rte_free(flow); 2443 rte_errno = ret; /* Restore rte_errno. */ 2444 return NULL; 2445 } 2446 2447 /** 2448 * Create a flow. 2449 * 2450 * @see rte_flow_create() 2451 * @see rte_flow_ops 2452 */ 2453 struct rte_flow * 2454 mlx5_flow_create(struct rte_eth_dev *dev, 2455 const struct rte_flow_attr *attr, 2456 const struct rte_flow_item items[], 2457 const struct rte_flow_action actions[], 2458 struct rte_flow_error *error) 2459 { 2460 struct mlx5_priv *priv = dev->data->dev_private; 2461 2462 return flow_list_create(dev, &priv->flows, 2463 attr, items, actions, error); 2464 } 2465 2466 /** 2467 * Destroy a flow in a list. 2468 * 2469 * @param dev 2470 * Pointer to Ethernet device. 2471 * @param list 2472 * Pointer to a TAILQ flow list. 2473 * @param[in] flow 2474 * Flow to destroy. 2475 */ 2476 static void 2477 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2478 struct rte_flow *flow) 2479 { 2480 /* 2481 * Update RX queue flags only if port is started, otherwise it is 2482 * already clean. 2483 */ 2484 if (dev->data->dev_started) 2485 flow_rxq_flags_trim(dev, flow); 2486 flow_drv_destroy(dev, flow); 2487 TAILQ_REMOVE(list, flow, next); 2488 rte_free(flow->fdir); 2489 rte_free(flow); 2490 } 2491 2492 /** 2493 * Destroy all flows. 2494 * 2495 * @param dev 2496 * Pointer to Ethernet device. 2497 * @param list 2498 * Pointer to a TAILQ flow list. 2499 */ 2500 void 2501 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 2502 { 2503 while (!TAILQ_EMPTY(list)) { 2504 struct rte_flow *flow; 2505 2506 flow = TAILQ_FIRST(list); 2507 flow_list_destroy(dev, list, flow); 2508 } 2509 } 2510 2511 /** 2512 * Remove all flows. 2513 * 2514 * @param dev 2515 * Pointer to Ethernet device. 2516 * @param list 2517 * Pointer to a TAILQ flow list. 2518 */ 2519 void 2520 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 2521 { 2522 struct rte_flow *flow; 2523 2524 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) 2525 flow_drv_remove(dev, flow); 2526 flow_rxq_flags_clear(dev); 2527 } 2528 2529 /** 2530 * Add all flows. 2531 * 2532 * @param dev 2533 * Pointer to Ethernet device. 2534 * @param list 2535 * Pointer to a TAILQ flow list. 2536 * 2537 * @return 2538 * 0 on success, a negative errno value otherwise and rte_errno is set. 2539 */ 2540 int 2541 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 2542 { 2543 struct rte_flow *flow; 2544 struct rte_flow_error error; 2545 int ret = 0; 2546 2547 TAILQ_FOREACH(flow, list, next) { 2548 ret = flow_drv_apply(dev, flow, &error); 2549 if (ret < 0) 2550 goto error; 2551 flow_rxq_flags_set(dev, flow); 2552 } 2553 return 0; 2554 error: 2555 ret = rte_errno; /* Save rte_errno before cleanup. */ 2556 mlx5_flow_stop(dev, list); 2557 rte_errno = ret; /* Restore rte_errno. */ 2558 return -rte_errno; 2559 } 2560 2561 /** 2562 * Verify the flow list is empty 2563 * 2564 * @param dev 2565 * Pointer to Ethernet device. 2566 * 2567 * @return the number of flows not released. 2568 */ 2569 int 2570 mlx5_flow_verify(struct rte_eth_dev *dev) 2571 { 2572 struct mlx5_priv *priv = dev->data->dev_private; 2573 struct rte_flow *flow; 2574 int ret = 0; 2575 2576 TAILQ_FOREACH(flow, &priv->flows, next) { 2577 DRV_LOG(DEBUG, "port %u flow %p still referenced", 2578 dev->data->port_id, (void *)flow); 2579 ++ret; 2580 } 2581 return ret; 2582 } 2583 2584 /** 2585 * Enable a control flow configured from the control plane. 2586 * 2587 * @param dev 2588 * Pointer to Ethernet device. 2589 * @param eth_spec 2590 * An Ethernet flow spec to apply. 2591 * @param eth_mask 2592 * An Ethernet flow mask to apply. 2593 * @param vlan_spec 2594 * A VLAN flow spec to apply. 2595 * @param vlan_mask 2596 * A VLAN flow mask to apply. 2597 * 2598 * @return 2599 * 0 on success, a negative errno value otherwise and rte_errno is set. 2600 */ 2601 int 2602 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2603 struct rte_flow_item_eth *eth_spec, 2604 struct rte_flow_item_eth *eth_mask, 2605 struct rte_flow_item_vlan *vlan_spec, 2606 struct rte_flow_item_vlan *vlan_mask) 2607 { 2608 struct mlx5_priv *priv = dev->data->dev_private; 2609 const struct rte_flow_attr attr = { 2610 .ingress = 1, 2611 .priority = MLX5_FLOW_PRIO_RSVD, 2612 }; 2613 struct rte_flow_item items[] = { 2614 { 2615 .type = RTE_FLOW_ITEM_TYPE_ETH, 2616 .spec = eth_spec, 2617 .last = NULL, 2618 .mask = eth_mask, 2619 }, 2620 { 2621 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2622 RTE_FLOW_ITEM_TYPE_END, 2623 .spec = vlan_spec, 2624 .last = NULL, 2625 .mask = vlan_mask, 2626 }, 2627 { 2628 .type = RTE_FLOW_ITEM_TYPE_END, 2629 }, 2630 }; 2631 uint16_t queue[priv->reta_idx_n]; 2632 struct rte_flow_action_rss action_rss = { 2633 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 2634 .level = 0, 2635 .types = priv->rss_conf.rss_hf, 2636 .key_len = priv->rss_conf.rss_key_len, 2637 .queue_num = priv->reta_idx_n, 2638 .key = priv->rss_conf.rss_key, 2639 .queue = queue, 2640 }; 2641 struct rte_flow_action actions[] = { 2642 { 2643 .type = RTE_FLOW_ACTION_TYPE_RSS, 2644 .conf = &action_rss, 2645 }, 2646 { 2647 .type = RTE_FLOW_ACTION_TYPE_END, 2648 }, 2649 }; 2650 struct rte_flow *flow; 2651 struct rte_flow_error error; 2652 unsigned int i; 2653 2654 if (!priv->reta_idx_n || !priv->rxqs_n) { 2655 return 0; 2656 } 2657 for (i = 0; i != priv->reta_idx_n; ++i) 2658 queue[i] = (*priv->reta_idx)[i]; 2659 flow = flow_list_create(dev, &priv->ctrl_flows, 2660 &attr, items, actions, &error); 2661 if (!flow) 2662 return -rte_errno; 2663 return 0; 2664 } 2665 2666 /** 2667 * Enable a flow control configured from the control plane. 2668 * 2669 * @param dev 2670 * Pointer to Ethernet device. 2671 * @param eth_spec 2672 * An Ethernet flow spec to apply. 2673 * @param eth_mask 2674 * An Ethernet flow mask to apply. 2675 * 2676 * @return 2677 * 0 on success, a negative errno value otherwise and rte_errno is set. 2678 */ 2679 int 2680 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2681 struct rte_flow_item_eth *eth_spec, 2682 struct rte_flow_item_eth *eth_mask) 2683 { 2684 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2685 } 2686 2687 /** 2688 * Destroy a flow. 2689 * 2690 * @see rte_flow_destroy() 2691 * @see rte_flow_ops 2692 */ 2693 int 2694 mlx5_flow_destroy(struct rte_eth_dev *dev, 2695 struct rte_flow *flow, 2696 struct rte_flow_error *error __rte_unused) 2697 { 2698 struct mlx5_priv *priv = dev->data->dev_private; 2699 2700 flow_list_destroy(dev, &priv->flows, flow); 2701 return 0; 2702 } 2703 2704 /** 2705 * Destroy all flows. 2706 * 2707 * @see rte_flow_flush() 2708 * @see rte_flow_ops 2709 */ 2710 int 2711 mlx5_flow_flush(struct rte_eth_dev *dev, 2712 struct rte_flow_error *error __rte_unused) 2713 { 2714 struct mlx5_priv *priv = dev->data->dev_private; 2715 2716 mlx5_flow_list_flush(dev, &priv->flows); 2717 return 0; 2718 } 2719 2720 /** 2721 * Isolated mode. 2722 * 2723 * @see rte_flow_isolate() 2724 * @see rte_flow_ops 2725 */ 2726 int 2727 mlx5_flow_isolate(struct rte_eth_dev *dev, 2728 int enable, 2729 struct rte_flow_error *error) 2730 { 2731 struct mlx5_priv *priv = dev->data->dev_private; 2732 2733 if (dev->data->dev_started) { 2734 rte_flow_error_set(error, EBUSY, 2735 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2736 NULL, 2737 "port must be stopped first"); 2738 return -rte_errno; 2739 } 2740 priv->isolated = !!enable; 2741 if (enable) 2742 dev->dev_ops = &mlx5_dev_ops_isolate; 2743 else 2744 dev->dev_ops = &mlx5_dev_ops; 2745 return 0; 2746 } 2747 2748 /** 2749 * Query a flow. 2750 * 2751 * @see rte_flow_query() 2752 * @see rte_flow_ops 2753 */ 2754 static int 2755 flow_drv_query(struct rte_eth_dev *dev, 2756 struct rte_flow *flow, 2757 const struct rte_flow_action *actions, 2758 void *data, 2759 struct rte_flow_error *error) 2760 { 2761 const struct mlx5_flow_driver_ops *fops; 2762 enum mlx5_flow_drv_type ftype = flow->drv_type; 2763 2764 assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 2765 fops = flow_get_drv_ops(ftype); 2766 2767 return fops->query(dev, flow, actions, data, error); 2768 } 2769 2770 /** 2771 * Query a flow. 2772 * 2773 * @see rte_flow_query() 2774 * @see rte_flow_ops 2775 */ 2776 int 2777 mlx5_flow_query(struct rte_eth_dev *dev, 2778 struct rte_flow *flow, 2779 const struct rte_flow_action *actions, 2780 void *data, 2781 struct rte_flow_error *error) 2782 { 2783 int ret; 2784 2785 ret = flow_drv_query(dev, flow, actions, data, error); 2786 if (ret < 0) 2787 return ret; 2788 return 0; 2789 } 2790 2791 /** 2792 * Convert a flow director filter to a generic flow. 2793 * 2794 * @param dev 2795 * Pointer to Ethernet device. 2796 * @param fdir_filter 2797 * Flow director filter to add. 2798 * @param attributes 2799 * Generic flow parameters structure. 2800 * 2801 * @return 2802 * 0 on success, a negative errno value otherwise and rte_errno is set. 2803 */ 2804 static int 2805 flow_fdir_filter_convert(struct rte_eth_dev *dev, 2806 const struct rte_eth_fdir_filter *fdir_filter, 2807 struct mlx5_fdir *attributes) 2808 { 2809 struct mlx5_priv *priv = dev->data->dev_private; 2810 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2811 const struct rte_eth_fdir_masks *mask = 2812 &dev->data->dev_conf.fdir_conf.mask; 2813 2814 /* Validate queue number. */ 2815 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2816 DRV_LOG(ERR, "port %u invalid queue number %d", 2817 dev->data->port_id, fdir_filter->action.rx_queue); 2818 rte_errno = EINVAL; 2819 return -rte_errno; 2820 } 2821 attributes->attr.ingress = 1; 2822 attributes->items[0] = (struct rte_flow_item) { 2823 .type = RTE_FLOW_ITEM_TYPE_ETH, 2824 .spec = &attributes->l2, 2825 .mask = &attributes->l2_mask, 2826 }; 2827 switch (fdir_filter->action.behavior) { 2828 case RTE_ETH_FDIR_ACCEPT: 2829 attributes->actions[0] = (struct rte_flow_action){ 2830 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2831 .conf = &attributes->queue, 2832 }; 2833 break; 2834 case RTE_ETH_FDIR_REJECT: 2835 attributes->actions[0] = (struct rte_flow_action){ 2836 .type = RTE_FLOW_ACTION_TYPE_DROP, 2837 }; 2838 break; 2839 default: 2840 DRV_LOG(ERR, "port %u invalid behavior %d", 2841 dev->data->port_id, 2842 fdir_filter->action.behavior); 2843 rte_errno = ENOTSUP; 2844 return -rte_errno; 2845 } 2846 attributes->queue.index = fdir_filter->action.rx_queue; 2847 /* Handle L3. */ 2848 switch (fdir_filter->input.flow_type) { 2849 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2850 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2851 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2852 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 2853 .src_addr = input->flow.ip4_flow.src_ip, 2854 .dst_addr = input->flow.ip4_flow.dst_ip, 2855 .time_to_live = input->flow.ip4_flow.ttl, 2856 .type_of_service = input->flow.ip4_flow.tos, 2857 }; 2858 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 2859 .src_addr = mask->ipv4_mask.src_ip, 2860 .dst_addr = mask->ipv4_mask.dst_ip, 2861 .time_to_live = mask->ipv4_mask.ttl, 2862 .type_of_service = mask->ipv4_mask.tos, 2863 .next_proto_id = mask->ipv4_mask.proto, 2864 }; 2865 attributes->items[1] = (struct rte_flow_item){ 2866 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2867 .spec = &attributes->l3, 2868 .mask = &attributes->l3_mask, 2869 }; 2870 break; 2871 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2872 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2873 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2874 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 2875 .hop_limits = input->flow.ipv6_flow.hop_limits, 2876 .proto = input->flow.ipv6_flow.proto, 2877 }; 2878 2879 memcpy(attributes->l3.ipv6.hdr.src_addr, 2880 input->flow.ipv6_flow.src_ip, 2881 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2882 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2883 input->flow.ipv6_flow.dst_ip, 2884 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2885 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 2886 mask->ipv6_mask.src_ip, 2887 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2888 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 2889 mask->ipv6_mask.dst_ip, 2890 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2891 attributes->items[1] = (struct rte_flow_item){ 2892 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2893 .spec = &attributes->l3, 2894 .mask = &attributes->l3_mask, 2895 }; 2896 break; 2897 default: 2898 DRV_LOG(ERR, "port %u invalid flow type%d", 2899 dev->data->port_id, fdir_filter->input.flow_type); 2900 rte_errno = ENOTSUP; 2901 return -rte_errno; 2902 } 2903 /* Handle L4. */ 2904 switch (fdir_filter->input.flow_type) { 2905 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2906 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 2907 .src_port = input->flow.udp4_flow.src_port, 2908 .dst_port = input->flow.udp4_flow.dst_port, 2909 }; 2910 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 2911 .src_port = mask->src_port_mask, 2912 .dst_port = mask->dst_port_mask, 2913 }; 2914 attributes->items[2] = (struct rte_flow_item){ 2915 .type = RTE_FLOW_ITEM_TYPE_UDP, 2916 .spec = &attributes->l4, 2917 .mask = &attributes->l4_mask, 2918 }; 2919 break; 2920 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2921 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 2922 .src_port = input->flow.tcp4_flow.src_port, 2923 .dst_port = input->flow.tcp4_flow.dst_port, 2924 }; 2925 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 2926 .src_port = mask->src_port_mask, 2927 .dst_port = mask->dst_port_mask, 2928 }; 2929 attributes->items[2] = (struct rte_flow_item){ 2930 .type = RTE_FLOW_ITEM_TYPE_TCP, 2931 .spec = &attributes->l4, 2932 .mask = &attributes->l4_mask, 2933 }; 2934 break; 2935 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2936 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 2937 .src_port = input->flow.udp6_flow.src_port, 2938 .dst_port = input->flow.udp6_flow.dst_port, 2939 }; 2940 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 2941 .src_port = mask->src_port_mask, 2942 .dst_port = mask->dst_port_mask, 2943 }; 2944 attributes->items[2] = (struct rte_flow_item){ 2945 .type = RTE_FLOW_ITEM_TYPE_UDP, 2946 .spec = &attributes->l4, 2947 .mask = &attributes->l4_mask, 2948 }; 2949 break; 2950 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2951 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 2952 .src_port = input->flow.tcp6_flow.src_port, 2953 .dst_port = input->flow.tcp6_flow.dst_port, 2954 }; 2955 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 2956 .src_port = mask->src_port_mask, 2957 .dst_port = mask->dst_port_mask, 2958 }; 2959 attributes->items[2] = (struct rte_flow_item){ 2960 .type = RTE_FLOW_ITEM_TYPE_TCP, 2961 .spec = &attributes->l4, 2962 .mask = &attributes->l4_mask, 2963 }; 2964 break; 2965 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2966 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2967 break; 2968 default: 2969 DRV_LOG(ERR, "port %u invalid flow type%d", 2970 dev->data->port_id, fdir_filter->input.flow_type); 2971 rte_errno = ENOTSUP; 2972 return -rte_errno; 2973 } 2974 return 0; 2975 } 2976 2977 #define FLOW_FDIR_CMP(f1, f2, fld) \ 2978 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 2979 2980 /** 2981 * Compare two FDIR flows. If items and actions are identical, the two flows are 2982 * regarded as same. 2983 * 2984 * @param dev 2985 * Pointer to Ethernet device. 2986 * @param f1 2987 * FDIR flow to compare. 2988 * @param f2 2989 * FDIR flow to compare. 2990 * 2991 * @return 2992 * Zero on match, 1 otherwise. 2993 */ 2994 static int 2995 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 2996 { 2997 if (FLOW_FDIR_CMP(f1, f2, attr) || 2998 FLOW_FDIR_CMP(f1, f2, l2) || 2999 FLOW_FDIR_CMP(f1, f2, l2_mask) || 3000 FLOW_FDIR_CMP(f1, f2, l3) || 3001 FLOW_FDIR_CMP(f1, f2, l3_mask) || 3002 FLOW_FDIR_CMP(f1, f2, l4) || 3003 FLOW_FDIR_CMP(f1, f2, l4_mask) || 3004 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 3005 return 1; 3006 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 3007 FLOW_FDIR_CMP(f1, f2, queue)) 3008 return 1; 3009 return 0; 3010 } 3011 3012 /** 3013 * Search device flow list to find out a matched FDIR flow. 3014 * 3015 * @param dev 3016 * Pointer to Ethernet device. 3017 * @param fdir_flow 3018 * FDIR flow to lookup. 3019 * 3020 * @return 3021 * Pointer of flow if found, NULL otherwise. 3022 */ 3023 static struct rte_flow * 3024 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 3025 { 3026 struct mlx5_priv *priv = dev->data->dev_private; 3027 struct rte_flow *flow = NULL; 3028 3029 assert(fdir_flow); 3030 TAILQ_FOREACH(flow, &priv->flows, next) { 3031 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 3032 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 3033 dev->data->port_id, (void *)flow); 3034 break; 3035 } 3036 } 3037 return flow; 3038 } 3039 3040 /** 3041 * Add new flow director filter and store it in list. 3042 * 3043 * @param dev 3044 * Pointer to Ethernet device. 3045 * @param fdir_filter 3046 * Flow director filter to add. 3047 * 3048 * @return 3049 * 0 on success, a negative errno value otherwise and rte_errno is set. 3050 */ 3051 static int 3052 flow_fdir_filter_add(struct rte_eth_dev *dev, 3053 const struct rte_eth_fdir_filter *fdir_filter) 3054 { 3055 struct mlx5_priv *priv = dev->data->dev_private; 3056 struct mlx5_fdir *fdir_flow; 3057 struct rte_flow *flow; 3058 int ret; 3059 3060 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 3061 if (!fdir_flow) { 3062 rte_errno = ENOMEM; 3063 return -rte_errno; 3064 } 3065 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 3066 if (ret) 3067 goto error; 3068 flow = flow_fdir_filter_lookup(dev, fdir_flow); 3069 if (flow) { 3070 rte_errno = EEXIST; 3071 goto error; 3072 } 3073 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 3074 fdir_flow->items, fdir_flow->actions, NULL); 3075 if (!flow) 3076 goto error; 3077 assert(!flow->fdir); 3078 flow->fdir = fdir_flow; 3079 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 3080 dev->data->port_id, (void *)flow); 3081 return 0; 3082 error: 3083 rte_free(fdir_flow); 3084 return -rte_errno; 3085 } 3086 3087 /** 3088 * Delete specific filter. 3089 * 3090 * @param dev 3091 * Pointer to Ethernet device. 3092 * @param fdir_filter 3093 * Filter to be deleted. 3094 * 3095 * @return 3096 * 0 on success, a negative errno value otherwise and rte_errno is set. 3097 */ 3098 static int 3099 flow_fdir_filter_delete(struct rte_eth_dev *dev, 3100 const struct rte_eth_fdir_filter *fdir_filter) 3101 { 3102 struct mlx5_priv *priv = dev->data->dev_private; 3103 struct rte_flow *flow; 3104 struct mlx5_fdir fdir_flow = { 3105 .attr.group = 0, 3106 }; 3107 int ret; 3108 3109 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 3110 if (ret) 3111 return -rte_errno; 3112 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 3113 if (!flow) { 3114 rte_errno = ENOENT; 3115 return -rte_errno; 3116 } 3117 flow_list_destroy(dev, &priv->flows, flow); 3118 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 3119 dev->data->port_id, (void *)flow); 3120 return 0; 3121 } 3122 3123 /** 3124 * Update queue for specific filter. 3125 * 3126 * @param dev 3127 * Pointer to Ethernet device. 3128 * @param fdir_filter 3129 * Filter to be updated. 3130 * 3131 * @return 3132 * 0 on success, a negative errno value otherwise and rte_errno is set. 3133 */ 3134 static int 3135 flow_fdir_filter_update(struct rte_eth_dev *dev, 3136 const struct rte_eth_fdir_filter *fdir_filter) 3137 { 3138 int ret; 3139 3140 ret = flow_fdir_filter_delete(dev, fdir_filter); 3141 if (ret) 3142 return ret; 3143 return flow_fdir_filter_add(dev, fdir_filter); 3144 } 3145 3146 /** 3147 * Flush all filters. 3148 * 3149 * @param dev 3150 * Pointer to Ethernet device. 3151 */ 3152 static void 3153 flow_fdir_filter_flush(struct rte_eth_dev *dev) 3154 { 3155 struct mlx5_priv *priv = dev->data->dev_private; 3156 3157 mlx5_flow_list_flush(dev, &priv->flows); 3158 } 3159 3160 /** 3161 * Get flow director information. 3162 * 3163 * @param dev 3164 * Pointer to Ethernet device. 3165 * @param[out] fdir_info 3166 * Resulting flow director information. 3167 */ 3168 static void 3169 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 3170 { 3171 struct rte_eth_fdir_masks *mask = 3172 &dev->data->dev_conf.fdir_conf.mask; 3173 3174 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 3175 fdir_info->guarant_spc = 0; 3176 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 3177 fdir_info->max_flexpayload = 0; 3178 fdir_info->flow_types_mask[0] = 0; 3179 fdir_info->flex_payload_unit = 0; 3180 fdir_info->max_flex_payload_segment_num = 0; 3181 fdir_info->flex_payload_limit = 0; 3182 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 3183 } 3184 3185 /** 3186 * Deal with flow director operations. 3187 * 3188 * @param dev 3189 * Pointer to Ethernet device. 3190 * @param filter_op 3191 * Operation to perform. 3192 * @param arg 3193 * Pointer to operation-specific structure. 3194 * 3195 * @return 3196 * 0 on success, a negative errno value otherwise and rte_errno is set. 3197 */ 3198 static int 3199 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 3200 void *arg) 3201 { 3202 enum rte_fdir_mode fdir_mode = 3203 dev->data->dev_conf.fdir_conf.mode; 3204 3205 if (filter_op == RTE_ETH_FILTER_NOP) 3206 return 0; 3207 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 3208 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 3209 DRV_LOG(ERR, "port %u flow director mode %d not supported", 3210 dev->data->port_id, fdir_mode); 3211 rte_errno = EINVAL; 3212 return -rte_errno; 3213 } 3214 switch (filter_op) { 3215 case RTE_ETH_FILTER_ADD: 3216 return flow_fdir_filter_add(dev, arg); 3217 case RTE_ETH_FILTER_UPDATE: 3218 return flow_fdir_filter_update(dev, arg); 3219 case RTE_ETH_FILTER_DELETE: 3220 return flow_fdir_filter_delete(dev, arg); 3221 case RTE_ETH_FILTER_FLUSH: 3222 flow_fdir_filter_flush(dev); 3223 break; 3224 case RTE_ETH_FILTER_INFO: 3225 flow_fdir_info_get(dev, arg); 3226 break; 3227 default: 3228 DRV_LOG(DEBUG, "port %u unknown operation %u", 3229 dev->data->port_id, filter_op); 3230 rte_errno = EINVAL; 3231 return -rte_errno; 3232 } 3233 return 0; 3234 } 3235 3236 /** 3237 * Manage filter operations. 3238 * 3239 * @param dev 3240 * Pointer to Ethernet device structure. 3241 * @param filter_type 3242 * Filter type. 3243 * @param filter_op 3244 * Operation to perform. 3245 * @param arg 3246 * Pointer to operation-specific structure. 3247 * 3248 * @return 3249 * 0 on success, a negative errno value otherwise and rte_errno is set. 3250 */ 3251 int 3252 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 3253 enum rte_filter_type filter_type, 3254 enum rte_filter_op filter_op, 3255 void *arg) 3256 { 3257 switch (filter_type) { 3258 case RTE_ETH_FILTER_GENERIC: 3259 if (filter_op != RTE_ETH_FILTER_GET) { 3260 rte_errno = EINVAL; 3261 return -rte_errno; 3262 } 3263 *(const void **)arg = &mlx5_flow_ops; 3264 return 0; 3265 case RTE_ETH_FILTER_FDIR: 3266 return flow_fdir_ctrl_func(dev, filter_op, arg); 3267 default: 3268 DRV_LOG(ERR, "port %u filter type (%d) not supported", 3269 dev->data->port_id, filter_type); 3270 rte_errno = ENOTSUP; 3271 return -rte_errno; 3272 } 3273 return 0; 3274 } 3275 3276 #define MLX5_POOL_QUERY_FREQ_US 1000000 3277 3278 /** 3279 * Set the periodic procedure for triggering asynchronous batch queries for all 3280 * the counter pools. 3281 * 3282 * @param[in] sh 3283 * Pointer to mlx5_ibv_shared object. 3284 */ 3285 void 3286 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 3287 { 3288 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 3289 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 3290 uint32_t us; 3291 3292 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 3293 pools_n += rte_atomic16_read(&cont->n_valid); 3294 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 3295 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us\n", pools_n, us); 3296 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 3297 sh->cmng.query_thread_on = 0; 3298 DRV_LOG(ERR, "Cannot reinitialize query alarm\n"); 3299 } else { 3300 sh->cmng.query_thread_on = 1; 3301 } 3302 } 3303 3304 /** 3305 * The periodic procedure for triggering asynchronous batch queries for all the 3306 * counter pools. This function is probably called by the host thread. 3307 * 3308 * @param[in] arg 3309 * The parameter for the alarm process. 3310 */ 3311 void 3312 mlx5_flow_query_alarm(void *arg) 3313 { 3314 struct mlx5_ibv_shared *sh = arg; 3315 struct mlx5_devx_obj *dcs; 3316 uint16_t offset; 3317 int ret; 3318 uint8_t batch = sh->cmng.batch; 3319 uint16_t pool_index = sh->cmng.pool_index; 3320 struct mlx5_pools_container *cont; 3321 struct mlx5_pools_container *mcont; 3322 struct mlx5_flow_counter_pool *pool; 3323 3324 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 3325 goto set_alarm; 3326 next_container: 3327 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 3328 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 3329 /* Check if resize was done and need to flip a container. */ 3330 if (cont != mcont) { 3331 if (cont->pools) { 3332 /* Clean the old container. */ 3333 rte_free(cont->pools); 3334 memset(cont, 0, sizeof(*cont)); 3335 } 3336 rte_cio_wmb(); 3337 /* Flip the host container. */ 3338 sh->cmng.mhi[batch] ^= (uint8_t)2; 3339 cont = mcont; 3340 } 3341 if (!cont->pools) { 3342 /* 2 empty containers case is unexpected. */ 3343 if (unlikely(batch != sh->cmng.batch)) 3344 goto set_alarm; 3345 batch ^= 0x1; 3346 pool_index = 0; 3347 goto next_container; 3348 } 3349 pool = cont->pools[pool_index]; 3350 if (pool->raw_hw) 3351 /* There is a pool query in progress. */ 3352 goto set_alarm; 3353 pool->raw_hw = 3354 LIST_FIRST(&sh->cmng.free_stat_raws); 3355 if (!pool->raw_hw) 3356 /* No free counter statistics raw memory. */ 3357 goto set_alarm; 3358 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 3359 (&pool->a64_dcs); 3360 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 3361 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 3362 offset, NULL, NULL, 3363 pool->raw_hw->mem_mng->dm->id, 3364 (void *)(uintptr_t) 3365 (pool->raw_hw->data + offset), 3366 sh->devx_comp, 3367 (uint64_t)(uintptr_t)pool); 3368 if (ret) { 3369 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 3370 " %d\n", pool->min_dcs->id); 3371 pool->raw_hw = NULL; 3372 goto set_alarm; 3373 } 3374 pool->raw_hw->min_dcs_id = dcs->id; 3375 LIST_REMOVE(pool->raw_hw, next); 3376 sh->cmng.pending_queries++; 3377 pool_index++; 3378 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 3379 batch ^= 0x1; 3380 pool_index = 0; 3381 } 3382 set_alarm: 3383 sh->cmng.batch = batch; 3384 sh->cmng.pool_index = pool_index; 3385 mlx5_set_query_alarm(sh); 3386 } 3387 3388 /** 3389 * Handler for the HW respond about ready values from an asynchronous batch 3390 * query. This function is probably called by the host thread. 3391 * 3392 * @param[in] sh 3393 * The pointer to the shared IB device context. 3394 * @param[in] async_id 3395 * The Devx async ID. 3396 * @param[in] status 3397 * The status of the completion. 3398 */ 3399 void 3400 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 3401 uint64_t async_id, int status) 3402 { 3403 struct mlx5_flow_counter_pool *pool = 3404 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 3405 struct mlx5_counter_stats_raw *raw_to_free; 3406 3407 if (unlikely(status)) { 3408 raw_to_free = pool->raw_hw; 3409 } else { 3410 raw_to_free = pool->raw; 3411 rte_spinlock_lock(&pool->sl); 3412 pool->raw = pool->raw_hw; 3413 rte_spinlock_unlock(&pool->sl); 3414 rte_atomic64_add(&pool->query_gen, 1); 3415 /* Be sure the new raw counters data is updated in memory. */ 3416 rte_cio_wmb(); 3417 } 3418 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 3419 pool->raw_hw = NULL; 3420 sh->cmng.pending_queries--; 3421 } 3422