1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 12 /* Verbs header. */ 13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 14 #ifdef PEDANTIC 15 #pragma GCC diagnostic ignored "-Wpedantic" 16 #endif 17 #include <infiniband/verbs.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_common.h> 23 #include <rte_ether.h> 24 #include <rte_eth_ctrl.h> 25 #include <rte_ethdev_driver.h> 26 #include <rte_flow.h> 27 #include <rte_flow_driver.h> 28 #include <rte_malloc.h> 29 #include <rte_ip.h> 30 31 #include "mlx5.h" 32 #include "mlx5_defs.h" 33 #include "mlx5_flow.h" 34 #include "mlx5_glue.h" 35 #include "mlx5_prm.h" 36 #include "mlx5_rxtx.h" 37 38 /* Dev ops structure defined in mlx5.c */ 39 extern const struct eth_dev_ops mlx5_dev_ops; 40 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 41 42 /** Device flow drivers. */ 43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 44 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 45 #endif 46 extern const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops; 47 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 48 49 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 50 51 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 52 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 53 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 54 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 55 #endif 56 [MLX5_FLOW_TYPE_TCF] = &mlx5_flow_tcf_drv_ops, 57 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 58 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 59 }; 60 61 enum mlx5_expansion { 62 MLX5_EXPANSION_ROOT, 63 MLX5_EXPANSION_ROOT_OUTER, 64 MLX5_EXPANSION_ROOT_ETH_VLAN, 65 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 66 MLX5_EXPANSION_OUTER_ETH, 67 MLX5_EXPANSION_OUTER_ETH_VLAN, 68 MLX5_EXPANSION_OUTER_VLAN, 69 MLX5_EXPANSION_OUTER_IPV4, 70 MLX5_EXPANSION_OUTER_IPV4_UDP, 71 MLX5_EXPANSION_OUTER_IPV4_TCP, 72 MLX5_EXPANSION_OUTER_IPV6, 73 MLX5_EXPANSION_OUTER_IPV6_UDP, 74 MLX5_EXPANSION_OUTER_IPV6_TCP, 75 MLX5_EXPANSION_VXLAN, 76 MLX5_EXPANSION_VXLAN_GPE, 77 MLX5_EXPANSION_GRE, 78 MLX5_EXPANSION_MPLS, 79 MLX5_EXPANSION_ETH, 80 MLX5_EXPANSION_ETH_VLAN, 81 MLX5_EXPANSION_VLAN, 82 MLX5_EXPANSION_IPV4, 83 MLX5_EXPANSION_IPV4_UDP, 84 MLX5_EXPANSION_IPV4_TCP, 85 MLX5_EXPANSION_IPV6, 86 MLX5_EXPANSION_IPV6_UDP, 87 MLX5_EXPANSION_IPV6_TCP, 88 }; 89 90 /** Supported expansion of items. */ 91 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 92 [MLX5_EXPANSION_ROOT] = { 93 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 94 MLX5_EXPANSION_IPV4, 95 MLX5_EXPANSION_IPV6), 96 .type = RTE_FLOW_ITEM_TYPE_END, 97 }, 98 [MLX5_EXPANSION_ROOT_OUTER] = { 99 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 100 MLX5_EXPANSION_OUTER_IPV4, 101 MLX5_EXPANSION_OUTER_IPV6), 102 .type = RTE_FLOW_ITEM_TYPE_END, 103 }, 104 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 105 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 106 .type = RTE_FLOW_ITEM_TYPE_END, 107 }, 108 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 109 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 110 .type = RTE_FLOW_ITEM_TYPE_END, 111 }, 112 [MLX5_EXPANSION_OUTER_ETH] = { 113 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 114 MLX5_EXPANSION_OUTER_IPV6, 115 MLX5_EXPANSION_MPLS), 116 .type = RTE_FLOW_ITEM_TYPE_ETH, 117 .rss_types = 0, 118 }, 119 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 120 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 121 .type = RTE_FLOW_ITEM_TYPE_ETH, 122 .rss_types = 0, 123 }, 124 [MLX5_EXPANSION_OUTER_VLAN] = { 125 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 126 MLX5_EXPANSION_OUTER_IPV6), 127 .type = RTE_FLOW_ITEM_TYPE_VLAN, 128 }, 129 [MLX5_EXPANSION_OUTER_IPV4] = { 130 .next = RTE_FLOW_EXPAND_RSS_NEXT 131 (MLX5_EXPANSION_OUTER_IPV4_UDP, 132 MLX5_EXPANSION_OUTER_IPV4_TCP, 133 MLX5_EXPANSION_GRE), 134 .type = RTE_FLOW_ITEM_TYPE_IPV4, 135 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 136 ETH_RSS_NONFRAG_IPV4_OTHER, 137 }, 138 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 139 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 140 MLX5_EXPANSION_VXLAN_GPE), 141 .type = RTE_FLOW_ITEM_TYPE_UDP, 142 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 143 }, 144 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 145 .type = RTE_FLOW_ITEM_TYPE_TCP, 146 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 147 }, 148 [MLX5_EXPANSION_OUTER_IPV6] = { 149 .next = RTE_FLOW_EXPAND_RSS_NEXT 150 (MLX5_EXPANSION_OUTER_IPV6_UDP, 151 MLX5_EXPANSION_OUTER_IPV6_TCP), 152 .type = RTE_FLOW_ITEM_TYPE_IPV6, 153 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 154 ETH_RSS_NONFRAG_IPV6_OTHER, 155 }, 156 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 157 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 158 MLX5_EXPANSION_VXLAN_GPE), 159 .type = RTE_FLOW_ITEM_TYPE_UDP, 160 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 161 }, 162 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 163 .type = RTE_FLOW_ITEM_TYPE_TCP, 164 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 165 }, 166 [MLX5_EXPANSION_VXLAN] = { 167 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 168 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 169 }, 170 [MLX5_EXPANSION_VXLAN_GPE] = { 171 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 172 MLX5_EXPANSION_IPV4, 173 MLX5_EXPANSION_IPV6), 174 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 175 }, 176 [MLX5_EXPANSION_GRE] = { 177 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 178 .type = RTE_FLOW_ITEM_TYPE_GRE, 179 }, 180 [MLX5_EXPANSION_MPLS] = { 181 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 182 MLX5_EXPANSION_IPV6), 183 .type = RTE_FLOW_ITEM_TYPE_MPLS, 184 }, 185 [MLX5_EXPANSION_ETH] = { 186 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 187 MLX5_EXPANSION_IPV6), 188 .type = RTE_FLOW_ITEM_TYPE_ETH, 189 }, 190 [MLX5_EXPANSION_ETH_VLAN] = { 191 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 192 .type = RTE_FLOW_ITEM_TYPE_ETH, 193 }, 194 [MLX5_EXPANSION_VLAN] = { 195 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 196 MLX5_EXPANSION_IPV6), 197 .type = RTE_FLOW_ITEM_TYPE_VLAN, 198 }, 199 [MLX5_EXPANSION_IPV4] = { 200 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 201 MLX5_EXPANSION_IPV4_TCP), 202 .type = RTE_FLOW_ITEM_TYPE_IPV4, 203 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 204 ETH_RSS_NONFRAG_IPV4_OTHER, 205 }, 206 [MLX5_EXPANSION_IPV4_UDP] = { 207 .type = RTE_FLOW_ITEM_TYPE_UDP, 208 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 209 }, 210 [MLX5_EXPANSION_IPV4_TCP] = { 211 .type = RTE_FLOW_ITEM_TYPE_TCP, 212 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 213 }, 214 [MLX5_EXPANSION_IPV6] = { 215 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 216 MLX5_EXPANSION_IPV6_TCP), 217 .type = RTE_FLOW_ITEM_TYPE_IPV6, 218 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 219 ETH_RSS_NONFRAG_IPV6_OTHER, 220 }, 221 [MLX5_EXPANSION_IPV6_UDP] = { 222 .type = RTE_FLOW_ITEM_TYPE_UDP, 223 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 224 }, 225 [MLX5_EXPANSION_IPV6_TCP] = { 226 .type = RTE_FLOW_ITEM_TYPE_TCP, 227 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 228 }, 229 }; 230 231 static const struct rte_flow_ops mlx5_flow_ops = { 232 .validate = mlx5_flow_validate, 233 .create = mlx5_flow_create, 234 .destroy = mlx5_flow_destroy, 235 .flush = mlx5_flow_flush, 236 .isolate = mlx5_flow_isolate, 237 .query = mlx5_flow_query, 238 }; 239 240 /* Convert FDIR request to Generic flow. */ 241 struct mlx5_fdir { 242 struct rte_flow_attr attr; 243 struct rte_flow_item items[4]; 244 struct rte_flow_item_eth l2; 245 struct rte_flow_item_eth l2_mask; 246 union { 247 struct rte_flow_item_ipv4 ipv4; 248 struct rte_flow_item_ipv6 ipv6; 249 } l3; 250 union { 251 struct rte_flow_item_ipv4 ipv4; 252 struct rte_flow_item_ipv6 ipv6; 253 } l3_mask; 254 union { 255 struct rte_flow_item_udp udp; 256 struct rte_flow_item_tcp tcp; 257 } l4; 258 union { 259 struct rte_flow_item_udp udp; 260 struct rte_flow_item_tcp tcp; 261 } l4_mask; 262 struct rte_flow_action actions[2]; 263 struct rte_flow_action_queue queue; 264 }; 265 266 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 267 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 268 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 269 }; 270 271 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 272 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 273 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 274 { 9, 10, 11 }, { 12, 13, 14 }, 275 }; 276 277 /* Tunnel information. */ 278 struct mlx5_flow_tunnel_info { 279 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 280 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 281 }; 282 283 static struct mlx5_flow_tunnel_info tunnels_info[] = { 284 { 285 .tunnel = MLX5_FLOW_LAYER_VXLAN, 286 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 287 }, 288 { 289 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 290 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 291 }, 292 { 293 .tunnel = MLX5_FLOW_LAYER_GRE, 294 .ptype = RTE_PTYPE_TUNNEL_GRE, 295 }, 296 { 297 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 298 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 299 }, 300 { 301 .tunnel = MLX5_FLOW_LAYER_MPLS, 302 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 303 }, 304 }; 305 306 /** 307 * Discover the maximum number of priority available. 308 * 309 * @param[in] dev 310 * Pointer to the Ethernet device structure. 311 * 312 * @return 313 * number of supported flow priority on success, a negative errno 314 * value otherwise and rte_errno is set. 315 */ 316 int 317 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 318 { 319 struct mlx5_priv *priv = dev->data->dev_private; 320 struct { 321 struct ibv_flow_attr attr; 322 struct ibv_flow_spec_eth eth; 323 struct ibv_flow_spec_action_drop drop; 324 } flow_attr = { 325 .attr = { 326 .num_of_specs = 2, 327 .port = (uint8_t)priv->ibv_port, 328 }, 329 .eth = { 330 .type = IBV_FLOW_SPEC_ETH, 331 .size = sizeof(struct ibv_flow_spec_eth), 332 }, 333 .drop = { 334 .size = sizeof(struct ibv_flow_spec_action_drop), 335 .type = IBV_FLOW_SPEC_ACTION_DROP, 336 }, 337 }; 338 struct ibv_flow *flow; 339 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 340 uint16_t vprio[] = { 8, 16 }; 341 int i; 342 int priority = 0; 343 344 if (!drop) { 345 rte_errno = ENOTSUP; 346 return -rte_errno; 347 } 348 for (i = 0; i != RTE_DIM(vprio); i++) { 349 flow_attr.attr.priority = vprio[i] - 1; 350 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 351 if (!flow) 352 break; 353 claim_zero(mlx5_glue->destroy_flow(flow)); 354 priority = vprio[i]; 355 } 356 mlx5_hrxq_drop_release(dev); 357 switch (priority) { 358 case 8: 359 priority = RTE_DIM(priority_map_3); 360 break; 361 case 16: 362 priority = RTE_DIM(priority_map_5); 363 break; 364 default: 365 rte_errno = ENOTSUP; 366 DRV_LOG(ERR, 367 "port %u verbs maximum priority: %d expected 8/16", 368 dev->data->port_id, priority); 369 return -rte_errno; 370 } 371 DRV_LOG(INFO, "port %u flow maximum priority: %d", 372 dev->data->port_id, priority); 373 return priority; 374 } 375 376 /** 377 * Adjust flow priority based on the highest layer and the request priority. 378 * 379 * @param[in] dev 380 * Pointer to the Ethernet device structure. 381 * @param[in] priority 382 * The rule base priority. 383 * @param[in] subpriority 384 * The priority based on the items. 385 * 386 * @return 387 * The new priority. 388 */ 389 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 390 uint32_t subpriority) 391 { 392 uint32_t res = 0; 393 struct mlx5_priv *priv = dev->data->dev_private; 394 395 switch (priv->config.flow_prio) { 396 case RTE_DIM(priority_map_3): 397 res = priority_map_3[priority][subpriority]; 398 break; 399 case RTE_DIM(priority_map_5): 400 res = priority_map_5[priority][subpriority]; 401 break; 402 } 403 return res; 404 } 405 406 /** 407 * Verify the @p item specifications (spec, last, mask) are compatible with the 408 * NIC capabilities. 409 * 410 * @param[in] item 411 * Item specification. 412 * @param[in] mask 413 * @p item->mask or flow default bit-masks. 414 * @param[in] nic_mask 415 * Bit-masks covering supported fields by the NIC to compare with user mask. 416 * @param[in] size 417 * Bit-masks size in bytes. 418 * @param[out] error 419 * Pointer to error structure. 420 * 421 * @return 422 * 0 on success, a negative errno value otherwise and rte_errno is set. 423 */ 424 int 425 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 426 const uint8_t *mask, 427 const uint8_t *nic_mask, 428 unsigned int size, 429 struct rte_flow_error *error) 430 { 431 unsigned int i; 432 433 assert(nic_mask); 434 for (i = 0; i < size; ++i) 435 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 436 return rte_flow_error_set(error, ENOTSUP, 437 RTE_FLOW_ERROR_TYPE_ITEM, 438 item, 439 "mask enables non supported" 440 " bits"); 441 if (!item->spec && (item->mask || item->last)) 442 return rte_flow_error_set(error, EINVAL, 443 RTE_FLOW_ERROR_TYPE_ITEM, item, 444 "mask/last without a spec is not" 445 " supported"); 446 if (item->spec && item->last) { 447 uint8_t spec[size]; 448 uint8_t last[size]; 449 unsigned int i; 450 int ret; 451 452 for (i = 0; i < size; ++i) { 453 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 454 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 455 } 456 ret = memcmp(spec, last, size); 457 if (ret != 0) 458 return rte_flow_error_set(error, EINVAL, 459 RTE_FLOW_ERROR_TYPE_ITEM, 460 item, 461 "range is not valid"); 462 } 463 return 0; 464 } 465 466 /** 467 * Adjust the hash fields according to the @p flow information. 468 * 469 * @param[in] dev_flow. 470 * Pointer to the mlx5_flow. 471 * @param[in] tunnel 472 * 1 when the hash field is for a tunnel item. 473 * @param[in] layer_types 474 * ETH_RSS_* types. 475 * @param[in] hash_fields 476 * Item hash fields. 477 * 478 * @return 479 * The hash fileds that should be used. 480 */ 481 uint64_t 482 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 483 int tunnel __rte_unused, uint64_t layer_types, 484 uint64_t hash_fields) 485 { 486 struct rte_flow *flow = dev_flow->flow; 487 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 488 int rss_request_inner = flow->rss.level >= 2; 489 490 /* Check RSS hash level for tunnel. */ 491 if (tunnel && rss_request_inner) 492 hash_fields |= IBV_RX_HASH_INNER; 493 else if (tunnel || rss_request_inner) 494 return 0; 495 #endif 496 /* Check if requested layer matches RSS hash fields. */ 497 if (!(flow->rss.types & layer_types)) 498 return 0; 499 return hash_fields; 500 } 501 502 /** 503 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 504 * if several tunnel rules are used on this queue, the tunnel ptype will be 505 * cleared. 506 * 507 * @param rxq_ctrl 508 * Rx queue to update. 509 */ 510 static void 511 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 512 { 513 unsigned int i; 514 uint32_t tunnel_ptype = 0; 515 516 /* Look up for the ptype to use. */ 517 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 518 if (!rxq_ctrl->flow_tunnels_n[i]) 519 continue; 520 if (!tunnel_ptype) { 521 tunnel_ptype = tunnels_info[i].ptype; 522 } else { 523 tunnel_ptype = 0; 524 break; 525 } 526 } 527 rxq_ctrl->rxq.tunnel = tunnel_ptype; 528 } 529 530 /** 531 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 532 * flow. 533 * 534 * @param[in] dev 535 * Pointer to the Ethernet device structure. 536 * @param[in] dev_flow 537 * Pointer to device flow structure. 538 */ 539 static void 540 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 541 { 542 struct mlx5_priv *priv = dev->data->dev_private; 543 struct rte_flow *flow = dev_flow->flow; 544 const int mark = !!(flow->actions & 545 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 546 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 547 unsigned int i; 548 549 for (i = 0; i != flow->rss.queue_num; ++i) { 550 int idx = (*flow->queue)[i]; 551 struct mlx5_rxq_ctrl *rxq_ctrl = 552 container_of((*priv->rxqs)[idx], 553 struct mlx5_rxq_ctrl, rxq); 554 555 if (mark) { 556 rxq_ctrl->rxq.mark = 1; 557 rxq_ctrl->flow_mark_n++; 558 } 559 if (tunnel) { 560 unsigned int j; 561 562 /* Increase the counter matching the flow. */ 563 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 564 if ((tunnels_info[j].tunnel & 565 dev_flow->layers) == 566 tunnels_info[j].tunnel) { 567 rxq_ctrl->flow_tunnels_n[j]++; 568 break; 569 } 570 } 571 flow_rxq_tunnel_ptype_update(rxq_ctrl); 572 } 573 } 574 } 575 576 /** 577 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 578 * 579 * @param[in] dev 580 * Pointer to the Ethernet device structure. 581 * @param[in] flow 582 * Pointer to flow structure. 583 */ 584 static void 585 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 586 { 587 struct mlx5_flow *dev_flow; 588 589 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 590 flow_drv_rxq_flags_set(dev, dev_flow); 591 } 592 593 /** 594 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 595 * device flow if no other flow uses it with the same kind of request. 596 * 597 * @param dev 598 * Pointer to Ethernet device. 599 * @param[in] dev_flow 600 * Pointer to the device flow. 601 */ 602 static void 603 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 604 { 605 struct mlx5_priv *priv = dev->data->dev_private; 606 struct rte_flow *flow = dev_flow->flow; 607 const int mark = !!(flow->actions & 608 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 609 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 610 unsigned int i; 611 612 assert(dev->data->dev_started); 613 for (i = 0; i != flow->rss.queue_num; ++i) { 614 int idx = (*flow->queue)[i]; 615 struct mlx5_rxq_ctrl *rxq_ctrl = 616 container_of((*priv->rxqs)[idx], 617 struct mlx5_rxq_ctrl, rxq); 618 619 if (mark) { 620 rxq_ctrl->flow_mark_n--; 621 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 622 } 623 if (tunnel) { 624 unsigned int j; 625 626 /* Decrease the counter matching the flow. */ 627 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 628 if ((tunnels_info[j].tunnel & 629 dev_flow->layers) == 630 tunnels_info[j].tunnel) { 631 rxq_ctrl->flow_tunnels_n[j]--; 632 break; 633 } 634 } 635 flow_rxq_tunnel_ptype_update(rxq_ctrl); 636 } 637 } 638 } 639 640 /** 641 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 642 * @p flow if no other flow uses it with the same kind of request. 643 * 644 * @param dev 645 * Pointer to Ethernet device. 646 * @param[in] flow 647 * Pointer to the flow. 648 */ 649 static void 650 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 651 { 652 struct mlx5_flow *dev_flow; 653 654 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 655 flow_drv_rxq_flags_trim(dev, dev_flow); 656 } 657 658 /** 659 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 660 * 661 * @param dev 662 * Pointer to Ethernet device. 663 */ 664 static void 665 flow_rxq_flags_clear(struct rte_eth_dev *dev) 666 { 667 struct mlx5_priv *priv = dev->data->dev_private; 668 unsigned int i; 669 670 for (i = 0; i != priv->rxqs_n; ++i) { 671 struct mlx5_rxq_ctrl *rxq_ctrl; 672 unsigned int j; 673 674 if (!(*priv->rxqs)[i]) 675 continue; 676 rxq_ctrl = container_of((*priv->rxqs)[i], 677 struct mlx5_rxq_ctrl, rxq); 678 rxq_ctrl->flow_mark_n = 0; 679 rxq_ctrl->rxq.mark = 0; 680 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 681 rxq_ctrl->flow_tunnels_n[j] = 0; 682 rxq_ctrl->rxq.tunnel = 0; 683 } 684 } 685 686 /* 687 * Validate the flag action. 688 * 689 * @param[in] action_flags 690 * Bit-fields that holds the actions detected until now. 691 * @param[in] attr 692 * Attributes of flow that includes this action. 693 * @param[out] error 694 * Pointer to error structure. 695 * 696 * @return 697 * 0 on success, a negative errno value otherwise and rte_errno is set. 698 */ 699 int 700 mlx5_flow_validate_action_flag(uint64_t action_flags, 701 const struct rte_flow_attr *attr, 702 struct rte_flow_error *error) 703 { 704 705 if (action_flags & MLX5_FLOW_ACTION_DROP) 706 return rte_flow_error_set(error, EINVAL, 707 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 708 "can't drop and flag in same flow"); 709 if (action_flags & MLX5_FLOW_ACTION_MARK) 710 return rte_flow_error_set(error, EINVAL, 711 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 712 "can't mark and flag in same flow"); 713 if (action_flags & MLX5_FLOW_ACTION_FLAG) 714 return rte_flow_error_set(error, EINVAL, 715 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 716 "can't have 2 flag" 717 " actions in same flow"); 718 if (attr->egress) 719 return rte_flow_error_set(error, ENOTSUP, 720 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 721 "flag action not supported for " 722 "egress"); 723 return 0; 724 } 725 726 /* 727 * Validate the mark action. 728 * 729 * @param[in] action 730 * Pointer to the queue action. 731 * @param[in] action_flags 732 * Bit-fields that holds the actions detected until now. 733 * @param[in] attr 734 * Attributes of flow that includes this action. 735 * @param[out] error 736 * Pointer to error structure. 737 * 738 * @return 739 * 0 on success, a negative errno value otherwise and rte_errno is set. 740 */ 741 int 742 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 743 uint64_t action_flags, 744 const struct rte_flow_attr *attr, 745 struct rte_flow_error *error) 746 { 747 const struct rte_flow_action_mark *mark = action->conf; 748 749 if (!mark) 750 return rte_flow_error_set(error, EINVAL, 751 RTE_FLOW_ERROR_TYPE_ACTION, 752 action, 753 "configuration cannot be null"); 754 if (mark->id >= MLX5_FLOW_MARK_MAX) 755 return rte_flow_error_set(error, EINVAL, 756 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 757 &mark->id, 758 "mark id must in 0 <= id < " 759 RTE_STR(MLX5_FLOW_MARK_MAX)); 760 if (action_flags & MLX5_FLOW_ACTION_DROP) 761 return rte_flow_error_set(error, EINVAL, 762 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 763 "can't drop and mark in same flow"); 764 if (action_flags & MLX5_FLOW_ACTION_FLAG) 765 return rte_flow_error_set(error, EINVAL, 766 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 767 "can't flag and mark in same flow"); 768 if (action_flags & MLX5_FLOW_ACTION_MARK) 769 return rte_flow_error_set(error, EINVAL, 770 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 771 "can't have 2 mark actions in same" 772 " flow"); 773 if (attr->egress) 774 return rte_flow_error_set(error, ENOTSUP, 775 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 776 "mark action not supported for " 777 "egress"); 778 return 0; 779 } 780 781 /* 782 * Validate the drop action. 783 * 784 * @param[in] action_flags 785 * Bit-fields that holds the actions detected until now. 786 * @param[in] attr 787 * Attributes of flow that includes this action. 788 * @param[out] error 789 * Pointer to error structure. 790 * 791 * @return 792 * 0 on success, a negative errno value otherwise and rte_errno is set. 793 */ 794 int 795 mlx5_flow_validate_action_drop(uint64_t action_flags, 796 const struct rte_flow_attr *attr, 797 struct rte_flow_error *error) 798 { 799 if (action_flags & MLX5_FLOW_ACTION_FLAG) 800 return rte_flow_error_set(error, EINVAL, 801 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 802 "can't drop and flag in same flow"); 803 if (action_flags & MLX5_FLOW_ACTION_MARK) 804 return rte_flow_error_set(error, EINVAL, 805 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 806 "can't drop and mark in same flow"); 807 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 808 return rte_flow_error_set(error, EINVAL, 809 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 810 "can't have 2 fate actions in" 811 " same flow"); 812 if (attr->egress) 813 return rte_flow_error_set(error, ENOTSUP, 814 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 815 "drop action not supported for " 816 "egress"); 817 return 0; 818 } 819 820 /* 821 * Validate the queue action. 822 * 823 * @param[in] action 824 * Pointer to the queue action. 825 * @param[in] action_flags 826 * Bit-fields that holds the actions detected until now. 827 * @param[in] dev 828 * Pointer to the Ethernet device structure. 829 * @param[in] attr 830 * Attributes of flow that includes this action. 831 * @param[out] error 832 * Pointer to error structure. 833 * 834 * @return 835 * 0 on success, a negative errno value otherwise and rte_errno is set. 836 */ 837 int 838 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 839 uint64_t action_flags, 840 struct rte_eth_dev *dev, 841 const struct rte_flow_attr *attr, 842 struct rte_flow_error *error) 843 { 844 struct mlx5_priv *priv = dev->data->dev_private; 845 const struct rte_flow_action_queue *queue = action->conf; 846 847 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 848 return rte_flow_error_set(error, EINVAL, 849 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 850 "can't have 2 fate actions in" 851 " same flow"); 852 if (!priv->rxqs_n) 853 return rte_flow_error_set(error, EINVAL, 854 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 855 NULL, "No Rx queues configured"); 856 if (queue->index >= priv->rxqs_n) 857 return rte_flow_error_set(error, EINVAL, 858 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 859 &queue->index, 860 "queue index out of range"); 861 if (!(*priv->rxqs)[queue->index]) 862 return rte_flow_error_set(error, EINVAL, 863 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 864 &queue->index, 865 "queue is not configured"); 866 if (attr->egress) 867 return rte_flow_error_set(error, ENOTSUP, 868 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 869 "queue action not supported for " 870 "egress"); 871 return 0; 872 } 873 874 /* 875 * Validate the rss action. 876 * 877 * @param[in] action 878 * Pointer to the queue action. 879 * @param[in] action_flags 880 * Bit-fields that holds the actions detected until now. 881 * @param[in] dev 882 * Pointer to the Ethernet device structure. 883 * @param[in] attr 884 * Attributes of flow that includes this action. 885 * @param[out] error 886 * Pointer to error structure. 887 * 888 * @return 889 * 0 on success, a negative errno value otherwise and rte_errno is set. 890 */ 891 int 892 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 893 uint64_t action_flags, 894 struct rte_eth_dev *dev, 895 const struct rte_flow_attr *attr, 896 struct rte_flow_error *error) 897 { 898 struct mlx5_priv *priv = dev->data->dev_private; 899 const struct rte_flow_action_rss *rss = action->conf; 900 unsigned int i; 901 902 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 903 return rte_flow_error_set(error, EINVAL, 904 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 905 "can't have 2 fate actions" 906 " in same flow"); 907 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 908 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 909 return rte_flow_error_set(error, ENOTSUP, 910 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 911 &rss->func, 912 "RSS hash function not supported"); 913 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 914 if (rss->level > 2) 915 #else 916 if (rss->level > 1) 917 #endif 918 return rte_flow_error_set(error, ENOTSUP, 919 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 920 &rss->level, 921 "tunnel RSS is not supported"); 922 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 923 if (rss->key_len == 0 && rss->key != NULL) 924 return rte_flow_error_set(error, ENOTSUP, 925 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 926 &rss->key_len, 927 "RSS hash key length 0"); 928 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 929 return rte_flow_error_set(error, ENOTSUP, 930 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 931 &rss->key_len, 932 "RSS hash key too small"); 933 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 934 return rte_flow_error_set(error, ENOTSUP, 935 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 936 &rss->key_len, 937 "RSS hash key too large"); 938 if (rss->queue_num > priv->config.ind_table_max_size) 939 return rte_flow_error_set(error, ENOTSUP, 940 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 941 &rss->queue_num, 942 "number of queues too large"); 943 if (rss->types & MLX5_RSS_HF_MASK) 944 return rte_flow_error_set(error, ENOTSUP, 945 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 946 &rss->types, 947 "some RSS protocols are not" 948 " supported"); 949 if (!priv->rxqs_n) 950 return rte_flow_error_set(error, EINVAL, 951 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 952 NULL, "No Rx queues configured"); 953 if (!rss->queue_num) 954 return rte_flow_error_set(error, EINVAL, 955 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 956 NULL, "No queues configured"); 957 for (i = 0; i != rss->queue_num; ++i) { 958 if (!(*priv->rxqs)[rss->queue[i]]) 959 return rte_flow_error_set 960 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 961 &rss->queue[i], "queue is not configured"); 962 } 963 if (attr->egress) 964 return rte_flow_error_set(error, ENOTSUP, 965 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 966 "rss action not supported for " 967 "egress"); 968 return 0; 969 } 970 971 /* 972 * Validate the count action. 973 * 974 * @param[in] dev 975 * Pointer to the Ethernet device structure. 976 * @param[in] attr 977 * Attributes of flow that includes this action. 978 * @param[out] error 979 * Pointer to error structure. 980 * 981 * @return 982 * 0 on success, a negative errno value otherwise and rte_errno is set. 983 */ 984 int 985 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 986 const struct rte_flow_attr *attr, 987 struct rte_flow_error *error) 988 { 989 if (attr->egress) 990 return rte_flow_error_set(error, ENOTSUP, 991 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 992 "count action not supported for " 993 "egress"); 994 return 0; 995 } 996 997 /** 998 * Verify the @p attributes will be correctly understood by the NIC and store 999 * them in the @p flow if everything is correct. 1000 * 1001 * @param[in] dev 1002 * Pointer to the Ethernet device structure. 1003 * @param[in] attributes 1004 * Pointer to flow attributes 1005 * @param[out] error 1006 * Pointer to error structure. 1007 * 1008 * @return 1009 * 0 on success, a negative errno value otherwise and rte_errno is set. 1010 */ 1011 int 1012 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1013 const struct rte_flow_attr *attributes, 1014 struct rte_flow_error *error) 1015 { 1016 struct mlx5_priv *priv = dev->data->dev_private; 1017 uint32_t priority_max = priv->config.flow_prio - 1; 1018 1019 if (attributes->group) 1020 return rte_flow_error_set(error, ENOTSUP, 1021 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1022 NULL, "groups is not supported"); 1023 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1024 attributes->priority >= priority_max) 1025 return rte_flow_error_set(error, ENOTSUP, 1026 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1027 NULL, "priority out of range"); 1028 if (attributes->egress) 1029 return rte_flow_error_set(error, ENOTSUP, 1030 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1031 "egress is not supported"); 1032 if (attributes->transfer) 1033 return rte_flow_error_set(error, ENOTSUP, 1034 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1035 NULL, "transfer is not supported"); 1036 if (!attributes->ingress) 1037 return rte_flow_error_set(error, EINVAL, 1038 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1039 NULL, 1040 "ingress attribute is mandatory"); 1041 return 0; 1042 } 1043 1044 /** 1045 * Validate Ethernet item. 1046 * 1047 * @param[in] item 1048 * Item specification. 1049 * @param[in] item_flags 1050 * Bit-fields that holds the items detected until now. 1051 * @param[out] error 1052 * Pointer to error structure. 1053 * 1054 * @return 1055 * 0 on success, a negative errno value otherwise and rte_errno is set. 1056 */ 1057 int 1058 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1059 uint64_t item_flags, 1060 struct rte_flow_error *error) 1061 { 1062 const struct rte_flow_item_eth *mask = item->mask; 1063 const struct rte_flow_item_eth nic_mask = { 1064 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1065 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1066 .type = RTE_BE16(0xffff), 1067 }; 1068 int ret; 1069 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1070 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1071 MLX5_FLOW_LAYER_OUTER_L2; 1072 1073 if (item_flags & ethm) 1074 return rte_flow_error_set(error, ENOTSUP, 1075 RTE_FLOW_ERROR_TYPE_ITEM, item, 1076 "multiple L2 layers not supported"); 1077 if (!mask) 1078 mask = &rte_flow_item_eth_mask; 1079 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1080 (const uint8_t *)&nic_mask, 1081 sizeof(struct rte_flow_item_eth), 1082 error); 1083 return ret; 1084 } 1085 1086 /** 1087 * Validate VLAN item. 1088 * 1089 * @param[in] item 1090 * Item specification. 1091 * @param[in] item_flags 1092 * Bit-fields that holds the items detected until now. 1093 * @param[out] error 1094 * Pointer to error structure. 1095 * 1096 * @return 1097 * 0 on success, a negative errno value otherwise and rte_errno is set. 1098 */ 1099 int 1100 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1101 uint64_t item_flags, 1102 struct rte_flow_error *error) 1103 { 1104 const struct rte_flow_item_vlan *spec = item->spec; 1105 const struct rte_flow_item_vlan *mask = item->mask; 1106 const struct rte_flow_item_vlan nic_mask = { 1107 .tci = RTE_BE16(0x0fff), 1108 .inner_type = RTE_BE16(0xffff), 1109 }; 1110 uint16_t vlan_tag = 0; 1111 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1112 int ret; 1113 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1114 MLX5_FLOW_LAYER_INNER_L4) : 1115 (MLX5_FLOW_LAYER_OUTER_L3 | 1116 MLX5_FLOW_LAYER_OUTER_L4); 1117 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1118 MLX5_FLOW_LAYER_OUTER_VLAN; 1119 1120 if (item_flags & vlanm) 1121 return rte_flow_error_set(error, EINVAL, 1122 RTE_FLOW_ERROR_TYPE_ITEM, item, 1123 "multiple VLAN layers not supported"); 1124 else if ((item_flags & l34m) != 0) 1125 return rte_flow_error_set(error, EINVAL, 1126 RTE_FLOW_ERROR_TYPE_ITEM, item, 1127 "L2 layer cannot follow L3/L4 layer"); 1128 if (!mask) 1129 mask = &rte_flow_item_vlan_mask; 1130 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1131 (const uint8_t *)&nic_mask, 1132 sizeof(struct rte_flow_item_vlan), 1133 error); 1134 if (ret) 1135 return ret; 1136 if (spec) { 1137 vlan_tag = spec->tci; 1138 vlan_tag &= mask->tci; 1139 } 1140 /* 1141 * From verbs perspective an empty VLAN is equivalent 1142 * to a packet without VLAN layer. 1143 */ 1144 if (!vlan_tag) 1145 return rte_flow_error_set(error, EINVAL, 1146 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1147 item->spec, 1148 "VLAN cannot be empty"); 1149 return 0; 1150 } 1151 1152 /** 1153 * Validate IPV4 item. 1154 * 1155 * @param[in] item 1156 * Item specification. 1157 * @param[in] item_flags 1158 * Bit-fields that holds the items detected until now. 1159 * @param[in] acc_mask 1160 * Acceptable mask, if NULL default internal default mask 1161 * will be used to check whether item fields are supported. 1162 * @param[out] error 1163 * Pointer to error structure. 1164 * 1165 * @return 1166 * 0 on success, a negative errno value otherwise and rte_errno is set. 1167 */ 1168 int 1169 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1170 uint64_t item_flags, 1171 const struct rte_flow_item_ipv4 *acc_mask, 1172 struct rte_flow_error *error) 1173 { 1174 const struct rte_flow_item_ipv4 *mask = item->mask; 1175 const struct rte_flow_item_ipv4 nic_mask = { 1176 .hdr = { 1177 .src_addr = RTE_BE32(0xffffffff), 1178 .dst_addr = RTE_BE32(0xffffffff), 1179 .type_of_service = 0xff, 1180 .next_proto_id = 0xff, 1181 }, 1182 }; 1183 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1184 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1185 MLX5_FLOW_LAYER_OUTER_L3; 1186 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1187 MLX5_FLOW_LAYER_OUTER_L4; 1188 int ret; 1189 1190 if (item_flags & l3m) 1191 return rte_flow_error_set(error, ENOTSUP, 1192 RTE_FLOW_ERROR_TYPE_ITEM, item, 1193 "multiple L3 layers not supported"); 1194 else if (item_flags & l4m) 1195 return rte_flow_error_set(error, EINVAL, 1196 RTE_FLOW_ERROR_TYPE_ITEM, item, 1197 "L3 cannot follow an L4 layer."); 1198 if (!mask) 1199 mask = &rte_flow_item_ipv4_mask; 1200 else if (mask->hdr.next_proto_id != 0 && 1201 mask->hdr.next_proto_id != 0xff) 1202 return rte_flow_error_set(error, EINVAL, 1203 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1204 "partial mask is not supported" 1205 " for protocol"); 1206 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1207 acc_mask ? (const uint8_t *)acc_mask 1208 : (const uint8_t *)&nic_mask, 1209 sizeof(struct rte_flow_item_ipv4), 1210 error); 1211 if (ret < 0) 1212 return ret; 1213 return 0; 1214 } 1215 1216 /** 1217 * Validate IPV6 item. 1218 * 1219 * @param[in] item 1220 * Item specification. 1221 * @param[in] item_flags 1222 * Bit-fields that holds the items detected until now. 1223 * @param[in] acc_mask 1224 * Acceptable mask, if NULL default internal default mask 1225 * will be used to check whether item fields are supported. 1226 * @param[out] error 1227 * Pointer to error structure. 1228 * 1229 * @return 1230 * 0 on success, a negative errno value otherwise and rte_errno is set. 1231 */ 1232 int 1233 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1234 uint64_t item_flags, 1235 const struct rte_flow_item_ipv6 *acc_mask, 1236 struct rte_flow_error *error) 1237 { 1238 const struct rte_flow_item_ipv6 *mask = item->mask; 1239 const struct rte_flow_item_ipv6 nic_mask = { 1240 .hdr = { 1241 .src_addr = 1242 "\xff\xff\xff\xff\xff\xff\xff\xff" 1243 "\xff\xff\xff\xff\xff\xff\xff\xff", 1244 .dst_addr = 1245 "\xff\xff\xff\xff\xff\xff\xff\xff" 1246 "\xff\xff\xff\xff\xff\xff\xff\xff", 1247 .vtc_flow = RTE_BE32(0xffffffff), 1248 .proto = 0xff, 1249 .hop_limits = 0xff, 1250 }, 1251 }; 1252 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1253 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1254 MLX5_FLOW_LAYER_OUTER_L3; 1255 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1256 MLX5_FLOW_LAYER_OUTER_L4; 1257 int ret; 1258 1259 if (item_flags & l3m) 1260 return rte_flow_error_set(error, ENOTSUP, 1261 RTE_FLOW_ERROR_TYPE_ITEM, item, 1262 "multiple L3 layers not supported"); 1263 else if (item_flags & l4m) 1264 return rte_flow_error_set(error, EINVAL, 1265 RTE_FLOW_ERROR_TYPE_ITEM, item, 1266 "L3 cannot follow an L4 layer."); 1267 if (!mask) 1268 mask = &rte_flow_item_ipv6_mask; 1269 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1270 acc_mask ? (const uint8_t *)acc_mask 1271 : (const uint8_t *)&nic_mask, 1272 sizeof(struct rte_flow_item_ipv6), 1273 error); 1274 if (ret < 0) 1275 return ret; 1276 return 0; 1277 } 1278 1279 /** 1280 * Validate UDP item. 1281 * 1282 * @param[in] item 1283 * Item specification. 1284 * @param[in] item_flags 1285 * Bit-fields that holds the items detected until now. 1286 * @param[in] target_protocol 1287 * The next protocol in the previous item. 1288 * @param[in] flow_mask 1289 * mlx5 flow-specific (TCF, DV, verbs, etc.) supported header fields mask. 1290 * @param[out] error 1291 * Pointer to error structure. 1292 * 1293 * @return 1294 * 0 on success, a negative errno value otherwise and rte_errno is set. 1295 */ 1296 int 1297 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1298 uint64_t item_flags, 1299 uint8_t target_protocol, 1300 struct rte_flow_error *error) 1301 { 1302 const struct rte_flow_item_udp *mask = item->mask; 1303 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1304 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1305 MLX5_FLOW_LAYER_OUTER_L3; 1306 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1307 MLX5_FLOW_LAYER_OUTER_L4; 1308 int ret; 1309 1310 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1311 return rte_flow_error_set(error, EINVAL, 1312 RTE_FLOW_ERROR_TYPE_ITEM, item, 1313 "protocol filtering not compatible" 1314 " with UDP layer"); 1315 if (!(item_flags & l3m)) 1316 return rte_flow_error_set(error, EINVAL, 1317 RTE_FLOW_ERROR_TYPE_ITEM, item, 1318 "L3 is mandatory to filter on L4"); 1319 if (item_flags & l4m) 1320 return rte_flow_error_set(error, EINVAL, 1321 RTE_FLOW_ERROR_TYPE_ITEM, item, 1322 "multiple L4 layers not supported"); 1323 if (!mask) 1324 mask = &rte_flow_item_udp_mask; 1325 ret = mlx5_flow_item_acceptable 1326 (item, (const uint8_t *)mask, 1327 (const uint8_t *)&rte_flow_item_udp_mask, 1328 sizeof(struct rte_flow_item_udp), error); 1329 if (ret < 0) 1330 return ret; 1331 return 0; 1332 } 1333 1334 /** 1335 * Validate TCP item. 1336 * 1337 * @param[in] item 1338 * Item specification. 1339 * @param[in] item_flags 1340 * Bit-fields that holds the items detected until now. 1341 * @param[in] target_protocol 1342 * The next protocol in the previous item. 1343 * @param[out] error 1344 * Pointer to error structure. 1345 * 1346 * @return 1347 * 0 on success, a negative errno value otherwise and rte_errno is set. 1348 */ 1349 int 1350 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1351 uint64_t item_flags, 1352 uint8_t target_protocol, 1353 const struct rte_flow_item_tcp *flow_mask, 1354 struct rte_flow_error *error) 1355 { 1356 const struct rte_flow_item_tcp *mask = item->mask; 1357 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1358 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1359 MLX5_FLOW_LAYER_OUTER_L3; 1360 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1361 MLX5_FLOW_LAYER_OUTER_L4; 1362 int ret; 1363 1364 assert(flow_mask); 1365 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1366 return rte_flow_error_set(error, EINVAL, 1367 RTE_FLOW_ERROR_TYPE_ITEM, item, 1368 "protocol filtering not compatible" 1369 " with TCP layer"); 1370 if (!(item_flags & l3m)) 1371 return rte_flow_error_set(error, EINVAL, 1372 RTE_FLOW_ERROR_TYPE_ITEM, item, 1373 "L3 is mandatory to filter on L4"); 1374 if (item_flags & l4m) 1375 return rte_flow_error_set(error, EINVAL, 1376 RTE_FLOW_ERROR_TYPE_ITEM, item, 1377 "multiple L4 layers not supported"); 1378 if (!mask) 1379 mask = &rte_flow_item_tcp_mask; 1380 ret = mlx5_flow_item_acceptable 1381 (item, (const uint8_t *)mask, 1382 (const uint8_t *)flow_mask, 1383 sizeof(struct rte_flow_item_tcp), error); 1384 if (ret < 0) 1385 return ret; 1386 return 0; 1387 } 1388 1389 /** 1390 * Validate VXLAN item. 1391 * 1392 * @param[in] item 1393 * Item specification. 1394 * @param[in] item_flags 1395 * Bit-fields that holds the items detected until now. 1396 * @param[in] target_protocol 1397 * The next protocol in the previous item. 1398 * @param[out] error 1399 * Pointer to error structure. 1400 * 1401 * @return 1402 * 0 on success, a negative errno value otherwise and rte_errno is set. 1403 */ 1404 int 1405 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1406 uint64_t item_flags, 1407 struct rte_flow_error *error) 1408 { 1409 const struct rte_flow_item_vxlan *spec = item->spec; 1410 const struct rte_flow_item_vxlan *mask = item->mask; 1411 int ret; 1412 union vni { 1413 uint32_t vlan_id; 1414 uint8_t vni[4]; 1415 } id = { .vlan_id = 0, }; 1416 uint32_t vlan_id = 0; 1417 1418 1419 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1420 return rte_flow_error_set(error, ENOTSUP, 1421 RTE_FLOW_ERROR_TYPE_ITEM, item, 1422 "multiple tunnel layers not" 1423 " supported"); 1424 /* 1425 * Verify only UDPv4 is present as defined in 1426 * https://tools.ietf.org/html/rfc7348 1427 */ 1428 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1429 return rte_flow_error_set(error, EINVAL, 1430 RTE_FLOW_ERROR_TYPE_ITEM, item, 1431 "no outer UDP layer found"); 1432 if (!mask) 1433 mask = &rte_flow_item_vxlan_mask; 1434 ret = mlx5_flow_item_acceptable 1435 (item, (const uint8_t *)mask, 1436 (const uint8_t *)&rte_flow_item_vxlan_mask, 1437 sizeof(struct rte_flow_item_vxlan), 1438 error); 1439 if (ret < 0) 1440 return ret; 1441 if (spec) { 1442 memcpy(&id.vni[1], spec->vni, 3); 1443 vlan_id = id.vlan_id; 1444 memcpy(&id.vni[1], mask->vni, 3); 1445 vlan_id &= id.vlan_id; 1446 } 1447 /* 1448 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if 1449 * only this layer is defined in the Verbs specification it is 1450 * interpreted as wildcard and all packets will match this 1451 * rule, if it follows a full stack layer (ex: eth / ipv4 / 1452 * udp), all packets matching the layers before will also 1453 * match this rule. To avoid such situation, VNI 0 is 1454 * currently refused. 1455 */ 1456 if (!vlan_id) 1457 return rte_flow_error_set(error, ENOTSUP, 1458 RTE_FLOW_ERROR_TYPE_ITEM, item, 1459 "VXLAN vni cannot be 0"); 1460 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1461 return rte_flow_error_set(error, ENOTSUP, 1462 RTE_FLOW_ERROR_TYPE_ITEM, item, 1463 "VXLAN tunnel must be fully defined"); 1464 return 0; 1465 } 1466 1467 /** 1468 * Validate VXLAN_GPE item. 1469 * 1470 * @param[in] item 1471 * Item specification. 1472 * @param[in] item_flags 1473 * Bit-fields that holds the items detected until now. 1474 * @param[in] priv 1475 * Pointer to the private data structure. 1476 * @param[in] target_protocol 1477 * The next protocol in the previous item. 1478 * @param[out] error 1479 * Pointer to error structure. 1480 * 1481 * @return 1482 * 0 on success, a negative errno value otherwise and rte_errno is set. 1483 */ 1484 int 1485 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1486 uint64_t item_flags, 1487 struct rte_eth_dev *dev, 1488 struct rte_flow_error *error) 1489 { 1490 struct mlx5_priv *priv = dev->data->dev_private; 1491 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1492 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1493 int ret; 1494 union vni { 1495 uint32_t vlan_id; 1496 uint8_t vni[4]; 1497 } id = { .vlan_id = 0, }; 1498 uint32_t vlan_id = 0; 1499 1500 if (!priv->config.l3_vxlan_en) 1501 return rte_flow_error_set(error, ENOTSUP, 1502 RTE_FLOW_ERROR_TYPE_ITEM, item, 1503 "L3 VXLAN is not enabled by device" 1504 " parameter and/or not configured in" 1505 " firmware"); 1506 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1507 return rte_flow_error_set(error, ENOTSUP, 1508 RTE_FLOW_ERROR_TYPE_ITEM, item, 1509 "multiple tunnel layers not" 1510 " supported"); 1511 /* 1512 * Verify only UDPv4 is present as defined in 1513 * https://tools.ietf.org/html/rfc7348 1514 */ 1515 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1516 return rte_flow_error_set(error, EINVAL, 1517 RTE_FLOW_ERROR_TYPE_ITEM, item, 1518 "no outer UDP layer found"); 1519 if (!mask) 1520 mask = &rte_flow_item_vxlan_gpe_mask; 1521 ret = mlx5_flow_item_acceptable 1522 (item, (const uint8_t *)mask, 1523 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1524 sizeof(struct rte_flow_item_vxlan_gpe), 1525 error); 1526 if (ret < 0) 1527 return ret; 1528 if (spec) { 1529 if (spec->protocol) 1530 return rte_flow_error_set(error, ENOTSUP, 1531 RTE_FLOW_ERROR_TYPE_ITEM, 1532 item, 1533 "VxLAN-GPE protocol" 1534 " not supported"); 1535 memcpy(&id.vni[1], spec->vni, 3); 1536 vlan_id = id.vlan_id; 1537 memcpy(&id.vni[1], mask->vni, 3); 1538 vlan_id &= id.vlan_id; 1539 } 1540 /* 1541 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1542 * layer is defined in the Verbs specification it is interpreted as 1543 * wildcard and all packets will match this rule, if it follows a full 1544 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1545 * before will also match this rule. To avoid such situation, VNI 0 1546 * is currently refused. 1547 */ 1548 if (!vlan_id) 1549 return rte_flow_error_set(error, ENOTSUP, 1550 RTE_FLOW_ERROR_TYPE_ITEM, item, 1551 "VXLAN-GPE vni cannot be 0"); 1552 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1553 return rte_flow_error_set(error, ENOTSUP, 1554 RTE_FLOW_ERROR_TYPE_ITEM, item, 1555 "VXLAN-GPE tunnel must be fully" 1556 " defined"); 1557 return 0; 1558 } 1559 1560 /** 1561 * Validate GRE item. 1562 * 1563 * @param[in] item 1564 * Item specification. 1565 * @param[in] item_flags 1566 * Bit flags to mark detected items. 1567 * @param[in] target_protocol 1568 * The next protocol in the previous item. 1569 * @param[out] error 1570 * Pointer to error structure. 1571 * 1572 * @return 1573 * 0 on success, a negative errno value otherwise and rte_errno is set. 1574 */ 1575 int 1576 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 1577 uint64_t item_flags, 1578 uint8_t target_protocol, 1579 struct rte_flow_error *error) 1580 { 1581 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 1582 const struct rte_flow_item_gre *mask = item->mask; 1583 int ret; 1584 1585 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 1586 return rte_flow_error_set(error, EINVAL, 1587 RTE_FLOW_ERROR_TYPE_ITEM, item, 1588 "protocol filtering not compatible" 1589 " with this GRE layer"); 1590 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1591 return rte_flow_error_set(error, ENOTSUP, 1592 RTE_FLOW_ERROR_TYPE_ITEM, item, 1593 "multiple tunnel layers not" 1594 " supported"); 1595 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 1596 return rte_flow_error_set(error, ENOTSUP, 1597 RTE_FLOW_ERROR_TYPE_ITEM, item, 1598 "L3 Layer is missing"); 1599 if (!mask) 1600 mask = &rte_flow_item_gre_mask; 1601 ret = mlx5_flow_item_acceptable 1602 (item, (const uint8_t *)mask, 1603 (const uint8_t *)&rte_flow_item_gre_mask, 1604 sizeof(struct rte_flow_item_gre), error); 1605 if (ret < 0) 1606 return ret; 1607 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 1608 if (spec && (spec->protocol & mask->protocol)) 1609 return rte_flow_error_set(error, ENOTSUP, 1610 RTE_FLOW_ERROR_TYPE_ITEM, item, 1611 "without MPLS support the" 1612 " specification cannot be used for" 1613 " filtering"); 1614 #endif 1615 return 0; 1616 } 1617 1618 /** 1619 * Validate MPLS item. 1620 * 1621 * @param[in] dev 1622 * Pointer to the rte_eth_dev structure. 1623 * @param[in] item 1624 * Item specification. 1625 * @param[in] item_flags 1626 * Bit-fields that holds the items detected until now. 1627 * @param[in] prev_layer 1628 * The protocol layer indicated in previous item. 1629 * @param[out] error 1630 * Pointer to error structure. 1631 * 1632 * @return 1633 * 0 on success, a negative errno value otherwise and rte_errno is set. 1634 */ 1635 int 1636 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 1637 const struct rte_flow_item *item __rte_unused, 1638 uint64_t item_flags __rte_unused, 1639 uint64_t prev_layer __rte_unused, 1640 struct rte_flow_error *error) 1641 { 1642 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1643 const struct rte_flow_item_mpls *mask = item->mask; 1644 struct mlx5_priv *priv = dev->data->dev_private; 1645 int ret; 1646 1647 if (!priv->config.mpls_en) 1648 return rte_flow_error_set(error, ENOTSUP, 1649 RTE_FLOW_ERROR_TYPE_ITEM, item, 1650 "MPLS not supported or" 1651 " disabled in firmware" 1652 " configuration."); 1653 /* MPLS over IP, UDP, GRE is allowed */ 1654 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 1655 MLX5_FLOW_LAYER_OUTER_L4_UDP | 1656 MLX5_FLOW_LAYER_GRE))) 1657 return rte_flow_error_set(error, EINVAL, 1658 RTE_FLOW_ERROR_TYPE_ITEM, item, 1659 "protocol filtering not compatible" 1660 " with MPLS layer"); 1661 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 1662 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 1663 !(item_flags & MLX5_FLOW_LAYER_GRE)) 1664 return rte_flow_error_set(error, ENOTSUP, 1665 RTE_FLOW_ERROR_TYPE_ITEM, item, 1666 "multiple tunnel layers not" 1667 " supported"); 1668 if (!mask) 1669 mask = &rte_flow_item_mpls_mask; 1670 ret = mlx5_flow_item_acceptable 1671 (item, (const uint8_t *)mask, 1672 (const uint8_t *)&rte_flow_item_mpls_mask, 1673 sizeof(struct rte_flow_item_mpls), error); 1674 if (ret < 0) 1675 return ret; 1676 return 0; 1677 #endif 1678 return rte_flow_error_set(error, ENOTSUP, 1679 RTE_FLOW_ERROR_TYPE_ITEM, item, 1680 "MPLS is not supported by Verbs, please" 1681 " update."); 1682 } 1683 1684 static int 1685 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 1686 const struct rte_flow_attr *attr __rte_unused, 1687 const struct rte_flow_item items[] __rte_unused, 1688 const struct rte_flow_action actions[] __rte_unused, 1689 struct rte_flow_error *error __rte_unused) 1690 { 1691 rte_errno = ENOTSUP; 1692 return -rte_errno; 1693 } 1694 1695 static struct mlx5_flow * 1696 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, 1697 const struct rte_flow_item items[] __rte_unused, 1698 const struct rte_flow_action actions[] __rte_unused, 1699 struct rte_flow_error *error __rte_unused) 1700 { 1701 rte_errno = ENOTSUP; 1702 return NULL; 1703 } 1704 1705 static int 1706 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 1707 struct mlx5_flow *dev_flow __rte_unused, 1708 const struct rte_flow_attr *attr __rte_unused, 1709 const struct rte_flow_item items[] __rte_unused, 1710 const struct rte_flow_action actions[] __rte_unused, 1711 struct rte_flow_error *error __rte_unused) 1712 { 1713 rte_errno = ENOTSUP; 1714 return -rte_errno; 1715 } 1716 1717 static int 1718 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 1719 struct rte_flow *flow __rte_unused, 1720 struct rte_flow_error *error __rte_unused) 1721 { 1722 rte_errno = ENOTSUP; 1723 return -rte_errno; 1724 } 1725 1726 static void 1727 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 1728 struct rte_flow *flow __rte_unused) 1729 { 1730 } 1731 1732 static void 1733 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 1734 struct rte_flow *flow __rte_unused) 1735 { 1736 } 1737 1738 static int 1739 flow_null_query(struct rte_eth_dev *dev __rte_unused, 1740 struct rte_flow *flow __rte_unused, 1741 const struct rte_flow_action *actions __rte_unused, 1742 void *data __rte_unused, 1743 struct rte_flow_error *error __rte_unused) 1744 { 1745 rte_errno = ENOTSUP; 1746 return -rte_errno; 1747 } 1748 1749 /* Void driver to protect from null pointer reference. */ 1750 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 1751 .validate = flow_null_validate, 1752 .prepare = flow_null_prepare, 1753 .translate = flow_null_translate, 1754 .apply = flow_null_apply, 1755 .remove = flow_null_remove, 1756 .destroy = flow_null_destroy, 1757 .query = flow_null_query, 1758 }; 1759 1760 /** 1761 * Select flow driver type according to flow attributes and device 1762 * configuration. 1763 * 1764 * @param[in] dev 1765 * Pointer to the dev structure. 1766 * @param[in] attr 1767 * Pointer to the flow attributes. 1768 * 1769 * @return 1770 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 1771 */ 1772 static enum mlx5_flow_drv_type 1773 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 1774 { 1775 struct mlx5_priv *priv = dev->data->dev_private; 1776 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 1777 1778 if (attr->transfer) 1779 type = MLX5_FLOW_TYPE_TCF; 1780 else 1781 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 1782 MLX5_FLOW_TYPE_VERBS; 1783 return type; 1784 } 1785 1786 #define flow_get_drv_ops(type) flow_drv_ops[type] 1787 1788 /** 1789 * Flow driver validation API. This abstracts calling driver specific functions. 1790 * The type of flow driver is determined according to flow attributes. 1791 * 1792 * @param[in] dev 1793 * Pointer to the dev structure. 1794 * @param[in] attr 1795 * Pointer to the flow attributes. 1796 * @param[in] items 1797 * Pointer to the list of items. 1798 * @param[in] actions 1799 * Pointer to the list of actions. 1800 * @param[out] error 1801 * Pointer to the error structure. 1802 * 1803 * @return 1804 * 0 on success, a negative errno value otherwise and rte_errno is set. 1805 */ 1806 static inline int 1807 flow_drv_validate(struct rte_eth_dev *dev, 1808 const struct rte_flow_attr *attr, 1809 const struct rte_flow_item items[], 1810 const struct rte_flow_action actions[], 1811 struct rte_flow_error *error) 1812 { 1813 const struct mlx5_flow_driver_ops *fops; 1814 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 1815 1816 fops = flow_get_drv_ops(type); 1817 return fops->validate(dev, attr, items, actions, error); 1818 } 1819 1820 /** 1821 * Flow driver preparation API. This abstracts calling driver specific 1822 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 1823 * calculates the size of memory required for device flow, allocates the memory, 1824 * initializes the device flow and returns the pointer. 1825 * 1826 * @note 1827 * This function initializes device flow structure such as dv, tcf or verbs in 1828 * struct mlx5_flow. However, it is caller's responsibility to initialize the 1829 * rest. For example, adding returning device flow to flow->dev_flow list and 1830 * setting backward reference to the flow should be done out of this function. 1831 * layers field is not filled either. 1832 * 1833 * @param[in] attr 1834 * Pointer to the flow attributes. 1835 * @param[in] items 1836 * Pointer to the list of items. 1837 * @param[in] actions 1838 * Pointer to the list of actions. 1839 * @param[out] error 1840 * Pointer to the error structure. 1841 * 1842 * @return 1843 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 1844 */ 1845 static inline struct mlx5_flow * 1846 flow_drv_prepare(const struct rte_flow *flow, 1847 const struct rte_flow_attr *attr, 1848 const struct rte_flow_item items[], 1849 const struct rte_flow_action actions[], 1850 struct rte_flow_error *error) 1851 { 1852 const struct mlx5_flow_driver_ops *fops; 1853 enum mlx5_flow_drv_type type = flow->drv_type; 1854 1855 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 1856 fops = flow_get_drv_ops(type); 1857 return fops->prepare(attr, items, actions, error); 1858 } 1859 1860 /** 1861 * Flow driver translation API. This abstracts calling driver specific 1862 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 1863 * translates a generic flow into a driver flow. flow_drv_prepare() must 1864 * precede. 1865 * 1866 * @note 1867 * dev_flow->layers could be filled as a result of parsing during translation 1868 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 1869 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 1870 * flow->actions could be overwritten even though all the expanded dev_flows 1871 * have the same actions. 1872 * 1873 * @param[in] dev 1874 * Pointer to the rte dev structure. 1875 * @param[in, out] dev_flow 1876 * Pointer to the mlx5 flow. 1877 * @param[in] attr 1878 * Pointer to the flow attributes. 1879 * @param[in] items 1880 * Pointer to the list of items. 1881 * @param[in] actions 1882 * Pointer to the list of actions. 1883 * @param[out] error 1884 * Pointer to the error structure. 1885 * 1886 * @return 1887 * 0 on success, a negative errno value otherwise and rte_errno is set. 1888 */ 1889 static inline int 1890 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 1891 const struct rte_flow_attr *attr, 1892 const struct rte_flow_item items[], 1893 const struct rte_flow_action actions[], 1894 struct rte_flow_error *error) 1895 { 1896 const struct mlx5_flow_driver_ops *fops; 1897 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 1898 1899 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 1900 fops = flow_get_drv_ops(type); 1901 return fops->translate(dev, dev_flow, attr, items, actions, error); 1902 } 1903 1904 /** 1905 * Flow driver apply API. This abstracts calling driver specific functions. 1906 * Parent flow (rte_flow) should have driver type (drv_type). It applies 1907 * translated driver flows on to device. flow_drv_translate() must precede. 1908 * 1909 * @param[in] dev 1910 * Pointer to Ethernet device structure. 1911 * @param[in, out] flow 1912 * Pointer to flow structure. 1913 * @param[out] error 1914 * Pointer to error structure. 1915 * 1916 * @return 1917 * 0 on success, a negative errno value otherwise and rte_errno is set. 1918 */ 1919 static inline int 1920 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 1921 struct rte_flow_error *error) 1922 { 1923 const struct mlx5_flow_driver_ops *fops; 1924 enum mlx5_flow_drv_type type = flow->drv_type; 1925 1926 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 1927 fops = flow_get_drv_ops(type); 1928 return fops->apply(dev, flow, error); 1929 } 1930 1931 /** 1932 * Flow driver remove API. This abstracts calling driver specific functions. 1933 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 1934 * on device. All the resources of the flow should be freed by calling 1935 * flow_drv_destroy(). 1936 * 1937 * @param[in] dev 1938 * Pointer to Ethernet device. 1939 * @param[in, out] flow 1940 * Pointer to flow structure. 1941 */ 1942 static inline void 1943 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 1944 { 1945 const struct mlx5_flow_driver_ops *fops; 1946 enum mlx5_flow_drv_type type = flow->drv_type; 1947 1948 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 1949 fops = flow_get_drv_ops(type); 1950 fops->remove(dev, flow); 1951 } 1952 1953 /** 1954 * Flow driver destroy API. This abstracts calling driver specific functions. 1955 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 1956 * on device and releases resources of the flow. 1957 * 1958 * @param[in] dev 1959 * Pointer to Ethernet device. 1960 * @param[in, out] flow 1961 * Pointer to flow structure. 1962 */ 1963 static inline void 1964 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 1965 { 1966 const struct mlx5_flow_driver_ops *fops; 1967 enum mlx5_flow_drv_type type = flow->drv_type; 1968 1969 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 1970 fops = flow_get_drv_ops(type); 1971 fops->destroy(dev, flow); 1972 } 1973 1974 /** 1975 * Validate a flow supported by the NIC. 1976 * 1977 * @see rte_flow_validate() 1978 * @see rte_flow_ops 1979 */ 1980 int 1981 mlx5_flow_validate(struct rte_eth_dev *dev, 1982 const struct rte_flow_attr *attr, 1983 const struct rte_flow_item items[], 1984 const struct rte_flow_action actions[], 1985 struct rte_flow_error *error) 1986 { 1987 int ret; 1988 1989 ret = flow_drv_validate(dev, attr, items, actions, error); 1990 if (ret < 0) 1991 return ret; 1992 return 0; 1993 } 1994 1995 /** 1996 * Get RSS action from the action list. 1997 * 1998 * @param[in] actions 1999 * Pointer to the list of actions. 2000 * 2001 * @return 2002 * Pointer to the RSS action if exist, else return NULL. 2003 */ 2004 static const struct rte_flow_action_rss* 2005 flow_get_rss_action(const struct rte_flow_action actions[]) 2006 { 2007 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2008 switch (actions->type) { 2009 case RTE_FLOW_ACTION_TYPE_RSS: 2010 return (const struct rte_flow_action_rss *) 2011 actions->conf; 2012 default: 2013 break; 2014 } 2015 } 2016 return NULL; 2017 } 2018 2019 static unsigned int 2020 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2021 { 2022 const struct rte_flow_item *item; 2023 unsigned int has_vlan = 0; 2024 2025 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2026 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2027 has_vlan = 1; 2028 break; 2029 } 2030 } 2031 if (has_vlan) 2032 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2033 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2034 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2035 MLX5_EXPANSION_ROOT_OUTER; 2036 } 2037 2038 /** 2039 * Create a flow and add it to @p list. 2040 * 2041 * @param dev 2042 * Pointer to Ethernet device. 2043 * @param list 2044 * Pointer to a TAILQ flow list. 2045 * @param[in] attr 2046 * Flow rule attributes. 2047 * @param[in] items 2048 * Pattern specification (list terminated by the END pattern item). 2049 * @param[in] actions 2050 * Associated actions (list terminated by the END action). 2051 * @param[out] error 2052 * Perform verbose error reporting if not NULL. 2053 * 2054 * @return 2055 * A flow on success, NULL otherwise and rte_errno is set. 2056 */ 2057 static struct rte_flow * 2058 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2059 const struct rte_flow_attr *attr, 2060 const struct rte_flow_item items[], 2061 const struct rte_flow_action actions[], 2062 struct rte_flow_error *error) 2063 { 2064 struct rte_flow *flow = NULL; 2065 struct mlx5_flow *dev_flow; 2066 const struct rte_flow_action_rss *rss; 2067 union { 2068 struct rte_flow_expand_rss buf; 2069 uint8_t buffer[2048]; 2070 } expand_buffer; 2071 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 2072 int ret; 2073 uint32_t i; 2074 uint32_t flow_size; 2075 2076 ret = flow_drv_validate(dev, attr, items, actions, error); 2077 if (ret < 0) 2078 return NULL; 2079 flow_size = sizeof(struct rte_flow); 2080 rss = flow_get_rss_action(actions); 2081 if (rss) 2082 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 2083 sizeof(void *)); 2084 else 2085 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 2086 flow = rte_calloc(__func__, 1, flow_size, 0); 2087 flow->drv_type = flow_get_drv_type(dev, attr); 2088 flow->ingress = attr->ingress; 2089 assert(flow->drv_type > MLX5_FLOW_TYPE_MIN && 2090 flow->drv_type < MLX5_FLOW_TYPE_MAX); 2091 flow->queue = (void *)(flow + 1); 2092 LIST_INIT(&flow->dev_flows); 2093 if (rss && rss->types) { 2094 unsigned int graph_root; 2095 2096 graph_root = find_graph_root(items, rss->level); 2097 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 2098 items, rss->types, 2099 mlx5_support_expansion, 2100 graph_root); 2101 assert(ret > 0 && 2102 (unsigned int)ret < sizeof(expand_buffer.buffer)); 2103 } else { 2104 buf->entries = 1; 2105 buf->entry[0].pattern = (void *)(uintptr_t)items; 2106 } 2107 for (i = 0; i < buf->entries; ++i) { 2108 dev_flow = flow_drv_prepare(flow, attr, buf->entry[i].pattern, 2109 actions, error); 2110 if (!dev_flow) 2111 goto error; 2112 dev_flow->flow = flow; 2113 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 2114 ret = flow_drv_translate(dev, dev_flow, attr, 2115 buf->entry[i].pattern, 2116 actions, error); 2117 if (ret < 0) 2118 goto error; 2119 } 2120 if (dev->data->dev_started) { 2121 ret = flow_drv_apply(dev, flow, error); 2122 if (ret < 0) 2123 goto error; 2124 } 2125 TAILQ_INSERT_TAIL(list, flow, next); 2126 flow_rxq_flags_set(dev, flow); 2127 return flow; 2128 error: 2129 ret = rte_errno; /* Save rte_errno before cleanup. */ 2130 assert(flow); 2131 flow_drv_destroy(dev, flow); 2132 rte_free(flow); 2133 rte_errno = ret; /* Restore rte_errno. */ 2134 return NULL; 2135 } 2136 2137 /** 2138 * Create a flow. 2139 * 2140 * @see rte_flow_create() 2141 * @see rte_flow_ops 2142 */ 2143 struct rte_flow * 2144 mlx5_flow_create(struct rte_eth_dev *dev, 2145 const struct rte_flow_attr *attr, 2146 const struct rte_flow_item items[], 2147 const struct rte_flow_action actions[], 2148 struct rte_flow_error *error) 2149 { 2150 struct mlx5_priv *priv = (struct mlx5_priv *)dev->data->dev_private; 2151 2152 return flow_list_create(dev, &priv->flows, 2153 attr, items, actions, error); 2154 } 2155 2156 /** 2157 * Destroy a flow in a list. 2158 * 2159 * @param dev 2160 * Pointer to Ethernet device. 2161 * @param list 2162 * Pointer to a TAILQ flow list. 2163 * @param[in] flow 2164 * Flow to destroy. 2165 */ 2166 static void 2167 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2168 struct rte_flow *flow) 2169 { 2170 /* 2171 * Update RX queue flags only if port is started, otherwise it is 2172 * already clean. 2173 */ 2174 if (dev->data->dev_started) 2175 flow_rxq_flags_trim(dev, flow); 2176 flow_drv_destroy(dev, flow); 2177 TAILQ_REMOVE(list, flow, next); 2178 rte_free(flow->fdir); 2179 rte_free(flow); 2180 } 2181 2182 /** 2183 * Destroy all flows. 2184 * 2185 * @param dev 2186 * Pointer to Ethernet device. 2187 * @param list 2188 * Pointer to a TAILQ flow list. 2189 */ 2190 void 2191 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 2192 { 2193 while (!TAILQ_EMPTY(list)) { 2194 struct rte_flow *flow; 2195 2196 flow = TAILQ_FIRST(list); 2197 flow_list_destroy(dev, list, flow); 2198 } 2199 } 2200 2201 /** 2202 * Remove all flows. 2203 * 2204 * @param dev 2205 * Pointer to Ethernet device. 2206 * @param list 2207 * Pointer to a TAILQ flow list. 2208 */ 2209 void 2210 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 2211 { 2212 struct rte_flow *flow; 2213 2214 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) 2215 flow_drv_remove(dev, flow); 2216 flow_rxq_flags_clear(dev); 2217 } 2218 2219 /** 2220 * Add all flows. 2221 * 2222 * @param dev 2223 * Pointer to Ethernet device. 2224 * @param list 2225 * Pointer to a TAILQ flow list. 2226 * 2227 * @return 2228 * 0 on success, a negative errno value otherwise and rte_errno is set. 2229 */ 2230 int 2231 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 2232 { 2233 struct rte_flow *flow; 2234 struct rte_flow_error error; 2235 int ret = 0; 2236 2237 TAILQ_FOREACH(flow, list, next) { 2238 ret = flow_drv_apply(dev, flow, &error); 2239 if (ret < 0) 2240 goto error; 2241 flow_rxq_flags_set(dev, flow); 2242 } 2243 return 0; 2244 error: 2245 ret = rte_errno; /* Save rte_errno before cleanup. */ 2246 mlx5_flow_stop(dev, list); 2247 rte_errno = ret; /* Restore rte_errno. */ 2248 return -rte_errno; 2249 } 2250 2251 /** 2252 * Verify the flow list is empty 2253 * 2254 * @param dev 2255 * Pointer to Ethernet device. 2256 * 2257 * @return the number of flows not released. 2258 */ 2259 int 2260 mlx5_flow_verify(struct rte_eth_dev *dev) 2261 { 2262 struct mlx5_priv *priv = dev->data->dev_private; 2263 struct rte_flow *flow; 2264 int ret = 0; 2265 2266 TAILQ_FOREACH(flow, &priv->flows, next) { 2267 DRV_LOG(DEBUG, "port %u flow %p still referenced", 2268 dev->data->port_id, (void *)flow); 2269 ++ret; 2270 } 2271 return ret; 2272 } 2273 2274 /** 2275 * Enable a control flow configured from the control plane. 2276 * 2277 * @param dev 2278 * Pointer to Ethernet device. 2279 * @param eth_spec 2280 * An Ethernet flow spec to apply. 2281 * @param eth_mask 2282 * An Ethernet flow mask to apply. 2283 * @param vlan_spec 2284 * A VLAN flow spec to apply. 2285 * @param vlan_mask 2286 * A VLAN flow mask to apply. 2287 * 2288 * @return 2289 * 0 on success, a negative errno value otherwise and rte_errno is set. 2290 */ 2291 int 2292 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2293 struct rte_flow_item_eth *eth_spec, 2294 struct rte_flow_item_eth *eth_mask, 2295 struct rte_flow_item_vlan *vlan_spec, 2296 struct rte_flow_item_vlan *vlan_mask) 2297 { 2298 struct mlx5_priv *priv = dev->data->dev_private; 2299 const struct rte_flow_attr attr = { 2300 .ingress = 1, 2301 .priority = MLX5_FLOW_PRIO_RSVD, 2302 }; 2303 struct rte_flow_item items[] = { 2304 { 2305 .type = RTE_FLOW_ITEM_TYPE_ETH, 2306 .spec = eth_spec, 2307 .last = NULL, 2308 .mask = eth_mask, 2309 }, 2310 { 2311 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2312 RTE_FLOW_ITEM_TYPE_END, 2313 .spec = vlan_spec, 2314 .last = NULL, 2315 .mask = vlan_mask, 2316 }, 2317 { 2318 .type = RTE_FLOW_ITEM_TYPE_END, 2319 }, 2320 }; 2321 uint16_t queue[priv->reta_idx_n]; 2322 struct rte_flow_action_rss action_rss = { 2323 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 2324 .level = 0, 2325 .types = priv->rss_conf.rss_hf, 2326 .key_len = priv->rss_conf.rss_key_len, 2327 .queue_num = priv->reta_idx_n, 2328 .key = priv->rss_conf.rss_key, 2329 .queue = queue, 2330 }; 2331 struct rte_flow_action actions[] = { 2332 { 2333 .type = RTE_FLOW_ACTION_TYPE_RSS, 2334 .conf = &action_rss, 2335 }, 2336 { 2337 .type = RTE_FLOW_ACTION_TYPE_END, 2338 }, 2339 }; 2340 struct rte_flow *flow; 2341 struct rte_flow_error error; 2342 unsigned int i; 2343 2344 if (!priv->reta_idx_n || !priv->rxqs_n) { 2345 return 0; 2346 } 2347 for (i = 0; i != priv->reta_idx_n; ++i) 2348 queue[i] = (*priv->reta_idx)[i]; 2349 flow = flow_list_create(dev, &priv->ctrl_flows, 2350 &attr, items, actions, &error); 2351 if (!flow) 2352 return -rte_errno; 2353 return 0; 2354 } 2355 2356 /** 2357 * Enable a flow control configured from the control plane. 2358 * 2359 * @param dev 2360 * Pointer to Ethernet device. 2361 * @param eth_spec 2362 * An Ethernet flow spec to apply. 2363 * @param eth_mask 2364 * An Ethernet flow mask to apply. 2365 * 2366 * @return 2367 * 0 on success, a negative errno value otherwise and rte_errno is set. 2368 */ 2369 int 2370 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2371 struct rte_flow_item_eth *eth_spec, 2372 struct rte_flow_item_eth *eth_mask) 2373 { 2374 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2375 } 2376 2377 /** 2378 * Destroy a flow. 2379 * 2380 * @see rte_flow_destroy() 2381 * @see rte_flow_ops 2382 */ 2383 int 2384 mlx5_flow_destroy(struct rte_eth_dev *dev, 2385 struct rte_flow *flow, 2386 struct rte_flow_error *error __rte_unused) 2387 { 2388 struct mlx5_priv *priv = dev->data->dev_private; 2389 2390 flow_list_destroy(dev, &priv->flows, flow); 2391 return 0; 2392 } 2393 2394 /** 2395 * Destroy all flows. 2396 * 2397 * @see rte_flow_flush() 2398 * @see rte_flow_ops 2399 */ 2400 int 2401 mlx5_flow_flush(struct rte_eth_dev *dev, 2402 struct rte_flow_error *error __rte_unused) 2403 { 2404 struct mlx5_priv *priv = dev->data->dev_private; 2405 2406 mlx5_flow_list_flush(dev, &priv->flows); 2407 return 0; 2408 } 2409 2410 /** 2411 * Isolated mode. 2412 * 2413 * @see rte_flow_isolate() 2414 * @see rte_flow_ops 2415 */ 2416 int 2417 mlx5_flow_isolate(struct rte_eth_dev *dev, 2418 int enable, 2419 struct rte_flow_error *error) 2420 { 2421 struct mlx5_priv *priv = dev->data->dev_private; 2422 2423 if (dev->data->dev_started) { 2424 rte_flow_error_set(error, EBUSY, 2425 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2426 NULL, 2427 "port must be stopped first"); 2428 return -rte_errno; 2429 } 2430 priv->isolated = !!enable; 2431 if (enable) 2432 dev->dev_ops = &mlx5_dev_ops_isolate; 2433 else 2434 dev->dev_ops = &mlx5_dev_ops; 2435 return 0; 2436 } 2437 2438 /** 2439 * Query a flow. 2440 * 2441 * @see rte_flow_query() 2442 * @see rte_flow_ops 2443 */ 2444 static int 2445 flow_drv_query(struct rte_eth_dev *dev, 2446 struct rte_flow *flow, 2447 const struct rte_flow_action *actions, 2448 void *data, 2449 struct rte_flow_error *error) 2450 { 2451 const struct mlx5_flow_driver_ops *fops; 2452 enum mlx5_flow_drv_type ftype = flow->drv_type; 2453 2454 assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 2455 fops = flow_get_drv_ops(ftype); 2456 2457 return fops->query(dev, flow, actions, data, error); 2458 } 2459 2460 /** 2461 * Query a flow. 2462 * 2463 * @see rte_flow_query() 2464 * @see rte_flow_ops 2465 */ 2466 int 2467 mlx5_flow_query(struct rte_eth_dev *dev, 2468 struct rte_flow *flow, 2469 const struct rte_flow_action *actions, 2470 void *data, 2471 struct rte_flow_error *error) 2472 { 2473 int ret; 2474 2475 ret = flow_drv_query(dev, flow, actions, data, error); 2476 if (ret < 0) 2477 return ret; 2478 return 0; 2479 } 2480 2481 /** 2482 * Convert a flow director filter to a generic flow. 2483 * 2484 * @param dev 2485 * Pointer to Ethernet device. 2486 * @param fdir_filter 2487 * Flow director filter to add. 2488 * @param attributes 2489 * Generic flow parameters structure. 2490 * 2491 * @return 2492 * 0 on success, a negative errno value otherwise and rte_errno is set. 2493 */ 2494 static int 2495 flow_fdir_filter_convert(struct rte_eth_dev *dev, 2496 const struct rte_eth_fdir_filter *fdir_filter, 2497 struct mlx5_fdir *attributes) 2498 { 2499 struct mlx5_priv *priv = dev->data->dev_private; 2500 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2501 const struct rte_eth_fdir_masks *mask = 2502 &dev->data->dev_conf.fdir_conf.mask; 2503 2504 /* Validate queue number. */ 2505 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2506 DRV_LOG(ERR, "port %u invalid queue number %d", 2507 dev->data->port_id, fdir_filter->action.rx_queue); 2508 rte_errno = EINVAL; 2509 return -rte_errno; 2510 } 2511 attributes->attr.ingress = 1; 2512 attributes->items[0] = (struct rte_flow_item) { 2513 .type = RTE_FLOW_ITEM_TYPE_ETH, 2514 .spec = &attributes->l2, 2515 .mask = &attributes->l2_mask, 2516 }; 2517 switch (fdir_filter->action.behavior) { 2518 case RTE_ETH_FDIR_ACCEPT: 2519 attributes->actions[0] = (struct rte_flow_action){ 2520 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2521 .conf = &attributes->queue, 2522 }; 2523 break; 2524 case RTE_ETH_FDIR_REJECT: 2525 attributes->actions[0] = (struct rte_flow_action){ 2526 .type = RTE_FLOW_ACTION_TYPE_DROP, 2527 }; 2528 break; 2529 default: 2530 DRV_LOG(ERR, "port %u invalid behavior %d", 2531 dev->data->port_id, 2532 fdir_filter->action.behavior); 2533 rte_errno = ENOTSUP; 2534 return -rte_errno; 2535 } 2536 attributes->queue.index = fdir_filter->action.rx_queue; 2537 /* Handle L3. */ 2538 switch (fdir_filter->input.flow_type) { 2539 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2540 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2541 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2542 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2543 .src_addr = input->flow.ip4_flow.src_ip, 2544 .dst_addr = input->flow.ip4_flow.dst_ip, 2545 .time_to_live = input->flow.ip4_flow.ttl, 2546 .type_of_service = input->flow.ip4_flow.tos, 2547 }; 2548 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){ 2549 .src_addr = mask->ipv4_mask.src_ip, 2550 .dst_addr = mask->ipv4_mask.dst_ip, 2551 .time_to_live = mask->ipv4_mask.ttl, 2552 .type_of_service = mask->ipv4_mask.tos, 2553 .next_proto_id = mask->ipv4_mask.proto, 2554 }; 2555 attributes->items[1] = (struct rte_flow_item){ 2556 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2557 .spec = &attributes->l3, 2558 .mask = &attributes->l3_mask, 2559 }; 2560 break; 2561 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2562 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2563 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2564 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2565 .hop_limits = input->flow.ipv6_flow.hop_limits, 2566 .proto = input->flow.ipv6_flow.proto, 2567 }; 2568 2569 memcpy(attributes->l3.ipv6.hdr.src_addr, 2570 input->flow.ipv6_flow.src_ip, 2571 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2572 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2573 input->flow.ipv6_flow.dst_ip, 2574 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2575 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 2576 mask->ipv6_mask.src_ip, 2577 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2578 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 2579 mask->ipv6_mask.dst_ip, 2580 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2581 attributes->items[1] = (struct rte_flow_item){ 2582 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2583 .spec = &attributes->l3, 2584 .mask = &attributes->l3_mask, 2585 }; 2586 break; 2587 default: 2588 DRV_LOG(ERR, "port %u invalid flow type%d", 2589 dev->data->port_id, fdir_filter->input.flow_type); 2590 rte_errno = ENOTSUP; 2591 return -rte_errno; 2592 } 2593 /* Handle L4. */ 2594 switch (fdir_filter->input.flow_type) { 2595 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2596 attributes->l4.udp.hdr = (struct udp_hdr){ 2597 .src_port = input->flow.udp4_flow.src_port, 2598 .dst_port = input->flow.udp4_flow.dst_port, 2599 }; 2600 attributes->l4_mask.udp.hdr = (struct udp_hdr){ 2601 .src_port = mask->src_port_mask, 2602 .dst_port = mask->dst_port_mask, 2603 }; 2604 attributes->items[2] = (struct rte_flow_item){ 2605 .type = RTE_FLOW_ITEM_TYPE_UDP, 2606 .spec = &attributes->l4, 2607 .mask = &attributes->l4_mask, 2608 }; 2609 break; 2610 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2611 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2612 .src_port = input->flow.tcp4_flow.src_port, 2613 .dst_port = input->flow.tcp4_flow.dst_port, 2614 }; 2615 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ 2616 .src_port = mask->src_port_mask, 2617 .dst_port = mask->dst_port_mask, 2618 }; 2619 attributes->items[2] = (struct rte_flow_item){ 2620 .type = RTE_FLOW_ITEM_TYPE_TCP, 2621 .spec = &attributes->l4, 2622 .mask = &attributes->l4_mask, 2623 }; 2624 break; 2625 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2626 attributes->l4.udp.hdr = (struct udp_hdr){ 2627 .src_port = input->flow.udp6_flow.src_port, 2628 .dst_port = input->flow.udp6_flow.dst_port, 2629 }; 2630 attributes->l4_mask.udp.hdr = (struct udp_hdr){ 2631 .src_port = mask->src_port_mask, 2632 .dst_port = mask->dst_port_mask, 2633 }; 2634 attributes->items[2] = (struct rte_flow_item){ 2635 .type = RTE_FLOW_ITEM_TYPE_UDP, 2636 .spec = &attributes->l4, 2637 .mask = &attributes->l4_mask, 2638 }; 2639 break; 2640 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2641 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2642 .src_port = input->flow.tcp6_flow.src_port, 2643 .dst_port = input->flow.tcp6_flow.dst_port, 2644 }; 2645 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ 2646 .src_port = mask->src_port_mask, 2647 .dst_port = mask->dst_port_mask, 2648 }; 2649 attributes->items[2] = (struct rte_flow_item){ 2650 .type = RTE_FLOW_ITEM_TYPE_TCP, 2651 .spec = &attributes->l4, 2652 .mask = &attributes->l4_mask, 2653 }; 2654 break; 2655 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2656 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2657 break; 2658 default: 2659 DRV_LOG(ERR, "port %u invalid flow type%d", 2660 dev->data->port_id, fdir_filter->input.flow_type); 2661 rte_errno = ENOTSUP; 2662 return -rte_errno; 2663 } 2664 return 0; 2665 } 2666 2667 #define FLOW_FDIR_CMP(f1, f2, fld) \ 2668 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 2669 2670 /** 2671 * Compare two FDIR flows. If items and actions are identical, the two flows are 2672 * regarded as same. 2673 * 2674 * @param dev 2675 * Pointer to Ethernet device. 2676 * @param f1 2677 * FDIR flow to compare. 2678 * @param f2 2679 * FDIR flow to compare. 2680 * 2681 * @return 2682 * Zero on match, 1 otherwise. 2683 */ 2684 static int 2685 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 2686 { 2687 if (FLOW_FDIR_CMP(f1, f2, attr) || 2688 FLOW_FDIR_CMP(f1, f2, l2) || 2689 FLOW_FDIR_CMP(f1, f2, l2_mask) || 2690 FLOW_FDIR_CMP(f1, f2, l3) || 2691 FLOW_FDIR_CMP(f1, f2, l3_mask) || 2692 FLOW_FDIR_CMP(f1, f2, l4) || 2693 FLOW_FDIR_CMP(f1, f2, l4_mask) || 2694 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 2695 return 1; 2696 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 2697 FLOW_FDIR_CMP(f1, f2, queue)) 2698 return 1; 2699 return 0; 2700 } 2701 2702 /** 2703 * Search device flow list to find out a matched FDIR flow. 2704 * 2705 * @param dev 2706 * Pointer to Ethernet device. 2707 * @param fdir_flow 2708 * FDIR flow to lookup. 2709 * 2710 * @return 2711 * Pointer of flow if found, NULL otherwise. 2712 */ 2713 static struct rte_flow * 2714 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 2715 { 2716 struct mlx5_priv *priv = dev->data->dev_private; 2717 struct rte_flow *flow = NULL; 2718 2719 assert(fdir_flow); 2720 TAILQ_FOREACH(flow, &priv->flows, next) { 2721 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 2722 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 2723 dev->data->port_id, (void *)flow); 2724 break; 2725 } 2726 } 2727 return flow; 2728 } 2729 2730 /** 2731 * Add new flow director filter and store it in list. 2732 * 2733 * @param dev 2734 * Pointer to Ethernet device. 2735 * @param fdir_filter 2736 * Flow director filter to add. 2737 * 2738 * @return 2739 * 0 on success, a negative errno value otherwise and rte_errno is set. 2740 */ 2741 static int 2742 flow_fdir_filter_add(struct rte_eth_dev *dev, 2743 const struct rte_eth_fdir_filter *fdir_filter) 2744 { 2745 struct mlx5_priv *priv = dev->data->dev_private; 2746 struct mlx5_fdir *fdir_flow; 2747 struct rte_flow *flow; 2748 int ret; 2749 2750 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 2751 if (!fdir_flow) { 2752 rte_errno = ENOMEM; 2753 return -rte_errno; 2754 } 2755 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 2756 if (ret) 2757 goto error; 2758 flow = flow_fdir_filter_lookup(dev, fdir_flow); 2759 if (flow) { 2760 rte_errno = EEXIST; 2761 goto error; 2762 } 2763 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 2764 fdir_flow->items, fdir_flow->actions, NULL); 2765 if (!flow) 2766 goto error; 2767 assert(!flow->fdir); 2768 flow->fdir = fdir_flow; 2769 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 2770 dev->data->port_id, (void *)flow); 2771 return 0; 2772 error: 2773 rte_free(fdir_flow); 2774 return -rte_errno; 2775 } 2776 2777 /** 2778 * Delete specific filter. 2779 * 2780 * @param dev 2781 * Pointer to Ethernet device. 2782 * @param fdir_filter 2783 * Filter to be deleted. 2784 * 2785 * @return 2786 * 0 on success, a negative errno value otherwise and rte_errno is set. 2787 */ 2788 static int 2789 flow_fdir_filter_delete(struct rte_eth_dev *dev, 2790 const struct rte_eth_fdir_filter *fdir_filter) 2791 { 2792 struct mlx5_priv *priv = dev->data->dev_private; 2793 struct rte_flow *flow; 2794 struct mlx5_fdir fdir_flow = { 2795 .attr.group = 0, 2796 }; 2797 int ret; 2798 2799 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 2800 if (ret) 2801 return -rte_errno; 2802 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 2803 if (!flow) { 2804 rte_errno = ENOENT; 2805 return -rte_errno; 2806 } 2807 flow_list_destroy(dev, &priv->flows, flow); 2808 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 2809 dev->data->port_id, (void *)flow); 2810 return 0; 2811 } 2812 2813 /** 2814 * Update queue for specific filter. 2815 * 2816 * @param dev 2817 * Pointer to Ethernet device. 2818 * @param fdir_filter 2819 * Filter to be updated. 2820 * 2821 * @return 2822 * 0 on success, a negative errno value otherwise and rte_errno is set. 2823 */ 2824 static int 2825 flow_fdir_filter_update(struct rte_eth_dev *dev, 2826 const struct rte_eth_fdir_filter *fdir_filter) 2827 { 2828 int ret; 2829 2830 ret = flow_fdir_filter_delete(dev, fdir_filter); 2831 if (ret) 2832 return ret; 2833 return flow_fdir_filter_add(dev, fdir_filter); 2834 } 2835 2836 /** 2837 * Flush all filters. 2838 * 2839 * @param dev 2840 * Pointer to Ethernet device. 2841 */ 2842 static void 2843 flow_fdir_filter_flush(struct rte_eth_dev *dev) 2844 { 2845 struct mlx5_priv *priv = dev->data->dev_private; 2846 2847 mlx5_flow_list_flush(dev, &priv->flows); 2848 } 2849 2850 /** 2851 * Get flow director information. 2852 * 2853 * @param dev 2854 * Pointer to Ethernet device. 2855 * @param[out] fdir_info 2856 * Resulting flow director information. 2857 */ 2858 static void 2859 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 2860 { 2861 struct rte_eth_fdir_masks *mask = 2862 &dev->data->dev_conf.fdir_conf.mask; 2863 2864 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 2865 fdir_info->guarant_spc = 0; 2866 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 2867 fdir_info->max_flexpayload = 0; 2868 fdir_info->flow_types_mask[0] = 0; 2869 fdir_info->flex_payload_unit = 0; 2870 fdir_info->max_flex_payload_segment_num = 0; 2871 fdir_info->flex_payload_limit = 0; 2872 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 2873 } 2874 2875 /** 2876 * Deal with flow director operations. 2877 * 2878 * @param dev 2879 * Pointer to Ethernet device. 2880 * @param filter_op 2881 * Operation to perform. 2882 * @param arg 2883 * Pointer to operation-specific structure. 2884 * 2885 * @return 2886 * 0 on success, a negative errno value otherwise and rte_errno is set. 2887 */ 2888 static int 2889 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 2890 void *arg) 2891 { 2892 enum rte_fdir_mode fdir_mode = 2893 dev->data->dev_conf.fdir_conf.mode; 2894 2895 if (filter_op == RTE_ETH_FILTER_NOP) 2896 return 0; 2897 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 2898 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 2899 DRV_LOG(ERR, "port %u flow director mode %d not supported", 2900 dev->data->port_id, fdir_mode); 2901 rte_errno = EINVAL; 2902 return -rte_errno; 2903 } 2904 switch (filter_op) { 2905 case RTE_ETH_FILTER_ADD: 2906 return flow_fdir_filter_add(dev, arg); 2907 case RTE_ETH_FILTER_UPDATE: 2908 return flow_fdir_filter_update(dev, arg); 2909 case RTE_ETH_FILTER_DELETE: 2910 return flow_fdir_filter_delete(dev, arg); 2911 case RTE_ETH_FILTER_FLUSH: 2912 flow_fdir_filter_flush(dev); 2913 break; 2914 case RTE_ETH_FILTER_INFO: 2915 flow_fdir_info_get(dev, arg); 2916 break; 2917 default: 2918 DRV_LOG(DEBUG, "port %u unknown operation %u", 2919 dev->data->port_id, filter_op); 2920 rte_errno = EINVAL; 2921 return -rte_errno; 2922 } 2923 return 0; 2924 } 2925 2926 /** 2927 * Manage filter operations. 2928 * 2929 * @param dev 2930 * Pointer to Ethernet device structure. 2931 * @param filter_type 2932 * Filter type. 2933 * @param filter_op 2934 * Operation to perform. 2935 * @param arg 2936 * Pointer to operation-specific structure. 2937 * 2938 * @return 2939 * 0 on success, a negative errno value otherwise and rte_errno is set. 2940 */ 2941 int 2942 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 2943 enum rte_filter_type filter_type, 2944 enum rte_filter_op filter_op, 2945 void *arg) 2946 { 2947 switch (filter_type) { 2948 case RTE_ETH_FILTER_GENERIC: 2949 if (filter_op != RTE_ETH_FILTER_GET) { 2950 rte_errno = EINVAL; 2951 return -rte_errno; 2952 } 2953 *(const void **)arg = &mlx5_flow_ops; 2954 return 0; 2955 case RTE_ETH_FILTER_FDIR: 2956 return flow_fdir_ctrl_func(dev, filter_op, arg); 2957 default: 2958 DRV_LOG(ERR, "port %u filter type (%d) not supported", 2959 dev->data->port_id, filter_type); 2960 rte_errno = ENOTSUP; 2961 return -rte_errno; 2962 } 2963 return 0; 2964 } 2965