1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <sys/queue.h> 7 #include <stdint.h> 8 #include <string.h> 9 10 /* Verbs header. */ 11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 12 #ifdef PEDANTIC 13 #pragma GCC diagnostic ignored "-Wpedantic" 14 #endif 15 #include <infiniband/verbs.h> 16 #ifdef PEDANTIC 17 #pragma GCC diagnostic error "-Wpedantic" 18 #endif 19 20 #include <rte_common.h> 21 #include <rte_eth_ctrl.h> 22 #include <rte_ethdev_driver.h> 23 #include <rte_flow.h> 24 #include <rte_flow_driver.h> 25 #include <rte_malloc.h> 26 #include <rte_ip.h> 27 28 #include "mlx5.h" 29 #include "mlx5_defs.h" 30 #include "mlx5_prm.h" 31 #include "mlx5_glue.h" 32 33 /* Define minimal priority for control plane flows. */ 34 #define MLX5_CTRL_FLOW_PRIORITY 4 35 36 /* Internet Protocol versions. */ 37 #define MLX5_IPV4 4 38 #define MLX5_IPV6 6 39 40 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 41 struct ibv_flow_spec_counter_action { 42 int dummy; 43 }; 44 #endif 45 46 /* Dev ops structure defined in mlx5.c */ 47 extern const struct eth_dev_ops mlx5_dev_ops; 48 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 49 50 /** Structure give to the conversion functions. */ 51 struct mlx5_flow_data { 52 struct mlx5_flow_parse *parser; /** Parser context. */ 53 struct rte_flow_error *error; /** Error context. */ 54 }; 55 56 static int 57 mlx5_flow_create_eth(const struct rte_flow_item *item, 58 const void *default_mask, 59 struct mlx5_flow_data *data); 60 61 static int 62 mlx5_flow_create_vlan(const struct rte_flow_item *item, 63 const void *default_mask, 64 struct mlx5_flow_data *data); 65 66 static int 67 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 68 const void *default_mask, 69 struct mlx5_flow_data *data); 70 71 static int 72 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 73 const void *default_mask, 74 struct mlx5_flow_data *data); 75 76 static int 77 mlx5_flow_create_udp(const struct rte_flow_item *item, 78 const void *default_mask, 79 struct mlx5_flow_data *data); 80 81 static int 82 mlx5_flow_create_tcp(const struct rte_flow_item *item, 83 const void *default_mask, 84 struct mlx5_flow_data *data); 85 86 static int 87 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 88 const void *default_mask, 89 struct mlx5_flow_data *data); 90 91 struct mlx5_flow_parse; 92 93 static void 94 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 95 unsigned int size); 96 97 static int 98 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); 99 100 static int 101 mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser); 102 103 /* Hash RX queue types. */ 104 enum hash_rxq_type { 105 HASH_RXQ_TCPV4, 106 HASH_RXQ_UDPV4, 107 HASH_RXQ_IPV4, 108 HASH_RXQ_TCPV6, 109 HASH_RXQ_UDPV6, 110 HASH_RXQ_IPV6, 111 HASH_RXQ_ETH, 112 }; 113 114 /* Initialization data for hash RX queue. */ 115 struct hash_rxq_init { 116 uint64_t hash_fields; /* Fields that participate in the hash. */ 117 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ 118 unsigned int flow_priority; /* Flow priority to use. */ 119 unsigned int ip_version; /* Internet protocol. */ 120 }; 121 122 /* Initialization data for hash RX queues. */ 123 const struct hash_rxq_init hash_rxq_init[] = { 124 [HASH_RXQ_TCPV4] = { 125 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 126 IBV_RX_HASH_DST_IPV4 | 127 IBV_RX_HASH_SRC_PORT_TCP | 128 IBV_RX_HASH_DST_PORT_TCP), 129 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, 130 .flow_priority = 1, 131 .ip_version = MLX5_IPV4, 132 }, 133 [HASH_RXQ_UDPV4] = { 134 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 135 IBV_RX_HASH_DST_IPV4 | 136 IBV_RX_HASH_SRC_PORT_UDP | 137 IBV_RX_HASH_DST_PORT_UDP), 138 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, 139 .flow_priority = 1, 140 .ip_version = MLX5_IPV4, 141 }, 142 [HASH_RXQ_IPV4] = { 143 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 144 IBV_RX_HASH_DST_IPV4), 145 .dpdk_rss_hf = (ETH_RSS_IPV4 | 146 ETH_RSS_FRAG_IPV4), 147 .flow_priority = 2, 148 .ip_version = MLX5_IPV4, 149 }, 150 [HASH_RXQ_TCPV6] = { 151 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 152 IBV_RX_HASH_DST_IPV6 | 153 IBV_RX_HASH_SRC_PORT_TCP | 154 IBV_RX_HASH_DST_PORT_TCP), 155 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, 156 .flow_priority = 1, 157 .ip_version = MLX5_IPV6, 158 }, 159 [HASH_RXQ_UDPV6] = { 160 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 161 IBV_RX_HASH_DST_IPV6 | 162 IBV_RX_HASH_SRC_PORT_UDP | 163 IBV_RX_HASH_DST_PORT_UDP), 164 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, 165 .flow_priority = 1, 166 .ip_version = MLX5_IPV6, 167 }, 168 [HASH_RXQ_IPV6] = { 169 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 170 IBV_RX_HASH_DST_IPV6), 171 .dpdk_rss_hf = (ETH_RSS_IPV6 | 172 ETH_RSS_FRAG_IPV6), 173 .flow_priority = 2, 174 .ip_version = MLX5_IPV6, 175 }, 176 [HASH_RXQ_ETH] = { 177 .hash_fields = 0, 178 .dpdk_rss_hf = 0, 179 .flow_priority = 3, 180 }, 181 }; 182 183 /* Number of entries in hash_rxq_init[]. */ 184 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); 185 186 /** Structure for holding counter stats. */ 187 struct mlx5_flow_counter_stats { 188 uint64_t hits; /**< Number of packets matched by the rule. */ 189 uint64_t bytes; /**< Number of bytes matched by the rule. */ 190 }; 191 192 /** Structure for Drop queue. */ 193 struct mlx5_hrxq_drop { 194 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 195 struct ibv_qp *qp; /**< Verbs queue pair. */ 196 struct ibv_wq *wq; /**< Verbs work queue. */ 197 struct ibv_cq *cq; /**< Verbs completion queue. */ 198 }; 199 200 /* Flows structures. */ 201 struct mlx5_flow { 202 uint64_t hash_fields; /**< Fields that participate in the hash. */ 203 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 204 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 205 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */ 206 }; 207 208 /* Drop flows structures. */ 209 struct mlx5_flow_drop { 210 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 211 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 212 }; 213 214 struct rte_flow { 215 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ 216 uint32_t mark:1; /**< Set if the flow is marked. */ 217 uint32_t drop:1; /**< Drop queue. */ 218 struct rte_flow_action_rss rss_conf; /**< RSS configuration */ 219 uint16_t (*queues)[]; /**< Queues indexes to use. */ 220 uint8_t rss_key[40]; /**< copy of the RSS key. */ 221 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */ 222 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */ 223 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)]; 224 /**< Flow with Rx queue. */ 225 }; 226 227 /** Static initializer for items. */ 228 #define ITEMS(...) \ 229 (const enum rte_flow_item_type []){ \ 230 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ 231 } 232 233 /** Structure to generate a simple graph of layers supported by the NIC. */ 234 struct mlx5_flow_items { 235 /** List of possible actions for these items. */ 236 const enum rte_flow_action_type *const actions; 237 /** Bit-masks corresponding to the possibilities for the item. */ 238 const void *mask; 239 /** 240 * Default bit-masks to use when item->mask is not provided. When 241 * \default_mask is also NULL, the full supported bit-mask (\mask) is 242 * used instead. 243 */ 244 const void *default_mask; 245 /** Bit-masks size in bytes. */ 246 const unsigned int mask_sz; 247 /** 248 * Conversion function from rte_flow to NIC specific flow. 249 * 250 * @param item 251 * rte_flow item to convert. 252 * @param default_mask 253 * Default bit-masks to use when item->mask is not provided. 254 * @param data 255 * Internal structure to store the conversion. 256 * 257 * @return 258 * 0 on success, a negative errno value otherwise and rte_errno is 259 * set. 260 */ 261 int (*convert)(const struct rte_flow_item *item, 262 const void *default_mask, 263 struct mlx5_flow_data *data); 264 /** Size in bytes of the destination structure. */ 265 const unsigned int dst_sz; 266 /** List of possible following items. */ 267 const enum rte_flow_item_type *const items; 268 }; 269 270 /** Valid action for this PMD. */ 271 static const enum rte_flow_action_type valid_actions[] = { 272 RTE_FLOW_ACTION_TYPE_DROP, 273 RTE_FLOW_ACTION_TYPE_QUEUE, 274 RTE_FLOW_ACTION_TYPE_MARK, 275 RTE_FLOW_ACTION_TYPE_FLAG, 276 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 277 RTE_FLOW_ACTION_TYPE_COUNT, 278 #endif 279 RTE_FLOW_ACTION_TYPE_END, 280 }; 281 282 /** Graph of supported items and associated actions. */ 283 static const struct mlx5_flow_items mlx5_flow_items[] = { 284 [RTE_FLOW_ITEM_TYPE_END] = { 285 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, 286 RTE_FLOW_ITEM_TYPE_VXLAN), 287 }, 288 [RTE_FLOW_ITEM_TYPE_ETH] = { 289 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, 290 RTE_FLOW_ITEM_TYPE_IPV4, 291 RTE_FLOW_ITEM_TYPE_IPV6), 292 .actions = valid_actions, 293 .mask = &(const struct rte_flow_item_eth){ 294 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 295 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 296 .type = -1, 297 }, 298 .default_mask = &rte_flow_item_eth_mask, 299 .mask_sz = sizeof(struct rte_flow_item_eth), 300 .convert = mlx5_flow_create_eth, 301 .dst_sz = sizeof(struct ibv_flow_spec_eth), 302 }, 303 [RTE_FLOW_ITEM_TYPE_VLAN] = { 304 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, 305 RTE_FLOW_ITEM_TYPE_IPV6), 306 .actions = valid_actions, 307 .mask = &(const struct rte_flow_item_vlan){ 308 .tci = -1, 309 }, 310 .default_mask = &rte_flow_item_vlan_mask, 311 .mask_sz = sizeof(struct rte_flow_item_vlan), 312 .convert = mlx5_flow_create_vlan, 313 .dst_sz = 0, 314 }, 315 [RTE_FLOW_ITEM_TYPE_IPV4] = { 316 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 317 RTE_FLOW_ITEM_TYPE_TCP), 318 .actions = valid_actions, 319 .mask = &(const struct rte_flow_item_ipv4){ 320 .hdr = { 321 .src_addr = -1, 322 .dst_addr = -1, 323 .type_of_service = -1, 324 .next_proto_id = -1, 325 }, 326 }, 327 .default_mask = &rte_flow_item_ipv4_mask, 328 .mask_sz = sizeof(struct rte_flow_item_ipv4), 329 .convert = mlx5_flow_create_ipv4, 330 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), 331 }, 332 [RTE_FLOW_ITEM_TYPE_IPV6] = { 333 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 334 RTE_FLOW_ITEM_TYPE_TCP), 335 .actions = valid_actions, 336 .mask = &(const struct rte_flow_item_ipv6){ 337 .hdr = { 338 .src_addr = { 339 0xff, 0xff, 0xff, 0xff, 340 0xff, 0xff, 0xff, 0xff, 341 0xff, 0xff, 0xff, 0xff, 342 0xff, 0xff, 0xff, 0xff, 343 }, 344 .dst_addr = { 345 0xff, 0xff, 0xff, 0xff, 346 0xff, 0xff, 0xff, 0xff, 347 0xff, 0xff, 0xff, 0xff, 348 0xff, 0xff, 0xff, 0xff, 349 }, 350 .vtc_flow = -1, 351 .proto = -1, 352 .hop_limits = -1, 353 }, 354 }, 355 .default_mask = &rte_flow_item_ipv6_mask, 356 .mask_sz = sizeof(struct rte_flow_item_ipv6), 357 .convert = mlx5_flow_create_ipv6, 358 .dst_sz = sizeof(struct ibv_flow_spec_ipv6), 359 }, 360 [RTE_FLOW_ITEM_TYPE_UDP] = { 361 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), 362 .actions = valid_actions, 363 .mask = &(const struct rte_flow_item_udp){ 364 .hdr = { 365 .src_port = -1, 366 .dst_port = -1, 367 }, 368 }, 369 .default_mask = &rte_flow_item_udp_mask, 370 .mask_sz = sizeof(struct rte_flow_item_udp), 371 .convert = mlx5_flow_create_udp, 372 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 373 }, 374 [RTE_FLOW_ITEM_TYPE_TCP] = { 375 .actions = valid_actions, 376 .mask = &(const struct rte_flow_item_tcp){ 377 .hdr = { 378 .src_port = -1, 379 .dst_port = -1, 380 }, 381 }, 382 .default_mask = &rte_flow_item_tcp_mask, 383 .mask_sz = sizeof(struct rte_flow_item_tcp), 384 .convert = mlx5_flow_create_tcp, 385 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 386 }, 387 [RTE_FLOW_ITEM_TYPE_VXLAN] = { 388 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), 389 .actions = valid_actions, 390 .mask = &(const struct rte_flow_item_vxlan){ 391 .vni = "\xff\xff\xff", 392 }, 393 .default_mask = &rte_flow_item_vxlan_mask, 394 .mask_sz = sizeof(struct rte_flow_item_vxlan), 395 .convert = mlx5_flow_create_vxlan, 396 .dst_sz = sizeof(struct ibv_flow_spec_tunnel), 397 }, 398 }; 399 400 /** Structure to pass to the conversion function. */ 401 struct mlx5_flow_parse { 402 uint32_t inner; /**< Set once VXLAN is encountered. */ 403 uint32_t create:1; 404 /**< Whether resources should remain after a validate. */ 405 uint32_t drop:1; /**< Target is a drop queue. */ 406 uint32_t mark:1; /**< Mark is present in the flow. */ 407 uint32_t count:1; /**< Count is present in the flow. */ 408 uint32_t mark_id; /**< Mark identifier. */ 409 struct rte_flow_action_rss rss_conf; /**< RSS configuration */ 410 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ 411 uint8_t rss_key[40]; /**< copy of the RSS key. */ 412 enum hash_rxq_type layer; /**< Last pattern layer detected. */ 413 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */ 414 struct { 415 struct ibv_flow_attr *ibv_attr; 416 /**< Pointer to Verbs attributes. */ 417 unsigned int offset; 418 /**< Current position or total size of the attribute. */ 419 } queue[RTE_DIM(hash_rxq_init)]; 420 }; 421 422 static const struct rte_flow_ops mlx5_flow_ops = { 423 .validate = mlx5_flow_validate, 424 .create = mlx5_flow_create, 425 .destroy = mlx5_flow_destroy, 426 .flush = mlx5_flow_flush, 427 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 428 .query = mlx5_flow_query, 429 #else 430 .query = NULL, 431 #endif 432 .isolate = mlx5_flow_isolate, 433 }; 434 435 /* Convert FDIR request to Generic flow. */ 436 struct mlx5_fdir { 437 struct rte_flow_attr attr; 438 struct rte_flow_action actions[2]; 439 struct rte_flow_item items[4]; 440 struct rte_flow_item_eth l2; 441 struct rte_flow_item_eth l2_mask; 442 union { 443 struct rte_flow_item_ipv4 ipv4; 444 struct rte_flow_item_ipv6 ipv6; 445 } l3; 446 union { 447 struct rte_flow_item_ipv4 ipv4; 448 struct rte_flow_item_ipv6 ipv6; 449 } l3_mask; 450 union { 451 struct rte_flow_item_udp udp; 452 struct rte_flow_item_tcp tcp; 453 } l4; 454 union { 455 struct rte_flow_item_udp udp; 456 struct rte_flow_item_tcp tcp; 457 } l4_mask; 458 struct rte_flow_action_queue queue; 459 }; 460 461 /* Verbs specification header. */ 462 struct ibv_spec_header { 463 enum ibv_flow_spec_type type; 464 uint16_t size; 465 }; 466 467 /** 468 * Check support for a given item. 469 * 470 * @param item[in] 471 * Item specification. 472 * @param mask[in] 473 * Bit-masks covering supported fields to compare with spec, last and mask in 474 * \item. 475 * @param size 476 * Bit-Mask size in bytes. 477 * 478 * @return 479 * 0 on success, a negative errno value otherwise and rte_errno is set. 480 */ 481 static int 482 mlx5_flow_item_validate(const struct rte_flow_item *item, 483 const uint8_t *mask, unsigned int size) 484 { 485 if (!item->spec && (item->mask || item->last)) { 486 rte_errno = EINVAL; 487 return -rte_errno; 488 } 489 if (item->spec && !item->mask) { 490 unsigned int i; 491 const uint8_t *spec = item->spec; 492 493 for (i = 0; i < size; ++i) 494 if ((spec[i] | mask[i]) != mask[i]) { 495 rte_errno = EINVAL; 496 return -rte_errno; 497 } 498 } 499 if (item->last && !item->mask) { 500 unsigned int i; 501 const uint8_t *spec = item->last; 502 503 for (i = 0; i < size; ++i) 504 if ((spec[i] | mask[i]) != mask[i]) { 505 rte_errno = EINVAL; 506 return -rte_errno; 507 } 508 } 509 if (item->mask) { 510 unsigned int i; 511 const uint8_t *spec = item->spec; 512 513 for (i = 0; i < size; ++i) 514 if ((spec[i] | mask[i]) != mask[i]) { 515 rte_errno = EINVAL; 516 return -rte_errno; 517 } 518 } 519 if (item->spec && item->last) { 520 uint8_t spec[size]; 521 uint8_t last[size]; 522 const uint8_t *apply = mask; 523 unsigned int i; 524 int ret; 525 526 if (item->mask) 527 apply = item->mask; 528 for (i = 0; i < size; ++i) { 529 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; 530 last[i] = ((const uint8_t *)item->last)[i] & apply[i]; 531 } 532 ret = memcmp(spec, last, size); 533 if (ret != 0) { 534 rte_errno = EINVAL; 535 return -rte_errno; 536 } 537 } 538 return 0; 539 } 540 541 /** 542 * Extract attribute to the parser. 543 * 544 * @param[in] attr 545 * Flow rule attributes. 546 * @param[out] error 547 * Perform verbose error reporting if not NULL. 548 * 549 * @return 550 * 0 on success, a negative errno value otherwise and rte_errno is set. 551 */ 552 static int 553 mlx5_flow_convert_attributes(const struct rte_flow_attr *attr, 554 struct rte_flow_error *error) 555 { 556 if (attr->group) { 557 rte_flow_error_set(error, ENOTSUP, 558 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 559 NULL, 560 "groups are not supported"); 561 return -rte_errno; 562 } 563 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) { 564 rte_flow_error_set(error, ENOTSUP, 565 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 566 NULL, 567 "priorities are not supported"); 568 return -rte_errno; 569 } 570 if (attr->egress) { 571 rte_flow_error_set(error, ENOTSUP, 572 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 573 NULL, 574 "egress is not supported"); 575 return -rte_errno; 576 } 577 if (!attr->ingress) { 578 rte_flow_error_set(error, ENOTSUP, 579 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 580 NULL, 581 "only ingress is supported"); 582 return -rte_errno; 583 } 584 return 0; 585 } 586 587 /** 588 * Extract actions request to the parser. 589 * 590 * @param dev 591 * Pointer to Ethernet device. 592 * @param[in] actions 593 * Associated actions (list terminated by the END action). 594 * @param[out] error 595 * Perform verbose error reporting if not NULL. 596 * @param[in, out] parser 597 * Internal parser structure. 598 * 599 * @return 600 * 0 on success, a negative errno value otherwise and rte_errno is set. 601 */ 602 static int 603 mlx5_flow_convert_actions(struct rte_eth_dev *dev, 604 const struct rte_flow_action actions[], 605 struct rte_flow_error *error, 606 struct mlx5_flow_parse *parser) 607 { 608 enum { FATE = 1, MARK = 2, COUNT = 4, }; 609 uint32_t overlap = 0; 610 struct priv *priv = dev->data->dev_private; 611 612 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { 613 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { 614 continue; 615 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { 616 if (overlap & FATE) 617 goto exit_action_overlap; 618 overlap |= FATE; 619 parser->drop = 1; 620 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 621 const struct rte_flow_action_queue *queue = 622 (const struct rte_flow_action_queue *) 623 actions->conf; 624 625 if (overlap & FATE) 626 goto exit_action_overlap; 627 overlap |= FATE; 628 if (!queue || (queue->index > (priv->rxqs_n - 1))) 629 goto exit_action_not_supported; 630 parser->queues[0] = queue->index; 631 parser->rss_conf = (struct rte_flow_action_rss){ 632 .queue_num = 1, 633 .queue = parser->queues, 634 }; 635 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { 636 const struct rte_flow_action_rss *rss = 637 (const struct rte_flow_action_rss *) 638 actions->conf; 639 const uint8_t *rss_key; 640 uint32_t rss_key_len; 641 uint16_t n; 642 643 if (overlap & FATE) 644 goto exit_action_overlap; 645 overlap |= FATE; 646 if (rss->func && 647 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) { 648 rte_flow_error_set(error, EINVAL, 649 RTE_FLOW_ERROR_TYPE_ACTION, 650 actions, 651 "the only supported RSS hash" 652 " function is Toeplitz"); 653 return -rte_errno; 654 } 655 if (rss->types & MLX5_RSS_HF_MASK) { 656 rte_flow_error_set(error, EINVAL, 657 RTE_FLOW_ERROR_TYPE_ACTION, 658 actions, 659 "unsupported RSS type" 660 " requested"); 661 return -rte_errno; 662 } 663 if (rss->key_len) { 664 rss_key_len = rss->key_len; 665 rss_key = rss->key; 666 } else { 667 rss_key_len = rss_hash_default_key_len; 668 rss_key = rss_hash_default_key; 669 } 670 if (rss_key_len != RTE_DIM(parser->rss_key)) { 671 rte_flow_error_set(error, EINVAL, 672 RTE_FLOW_ERROR_TYPE_ACTION, 673 actions, 674 "RSS hash key must be" 675 " exactly 40 bytes long"); 676 return -rte_errno; 677 } 678 if (!rss->queue_num) { 679 rte_flow_error_set(error, EINVAL, 680 RTE_FLOW_ERROR_TYPE_ACTION, 681 actions, 682 "no valid queues"); 683 return -rte_errno; 684 } 685 if (rss->queue_num > RTE_DIM(parser->queues)) { 686 rte_flow_error_set(error, EINVAL, 687 RTE_FLOW_ERROR_TYPE_ACTION, 688 actions, 689 "too many queues for RSS" 690 " context"); 691 return -rte_errno; 692 } 693 for (n = 0; n < rss->queue_num; ++n) { 694 if (rss->queue[n] >= priv->rxqs_n) { 695 rte_flow_error_set(error, EINVAL, 696 RTE_FLOW_ERROR_TYPE_ACTION, 697 actions, 698 "queue id > number of" 699 " queues"); 700 return -rte_errno; 701 } 702 } 703 parser->rss_conf = (struct rte_flow_action_rss){ 704 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 705 .types = rss->types, 706 .key_len = rss_key_len, 707 .queue_num = rss->queue_num, 708 .key = memcpy(parser->rss_key, rss_key, 709 sizeof(*rss_key) * rss_key_len), 710 .queue = memcpy(parser->queues, rss->queue, 711 sizeof(*rss->queue) * 712 rss->queue_num), 713 }; 714 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { 715 const struct rte_flow_action_mark *mark = 716 (const struct rte_flow_action_mark *) 717 actions->conf; 718 719 if (overlap & MARK) 720 goto exit_action_overlap; 721 overlap |= MARK; 722 if (!mark) { 723 rte_flow_error_set(error, EINVAL, 724 RTE_FLOW_ERROR_TYPE_ACTION, 725 actions, 726 "mark must be defined"); 727 return -rte_errno; 728 } else if (mark->id >= MLX5_FLOW_MARK_MAX) { 729 rte_flow_error_set(error, ENOTSUP, 730 RTE_FLOW_ERROR_TYPE_ACTION, 731 actions, 732 "mark must be between 0" 733 " and 16777199"); 734 return -rte_errno; 735 } 736 parser->mark = 1; 737 parser->mark_id = mark->id; 738 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { 739 if (overlap & MARK) 740 goto exit_action_overlap; 741 overlap |= MARK; 742 parser->mark = 1; 743 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && 744 priv->config.flow_counter_en) { 745 if (overlap & COUNT) 746 goto exit_action_overlap; 747 overlap |= COUNT; 748 parser->count = 1; 749 } else { 750 goto exit_action_not_supported; 751 } 752 } 753 /* When fate is unknown, drop traffic. */ 754 if (!(overlap & FATE)) 755 parser->drop = 1; 756 if (parser->drop && parser->mark) 757 parser->mark = 0; 758 if (!parser->rss_conf.queue_num && !parser->drop) { 759 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, 760 NULL, "no valid action"); 761 return -rte_errno; 762 } 763 return 0; 764 exit_action_not_supported: 765 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 766 actions, "action not supported"); 767 return -rte_errno; 768 exit_action_overlap: 769 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 770 actions, "overlapping actions are not supported"); 771 return -rte_errno; 772 } 773 774 /** 775 * Validate items. 776 * 777 * @param[in] items 778 * Pattern specification (list terminated by the END pattern item). 779 * @param[out] error 780 * Perform verbose error reporting if not NULL. 781 * @param[in, out] parser 782 * Internal parser structure. 783 * 784 * @return 785 * 0 on success, a negative errno value otherwise and rte_errno is set. 786 */ 787 static int 788 mlx5_flow_convert_items_validate(const struct rte_flow_item items[], 789 struct rte_flow_error *error, 790 struct mlx5_flow_parse *parser) 791 { 792 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 793 unsigned int i; 794 int ret = 0; 795 796 /* Initialise the offsets to start after verbs attribute. */ 797 for (i = 0; i != hash_rxq_init_n; ++i) 798 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 799 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 800 const struct mlx5_flow_items *token = NULL; 801 unsigned int n; 802 803 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 804 continue; 805 for (i = 0; 806 cur_item->items && 807 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; 808 ++i) { 809 if (cur_item->items[i] == items->type) { 810 token = &mlx5_flow_items[items->type]; 811 break; 812 } 813 } 814 if (!token) { 815 ret = -ENOTSUP; 816 goto exit_item_not_supported; 817 } 818 cur_item = token; 819 ret = mlx5_flow_item_validate(items, 820 (const uint8_t *)cur_item->mask, 821 cur_item->mask_sz); 822 if (ret) 823 goto exit_item_not_supported; 824 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 825 if (parser->inner) { 826 rte_flow_error_set(error, ENOTSUP, 827 RTE_FLOW_ERROR_TYPE_ITEM, 828 items, 829 "cannot recognize multiple" 830 " VXLAN encapsulations"); 831 return -rte_errno; 832 } 833 parser->inner = IBV_FLOW_SPEC_INNER; 834 } 835 if (parser->drop) { 836 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; 837 } else { 838 for (n = 0; n != hash_rxq_init_n; ++n) 839 parser->queue[n].offset += cur_item->dst_sz; 840 } 841 } 842 if (parser->drop) { 843 parser->queue[HASH_RXQ_ETH].offset += 844 sizeof(struct ibv_flow_spec_action_drop); 845 } 846 if (parser->mark) { 847 for (i = 0; i != hash_rxq_init_n; ++i) 848 parser->queue[i].offset += 849 sizeof(struct ibv_flow_spec_action_tag); 850 } 851 if (parser->count) { 852 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 853 854 for (i = 0; i != hash_rxq_init_n; ++i) 855 parser->queue[i].offset += size; 856 } 857 return 0; 858 exit_item_not_supported: 859 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM, 860 items, "item not supported"); 861 } 862 863 /** 864 * Allocate memory space to store verbs flow attributes. 865 * 866 * @param[in] size 867 * Amount of byte to allocate. 868 * @param[out] error 869 * Perform verbose error reporting if not NULL. 870 * 871 * @return 872 * A verbs flow attribute on success, NULL otherwise and rte_errno is set. 873 */ 874 static struct ibv_flow_attr * 875 mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error) 876 { 877 struct ibv_flow_attr *ibv_attr; 878 879 ibv_attr = rte_calloc(__func__, 1, size, 0); 880 if (!ibv_attr) { 881 rte_flow_error_set(error, ENOMEM, 882 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 883 NULL, 884 "cannot allocate verbs spec attributes"); 885 return NULL; 886 } 887 return ibv_attr; 888 } 889 890 /** 891 * Make inner packet matching with an higher priority from the non Inner 892 * matching. 893 * 894 * @param[in, out] parser 895 * Internal parser structure. 896 * @param attr 897 * User flow attribute. 898 */ 899 static void 900 mlx5_flow_update_priority(struct mlx5_flow_parse *parser, 901 const struct rte_flow_attr *attr) 902 { 903 unsigned int i; 904 905 if (parser->drop) { 906 parser->queue[HASH_RXQ_ETH].ibv_attr->priority = 907 attr->priority + 908 hash_rxq_init[HASH_RXQ_ETH].flow_priority; 909 return; 910 } 911 for (i = 0; i != hash_rxq_init_n; ++i) { 912 if (parser->queue[i].ibv_attr) { 913 parser->queue[i].ibv_attr->priority = 914 attr->priority + 915 hash_rxq_init[i].flow_priority - 916 (parser->inner ? 1 : 0); 917 } 918 } 919 } 920 921 /** 922 * Finalise verbs flow attributes. 923 * 924 * @param[in, out] parser 925 * Internal parser structure. 926 */ 927 static void 928 mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser) 929 { 930 const unsigned int ipv4 = 931 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; 932 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6; 933 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 934 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4; 935 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4; 936 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 937 unsigned int i; 938 939 /* Remove any other flow not matching the pattern. */ 940 if (parser->rss_conf.queue_num == 1 && !parser->rss_conf.types) { 941 for (i = 0; i != hash_rxq_init_n; ++i) { 942 if (i == HASH_RXQ_ETH) 943 continue; 944 rte_free(parser->queue[i].ibv_attr); 945 parser->queue[i].ibv_attr = NULL; 946 } 947 return; 948 } 949 if (parser->layer == HASH_RXQ_ETH) { 950 goto fill; 951 } else { 952 /* 953 * This layer becomes useless as the pattern define under 954 * layers. 955 */ 956 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr); 957 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 958 } 959 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */ 960 for (i = ohmin; i != (ohmax + 1); ++i) { 961 if (!parser->queue[i].ibv_attr) 962 continue; 963 rte_free(parser->queue[i].ibv_attr); 964 parser->queue[i].ibv_attr = NULL; 965 } 966 /* Remove impossible flow according to the RSS configuration. */ 967 if (hash_rxq_init[parser->layer].dpdk_rss_hf & 968 parser->rss_conf.types) { 969 /* Remove any other flow. */ 970 for (i = hmin; i != (hmax + 1); ++i) { 971 if ((i == parser->layer) || 972 (!parser->queue[i].ibv_attr)) 973 continue; 974 rte_free(parser->queue[i].ibv_attr); 975 parser->queue[i].ibv_attr = NULL; 976 } 977 } else if (!parser->queue[ip].ibv_attr) { 978 /* no RSS possible with the current configuration. */ 979 parser->rss_conf.queue_num = 1; 980 return; 981 } 982 fill: 983 /* 984 * Fill missing layers in verbs specifications, or compute the correct 985 * offset to allocate the memory space for the attributes and 986 * specifications. 987 */ 988 for (i = 0; i != hash_rxq_init_n - 1; ++i) { 989 union { 990 struct ibv_flow_spec_ipv4_ext ipv4; 991 struct ibv_flow_spec_ipv6 ipv6; 992 struct ibv_flow_spec_tcp_udp udp_tcp; 993 } specs; 994 void *dst; 995 uint16_t size; 996 997 if (i == parser->layer) 998 continue; 999 if (parser->layer == HASH_RXQ_ETH) { 1000 if (hash_rxq_init[i].ip_version == MLX5_IPV4) { 1001 size = sizeof(struct ibv_flow_spec_ipv4_ext); 1002 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){ 1003 .type = IBV_FLOW_SPEC_IPV4_EXT, 1004 .size = size, 1005 }; 1006 } else { 1007 size = sizeof(struct ibv_flow_spec_ipv6); 1008 specs.ipv6 = (struct ibv_flow_spec_ipv6){ 1009 .type = IBV_FLOW_SPEC_IPV6, 1010 .size = size, 1011 }; 1012 } 1013 if (parser->queue[i].ibv_attr) { 1014 dst = (void *)((uintptr_t) 1015 parser->queue[i].ibv_attr + 1016 parser->queue[i].offset); 1017 memcpy(dst, &specs, size); 1018 ++parser->queue[i].ibv_attr->num_of_specs; 1019 } 1020 parser->queue[i].offset += size; 1021 } 1022 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) || 1023 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) { 1024 size = sizeof(struct ibv_flow_spec_tcp_udp); 1025 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) { 1026 .type = ((i == HASH_RXQ_UDPV4 || 1027 i == HASH_RXQ_UDPV6) ? 1028 IBV_FLOW_SPEC_UDP : 1029 IBV_FLOW_SPEC_TCP), 1030 .size = size, 1031 }; 1032 if (parser->queue[i].ibv_attr) { 1033 dst = (void *)((uintptr_t) 1034 parser->queue[i].ibv_attr + 1035 parser->queue[i].offset); 1036 memcpy(dst, &specs, size); 1037 ++parser->queue[i].ibv_attr->num_of_specs; 1038 } 1039 parser->queue[i].offset += size; 1040 } 1041 } 1042 } 1043 1044 /** 1045 * Validate and convert a flow supported by the NIC. 1046 * 1047 * @param dev 1048 * Pointer to Ethernet device. 1049 * @param[in] attr 1050 * Flow rule attributes. 1051 * @param[in] pattern 1052 * Pattern specification (list terminated by the END pattern item). 1053 * @param[in] actions 1054 * Associated actions (list terminated by the END action). 1055 * @param[out] error 1056 * Perform verbose error reporting if not NULL. 1057 * @param[in, out] parser 1058 * Internal parser structure. 1059 * 1060 * @return 1061 * 0 on success, a negative errno value otherwise and rte_errno is set. 1062 */ 1063 static int 1064 mlx5_flow_convert(struct rte_eth_dev *dev, 1065 const struct rte_flow_attr *attr, 1066 const struct rte_flow_item items[], 1067 const struct rte_flow_action actions[], 1068 struct rte_flow_error *error, 1069 struct mlx5_flow_parse *parser) 1070 { 1071 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 1072 unsigned int i; 1073 int ret; 1074 1075 /* First step. Validate the attributes, items and actions. */ 1076 *parser = (struct mlx5_flow_parse){ 1077 .create = parser->create, 1078 .layer = HASH_RXQ_ETH, 1079 .mark_id = MLX5_FLOW_MARK_DEFAULT, 1080 }; 1081 ret = mlx5_flow_convert_attributes(attr, error); 1082 if (ret) 1083 return ret; 1084 ret = mlx5_flow_convert_actions(dev, actions, error, parser); 1085 if (ret) 1086 return ret; 1087 ret = mlx5_flow_convert_items_validate(items, error, parser); 1088 if (ret) 1089 return ret; 1090 mlx5_flow_convert_finalise(parser); 1091 /* 1092 * Second step. 1093 * Allocate the memory space to store verbs specifications. 1094 */ 1095 if (parser->drop) { 1096 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; 1097 1098 parser->queue[HASH_RXQ_ETH].ibv_attr = 1099 mlx5_flow_convert_allocate(offset, error); 1100 if (!parser->queue[HASH_RXQ_ETH].ibv_attr) 1101 goto exit_enomem; 1102 parser->queue[HASH_RXQ_ETH].offset = 1103 sizeof(struct ibv_flow_attr); 1104 } else { 1105 for (i = 0; i != hash_rxq_init_n; ++i) { 1106 unsigned int offset; 1107 1108 if (!(parser->rss_conf.types & 1109 hash_rxq_init[i].dpdk_rss_hf) && 1110 (i != HASH_RXQ_ETH)) 1111 continue; 1112 offset = parser->queue[i].offset; 1113 parser->queue[i].ibv_attr = 1114 mlx5_flow_convert_allocate(offset, error); 1115 if (!parser->queue[i].ibv_attr) 1116 goto exit_enomem; 1117 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 1118 } 1119 } 1120 /* Third step. Conversion parse, fill the specifications. */ 1121 parser->inner = 0; 1122 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 1123 struct mlx5_flow_data data = { 1124 .parser = parser, 1125 .error = error, 1126 }; 1127 1128 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 1129 continue; 1130 cur_item = &mlx5_flow_items[items->type]; 1131 ret = cur_item->convert(items, 1132 (cur_item->default_mask ? 1133 cur_item->default_mask : 1134 cur_item->mask), 1135 &data); 1136 if (ret) 1137 goto exit_free; 1138 } 1139 if (parser->mark) 1140 mlx5_flow_create_flag_mark(parser, parser->mark_id); 1141 if (parser->count && parser->create) { 1142 mlx5_flow_create_count(dev, parser); 1143 if (!parser->cs) 1144 goto exit_count_error; 1145 } 1146 /* 1147 * Last step. Complete missing specification to reach the RSS 1148 * configuration. 1149 */ 1150 if (!parser->drop) 1151 mlx5_flow_convert_finalise(parser); 1152 mlx5_flow_update_priority(parser, attr); 1153 exit_free: 1154 /* Only verification is expected, all resources should be released. */ 1155 if (!parser->create) { 1156 for (i = 0; i != hash_rxq_init_n; ++i) { 1157 if (parser->queue[i].ibv_attr) { 1158 rte_free(parser->queue[i].ibv_attr); 1159 parser->queue[i].ibv_attr = NULL; 1160 } 1161 } 1162 } 1163 return ret; 1164 exit_enomem: 1165 for (i = 0; i != hash_rxq_init_n; ++i) { 1166 if (parser->queue[i].ibv_attr) { 1167 rte_free(parser->queue[i].ibv_attr); 1168 parser->queue[i].ibv_attr = NULL; 1169 } 1170 } 1171 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1172 NULL, "cannot allocate verbs spec attributes"); 1173 return -rte_errno; 1174 exit_count_error: 1175 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1176 NULL, "cannot create counter"); 1177 return -rte_errno; 1178 } 1179 1180 /** 1181 * Copy the specification created into the flow. 1182 * 1183 * @param parser 1184 * Internal parser structure. 1185 * @param src 1186 * Create specification. 1187 * @param size 1188 * Size in bytes of the specification to copy. 1189 */ 1190 static void 1191 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 1192 unsigned int size) 1193 { 1194 unsigned int i; 1195 void *dst; 1196 1197 for (i = 0; i != hash_rxq_init_n; ++i) { 1198 if (!parser->queue[i].ibv_attr) 1199 continue; 1200 /* Specification must be the same l3 type or none. */ 1201 if (parser->layer == HASH_RXQ_ETH || 1202 (hash_rxq_init[parser->layer].ip_version == 1203 hash_rxq_init[i].ip_version) || 1204 (hash_rxq_init[i].ip_version == 0)) { 1205 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1206 parser->queue[i].offset); 1207 memcpy(dst, src, size); 1208 ++parser->queue[i].ibv_attr->num_of_specs; 1209 parser->queue[i].offset += size; 1210 } 1211 } 1212 } 1213 1214 /** 1215 * Convert Ethernet item to Verbs specification. 1216 * 1217 * @param item[in] 1218 * Item specification. 1219 * @param default_mask[in] 1220 * Default bit-masks to use when item->mask is not provided. 1221 * @param data[in, out] 1222 * User structure. 1223 * 1224 * @return 1225 * 0 on success, a negative errno value otherwise and rte_errno is set. 1226 */ 1227 static int 1228 mlx5_flow_create_eth(const struct rte_flow_item *item, 1229 const void *default_mask, 1230 struct mlx5_flow_data *data) 1231 { 1232 const struct rte_flow_item_eth *spec = item->spec; 1233 const struct rte_flow_item_eth *mask = item->mask; 1234 struct mlx5_flow_parse *parser = data->parser; 1235 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1236 struct ibv_flow_spec_eth eth = { 1237 .type = parser->inner | IBV_FLOW_SPEC_ETH, 1238 .size = eth_size, 1239 }; 1240 1241 /* Don't update layer for the inner pattern. */ 1242 if (!parser->inner) 1243 parser->layer = HASH_RXQ_ETH; 1244 if (spec) { 1245 unsigned int i; 1246 1247 if (!mask) 1248 mask = default_mask; 1249 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); 1250 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); 1251 eth.val.ether_type = spec->type; 1252 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); 1253 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); 1254 eth.mask.ether_type = mask->type; 1255 /* Remove unwanted bits from values. */ 1256 for (i = 0; i < ETHER_ADDR_LEN; ++i) { 1257 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 1258 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 1259 } 1260 eth.val.ether_type &= eth.mask.ether_type; 1261 } 1262 mlx5_flow_create_copy(parser, ð, eth_size); 1263 return 0; 1264 } 1265 1266 /** 1267 * Convert VLAN item to Verbs specification. 1268 * 1269 * @param item[in] 1270 * Item specification. 1271 * @param default_mask[in] 1272 * Default bit-masks to use when item->mask is not provided. 1273 * @param data[in, out] 1274 * User structure. 1275 * 1276 * @return 1277 * 0 on success, a negative errno value otherwise and rte_errno is set. 1278 */ 1279 static int 1280 mlx5_flow_create_vlan(const struct rte_flow_item *item, 1281 const void *default_mask, 1282 struct mlx5_flow_data *data) 1283 { 1284 const struct rte_flow_item_vlan *spec = item->spec; 1285 const struct rte_flow_item_vlan *mask = item->mask; 1286 struct mlx5_flow_parse *parser = data->parser; 1287 struct ibv_flow_spec_eth *eth; 1288 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1289 1290 if (spec) { 1291 unsigned int i; 1292 if (!mask) 1293 mask = default_mask; 1294 1295 for (i = 0; i != hash_rxq_init_n; ++i) { 1296 if (!parser->queue[i].ibv_attr) 1297 continue; 1298 1299 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1300 parser->queue[i].offset - eth_size); 1301 eth->val.vlan_tag = spec->tci; 1302 eth->mask.vlan_tag = mask->tci; 1303 eth->val.vlan_tag &= eth->mask.vlan_tag; 1304 /* 1305 * From verbs perspective an empty VLAN is equivalent 1306 * to a packet without VLAN layer. 1307 */ 1308 if (!eth->mask.vlan_tag) 1309 goto error; 1310 } 1311 return 0; 1312 } 1313 error: 1314 return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, 1315 item, "VLAN cannot be empty"); 1316 } 1317 1318 /** 1319 * Convert IPv4 item to Verbs specification. 1320 * 1321 * @param item[in] 1322 * Item specification. 1323 * @param default_mask[in] 1324 * Default bit-masks to use when item->mask is not provided. 1325 * @param data[in, out] 1326 * User structure. 1327 * 1328 * @return 1329 * 0 on success, a negative errno value otherwise and rte_errno is set. 1330 */ 1331 static int 1332 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 1333 const void *default_mask, 1334 struct mlx5_flow_data *data) 1335 { 1336 const struct rte_flow_item_ipv4 *spec = item->spec; 1337 const struct rte_flow_item_ipv4 *mask = item->mask; 1338 struct mlx5_flow_parse *parser = data->parser; 1339 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); 1340 struct ibv_flow_spec_ipv4_ext ipv4 = { 1341 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, 1342 .size = ipv4_size, 1343 }; 1344 1345 /* Don't update layer for the inner pattern. */ 1346 if (!parser->inner) 1347 parser->layer = HASH_RXQ_IPV4; 1348 if (spec) { 1349 if (!mask) 1350 mask = default_mask; 1351 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 1352 .src_ip = spec->hdr.src_addr, 1353 .dst_ip = spec->hdr.dst_addr, 1354 .proto = spec->hdr.next_proto_id, 1355 .tos = spec->hdr.type_of_service, 1356 }; 1357 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 1358 .src_ip = mask->hdr.src_addr, 1359 .dst_ip = mask->hdr.dst_addr, 1360 .proto = mask->hdr.next_proto_id, 1361 .tos = mask->hdr.type_of_service, 1362 }; 1363 /* Remove unwanted bits from values. */ 1364 ipv4.val.src_ip &= ipv4.mask.src_ip; 1365 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 1366 ipv4.val.proto &= ipv4.mask.proto; 1367 ipv4.val.tos &= ipv4.mask.tos; 1368 } 1369 mlx5_flow_create_copy(parser, &ipv4, ipv4_size); 1370 return 0; 1371 } 1372 1373 /** 1374 * Convert IPv6 item to Verbs specification. 1375 * 1376 * @param item[in] 1377 * Item specification. 1378 * @param default_mask[in] 1379 * Default bit-masks to use when item->mask is not provided. 1380 * @param data[in, out] 1381 * User structure. 1382 * 1383 * @return 1384 * 0 on success, a negative errno value otherwise and rte_errno is set. 1385 */ 1386 static int 1387 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 1388 const void *default_mask, 1389 struct mlx5_flow_data *data) 1390 { 1391 const struct rte_flow_item_ipv6 *spec = item->spec; 1392 const struct rte_flow_item_ipv6 *mask = item->mask; 1393 struct mlx5_flow_parse *parser = data->parser; 1394 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); 1395 struct ibv_flow_spec_ipv6 ipv6 = { 1396 .type = parser->inner | IBV_FLOW_SPEC_IPV6, 1397 .size = ipv6_size, 1398 }; 1399 1400 /* Don't update layer for the inner pattern. */ 1401 if (!parser->inner) 1402 parser->layer = HASH_RXQ_IPV6; 1403 if (spec) { 1404 unsigned int i; 1405 uint32_t vtc_flow_val; 1406 uint32_t vtc_flow_mask; 1407 1408 if (!mask) 1409 mask = default_mask; 1410 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 1411 RTE_DIM(ipv6.val.src_ip)); 1412 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 1413 RTE_DIM(ipv6.val.dst_ip)); 1414 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 1415 RTE_DIM(ipv6.mask.src_ip)); 1416 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 1417 RTE_DIM(ipv6.mask.dst_ip)); 1418 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); 1419 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); 1420 ipv6.val.flow_label = 1421 rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >> 1422 IPV6_HDR_FL_SHIFT); 1423 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >> 1424 IPV6_HDR_TC_SHIFT; 1425 ipv6.val.next_hdr = spec->hdr.proto; 1426 ipv6.val.hop_limit = spec->hdr.hop_limits; 1427 ipv6.mask.flow_label = 1428 rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >> 1429 IPV6_HDR_FL_SHIFT); 1430 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >> 1431 IPV6_HDR_TC_SHIFT; 1432 ipv6.mask.next_hdr = mask->hdr.proto; 1433 ipv6.mask.hop_limit = mask->hdr.hop_limits; 1434 /* Remove unwanted bits from values. */ 1435 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 1436 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 1437 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 1438 } 1439 ipv6.val.flow_label &= ipv6.mask.flow_label; 1440 ipv6.val.traffic_class &= ipv6.mask.traffic_class; 1441 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 1442 ipv6.val.hop_limit &= ipv6.mask.hop_limit; 1443 } 1444 mlx5_flow_create_copy(parser, &ipv6, ipv6_size); 1445 return 0; 1446 } 1447 1448 /** 1449 * Convert UDP item to Verbs specification. 1450 * 1451 * @param item[in] 1452 * Item specification. 1453 * @param default_mask[in] 1454 * Default bit-masks to use when item->mask is not provided. 1455 * @param data[in, out] 1456 * User structure. 1457 * 1458 * @return 1459 * 0 on success, a negative errno value otherwise and rte_errno is set. 1460 */ 1461 static int 1462 mlx5_flow_create_udp(const struct rte_flow_item *item, 1463 const void *default_mask, 1464 struct mlx5_flow_data *data) 1465 { 1466 const struct rte_flow_item_udp *spec = item->spec; 1467 const struct rte_flow_item_udp *mask = item->mask; 1468 struct mlx5_flow_parse *parser = data->parser; 1469 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1470 struct ibv_flow_spec_tcp_udp udp = { 1471 .type = parser->inner | IBV_FLOW_SPEC_UDP, 1472 .size = udp_size, 1473 }; 1474 1475 /* Don't update layer for the inner pattern. */ 1476 if (!parser->inner) { 1477 if (parser->layer == HASH_RXQ_IPV4) 1478 parser->layer = HASH_RXQ_UDPV4; 1479 else 1480 parser->layer = HASH_RXQ_UDPV6; 1481 } 1482 if (spec) { 1483 if (!mask) 1484 mask = default_mask; 1485 udp.val.dst_port = spec->hdr.dst_port; 1486 udp.val.src_port = spec->hdr.src_port; 1487 udp.mask.dst_port = mask->hdr.dst_port; 1488 udp.mask.src_port = mask->hdr.src_port; 1489 /* Remove unwanted bits from values. */ 1490 udp.val.src_port &= udp.mask.src_port; 1491 udp.val.dst_port &= udp.mask.dst_port; 1492 } 1493 mlx5_flow_create_copy(parser, &udp, udp_size); 1494 return 0; 1495 } 1496 1497 /** 1498 * Convert TCP item to Verbs specification. 1499 * 1500 * @param item[in] 1501 * Item specification. 1502 * @param default_mask[in] 1503 * Default bit-masks to use when item->mask is not provided. 1504 * @param data[in, out] 1505 * User structure. 1506 * 1507 * @return 1508 * 0 on success, a negative errno value otherwise and rte_errno is set. 1509 */ 1510 static int 1511 mlx5_flow_create_tcp(const struct rte_flow_item *item, 1512 const void *default_mask, 1513 struct mlx5_flow_data *data) 1514 { 1515 const struct rte_flow_item_tcp *spec = item->spec; 1516 const struct rte_flow_item_tcp *mask = item->mask; 1517 struct mlx5_flow_parse *parser = data->parser; 1518 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1519 struct ibv_flow_spec_tcp_udp tcp = { 1520 .type = parser->inner | IBV_FLOW_SPEC_TCP, 1521 .size = tcp_size, 1522 }; 1523 1524 /* Don't update layer for the inner pattern. */ 1525 if (!parser->inner) { 1526 if (parser->layer == HASH_RXQ_IPV4) 1527 parser->layer = HASH_RXQ_TCPV4; 1528 else 1529 parser->layer = HASH_RXQ_TCPV6; 1530 } 1531 if (spec) { 1532 if (!mask) 1533 mask = default_mask; 1534 tcp.val.dst_port = spec->hdr.dst_port; 1535 tcp.val.src_port = spec->hdr.src_port; 1536 tcp.mask.dst_port = mask->hdr.dst_port; 1537 tcp.mask.src_port = mask->hdr.src_port; 1538 /* Remove unwanted bits from values. */ 1539 tcp.val.src_port &= tcp.mask.src_port; 1540 tcp.val.dst_port &= tcp.mask.dst_port; 1541 } 1542 mlx5_flow_create_copy(parser, &tcp, tcp_size); 1543 return 0; 1544 } 1545 1546 /** 1547 * Convert VXLAN item to Verbs specification. 1548 * 1549 * @param item[in] 1550 * Item specification. 1551 * @param default_mask[in] 1552 * Default bit-masks to use when item->mask is not provided. 1553 * @param data[in, out] 1554 * User structure. 1555 * 1556 * @return 1557 * 0 on success, a negative errno value otherwise and rte_errno is set. 1558 */ 1559 static int 1560 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 1561 const void *default_mask, 1562 struct mlx5_flow_data *data) 1563 { 1564 const struct rte_flow_item_vxlan *spec = item->spec; 1565 const struct rte_flow_item_vxlan *mask = item->mask; 1566 struct mlx5_flow_parse *parser = data->parser; 1567 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 1568 struct ibv_flow_spec_tunnel vxlan = { 1569 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, 1570 .size = size, 1571 }; 1572 union vni { 1573 uint32_t vlan_id; 1574 uint8_t vni[4]; 1575 } id; 1576 1577 id.vni[0] = 0; 1578 parser->inner = IBV_FLOW_SPEC_INNER; 1579 if (spec) { 1580 if (!mask) 1581 mask = default_mask; 1582 memcpy(&id.vni[1], spec->vni, 3); 1583 vxlan.val.tunnel_id = id.vlan_id; 1584 memcpy(&id.vni[1], mask->vni, 3); 1585 vxlan.mask.tunnel_id = id.vlan_id; 1586 /* Remove unwanted bits from values. */ 1587 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 1588 } 1589 /* 1590 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1591 * layer is defined in the Verbs specification it is interpreted as 1592 * wildcard and all packets will match this rule, if it follows a full 1593 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1594 * before will also match this rule. 1595 * To avoid such situation, VNI 0 is currently refused. 1596 */ 1597 if (!vxlan.val.tunnel_id) 1598 return rte_flow_error_set(data->error, EINVAL, 1599 RTE_FLOW_ERROR_TYPE_ITEM, 1600 item, 1601 "VxLAN vni cannot be 0"); 1602 mlx5_flow_create_copy(parser, &vxlan, size); 1603 return 0; 1604 } 1605 1606 /** 1607 * Convert mark/flag action to Verbs specification. 1608 * 1609 * @param parser 1610 * Internal parser structure. 1611 * @param mark_id 1612 * Mark identifier. 1613 * 1614 * @return 1615 * 0 on success, a negative errno value otherwise and rte_errno is set. 1616 */ 1617 static int 1618 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) 1619 { 1620 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1621 struct ibv_flow_spec_action_tag tag = { 1622 .type = IBV_FLOW_SPEC_ACTION_TAG, 1623 .size = size, 1624 .tag_id = mlx5_flow_mark_set(mark_id), 1625 }; 1626 1627 assert(parser->mark); 1628 mlx5_flow_create_copy(parser, &tag, size); 1629 return 0; 1630 } 1631 1632 /** 1633 * Convert count action to Verbs specification. 1634 * 1635 * @param dev 1636 * Pointer to Ethernet device. 1637 * @param parser 1638 * Pointer to MLX5 flow parser structure. 1639 * 1640 * @return 1641 * 0 on success, a negative errno value otherwise and rte_errno is set. 1642 */ 1643 static int 1644 mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused, 1645 struct mlx5_flow_parse *parser __rte_unused) 1646 { 1647 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 1648 struct priv *priv = dev->data->dev_private; 1649 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1650 struct ibv_counter_set_init_attr init_attr = {0}; 1651 struct ibv_flow_spec_counter_action counter = { 1652 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1653 .size = size, 1654 .counter_set_handle = 0, 1655 }; 1656 1657 init_attr.counter_set_id = 0; 1658 parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr); 1659 if (!parser->cs) { 1660 rte_errno = EINVAL; 1661 return -rte_errno; 1662 } 1663 counter.counter_set_handle = parser->cs->handle; 1664 mlx5_flow_create_copy(parser, &counter, size); 1665 #endif 1666 return 0; 1667 } 1668 1669 /** 1670 * Complete flow rule creation with a drop queue. 1671 * 1672 * @param dev 1673 * Pointer to Ethernet device. 1674 * @param parser 1675 * Internal parser structure. 1676 * @param flow 1677 * Pointer to the rte_flow. 1678 * @param[out] error 1679 * Perform verbose error reporting if not NULL. 1680 * 1681 * @return 1682 * 0 on success, a negative errno value otherwise and rte_errno is set. 1683 */ 1684 static int 1685 mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev, 1686 struct mlx5_flow_parse *parser, 1687 struct rte_flow *flow, 1688 struct rte_flow_error *error) 1689 { 1690 struct priv *priv = dev->data->dev_private; 1691 struct ibv_flow_spec_action_drop *drop; 1692 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1693 1694 assert(priv->pd); 1695 assert(priv->ctx); 1696 flow->drop = 1; 1697 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr + 1698 parser->queue[HASH_RXQ_ETH].offset); 1699 *drop = (struct ibv_flow_spec_action_drop){ 1700 .type = IBV_FLOW_SPEC_ACTION_DROP, 1701 .size = size, 1702 }; 1703 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs; 1704 parser->queue[HASH_RXQ_ETH].offset += size; 1705 flow->frxq[HASH_RXQ_ETH].ibv_attr = 1706 parser->queue[HASH_RXQ_ETH].ibv_attr; 1707 if (parser->count) 1708 flow->cs = parser->cs; 1709 if (!priv->dev->data->dev_started) 1710 return 0; 1711 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 1712 flow->frxq[HASH_RXQ_ETH].ibv_flow = 1713 mlx5_glue->create_flow(priv->flow_drop_queue->qp, 1714 flow->frxq[HASH_RXQ_ETH].ibv_attr); 1715 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { 1716 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1717 NULL, "flow rule creation failure"); 1718 goto error; 1719 } 1720 return 0; 1721 error: 1722 assert(flow); 1723 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) { 1724 claim_zero(mlx5_glue->destroy_flow 1725 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 1726 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; 1727 } 1728 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) { 1729 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); 1730 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL; 1731 } 1732 if (flow->cs) { 1733 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 1734 flow->cs = NULL; 1735 parser->cs = NULL; 1736 } 1737 return -rte_errno; 1738 } 1739 1740 /** 1741 * Create hash Rx queues when RSS is enabled. 1742 * 1743 * @param dev 1744 * Pointer to Ethernet device. 1745 * @param parser 1746 * Internal parser structure. 1747 * @param flow 1748 * Pointer to the rte_flow. 1749 * @param[out] error 1750 * Perform verbose error reporting if not NULL. 1751 * 1752 * @return 1753 * 0 on success, a negative errno value otherwise and rte_errno is set. 1754 */ 1755 static int 1756 mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev, 1757 struct mlx5_flow_parse *parser, 1758 struct rte_flow *flow, 1759 struct rte_flow_error *error) 1760 { 1761 struct priv *priv = dev->data->dev_private; 1762 unsigned int i; 1763 1764 for (i = 0; i != hash_rxq_init_n; ++i) { 1765 uint64_t hash_fields; 1766 1767 if (!parser->queue[i].ibv_attr) 1768 continue; 1769 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; 1770 parser->queue[i].ibv_attr = NULL; 1771 hash_fields = hash_rxq_init[i].hash_fields; 1772 if (!priv->dev->data->dev_started) 1773 continue; 1774 flow->frxq[i].hrxq = 1775 mlx5_hrxq_get(dev, 1776 parser->rss_conf.key, 1777 parser->rss_conf.key_len, 1778 hash_fields, 1779 parser->rss_conf.queue, 1780 parser->rss_conf.queue_num); 1781 if (flow->frxq[i].hrxq) 1782 continue; 1783 flow->frxq[i].hrxq = 1784 mlx5_hrxq_new(dev, 1785 parser->rss_conf.key, 1786 parser->rss_conf.key_len, 1787 hash_fields, 1788 parser->rss_conf.queue, 1789 parser->rss_conf.queue_num); 1790 if (!flow->frxq[i].hrxq) { 1791 return rte_flow_error_set(error, ENOMEM, 1792 RTE_FLOW_ERROR_TYPE_HANDLE, 1793 NULL, 1794 "cannot create hash rxq"); 1795 } 1796 } 1797 return 0; 1798 } 1799 1800 /** 1801 * Complete flow rule creation. 1802 * 1803 * @param dev 1804 * Pointer to Ethernet device. 1805 * @param parser 1806 * Internal parser structure. 1807 * @param flow 1808 * Pointer to the rte_flow. 1809 * @param[out] error 1810 * Perform verbose error reporting if not NULL. 1811 * 1812 * @return 1813 * 0 on success, a negative errno value otherwise and rte_errno is set. 1814 */ 1815 static int 1816 mlx5_flow_create_action_queue(struct rte_eth_dev *dev, 1817 struct mlx5_flow_parse *parser, 1818 struct rte_flow *flow, 1819 struct rte_flow_error *error) 1820 { 1821 struct priv *priv = dev->data->dev_private; 1822 int ret; 1823 unsigned int i; 1824 unsigned int flows_n = 0; 1825 1826 assert(priv->pd); 1827 assert(priv->ctx); 1828 assert(!parser->drop); 1829 ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error); 1830 if (ret) 1831 goto error; 1832 if (parser->count) 1833 flow->cs = parser->cs; 1834 if (!priv->dev->data->dev_started) 1835 return 0; 1836 for (i = 0; i != hash_rxq_init_n; ++i) { 1837 if (!flow->frxq[i].hrxq) 1838 continue; 1839 flow->frxq[i].ibv_flow = 1840 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp, 1841 flow->frxq[i].ibv_attr); 1842 if (!flow->frxq[i].ibv_flow) { 1843 rte_flow_error_set(error, ENOMEM, 1844 RTE_FLOW_ERROR_TYPE_HANDLE, 1845 NULL, "flow rule creation failure"); 1846 goto error; 1847 } 1848 ++flows_n; 1849 DRV_LOG(DEBUG, "port %u %p type %d QP %p ibv_flow %p", 1850 dev->data->port_id, 1851 (void *)flow, i, 1852 (void *)flow->frxq[i].hrxq, 1853 (void *)flow->frxq[i].ibv_flow); 1854 } 1855 if (!flows_n) { 1856 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, 1857 NULL, "internal error in flow creation"); 1858 goto error; 1859 } 1860 for (i = 0; i != parser->rss_conf.queue_num; ++i) { 1861 struct mlx5_rxq_data *q = 1862 (*priv->rxqs)[parser->rss_conf.queue[i]]; 1863 1864 q->mark |= parser->mark; 1865 } 1866 return 0; 1867 error: 1868 ret = rte_errno; /* Save rte_errno before cleanup. */ 1869 assert(flow); 1870 for (i = 0; i != hash_rxq_init_n; ++i) { 1871 if (flow->frxq[i].ibv_flow) { 1872 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; 1873 1874 claim_zero(mlx5_glue->destroy_flow(ibv_flow)); 1875 } 1876 if (flow->frxq[i].hrxq) 1877 mlx5_hrxq_release(dev, flow->frxq[i].hrxq); 1878 if (flow->frxq[i].ibv_attr) 1879 rte_free(flow->frxq[i].ibv_attr); 1880 } 1881 if (flow->cs) { 1882 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 1883 flow->cs = NULL; 1884 parser->cs = NULL; 1885 } 1886 rte_errno = ret; /* Restore rte_errno. */ 1887 return -rte_errno; 1888 } 1889 1890 /** 1891 * Convert a flow. 1892 * 1893 * @param dev 1894 * Pointer to Ethernet device. 1895 * @param list 1896 * Pointer to a TAILQ flow list. 1897 * @param[in] attr 1898 * Flow rule attributes. 1899 * @param[in] pattern 1900 * Pattern specification (list terminated by the END pattern item). 1901 * @param[in] actions 1902 * Associated actions (list terminated by the END action). 1903 * @param[out] error 1904 * Perform verbose error reporting if not NULL. 1905 * 1906 * @return 1907 * A flow on success, NULL otherwise and rte_errno is set. 1908 */ 1909 static struct rte_flow * 1910 mlx5_flow_list_create(struct rte_eth_dev *dev, 1911 struct mlx5_flows *list, 1912 const struct rte_flow_attr *attr, 1913 const struct rte_flow_item items[], 1914 const struct rte_flow_action actions[], 1915 struct rte_flow_error *error) 1916 { 1917 struct mlx5_flow_parse parser = { .create = 1, }; 1918 struct rte_flow *flow = NULL; 1919 unsigned int i; 1920 int ret; 1921 1922 ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser); 1923 if (ret) 1924 goto exit; 1925 flow = rte_calloc(__func__, 1, 1926 sizeof(*flow) + 1927 parser.rss_conf.queue_num * sizeof(uint16_t), 1928 0); 1929 if (!flow) { 1930 rte_flow_error_set(error, ENOMEM, 1931 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1932 NULL, 1933 "cannot allocate flow memory"); 1934 return NULL; 1935 } 1936 /* Copy configuration. */ 1937 flow->queues = (uint16_t (*)[])(flow + 1); 1938 flow->rss_conf = (struct rte_flow_action_rss){ 1939 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 1940 .types = parser.rss_conf.types, 1941 .key_len = parser.rss_conf.key_len, 1942 .queue_num = parser.rss_conf.queue_num, 1943 .key = memcpy(flow->rss_key, parser.rss_conf.key, 1944 sizeof(*parser.rss_conf.key) * 1945 parser.rss_conf.key_len), 1946 .queue = memcpy(flow->queues, parser.rss_conf.queue, 1947 sizeof(*parser.rss_conf.queue) * 1948 parser.rss_conf.queue_num), 1949 }; 1950 flow->mark = parser.mark; 1951 /* finalise the flow. */ 1952 if (parser.drop) 1953 ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow, 1954 error); 1955 else 1956 ret = mlx5_flow_create_action_queue(dev, &parser, flow, error); 1957 if (ret) 1958 goto exit; 1959 TAILQ_INSERT_TAIL(list, flow, next); 1960 DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id, 1961 (void *)flow); 1962 return flow; 1963 exit: 1964 DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id, 1965 error->message); 1966 for (i = 0; i != hash_rxq_init_n; ++i) { 1967 if (parser.queue[i].ibv_attr) 1968 rte_free(parser.queue[i].ibv_attr); 1969 } 1970 rte_free(flow); 1971 return NULL; 1972 } 1973 1974 /** 1975 * Validate a flow supported by the NIC. 1976 * 1977 * @see rte_flow_validate() 1978 * @see rte_flow_ops 1979 */ 1980 int 1981 mlx5_flow_validate(struct rte_eth_dev *dev, 1982 const struct rte_flow_attr *attr, 1983 const struct rte_flow_item items[], 1984 const struct rte_flow_action actions[], 1985 struct rte_flow_error *error) 1986 { 1987 struct mlx5_flow_parse parser = { .create = 0, }; 1988 1989 return mlx5_flow_convert(dev, attr, items, actions, error, &parser); 1990 } 1991 1992 /** 1993 * Create a flow. 1994 * 1995 * @see rte_flow_create() 1996 * @see rte_flow_ops 1997 */ 1998 struct rte_flow * 1999 mlx5_flow_create(struct rte_eth_dev *dev, 2000 const struct rte_flow_attr *attr, 2001 const struct rte_flow_item items[], 2002 const struct rte_flow_action actions[], 2003 struct rte_flow_error *error) 2004 { 2005 struct priv *priv = dev->data->dev_private; 2006 2007 return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions, 2008 error); 2009 } 2010 2011 /** 2012 * Destroy a flow in a list. 2013 * 2014 * @param dev 2015 * Pointer to Ethernet device. 2016 * @param list 2017 * Pointer to a TAILQ flow list. 2018 * @param[in] flow 2019 * Flow to destroy. 2020 */ 2021 static void 2022 mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2023 struct rte_flow *flow) 2024 { 2025 struct priv *priv = dev->data->dev_private; 2026 unsigned int i; 2027 2028 if (flow->drop || !flow->mark) 2029 goto free; 2030 for (i = 0; i != flow->rss_conf.queue_num; ++i) { 2031 struct rte_flow *tmp; 2032 int mark = 0; 2033 2034 /* 2035 * To remove the mark from the queue, the queue must not be 2036 * present in any other marked flow (RSS or not). 2037 */ 2038 TAILQ_FOREACH(tmp, list, next) { 2039 unsigned int j; 2040 uint16_t *tqs = NULL; 2041 uint16_t tq_n = 0; 2042 2043 if (!tmp->mark) 2044 continue; 2045 for (j = 0; j != hash_rxq_init_n; ++j) { 2046 if (!tmp->frxq[j].hrxq) 2047 continue; 2048 tqs = tmp->frxq[j].hrxq->ind_table->queues; 2049 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n; 2050 } 2051 if (!tq_n) 2052 continue; 2053 for (j = 0; (j != tq_n) && !mark; j++) 2054 if (tqs[j] == (*flow->queues)[i]) 2055 mark = 1; 2056 } 2057 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark; 2058 } 2059 free: 2060 if (flow->drop) { 2061 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) 2062 claim_zero(mlx5_glue->destroy_flow 2063 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 2064 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); 2065 } else { 2066 for (i = 0; i != hash_rxq_init_n; ++i) { 2067 struct mlx5_flow *frxq = &flow->frxq[i]; 2068 2069 if (frxq->ibv_flow) 2070 claim_zero(mlx5_glue->destroy_flow 2071 (frxq->ibv_flow)); 2072 if (frxq->hrxq) 2073 mlx5_hrxq_release(dev, frxq->hrxq); 2074 if (frxq->ibv_attr) 2075 rte_free(frxq->ibv_attr); 2076 } 2077 } 2078 if (flow->cs) { 2079 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 2080 flow->cs = NULL; 2081 } 2082 TAILQ_REMOVE(list, flow, next); 2083 DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id, 2084 (void *)flow); 2085 rte_free(flow); 2086 } 2087 2088 /** 2089 * Destroy all flows. 2090 * 2091 * @param dev 2092 * Pointer to Ethernet device. 2093 * @param list 2094 * Pointer to a TAILQ flow list. 2095 */ 2096 void 2097 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 2098 { 2099 while (!TAILQ_EMPTY(list)) { 2100 struct rte_flow *flow; 2101 2102 flow = TAILQ_FIRST(list); 2103 mlx5_flow_list_destroy(dev, list, flow); 2104 } 2105 } 2106 2107 /** 2108 * Create drop queue. 2109 * 2110 * @param dev 2111 * Pointer to Ethernet device. 2112 * 2113 * @return 2114 * 0 on success, a negative errno value otherwise and rte_errno is set. 2115 */ 2116 int 2117 mlx5_flow_create_drop_queue(struct rte_eth_dev *dev) 2118 { 2119 struct priv *priv = dev->data->dev_private; 2120 struct mlx5_hrxq_drop *fdq = NULL; 2121 2122 assert(priv->pd); 2123 assert(priv->ctx); 2124 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); 2125 if (!fdq) { 2126 DRV_LOG(WARNING, 2127 "port %u cannot allocate memory for drop queue", 2128 dev->data->port_id); 2129 rte_errno = ENOMEM; 2130 return -rte_errno; 2131 } 2132 fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0); 2133 if (!fdq->cq) { 2134 DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue", 2135 dev->data->port_id); 2136 rte_errno = errno; 2137 goto error; 2138 } 2139 fdq->wq = mlx5_glue->create_wq 2140 (priv->ctx, 2141 &(struct ibv_wq_init_attr){ 2142 .wq_type = IBV_WQT_RQ, 2143 .max_wr = 1, 2144 .max_sge = 1, 2145 .pd = priv->pd, 2146 .cq = fdq->cq, 2147 }); 2148 if (!fdq->wq) { 2149 DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue", 2150 dev->data->port_id); 2151 rte_errno = errno; 2152 goto error; 2153 } 2154 fdq->ind_table = mlx5_glue->create_rwq_ind_table 2155 (priv->ctx, 2156 &(struct ibv_rwq_ind_table_init_attr){ 2157 .log_ind_tbl_size = 0, 2158 .ind_tbl = &fdq->wq, 2159 .comp_mask = 0, 2160 }); 2161 if (!fdq->ind_table) { 2162 DRV_LOG(WARNING, 2163 "port %u cannot allocate indirection table for drop" 2164 " queue", 2165 dev->data->port_id); 2166 rte_errno = errno; 2167 goto error; 2168 } 2169 fdq->qp = mlx5_glue->create_qp_ex 2170 (priv->ctx, 2171 &(struct ibv_qp_init_attr_ex){ 2172 .qp_type = IBV_QPT_RAW_PACKET, 2173 .comp_mask = 2174 IBV_QP_INIT_ATTR_PD | 2175 IBV_QP_INIT_ATTR_IND_TABLE | 2176 IBV_QP_INIT_ATTR_RX_HASH, 2177 .rx_hash_conf = (struct ibv_rx_hash_conf){ 2178 .rx_hash_function = 2179 IBV_RX_HASH_FUNC_TOEPLITZ, 2180 .rx_hash_key_len = rss_hash_default_key_len, 2181 .rx_hash_key = rss_hash_default_key, 2182 .rx_hash_fields_mask = 0, 2183 }, 2184 .rwq_ind_tbl = fdq->ind_table, 2185 .pd = priv->pd 2186 }); 2187 if (!fdq->qp) { 2188 DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue", 2189 dev->data->port_id); 2190 rte_errno = errno; 2191 goto error; 2192 } 2193 priv->flow_drop_queue = fdq; 2194 return 0; 2195 error: 2196 if (fdq->qp) 2197 claim_zero(mlx5_glue->destroy_qp(fdq->qp)); 2198 if (fdq->ind_table) 2199 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table)); 2200 if (fdq->wq) 2201 claim_zero(mlx5_glue->destroy_wq(fdq->wq)); 2202 if (fdq->cq) 2203 claim_zero(mlx5_glue->destroy_cq(fdq->cq)); 2204 if (fdq) 2205 rte_free(fdq); 2206 priv->flow_drop_queue = NULL; 2207 return -rte_errno; 2208 } 2209 2210 /** 2211 * Delete drop queue. 2212 * 2213 * @param dev 2214 * Pointer to Ethernet device. 2215 */ 2216 void 2217 mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev) 2218 { 2219 struct priv *priv = dev->data->dev_private; 2220 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; 2221 2222 if (!fdq) 2223 return; 2224 if (fdq->qp) 2225 claim_zero(mlx5_glue->destroy_qp(fdq->qp)); 2226 if (fdq->ind_table) 2227 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table)); 2228 if (fdq->wq) 2229 claim_zero(mlx5_glue->destroy_wq(fdq->wq)); 2230 if (fdq->cq) 2231 claim_zero(mlx5_glue->destroy_cq(fdq->cq)); 2232 rte_free(fdq); 2233 priv->flow_drop_queue = NULL; 2234 } 2235 2236 /** 2237 * Remove all flows. 2238 * 2239 * @param dev 2240 * Pointer to Ethernet device. 2241 * @param list 2242 * Pointer to a TAILQ flow list. 2243 */ 2244 void 2245 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 2246 { 2247 struct priv *priv = dev->data->dev_private; 2248 struct rte_flow *flow; 2249 2250 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 2251 unsigned int i; 2252 struct mlx5_ind_table_ibv *ind_tbl = NULL; 2253 2254 if (flow->drop) { 2255 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) 2256 continue; 2257 claim_zero(mlx5_glue->destroy_flow 2258 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 2259 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; 2260 DRV_LOG(DEBUG, "port %u flow %p removed", 2261 dev->data->port_id, (void *)flow); 2262 /* Next flow. */ 2263 continue; 2264 } 2265 /* Verify the flow has not already been cleaned. */ 2266 for (i = 0; i != hash_rxq_init_n; ++i) { 2267 if (!flow->frxq[i].ibv_flow) 2268 continue; 2269 /* 2270 * Indirection table may be necessary to remove the 2271 * flags in the Rx queues. 2272 * This helps to speed-up the process by avoiding 2273 * another loop. 2274 */ 2275 ind_tbl = flow->frxq[i].hrxq->ind_table; 2276 break; 2277 } 2278 if (i == hash_rxq_init_n) 2279 return; 2280 if (flow->mark) { 2281 assert(ind_tbl); 2282 for (i = 0; i != ind_tbl->queues_n; ++i) 2283 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; 2284 } 2285 for (i = 0; i != hash_rxq_init_n; ++i) { 2286 if (!flow->frxq[i].ibv_flow) 2287 continue; 2288 claim_zero(mlx5_glue->destroy_flow 2289 (flow->frxq[i].ibv_flow)); 2290 flow->frxq[i].ibv_flow = NULL; 2291 mlx5_hrxq_release(dev, flow->frxq[i].hrxq); 2292 flow->frxq[i].hrxq = NULL; 2293 } 2294 DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id, 2295 (void *)flow); 2296 } 2297 } 2298 2299 /** 2300 * Add all flows. 2301 * 2302 * @param dev 2303 * Pointer to Ethernet device. 2304 * @param list 2305 * Pointer to a TAILQ flow list. 2306 * 2307 * @return 2308 * 0 on success, a negative errno value otherwise and rte_errno is set. 2309 */ 2310 int 2311 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 2312 { 2313 struct priv *priv = dev->data->dev_private; 2314 struct rte_flow *flow; 2315 2316 TAILQ_FOREACH(flow, list, next) { 2317 unsigned int i; 2318 2319 if (flow->drop) { 2320 flow->frxq[HASH_RXQ_ETH].ibv_flow = 2321 mlx5_glue->create_flow 2322 (priv->flow_drop_queue->qp, 2323 flow->frxq[HASH_RXQ_ETH].ibv_attr); 2324 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { 2325 DRV_LOG(DEBUG, 2326 "port %u flow %p cannot be applied", 2327 dev->data->port_id, (void *)flow); 2328 rte_errno = EINVAL; 2329 return -rte_errno; 2330 } 2331 DRV_LOG(DEBUG, "port %u flow %p applied", 2332 dev->data->port_id, (void *)flow); 2333 /* Next flow. */ 2334 continue; 2335 } 2336 for (i = 0; i != hash_rxq_init_n; ++i) { 2337 if (!flow->frxq[i].ibv_attr) 2338 continue; 2339 flow->frxq[i].hrxq = 2340 mlx5_hrxq_get(dev, flow->rss_conf.key, 2341 flow->rss_conf.key_len, 2342 hash_rxq_init[i].hash_fields, 2343 flow->rss_conf.queue, 2344 flow->rss_conf.queue_num); 2345 if (flow->frxq[i].hrxq) 2346 goto flow_create; 2347 flow->frxq[i].hrxq = 2348 mlx5_hrxq_new(dev, flow->rss_conf.key, 2349 flow->rss_conf.key_len, 2350 hash_rxq_init[i].hash_fields, 2351 flow->rss_conf.queue, 2352 flow->rss_conf.queue_num); 2353 if (!flow->frxq[i].hrxq) { 2354 DRV_LOG(DEBUG, 2355 "port %u flow %p cannot be applied", 2356 dev->data->port_id, (void *)flow); 2357 rte_errno = EINVAL; 2358 return -rte_errno; 2359 } 2360 flow_create: 2361 flow->frxq[i].ibv_flow = 2362 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp, 2363 flow->frxq[i].ibv_attr); 2364 if (!flow->frxq[i].ibv_flow) { 2365 DRV_LOG(DEBUG, 2366 "port %u flow %p cannot be applied", 2367 dev->data->port_id, (void *)flow); 2368 rte_errno = EINVAL; 2369 return -rte_errno; 2370 } 2371 DRV_LOG(DEBUG, "port %u flow %p applied", 2372 dev->data->port_id, (void *)flow); 2373 } 2374 if (!flow->mark) 2375 continue; 2376 for (i = 0; i != flow->rss_conf.queue_num; ++i) 2377 (*priv->rxqs)[flow->rss_conf.queue[i]]->mark = 1; 2378 } 2379 return 0; 2380 } 2381 2382 /** 2383 * Verify the flow list is empty 2384 * 2385 * @param dev 2386 * Pointer to Ethernet device. 2387 * 2388 * @return the number of flows not released. 2389 */ 2390 int 2391 mlx5_flow_verify(struct rte_eth_dev *dev) 2392 { 2393 struct priv *priv = dev->data->dev_private; 2394 struct rte_flow *flow; 2395 int ret = 0; 2396 2397 TAILQ_FOREACH(flow, &priv->flows, next) { 2398 DRV_LOG(DEBUG, "port %u flow %p still referenced", 2399 dev->data->port_id, (void *)flow); 2400 ++ret; 2401 } 2402 return ret; 2403 } 2404 2405 /** 2406 * Enable a control flow configured from the control plane. 2407 * 2408 * @param dev 2409 * Pointer to Ethernet device. 2410 * @param eth_spec 2411 * An Ethernet flow spec to apply. 2412 * @param eth_mask 2413 * An Ethernet flow mask to apply. 2414 * @param vlan_spec 2415 * A VLAN flow spec to apply. 2416 * @param vlan_mask 2417 * A VLAN flow mask to apply. 2418 * 2419 * @return 2420 * 0 on success, a negative errno value otherwise and rte_errno is set. 2421 */ 2422 int 2423 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2424 struct rte_flow_item_eth *eth_spec, 2425 struct rte_flow_item_eth *eth_mask, 2426 struct rte_flow_item_vlan *vlan_spec, 2427 struct rte_flow_item_vlan *vlan_mask) 2428 { 2429 struct priv *priv = dev->data->dev_private; 2430 const struct rte_flow_attr attr = { 2431 .ingress = 1, 2432 .priority = MLX5_CTRL_FLOW_PRIORITY, 2433 }; 2434 struct rte_flow_item items[] = { 2435 { 2436 .type = RTE_FLOW_ITEM_TYPE_ETH, 2437 .spec = eth_spec, 2438 .last = NULL, 2439 .mask = eth_mask, 2440 }, 2441 { 2442 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2443 RTE_FLOW_ITEM_TYPE_END, 2444 .spec = vlan_spec, 2445 .last = NULL, 2446 .mask = vlan_mask, 2447 }, 2448 { 2449 .type = RTE_FLOW_ITEM_TYPE_END, 2450 }, 2451 }; 2452 uint16_t queue[priv->reta_idx_n]; 2453 struct rte_flow_action_rss action_rss = { 2454 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 2455 .types = priv->rss_conf.rss_hf, 2456 .key_len = priv->rss_conf.rss_key_len, 2457 .queue_num = priv->reta_idx_n, 2458 .key = priv->rss_conf.rss_key, 2459 .queue = queue, 2460 }; 2461 struct rte_flow_action actions[] = { 2462 { 2463 .type = RTE_FLOW_ACTION_TYPE_RSS, 2464 .conf = &action_rss, 2465 }, 2466 { 2467 .type = RTE_FLOW_ACTION_TYPE_END, 2468 }, 2469 }; 2470 struct rte_flow *flow; 2471 struct rte_flow_error error; 2472 unsigned int i; 2473 2474 if (!priv->reta_idx_n) { 2475 rte_errno = EINVAL; 2476 return -rte_errno; 2477 } 2478 for (i = 0; i != priv->reta_idx_n; ++i) 2479 queue[i] = (*priv->reta_idx)[i]; 2480 flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items, 2481 actions, &error); 2482 if (!flow) 2483 return -rte_errno; 2484 return 0; 2485 } 2486 2487 /** 2488 * Enable a flow control configured from the control plane. 2489 * 2490 * @param dev 2491 * Pointer to Ethernet device. 2492 * @param eth_spec 2493 * An Ethernet flow spec to apply. 2494 * @param eth_mask 2495 * An Ethernet flow mask to apply. 2496 * 2497 * @return 2498 * 0 on success, a negative errno value otherwise and rte_errno is set. 2499 */ 2500 int 2501 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2502 struct rte_flow_item_eth *eth_spec, 2503 struct rte_flow_item_eth *eth_mask) 2504 { 2505 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2506 } 2507 2508 /** 2509 * Destroy a flow. 2510 * 2511 * @see rte_flow_destroy() 2512 * @see rte_flow_ops 2513 */ 2514 int 2515 mlx5_flow_destroy(struct rte_eth_dev *dev, 2516 struct rte_flow *flow, 2517 struct rte_flow_error *error __rte_unused) 2518 { 2519 struct priv *priv = dev->data->dev_private; 2520 2521 mlx5_flow_list_destroy(dev, &priv->flows, flow); 2522 return 0; 2523 } 2524 2525 /** 2526 * Destroy all flows. 2527 * 2528 * @see rte_flow_flush() 2529 * @see rte_flow_ops 2530 */ 2531 int 2532 mlx5_flow_flush(struct rte_eth_dev *dev, 2533 struct rte_flow_error *error __rte_unused) 2534 { 2535 struct priv *priv = dev->data->dev_private; 2536 2537 mlx5_flow_list_flush(dev, &priv->flows); 2538 return 0; 2539 } 2540 2541 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 2542 /** 2543 * Query flow counter. 2544 * 2545 * @param cs 2546 * the counter set. 2547 * @param counter_value 2548 * returned data from the counter. 2549 * 2550 * @return 2551 * 0 on success, a negative errno value otherwise and rte_errno is set. 2552 */ 2553 static int 2554 mlx5_flow_query_count(struct ibv_counter_set *cs, 2555 struct mlx5_flow_counter_stats *counter_stats, 2556 struct rte_flow_query_count *query_count, 2557 struct rte_flow_error *error) 2558 { 2559 uint64_t counters[2]; 2560 struct ibv_query_counter_set_attr query_cs_attr = { 2561 .cs = cs, 2562 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 2563 }; 2564 struct ibv_counter_set_data query_out = { 2565 .out = counters, 2566 .outlen = 2 * sizeof(uint64_t), 2567 }; 2568 int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out); 2569 2570 if (err) 2571 return rte_flow_error_set(error, err, 2572 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2573 NULL, 2574 "cannot read counter"); 2575 query_count->hits_set = 1; 2576 query_count->bytes_set = 1; 2577 query_count->hits = counters[0] - counter_stats->hits; 2578 query_count->bytes = counters[1] - counter_stats->bytes; 2579 if (query_count->reset) { 2580 counter_stats->hits = counters[0]; 2581 counter_stats->bytes = counters[1]; 2582 } 2583 return 0; 2584 } 2585 2586 /** 2587 * Query a flows. 2588 * 2589 * @see rte_flow_query() 2590 * @see rte_flow_ops 2591 */ 2592 int 2593 mlx5_flow_query(struct rte_eth_dev *dev __rte_unused, 2594 struct rte_flow *flow, 2595 enum rte_flow_action_type action __rte_unused, 2596 void *data, 2597 struct rte_flow_error *error) 2598 { 2599 if (flow->cs) { 2600 int ret; 2601 2602 ret = mlx5_flow_query_count(flow->cs, 2603 &flow->counter_stats, 2604 (struct rte_flow_query_count *)data, 2605 error); 2606 if (ret) 2607 return ret; 2608 } else { 2609 return rte_flow_error_set(error, EINVAL, 2610 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2611 NULL, 2612 "no counter found for flow"); 2613 } 2614 return 0; 2615 } 2616 #endif 2617 2618 /** 2619 * Isolated mode. 2620 * 2621 * @see rte_flow_isolate() 2622 * @see rte_flow_ops 2623 */ 2624 int 2625 mlx5_flow_isolate(struct rte_eth_dev *dev, 2626 int enable, 2627 struct rte_flow_error *error) 2628 { 2629 struct priv *priv = dev->data->dev_private; 2630 2631 if (dev->data->dev_started) { 2632 rte_flow_error_set(error, EBUSY, 2633 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2634 NULL, 2635 "port must be stopped first"); 2636 return -rte_errno; 2637 } 2638 priv->isolated = !!enable; 2639 if (enable) 2640 priv->dev->dev_ops = &mlx5_dev_ops_isolate; 2641 else 2642 priv->dev->dev_ops = &mlx5_dev_ops; 2643 return 0; 2644 } 2645 2646 /** 2647 * Convert a flow director filter to a generic flow. 2648 * 2649 * @param dev 2650 * Pointer to Ethernet device. 2651 * @param fdir_filter 2652 * Flow director filter to add. 2653 * @param attributes 2654 * Generic flow parameters structure. 2655 * 2656 * @return 2657 * 0 on success, a negative errno value otherwise and rte_errno is set. 2658 */ 2659 static int 2660 mlx5_fdir_filter_convert(struct rte_eth_dev *dev, 2661 const struct rte_eth_fdir_filter *fdir_filter, 2662 struct mlx5_fdir *attributes) 2663 { 2664 struct priv *priv = dev->data->dev_private; 2665 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2666 const struct rte_eth_fdir_masks *mask = 2667 &dev->data->dev_conf.fdir_conf.mask; 2668 2669 /* Validate queue number. */ 2670 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2671 DRV_LOG(ERR, "port %u invalid queue number %d", 2672 dev->data->port_id, fdir_filter->action.rx_queue); 2673 rte_errno = EINVAL; 2674 return -rte_errno; 2675 } 2676 attributes->attr.ingress = 1; 2677 attributes->items[0] = (struct rte_flow_item) { 2678 .type = RTE_FLOW_ITEM_TYPE_ETH, 2679 .spec = &attributes->l2, 2680 .mask = &attributes->l2_mask, 2681 }; 2682 switch (fdir_filter->action.behavior) { 2683 case RTE_ETH_FDIR_ACCEPT: 2684 attributes->actions[0] = (struct rte_flow_action){ 2685 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2686 .conf = &attributes->queue, 2687 }; 2688 break; 2689 case RTE_ETH_FDIR_REJECT: 2690 attributes->actions[0] = (struct rte_flow_action){ 2691 .type = RTE_FLOW_ACTION_TYPE_DROP, 2692 }; 2693 break; 2694 default: 2695 DRV_LOG(ERR, "port %u invalid behavior %d", 2696 dev->data->port_id, 2697 fdir_filter->action.behavior); 2698 rte_errno = ENOTSUP; 2699 return -rte_errno; 2700 } 2701 attributes->queue.index = fdir_filter->action.rx_queue; 2702 /* Handle L3. */ 2703 switch (fdir_filter->input.flow_type) { 2704 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2705 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2706 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2707 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2708 .src_addr = input->flow.ip4_flow.src_ip, 2709 .dst_addr = input->flow.ip4_flow.dst_ip, 2710 .time_to_live = input->flow.ip4_flow.ttl, 2711 .type_of_service = input->flow.ip4_flow.tos, 2712 .next_proto_id = input->flow.ip4_flow.proto, 2713 }; 2714 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){ 2715 .src_addr = mask->ipv4_mask.src_ip, 2716 .dst_addr = mask->ipv4_mask.dst_ip, 2717 .time_to_live = mask->ipv4_mask.ttl, 2718 .type_of_service = mask->ipv4_mask.tos, 2719 .next_proto_id = mask->ipv4_mask.proto, 2720 }; 2721 attributes->items[1] = (struct rte_flow_item){ 2722 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2723 .spec = &attributes->l3, 2724 .mask = &attributes->l3_mask, 2725 }; 2726 break; 2727 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2728 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2729 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2730 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2731 .hop_limits = input->flow.ipv6_flow.hop_limits, 2732 .proto = input->flow.ipv6_flow.proto, 2733 }; 2734 2735 memcpy(attributes->l3.ipv6.hdr.src_addr, 2736 input->flow.ipv6_flow.src_ip, 2737 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2738 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2739 input->flow.ipv6_flow.dst_ip, 2740 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2741 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 2742 mask->ipv6_mask.src_ip, 2743 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2744 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 2745 mask->ipv6_mask.dst_ip, 2746 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2747 attributes->items[1] = (struct rte_flow_item){ 2748 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2749 .spec = &attributes->l3, 2750 .mask = &attributes->l3_mask, 2751 }; 2752 break; 2753 default: 2754 DRV_LOG(ERR, "port %u invalid flow type%d", 2755 dev->data->port_id, fdir_filter->input.flow_type); 2756 rte_errno = ENOTSUP; 2757 return -rte_errno; 2758 } 2759 /* Handle L4. */ 2760 switch (fdir_filter->input.flow_type) { 2761 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2762 attributes->l4.udp.hdr = (struct udp_hdr){ 2763 .src_port = input->flow.udp4_flow.src_port, 2764 .dst_port = input->flow.udp4_flow.dst_port, 2765 }; 2766 attributes->l4_mask.udp.hdr = (struct udp_hdr){ 2767 .src_port = mask->src_port_mask, 2768 .dst_port = mask->dst_port_mask, 2769 }; 2770 attributes->items[2] = (struct rte_flow_item){ 2771 .type = RTE_FLOW_ITEM_TYPE_UDP, 2772 .spec = &attributes->l4, 2773 .mask = &attributes->l4_mask, 2774 }; 2775 break; 2776 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2777 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2778 .src_port = input->flow.tcp4_flow.src_port, 2779 .dst_port = input->flow.tcp4_flow.dst_port, 2780 }; 2781 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ 2782 .src_port = mask->src_port_mask, 2783 .dst_port = mask->dst_port_mask, 2784 }; 2785 attributes->items[2] = (struct rte_flow_item){ 2786 .type = RTE_FLOW_ITEM_TYPE_TCP, 2787 .spec = &attributes->l4, 2788 .mask = &attributes->l4_mask, 2789 }; 2790 break; 2791 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2792 attributes->l4.udp.hdr = (struct udp_hdr){ 2793 .src_port = input->flow.udp6_flow.src_port, 2794 .dst_port = input->flow.udp6_flow.dst_port, 2795 }; 2796 attributes->l4_mask.udp.hdr = (struct udp_hdr){ 2797 .src_port = mask->src_port_mask, 2798 .dst_port = mask->dst_port_mask, 2799 }; 2800 attributes->items[2] = (struct rte_flow_item){ 2801 .type = RTE_FLOW_ITEM_TYPE_UDP, 2802 .spec = &attributes->l4, 2803 .mask = &attributes->l4_mask, 2804 }; 2805 break; 2806 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2807 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2808 .src_port = input->flow.tcp6_flow.src_port, 2809 .dst_port = input->flow.tcp6_flow.dst_port, 2810 }; 2811 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ 2812 .src_port = mask->src_port_mask, 2813 .dst_port = mask->dst_port_mask, 2814 }; 2815 attributes->items[2] = (struct rte_flow_item){ 2816 .type = RTE_FLOW_ITEM_TYPE_TCP, 2817 .spec = &attributes->l4, 2818 .mask = &attributes->l4_mask, 2819 }; 2820 break; 2821 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2822 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2823 break; 2824 default: 2825 DRV_LOG(ERR, "port %u invalid flow type%d", 2826 dev->data->port_id, fdir_filter->input.flow_type); 2827 rte_errno = ENOTSUP; 2828 return -rte_errno; 2829 } 2830 return 0; 2831 } 2832 2833 /** 2834 * Add new flow director filter and store it in list. 2835 * 2836 * @param dev 2837 * Pointer to Ethernet device. 2838 * @param fdir_filter 2839 * Flow director filter to add. 2840 * 2841 * @return 2842 * 0 on success, a negative errno value otherwise and rte_errno is set. 2843 */ 2844 static int 2845 mlx5_fdir_filter_add(struct rte_eth_dev *dev, 2846 const struct rte_eth_fdir_filter *fdir_filter) 2847 { 2848 struct priv *priv = dev->data->dev_private; 2849 struct mlx5_fdir attributes = { 2850 .attr.group = 0, 2851 .l2_mask = { 2852 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2853 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2854 .type = 0, 2855 }, 2856 }; 2857 struct mlx5_flow_parse parser = { 2858 .layer = HASH_RXQ_ETH, 2859 }; 2860 struct rte_flow_error error; 2861 struct rte_flow *flow; 2862 int ret; 2863 2864 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); 2865 if (ret) 2866 return ret; 2867 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, 2868 attributes.actions, &error, &parser); 2869 if (ret) 2870 return ret; 2871 flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr, 2872 attributes.items, attributes.actions, 2873 &error); 2874 if (flow) { 2875 DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id, 2876 (void *)flow); 2877 return 0; 2878 } 2879 return -rte_errno; 2880 } 2881 2882 /** 2883 * Delete specific filter. 2884 * 2885 * @param dev 2886 * Pointer to Ethernet device. 2887 * @param fdir_filter 2888 * Filter to be deleted. 2889 * 2890 * @return 2891 * 0 on success, a negative errno value otherwise and rte_errno is set. 2892 */ 2893 static int 2894 mlx5_fdir_filter_delete(struct rte_eth_dev *dev, 2895 const struct rte_eth_fdir_filter *fdir_filter) 2896 { 2897 struct priv *priv = dev->data->dev_private; 2898 struct mlx5_fdir attributes = { 2899 .attr.group = 0, 2900 }; 2901 struct mlx5_flow_parse parser = { 2902 .create = 1, 2903 .layer = HASH_RXQ_ETH, 2904 }; 2905 struct rte_flow_error error; 2906 struct rte_flow *flow; 2907 unsigned int i; 2908 int ret; 2909 2910 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); 2911 if (ret) 2912 return ret; 2913 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, 2914 attributes.actions, &error, &parser); 2915 if (ret) 2916 goto exit; 2917 /* 2918 * Special case for drop action which is only set in the 2919 * specifications when the flow is created. In this situation the 2920 * drop specification is missing. 2921 */ 2922 if (parser.drop) { 2923 struct ibv_flow_spec_action_drop *drop; 2924 2925 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr + 2926 parser.queue[HASH_RXQ_ETH].offset); 2927 *drop = (struct ibv_flow_spec_action_drop){ 2928 .type = IBV_FLOW_SPEC_ACTION_DROP, 2929 .size = sizeof(struct ibv_flow_spec_action_drop), 2930 }; 2931 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++; 2932 } 2933 TAILQ_FOREACH(flow, &priv->flows, next) { 2934 struct ibv_flow_attr *attr; 2935 struct ibv_spec_header *attr_h; 2936 void *spec; 2937 struct ibv_flow_attr *flow_attr; 2938 struct ibv_spec_header *flow_h; 2939 void *flow_spec; 2940 unsigned int specs_n; 2941 2942 attr = parser.queue[HASH_RXQ_ETH].ibv_attr; 2943 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; 2944 /* Compare first the attributes. */ 2945 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) 2946 continue; 2947 if (attr->num_of_specs == 0) 2948 continue; 2949 spec = (void *)((uintptr_t)attr + 2950 sizeof(struct ibv_flow_attr)); 2951 flow_spec = (void *)((uintptr_t)flow_attr + 2952 sizeof(struct ibv_flow_attr)); 2953 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs); 2954 for (i = 0; i != specs_n; ++i) { 2955 attr_h = spec; 2956 flow_h = flow_spec; 2957 if (memcmp(spec, flow_spec, 2958 RTE_MIN(attr_h->size, flow_h->size))) 2959 goto wrong_flow; 2960 spec = (void *)((uintptr_t)spec + attr_h->size); 2961 flow_spec = (void *)((uintptr_t)flow_spec + 2962 flow_h->size); 2963 } 2964 /* At this point, the flow match. */ 2965 break; 2966 wrong_flow: 2967 /* The flow does not match. */ 2968 continue; 2969 } 2970 ret = rte_errno; /* Save rte_errno before cleanup. */ 2971 if (flow) 2972 mlx5_flow_list_destroy(dev, &priv->flows, flow); 2973 exit: 2974 for (i = 0; i != hash_rxq_init_n; ++i) { 2975 if (parser.queue[i].ibv_attr) 2976 rte_free(parser.queue[i].ibv_attr); 2977 } 2978 rte_errno = ret; /* Restore rte_errno. */ 2979 return -rte_errno; 2980 } 2981 2982 /** 2983 * Update queue for specific filter. 2984 * 2985 * @param dev 2986 * Pointer to Ethernet device. 2987 * @param fdir_filter 2988 * Filter to be updated. 2989 * 2990 * @return 2991 * 0 on success, a negative errno value otherwise and rte_errno is set. 2992 */ 2993 static int 2994 mlx5_fdir_filter_update(struct rte_eth_dev *dev, 2995 const struct rte_eth_fdir_filter *fdir_filter) 2996 { 2997 int ret; 2998 2999 ret = mlx5_fdir_filter_delete(dev, fdir_filter); 3000 if (ret) 3001 return ret; 3002 return mlx5_fdir_filter_add(dev, fdir_filter); 3003 } 3004 3005 /** 3006 * Flush all filters. 3007 * 3008 * @param dev 3009 * Pointer to Ethernet device. 3010 */ 3011 static void 3012 mlx5_fdir_filter_flush(struct rte_eth_dev *dev) 3013 { 3014 struct priv *priv = dev->data->dev_private; 3015 3016 mlx5_flow_list_flush(dev, &priv->flows); 3017 } 3018 3019 /** 3020 * Get flow director information. 3021 * 3022 * @param dev 3023 * Pointer to Ethernet device. 3024 * @param[out] fdir_info 3025 * Resulting flow director information. 3026 */ 3027 static void 3028 mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 3029 { 3030 struct priv *priv = dev->data->dev_private; 3031 struct rte_eth_fdir_masks *mask = 3032 &priv->dev->data->dev_conf.fdir_conf.mask; 3033 3034 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; 3035 fdir_info->guarant_spc = 0; 3036 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 3037 fdir_info->max_flexpayload = 0; 3038 fdir_info->flow_types_mask[0] = 0; 3039 fdir_info->flex_payload_unit = 0; 3040 fdir_info->max_flex_payload_segment_num = 0; 3041 fdir_info->flex_payload_limit = 0; 3042 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 3043 } 3044 3045 /** 3046 * Deal with flow director operations. 3047 * 3048 * @param dev 3049 * Pointer to Ethernet device. 3050 * @param filter_op 3051 * Operation to perform. 3052 * @param arg 3053 * Pointer to operation-specific structure. 3054 * 3055 * @return 3056 * 0 on success, a negative errno value otherwise and rte_errno is set. 3057 */ 3058 static int 3059 mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 3060 void *arg) 3061 { 3062 struct priv *priv = dev->data->dev_private; 3063 enum rte_fdir_mode fdir_mode = 3064 priv->dev->data->dev_conf.fdir_conf.mode; 3065 3066 if (filter_op == RTE_ETH_FILTER_NOP) 3067 return 0; 3068 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 3069 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 3070 DRV_LOG(ERR, "port %u flow director mode %d not supported", 3071 dev->data->port_id, fdir_mode); 3072 rte_errno = EINVAL; 3073 return -rte_errno; 3074 } 3075 switch (filter_op) { 3076 case RTE_ETH_FILTER_ADD: 3077 return mlx5_fdir_filter_add(dev, arg); 3078 case RTE_ETH_FILTER_UPDATE: 3079 return mlx5_fdir_filter_update(dev, arg); 3080 case RTE_ETH_FILTER_DELETE: 3081 return mlx5_fdir_filter_delete(dev, arg); 3082 case RTE_ETH_FILTER_FLUSH: 3083 mlx5_fdir_filter_flush(dev); 3084 break; 3085 case RTE_ETH_FILTER_INFO: 3086 mlx5_fdir_info_get(dev, arg); 3087 break; 3088 default: 3089 DRV_LOG(DEBUG, "port %u unknown operation %u", 3090 dev->data->port_id, filter_op); 3091 rte_errno = EINVAL; 3092 return -rte_errno; 3093 } 3094 return 0; 3095 } 3096 3097 /** 3098 * Manage filter operations. 3099 * 3100 * @param dev 3101 * Pointer to Ethernet device structure. 3102 * @param filter_type 3103 * Filter type. 3104 * @param filter_op 3105 * Operation to perform. 3106 * @param arg 3107 * Pointer to operation-specific structure. 3108 * 3109 * @return 3110 * 0 on success, a negative errno value otherwise and rte_errno is set. 3111 */ 3112 int 3113 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 3114 enum rte_filter_type filter_type, 3115 enum rte_filter_op filter_op, 3116 void *arg) 3117 { 3118 switch (filter_type) { 3119 case RTE_ETH_FILTER_GENERIC: 3120 if (filter_op != RTE_ETH_FILTER_GET) { 3121 rte_errno = EINVAL; 3122 return -rte_errno; 3123 } 3124 *(const void **)arg = &mlx5_flow_ops; 3125 return 0; 3126 case RTE_ETH_FILTER_FDIR: 3127 return mlx5_fdir_ctrl_func(dev, filter_op, arg); 3128 default: 3129 DRV_LOG(ERR, "port %u filter type (%d) not supported", 3130 dev->data->port_id, filter_type); 3131 rte_errno = ENOTSUP; 3132 return -rte_errno; 3133 } 3134 return 0; 3135 } 3136