1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <sys/queue.h> 7 #include <stdint.h> 8 #include <string.h> 9 10 /* Verbs header. */ 11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 12 #ifdef PEDANTIC 13 #pragma GCC diagnostic ignored "-Wpedantic" 14 #endif 15 #include <infiniband/verbs.h> 16 #ifdef PEDANTIC 17 #pragma GCC diagnostic error "-Wpedantic" 18 #endif 19 20 #include <rte_common.h> 21 #include <rte_ether.h> 22 #include <rte_eth_ctrl.h> 23 #include <rte_ethdev_driver.h> 24 #include <rte_flow.h> 25 #include <rte_flow_driver.h> 26 #include <rte_malloc.h> 27 #include <rte_ip.h> 28 29 #include "mlx5.h" 30 #include "mlx5_defs.h" 31 #include "mlx5_prm.h" 32 #include "mlx5_glue.h" 33 34 /* Define minimal priority for control plane flows. */ 35 #define MLX5_CTRL_FLOW_PRIORITY 4 36 37 /* Internet Protocol versions. */ 38 #define MLX5_IPV4 4 39 #define MLX5_IPV6 6 40 41 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 42 struct ibv_flow_spec_counter_action { 43 int dummy; 44 }; 45 #endif 46 47 /* Dev ops structure defined in mlx5.c */ 48 extern const struct eth_dev_ops mlx5_dev_ops; 49 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 50 51 /** Structure give to the conversion functions. */ 52 struct mlx5_flow_data { 53 struct mlx5_flow_parse *parser; /** Parser context. */ 54 struct rte_flow_error *error; /** Error context. */ 55 }; 56 57 static int 58 mlx5_flow_create_eth(const struct rte_flow_item *item, 59 const void *default_mask, 60 struct mlx5_flow_data *data); 61 62 static int 63 mlx5_flow_create_vlan(const struct rte_flow_item *item, 64 const void *default_mask, 65 struct mlx5_flow_data *data); 66 67 static int 68 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 69 const void *default_mask, 70 struct mlx5_flow_data *data); 71 72 static int 73 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 74 const void *default_mask, 75 struct mlx5_flow_data *data); 76 77 static int 78 mlx5_flow_create_udp(const struct rte_flow_item *item, 79 const void *default_mask, 80 struct mlx5_flow_data *data); 81 82 static int 83 mlx5_flow_create_tcp(const struct rte_flow_item *item, 84 const void *default_mask, 85 struct mlx5_flow_data *data); 86 87 static int 88 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 89 const void *default_mask, 90 struct mlx5_flow_data *data); 91 92 struct mlx5_flow_parse; 93 94 static void 95 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 96 unsigned int size); 97 98 static int 99 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); 100 101 static int 102 mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser); 103 104 /* Hash RX queue types. */ 105 enum hash_rxq_type { 106 HASH_RXQ_TCPV4, 107 HASH_RXQ_UDPV4, 108 HASH_RXQ_IPV4, 109 HASH_RXQ_TCPV6, 110 HASH_RXQ_UDPV6, 111 HASH_RXQ_IPV6, 112 HASH_RXQ_ETH, 113 }; 114 115 /* Initialization data for hash RX queue. */ 116 struct hash_rxq_init { 117 uint64_t hash_fields; /* Fields that participate in the hash. */ 118 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ 119 unsigned int flow_priority; /* Flow priority to use. */ 120 unsigned int ip_version; /* Internet protocol. */ 121 }; 122 123 /* Initialization data for hash RX queues. */ 124 const struct hash_rxq_init hash_rxq_init[] = { 125 [HASH_RXQ_TCPV4] = { 126 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 127 IBV_RX_HASH_DST_IPV4 | 128 IBV_RX_HASH_SRC_PORT_TCP | 129 IBV_RX_HASH_DST_PORT_TCP), 130 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, 131 .flow_priority = 1, 132 .ip_version = MLX5_IPV4, 133 }, 134 [HASH_RXQ_UDPV4] = { 135 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 136 IBV_RX_HASH_DST_IPV4 | 137 IBV_RX_HASH_SRC_PORT_UDP | 138 IBV_RX_HASH_DST_PORT_UDP), 139 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, 140 .flow_priority = 1, 141 .ip_version = MLX5_IPV4, 142 }, 143 [HASH_RXQ_IPV4] = { 144 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 145 IBV_RX_HASH_DST_IPV4), 146 .dpdk_rss_hf = (ETH_RSS_IPV4 | 147 ETH_RSS_FRAG_IPV4), 148 .flow_priority = 2, 149 .ip_version = MLX5_IPV4, 150 }, 151 [HASH_RXQ_TCPV6] = { 152 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 153 IBV_RX_HASH_DST_IPV6 | 154 IBV_RX_HASH_SRC_PORT_TCP | 155 IBV_RX_HASH_DST_PORT_TCP), 156 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, 157 .flow_priority = 1, 158 .ip_version = MLX5_IPV6, 159 }, 160 [HASH_RXQ_UDPV6] = { 161 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 162 IBV_RX_HASH_DST_IPV6 | 163 IBV_RX_HASH_SRC_PORT_UDP | 164 IBV_RX_HASH_DST_PORT_UDP), 165 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, 166 .flow_priority = 1, 167 .ip_version = MLX5_IPV6, 168 }, 169 [HASH_RXQ_IPV6] = { 170 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 171 IBV_RX_HASH_DST_IPV6), 172 .dpdk_rss_hf = (ETH_RSS_IPV6 | 173 ETH_RSS_FRAG_IPV6), 174 .flow_priority = 2, 175 .ip_version = MLX5_IPV6, 176 }, 177 [HASH_RXQ_ETH] = { 178 .hash_fields = 0, 179 .dpdk_rss_hf = 0, 180 .flow_priority = 3, 181 }, 182 }; 183 184 /* Number of entries in hash_rxq_init[]. */ 185 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); 186 187 /** Structure for holding counter stats. */ 188 struct mlx5_flow_counter_stats { 189 uint64_t hits; /**< Number of packets matched by the rule. */ 190 uint64_t bytes; /**< Number of bytes matched by the rule. */ 191 }; 192 193 /** Structure for Drop queue. */ 194 struct mlx5_hrxq_drop { 195 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 196 struct ibv_qp *qp; /**< Verbs queue pair. */ 197 struct ibv_wq *wq; /**< Verbs work queue. */ 198 struct ibv_cq *cq; /**< Verbs completion queue. */ 199 }; 200 201 /* Flows structures. */ 202 struct mlx5_flow { 203 uint64_t hash_fields; /**< Fields that participate in the hash. */ 204 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 205 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 206 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */ 207 }; 208 209 /* Drop flows structures. */ 210 struct mlx5_flow_drop { 211 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 212 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 213 }; 214 215 struct rte_flow { 216 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ 217 uint32_t mark:1; /**< Set if the flow is marked. */ 218 uint32_t drop:1; /**< Drop queue. */ 219 struct rte_flow_action_rss rss_conf; /**< RSS configuration */ 220 uint16_t (*queues)[]; /**< Queues indexes to use. */ 221 uint8_t rss_key[40]; /**< copy of the RSS key. */ 222 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */ 223 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */ 224 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)]; 225 /**< Flow with Rx queue. */ 226 }; 227 228 /** Static initializer for items. */ 229 #define ITEMS(...) \ 230 (const enum rte_flow_item_type []){ \ 231 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ 232 } 233 234 /** Structure to generate a simple graph of layers supported by the NIC. */ 235 struct mlx5_flow_items { 236 /** List of possible actions for these items. */ 237 const enum rte_flow_action_type *const actions; 238 /** Bit-masks corresponding to the possibilities for the item. */ 239 const void *mask; 240 /** 241 * Default bit-masks to use when item->mask is not provided. When 242 * \default_mask is also NULL, the full supported bit-mask (\mask) is 243 * used instead. 244 */ 245 const void *default_mask; 246 /** Bit-masks size in bytes. */ 247 const unsigned int mask_sz; 248 /** 249 * Conversion function from rte_flow to NIC specific flow. 250 * 251 * @param item 252 * rte_flow item to convert. 253 * @param default_mask 254 * Default bit-masks to use when item->mask is not provided. 255 * @param data 256 * Internal structure to store the conversion. 257 * 258 * @return 259 * 0 on success, a negative errno value otherwise and rte_errno is 260 * set. 261 */ 262 int (*convert)(const struct rte_flow_item *item, 263 const void *default_mask, 264 struct mlx5_flow_data *data); 265 /** Size in bytes of the destination structure. */ 266 const unsigned int dst_sz; 267 /** List of possible following items. */ 268 const enum rte_flow_item_type *const items; 269 }; 270 271 /** Valid action for this PMD. */ 272 static const enum rte_flow_action_type valid_actions[] = { 273 RTE_FLOW_ACTION_TYPE_DROP, 274 RTE_FLOW_ACTION_TYPE_QUEUE, 275 RTE_FLOW_ACTION_TYPE_MARK, 276 RTE_FLOW_ACTION_TYPE_FLAG, 277 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 278 RTE_FLOW_ACTION_TYPE_COUNT, 279 #endif 280 RTE_FLOW_ACTION_TYPE_END, 281 }; 282 283 /** Graph of supported items and associated actions. */ 284 static const struct mlx5_flow_items mlx5_flow_items[] = { 285 [RTE_FLOW_ITEM_TYPE_END] = { 286 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, 287 RTE_FLOW_ITEM_TYPE_VXLAN), 288 }, 289 [RTE_FLOW_ITEM_TYPE_ETH] = { 290 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, 291 RTE_FLOW_ITEM_TYPE_IPV4, 292 RTE_FLOW_ITEM_TYPE_IPV6), 293 .actions = valid_actions, 294 .mask = &(const struct rte_flow_item_eth){ 295 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 296 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 297 .type = -1, 298 }, 299 .default_mask = &rte_flow_item_eth_mask, 300 .mask_sz = sizeof(struct rte_flow_item_eth), 301 .convert = mlx5_flow_create_eth, 302 .dst_sz = sizeof(struct ibv_flow_spec_eth), 303 }, 304 [RTE_FLOW_ITEM_TYPE_VLAN] = { 305 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, 306 RTE_FLOW_ITEM_TYPE_IPV6), 307 .actions = valid_actions, 308 .mask = &(const struct rte_flow_item_vlan){ 309 .tci = -1, 310 .inner_type = -1, 311 }, 312 .default_mask = &rte_flow_item_vlan_mask, 313 .mask_sz = sizeof(struct rte_flow_item_vlan), 314 .convert = mlx5_flow_create_vlan, 315 .dst_sz = 0, 316 }, 317 [RTE_FLOW_ITEM_TYPE_IPV4] = { 318 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 319 RTE_FLOW_ITEM_TYPE_TCP), 320 .actions = valid_actions, 321 .mask = &(const struct rte_flow_item_ipv4){ 322 .hdr = { 323 .src_addr = -1, 324 .dst_addr = -1, 325 .type_of_service = -1, 326 .next_proto_id = -1, 327 }, 328 }, 329 .default_mask = &rte_flow_item_ipv4_mask, 330 .mask_sz = sizeof(struct rte_flow_item_ipv4), 331 .convert = mlx5_flow_create_ipv4, 332 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), 333 }, 334 [RTE_FLOW_ITEM_TYPE_IPV6] = { 335 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 336 RTE_FLOW_ITEM_TYPE_TCP), 337 .actions = valid_actions, 338 .mask = &(const struct rte_flow_item_ipv6){ 339 .hdr = { 340 .src_addr = { 341 0xff, 0xff, 0xff, 0xff, 342 0xff, 0xff, 0xff, 0xff, 343 0xff, 0xff, 0xff, 0xff, 344 0xff, 0xff, 0xff, 0xff, 345 }, 346 .dst_addr = { 347 0xff, 0xff, 0xff, 0xff, 348 0xff, 0xff, 0xff, 0xff, 349 0xff, 0xff, 0xff, 0xff, 350 0xff, 0xff, 0xff, 0xff, 351 }, 352 .vtc_flow = -1, 353 .proto = -1, 354 .hop_limits = -1, 355 }, 356 }, 357 .default_mask = &rte_flow_item_ipv6_mask, 358 .mask_sz = sizeof(struct rte_flow_item_ipv6), 359 .convert = mlx5_flow_create_ipv6, 360 .dst_sz = sizeof(struct ibv_flow_spec_ipv6), 361 }, 362 [RTE_FLOW_ITEM_TYPE_UDP] = { 363 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), 364 .actions = valid_actions, 365 .mask = &(const struct rte_flow_item_udp){ 366 .hdr = { 367 .src_port = -1, 368 .dst_port = -1, 369 }, 370 }, 371 .default_mask = &rte_flow_item_udp_mask, 372 .mask_sz = sizeof(struct rte_flow_item_udp), 373 .convert = mlx5_flow_create_udp, 374 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 375 }, 376 [RTE_FLOW_ITEM_TYPE_TCP] = { 377 .actions = valid_actions, 378 .mask = &(const struct rte_flow_item_tcp){ 379 .hdr = { 380 .src_port = -1, 381 .dst_port = -1, 382 }, 383 }, 384 .default_mask = &rte_flow_item_tcp_mask, 385 .mask_sz = sizeof(struct rte_flow_item_tcp), 386 .convert = mlx5_flow_create_tcp, 387 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 388 }, 389 [RTE_FLOW_ITEM_TYPE_VXLAN] = { 390 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), 391 .actions = valid_actions, 392 .mask = &(const struct rte_flow_item_vxlan){ 393 .vni = "\xff\xff\xff", 394 }, 395 .default_mask = &rte_flow_item_vxlan_mask, 396 .mask_sz = sizeof(struct rte_flow_item_vxlan), 397 .convert = mlx5_flow_create_vxlan, 398 .dst_sz = sizeof(struct ibv_flow_spec_tunnel), 399 }, 400 }; 401 402 /** Structure to pass to the conversion function. */ 403 struct mlx5_flow_parse { 404 uint32_t inner; /**< Set once VXLAN is encountered. */ 405 uint32_t create:1; 406 /**< Whether resources should remain after a validate. */ 407 uint32_t drop:1; /**< Target is a drop queue. */ 408 uint32_t mark:1; /**< Mark is present in the flow. */ 409 uint32_t count:1; /**< Count is present in the flow. */ 410 uint32_t mark_id; /**< Mark identifier. */ 411 struct rte_flow_action_rss rss_conf; /**< RSS configuration */ 412 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ 413 uint8_t rss_key[40]; /**< copy of the RSS key. */ 414 enum hash_rxq_type layer; /**< Last pattern layer detected. */ 415 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */ 416 struct { 417 struct ibv_flow_attr *ibv_attr; 418 /**< Pointer to Verbs attributes. */ 419 unsigned int offset; 420 /**< Current position or total size of the attribute. */ 421 } queue[RTE_DIM(hash_rxq_init)]; 422 }; 423 424 static const struct rte_flow_ops mlx5_flow_ops = { 425 .validate = mlx5_flow_validate, 426 .create = mlx5_flow_create, 427 .destroy = mlx5_flow_destroy, 428 .flush = mlx5_flow_flush, 429 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 430 .query = mlx5_flow_query, 431 #else 432 .query = NULL, 433 #endif 434 .isolate = mlx5_flow_isolate, 435 }; 436 437 /* Convert FDIR request to Generic flow. */ 438 struct mlx5_fdir { 439 struct rte_flow_attr attr; 440 struct rte_flow_action actions[2]; 441 struct rte_flow_item items[4]; 442 struct rte_flow_item_eth l2; 443 struct rte_flow_item_eth l2_mask; 444 union { 445 struct rte_flow_item_ipv4 ipv4; 446 struct rte_flow_item_ipv6 ipv6; 447 } l3; 448 union { 449 struct rte_flow_item_ipv4 ipv4; 450 struct rte_flow_item_ipv6 ipv6; 451 } l3_mask; 452 union { 453 struct rte_flow_item_udp udp; 454 struct rte_flow_item_tcp tcp; 455 } l4; 456 union { 457 struct rte_flow_item_udp udp; 458 struct rte_flow_item_tcp tcp; 459 } l4_mask; 460 struct rte_flow_action_queue queue; 461 }; 462 463 /* Verbs specification header. */ 464 struct ibv_spec_header { 465 enum ibv_flow_spec_type type; 466 uint16_t size; 467 }; 468 469 /** 470 * Check support for a given item. 471 * 472 * @param item[in] 473 * Item specification. 474 * @param mask[in] 475 * Bit-masks covering supported fields to compare with spec, last and mask in 476 * \item. 477 * @param size 478 * Bit-Mask size in bytes. 479 * 480 * @return 481 * 0 on success, a negative errno value otherwise and rte_errno is set. 482 */ 483 static int 484 mlx5_flow_item_validate(const struct rte_flow_item *item, 485 const uint8_t *mask, unsigned int size) 486 { 487 if (!item->spec && (item->mask || item->last)) { 488 rte_errno = EINVAL; 489 return -rte_errno; 490 } 491 if (item->spec && !item->mask) { 492 unsigned int i; 493 const uint8_t *spec = item->spec; 494 495 for (i = 0; i < size; ++i) 496 if ((spec[i] | mask[i]) != mask[i]) { 497 rte_errno = EINVAL; 498 return -rte_errno; 499 } 500 } 501 if (item->last && !item->mask) { 502 unsigned int i; 503 const uint8_t *spec = item->last; 504 505 for (i = 0; i < size; ++i) 506 if ((spec[i] | mask[i]) != mask[i]) { 507 rte_errno = EINVAL; 508 return -rte_errno; 509 } 510 } 511 if (item->mask) { 512 unsigned int i; 513 const uint8_t *spec = item->spec; 514 515 for (i = 0; i < size; ++i) 516 if ((spec[i] | mask[i]) != mask[i]) { 517 rte_errno = EINVAL; 518 return -rte_errno; 519 } 520 } 521 if (item->spec && item->last) { 522 uint8_t spec[size]; 523 uint8_t last[size]; 524 const uint8_t *apply = mask; 525 unsigned int i; 526 int ret; 527 528 if (item->mask) 529 apply = item->mask; 530 for (i = 0; i < size; ++i) { 531 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; 532 last[i] = ((const uint8_t *)item->last)[i] & apply[i]; 533 } 534 ret = memcmp(spec, last, size); 535 if (ret != 0) { 536 rte_errno = EINVAL; 537 return -rte_errno; 538 } 539 } 540 return 0; 541 } 542 543 /** 544 * Extract attribute to the parser. 545 * 546 * @param[in] attr 547 * Flow rule attributes. 548 * @param[out] error 549 * Perform verbose error reporting if not NULL. 550 * 551 * @return 552 * 0 on success, a negative errno value otherwise and rte_errno is set. 553 */ 554 static int 555 mlx5_flow_convert_attributes(const struct rte_flow_attr *attr, 556 struct rte_flow_error *error) 557 { 558 if (attr->group) { 559 rte_flow_error_set(error, ENOTSUP, 560 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 561 NULL, 562 "groups are not supported"); 563 return -rte_errno; 564 } 565 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) { 566 rte_flow_error_set(error, ENOTSUP, 567 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 568 NULL, 569 "priorities are not supported"); 570 return -rte_errno; 571 } 572 if (attr->egress) { 573 rte_flow_error_set(error, ENOTSUP, 574 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 575 NULL, 576 "egress is not supported"); 577 return -rte_errno; 578 } 579 if (attr->transfer) { 580 rte_flow_error_set(error, ENOTSUP, 581 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 582 NULL, 583 "transfer is not supported"); 584 return -rte_errno; 585 } 586 if (!attr->ingress) { 587 rte_flow_error_set(error, ENOTSUP, 588 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 589 NULL, 590 "only ingress is supported"); 591 return -rte_errno; 592 } 593 return 0; 594 } 595 596 /** 597 * Extract actions request to the parser. 598 * 599 * @param dev 600 * Pointer to Ethernet device. 601 * @param[in] actions 602 * Associated actions (list terminated by the END action). 603 * @param[out] error 604 * Perform verbose error reporting if not NULL. 605 * @param[in, out] parser 606 * Internal parser structure. 607 * 608 * @return 609 * 0 on success, a negative errno value otherwise and rte_errno is set. 610 */ 611 static int 612 mlx5_flow_convert_actions(struct rte_eth_dev *dev, 613 const struct rte_flow_action actions[], 614 struct rte_flow_error *error, 615 struct mlx5_flow_parse *parser) 616 { 617 enum { FATE = 1, MARK = 2, COUNT = 4, }; 618 uint32_t overlap = 0; 619 struct priv *priv = dev->data->dev_private; 620 621 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { 622 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { 623 continue; 624 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { 625 if (overlap & FATE) 626 goto exit_action_overlap; 627 overlap |= FATE; 628 parser->drop = 1; 629 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 630 const struct rte_flow_action_queue *queue = 631 (const struct rte_flow_action_queue *) 632 actions->conf; 633 634 if (overlap & FATE) 635 goto exit_action_overlap; 636 overlap |= FATE; 637 if (!queue || (queue->index > (priv->rxqs_n - 1))) 638 goto exit_action_not_supported; 639 parser->queues[0] = queue->index; 640 parser->rss_conf = (struct rte_flow_action_rss){ 641 .queue_num = 1, 642 .queue = parser->queues, 643 }; 644 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { 645 const struct rte_flow_action_rss *rss = 646 (const struct rte_flow_action_rss *) 647 actions->conf; 648 const uint8_t *rss_key; 649 uint32_t rss_key_len; 650 uint16_t n; 651 652 if (overlap & FATE) 653 goto exit_action_overlap; 654 overlap |= FATE; 655 if (rss->func && 656 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) { 657 rte_flow_error_set(error, EINVAL, 658 RTE_FLOW_ERROR_TYPE_ACTION, 659 actions, 660 "the only supported RSS hash" 661 " function is Toeplitz"); 662 return -rte_errno; 663 } 664 if (rss->level) { 665 rte_flow_error_set(error, EINVAL, 666 RTE_FLOW_ERROR_TYPE_ACTION, 667 actions, 668 "a nonzero RSS encapsulation" 669 " level is not supported"); 670 return -rte_errno; 671 } 672 if (rss->types & MLX5_RSS_HF_MASK) { 673 rte_flow_error_set(error, EINVAL, 674 RTE_FLOW_ERROR_TYPE_ACTION, 675 actions, 676 "unsupported RSS type" 677 " requested"); 678 return -rte_errno; 679 } 680 if (rss->key_len) { 681 rss_key_len = rss->key_len; 682 rss_key = rss->key; 683 } else { 684 rss_key_len = rss_hash_default_key_len; 685 rss_key = rss_hash_default_key; 686 } 687 if (rss_key_len != RTE_DIM(parser->rss_key)) { 688 rte_flow_error_set(error, EINVAL, 689 RTE_FLOW_ERROR_TYPE_ACTION, 690 actions, 691 "RSS hash key must be" 692 " exactly 40 bytes long"); 693 return -rte_errno; 694 } 695 if (!rss->queue_num) { 696 rte_flow_error_set(error, EINVAL, 697 RTE_FLOW_ERROR_TYPE_ACTION, 698 actions, 699 "no valid queues"); 700 return -rte_errno; 701 } 702 if (rss->queue_num > RTE_DIM(parser->queues)) { 703 rte_flow_error_set(error, EINVAL, 704 RTE_FLOW_ERROR_TYPE_ACTION, 705 actions, 706 "too many queues for RSS" 707 " context"); 708 return -rte_errno; 709 } 710 for (n = 0; n < rss->queue_num; ++n) { 711 if (rss->queue[n] >= priv->rxqs_n) { 712 rte_flow_error_set(error, EINVAL, 713 RTE_FLOW_ERROR_TYPE_ACTION, 714 actions, 715 "queue id > number of" 716 " queues"); 717 return -rte_errno; 718 } 719 } 720 parser->rss_conf = (struct rte_flow_action_rss){ 721 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 722 .level = 0, 723 .types = rss->types, 724 .key_len = rss_key_len, 725 .queue_num = rss->queue_num, 726 .key = memcpy(parser->rss_key, rss_key, 727 sizeof(*rss_key) * rss_key_len), 728 .queue = memcpy(parser->queues, rss->queue, 729 sizeof(*rss->queue) * 730 rss->queue_num), 731 }; 732 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { 733 const struct rte_flow_action_mark *mark = 734 (const struct rte_flow_action_mark *) 735 actions->conf; 736 737 if (overlap & MARK) 738 goto exit_action_overlap; 739 overlap |= MARK; 740 if (!mark) { 741 rte_flow_error_set(error, EINVAL, 742 RTE_FLOW_ERROR_TYPE_ACTION, 743 actions, 744 "mark must be defined"); 745 return -rte_errno; 746 } else if (mark->id >= MLX5_FLOW_MARK_MAX) { 747 rte_flow_error_set(error, ENOTSUP, 748 RTE_FLOW_ERROR_TYPE_ACTION, 749 actions, 750 "mark must be between 0" 751 " and 16777199"); 752 return -rte_errno; 753 } 754 parser->mark = 1; 755 parser->mark_id = mark->id; 756 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { 757 if (overlap & MARK) 758 goto exit_action_overlap; 759 overlap |= MARK; 760 parser->mark = 1; 761 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && 762 priv->config.flow_counter_en) { 763 if (overlap & COUNT) 764 goto exit_action_overlap; 765 overlap |= COUNT; 766 parser->count = 1; 767 } else { 768 goto exit_action_not_supported; 769 } 770 } 771 /* When fate is unknown, drop traffic. */ 772 if (!(overlap & FATE)) 773 parser->drop = 1; 774 if (parser->drop && parser->mark) 775 parser->mark = 0; 776 if (!parser->rss_conf.queue_num && !parser->drop) { 777 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, 778 NULL, "no valid action"); 779 return -rte_errno; 780 } 781 return 0; 782 exit_action_not_supported: 783 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 784 actions, "action not supported"); 785 return -rte_errno; 786 exit_action_overlap: 787 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 788 actions, "overlapping actions are not supported"); 789 return -rte_errno; 790 } 791 792 /** 793 * Validate items. 794 * 795 * @param[in] items 796 * Pattern specification (list terminated by the END pattern item). 797 * @param[out] error 798 * Perform verbose error reporting if not NULL. 799 * @param[in, out] parser 800 * Internal parser structure. 801 * 802 * @return 803 * 0 on success, a negative errno value otherwise and rte_errno is set. 804 */ 805 static int 806 mlx5_flow_convert_items_validate(const struct rte_flow_item items[], 807 struct rte_flow_error *error, 808 struct mlx5_flow_parse *parser) 809 { 810 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 811 unsigned int i; 812 int ret = 0; 813 814 /* Initialise the offsets to start after verbs attribute. */ 815 for (i = 0; i != hash_rxq_init_n; ++i) 816 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 817 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 818 const struct mlx5_flow_items *token = NULL; 819 unsigned int n; 820 821 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 822 continue; 823 for (i = 0; 824 cur_item->items && 825 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; 826 ++i) { 827 if (cur_item->items[i] == items->type) { 828 token = &mlx5_flow_items[items->type]; 829 break; 830 } 831 } 832 if (!token) { 833 ret = -ENOTSUP; 834 goto exit_item_not_supported; 835 } 836 cur_item = token; 837 ret = mlx5_flow_item_validate(items, 838 (const uint8_t *)cur_item->mask, 839 cur_item->mask_sz); 840 if (ret) 841 goto exit_item_not_supported; 842 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 843 if (parser->inner) { 844 rte_flow_error_set(error, ENOTSUP, 845 RTE_FLOW_ERROR_TYPE_ITEM, 846 items, 847 "cannot recognize multiple" 848 " VXLAN encapsulations"); 849 return -rte_errno; 850 } 851 parser->inner = IBV_FLOW_SPEC_INNER; 852 } 853 if (parser->drop) { 854 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; 855 } else { 856 for (n = 0; n != hash_rxq_init_n; ++n) 857 parser->queue[n].offset += cur_item->dst_sz; 858 } 859 } 860 if (parser->drop) { 861 parser->queue[HASH_RXQ_ETH].offset += 862 sizeof(struct ibv_flow_spec_action_drop); 863 } 864 if (parser->mark) { 865 for (i = 0; i != hash_rxq_init_n; ++i) 866 parser->queue[i].offset += 867 sizeof(struct ibv_flow_spec_action_tag); 868 } 869 if (parser->count) { 870 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 871 872 for (i = 0; i != hash_rxq_init_n; ++i) 873 parser->queue[i].offset += size; 874 } 875 return 0; 876 exit_item_not_supported: 877 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM, 878 items, "item not supported"); 879 } 880 881 /** 882 * Allocate memory space to store verbs flow attributes. 883 * 884 * @param[in] size 885 * Amount of byte to allocate. 886 * @param[out] error 887 * Perform verbose error reporting if not NULL. 888 * 889 * @return 890 * A verbs flow attribute on success, NULL otherwise and rte_errno is set. 891 */ 892 static struct ibv_flow_attr * 893 mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error) 894 { 895 struct ibv_flow_attr *ibv_attr; 896 897 ibv_attr = rte_calloc(__func__, 1, size, 0); 898 if (!ibv_attr) { 899 rte_flow_error_set(error, ENOMEM, 900 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 901 NULL, 902 "cannot allocate verbs spec attributes"); 903 return NULL; 904 } 905 return ibv_attr; 906 } 907 908 /** 909 * Make inner packet matching with an higher priority from the non Inner 910 * matching. 911 * 912 * @param[in, out] parser 913 * Internal parser structure. 914 * @param attr 915 * User flow attribute. 916 */ 917 static void 918 mlx5_flow_update_priority(struct mlx5_flow_parse *parser, 919 const struct rte_flow_attr *attr) 920 { 921 unsigned int i; 922 923 if (parser->drop) { 924 parser->queue[HASH_RXQ_ETH].ibv_attr->priority = 925 attr->priority + 926 hash_rxq_init[HASH_RXQ_ETH].flow_priority; 927 return; 928 } 929 for (i = 0; i != hash_rxq_init_n; ++i) { 930 if (parser->queue[i].ibv_attr) { 931 parser->queue[i].ibv_attr->priority = 932 attr->priority + 933 hash_rxq_init[i].flow_priority - 934 (parser->inner ? 1 : 0); 935 } 936 } 937 } 938 939 /** 940 * Finalise verbs flow attributes. 941 * 942 * @param[in, out] parser 943 * Internal parser structure. 944 */ 945 static void 946 mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser) 947 { 948 const unsigned int ipv4 = 949 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; 950 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6; 951 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 952 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4; 953 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4; 954 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 955 unsigned int i; 956 957 /* Remove any other flow not matching the pattern. */ 958 if (parser->rss_conf.queue_num == 1 && !parser->rss_conf.types) { 959 for (i = 0; i != hash_rxq_init_n; ++i) { 960 if (i == HASH_RXQ_ETH) 961 continue; 962 rte_free(parser->queue[i].ibv_attr); 963 parser->queue[i].ibv_attr = NULL; 964 } 965 return; 966 } 967 if (parser->layer == HASH_RXQ_ETH) { 968 goto fill; 969 } else { 970 /* 971 * This layer becomes useless as the pattern define under 972 * layers. 973 */ 974 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr); 975 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 976 } 977 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */ 978 for (i = ohmin; i != (ohmax + 1); ++i) { 979 if (!parser->queue[i].ibv_attr) 980 continue; 981 rte_free(parser->queue[i].ibv_attr); 982 parser->queue[i].ibv_attr = NULL; 983 } 984 /* Remove impossible flow according to the RSS configuration. */ 985 if (hash_rxq_init[parser->layer].dpdk_rss_hf & 986 parser->rss_conf.types) { 987 /* Remove any other flow. */ 988 for (i = hmin; i != (hmax + 1); ++i) { 989 if ((i == parser->layer) || 990 (!parser->queue[i].ibv_attr)) 991 continue; 992 rte_free(parser->queue[i].ibv_attr); 993 parser->queue[i].ibv_attr = NULL; 994 } 995 } else if (!parser->queue[ip].ibv_attr) { 996 /* no RSS possible with the current configuration. */ 997 parser->rss_conf.queue_num = 1; 998 return; 999 } 1000 fill: 1001 /* 1002 * Fill missing layers in verbs specifications, or compute the correct 1003 * offset to allocate the memory space for the attributes and 1004 * specifications. 1005 */ 1006 for (i = 0; i != hash_rxq_init_n - 1; ++i) { 1007 union { 1008 struct ibv_flow_spec_ipv4_ext ipv4; 1009 struct ibv_flow_spec_ipv6 ipv6; 1010 struct ibv_flow_spec_tcp_udp udp_tcp; 1011 } specs; 1012 void *dst; 1013 uint16_t size; 1014 1015 if (i == parser->layer) 1016 continue; 1017 if (parser->layer == HASH_RXQ_ETH) { 1018 if (hash_rxq_init[i].ip_version == MLX5_IPV4) { 1019 size = sizeof(struct ibv_flow_spec_ipv4_ext); 1020 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){ 1021 .type = IBV_FLOW_SPEC_IPV4_EXT, 1022 .size = size, 1023 }; 1024 } else { 1025 size = sizeof(struct ibv_flow_spec_ipv6); 1026 specs.ipv6 = (struct ibv_flow_spec_ipv6){ 1027 .type = IBV_FLOW_SPEC_IPV6, 1028 .size = size, 1029 }; 1030 } 1031 if (parser->queue[i].ibv_attr) { 1032 dst = (void *)((uintptr_t) 1033 parser->queue[i].ibv_attr + 1034 parser->queue[i].offset); 1035 memcpy(dst, &specs, size); 1036 ++parser->queue[i].ibv_attr->num_of_specs; 1037 } 1038 parser->queue[i].offset += size; 1039 } 1040 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) || 1041 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) { 1042 size = sizeof(struct ibv_flow_spec_tcp_udp); 1043 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) { 1044 .type = ((i == HASH_RXQ_UDPV4 || 1045 i == HASH_RXQ_UDPV6) ? 1046 IBV_FLOW_SPEC_UDP : 1047 IBV_FLOW_SPEC_TCP), 1048 .size = size, 1049 }; 1050 if (parser->queue[i].ibv_attr) { 1051 dst = (void *)((uintptr_t) 1052 parser->queue[i].ibv_attr + 1053 parser->queue[i].offset); 1054 memcpy(dst, &specs, size); 1055 ++parser->queue[i].ibv_attr->num_of_specs; 1056 } 1057 parser->queue[i].offset += size; 1058 } 1059 } 1060 } 1061 1062 /** 1063 * Validate and convert a flow supported by the NIC. 1064 * 1065 * @param dev 1066 * Pointer to Ethernet device. 1067 * @param[in] attr 1068 * Flow rule attributes. 1069 * @param[in] pattern 1070 * Pattern specification (list terminated by the END pattern item). 1071 * @param[in] actions 1072 * Associated actions (list terminated by the END action). 1073 * @param[out] error 1074 * Perform verbose error reporting if not NULL. 1075 * @param[in, out] parser 1076 * Internal parser structure. 1077 * 1078 * @return 1079 * 0 on success, a negative errno value otherwise and rte_errno is set. 1080 */ 1081 static int 1082 mlx5_flow_convert(struct rte_eth_dev *dev, 1083 const struct rte_flow_attr *attr, 1084 const struct rte_flow_item items[], 1085 const struct rte_flow_action actions[], 1086 struct rte_flow_error *error, 1087 struct mlx5_flow_parse *parser) 1088 { 1089 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 1090 unsigned int i; 1091 int ret; 1092 1093 /* First step. Validate the attributes, items and actions. */ 1094 *parser = (struct mlx5_flow_parse){ 1095 .create = parser->create, 1096 .layer = HASH_RXQ_ETH, 1097 .mark_id = MLX5_FLOW_MARK_DEFAULT, 1098 }; 1099 ret = mlx5_flow_convert_attributes(attr, error); 1100 if (ret) 1101 return ret; 1102 ret = mlx5_flow_convert_actions(dev, actions, error, parser); 1103 if (ret) 1104 return ret; 1105 ret = mlx5_flow_convert_items_validate(items, error, parser); 1106 if (ret) 1107 return ret; 1108 mlx5_flow_convert_finalise(parser); 1109 /* 1110 * Second step. 1111 * Allocate the memory space to store verbs specifications. 1112 */ 1113 if (parser->drop) { 1114 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; 1115 1116 parser->queue[HASH_RXQ_ETH].ibv_attr = 1117 mlx5_flow_convert_allocate(offset, error); 1118 if (!parser->queue[HASH_RXQ_ETH].ibv_attr) 1119 goto exit_enomem; 1120 parser->queue[HASH_RXQ_ETH].offset = 1121 sizeof(struct ibv_flow_attr); 1122 } else { 1123 for (i = 0; i != hash_rxq_init_n; ++i) { 1124 unsigned int offset; 1125 1126 if (!(parser->rss_conf.types & 1127 hash_rxq_init[i].dpdk_rss_hf) && 1128 (i != HASH_RXQ_ETH)) 1129 continue; 1130 offset = parser->queue[i].offset; 1131 parser->queue[i].ibv_attr = 1132 mlx5_flow_convert_allocate(offset, error); 1133 if (!parser->queue[i].ibv_attr) 1134 goto exit_enomem; 1135 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 1136 } 1137 } 1138 /* Third step. Conversion parse, fill the specifications. */ 1139 parser->inner = 0; 1140 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 1141 struct mlx5_flow_data data = { 1142 .parser = parser, 1143 .error = error, 1144 }; 1145 1146 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 1147 continue; 1148 cur_item = &mlx5_flow_items[items->type]; 1149 ret = cur_item->convert(items, 1150 (cur_item->default_mask ? 1151 cur_item->default_mask : 1152 cur_item->mask), 1153 &data); 1154 if (ret) 1155 goto exit_free; 1156 } 1157 if (parser->mark) 1158 mlx5_flow_create_flag_mark(parser, parser->mark_id); 1159 if (parser->count && parser->create) { 1160 mlx5_flow_create_count(dev, parser); 1161 if (!parser->cs) 1162 goto exit_count_error; 1163 } 1164 /* 1165 * Last step. Complete missing specification to reach the RSS 1166 * configuration. 1167 */ 1168 if (!parser->drop) 1169 mlx5_flow_convert_finalise(parser); 1170 mlx5_flow_update_priority(parser, attr); 1171 exit_free: 1172 /* Only verification is expected, all resources should be released. */ 1173 if (!parser->create) { 1174 for (i = 0; i != hash_rxq_init_n; ++i) { 1175 if (parser->queue[i].ibv_attr) { 1176 rte_free(parser->queue[i].ibv_attr); 1177 parser->queue[i].ibv_attr = NULL; 1178 } 1179 } 1180 } 1181 return ret; 1182 exit_enomem: 1183 for (i = 0; i != hash_rxq_init_n; ++i) { 1184 if (parser->queue[i].ibv_attr) { 1185 rte_free(parser->queue[i].ibv_attr); 1186 parser->queue[i].ibv_attr = NULL; 1187 } 1188 } 1189 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1190 NULL, "cannot allocate verbs spec attributes"); 1191 return -rte_errno; 1192 exit_count_error: 1193 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1194 NULL, "cannot create counter"); 1195 return -rte_errno; 1196 } 1197 1198 /** 1199 * Copy the specification created into the flow. 1200 * 1201 * @param parser 1202 * Internal parser structure. 1203 * @param src 1204 * Create specification. 1205 * @param size 1206 * Size in bytes of the specification to copy. 1207 */ 1208 static void 1209 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 1210 unsigned int size) 1211 { 1212 unsigned int i; 1213 void *dst; 1214 1215 for (i = 0; i != hash_rxq_init_n; ++i) { 1216 if (!parser->queue[i].ibv_attr) 1217 continue; 1218 /* Specification must be the same l3 type or none. */ 1219 if (parser->layer == HASH_RXQ_ETH || 1220 (hash_rxq_init[parser->layer].ip_version == 1221 hash_rxq_init[i].ip_version) || 1222 (hash_rxq_init[i].ip_version == 0)) { 1223 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1224 parser->queue[i].offset); 1225 memcpy(dst, src, size); 1226 ++parser->queue[i].ibv_attr->num_of_specs; 1227 parser->queue[i].offset += size; 1228 } 1229 } 1230 } 1231 1232 /** 1233 * Convert Ethernet item to Verbs specification. 1234 * 1235 * @param item[in] 1236 * Item specification. 1237 * @param default_mask[in] 1238 * Default bit-masks to use when item->mask is not provided. 1239 * @param data[in, out] 1240 * User structure. 1241 * 1242 * @return 1243 * 0 on success, a negative errno value otherwise and rte_errno is set. 1244 */ 1245 static int 1246 mlx5_flow_create_eth(const struct rte_flow_item *item, 1247 const void *default_mask, 1248 struct mlx5_flow_data *data) 1249 { 1250 const struct rte_flow_item_eth *spec = item->spec; 1251 const struct rte_flow_item_eth *mask = item->mask; 1252 struct mlx5_flow_parse *parser = data->parser; 1253 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1254 struct ibv_flow_spec_eth eth = { 1255 .type = parser->inner | IBV_FLOW_SPEC_ETH, 1256 .size = eth_size, 1257 }; 1258 1259 /* Don't update layer for the inner pattern. */ 1260 if (!parser->inner) 1261 parser->layer = HASH_RXQ_ETH; 1262 if (spec) { 1263 unsigned int i; 1264 1265 if (!mask) 1266 mask = default_mask; 1267 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); 1268 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); 1269 eth.val.ether_type = spec->type; 1270 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); 1271 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); 1272 eth.mask.ether_type = mask->type; 1273 /* Remove unwanted bits from values. */ 1274 for (i = 0; i < ETHER_ADDR_LEN; ++i) { 1275 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 1276 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 1277 } 1278 eth.val.ether_type &= eth.mask.ether_type; 1279 } 1280 mlx5_flow_create_copy(parser, ð, eth_size); 1281 return 0; 1282 } 1283 1284 /** 1285 * Convert VLAN item to Verbs specification. 1286 * 1287 * @param item[in] 1288 * Item specification. 1289 * @param default_mask[in] 1290 * Default bit-masks to use when item->mask is not provided. 1291 * @param data[in, out] 1292 * User structure. 1293 * 1294 * @return 1295 * 0 on success, a negative errno value otherwise and rte_errno is set. 1296 */ 1297 static int 1298 mlx5_flow_create_vlan(const struct rte_flow_item *item, 1299 const void *default_mask, 1300 struct mlx5_flow_data *data) 1301 { 1302 const struct rte_flow_item_vlan *spec = item->spec; 1303 const struct rte_flow_item_vlan *mask = item->mask; 1304 struct mlx5_flow_parse *parser = data->parser; 1305 struct ibv_flow_spec_eth *eth; 1306 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1307 const char *msg = "VLAN cannot be empty"; 1308 1309 if (spec) { 1310 unsigned int i; 1311 if (!mask) 1312 mask = default_mask; 1313 1314 for (i = 0; i != hash_rxq_init_n; ++i) { 1315 if (!parser->queue[i].ibv_attr) 1316 continue; 1317 1318 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1319 parser->queue[i].offset - eth_size); 1320 eth->val.vlan_tag = spec->tci; 1321 eth->mask.vlan_tag = mask->tci; 1322 eth->val.vlan_tag &= eth->mask.vlan_tag; 1323 /* 1324 * From verbs perspective an empty VLAN is equivalent 1325 * to a packet without VLAN layer. 1326 */ 1327 if (!eth->mask.vlan_tag) 1328 goto error; 1329 /* Outer TPID cannot be matched. */ 1330 if (eth->mask.ether_type) { 1331 msg = "VLAN TPID matching is not supported"; 1332 goto error; 1333 } 1334 eth->val.ether_type = spec->inner_type; 1335 eth->mask.ether_type = mask->inner_type; 1336 eth->val.ether_type &= eth->mask.ether_type; 1337 } 1338 return 0; 1339 } 1340 error: 1341 return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, 1342 item, msg); 1343 } 1344 1345 /** 1346 * Convert IPv4 item to Verbs specification. 1347 * 1348 * @param item[in] 1349 * Item specification. 1350 * @param default_mask[in] 1351 * Default bit-masks to use when item->mask is not provided. 1352 * @param data[in, out] 1353 * User structure. 1354 * 1355 * @return 1356 * 0 on success, a negative errno value otherwise and rte_errno is set. 1357 */ 1358 static int 1359 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 1360 const void *default_mask, 1361 struct mlx5_flow_data *data) 1362 { 1363 const struct rte_flow_item_ipv4 *spec = item->spec; 1364 const struct rte_flow_item_ipv4 *mask = item->mask; 1365 struct mlx5_flow_parse *parser = data->parser; 1366 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); 1367 struct ibv_flow_spec_ipv4_ext ipv4 = { 1368 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, 1369 .size = ipv4_size, 1370 }; 1371 1372 /* Don't update layer for the inner pattern. */ 1373 if (!parser->inner) 1374 parser->layer = HASH_RXQ_IPV4; 1375 if (spec) { 1376 if (!mask) 1377 mask = default_mask; 1378 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 1379 .src_ip = spec->hdr.src_addr, 1380 .dst_ip = spec->hdr.dst_addr, 1381 .proto = spec->hdr.next_proto_id, 1382 .tos = spec->hdr.type_of_service, 1383 }; 1384 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 1385 .src_ip = mask->hdr.src_addr, 1386 .dst_ip = mask->hdr.dst_addr, 1387 .proto = mask->hdr.next_proto_id, 1388 .tos = mask->hdr.type_of_service, 1389 }; 1390 /* Remove unwanted bits from values. */ 1391 ipv4.val.src_ip &= ipv4.mask.src_ip; 1392 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 1393 ipv4.val.proto &= ipv4.mask.proto; 1394 ipv4.val.tos &= ipv4.mask.tos; 1395 } 1396 mlx5_flow_create_copy(parser, &ipv4, ipv4_size); 1397 return 0; 1398 } 1399 1400 /** 1401 * Convert IPv6 item to Verbs specification. 1402 * 1403 * @param item[in] 1404 * Item specification. 1405 * @param default_mask[in] 1406 * Default bit-masks to use when item->mask is not provided. 1407 * @param data[in, out] 1408 * User structure. 1409 * 1410 * @return 1411 * 0 on success, a negative errno value otherwise and rte_errno is set. 1412 */ 1413 static int 1414 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 1415 const void *default_mask, 1416 struct mlx5_flow_data *data) 1417 { 1418 const struct rte_flow_item_ipv6 *spec = item->spec; 1419 const struct rte_flow_item_ipv6 *mask = item->mask; 1420 struct mlx5_flow_parse *parser = data->parser; 1421 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); 1422 struct ibv_flow_spec_ipv6 ipv6 = { 1423 .type = parser->inner | IBV_FLOW_SPEC_IPV6, 1424 .size = ipv6_size, 1425 }; 1426 1427 /* Don't update layer for the inner pattern. */ 1428 if (!parser->inner) 1429 parser->layer = HASH_RXQ_IPV6; 1430 if (spec) { 1431 unsigned int i; 1432 uint32_t vtc_flow_val; 1433 uint32_t vtc_flow_mask; 1434 1435 if (!mask) 1436 mask = default_mask; 1437 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 1438 RTE_DIM(ipv6.val.src_ip)); 1439 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 1440 RTE_DIM(ipv6.val.dst_ip)); 1441 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 1442 RTE_DIM(ipv6.mask.src_ip)); 1443 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 1444 RTE_DIM(ipv6.mask.dst_ip)); 1445 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); 1446 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); 1447 ipv6.val.flow_label = 1448 rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >> 1449 IPV6_HDR_FL_SHIFT); 1450 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >> 1451 IPV6_HDR_TC_SHIFT; 1452 ipv6.val.next_hdr = spec->hdr.proto; 1453 ipv6.val.hop_limit = spec->hdr.hop_limits; 1454 ipv6.mask.flow_label = 1455 rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >> 1456 IPV6_HDR_FL_SHIFT); 1457 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >> 1458 IPV6_HDR_TC_SHIFT; 1459 ipv6.mask.next_hdr = mask->hdr.proto; 1460 ipv6.mask.hop_limit = mask->hdr.hop_limits; 1461 /* Remove unwanted bits from values. */ 1462 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 1463 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 1464 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 1465 } 1466 ipv6.val.flow_label &= ipv6.mask.flow_label; 1467 ipv6.val.traffic_class &= ipv6.mask.traffic_class; 1468 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 1469 ipv6.val.hop_limit &= ipv6.mask.hop_limit; 1470 } 1471 mlx5_flow_create_copy(parser, &ipv6, ipv6_size); 1472 return 0; 1473 } 1474 1475 /** 1476 * Convert UDP item to Verbs specification. 1477 * 1478 * @param item[in] 1479 * Item specification. 1480 * @param default_mask[in] 1481 * Default bit-masks to use when item->mask is not provided. 1482 * @param data[in, out] 1483 * User structure. 1484 * 1485 * @return 1486 * 0 on success, a negative errno value otherwise and rte_errno is set. 1487 */ 1488 static int 1489 mlx5_flow_create_udp(const struct rte_flow_item *item, 1490 const void *default_mask, 1491 struct mlx5_flow_data *data) 1492 { 1493 const struct rte_flow_item_udp *spec = item->spec; 1494 const struct rte_flow_item_udp *mask = item->mask; 1495 struct mlx5_flow_parse *parser = data->parser; 1496 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1497 struct ibv_flow_spec_tcp_udp udp = { 1498 .type = parser->inner | IBV_FLOW_SPEC_UDP, 1499 .size = udp_size, 1500 }; 1501 1502 /* Don't update layer for the inner pattern. */ 1503 if (!parser->inner) { 1504 if (parser->layer == HASH_RXQ_IPV4) 1505 parser->layer = HASH_RXQ_UDPV4; 1506 else 1507 parser->layer = HASH_RXQ_UDPV6; 1508 } 1509 if (spec) { 1510 if (!mask) 1511 mask = default_mask; 1512 udp.val.dst_port = spec->hdr.dst_port; 1513 udp.val.src_port = spec->hdr.src_port; 1514 udp.mask.dst_port = mask->hdr.dst_port; 1515 udp.mask.src_port = mask->hdr.src_port; 1516 /* Remove unwanted bits from values. */ 1517 udp.val.src_port &= udp.mask.src_port; 1518 udp.val.dst_port &= udp.mask.dst_port; 1519 } 1520 mlx5_flow_create_copy(parser, &udp, udp_size); 1521 return 0; 1522 } 1523 1524 /** 1525 * Convert TCP item to Verbs specification. 1526 * 1527 * @param item[in] 1528 * Item specification. 1529 * @param default_mask[in] 1530 * Default bit-masks to use when item->mask is not provided. 1531 * @param data[in, out] 1532 * User structure. 1533 * 1534 * @return 1535 * 0 on success, a negative errno value otherwise and rte_errno is set. 1536 */ 1537 static int 1538 mlx5_flow_create_tcp(const struct rte_flow_item *item, 1539 const void *default_mask, 1540 struct mlx5_flow_data *data) 1541 { 1542 const struct rte_flow_item_tcp *spec = item->spec; 1543 const struct rte_flow_item_tcp *mask = item->mask; 1544 struct mlx5_flow_parse *parser = data->parser; 1545 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1546 struct ibv_flow_spec_tcp_udp tcp = { 1547 .type = parser->inner | IBV_FLOW_SPEC_TCP, 1548 .size = tcp_size, 1549 }; 1550 1551 /* Don't update layer for the inner pattern. */ 1552 if (!parser->inner) { 1553 if (parser->layer == HASH_RXQ_IPV4) 1554 parser->layer = HASH_RXQ_TCPV4; 1555 else 1556 parser->layer = HASH_RXQ_TCPV6; 1557 } 1558 if (spec) { 1559 if (!mask) 1560 mask = default_mask; 1561 tcp.val.dst_port = spec->hdr.dst_port; 1562 tcp.val.src_port = spec->hdr.src_port; 1563 tcp.mask.dst_port = mask->hdr.dst_port; 1564 tcp.mask.src_port = mask->hdr.src_port; 1565 /* Remove unwanted bits from values. */ 1566 tcp.val.src_port &= tcp.mask.src_port; 1567 tcp.val.dst_port &= tcp.mask.dst_port; 1568 } 1569 mlx5_flow_create_copy(parser, &tcp, tcp_size); 1570 return 0; 1571 } 1572 1573 /** 1574 * Convert VXLAN item to Verbs specification. 1575 * 1576 * @param item[in] 1577 * Item specification. 1578 * @param default_mask[in] 1579 * Default bit-masks to use when item->mask is not provided. 1580 * @param data[in, out] 1581 * User structure. 1582 * 1583 * @return 1584 * 0 on success, a negative errno value otherwise and rte_errno is set. 1585 */ 1586 static int 1587 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 1588 const void *default_mask, 1589 struct mlx5_flow_data *data) 1590 { 1591 const struct rte_flow_item_vxlan *spec = item->spec; 1592 const struct rte_flow_item_vxlan *mask = item->mask; 1593 struct mlx5_flow_parse *parser = data->parser; 1594 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 1595 struct ibv_flow_spec_tunnel vxlan = { 1596 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, 1597 .size = size, 1598 }; 1599 union vni { 1600 uint32_t vlan_id; 1601 uint8_t vni[4]; 1602 } id; 1603 1604 id.vni[0] = 0; 1605 parser->inner = IBV_FLOW_SPEC_INNER; 1606 if (spec) { 1607 if (!mask) 1608 mask = default_mask; 1609 memcpy(&id.vni[1], spec->vni, 3); 1610 vxlan.val.tunnel_id = id.vlan_id; 1611 memcpy(&id.vni[1], mask->vni, 3); 1612 vxlan.mask.tunnel_id = id.vlan_id; 1613 /* Remove unwanted bits from values. */ 1614 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 1615 } 1616 /* 1617 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1618 * layer is defined in the Verbs specification it is interpreted as 1619 * wildcard and all packets will match this rule, if it follows a full 1620 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1621 * before will also match this rule. 1622 * To avoid such situation, VNI 0 is currently refused. 1623 */ 1624 if (!vxlan.val.tunnel_id) 1625 return rte_flow_error_set(data->error, EINVAL, 1626 RTE_FLOW_ERROR_TYPE_ITEM, 1627 item, 1628 "VxLAN vni cannot be 0"); 1629 mlx5_flow_create_copy(parser, &vxlan, size); 1630 return 0; 1631 } 1632 1633 /** 1634 * Convert mark/flag action to Verbs specification. 1635 * 1636 * @param parser 1637 * Internal parser structure. 1638 * @param mark_id 1639 * Mark identifier. 1640 * 1641 * @return 1642 * 0 on success, a negative errno value otherwise and rte_errno is set. 1643 */ 1644 static int 1645 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) 1646 { 1647 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1648 struct ibv_flow_spec_action_tag tag = { 1649 .type = IBV_FLOW_SPEC_ACTION_TAG, 1650 .size = size, 1651 .tag_id = mlx5_flow_mark_set(mark_id), 1652 }; 1653 1654 assert(parser->mark); 1655 mlx5_flow_create_copy(parser, &tag, size); 1656 return 0; 1657 } 1658 1659 /** 1660 * Convert count action to Verbs specification. 1661 * 1662 * @param dev 1663 * Pointer to Ethernet device. 1664 * @param parser 1665 * Pointer to MLX5 flow parser structure. 1666 * 1667 * @return 1668 * 0 on success, a negative errno value otherwise and rte_errno is set. 1669 */ 1670 static int 1671 mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused, 1672 struct mlx5_flow_parse *parser __rte_unused) 1673 { 1674 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 1675 struct priv *priv = dev->data->dev_private; 1676 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1677 struct ibv_counter_set_init_attr init_attr = {0}; 1678 struct ibv_flow_spec_counter_action counter = { 1679 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1680 .size = size, 1681 .counter_set_handle = 0, 1682 }; 1683 1684 init_attr.counter_set_id = 0; 1685 parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr); 1686 if (!parser->cs) { 1687 rte_errno = EINVAL; 1688 return -rte_errno; 1689 } 1690 counter.counter_set_handle = parser->cs->handle; 1691 mlx5_flow_create_copy(parser, &counter, size); 1692 #endif 1693 return 0; 1694 } 1695 1696 /** 1697 * Complete flow rule creation with a drop queue. 1698 * 1699 * @param dev 1700 * Pointer to Ethernet device. 1701 * @param parser 1702 * Internal parser structure. 1703 * @param flow 1704 * Pointer to the rte_flow. 1705 * @param[out] error 1706 * Perform verbose error reporting if not NULL. 1707 * 1708 * @return 1709 * 0 on success, a negative errno value otherwise and rte_errno is set. 1710 */ 1711 static int 1712 mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev, 1713 struct mlx5_flow_parse *parser, 1714 struct rte_flow *flow, 1715 struct rte_flow_error *error) 1716 { 1717 struct priv *priv = dev->data->dev_private; 1718 struct ibv_flow_spec_action_drop *drop; 1719 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1720 1721 assert(priv->pd); 1722 assert(priv->ctx); 1723 flow->drop = 1; 1724 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr + 1725 parser->queue[HASH_RXQ_ETH].offset); 1726 *drop = (struct ibv_flow_spec_action_drop){ 1727 .type = IBV_FLOW_SPEC_ACTION_DROP, 1728 .size = size, 1729 }; 1730 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs; 1731 parser->queue[HASH_RXQ_ETH].offset += size; 1732 flow->frxq[HASH_RXQ_ETH].ibv_attr = 1733 parser->queue[HASH_RXQ_ETH].ibv_attr; 1734 if (parser->count) 1735 flow->cs = parser->cs; 1736 if (!priv->dev->data->dev_started) 1737 return 0; 1738 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 1739 flow->frxq[HASH_RXQ_ETH].ibv_flow = 1740 mlx5_glue->create_flow(priv->flow_drop_queue->qp, 1741 flow->frxq[HASH_RXQ_ETH].ibv_attr); 1742 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { 1743 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1744 NULL, "flow rule creation failure"); 1745 goto error; 1746 } 1747 return 0; 1748 error: 1749 assert(flow); 1750 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) { 1751 claim_zero(mlx5_glue->destroy_flow 1752 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 1753 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; 1754 } 1755 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) { 1756 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); 1757 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL; 1758 } 1759 if (flow->cs) { 1760 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 1761 flow->cs = NULL; 1762 parser->cs = NULL; 1763 } 1764 return -rte_errno; 1765 } 1766 1767 /** 1768 * Create hash Rx queues when RSS is enabled. 1769 * 1770 * @param dev 1771 * Pointer to Ethernet device. 1772 * @param parser 1773 * Internal parser structure. 1774 * @param flow 1775 * Pointer to the rte_flow. 1776 * @param[out] error 1777 * Perform verbose error reporting if not NULL. 1778 * 1779 * @return 1780 * 0 on success, a negative errno value otherwise and rte_errno is set. 1781 */ 1782 static int 1783 mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev, 1784 struct mlx5_flow_parse *parser, 1785 struct rte_flow *flow, 1786 struct rte_flow_error *error) 1787 { 1788 struct priv *priv = dev->data->dev_private; 1789 unsigned int i; 1790 1791 for (i = 0; i != hash_rxq_init_n; ++i) { 1792 uint64_t hash_fields; 1793 1794 if (!parser->queue[i].ibv_attr) 1795 continue; 1796 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; 1797 parser->queue[i].ibv_attr = NULL; 1798 hash_fields = hash_rxq_init[i].hash_fields; 1799 if (!priv->dev->data->dev_started) 1800 continue; 1801 flow->frxq[i].hrxq = 1802 mlx5_hrxq_get(dev, 1803 parser->rss_conf.key, 1804 parser->rss_conf.key_len, 1805 hash_fields, 1806 parser->rss_conf.queue, 1807 parser->rss_conf.queue_num); 1808 if (flow->frxq[i].hrxq) 1809 continue; 1810 flow->frxq[i].hrxq = 1811 mlx5_hrxq_new(dev, 1812 parser->rss_conf.key, 1813 parser->rss_conf.key_len, 1814 hash_fields, 1815 parser->rss_conf.queue, 1816 parser->rss_conf.queue_num); 1817 if (!flow->frxq[i].hrxq) { 1818 return rte_flow_error_set(error, ENOMEM, 1819 RTE_FLOW_ERROR_TYPE_HANDLE, 1820 NULL, 1821 "cannot create hash rxq"); 1822 } 1823 } 1824 return 0; 1825 } 1826 1827 /** 1828 * Complete flow rule creation. 1829 * 1830 * @param dev 1831 * Pointer to Ethernet device. 1832 * @param parser 1833 * Internal parser structure. 1834 * @param flow 1835 * Pointer to the rte_flow. 1836 * @param[out] error 1837 * Perform verbose error reporting if not NULL. 1838 * 1839 * @return 1840 * 0 on success, a negative errno value otherwise and rte_errno is set. 1841 */ 1842 static int 1843 mlx5_flow_create_action_queue(struct rte_eth_dev *dev, 1844 struct mlx5_flow_parse *parser, 1845 struct rte_flow *flow, 1846 struct rte_flow_error *error) 1847 { 1848 struct priv *priv = dev->data->dev_private; 1849 int ret; 1850 unsigned int i; 1851 unsigned int flows_n = 0; 1852 1853 assert(priv->pd); 1854 assert(priv->ctx); 1855 assert(!parser->drop); 1856 ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error); 1857 if (ret) 1858 goto error; 1859 if (parser->count) 1860 flow->cs = parser->cs; 1861 if (!priv->dev->data->dev_started) 1862 return 0; 1863 for (i = 0; i != hash_rxq_init_n; ++i) { 1864 if (!flow->frxq[i].hrxq) 1865 continue; 1866 flow->frxq[i].ibv_flow = 1867 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp, 1868 flow->frxq[i].ibv_attr); 1869 if (!flow->frxq[i].ibv_flow) { 1870 rte_flow_error_set(error, ENOMEM, 1871 RTE_FLOW_ERROR_TYPE_HANDLE, 1872 NULL, "flow rule creation failure"); 1873 goto error; 1874 } 1875 ++flows_n; 1876 DRV_LOG(DEBUG, "port %u %p type %d QP %p ibv_flow %p", 1877 dev->data->port_id, 1878 (void *)flow, i, 1879 (void *)flow->frxq[i].hrxq, 1880 (void *)flow->frxq[i].ibv_flow); 1881 } 1882 if (!flows_n) { 1883 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, 1884 NULL, "internal error in flow creation"); 1885 goto error; 1886 } 1887 for (i = 0; i != parser->rss_conf.queue_num; ++i) { 1888 struct mlx5_rxq_data *q = 1889 (*priv->rxqs)[parser->rss_conf.queue[i]]; 1890 1891 q->mark |= parser->mark; 1892 } 1893 return 0; 1894 error: 1895 ret = rte_errno; /* Save rte_errno before cleanup. */ 1896 assert(flow); 1897 for (i = 0; i != hash_rxq_init_n; ++i) { 1898 if (flow->frxq[i].ibv_flow) { 1899 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; 1900 1901 claim_zero(mlx5_glue->destroy_flow(ibv_flow)); 1902 } 1903 if (flow->frxq[i].hrxq) 1904 mlx5_hrxq_release(dev, flow->frxq[i].hrxq); 1905 if (flow->frxq[i].ibv_attr) 1906 rte_free(flow->frxq[i].ibv_attr); 1907 } 1908 if (flow->cs) { 1909 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 1910 flow->cs = NULL; 1911 parser->cs = NULL; 1912 } 1913 rte_errno = ret; /* Restore rte_errno. */ 1914 return -rte_errno; 1915 } 1916 1917 /** 1918 * Convert a flow. 1919 * 1920 * @param dev 1921 * Pointer to Ethernet device. 1922 * @param list 1923 * Pointer to a TAILQ flow list. 1924 * @param[in] attr 1925 * Flow rule attributes. 1926 * @param[in] pattern 1927 * Pattern specification (list terminated by the END pattern item). 1928 * @param[in] actions 1929 * Associated actions (list terminated by the END action). 1930 * @param[out] error 1931 * Perform verbose error reporting if not NULL. 1932 * 1933 * @return 1934 * A flow on success, NULL otherwise and rte_errno is set. 1935 */ 1936 static struct rte_flow * 1937 mlx5_flow_list_create(struct rte_eth_dev *dev, 1938 struct mlx5_flows *list, 1939 const struct rte_flow_attr *attr, 1940 const struct rte_flow_item items[], 1941 const struct rte_flow_action actions[], 1942 struct rte_flow_error *error) 1943 { 1944 struct mlx5_flow_parse parser = { .create = 1, }; 1945 struct rte_flow *flow = NULL; 1946 unsigned int i; 1947 int ret; 1948 1949 ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser); 1950 if (ret) 1951 goto exit; 1952 flow = rte_calloc(__func__, 1, 1953 sizeof(*flow) + 1954 parser.rss_conf.queue_num * sizeof(uint16_t), 1955 0); 1956 if (!flow) { 1957 rte_flow_error_set(error, ENOMEM, 1958 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1959 NULL, 1960 "cannot allocate flow memory"); 1961 return NULL; 1962 } 1963 /* Copy configuration. */ 1964 flow->queues = (uint16_t (*)[])(flow + 1); 1965 flow->rss_conf = (struct rte_flow_action_rss){ 1966 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 1967 .level = 0, 1968 .types = parser.rss_conf.types, 1969 .key_len = parser.rss_conf.key_len, 1970 .queue_num = parser.rss_conf.queue_num, 1971 .key = memcpy(flow->rss_key, parser.rss_conf.key, 1972 sizeof(*parser.rss_conf.key) * 1973 parser.rss_conf.key_len), 1974 .queue = memcpy(flow->queues, parser.rss_conf.queue, 1975 sizeof(*parser.rss_conf.queue) * 1976 parser.rss_conf.queue_num), 1977 }; 1978 flow->mark = parser.mark; 1979 /* finalise the flow. */ 1980 if (parser.drop) 1981 ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow, 1982 error); 1983 else 1984 ret = mlx5_flow_create_action_queue(dev, &parser, flow, error); 1985 if (ret) 1986 goto exit; 1987 TAILQ_INSERT_TAIL(list, flow, next); 1988 DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id, 1989 (void *)flow); 1990 return flow; 1991 exit: 1992 DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id, 1993 error->message); 1994 for (i = 0; i != hash_rxq_init_n; ++i) { 1995 if (parser.queue[i].ibv_attr) 1996 rte_free(parser.queue[i].ibv_attr); 1997 } 1998 rte_free(flow); 1999 return NULL; 2000 } 2001 2002 /** 2003 * Validate a flow supported by the NIC. 2004 * 2005 * @see rte_flow_validate() 2006 * @see rte_flow_ops 2007 */ 2008 int 2009 mlx5_flow_validate(struct rte_eth_dev *dev, 2010 const struct rte_flow_attr *attr, 2011 const struct rte_flow_item items[], 2012 const struct rte_flow_action actions[], 2013 struct rte_flow_error *error) 2014 { 2015 struct mlx5_flow_parse parser = { .create = 0, }; 2016 2017 return mlx5_flow_convert(dev, attr, items, actions, error, &parser); 2018 } 2019 2020 /** 2021 * Create a flow. 2022 * 2023 * @see rte_flow_create() 2024 * @see rte_flow_ops 2025 */ 2026 struct rte_flow * 2027 mlx5_flow_create(struct rte_eth_dev *dev, 2028 const struct rte_flow_attr *attr, 2029 const struct rte_flow_item items[], 2030 const struct rte_flow_action actions[], 2031 struct rte_flow_error *error) 2032 { 2033 struct priv *priv = dev->data->dev_private; 2034 2035 return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions, 2036 error); 2037 } 2038 2039 /** 2040 * Destroy a flow in a list. 2041 * 2042 * @param dev 2043 * Pointer to Ethernet device. 2044 * @param list 2045 * Pointer to a TAILQ flow list. 2046 * @param[in] flow 2047 * Flow to destroy. 2048 */ 2049 static void 2050 mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2051 struct rte_flow *flow) 2052 { 2053 struct priv *priv = dev->data->dev_private; 2054 unsigned int i; 2055 2056 if (flow->drop || !flow->mark) 2057 goto free; 2058 for (i = 0; i != flow->rss_conf.queue_num; ++i) { 2059 struct rte_flow *tmp; 2060 int mark = 0; 2061 2062 /* 2063 * To remove the mark from the queue, the queue must not be 2064 * present in any other marked flow (RSS or not). 2065 */ 2066 TAILQ_FOREACH(tmp, list, next) { 2067 unsigned int j; 2068 uint16_t *tqs = NULL; 2069 uint16_t tq_n = 0; 2070 2071 if (!tmp->mark) 2072 continue; 2073 for (j = 0; j != hash_rxq_init_n; ++j) { 2074 if (!tmp->frxq[j].hrxq) 2075 continue; 2076 tqs = tmp->frxq[j].hrxq->ind_table->queues; 2077 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n; 2078 } 2079 if (!tq_n) 2080 continue; 2081 for (j = 0; (j != tq_n) && !mark; j++) 2082 if (tqs[j] == (*flow->queues)[i]) 2083 mark = 1; 2084 } 2085 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark; 2086 } 2087 free: 2088 if (flow->drop) { 2089 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) 2090 claim_zero(mlx5_glue->destroy_flow 2091 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 2092 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); 2093 } else { 2094 for (i = 0; i != hash_rxq_init_n; ++i) { 2095 struct mlx5_flow *frxq = &flow->frxq[i]; 2096 2097 if (frxq->ibv_flow) 2098 claim_zero(mlx5_glue->destroy_flow 2099 (frxq->ibv_flow)); 2100 if (frxq->hrxq) 2101 mlx5_hrxq_release(dev, frxq->hrxq); 2102 if (frxq->ibv_attr) 2103 rte_free(frxq->ibv_attr); 2104 } 2105 } 2106 if (flow->cs) { 2107 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 2108 flow->cs = NULL; 2109 } 2110 TAILQ_REMOVE(list, flow, next); 2111 DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id, 2112 (void *)flow); 2113 rte_free(flow); 2114 } 2115 2116 /** 2117 * Destroy all flows. 2118 * 2119 * @param dev 2120 * Pointer to Ethernet device. 2121 * @param list 2122 * Pointer to a TAILQ flow list. 2123 */ 2124 void 2125 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 2126 { 2127 while (!TAILQ_EMPTY(list)) { 2128 struct rte_flow *flow; 2129 2130 flow = TAILQ_FIRST(list); 2131 mlx5_flow_list_destroy(dev, list, flow); 2132 } 2133 } 2134 2135 /** 2136 * Create drop queue. 2137 * 2138 * @param dev 2139 * Pointer to Ethernet device. 2140 * 2141 * @return 2142 * 0 on success, a negative errno value otherwise and rte_errno is set. 2143 */ 2144 int 2145 mlx5_flow_create_drop_queue(struct rte_eth_dev *dev) 2146 { 2147 struct priv *priv = dev->data->dev_private; 2148 struct mlx5_hrxq_drop *fdq = NULL; 2149 2150 assert(priv->pd); 2151 assert(priv->ctx); 2152 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); 2153 if (!fdq) { 2154 DRV_LOG(WARNING, 2155 "port %u cannot allocate memory for drop queue", 2156 dev->data->port_id); 2157 rte_errno = ENOMEM; 2158 return -rte_errno; 2159 } 2160 fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0); 2161 if (!fdq->cq) { 2162 DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue", 2163 dev->data->port_id); 2164 rte_errno = errno; 2165 goto error; 2166 } 2167 fdq->wq = mlx5_glue->create_wq 2168 (priv->ctx, 2169 &(struct ibv_wq_init_attr){ 2170 .wq_type = IBV_WQT_RQ, 2171 .max_wr = 1, 2172 .max_sge = 1, 2173 .pd = priv->pd, 2174 .cq = fdq->cq, 2175 }); 2176 if (!fdq->wq) { 2177 DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue", 2178 dev->data->port_id); 2179 rte_errno = errno; 2180 goto error; 2181 } 2182 fdq->ind_table = mlx5_glue->create_rwq_ind_table 2183 (priv->ctx, 2184 &(struct ibv_rwq_ind_table_init_attr){ 2185 .log_ind_tbl_size = 0, 2186 .ind_tbl = &fdq->wq, 2187 .comp_mask = 0, 2188 }); 2189 if (!fdq->ind_table) { 2190 DRV_LOG(WARNING, 2191 "port %u cannot allocate indirection table for drop" 2192 " queue", 2193 dev->data->port_id); 2194 rte_errno = errno; 2195 goto error; 2196 } 2197 fdq->qp = mlx5_glue->create_qp_ex 2198 (priv->ctx, 2199 &(struct ibv_qp_init_attr_ex){ 2200 .qp_type = IBV_QPT_RAW_PACKET, 2201 .comp_mask = 2202 IBV_QP_INIT_ATTR_PD | 2203 IBV_QP_INIT_ATTR_IND_TABLE | 2204 IBV_QP_INIT_ATTR_RX_HASH, 2205 .rx_hash_conf = (struct ibv_rx_hash_conf){ 2206 .rx_hash_function = 2207 IBV_RX_HASH_FUNC_TOEPLITZ, 2208 .rx_hash_key_len = rss_hash_default_key_len, 2209 .rx_hash_key = rss_hash_default_key, 2210 .rx_hash_fields_mask = 0, 2211 }, 2212 .rwq_ind_tbl = fdq->ind_table, 2213 .pd = priv->pd 2214 }); 2215 if (!fdq->qp) { 2216 DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue", 2217 dev->data->port_id); 2218 rte_errno = errno; 2219 goto error; 2220 } 2221 priv->flow_drop_queue = fdq; 2222 return 0; 2223 error: 2224 if (fdq->qp) 2225 claim_zero(mlx5_glue->destroy_qp(fdq->qp)); 2226 if (fdq->ind_table) 2227 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table)); 2228 if (fdq->wq) 2229 claim_zero(mlx5_glue->destroy_wq(fdq->wq)); 2230 if (fdq->cq) 2231 claim_zero(mlx5_glue->destroy_cq(fdq->cq)); 2232 if (fdq) 2233 rte_free(fdq); 2234 priv->flow_drop_queue = NULL; 2235 return -rte_errno; 2236 } 2237 2238 /** 2239 * Delete drop queue. 2240 * 2241 * @param dev 2242 * Pointer to Ethernet device. 2243 */ 2244 void 2245 mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev) 2246 { 2247 struct priv *priv = dev->data->dev_private; 2248 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; 2249 2250 if (!fdq) 2251 return; 2252 if (fdq->qp) 2253 claim_zero(mlx5_glue->destroy_qp(fdq->qp)); 2254 if (fdq->ind_table) 2255 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table)); 2256 if (fdq->wq) 2257 claim_zero(mlx5_glue->destroy_wq(fdq->wq)); 2258 if (fdq->cq) 2259 claim_zero(mlx5_glue->destroy_cq(fdq->cq)); 2260 rte_free(fdq); 2261 priv->flow_drop_queue = NULL; 2262 } 2263 2264 /** 2265 * Remove all flows. 2266 * 2267 * @param dev 2268 * Pointer to Ethernet device. 2269 * @param list 2270 * Pointer to a TAILQ flow list. 2271 */ 2272 void 2273 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 2274 { 2275 struct priv *priv = dev->data->dev_private; 2276 struct rte_flow *flow; 2277 2278 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 2279 unsigned int i; 2280 struct mlx5_ind_table_ibv *ind_tbl = NULL; 2281 2282 if (flow->drop) { 2283 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) 2284 continue; 2285 claim_zero(mlx5_glue->destroy_flow 2286 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 2287 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; 2288 DRV_LOG(DEBUG, "port %u flow %p removed", 2289 dev->data->port_id, (void *)flow); 2290 /* Next flow. */ 2291 continue; 2292 } 2293 /* Verify the flow has not already been cleaned. */ 2294 for (i = 0; i != hash_rxq_init_n; ++i) { 2295 if (!flow->frxq[i].ibv_flow) 2296 continue; 2297 /* 2298 * Indirection table may be necessary to remove the 2299 * flags in the Rx queues. 2300 * This helps to speed-up the process by avoiding 2301 * another loop. 2302 */ 2303 ind_tbl = flow->frxq[i].hrxq->ind_table; 2304 break; 2305 } 2306 if (i == hash_rxq_init_n) 2307 return; 2308 if (flow->mark) { 2309 assert(ind_tbl); 2310 for (i = 0; i != ind_tbl->queues_n; ++i) 2311 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; 2312 } 2313 for (i = 0; i != hash_rxq_init_n; ++i) { 2314 if (!flow->frxq[i].ibv_flow) 2315 continue; 2316 claim_zero(mlx5_glue->destroy_flow 2317 (flow->frxq[i].ibv_flow)); 2318 flow->frxq[i].ibv_flow = NULL; 2319 mlx5_hrxq_release(dev, flow->frxq[i].hrxq); 2320 flow->frxq[i].hrxq = NULL; 2321 } 2322 DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id, 2323 (void *)flow); 2324 } 2325 } 2326 2327 /** 2328 * Add all flows. 2329 * 2330 * @param dev 2331 * Pointer to Ethernet device. 2332 * @param list 2333 * Pointer to a TAILQ flow list. 2334 * 2335 * @return 2336 * 0 on success, a negative errno value otherwise and rte_errno is set. 2337 */ 2338 int 2339 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 2340 { 2341 struct priv *priv = dev->data->dev_private; 2342 struct rte_flow *flow; 2343 2344 TAILQ_FOREACH(flow, list, next) { 2345 unsigned int i; 2346 2347 if (flow->drop) { 2348 flow->frxq[HASH_RXQ_ETH].ibv_flow = 2349 mlx5_glue->create_flow 2350 (priv->flow_drop_queue->qp, 2351 flow->frxq[HASH_RXQ_ETH].ibv_attr); 2352 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { 2353 DRV_LOG(DEBUG, 2354 "port %u flow %p cannot be applied", 2355 dev->data->port_id, (void *)flow); 2356 rte_errno = EINVAL; 2357 return -rte_errno; 2358 } 2359 DRV_LOG(DEBUG, "port %u flow %p applied", 2360 dev->data->port_id, (void *)flow); 2361 /* Next flow. */ 2362 continue; 2363 } 2364 for (i = 0; i != hash_rxq_init_n; ++i) { 2365 if (!flow->frxq[i].ibv_attr) 2366 continue; 2367 flow->frxq[i].hrxq = 2368 mlx5_hrxq_get(dev, flow->rss_conf.key, 2369 flow->rss_conf.key_len, 2370 hash_rxq_init[i].hash_fields, 2371 flow->rss_conf.queue, 2372 flow->rss_conf.queue_num); 2373 if (flow->frxq[i].hrxq) 2374 goto flow_create; 2375 flow->frxq[i].hrxq = 2376 mlx5_hrxq_new(dev, flow->rss_conf.key, 2377 flow->rss_conf.key_len, 2378 hash_rxq_init[i].hash_fields, 2379 flow->rss_conf.queue, 2380 flow->rss_conf.queue_num); 2381 if (!flow->frxq[i].hrxq) { 2382 DRV_LOG(DEBUG, 2383 "port %u flow %p cannot be applied", 2384 dev->data->port_id, (void *)flow); 2385 rte_errno = EINVAL; 2386 return -rte_errno; 2387 } 2388 flow_create: 2389 flow->frxq[i].ibv_flow = 2390 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp, 2391 flow->frxq[i].ibv_attr); 2392 if (!flow->frxq[i].ibv_flow) { 2393 DRV_LOG(DEBUG, 2394 "port %u flow %p cannot be applied", 2395 dev->data->port_id, (void *)flow); 2396 rte_errno = EINVAL; 2397 return -rte_errno; 2398 } 2399 DRV_LOG(DEBUG, "port %u flow %p applied", 2400 dev->data->port_id, (void *)flow); 2401 } 2402 if (!flow->mark) 2403 continue; 2404 for (i = 0; i != flow->rss_conf.queue_num; ++i) 2405 (*priv->rxqs)[flow->rss_conf.queue[i]]->mark = 1; 2406 } 2407 return 0; 2408 } 2409 2410 /** 2411 * Verify the flow list is empty 2412 * 2413 * @param dev 2414 * Pointer to Ethernet device. 2415 * 2416 * @return the number of flows not released. 2417 */ 2418 int 2419 mlx5_flow_verify(struct rte_eth_dev *dev) 2420 { 2421 struct priv *priv = dev->data->dev_private; 2422 struct rte_flow *flow; 2423 int ret = 0; 2424 2425 TAILQ_FOREACH(flow, &priv->flows, next) { 2426 DRV_LOG(DEBUG, "port %u flow %p still referenced", 2427 dev->data->port_id, (void *)flow); 2428 ++ret; 2429 } 2430 return ret; 2431 } 2432 2433 /** 2434 * Enable a control flow configured from the control plane. 2435 * 2436 * @param dev 2437 * Pointer to Ethernet device. 2438 * @param eth_spec 2439 * An Ethernet flow spec to apply. 2440 * @param eth_mask 2441 * An Ethernet flow mask to apply. 2442 * @param vlan_spec 2443 * A VLAN flow spec to apply. 2444 * @param vlan_mask 2445 * A VLAN flow mask to apply. 2446 * 2447 * @return 2448 * 0 on success, a negative errno value otherwise and rte_errno is set. 2449 */ 2450 int 2451 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2452 struct rte_flow_item_eth *eth_spec, 2453 struct rte_flow_item_eth *eth_mask, 2454 struct rte_flow_item_vlan *vlan_spec, 2455 struct rte_flow_item_vlan *vlan_mask) 2456 { 2457 struct priv *priv = dev->data->dev_private; 2458 const struct rte_flow_attr attr = { 2459 .ingress = 1, 2460 .priority = MLX5_CTRL_FLOW_PRIORITY, 2461 }; 2462 struct rte_flow_item items[] = { 2463 { 2464 .type = RTE_FLOW_ITEM_TYPE_ETH, 2465 .spec = eth_spec, 2466 .last = NULL, 2467 .mask = eth_mask, 2468 }, 2469 { 2470 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2471 RTE_FLOW_ITEM_TYPE_END, 2472 .spec = vlan_spec, 2473 .last = NULL, 2474 .mask = vlan_mask, 2475 }, 2476 { 2477 .type = RTE_FLOW_ITEM_TYPE_END, 2478 }, 2479 }; 2480 uint16_t queue[priv->reta_idx_n]; 2481 struct rte_flow_action_rss action_rss = { 2482 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 2483 .level = 0, 2484 .types = priv->rss_conf.rss_hf, 2485 .key_len = priv->rss_conf.rss_key_len, 2486 .queue_num = priv->reta_idx_n, 2487 .key = priv->rss_conf.rss_key, 2488 .queue = queue, 2489 }; 2490 struct rte_flow_action actions[] = { 2491 { 2492 .type = RTE_FLOW_ACTION_TYPE_RSS, 2493 .conf = &action_rss, 2494 }, 2495 { 2496 .type = RTE_FLOW_ACTION_TYPE_END, 2497 }, 2498 }; 2499 struct rte_flow *flow; 2500 struct rte_flow_error error; 2501 unsigned int i; 2502 2503 if (!priv->reta_idx_n) { 2504 rte_errno = EINVAL; 2505 return -rte_errno; 2506 } 2507 for (i = 0; i != priv->reta_idx_n; ++i) 2508 queue[i] = (*priv->reta_idx)[i]; 2509 flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items, 2510 actions, &error); 2511 if (!flow) 2512 return -rte_errno; 2513 return 0; 2514 } 2515 2516 /** 2517 * Enable a flow control configured from the control plane. 2518 * 2519 * @param dev 2520 * Pointer to Ethernet device. 2521 * @param eth_spec 2522 * An Ethernet flow spec to apply. 2523 * @param eth_mask 2524 * An Ethernet flow mask to apply. 2525 * 2526 * @return 2527 * 0 on success, a negative errno value otherwise and rte_errno is set. 2528 */ 2529 int 2530 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2531 struct rte_flow_item_eth *eth_spec, 2532 struct rte_flow_item_eth *eth_mask) 2533 { 2534 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2535 } 2536 2537 /** 2538 * Destroy a flow. 2539 * 2540 * @see rte_flow_destroy() 2541 * @see rte_flow_ops 2542 */ 2543 int 2544 mlx5_flow_destroy(struct rte_eth_dev *dev, 2545 struct rte_flow *flow, 2546 struct rte_flow_error *error __rte_unused) 2547 { 2548 struct priv *priv = dev->data->dev_private; 2549 2550 mlx5_flow_list_destroy(dev, &priv->flows, flow); 2551 return 0; 2552 } 2553 2554 /** 2555 * Destroy all flows. 2556 * 2557 * @see rte_flow_flush() 2558 * @see rte_flow_ops 2559 */ 2560 int 2561 mlx5_flow_flush(struct rte_eth_dev *dev, 2562 struct rte_flow_error *error __rte_unused) 2563 { 2564 struct priv *priv = dev->data->dev_private; 2565 2566 mlx5_flow_list_flush(dev, &priv->flows); 2567 return 0; 2568 } 2569 2570 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 2571 /** 2572 * Query flow counter. 2573 * 2574 * @param cs 2575 * the counter set. 2576 * @param counter_value 2577 * returned data from the counter. 2578 * 2579 * @return 2580 * 0 on success, a negative errno value otherwise and rte_errno is set. 2581 */ 2582 static int 2583 mlx5_flow_query_count(struct ibv_counter_set *cs, 2584 struct mlx5_flow_counter_stats *counter_stats, 2585 struct rte_flow_query_count *query_count, 2586 struct rte_flow_error *error) 2587 { 2588 uint64_t counters[2]; 2589 struct ibv_query_counter_set_attr query_cs_attr = { 2590 .cs = cs, 2591 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 2592 }; 2593 struct ibv_counter_set_data query_out = { 2594 .out = counters, 2595 .outlen = 2 * sizeof(uint64_t), 2596 }; 2597 int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out); 2598 2599 if (err) 2600 return rte_flow_error_set(error, err, 2601 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2602 NULL, 2603 "cannot read counter"); 2604 query_count->hits_set = 1; 2605 query_count->bytes_set = 1; 2606 query_count->hits = counters[0] - counter_stats->hits; 2607 query_count->bytes = counters[1] - counter_stats->bytes; 2608 if (query_count->reset) { 2609 counter_stats->hits = counters[0]; 2610 counter_stats->bytes = counters[1]; 2611 } 2612 return 0; 2613 } 2614 2615 /** 2616 * Query a flows. 2617 * 2618 * @see rte_flow_query() 2619 * @see rte_flow_ops 2620 */ 2621 int 2622 mlx5_flow_query(struct rte_eth_dev *dev __rte_unused, 2623 struct rte_flow *flow, 2624 enum rte_flow_action_type action __rte_unused, 2625 void *data, 2626 struct rte_flow_error *error) 2627 { 2628 if (flow->cs) { 2629 int ret; 2630 2631 ret = mlx5_flow_query_count(flow->cs, 2632 &flow->counter_stats, 2633 (struct rte_flow_query_count *)data, 2634 error); 2635 if (ret) 2636 return ret; 2637 } else { 2638 return rte_flow_error_set(error, EINVAL, 2639 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2640 NULL, 2641 "no counter found for flow"); 2642 } 2643 return 0; 2644 } 2645 #endif 2646 2647 /** 2648 * Isolated mode. 2649 * 2650 * @see rte_flow_isolate() 2651 * @see rte_flow_ops 2652 */ 2653 int 2654 mlx5_flow_isolate(struct rte_eth_dev *dev, 2655 int enable, 2656 struct rte_flow_error *error) 2657 { 2658 struct priv *priv = dev->data->dev_private; 2659 2660 if (dev->data->dev_started) { 2661 rte_flow_error_set(error, EBUSY, 2662 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2663 NULL, 2664 "port must be stopped first"); 2665 return -rte_errno; 2666 } 2667 priv->isolated = !!enable; 2668 if (enable) 2669 priv->dev->dev_ops = &mlx5_dev_ops_isolate; 2670 else 2671 priv->dev->dev_ops = &mlx5_dev_ops; 2672 return 0; 2673 } 2674 2675 /** 2676 * Convert a flow director filter to a generic flow. 2677 * 2678 * @param dev 2679 * Pointer to Ethernet device. 2680 * @param fdir_filter 2681 * Flow director filter to add. 2682 * @param attributes 2683 * Generic flow parameters structure. 2684 * 2685 * @return 2686 * 0 on success, a negative errno value otherwise and rte_errno is set. 2687 */ 2688 static int 2689 mlx5_fdir_filter_convert(struct rte_eth_dev *dev, 2690 const struct rte_eth_fdir_filter *fdir_filter, 2691 struct mlx5_fdir *attributes) 2692 { 2693 struct priv *priv = dev->data->dev_private; 2694 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2695 const struct rte_eth_fdir_masks *mask = 2696 &dev->data->dev_conf.fdir_conf.mask; 2697 2698 /* Validate queue number. */ 2699 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2700 DRV_LOG(ERR, "port %u invalid queue number %d", 2701 dev->data->port_id, fdir_filter->action.rx_queue); 2702 rte_errno = EINVAL; 2703 return -rte_errno; 2704 } 2705 attributes->attr.ingress = 1; 2706 attributes->items[0] = (struct rte_flow_item) { 2707 .type = RTE_FLOW_ITEM_TYPE_ETH, 2708 .spec = &attributes->l2, 2709 .mask = &attributes->l2_mask, 2710 }; 2711 switch (fdir_filter->action.behavior) { 2712 case RTE_ETH_FDIR_ACCEPT: 2713 attributes->actions[0] = (struct rte_flow_action){ 2714 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2715 .conf = &attributes->queue, 2716 }; 2717 break; 2718 case RTE_ETH_FDIR_REJECT: 2719 attributes->actions[0] = (struct rte_flow_action){ 2720 .type = RTE_FLOW_ACTION_TYPE_DROP, 2721 }; 2722 break; 2723 default: 2724 DRV_LOG(ERR, "port %u invalid behavior %d", 2725 dev->data->port_id, 2726 fdir_filter->action.behavior); 2727 rte_errno = ENOTSUP; 2728 return -rte_errno; 2729 } 2730 attributes->queue.index = fdir_filter->action.rx_queue; 2731 /* Handle L3. */ 2732 switch (fdir_filter->input.flow_type) { 2733 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2734 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2735 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2736 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2737 .src_addr = input->flow.ip4_flow.src_ip, 2738 .dst_addr = input->flow.ip4_flow.dst_ip, 2739 .time_to_live = input->flow.ip4_flow.ttl, 2740 .type_of_service = input->flow.ip4_flow.tos, 2741 .next_proto_id = input->flow.ip4_flow.proto, 2742 }; 2743 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){ 2744 .src_addr = mask->ipv4_mask.src_ip, 2745 .dst_addr = mask->ipv4_mask.dst_ip, 2746 .time_to_live = mask->ipv4_mask.ttl, 2747 .type_of_service = mask->ipv4_mask.tos, 2748 .next_proto_id = mask->ipv4_mask.proto, 2749 }; 2750 attributes->items[1] = (struct rte_flow_item){ 2751 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2752 .spec = &attributes->l3, 2753 .mask = &attributes->l3_mask, 2754 }; 2755 break; 2756 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2757 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2758 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2759 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2760 .hop_limits = input->flow.ipv6_flow.hop_limits, 2761 .proto = input->flow.ipv6_flow.proto, 2762 }; 2763 2764 memcpy(attributes->l3.ipv6.hdr.src_addr, 2765 input->flow.ipv6_flow.src_ip, 2766 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2767 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2768 input->flow.ipv6_flow.dst_ip, 2769 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2770 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 2771 mask->ipv6_mask.src_ip, 2772 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2773 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 2774 mask->ipv6_mask.dst_ip, 2775 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2776 attributes->items[1] = (struct rte_flow_item){ 2777 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2778 .spec = &attributes->l3, 2779 .mask = &attributes->l3_mask, 2780 }; 2781 break; 2782 default: 2783 DRV_LOG(ERR, "port %u invalid flow type%d", 2784 dev->data->port_id, fdir_filter->input.flow_type); 2785 rte_errno = ENOTSUP; 2786 return -rte_errno; 2787 } 2788 /* Handle L4. */ 2789 switch (fdir_filter->input.flow_type) { 2790 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2791 attributes->l4.udp.hdr = (struct udp_hdr){ 2792 .src_port = input->flow.udp4_flow.src_port, 2793 .dst_port = input->flow.udp4_flow.dst_port, 2794 }; 2795 attributes->l4_mask.udp.hdr = (struct udp_hdr){ 2796 .src_port = mask->src_port_mask, 2797 .dst_port = mask->dst_port_mask, 2798 }; 2799 attributes->items[2] = (struct rte_flow_item){ 2800 .type = RTE_FLOW_ITEM_TYPE_UDP, 2801 .spec = &attributes->l4, 2802 .mask = &attributes->l4_mask, 2803 }; 2804 break; 2805 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2806 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2807 .src_port = input->flow.tcp4_flow.src_port, 2808 .dst_port = input->flow.tcp4_flow.dst_port, 2809 }; 2810 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ 2811 .src_port = mask->src_port_mask, 2812 .dst_port = mask->dst_port_mask, 2813 }; 2814 attributes->items[2] = (struct rte_flow_item){ 2815 .type = RTE_FLOW_ITEM_TYPE_TCP, 2816 .spec = &attributes->l4, 2817 .mask = &attributes->l4_mask, 2818 }; 2819 break; 2820 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2821 attributes->l4.udp.hdr = (struct udp_hdr){ 2822 .src_port = input->flow.udp6_flow.src_port, 2823 .dst_port = input->flow.udp6_flow.dst_port, 2824 }; 2825 attributes->l4_mask.udp.hdr = (struct udp_hdr){ 2826 .src_port = mask->src_port_mask, 2827 .dst_port = mask->dst_port_mask, 2828 }; 2829 attributes->items[2] = (struct rte_flow_item){ 2830 .type = RTE_FLOW_ITEM_TYPE_UDP, 2831 .spec = &attributes->l4, 2832 .mask = &attributes->l4_mask, 2833 }; 2834 break; 2835 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2836 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2837 .src_port = input->flow.tcp6_flow.src_port, 2838 .dst_port = input->flow.tcp6_flow.dst_port, 2839 }; 2840 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ 2841 .src_port = mask->src_port_mask, 2842 .dst_port = mask->dst_port_mask, 2843 }; 2844 attributes->items[2] = (struct rte_flow_item){ 2845 .type = RTE_FLOW_ITEM_TYPE_TCP, 2846 .spec = &attributes->l4, 2847 .mask = &attributes->l4_mask, 2848 }; 2849 break; 2850 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2851 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2852 break; 2853 default: 2854 DRV_LOG(ERR, "port %u invalid flow type%d", 2855 dev->data->port_id, fdir_filter->input.flow_type); 2856 rte_errno = ENOTSUP; 2857 return -rte_errno; 2858 } 2859 return 0; 2860 } 2861 2862 /** 2863 * Add new flow director filter and store it in list. 2864 * 2865 * @param dev 2866 * Pointer to Ethernet device. 2867 * @param fdir_filter 2868 * Flow director filter to add. 2869 * 2870 * @return 2871 * 0 on success, a negative errno value otherwise and rte_errno is set. 2872 */ 2873 static int 2874 mlx5_fdir_filter_add(struct rte_eth_dev *dev, 2875 const struct rte_eth_fdir_filter *fdir_filter) 2876 { 2877 struct priv *priv = dev->data->dev_private; 2878 struct mlx5_fdir attributes = { 2879 .attr.group = 0, 2880 .l2_mask = { 2881 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2882 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2883 .type = 0, 2884 }, 2885 }; 2886 struct mlx5_flow_parse parser = { 2887 .layer = HASH_RXQ_ETH, 2888 }; 2889 struct rte_flow_error error; 2890 struct rte_flow *flow; 2891 int ret; 2892 2893 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); 2894 if (ret) 2895 return ret; 2896 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, 2897 attributes.actions, &error, &parser); 2898 if (ret) 2899 return ret; 2900 flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr, 2901 attributes.items, attributes.actions, 2902 &error); 2903 if (flow) { 2904 DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id, 2905 (void *)flow); 2906 return 0; 2907 } 2908 return -rte_errno; 2909 } 2910 2911 /** 2912 * Delete specific filter. 2913 * 2914 * @param dev 2915 * Pointer to Ethernet device. 2916 * @param fdir_filter 2917 * Filter to be deleted. 2918 * 2919 * @return 2920 * 0 on success, a negative errno value otherwise and rte_errno is set. 2921 */ 2922 static int 2923 mlx5_fdir_filter_delete(struct rte_eth_dev *dev, 2924 const struct rte_eth_fdir_filter *fdir_filter) 2925 { 2926 struct priv *priv = dev->data->dev_private; 2927 struct mlx5_fdir attributes = { 2928 .attr.group = 0, 2929 }; 2930 struct mlx5_flow_parse parser = { 2931 .create = 1, 2932 .layer = HASH_RXQ_ETH, 2933 }; 2934 struct rte_flow_error error; 2935 struct rte_flow *flow; 2936 unsigned int i; 2937 int ret; 2938 2939 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); 2940 if (ret) 2941 return ret; 2942 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, 2943 attributes.actions, &error, &parser); 2944 if (ret) 2945 goto exit; 2946 /* 2947 * Special case for drop action which is only set in the 2948 * specifications when the flow is created. In this situation the 2949 * drop specification is missing. 2950 */ 2951 if (parser.drop) { 2952 struct ibv_flow_spec_action_drop *drop; 2953 2954 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr + 2955 parser.queue[HASH_RXQ_ETH].offset); 2956 *drop = (struct ibv_flow_spec_action_drop){ 2957 .type = IBV_FLOW_SPEC_ACTION_DROP, 2958 .size = sizeof(struct ibv_flow_spec_action_drop), 2959 }; 2960 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++; 2961 } 2962 TAILQ_FOREACH(flow, &priv->flows, next) { 2963 struct ibv_flow_attr *attr; 2964 struct ibv_spec_header *attr_h; 2965 void *spec; 2966 struct ibv_flow_attr *flow_attr; 2967 struct ibv_spec_header *flow_h; 2968 void *flow_spec; 2969 unsigned int specs_n; 2970 2971 attr = parser.queue[HASH_RXQ_ETH].ibv_attr; 2972 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; 2973 /* Compare first the attributes. */ 2974 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) 2975 continue; 2976 if (attr->num_of_specs == 0) 2977 continue; 2978 spec = (void *)((uintptr_t)attr + 2979 sizeof(struct ibv_flow_attr)); 2980 flow_spec = (void *)((uintptr_t)flow_attr + 2981 sizeof(struct ibv_flow_attr)); 2982 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs); 2983 for (i = 0; i != specs_n; ++i) { 2984 attr_h = spec; 2985 flow_h = flow_spec; 2986 if (memcmp(spec, flow_spec, 2987 RTE_MIN(attr_h->size, flow_h->size))) 2988 goto wrong_flow; 2989 spec = (void *)((uintptr_t)spec + attr_h->size); 2990 flow_spec = (void *)((uintptr_t)flow_spec + 2991 flow_h->size); 2992 } 2993 /* At this point, the flow match. */ 2994 break; 2995 wrong_flow: 2996 /* The flow does not match. */ 2997 continue; 2998 } 2999 ret = rte_errno; /* Save rte_errno before cleanup. */ 3000 if (flow) 3001 mlx5_flow_list_destroy(dev, &priv->flows, flow); 3002 exit: 3003 for (i = 0; i != hash_rxq_init_n; ++i) { 3004 if (parser.queue[i].ibv_attr) 3005 rte_free(parser.queue[i].ibv_attr); 3006 } 3007 rte_errno = ret; /* Restore rte_errno. */ 3008 return -rte_errno; 3009 } 3010 3011 /** 3012 * Update queue for specific filter. 3013 * 3014 * @param dev 3015 * Pointer to Ethernet device. 3016 * @param fdir_filter 3017 * Filter to be updated. 3018 * 3019 * @return 3020 * 0 on success, a negative errno value otherwise and rte_errno is set. 3021 */ 3022 static int 3023 mlx5_fdir_filter_update(struct rte_eth_dev *dev, 3024 const struct rte_eth_fdir_filter *fdir_filter) 3025 { 3026 int ret; 3027 3028 ret = mlx5_fdir_filter_delete(dev, fdir_filter); 3029 if (ret) 3030 return ret; 3031 return mlx5_fdir_filter_add(dev, fdir_filter); 3032 } 3033 3034 /** 3035 * Flush all filters. 3036 * 3037 * @param dev 3038 * Pointer to Ethernet device. 3039 */ 3040 static void 3041 mlx5_fdir_filter_flush(struct rte_eth_dev *dev) 3042 { 3043 struct priv *priv = dev->data->dev_private; 3044 3045 mlx5_flow_list_flush(dev, &priv->flows); 3046 } 3047 3048 /** 3049 * Get flow director information. 3050 * 3051 * @param dev 3052 * Pointer to Ethernet device. 3053 * @param[out] fdir_info 3054 * Resulting flow director information. 3055 */ 3056 static void 3057 mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 3058 { 3059 struct priv *priv = dev->data->dev_private; 3060 struct rte_eth_fdir_masks *mask = 3061 &priv->dev->data->dev_conf.fdir_conf.mask; 3062 3063 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; 3064 fdir_info->guarant_spc = 0; 3065 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 3066 fdir_info->max_flexpayload = 0; 3067 fdir_info->flow_types_mask[0] = 0; 3068 fdir_info->flex_payload_unit = 0; 3069 fdir_info->max_flex_payload_segment_num = 0; 3070 fdir_info->flex_payload_limit = 0; 3071 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 3072 } 3073 3074 /** 3075 * Deal with flow director operations. 3076 * 3077 * @param dev 3078 * Pointer to Ethernet device. 3079 * @param filter_op 3080 * Operation to perform. 3081 * @param arg 3082 * Pointer to operation-specific structure. 3083 * 3084 * @return 3085 * 0 on success, a negative errno value otherwise and rte_errno is set. 3086 */ 3087 static int 3088 mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 3089 void *arg) 3090 { 3091 struct priv *priv = dev->data->dev_private; 3092 enum rte_fdir_mode fdir_mode = 3093 priv->dev->data->dev_conf.fdir_conf.mode; 3094 3095 if (filter_op == RTE_ETH_FILTER_NOP) 3096 return 0; 3097 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 3098 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 3099 DRV_LOG(ERR, "port %u flow director mode %d not supported", 3100 dev->data->port_id, fdir_mode); 3101 rte_errno = EINVAL; 3102 return -rte_errno; 3103 } 3104 switch (filter_op) { 3105 case RTE_ETH_FILTER_ADD: 3106 return mlx5_fdir_filter_add(dev, arg); 3107 case RTE_ETH_FILTER_UPDATE: 3108 return mlx5_fdir_filter_update(dev, arg); 3109 case RTE_ETH_FILTER_DELETE: 3110 return mlx5_fdir_filter_delete(dev, arg); 3111 case RTE_ETH_FILTER_FLUSH: 3112 mlx5_fdir_filter_flush(dev); 3113 break; 3114 case RTE_ETH_FILTER_INFO: 3115 mlx5_fdir_info_get(dev, arg); 3116 break; 3117 default: 3118 DRV_LOG(DEBUG, "port %u unknown operation %u", 3119 dev->data->port_id, filter_op); 3120 rte_errno = EINVAL; 3121 return -rte_errno; 3122 } 3123 return 0; 3124 } 3125 3126 /** 3127 * Manage filter operations. 3128 * 3129 * @param dev 3130 * Pointer to Ethernet device structure. 3131 * @param filter_type 3132 * Filter type. 3133 * @param filter_op 3134 * Operation to perform. 3135 * @param arg 3136 * Pointer to operation-specific structure. 3137 * 3138 * @return 3139 * 0 on success, a negative errno value otherwise and rte_errno is set. 3140 */ 3141 int 3142 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 3143 enum rte_filter_type filter_type, 3144 enum rte_filter_op filter_op, 3145 void *arg) 3146 { 3147 switch (filter_type) { 3148 case RTE_ETH_FILTER_GENERIC: 3149 if (filter_op != RTE_ETH_FILTER_GET) { 3150 rte_errno = EINVAL; 3151 return -rte_errno; 3152 } 3153 *(const void **)arg = &mlx5_flow_ops; 3154 return 0; 3155 case RTE_ETH_FILTER_FDIR: 3156 return mlx5_fdir_ctrl_func(dev, filter_op, arg); 3157 default: 3158 DRV_LOG(ERR, "port %u filter type (%d) not supported", 3159 dev->data->port_id, filter_type); 3160 rte_errno = ENOTSUP; 3161 return -rte_errno; 3162 } 3163 return 0; 3164 } 3165