1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <sys/queue.h> 7 #include <string.h> 8 9 /* Verbs header. */ 10 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 11 #ifdef PEDANTIC 12 #pragma GCC diagnostic ignored "-Wpedantic" 13 #endif 14 #include <infiniband/verbs.h> 15 #ifdef PEDANTIC 16 #pragma GCC diagnostic error "-Wpedantic" 17 #endif 18 19 #include <rte_common.h> 20 #include <rte_ethdev_driver.h> 21 #include <rte_flow.h> 22 #include <rte_flow_driver.h> 23 #include <rte_malloc.h> 24 #include <rte_ip.h> 25 26 #include "mlx5.h" 27 #include "mlx5_defs.h" 28 #include "mlx5_prm.h" 29 #include "mlx5_glue.h" 30 31 /* Define minimal priority for control plane flows. */ 32 #define MLX5_CTRL_FLOW_PRIORITY 4 33 34 /* Internet Protocol versions. */ 35 #define MLX5_IPV4 4 36 #define MLX5_IPV6 6 37 38 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 39 struct ibv_flow_spec_counter_action { 40 int dummy; 41 }; 42 #endif 43 44 /* Dev ops structure defined in mlx5.c */ 45 extern const struct eth_dev_ops mlx5_dev_ops; 46 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 47 48 /** Structure give to the conversion functions. */ 49 struct mlx5_flow_data { 50 struct mlx5_flow_parse *parser; /** Parser context. */ 51 struct rte_flow_error *error; /** Error context. */ 52 }; 53 54 static int 55 mlx5_flow_create_eth(const struct rte_flow_item *item, 56 const void *default_mask, 57 struct mlx5_flow_data *data); 58 59 static int 60 mlx5_flow_create_vlan(const struct rte_flow_item *item, 61 const void *default_mask, 62 struct mlx5_flow_data *data); 63 64 static int 65 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 66 const void *default_mask, 67 struct mlx5_flow_data *data); 68 69 static int 70 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 71 const void *default_mask, 72 struct mlx5_flow_data *data); 73 74 static int 75 mlx5_flow_create_udp(const struct rte_flow_item *item, 76 const void *default_mask, 77 struct mlx5_flow_data *data); 78 79 static int 80 mlx5_flow_create_tcp(const struct rte_flow_item *item, 81 const void *default_mask, 82 struct mlx5_flow_data *data); 83 84 static int 85 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 86 const void *default_mask, 87 struct mlx5_flow_data *data); 88 89 struct mlx5_flow_parse; 90 91 static void 92 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 93 unsigned int size); 94 95 static int 96 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); 97 98 static int 99 mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser); 100 101 /* Hash RX queue types. */ 102 enum hash_rxq_type { 103 HASH_RXQ_TCPV4, 104 HASH_RXQ_UDPV4, 105 HASH_RXQ_IPV4, 106 HASH_RXQ_TCPV6, 107 HASH_RXQ_UDPV6, 108 HASH_RXQ_IPV6, 109 HASH_RXQ_ETH, 110 }; 111 112 /* Initialization data for hash RX queue. */ 113 struct hash_rxq_init { 114 uint64_t hash_fields; /* Fields that participate in the hash. */ 115 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ 116 unsigned int flow_priority; /* Flow priority to use. */ 117 unsigned int ip_version; /* Internet protocol. */ 118 }; 119 120 /* Initialization data for hash RX queues. */ 121 const struct hash_rxq_init hash_rxq_init[] = { 122 [HASH_RXQ_TCPV4] = { 123 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 124 IBV_RX_HASH_DST_IPV4 | 125 IBV_RX_HASH_SRC_PORT_TCP | 126 IBV_RX_HASH_DST_PORT_TCP), 127 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, 128 .flow_priority = 1, 129 .ip_version = MLX5_IPV4, 130 }, 131 [HASH_RXQ_UDPV4] = { 132 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 133 IBV_RX_HASH_DST_IPV4 | 134 IBV_RX_HASH_SRC_PORT_UDP | 135 IBV_RX_HASH_DST_PORT_UDP), 136 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, 137 .flow_priority = 1, 138 .ip_version = MLX5_IPV4, 139 }, 140 [HASH_RXQ_IPV4] = { 141 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 142 IBV_RX_HASH_DST_IPV4), 143 .dpdk_rss_hf = (ETH_RSS_IPV4 | 144 ETH_RSS_FRAG_IPV4), 145 .flow_priority = 2, 146 .ip_version = MLX5_IPV4, 147 }, 148 [HASH_RXQ_TCPV6] = { 149 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 150 IBV_RX_HASH_DST_IPV6 | 151 IBV_RX_HASH_SRC_PORT_TCP | 152 IBV_RX_HASH_DST_PORT_TCP), 153 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, 154 .flow_priority = 1, 155 .ip_version = MLX5_IPV6, 156 }, 157 [HASH_RXQ_UDPV6] = { 158 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 159 IBV_RX_HASH_DST_IPV6 | 160 IBV_RX_HASH_SRC_PORT_UDP | 161 IBV_RX_HASH_DST_PORT_UDP), 162 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, 163 .flow_priority = 1, 164 .ip_version = MLX5_IPV6, 165 }, 166 [HASH_RXQ_IPV6] = { 167 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 168 IBV_RX_HASH_DST_IPV6), 169 .dpdk_rss_hf = (ETH_RSS_IPV6 | 170 ETH_RSS_FRAG_IPV6), 171 .flow_priority = 2, 172 .ip_version = MLX5_IPV6, 173 }, 174 [HASH_RXQ_ETH] = { 175 .hash_fields = 0, 176 .dpdk_rss_hf = 0, 177 .flow_priority = 3, 178 }, 179 }; 180 181 /* Number of entries in hash_rxq_init[]. */ 182 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); 183 184 /** Structure for holding counter stats. */ 185 struct mlx5_flow_counter_stats { 186 uint64_t hits; /**< Number of packets matched by the rule. */ 187 uint64_t bytes; /**< Number of bytes matched by the rule. */ 188 }; 189 190 /** Structure for Drop queue. */ 191 struct mlx5_hrxq_drop { 192 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 193 struct ibv_qp *qp; /**< Verbs queue pair. */ 194 struct ibv_wq *wq; /**< Verbs work queue. */ 195 struct ibv_cq *cq; /**< Verbs completion queue. */ 196 }; 197 198 /* Flows structures. */ 199 struct mlx5_flow { 200 uint64_t hash_fields; /**< Fields that participate in the hash. */ 201 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 202 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 203 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */ 204 }; 205 206 /* Drop flows structures. */ 207 struct mlx5_flow_drop { 208 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 209 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 210 }; 211 212 struct rte_flow { 213 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ 214 uint32_t mark:1; /**< Set if the flow is marked. */ 215 uint32_t drop:1; /**< Drop queue. */ 216 uint16_t queues_n; /**< Number of entries in queue[]. */ 217 uint16_t (*queues)[]; /**< Queues indexes to use. */ 218 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 219 uint8_t rss_key[40]; /**< copy of the RSS key. */ 220 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */ 221 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */ 222 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)]; 223 /**< Flow with Rx queue. */ 224 }; 225 226 /** Static initializer for items. */ 227 #define ITEMS(...) \ 228 (const enum rte_flow_item_type []){ \ 229 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ 230 } 231 232 /** Structure to generate a simple graph of layers supported by the NIC. */ 233 struct mlx5_flow_items { 234 /** List of possible actions for these items. */ 235 const enum rte_flow_action_type *const actions; 236 /** Bit-masks corresponding to the possibilities for the item. */ 237 const void *mask; 238 /** 239 * Default bit-masks to use when item->mask is not provided. When 240 * \default_mask is also NULL, the full supported bit-mask (\mask) is 241 * used instead. 242 */ 243 const void *default_mask; 244 /** Bit-masks size in bytes. */ 245 const unsigned int mask_sz; 246 /** 247 * Conversion function from rte_flow to NIC specific flow. 248 * 249 * @param item 250 * rte_flow item to convert. 251 * @param default_mask 252 * Default bit-masks to use when item->mask is not provided. 253 * @param data 254 * Internal structure to store the conversion. 255 * 256 * @return 257 * 0 on success, a negative errno value otherwise and rte_errno is 258 * set. 259 */ 260 int (*convert)(const struct rte_flow_item *item, 261 const void *default_mask, 262 struct mlx5_flow_data *data); 263 /** Size in bytes of the destination structure. */ 264 const unsigned int dst_sz; 265 /** List of possible following items. */ 266 const enum rte_flow_item_type *const items; 267 }; 268 269 /** Valid action for this PMD. */ 270 static const enum rte_flow_action_type valid_actions[] = { 271 RTE_FLOW_ACTION_TYPE_DROP, 272 RTE_FLOW_ACTION_TYPE_QUEUE, 273 RTE_FLOW_ACTION_TYPE_MARK, 274 RTE_FLOW_ACTION_TYPE_FLAG, 275 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 276 RTE_FLOW_ACTION_TYPE_COUNT, 277 #endif 278 RTE_FLOW_ACTION_TYPE_END, 279 }; 280 281 /** Graph of supported items and associated actions. */ 282 static const struct mlx5_flow_items mlx5_flow_items[] = { 283 [RTE_FLOW_ITEM_TYPE_END] = { 284 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, 285 RTE_FLOW_ITEM_TYPE_VXLAN), 286 }, 287 [RTE_FLOW_ITEM_TYPE_ETH] = { 288 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, 289 RTE_FLOW_ITEM_TYPE_IPV4, 290 RTE_FLOW_ITEM_TYPE_IPV6), 291 .actions = valid_actions, 292 .mask = &(const struct rte_flow_item_eth){ 293 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 294 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 295 .type = -1, 296 }, 297 .default_mask = &rte_flow_item_eth_mask, 298 .mask_sz = sizeof(struct rte_flow_item_eth), 299 .convert = mlx5_flow_create_eth, 300 .dst_sz = sizeof(struct ibv_flow_spec_eth), 301 }, 302 [RTE_FLOW_ITEM_TYPE_VLAN] = { 303 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, 304 RTE_FLOW_ITEM_TYPE_IPV6), 305 .actions = valid_actions, 306 .mask = &(const struct rte_flow_item_vlan){ 307 .tci = -1, 308 }, 309 .default_mask = &rte_flow_item_vlan_mask, 310 .mask_sz = sizeof(struct rte_flow_item_vlan), 311 .convert = mlx5_flow_create_vlan, 312 .dst_sz = 0, 313 }, 314 [RTE_FLOW_ITEM_TYPE_IPV4] = { 315 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 316 RTE_FLOW_ITEM_TYPE_TCP), 317 .actions = valid_actions, 318 .mask = &(const struct rte_flow_item_ipv4){ 319 .hdr = { 320 .src_addr = -1, 321 .dst_addr = -1, 322 .type_of_service = -1, 323 .next_proto_id = -1, 324 }, 325 }, 326 .default_mask = &rte_flow_item_ipv4_mask, 327 .mask_sz = sizeof(struct rte_flow_item_ipv4), 328 .convert = mlx5_flow_create_ipv4, 329 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), 330 }, 331 [RTE_FLOW_ITEM_TYPE_IPV6] = { 332 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 333 RTE_FLOW_ITEM_TYPE_TCP), 334 .actions = valid_actions, 335 .mask = &(const struct rte_flow_item_ipv6){ 336 .hdr = { 337 .src_addr = { 338 0xff, 0xff, 0xff, 0xff, 339 0xff, 0xff, 0xff, 0xff, 340 0xff, 0xff, 0xff, 0xff, 341 0xff, 0xff, 0xff, 0xff, 342 }, 343 .dst_addr = { 344 0xff, 0xff, 0xff, 0xff, 345 0xff, 0xff, 0xff, 0xff, 346 0xff, 0xff, 0xff, 0xff, 347 0xff, 0xff, 0xff, 0xff, 348 }, 349 .vtc_flow = -1, 350 .proto = -1, 351 .hop_limits = -1, 352 }, 353 }, 354 .default_mask = &rte_flow_item_ipv6_mask, 355 .mask_sz = sizeof(struct rte_flow_item_ipv6), 356 .convert = mlx5_flow_create_ipv6, 357 .dst_sz = sizeof(struct ibv_flow_spec_ipv6), 358 }, 359 [RTE_FLOW_ITEM_TYPE_UDP] = { 360 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), 361 .actions = valid_actions, 362 .mask = &(const struct rte_flow_item_udp){ 363 .hdr = { 364 .src_port = -1, 365 .dst_port = -1, 366 }, 367 }, 368 .default_mask = &rte_flow_item_udp_mask, 369 .mask_sz = sizeof(struct rte_flow_item_udp), 370 .convert = mlx5_flow_create_udp, 371 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 372 }, 373 [RTE_FLOW_ITEM_TYPE_TCP] = { 374 .actions = valid_actions, 375 .mask = &(const struct rte_flow_item_tcp){ 376 .hdr = { 377 .src_port = -1, 378 .dst_port = -1, 379 }, 380 }, 381 .default_mask = &rte_flow_item_tcp_mask, 382 .mask_sz = sizeof(struct rte_flow_item_tcp), 383 .convert = mlx5_flow_create_tcp, 384 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 385 }, 386 [RTE_FLOW_ITEM_TYPE_VXLAN] = { 387 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), 388 .actions = valid_actions, 389 .mask = &(const struct rte_flow_item_vxlan){ 390 .vni = "\xff\xff\xff", 391 }, 392 .default_mask = &rte_flow_item_vxlan_mask, 393 .mask_sz = sizeof(struct rte_flow_item_vxlan), 394 .convert = mlx5_flow_create_vxlan, 395 .dst_sz = sizeof(struct ibv_flow_spec_tunnel), 396 }, 397 }; 398 399 /** Structure to pass to the conversion function. */ 400 struct mlx5_flow_parse { 401 uint32_t inner; /**< Set once VXLAN is encountered. */ 402 uint32_t create:1; 403 /**< Whether resources should remain after a validate. */ 404 uint32_t drop:1; /**< Target is a drop queue. */ 405 uint32_t mark:1; /**< Mark is present in the flow. */ 406 uint32_t count:1; /**< Count is present in the flow. */ 407 uint32_t mark_id; /**< Mark identifier. */ 408 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ 409 uint16_t queues_n; /**< Number of entries in queue[]. */ 410 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 411 uint8_t rss_key[40]; /**< copy of the RSS key. */ 412 enum hash_rxq_type layer; /**< Last pattern layer detected. */ 413 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */ 414 struct { 415 struct ibv_flow_attr *ibv_attr; 416 /**< Pointer to Verbs attributes. */ 417 unsigned int offset; 418 /**< Current position or total size of the attribute. */ 419 } queue[RTE_DIM(hash_rxq_init)]; 420 }; 421 422 static const struct rte_flow_ops mlx5_flow_ops = { 423 .validate = mlx5_flow_validate, 424 .create = mlx5_flow_create, 425 .destroy = mlx5_flow_destroy, 426 .flush = mlx5_flow_flush, 427 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 428 .query = mlx5_flow_query, 429 #else 430 .query = NULL, 431 #endif 432 .isolate = mlx5_flow_isolate, 433 }; 434 435 /* Convert FDIR request to Generic flow. */ 436 struct mlx5_fdir { 437 struct rte_flow_attr attr; 438 struct rte_flow_action actions[2]; 439 struct rte_flow_item items[4]; 440 struct rte_flow_item_eth l2; 441 struct rte_flow_item_eth l2_mask; 442 union { 443 struct rte_flow_item_ipv4 ipv4; 444 struct rte_flow_item_ipv6 ipv6; 445 } l3; 446 union { 447 struct rte_flow_item_ipv4 ipv4; 448 struct rte_flow_item_ipv6 ipv6; 449 } l3_mask; 450 union { 451 struct rte_flow_item_udp udp; 452 struct rte_flow_item_tcp tcp; 453 } l4; 454 union { 455 struct rte_flow_item_udp udp; 456 struct rte_flow_item_tcp tcp; 457 } l4_mask; 458 struct rte_flow_action_queue queue; 459 }; 460 461 /* Verbs specification header. */ 462 struct ibv_spec_header { 463 enum ibv_flow_spec_type type; 464 uint16_t size; 465 }; 466 467 /** 468 * Check support for a given item. 469 * 470 * @param item[in] 471 * Item specification. 472 * @param mask[in] 473 * Bit-masks covering supported fields to compare with spec, last and mask in 474 * \item. 475 * @param size 476 * Bit-Mask size in bytes. 477 * 478 * @return 479 * 0 on success, a negative errno value otherwise and rte_errno is set. 480 */ 481 static int 482 mlx5_flow_item_validate(const struct rte_flow_item *item, 483 const uint8_t *mask, unsigned int size) 484 { 485 if (!item->spec && (item->mask || item->last)) { 486 rte_errno = EINVAL; 487 return -rte_errno; 488 } 489 if (item->spec && !item->mask) { 490 unsigned int i; 491 const uint8_t *spec = item->spec; 492 493 for (i = 0; i < size; ++i) 494 if ((spec[i] | mask[i]) != mask[i]) { 495 rte_errno = EINVAL; 496 return -rte_errno; 497 } 498 } 499 if (item->last && !item->mask) { 500 unsigned int i; 501 const uint8_t *spec = item->last; 502 503 for (i = 0; i < size; ++i) 504 if ((spec[i] | mask[i]) != mask[i]) { 505 rte_errno = EINVAL; 506 return -rte_errno; 507 } 508 } 509 if (item->mask) { 510 unsigned int i; 511 const uint8_t *spec = item->spec; 512 513 for (i = 0; i < size; ++i) 514 if ((spec[i] | mask[i]) != mask[i]) { 515 rte_errno = EINVAL; 516 return -rte_errno; 517 } 518 } 519 if (item->spec && item->last) { 520 uint8_t spec[size]; 521 uint8_t last[size]; 522 const uint8_t *apply = mask; 523 unsigned int i; 524 int ret; 525 526 if (item->mask) 527 apply = item->mask; 528 for (i = 0; i < size; ++i) { 529 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; 530 last[i] = ((const uint8_t *)item->last)[i] & apply[i]; 531 } 532 ret = memcmp(spec, last, size); 533 if (ret != 0) { 534 rte_errno = EINVAL; 535 return -rte_errno; 536 } 537 } 538 return 0; 539 } 540 541 /** 542 * Copy the RSS configuration from the user ones, of the rss_conf is null, 543 * uses the driver one. 544 * 545 * @param parser 546 * Internal parser structure. 547 * @param rss_conf 548 * User RSS configuration to save. 549 * 550 * @return 551 * 0 on success, a negative errno value otherwise and rte_errno is set. 552 */ 553 static int 554 mlx5_flow_convert_rss_conf(struct mlx5_flow_parse *parser, 555 const struct rte_eth_rss_conf *rss_conf) 556 { 557 /* 558 * This function is also called at the beginning of 559 * mlx5_flow_convert_actions() to initialize the parser with the 560 * device default RSS configuration. 561 */ 562 if (rss_conf) { 563 if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) { 564 rte_errno = EINVAL; 565 return -rte_errno; 566 } 567 if (rss_conf->rss_key_len != 40) { 568 rte_errno = EINVAL; 569 return -rte_errno; 570 } 571 if (rss_conf->rss_key_len && rss_conf->rss_key) { 572 parser->rss_conf.rss_key_len = rss_conf->rss_key_len; 573 memcpy(parser->rss_key, rss_conf->rss_key, 574 rss_conf->rss_key_len); 575 parser->rss_conf.rss_key = parser->rss_key; 576 } 577 parser->rss_conf.rss_hf = rss_conf->rss_hf; 578 } 579 return 0; 580 } 581 582 /** 583 * Extract attribute to the parser. 584 * 585 * @param[in] attr 586 * Flow rule attributes. 587 * @param[out] error 588 * Perform verbose error reporting if not NULL. 589 * 590 * @return 591 * 0 on success, a negative errno value otherwise and rte_errno is set. 592 */ 593 static int 594 mlx5_flow_convert_attributes(const struct rte_flow_attr *attr, 595 struct rte_flow_error *error) 596 { 597 if (attr->group) { 598 rte_flow_error_set(error, ENOTSUP, 599 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 600 NULL, 601 "groups are not supported"); 602 return -rte_errno; 603 } 604 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) { 605 rte_flow_error_set(error, ENOTSUP, 606 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 607 NULL, 608 "priorities are not supported"); 609 return -rte_errno; 610 } 611 if (attr->egress) { 612 rte_flow_error_set(error, ENOTSUP, 613 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 614 NULL, 615 "egress is not supported"); 616 return -rte_errno; 617 } 618 if (!attr->ingress) { 619 rte_flow_error_set(error, ENOTSUP, 620 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 621 NULL, 622 "only ingress is supported"); 623 return -rte_errno; 624 } 625 return 0; 626 } 627 628 /** 629 * Extract actions request to the parser. 630 * 631 * @param dev 632 * Pointer to Ethernet device. 633 * @param[in] actions 634 * Associated actions (list terminated by the END action). 635 * @param[out] error 636 * Perform verbose error reporting if not NULL. 637 * @param[in, out] parser 638 * Internal parser structure. 639 * 640 * @return 641 * 0 on success, a negative errno value otherwise and rte_errno is set. 642 */ 643 static int 644 mlx5_flow_convert_actions(struct rte_eth_dev *dev, 645 const struct rte_flow_action actions[], 646 struct rte_flow_error *error, 647 struct mlx5_flow_parse *parser) 648 { 649 struct priv *priv = dev->data->dev_private; 650 int ret; 651 652 /* 653 * Add default RSS configuration necessary for Verbs to create QP even 654 * if no RSS is necessary. 655 */ 656 ret = mlx5_flow_convert_rss_conf(parser, 657 (const struct rte_eth_rss_conf *) 658 &priv->rss_conf); 659 if (ret) 660 return ret; 661 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { 662 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { 663 continue; 664 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { 665 parser->drop = 1; 666 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 667 const struct rte_flow_action_queue *queue = 668 (const struct rte_flow_action_queue *) 669 actions->conf; 670 uint16_t n; 671 uint16_t found = 0; 672 673 if (!queue || (queue->index > (priv->rxqs_n - 1))) 674 goto exit_action_not_supported; 675 for (n = 0; n < parser->queues_n; ++n) { 676 if (parser->queues[n] == queue->index) { 677 found = 1; 678 break; 679 } 680 } 681 if (parser->queues_n > 1 && !found) { 682 rte_flow_error_set(error, ENOTSUP, 683 RTE_FLOW_ERROR_TYPE_ACTION, 684 actions, 685 "queue action not in RSS queues"); 686 return -rte_errno; 687 } 688 if (!found) { 689 parser->queues_n = 1; 690 parser->queues[0] = queue->index; 691 } 692 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { 693 const struct rte_flow_action_rss *rss = 694 (const struct rte_flow_action_rss *) 695 actions->conf; 696 uint16_t n; 697 698 if (!rss || !rss->num) { 699 rte_flow_error_set(error, EINVAL, 700 RTE_FLOW_ERROR_TYPE_ACTION, 701 actions, 702 "no valid queues"); 703 return -rte_errno; 704 } 705 if (parser->queues_n == 1) { 706 uint16_t found = 0; 707 708 assert(parser->queues_n); 709 for (n = 0; n < rss->num; ++n) { 710 if (parser->queues[0] == 711 rss->queue[n]) { 712 found = 1; 713 break; 714 } 715 } 716 if (!found) { 717 rte_flow_error_set(error, ENOTSUP, 718 RTE_FLOW_ERROR_TYPE_ACTION, 719 actions, 720 "queue action not in RSS" 721 " queues"); 722 return -rte_errno; 723 } 724 } 725 if (rss->num > RTE_DIM(parser->queues)) { 726 rte_flow_error_set(error, EINVAL, 727 RTE_FLOW_ERROR_TYPE_ACTION, 728 actions, 729 "too many queues for RSS" 730 " context"); 731 return -rte_errno; 732 } 733 for (n = 0; n < rss->num; ++n) { 734 if (rss->queue[n] >= priv->rxqs_n) { 735 rte_flow_error_set(error, EINVAL, 736 RTE_FLOW_ERROR_TYPE_ACTION, 737 actions, 738 "queue id > number of" 739 " queues"); 740 return -rte_errno; 741 } 742 } 743 for (n = 0; n < rss->num; ++n) 744 parser->queues[n] = rss->queue[n]; 745 parser->queues_n = rss->num; 746 if (mlx5_flow_convert_rss_conf(parser, rss->rss_conf)) { 747 rte_flow_error_set(error, EINVAL, 748 RTE_FLOW_ERROR_TYPE_ACTION, 749 actions, 750 "wrong RSS configuration"); 751 return -rte_errno; 752 } 753 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { 754 const struct rte_flow_action_mark *mark = 755 (const struct rte_flow_action_mark *) 756 actions->conf; 757 758 if (!mark) { 759 rte_flow_error_set(error, EINVAL, 760 RTE_FLOW_ERROR_TYPE_ACTION, 761 actions, 762 "mark must be defined"); 763 return -rte_errno; 764 } else if (mark->id >= MLX5_FLOW_MARK_MAX) { 765 rte_flow_error_set(error, ENOTSUP, 766 RTE_FLOW_ERROR_TYPE_ACTION, 767 actions, 768 "mark must be between 0" 769 " and 16777199"); 770 return -rte_errno; 771 } 772 parser->mark = 1; 773 parser->mark_id = mark->id; 774 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { 775 parser->mark = 1; 776 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && 777 priv->config.flow_counter_en) { 778 parser->count = 1; 779 } else { 780 goto exit_action_not_supported; 781 } 782 } 783 if (parser->drop && parser->mark) 784 parser->mark = 0; 785 if (!parser->queues_n && !parser->drop) { 786 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, 787 NULL, "no valid action"); 788 return -rte_errno; 789 } 790 return 0; 791 exit_action_not_supported: 792 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 793 actions, "action not supported"); 794 return -rte_errno; 795 } 796 797 /** 798 * Validate items. 799 * 800 * @param[in] items 801 * Pattern specification (list terminated by the END pattern item). 802 * @param[out] error 803 * Perform verbose error reporting if not NULL. 804 * @param[in, out] parser 805 * Internal parser structure. 806 * 807 * @return 808 * 0 on success, a negative errno value otherwise and rte_errno is set. 809 */ 810 static int 811 mlx5_flow_convert_items_validate(const struct rte_flow_item items[], 812 struct rte_flow_error *error, 813 struct mlx5_flow_parse *parser) 814 { 815 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 816 unsigned int i; 817 int ret = 0; 818 819 /* Initialise the offsets to start after verbs attribute. */ 820 for (i = 0; i != hash_rxq_init_n; ++i) 821 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 822 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 823 const struct mlx5_flow_items *token = NULL; 824 unsigned int n; 825 826 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 827 continue; 828 for (i = 0; 829 cur_item->items && 830 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; 831 ++i) { 832 if (cur_item->items[i] == items->type) { 833 token = &mlx5_flow_items[items->type]; 834 break; 835 } 836 } 837 if (!token) { 838 ret = -ENOTSUP; 839 goto exit_item_not_supported; 840 } 841 cur_item = token; 842 ret = mlx5_flow_item_validate(items, 843 (const uint8_t *)cur_item->mask, 844 cur_item->mask_sz); 845 if (ret) 846 goto exit_item_not_supported; 847 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 848 if (parser->inner) { 849 rte_flow_error_set(error, ENOTSUP, 850 RTE_FLOW_ERROR_TYPE_ITEM, 851 items, 852 "cannot recognize multiple" 853 " VXLAN encapsulations"); 854 return -rte_errno; 855 } 856 parser->inner = IBV_FLOW_SPEC_INNER; 857 } 858 if (parser->drop) { 859 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; 860 } else { 861 for (n = 0; n != hash_rxq_init_n; ++n) 862 parser->queue[n].offset += cur_item->dst_sz; 863 } 864 } 865 if (parser->drop) { 866 parser->queue[HASH_RXQ_ETH].offset += 867 sizeof(struct ibv_flow_spec_action_drop); 868 } 869 if (parser->mark) { 870 for (i = 0; i != hash_rxq_init_n; ++i) 871 parser->queue[i].offset += 872 sizeof(struct ibv_flow_spec_action_tag); 873 } 874 if (parser->count) { 875 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 876 877 for (i = 0; i != hash_rxq_init_n; ++i) 878 parser->queue[i].offset += size; 879 } 880 return 0; 881 exit_item_not_supported: 882 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM, 883 items, "item not supported"); 884 } 885 886 /** 887 * Allocate memory space to store verbs flow attributes. 888 * 889 * @param[in] size 890 * Amount of byte to allocate. 891 * @param[out] error 892 * Perform verbose error reporting if not NULL. 893 * 894 * @return 895 * A verbs flow attribute on success, NULL otherwise and rte_errno is set. 896 */ 897 static struct ibv_flow_attr * 898 mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error) 899 { 900 struct ibv_flow_attr *ibv_attr; 901 902 ibv_attr = rte_calloc(__func__, 1, size, 0); 903 if (!ibv_attr) { 904 rte_flow_error_set(error, ENOMEM, 905 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 906 NULL, 907 "cannot allocate verbs spec attributes"); 908 return NULL; 909 } 910 return ibv_attr; 911 } 912 913 /** 914 * Make inner packet matching with an higher priority from the non Inner 915 * matching. 916 * 917 * @param[in, out] parser 918 * Internal parser structure. 919 * @param attr 920 * User flow attribute. 921 */ 922 static void 923 mlx5_flow_update_priority(struct mlx5_flow_parse *parser, 924 const struct rte_flow_attr *attr) 925 { 926 unsigned int i; 927 928 if (parser->drop) { 929 parser->queue[HASH_RXQ_ETH].ibv_attr->priority = 930 attr->priority + 931 hash_rxq_init[HASH_RXQ_ETH].flow_priority; 932 return; 933 } 934 for (i = 0; i != hash_rxq_init_n; ++i) { 935 if (parser->queue[i].ibv_attr) { 936 parser->queue[i].ibv_attr->priority = 937 attr->priority + 938 hash_rxq_init[i].flow_priority - 939 (parser->inner ? 1 : 0); 940 } 941 } 942 } 943 944 /** 945 * Finalise verbs flow attributes. 946 * 947 * @param[in, out] parser 948 * Internal parser structure. 949 */ 950 static void 951 mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser) 952 { 953 const unsigned int ipv4 = 954 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; 955 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6; 956 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 957 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4; 958 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4; 959 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 960 unsigned int i; 961 962 /* Remove any other flow not matching the pattern. */ 963 if (parser->queues_n == 1 && !parser->rss_conf.rss_hf) { 964 for (i = 0; i != hash_rxq_init_n; ++i) { 965 if (i == HASH_RXQ_ETH) 966 continue; 967 rte_free(parser->queue[i].ibv_attr); 968 parser->queue[i].ibv_attr = NULL; 969 } 970 return; 971 } 972 if (parser->layer == HASH_RXQ_ETH) { 973 goto fill; 974 } else { 975 /* 976 * This layer becomes useless as the pattern define under 977 * layers. 978 */ 979 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr); 980 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 981 } 982 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */ 983 for (i = ohmin; i != (ohmax + 1); ++i) { 984 if (!parser->queue[i].ibv_attr) 985 continue; 986 rte_free(parser->queue[i].ibv_attr); 987 parser->queue[i].ibv_attr = NULL; 988 } 989 /* Remove impossible flow according to the RSS configuration. */ 990 if (hash_rxq_init[parser->layer].dpdk_rss_hf & 991 parser->rss_conf.rss_hf) { 992 /* Remove any other flow. */ 993 for (i = hmin; i != (hmax + 1); ++i) { 994 if ((i == parser->layer) || 995 (!parser->queue[i].ibv_attr)) 996 continue; 997 rte_free(parser->queue[i].ibv_attr); 998 parser->queue[i].ibv_attr = NULL; 999 } 1000 } else if (!parser->queue[ip].ibv_attr) { 1001 /* no RSS possible with the current configuration. */ 1002 parser->queues_n = 1; 1003 return; 1004 } 1005 fill: 1006 /* 1007 * Fill missing layers in verbs specifications, or compute the correct 1008 * offset to allocate the memory space for the attributes and 1009 * specifications. 1010 */ 1011 for (i = 0; i != hash_rxq_init_n - 1; ++i) { 1012 union { 1013 struct ibv_flow_spec_ipv4_ext ipv4; 1014 struct ibv_flow_spec_ipv6 ipv6; 1015 struct ibv_flow_spec_tcp_udp udp_tcp; 1016 } specs; 1017 void *dst; 1018 uint16_t size; 1019 1020 if (i == parser->layer) 1021 continue; 1022 if (parser->layer == HASH_RXQ_ETH) { 1023 if (hash_rxq_init[i].ip_version == MLX5_IPV4) { 1024 size = sizeof(struct ibv_flow_spec_ipv4_ext); 1025 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){ 1026 .type = IBV_FLOW_SPEC_IPV4_EXT, 1027 .size = size, 1028 }; 1029 } else { 1030 size = sizeof(struct ibv_flow_spec_ipv6); 1031 specs.ipv6 = (struct ibv_flow_spec_ipv6){ 1032 .type = IBV_FLOW_SPEC_IPV6, 1033 .size = size, 1034 }; 1035 } 1036 if (parser->queue[i].ibv_attr) { 1037 dst = (void *)((uintptr_t) 1038 parser->queue[i].ibv_attr + 1039 parser->queue[i].offset); 1040 memcpy(dst, &specs, size); 1041 ++parser->queue[i].ibv_attr->num_of_specs; 1042 } 1043 parser->queue[i].offset += size; 1044 } 1045 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) || 1046 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) { 1047 size = sizeof(struct ibv_flow_spec_tcp_udp); 1048 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) { 1049 .type = ((i == HASH_RXQ_UDPV4 || 1050 i == HASH_RXQ_UDPV6) ? 1051 IBV_FLOW_SPEC_UDP : 1052 IBV_FLOW_SPEC_TCP), 1053 .size = size, 1054 }; 1055 if (parser->queue[i].ibv_attr) { 1056 dst = (void *)((uintptr_t) 1057 parser->queue[i].ibv_attr + 1058 parser->queue[i].offset); 1059 memcpy(dst, &specs, size); 1060 ++parser->queue[i].ibv_attr->num_of_specs; 1061 } 1062 parser->queue[i].offset += size; 1063 } 1064 } 1065 } 1066 1067 /** 1068 * Validate and convert a flow supported by the NIC. 1069 * 1070 * @param dev 1071 * Pointer to Ethernet device. 1072 * @param[in] attr 1073 * Flow rule attributes. 1074 * @param[in] pattern 1075 * Pattern specification (list terminated by the END pattern item). 1076 * @param[in] actions 1077 * Associated actions (list terminated by the END action). 1078 * @param[out] error 1079 * Perform verbose error reporting if not NULL. 1080 * @param[in, out] parser 1081 * Internal parser structure. 1082 * 1083 * @return 1084 * 0 on success, a negative errno value otherwise and rte_errno is set. 1085 */ 1086 static int 1087 mlx5_flow_convert(struct rte_eth_dev *dev, 1088 const struct rte_flow_attr *attr, 1089 const struct rte_flow_item items[], 1090 const struct rte_flow_action actions[], 1091 struct rte_flow_error *error, 1092 struct mlx5_flow_parse *parser) 1093 { 1094 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 1095 unsigned int i; 1096 int ret; 1097 1098 /* First step. Validate the attributes, items and actions. */ 1099 *parser = (struct mlx5_flow_parse){ 1100 .create = parser->create, 1101 .layer = HASH_RXQ_ETH, 1102 .mark_id = MLX5_FLOW_MARK_DEFAULT, 1103 }; 1104 ret = mlx5_flow_convert_attributes(attr, error); 1105 if (ret) 1106 return ret; 1107 ret = mlx5_flow_convert_actions(dev, actions, error, parser); 1108 if (ret) 1109 return ret; 1110 ret = mlx5_flow_convert_items_validate(items, error, parser); 1111 if (ret) 1112 return ret; 1113 mlx5_flow_convert_finalise(parser); 1114 /* 1115 * Second step. 1116 * Allocate the memory space to store verbs specifications. 1117 */ 1118 if (parser->drop) { 1119 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; 1120 1121 parser->queue[HASH_RXQ_ETH].ibv_attr = 1122 mlx5_flow_convert_allocate(offset, error); 1123 if (!parser->queue[HASH_RXQ_ETH].ibv_attr) 1124 goto exit_enomem; 1125 parser->queue[HASH_RXQ_ETH].offset = 1126 sizeof(struct ibv_flow_attr); 1127 } else { 1128 for (i = 0; i != hash_rxq_init_n; ++i) { 1129 unsigned int offset; 1130 1131 if (!(parser->rss_conf.rss_hf & 1132 hash_rxq_init[i].dpdk_rss_hf) && 1133 (i != HASH_RXQ_ETH)) 1134 continue; 1135 offset = parser->queue[i].offset; 1136 parser->queue[i].ibv_attr = 1137 mlx5_flow_convert_allocate(offset, error); 1138 if (!parser->queue[i].ibv_attr) 1139 goto exit_enomem; 1140 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 1141 } 1142 } 1143 /* Third step. Conversion parse, fill the specifications. */ 1144 parser->inner = 0; 1145 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 1146 struct mlx5_flow_data data = { 1147 .parser = parser, 1148 .error = error, 1149 }; 1150 1151 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 1152 continue; 1153 cur_item = &mlx5_flow_items[items->type]; 1154 ret = cur_item->convert(items, 1155 (cur_item->default_mask ? 1156 cur_item->default_mask : 1157 cur_item->mask), 1158 &data); 1159 if (ret) 1160 goto exit_free; 1161 } 1162 if (parser->mark) 1163 mlx5_flow_create_flag_mark(parser, parser->mark_id); 1164 if (parser->count && parser->create) { 1165 mlx5_flow_create_count(dev, parser); 1166 if (!parser->cs) 1167 goto exit_count_error; 1168 } 1169 /* 1170 * Last step. Complete missing specification to reach the RSS 1171 * configuration. 1172 */ 1173 if (!parser->drop) 1174 mlx5_flow_convert_finalise(parser); 1175 mlx5_flow_update_priority(parser, attr); 1176 exit_free: 1177 /* Only verification is expected, all resources should be released. */ 1178 if (!parser->create) { 1179 for (i = 0; i != hash_rxq_init_n; ++i) { 1180 if (parser->queue[i].ibv_attr) { 1181 rte_free(parser->queue[i].ibv_attr); 1182 parser->queue[i].ibv_attr = NULL; 1183 } 1184 } 1185 } 1186 return ret; 1187 exit_enomem: 1188 for (i = 0; i != hash_rxq_init_n; ++i) { 1189 if (parser->queue[i].ibv_attr) { 1190 rte_free(parser->queue[i].ibv_attr); 1191 parser->queue[i].ibv_attr = NULL; 1192 } 1193 } 1194 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1195 NULL, "cannot allocate verbs spec attributes"); 1196 return -rte_errno; 1197 exit_count_error: 1198 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1199 NULL, "cannot create counter"); 1200 return -rte_errno; 1201 } 1202 1203 /** 1204 * Copy the specification created into the flow. 1205 * 1206 * @param parser 1207 * Internal parser structure. 1208 * @param src 1209 * Create specification. 1210 * @param size 1211 * Size in bytes of the specification to copy. 1212 */ 1213 static void 1214 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 1215 unsigned int size) 1216 { 1217 unsigned int i; 1218 void *dst; 1219 1220 for (i = 0; i != hash_rxq_init_n; ++i) { 1221 if (!parser->queue[i].ibv_attr) 1222 continue; 1223 /* Specification must be the same l3 type or none. */ 1224 if (parser->layer == HASH_RXQ_ETH || 1225 (hash_rxq_init[parser->layer].ip_version == 1226 hash_rxq_init[i].ip_version) || 1227 (hash_rxq_init[i].ip_version == 0)) { 1228 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1229 parser->queue[i].offset); 1230 memcpy(dst, src, size); 1231 ++parser->queue[i].ibv_attr->num_of_specs; 1232 parser->queue[i].offset += size; 1233 } 1234 } 1235 } 1236 1237 /** 1238 * Convert Ethernet item to Verbs specification. 1239 * 1240 * @param item[in] 1241 * Item specification. 1242 * @param default_mask[in] 1243 * Default bit-masks to use when item->mask is not provided. 1244 * @param data[in, out] 1245 * User structure. 1246 * 1247 * @return 1248 * 0 on success, a negative errno value otherwise and rte_errno is set. 1249 */ 1250 static int 1251 mlx5_flow_create_eth(const struct rte_flow_item *item, 1252 const void *default_mask, 1253 struct mlx5_flow_data *data) 1254 { 1255 const struct rte_flow_item_eth *spec = item->spec; 1256 const struct rte_flow_item_eth *mask = item->mask; 1257 struct mlx5_flow_parse *parser = data->parser; 1258 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1259 struct ibv_flow_spec_eth eth = { 1260 .type = parser->inner | IBV_FLOW_SPEC_ETH, 1261 .size = eth_size, 1262 }; 1263 1264 /* Don't update layer for the inner pattern. */ 1265 if (!parser->inner) 1266 parser->layer = HASH_RXQ_ETH; 1267 if (spec) { 1268 unsigned int i; 1269 1270 if (!mask) 1271 mask = default_mask; 1272 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); 1273 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); 1274 eth.val.ether_type = spec->type; 1275 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); 1276 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); 1277 eth.mask.ether_type = mask->type; 1278 /* Remove unwanted bits from values. */ 1279 for (i = 0; i < ETHER_ADDR_LEN; ++i) { 1280 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 1281 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 1282 } 1283 eth.val.ether_type &= eth.mask.ether_type; 1284 } 1285 mlx5_flow_create_copy(parser, ð, eth_size); 1286 return 0; 1287 } 1288 1289 /** 1290 * Convert VLAN item to Verbs specification. 1291 * 1292 * @param item[in] 1293 * Item specification. 1294 * @param default_mask[in] 1295 * Default bit-masks to use when item->mask is not provided. 1296 * @param data[in, out] 1297 * User structure. 1298 * 1299 * @return 1300 * 0 on success, a negative errno value otherwise and rte_errno is set. 1301 */ 1302 static int 1303 mlx5_flow_create_vlan(const struct rte_flow_item *item, 1304 const void *default_mask, 1305 struct mlx5_flow_data *data) 1306 { 1307 const struct rte_flow_item_vlan *spec = item->spec; 1308 const struct rte_flow_item_vlan *mask = item->mask; 1309 struct mlx5_flow_parse *parser = data->parser; 1310 struct ibv_flow_spec_eth *eth; 1311 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1312 1313 if (spec) { 1314 unsigned int i; 1315 if (!mask) 1316 mask = default_mask; 1317 1318 for (i = 0; i != hash_rxq_init_n; ++i) { 1319 if (!parser->queue[i].ibv_attr) 1320 continue; 1321 1322 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1323 parser->queue[i].offset - eth_size); 1324 eth->val.vlan_tag = spec->tci; 1325 eth->mask.vlan_tag = mask->tci; 1326 eth->val.vlan_tag &= eth->mask.vlan_tag; 1327 /* 1328 * From verbs perspective an empty VLAN is equivalent 1329 * to a packet without VLAN layer. 1330 */ 1331 if (!eth->mask.vlan_tag) 1332 goto error; 1333 } 1334 return 0; 1335 } 1336 error: 1337 return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, 1338 item, "VLAN cannot be empty"); 1339 } 1340 1341 /** 1342 * Convert IPv4 item to Verbs specification. 1343 * 1344 * @param item[in] 1345 * Item specification. 1346 * @param default_mask[in] 1347 * Default bit-masks to use when item->mask is not provided. 1348 * @param data[in, out] 1349 * User structure. 1350 * 1351 * @return 1352 * 0 on success, a negative errno value otherwise and rte_errno is set. 1353 */ 1354 static int 1355 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 1356 const void *default_mask, 1357 struct mlx5_flow_data *data) 1358 { 1359 const struct rte_flow_item_ipv4 *spec = item->spec; 1360 const struct rte_flow_item_ipv4 *mask = item->mask; 1361 struct mlx5_flow_parse *parser = data->parser; 1362 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); 1363 struct ibv_flow_spec_ipv4_ext ipv4 = { 1364 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, 1365 .size = ipv4_size, 1366 }; 1367 1368 /* Don't update layer for the inner pattern. */ 1369 if (!parser->inner) 1370 parser->layer = HASH_RXQ_IPV4; 1371 if (spec) { 1372 if (!mask) 1373 mask = default_mask; 1374 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 1375 .src_ip = spec->hdr.src_addr, 1376 .dst_ip = spec->hdr.dst_addr, 1377 .proto = spec->hdr.next_proto_id, 1378 .tos = spec->hdr.type_of_service, 1379 }; 1380 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 1381 .src_ip = mask->hdr.src_addr, 1382 .dst_ip = mask->hdr.dst_addr, 1383 .proto = mask->hdr.next_proto_id, 1384 .tos = mask->hdr.type_of_service, 1385 }; 1386 /* Remove unwanted bits from values. */ 1387 ipv4.val.src_ip &= ipv4.mask.src_ip; 1388 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 1389 ipv4.val.proto &= ipv4.mask.proto; 1390 ipv4.val.tos &= ipv4.mask.tos; 1391 } 1392 mlx5_flow_create_copy(parser, &ipv4, ipv4_size); 1393 return 0; 1394 } 1395 1396 /** 1397 * Convert IPv6 item to Verbs specification. 1398 * 1399 * @param item[in] 1400 * Item specification. 1401 * @param default_mask[in] 1402 * Default bit-masks to use when item->mask is not provided. 1403 * @param data[in, out] 1404 * User structure. 1405 * 1406 * @return 1407 * 0 on success, a negative errno value otherwise and rte_errno is set. 1408 */ 1409 static int 1410 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 1411 const void *default_mask, 1412 struct mlx5_flow_data *data) 1413 { 1414 const struct rte_flow_item_ipv6 *spec = item->spec; 1415 const struct rte_flow_item_ipv6 *mask = item->mask; 1416 struct mlx5_flow_parse *parser = data->parser; 1417 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); 1418 struct ibv_flow_spec_ipv6 ipv6 = { 1419 .type = parser->inner | IBV_FLOW_SPEC_IPV6, 1420 .size = ipv6_size, 1421 }; 1422 1423 /* Don't update layer for the inner pattern. */ 1424 if (!parser->inner) 1425 parser->layer = HASH_RXQ_IPV6; 1426 if (spec) { 1427 unsigned int i; 1428 uint32_t vtc_flow_val; 1429 uint32_t vtc_flow_mask; 1430 1431 if (!mask) 1432 mask = default_mask; 1433 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 1434 RTE_DIM(ipv6.val.src_ip)); 1435 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 1436 RTE_DIM(ipv6.val.dst_ip)); 1437 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 1438 RTE_DIM(ipv6.mask.src_ip)); 1439 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 1440 RTE_DIM(ipv6.mask.dst_ip)); 1441 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); 1442 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); 1443 ipv6.val.flow_label = 1444 rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >> 1445 IPV6_HDR_FL_SHIFT); 1446 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >> 1447 IPV6_HDR_TC_SHIFT; 1448 ipv6.val.next_hdr = spec->hdr.proto; 1449 ipv6.val.hop_limit = spec->hdr.hop_limits; 1450 ipv6.mask.flow_label = 1451 rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >> 1452 IPV6_HDR_FL_SHIFT); 1453 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >> 1454 IPV6_HDR_TC_SHIFT; 1455 ipv6.mask.next_hdr = mask->hdr.proto; 1456 ipv6.mask.hop_limit = mask->hdr.hop_limits; 1457 /* Remove unwanted bits from values. */ 1458 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 1459 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 1460 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 1461 } 1462 ipv6.val.flow_label &= ipv6.mask.flow_label; 1463 ipv6.val.traffic_class &= ipv6.mask.traffic_class; 1464 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 1465 ipv6.val.hop_limit &= ipv6.mask.hop_limit; 1466 } 1467 mlx5_flow_create_copy(parser, &ipv6, ipv6_size); 1468 return 0; 1469 } 1470 1471 /** 1472 * Convert UDP item to Verbs specification. 1473 * 1474 * @param item[in] 1475 * Item specification. 1476 * @param default_mask[in] 1477 * Default bit-masks to use when item->mask is not provided. 1478 * @param data[in, out] 1479 * User structure. 1480 * 1481 * @return 1482 * 0 on success, a negative errno value otherwise and rte_errno is set. 1483 */ 1484 static int 1485 mlx5_flow_create_udp(const struct rte_flow_item *item, 1486 const void *default_mask, 1487 struct mlx5_flow_data *data) 1488 { 1489 const struct rte_flow_item_udp *spec = item->spec; 1490 const struct rte_flow_item_udp *mask = item->mask; 1491 struct mlx5_flow_parse *parser = data->parser; 1492 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1493 struct ibv_flow_spec_tcp_udp udp = { 1494 .type = parser->inner | IBV_FLOW_SPEC_UDP, 1495 .size = udp_size, 1496 }; 1497 1498 /* Don't update layer for the inner pattern. */ 1499 if (!parser->inner) { 1500 if (parser->layer == HASH_RXQ_IPV4) 1501 parser->layer = HASH_RXQ_UDPV4; 1502 else 1503 parser->layer = HASH_RXQ_UDPV6; 1504 } 1505 if (spec) { 1506 if (!mask) 1507 mask = default_mask; 1508 udp.val.dst_port = spec->hdr.dst_port; 1509 udp.val.src_port = spec->hdr.src_port; 1510 udp.mask.dst_port = mask->hdr.dst_port; 1511 udp.mask.src_port = mask->hdr.src_port; 1512 /* Remove unwanted bits from values. */ 1513 udp.val.src_port &= udp.mask.src_port; 1514 udp.val.dst_port &= udp.mask.dst_port; 1515 } 1516 mlx5_flow_create_copy(parser, &udp, udp_size); 1517 return 0; 1518 } 1519 1520 /** 1521 * Convert TCP item to Verbs specification. 1522 * 1523 * @param item[in] 1524 * Item specification. 1525 * @param default_mask[in] 1526 * Default bit-masks to use when item->mask is not provided. 1527 * @param data[in, out] 1528 * User structure. 1529 * 1530 * @return 1531 * 0 on success, a negative errno value otherwise and rte_errno is set. 1532 */ 1533 static int 1534 mlx5_flow_create_tcp(const struct rte_flow_item *item, 1535 const void *default_mask, 1536 struct mlx5_flow_data *data) 1537 { 1538 const struct rte_flow_item_tcp *spec = item->spec; 1539 const struct rte_flow_item_tcp *mask = item->mask; 1540 struct mlx5_flow_parse *parser = data->parser; 1541 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1542 struct ibv_flow_spec_tcp_udp tcp = { 1543 .type = parser->inner | IBV_FLOW_SPEC_TCP, 1544 .size = tcp_size, 1545 }; 1546 1547 /* Don't update layer for the inner pattern. */ 1548 if (!parser->inner) { 1549 if (parser->layer == HASH_RXQ_IPV4) 1550 parser->layer = HASH_RXQ_TCPV4; 1551 else 1552 parser->layer = HASH_RXQ_TCPV6; 1553 } 1554 if (spec) { 1555 if (!mask) 1556 mask = default_mask; 1557 tcp.val.dst_port = spec->hdr.dst_port; 1558 tcp.val.src_port = spec->hdr.src_port; 1559 tcp.mask.dst_port = mask->hdr.dst_port; 1560 tcp.mask.src_port = mask->hdr.src_port; 1561 /* Remove unwanted bits from values. */ 1562 tcp.val.src_port &= tcp.mask.src_port; 1563 tcp.val.dst_port &= tcp.mask.dst_port; 1564 } 1565 mlx5_flow_create_copy(parser, &tcp, tcp_size); 1566 return 0; 1567 } 1568 1569 /** 1570 * Convert VXLAN item to Verbs specification. 1571 * 1572 * @param item[in] 1573 * Item specification. 1574 * @param default_mask[in] 1575 * Default bit-masks to use when item->mask is not provided. 1576 * @param data[in, out] 1577 * User structure. 1578 * 1579 * @return 1580 * 0 on success, a negative errno value otherwise and rte_errno is set. 1581 */ 1582 static int 1583 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 1584 const void *default_mask, 1585 struct mlx5_flow_data *data) 1586 { 1587 const struct rte_flow_item_vxlan *spec = item->spec; 1588 const struct rte_flow_item_vxlan *mask = item->mask; 1589 struct mlx5_flow_parse *parser = data->parser; 1590 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 1591 struct ibv_flow_spec_tunnel vxlan = { 1592 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, 1593 .size = size, 1594 }; 1595 union vni { 1596 uint32_t vlan_id; 1597 uint8_t vni[4]; 1598 } id; 1599 1600 id.vni[0] = 0; 1601 parser->inner = IBV_FLOW_SPEC_INNER; 1602 if (spec) { 1603 if (!mask) 1604 mask = default_mask; 1605 memcpy(&id.vni[1], spec->vni, 3); 1606 vxlan.val.tunnel_id = id.vlan_id; 1607 memcpy(&id.vni[1], mask->vni, 3); 1608 vxlan.mask.tunnel_id = id.vlan_id; 1609 /* Remove unwanted bits from values. */ 1610 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 1611 } 1612 /* 1613 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1614 * layer is defined in the Verbs specification it is interpreted as 1615 * wildcard and all packets will match this rule, if it follows a full 1616 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1617 * before will also match this rule. 1618 * To avoid such situation, VNI 0 is currently refused. 1619 */ 1620 if (!vxlan.val.tunnel_id) 1621 return rte_flow_error_set(data->error, EINVAL, 1622 RTE_FLOW_ERROR_TYPE_ITEM, 1623 item, 1624 "VxLAN vni cannot be 0"); 1625 mlx5_flow_create_copy(parser, &vxlan, size); 1626 return 0; 1627 } 1628 1629 /** 1630 * Convert mark/flag action to Verbs specification. 1631 * 1632 * @param parser 1633 * Internal parser structure. 1634 * @param mark_id 1635 * Mark identifier. 1636 * 1637 * @return 1638 * 0 on success, a negative errno value otherwise and rte_errno is set. 1639 */ 1640 static int 1641 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) 1642 { 1643 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1644 struct ibv_flow_spec_action_tag tag = { 1645 .type = IBV_FLOW_SPEC_ACTION_TAG, 1646 .size = size, 1647 .tag_id = mlx5_flow_mark_set(mark_id), 1648 }; 1649 1650 assert(parser->mark); 1651 mlx5_flow_create_copy(parser, &tag, size); 1652 return 0; 1653 } 1654 1655 /** 1656 * Convert count action to Verbs specification. 1657 * 1658 * @param dev 1659 * Pointer to Ethernet device. 1660 * @param parser 1661 * Pointer to MLX5 flow parser structure. 1662 * 1663 * @return 1664 * 0 on success, a negative errno value otherwise and rte_errno is set. 1665 */ 1666 static int 1667 mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused, 1668 struct mlx5_flow_parse *parser __rte_unused) 1669 { 1670 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 1671 struct priv *priv = dev->data->dev_private; 1672 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1673 struct ibv_counter_set_init_attr init_attr = {0}; 1674 struct ibv_flow_spec_counter_action counter = { 1675 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1676 .size = size, 1677 .counter_set_handle = 0, 1678 }; 1679 1680 init_attr.counter_set_id = 0; 1681 parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr); 1682 if (!parser->cs) { 1683 rte_errno = EINVAL; 1684 return -rte_errno; 1685 } 1686 counter.counter_set_handle = parser->cs->handle; 1687 mlx5_flow_create_copy(parser, &counter, size); 1688 #endif 1689 return 0; 1690 } 1691 1692 /** 1693 * Complete flow rule creation with a drop queue. 1694 * 1695 * @param dev 1696 * Pointer to Ethernet device. 1697 * @param parser 1698 * Internal parser structure. 1699 * @param flow 1700 * Pointer to the rte_flow. 1701 * @param[out] error 1702 * Perform verbose error reporting if not NULL. 1703 * 1704 * @return 1705 * 0 on success, a negative errno value otherwise and rte_errno is set. 1706 */ 1707 static int 1708 mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev, 1709 struct mlx5_flow_parse *parser, 1710 struct rte_flow *flow, 1711 struct rte_flow_error *error) 1712 { 1713 struct priv *priv = dev->data->dev_private; 1714 struct ibv_flow_spec_action_drop *drop; 1715 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1716 1717 assert(priv->pd); 1718 assert(priv->ctx); 1719 flow->drop = 1; 1720 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr + 1721 parser->queue[HASH_RXQ_ETH].offset); 1722 *drop = (struct ibv_flow_spec_action_drop){ 1723 .type = IBV_FLOW_SPEC_ACTION_DROP, 1724 .size = size, 1725 }; 1726 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs; 1727 parser->queue[HASH_RXQ_ETH].offset += size; 1728 flow->frxq[HASH_RXQ_ETH].ibv_attr = 1729 parser->queue[HASH_RXQ_ETH].ibv_attr; 1730 if (parser->count) 1731 flow->cs = parser->cs; 1732 if (!priv->dev->data->dev_started) 1733 return 0; 1734 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 1735 flow->frxq[HASH_RXQ_ETH].ibv_flow = 1736 mlx5_glue->create_flow(priv->flow_drop_queue->qp, 1737 flow->frxq[HASH_RXQ_ETH].ibv_attr); 1738 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { 1739 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1740 NULL, "flow rule creation failure"); 1741 goto error; 1742 } 1743 return 0; 1744 error: 1745 assert(flow); 1746 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) { 1747 claim_zero(mlx5_glue->destroy_flow 1748 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 1749 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; 1750 } 1751 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) { 1752 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); 1753 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL; 1754 } 1755 if (flow->cs) { 1756 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 1757 flow->cs = NULL; 1758 parser->cs = NULL; 1759 } 1760 return -rte_errno; 1761 } 1762 1763 /** 1764 * Create hash Rx queues when RSS is enabled. 1765 * 1766 * @param dev 1767 * Pointer to Ethernet device. 1768 * @param parser 1769 * Internal parser structure. 1770 * @param flow 1771 * Pointer to the rte_flow. 1772 * @param[out] error 1773 * Perform verbose error reporting if not NULL. 1774 * 1775 * @return 1776 * 0 on success, a negative errno value otherwise and rte_errno is set. 1777 */ 1778 static int 1779 mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev, 1780 struct mlx5_flow_parse *parser, 1781 struct rte_flow *flow, 1782 struct rte_flow_error *error) 1783 { 1784 struct priv *priv = dev->data->dev_private; 1785 unsigned int i; 1786 1787 for (i = 0; i != hash_rxq_init_n; ++i) { 1788 uint64_t hash_fields; 1789 1790 if (!parser->queue[i].ibv_attr) 1791 continue; 1792 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; 1793 parser->queue[i].ibv_attr = NULL; 1794 hash_fields = hash_rxq_init[i].hash_fields; 1795 if (!priv->dev->data->dev_started) 1796 continue; 1797 flow->frxq[i].hrxq = 1798 mlx5_hrxq_get(dev, 1799 parser->rss_conf.rss_key, 1800 parser->rss_conf.rss_key_len, 1801 hash_fields, 1802 parser->queues, 1803 parser->queues_n); 1804 if (flow->frxq[i].hrxq) 1805 continue; 1806 flow->frxq[i].hrxq = 1807 mlx5_hrxq_new(dev, 1808 parser->rss_conf.rss_key, 1809 parser->rss_conf.rss_key_len, 1810 hash_fields, 1811 parser->queues, 1812 parser->queues_n); 1813 if (!flow->frxq[i].hrxq) { 1814 return rte_flow_error_set(error, ENOMEM, 1815 RTE_FLOW_ERROR_TYPE_HANDLE, 1816 NULL, 1817 "cannot create hash rxq"); 1818 } 1819 } 1820 return 0; 1821 } 1822 1823 /** 1824 * Complete flow rule creation. 1825 * 1826 * @param dev 1827 * Pointer to Ethernet device. 1828 * @param parser 1829 * Internal parser structure. 1830 * @param flow 1831 * Pointer to the rte_flow. 1832 * @param[out] error 1833 * Perform verbose error reporting if not NULL. 1834 * 1835 * @return 1836 * 0 on success, a negative errno value otherwise and rte_errno is set. 1837 */ 1838 static int 1839 mlx5_flow_create_action_queue(struct rte_eth_dev *dev, 1840 struct mlx5_flow_parse *parser, 1841 struct rte_flow *flow, 1842 struct rte_flow_error *error) 1843 { 1844 struct priv *priv = dev->data->dev_private; 1845 int ret; 1846 unsigned int i; 1847 unsigned int flows_n = 0; 1848 1849 assert(priv->pd); 1850 assert(priv->ctx); 1851 assert(!parser->drop); 1852 ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error); 1853 if (ret) 1854 goto error; 1855 if (parser->count) 1856 flow->cs = parser->cs; 1857 if (!priv->dev->data->dev_started) 1858 return 0; 1859 for (i = 0; i != hash_rxq_init_n; ++i) { 1860 if (!flow->frxq[i].hrxq) 1861 continue; 1862 flow->frxq[i].ibv_flow = 1863 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp, 1864 flow->frxq[i].ibv_attr); 1865 if (!flow->frxq[i].ibv_flow) { 1866 rte_flow_error_set(error, ENOMEM, 1867 RTE_FLOW_ERROR_TYPE_HANDLE, 1868 NULL, "flow rule creation failure"); 1869 goto error; 1870 } 1871 ++flows_n; 1872 DRV_LOG(DEBUG, "port %u %p type %d QP %p ibv_flow %p", 1873 dev->data->port_id, 1874 (void *)flow, i, 1875 (void *)flow->frxq[i].hrxq, 1876 (void *)flow->frxq[i].ibv_flow); 1877 } 1878 if (!flows_n) { 1879 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, 1880 NULL, "internal error in flow creation"); 1881 goto error; 1882 } 1883 for (i = 0; i != parser->queues_n; ++i) { 1884 struct mlx5_rxq_data *q = 1885 (*priv->rxqs)[parser->queues[i]]; 1886 1887 q->mark |= parser->mark; 1888 } 1889 return 0; 1890 error: 1891 ret = rte_errno; /* Save rte_errno before cleanup. */ 1892 assert(flow); 1893 for (i = 0; i != hash_rxq_init_n; ++i) { 1894 if (flow->frxq[i].ibv_flow) { 1895 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; 1896 1897 claim_zero(mlx5_glue->destroy_flow(ibv_flow)); 1898 } 1899 if (flow->frxq[i].hrxq) 1900 mlx5_hrxq_release(dev, flow->frxq[i].hrxq); 1901 if (flow->frxq[i].ibv_attr) 1902 rte_free(flow->frxq[i].ibv_attr); 1903 } 1904 if (flow->cs) { 1905 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 1906 flow->cs = NULL; 1907 parser->cs = NULL; 1908 } 1909 rte_errno = ret; /* Restore rte_errno. */ 1910 return -rte_errno; 1911 } 1912 1913 /** 1914 * Convert a flow. 1915 * 1916 * @param dev 1917 * Pointer to Ethernet device. 1918 * @param list 1919 * Pointer to a TAILQ flow list. 1920 * @param[in] attr 1921 * Flow rule attributes. 1922 * @param[in] pattern 1923 * Pattern specification (list terminated by the END pattern item). 1924 * @param[in] actions 1925 * Associated actions (list terminated by the END action). 1926 * @param[out] error 1927 * Perform verbose error reporting if not NULL. 1928 * 1929 * @return 1930 * A flow on success, NULL otherwise and rte_errno is set. 1931 */ 1932 static struct rte_flow * 1933 mlx5_flow_list_create(struct rte_eth_dev *dev, 1934 struct mlx5_flows *list, 1935 const struct rte_flow_attr *attr, 1936 const struct rte_flow_item items[], 1937 const struct rte_flow_action actions[], 1938 struct rte_flow_error *error) 1939 { 1940 struct mlx5_flow_parse parser = { .create = 1, }; 1941 struct rte_flow *flow = NULL; 1942 unsigned int i; 1943 int ret; 1944 1945 ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser); 1946 if (ret) 1947 goto exit; 1948 flow = rte_calloc(__func__, 1, 1949 sizeof(*flow) + parser.queues_n * sizeof(uint16_t), 1950 0); 1951 if (!flow) { 1952 rte_flow_error_set(error, ENOMEM, 1953 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1954 NULL, 1955 "cannot allocate flow memory"); 1956 return NULL; 1957 } 1958 /* Copy queues configuration. */ 1959 flow->queues = (uint16_t (*)[])(flow + 1); 1960 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t)); 1961 flow->queues_n = parser.queues_n; 1962 flow->mark = parser.mark; 1963 /* Copy RSS configuration. */ 1964 flow->rss_conf = parser.rss_conf; 1965 flow->rss_conf.rss_key = flow->rss_key; 1966 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len); 1967 /* finalise the flow. */ 1968 if (parser.drop) 1969 ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow, 1970 error); 1971 else 1972 ret = mlx5_flow_create_action_queue(dev, &parser, flow, error); 1973 if (ret) 1974 goto exit; 1975 TAILQ_INSERT_TAIL(list, flow, next); 1976 DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id, 1977 (void *)flow); 1978 return flow; 1979 exit: 1980 DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id, 1981 error->message); 1982 for (i = 0; i != hash_rxq_init_n; ++i) { 1983 if (parser.queue[i].ibv_attr) 1984 rte_free(parser.queue[i].ibv_attr); 1985 } 1986 rte_free(flow); 1987 return NULL; 1988 } 1989 1990 /** 1991 * Validate a flow supported by the NIC. 1992 * 1993 * @see rte_flow_validate() 1994 * @see rte_flow_ops 1995 */ 1996 int 1997 mlx5_flow_validate(struct rte_eth_dev *dev, 1998 const struct rte_flow_attr *attr, 1999 const struct rte_flow_item items[], 2000 const struct rte_flow_action actions[], 2001 struct rte_flow_error *error) 2002 { 2003 struct mlx5_flow_parse parser = { .create = 0, }; 2004 2005 return mlx5_flow_convert(dev, attr, items, actions, error, &parser); 2006 } 2007 2008 /** 2009 * Create a flow. 2010 * 2011 * @see rte_flow_create() 2012 * @see rte_flow_ops 2013 */ 2014 struct rte_flow * 2015 mlx5_flow_create(struct rte_eth_dev *dev, 2016 const struct rte_flow_attr *attr, 2017 const struct rte_flow_item items[], 2018 const struct rte_flow_action actions[], 2019 struct rte_flow_error *error) 2020 { 2021 struct priv *priv = dev->data->dev_private; 2022 2023 return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions, 2024 error); 2025 } 2026 2027 /** 2028 * Destroy a flow in a list. 2029 * 2030 * @param dev 2031 * Pointer to Ethernet device. 2032 * @param list 2033 * Pointer to a TAILQ flow list. 2034 * @param[in] flow 2035 * Flow to destroy. 2036 */ 2037 static void 2038 mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2039 struct rte_flow *flow) 2040 { 2041 struct priv *priv = dev->data->dev_private; 2042 unsigned int i; 2043 2044 if (flow->drop || !flow->mark) 2045 goto free; 2046 for (i = 0; i != flow->queues_n; ++i) { 2047 struct rte_flow *tmp; 2048 int mark = 0; 2049 2050 /* 2051 * To remove the mark from the queue, the queue must not be 2052 * present in any other marked flow (RSS or not). 2053 */ 2054 TAILQ_FOREACH(tmp, list, next) { 2055 unsigned int j; 2056 uint16_t *tqs = NULL; 2057 uint16_t tq_n = 0; 2058 2059 if (!tmp->mark) 2060 continue; 2061 for (j = 0; j != hash_rxq_init_n; ++j) { 2062 if (!tmp->frxq[j].hrxq) 2063 continue; 2064 tqs = tmp->frxq[j].hrxq->ind_table->queues; 2065 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n; 2066 } 2067 if (!tq_n) 2068 continue; 2069 for (j = 0; (j != tq_n) && !mark; j++) 2070 if (tqs[j] == (*flow->queues)[i]) 2071 mark = 1; 2072 } 2073 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark; 2074 } 2075 free: 2076 if (flow->drop) { 2077 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) 2078 claim_zero(mlx5_glue->destroy_flow 2079 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 2080 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); 2081 } else { 2082 for (i = 0; i != hash_rxq_init_n; ++i) { 2083 struct mlx5_flow *frxq = &flow->frxq[i]; 2084 2085 if (frxq->ibv_flow) 2086 claim_zero(mlx5_glue->destroy_flow 2087 (frxq->ibv_flow)); 2088 if (frxq->hrxq) 2089 mlx5_hrxq_release(dev, frxq->hrxq); 2090 if (frxq->ibv_attr) 2091 rte_free(frxq->ibv_attr); 2092 } 2093 } 2094 if (flow->cs) { 2095 claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); 2096 flow->cs = NULL; 2097 } 2098 TAILQ_REMOVE(list, flow, next); 2099 DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id, 2100 (void *)flow); 2101 rte_free(flow); 2102 } 2103 2104 /** 2105 * Destroy all flows. 2106 * 2107 * @param dev 2108 * Pointer to Ethernet device. 2109 * @param list 2110 * Pointer to a TAILQ flow list. 2111 */ 2112 void 2113 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 2114 { 2115 while (!TAILQ_EMPTY(list)) { 2116 struct rte_flow *flow; 2117 2118 flow = TAILQ_FIRST(list); 2119 mlx5_flow_list_destroy(dev, list, flow); 2120 } 2121 } 2122 2123 /** 2124 * Create drop queue. 2125 * 2126 * @param dev 2127 * Pointer to Ethernet device. 2128 * 2129 * @return 2130 * 0 on success, a negative errno value otherwise and rte_errno is set. 2131 */ 2132 int 2133 mlx5_flow_create_drop_queue(struct rte_eth_dev *dev) 2134 { 2135 struct priv *priv = dev->data->dev_private; 2136 struct mlx5_hrxq_drop *fdq = NULL; 2137 2138 assert(priv->pd); 2139 assert(priv->ctx); 2140 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); 2141 if (!fdq) { 2142 DRV_LOG(WARNING, 2143 "port %u cannot allocate memory for drop queue", 2144 dev->data->port_id); 2145 rte_errno = ENOMEM; 2146 return -rte_errno; 2147 } 2148 fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0); 2149 if (!fdq->cq) { 2150 DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue", 2151 dev->data->port_id); 2152 rte_errno = errno; 2153 goto error; 2154 } 2155 fdq->wq = mlx5_glue->create_wq 2156 (priv->ctx, 2157 &(struct ibv_wq_init_attr){ 2158 .wq_type = IBV_WQT_RQ, 2159 .max_wr = 1, 2160 .max_sge = 1, 2161 .pd = priv->pd, 2162 .cq = fdq->cq, 2163 }); 2164 if (!fdq->wq) { 2165 DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue", 2166 dev->data->port_id); 2167 rte_errno = errno; 2168 goto error; 2169 } 2170 fdq->ind_table = mlx5_glue->create_rwq_ind_table 2171 (priv->ctx, 2172 &(struct ibv_rwq_ind_table_init_attr){ 2173 .log_ind_tbl_size = 0, 2174 .ind_tbl = &fdq->wq, 2175 .comp_mask = 0, 2176 }); 2177 if (!fdq->ind_table) { 2178 DRV_LOG(WARNING, 2179 "port %u cannot allocate indirection table for drop" 2180 " queue", 2181 dev->data->port_id); 2182 rte_errno = errno; 2183 goto error; 2184 } 2185 fdq->qp = mlx5_glue->create_qp_ex 2186 (priv->ctx, 2187 &(struct ibv_qp_init_attr_ex){ 2188 .qp_type = IBV_QPT_RAW_PACKET, 2189 .comp_mask = 2190 IBV_QP_INIT_ATTR_PD | 2191 IBV_QP_INIT_ATTR_IND_TABLE | 2192 IBV_QP_INIT_ATTR_RX_HASH, 2193 .rx_hash_conf = (struct ibv_rx_hash_conf){ 2194 .rx_hash_function = 2195 IBV_RX_HASH_FUNC_TOEPLITZ, 2196 .rx_hash_key_len = rss_hash_default_key_len, 2197 .rx_hash_key = rss_hash_default_key, 2198 .rx_hash_fields_mask = 0, 2199 }, 2200 .rwq_ind_tbl = fdq->ind_table, 2201 .pd = priv->pd 2202 }); 2203 if (!fdq->qp) { 2204 DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue", 2205 dev->data->port_id); 2206 rte_errno = errno; 2207 goto error; 2208 } 2209 priv->flow_drop_queue = fdq; 2210 return 0; 2211 error: 2212 if (fdq->qp) 2213 claim_zero(mlx5_glue->destroy_qp(fdq->qp)); 2214 if (fdq->ind_table) 2215 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table)); 2216 if (fdq->wq) 2217 claim_zero(mlx5_glue->destroy_wq(fdq->wq)); 2218 if (fdq->cq) 2219 claim_zero(mlx5_glue->destroy_cq(fdq->cq)); 2220 if (fdq) 2221 rte_free(fdq); 2222 priv->flow_drop_queue = NULL; 2223 return -rte_errno; 2224 } 2225 2226 /** 2227 * Delete drop queue. 2228 * 2229 * @param dev 2230 * Pointer to Ethernet device. 2231 */ 2232 void 2233 mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev) 2234 { 2235 struct priv *priv = dev->data->dev_private; 2236 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; 2237 2238 if (!fdq) 2239 return; 2240 if (fdq->qp) 2241 claim_zero(mlx5_glue->destroy_qp(fdq->qp)); 2242 if (fdq->ind_table) 2243 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table)); 2244 if (fdq->wq) 2245 claim_zero(mlx5_glue->destroy_wq(fdq->wq)); 2246 if (fdq->cq) 2247 claim_zero(mlx5_glue->destroy_cq(fdq->cq)); 2248 rte_free(fdq); 2249 priv->flow_drop_queue = NULL; 2250 } 2251 2252 /** 2253 * Remove all flows. 2254 * 2255 * @param dev 2256 * Pointer to Ethernet device. 2257 * @param list 2258 * Pointer to a TAILQ flow list. 2259 */ 2260 void 2261 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 2262 { 2263 struct priv *priv = dev->data->dev_private; 2264 struct rte_flow *flow; 2265 2266 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 2267 unsigned int i; 2268 struct mlx5_ind_table_ibv *ind_tbl = NULL; 2269 2270 if (flow->drop) { 2271 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) 2272 continue; 2273 claim_zero(mlx5_glue->destroy_flow 2274 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 2275 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; 2276 DRV_LOG(DEBUG, "port %u flow %p removed", 2277 dev->data->port_id, (void *)flow); 2278 /* Next flow. */ 2279 continue; 2280 } 2281 /* Verify the flow has not already been cleaned. */ 2282 for (i = 0; i != hash_rxq_init_n; ++i) { 2283 if (!flow->frxq[i].ibv_flow) 2284 continue; 2285 /* 2286 * Indirection table may be necessary to remove the 2287 * flags in the Rx queues. 2288 * This helps to speed-up the process by avoiding 2289 * another loop. 2290 */ 2291 ind_tbl = flow->frxq[i].hrxq->ind_table; 2292 break; 2293 } 2294 if (i == hash_rxq_init_n) 2295 return; 2296 if (flow->mark) { 2297 assert(ind_tbl); 2298 for (i = 0; i != ind_tbl->queues_n; ++i) 2299 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; 2300 } 2301 for (i = 0; i != hash_rxq_init_n; ++i) { 2302 if (!flow->frxq[i].ibv_flow) 2303 continue; 2304 claim_zero(mlx5_glue->destroy_flow 2305 (flow->frxq[i].ibv_flow)); 2306 flow->frxq[i].ibv_flow = NULL; 2307 mlx5_hrxq_release(dev, flow->frxq[i].hrxq); 2308 flow->frxq[i].hrxq = NULL; 2309 } 2310 DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id, 2311 (void *)flow); 2312 } 2313 } 2314 2315 /** 2316 * Add all flows. 2317 * 2318 * @param dev 2319 * Pointer to Ethernet device. 2320 * @param list 2321 * Pointer to a TAILQ flow list. 2322 * 2323 * @return 2324 * 0 on success, a negative errno value otherwise and rte_errno is set. 2325 */ 2326 int 2327 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 2328 { 2329 struct priv *priv = dev->data->dev_private; 2330 struct rte_flow *flow; 2331 2332 TAILQ_FOREACH(flow, list, next) { 2333 unsigned int i; 2334 2335 if (flow->drop) { 2336 flow->frxq[HASH_RXQ_ETH].ibv_flow = 2337 mlx5_glue->create_flow 2338 (priv->flow_drop_queue->qp, 2339 flow->frxq[HASH_RXQ_ETH].ibv_attr); 2340 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { 2341 DRV_LOG(DEBUG, 2342 "port %u flow %p cannot be applied", 2343 dev->data->port_id, (void *)flow); 2344 rte_errno = EINVAL; 2345 return -rte_errno; 2346 } 2347 DRV_LOG(DEBUG, "port %u flow %p applied", 2348 dev->data->port_id, (void *)flow); 2349 /* Next flow. */ 2350 continue; 2351 } 2352 for (i = 0; i != hash_rxq_init_n; ++i) { 2353 if (!flow->frxq[i].ibv_attr) 2354 continue; 2355 flow->frxq[i].hrxq = 2356 mlx5_hrxq_get(dev, flow->rss_conf.rss_key, 2357 flow->rss_conf.rss_key_len, 2358 hash_rxq_init[i].hash_fields, 2359 (*flow->queues), 2360 flow->queues_n); 2361 if (flow->frxq[i].hrxq) 2362 goto flow_create; 2363 flow->frxq[i].hrxq = 2364 mlx5_hrxq_new(dev, flow->rss_conf.rss_key, 2365 flow->rss_conf.rss_key_len, 2366 hash_rxq_init[i].hash_fields, 2367 (*flow->queues), 2368 flow->queues_n); 2369 if (!flow->frxq[i].hrxq) { 2370 DRV_LOG(DEBUG, 2371 "port %u flow %p cannot be applied", 2372 dev->data->port_id, (void *)flow); 2373 rte_errno = EINVAL; 2374 return -rte_errno; 2375 } 2376 flow_create: 2377 flow->frxq[i].ibv_flow = 2378 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp, 2379 flow->frxq[i].ibv_attr); 2380 if (!flow->frxq[i].ibv_flow) { 2381 DRV_LOG(DEBUG, 2382 "port %u flow %p cannot be applied", 2383 dev->data->port_id, (void *)flow); 2384 rte_errno = EINVAL; 2385 return -rte_errno; 2386 } 2387 DRV_LOG(DEBUG, "port %u flow %p applied", 2388 dev->data->port_id, (void *)flow); 2389 } 2390 if (!flow->mark) 2391 continue; 2392 for (i = 0; i != flow->queues_n; ++i) 2393 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1; 2394 } 2395 return 0; 2396 } 2397 2398 /** 2399 * Verify the flow list is empty 2400 * 2401 * @param dev 2402 * Pointer to Ethernet device. 2403 * 2404 * @return the number of flows not released. 2405 */ 2406 int 2407 mlx5_flow_verify(struct rte_eth_dev *dev) 2408 { 2409 struct priv *priv = dev->data->dev_private; 2410 struct rte_flow *flow; 2411 int ret = 0; 2412 2413 TAILQ_FOREACH(flow, &priv->flows, next) { 2414 DRV_LOG(DEBUG, "port %u flow %p still referenced", 2415 dev->data->port_id, (void *)flow); 2416 ++ret; 2417 } 2418 return ret; 2419 } 2420 2421 /** 2422 * Enable a control flow configured from the control plane. 2423 * 2424 * @param dev 2425 * Pointer to Ethernet device. 2426 * @param eth_spec 2427 * An Ethernet flow spec to apply. 2428 * @param eth_mask 2429 * An Ethernet flow mask to apply. 2430 * @param vlan_spec 2431 * A VLAN flow spec to apply. 2432 * @param vlan_mask 2433 * A VLAN flow mask to apply. 2434 * 2435 * @return 2436 * 0 on success, a negative errno value otherwise and rte_errno is set. 2437 */ 2438 int 2439 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2440 struct rte_flow_item_eth *eth_spec, 2441 struct rte_flow_item_eth *eth_mask, 2442 struct rte_flow_item_vlan *vlan_spec, 2443 struct rte_flow_item_vlan *vlan_mask) 2444 { 2445 struct priv *priv = dev->data->dev_private; 2446 const struct rte_flow_attr attr = { 2447 .ingress = 1, 2448 .priority = MLX5_CTRL_FLOW_PRIORITY, 2449 }; 2450 struct rte_flow_item items[] = { 2451 { 2452 .type = RTE_FLOW_ITEM_TYPE_ETH, 2453 .spec = eth_spec, 2454 .last = NULL, 2455 .mask = eth_mask, 2456 }, 2457 { 2458 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2459 RTE_FLOW_ITEM_TYPE_END, 2460 .spec = vlan_spec, 2461 .last = NULL, 2462 .mask = vlan_mask, 2463 }, 2464 { 2465 .type = RTE_FLOW_ITEM_TYPE_END, 2466 }, 2467 }; 2468 struct rte_flow_action actions[] = { 2469 { 2470 .type = RTE_FLOW_ACTION_TYPE_RSS, 2471 }, 2472 { 2473 .type = RTE_FLOW_ACTION_TYPE_END, 2474 }, 2475 }; 2476 struct rte_flow *flow; 2477 struct rte_flow_error error; 2478 unsigned int i; 2479 union { 2480 struct rte_flow_action_rss rss; 2481 struct { 2482 const struct rte_eth_rss_conf *rss_conf; 2483 uint16_t num; 2484 uint16_t queue[RTE_MAX_QUEUES_PER_PORT]; 2485 } local; 2486 } action_rss; 2487 2488 if (!priv->reta_idx_n) { 2489 rte_errno = EINVAL; 2490 return -rte_errno; 2491 } 2492 for (i = 0; i != priv->reta_idx_n; ++i) 2493 action_rss.local.queue[i] = (*priv->reta_idx)[i]; 2494 action_rss.local.rss_conf = &priv->rss_conf; 2495 action_rss.local.num = priv->reta_idx_n; 2496 actions[0].conf = (const void *)&action_rss.rss; 2497 flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items, 2498 actions, &error); 2499 if (!flow) 2500 return -rte_errno; 2501 return 0; 2502 } 2503 2504 /** 2505 * Enable a flow control configured from the control plane. 2506 * 2507 * @param dev 2508 * Pointer to Ethernet device. 2509 * @param eth_spec 2510 * An Ethernet flow spec to apply. 2511 * @param eth_mask 2512 * An Ethernet flow mask to apply. 2513 * 2514 * @return 2515 * 0 on success, a negative errno value otherwise and rte_errno is set. 2516 */ 2517 int 2518 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2519 struct rte_flow_item_eth *eth_spec, 2520 struct rte_flow_item_eth *eth_mask) 2521 { 2522 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2523 } 2524 2525 /** 2526 * Destroy a flow. 2527 * 2528 * @see rte_flow_destroy() 2529 * @see rte_flow_ops 2530 */ 2531 int 2532 mlx5_flow_destroy(struct rte_eth_dev *dev, 2533 struct rte_flow *flow, 2534 struct rte_flow_error *error __rte_unused) 2535 { 2536 struct priv *priv = dev->data->dev_private; 2537 2538 mlx5_flow_list_destroy(dev, &priv->flows, flow); 2539 return 0; 2540 } 2541 2542 /** 2543 * Destroy all flows. 2544 * 2545 * @see rte_flow_flush() 2546 * @see rte_flow_ops 2547 */ 2548 int 2549 mlx5_flow_flush(struct rte_eth_dev *dev, 2550 struct rte_flow_error *error __rte_unused) 2551 { 2552 struct priv *priv = dev->data->dev_private; 2553 2554 mlx5_flow_list_flush(dev, &priv->flows); 2555 return 0; 2556 } 2557 2558 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 2559 /** 2560 * Query flow counter. 2561 * 2562 * @param cs 2563 * the counter set. 2564 * @param counter_value 2565 * returned data from the counter. 2566 * 2567 * @return 2568 * 0 on success, a negative errno value otherwise and rte_errno is set. 2569 */ 2570 static int 2571 mlx5_flow_query_count(struct ibv_counter_set *cs, 2572 struct mlx5_flow_counter_stats *counter_stats, 2573 struct rte_flow_query_count *query_count, 2574 struct rte_flow_error *error) 2575 { 2576 uint64_t counters[2]; 2577 struct ibv_query_counter_set_attr query_cs_attr = { 2578 .cs = cs, 2579 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 2580 }; 2581 struct ibv_counter_set_data query_out = { 2582 .out = counters, 2583 .outlen = 2 * sizeof(uint64_t), 2584 }; 2585 int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out); 2586 2587 if (err) 2588 return rte_flow_error_set(error, err, 2589 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2590 NULL, 2591 "cannot read counter"); 2592 query_count->hits_set = 1; 2593 query_count->bytes_set = 1; 2594 query_count->hits = counters[0] - counter_stats->hits; 2595 query_count->bytes = counters[1] - counter_stats->bytes; 2596 if (query_count->reset) { 2597 counter_stats->hits = counters[0]; 2598 counter_stats->bytes = counters[1]; 2599 } 2600 return 0; 2601 } 2602 2603 /** 2604 * Query a flows. 2605 * 2606 * @see rte_flow_query() 2607 * @see rte_flow_ops 2608 */ 2609 int 2610 mlx5_flow_query(struct rte_eth_dev *dev __rte_unused, 2611 struct rte_flow *flow, 2612 enum rte_flow_action_type action __rte_unused, 2613 void *data, 2614 struct rte_flow_error *error) 2615 { 2616 if (flow->cs) { 2617 int ret; 2618 2619 ret = mlx5_flow_query_count(flow->cs, 2620 &flow->counter_stats, 2621 (struct rte_flow_query_count *)data, 2622 error); 2623 if (ret) 2624 return ret; 2625 } else { 2626 return rte_flow_error_set(error, EINVAL, 2627 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2628 NULL, 2629 "no counter found for flow"); 2630 } 2631 return 0; 2632 } 2633 #endif 2634 2635 /** 2636 * Isolated mode. 2637 * 2638 * @see rte_flow_isolate() 2639 * @see rte_flow_ops 2640 */ 2641 int 2642 mlx5_flow_isolate(struct rte_eth_dev *dev, 2643 int enable, 2644 struct rte_flow_error *error) 2645 { 2646 struct priv *priv = dev->data->dev_private; 2647 2648 if (dev->data->dev_started) { 2649 rte_flow_error_set(error, EBUSY, 2650 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2651 NULL, 2652 "port must be stopped first"); 2653 return -rte_errno; 2654 } 2655 priv->isolated = !!enable; 2656 if (enable) 2657 priv->dev->dev_ops = &mlx5_dev_ops_isolate; 2658 else 2659 priv->dev->dev_ops = &mlx5_dev_ops; 2660 return 0; 2661 } 2662 2663 /** 2664 * Convert a flow director filter to a generic flow. 2665 * 2666 * @param dev 2667 * Pointer to Ethernet device. 2668 * @param fdir_filter 2669 * Flow director filter to add. 2670 * @param attributes 2671 * Generic flow parameters structure. 2672 * 2673 * @return 2674 * 0 on success, a negative errno value otherwise and rte_errno is set. 2675 */ 2676 static int 2677 mlx5_fdir_filter_convert(struct rte_eth_dev *dev, 2678 const struct rte_eth_fdir_filter *fdir_filter, 2679 struct mlx5_fdir *attributes) 2680 { 2681 struct priv *priv = dev->data->dev_private; 2682 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2683 const struct rte_eth_fdir_masks *mask = 2684 &dev->data->dev_conf.fdir_conf.mask; 2685 2686 /* Validate queue number. */ 2687 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2688 DRV_LOG(ERR, "port %u invalid queue number %d", 2689 dev->data->port_id, fdir_filter->action.rx_queue); 2690 rte_errno = EINVAL; 2691 return -rte_errno; 2692 } 2693 attributes->attr.ingress = 1; 2694 attributes->items[0] = (struct rte_flow_item) { 2695 .type = RTE_FLOW_ITEM_TYPE_ETH, 2696 .spec = &attributes->l2, 2697 .mask = &attributes->l2_mask, 2698 }; 2699 switch (fdir_filter->action.behavior) { 2700 case RTE_ETH_FDIR_ACCEPT: 2701 attributes->actions[0] = (struct rte_flow_action){ 2702 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2703 .conf = &attributes->queue, 2704 }; 2705 break; 2706 case RTE_ETH_FDIR_REJECT: 2707 attributes->actions[0] = (struct rte_flow_action){ 2708 .type = RTE_FLOW_ACTION_TYPE_DROP, 2709 }; 2710 break; 2711 default: 2712 DRV_LOG(ERR, "port %u invalid behavior %d", 2713 dev->data->port_id, 2714 fdir_filter->action.behavior); 2715 rte_errno = ENOTSUP; 2716 return -rte_errno; 2717 } 2718 attributes->queue.index = fdir_filter->action.rx_queue; 2719 /* Handle L3. */ 2720 switch (fdir_filter->input.flow_type) { 2721 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2722 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2723 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2724 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2725 .src_addr = input->flow.ip4_flow.src_ip, 2726 .dst_addr = input->flow.ip4_flow.dst_ip, 2727 .time_to_live = input->flow.ip4_flow.ttl, 2728 .type_of_service = input->flow.ip4_flow.tos, 2729 .next_proto_id = input->flow.ip4_flow.proto, 2730 }; 2731 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){ 2732 .src_addr = mask->ipv4_mask.src_ip, 2733 .dst_addr = mask->ipv4_mask.dst_ip, 2734 .time_to_live = mask->ipv4_mask.ttl, 2735 .type_of_service = mask->ipv4_mask.tos, 2736 .next_proto_id = mask->ipv4_mask.proto, 2737 }; 2738 attributes->items[1] = (struct rte_flow_item){ 2739 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2740 .spec = &attributes->l3, 2741 .mask = &attributes->l3_mask, 2742 }; 2743 break; 2744 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2745 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2746 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2747 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2748 .hop_limits = input->flow.ipv6_flow.hop_limits, 2749 .proto = input->flow.ipv6_flow.proto, 2750 }; 2751 2752 memcpy(attributes->l3.ipv6.hdr.src_addr, 2753 input->flow.ipv6_flow.src_ip, 2754 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2755 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2756 input->flow.ipv6_flow.dst_ip, 2757 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2758 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 2759 mask->ipv6_mask.src_ip, 2760 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2761 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 2762 mask->ipv6_mask.dst_ip, 2763 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 2764 attributes->items[1] = (struct rte_flow_item){ 2765 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2766 .spec = &attributes->l3, 2767 .mask = &attributes->l3_mask, 2768 }; 2769 break; 2770 default: 2771 DRV_LOG(ERR, "port %u invalid flow type%d", 2772 dev->data->port_id, fdir_filter->input.flow_type); 2773 rte_errno = ENOTSUP; 2774 return -rte_errno; 2775 } 2776 /* Handle L4. */ 2777 switch (fdir_filter->input.flow_type) { 2778 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2779 attributes->l4.udp.hdr = (struct udp_hdr){ 2780 .src_port = input->flow.udp4_flow.src_port, 2781 .dst_port = input->flow.udp4_flow.dst_port, 2782 }; 2783 attributes->l4_mask.udp.hdr = (struct udp_hdr){ 2784 .src_port = mask->src_port_mask, 2785 .dst_port = mask->dst_port_mask, 2786 }; 2787 attributes->items[2] = (struct rte_flow_item){ 2788 .type = RTE_FLOW_ITEM_TYPE_UDP, 2789 .spec = &attributes->l4, 2790 .mask = &attributes->l4_mask, 2791 }; 2792 break; 2793 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2794 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2795 .src_port = input->flow.tcp4_flow.src_port, 2796 .dst_port = input->flow.tcp4_flow.dst_port, 2797 }; 2798 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ 2799 .src_port = mask->src_port_mask, 2800 .dst_port = mask->dst_port_mask, 2801 }; 2802 attributes->items[2] = (struct rte_flow_item){ 2803 .type = RTE_FLOW_ITEM_TYPE_TCP, 2804 .spec = &attributes->l4, 2805 .mask = &attributes->l4_mask, 2806 }; 2807 break; 2808 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2809 attributes->l4.udp.hdr = (struct udp_hdr){ 2810 .src_port = input->flow.udp6_flow.src_port, 2811 .dst_port = input->flow.udp6_flow.dst_port, 2812 }; 2813 attributes->l4_mask.udp.hdr = (struct udp_hdr){ 2814 .src_port = mask->src_port_mask, 2815 .dst_port = mask->dst_port_mask, 2816 }; 2817 attributes->items[2] = (struct rte_flow_item){ 2818 .type = RTE_FLOW_ITEM_TYPE_UDP, 2819 .spec = &attributes->l4, 2820 .mask = &attributes->l4_mask, 2821 }; 2822 break; 2823 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2824 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2825 .src_port = input->flow.tcp6_flow.src_port, 2826 .dst_port = input->flow.tcp6_flow.dst_port, 2827 }; 2828 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ 2829 .src_port = mask->src_port_mask, 2830 .dst_port = mask->dst_port_mask, 2831 }; 2832 attributes->items[2] = (struct rte_flow_item){ 2833 .type = RTE_FLOW_ITEM_TYPE_TCP, 2834 .spec = &attributes->l4, 2835 .mask = &attributes->l4_mask, 2836 }; 2837 break; 2838 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2839 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2840 break; 2841 default: 2842 DRV_LOG(ERR, "port %u invalid flow type%d", 2843 dev->data->port_id, fdir_filter->input.flow_type); 2844 rte_errno = ENOTSUP; 2845 return -rte_errno; 2846 } 2847 return 0; 2848 } 2849 2850 /** 2851 * Add new flow director filter and store it in list. 2852 * 2853 * @param dev 2854 * Pointer to Ethernet device. 2855 * @param fdir_filter 2856 * Flow director filter to add. 2857 * 2858 * @return 2859 * 0 on success, a negative errno value otherwise and rte_errno is set. 2860 */ 2861 static int 2862 mlx5_fdir_filter_add(struct rte_eth_dev *dev, 2863 const struct rte_eth_fdir_filter *fdir_filter) 2864 { 2865 struct priv *priv = dev->data->dev_private; 2866 struct mlx5_fdir attributes = { 2867 .attr.group = 0, 2868 .l2_mask = { 2869 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2870 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2871 .type = 0, 2872 }, 2873 }; 2874 struct mlx5_flow_parse parser = { 2875 .layer = HASH_RXQ_ETH, 2876 }; 2877 struct rte_flow_error error; 2878 struct rte_flow *flow; 2879 int ret; 2880 2881 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); 2882 if (ret) 2883 return ret; 2884 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, 2885 attributes.actions, &error, &parser); 2886 if (ret) 2887 return ret; 2888 flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr, 2889 attributes.items, attributes.actions, 2890 &error); 2891 if (flow) { 2892 DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id, 2893 (void *)flow); 2894 return 0; 2895 } 2896 return -rte_errno; 2897 } 2898 2899 /** 2900 * Delete specific filter. 2901 * 2902 * @param dev 2903 * Pointer to Ethernet device. 2904 * @param fdir_filter 2905 * Filter to be deleted. 2906 * 2907 * @return 2908 * 0 on success, a negative errno value otherwise and rte_errno is set. 2909 */ 2910 static int 2911 mlx5_fdir_filter_delete(struct rte_eth_dev *dev, 2912 const struct rte_eth_fdir_filter *fdir_filter) 2913 { 2914 struct priv *priv = dev->data->dev_private; 2915 struct mlx5_fdir attributes = { 2916 .attr.group = 0, 2917 }; 2918 struct mlx5_flow_parse parser = { 2919 .create = 1, 2920 .layer = HASH_RXQ_ETH, 2921 }; 2922 struct rte_flow_error error; 2923 struct rte_flow *flow; 2924 unsigned int i; 2925 int ret; 2926 2927 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); 2928 if (ret) 2929 return ret; 2930 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, 2931 attributes.actions, &error, &parser); 2932 if (ret) 2933 goto exit; 2934 /* 2935 * Special case for drop action which is only set in the 2936 * specifications when the flow is created. In this situation the 2937 * drop specification is missing. 2938 */ 2939 if (parser.drop) { 2940 struct ibv_flow_spec_action_drop *drop; 2941 2942 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr + 2943 parser.queue[HASH_RXQ_ETH].offset); 2944 *drop = (struct ibv_flow_spec_action_drop){ 2945 .type = IBV_FLOW_SPEC_ACTION_DROP, 2946 .size = sizeof(struct ibv_flow_spec_action_drop), 2947 }; 2948 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++; 2949 } 2950 TAILQ_FOREACH(flow, &priv->flows, next) { 2951 struct ibv_flow_attr *attr; 2952 struct ibv_spec_header *attr_h; 2953 void *spec; 2954 struct ibv_flow_attr *flow_attr; 2955 struct ibv_spec_header *flow_h; 2956 void *flow_spec; 2957 unsigned int specs_n; 2958 2959 attr = parser.queue[HASH_RXQ_ETH].ibv_attr; 2960 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; 2961 /* Compare first the attributes. */ 2962 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) 2963 continue; 2964 if (attr->num_of_specs == 0) 2965 continue; 2966 spec = (void *)((uintptr_t)attr + 2967 sizeof(struct ibv_flow_attr)); 2968 flow_spec = (void *)((uintptr_t)flow_attr + 2969 sizeof(struct ibv_flow_attr)); 2970 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs); 2971 for (i = 0; i != specs_n; ++i) { 2972 attr_h = spec; 2973 flow_h = flow_spec; 2974 if (memcmp(spec, flow_spec, 2975 RTE_MIN(attr_h->size, flow_h->size))) 2976 goto wrong_flow; 2977 spec = (void *)((uintptr_t)spec + attr_h->size); 2978 flow_spec = (void *)((uintptr_t)flow_spec + 2979 flow_h->size); 2980 } 2981 /* At this point, the flow match. */ 2982 break; 2983 wrong_flow: 2984 /* The flow does not match. */ 2985 continue; 2986 } 2987 ret = rte_errno; /* Save rte_errno before cleanup. */ 2988 if (flow) 2989 mlx5_flow_list_destroy(dev, &priv->flows, flow); 2990 exit: 2991 for (i = 0; i != hash_rxq_init_n; ++i) { 2992 if (parser.queue[i].ibv_attr) 2993 rte_free(parser.queue[i].ibv_attr); 2994 } 2995 rte_errno = ret; /* Restore rte_errno. */ 2996 return -rte_errno; 2997 } 2998 2999 /** 3000 * Update queue for specific filter. 3001 * 3002 * @param dev 3003 * Pointer to Ethernet device. 3004 * @param fdir_filter 3005 * Filter to be updated. 3006 * 3007 * @return 3008 * 0 on success, a negative errno value otherwise and rte_errno is set. 3009 */ 3010 static int 3011 mlx5_fdir_filter_update(struct rte_eth_dev *dev, 3012 const struct rte_eth_fdir_filter *fdir_filter) 3013 { 3014 int ret; 3015 3016 ret = mlx5_fdir_filter_delete(dev, fdir_filter); 3017 if (ret) 3018 return ret; 3019 return mlx5_fdir_filter_add(dev, fdir_filter); 3020 } 3021 3022 /** 3023 * Flush all filters. 3024 * 3025 * @param dev 3026 * Pointer to Ethernet device. 3027 */ 3028 static void 3029 mlx5_fdir_filter_flush(struct rte_eth_dev *dev) 3030 { 3031 struct priv *priv = dev->data->dev_private; 3032 3033 mlx5_flow_list_flush(dev, &priv->flows); 3034 } 3035 3036 /** 3037 * Get flow director information. 3038 * 3039 * @param dev 3040 * Pointer to Ethernet device. 3041 * @param[out] fdir_info 3042 * Resulting flow director information. 3043 */ 3044 static void 3045 mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 3046 { 3047 struct priv *priv = dev->data->dev_private; 3048 struct rte_eth_fdir_masks *mask = 3049 &priv->dev->data->dev_conf.fdir_conf.mask; 3050 3051 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; 3052 fdir_info->guarant_spc = 0; 3053 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 3054 fdir_info->max_flexpayload = 0; 3055 fdir_info->flow_types_mask[0] = 0; 3056 fdir_info->flex_payload_unit = 0; 3057 fdir_info->max_flex_payload_segment_num = 0; 3058 fdir_info->flex_payload_limit = 0; 3059 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 3060 } 3061 3062 /** 3063 * Deal with flow director operations. 3064 * 3065 * @param dev 3066 * Pointer to Ethernet device. 3067 * @param filter_op 3068 * Operation to perform. 3069 * @param arg 3070 * Pointer to operation-specific structure. 3071 * 3072 * @return 3073 * 0 on success, a negative errno value otherwise and rte_errno is set. 3074 */ 3075 static int 3076 mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 3077 void *arg) 3078 { 3079 struct priv *priv = dev->data->dev_private; 3080 enum rte_fdir_mode fdir_mode = 3081 priv->dev->data->dev_conf.fdir_conf.mode; 3082 3083 if (filter_op == RTE_ETH_FILTER_NOP) 3084 return 0; 3085 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 3086 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 3087 DRV_LOG(ERR, "port %u flow director mode %d not supported", 3088 dev->data->port_id, fdir_mode); 3089 rte_errno = EINVAL; 3090 return -rte_errno; 3091 } 3092 switch (filter_op) { 3093 case RTE_ETH_FILTER_ADD: 3094 return mlx5_fdir_filter_add(dev, arg); 3095 case RTE_ETH_FILTER_UPDATE: 3096 return mlx5_fdir_filter_update(dev, arg); 3097 case RTE_ETH_FILTER_DELETE: 3098 return mlx5_fdir_filter_delete(dev, arg); 3099 case RTE_ETH_FILTER_FLUSH: 3100 mlx5_fdir_filter_flush(dev); 3101 break; 3102 case RTE_ETH_FILTER_INFO: 3103 mlx5_fdir_info_get(dev, arg); 3104 break; 3105 default: 3106 DRV_LOG(DEBUG, "port %u unknown operation %u", 3107 dev->data->port_id, filter_op); 3108 rte_errno = EINVAL; 3109 return -rte_errno; 3110 } 3111 return 0; 3112 } 3113 3114 /** 3115 * Manage filter operations. 3116 * 3117 * @param dev 3118 * Pointer to Ethernet device structure. 3119 * @param filter_type 3120 * Filter type. 3121 * @param filter_op 3122 * Operation to perform. 3123 * @param arg 3124 * Pointer to operation-specific structure. 3125 * 3126 * @return 3127 * 0 on success, a negative errno value otherwise and rte_errno is set. 3128 */ 3129 int 3130 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 3131 enum rte_filter_type filter_type, 3132 enum rte_filter_op filter_op, 3133 void *arg) 3134 { 3135 switch (filter_type) { 3136 case RTE_ETH_FILTER_GENERIC: 3137 if (filter_op != RTE_ETH_FILTER_GET) { 3138 rte_errno = EINVAL; 3139 return -rte_errno; 3140 } 3141 *(const void **)arg = &mlx5_flow_ops; 3142 return 0; 3143 case RTE_ETH_FILTER_FDIR: 3144 return mlx5_fdir_ctrl_func(dev, filter_op, arg); 3145 default: 3146 DRV_LOG(ERR, "port %u filter type (%d) not supported", 3147 dev->data->port_id, filter_type); 3148 rte_errno = ENOTSUP; 3149 return -rte_errno; 3150 } 3151 return 0; 3152 } 3153