1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2016 6WIND S.A. 5 * Copyright 2016 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/queue.h> 35 #include <string.h> 36 37 /* Verbs header. */ 38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 39 #ifdef PEDANTIC 40 #pragma GCC diagnostic ignored "-Wpedantic" 41 #endif 42 #include <infiniband/verbs.h> 43 #ifdef PEDANTIC 44 #pragma GCC diagnostic error "-Wpedantic" 45 #endif 46 47 #include <rte_ethdev.h> 48 #include <rte_flow.h> 49 #include <rte_flow_driver.h> 50 #include <rte_malloc.h> 51 52 #include "mlx5.h" 53 #include "mlx5_prm.h" 54 55 /* Define minimal priority for control plane flows. */ 56 #define MLX5_CTRL_FLOW_PRIORITY 4 57 58 /* Internet Protocol versions. */ 59 #define MLX5_IPV4 4 60 #define MLX5_IPV6 6 61 62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 63 struct ibv_counter_set_init_attr { 64 int dummy; 65 }; 66 struct ibv_flow_spec_counter_action { 67 int dummy; 68 }; 69 struct ibv_counter_set { 70 int dummy; 71 }; 72 73 static inline int 74 ibv_destroy_counter_set(struct ibv_counter_set *cs) 75 { 76 (void)cs; 77 return -ENOTSUP; 78 } 79 #endif 80 81 /* Dev ops structure defined in mlx5.c */ 82 extern const struct eth_dev_ops mlx5_dev_ops; 83 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 84 85 static int 86 mlx5_flow_create_eth(const struct rte_flow_item *item, 87 const void *default_mask, 88 void *data); 89 90 static int 91 mlx5_flow_create_vlan(const struct rte_flow_item *item, 92 const void *default_mask, 93 void *data); 94 95 static int 96 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 97 const void *default_mask, 98 void *data); 99 100 static int 101 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 102 const void *default_mask, 103 void *data); 104 105 static int 106 mlx5_flow_create_udp(const struct rte_flow_item *item, 107 const void *default_mask, 108 void *data); 109 110 static int 111 mlx5_flow_create_tcp(const struct rte_flow_item *item, 112 const void *default_mask, 113 void *data); 114 115 static int 116 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 117 const void *default_mask, 118 void *data); 119 120 struct mlx5_flow_parse; 121 122 static void 123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 124 unsigned int size); 125 126 static int 127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); 128 129 static int 130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser); 131 132 /* Hash RX queue types. */ 133 enum hash_rxq_type { 134 HASH_RXQ_TCPV4, 135 HASH_RXQ_UDPV4, 136 HASH_RXQ_IPV4, 137 HASH_RXQ_TCPV6, 138 HASH_RXQ_UDPV6, 139 HASH_RXQ_IPV6, 140 HASH_RXQ_ETH, 141 }; 142 143 /* Initialization data for hash RX queue. */ 144 struct hash_rxq_init { 145 uint64_t hash_fields; /* Fields that participate in the hash. */ 146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ 147 unsigned int flow_priority; /* Flow priority to use. */ 148 unsigned int ip_version; /* Internet protocol. */ 149 }; 150 151 /* Initialization data for hash RX queues. */ 152 const struct hash_rxq_init hash_rxq_init[] = { 153 [HASH_RXQ_TCPV4] = { 154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 155 IBV_RX_HASH_DST_IPV4 | 156 IBV_RX_HASH_SRC_PORT_TCP | 157 IBV_RX_HASH_DST_PORT_TCP), 158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, 159 .flow_priority = 0, 160 .ip_version = MLX5_IPV4, 161 }, 162 [HASH_RXQ_UDPV4] = { 163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 164 IBV_RX_HASH_DST_IPV4 | 165 IBV_RX_HASH_SRC_PORT_UDP | 166 IBV_RX_HASH_DST_PORT_UDP), 167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, 168 .flow_priority = 0, 169 .ip_version = MLX5_IPV4, 170 }, 171 [HASH_RXQ_IPV4] = { 172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 173 IBV_RX_HASH_DST_IPV4), 174 .dpdk_rss_hf = (ETH_RSS_IPV4 | 175 ETH_RSS_FRAG_IPV4), 176 .flow_priority = 1, 177 .ip_version = MLX5_IPV4, 178 }, 179 [HASH_RXQ_TCPV6] = { 180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 181 IBV_RX_HASH_DST_IPV6 | 182 IBV_RX_HASH_SRC_PORT_TCP | 183 IBV_RX_HASH_DST_PORT_TCP), 184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, 185 .flow_priority = 0, 186 .ip_version = MLX5_IPV6, 187 }, 188 [HASH_RXQ_UDPV6] = { 189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 190 IBV_RX_HASH_DST_IPV6 | 191 IBV_RX_HASH_SRC_PORT_UDP | 192 IBV_RX_HASH_DST_PORT_UDP), 193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, 194 .flow_priority = 0, 195 .ip_version = MLX5_IPV6, 196 }, 197 [HASH_RXQ_IPV6] = { 198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 199 IBV_RX_HASH_DST_IPV6), 200 .dpdk_rss_hf = (ETH_RSS_IPV6 | 201 ETH_RSS_FRAG_IPV6), 202 .flow_priority = 1, 203 .ip_version = MLX5_IPV6, 204 }, 205 [HASH_RXQ_ETH] = { 206 .hash_fields = 0, 207 .dpdk_rss_hf = 0, 208 .flow_priority = 2, 209 }, 210 }; 211 212 /* Number of entries in hash_rxq_init[]. */ 213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); 214 215 /** Structure for holding counter stats. */ 216 struct mlx5_flow_counter_stats { 217 uint64_t hits; /**< Number of packets matched by the rule. */ 218 uint64_t bytes; /**< Number of bytes matched by the rule. */ 219 }; 220 221 /** Structure for Drop queue. */ 222 struct mlx5_hrxq_drop { 223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 224 struct ibv_qp *qp; /**< Verbs queue pair. */ 225 struct ibv_wq *wq; /**< Verbs work queue. */ 226 struct ibv_cq *cq; /**< Verbs completion queue. */ 227 }; 228 229 /* Flows structures. */ 230 struct mlx5_flow { 231 uint64_t hash_fields; /**< Fields that participate in the hash. */ 232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 233 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */ 235 }; 236 237 /* Drop flows structures. */ 238 struct mlx5_flow_drop { 239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 240 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 241 }; 242 243 struct rte_flow { 244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ 245 uint32_t mark:1; /**< Set if the flow is marked. */ 246 uint32_t drop:1; /**< Drop queue. */ 247 uint16_t queues_n; /**< Number of entries in queue[]. */ 248 uint16_t (*queues)[]; /**< Queues indexes to use. */ 249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 250 uint8_t rss_key[40]; /**< copy of the RSS key. */ 251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */ 252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */ 253 union { 254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)]; 255 /**< Flow with Rx queue. */ 256 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */ 257 }; 258 }; 259 260 /** Static initializer for items. */ 261 #define ITEMS(...) \ 262 (const enum rte_flow_item_type []){ \ 263 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ 264 } 265 266 /** Structure to generate a simple graph of layers supported by the NIC. */ 267 struct mlx5_flow_items { 268 /** List of possible actions for these items. */ 269 const enum rte_flow_action_type *const actions; 270 /** Bit-masks corresponding to the possibilities for the item. */ 271 const void *mask; 272 /** 273 * Default bit-masks to use when item->mask is not provided. When 274 * \default_mask is also NULL, the full supported bit-mask (\mask) is 275 * used instead. 276 */ 277 const void *default_mask; 278 /** Bit-masks size in bytes. */ 279 const unsigned int mask_sz; 280 /** 281 * Conversion function from rte_flow to NIC specific flow. 282 * 283 * @param item 284 * rte_flow item to convert. 285 * @param default_mask 286 * Default bit-masks to use when item->mask is not provided. 287 * @param data 288 * Internal structure to store the conversion. 289 * 290 * @return 291 * 0 on success, negative value otherwise. 292 */ 293 int (*convert)(const struct rte_flow_item *item, 294 const void *default_mask, 295 void *data); 296 /** Size in bytes of the destination structure. */ 297 const unsigned int dst_sz; 298 /** List of possible following items. */ 299 const enum rte_flow_item_type *const items; 300 }; 301 302 /** Valid action for this PMD. */ 303 static const enum rte_flow_action_type valid_actions[] = { 304 RTE_FLOW_ACTION_TYPE_DROP, 305 RTE_FLOW_ACTION_TYPE_QUEUE, 306 RTE_FLOW_ACTION_TYPE_MARK, 307 RTE_FLOW_ACTION_TYPE_FLAG, 308 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 309 RTE_FLOW_ACTION_TYPE_COUNT, 310 #endif 311 RTE_FLOW_ACTION_TYPE_END, 312 }; 313 314 /** Graph of supported items and associated actions. */ 315 static const struct mlx5_flow_items mlx5_flow_items[] = { 316 [RTE_FLOW_ITEM_TYPE_END] = { 317 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, 318 RTE_FLOW_ITEM_TYPE_VXLAN), 319 }, 320 [RTE_FLOW_ITEM_TYPE_ETH] = { 321 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, 322 RTE_FLOW_ITEM_TYPE_IPV4, 323 RTE_FLOW_ITEM_TYPE_IPV6), 324 .actions = valid_actions, 325 .mask = &(const struct rte_flow_item_eth){ 326 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 327 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 328 .type = -1, 329 }, 330 .default_mask = &rte_flow_item_eth_mask, 331 .mask_sz = sizeof(struct rte_flow_item_eth), 332 .convert = mlx5_flow_create_eth, 333 .dst_sz = sizeof(struct ibv_flow_spec_eth), 334 }, 335 [RTE_FLOW_ITEM_TYPE_VLAN] = { 336 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, 337 RTE_FLOW_ITEM_TYPE_IPV6), 338 .actions = valid_actions, 339 .mask = &(const struct rte_flow_item_vlan){ 340 .tci = -1, 341 }, 342 .default_mask = &rte_flow_item_vlan_mask, 343 .mask_sz = sizeof(struct rte_flow_item_vlan), 344 .convert = mlx5_flow_create_vlan, 345 .dst_sz = 0, 346 }, 347 [RTE_FLOW_ITEM_TYPE_IPV4] = { 348 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 349 RTE_FLOW_ITEM_TYPE_TCP), 350 .actions = valid_actions, 351 .mask = &(const struct rte_flow_item_ipv4){ 352 .hdr = { 353 .src_addr = -1, 354 .dst_addr = -1, 355 .type_of_service = -1, 356 .next_proto_id = -1, 357 }, 358 }, 359 .default_mask = &rte_flow_item_ipv4_mask, 360 .mask_sz = sizeof(struct rte_flow_item_ipv4), 361 .convert = mlx5_flow_create_ipv4, 362 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), 363 }, 364 [RTE_FLOW_ITEM_TYPE_IPV6] = { 365 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 366 RTE_FLOW_ITEM_TYPE_TCP), 367 .actions = valid_actions, 368 .mask = &(const struct rte_flow_item_ipv6){ 369 .hdr = { 370 .src_addr = { 371 0xff, 0xff, 0xff, 0xff, 372 0xff, 0xff, 0xff, 0xff, 373 0xff, 0xff, 0xff, 0xff, 374 0xff, 0xff, 0xff, 0xff, 375 }, 376 .dst_addr = { 377 0xff, 0xff, 0xff, 0xff, 378 0xff, 0xff, 0xff, 0xff, 379 0xff, 0xff, 0xff, 0xff, 380 0xff, 0xff, 0xff, 0xff, 381 }, 382 .vtc_flow = -1, 383 .proto = -1, 384 .hop_limits = -1, 385 }, 386 }, 387 .default_mask = &rte_flow_item_ipv6_mask, 388 .mask_sz = sizeof(struct rte_flow_item_ipv6), 389 .convert = mlx5_flow_create_ipv6, 390 .dst_sz = sizeof(struct ibv_flow_spec_ipv6), 391 }, 392 [RTE_FLOW_ITEM_TYPE_UDP] = { 393 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), 394 .actions = valid_actions, 395 .mask = &(const struct rte_flow_item_udp){ 396 .hdr = { 397 .src_port = -1, 398 .dst_port = -1, 399 }, 400 }, 401 .default_mask = &rte_flow_item_udp_mask, 402 .mask_sz = sizeof(struct rte_flow_item_udp), 403 .convert = mlx5_flow_create_udp, 404 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 405 }, 406 [RTE_FLOW_ITEM_TYPE_TCP] = { 407 .actions = valid_actions, 408 .mask = &(const struct rte_flow_item_tcp){ 409 .hdr = { 410 .src_port = -1, 411 .dst_port = -1, 412 }, 413 }, 414 .default_mask = &rte_flow_item_tcp_mask, 415 .mask_sz = sizeof(struct rte_flow_item_tcp), 416 .convert = mlx5_flow_create_tcp, 417 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 418 }, 419 [RTE_FLOW_ITEM_TYPE_VXLAN] = { 420 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), 421 .actions = valid_actions, 422 .mask = &(const struct rte_flow_item_vxlan){ 423 .vni = "\xff\xff\xff", 424 }, 425 .default_mask = &rte_flow_item_vxlan_mask, 426 .mask_sz = sizeof(struct rte_flow_item_vxlan), 427 .convert = mlx5_flow_create_vxlan, 428 .dst_sz = sizeof(struct ibv_flow_spec_tunnel), 429 }, 430 }; 431 432 /** Structure to pass to the conversion function. */ 433 struct mlx5_flow_parse { 434 uint32_t inner; /**< Set once VXLAN is encountered. */ 435 uint32_t create:1; 436 /**< Whether resources should remain after a validate. */ 437 uint32_t drop:1; /**< Target is a drop queue. */ 438 uint32_t mark:1; /**< Mark is present in the flow. */ 439 uint32_t count:1; /**< Count is present in the flow. */ 440 uint32_t mark_id; /**< Mark identifier. */ 441 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ 442 uint16_t queues_n; /**< Number of entries in queue[]. */ 443 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 444 uint8_t rss_key[40]; /**< copy of the RSS key. */ 445 enum hash_rxq_type layer; /**< Last pattern layer detected. */ 446 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */ 447 union { 448 struct { 449 struct ibv_flow_attr *ibv_attr; 450 /**< Pointer to Verbs attributes. */ 451 unsigned int offset; 452 /**< Current position or total size of the attribute. */ 453 } queue[RTE_DIM(hash_rxq_init)]; 454 struct { 455 struct ibv_flow_attr *ibv_attr; 456 /**< Pointer to Verbs attributes. */ 457 unsigned int offset; 458 /**< Current position or total size of the attribute. */ 459 } drop_q; 460 }; 461 }; 462 463 static const struct rte_flow_ops mlx5_flow_ops = { 464 .validate = mlx5_flow_validate, 465 .create = mlx5_flow_create, 466 .destroy = mlx5_flow_destroy, 467 .flush = mlx5_flow_flush, 468 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 469 .query = mlx5_flow_query, 470 #else 471 .query = NULL, 472 #endif 473 .isolate = mlx5_flow_isolate, 474 }; 475 476 /* Convert FDIR request to Generic flow. */ 477 struct mlx5_fdir { 478 struct rte_flow_attr attr; 479 struct rte_flow_action actions[2]; 480 struct rte_flow_item items[4]; 481 struct rte_flow_item_eth l2; 482 union { 483 struct rte_flow_item_ipv4 ipv4; 484 struct rte_flow_item_ipv6 ipv6; 485 } l3; 486 union { 487 struct rte_flow_item_udp udp; 488 struct rte_flow_item_tcp tcp; 489 } l4; 490 struct rte_flow_action_queue queue; 491 }; 492 493 /* Verbs specification header. */ 494 struct ibv_spec_header { 495 enum ibv_flow_spec_type type; 496 uint16_t size; 497 }; 498 499 /** 500 * Check support for a given item. 501 * 502 * @param item[in] 503 * Item specification. 504 * @param mask[in] 505 * Bit-masks covering supported fields to compare with spec, last and mask in 506 * \item. 507 * @param size 508 * Bit-Mask size in bytes. 509 * 510 * @return 511 * 0 on success. 512 */ 513 static int 514 mlx5_flow_item_validate(const struct rte_flow_item *item, 515 const uint8_t *mask, unsigned int size) 516 { 517 int ret = 0; 518 519 if (!item->spec && (item->mask || item->last)) 520 return -1; 521 if (item->spec && !item->mask) { 522 unsigned int i; 523 const uint8_t *spec = item->spec; 524 525 for (i = 0; i < size; ++i) 526 if ((spec[i] | mask[i]) != mask[i]) 527 return -1; 528 } 529 if (item->last && !item->mask) { 530 unsigned int i; 531 const uint8_t *spec = item->last; 532 533 for (i = 0; i < size; ++i) 534 if ((spec[i] | mask[i]) != mask[i]) 535 return -1; 536 } 537 if (item->mask) { 538 unsigned int i; 539 const uint8_t *spec = item->mask; 540 541 for (i = 0; i < size; ++i) 542 if ((spec[i] | mask[i]) != mask[i]) 543 return -1; 544 } 545 if (item->spec && item->last) { 546 uint8_t spec[size]; 547 uint8_t last[size]; 548 const uint8_t *apply = mask; 549 unsigned int i; 550 551 if (item->mask) 552 apply = item->mask; 553 for (i = 0; i < size; ++i) { 554 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; 555 last[i] = ((const uint8_t *)item->last)[i] & apply[i]; 556 } 557 ret = memcmp(spec, last, size); 558 } 559 return ret; 560 } 561 562 /** 563 * Copy the RSS configuration from the user ones. 564 * 565 * @param priv 566 * Pointer to private structure. 567 * @param parser 568 * Internal parser structure. 569 * @param rss_conf 570 * User RSS configuration to save. 571 * 572 * @return 573 * 0 on success, errno value on failure. 574 */ 575 static int 576 priv_flow_convert_rss_conf(struct priv *priv, 577 struct mlx5_flow_parse *parser, 578 const struct rte_eth_rss_conf *rss_conf) 579 { 580 const struct rte_eth_rss_conf *rss = 581 rss_conf ? rss_conf : &priv->rss_conf; 582 583 if (rss->rss_key_len > 40) 584 return EINVAL; 585 parser->rss_conf.rss_key_len = rss->rss_key_len; 586 parser->rss_conf.rss_hf = rss->rss_hf; 587 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len); 588 parser->rss_conf.rss_key = parser->rss_key; 589 return 0; 590 } 591 592 /** 593 * Extract attribute to the parser. 594 * 595 * @param priv 596 * Pointer to private structure. 597 * @param[in] attr 598 * Flow rule attributes. 599 * @param[out] error 600 * Perform verbose error reporting if not NULL. 601 * @param[in, out] parser 602 * Internal parser structure. 603 * 604 * @return 605 * 0 on success, a negative errno value otherwise and rte_errno is set. 606 */ 607 static int 608 priv_flow_convert_attributes(struct priv *priv, 609 const struct rte_flow_attr *attr, 610 struct rte_flow_error *error, 611 struct mlx5_flow_parse *parser) 612 { 613 (void)priv; 614 (void)parser; 615 if (attr->group) { 616 rte_flow_error_set(error, ENOTSUP, 617 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 618 NULL, 619 "groups are not supported"); 620 return -rte_errno; 621 } 622 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) { 623 rte_flow_error_set(error, ENOTSUP, 624 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 625 NULL, 626 "priorities are not supported"); 627 return -rte_errno; 628 } 629 if (attr->egress) { 630 rte_flow_error_set(error, ENOTSUP, 631 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 632 NULL, 633 "egress is not supported"); 634 return -rte_errno; 635 } 636 if (!attr->ingress) { 637 rte_flow_error_set(error, ENOTSUP, 638 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 639 NULL, 640 "only ingress is supported"); 641 return -rte_errno; 642 } 643 return 0; 644 } 645 646 /** 647 * Extract actions request to the parser. 648 * 649 * @param priv 650 * Pointer to private structure. 651 * @param[in] actions 652 * Associated actions (list terminated by the END action). 653 * @param[out] error 654 * Perform verbose error reporting if not NULL. 655 * @param[in, out] parser 656 * Internal parser structure. 657 * 658 * @return 659 * 0 on success, a negative errno value otherwise and rte_errno is set. 660 */ 661 static int 662 priv_flow_convert_actions(struct priv *priv, 663 const struct rte_flow_action actions[], 664 struct rte_flow_error *error, 665 struct mlx5_flow_parse *parser) 666 { 667 /* 668 * Add default RSS configuration necessary for Verbs to create QP even 669 * if no RSS is necessary. 670 */ 671 priv_flow_convert_rss_conf(priv, parser, 672 (const struct rte_eth_rss_conf *) 673 &priv->rss_conf); 674 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { 675 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { 676 continue; 677 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { 678 parser->drop = 1; 679 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 680 const struct rte_flow_action_queue *queue = 681 (const struct rte_flow_action_queue *) 682 actions->conf; 683 uint16_t n; 684 uint16_t found = 0; 685 686 if (!queue || (queue->index > (priv->rxqs_n - 1))) 687 goto exit_action_not_supported; 688 for (n = 0; n < parser->queues_n; ++n) { 689 if (parser->queues[n] == queue->index) { 690 found = 1; 691 break; 692 } 693 } 694 if (parser->queues_n > 1 && !found) { 695 rte_flow_error_set(error, ENOTSUP, 696 RTE_FLOW_ERROR_TYPE_ACTION, 697 actions, 698 "queue action not in RSS queues"); 699 return -rte_errno; 700 } 701 if (!found) { 702 parser->queues_n = 1; 703 parser->queues[0] = queue->index; 704 } 705 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { 706 const struct rte_flow_action_rss *rss = 707 (const struct rte_flow_action_rss *) 708 actions->conf; 709 uint16_t n; 710 711 if (!rss || !rss->num) { 712 rte_flow_error_set(error, EINVAL, 713 RTE_FLOW_ERROR_TYPE_ACTION, 714 actions, 715 "no valid queues"); 716 return -rte_errno; 717 } 718 if (parser->queues_n == 1) { 719 uint16_t found = 0; 720 721 assert(parser->queues_n); 722 for (n = 0; n < rss->num; ++n) { 723 if (parser->queues[0] == 724 rss->queue[n]) { 725 found = 1; 726 break; 727 } 728 } 729 if (!found) { 730 rte_flow_error_set(error, ENOTSUP, 731 RTE_FLOW_ERROR_TYPE_ACTION, 732 actions, 733 "queue action not in RSS" 734 " queues"); 735 return -rte_errno; 736 } 737 } 738 for (n = 0; n < rss->num; ++n) { 739 if (rss->queue[n] >= priv->rxqs_n) { 740 rte_flow_error_set(error, EINVAL, 741 RTE_FLOW_ERROR_TYPE_ACTION, 742 actions, 743 "queue id > number of" 744 " queues"); 745 return -rte_errno; 746 } 747 } 748 for (n = 0; n < rss->num; ++n) 749 parser->queues[n] = rss->queue[n]; 750 parser->queues_n = rss->num; 751 if (priv_flow_convert_rss_conf(priv, parser, 752 rss->rss_conf)) { 753 rte_flow_error_set(error, EINVAL, 754 RTE_FLOW_ERROR_TYPE_ACTION, 755 actions, 756 "wrong RSS configuration"); 757 return -rte_errno; 758 } 759 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { 760 const struct rte_flow_action_mark *mark = 761 (const struct rte_flow_action_mark *) 762 actions->conf; 763 764 if (!mark) { 765 rte_flow_error_set(error, EINVAL, 766 RTE_FLOW_ERROR_TYPE_ACTION, 767 actions, 768 "mark must be defined"); 769 return -rte_errno; 770 } else if (mark->id >= MLX5_FLOW_MARK_MAX) { 771 rte_flow_error_set(error, ENOTSUP, 772 RTE_FLOW_ERROR_TYPE_ACTION, 773 actions, 774 "mark must be between 0" 775 " and 16777199"); 776 return -rte_errno; 777 } 778 parser->mark = 1; 779 parser->mark_id = mark->id; 780 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { 781 parser->mark = 1; 782 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && 783 priv->counter_set_supported) { 784 parser->count = 1; 785 } else { 786 goto exit_action_not_supported; 787 } 788 } 789 if (!parser->queues_n && !parser->drop) { 790 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, 791 NULL, "no valid action"); 792 return -rte_errno; 793 } 794 return 0; 795 exit_action_not_supported: 796 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 797 actions, "action not supported"); 798 return -rte_errno; 799 } 800 801 /** 802 * Validate items. 803 * 804 * @param priv 805 * Pointer to private structure. 806 * @param[in] items 807 * Pattern specification (list terminated by the END pattern item). 808 * @param[out] error 809 * Perform verbose error reporting if not NULL. 810 * @param[in, out] parser 811 * Internal parser structure. 812 * 813 * @return 814 * 0 on success, a negative errno value otherwise and rte_errno is set. 815 */ 816 static int 817 priv_flow_convert_items_validate(struct priv *priv, 818 const struct rte_flow_item items[], 819 struct rte_flow_error *error, 820 struct mlx5_flow_parse *parser) 821 { 822 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 823 unsigned int i; 824 825 (void)priv; 826 /* Initialise the offsets to start after verbs attribute. */ 827 if (parser->drop) { 828 parser->drop_q.offset = sizeof(struct ibv_flow_attr); 829 } else { 830 for (i = 0; i != hash_rxq_init_n; ++i) 831 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 832 } 833 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 834 const struct mlx5_flow_items *token = NULL; 835 unsigned int n; 836 int err; 837 838 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 839 continue; 840 for (i = 0; 841 cur_item->items && 842 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; 843 ++i) { 844 if (cur_item->items[i] == items->type) { 845 token = &mlx5_flow_items[items->type]; 846 break; 847 } 848 } 849 if (!token) 850 goto exit_item_not_supported; 851 cur_item = token; 852 err = mlx5_flow_item_validate(items, 853 (const uint8_t *)cur_item->mask, 854 cur_item->mask_sz); 855 if (err) 856 goto exit_item_not_supported; 857 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 858 if (parser->inner) { 859 rte_flow_error_set(error, ENOTSUP, 860 RTE_FLOW_ERROR_TYPE_ITEM, 861 items, 862 "cannot recognize multiple" 863 " VXLAN encapsulations"); 864 return -rte_errno; 865 } 866 parser->inner = 1; 867 } 868 if (parser->drop) { 869 parser->drop_q.offset += cur_item->dst_sz; 870 } else if (parser->queues_n == 1) { 871 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; 872 } else { 873 for (n = 0; n != hash_rxq_init_n; ++n) 874 parser->queue[n].offset += cur_item->dst_sz; 875 } 876 } 877 if (parser->mark) { 878 for (i = 0; i != hash_rxq_init_n; ++i) 879 parser->queue[i].offset += 880 sizeof(struct ibv_flow_spec_action_tag); 881 } 882 if (parser->count) { 883 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 884 885 if (parser->drop) { 886 parser->drop_q.offset += size; 887 } else { 888 for (i = 0; i != hash_rxq_init_n; ++i) 889 parser->queue[i].offset += size; 890 } 891 } 892 return 0; 893 exit_item_not_supported: 894 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 895 items, "item not supported"); 896 return -rte_errno; 897 } 898 899 /** 900 * Allocate memory space to store verbs flow attributes. 901 * 902 * @param priv 903 * Pointer to private structure. 904 * @param[in] priority 905 * Flow priority. 906 * @param[in] size 907 * Amount of byte to allocate. 908 * @param[out] error 909 * Perform verbose error reporting if not NULL. 910 * 911 * @return 912 * A verbs flow attribute on success, NULL otherwise. 913 */ 914 static struct ibv_flow_attr* 915 priv_flow_convert_allocate(struct priv *priv, 916 unsigned int priority, 917 unsigned int size, 918 struct rte_flow_error *error) 919 { 920 struct ibv_flow_attr *ibv_attr; 921 922 (void)priv; 923 ibv_attr = rte_calloc(__func__, 1, size, 0); 924 if (!ibv_attr) { 925 rte_flow_error_set(error, ENOMEM, 926 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 927 NULL, 928 "cannot allocate verbs spec attributes."); 929 return NULL; 930 } 931 ibv_attr->priority = priority; 932 return ibv_attr; 933 } 934 935 /** 936 * Finalise verbs flow attributes. 937 * 938 * @param priv 939 * Pointer to private structure. 940 * @param[in, out] parser 941 * Internal parser structure. 942 */ 943 static void 944 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) 945 { 946 const unsigned int ipv4 = 947 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; 948 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6; 949 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 950 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4; 951 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4; 952 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 953 unsigned int i; 954 955 (void)priv; 956 if (parser->layer == HASH_RXQ_ETH) { 957 goto fill; 958 } else { 959 /* 960 * This layer becomes useless as the pattern define under 961 * layers. 962 */ 963 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr); 964 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 965 } 966 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */ 967 for (i = ohmin; i != (ohmax + 1); ++i) { 968 if (!parser->queue[i].ibv_attr) 969 continue; 970 rte_free(parser->queue[i].ibv_attr); 971 parser->queue[i].ibv_attr = NULL; 972 } 973 /* Remove impossible flow according to the RSS configuration. */ 974 if (hash_rxq_init[parser->layer].dpdk_rss_hf & 975 parser->rss_conf.rss_hf) { 976 /* Remove any other flow. */ 977 for (i = hmin; i != (hmax + 1); ++i) { 978 if ((i == parser->layer) || 979 (!parser->queue[i].ibv_attr)) 980 continue; 981 rte_free(parser->queue[i].ibv_attr); 982 parser->queue[i].ibv_attr = NULL; 983 } 984 } else if (!parser->queue[ip].ibv_attr) { 985 /* no RSS possible with the current configuration. */ 986 parser->queues_n = 1; 987 return; 988 } 989 fill: 990 /* 991 * Fill missing layers in verbs specifications, or compute the correct 992 * offset to allocate the memory space for the attributes and 993 * specifications. 994 */ 995 for (i = 0; i != hash_rxq_init_n - 1; ++i) { 996 union { 997 struct ibv_flow_spec_ipv4_ext ipv4; 998 struct ibv_flow_spec_ipv6 ipv6; 999 struct ibv_flow_spec_tcp_udp udp_tcp; 1000 } specs; 1001 void *dst; 1002 uint16_t size; 1003 1004 if (i == parser->layer) 1005 continue; 1006 if (parser->layer == HASH_RXQ_ETH) { 1007 if (hash_rxq_init[i].ip_version == MLX5_IPV4) { 1008 size = sizeof(struct ibv_flow_spec_ipv4_ext); 1009 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){ 1010 .type = IBV_FLOW_SPEC_IPV4_EXT | 1011 parser->inner, 1012 .size = size, 1013 }; 1014 } else { 1015 size = sizeof(struct ibv_flow_spec_ipv6); 1016 specs.ipv6 = (struct ibv_flow_spec_ipv6){ 1017 .type = IBV_FLOW_SPEC_IPV6 | 1018 parser->inner, 1019 .size = size, 1020 }; 1021 } 1022 if (parser->queue[i].ibv_attr) { 1023 dst = (void *)((uintptr_t) 1024 parser->queue[i].ibv_attr + 1025 parser->queue[i].offset); 1026 memcpy(dst, &specs, size); 1027 ++parser->queue[i].ibv_attr->num_of_specs; 1028 } 1029 parser->queue[i].offset += size; 1030 } 1031 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) || 1032 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) { 1033 size = sizeof(struct ibv_flow_spec_tcp_udp); 1034 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) { 1035 .type = ((i == HASH_RXQ_UDPV4 || 1036 i == HASH_RXQ_UDPV6) ? 1037 IBV_FLOW_SPEC_UDP : 1038 IBV_FLOW_SPEC_TCP) | 1039 parser->inner, 1040 .size = size, 1041 }; 1042 if (parser->queue[i].ibv_attr) { 1043 dst = (void *)((uintptr_t) 1044 parser->queue[i].ibv_attr + 1045 parser->queue[i].offset); 1046 memcpy(dst, &specs, size); 1047 ++parser->queue[i].ibv_attr->num_of_specs; 1048 } 1049 parser->queue[i].offset += size; 1050 } 1051 } 1052 } 1053 1054 /** 1055 * Validate and convert a flow supported by the NIC. 1056 * 1057 * @param priv 1058 * Pointer to private structure. 1059 * @param[in] attr 1060 * Flow rule attributes. 1061 * @param[in] pattern 1062 * Pattern specification (list terminated by the END pattern item). 1063 * @param[in] actions 1064 * Associated actions (list terminated by the END action). 1065 * @param[out] error 1066 * Perform verbose error reporting if not NULL. 1067 * @param[in, out] parser 1068 * Internal parser structure. 1069 * 1070 * @return 1071 * 0 on success, a negative errno value otherwise and rte_errno is set. 1072 */ 1073 static int 1074 priv_flow_convert(struct priv *priv, 1075 const struct rte_flow_attr *attr, 1076 const struct rte_flow_item items[], 1077 const struct rte_flow_action actions[], 1078 struct rte_flow_error *error, 1079 struct mlx5_flow_parse *parser) 1080 { 1081 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 1082 unsigned int i; 1083 int ret; 1084 1085 /* First step. Validate the attributes, items and actions. */ 1086 *parser = (struct mlx5_flow_parse){ 1087 .create = parser->create, 1088 .layer = HASH_RXQ_ETH, 1089 .mark_id = MLX5_FLOW_MARK_DEFAULT, 1090 }; 1091 ret = priv_flow_convert_attributes(priv, attr, error, parser); 1092 if (ret) 1093 return ret; 1094 ret = priv_flow_convert_actions(priv, actions, error, parser); 1095 if (ret) 1096 return ret; 1097 ret = priv_flow_convert_items_validate(priv, items, error, parser); 1098 if (ret) 1099 return ret; 1100 priv_flow_convert_finalise(priv, parser); 1101 /* 1102 * Second step. 1103 * Allocate the memory space to store verbs specifications. 1104 */ 1105 if (parser->drop) { 1106 parser->drop_q.ibv_attr = 1107 priv_flow_convert_allocate(priv, attr->priority, 1108 parser->drop_q.offset, 1109 error); 1110 if (!parser->drop_q.ibv_attr) 1111 return ENOMEM; 1112 parser->drop_q.offset = sizeof(struct ibv_flow_attr); 1113 } else if (parser->queues_n == 1) { 1114 unsigned int priority = 1115 attr->priority + 1116 hash_rxq_init[HASH_RXQ_ETH].flow_priority; 1117 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; 1118 1119 parser->queue[HASH_RXQ_ETH].ibv_attr = 1120 priv_flow_convert_allocate(priv, priority, 1121 offset, error); 1122 if (!parser->queue[HASH_RXQ_ETH].ibv_attr) 1123 return ENOMEM; 1124 parser->queue[HASH_RXQ_ETH].offset = 1125 sizeof(struct ibv_flow_attr); 1126 } else { 1127 for (i = 0; i != hash_rxq_init_n; ++i) { 1128 unsigned int priority = 1129 attr->priority + 1130 hash_rxq_init[i].flow_priority; 1131 unsigned int offset; 1132 1133 if (!(parser->rss_conf.rss_hf & 1134 hash_rxq_init[i].dpdk_rss_hf) && 1135 (i != HASH_RXQ_ETH)) 1136 continue; 1137 offset = parser->queue[i].offset; 1138 parser->queue[i].ibv_attr = 1139 priv_flow_convert_allocate(priv, priority, 1140 offset, error); 1141 if (!parser->queue[i].ibv_attr) 1142 goto exit_enomem; 1143 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 1144 } 1145 } 1146 /* Third step. Conversion parse, fill the specifications. */ 1147 parser->inner = 0; 1148 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 1149 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 1150 continue; 1151 cur_item = &mlx5_flow_items[items->type]; 1152 ret = cur_item->convert(items, 1153 (cur_item->default_mask ? 1154 cur_item->default_mask : 1155 cur_item->mask), 1156 parser); 1157 if (ret) { 1158 rte_flow_error_set(error, ENOTSUP, 1159 RTE_FLOW_ERROR_TYPE_ITEM, 1160 items, "item not supported"); 1161 goto exit_free; 1162 } 1163 } 1164 if (parser->mark) 1165 mlx5_flow_create_flag_mark(parser, parser->mark_id); 1166 if (parser->count && parser->create) { 1167 mlx5_flow_create_count(priv, parser); 1168 if (!parser->cs) 1169 goto exit_count_error; 1170 } 1171 /* 1172 * Last step. Complete missing specification to reach the RSS 1173 * configuration. 1174 */ 1175 if (parser->queues_n > 1) 1176 priv_flow_convert_finalise(priv, parser); 1177 exit_free: 1178 /* Only verification is expected, all resources should be released. */ 1179 if (!parser->create) { 1180 if (parser->drop) { 1181 rte_free(parser->drop_q.ibv_attr); 1182 parser->drop_q.ibv_attr = NULL; 1183 } 1184 for (i = 0; i != hash_rxq_init_n; ++i) { 1185 if (parser->queue[i].ibv_attr) { 1186 rte_free(parser->queue[i].ibv_attr); 1187 parser->queue[i].ibv_attr = NULL; 1188 } 1189 } 1190 } 1191 return ret; 1192 exit_enomem: 1193 for (i = 0; i != hash_rxq_init_n; ++i) { 1194 if (parser->queue[i].ibv_attr) { 1195 rte_free(parser->queue[i].ibv_attr); 1196 parser->queue[i].ibv_attr = NULL; 1197 } 1198 } 1199 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1200 NULL, "cannot allocate verbs spec attributes."); 1201 return ret; 1202 exit_count_error: 1203 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1204 NULL, "cannot create counter."); 1205 return rte_errno; 1206 } 1207 1208 /** 1209 * Copy the specification created into the flow. 1210 * 1211 * @param parser 1212 * Internal parser structure. 1213 * @param src 1214 * Create specification. 1215 * @param size 1216 * Size in bytes of the specification to copy. 1217 */ 1218 static void 1219 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 1220 unsigned int size) 1221 { 1222 unsigned int i; 1223 void *dst; 1224 1225 if (parser->drop) { 1226 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1227 parser->drop_q.offset); 1228 memcpy(dst, src, size); 1229 ++parser->drop_q.ibv_attr->num_of_specs; 1230 parser->drop_q.offset += size; 1231 return; 1232 } 1233 for (i = 0; i != hash_rxq_init_n; ++i) { 1234 if (!parser->queue[i].ibv_attr) 1235 continue; 1236 /* Specification must be the same l3 type or none. */ 1237 if (parser->layer == HASH_RXQ_ETH || 1238 (hash_rxq_init[parser->layer].ip_version == 1239 hash_rxq_init[i].ip_version) || 1240 (hash_rxq_init[i].ip_version == 0)) { 1241 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1242 parser->queue[i].offset); 1243 memcpy(dst, src, size); 1244 ++parser->queue[i].ibv_attr->num_of_specs; 1245 parser->queue[i].offset += size; 1246 } 1247 } 1248 } 1249 1250 /** 1251 * Convert Ethernet item to Verbs specification. 1252 * 1253 * @param item[in] 1254 * Item specification. 1255 * @param default_mask[in] 1256 * Default bit-masks to use when item->mask is not provided. 1257 * @param data[in, out] 1258 * User structure. 1259 */ 1260 static int 1261 mlx5_flow_create_eth(const struct rte_flow_item *item, 1262 const void *default_mask, 1263 void *data) 1264 { 1265 const struct rte_flow_item_eth *spec = item->spec; 1266 const struct rte_flow_item_eth *mask = item->mask; 1267 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1268 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1269 struct ibv_flow_spec_eth eth = { 1270 .type = parser->inner | IBV_FLOW_SPEC_ETH, 1271 .size = eth_size, 1272 }; 1273 1274 parser->layer = HASH_RXQ_ETH; 1275 if (spec) { 1276 unsigned int i; 1277 1278 if (!mask) 1279 mask = default_mask; 1280 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); 1281 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); 1282 eth.val.ether_type = spec->type; 1283 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); 1284 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); 1285 eth.mask.ether_type = mask->type; 1286 /* Remove unwanted bits from values. */ 1287 for (i = 0; i < ETHER_ADDR_LEN; ++i) { 1288 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 1289 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 1290 } 1291 eth.val.ether_type &= eth.mask.ether_type; 1292 } 1293 mlx5_flow_create_copy(parser, ð, eth_size); 1294 return 0; 1295 } 1296 1297 /** 1298 * Convert VLAN item to Verbs specification. 1299 * 1300 * @param item[in] 1301 * Item specification. 1302 * @param default_mask[in] 1303 * Default bit-masks to use when item->mask is not provided. 1304 * @param data[in, out] 1305 * User structure. 1306 */ 1307 static int 1308 mlx5_flow_create_vlan(const struct rte_flow_item *item, 1309 const void *default_mask, 1310 void *data) 1311 { 1312 const struct rte_flow_item_vlan *spec = item->spec; 1313 const struct rte_flow_item_vlan *mask = item->mask; 1314 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1315 struct ibv_flow_spec_eth *eth; 1316 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1317 1318 if (spec) { 1319 unsigned int i; 1320 if (!mask) 1321 mask = default_mask; 1322 1323 if (parser->drop) { 1324 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1325 parser->drop_q.offset - eth_size); 1326 eth->val.vlan_tag = spec->tci; 1327 eth->mask.vlan_tag = mask->tci; 1328 eth->val.vlan_tag &= eth->mask.vlan_tag; 1329 return 0; 1330 } 1331 for (i = 0; i != hash_rxq_init_n; ++i) { 1332 if (!parser->queue[i].ibv_attr) 1333 continue; 1334 1335 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1336 parser->queue[i].offset - eth_size); 1337 eth->val.vlan_tag = spec->tci; 1338 eth->mask.vlan_tag = mask->tci; 1339 eth->val.vlan_tag &= eth->mask.vlan_tag; 1340 } 1341 } 1342 return 0; 1343 } 1344 1345 /** 1346 * Convert IPv4 item to Verbs specification. 1347 * 1348 * @param item[in] 1349 * Item specification. 1350 * @param default_mask[in] 1351 * Default bit-masks to use when item->mask is not provided. 1352 * @param data[in, out] 1353 * User structure. 1354 */ 1355 static int 1356 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 1357 const void *default_mask, 1358 void *data) 1359 { 1360 const struct rte_flow_item_ipv4 *spec = item->spec; 1361 const struct rte_flow_item_ipv4 *mask = item->mask; 1362 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1363 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); 1364 struct ibv_flow_spec_ipv4_ext ipv4 = { 1365 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, 1366 .size = ipv4_size, 1367 }; 1368 1369 parser->layer = HASH_RXQ_IPV4; 1370 if (spec) { 1371 if (!mask) 1372 mask = default_mask; 1373 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 1374 .src_ip = spec->hdr.src_addr, 1375 .dst_ip = spec->hdr.dst_addr, 1376 .proto = spec->hdr.next_proto_id, 1377 .tos = spec->hdr.type_of_service, 1378 }; 1379 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 1380 .src_ip = mask->hdr.src_addr, 1381 .dst_ip = mask->hdr.dst_addr, 1382 .proto = mask->hdr.next_proto_id, 1383 .tos = mask->hdr.type_of_service, 1384 }; 1385 /* Remove unwanted bits from values. */ 1386 ipv4.val.src_ip &= ipv4.mask.src_ip; 1387 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 1388 ipv4.val.proto &= ipv4.mask.proto; 1389 ipv4.val.tos &= ipv4.mask.tos; 1390 } 1391 mlx5_flow_create_copy(parser, &ipv4, ipv4_size); 1392 return 0; 1393 } 1394 1395 /** 1396 * Convert IPv6 item to Verbs specification. 1397 * 1398 * @param item[in] 1399 * Item specification. 1400 * @param default_mask[in] 1401 * Default bit-masks to use when item->mask is not provided. 1402 * @param data[in, out] 1403 * User structure. 1404 */ 1405 static int 1406 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 1407 const void *default_mask, 1408 void *data) 1409 { 1410 const struct rte_flow_item_ipv6 *spec = item->spec; 1411 const struct rte_flow_item_ipv6 *mask = item->mask; 1412 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1413 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); 1414 struct ibv_flow_spec_ipv6 ipv6 = { 1415 .type = parser->inner | IBV_FLOW_SPEC_IPV6, 1416 .size = ipv6_size, 1417 }; 1418 1419 parser->layer = HASH_RXQ_IPV6; 1420 if (spec) { 1421 unsigned int i; 1422 1423 if (!mask) 1424 mask = default_mask; 1425 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 1426 RTE_DIM(ipv6.val.src_ip)); 1427 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 1428 RTE_DIM(ipv6.val.dst_ip)); 1429 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 1430 RTE_DIM(ipv6.mask.src_ip)); 1431 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 1432 RTE_DIM(ipv6.mask.dst_ip)); 1433 ipv6.mask.flow_label = mask->hdr.vtc_flow; 1434 ipv6.mask.next_hdr = mask->hdr.proto; 1435 ipv6.mask.hop_limit = mask->hdr.hop_limits; 1436 /* Remove unwanted bits from values. */ 1437 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 1438 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 1439 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 1440 } 1441 ipv6.val.flow_label &= ipv6.mask.flow_label; 1442 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 1443 ipv6.val.hop_limit &= ipv6.mask.hop_limit; 1444 } 1445 mlx5_flow_create_copy(parser, &ipv6, ipv6_size); 1446 return 0; 1447 } 1448 1449 /** 1450 * Convert UDP item to Verbs specification. 1451 * 1452 * @param item[in] 1453 * Item specification. 1454 * @param default_mask[in] 1455 * Default bit-masks to use when item->mask is not provided. 1456 * @param data[in, out] 1457 * User structure. 1458 */ 1459 static int 1460 mlx5_flow_create_udp(const struct rte_flow_item *item, 1461 const void *default_mask, 1462 void *data) 1463 { 1464 const struct rte_flow_item_udp *spec = item->spec; 1465 const struct rte_flow_item_udp *mask = item->mask; 1466 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1467 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1468 struct ibv_flow_spec_tcp_udp udp = { 1469 .type = parser->inner | IBV_FLOW_SPEC_UDP, 1470 .size = udp_size, 1471 }; 1472 1473 if (parser->layer == HASH_RXQ_IPV4) 1474 parser->layer = HASH_RXQ_UDPV4; 1475 else 1476 parser->layer = HASH_RXQ_UDPV6; 1477 if (spec) { 1478 if (!mask) 1479 mask = default_mask; 1480 udp.val.dst_port = spec->hdr.dst_port; 1481 udp.val.src_port = spec->hdr.src_port; 1482 udp.mask.dst_port = mask->hdr.dst_port; 1483 udp.mask.src_port = mask->hdr.src_port; 1484 /* Remove unwanted bits from values. */ 1485 udp.val.src_port &= udp.mask.src_port; 1486 udp.val.dst_port &= udp.mask.dst_port; 1487 } 1488 mlx5_flow_create_copy(parser, &udp, udp_size); 1489 return 0; 1490 } 1491 1492 /** 1493 * Convert TCP item to Verbs specification. 1494 * 1495 * @param item[in] 1496 * Item specification. 1497 * @param default_mask[in] 1498 * Default bit-masks to use when item->mask is not provided. 1499 * @param data[in, out] 1500 * User structure. 1501 */ 1502 static int 1503 mlx5_flow_create_tcp(const struct rte_flow_item *item, 1504 const void *default_mask, 1505 void *data) 1506 { 1507 const struct rte_flow_item_tcp *spec = item->spec; 1508 const struct rte_flow_item_tcp *mask = item->mask; 1509 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1510 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1511 struct ibv_flow_spec_tcp_udp tcp = { 1512 .type = parser->inner | IBV_FLOW_SPEC_TCP, 1513 .size = tcp_size, 1514 }; 1515 1516 if (parser->layer == HASH_RXQ_IPV4) 1517 parser->layer = HASH_RXQ_TCPV4; 1518 else 1519 parser->layer = HASH_RXQ_TCPV6; 1520 if (spec) { 1521 if (!mask) 1522 mask = default_mask; 1523 tcp.val.dst_port = spec->hdr.dst_port; 1524 tcp.val.src_port = spec->hdr.src_port; 1525 tcp.mask.dst_port = mask->hdr.dst_port; 1526 tcp.mask.src_port = mask->hdr.src_port; 1527 /* Remove unwanted bits from values. */ 1528 tcp.val.src_port &= tcp.mask.src_port; 1529 tcp.val.dst_port &= tcp.mask.dst_port; 1530 } 1531 mlx5_flow_create_copy(parser, &tcp, tcp_size); 1532 return 0; 1533 } 1534 1535 /** 1536 * Convert VXLAN item to Verbs specification. 1537 * 1538 * @param item[in] 1539 * Item specification. 1540 * @param default_mask[in] 1541 * Default bit-masks to use when item->mask is not provided. 1542 * @param data[in, out] 1543 * User structure. 1544 */ 1545 static int 1546 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 1547 const void *default_mask, 1548 void *data) 1549 { 1550 const struct rte_flow_item_vxlan *spec = item->spec; 1551 const struct rte_flow_item_vxlan *mask = item->mask; 1552 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1553 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 1554 struct ibv_flow_spec_tunnel vxlan = { 1555 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, 1556 .size = size, 1557 }; 1558 union vni { 1559 uint32_t vlan_id; 1560 uint8_t vni[4]; 1561 } id; 1562 1563 id.vni[0] = 0; 1564 parser->inner = IBV_FLOW_SPEC_INNER; 1565 if (spec) { 1566 if (!mask) 1567 mask = default_mask; 1568 memcpy(&id.vni[1], spec->vni, 3); 1569 vxlan.val.tunnel_id = id.vlan_id; 1570 memcpy(&id.vni[1], mask->vni, 3); 1571 vxlan.mask.tunnel_id = id.vlan_id; 1572 /* Remove unwanted bits from values. */ 1573 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 1574 } 1575 mlx5_flow_create_copy(parser, &vxlan, size); 1576 return 0; 1577 } 1578 1579 /** 1580 * Convert mark/flag action to Verbs specification. 1581 * 1582 * @param parser 1583 * Internal parser structure. 1584 * @param mark_id 1585 * Mark identifier. 1586 */ 1587 static int 1588 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) 1589 { 1590 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1591 struct ibv_flow_spec_action_tag tag = { 1592 .type = IBV_FLOW_SPEC_ACTION_TAG, 1593 .size = size, 1594 .tag_id = mlx5_flow_mark_set(mark_id), 1595 }; 1596 1597 assert(parser->mark); 1598 mlx5_flow_create_copy(parser, &tag, size); 1599 return 0; 1600 } 1601 1602 /** 1603 * Convert count action to Verbs specification. 1604 * 1605 * @param priv 1606 * Pointer to private structure. 1607 * @param parser 1608 * Pointer to MLX5 flow parser structure. 1609 * 1610 * @return 1611 * 0 on success, errno value on failure. 1612 */ 1613 static int 1614 mlx5_flow_create_count(struct priv *priv __rte_unused, 1615 struct mlx5_flow_parse *parser __rte_unused) 1616 { 1617 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 1618 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1619 struct ibv_counter_set_init_attr init_attr = {0}; 1620 struct ibv_flow_spec_counter_action counter = { 1621 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1622 .size = size, 1623 .counter_set_handle = 0, 1624 }; 1625 1626 init_attr.counter_set_id = 0; 1627 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr); 1628 if (!parser->cs) 1629 return EINVAL; 1630 counter.counter_set_handle = parser->cs->handle; 1631 mlx5_flow_create_copy(parser, &counter, size); 1632 #endif 1633 return 0; 1634 } 1635 1636 /** 1637 * Complete flow rule creation with a drop queue. 1638 * 1639 * @param priv 1640 * Pointer to private structure. 1641 * @param parser 1642 * Internal parser structure. 1643 * @param flow 1644 * Pointer to the rte_flow. 1645 * @param[out] error 1646 * Perform verbose error reporting if not NULL. 1647 * 1648 * @return 1649 * 0 on success, errno value on failure. 1650 */ 1651 static int 1652 priv_flow_create_action_queue_drop(struct priv *priv, 1653 struct mlx5_flow_parse *parser, 1654 struct rte_flow *flow, 1655 struct rte_flow_error *error) 1656 { 1657 struct ibv_flow_spec_action_drop *drop; 1658 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1659 int err = 0; 1660 1661 assert(priv->pd); 1662 assert(priv->ctx); 1663 flow->drop = 1; 1664 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1665 parser->drop_q.offset); 1666 *drop = (struct ibv_flow_spec_action_drop){ 1667 .type = IBV_FLOW_SPEC_ACTION_DROP, 1668 .size = size, 1669 }; 1670 ++parser->drop_q.ibv_attr->num_of_specs; 1671 parser->drop_q.offset += size; 1672 if (!priv->dev->data->dev_started) 1673 return 0; 1674 flow->drxq.ibv_attr = parser->drop_q.ibv_attr; 1675 parser->drop_q.ibv_attr = NULL; 1676 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp, 1677 flow->drxq.ibv_attr); 1678 if (parser->count) 1679 flow->cs = parser->cs; 1680 if (!flow->drxq.ibv_flow) { 1681 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1682 NULL, "flow rule creation failure"); 1683 err = ENOMEM; 1684 goto error; 1685 } 1686 return 0; 1687 error: 1688 assert(flow); 1689 if (flow->drxq.ibv_flow) { 1690 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 1691 flow->drxq.ibv_flow = NULL; 1692 } 1693 if (flow->drxq.ibv_attr) { 1694 rte_free(flow->drxq.ibv_attr); 1695 flow->drxq.ibv_attr = NULL; 1696 } 1697 if (flow->cs) { 1698 claim_zero(ibv_destroy_counter_set(flow->cs)); 1699 flow->cs = NULL; 1700 parser->cs = NULL; 1701 } 1702 return err; 1703 } 1704 1705 /** 1706 * Create hash Rx queues when RSS is enabled. 1707 * 1708 * @param priv 1709 * Pointer to private structure. 1710 * @param parser 1711 * Internal parser structure. 1712 * @param flow 1713 * Pointer to the rte_flow. 1714 * @param[out] error 1715 * Perform verbose error reporting if not NULL. 1716 * 1717 * @return 1718 * 0 on success, a errno value otherwise and rte_errno is set. 1719 */ 1720 static int 1721 priv_flow_create_action_queue_rss(struct priv *priv, 1722 struct mlx5_flow_parse *parser, 1723 struct rte_flow *flow, 1724 struct rte_flow_error *error) 1725 { 1726 unsigned int i; 1727 1728 for (i = 0; i != hash_rxq_init_n; ++i) { 1729 uint64_t hash_fields; 1730 1731 if (!parser->queue[i].ibv_attr) 1732 continue; 1733 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; 1734 parser->queue[i].ibv_attr = NULL; 1735 hash_fields = hash_rxq_init[i].hash_fields; 1736 flow->frxq[i].hrxq = 1737 mlx5_priv_hrxq_get(priv, 1738 parser->rss_conf.rss_key, 1739 parser->rss_conf.rss_key_len, 1740 hash_fields, 1741 parser->queues, 1742 hash_fields ? parser->queues_n : 1); 1743 if (flow->frxq[i].hrxq) 1744 continue; 1745 flow->frxq[i].hrxq = 1746 mlx5_priv_hrxq_new(priv, 1747 parser->rss_conf.rss_key, 1748 parser->rss_conf.rss_key_len, 1749 hash_fields, 1750 parser->queues, 1751 hash_fields ? parser->queues_n : 1); 1752 if (!flow->frxq[i].hrxq) { 1753 rte_flow_error_set(error, ENOMEM, 1754 RTE_FLOW_ERROR_TYPE_HANDLE, 1755 NULL, "cannot create hash rxq"); 1756 return ENOMEM; 1757 } 1758 } 1759 return 0; 1760 } 1761 1762 /** 1763 * Complete flow rule creation. 1764 * 1765 * @param priv 1766 * Pointer to private structure. 1767 * @param parser 1768 * Internal parser structure. 1769 * @param flow 1770 * Pointer to the rte_flow. 1771 * @param[out] error 1772 * Perform verbose error reporting if not NULL. 1773 * 1774 * @return 1775 * 0 on success, a errno value otherwise and rte_errno is set. 1776 */ 1777 static int 1778 priv_flow_create_action_queue(struct priv *priv, 1779 struct mlx5_flow_parse *parser, 1780 struct rte_flow *flow, 1781 struct rte_flow_error *error) 1782 { 1783 int err = 0; 1784 unsigned int i; 1785 1786 assert(priv->pd); 1787 assert(priv->ctx); 1788 assert(!parser->drop); 1789 err = priv_flow_create_action_queue_rss(priv, parser, flow, error); 1790 if (err) 1791 goto error; 1792 if (parser->count) 1793 flow->cs = parser->cs; 1794 if (!priv->dev->data->dev_started) 1795 return 0; 1796 for (i = 0; i != hash_rxq_init_n; ++i) { 1797 if (!flow->frxq[i].hrxq) 1798 continue; 1799 flow->frxq[i].ibv_flow = 1800 ibv_create_flow(flow->frxq[i].hrxq->qp, 1801 flow->frxq[i].ibv_attr); 1802 if (!flow->frxq[i].ibv_flow) { 1803 rte_flow_error_set(error, ENOMEM, 1804 RTE_FLOW_ERROR_TYPE_HANDLE, 1805 NULL, "flow rule creation failure"); 1806 err = ENOMEM; 1807 goto error; 1808 } 1809 DEBUG("%p type %d QP %p ibv_flow %p", 1810 (void *)flow, i, 1811 (void *)flow->frxq[i].hrxq, 1812 (void *)flow->frxq[i].ibv_flow); 1813 } 1814 for (i = 0; i != parser->queues_n; ++i) { 1815 struct mlx5_rxq_data *q = 1816 (*priv->rxqs)[parser->queues[i]]; 1817 1818 q->mark |= parser->mark; 1819 } 1820 return 0; 1821 error: 1822 assert(flow); 1823 for (i = 0; i != hash_rxq_init_n; ++i) { 1824 if (flow->frxq[i].ibv_flow) { 1825 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; 1826 1827 claim_zero(ibv_destroy_flow(ibv_flow)); 1828 } 1829 if (flow->frxq[i].hrxq) 1830 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); 1831 if (flow->frxq[i].ibv_attr) 1832 rte_free(flow->frxq[i].ibv_attr); 1833 } 1834 if (flow->cs) { 1835 claim_zero(ibv_destroy_counter_set(flow->cs)); 1836 flow->cs = NULL; 1837 parser->cs = NULL; 1838 } 1839 return err; 1840 } 1841 1842 /** 1843 * Convert a flow. 1844 * 1845 * @param priv 1846 * Pointer to private structure. 1847 * @param list 1848 * Pointer to a TAILQ flow list. 1849 * @param[in] attr 1850 * Flow rule attributes. 1851 * @param[in] pattern 1852 * Pattern specification (list terminated by the END pattern item). 1853 * @param[in] actions 1854 * Associated actions (list terminated by the END action). 1855 * @param[out] error 1856 * Perform verbose error reporting if not NULL. 1857 * 1858 * @return 1859 * A flow on success, NULL otherwise. 1860 */ 1861 static struct rte_flow * 1862 priv_flow_create(struct priv *priv, 1863 struct mlx5_flows *list, 1864 const struct rte_flow_attr *attr, 1865 const struct rte_flow_item items[], 1866 const struct rte_flow_action actions[], 1867 struct rte_flow_error *error) 1868 { 1869 struct mlx5_flow_parse parser = { .create = 1, }; 1870 struct rte_flow *flow = NULL; 1871 unsigned int i; 1872 int err; 1873 1874 err = priv_flow_convert(priv, attr, items, actions, error, &parser); 1875 if (err) 1876 goto exit; 1877 flow = rte_calloc(__func__, 1, 1878 sizeof(*flow) + parser.queues_n * sizeof(uint16_t), 1879 0); 1880 if (!flow) { 1881 rte_flow_error_set(error, ENOMEM, 1882 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1883 NULL, 1884 "cannot allocate flow memory"); 1885 return NULL; 1886 } 1887 /* Copy queues configuration. */ 1888 flow->queues = (uint16_t (*)[])(flow + 1); 1889 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t)); 1890 flow->queues_n = parser.queues_n; 1891 /* Copy RSS configuration. */ 1892 flow->rss_conf = parser.rss_conf; 1893 flow->rss_conf.rss_key = flow->rss_key; 1894 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len); 1895 /* finalise the flow. */ 1896 if (parser.drop) 1897 err = priv_flow_create_action_queue_drop(priv, &parser, flow, 1898 error); 1899 else 1900 err = priv_flow_create_action_queue(priv, &parser, flow, error); 1901 if (err) 1902 goto exit; 1903 TAILQ_INSERT_TAIL(list, flow, next); 1904 DEBUG("Flow created %p", (void *)flow); 1905 return flow; 1906 exit: 1907 if (parser.drop) { 1908 rte_free(parser.drop_q.ibv_attr); 1909 } else { 1910 for (i = 0; i != hash_rxq_init_n; ++i) { 1911 if (parser.queue[i].ibv_attr) 1912 rte_free(parser.queue[i].ibv_attr); 1913 } 1914 } 1915 rte_free(flow); 1916 return NULL; 1917 } 1918 1919 /** 1920 * Validate a flow supported by the NIC. 1921 * 1922 * @see rte_flow_validate() 1923 * @see rte_flow_ops 1924 */ 1925 int 1926 mlx5_flow_validate(struct rte_eth_dev *dev, 1927 const struct rte_flow_attr *attr, 1928 const struct rte_flow_item items[], 1929 const struct rte_flow_action actions[], 1930 struct rte_flow_error *error) 1931 { 1932 struct priv *priv = dev->data->dev_private; 1933 int ret; 1934 struct mlx5_flow_parse parser = { .create = 0, }; 1935 1936 priv_lock(priv); 1937 ret = priv_flow_convert(priv, attr, items, actions, error, &parser); 1938 priv_unlock(priv); 1939 return ret; 1940 } 1941 1942 /** 1943 * Create a flow. 1944 * 1945 * @see rte_flow_create() 1946 * @see rte_flow_ops 1947 */ 1948 struct rte_flow * 1949 mlx5_flow_create(struct rte_eth_dev *dev, 1950 const struct rte_flow_attr *attr, 1951 const struct rte_flow_item items[], 1952 const struct rte_flow_action actions[], 1953 struct rte_flow_error *error) 1954 { 1955 struct priv *priv = dev->data->dev_private; 1956 struct rte_flow *flow; 1957 1958 priv_lock(priv); 1959 flow = priv_flow_create(priv, &priv->flows, attr, items, actions, 1960 error); 1961 priv_unlock(priv); 1962 return flow; 1963 } 1964 1965 /** 1966 * Destroy a flow. 1967 * 1968 * @param priv 1969 * Pointer to private structure. 1970 * @param list 1971 * Pointer to a TAILQ flow list. 1972 * @param[in] flow 1973 * Flow to destroy. 1974 */ 1975 static void 1976 priv_flow_destroy(struct priv *priv, 1977 struct mlx5_flows *list, 1978 struct rte_flow *flow) 1979 { 1980 unsigned int i; 1981 1982 if (flow->cs) { 1983 claim_zero(ibv_destroy_counter_set(flow->cs)); 1984 flow->cs = NULL; 1985 } 1986 if (flow->drop || !flow->mark) 1987 goto free; 1988 for (i = 0; i != flow->queues_n; ++i) { 1989 struct rte_flow *tmp; 1990 int mark = 0; 1991 1992 /* 1993 * To remove the mark from the queue, the queue must not be 1994 * present in any other marked flow (RSS or not). 1995 */ 1996 TAILQ_FOREACH(tmp, list, next) { 1997 unsigned int j; 1998 uint16_t *tqs = NULL; 1999 uint16_t tq_n = 0; 2000 2001 if (!tmp->mark) 2002 continue; 2003 for (j = 0; j != hash_rxq_init_n; ++j) { 2004 if (!tmp->frxq[j].hrxq) 2005 continue; 2006 tqs = tmp->frxq[j].hrxq->ind_table->queues; 2007 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n; 2008 } 2009 if (!tq_n) 2010 continue; 2011 for (j = 0; (j != tq_n) && !mark; j++) 2012 if (tqs[j] == (*flow->queues)[i]) 2013 mark = 1; 2014 } 2015 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark; 2016 } 2017 free: 2018 if (flow->drop) { 2019 if (flow->drxq.ibv_flow) 2020 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 2021 rte_free(flow->drxq.ibv_attr); 2022 } else { 2023 for (i = 0; i != hash_rxq_init_n; ++i) { 2024 struct mlx5_flow *frxq = &flow->frxq[i]; 2025 2026 if (frxq->ibv_flow) 2027 claim_zero(ibv_destroy_flow(frxq->ibv_flow)); 2028 if (frxq->hrxq) 2029 mlx5_priv_hrxq_release(priv, frxq->hrxq); 2030 if (frxq->ibv_attr) 2031 rte_free(frxq->ibv_attr); 2032 } 2033 } 2034 TAILQ_REMOVE(list, flow, next); 2035 DEBUG("Flow destroyed %p", (void *)flow); 2036 rte_free(flow); 2037 } 2038 2039 /** 2040 * Destroy all flows. 2041 * 2042 * @param priv 2043 * Pointer to private structure. 2044 * @param list 2045 * Pointer to a TAILQ flow list. 2046 */ 2047 void 2048 priv_flow_flush(struct priv *priv, struct mlx5_flows *list) 2049 { 2050 while (!TAILQ_EMPTY(list)) { 2051 struct rte_flow *flow; 2052 2053 flow = TAILQ_FIRST(list); 2054 priv_flow_destroy(priv, list, flow); 2055 } 2056 } 2057 2058 /** 2059 * Create drop queue. 2060 * 2061 * @param priv 2062 * Pointer to private structure. 2063 * 2064 * @return 2065 * 0 on success. 2066 */ 2067 int 2068 priv_flow_create_drop_queue(struct priv *priv) 2069 { 2070 struct mlx5_hrxq_drop *fdq = NULL; 2071 2072 assert(priv->pd); 2073 assert(priv->ctx); 2074 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); 2075 if (!fdq) { 2076 WARN("cannot allocate memory for drop queue"); 2077 goto error; 2078 } 2079 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); 2080 if (!fdq->cq) { 2081 WARN("cannot allocate CQ for drop queue"); 2082 goto error; 2083 } 2084 fdq->wq = ibv_create_wq(priv->ctx, 2085 &(struct ibv_wq_init_attr){ 2086 .wq_type = IBV_WQT_RQ, 2087 .max_wr = 1, 2088 .max_sge = 1, 2089 .pd = priv->pd, 2090 .cq = fdq->cq, 2091 }); 2092 if (!fdq->wq) { 2093 WARN("cannot allocate WQ for drop queue"); 2094 goto error; 2095 } 2096 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx, 2097 &(struct ibv_rwq_ind_table_init_attr){ 2098 .log_ind_tbl_size = 0, 2099 .ind_tbl = &fdq->wq, 2100 .comp_mask = 0, 2101 }); 2102 if (!fdq->ind_table) { 2103 WARN("cannot allocate indirection table for drop queue"); 2104 goto error; 2105 } 2106 fdq->qp = ibv_create_qp_ex(priv->ctx, 2107 &(struct ibv_qp_init_attr_ex){ 2108 .qp_type = IBV_QPT_RAW_PACKET, 2109 .comp_mask = 2110 IBV_QP_INIT_ATTR_PD | 2111 IBV_QP_INIT_ATTR_IND_TABLE | 2112 IBV_QP_INIT_ATTR_RX_HASH, 2113 .rx_hash_conf = (struct ibv_rx_hash_conf){ 2114 .rx_hash_function = 2115 IBV_RX_HASH_FUNC_TOEPLITZ, 2116 .rx_hash_key_len = rss_hash_default_key_len, 2117 .rx_hash_key = rss_hash_default_key, 2118 .rx_hash_fields_mask = 0, 2119 }, 2120 .rwq_ind_tbl = fdq->ind_table, 2121 .pd = priv->pd 2122 }); 2123 if (!fdq->qp) { 2124 WARN("cannot allocate QP for drop queue"); 2125 goto error; 2126 } 2127 priv->flow_drop_queue = fdq; 2128 return 0; 2129 error: 2130 if (fdq->qp) 2131 claim_zero(ibv_destroy_qp(fdq->qp)); 2132 if (fdq->ind_table) 2133 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 2134 if (fdq->wq) 2135 claim_zero(ibv_destroy_wq(fdq->wq)); 2136 if (fdq->cq) 2137 claim_zero(ibv_destroy_cq(fdq->cq)); 2138 if (fdq) 2139 rte_free(fdq); 2140 priv->flow_drop_queue = NULL; 2141 return -1; 2142 } 2143 2144 /** 2145 * Delete drop queue. 2146 * 2147 * @param priv 2148 * Pointer to private structure. 2149 */ 2150 void 2151 priv_flow_delete_drop_queue(struct priv *priv) 2152 { 2153 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; 2154 2155 if (!fdq) 2156 return; 2157 if (fdq->qp) 2158 claim_zero(ibv_destroy_qp(fdq->qp)); 2159 if (fdq->ind_table) 2160 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 2161 if (fdq->wq) 2162 claim_zero(ibv_destroy_wq(fdq->wq)); 2163 if (fdq->cq) 2164 claim_zero(ibv_destroy_cq(fdq->cq)); 2165 rte_free(fdq); 2166 priv->flow_drop_queue = NULL; 2167 } 2168 2169 /** 2170 * Remove all flows. 2171 * 2172 * @param priv 2173 * Pointer to private structure. 2174 * @param list 2175 * Pointer to a TAILQ flow list. 2176 */ 2177 void 2178 priv_flow_stop(struct priv *priv, struct mlx5_flows *list) 2179 { 2180 struct rte_flow *flow; 2181 2182 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 2183 unsigned int i; 2184 2185 if (flow->drop) { 2186 if (!flow->drxq.ibv_flow) 2187 continue; 2188 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 2189 flow->drxq.ibv_flow = NULL; 2190 /* Next flow. */ 2191 continue; 2192 } 2193 if (flow->mark) { 2194 struct mlx5_ind_table_ibv *ind_tbl = NULL; 2195 2196 for (i = 0; i != hash_rxq_init_n; ++i) { 2197 if (!flow->frxq[i].hrxq) 2198 continue; 2199 ind_tbl = flow->frxq[i].hrxq->ind_table; 2200 } 2201 assert(ind_tbl); 2202 for (i = 0; i != ind_tbl->queues_n; ++i) 2203 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; 2204 } 2205 for (i = 0; i != hash_rxq_init_n; ++i) { 2206 if (!flow->frxq[i].ibv_flow) 2207 continue; 2208 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow)); 2209 flow->frxq[i].ibv_flow = NULL; 2210 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); 2211 flow->frxq[i].hrxq = NULL; 2212 } 2213 DEBUG("Flow %p removed", (void *)flow); 2214 } 2215 } 2216 2217 /** 2218 * Add all flows. 2219 * 2220 * @param priv 2221 * Pointer to private structure. 2222 * @param list 2223 * Pointer to a TAILQ flow list. 2224 * 2225 * @return 2226 * 0 on success, a errno value otherwise and rte_errno is set. 2227 */ 2228 int 2229 priv_flow_start(struct priv *priv, struct mlx5_flows *list) 2230 { 2231 struct rte_flow *flow; 2232 2233 TAILQ_FOREACH(flow, list, next) { 2234 unsigned int i; 2235 2236 if (flow->drop) { 2237 flow->drxq.ibv_flow = 2238 ibv_create_flow(priv->flow_drop_queue->qp, 2239 flow->drxq.ibv_attr); 2240 if (!flow->drxq.ibv_flow) { 2241 DEBUG("Flow %p cannot be applied", 2242 (void *)flow); 2243 rte_errno = EINVAL; 2244 return rte_errno; 2245 } 2246 DEBUG("Flow %p applied", (void *)flow); 2247 /* Next flow. */ 2248 continue; 2249 } 2250 for (i = 0; i != hash_rxq_init_n; ++i) { 2251 if (!flow->frxq[i].ibv_attr) 2252 continue; 2253 flow->frxq[i].hrxq = 2254 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key, 2255 flow->rss_conf.rss_key_len, 2256 hash_rxq_init[i].hash_fields, 2257 (*flow->queues), 2258 flow->queues_n); 2259 if (flow->frxq[i].hrxq) 2260 goto flow_create; 2261 flow->frxq[i].hrxq = 2262 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key, 2263 flow->rss_conf.rss_key_len, 2264 hash_rxq_init[i].hash_fields, 2265 (*flow->queues), 2266 flow->queues_n); 2267 if (!flow->frxq[i].hrxq) { 2268 DEBUG("Flow %p cannot be applied", 2269 (void *)flow); 2270 rte_errno = EINVAL; 2271 return rte_errno; 2272 } 2273 flow_create: 2274 flow->frxq[i].ibv_flow = 2275 ibv_create_flow(flow->frxq[i].hrxq->qp, 2276 flow->frxq[i].ibv_attr); 2277 if (!flow->frxq[i].ibv_flow) { 2278 DEBUG("Flow %p cannot be applied", 2279 (void *)flow); 2280 rte_errno = EINVAL; 2281 return rte_errno; 2282 } 2283 DEBUG("Flow %p applied", (void *)flow); 2284 } 2285 if (!flow->mark) 2286 continue; 2287 for (i = 0; i != flow->queues_n; ++i) 2288 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1; 2289 } 2290 return 0; 2291 } 2292 2293 /** 2294 * Verify the flow list is empty 2295 * 2296 * @param priv 2297 * Pointer to private structure. 2298 * 2299 * @return the number of flows not released. 2300 */ 2301 int 2302 priv_flow_verify(struct priv *priv) 2303 { 2304 struct rte_flow *flow; 2305 int ret = 0; 2306 2307 TAILQ_FOREACH(flow, &priv->flows, next) { 2308 DEBUG("%p: flow %p still referenced", (void *)priv, 2309 (void *)flow); 2310 ++ret; 2311 } 2312 return ret; 2313 } 2314 2315 /** 2316 * Enable a control flow configured from the control plane. 2317 * 2318 * @param dev 2319 * Pointer to Ethernet device. 2320 * @param eth_spec 2321 * An Ethernet flow spec to apply. 2322 * @param eth_mask 2323 * An Ethernet flow mask to apply. 2324 * @param vlan_spec 2325 * A VLAN flow spec to apply. 2326 * @param vlan_mask 2327 * A VLAN flow mask to apply. 2328 * 2329 * @return 2330 * 0 on success. 2331 */ 2332 int 2333 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2334 struct rte_flow_item_eth *eth_spec, 2335 struct rte_flow_item_eth *eth_mask, 2336 struct rte_flow_item_vlan *vlan_spec, 2337 struct rte_flow_item_vlan *vlan_mask) 2338 { 2339 struct priv *priv = dev->data->dev_private; 2340 const struct rte_flow_attr attr = { 2341 .ingress = 1, 2342 .priority = MLX5_CTRL_FLOW_PRIORITY, 2343 }; 2344 struct rte_flow_item items[] = { 2345 { 2346 .type = RTE_FLOW_ITEM_TYPE_ETH, 2347 .spec = eth_spec, 2348 .last = NULL, 2349 .mask = eth_mask, 2350 }, 2351 { 2352 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2353 RTE_FLOW_ITEM_TYPE_END, 2354 .spec = vlan_spec, 2355 .last = NULL, 2356 .mask = vlan_mask, 2357 }, 2358 { 2359 .type = RTE_FLOW_ITEM_TYPE_END, 2360 }, 2361 }; 2362 struct rte_flow_action actions[] = { 2363 { 2364 .type = RTE_FLOW_ACTION_TYPE_RSS, 2365 }, 2366 { 2367 .type = RTE_FLOW_ACTION_TYPE_END, 2368 }, 2369 }; 2370 struct rte_flow *flow; 2371 struct rte_flow_error error; 2372 unsigned int i; 2373 union { 2374 struct rte_flow_action_rss rss; 2375 struct { 2376 const struct rte_eth_rss_conf *rss_conf; 2377 uint16_t num; 2378 uint16_t queue[RTE_MAX_QUEUES_PER_PORT]; 2379 } local; 2380 } action_rss; 2381 2382 if (!priv->reta_idx_n) 2383 return EINVAL; 2384 for (i = 0; i != priv->reta_idx_n; ++i) 2385 action_rss.local.queue[i] = (*priv->reta_idx)[i]; 2386 action_rss.local.rss_conf = &priv->rss_conf; 2387 action_rss.local.num = priv->reta_idx_n; 2388 actions[0].conf = (const void *)&action_rss.rss; 2389 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions, 2390 &error); 2391 if (!flow) 2392 return rte_errno; 2393 return 0; 2394 } 2395 2396 /** 2397 * Enable a flow control configured from the control plane. 2398 * 2399 * @param dev 2400 * Pointer to Ethernet device. 2401 * @param eth_spec 2402 * An Ethernet flow spec to apply. 2403 * @param eth_mask 2404 * An Ethernet flow mask to apply. 2405 * 2406 * @return 2407 * 0 on success. 2408 */ 2409 int 2410 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2411 struct rte_flow_item_eth *eth_spec, 2412 struct rte_flow_item_eth *eth_mask) 2413 { 2414 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2415 } 2416 2417 /** 2418 * Destroy a flow. 2419 * 2420 * @see rte_flow_destroy() 2421 * @see rte_flow_ops 2422 */ 2423 int 2424 mlx5_flow_destroy(struct rte_eth_dev *dev, 2425 struct rte_flow *flow, 2426 struct rte_flow_error *error) 2427 { 2428 struct priv *priv = dev->data->dev_private; 2429 2430 (void)error; 2431 priv_lock(priv); 2432 priv_flow_destroy(priv, &priv->flows, flow); 2433 priv_unlock(priv); 2434 return 0; 2435 } 2436 2437 /** 2438 * Destroy all flows. 2439 * 2440 * @see rte_flow_flush() 2441 * @see rte_flow_ops 2442 */ 2443 int 2444 mlx5_flow_flush(struct rte_eth_dev *dev, 2445 struct rte_flow_error *error) 2446 { 2447 struct priv *priv = dev->data->dev_private; 2448 2449 (void)error; 2450 priv_lock(priv); 2451 priv_flow_flush(priv, &priv->flows); 2452 priv_unlock(priv); 2453 return 0; 2454 } 2455 2456 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 2457 /** 2458 * Query flow counter. 2459 * 2460 * @param cs 2461 * the counter set. 2462 * @param counter_value 2463 * returned data from the counter. 2464 * 2465 * @return 2466 * 0 on success, a errno value otherwise and rte_errno is set. 2467 */ 2468 static int 2469 priv_flow_query_count(struct ibv_counter_set *cs, 2470 struct mlx5_flow_counter_stats *counter_stats, 2471 struct rte_flow_query_count *query_count, 2472 struct rte_flow_error *error) 2473 { 2474 uint64_t counters[2]; 2475 struct ibv_query_counter_set_attr query_cs_attr = { 2476 .cs = cs, 2477 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 2478 }; 2479 struct ibv_counter_set_data query_out = { 2480 .out = counters, 2481 .outlen = 2 * sizeof(uint64_t), 2482 }; 2483 int res = ibv_query_counter_set(&query_cs_attr, &query_out); 2484 2485 if (res) { 2486 rte_flow_error_set(error, -res, 2487 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2488 NULL, 2489 "cannot read counter"); 2490 return -res; 2491 } 2492 query_count->hits_set = 1; 2493 query_count->bytes_set = 1; 2494 query_count->hits = counters[0] - counter_stats->hits; 2495 query_count->bytes = counters[1] - counter_stats->bytes; 2496 if (query_count->reset) { 2497 counter_stats->hits = counters[0]; 2498 counter_stats->bytes = counters[1]; 2499 } 2500 return 0; 2501 } 2502 2503 /** 2504 * Query a flows. 2505 * 2506 * @see rte_flow_query() 2507 * @see rte_flow_ops 2508 */ 2509 int 2510 mlx5_flow_query(struct rte_eth_dev *dev, 2511 struct rte_flow *flow, 2512 enum rte_flow_action_type action __rte_unused, 2513 void *data, 2514 struct rte_flow_error *error) 2515 { 2516 struct priv *priv = dev->data->dev_private; 2517 int res = EINVAL; 2518 2519 priv_lock(priv); 2520 if (flow->cs) { 2521 res = priv_flow_query_count(flow->cs, 2522 &flow->counter_stats, 2523 (struct rte_flow_query_count *)data, 2524 error); 2525 } else { 2526 rte_flow_error_set(error, res, 2527 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2528 NULL, 2529 "no counter found for flow"); 2530 } 2531 priv_unlock(priv); 2532 return -res; 2533 } 2534 #endif 2535 2536 /** 2537 * Isolated mode. 2538 * 2539 * @see rte_flow_isolate() 2540 * @see rte_flow_ops 2541 */ 2542 int 2543 mlx5_flow_isolate(struct rte_eth_dev *dev, 2544 int enable, 2545 struct rte_flow_error *error) 2546 { 2547 struct priv *priv = dev->data->dev_private; 2548 2549 priv_lock(priv); 2550 if (dev->data->dev_started) { 2551 rte_flow_error_set(error, EBUSY, 2552 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2553 NULL, 2554 "port must be stopped first"); 2555 priv_unlock(priv); 2556 return -rte_errno; 2557 } 2558 priv->isolated = !!enable; 2559 if (enable) 2560 priv->dev->dev_ops = &mlx5_dev_ops_isolate; 2561 else 2562 priv->dev->dev_ops = &mlx5_dev_ops; 2563 priv_unlock(priv); 2564 return 0; 2565 } 2566 2567 /** 2568 * Convert a flow director filter to a generic flow. 2569 * 2570 * @param priv 2571 * Private structure. 2572 * @param fdir_filter 2573 * Flow director filter to add. 2574 * @param attributes 2575 * Generic flow parameters structure. 2576 * 2577 * @return 2578 * 0 on success, errno value on error. 2579 */ 2580 static int 2581 priv_fdir_filter_convert(struct priv *priv, 2582 const struct rte_eth_fdir_filter *fdir_filter, 2583 struct mlx5_fdir *attributes) 2584 { 2585 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2586 2587 /* Validate queue number. */ 2588 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2589 ERROR("invalid queue number %d", fdir_filter->action.rx_queue); 2590 return EINVAL; 2591 } 2592 /* Validate the behavior. */ 2593 if (fdir_filter->action.behavior != RTE_ETH_FDIR_ACCEPT) { 2594 ERROR("invalid behavior %d", fdir_filter->action.behavior); 2595 return ENOTSUP; 2596 } 2597 attributes->attr.ingress = 1; 2598 attributes->items[0] = (struct rte_flow_item) { 2599 .type = RTE_FLOW_ITEM_TYPE_ETH, 2600 .spec = &attributes->l2, 2601 }; 2602 attributes->actions[0] = (struct rte_flow_action){ 2603 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2604 .conf = &attributes->queue, 2605 }; 2606 attributes->queue.index = fdir_filter->action.rx_queue; 2607 switch (fdir_filter->input.flow_type) { 2608 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2609 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2610 .src_addr = input->flow.udp4_flow.ip.src_ip, 2611 .dst_addr = input->flow.udp4_flow.ip.dst_ip, 2612 .time_to_live = input->flow.udp4_flow.ip.ttl, 2613 .type_of_service = input->flow.udp4_flow.ip.tos, 2614 .next_proto_id = input->flow.udp4_flow.ip.proto, 2615 }; 2616 attributes->l4.udp.hdr = (struct udp_hdr){ 2617 .src_port = input->flow.udp4_flow.src_port, 2618 .dst_port = input->flow.udp4_flow.dst_port, 2619 }; 2620 attributes->items[1] = (struct rte_flow_item){ 2621 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2622 .spec = &attributes->l3, 2623 }; 2624 attributes->items[2] = (struct rte_flow_item){ 2625 .type = RTE_FLOW_ITEM_TYPE_UDP, 2626 .spec = &attributes->l4, 2627 }; 2628 break; 2629 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2630 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2631 .src_addr = input->flow.tcp4_flow.ip.src_ip, 2632 .dst_addr = input->flow.tcp4_flow.ip.dst_ip, 2633 .time_to_live = input->flow.tcp4_flow.ip.ttl, 2634 .type_of_service = input->flow.tcp4_flow.ip.tos, 2635 .next_proto_id = input->flow.tcp4_flow.ip.proto, 2636 }; 2637 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2638 .src_port = input->flow.tcp4_flow.src_port, 2639 .dst_port = input->flow.tcp4_flow.dst_port, 2640 }; 2641 attributes->items[1] = (struct rte_flow_item){ 2642 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2643 .spec = &attributes->l3, 2644 }; 2645 attributes->items[2] = (struct rte_flow_item){ 2646 .type = RTE_FLOW_ITEM_TYPE_TCP, 2647 .spec = &attributes->l4, 2648 }; 2649 break; 2650 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2651 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2652 .src_addr = input->flow.ip4_flow.src_ip, 2653 .dst_addr = input->flow.ip4_flow.dst_ip, 2654 .time_to_live = input->flow.ip4_flow.ttl, 2655 .type_of_service = input->flow.ip4_flow.tos, 2656 .next_proto_id = input->flow.ip4_flow.proto, 2657 }; 2658 attributes->items[1] = (struct rte_flow_item){ 2659 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2660 .spec = &attributes->l3, 2661 }; 2662 break; 2663 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2664 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2665 .hop_limits = input->flow.udp6_flow.ip.hop_limits, 2666 .proto = input->flow.udp6_flow.ip.proto, 2667 }; 2668 memcpy(attributes->l3.ipv6.hdr.src_addr, 2669 input->flow.udp6_flow.ip.src_ip, 2670 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2671 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2672 input->flow.udp6_flow.ip.dst_ip, 2673 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2674 attributes->l4.udp.hdr = (struct udp_hdr){ 2675 .src_port = input->flow.udp6_flow.src_port, 2676 .dst_port = input->flow.udp6_flow.dst_port, 2677 }; 2678 attributes->items[1] = (struct rte_flow_item){ 2679 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2680 .spec = &attributes->l3, 2681 }; 2682 attributes->items[2] = (struct rte_flow_item){ 2683 .type = RTE_FLOW_ITEM_TYPE_UDP, 2684 .spec = &attributes->l4, 2685 }; 2686 break; 2687 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2688 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2689 .hop_limits = input->flow.tcp6_flow.ip.hop_limits, 2690 .proto = input->flow.tcp6_flow.ip.proto, 2691 }; 2692 memcpy(attributes->l3.ipv6.hdr.src_addr, 2693 input->flow.tcp6_flow.ip.src_ip, 2694 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2695 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2696 input->flow.tcp6_flow.ip.dst_ip, 2697 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2698 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2699 .src_port = input->flow.tcp6_flow.src_port, 2700 .dst_port = input->flow.tcp6_flow.dst_port, 2701 }; 2702 attributes->items[1] = (struct rte_flow_item){ 2703 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2704 .spec = &attributes->l3, 2705 }; 2706 attributes->items[2] = (struct rte_flow_item){ 2707 .type = RTE_FLOW_ITEM_TYPE_UDP, 2708 .spec = &attributes->l4, 2709 }; 2710 break; 2711 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2712 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2713 .hop_limits = input->flow.ipv6_flow.hop_limits, 2714 .proto = input->flow.ipv6_flow.proto, 2715 }; 2716 memcpy(attributes->l3.ipv6.hdr.src_addr, 2717 input->flow.ipv6_flow.src_ip, 2718 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2719 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2720 input->flow.ipv6_flow.dst_ip, 2721 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2722 attributes->items[1] = (struct rte_flow_item){ 2723 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2724 .spec = &attributes->l3, 2725 }; 2726 break; 2727 default: 2728 ERROR("invalid flow type%d", 2729 fdir_filter->input.flow_type); 2730 return ENOTSUP; 2731 } 2732 return 0; 2733 } 2734 2735 /** 2736 * Add new flow director filter and store it in list. 2737 * 2738 * @param priv 2739 * Private structure. 2740 * @param fdir_filter 2741 * Flow director filter to add. 2742 * 2743 * @return 2744 * 0 on success, errno value on failure. 2745 */ 2746 static int 2747 priv_fdir_filter_add(struct priv *priv, 2748 const struct rte_eth_fdir_filter *fdir_filter) 2749 { 2750 struct mlx5_fdir attributes = { 2751 .attr.group = 0, 2752 }; 2753 struct mlx5_flow_parse parser = { 2754 .layer = HASH_RXQ_ETH, 2755 }; 2756 struct rte_flow_error error; 2757 struct rte_flow *flow; 2758 int ret; 2759 2760 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); 2761 if (ret) 2762 return -ret; 2763 ret = priv_flow_convert(priv, &attributes.attr, attributes.items, 2764 attributes.actions, &error, &parser); 2765 if (ret) 2766 return -ret; 2767 flow = priv_flow_create(priv, 2768 &priv->flows, 2769 &attributes.attr, 2770 attributes.items, 2771 attributes.actions, 2772 &error); 2773 if (flow) { 2774 TAILQ_INSERT_TAIL(&priv->flows, flow, next); 2775 DEBUG("FDIR created %p", (void *)flow); 2776 return 0; 2777 } 2778 return ENOTSUP; 2779 } 2780 2781 /** 2782 * Delete specific filter. 2783 * 2784 * @param priv 2785 * Private structure. 2786 * @param fdir_filter 2787 * Filter to be deleted. 2788 * 2789 * @return 2790 * 0 on success, errno value on failure. 2791 */ 2792 static int 2793 priv_fdir_filter_delete(struct priv *priv, 2794 const struct rte_eth_fdir_filter *fdir_filter) 2795 { 2796 struct mlx5_fdir attributes; 2797 struct mlx5_flow_parse parser = { 2798 .create = 1, 2799 .layer = HASH_RXQ_ETH, 2800 }; 2801 struct rte_flow_error error; 2802 struct rte_flow *flow; 2803 unsigned int i; 2804 int ret; 2805 2806 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); 2807 if (ret) 2808 return -ret; 2809 ret = priv_flow_convert(priv, &attributes.attr, attributes.items, 2810 attributes.actions, &error, &parser); 2811 if (ret) 2812 goto exit; 2813 TAILQ_FOREACH(flow, &priv->flows, next) { 2814 struct ibv_flow_attr *attr; 2815 struct ibv_spec_header *attr_h; 2816 void *spec; 2817 struct ibv_flow_attr *flow_attr; 2818 struct ibv_spec_header *flow_h; 2819 void *flow_spec; 2820 unsigned int specs_n; 2821 2822 if (parser.drop) 2823 attr = parser.drop_q.ibv_attr; 2824 else 2825 attr = parser.queue[HASH_RXQ_ETH].ibv_attr; 2826 if (flow->drop) 2827 flow_attr = flow->drxq.ibv_attr; 2828 else 2829 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; 2830 /* Compare first the attributes. */ 2831 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) 2832 continue; 2833 if (attr->num_of_specs == 0) 2834 continue; 2835 spec = (void *)((uintptr_t)attr + 2836 sizeof(struct ibv_flow_attr)); 2837 flow_spec = (void *)((uintptr_t)flow_attr + 2838 sizeof(struct ibv_flow_attr)); 2839 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs); 2840 for (i = 0; i != specs_n; ++i) { 2841 attr_h = spec; 2842 flow_h = flow_spec; 2843 if (memcmp(spec, flow_spec, 2844 RTE_MIN(attr_h->size, flow_h->size))) 2845 continue; 2846 spec = (void *)((uintptr_t)attr + attr_h->size); 2847 flow_spec = (void *)((uintptr_t)flow_attr + 2848 flow_h->size); 2849 } 2850 /* At this point, the flow match. */ 2851 break; 2852 } 2853 if (flow) 2854 priv_flow_destroy(priv, &priv->flows, flow); 2855 exit: 2856 if (parser.drop) { 2857 rte_free(parser.drop_q.ibv_attr); 2858 } else { 2859 for (i = 0; i != hash_rxq_init_n; ++i) { 2860 if (parser.queue[i].ibv_attr) 2861 rte_free(parser.queue[i].ibv_attr); 2862 } 2863 } 2864 return -ret; 2865 } 2866 2867 /** 2868 * Update queue for specific filter. 2869 * 2870 * @param priv 2871 * Private structure. 2872 * @param fdir_filter 2873 * Filter to be updated. 2874 * 2875 * @return 2876 * 0 on success, errno value on failure. 2877 */ 2878 static int 2879 priv_fdir_filter_update(struct priv *priv, 2880 const struct rte_eth_fdir_filter *fdir_filter) 2881 { 2882 int ret; 2883 2884 ret = priv_fdir_filter_delete(priv, fdir_filter); 2885 if (ret) 2886 return ret; 2887 ret = priv_fdir_filter_add(priv, fdir_filter); 2888 return ret; 2889 } 2890 2891 /** 2892 * Flush all filters. 2893 * 2894 * @param priv 2895 * Private structure. 2896 */ 2897 static void 2898 priv_fdir_filter_flush(struct priv *priv) 2899 { 2900 priv_flow_flush(priv, &priv->flows); 2901 } 2902 2903 /** 2904 * Get flow director information. 2905 * 2906 * @param priv 2907 * Private structure. 2908 * @param[out] fdir_info 2909 * Resulting flow director information. 2910 */ 2911 static void 2912 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) 2913 { 2914 struct rte_eth_fdir_masks *mask = 2915 &priv->dev->data->dev_conf.fdir_conf.mask; 2916 2917 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; 2918 fdir_info->guarant_spc = 0; 2919 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 2920 fdir_info->max_flexpayload = 0; 2921 fdir_info->flow_types_mask[0] = 0; 2922 fdir_info->flex_payload_unit = 0; 2923 fdir_info->max_flex_payload_segment_num = 0; 2924 fdir_info->flex_payload_limit = 0; 2925 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 2926 } 2927 2928 /** 2929 * Deal with flow director operations. 2930 * 2931 * @param priv 2932 * Pointer to private structure. 2933 * @param filter_op 2934 * Operation to perform. 2935 * @param arg 2936 * Pointer to operation-specific structure. 2937 * 2938 * @return 2939 * 0 on success, errno value on failure. 2940 */ 2941 static int 2942 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) 2943 { 2944 enum rte_fdir_mode fdir_mode = 2945 priv->dev->data->dev_conf.fdir_conf.mode; 2946 int ret = 0; 2947 2948 if (filter_op == RTE_ETH_FILTER_NOP) 2949 return 0; 2950 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 2951 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 2952 ERROR("%p: flow director mode %d not supported", 2953 (void *)priv, fdir_mode); 2954 return EINVAL; 2955 } 2956 switch (filter_op) { 2957 case RTE_ETH_FILTER_ADD: 2958 ret = priv_fdir_filter_add(priv, arg); 2959 break; 2960 case RTE_ETH_FILTER_UPDATE: 2961 ret = priv_fdir_filter_update(priv, arg); 2962 break; 2963 case RTE_ETH_FILTER_DELETE: 2964 ret = priv_fdir_filter_delete(priv, arg); 2965 break; 2966 case RTE_ETH_FILTER_FLUSH: 2967 priv_fdir_filter_flush(priv); 2968 break; 2969 case RTE_ETH_FILTER_INFO: 2970 priv_fdir_info_get(priv, arg); 2971 break; 2972 default: 2973 DEBUG("%p: unknown operation %u", (void *)priv, 2974 filter_op); 2975 ret = EINVAL; 2976 break; 2977 } 2978 return ret; 2979 } 2980 2981 /** 2982 * Manage filter operations. 2983 * 2984 * @param dev 2985 * Pointer to Ethernet device structure. 2986 * @param filter_type 2987 * Filter type. 2988 * @param filter_op 2989 * Operation to perform. 2990 * @param arg 2991 * Pointer to operation-specific structure. 2992 * 2993 * @return 2994 * 0 on success, negative errno value on failure. 2995 */ 2996 int 2997 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 2998 enum rte_filter_type filter_type, 2999 enum rte_filter_op filter_op, 3000 void *arg) 3001 { 3002 int ret = EINVAL; 3003 struct priv *priv = dev->data->dev_private; 3004 3005 switch (filter_type) { 3006 case RTE_ETH_FILTER_GENERIC: 3007 if (filter_op != RTE_ETH_FILTER_GET) 3008 return -EINVAL; 3009 *(const void **)arg = &mlx5_flow_ops; 3010 return 0; 3011 case RTE_ETH_FILTER_FDIR: 3012 priv_lock(priv); 3013 ret = priv_fdir_ctrl_func(priv, filter_op, arg); 3014 priv_unlock(priv); 3015 break; 3016 default: 3017 ERROR("%p: filter type (%d) not supported", 3018 (void *)dev, filter_type); 3019 break; 3020 } 3021 return -ret; 3022 } 3023