1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2016 6WIND S.A. 5 * Copyright 2016 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/queue.h> 35 #include <string.h> 36 37 /* Verbs header. */ 38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 39 #ifdef PEDANTIC 40 #pragma GCC diagnostic ignored "-Wpedantic" 41 #endif 42 #include <infiniband/verbs.h> 43 #ifdef PEDANTIC 44 #pragma GCC diagnostic error "-Wpedantic" 45 #endif 46 47 #include <rte_ethdev.h> 48 #include <rte_flow.h> 49 #include <rte_flow_driver.h> 50 #include <rte_malloc.h> 51 52 #include "mlx5.h" 53 #include "mlx5_defs.h" 54 #include "mlx5_prm.h" 55 56 /* Define minimal priority for control plane flows. */ 57 #define MLX5_CTRL_FLOW_PRIORITY 4 58 59 /* Internet Protocol versions. */ 60 #define MLX5_IPV4 4 61 #define MLX5_IPV6 6 62 63 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 64 struct ibv_counter_set_init_attr { 65 int dummy; 66 }; 67 struct ibv_flow_spec_counter_action { 68 int dummy; 69 }; 70 struct ibv_counter_set { 71 int dummy; 72 }; 73 74 static inline int 75 ibv_destroy_counter_set(struct ibv_counter_set *cs) 76 { 77 (void)cs; 78 return -ENOTSUP; 79 } 80 #endif 81 82 /* Dev ops structure defined in mlx5.c */ 83 extern const struct eth_dev_ops mlx5_dev_ops; 84 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 85 86 static int 87 mlx5_flow_create_eth(const struct rte_flow_item *item, 88 const void *default_mask, 89 void *data); 90 91 static int 92 mlx5_flow_create_vlan(const struct rte_flow_item *item, 93 const void *default_mask, 94 void *data); 95 96 static int 97 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 98 const void *default_mask, 99 void *data); 100 101 static int 102 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 103 const void *default_mask, 104 void *data); 105 106 static int 107 mlx5_flow_create_udp(const struct rte_flow_item *item, 108 const void *default_mask, 109 void *data); 110 111 static int 112 mlx5_flow_create_tcp(const struct rte_flow_item *item, 113 const void *default_mask, 114 void *data); 115 116 static int 117 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 118 const void *default_mask, 119 void *data); 120 121 struct mlx5_flow_parse; 122 123 static void 124 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 125 unsigned int size); 126 127 static int 128 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); 129 130 static int 131 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser); 132 133 /* Hash RX queue types. */ 134 enum hash_rxq_type { 135 HASH_RXQ_TCPV4, 136 HASH_RXQ_UDPV4, 137 HASH_RXQ_IPV4, 138 HASH_RXQ_TCPV6, 139 HASH_RXQ_UDPV6, 140 HASH_RXQ_IPV6, 141 HASH_RXQ_ETH, 142 }; 143 144 /* Initialization data for hash RX queue. */ 145 struct hash_rxq_init { 146 uint64_t hash_fields; /* Fields that participate in the hash. */ 147 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ 148 unsigned int flow_priority; /* Flow priority to use. */ 149 unsigned int ip_version; /* Internet protocol. */ 150 }; 151 152 /* Initialization data for hash RX queues. */ 153 const struct hash_rxq_init hash_rxq_init[] = { 154 [HASH_RXQ_TCPV4] = { 155 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 156 IBV_RX_HASH_DST_IPV4 | 157 IBV_RX_HASH_SRC_PORT_TCP | 158 IBV_RX_HASH_DST_PORT_TCP), 159 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, 160 .flow_priority = 0, 161 .ip_version = MLX5_IPV4, 162 }, 163 [HASH_RXQ_UDPV4] = { 164 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 165 IBV_RX_HASH_DST_IPV4 | 166 IBV_RX_HASH_SRC_PORT_UDP | 167 IBV_RX_HASH_DST_PORT_UDP), 168 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, 169 .flow_priority = 0, 170 .ip_version = MLX5_IPV4, 171 }, 172 [HASH_RXQ_IPV4] = { 173 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 174 IBV_RX_HASH_DST_IPV4), 175 .dpdk_rss_hf = (ETH_RSS_IPV4 | 176 ETH_RSS_FRAG_IPV4), 177 .flow_priority = 1, 178 .ip_version = MLX5_IPV4, 179 }, 180 [HASH_RXQ_TCPV6] = { 181 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 182 IBV_RX_HASH_DST_IPV6 | 183 IBV_RX_HASH_SRC_PORT_TCP | 184 IBV_RX_HASH_DST_PORT_TCP), 185 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, 186 .flow_priority = 0, 187 .ip_version = MLX5_IPV6, 188 }, 189 [HASH_RXQ_UDPV6] = { 190 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 191 IBV_RX_HASH_DST_IPV6 | 192 IBV_RX_HASH_SRC_PORT_UDP | 193 IBV_RX_HASH_DST_PORT_UDP), 194 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, 195 .flow_priority = 0, 196 .ip_version = MLX5_IPV6, 197 }, 198 [HASH_RXQ_IPV6] = { 199 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 200 IBV_RX_HASH_DST_IPV6), 201 .dpdk_rss_hf = (ETH_RSS_IPV6 | 202 ETH_RSS_FRAG_IPV6), 203 .flow_priority = 1, 204 .ip_version = MLX5_IPV6, 205 }, 206 [HASH_RXQ_ETH] = { 207 .hash_fields = 0, 208 .dpdk_rss_hf = 0, 209 .flow_priority = 2, 210 }, 211 }; 212 213 /* Number of entries in hash_rxq_init[]. */ 214 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); 215 216 /** Structure for holding counter stats. */ 217 struct mlx5_flow_counter_stats { 218 uint64_t hits; /**< Number of packets matched by the rule. */ 219 uint64_t bytes; /**< Number of bytes matched by the rule. */ 220 }; 221 222 /** Structure for Drop queue. */ 223 struct mlx5_hrxq_drop { 224 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 225 struct ibv_qp *qp; /**< Verbs queue pair. */ 226 struct ibv_wq *wq; /**< Verbs work queue. */ 227 struct ibv_cq *cq; /**< Verbs completion queue. */ 228 }; 229 230 /* Flows structures. */ 231 struct mlx5_flow { 232 uint64_t hash_fields; /**< Fields that participate in the hash. */ 233 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 234 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 235 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */ 236 }; 237 238 /* Drop flows structures. */ 239 struct mlx5_flow_drop { 240 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 241 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 242 }; 243 244 struct rte_flow { 245 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ 246 uint32_t mark:1; /**< Set if the flow is marked. */ 247 uint32_t drop:1; /**< Drop queue. */ 248 uint16_t queues_n; /**< Number of entries in queue[]. */ 249 uint16_t (*queues)[]; /**< Queues indexes to use. */ 250 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 251 uint8_t rss_key[40]; /**< copy of the RSS key. */ 252 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */ 253 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */ 254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)]; 255 /**< Flow with Rx queue. */ 256 }; 257 258 /** Static initializer for items. */ 259 #define ITEMS(...) \ 260 (const enum rte_flow_item_type []){ \ 261 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ 262 } 263 264 /** Structure to generate a simple graph of layers supported by the NIC. */ 265 struct mlx5_flow_items { 266 /** List of possible actions for these items. */ 267 const enum rte_flow_action_type *const actions; 268 /** Bit-masks corresponding to the possibilities for the item. */ 269 const void *mask; 270 /** 271 * Default bit-masks to use when item->mask is not provided. When 272 * \default_mask is also NULL, the full supported bit-mask (\mask) is 273 * used instead. 274 */ 275 const void *default_mask; 276 /** Bit-masks size in bytes. */ 277 const unsigned int mask_sz; 278 /** 279 * Conversion function from rte_flow to NIC specific flow. 280 * 281 * @param item 282 * rte_flow item to convert. 283 * @param default_mask 284 * Default bit-masks to use when item->mask is not provided. 285 * @param data 286 * Internal structure to store the conversion. 287 * 288 * @return 289 * 0 on success, negative value otherwise. 290 */ 291 int (*convert)(const struct rte_flow_item *item, 292 const void *default_mask, 293 void *data); 294 /** Size in bytes of the destination structure. */ 295 const unsigned int dst_sz; 296 /** List of possible following items. */ 297 const enum rte_flow_item_type *const items; 298 }; 299 300 /** Valid action for this PMD. */ 301 static const enum rte_flow_action_type valid_actions[] = { 302 RTE_FLOW_ACTION_TYPE_DROP, 303 RTE_FLOW_ACTION_TYPE_QUEUE, 304 RTE_FLOW_ACTION_TYPE_MARK, 305 RTE_FLOW_ACTION_TYPE_FLAG, 306 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 307 RTE_FLOW_ACTION_TYPE_COUNT, 308 #endif 309 RTE_FLOW_ACTION_TYPE_END, 310 }; 311 312 /** Graph of supported items and associated actions. */ 313 static const struct mlx5_flow_items mlx5_flow_items[] = { 314 [RTE_FLOW_ITEM_TYPE_END] = { 315 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, 316 RTE_FLOW_ITEM_TYPE_VXLAN), 317 }, 318 [RTE_FLOW_ITEM_TYPE_ETH] = { 319 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, 320 RTE_FLOW_ITEM_TYPE_IPV4, 321 RTE_FLOW_ITEM_TYPE_IPV6), 322 .actions = valid_actions, 323 .mask = &(const struct rte_flow_item_eth){ 324 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 325 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 326 .type = -1, 327 }, 328 .default_mask = &rte_flow_item_eth_mask, 329 .mask_sz = sizeof(struct rte_flow_item_eth), 330 .convert = mlx5_flow_create_eth, 331 .dst_sz = sizeof(struct ibv_flow_spec_eth), 332 }, 333 [RTE_FLOW_ITEM_TYPE_VLAN] = { 334 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, 335 RTE_FLOW_ITEM_TYPE_IPV6), 336 .actions = valid_actions, 337 .mask = &(const struct rte_flow_item_vlan){ 338 .tci = -1, 339 }, 340 .default_mask = &rte_flow_item_vlan_mask, 341 .mask_sz = sizeof(struct rte_flow_item_vlan), 342 .convert = mlx5_flow_create_vlan, 343 .dst_sz = 0, 344 }, 345 [RTE_FLOW_ITEM_TYPE_IPV4] = { 346 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 347 RTE_FLOW_ITEM_TYPE_TCP), 348 .actions = valid_actions, 349 .mask = &(const struct rte_flow_item_ipv4){ 350 .hdr = { 351 .src_addr = -1, 352 .dst_addr = -1, 353 .type_of_service = -1, 354 .next_proto_id = -1, 355 }, 356 }, 357 .default_mask = &rte_flow_item_ipv4_mask, 358 .mask_sz = sizeof(struct rte_flow_item_ipv4), 359 .convert = mlx5_flow_create_ipv4, 360 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), 361 }, 362 [RTE_FLOW_ITEM_TYPE_IPV6] = { 363 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 364 RTE_FLOW_ITEM_TYPE_TCP), 365 .actions = valid_actions, 366 .mask = &(const struct rte_flow_item_ipv6){ 367 .hdr = { 368 .src_addr = { 369 0xff, 0xff, 0xff, 0xff, 370 0xff, 0xff, 0xff, 0xff, 371 0xff, 0xff, 0xff, 0xff, 372 0xff, 0xff, 0xff, 0xff, 373 }, 374 .dst_addr = { 375 0xff, 0xff, 0xff, 0xff, 376 0xff, 0xff, 0xff, 0xff, 377 0xff, 0xff, 0xff, 0xff, 378 0xff, 0xff, 0xff, 0xff, 379 }, 380 .vtc_flow = -1, 381 .proto = -1, 382 .hop_limits = -1, 383 }, 384 }, 385 .default_mask = &rte_flow_item_ipv6_mask, 386 .mask_sz = sizeof(struct rte_flow_item_ipv6), 387 .convert = mlx5_flow_create_ipv6, 388 .dst_sz = sizeof(struct ibv_flow_spec_ipv6), 389 }, 390 [RTE_FLOW_ITEM_TYPE_UDP] = { 391 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), 392 .actions = valid_actions, 393 .mask = &(const struct rte_flow_item_udp){ 394 .hdr = { 395 .src_port = -1, 396 .dst_port = -1, 397 }, 398 }, 399 .default_mask = &rte_flow_item_udp_mask, 400 .mask_sz = sizeof(struct rte_flow_item_udp), 401 .convert = mlx5_flow_create_udp, 402 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 403 }, 404 [RTE_FLOW_ITEM_TYPE_TCP] = { 405 .actions = valid_actions, 406 .mask = &(const struct rte_flow_item_tcp){ 407 .hdr = { 408 .src_port = -1, 409 .dst_port = -1, 410 }, 411 }, 412 .default_mask = &rte_flow_item_tcp_mask, 413 .mask_sz = sizeof(struct rte_flow_item_tcp), 414 .convert = mlx5_flow_create_tcp, 415 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 416 }, 417 [RTE_FLOW_ITEM_TYPE_VXLAN] = { 418 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), 419 .actions = valid_actions, 420 .mask = &(const struct rte_flow_item_vxlan){ 421 .vni = "\xff\xff\xff", 422 }, 423 .default_mask = &rte_flow_item_vxlan_mask, 424 .mask_sz = sizeof(struct rte_flow_item_vxlan), 425 .convert = mlx5_flow_create_vxlan, 426 .dst_sz = sizeof(struct ibv_flow_spec_tunnel), 427 }, 428 }; 429 430 /** Structure to pass to the conversion function. */ 431 struct mlx5_flow_parse { 432 uint32_t inner; /**< Set once VXLAN is encountered. */ 433 uint32_t allmulti:1; /**< Set once allmulti dst MAC is encountered. */ 434 uint32_t create:1; 435 /**< Whether resources should remain after a validate. */ 436 uint32_t drop:1; /**< Target is a drop queue. */ 437 uint32_t mark:1; /**< Mark is present in the flow. */ 438 uint32_t count:1; /**< Count is present in the flow. */ 439 uint32_t mark_id; /**< Mark identifier. */ 440 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ 441 uint16_t queues_n; /**< Number of entries in queue[]. */ 442 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 443 uint8_t rss_key[40]; /**< copy of the RSS key. */ 444 enum hash_rxq_type layer; /**< Last pattern layer detected. */ 445 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */ 446 struct { 447 struct ibv_flow_attr *ibv_attr; 448 /**< Pointer to Verbs attributes. */ 449 unsigned int offset; 450 /**< Current position or total size of the attribute. */ 451 } queue[RTE_DIM(hash_rxq_init)]; 452 }; 453 454 static const struct rte_flow_ops mlx5_flow_ops = { 455 .validate = mlx5_flow_validate, 456 .create = mlx5_flow_create, 457 .destroy = mlx5_flow_destroy, 458 .flush = mlx5_flow_flush, 459 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 460 .query = mlx5_flow_query, 461 #else 462 .query = NULL, 463 #endif 464 .isolate = mlx5_flow_isolate, 465 }; 466 467 /* Convert FDIR request to Generic flow. */ 468 struct mlx5_fdir { 469 struct rte_flow_attr attr; 470 struct rte_flow_action actions[2]; 471 struct rte_flow_item items[4]; 472 struct rte_flow_item_eth l2; 473 struct rte_flow_item_eth l2_mask; 474 union { 475 struct rte_flow_item_ipv4 ipv4; 476 struct rte_flow_item_ipv6 ipv6; 477 } l3; 478 union { 479 struct rte_flow_item_udp udp; 480 struct rte_flow_item_tcp tcp; 481 } l4; 482 struct rte_flow_action_queue queue; 483 }; 484 485 /* Verbs specification header. */ 486 struct ibv_spec_header { 487 enum ibv_flow_spec_type type; 488 uint16_t size; 489 }; 490 491 /** 492 * Check support for a given item. 493 * 494 * @param item[in] 495 * Item specification. 496 * @param mask[in] 497 * Bit-masks covering supported fields to compare with spec, last and mask in 498 * \item. 499 * @param size 500 * Bit-Mask size in bytes. 501 * 502 * @return 503 * 0 on success. 504 */ 505 static int 506 mlx5_flow_item_validate(const struct rte_flow_item *item, 507 const uint8_t *mask, unsigned int size) 508 { 509 int ret = 0; 510 511 if (!item->spec && (item->mask || item->last)) 512 return -1; 513 if (item->spec && !item->mask) { 514 unsigned int i; 515 const uint8_t *spec = item->spec; 516 517 for (i = 0; i < size; ++i) 518 if ((spec[i] | mask[i]) != mask[i]) 519 return -1; 520 } 521 if (item->last && !item->mask) { 522 unsigned int i; 523 const uint8_t *spec = item->last; 524 525 for (i = 0; i < size; ++i) 526 if ((spec[i] | mask[i]) != mask[i]) 527 return -1; 528 } 529 if (item->mask) { 530 unsigned int i; 531 const uint8_t *spec = item->mask; 532 533 for (i = 0; i < size; ++i) 534 if ((spec[i] | mask[i]) != mask[i]) 535 return -1; 536 } 537 if (item->spec && item->last) { 538 uint8_t spec[size]; 539 uint8_t last[size]; 540 const uint8_t *apply = mask; 541 unsigned int i; 542 543 if (item->mask) 544 apply = item->mask; 545 for (i = 0; i < size; ++i) { 546 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; 547 last[i] = ((const uint8_t *)item->last)[i] & apply[i]; 548 } 549 ret = memcmp(spec, last, size); 550 } 551 return ret; 552 } 553 554 /** 555 * Copy the RSS configuration from the user ones. 556 * 557 * @param priv 558 * Pointer to private structure. 559 * @param parser 560 * Internal parser structure. 561 * @param rss_conf 562 * User RSS configuration to save. 563 * 564 * @return 565 * 0 on success, errno value on failure. 566 */ 567 static int 568 priv_flow_convert_rss_conf(struct priv *priv, 569 struct mlx5_flow_parse *parser, 570 const struct rte_eth_rss_conf *rss_conf) 571 { 572 const struct rte_eth_rss_conf *rss; 573 574 if (rss_conf) { 575 if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) 576 return EINVAL; 577 rss = rss_conf; 578 } else { 579 rss = &priv->rss_conf; 580 } 581 if (rss->rss_key_len > 40) 582 return EINVAL; 583 parser->rss_conf.rss_key_len = rss->rss_key_len; 584 parser->rss_conf.rss_hf = rss->rss_hf; 585 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len); 586 parser->rss_conf.rss_key = parser->rss_key; 587 return 0; 588 } 589 590 /** 591 * Extract attribute to the parser. 592 * 593 * @param priv 594 * Pointer to private structure. 595 * @param[in] attr 596 * Flow rule attributes. 597 * @param[out] error 598 * Perform verbose error reporting if not NULL. 599 * @param[in, out] parser 600 * Internal parser structure. 601 * 602 * @return 603 * 0 on success, a negative errno value otherwise and rte_errno is set. 604 */ 605 static int 606 priv_flow_convert_attributes(struct priv *priv, 607 const struct rte_flow_attr *attr, 608 struct rte_flow_error *error, 609 struct mlx5_flow_parse *parser) 610 { 611 (void)priv; 612 (void)parser; 613 if (attr->group) { 614 rte_flow_error_set(error, ENOTSUP, 615 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 616 NULL, 617 "groups are not supported"); 618 return -rte_errno; 619 } 620 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) { 621 rte_flow_error_set(error, ENOTSUP, 622 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 623 NULL, 624 "priorities are not supported"); 625 return -rte_errno; 626 } 627 if (attr->egress) { 628 rte_flow_error_set(error, ENOTSUP, 629 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 630 NULL, 631 "egress is not supported"); 632 return -rte_errno; 633 } 634 if (!attr->ingress) { 635 rte_flow_error_set(error, ENOTSUP, 636 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 637 NULL, 638 "only ingress is supported"); 639 return -rte_errno; 640 } 641 return 0; 642 } 643 644 /** 645 * Extract actions request to the parser. 646 * 647 * @param priv 648 * Pointer to private structure. 649 * @param[in] actions 650 * Associated actions (list terminated by the END action). 651 * @param[out] error 652 * Perform verbose error reporting if not NULL. 653 * @param[in, out] parser 654 * Internal parser structure. 655 * 656 * @return 657 * 0 on success, a negative errno value otherwise and rte_errno is set. 658 */ 659 static int 660 priv_flow_convert_actions(struct priv *priv, 661 const struct rte_flow_action actions[], 662 struct rte_flow_error *error, 663 struct mlx5_flow_parse *parser) 664 { 665 /* 666 * Add default RSS configuration necessary for Verbs to create QP even 667 * if no RSS is necessary. 668 */ 669 priv_flow_convert_rss_conf(priv, parser, 670 (const struct rte_eth_rss_conf *) 671 &priv->rss_conf); 672 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { 673 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { 674 continue; 675 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { 676 parser->drop = 1; 677 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 678 const struct rte_flow_action_queue *queue = 679 (const struct rte_flow_action_queue *) 680 actions->conf; 681 uint16_t n; 682 uint16_t found = 0; 683 684 if (!queue || (queue->index > (priv->rxqs_n - 1))) 685 goto exit_action_not_supported; 686 for (n = 0; n < parser->queues_n; ++n) { 687 if (parser->queues[n] == queue->index) { 688 found = 1; 689 break; 690 } 691 } 692 if (parser->queues_n > 1 && !found) { 693 rte_flow_error_set(error, ENOTSUP, 694 RTE_FLOW_ERROR_TYPE_ACTION, 695 actions, 696 "queue action not in RSS queues"); 697 return -rte_errno; 698 } 699 if (!found) { 700 parser->queues_n = 1; 701 parser->queues[0] = queue->index; 702 } 703 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { 704 const struct rte_flow_action_rss *rss = 705 (const struct rte_flow_action_rss *) 706 actions->conf; 707 uint16_t n; 708 709 if (!rss || !rss->num) { 710 rte_flow_error_set(error, EINVAL, 711 RTE_FLOW_ERROR_TYPE_ACTION, 712 actions, 713 "no valid queues"); 714 return -rte_errno; 715 } 716 if (parser->queues_n == 1) { 717 uint16_t found = 0; 718 719 assert(parser->queues_n); 720 for (n = 0; n < rss->num; ++n) { 721 if (parser->queues[0] == 722 rss->queue[n]) { 723 found = 1; 724 break; 725 } 726 } 727 if (!found) { 728 rte_flow_error_set(error, ENOTSUP, 729 RTE_FLOW_ERROR_TYPE_ACTION, 730 actions, 731 "queue action not in RSS" 732 " queues"); 733 return -rte_errno; 734 } 735 } 736 for (n = 0; n < rss->num; ++n) { 737 if (rss->queue[n] >= priv->rxqs_n) { 738 rte_flow_error_set(error, EINVAL, 739 RTE_FLOW_ERROR_TYPE_ACTION, 740 actions, 741 "queue id > number of" 742 " queues"); 743 return -rte_errno; 744 } 745 } 746 for (n = 0; n < rss->num; ++n) 747 parser->queues[n] = rss->queue[n]; 748 parser->queues_n = rss->num; 749 if (priv_flow_convert_rss_conf(priv, parser, 750 rss->rss_conf)) { 751 rte_flow_error_set(error, EINVAL, 752 RTE_FLOW_ERROR_TYPE_ACTION, 753 actions, 754 "wrong RSS configuration"); 755 return -rte_errno; 756 } 757 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { 758 const struct rte_flow_action_mark *mark = 759 (const struct rte_flow_action_mark *) 760 actions->conf; 761 762 if (!mark) { 763 rte_flow_error_set(error, EINVAL, 764 RTE_FLOW_ERROR_TYPE_ACTION, 765 actions, 766 "mark must be defined"); 767 return -rte_errno; 768 } else if (mark->id >= MLX5_FLOW_MARK_MAX) { 769 rte_flow_error_set(error, ENOTSUP, 770 RTE_FLOW_ERROR_TYPE_ACTION, 771 actions, 772 "mark must be between 0" 773 " and 16777199"); 774 return -rte_errno; 775 } 776 parser->mark = 1; 777 parser->mark_id = mark->id; 778 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { 779 parser->mark = 1; 780 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && 781 priv->config.flow_counter_en) { 782 parser->count = 1; 783 } else { 784 goto exit_action_not_supported; 785 } 786 } 787 if (parser->drop && parser->mark) 788 parser->mark = 0; 789 if (!parser->queues_n && !parser->drop) { 790 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, 791 NULL, "no valid action"); 792 return -rte_errno; 793 } 794 return 0; 795 exit_action_not_supported: 796 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 797 actions, "action not supported"); 798 return -rte_errno; 799 } 800 801 /** 802 * Validate items. 803 * 804 * @param priv 805 * Pointer to private structure. 806 * @param[in] items 807 * Pattern specification (list terminated by the END pattern item). 808 * @param[out] error 809 * Perform verbose error reporting if not NULL. 810 * @param[in, out] parser 811 * Internal parser structure. 812 * 813 * @return 814 * 0 on success, a negative errno value otherwise and rte_errno is set. 815 */ 816 static int 817 priv_flow_convert_items_validate(struct priv *priv, 818 const struct rte_flow_item items[], 819 struct rte_flow_error *error, 820 struct mlx5_flow_parse *parser) 821 { 822 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 823 unsigned int i; 824 825 (void)priv; 826 /* Initialise the offsets to start after verbs attribute. */ 827 for (i = 0; i != hash_rxq_init_n; ++i) 828 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 829 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 830 const struct mlx5_flow_items *token = NULL; 831 unsigned int n; 832 int err; 833 834 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 835 continue; 836 for (i = 0; 837 cur_item->items && 838 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; 839 ++i) { 840 if (cur_item->items[i] == items->type) { 841 token = &mlx5_flow_items[items->type]; 842 break; 843 } 844 } 845 if (!token) 846 goto exit_item_not_supported; 847 cur_item = token; 848 err = mlx5_flow_item_validate(items, 849 (const uint8_t *)cur_item->mask, 850 cur_item->mask_sz); 851 if (err) 852 goto exit_item_not_supported; 853 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 854 if (parser->inner) { 855 rte_flow_error_set(error, ENOTSUP, 856 RTE_FLOW_ERROR_TYPE_ITEM, 857 items, 858 "cannot recognize multiple" 859 " VXLAN encapsulations"); 860 return -rte_errno; 861 } 862 parser->inner = IBV_FLOW_SPEC_INNER; 863 } 864 if (parser->drop || parser->queues_n == 1) { 865 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; 866 } else { 867 for (n = 0; n != hash_rxq_init_n; ++n) 868 parser->queue[n].offset += cur_item->dst_sz; 869 } 870 } 871 if (parser->drop) { 872 parser->queue[HASH_RXQ_ETH].offset += 873 sizeof(struct ibv_flow_spec_action_drop); 874 } 875 if (parser->mark) { 876 for (i = 0; i != hash_rxq_init_n; ++i) 877 parser->queue[i].offset += 878 sizeof(struct ibv_flow_spec_action_tag); 879 } 880 if (parser->count) { 881 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 882 883 for (i = 0; i != hash_rxq_init_n; ++i) 884 parser->queue[i].offset += size; 885 } 886 return 0; 887 exit_item_not_supported: 888 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 889 items, "item not supported"); 890 return -rte_errno; 891 } 892 893 /** 894 * Allocate memory space to store verbs flow attributes. 895 * 896 * @param priv 897 * Pointer to private structure. 898 * @param[in] priority 899 * Flow priority. 900 * @param[in] size 901 * Amount of byte to allocate. 902 * @param[out] error 903 * Perform verbose error reporting if not NULL. 904 * 905 * @return 906 * A verbs flow attribute on success, NULL otherwise. 907 */ 908 static struct ibv_flow_attr* 909 priv_flow_convert_allocate(struct priv *priv, 910 unsigned int priority, 911 unsigned int size, 912 struct rte_flow_error *error) 913 { 914 struct ibv_flow_attr *ibv_attr; 915 916 (void)priv; 917 ibv_attr = rte_calloc(__func__, 1, size, 0); 918 if (!ibv_attr) { 919 rte_flow_error_set(error, ENOMEM, 920 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 921 NULL, 922 "cannot allocate verbs spec attributes."); 923 return NULL; 924 } 925 ibv_attr->priority = priority; 926 return ibv_attr; 927 } 928 929 /** 930 * Finalise verbs flow attributes. 931 * 932 * @param priv 933 * Pointer to private structure. 934 * @param[in, out] parser 935 * Internal parser structure. 936 */ 937 static void 938 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) 939 { 940 const unsigned int ipv4 = 941 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; 942 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6; 943 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 944 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4; 945 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4; 946 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 947 unsigned int i; 948 949 (void)priv; 950 if (parser->layer == HASH_RXQ_ETH) { 951 goto fill; 952 } else { 953 /* 954 * This layer becomes useless as the pattern define under 955 * layers. 956 */ 957 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr); 958 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 959 } 960 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */ 961 for (i = ohmin; i != (ohmax + 1); ++i) { 962 if (!parser->queue[i].ibv_attr) 963 continue; 964 rte_free(parser->queue[i].ibv_attr); 965 parser->queue[i].ibv_attr = NULL; 966 } 967 /* Remove impossible flow according to the RSS configuration. */ 968 if (hash_rxq_init[parser->layer].dpdk_rss_hf & 969 parser->rss_conf.rss_hf) { 970 /* Remove any other flow. */ 971 for (i = hmin; i != (hmax + 1); ++i) { 972 if ((i == parser->layer) || 973 (!parser->queue[i].ibv_attr)) 974 continue; 975 rte_free(parser->queue[i].ibv_attr); 976 parser->queue[i].ibv_attr = NULL; 977 } 978 } else if (!parser->queue[ip].ibv_attr) { 979 /* no RSS possible with the current configuration. */ 980 parser->queues_n = 1; 981 return; 982 } 983 fill: 984 /* 985 * Fill missing layers in verbs specifications, or compute the correct 986 * offset to allocate the memory space for the attributes and 987 * specifications. 988 */ 989 for (i = 0; i != hash_rxq_init_n - 1; ++i) { 990 union { 991 struct ibv_flow_spec_ipv4_ext ipv4; 992 struct ibv_flow_spec_ipv6 ipv6; 993 struct ibv_flow_spec_tcp_udp udp_tcp; 994 } specs; 995 void *dst; 996 uint16_t size; 997 998 if (i == parser->layer) 999 continue; 1000 if (parser->layer == HASH_RXQ_ETH) { 1001 if (hash_rxq_init[i].ip_version == MLX5_IPV4) { 1002 size = sizeof(struct ibv_flow_spec_ipv4_ext); 1003 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){ 1004 .type = IBV_FLOW_SPEC_IPV4_EXT, 1005 .size = size, 1006 }; 1007 } else { 1008 size = sizeof(struct ibv_flow_spec_ipv6); 1009 specs.ipv6 = (struct ibv_flow_spec_ipv6){ 1010 .type = IBV_FLOW_SPEC_IPV6, 1011 .size = size, 1012 }; 1013 } 1014 if (parser->queue[i].ibv_attr) { 1015 dst = (void *)((uintptr_t) 1016 parser->queue[i].ibv_attr + 1017 parser->queue[i].offset); 1018 memcpy(dst, &specs, size); 1019 ++parser->queue[i].ibv_attr->num_of_specs; 1020 } 1021 parser->queue[i].offset += size; 1022 } 1023 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) || 1024 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) { 1025 size = sizeof(struct ibv_flow_spec_tcp_udp); 1026 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) { 1027 .type = ((i == HASH_RXQ_UDPV4 || 1028 i == HASH_RXQ_UDPV6) ? 1029 IBV_FLOW_SPEC_UDP : 1030 IBV_FLOW_SPEC_TCP), 1031 .size = size, 1032 }; 1033 if (parser->queue[i].ibv_attr) { 1034 dst = (void *)((uintptr_t) 1035 parser->queue[i].ibv_attr + 1036 parser->queue[i].offset); 1037 memcpy(dst, &specs, size); 1038 ++parser->queue[i].ibv_attr->num_of_specs; 1039 } 1040 parser->queue[i].offset += size; 1041 } 1042 } 1043 } 1044 1045 /** 1046 * Validate and convert a flow supported by the NIC. 1047 * 1048 * @param priv 1049 * Pointer to private structure. 1050 * @param[in] attr 1051 * Flow rule attributes. 1052 * @param[in] pattern 1053 * Pattern specification (list terminated by the END pattern item). 1054 * @param[in] actions 1055 * Associated actions (list terminated by the END action). 1056 * @param[out] error 1057 * Perform verbose error reporting if not NULL. 1058 * @param[in, out] parser 1059 * Internal parser structure. 1060 * 1061 * @return 1062 * 0 on success, a negative errno value otherwise and rte_errno is set. 1063 */ 1064 static int 1065 priv_flow_convert(struct priv *priv, 1066 const struct rte_flow_attr *attr, 1067 const struct rte_flow_item items[], 1068 const struct rte_flow_action actions[], 1069 struct rte_flow_error *error, 1070 struct mlx5_flow_parse *parser) 1071 { 1072 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 1073 unsigned int i; 1074 int ret; 1075 1076 /* First step. Validate the attributes, items and actions. */ 1077 *parser = (struct mlx5_flow_parse){ 1078 .create = parser->create, 1079 .layer = HASH_RXQ_ETH, 1080 .mark_id = MLX5_FLOW_MARK_DEFAULT, 1081 }; 1082 ret = priv_flow_convert_attributes(priv, attr, error, parser); 1083 if (ret) 1084 return ret; 1085 ret = priv_flow_convert_actions(priv, actions, error, parser); 1086 if (ret) 1087 return ret; 1088 ret = priv_flow_convert_items_validate(priv, items, error, parser); 1089 if (ret) 1090 return ret; 1091 priv_flow_convert_finalise(priv, parser); 1092 /* 1093 * Second step. 1094 * Allocate the memory space to store verbs specifications. 1095 */ 1096 if (parser->drop || parser->queues_n == 1) { 1097 unsigned int priority = 1098 attr->priority + 1099 hash_rxq_init[HASH_RXQ_ETH].flow_priority; 1100 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; 1101 1102 parser->queue[HASH_RXQ_ETH].ibv_attr = 1103 priv_flow_convert_allocate(priv, priority, 1104 offset, error); 1105 if (!parser->queue[HASH_RXQ_ETH].ibv_attr) 1106 return ENOMEM; 1107 parser->queue[HASH_RXQ_ETH].offset = 1108 sizeof(struct ibv_flow_attr); 1109 } else { 1110 for (i = 0; i != hash_rxq_init_n; ++i) { 1111 unsigned int priority = 1112 attr->priority + 1113 hash_rxq_init[i].flow_priority; 1114 unsigned int offset; 1115 1116 if (!(parser->rss_conf.rss_hf & 1117 hash_rxq_init[i].dpdk_rss_hf) && 1118 (i != HASH_RXQ_ETH)) 1119 continue; 1120 offset = parser->queue[i].offset; 1121 parser->queue[i].ibv_attr = 1122 priv_flow_convert_allocate(priv, priority, 1123 offset, error); 1124 if (!parser->queue[i].ibv_attr) 1125 goto exit_enomem; 1126 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 1127 } 1128 } 1129 /* Third step. Conversion parse, fill the specifications. */ 1130 parser->inner = 0; 1131 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 1132 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 1133 continue; 1134 cur_item = &mlx5_flow_items[items->type]; 1135 ret = cur_item->convert(items, 1136 (cur_item->default_mask ? 1137 cur_item->default_mask : 1138 cur_item->mask), 1139 parser); 1140 if (ret) { 1141 rte_flow_error_set(error, ret, 1142 RTE_FLOW_ERROR_TYPE_ITEM, 1143 items, "item not supported"); 1144 goto exit_free; 1145 } 1146 } 1147 if (parser->mark) 1148 mlx5_flow_create_flag_mark(parser, parser->mark_id); 1149 if (parser->count && parser->create) { 1150 mlx5_flow_create_count(priv, parser); 1151 if (!parser->cs) 1152 goto exit_count_error; 1153 } 1154 /* 1155 * Last step. Complete missing specification to reach the RSS 1156 * configuration. 1157 */ 1158 if (parser->queues_n > 1) { 1159 priv_flow_convert_finalise(priv, parser); 1160 } else { 1161 /* 1162 * Action queue have their priority overridden with 1163 * Ethernet priority, this priority needs to be adjusted to 1164 * their most specific layer priority. 1165 */ 1166 parser->queue[HASH_RXQ_ETH].ibv_attr->priority = 1167 attr->priority + 1168 hash_rxq_init[parser->layer].flow_priority; 1169 } 1170 exit_free: 1171 /* Only verification is expected, all resources should be released. */ 1172 if (!parser->create) { 1173 for (i = 0; i != hash_rxq_init_n; ++i) { 1174 if (parser->queue[i].ibv_attr) { 1175 rte_free(parser->queue[i].ibv_attr); 1176 parser->queue[i].ibv_attr = NULL; 1177 } 1178 } 1179 } 1180 if (parser->allmulti && 1181 parser->layer == HASH_RXQ_ETH) { 1182 for (i = 0; i != hash_rxq_init_n; ++i) { 1183 if (!parser->queue[i].ibv_attr) 1184 continue; 1185 if (parser->queue[i].ibv_attr->num_of_specs != 1) 1186 break; 1187 parser->queue[i].ibv_attr->type = 1188 IBV_FLOW_ATTR_MC_DEFAULT; 1189 } 1190 } 1191 return ret; 1192 exit_enomem: 1193 for (i = 0; i != hash_rxq_init_n; ++i) { 1194 if (parser->queue[i].ibv_attr) { 1195 rte_free(parser->queue[i].ibv_attr); 1196 parser->queue[i].ibv_attr = NULL; 1197 } 1198 } 1199 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1200 NULL, "cannot allocate verbs spec attributes."); 1201 return ret; 1202 exit_count_error: 1203 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1204 NULL, "cannot create counter."); 1205 return rte_errno; 1206 } 1207 1208 /** 1209 * Copy the specification created into the flow. 1210 * 1211 * @param parser 1212 * Internal parser structure. 1213 * @param src 1214 * Create specification. 1215 * @param size 1216 * Size in bytes of the specification to copy. 1217 */ 1218 static void 1219 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 1220 unsigned int size) 1221 { 1222 unsigned int i; 1223 void *dst; 1224 1225 for (i = 0; i != hash_rxq_init_n; ++i) { 1226 if (!parser->queue[i].ibv_attr) 1227 continue; 1228 /* Specification must be the same l3 type or none. */ 1229 if (parser->layer == HASH_RXQ_ETH || 1230 (hash_rxq_init[parser->layer].ip_version == 1231 hash_rxq_init[i].ip_version) || 1232 (hash_rxq_init[i].ip_version == 0)) { 1233 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1234 parser->queue[i].offset); 1235 memcpy(dst, src, size); 1236 ++parser->queue[i].ibv_attr->num_of_specs; 1237 parser->queue[i].offset += size; 1238 } 1239 } 1240 } 1241 1242 /** 1243 * Convert Ethernet item to Verbs specification. 1244 * 1245 * @param item[in] 1246 * Item specification. 1247 * @param default_mask[in] 1248 * Default bit-masks to use when item->mask is not provided. 1249 * @param data[in, out] 1250 * User structure. 1251 */ 1252 static int 1253 mlx5_flow_create_eth(const struct rte_flow_item *item, 1254 const void *default_mask, 1255 void *data) 1256 { 1257 const struct rte_flow_item_eth *spec = item->spec; 1258 const struct rte_flow_item_eth *mask = item->mask; 1259 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1260 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1261 struct ibv_flow_spec_eth eth = { 1262 .type = parser->inner | IBV_FLOW_SPEC_ETH, 1263 .size = eth_size, 1264 }; 1265 1266 /* Don't update layer for the inner pattern. */ 1267 if (!parser->inner) 1268 parser->layer = HASH_RXQ_ETH; 1269 if (spec) { 1270 unsigned int i; 1271 1272 if (!mask) 1273 mask = default_mask; 1274 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); 1275 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); 1276 eth.val.ether_type = spec->type; 1277 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); 1278 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); 1279 eth.mask.ether_type = mask->type; 1280 /* Remove unwanted bits from values. */ 1281 for (i = 0; i < ETHER_ADDR_LEN; ++i) { 1282 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 1283 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 1284 } 1285 eth.val.ether_type &= eth.mask.ether_type; 1286 } 1287 mlx5_flow_create_copy(parser, ð, eth_size); 1288 parser->allmulti = eth.val.dst_mac[0] & 1; 1289 return 0; 1290 } 1291 1292 /** 1293 * Convert VLAN item to Verbs specification. 1294 * 1295 * @param item[in] 1296 * Item specification. 1297 * @param default_mask[in] 1298 * Default bit-masks to use when item->mask is not provided. 1299 * @param data[in, out] 1300 * User structure. 1301 */ 1302 static int 1303 mlx5_flow_create_vlan(const struct rte_flow_item *item, 1304 const void *default_mask, 1305 void *data) 1306 { 1307 const struct rte_flow_item_vlan *spec = item->spec; 1308 const struct rte_flow_item_vlan *mask = item->mask; 1309 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1310 struct ibv_flow_spec_eth *eth; 1311 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1312 1313 if (spec) { 1314 unsigned int i; 1315 if (!mask) 1316 mask = default_mask; 1317 1318 for (i = 0; i != hash_rxq_init_n; ++i) { 1319 if (!parser->queue[i].ibv_attr) 1320 continue; 1321 1322 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1323 parser->queue[i].offset - eth_size); 1324 eth->val.vlan_tag = spec->tci; 1325 eth->mask.vlan_tag = mask->tci; 1326 eth->val.vlan_tag &= eth->mask.vlan_tag; 1327 } 1328 } 1329 return 0; 1330 } 1331 1332 /** 1333 * Convert IPv4 item to Verbs specification. 1334 * 1335 * @param item[in] 1336 * Item specification. 1337 * @param default_mask[in] 1338 * Default bit-masks to use when item->mask is not provided. 1339 * @param data[in, out] 1340 * User structure. 1341 */ 1342 static int 1343 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 1344 const void *default_mask, 1345 void *data) 1346 { 1347 const struct rte_flow_item_ipv4 *spec = item->spec; 1348 const struct rte_flow_item_ipv4 *mask = item->mask; 1349 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1350 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); 1351 struct ibv_flow_spec_ipv4_ext ipv4 = { 1352 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, 1353 .size = ipv4_size, 1354 }; 1355 1356 /* Don't update layer for the inner pattern. */ 1357 if (!parser->inner) 1358 parser->layer = HASH_RXQ_IPV4; 1359 if (spec) { 1360 if (!mask) 1361 mask = default_mask; 1362 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 1363 .src_ip = spec->hdr.src_addr, 1364 .dst_ip = spec->hdr.dst_addr, 1365 .proto = spec->hdr.next_proto_id, 1366 .tos = spec->hdr.type_of_service, 1367 }; 1368 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 1369 .src_ip = mask->hdr.src_addr, 1370 .dst_ip = mask->hdr.dst_addr, 1371 .proto = mask->hdr.next_proto_id, 1372 .tos = mask->hdr.type_of_service, 1373 }; 1374 /* Remove unwanted bits from values. */ 1375 ipv4.val.src_ip &= ipv4.mask.src_ip; 1376 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 1377 ipv4.val.proto &= ipv4.mask.proto; 1378 ipv4.val.tos &= ipv4.mask.tos; 1379 } 1380 mlx5_flow_create_copy(parser, &ipv4, ipv4_size); 1381 return 0; 1382 } 1383 1384 /** 1385 * Convert IPv6 item to Verbs specification. 1386 * 1387 * @param item[in] 1388 * Item specification. 1389 * @param default_mask[in] 1390 * Default bit-masks to use when item->mask is not provided. 1391 * @param data[in, out] 1392 * User structure. 1393 */ 1394 static int 1395 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 1396 const void *default_mask, 1397 void *data) 1398 { 1399 const struct rte_flow_item_ipv6 *spec = item->spec; 1400 const struct rte_flow_item_ipv6 *mask = item->mask; 1401 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1402 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); 1403 struct ibv_flow_spec_ipv6 ipv6 = { 1404 .type = parser->inner | IBV_FLOW_SPEC_IPV6, 1405 .size = ipv6_size, 1406 }; 1407 1408 /* Don't update layer for the inner pattern. */ 1409 if (!parser->inner) 1410 parser->layer = HASH_RXQ_IPV6; 1411 if (spec) { 1412 unsigned int i; 1413 1414 if (!mask) 1415 mask = default_mask; 1416 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 1417 RTE_DIM(ipv6.val.src_ip)); 1418 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 1419 RTE_DIM(ipv6.val.dst_ip)); 1420 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 1421 RTE_DIM(ipv6.mask.src_ip)); 1422 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 1423 RTE_DIM(ipv6.mask.dst_ip)); 1424 ipv6.mask.flow_label = mask->hdr.vtc_flow; 1425 ipv6.mask.next_hdr = mask->hdr.proto; 1426 ipv6.mask.hop_limit = mask->hdr.hop_limits; 1427 /* Remove unwanted bits from values. */ 1428 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 1429 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 1430 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 1431 } 1432 ipv6.val.flow_label &= ipv6.mask.flow_label; 1433 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 1434 ipv6.val.hop_limit &= ipv6.mask.hop_limit; 1435 } 1436 mlx5_flow_create_copy(parser, &ipv6, ipv6_size); 1437 return 0; 1438 } 1439 1440 /** 1441 * Convert UDP item to Verbs specification. 1442 * 1443 * @param item[in] 1444 * Item specification. 1445 * @param default_mask[in] 1446 * Default bit-masks to use when item->mask is not provided. 1447 * @param data[in, out] 1448 * User structure. 1449 */ 1450 static int 1451 mlx5_flow_create_udp(const struct rte_flow_item *item, 1452 const void *default_mask, 1453 void *data) 1454 { 1455 const struct rte_flow_item_udp *spec = item->spec; 1456 const struct rte_flow_item_udp *mask = item->mask; 1457 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1458 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1459 struct ibv_flow_spec_tcp_udp udp = { 1460 .type = parser->inner | IBV_FLOW_SPEC_UDP, 1461 .size = udp_size, 1462 }; 1463 1464 /* Don't update layer for the inner pattern. */ 1465 if (!parser->inner) { 1466 if (parser->layer == HASH_RXQ_IPV4) 1467 parser->layer = HASH_RXQ_UDPV4; 1468 else 1469 parser->layer = HASH_RXQ_UDPV6; 1470 } 1471 if (spec) { 1472 if (!mask) 1473 mask = default_mask; 1474 udp.val.dst_port = spec->hdr.dst_port; 1475 udp.val.src_port = spec->hdr.src_port; 1476 udp.mask.dst_port = mask->hdr.dst_port; 1477 udp.mask.src_port = mask->hdr.src_port; 1478 /* Remove unwanted bits from values. */ 1479 udp.val.src_port &= udp.mask.src_port; 1480 udp.val.dst_port &= udp.mask.dst_port; 1481 } 1482 mlx5_flow_create_copy(parser, &udp, udp_size); 1483 return 0; 1484 } 1485 1486 /** 1487 * Convert TCP item to Verbs specification. 1488 * 1489 * @param item[in] 1490 * Item specification. 1491 * @param default_mask[in] 1492 * Default bit-masks to use when item->mask is not provided. 1493 * @param data[in, out] 1494 * User structure. 1495 */ 1496 static int 1497 mlx5_flow_create_tcp(const struct rte_flow_item *item, 1498 const void *default_mask, 1499 void *data) 1500 { 1501 const struct rte_flow_item_tcp *spec = item->spec; 1502 const struct rte_flow_item_tcp *mask = item->mask; 1503 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1504 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1505 struct ibv_flow_spec_tcp_udp tcp = { 1506 .type = parser->inner | IBV_FLOW_SPEC_TCP, 1507 .size = tcp_size, 1508 }; 1509 1510 /* Don't update layer for the inner pattern. */ 1511 if (!parser->inner) { 1512 if (parser->layer == HASH_RXQ_IPV4) 1513 parser->layer = HASH_RXQ_TCPV4; 1514 else 1515 parser->layer = HASH_RXQ_TCPV6; 1516 } 1517 if (spec) { 1518 if (!mask) 1519 mask = default_mask; 1520 tcp.val.dst_port = spec->hdr.dst_port; 1521 tcp.val.src_port = spec->hdr.src_port; 1522 tcp.mask.dst_port = mask->hdr.dst_port; 1523 tcp.mask.src_port = mask->hdr.src_port; 1524 /* Remove unwanted bits from values. */ 1525 tcp.val.src_port &= tcp.mask.src_port; 1526 tcp.val.dst_port &= tcp.mask.dst_port; 1527 } 1528 mlx5_flow_create_copy(parser, &tcp, tcp_size); 1529 return 0; 1530 } 1531 1532 /** 1533 * Convert VXLAN item to Verbs specification. 1534 * 1535 * @param item[in] 1536 * Item specification. 1537 * @param default_mask[in] 1538 * Default bit-masks to use when item->mask is not provided. 1539 * @param data[in, out] 1540 * User structure. 1541 */ 1542 static int 1543 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 1544 const void *default_mask, 1545 void *data) 1546 { 1547 const struct rte_flow_item_vxlan *spec = item->spec; 1548 const struct rte_flow_item_vxlan *mask = item->mask; 1549 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1550 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 1551 struct ibv_flow_spec_tunnel vxlan = { 1552 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, 1553 .size = size, 1554 }; 1555 union vni { 1556 uint32_t vlan_id; 1557 uint8_t vni[4]; 1558 } id; 1559 1560 id.vni[0] = 0; 1561 parser->inner = IBV_FLOW_SPEC_INNER; 1562 if (spec) { 1563 if (!mask) 1564 mask = default_mask; 1565 memcpy(&id.vni[1], spec->vni, 3); 1566 vxlan.val.tunnel_id = id.vlan_id; 1567 memcpy(&id.vni[1], mask->vni, 3); 1568 vxlan.mask.tunnel_id = id.vlan_id; 1569 /* Remove unwanted bits from values. */ 1570 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 1571 } 1572 /* 1573 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1574 * layer is defined in the Verbs specification it is interpreted as 1575 * wildcard and all packets will match this rule, if it follows a full 1576 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1577 * before will also match this rule. 1578 * To avoid such situation, VNI 0 is currently refused. 1579 */ 1580 if (!vxlan.val.tunnel_id) 1581 return EINVAL; 1582 mlx5_flow_create_copy(parser, &vxlan, size); 1583 return 0; 1584 } 1585 1586 /** 1587 * Convert mark/flag action to Verbs specification. 1588 * 1589 * @param parser 1590 * Internal parser structure. 1591 * @param mark_id 1592 * Mark identifier. 1593 */ 1594 static int 1595 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) 1596 { 1597 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1598 struct ibv_flow_spec_action_tag tag = { 1599 .type = IBV_FLOW_SPEC_ACTION_TAG, 1600 .size = size, 1601 .tag_id = mlx5_flow_mark_set(mark_id), 1602 }; 1603 1604 assert(parser->mark); 1605 mlx5_flow_create_copy(parser, &tag, size); 1606 return 0; 1607 } 1608 1609 /** 1610 * Convert count action to Verbs specification. 1611 * 1612 * @param priv 1613 * Pointer to private structure. 1614 * @param parser 1615 * Pointer to MLX5 flow parser structure. 1616 * 1617 * @return 1618 * 0 on success, errno value on failure. 1619 */ 1620 static int 1621 mlx5_flow_create_count(struct priv *priv __rte_unused, 1622 struct mlx5_flow_parse *parser __rte_unused) 1623 { 1624 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 1625 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1626 struct ibv_counter_set_init_attr init_attr = {0}; 1627 struct ibv_flow_spec_counter_action counter = { 1628 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1629 .size = size, 1630 .counter_set_handle = 0, 1631 }; 1632 1633 init_attr.counter_set_id = 0; 1634 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr); 1635 if (!parser->cs) 1636 return EINVAL; 1637 counter.counter_set_handle = parser->cs->handle; 1638 mlx5_flow_create_copy(parser, &counter, size); 1639 #endif 1640 return 0; 1641 } 1642 1643 /** 1644 * Complete flow rule creation with a drop queue. 1645 * 1646 * @param priv 1647 * Pointer to private structure. 1648 * @param parser 1649 * Internal parser structure. 1650 * @param flow 1651 * Pointer to the rte_flow. 1652 * @param[out] error 1653 * Perform verbose error reporting if not NULL. 1654 * 1655 * @return 1656 * 0 on success, errno value on failure. 1657 */ 1658 static int 1659 priv_flow_create_action_queue_drop(struct priv *priv, 1660 struct mlx5_flow_parse *parser, 1661 struct rte_flow *flow, 1662 struct rte_flow_error *error) 1663 { 1664 struct ibv_flow_spec_action_drop *drop; 1665 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1666 int err = 0; 1667 1668 assert(priv->pd); 1669 assert(priv->ctx); 1670 flow->drop = 1; 1671 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr + 1672 parser->queue[HASH_RXQ_ETH].offset); 1673 *drop = (struct ibv_flow_spec_action_drop){ 1674 .type = IBV_FLOW_SPEC_ACTION_DROP, 1675 .size = size, 1676 }; 1677 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs; 1678 parser->queue[HASH_RXQ_ETH].offset += size; 1679 flow->frxq[HASH_RXQ_ETH].ibv_attr = 1680 parser->queue[HASH_RXQ_ETH].ibv_attr; 1681 if (parser->count) 1682 flow->cs = parser->cs; 1683 if (!priv->dev->data->dev_started) 1684 return 0; 1685 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 1686 flow->frxq[HASH_RXQ_ETH].ibv_flow = 1687 ibv_create_flow(priv->flow_drop_queue->qp, 1688 flow->frxq[HASH_RXQ_ETH].ibv_attr); 1689 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { 1690 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1691 NULL, "flow rule creation failure"); 1692 err = ENOMEM; 1693 goto error; 1694 } 1695 return 0; 1696 error: 1697 assert(flow); 1698 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) { 1699 claim_zero(ibv_destroy_flow(flow->frxq[HASH_RXQ_ETH].ibv_flow)); 1700 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; 1701 } 1702 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) { 1703 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); 1704 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL; 1705 } 1706 if (flow->cs) { 1707 claim_zero(ibv_destroy_counter_set(flow->cs)); 1708 flow->cs = NULL; 1709 parser->cs = NULL; 1710 } 1711 return err; 1712 } 1713 1714 /** 1715 * Create hash Rx queues when RSS is enabled. 1716 * 1717 * @param priv 1718 * Pointer to private structure. 1719 * @param parser 1720 * Internal parser structure. 1721 * @param flow 1722 * Pointer to the rte_flow. 1723 * @param[out] error 1724 * Perform verbose error reporting if not NULL. 1725 * 1726 * @return 1727 * 0 on success, a errno value otherwise and rte_errno is set. 1728 */ 1729 static int 1730 priv_flow_create_action_queue_rss(struct priv *priv, 1731 struct mlx5_flow_parse *parser, 1732 struct rte_flow *flow, 1733 struct rte_flow_error *error) 1734 { 1735 unsigned int i; 1736 1737 for (i = 0; i != hash_rxq_init_n; ++i) { 1738 uint64_t hash_fields; 1739 1740 if (!parser->queue[i].ibv_attr) 1741 continue; 1742 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; 1743 parser->queue[i].ibv_attr = NULL; 1744 hash_fields = hash_rxq_init[i].hash_fields; 1745 if (!priv->dev->data->dev_started) 1746 continue; 1747 flow->frxq[i].hrxq = 1748 mlx5_priv_hrxq_get(priv, 1749 parser->rss_conf.rss_key, 1750 parser->rss_conf.rss_key_len, 1751 hash_fields, 1752 parser->queues, 1753 parser->queues_n); 1754 if (flow->frxq[i].hrxq) 1755 continue; 1756 flow->frxq[i].hrxq = 1757 mlx5_priv_hrxq_new(priv, 1758 parser->rss_conf.rss_key, 1759 parser->rss_conf.rss_key_len, 1760 hash_fields, 1761 parser->queues, 1762 parser->queues_n); 1763 if (!flow->frxq[i].hrxq) { 1764 rte_flow_error_set(error, ENOMEM, 1765 RTE_FLOW_ERROR_TYPE_HANDLE, 1766 NULL, "cannot create hash rxq"); 1767 return ENOMEM; 1768 } 1769 } 1770 return 0; 1771 } 1772 1773 /** 1774 * Complete flow rule creation. 1775 * 1776 * @param priv 1777 * Pointer to private structure. 1778 * @param parser 1779 * Internal parser structure. 1780 * @param flow 1781 * Pointer to the rte_flow. 1782 * @param[out] error 1783 * Perform verbose error reporting if not NULL. 1784 * 1785 * @return 1786 * 0 on success, a errno value otherwise and rte_errno is set. 1787 */ 1788 static int 1789 priv_flow_create_action_queue(struct priv *priv, 1790 struct mlx5_flow_parse *parser, 1791 struct rte_flow *flow, 1792 struct rte_flow_error *error) 1793 { 1794 int err = 0; 1795 unsigned int i; 1796 1797 assert(priv->pd); 1798 assert(priv->ctx); 1799 assert(!parser->drop); 1800 err = priv_flow_create_action_queue_rss(priv, parser, flow, error); 1801 if (err) 1802 goto error; 1803 if (parser->count) 1804 flow->cs = parser->cs; 1805 if (!priv->dev->data->dev_started) 1806 return 0; 1807 for (i = 0; i != hash_rxq_init_n; ++i) { 1808 if (!flow->frxq[i].hrxq) 1809 continue; 1810 flow->frxq[i].ibv_flow = 1811 ibv_create_flow(flow->frxq[i].hrxq->qp, 1812 flow->frxq[i].ibv_attr); 1813 if (!flow->frxq[i].ibv_flow) { 1814 rte_flow_error_set(error, ENOMEM, 1815 RTE_FLOW_ERROR_TYPE_HANDLE, 1816 NULL, "flow rule creation failure"); 1817 err = ENOMEM; 1818 goto error; 1819 } 1820 DEBUG("%p type %d QP %p ibv_flow %p", 1821 (void *)flow, i, 1822 (void *)flow->frxq[i].hrxq, 1823 (void *)flow->frxq[i].ibv_flow); 1824 } 1825 for (i = 0; i != parser->queues_n; ++i) { 1826 struct mlx5_rxq_data *q = 1827 (*priv->rxqs)[parser->queues[i]]; 1828 1829 q->mark |= parser->mark; 1830 } 1831 return 0; 1832 error: 1833 assert(flow); 1834 for (i = 0; i != hash_rxq_init_n; ++i) { 1835 if (flow->frxq[i].ibv_flow) { 1836 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; 1837 1838 claim_zero(ibv_destroy_flow(ibv_flow)); 1839 } 1840 if (flow->frxq[i].hrxq) 1841 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); 1842 if (flow->frxq[i].ibv_attr) 1843 rte_free(flow->frxq[i].ibv_attr); 1844 } 1845 if (flow->cs) { 1846 claim_zero(ibv_destroy_counter_set(flow->cs)); 1847 flow->cs = NULL; 1848 parser->cs = NULL; 1849 } 1850 return err; 1851 } 1852 1853 /** 1854 * Convert a flow. 1855 * 1856 * @param priv 1857 * Pointer to private structure. 1858 * @param list 1859 * Pointer to a TAILQ flow list. 1860 * @param[in] attr 1861 * Flow rule attributes. 1862 * @param[in] pattern 1863 * Pattern specification (list terminated by the END pattern item). 1864 * @param[in] actions 1865 * Associated actions (list terminated by the END action). 1866 * @param[out] error 1867 * Perform verbose error reporting if not NULL. 1868 * 1869 * @return 1870 * A flow on success, NULL otherwise. 1871 */ 1872 static struct rte_flow * 1873 priv_flow_create(struct priv *priv, 1874 struct mlx5_flows *list, 1875 const struct rte_flow_attr *attr, 1876 const struct rte_flow_item items[], 1877 const struct rte_flow_action actions[], 1878 struct rte_flow_error *error) 1879 { 1880 struct mlx5_flow_parse parser = { .create = 1, }; 1881 struct rte_flow *flow = NULL; 1882 unsigned int i; 1883 int err; 1884 1885 err = priv_flow_convert(priv, attr, items, actions, error, &parser); 1886 if (err) 1887 goto exit; 1888 flow = rte_calloc(__func__, 1, 1889 sizeof(*flow) + parser.queues_n * sizeof(uint16_t), 1890 0); 1891 if (!flow) { 1892 rte_flow_error_set(error, ENOMEM, 1893 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1894 NULL, 1895 "cannot allocate flow memory"); 1896 return NULL; 1897 } 1898 /* Copy queues configuration. */ 1899 flow->queues = (uint16_t (*)[])(flow + 1); 1900 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t)); 1901 flow->queues_n = parser.queues_n; 1902 flow->mark = parser.mark; 1903 /* Copy RSS configuration. */ 1904 flow->rss_conf = parser.rss_conf; 1905 flow->rss_conf.rss_key = flow->rss_key; 1906 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len); 1907 /* finalise the flow. */ 1908 if (parser.drop) 1909 err = priv_flow_create_action_queue_drop(priv, &parser, flow, 1910 error); 1911 else 1912 err = priv_flow_create_action_queue(priv, &parser, flow, error); 1913 if (err) 1914 goto exit; 1915 TAILQ_INSERT_TAIL(list, flow, next); 1916 DEBUG("Flow created %p", (void *)flow); 1917 return flow; 1918 exit: 1919 for (i = 0; i != hash_rxq_init_n; ++i) { 1920 if (parser.queue[i].ibv_attr) 1921 rte_free(parser.queue[i].ibv_attr); 1922 } 1923 rte_free(flow); 1924 return NULL; 1925 } 1926 1927 /** 1928 * Validate a flow supported by the NIC. 1929 * 1930 * @see rte_flow_validate() 1931 * @see rte_flow_ops 1932 */ 1933 int 1934 mlx5_flow_validate(struct rte_eth_dev *dev, 1935 const struct rte_flow_attr *attr, 1936 const struct rte_flow_item items[], 1937 const struct rte_flow_action actions[], 1938 struct rte_flow_error *error) 1939 { 1940 struct priv *priv = dev->data->dev_private; 1941 int ret; 1942 struct mlx5_flow_parse parser = { .create = 0, }; 1943 1944 priv_lock(priv); 1945 ret = priv_flow_convert(priv, attr, items, actions, error, &parser); 1946 priv_unlock(priv); 1947 return ret; 1948 } 1949 1950 /** 1951 * Create a flow. 1952 * 1953 * @see rte_flow_create() 1954 * @see rte_flow_ops 1955 */ 1956 struct rte_flow * 1957 mlx5_flow_create(struct rte_eth_dev *dev, 1958 const struct rte_flow_attr *attr, 1959 const struct rte_flow_item items[], 1960 const struct rte_flow_action actions[], 1961 struct rte_flow_error *error) 1962 { 1963 struct priv *priv = dev->data->dev_private; 1964 struct rte_flow *flow; 1965 1966 priv_lock(priv); 1967 flow = priv_flow_create(priv, &priv->flows, attr, items, actions, 1968 error); 1969 priv_unlock(priv); 1970 return flow; 1971 } 1972 1973 /** 1974 * Destroy a flow. 1975 * 1976 * @param priv 1977 * Pointer to private structure. 1978 * @param list 1979 * Pointer to a TAILQ flow list. 1980 * @param[in] flow 1981 * Flow to destroy. 1982 */ 1983 static void 1984 priv_flow_destroy(struct priv *priv, 1985 struct mlx5_flows *list, 1986 struct rte_flow *flow) 1987 { 1988 unsigned int i; 1989 1990 if (flow->drop || !flow->mark) 1991 goto free; 1992 for (i = 0; i != flow->queues_n; ++i) { 1993 struct rte_flow *tmp; 1994 int mark = 0; 1995 1996 /* 1997 * To remove the mark from the queue, the queue must not be 1998 * present in any other marked flow (RSS or not). 1999 */ 2000 TAILQ_FOREACH(tmp, list, next) { 2001 unsigned int j; 2002 uint16_t *tqs = NULL; 2003 uint16_t tq_n = 0; 2004 2005 if (!tmp->mark) 2006 continue; 2007 for (j = 0; j != hash_rxq_init_n; ++j) { 2008 if (!tmp->frxq[j].hrxq) 2009 continue; 2010 tqs = tmp->frxq[j].hrxq->ind_table->queues; 2011 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n; 2012 } 2013 if (!tq_n) 2014 continue; 2015 for (j = 0; (j != tq_n) && !mark; j++) 2016 if (tqs[j] == (*flow->queues)[i]) 2017 mark = 1; 2018 } 2019 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark; 2020 } 2021 free: 2022 if (flow->drop) { 2023 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) 2024 claim_zero(ibv_destroy_flow 2025 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 2026 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); 2027 } else { 2028 for (i = 0; i != hash_rxq_init_n; ++i) { 2029 struct mlx5_flow *frxq = &flow->frxq[i]; 2030 2031 if (frxq->ibv_flow) 2032 claim_zero(ibv_destroy_flow(frxq->ibv_flow)); 2033 if (frxq->hrxq) 2034 mlx5_priv_hrxq_release(priv, frxq->hrxq); 2035 if (frxq->ibv_attr) 2036 rte_free(frxq->ibv_attr); 2037 } 2038 } 2039 if (flow->cs) { 2040 claim_zero(ibv_destroy_counter_set(flow->cs)); 2041 flow->cs = NULL; 2042 } 2043 TAILQ_REMOVE(list, flow, next); 2044 DEBUG("Flow destroyed %p", (void *)flow); 2045 rte_free(flow); 2046 } 2047 2048 /** 2049 * Destroy all flows. 2050 * 2051 * @param priv 2052 * Pointer to private structure. 2053 * @param list 2054 * Pointer to a TAILQ flow list. 2055 */ 2056 void 2057 priv_flow_flush(struct priv *priv, struct mlx5_flows *list) 2058 { 2059 while (!TAILQ_EMPTY(list)) { 2060 struct rte_flow *flow; 2061 2062 flow = TAILQ_FIRST(list); 2063 priv_flow_destroy(priv, list, flow); 2064 } 2065 } 2066 2067 /** 2068 * Create drop queue. 2069 * 2070 * @param priv 2071 * Pointer to private structure. 2072 * 2073 * @return 2074 * 0 on success. 2075 */ 2076 int 2077 priv_flow_create_drop_queue(struct priv *priv) 2078 { 2079 struct mlx5_hrxq_drop *fdq = NULL; 2080 2081 assert(priv->pd); 2082 assert(priv->ctx); 2083 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); 2084 if (!fdq) { 2085 WARN("cannot allocate memory for drop queue"); 2086 goto error; 2087 } 2088 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); 2089 if (!fdq->cq) { 2090 WARN("cannot allocate CQ for drop queue"); 2091 goto error; 2092 } 2093 fdq->wq = ibv_create_wq(priv->ctx, 2094 &(struct ibv_wq_init_attr){ 2095 .wq_type = IBV_WQT_RQ, 2096 .max_wr = 1, 2097 .max_sge = 1, 2098 .pd = priv->pd, 2099 .cq = fdq->cq, 2100 }); 2101 if (!fdq->wq) { 2102 WARN("cannot allocate WQ for drop queue"); 2103 goto error; 2104 } 2105 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx, 2106 &(struct ibv_rwq_ind_table_init_attr){ 2107 .log_ind_tbl_size = 0, 2108 .ind_tbl = &fdq->wq, 2109 .comp_mask = 0, 2110 }); 2111 if (!fdq->ind_table) { 2112 WARN("cannot allocate indirection table for drop queue"); 2113 goto error; 2114 } 2115 fdq->qp = ibv_create_qp_ex(priv->ctx, 2116 &(struct ibv_qp_init_attr_ex){ 2117 .qp_type = IBV_QPT_RAW_PACKET, 2118 .comp_mask = 2119 IBV_QP_INIT_ATTR_PD | 2120 IBV_QP_INIT_ATTR_IND_TABLE | 2121 IBV_QP_INIT_ATTR_RX_HASH, 2122 .rx_hash_conf = (struct ibv_rx_hash_conf){ 2123 .rx_hash_function = 2124 IBV_RX_HASH_FUNC_TOEPLITZ, 2125 .rx_hash_key_len = rss_hash_default_key_len, 2126 .rx_hash_key = rss_hash_default_key, 2127 .rx_hash_fields_mask = 0, 2128 }, 2129 .rwq_ind_tbl = fdq->ind_table, 2130 .pd = priv->pd 2131 }); 2132 if (!fdq->qp) { 2133 WARN("cannot allocate QP for drop queue"); 2134 goto error; 2135 } 2136 priv->flow_drop_queue = fdq; 2137 return 0; 2138 error: 2139 if (fdq->qp) 2140 claim_zero(ibv_destroy_qp(fdq->qp)); 2141 if (fdq->ind_table) 2142 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 2143 if (fdq->wq) 2144 claim_zero(ibv_destroy_wq(fdq->wq)); 2145 if (fdq->cq) 2146 claim_zero(ibv_destroy_cq(fdq->cq)); 2147 if (fdq) 2148 rte_free(fdq); 2149 priv->flow_drop_queue = NULL; 2150 return -1; 2151 } 2152 2153 /** 2154 * Delete drop queue. 2155 * 2156 * @param priv 2157 * Pointer to private structure. 2158 */ 2159 void 2160 priv_flow_delete_drop_queue(struct priv *priv) 2161 { 2162 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; 2163 2164 if (!fdq) 2165 return; 2166 if (fdq->qp) 2167 claim_zero(ibv_destroy_qp(fdq->qp)); 2168 if (fdq->ind_table) 2169 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 2170 if (fdq->wq) 2171 claim_zero(ibv_destroy_wq(fdq->wq)); 2172 if (fdq->cq) 2173 claim_zero(ibv_destroy_cq(fdq->cq)); 2174 rte_free(fdq); 2175 priv->flow_drop_queue = NULL; 2176 } 2177 2178 /** 2179 * Remove all flows. 2180 * 2181 * @param priv 2182 * Pointer to private structure. 2183 * @param list 2184 * Pointer to a TAILQ flow list. 2185 */ 2186 void 2187 priv_flow_stop(struct priv *priv, struct mlx5_flows *list) 2188 { 2189 struct rte_flow *flow; 2190 2191 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 2192 unsigned int i; 2193 2194 if (flow->drop) { 2195 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) 2196 continue; 2197 claim_zero(ibv_destroy_flow 2198 (flow->frxq[HASH_RXQ_ETH].ibv_flow)); 2199 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; 2200 /* Next flow. */ 2201 continue; 2202 } 2203 if (flow->mark) { 2204 struct mlx5_ind_table_ibv *ind_tbl = NULL; 2205 2206 for (i = 0; i != hash_rxq_init_n; ++i) { 2207 if (!flow->frxq[i].hrxq) 2208 continue; 2209 ind_tbl = flow->frxq[i].hrxq->ind_table; 2210 } 2211 assert(ind_tbl); 2212 for (i = 0; i != ind_tbl->queues_n; ++i) 2213 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; 2214 } 2215 for (i = 0; i != hash_rxq_init_n; ++i) { 2216 if (!flow->frxq[i].ibv_flow) 2217 continue; 2218 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow)); 2219 flow->frxq[i].ibv_flow = NULL; 2220 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); 2221 flow->frxq[i].hrxq = NULL; 2222 } 2223 DEBUG("Flow %p removed", (void *)flow); 2224 } 2225 } 2226 2227 /** 2228 * Add all flows. 2229 * 2230 * @param priv 2231 * Pointer to private structure. 2232 * @param list 2233 * Pointer to a TAILQ flow list. 2234 * 2235 * @return 2236 * 0 on success, a errno value otherwise and rte_errno is set. 2237 */ 2238 int 2239 priv_flow_start(struct priv *priv, struct mlx5_flows *list) 2240 { 2241 struct rte_flow *flow; 2242 2243 TAILQ_FOREACH(flow, list, next) { 2244 unsigned int i; 2245 2246 if (flow->drop) { 2247 flow->frxq[HASH_RXQ_ETH].ibv_flow = 2248 ibv_create_flow 2249 (priv->flow_drop_queue->qp, 2250 flow->frxq[HASH_RXQ_ETH].ibv_attr); 2251 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { 2252 DEBUG("Flow %p cannot be applied", 2253 (void *)flow); 2254 rte_errno = EINVAL; 2255 return rte_errno; 2256 } 2257 DEBUG("Flow %p applied", (void *)flow); 2258 /* Next flow. */ 2259 continue; 2260 } 2261 for (i = 0; i != hash_rxq_init_n; ++i) { 2262 if (!flow->frxq[i].ibv_attr) 2263 continue; 2264 flow->frxq[i].hrxq = 2265 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key, 2266 flow->rss_conf.rss_key_len, 2267 hash_rxq_init[i].hash_fields, 2268 (*flow->queues), 2269 flow->queues_n); 2270 if (flow->frxq[i].hrxq) 2271 goto flow_create; 2272 flow->frxq[i].hrxq = 2273 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key, 2274 flow->rss_conf.rss_key_len, 2275 hash_rxq_init[i].hash_fields, 2276 (*flow->queues), 2277 flow->queues_n); 2278 if (!flow->frxq[i].hrxq) { 2279 DEBUG("Flow %p cannot be applied", 2280 (void *)flow); 2281 rte_errno = EINVAL; 2282 return rte_errno; 2283 } 2284 flow_create: 2285 flow->frxq[i].ibv_flow = 2286 ibv_create_flow(flow->frxq[i].hrxq->qp, 2287 flow->frxq[i].ibv_attr); 2288 if (!flow->frxq[i].ibv_flow) { 2289 DEBUG("Flow %p cannot be applied", 2290 (void *)flow); 2291 rte_errno = EINVAL; 2292 return rte_errno; 2293 } 2294 DEBUG("Flow %p applied", (void *)flow); 2295 } 2296 if (!flow->mark) 2297 continue; 2298 for (i = 0; i != flow->queues_n; ++i) 2299 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1; 2300 } 2301 return 0; 2302 } 2303 2304 /** 2305 * Verify the flow list is empty 2306 * 2307 * @param priv 2308 * Pointer to private structure. 2309 * 2310 * @return the number of flows not released. 2311 */ 2312 int 2313 priv_flow_verify(struct priv *priv) 2314 { 2315 struct rte_flow *flow; 2316 int ret = 0; 2317 2318 TAILQ_FOREACH(flow, &priv->flows, next) { 2319 DEBUG("%p: flow %p still referenced", (void *)priv, 2320 (void *)flow); 2321 ++ret; 2322 } 2323 return ret; 2324 } 2325 2326 /** 2327 * Enable a control flow configured from the control plane. 2328 * 2329 * @param dev 2330 * Pointer to Ethernet device. 2331 * @param eth_spec 2332 * An Ethernet flow spec to apply. 2333 * @param eth_mask 2334 * An Ethernet flow mask to apply. 2335 * @param vlan_spec 2336 * A VLAN flow spec to apply. 2337 * @param vlan_mask 2338 * A VLAN flow mask to apply. 2339 * 2340 * @return 2341 * 0 on success. 2342 */ 2343 int 2344 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2345 struct rte_flow_item_eth *eth_spec, 2346 struct rte_flow_item_eth *eth_mask, 2347 struct rte_flow_item_vlan *vlan_spec, 2348 struct rte_flow_item_vlan *vlan_mask) 2349 { 2350 struct priv *priv = dev->data->dev_private; 2351 const struct rte_flow_attr attr = { 2352 .ingress = 1, 2353 .priority = MLX5_CTRL_FLOW_PRIORITY, 2354 }; 2355 struct rte_flow_item items[] = { 2356 { 2357 .type = RTE_FLOW_ITEM_TYPE_ETH, 2358 .spec = eth_spec, 2359 .last = NULL, 2360 .mask = eth_mask, 2361 }, 2362 { 2363 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2364 RTE_FLOW_ITEM_TYPE_END, 2365 .spec = vlan_spec, 2366 .last = NULL, 2367 .mask = vlan_mask, 2368 }, 2369 { 2370 .type = RTE_FLOW_ITEM_TYPE_END, 2371 }, 2372 }; 2373 struct rte_flow_action actions[] = { 2374 { 2375 .type = RTE_FLOW_ACTION_TYPE_RSS, 2376 }, 2377 { 2378 .type = RTE_FLOW_ACTION_TYPE_END, 2379 }, 2380 }; 2381 struct rte_flow *flow; 2382 struct rte_flow_error error; 2383 unsigned int i; 2384 union { 2385 struct rte_flow_action_rss rss; 2386 struct { 2387 const struct rte_eth_rss_conf *rss_conf; 2388 uint16_t num; 2389 uint16_t queue[RTE_MAX_QUEUES_PER_PORT]; 2390 } local; 2391 } action_rss; 2392 2393 if (!priv->reta_idx_n) 2394 return EINVAL; 2395 for (i = 0; i != priv->reta_idx_n; ++i) 2396 action_rss.local.queue[i] = (*priv->reta_idx)[i]; 2397 action_rss.local.rss_conf = &priv->rss_conf; 2398 action_rss.local.num = priv->reta_idx_n; 2399 actions[0].conf = (const void *)&action_rss.rss; 2400 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions, 2401 &error); 2402 if (!flow) 2403 return rte_errno; 2404 return 0; 2405 } 2406 2407 /** 2408 * Enable a flow control configured from the control plane. 2409 * 2410 * @param dev 2411 * Pointer to Ethernet device. 2412 * @param eth_spec 2413 * An Ethernet flow spec to apply. 2414 * @param eth_mask 2415 * An Ethernet flow mask to apply. 2416 * 2417 * @return 2418 * 0 on success. 2419 */ 2420 int 2421 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2422 struct rte_flow_item_eth *eth_spec, 2423 struct rte_flow_item_eth *eth_mask) 2424 { 2425 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2426 } 2427 2428 /** 2429 * Destroy a flow. 2430 * 2431 * @see rte_flow_destroy() 2432 * @see rte_flow_ops 2433 */ 2434 int 2435 mlx5_flow_destroy(struct rte_eth_dev *dev, 2436 struct rte_flow *flow, 2437 struct rte_flow_error *error) 2438 { 2439 struct priv *priv = dev->data->dev_private; 2440 2441 (void)error; 2442 priv_lock(priv); 2443 priv_flow_destroy(priv, &priv->flows, flow); 2444 priv_unlock(priv); 2445 return 0; 2446 } 2447 2448 /** 2449 * Destroy all flows. 2450 * 2451 * @see rte_flow_flush() 2452 * @see rte_flow_ops 2453 */ 2454 int 2455 mlx5_flow_flush(struct rte_eth_dev *dev, 2456 struct rte_flow_error *error) 2457 { 2458 struct priv *priv = dev->data->dev_private; 2459 2460 (void)error; 2461 priv_lock(priv); 2462 priv_flow_flush(priv, &priv->flows); 2463 priv_unlock(priv); 2464 return 0; 2465 } 2466 2467 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 2468 /** 2469 * Query flow counter. 2470 * 2471 * @param cs 2472 * the counter set. 2473 * @param counter_value 2474 * returned data from the counter. 2475 * 2476 * @return 2477 * 0 on success, a errno value otherwise and rte_errno is set. 2478 */ 2479 static int 2480 priv_flow_query_count(struct ibv_counter_set *cs, 2481 struct mlx5_flow_counter_stats *counter_stats, 2482 struct rte_flow_query_count *query_count, 2483 struct rte_flow_error *error) 2484 { 2485 uint64_t counters[2]; 2486 struct ibv_query_counter_set_attr query_cs_attr = { 2487 .cs = cs, 2488 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 2489 }; 2490 struct ibv_counter_set_data query_out = { 2491 .out = counters, 2492 .outlen = 2 * sizeof(uint64_t), 2493 }; 2494 int res = ibv_query_counter_set(&query_cs_attr, &query_out); 2495 2496 if (res) { 2497 rte_flow_error_set(error, -res, 2498 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2499 NULL, 2500 "cannot read counter"); 2501 return -res; 2502 } 2503 query_count->hits_set = 1; 2504 query_count->bytes_set = 1; 2505 query_count->hits = counters[0] - counter_stats->hits; 2506 query_count->bytes = counters[1] - counter_stats->bytes; 2507 if (query_count->reset) { 2508 counter_stats->hits = counters[0]; 2509 counter_stats->bytes = counters[1]; 2510 } 2511 return 0; 2512 } 2513 2514 /** 2515 * Query a flows. 2516 * 2517 * @see rte_flow_query() 2518 * @see rte_flow_ops 2519 */ 2520 int 2521 mlx5_flow_query(struct rte_eth_dev *dev, 2522 struct rte_flow *flow, 2523 enum rte_flow_action_type action __rte_unused, 2524 void *data, 2525 struct rte_flow_error *error) 2526 { 2527 struct priv *priv = dev->data->dev_private; 2528 int res = EINVAL; 2529 2530 priv_lock(priv); 2531 if (flow->cs) { 2532 res = priv_flow_query_count(flow->cs, 2533 &flow->counter_stats, 2534 (struct rte_flow_query_count *)data, 2535 error); 2536 } else { 2537 rte_flow_error_set(error, res, 2538 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2539 NULL, 2540 "no counter found for flow"); 2541 } 2542 priv_unlock(priv); 2543 return -res; 2544 } 2545 #endif 2546 2547 /** 2548 * Isolated mode. 2549 * 2550 * @see rte_flow_isolate() 2551 * @see rte_flow_ops 2552 */ 2553 int 2554 mlx5_flow_isolate(struct rte_eth_dev *dev, 2555 int enable, 2556 struct rte_flow_error *error) 2557 { 2558 struct priv *priv = dev->data->dev_private; 2559 2560 priv_lock(priv); 2561 if (dev->data->dev_started) { 2562 rte_flow_error_set(error, EBUSY, 2563 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2564 NULL, 2565 "port must be stopped first"); 2566 priv_unlock(priv); 2567 return -rte_errno; 2568 } 2569 priv->isolated = !!enable; 2570 if (enable) 2571 priv->dev->dev_ops = &mlx5_dev_ops_isolate; 2572 else 2573 priv->dev->dev_ops = &mlx5_dev_ops; 2574 priv_unlock(priv); 2575 return 0; 2576 } 2577 2578 /** 2579 * Convert a flow director filter to a generic flow. 2580 * 2581 * @param priv 2582 * Private structure. 2583 * @param fdir_filter 2584 * Flow director filter to add. 2585 * @param attributes 2586 * Generic flow parameters structure. 2587 * 2588 * @return 2589 * 0 on success, errno value on error. 2590 */ 2591 static int 2592 priv_fdir_filter_convert(struct priv *priv, 2593 const struct rte_eth_fdir_filter *fdir_filter, 2594 struct mlx5_fdir *attributes) 2595 { 2596 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2597 2598 /* Validate queue number. */ 2599 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2600 ERROR("invalid queue number %d", fdir_filter->action.rx_queue); 2601 return EINVAL; 2602 } 2603 attributes->attr.ingress = 1; 2604 attributes->items[0] = (struct rte_flow_item) { 2605 .type = RTE_FLOW_ITEM_TYPE_ETH, 2606 .spec = &attributes->l2, 2607 .mask = &attributes->l2_mask, 2608 }; 2609 switch (fdir_filter->action.behavior) { 2610 case RTE_ETH_FDIR_ACCEPT: 2611 attributes->actions[0] = (struct rte_flow_action){ 2612 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2613 .conf = &attributes->queue, 2614 }; 2615 break; 2616 case RTE_ETH_FDIR_REJECT: 2617 attributes->actions[0] = (struct rte_flow_action){ 2618 .type = RTE_FLOW_ACTION_TYPE_DROP, 2619 }; 2620 break; 2621 default: 2622 ERROR("invalid behavior %d", fdir_filter->action.behavior); 2623 return ENOTSUP; 2624 } 2625 attributes->queue.index = fdir_filter->action.rx_queue; 2626 switch (fdir_filter->input.flow_type) { 2627 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2628 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2629 .src_addr = input->flow.udp4_flow.ip.src_ip, 2630 .dst_addr = input->flow.udp4_flow.ip.dst_ip, 2631 .time_to_live = input->flow.udp4_flow.ip.ttl, 2632 .type_of_service = input->flow.udp4_flow.ip.tos, 2633 .next_proto_id = input->flow.udp4_flow.ip.proto, 2634 }; 2635 attributes->l4.udp.hdr = (struct udp_hdr){ 2636 .src_port = input->flow.udp4_flow.src_port, 2637 .dst_port = input->flow.udp4_flow.dst_port, 2638 }; 2639 attributes->items[1] = (struct rte_flow_item){ 2640 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2641 .spec = &attributes->l3, 2642 }; 2643 attributes->items[2] = (struct rte_flow_item){ 2644 .type = RTE_FLOW_ITEM_TYPE_UDP, 2645 .spec = &attributes->l4, 2646 }; 2647 break; 2648 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2649 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2650 .src_addr = input->flow.tcp4_flow.ip.src_ip, 2651 .dst_addr = input->flow.tcp4_flow.ip.dst_ip, 2652 .time_to_live = input->flow.tcp4_flow.ip.ttl, 2653 .type_of_service = input->flow.tcp4_flow.ip.tos, 2654 .next_proto_id = input->flow.tcp4_flow.ip.proto, 2655 }; 2656 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2657 .src_port = input->flow.tcp4_flow.src_port, 2658 .dst_port = input->flow.tcp4_flow.dst_port, 2659 }; 2660 attributes->items[1] = (struct rte_flow_item){ 2661 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2662 .spec = &attributes->l3, 2663 }; 2664 attributes->items[2] = (struct rte_flow_item){ 2665 .type = RTE_FLOW_ITEM_TYPE_TCP, 2666 .spec = &attributes->l4, 2667 }; 2668 break; 2669 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2670 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2671 .src_addr = input->flow.ip4_flow.src_ip, 2672 .dst_addr = input->flow.ip4_flow.dst_ip, 2673 .time_to_live = input->flow.ip4_flow.ttl, 2674 .type_of_service = input->flow.ip4_flow.tos, 2675 .next_proto_id = input->flow.ip4_flow.proto, 2676 }; 2677 attributes->items[1] = (struct rte_flow_item){ 2678 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2679 .spec = &attributes->l3, 2680 }; 2681 break; 2682 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2683 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2684 .hop_limits = input->flow.udp6_flow.ip.hop_limits, 2685 .proto = input->flow.udp6_flow.ip.proto, 2686 }; 2687 memcpy(attributes->l3.ipv6.hdr.src_addr, 2688 input->flow.udp6_flow.ip.src_ip, 2689 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2690 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2691 input->flow.udp6_flow.ip.dst_ip, 2692 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2693 attributes->l4.udp.hdr = (struct udp_hdr){ 2694 .src_port = input->flow.udp6_flow.src_port, 2695 .dst_port = input->flow.udp6_flow.dst_port, 2696 }; 2697 attributes->items[1] = (struct rte_flow_item){ 2698 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2699 .spec = &attributes->l3, 2700 }; 2701 attributes->items[2] = (struct rte_flow_item){ 2702 .type = RTE_FLOW_ITEM_TYPE_UDP, 2703 .spec = &attributes->l4, 2704 }; 2705 break; 2706 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2707 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2708 .hop_limits = input->flow.tcp6_flow.ip.hop_limits, 2709 .proto = input->flow.tcp6_flow.ip.proto, 2710 }; 2711 memcpy(attributes->l3.ipv6.hdr.src_addr, 2712 input->flow.tcp6_flow.ip.src_ip, 2713 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2714 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2715 input->flow.tcp6_flow.ip.dst_ip, 2716 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2717 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2718 .src_port = input->flow.tcp6_flow.src_port, 2719 .dst_port = input->flow.tcp6_flow.dst_port, 2720 }; 2721 attributes->items[1] = (struct rte_flow_item){ 2722 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2723 .spec = &attributes->l3, 2724 }; 2725 attributes->items[2] = (struct rte_flow_item){ 2726 .type = RTE_FLOW_ITEM_TYPE_TCP, 2727 .spec = &attributes->l4, 2728 }; 2729 break; 2730 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2731 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2732 .hop_limits = input->flow.ipv6_flow.hop_limits, 2733 .proto = input->flow.ipv6_flow.proto, 2734 }; 2735 memcpy(attributes->l3.ipv6.hdr.src_addr, 2736 input->flow.ipv6_flow.src_ip, 2737 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2738 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2739 input->flow.ipv6_flow.dst_ip, 2740 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2741 attributes->items[1] = (struct rte_flow_item){ 2742 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2743 .spec = &attributes->l3, 2744 }; 2745 break; 2746 default: 2747 ERROR("invalid flow type%d", 2748 fdir_filter->input.flow_type); 2749 return ENOTSUP; 2750 } 2751 return 0; 2752 } 2753 2754 /** 2755 * Add new flow director filter and store it in list. 2756 * 2757 * @param priv 2758 * Private structure. 2759 * @param fdir_filter 2760 * Flow director filter to add. 2761 * 2762 * @return 2763 * 0 on success, errno value on failure. 2764 */ 2765 static int 2766 priv_fdir_filter_add(struct priv *priv, 2767 const struct rte_eth_fdir_filter *fdir_filter) 2768 { 2769 struct mlx5_fdir attributes = { 2770 .attr.group = 0, 2771 .l2_mask = { 2772 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2773 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2774 .type = 0, 2775 }, 2776 }; 2777 struct mlx5_flow_parse parser = { 2778 .layer = HASH_RXQ_ETH, 2779 }; 2780 struct rte_flow_error error; 2781 struct rte_flow *flow; 2782 int ret; 2783 2784 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); 2785 if (ret) 2786 return -ret; 2787 ret = priv_flow_convert(priv, &attributes.attr, attributes.items, 2788 attributes.actions, &error, &parser); 2789 if (ret) 2790 return -ret; 2791 flow = priv_flow_create(priv, 2792 &priv->flows, 2793 &attributes.attr, 2794 attributes.items, 2795 attributes.actions, 2796 &error); 2797 if (flow) { 2798 DEBUG("FDIR created %p", (void *)flow); 2799 return 0; 2800 } 2801 return ENOTSUP; 2802 } 2803 2804 /** 2805 * Delete specific filter. 2806 * 2807 * @param priv 2808 * Private structure. 2809 * @param fdir_filter 2810 * Filter to be deleted. 2811 * 2812 * @return 2813 * 0 on success, errno value on failure. 2814 */ 2815 static int 2816 priv_fdir_filter_delete(struct priv *priv, 2817 const struct rte_eth_fdir_filter *fdir_filter) 2818 { 2819 struct mlx5_fdir attributes = { 2820 .attr.group = 0, 2821 }; 2822 struct mlx5_flow_parse parser = { 2823 .create = 1, 2824 .layer = HASH_RXQ_ETH, 2825 }; 2826 struct rte_flow_error error; 2827 struct rte_flow *flow; 2828 unsigned int i; 2829 int ret; 2830 2831 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); 2832 if (ret) 2833 return -ret; 2834 ret = priv_flow_convert(priv, &attributes.attr, attributes.items, 2835 attributes.actions, &error, &parser); 2836 if (ret) 2837 goto exit; 2838 /* 2839 * Special case for drop action which is only set in the 2840 * specifications when the flow is created. In this situation the 2841 * drop specification is missing. 2842 */ 2843 if (parser.drop) { 2844 struct ibv_flow_spec_action_drop *drop; 2845 2846 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr + 2847 parser.queue[HASH_RXQ_ETH].offset); 2848 *drop = (struct ibv_flow_spec_action_drop){ 2849 .type = IBV_FLOW_SPEC_ACTION_DROP, 2850 .size = sizeof(struct ibv_flow_spec_action_drop), 2851 }; 2852 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++; 2853 } 2854 TAILQ_FOREACH(flow, &priv->flows, next) { 2855 struct ibv_flow_attr *attr; 2856 struct ibv_spec_header *attr_h; 2857 void *spec; 2858 struct ibv_flow_attr *flow_attr; 2859 struct ibv_spec_header *flow_h; 2860 void *flow_spec; 2861 unsigned int specs_n; 2862 2863 attr = parser.queue[HASH_RXQ_ETH].ibv_attr; 2864 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; 2865 /* Compare first the attributes. */ 2866 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) 2867 continue; 2868 if (attr->num_of_specs == 0) 2869 continue; 2870 spec = (void *)((uintptr_t)attr + 2871 sizeof(struct ibv_flow_attr)); 2872 flow_spec = (void *)((uintptr_t)flow_attr + 2873 sizeof(struct ibv_flow_attr)); 2874 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs); 2875 for (i = 0; i != specs_n; ++i) { 2876 attr_h = spec; 2877 flow_h = flow_spec; 2878 if (memcmp(spec, flow_spec, 2879 RTE_MIN(attr_h->size, flow_h->size))) 2880 goto wrong_flow; 2881 spec = (void *)((uintptr_t)spec + attr_h->size); 2882 flow_spec = (void *)((uintptr_t)flow_spec + 2883 flow_h->size); 2884 } 2885 /* At this point, the flow match. */ 2886 break; 2887 wrong_flow: 2888 /* The flow does not match. */ 2889 continue; 2890 } 2891 if (flow) 2892 priv_flow_destroy(priv, &priv->flows, flow); 2893 exit: 2894 for (i = 0; i != hash_rxq_init_n; ++i) { 2895 if (parser.queue[i].ibv_attr) 2896 rte_free(parser.queue[i].ibv_attr); 2897 } 2898 return -ret; 2899 } 2900 2901 /** 2902 * Update queue for specific filter. 2903 * 2904 * @param priv 2905 * Private structure. 2906 * @param fdir_filter 2907 * Filter to be updated. 2908 * 2909 * @return 2910 * 0 on success, errno value on failure. 2911 */ 2912 static int 2913 priv_fdir_filter_update(struct priv *priv, 2914 const struct rte_eth_fdir_filter *fdir_filter) 2915 { 2916 int ret; 2917 2918 ret = priv_fdir_filter_delete(priv, fdir_filter); 2919 if (ret) 2920 return ret; 2921 ret = priv_fdir_filter_add(priv, fdir_filter); 2922 return ret; 2923 } 2924 2925 /** 2926 * Flush all filters. 2927 * 2928 * @param priv 2929 * Private structure. 2930 */ 2931 static void 2932 priv_fdir_filter_flush(struct priv *priv) 2933 { 2934 priv_flow_flush(priv, &priv->flows); 2935 } 2936 2937 /** 2938 * Get flow director information. 2939 * 2940 * @param priv 2941 * Private structure. 2942 * @param[out] fdir_info 2943 * Resulting flow director information. 2944 */ 2945 static void 2946 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) 2947 { 2948 struct rte_eth_fdir_masks *mask = 2949 &priv->dev->data->dev_conf.fdir_conf.mask; 2950 2951 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; 2952 fdir_info->guarant_spc = 0; 2953 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 2954 fdir_info->max_flexpayload = 0; 2955 fdir_info->flow_types_mask[0] = 0; 2956 fdir_info->flex_payload_unit = 0; 2957 fdir_info->max_flex_payload_segment_num = 0; 2958 fdir_info->flex_payload_limit = 0; 2959 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 2960 } 2961 2962 /** 2963 * Deal with flow director operations. 2964 * 2965 * @param priv 2966 * Pointer to private structure. 2967 * @param filter_op 2968 * Operation to perform. 2969 * @param arg 2970 * Pointer to operation-specific structure. 2971 * 2972 * @return 2973 * 0 on success, errno value on failure. 2974 */ 2975 static int 2976 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) 2977 { 2978 enum rte_fdir_mode fdir_mode = 2979 priv->dev->data->dev_conf.fdir_conf.mode; 2980 int ret = 0; 2981 2982 if (filter_op == RTE_ETH_FILTER_NOP) 2983 return 0; 2984 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 2985 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 2986 ERROR("%p: flow director mode %d not supported", 2987 (void *)priv, fdir_mode); 2988 return EINVAL; 2989 } 2990 switch (filter_op) { 2991 case RTE_ETH_FILTER_ADD: 2992 ret = priv_fdir_filter_add(priv, arg); 2993 break; 2994 case RTE_ETH_FILTER_UPDATE: 2995 ret = priv_fdir_filter_update(priv, arg); 2996 break; 2997 case RTE_ETH_FILTER_DELETE: 2998 ret = priv_fdir_filter_delete(priv, arg); 2999 break; 3000 case RTE_ETH_FILTER_FLUSH: 3001 priv_fdir_filter_flush(priv); 3002 break; 3003 case RTE_ETH_FILTER_INFO: 3004 priv_fdir_info_get(priv, arg); 3005 break; 3006 default: 3007 DEBUG("%p: unknown operation %u", (void *)priv, 3008 filter_op); 3009 ret = EINVAL; 3010 break; 3011 } 3012 return ret; 3013 } 3014 3015 /** 3016 * Manage filter operations. 3017 * 3018 * @param dev 3019 * Pointer to Ethernet device structure. 3020 * @param filter_type 3021 * Filter type. 3022 * @param filter_op 3023 * Operation to perform. 3024 * @param arg 3025 * Pointer to operation-specific structure. 3026 * 3027 * @return 3028 * 0 on success, negative errno value on failure. 3029 */ 3030 int 3031 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 3032 enum rte_filter_type filter_type, 3033 enum rte_filter_op filter_op, 3034 void *arg) 3035 { 3036 int ret = EINVAL; 3037 struct priv *priv = dev->data->dev_private; 3038 3039 switch (filter_type) { 3040 case RTE_ETH_FILTER_GENERIC: 3041 if (filter_op != RTE_ETH_FILTER_GET) 3042 return -EINVAL; 3043 *(const void **)arg = &mlx5_flow_ops; 3044 return 0; 3045 case RTE_ETH_FILTER_FDIR: 3046 priv_lock(priv); 3047 ret = priv_fdir_ctrl_func(priv, filter_op, arg); 3048 priv_unlock(priv); 3049 break; 3050 default: 3051 ERROR("%p: filter type (%d) not supported", 3052 (void *)dev, filter_type); 3053 break; 3054 } 3055 return -ret; 3056 } 3057