1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2016 6WIND S.A. 5 * Copyright 2016 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/queue.h> 35 #include <string.h> 36 37 /* Verbs header. */ 38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 39 #ifdef PEDANTIC 40 #pragma GCC diagnostic ignored "-Wpedantic" 41 #endif 42 #include <infiniband/verbs.h> 43 #ifdef PEDANTIC 44 #pragma GCC diagnostic error "-Wpedantic" 45 #endif 46 47 #include <rte_ethdev.h> 48 #include <rte_flow.h> 49 #include <rte_flow_driver.h> 50 #include <rte_malloc.h> 51 52 #include "mlx5.h" 53 #include "mlx5_prm.h" 54 55 /* Define minimal priority for control plane flows. */ 56 #define MLX5_CTRL_FLOW_PRIORITY 4 57 58 /* Internet Protocol versions. */ 59 #define MLX5_IPV4 4 60 #define MLX5_IPV6 6 61 62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 63 struct ibv_counter_set_init_attr { 64 int dummy; 65 }; 66 struct ibv_flow_spec_counter_action { 67 int dummy; 68 }; 69 struct ibv_counter_set { 70 int dummy; 71 }; 72 73 static inline int 74 ibv_destroy_counter_set(struct ibv_counter_set *cs) 75 { 76 (void)cs; 77 return -ENOTSUP; 78 } 79 #endif 80 81 /* Dev ops structure defined in mlx5.c */ 82 extern const struct eth_dev_ops mlx5_dev_ops; 83 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 84 85 static int 86 mlx5_flow_create_eth(const struct rte_flow_item *item, 87 const void *default_mask, 88 void *data); 89 90 static int 91 mlx5_flow_create_vlan(const struct rte_flow_item *item, 92 const void *default_mask, 93 void *data); 94 95 static int 96 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 97 const void *default_mask, 98 void *data); 99 100 static int 101 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 102 const void *default_mask, 103 void *data); 104 105 static int 106 mlx5_flow_create_udp(const struct rte_flow_item *item, 107 const void *default_mask, 108 void *data); 109 110 static int 111 mlx5_flow_create_tcp(const struct rte_flow_item *item, 112 const void *default_mask, 113 void *data); 114 115 static int 116 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 117 const void *default_mask, 118 void *data); 119 120 struct mlx5_flow_parse; 121 122 static void 123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 124 unsigned int size); 125 126 static int 127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); 128 129 static int 130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser); 131 132 /* Hash RX queue types. */ 133 enum hash_rxq_type { 134 HASH_RXQ_TCPV4, 135 HASH_RXQ_UDPV4, 136 HASH_RXQ_IPV4, 137 HASH_RXQ_TCPV6, 138 HASH_RXQ_UDPV6, 139 HASH_RXQ_IPV6, 140 HASH_RXQ_ETH, 141 }; 142 143 /* Initialization data for hash RX queue. */ 144 struct hash_rxq_init { 145 uint64_t hash_fields; /* Fields that participate in the hash. */ 146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ 147 unsigned int flow_priority; /* Flow priority to use. */ 148 unsigned int ip_version; /* Internet protocol. */ 149 }; 150 151 /* Initialization data for hash RX queues. */ 152 const struct hash_rxq_init hash_rxq_init[] = { 153 [HASH_RXQ_TCPV4] = { 154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 155 IBV_RX_HASH_DST_IPV4 | 156 IBV_RX_HASH_SRC_PORT_TCP | 157 IBV_RX_HASH_DST_PORT_TCP), 158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, 159 .flow_priority = 0, 160 .ip_version = MLX5_IPV4, 161 }, 162 [HASH_RXQ_UDPV4] = { 163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 164 IBV_RX_HASH_DST_IPV4 | 165 IBV_RX_HASH_SRC_PORT_UDP | 166 IBV_RX_HASH_DST_PORT_UDP), 167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, 168 .flow_priority = 0, 169 .ip_version = MLX5_IPV4, 170 }, 171 [HASH_RXQ_IPV4] = { 172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 173 IBV_RX_HASH_DST_IPV4), 174 .dpdk_rss_hf = (ETH_RSS_IPV4 | 175 ETH_RSS_FRAG_IPV4), 176 .flow_priority = 1, 177 .ip_version = MLX5_IPV4, 178 }, 179 [HASH_RXQ_TCPV6] = { 180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 181 IBV_RX_HASH_DST_IPV6 | 182 IBV_RX_HASH_SRC_PORT_TCP | 183 IBV_RX_HASH_DST_PORT_TCP), 184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, 185 .flow_priority = 0, 186 .ip_version = MLX5_IPV6, 187 }, 188 [HASH_RXQ_UDPV6] = { 189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 190 IBV_RX_HASH_DST_IPV6 | 191 IBV_RX_HASH_SRC_PORT_UDP | 192 IBV_RX_HASH_DST_PORT_UDP), 193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, 194 .flow_priority = 0, 195 .ip_version = MLX5_IPV6, 196 }, 197 [HASH_RXQ_IPV6] = { 198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 199 IBV_RX_HASH_DST_IPV6), 200 .dpdk_rss_hf = (ETH_RSS_IPV6 | 201 ETH_RSS_FRAG_IPV6), 202 .flow_priority = 1, 203 .ip_version = MLX5_IPV6, 204 }, 205 [HASH_RXQ_ETH] = { 206 .hash_fields = 0, 207 .dpdk_rss_hf = 0, 208 .flow_priority = 2, 209 }, 210 }; 211 212 /* Number of entries in hash_rxq_init[]. */ 213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); 214 215 /** Structure for holding counter stats. */ 216 struct mlx5_flow_counter_stats { 217 uint64_t hits; /**< Number of packets matched by the rule. */ 218 uint64_t bytes; /**< Number of bytes matched by the rule. */ 219 }; 220 221 /** Structure for Drop queue. */ 222 struct mlx5_hrxq_drop { 223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 224 struct ibv_qp *qp; /**< Verbs queue pair. */ 225 struct ibv_wq *wq; /**< Verbs work queue. */ 226 struct ibv_cq *cq; /**< Verbs completion queue. */ 227 }; 228 229 /* Flows structures. */ 230 struct mlx5_flow { 231 uint64_t hash_fields; /**< Fields that participate in the hash. */ 232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 233 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */ 235 }; 236 237 /* Drop flows structures. */ 238 struct mlx5_flow_drop { 239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 240 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 241 }; 242 243 struct rte_flow { 244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ 245 uint32_t mark:1; /**< Set if the flow is marked. */ 246 uint32_t drop:1; /**< Drop queue. */ 247 uint16_t queues_n; /**< Number of entries in queue[]. */ 248 uint16_t (*queues)[]; /**< Queues indexes to use. */ 249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 250 uint8_t rss_key[40]; /**< copy of the RSS key. */ 251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */ 252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */ 253 union { 254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)]; 255 /**< Flow with Rx queue. */ 256 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */ 257 }; 258 }; 259 260 /** Static initializer for items. */ 261 #define ITEMS(...) \ 262 (const enum rte_flow_item_type []){ \ 263 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ 264 } 265 266 /** Structure to generate a simple graph of layers supported by the NIC. */ 267 struct mlx5_flow_items { 268 /** List of possible actions for these items. */ 269 const enum rte_flow_action_type *const actions; 270 /** Bit-masks corresponding to the possibilities for the item. */ 271 const void *mask; 272 /** 273 * Default bit-masks to use when item->mask is not provided. When 274 * \default_mask is also NULL, the full supported bit-mask (\mask) is 275 * used instead. 276 */ 277 const void *default_mask; 278 /** Bit-masks size in bytes. */ 279 const unsigned int mask_sz; 280 /** 281 * Conversion function from rte_flow to NIC specific flow. 282 * 283 * @param item 284 * rte_flow item to convert. 285 * @param default_mask 286 * Default bit-masks to use when item->mask is not provided. 287 * @param data 288 * Internal structure to store the conversion. 289 * 290 * @return 291 * 0 on success, negative value otherwise. 292 */ 293 int (*convert)(const struct rte_flow_item *item, 294 const void *default_mask, 295 void *data); 296 /** Size in bytes of the destination structure. */ 297 const unsigned int dst_sz; 298 /** List of possible following items. */ 299 const enum rte_flow_item_type *const items; 300 }; 301 302 /** Valid action for this PMD. */ 303 static const enum rte_flow_action_type valid_actions[] = { 304 RTE_FLOW_ACTION_TYPE_DROP, 305 RTE_FLOW_ACTION_TYPE_QUEUE, 306 RTE_FLOW_ACTION_TYPE_MARK, 307 RTE_FLOW_ACTION_TYPE_FLAG, 308 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 309 RTE_FLOW_ACTION_TYPE_COUNT, 310 #endif 311 RTE_FLOW_ACTION_TYPE_END, 312 }; 313 314 /** Graph of supported items and associated actions. */ 315 static const struct mlx5_flow_items mlx5_flow_items[] = { 316 [RTE_FLOW_ITEM_TYPE_END] = { 317 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, 318 RTE_FLOW_ITEM_TYPE_VXLAN), 319 }, 320 [RTE_FLOW_ITEM_TYPE_ETH] = { 321 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, 322 RTE_FLOW_ITEM_TYPE_IPV4, 323 RTE_FLOW_ITEM_TYPE_IPV6), 324 .actions = valid_actions, 325 .mask = &(const struct rte_flow_item_eth){ 326 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 327 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 328 .type = -1, 329 }, 330 .default_mask = &rte_flow_item_eth_mask, 331 .mask_sz = sizeof(struct rte_flow_item_eth), 332 .convert = mlx5_flow_create_eth, 333 .dst_sz = sizeof(struct ibv_flow_spec_eth), 334 }, 335 [RTE_FLOW_ITEM_TYPE_VLAN] = { 336 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, 337 RTE_FLOW_ITEM_TYPE_IPV6), 338 .actions = valid_actions, 339 .mask = &(const struct rte_flow_item_vlan){ 340 .tci = -1, 341 }, 342 .default_mask = &rte_flow_item_vlan_mask, 343 .mask_sz = sizeof(struct rte_flow_item_vlan), 344 .convert = mlx5_flow_create_vlan, 345 .dst_sz = 0, 346 }, 347 [RTE_FLOW_ITEM_TYPE_IPV4] = { 348 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 349 RTE_FLOW_ITEM_TYPE_TCP), 350 .actions = valid_actions, 351 .mask = &(const struct rte_flow_item_ipv4){ 352 .hdr = { 353 .src_addr = -1, 354 .dst_addr = -1, 355 .type_of_service = -1, 356 .next_proto_id = -1, 357 }, 358 }, 359 .default_mask = &rte_flow_item_ipv4_mask, 360 .mask_sz = sizeof(struct rte_flow_item_ipv4), 361 .convert = mlx5_flow_create_ipv4, 362 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), 363 }, 364 [RTE_FLOW_ITEM_TYPE_IPV6] = { 365 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 366 RTE_FLOW_ITEM_TYPE_TCP), 367 .actions = valid_actions, 368 .mask = &(const struct rte_flow_item_ipv6){ 369 .hdr = { 370 .src_addr = { 371 0xff, 0xff, 0xff, 0xff, 372 0xff, 0xff, 0xff, 0xff, 373 0xff, 0xff, 0xff, 0xff, 374 0xff, 0xff, 0xff, 0xff, 375 }, 376 .dst_addr = { 377 0xff, 0xff, 0xff, 0xff, 378 0xff, 0xff, 0xff, 0xff, 379 0xff, 0xff, 0xff, 0xff, 380 0xff, 0xff, 0xff, 0xff, 381 }, 382 .vtc_flow = -1, 383 .proto = -1, 384 .hop_limits = -1, 385 }, 386 }, 387 .default_mask = &rte_flow_item_ipv6_mask, 388 .mask_sz = sizeof(struct rte_flow_item_ipv6), 389 .convert = mlx5_flow_create_ipv6, 390 .dst_sz = sizeof(struct ibv_flow_spec_ipv6), 391 }, 392 [RTE_FLOW_ITEM_TYPE_UDP] = { 393 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), 394 .actions = valid_actions, 395 .mask = &(const struct rte_flow_item_udp){ 396 .hdr = { 397 .src_port = -1, 398 .dst_port = -1, 399 }, 400 }, 401 .default_mask = &rte_flow_item_udp_mask, 402 .mask_sz = sizeof(struct rte_flow_item_udp), 403 .convert = mlx5_flow_create_udp, 404 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 405 }, 406 [RTE_FLOW_ITEM_TYPE_TCP] = { 407 .actions = valid_actions, 408 .mask = &(const struct rte_flow_item_tcp){ 409 .hdr = { 410 .src_port = -1, 411 .dst_port = -1, 412 }, 413 }, 414 .default_mask = &rte_flow_item_tcp_mask, 415 .mask_sz = sizeof(struct rte_flow_item_tcp), 416 .convert = mlx5_flow_create_tcp, 417 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 418 }, 419 [RTE_FLOW_ITEM_TYPE_VXLAN] = { 420 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), 421 .actions = valid_actions, 422 .mask = &(const struct rte_flow_item_vxlan){ 423 .vni = "\xff\xff\xff", 424 }, 425 .default_mask = &rte_flow_item_vxlan_mask, 426 .mask_sz = sizeof(struct rte_flow_item_vxlan), 427 .convert = mlx5_flow_create_vxlan, 428 .dst_sz = sizeof(struct ibv_flow_spec_tunnel), 429 }, 430 }; 431 432 /** Structure to pass to the conversion function. */ 433 struct mlx5_flow_parse { 434 uint32_t inner; /**< Set once VXLAN is encountered. */ 435 uint32_t create:1; 436 /**< Whether resources should remain after a validate. */ 437 uint32_t drop:1; /**< Target is a drop queue. */ 438 uint32_t mark:1; /**< Mark is present in the flow. */ 439 uint32_t count:1; /**< Count is present in the flow. */ 440 uint32_t mark_id; /**< Mark identifier. */ 441 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ 442 uint16_t queues_n; /**< Number of entries in queue[]. */ 443 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 444 uint8_t rss_key[40]; /**< copy of the RSS key. */ 445 enum hash_rxq_type layer; /**< Last pattern layer detected. */ 446 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */ 447 union { 448 struct { 449 struct ibv_flow_attr *ibv_attr; 450 /**< Pointer to Verbs attributes. */ 451 unsigned int offset; 452 /**< Current position or total size of the attribute. */ 453 } queue[RTE_DIM(hash_rxq_init)]; 454 struct { 455 struct ibv_flow_attr *ibv_attr; 456 /**< Pointer to Verbs attributes. */ 457 unsigned int offset; 458 /**< Current position or total size of the attribute. */ 459 } drop_q; 460 }; 461 }; 462 463 static const struct rte_flow_ops mlx5_flow_ops = { 464 .validate = mlx5_flow_validate, 465 .create = mlx5_flow_create, 466 .destroy = mlx5_flow_destroy, 467 .flush = mlx5_flow_flush, 468 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 469 .query = mlx5_flow_query, 470 #else 471 .query = NULL, 472 #endif 473 .isolate = mlx5_flow_isolate, 474 }; 475 476 /* Convert FDIR request to Generic flow. */ 477 struct mlx5_fdir { 478 struct rte_flow_attr attr; 479 struct rte_flow_action actions[2]; 480 struct rte_flow_item items[4]; 481 struct rte_flow_item_eth l2; 482 struct rte_flow_item_eth l2_mask; 483 union { 484 struct rte_flow_item_ipv4 ipv4; 485 struct rte_flow_item_ipv6 ipv6; 486 } l3; 487 union { 488 struct rte_flow_item_udp udp; 489 struct rte_flow_item_tcp tcp; 490 } l4; 491 struct rte_flow_action_queue queue; 492 }; 493 494 /* Verbs specification header. */ 495 struct ibv_spec_header { 496 enum ibv_flow_spec_type type; 497 uint16_t size; 498 }; 499 500 /** 501 * Check support for a given item. 502 * 503 * @param item[in] 504 * Item specification. 505 * @param mask[in] 506 * Bit-masks covering supported fields to compare with spec, last and mask in 507 * \item. 508 * @param size 509 * Bit-Mask size in bytes. 510 * 511 * @return 512 * 0 on success. 513 */ 514 static int 515 mlx5_flow_item_validate(const struct rte_flow_item *item, 516 const uint8_t *mask, unsigned int size) 517 { 518 int ret = 0; 519 520 if (!item->spec && (item->mask || item->last)) 521 return -1; 522 if (item->spec && !item->mask) { 523 unsigned int i; 524 const uint8_t *spec = item->spec; 525 526 for (i = 0; i < size; ++i) 527 if ((spec[i] | mask[i]) != mask[i]) 528 return -1; 529 } 530 if (item->last && !item->mask) { 531 unsigned int i; 532 const uint8_t *spec = item->last; 533 534 for (i = 0; i < size; ++i) 535 if ((spec[i] | mask[i]) != mask[i]) 536 return -1; 537 } 538 if (item->mask) { 539 unsigned int i; 540 const uint8_t *spec = item->mask; 541 542 for (i = 0; i < size; ++i) 543 if ((spec[i] | mask[i]) != mask[i]) 544 return -1; 545 } 546 if (item->spec && item->last) { 547 uint8_t spec[size]; 548 uint8_t last[size]; 549 const uint8_t *apply = mask; 550 unsigned int i; 551 552 if (item->mask) 553 apply = item->mask; 554 for (i = 0; i < size; ++i) { 555 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; 556 last[i] = ((const uint8_t *)item->last)[i] & apply[i]; 557 } 558 ret = memcmp(spec, last, size); 559 } 560 return ret; 561 } 562 563 /** 564 * Copy the RSS configuration from the user ones. 565 * 566 * @param priv 567 * Pointer to private structure. 568 * @param parser 569 * Internal parser structure. 570 * @param rss_conf 571 * User RSS configuration to save. 572 * 573 * @return 574 * 0 on success, errno value on failure. 575 */ 576 static int 577 priv_flow_convert_rss_conf(struct priv *priv, 578 struct mlx5_flow_parse *parser, 579 const struct rte_eth_rss_conf *rss_conf) 580 { 581 const struct rte_eth_rss_conf *rss = 582 rss_conf ? rss_conf : &priv->rss_conf; 583 584 if (rss->rss_key_len > 40) 585 return EINVAL; 586 parser->rss_conf.rss_key_len = rss->rss_key_len; 587 parser->rss_conf.rss_hf = rss->rss_hf; 588 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len); 589 parser->rss_conf.rss_key = parser->rss_key; 590 return 0; 591 } 592 593 /** 594 * Extract attribute to the parser. 595 * 596 * @param priv 597 * Pointer to private structure. 598 * @param[in] attr 599 * Flow rule attributes. 600 * @param[out] error 601 * Perform verbose error reporting if not NULL. 602 * @param[in, out] parser 603 * Internal parser structure. 604 * 605 * @return 606 * 0 on success, a negative errno value otherwise and rte_errno is set. 607 */ 608 static int 609 priv_flow_convert_attributes(struct priv *priv, 610 const struct rte_flow_attr *attr, 611 struct rte_flow_error *error, 612 struct mlx5_flow_parse *parser) 613 { 614 (void)priv; 615 (void)parser; 616 if (attr->group) { 617 rte_flow_error_set(error, ENOTSUP, 618 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 619 NULL, 620 "groups are not supported"); 621 return -rte_errno; 622 } 623 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) { 624 rte_flow_error_set(error, ENOTSUP, 625 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 626 NULL, 627 "priorities are not supported"); 628 return -rte_errno; 629 } 630 if (attr->egress) { 631 rte_flow_error_set(error, ENOTSUP, 632 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 633 NULL, 634 "egress is not supported"); 635 return -rte_errno; 636 } 637 if (!attr->ingress) { 638 rte_flow_error_set(error, ENOTSUP, 639 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 640 NULL, 641 "only ingress is supported"); 642 return -rte_errno; 643 } 644 return 0; 645 } 646 647 /** 648 * Extract actions request to the parser. 649 * 650 * @param priv 651 * Pointer to private structure. 652 * @param[in] actions 653 * Associated actions (list terminated by the END action). 654 * @param[out] error 655 * Perform verbose error reporting if not NULL. 656 * @param[in, out] parser 657 * Internal parser structure. 658 * 659 * @return 660 * 0 on success, a negative errno value otherwise and rte_errno is set. 661 */ 662 static int 663 priv_flow_convert_actions(struct priv *priv, 664 const struct rte_flow_action actions[], 665 struct rte_flow_error *error, 666 struct mlx5_flow_parse *parser) 667 { 668 /* 669 * Add default RSS configuration necessary for Verbs to create QP even 670 * if no RSS is necessary. 671 */ 672 priv_flow_convert_rss_conf(priv, parser, 673 (const struct rte_eth_rss_conf *) 674 &priv->rss_conf); 675 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { 676 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { 677 continue; 678 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { 679 parser->drop = 1; 680 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 681 const struct rte_flow_action_queue *queue = 682 (const struct rte_flow_action_queue *) 683 actions->conf; 684 uint16_t n; 685 uint16_t found = 0; 686 687 if (!queue || (queue->index > (priv->rxqs_n - 1))) 688 goto exit_action_not_supported; 689 for (n = 0; n < parser->queues_n; ++n) { 690 if (parser->queues[n] == queue->index) { 691 found = 1; 692 break; 693 } 694 } 695 if (parser->queues_n > 1 && !found) { 696 rte_flow_error_set(error, ENOTSUP, 697 RTE_FLOW_ERROR_TYPE_ACTION, 698 actions, 699 "queue action not in RSS queues"); 700 return -rte_errno; 701 } 702 if (!found) { 703 parser->queues_n = 1; 704 parser->queues[0] = queue->index; 705 } 706 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { 707 const struct rte_flow_action_rss *rss = 708 (const struct rte_flow_action_rss *) 709 actions->conf; 710 uint16_t n; 711 712 if (!rss || !rss->num) { 713 rte_flow_error_set(error, EINVAL, 714 RTE_FLOW_ERROR_TYPE_ACTION, 715 actions, 716 "no valid queues"); 717 return -rte_errno; 718 } 719 if (parser->queues_n == 1) { 720 uint16_t found = 0; 721 722 assert(parser->queues_n); 723 for (n = 0; n < rss->num; ++n) { 724 if (parser->queues[0] == 725 rss->queue[n]) { 726 found = 1; 727 break; 728 } 729 } 730 if (!found) { 731 rte_flow_error_set(error, ENOTSUP, 732 RTE_FLOW_ERROR_TYPE_ACTION, 733 actions, 734 "queue action not in RSS" 735 " queues"); 736 return -rte_errno; 737 } 738 } 739 for (n = 0; n < rss->num; ++n) { 740 if (rss->queue[n] >= priv->rxqs_n) { 741 rte_flow_error_set(error, EINVAL, 742 RTE_FLOW_ERROR_TYPE_ACTION, 743 actions, 744 "queue id > number of" 745 " queues"); 746 return -rte_errno; 747 } 748 } 749 for (n = 0; n < rss->num; ++n) 750 parser->queues[n] = rss->queue[n]; 751 parser->queues_n = rss->num; 752 if (priv_flow_convert_rss_conf(priv, parser, 753 rss->rss_conf)) { 754 rte_flow_error_set(error, EINVAL, 755 RTE_FLOW_ERROR_TYPE_ACTION, 756 actions, 757 "wrong RSS configuration"); 758 return -rte_errno; 759 } 760 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { 761 const struct rte_flow_action_mark *mark = 762 (const struct rte_flow_action_mark *) 763 actions->conf; 764 765 if (!mark) { 766 rte_flow_error_set(error, EINVAL, 767 RTE_FLOW_ERROR_TYPE_ACTION, 768 actions, 769 "mark must be defined"); 770 return -rte_errno; 771 } else if (mark->id >= MLX5_FLOW_MARK_MAX) { 772 rte_flow_error_set(error, ENOTSUP, 773 RTE_FLOW_ERROR_TYPE_ACTION, 774 actions, 775 "mark must be between 0" 776 " and 16777199"); 777 return -rte_errno; 778 } 779 parser->mark = 1; 780 parser->mark_id = mark->id; 781 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { 782 parser->mark = 1; 783 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && 784 priv->counter_set_supported) { 785 parser->count = 1; 786 } else { 787 goto exit_action_not_supported; 788 } 789 } 790 if (parser->drop && parser->mark) 791 parser->mark = 0; 792 if (!parser->queues_n && !parser->drop) { 793 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, 794 NULL, "no valid action"); 795 return -rte_errno; 796 } 797 return 0; 798 exit_action_not_supported: 799 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 800 actions, "action not supported"); 801 return -rte_errno; 802 } 803 804 /** 805 * Validate items. 806 * 807 * @param priv 808 * Pointer to private structure. 809 * @param[in] items 810 * Pattern specification (list terminated by the END pattern item). 811 * @param[out] error 812 * Perform verbose error reporting if not NULL. 813 * @param[in, out] parser 814 * Internal parser structure. 815 * 816 * @return 817 * 0 on success, a negative errno value otherwise and rte_errno is set. 818 */ 819 static int 820 priv_flow_convert_items_validate(struct priv *priv, 821 const struct rte_flow_item items[], 822 struct rte_flow_error *error, 823 struct mlx5_flow_parse *parser) 824 { 825 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 826 unsigned int i; 827 828 (void)priv; 829 /* Initialise the offsets to start after verbs attribute. */ 830 if (parser->drop) { 831 parser->drop_q.offset = sizeof(struct ibv_flow_attr); 832 } else { 833 for (i = 0; i != hash_rxq_init_n; ++i) 834 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 835 } 836 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 837 const struct mlx5_flow_items *token = NULL; 838 unsigned int n; 839 int err; 840 841 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 842 continue; 843 for (i = 0; 844 cur_item->items && 845 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; 846 ++i) { 847 if (cur_item->items[i] == items->type) { 848 token = &mlx5_flow_items[items->type]; 849 break; 850 } 851 } 852 if (!token) 853 goto exit_item_not_supported; 854 cur_item = token; 855 err = mlx5_flow_item_validate(items, 856 (const uint8_t *)cur_item->mask, 857 cur_item->mask_sz); 858 if (err) 859 goto exit_item_not_supported; 860 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 861 if (parser->inner) { 862 rte_flow_error_set(error, ENOTSUP, 863 RTE_FLOW_ERROR_TYPE_ITEM, 864 items, 865 "cannot recognize multiple" 866 " VXLAN encapsulations"); 867 return -rte_errno; 868 } 869 parser->inner = IBV_FLOW_SPEC_INNER; 870 } 871 if (parser->drop) { 872 parser->drop_q.offset += cur_item->dst_sz; 873 } else if (parser->queues_n == 1) { 874 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; 875 } else { 876 for (n = 0; n != hash_rxq_init_n; ++n) 877 parser->queue[n].offset += cur_item->dst_sz; 878 } 879 } 880 if (parser->mark) { 881 for (i = 0; i != hash_rxq_init_n; ++i) 882 parser->queue[i].offset += 883 sizeof(struct ibv_flow_spec_action_tag); 884 } 885 if (parser->count) { 886 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 887 888 if (parser->drop) { 889 parser->drop_q.offset += size; 890 } else { 891 for (i = 0; i != hash_rxq_init_n; ++i) 892 parser->queue[i].offset += size; 893 } 894 } 895 return 0; 896 exit_item_not_supported: 897 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 898 items, "item not supported"); 899 return -rte_errno; 900 } 901 902 /** 903 * Allocate memory space to store verbs flow attributes. 904 * 905 * @param priv 906 * Pointer to private structure. 907 * @param[in] priority 908 * Flow priority. 909 * @param[in] size 910 * Amount of byte to allocate. 911 * @param[out] error 912 * Perform verbose error reporting if not NULL. 913 * 914 * @return 915 * A verbs flow attribute on success, NULL otherwise. 916 */ 917 static struct ibv_flow_attr* 918 priv_flow_convert_allocate(struct priv *priv, 919 unsigned int priority, 920 unsigned int size, 921 struct rte_flow_error *error) 922 { 923 struct ibv_flow_attr *ibv_attr; 924 925 (void)priv; 926 ibv_attr = rte_calloc(__func__, 1, size, 0); 927 if (!ibv_attr) { 928 rte_flow_error_set(error, ENOMEM, 929 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 930 NULL, 931 "cannot allocate verbs spec attributes."); 932 return NULL; 933 } 934 ibv_attr->priority = priority; 935 return ibv_attr; 936 } 937 938 /** 939 * Finalise verbs flow attributes. 940 * 941 * @param priv 942 * Pointer to private structure. 943 * @param[in, out] parser 944 * Internal parser structure. 945 */ 946 static void 947 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) 948 { 949 const unsigned int ipv4 = 950 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; 951 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6; 952 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 953 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4; 954 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4; 955 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 956 unsigned int i; 957 958 (void)priv; 959 if (parser->layer == HASH_RXQ_ETH) { 960 goto fill; 961 } else { 962 /* 963 * This layer becomes useless as the pattern define under 964 * layers. 965 */ 966 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr); 967 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 968 } 969 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */ 970 for (i = ohmin; i != (ohmax + 1); ++i) { 971 if (!parser->queue[i].ibv_attr) 972 continue; 973 rte_free(parser->queue[i].ibv_attr); 974 parser->queue[i].ibv_attr = NULL; 975 } 976 /* Remove impossible flow according to the RSS configuration. */ 977 if (hash_rxq_init[parser->layer].dpdk_rss_hf & 978 parser->rss_conf.rss_hf) { 979 /* Remove any other flow. */ 980 for (i = hmin; i != (hmax + 1); ++i) { 981 if ((i == parser->layer) || 982 (!parser->queue[i].ibv_attr)) 983 continue; 984 rte_free(parser->queue[i].ibv_attr); 985 parser->queue[i].ibv_attr = NULL; 986 } 987 } else if (!parser->queue[ip].ibv_attr) { 988 /* no RSS possible with the current configuration. */ 989 parser->queues_n = 1; 990 return; 991 } 992 fill: 993 /* 994 * Fill missing layers in verbs specifications, or compute the correct 995 * offset to allocate the memory space for the attributes and 996 * specifications. 997 */ 998 for (i = 0; i != hash_rxq_init_n - 1; ++i) { 999 union { 1000 struct ibv_flow_spec_ipv4_ext ipv4; 1001 struct ibv_flow_spec_ipv6 ipv6; 1002 struct ibv_flow_spec_tcp_udp udp_tcp; 1003 } specs; 1004 void *dst; 1005 uint16_t size; 1006 1007 if (i == parser->layer) 1008 continue; 1009 if (parser->layer == HASH_RXQ_ETH) { 1010 if (hash_rxq_init[i].ip_version == MLX5_IPV4) { 1011 size = sizeof(struct ibv_flow_spec_ipv4_ext); 1012 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){ 1013 .type = IBV_FLOW_SPEC_IPV4_EXT, 1014 .size = size, 1015 }; 1016 } else { 1017 size = sizeof(struct ibv_flow_spec_ipv6); 1018 specs.ipv6 = (struct ibv_flow_spec_ipv6){ 1019 .type = IBV_FLOW_SPEC_IPV6, 1020 .size = size, 1021 }; 1022 } 1023 if (parser->queue[i].ibv_attr) { 1024 dst = (void *)((uintptr_t) 1025 parser->queue[i].ibv_attr + 1026 parser->queue[i].offset); 1027 memcpy(dst, &specs, size); 1028 ++parser->queue[i].ibv_attr->num_of_specs; 1029 } 1030 parser->queue[i].offset += size; 1031 } 1032 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) || 1033 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) { 1034 size = sizeof(struct ibv_flow_spec_tcp_udp); 1035 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) { 1036 .type = ((i == HASH_RXQ_UDPV4 || 1037 i == HASH_RXQ_UDPV6) ? 1038 IBV_FLOW_SPEC_UDP : 1039 IBV_FLOW_SPEC_TCP), 1040 .size = size, 1041 }; 1042 if (parser->queue[i].ibv_attr) { 1043 dst = (void *)((uintptr_t) 1044 parser->queue[i].ibv_attr + 1045 parser->queue[i].offset); 1046 memcpy(dst, &specs, size); 1047 ++parser->queue[i].ibv_attr->num_of_specs; 1048 } 1049 parser->queue[i].offset += size; 1050 } 1051 } 1052 } 1053 1054 /** 1055 * Validate and convert a flow supported by the NIC. 1056 * 1057 * @param priv 1058 * Pointer to private structure. 1059 * @param[in] attr 1060 * Flow rule attributes. 1061 * @param[in] pattern 1062 * Pattern specification (list terminated by the END pattern item). 1063 * @param[in] actions 1064 * Associated actions (list terminated by the END action). 1065 * @param[out] error 1066 * Perform verbose error reporting if not NULL. 1067 * @param[in, out] parser 1068 * Internal parser structure. 1069 * 1070 * @return 1071 * 0 on success, a negative errno value otherwise and rte_errno is set. 1072 */ 1073 static int 1074 priv_flow_convert(struct priv *priv, 1075 const struct rte_flow_attr *attr, 1076 const struct rte_flow_item items[], 1077 const struct rte_flow_action actions[], 1078 struct rte_flow_error *error, 1079 struct mlx5_flow_parse *parser) 1080 { 1081 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 1082 unsigned int i; 1083 int ret; 1084 1085 /* First step. Validate the attributes, items and actions. */ 1086 *parser = (struct mlx5_flow_parse){ 1087 .create = parser->create, 1088 .layer = HASH_RXQ_ETH, 1089 .mark_id = MLX5_FLOW_MARK_DEFAULT, 1090 }; 1091 ret = priv_flow_convert_attributes(priv, attr, error, parser); 1092 if (ret) 1093 return ret; 1094 ret = priv_flow_convert_actions(priv, actions, error, parser); 1095 if (ret) 1096 return ret; 1097 ret = priv_flow_convert_items_validate(priv, items, error, parser); 1098 if (ret) 1099 return ret; 1100 priv_flow_convert_finalise(priv, parser); 1101 /* 1102 * Second step. 1103 * Allocate the memory space to store verbs specifications. 1104 */ 1105 if (parser->drop) { 1106 parser->drop_q.ibv_attr = 1107 priv_flow_convert_allocate(priv, attr->priority, 1108 parser->drop_q.offset, 1109 error); 1110 if (!parser->drop_q.ibv_attr) 1111 return ENOMEM; 1112 parser->drop_q.offset = sizeof(struct ibv_flow_attr); 1113 } else if (parser->queues_n == 1) { 1114 unsigned int priority = 1115 attr->priority + 1116 hash_rxq_init[HASH_RXQ_ETH].flow_priority; 1117 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; 1118 1119 parser->queue[HASH_RXQ_ETH].ibv_attr = 1120 priv_flow_convert_allocate(priv, priority, 1121 offset, error); 1122 if (!parser->queue[HASH_RXQ_ETH].ibv_attr) 1123 return ENOMEM; 1124 parser->queue[HASH_RXQ_ETH].offset = 1125 sizeof(struct ibv_flow_attr); 1126 } else { 1127 for (i = 0; i != hash_rxq_init_n; ++i) { 1128 unsigned int priority = 1129 attr->priority + 1130 hash_rxq_init[i].flow_priority; 1131 unsigned int offset; 1132 1133 if (!(parser->rss_conf.rss_hf & 1134 hash_rxq_init[i].dpdk_rss_hf) && 1135 (i != HASH_RXQ_ETH)) 1136 continue; 1137 offset = parser->queue[i].offset; 1138 parser->queue[i].ibv_attr = 1139 priv_flow_convert_allocate(priv, priority, 1140 offset, error); 1141 if (!parser->queue[i].ibv_attr) 1142 goto exit_enomem; 1143 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 1144 } 1145 } 1146 /* Third step. Conversion parse, fill the specifications. */ 1147 parser->inner = 0; 1148 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 1149 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 1150 continue; 1151 cur_item = &mlx5_flow_items[items->type]; 1152 ret = cur_item->convert(items, 1153 (cur_item->default_mask ? 1154 cur_item->default_mask : 1155 cur_item->mask), 1156 parser); 1157 if (ret) { 1158 rte_flow_error_set(error, ret, 1159 RTE_FLOW_ERROR_TYPE_ITEM, 1160 items, "item not supported"); 1161 goto exit_free; 1162 } 1163 } 1164 if (parser->mark) 1165 mlx5_flow_create_flag_mark(parser, parser->mark_id); 1166 if (parser->count && parser->create) { 1167 mlx5_flow_create_count(priv, parser); 1168 if (!parser->cs) 1169 goto exit_count_error; 1170 } 1171 /* 1172 * Last step. Complete missing specification to reach the RSS 1173 * configuration. 1174 */ 1175 if (parser->drop) { 1176 /* 1177 * Drop queue priority needs to be adjusted to 1178 * their most specific layer priority. 1179 */ 1180 parser->drop_q.ibv_attr->priority = 1181 attr->priority + 1182 hash_rxq_init[parser->layer].flow_priority; 1183 } else if (parser->queues_n > 1) { 1184 priv_flow_convert_finalise(priv, parser); 1185 } else { 1186 /* 1187 * Action queue have their priority overridden with 1188 * Ethernet priority, this priority needs to be adjusted to 1189 * their most specific layer priority. 1190 */ 1191 parser->queue[HASH_RXQ_ETH].ibv_attr->priority = 1192 attr->priority + 1193 hash_rxq_init[parser->layer].flow_priority; 1194 } 1195 exit_free: 1196 /* Only verification is expected, all resources should be released. */ 1197 if (!parser->create) { 1198 if (parser->drop) { 1199 rte_free(parser->drop_q.ibv_attr); 1200 parser->drop_q.ibv_attr = NULL; 1201 } 1202 for (i = 0; i != hash_rxq_init_n; ++i) { 1203 if (parser->queue[i].ibv_attr) { 1204 rte_free(parser->queue[i].ibv_attr); 1205 parser->queue[i].ibv_attr = NULL; 1206 } 1207 } 1208 } 1209 return ret; 1210 exit_enomem: 1211 for (i = 0; i != hash_rxq_init_n; ++i) { 1212 if (parser->queue[i].ibv_attr) { 1213 rte_free(parser->queue[i].ibv_attr); 1214 parser->queue[i].ibv_attr = NULL; 1215 } 1216 } 1217 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1218 NULL, "cannot allocate verbs spec attributes."); 1219 return ret; 1220 exit_count_error: 1221 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1222 NULL, "cannot create counter."); 1223 return rte_errno; 1224 } 1225 1226 /** 1227 * Copy the specification created into the flow. 1228 * 1229 * @param parser 1230 * Internal parser structure. 1231 * @param src 1232 * Create specification. 1233 * @param size 1234 * Size in bytes of the specification to copy. 1235 */ 1236 static void 1237 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 1238 unsigned int size) 1239 { 1240 unsigned int i; 1241 void *dst; 1242 1243 if (parser->drop) { 1244 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1245 parser->drop_q.offset); 1246 memcpy(dst, src, size); 1247 ++parser->drop_q.ibv_attr->num_of_specs; 1248 parser->drop_q.offset += size; 1249 return; 1250 } 1251 for (i = 0; i != hash_rxq_init_n; ++i) { 1252 if (!parser->queue[i].ibv_attr) 1253 continue; 1254 /* Specification must be the same l3 type or none. */ 1255 if (parser->layer == HASH_RXQ_ETH || 1256 (hash_rxq_init[parser->layer].ip_version == 1257 hash_rxq_init[i].ip_version) || 1258 (hash_rxq_init[i].ip_version == 0)) { 1259 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1260 parser->queue[i].offset); 1261 memcpy(dst, src, size); 1262 ++parser->queue[i].ibv_attr->num_of_specs; 1263 parser->queue[i].offset += size; 1264 } 1265 } 1266 } 1267 1268 /** 1269 * Convert Ethernet item to Verbs specification. 1270 * 1271 * @param item[in] 1272 * Item specification. 1273 * @param default_mask[in] 1274 * Default bit-masks to use when item->mask is not provided. 1275 * @param data[in, out] 1276 * User structure. 1277 */ 1278 static int 1279 mlx5_flow_create_eth(const struct rte_flow_item *item, 1280 const void *default_mask, 1281 void *data) 1282 { 1283 const struct rte_flow_item_eth *spec = item->spec; 1284 const struct rte_flow_item_eth *mask = item->mask; 1285 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1286 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1287 struct ibv_flow_spec_eth eth = { 1288 .type = parser->inner | IBV_FLOW_SPEC_ETH, 1289 .size = eth_size, 1290 }; 1291 1292 /* Don't update layer for the inner pattern. */ 1293 if (!parser->inner) 1294 parser->layer = HASH_RXQ_ETH; 1295 if (spec) { 1296 unsigned int i; 1297 1298 if (!mask) 1299 mask = default_mask; 1300 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); 1301 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); 1302 eth.val.ether_type = spec->type; 1303 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); 1304 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); 1305 eth.mask.ether_type = mask->type; 1306 /* Remove unwanted bits from values. */ 1307 for (i = 0; i < ETHER_ADDR_LEN; ++i) { 1308 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 1309 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 1310 } 1311 eth.val.ether_type &= eth.mask.ether_type; 1312 } 1313 mlx5_flow_create_copy(parser, ð, eth_size); 1314 return 0; 1315 } 1316 1317 /** 1318 * Convert VLAN item to Verbs specification. 1319 * 1320 * @param item[in] 1321 * Item specification. 1322 * @param default_mask[in] 1323 * Default bit-masks to use when item->mask is not provided. 1324 * @param data[in, out] 1325 * User structure. 1326 */ 1327 static int 1328 mlx5_flow_create_vlan(const struct rte_flow_item *item, 1329 const void *default_mask, 1330 void *data) 1331 { 1332 const struct rte_flow_item_vlan *spec = item->spec; 1333 const struct rte_flow_item_vlan *mask = item->mask; 1334 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1335 struct ibv_flow_spec_eth *eth; 1336 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1337 1338 if (spec) { 1339 unsigned int i; 1340 if (!mask) 1341 mask = default_mask; 1342 1343 if (parser->drop) { 1344 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1345 parser->drop_q.offset - eth_size); 1346 eth->val.vlan_tag = spec->tci; 1347 eth->mask.vlan_tag = mask->tci; 1348 eth->val.vlan_tag &= eth->mask.vlan_tag; 1349 return 0; 1350 } 1351 for (i = 0; i != hash_rxq_init_n; ++i) { 1352 if (!parser->queue[i].ibv_attr) 1353 continue; 1354 1355 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1356 parser->queue[i].offset - eth_size); 1357 eth->val.vlan_tag = spec->tci; 1358 eth->mask.vlan_tag = mask->tci; 1359 eth->val.vlan_tag &= eth->mask.vlan_tag; 1360 } 1361 } 1362 return 0; 1363 } 1364 1365 /** 1366 * Convert IPv4 item to Verbs specification. 1367 * 1368 * @param item[in] 1369 * Item specification. 1370 * @param default_mask[in] 1371 * Default bit-masks to use when item->mask is not provided. 1372 * @param data[in, out] 1373 * User structure. 1374 */ 1375 static int 1376 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 1377 const void *default_mask, 1378 void *data) 1379 { 1380 const struct rte_flow_item_ipv4 *spec = item->spec; 1381 const struct rte_flow_item_ipv4 *mask = item->mask; 1382 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1383 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); 1384 struct ibv_flow_spec_ipv4_ext ipv4 = { 1385 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, 1386 .size = ipv4_size, 1387 }; 1388 1389 /* Don't update layer for the inner pattern. */ 1390 if (!parser->inner) 1391 parser->layer = HASH_RXQ_IPV4; 1392 if (spec) { 1393 if (!mask) 1394 mask = default_mask; 1395 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 1396 .src_ip = spec->hdr.src_addr, 1397 .dst_ip = spec->hdr.dst_addr, 1398 .proto = spec->hdr.next_proto_id, 1399 .tos = spec->hdr.type_of_service, 1400 }; 1401 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 1402 .src_ip = mask->hdr.src_addr, 1403 .dst_ip = mask->hdr.dst_addr, 1404 .proto = mask->hdr.next_proto_id, 1405 .tos = mask->hdr.type_of_service, 1406 }; 1407 /* Remove unwanted bits from values. */ 1408 ipv4.val.src_ip &= ipv4.mask.src_ip; 1409 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 1410 ipv4.val.proto &= ipv4.mask.proto; 1411 ipv4.val.tos &= ipv4.mask.tos; 1412 } 1413 mlx5_flow_create_copy(parser, &ipv4, ipv4_size); 1414 return 0; 1415 } 1416 1417 /** 1418 * Convert IPv6 item to Verbs specification. 1419 * 1420 * @param item[in] 1421 * Item specification. 1422 * @param default_mask[in] 1423 * Default bit-masks to use when item->mask is not provided. 1424 * @param data[in, out] 1425 * User structure. 1426 */ 1427 static int 1428 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 1429 const void *default_mask, 1430 void *data) 1431 { 1432 const struct rte_flow_item_ipv6 *spec = item->spec; 1433 const struct rte_flow_item_ipv6 *mask = item->mask; 1434 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1435 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); 1436 struct ibv_flow_spec_ipv6 ipv6 = { 1437 .type = parser->inner | IBV_FLOW_SPEC_IPV6, 1438 .size = ipv6_size, 1439 }; 1440 1441 /* Don't update layer for the inner pattern. */ 1442 if (!parser->inner) 1443 parser->layer = HASH_RXQ_IPV6; 1444 if (spec) { 1445 unsigned int i; 1446 1447 if (!mask) 1448 mask = default_mask; 1449 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 1450 RTE_DIM(ipv6.val.src_ip)); 1451 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 1452 RTE_DIM(ipv6.val.dst_ip)); 1453 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 1454 RTE_DIM(ipv6.mask.src_ip)); 1455 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 1456 RTE_DIM(ipv6.mask.dst_ip)); 1457 ipv6.mask.flow_label = mask->hdr.vtc_flow; 1458 ipv6.mask.next_hdr = mask->hdr.proto; 1459 ipv6.mask.hop_limit = mask->hdr.hop_limits; 1460 /* Remove unwanted bits from values. */ 1461 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 1462 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 1463 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 1464 } 1465 ipv6.val.flow_label &= ipv6.mask.flow_label; 1466 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 1467 ipv6.val.hop_limit &= ipv6.mask.hop_limit; 1468 } 1469 mlx5_flow_create_copy(parser, &ipv6, ipv6_size); 1470 return 0; 1471 } 1472 1473 /** 1474 * Convert UDP item to Verbs specification. 1475 * 1476 * @param item[in] 1477 * Item specification. 1478 * @param default_mask[in] 1479 * Default bit-masks to use when item->mask is not provided. 1480 * @param data[in, out] 1481 * User structure. 1482 */ 1483 static int 1484 mlx5_flow_create_udp(const struct rte_flow_item *item, 1485 const void *default_mask, 1486 void *data) 1487 { 1488 const struct rte_flow_item_udp *spec = item->spec; 1489 const struct rte_flow_item_udp *mask = item->mask; 1490 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1491 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1492 struct ibv_flow_spec_tcp_udp udp = { 1493 .type = parser->inner | IBV_FLOW_SPEC_UDP, 1494 .size = udp_size, 1495 }; 1496 1497 /* Don't update layer for the inner pattern. */ 1498 if (!parser->inner) { 1499 if (parser->layer == HASH_RXQ_IPV4) 1500 parser->layer = HASH_RXQ_UDPV4; 1501 else 1502 parser->layer = HASH_RXQ_UDPV6; 1503 } 1504 if (spec) { 1505 if (!mask) 1506 mask = default_mask; 1507 udp.val.dst_port = spec->hdr.dst_port; 1508 udp.val.src_port = spec->hdr.src_port; 1509 udp.mask.dst_port = mask->hdr.dst_port; 1510 udp.mask.src_port = mask->hdr.src_port; 1511 /* Remove unwanted bits from values. */ 1512 udp.val.src_port &= udp.mask.src_port; 1513 udp.val.dst_port &= udp.mask.dst_port; 1514 } 1515 mlx5_flow_create_copy(parser, &udp, udp_size); 1516 return 0; 1517 } 1518 1519 /** 1520 * Convert TCP item to Verbs specification. 1521 * 1522 * @param item[in] 1523 * Item specification. 1524 * @param default_mask[in] 1525 * Default bit-masks to use when item->mask is not provided. 1526 * @param data[in, out] 1527 * User structure. 1528 */ 1529 static int 1530 mlx5_flow_create_tcp(const struct rte_flow_item *item, 1531 const void *default_mask, 1532 void *data) 1533 { 1534 const struct rte_flow_item_tcp *spec = item->spec; 1535 const struct rte_flow_item_tcp *mask = item->mask; 1536 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1537 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1538 struct ibv_flow_spec_tcp_udp tcp = { 1539 .type = parser->inner | IBV_FLOW_SPEC_TCP, 1540 .size = tcp_size, 1541 }; 1542 1543 /* Don't update layer for the inner pattern. */ 1544 if (!parser->inner) { 1545 if (parser->layer == HASH_RXQ_IPV4) 1546 parser->layer = HASH_RXQ_TCPV4; 1547 else 1548 parser->layer = HASH_RXQ_TCPV6; 1549 } 1550 if (spec) { 1551 if (!mask) 1552 mask = default_mask; 1553 tcp.val.dst_port = spec->hdr.dst_port; 1554 tcp.val.src_port = spec->hdr.src_port; 1555 tcp.mask.dst_port = mask->hdr.dst_port; 1556 tcp.mask.src_port = mask->hdr.src_port; 1557 /* Remove unwanted bits from values. */ 1558 tcp.val.src_port &= tcp.mask.src_port; 1559 tcp.val.dst_port &= tcp.mask.dst_port; 1560 } 1561 mlx5_flow_create_copy(parser, &tcp, tcp_size); 1562 return 0; 1563 } 1564 1565 /** 1566 * Convert VXLAN item to Verbs specification. 1567 * 1568 * @param item[in] 1569 * Item specification. 1570 * @param default_mask[in] 1571 * Default bit-masks to use when item->mask is not provided. 1572 * @param data[in, out] 1573 * User structure. 1574 */ 1575 static int 1576 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 1577 const void *default_mask, 1578 void *data) 1579 { 1580 const struct rte_flow_item_vxlan *spec = item->spec; 1581 const struct rte_flow_item_vxlan *mask = item->mask; 1582 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1583 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 1584 struct ibv_flow_spec_tunnel vxlan = { 1585 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, 1586 .size = size, 1587 }; 1588 union vni { 1589 uint32_t vlan_id; 1590 uint8_t vni[4]; 1591 } id; 1592 1593 id.vni[0] = 0; 1594 parser->inner = IBV_FLOW_SPEC_INNER; 1595 if (spec) { 1596 if (!mask) 1597 mask = default_mask; 1598 memcpy(&id.vni[1], spec->vni, 3); 1599 vxlan.val.tunnel_id = id.vlan_id; 1600 memcpy(&id.vni[1], mask->vni, 3); 1601 vxlan.mask.tunnel_id = id.vlan_id; 1602 /* Remove unwanted bits from values. */ 1603 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 1604 } 1605 /* 1606 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1607 * layer is defined in the Verbs specification it is interpreted as 1608 * wildcard and all packets will match this rule, if it follows a full 1609 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1610 * before will also match this rule. 1611 * To avoid such situation, VNI 0 is currently refused. 1612 */ 1613 if (!vxlan.val.tunnel_id) 1614 return EINVAL; 1615 mlx5_flow_create_copy(parser, &vxlan, size); 1616 return 0; 1617 } 1618 1619 /** 1620 * Convert mark/flag action to Verbs specification. 1621 * 1622 * @param parser 1623 * Internal parser structure. 1624 * @param mark_id 1625 * Mark identifier. 1626 */ 1627 static int 1628 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) 1629 { 1630 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1631 struct ibv_flow_spec_action_tag tag = { 1632 .type = IBV_FLOW_SPEC_ACTION_TAG, 1633 .size = size, 1634 .tag_id = mlx5_flow_mark_set(mark_id), 1635 }; 1636 1637 assert(parser->mark); 1638 mlx5_flow_create_copy(parser, &tag, size); 1639 return 0; 1640 } 1641 1642 /** 1643 * Convert count action to Verbs specification. 1644 * 1645 * @param priv 1646 * Pointer to private structure. 1647 * @param parser 1648 * Pointer to MLX5 flow parser structure. 1649 * 1650 * @return 1651 * 0 on success, errno value on failure. 1652 */ 1653 static int 1654 mlx5_flow_create_count(struct priv *priv __rte_unused, 1655 struct mlx5_flow_parse *parser __rte_unused) 1656 { 1657 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 1658 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1659 struct ibv_counter_set_init_attr init_attr = {0}; 1660 struct ibv_flow_spec_counter_action counter = { 1661 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1662 .size = size, 1663 .counter_set_handle = 0, 1664 }; 1665 1666 init_attr.counter_set_id = 0; 1667 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr); 1668 if (!parser->cs) 1669 return EINVAL; 1670 counter.counter_set_handle = parser->cs->handle; 1671 mlx5_flow_create_copy(parser, &counter, size); 1672 #endif 1673 return 0; 1674 } 1675 1676 /** 1677 * Complete flow rule creation with a drop queue. 1678 * 1679 * @param priv 1680 * Pointer to private structure. 1681 * @param parser 1682 * Internal parser structure. 1683 * @param flow 1684 * Pointer to the rte_flow. 1685 * @param[out] error 1686 * Perform verbose error reporting if not NULL. 1687 * 1688 * @return 1689 * 0 on success, errno value on failure. 1690 */ 1691 static int 1692 priv_flow_create_action_queue_drop(struct priv *priv, 1693 struct mlx5_flow_parse *parser, 1694 struct rte_flow *flow, 1695 struct rte_flow_error *error) 1696 { 1697 struct ibv_flow_spec_action_drop *drop; 1698 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1699 int err = 0; 1700 1701 assert(priv->pd); 1702 assert(priv->ctx); 1703 flow->drop = 1; 1704 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1705 parser->drop_q.offset); 1706 *drop = (struct ibv_flow_spec_action_drop){ 1707 .type = IBV_FLOW_SPEC_ACTION_DROP, 1708 .size = size, 1709 }; 1710 ++parser->drop_q.ibv_attr->num_of_specs; 1711 parser->drop_q.offset += size; 1712 flow->drxq.ibv_attr = parser->drop_q.ibv_attr; 1713 if (parser->count) 1714 flow->cs = parser->cs; 1715 if (!priv->dev->data->dev_started) 1716 return 0; 1717 parser->drop_q.ibv_attr = NULL; 1718 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp, 1719 flow->drxq.ibv_attr); 1720 if (!flow->drxq.ibv_flow) { 1721 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1722 NULL, "flow rule creation failure"); 1723 err = ENOMEM; 1724 goto error; 1725 } 1726 return 0; 1727 error: 1728 assert(flow); 1729 if (flow->drxq.ibv_flow) { 1730 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 1731 flow->drxq.ibv_flow = NULL; 1732 } 1733 if (flow->drxq.ibv_attr) { 1734 rte_free(flow->drxq.ibv_attr); 1735 flow->drxq.ibv_attr = NULL; 1736 } 1737 if (flow->cs) { 1738 claim_zero(ibv_destroy_counter_set(flow->cs)); 1739 flow->cs = NULL; 1740 parser->cs = NULL; 1741 } 1742 return err; 1743 } 1744 1745 /** 1746 * Create hash Rx queues when RSS is enabled. 1747 * 1748 * @param priv 1749 * Pointer to private structure. 1750 * @param parser 1751 * Internal parser structure. 1752 * @param flow 1753 * Pointer to the rte_flow. 1754 * @param[out] error 1755 * Perform verbose error reporting if not NULL. 1756 * 1757 * @return 1758 * 0 on success, a errno value otherwise and rte_errno is set. 1759 */ 1760 static int 1761 priv_flow_create_action_queue_rss(struct priv *priv, 1762 struct mlx5_flow_parse *parser, 1763 struct rte_flow *flow, 1764 struct rte_flow_error *error) 1765 { 1766 unsigned int i; 1767 1768 for (i = 0; i != hash_rxq_init_n; ++i) { 1769 uint64_t hash_fields; 1770 1771 if (!parser->queue[i].ibv_attr) 1772 continue; 1773 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; 1774 parser->queue[i].ibv_attr = NULL; 1775 hash_fields = hash_rxq_init[i].hash_fields; 1776 if (!priv->dev->data->dev_started) 1777 continue; 1778 flow->frxq[i].hrxq = 1779 mlx5_priv_hrxq_get(priv, 1780 parser->rss_conf.rss_key, 1781 parser->rss_conf.rss_key_len, 1782 hash_fields, 1783 parser->queues, 1784 parser->queues_n); 1785 if (flow->frxq[i].hrxq) 1786 continue; 1787 flow->frxq[i].hrxq = 1788 mlx5_priv_hrxq_new(priv, 1789 parser->rss_conf.rss_key, 1790 parser->rss_conf.rss_key_len, 1791 hash_fields, 1792 parser->queues, 1793 parser->queues_n); 1794 if (!flow->frxq[i].hrxq) { 1795 rte_flow_error_set(error, ENOMEM, 1796 RTE_FLOW_ERROR_TYPE_HANDLE, 1797 NULL, "cannot create hash rxq"); 1798 return ENOMEM; 1799 } 1800 } 1801 return 0; 1802 } 1803 1804 /** 1805 * Complete flow rule creation. 1806 * 1807 * @param priv 1808 * Pointer to private structure. 1809 * @param parser 1810 * Internal parser structure. 1811 * @param flow 1812 * Pointer to the rte_flow. 1813 * @param[out] error 1814 * Perform verbose error reporting if not NULL. 1815 * 1816 * @return 1817 * 0 on success, a errno value otherwise and rte_errno is set. 1818 */ 1819 static int 1820 priv_flow_create_action_queue(struct priv *priv, 1821 struct mlx5_flow_parse *parser, 1822 struct rte_flow *flow, 1823 struct rte_flow_error *error) 1824 { 1825 int err = 0; 1826 unsigned int i; 1827 1828 assert(priv->pd); 1829 assert(priv->ctx); 1830 assert(!parser->drop); 1831 err = priv_flow_create_action_queue_rss(priv, parser, flow, error); 1832 if (err) 1833 goto error; 1834 if (parser->count) 1835 flow->cs = parser->cs; 1836 if (!priv->dev->data->dev_started) 1837 return 0; 1838 for (i = 0; i != hash_rxq_init_n; ++i) { 1839 if (!flow->frxq[i].hrxq) 1840 continue; 1841 flow->frxq[i].ibv_flow = 1842 ibv_create_flow(flow->frxq[i].hrxq->qp, 1843 flow->frxq[i].ibv_attr); 1844 if (!flow->frxq[i].ibv_flow) { 1845 rte_flow_error_set(error, ENOMEM, 1846 RTE_FLOW_ERROR_TYPE_HANDLE, 1847 NULL, "flow rule creation failure"); 1848 err = ENOMEM; 1849 goto error; 1850 } 1851 DEBUG("%p type %d QP %p ibv_flow %p", 1852 (void *)flow, i, 1853 (void *)flow->frxq[i].hrxq, 1854 (void *)flow->frxq[i].ibv_flow); 1855 } 1856 for (i = 0; i != parser->queues_n; ++i) { 1857 struct mlx5_rxq_data *q = 1858 (*priv->rxqs)[parser->queues[i]]; 1859 1860 q->mark |= parser->mark; 1861 } 1862 return 0; 1863 error: 1864 assert(flow); 1865 for (i = 0; i != hash_rxq_init_n; ++i) { 1866 if (flow->frxq[i].ibv_flow) { 1867 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; 1868 1869 claim_zero(ibv_destroy_flow(ibv_flow)); 1870 } 1871 if (flow->frxq[i].hrxq) 1872 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); 1873 if (flow->frxq[i].ibv_attr) 1874 rte_free(flow->frxq[i].ibv_attr); 1875 } 1876 if (flow->cs) { 1877 claim_zero(ibv_destroy_counter_set(flow->cs)); 1878 flow->cs = NULL; 1879 parser->cs = NULL; 1880 } 1881 return err; 1882 } 1883 1884 /** 1885 * Convert a flow. 1886 * 1887 * @param priv 1888 * Pointer to private structure. 1889 * @param list 1890 * Pointer to a TAILQ flow list. 1891 * @param[in] attr 1892 * Flow rule attributes. 1893 * @param[in] pattern 1894 * Pattern specification (list terminated by the END pattern item). 1895 * @param[in] actions 1896 * Associated actions (list terminated by the END action). 1897 * @param[out] error 1898 * Perform verbose error reporting if not NULL. 1899 * 1900 * @return 1901 * A flow on success, NULL otherwise. 1902 */ 1903 static struct rte_flow * 1904 priv_flow_create(struct priv *priv, 1905 struct mlx5_flows *list, 1906 const struct rte_flow_attr *attr, 1907 const struct rte_flow_item items[], 1908 const struct rte_flow_action actions[], 1909 struct rte_flow_error *error) 1910 { 1911 struct mlx5_flow_parse parser = { .create = 1, }; 1912 struct rte_flow *flow = NULL; 1913 unsigned int i; 1914 int err; 1915 1916 err = priv_flow_convert(priv, attr, items, actions, error, &parser); 1917 if (err) 1918 goto exit; 1919 flow = rte_calloc(__func__, 1, 1920 sizeof(*flow) + parser.queues_n * sizeof(uint16_t), 1921 0); 1922 if (!flow) { 1923 rte_flow_error_set(error, ENOMEM, 1924 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1925 NULL, 1926 "cannot allocate flow memory"); 1927 return NULL; 1928 } 1929 /* Copy queues configuration. */ 1930 flow->queues = (uint16_t (*)[])(flow + 1); 1931 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t)); 1932 flow->queues_n = parser.queues_n; 1933 /* Copy RSS configuration. */ 1934 flow->rss_conf = parser.rss_conf; 1935 flow->rss_conf.rss_key = flow->rss_key; 1936 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len); 1937 /* finalise the flow. */ 1938 if (parser.drop) 1939 err = priv_flow_create_action_queue_drop(priv, &parser, flow, 1940 error); 1941 else 1942 err = priv_flow_create_action_queue(priv, &parser, flow, error); 1943 if (err) 1944 goto exit; 1945 TAILQ_INSERT_TAIL(list, flow, next); 1946 DEBUG("Flow created %p", (void *)flow); 1947 return flow; 1948 exit: 1949 if (parser.drop) { 1950 rte_free(parser.drop_q.ibv_attr); 1951 } else { 1952 for (i = 0; i != hash_rxq_init_n; ++i) { 1953 if (parser.queue[i].ibv_attr) 1954 rte_free(parser.queue[i].ibv_attr); 1955 } 1956 } 1957 rte_free(flow); 1958 return NULL; 1959 } 1960 1961 /** 1962 * Validate a flow supported by the NIC. 1963 * 1964 * @see rte_flow_validate() 1965 * @see rte_flow_ops 1966 */ 1967 int 1968 mlx5_flow_validate(struct rte_eth_dev *dev, 1969 const struct rte_flow_attr *attr, 1970 const struct rte_flow_item items[], 1971 const struct rte_flow_action actions[], 1972 struct rte_flow_error *error) 1973 { 1974 struct priv *priv = dev->data->dev_private; 1975 int ret; 1976 struct mlx5_flow_parse parser = { .create = 0, }; 1977 1978 priv_lock(priv); 1979 ret = priv_flow_convert(priv, attr, items, actions, error, &parser); 1980 priv_unlock(priv); 1981 return ret; 1982 } 1983 1984 /** 1985 * Create a flow. 1986 * 1987 * @see rte_flow_create() 1988 * @see rte_flow_ops 1989 */ 1990 struct rte_flow * 1991 mlx5_flow_create(struct rte_eth_dev *dev, 1992 const struct rte_flow_attr *attr, 1993 const struct rte_flow_item items[], 1994 const struct rte_flow_action actions[], 1995 struct rte_flow_error *error) 1996 { 1997 struct priv *priv = dev->data->dev_private; 1998 struct rte_flow *flow; 1999 2000 priv_lock(priv); 2001 flow = priv_flow_create(priv, &priv->flows, attr, items, actions, 2002 error); 2003 priv_unlock(priv); 2004 return flow; 2005 } 2006 2007 /** 2008 * Destroy a flow. 2009 * 2010 * @param priv 2011 * Pointer to private structure. 2012 * @param list 2013 * Pointer to a TAILQ flow list. 2014 * @param[in] flow 2015 * Flow to destroy. 2016 */ 2017 static void 2018 priv_flow_destroy(struct priv *priv, 2019 struct mlx5_flows *list, 2020 struct rte_flow *flow) 2021 { 2022 unsigned int i; 2023 2024 if (flow->drop || !flow->mark) 2025 goto free; 2026 for (i = 0; i != flow->queues_n; ++i) { 2027 struct rte_flow *tmp; 2028 int mark = 0; 2029 2030 /* 2031 * To remove the mark from the queue, the queue must not be 2032 * present in any other marked flow (RSS or not). 2033 */ 2034 TAILQ_FOREACH(tmp, list, next) { 2035 unsigned int j; 2036 uint16_t *tqs = NULL; 2037 uint16_t tq_n = 0; 2038 2039 if (!tmp->mark) 2040 continue; 2041 for (j = 0; j != hash_rxq_init_n; ++j) { 2042 if (!tmp->frxq[j].hrxq) 2043 continue; 2044 tqs = tmp->frxq[j].hrxq->ind_table->queues; 2045 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n; 2046 } 2047 if (!tq_n) 2048 continue; 2049 for (j = 0; (j != tq_n) && !mark; j++) 2050 if (tqs[j] == (*flow->queues)[i]) 2051 mark = 1; 2052 } 2053 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark; 2054 } 2055 free: 2056 if (flow->drop) { 2057 if (flow->drxq.ibv_flow) 2058 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 2059 rte_free(flow->drxq.ibv_attr); 2060 } else { 2061 for (i = 0; i != hash_rxq_init_n; ++i) { 2062 struct mlx5_flow *frxq = &flow->frxq[i]; 2063 2064 if (frxq->ibv_flow) 2065 claim_zero(ibv_destroy_flow(frxq->ibv_flow)); 2066 if (frxq->hrxq) 2067 mlx5_priv_hrxq_release(priv, frxq->hrxq); 2068 if (frxq->ibv_attr) 2069 rte_free(frxq->ibv_attr); 2070 } 2071 } 2072 if (flow->cs) { 2073 claim_zero(ibv_destroy_counter_set(flow->cs)); 2074 flow->cs = NULL; 2075 } 2076 TAILQ_REMOVE(list, flow, next); 2077 DEBUG("Flow destroyed %p", (void *)flow); 2078 rte_free(flow); 2079 } 2080 2081 /** 2082 * Destroy all flows. 2083 * 2084 * @param priv 2085 * Pointer to private structure. 2086 * @param list 2087 * Pointer to a TAILQ flow list. 2088 */ 2089 void 2090 priv_flow_flush(struct priv *priv, struct mlx5_flows *list) 2091 { 2092 while (!TAILQ_EMPTY(list)) { 2093 struct rte_flow *flow; 2094 2095 flow = TAILQ_FIRST(list); 2096 priv_flow_destroy(priv, list, flow); 2097 } 2098 } 2099 2100 /** 2101 * Create drop queue. 2102 * 2103 * @param priv 2104 * Pointer to private structure. 2105 * 2106 * @return 2107 * 0 on success. 2108 */ 2109 int 2110 priv_flow_create_drop_queue(struct priv *priv) 2111 { 2112 struct mlx5_hrxq_drop *fdq = NULL; 2113 2114 assert(priv->pd); 2115 assert(priv->ctx); 2116 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); 2117 if (!fdq) { 2118 WARN("cannot allocate memory for drop queue"); 2119 goto error; 2120 } 2121 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); 2122 if (!fdq->cq) { 2123 WARN("cannot allocate CQ for drop queue"); 2124 goto error; 2125 } 2126 fdq->wq = ibv_create_wq(priv->ctx, 2127 &(struct ibv_wq_init_attr){ 2128 .wq_type = IBV_WQT_RQ, 2129 .max_wr = 1, 2130 .max_sge = 1, 2131 .pd = priv->pd, 2132 .cq = fdq->cq, 2133 }); 2134 if (!fdq->wq) { 2135 WARN("cannot allocate WQ for drop queue"); 2136 goto error; 2137 } 2138 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx, 2139 &(struct ibv_rwq_ind_table_init_attr){ 2140 .log_ind_tbl_size = 0, 2141 .ind_tbl = &fdq->wq, 2142 .comp_mask = 0, 2143 }); 2144 if (!fdq->ind_table) { 2145 WARN("cannot allocate indirection table for drop queue"); 2146 goto error; 2147 } 2148 fdq->qp = ibv_create_qp_ex(priv->ctx, 2149 &(struct ibv_qp_init_attr_ex){ 2150 .qp_type = IBV_QPT_RAW_PACKET, 2151 .comp_mask = 2152 IBV_QP_INIT_ATTR_PD | 2153 IBV_QP_INIT_ATTR_IND_TABLE | 2154 IBV_QP_INIT_ATTR_RX_HASH, 2155 .rx_hash_conf = (struct ibv_rx_hash_conf){ 2156 .rx_hash_function = 2157 IBV_RX_HASH_FUNC_TOEPLITZ, 2158 .rx_hash_key_len = rss_hash_default_key_len, 2159 .rx_hash_key = rss_hash_default_key, 2160 .rx_hash_fields_mask = 0, 2161 }, 2162 .rwq_ind_tbl = fdq->ind_table, 2163 .pd = priv->pd 2164 }); 2165 if (!fdq->qp) { 2166 WARN("cannot allocate QP for drop queue"); 2167 goto error; 2168 } 2169 priv->flow_drop_queue = fdq; 2170 return 0; 2171 error: 2172 if (fdq->qp) 2173 claim_zero(ibv_destroy_qp(fdq->qp)); 2174 if (fdq->ind_table) 2175 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 2176 if (fdq->wq) 2177 claim_zero(ibv_destroy_wq(fdq->wq)); 2178 if (fdq->cq) 2179 claim_zero(ibv_destroy_cq(fdq->cq)); 2180 if (fdq) 2181 rte_free(fdq); 2182 priv->flow_drop_queue = NULL; 2183 return -1; 2184 } 2185 2186 /** 2187 * Delete drop queue. 2188 * 2189 * @param priv 2190 * Pointer to private structure. 2191 */ 2192 void 2193 priv_flow_delete_drop_queue(struct priv *priv) 2194 { 2195 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; 2196 2197 if (!fdq) 2198 return; 2199 if (fdq->qp) 2200 claim_zero(ibv_destroy_qp(fdq->qp)); 2201 if (fdq->ind_table) 2202 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 2203 if (fdq->wq) 2204 claim_zero(ibv_destroy_wq(fdq->wq)); 2205 if (fdq->cq) 2206 claim_zero(ibv_destroy_cq(fdq->cq)); 2207 rte_free(fdq); 2208 priv->flow_drop_queue = NULL; 2209 } 2210 2211 /** 2212 * Remove all flows. 2213 * 2214 * @param priv 2215 * Pointer to private structure. 2216 * @param list 2217 * Pointer to a TAILQ flow list. 2218 */ 2219 void 2220 priv_flow_stop(struct priv *priv, struct mlx5_flows *list) 2221 { 2222 struct rte_flow *flow; 2223 2224 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 2225 unsigned int i; 2226 2227 if (flow->drop) { 2228 if (!flow->drxq.ibv_flow) 2229 continue; 2230 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 2231 flow->drxq.ibv_flow = NULL; 2232 /* Next flow. */ 2233 continue; 2234 } 2235 if (flow->mark) { 2236 struct mlx5_ind_table_ibv *ind_tbl = NULL; 2237 2238 for (i = 0; i != hash_rxq_init_n; ++i) { 2239 if (!flow->frxq[i].hrxq) 2240 continue; 2241 ind_tbl = flow->frxq[i].hrxq->ind_table; 2242 } 2243 assert(ind_tbl); 2244 for (i = 0; i != ind_tbl->queues_n; ++i) 2245 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; 2246 } 2247 for (i = 0; i != hash_rxq_init_n; ++i) { 2248 if (!flow->frxq[i].ibv_flow) 2249 continue; 2250 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow)); 2251 flow->frxq[i].ibv_flow = NULL; 2252 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); 2253 flow->frxq[i].hrxq = NULL; 2254 } 2255 DEBUG("Flow %p removed", (void *)flow); 2256 } 2257 } 2258 2259 /** 2260 * Add all flows. 2261 * 2262 * @param priv 2263 * Pointer to private structure. 2264 * @param list 2265 * Pointer to a TAILQ flow list. 2266 * 2267 * @return 2268 * 0 on success, a errno value otherwise and rte_errno is set. 2269 */ 2270 int 2271 priv_flow_start(struct priv *priv, struct mlx5_flows *list) 2272 { 2273 struct rte_flow *flow; 2274 2275 TAILQ_FOREACH(flow, list, next) { 2276 unsigned int i; 2277 2278 if (flow->drop) { 2279 flow->drxq.ibv_flow = 2280 ibv_create_flow(priv->flow_drop_queue->qp, 2281 flow->drxq.ibv_attr); 2282 if (!flow->drxq.ibv_flow) { 2283 DEBUG("Flow %p cannot be applied", 2284 (void *)flow); 2285 rte_errno = EINVAL; 2286 return rte_errno; 2287 } 2288 DEBUG("Flow %p applied", (void *)flow); 2289 /* Next flow. */ 2290 continue; 2291 } 2292 for (i = 0; i != hash_rxq_init_n; ++i) { 2293 if (!flow->frxq[i].ibv_attr) 2294 continue; 2295 flow->frxq[i].hrxq = 2296 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key, 2297 flow->rss_conf.rss_key_len, 2298 hash_rxq_init[i].hash_fields, 2299 (*flow->queues), 2300 flow->queues_n); 2301 if (flow->frxq[i].hrxq) 2302 goto flow_create; 2303 flow->frxq[i].hrxq = 2304 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key, 2305 flow->rss_conf.rss_key_len, 2306 hash_rxq_init[i].hash_fields, 2307 (*flow->queues), 2308 flow->queues_n); 2309 if (!flow->frxq[i].hrxq) { 2310 DEBUG("Flow %p cannot be applied", 2311 (void *)flow); 2312 rte_errno = EINVAL; 2313 return rte_errno; 2314 } 2315 flow_create: 2316 flow->frxq[i].ibv_flow = 2317 ibv_create_flow(flow->frxq[i].hrxq->qp, 2318 flow->frxq[i].ibv_attr); 2319 if (!flow->frxq[i].ibv_flow) { 2320 DEBUG("Flow %p cannot be applied", 2321 (void *)flow); 2322 rte_errno = EINVAL; 2323 return rte_errno; 2324 } 2325 DEBUG("Flow %p applied", (void *)flow); 2326 } 2327 if (!flow->mark) 2328 continue; 2329 for (i = 0; i != flow->queues_n; ++i) 2330 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1; 2331 } 2332 return 0; 2333 } 2334 2335 /** 2336 * Verify the flow list is empty 2337 * 2338 * @param priv 2339 * Pointer to private structure. 2340 * 2341 * @return the number of flows not released. 2342 */ 2343 int 2344 priv_flow_verify(struct priv *priv) 2345 { 2346 struct rte_flow *flow; 2347 int ret = 0; 2348 2349 TAILQ_FOREACH(flow, &priv->flows, next) { 2350 DEBUG("%p: flow %p still referenced", (void *)priv, 2351 (void *)flow); 2352 ++ret; 2353 } 2354 return ret; 2355 } 2356 2357 /** 2358 * Enable a control flow configured from the control plane. 2359 * 2360 * @param dev 2361 * Pointer to Ethernet device. 2362 * @param eth_spec 2363 * An Ethernet flow spec to apply. 2364 * @param eth_mask 2365 * An Ethernet flow mask to apply. 2366 * @param vlan_spec 2367 * A VLAN flow spec to apply. 2368 * @param vlan_mask 2369 * A VLAN flow mask to apply. 2370 * 2371 * @return 2372 * 0 on success. 2373 */ 2374 int 2375 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2376 struct rte_flow_item_eth *eth_spec, 2377 struct rte_flow_item_eth *eth_mask, 2378 struct rte_flow_item_vlan *vlan_spec, 2379 struct rte_flow_item_vlan *vlan_mask) 2380 { 2381 struct priv *priv = dev->data->dev_private; 2382 const struct rte_flow_attr attr = { 2383 .ingress = 1, 2384 .priority = MLX5_CTRL_FLOW_PRIORITY, 2385 }; 2386 struct rte_flow_item items[] = { 2387 { 2388 .type = RTE_FLOW_ITEM_TYPE_ETH, 2389 .spec = eth_spec, 2390 .last = NULL, 2391 .mask = eth_mask, 2392 }, 2393 { 2394 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2395 RTE_FLOW_ITEM_TYPE_END, 2396 .spec = vlan_spec, 2397 .last = NULL, 2398 .mask = vlan_mask, 2399 }, 2400 { 2401 .type = RTE_FLOW_ITEM_TYPE_END, 2402 }, 2403 }; 2404 struct rte_flow_action actions[] = { 2405 { 2406 .type = RTE_FLOW_ACTION_TYPE_RSS, 2407 }, 2408 { 2409 .type = RTE_FLOW_ACTION_TYPE_END, 2410 }, 2411 }; 2412 struct rte_flow *flow; 2413 struct rte_flow_error error; 2414 unsigned int i; 2415 union { 2416 struct rte_flow_action_rss rss; 2417 struct { 2418 const struct rte_eth_rss_conf *rss_conf; 2419 uint16_t num; 2420 uint16_t queue[RTE_MAX_QUEUES_PER_PORT]; 2421 } local; 2422 } action_rss; 2423 2424 if (!priv->reta_idx_n) 2425 return EINVAL; 2426 for (i = 0; i != priv->reta_idx_n; ++i) 2427 action_rss.local.queue[i] = (*priv->reta_idx)[i]; 2428 action_rss.local.rss_conf = &priv->rss_conf; 2429 action_rss.local.num = priv->reta_idx_n; 2430 actions[0].conf = (const void *)&action_rss.rss; 2431 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions, 2432 &error); 2433 if (!flow) 2434 return rte_errno; 2435 return 0; 2436 } 2437 2438 /** 2439 * Enable a flow control configured from the control plane. 2440 * 2441 * @param dev 2442 * Pointer to Ethernet device. 2443 * @param eth_spec 2444 * An Ethernet flow spec to apply. 2445 * @param eth_mask 2446 * An Ethernet flow mask to apply. 2447 * 2448 * @return 2449 * 0 on success. 2450 */ 2451 int 2452 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2453 struct rte_flow_item_eth *eth_spec, 2454 struct rte_flow_item_eth *eth_mask) 2455 { 2456 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2457 } 2458 2459 /** 2460 * Destroy a flow. 2461 * 2462 * @see rte_flow_destroy() 2463 * @see rte_flow_ops 2464 */ 2465 int 2466 mlx5_flow_destroy(struct rte_eth_dev *dev, 2467 struct rte_flow *flow, 2468 struct rte_flow_error *error) 2469 { 2470 struct priv *priv = dev->data->dev_private; 2471 2472 (void)error; 2473 priv_lock(priv); 2474 priv_flow_destroy(priv, &priv->flows, flow); 2475 priv_unlock(priv); 2476 return 0; 2477 } 2478 2479 /** 2480 * Destroy all flows. 2481 * 2482 * @see rte_flow_flush() 2483 * @see rte_flow_ops 2484 */ 2485 int 2486 mlx5_flow_flush(struct rte_eth_dev *dev, 2487 struct rte_flow_error *error) 2488 { 2489 struct priv *priv = dev->data->dev_private; 2490 2491 (void)error; 2492 priv_lock(priv); 2493 priv_flow_flush(priv, &priv->flows); 2494 priv_unlock(priv); 2495 return 0; 2496 } 2497 2498 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 2499 /** 2500 * Query flow counter. 2501 * 2502 * @param cs 2503 * the counter set. 2504 * @param counter_value 2505 * returned data from the counter. 2506 * 2507 * @return 2508 * 0 on success, a errno value otherwise and rte_errno is set. 2509 */ 2510 static int 2511 priv_flow_query_count(struct ibv_counter_set *cs, 2512 struct mlx5_flow_counter_stats *counter_stats, 2513 struct rte_flow_query_count *query_count, 2514 struct rte_flow_error *error) 2515 { 2516 uint64_t counters[2]; 2517 struct ibv_query_counter_set_attr query_cs_attr = { 2518 .cs = cs, 2519 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 2520 }; 2521 struct ibv_counter_set_data query_out = { 2522 .out = counters, 2523 .outlen = 2 * sizeof(uint64_t), 2524 }; 2525 int res = ibv_query_counter_set(&query_cs_attr, &query_out); 2526 2527 if (res) { 2528 rte_flow_error_set(error, -res, 2529 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2530 NULL, 2531 "cannot read counter"); 2532 return -res; 2533 } 2534 query_count->hits_set = 1; 2535 query_count->bytes_set = 1; 2536 query_count->hits = counters[0] - counter_stats->hits; 2537 query_count->bytes = counters[1] - counter_stats->bytes; 2538 if (query_count->reset) { 2539 counter_stats->hits = counters[0]; 2540 counter_stats->bytes = counters[1]; 2541 } 2542 return 0; 2543 } 2544 2545 /** 2546 * Query a flows. 2547 * 2548 * @see rte_flow_query() 2549 * @see rte_flow_ops 2550 */ 2551 int 2552 mlx5_flow_query(struct rte_eth_dev *dev, 2553 struct rte_flow *flow, 2554 enum rte_flow_action_type action __rte_unused, 2555 void *data, 2556 struct rte_flow_error *error) 2557 { 2558 struct priv *priv = dev->data->dev_private; 2559 int res = EINVAL; 2560 2561 priv_lock(priv); 2562 if (flow->cs) { 2563 res = priv_flow_query_count(flow->cs, 2564 &flow->counter_stats, 2565 (struct rte_flow_query_count *)data, 2566 error); 2567 } else { 2568 rte_flow_error_set(error, res, 2569 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2570 NULL, 2571 "no counter found for flow"); 2572 } 2573 priv_unlock(priv); 2574 return -res; 2575 } 2576 #endif 2577 2578 /** 2579 * Isolated mode. 2580 * 2581 * @see rte_flow_isolate() 2582 * @see rte_flow_ops 2583 */ 2584 int 2585 mlx5_flow_isolate(struct rte_eth_dev *dev, 2586 int enable, 2587 struct rte_flow_error *error) 2588 { 2589 struct priv *priv = dev->data->dev_private; 2590 2591 priv_lock(priv); 2592 if (dev->data->dev_started) { 2593 rte_flow_error_set(error, EBUSY, 2594 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2595 NULL, 2596 "port must be stopped first"); 2597 priv_unlock(priv); 2598 return -rte_errno; 2599 } 2600 priv->isolated = !!enable; 2601 if (enable) 2602 priv->dev->dev_ops = &mlx5_dev_ops_isolate; 2603 else 2604 priv->dev->dev_ops = &mlx5_dev_ops; 2605 priv_unlock(priv); 2606 return 0; 2607 } 2608 2609 /** 2610 * Convert a flow director filter to a generic flow. 2611 * 2612 * @param priv 2613 * Private structure. 2614 * @param fdir_filter 2615 * Flow director filter to add. 2616 * @param attributes 2617 * Generic flow parameters structure. 2618 * 2619 * @return 2620 * 0 on success, errno value on error. 2621 */ 2622 static int 2623 priv_fdir_filter_convert(struct priv *priv, 2624 const struct rte_eth_fdir_filter *fdir_filter, 2625 struct mlx5_fdir *attributes) 2626 { 2627 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2628 2629 /* Validate queue number. */ 2630 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2631 ERROR("invalid queue number %d", fdir_filter->action.rx_queue); 2632 return EINVAL; 2633 } 2634 attributes->attr.ingress = 1; 2635 attributes->items[0] = (struct rte_flow_item) { 2636 .type = RTE_FLOW_ITEM_TYPE_ETH, 2637 .spec = &attributes->l2, 2638 .mask = &attributes->l2_mask, 2639 }; 2640 switch (fdir_filter->action.behavior) { 2641 case RTE_ETH_FDIR_ACCEPT: 2642 attributes->actions[0] = (struct rte_flow_action){ 2643 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2644 .conf = &attributes->queue, 2645 }; 2646 break; 2647 case RTE_ETH_FDIR_REJECT: 2648 attributes->actions[0] = (struct rte_flow_action){ 2649 .type = RTE_FLOW_ACTION_TYPE_DROP, 2650 }; 2651 break; 2652 default: 2653 ERROR("invalid behavior %d", fdir_filter->action.behavior); 2654 return ENOTSUP; 2655 } 2656 attributes->queue.index = fdir_filter->action.rx_queue; 2657 switch (fdir_filter->input.flow_type) { 2658 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2659 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2660 .src_addr = input->flow.udp4_flow.ip.src_ip, 2661 .dst_addr = input->flow.udp4_flow.ip.dst_ip, 2662 .time_to_live = input->flow.udp4_flow.ip.ttl, 2663 .type_of_service = input->flow.udp4_flow.ip.tos, 2664 .next_proto_id = input->flow.udp4_flow.ip.proto, 2665 }; 2666 attributes->l4.udp.hdr = (struct udp_hdr){ 2667 .src_port = input->flow.udp4_flow.src_port, 2668 .dst_port = input->flow.udp4_flow.dst_port, 2669 }; 2670 attributes->items[1] = (struct rte_flow_item){ 2671 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2672 .spec = &attributes->l3, 2673 }; 2674 attributes->items[2] = (struct rte_flow_item){ 2675 .type = RTE_FLOW_ITEM_TYPE_UDP, 2676 .spec = &attributes->l4, 2677 }; 2678 break; 2679 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2680 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2681 .src_addr = input->flow.tcp4_flow.ip.src_ip, 2682 .dst_addr = input->flow.tcp4_flow.ip.dst_ip, 2683 .time_to_live = input->flow.tcp4_flow.ip.ttl, 2684 .type_of_service = input->flow.tcp4_flow.ip.tos, 2685 .next_proto_id = input->flow.tcp4_flow.ip.proto, 2686 }; 2687 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2688 .src_port = input->flow.tcp4_flow.src_port, 2689 .dst_port = input->flow.tcp4_flow.dst_port, 2690 }; 2691 attributes->items[1] = (struct rte_flow_item){ 2692 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2693 .spec = &attributes->l3, 2694 }; 2695 attributes->items[2] = (struct rte_flow_item){ 2696 .type = RTE_FLOW_ITEM_TYPE_TCP, 2697 .spec = &attributes->l4, 2698 }; 2699 break; 2700 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2701 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2702 .src_addr = input->flow.ip4_flow.src_ip, 2703 .dst_addr = input->flow.ip4_flow.dst_ip, 2704 .time_to_live = input->flow.ip4_flow.ttl, 2705 .type_of_service = input->flow.ip4_flow.tos, 2706 .next_proto_id = input->flow.ip4_flow.proto, 2707 }; 2708 attributes->items[1] = (struct rte_flow_item){ 2709 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2710 .spec = &attributes->l3, 2711 }; 2712 break; 2713 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2714 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2715 .hop_limits = input->flow.udp6_flow.ip.hop_limits, 2716 .proto = input->flow.udp6_flow.ip.proto, 2717 }; 2718 memcpy(attributes->l3.ipv6.hdr.src_addr, 2719 input->flow.udp6_flow.ip.src_ip, 2720 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2721 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2722 input->flow.udp6_flow.ip.dst_ip, 2723 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2724 attributes->l4.udp.hdr = (struct udp_hdr){ 2725 .src_port = input->flow.udp6_flow.src_port, 2726 .dst_port = input->flow.udp6_flow.dst_port, 2727 }; 2728 attributes->items[1] = (struct rte_flow_item){ 2729 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2730 .spec = &attributes->l3, 2731 }; 2732 attributes->items[2] = (struct rte_flow_item){ 2733 .type = RTE_FLOW_ITEM_TYPE_UDP, 2734 .spec = &attributes->l4, 2735 }; 2736 break; 2737 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2738 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2739 .hop_limits = input->flow.tcp6_flow.ip.hop_limits, 2740 .proto = input->flow.tcp6_flow.ip.proto, 2741 }; 2742 memcpy(attributes->l3.ipv6.hdr.src_addr, 2743 input->flow.tcp6_flow.ip.src_ip, 2744 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2745 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2746 input->flow.tcp6_flow.ip.dst_ip, 2747 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2748 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2749 .src_port = input->flow.tcp6_flow.src_port, 2750 .dst_port = input->flow.tcp6_flow.dst_port, 2751 }; 2752 attributes->items[1] = (struct rte_flow_item){ 2753 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2754 .spec = &attributes->l3, 2755 }; 2756 attributes->items[2] = (struct rte_flow_item){ 2757 .type = RTE_FLOW_ITEM_TYPE_TCP, 2758 .spec = &attributes->l4, 2759 }; 2760 break; 2761 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2762 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2763 .hop_limits = input->flow.ipv6_flow.hop_limits, 2764 .proto = input->flow.ipv6_flow.proto, 2765 }; 2766 memcpy(attributes->l3.ipv6.hdr.src_addr, 2767 input->flow.ipv6_flow.src_ip, 2768 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2769 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2770 input->flow.ipv6_flow.dst_ip, 2771 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2772 attributes->items[1] = (struct rte_flow_item){ 2773 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2774 .spec = &attributes->l3, 2775 }; 2776 break; 2777 default: 2778 ERROR("invalid flow type%d", 2779 fdir_filter->input.flow_type); 2780 return ENOTSUP; 2781 } 2782 return 0; 2783 } 2784 2785 /** 2786 * Add new flow director filter and store it in list. 2787 * 2788 * @param priv 2789 * Private structure. 2790 * @param fdir_filter 2791 * Flow director filter to add. 2792 * 2793 * @return 2794 * 0 on success, errno value on failure. 2795 */ 2796 static int 2797 priv_fdir_filter_add(struct priv *priv, 2798 const struct rte_eth_fdir_filter *fdir_filter) 2799 { 2800 struct mlx5_fdir attributes = { 2801 .attr.group = 0, 2802 .l2_mask = { 2803 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2804 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2805 .type = 0, 2806 }, 2807 }; 2808 struct mlx5_flow_parse parser = { 2809 .layer = HASH_RXQ_ETH, 2810 }; 2811 struct rte_flow_error error; 2812 struct rte_flow *flow; 2813 int ret; 2814 2815 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); 2816 if (ret) 2817 return -ret; 2818 ret = priv_flow_convert(priv, &attributes.attr, attributes.items, 2819 attributes.actions, &error, &parser); 2820 if (ret) 2821 return -ret; 2822 flow = priv_flow_create(priv, 2823 &priv->flows, 2824 &attributes.attr, 2825 attributes.items, 2826 attributes.actions, 2827 &error); 2828 if (flow) { 2829 DEBUG("FDIR created %p", (void *)flow); 2830 return 0; 2831 } 2832 return ENOTSUP; 2833 } 2834 2835 /** 2836 * Delete specific filter. 2837 * 2838 * @param priv 2839 * Private structure. 2840 * @param fdir_filter 2841 * Filter to be deleted. 2842 * 2843 * @return 2844 * 0 on success, errno value on failure. 2845 */ 2846 static int 2847 priv_fdir_filter_delete(struct priv *priv, 2848 const struct rte_eth_fdir_filter *fdir_filter) 2849 { 2850 struct mlx5_fdir attributes = { 2851 .attr.group = 0, 2852 }; 2853 struct mlx5_flow_parse parser = { 2854 .create = 1, 2855 .layer = HASH_RXQ_ETH, 2856 }; 2857 struct rte_flow_error error; 2858 struct rte_flow *flow; 2859 unsigned int i; 2860 int ret; 2861 2862 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); 2863 if (ret) 2864 return -ret; 2865 ret = priv_flow_convert(priv, &attributes.attr, attributes.items, 2866 attributes.actions, &error, &parser); 2867 if (ret) 2868 goto exit; 2869 TAILQ_FOREACH(flow, &priv->flows, next) { 2870 struct ibv_flow_attr *attr; 2871 struct ibv_spec_header *attr_h; 2872 void *spec; 2873 struct ibv_flow_attr *flow_attr; 2874 struct ibv_spec_header *flow_h; 2875 void *flow_spec; 2876 unsigned int specs_n; 2877 2878 if (parser.drop) 2879 attr = parser.drop_q.ibv_attr; 2880 else 2881 attr = parser.queue[HASH_RXQ_ETH].ibv_attr; 2882 if (flow->drop) 2883 flow_attr = flow->drxq.ibv_attr; 2884 else 2885 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; 2886 /* Compare first the attributes. */ 2887 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) 2888 continue; 2889 if (attr->num_of_specs == 0) 2890 continue; 2891 spec = (void *)((uintptr_t)attr + 2892 sizeof(struct ibv_flow_attr)); 2893 flow_spec = (void *)((uintptr_t)flow_attr + 2894 sizeof(struct ibv_flow_attr)); 2895 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs); 2896 for (i = 0; i != specs_n; ++i) { 2897 attr_h = spec; 2898 flow_h = flow_spec; 2899 if (memcmp(spec, flow_spec, 2900 RTE_MIN(attr_h->size, flow_h->size))) 2901 continue; 2902 spec = (void *)((uintptr_t)attr + attr_h->size); 2903 flow_spec = (void *)((uintptr_t)flow_attr + 2904 flow_h->size); 2905 } 2906 /* At this point, the flow match. */ 2907 break; 2908 } 2909 if (flow) 2910 priv_flow_destroy(priv, &priv->flows, flow); 2911 exit: 2912 if (parser.drop) { 2913 rte_free(parser.drop_q.ibv_attr); 2914 } else { 2915 for (i = 0; i != hash_rxq_init_n; ++i) { 2916 if (parser.queue[i].ibv_attr) 2917 rte_free(parser.queue[i].ibv_attr); 2918 } 2919 } 2920 return -ret; 2921 } 2922 2923 /** 2924 * Update queue for specific filter. 2925 * 2926 * @param priv 2927 * Private structure. 2928 * @param fdir_filter 2929 * Filter to be updated. 2930 * 2931 * @return 2932 * 0 on success, errno value on failure. 2933 */ 2934 static int 2935 priv_fdir_filter_update(struct priv *priv, 2936 const struct rte_eth_fdir_filter *fdir_filter) 2937 { 2938 int ret; 2939 2940 ret = priv_fdir_filter_delete(priv, fdir_filter); 2941 if (ret) 2942 return ret; 2943 ret = priv_fdir_filter_add(priv, fdir_filter); 2944 return ret; 2945 } 2946 2947 /** 2948 * Flush all filters. 2949 * 2950 * @param priv 2951 * Private structure. 2952 */ 2953 static void 2954 priv_fdir_filter_flush(struct priv *priv) 2955 { 2956 priv_flow_flush(priv, &priv->flows); 2957 } 2958 2959 /** 2960 * Get flow director information. 2961 * 2962 * @param priv 2963 * Private structure. 2964 * @param[out] fdir_info 2965 * Resulting flow director information. 2966 */ 2967 static void 2968 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) 2969 { 2970 struct rte_eth_fdir_masks *mask = 2971 &priv->dev->data->dev_conf.fdir_conf.mask; 2972 2973 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; 2974 fdir_info->guarant_spc = 0; 2975 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 2976 fdir_info->max_flexpayload = 0; 2977 fdir_info->flow_types_mask[0] = 0; 2978 fdir_info->flex_payload_unit = 0; 2979 fdir_info->max_flex_payload_segment_num = 0; 2980 fdir_info->flex_payload_limit = 0; 2981 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 2982 } 2983 2984 /** 2985 * Deal with flow director operations. 2986 * 2987 * @param priv 2988 * Pointer to private structure. 2989 * @param filter_op 2990 * Operation to perform. 2991 * @param arg 2992 * Pointer to operation-specific structure. 2993 * 2994 * @return 2995 * 0 on success, errno value on failure. 2996 */ 2997 static int 2998 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) 2999 { 3000 enum rte_fdir_mode fdir_mode = 3001 priv->dev->data->dev_conf.fdir_conf.mode; 3002 int ret = 0; 3003 3004 if (filter_op == RTE_ETH_FILTER_NOP) 3005 return 0; 3006 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 3007 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 3008 ERROR("%p: flow director mode %d not supported", 3009 (void *)priv, fdir_mode); 3010 return EINVAL; 3011 } 3012 switch (filter_op) { 3013 case RTE_ETH_FILTER_ADD: 3014 ret = priv_fdir_filter_add(priv, arg); 3015 break; 3016 case RTE_ETH_FILTER_UPDATE: 3017 ret = priv_fdir_filter_update(priv, arg); 3018 break; 3019 case RTE_ETH_FILTER_DELETE: 3020 ret = priv_fdir_filter_delete(priv, arg); 3021 break; 3022 case RTE_ETH_FILTER_FLUSH: 3023 priv_fdir_filter_flush(priv); 3024 break; 3025 case RTE_ETH_FILTER_INFO: 3026 priv_fdir_info_get(priv, arg); 3027 break; 3028 default: 3029 DEBUG("%p: unknown operation %u", (void *)priv, 3030 filter_op); 3031 ret = EINVAL; 3032 break; 3033 } 3034 return ret; 3035 } 3036 3037 /** 3038 * Manage filter operations. 3039 * 3040 * @param dev 3041 * Pointer to Ethernet device structure. 3042 * @param filter_type 3043 * Filter type. 3044 * @param filter_op 3045 * Operation to perform. 3046 * @param arg 3047 * Pointer to operation-specific structure. 3048 * 3049 * @return 3050 * 0 on success, negative errno value on failure. 3051 */ 3052 int 3053 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 3054 enum rte_filter_type filter_type, 3055 enum rte_filter_op filter_op, 3056 void *arg) 3057 { 3058 int ret = EINVAL; 3059 struct priv *priv = dev->data->dev_private; 3060 3061 switch (filter_type) { 3062 case RTE_ETH_FILTER_GENERIC: 3063 if (filter_op != RTE_ETH_FILTER_GET) 3064 return -EINVAL; 3065 *(const void **)arg = &mlx5_flow_ops; 3066 return 0; 3067 case RTE_ETH_FILTER_FDIR: 3068 priv_lock(priv); 3069 ret = priv_fdir_ctrl_func(priv, filter_op, arg); 3070 priv_unlock(priv); 3071 break; 3072 default: 3073 ERROR("%p: filter type (%d) not supported", 3074 (void *)dev, filter_type); 3075 break; 3076 } 3077 return -ret; 3078 } 3079