1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2016 6WIND S.A. 5 * Copyright 2016 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/queue.h> 35 #include <string.h> 36 37 /* Verbs header. */ 38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 39 #ifdef PEDANTIC 40 #pragma GCC diagnostic ignored "-Wpedantic" 41 #endif 42 #include <infiniband/verbs.h> 43 #ifdef PEDANTIC 44 #pragma GCC diagnostic error "-Wpedantic" 45 #endif 46 47 #include <rte_ethdev.h> 48 #include <rte_flow.h> 49 #include <rte_flow_driver.h> 50 #include <rte_malloc.h> 51 52 #include "mlx5.h" 53 #include "mlx5_prm.h" 54 55 /* Define minimal priority for control plane flows. */ 56 #define MLX5_CTRL_FLOW_PRIORITY 4 57 58 /* Internet Protocol versions. */ 59 #define MLX5_IPV4 4 60 #define MLX5_IPV6 6 61 62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 63 struct ibv_counter_set_init_attr { 64 int dummy; 65 }; 66 struct ibv_flow_spec_counter_action { 67 int dummy; 68 }; 69 struct ibv_counter_set { 70 int dummy; 71 }; 72 73 static inline int 74 ibv_destroy_counter_set(struct ibv_counter_set *cs) 75 { 76 (void)cs; 77 return -ENOTSUP; 78 } 79 #endif 80 81 /* Dev ops structure defined in mlx5.c */ 82 extern const struct eth_dev_ops mlx5_dev_ops; 83 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 84 85 static int 86 mlx5_flow_create_eth(const struct rte_flow_item *item, 87 const void *default_mask, 88 void *data); 89 90 static int 91 mlx5_flow_create_vlan(const struct rte_flow_item *item, 92 const void *default_mask, 93 void *data); 94 95 static int 96 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 97 const void *default_mask, 98 void *data); 99 100 static int 101 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 102 const void *default_mask, 103 void *data); 104 105 static int 106 mlx5_flow_create_udp(const struct rte_flow_item *item, 107 const void *default_mask, 108 void *data); 109 110 static int 111 mlx5_flow_create_tcp(const struct rte_flow_item *item, 112 const void *default_mask, 113 void *data); 114 115 static int 116 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 117 const void *default_mask, 118 void *data); 119 120 struct mlx5_flow_parse; 121 122 static void 123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 124 unsigned int size); 125 126 static int 127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); 128 129 static int 130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser); 131 132 /* Hash RX queue types. */ 133 enum hash_rxq_type { 134 HASH_RXQ_TCPV4, 135 HASH_RXQ_UDPV4, 136 HASH_RXQ_IPV4, 137 HASH_RXQ_TCPV6, 138 HASH_RXQ_UDPV6, 139 HASH_RXQ_IPV6, 140 HASH_RXQ_ETH, 141 }; 142 143 /* Initialization data for hash RX queue. */ 144 struct hash_rxq_init { 145 uint64_t hash_fields; /* Fields that participate in the hash. */ 146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ 147 unsigned int flow_priority; /* Flow priority to use. */ 148 unsigned int ip_version; /* Internet protocol. */ 149 }; 150 151 /* Initialization data for hash RX queues. */ 152 const struct hash_rxq_init hash_rxq_init[] = { 153 [HASH_RXQ_TCPV4] = { 154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 155 IBV_RX_HASH_DST_IPV4 | 156 IBV_RX_HASH_SRC_PORT_TCP | 157 IBV_RX_HASH_DST_PORT_TCP), 158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, 159 .flow_priority = 0, 160 .ip_version = MLX5_IPV4, 161 }, 162 [HASH_RXQ_UDPV4] = { 163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 164 IBV_RX_HASH_DST_IPV4 | 165 IBV_RX_HASH_SRC_PORT_UDP | 166 IBV_RX_HASH_DST_PORT_UDP), 167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, 168 .flow_priority = 0, 169 .ip_version = MLX5_IPV4, 170 }, 171 [HASH_RXQ_IPV4] = { 172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 | 173 IBV_RX_HASH_DST_IPV4), 174 .dpdk_rss_hf = (ETH_RSS_IPV4 | 175 ETH_RSS_FRAG_IPV4), 176 .flow_priority = 1, 177 .ip_version = MLX5_IPV4, 178 }, 179 [HASH_RXQ_TCPV6] = { 180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 181 IBV_RX_HASH_DST_IPV6 | 182 IBV_RX_HASH_SRC_PORT_TCP | 183 IBV_RX_HASH_DST_PORT_TCP), 184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, 185 .flow_priority = 0, 186 .ip_version = MLX5_IPV6, 187 }, 188 [HASH_RXQ_UDPV6] = { 189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 190 IBV_RX_HASH_DST_IPV6 | 191 IBV_RX_HASH_SRC_PORT_UDP | 192 IBV_RX_HASH_DST_PORT_UDP), 193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, 194 .flow_priority = 0, 195 .ip_version = MLX5_IPV6, 196 }, 197 [HASH_RXQ_IPV6] = { 198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 | 199 IBV_RX_HASH_DST_IPV6), 200 .dpdk_rss_hf = (ETH_RSS_IPV6 | 201 ETH_RSS_FRAG_IPV6), 202 .flow_priority = 1, 203 .ip_version = MLX5_IPV6, 204 }, 205 [HASH_RXQ_ETH] = { 206 .hash_fields = 0, 207 .dpdk_rss_hf = 0, 208 .flow_priority = 2, 209 }, 210 }; 211 212 /* Number of entries in hash_rxq_init[]. */ 213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); 214 215 /** Structure for holding counter stats. */ 216 struct mlx5_flow_counter_stats { 217 uint64_t hits; /**< Number of packets matched by the rule. */ 218 uint64_t bytes; /**< Number of bytes matched by the rule. */ 219 }; 220 221 /** Structure for Drop queue. */ 222 struct mlx5_hrxq_drop { 223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 224 struct ibv_qp *qp; /**< Verbs queue pair. */ 225 struct ibv_wq *wq; /**< Verbs work queue. */ 226 struct ibv_cq *cq; /**< Verbs completion queue. */ 227 }; 228 229 /* Flows structures. */ 230 struct mlx5_flow { 231 uint64_t hash_fields; /**< Fields that participate in the hash. */ 232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 233 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */ 235 }; 236 237 /* Drop flows structures. */ 238 struct mlx5_flow_drop { 239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 240 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 241 }; 242 243 struct rte_flow { 244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ 245 uint32_t mark:1; /**< Set if the flow is marked. */ 246 uint32_t drop:1; /**< Drop queue. */ 247 uint16_t queues_n; /**< Number of entries in queue[]. */ 248 uint16_t (*queues)[]; /**< Queues indexes to use. */ 249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 250 uint8_t rss_key[40]; /**< copy of the RSS key. */ 251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */ 252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */ 253 union { 254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)]; 255 /**< Flow with Rx queue. */ 256 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */ 257 }; 258 }; 259 260 /** Static initializer for items. */ 261 #define ITEMS(...) \ 262 (const enum rte_flow_item_type []){ \ 263 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ 264 } 265 266 /** Structure to generate a simple graph of layers supported by the NIC. */ 267 struct mlx5_flow_items { 268 /** List of possible actions for these items. */ 269 const enum rte_flow_action_type *const actions; 270 /** Bit-masks corresponding to the possibilities for the item. */ 271 const void *mask; 272 /** 273 * Default bit-masks to use when item->mask is not provided. When 274 * \default_mask is also NULL, the full supported bit-mask (\mask) is 275 * used instead. 276 */ 277 const void *default_mask; 278 /** Bit-masks size in bytes. */ 279 const unsigned int mask_sz; 280 /** 281 * Conversion function from rte_flow to NIC specific flow. 282 * 283 * @param item 284 * rte_flow item to convert. 285 * @param default_mask 286 * Default bit-masks to use when item->mask is not provided. 287 * @param data 288 * Internal structure to store the conversion. 289 * 290 * @return 291 * 0 on success, negative value otherwise. 292 */ 293 int (*convert)(const struct rte_flow_item *item, 294 const void *default_mask, 295 void *data); 296 /** Size in bytes of the destination structure. */ 297 const unsigned int dst_sz; 298 /** List of possible following items. */ 299 const enum rte_flow_item_type *const items; 300 }; 301 302 /** Valid action for this PMD. */ 303 static const enum rte_flow_action_type valid_actions[] = { 304 RTE_FLOW_ACTION_TYPE_DROP, 305 RTE_FLOW_ACTION_TYPE_QUEUE, 306 RTE_FLOW_ACTION_TYPE_MARK, 307 RTE_FLOW_ACTION_TYPE_FLAG, 308 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 309 RTE_FLOW_ACTION_TYPE_COUNT, 310 #endif 311 RTE_FLOW_ACTION_TYPE_END, 312 }; 313 314 /** Graph of supported items and associated actions. */ 315 static const struct mlx5_flow_items mlx5_flow_items[] = { 316 [RTE_FLOW_ITEM_TYPE_END] = { 317 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, 318 RTE_FLOW_ITEM_TYPE_VXLAN), 319 }, 320 [RTE_FLOW_ITEM_TYPE_ETH] = { 321 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, 322 RTE_FLOW_ITEM_TYPE_IPV4, 323 RTE_FLOW_ITEM_TYPE_IPV6), 324 .actions = valid_actions, 325 .mask = &(const struct rte_flow_item_eth){ 326 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 327 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 328 .type = -1, 329 }, 330 .default_mask = &rte_flow_item_eth_mask, 331 .mask_sz = sizeof(struct rte_flow_item_eth), 332 .convert = mlx5_flow_create_eth, 333 .dst_sz = sizeof(struct ibv_flow_spec_eth), 334 }, 335 [RTE_FLOW_ITEM_TYPE_VLAN] = { 336 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, 337 RTE_FLOW_ITEM_TYPE_IPV6), 338 .actions = valid_actions, 339 .mask = &(const struct rte_flow_item_vlan){ 340 .tci = -1, 341 }, 342 .default_mask = &rte_flow_item_vlan_mask, 343 .mask_sz = sizeof(struct rte_flow_item_vlan), 344 .convert = mlx5_flow_create_vlan, 345 .dst_sz = 0, 346 }, 347 [RTE_FLOW_ITEM_TYPE_IPV4] = { 348 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 349 RTE_FLOW_ITEM_TYPE_TCP), 350 .actions = valid_actions, 351 .mask = &(const struct rte_flow_item_ipv4){ 352 .hdr = { 353 .src_addr = -1, 354 .dst_addr = -1, 355 .type_of_service = -1, 356 .next_proto_id = -1, 357 }, 358 }, 359 .default_mask = &rte_flow_item_ipv4_mask, 360 .mask_sz = sizeof(struct rte_flow_item_ipv4), 361 .convert = mlx5_flow_create_ipv4, 362 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), 363 }, 364 [RTE_FLOW_ITEM_TYPE_IPV6] = { 365 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 366 RTE_FLOW_ITEM_TYPE_TCP), 367 .actions = valid_actions, 368 .mask = &(const struct rte_flow_item_ipv6){ 369 .hdr = { 370 .src_addr = { 371 0xff, 0xff, 0xff, 0xff, 372 0xff, 0xff, 0xff, 0xff, 373 0xff, 0xff, 0xff, 0xff, 374 0xff, 0xff, 0xff, 0xff, 375 }, 376 .dst_addr = { 377 0xff, 0xff, 0xff, 0xff, 378 0xff, 0xff, 0xff, 0xff, 379 0xff, 0xff, 0xff, 0xff, 380 0xff, 0xff, 0xff, 0xff, 381 }, 382 .vtc_flow = -1, 383 .proto = -1, 384 .hop_limits = -1, 385 }, 386 }, 387 .default_mask = &rte_flow_item_ipv6_mask, 388 .mask_sz = sizeof(struct rte_flow_item_ipv6), 389 .convert = mlx5_flow_create_ipv6, 390 .dst_sz = sizeof(struct ibv_flow_spec_ipv6), 391 }, 392 [RTE_FLOW_ITEM_TYPE_UDP] = { 393 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), 394 .actions = valid_actions, 395 .mask = &(const struct rte_flow_item_udp){ 396 .hdr = { 397 .src_port = -1, 398 .dst_port = -1, 399 }, 400 }, 401 .default_mask = &rte_flow_item_udp_mask, 402 .mask_sz = sizeof(struct rte_flow_item_udp), 403 .convert = mlx5_flow_create_udp, 404 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 405 }, 406 [RTE_FLOW_ITEM_TYPE_TCP] = { 407 .actions = valid_actions, 408 .mask = &(const struct rte_flow_item_tcp){ 409 .hdr = { 410 .src_port = -1, 411 .dst_port = -1, 412 }, 413 }, 414 .default_mask = &rte_flow_item_tcp_mask, 415 .mask_sz = sizeof(struct rte_flow_item_tcp), 416 .convert = mlx5_flow_create_tcp, 417 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 418 }, 419 [RTE_FLOW_ITEM_TYPE_VXLAN] = { 420 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), 421 .actions = valid_actions, 422 .mask = &(const struct rte_flow_item_vxlan){ 423 .vni = "\xff\xff\xff", 424 }, 425 .default_mask = &rte_flow_item_vxlan_mask, 426 .mask_sz = sizeof(struct rte_flow_item_vxlan), 427 .convert = mlx5_flow_create_vxlan, 428 .dst_sz = sizeof(struct ibv_flow_spec_tunnel), 429 }, 430 }; 431 432 /** Structure to pass to the conversion function. */ 433 struct mlx5_flow_parse { 434 uint32_t inner; /**< Set once VXLAN is encountered. */ 435 uint32_t create:1; 436 /**< Whether resources should remain after a validate. */ 437 uint32_t drop:1; /**< Target is a drop queue. */ 438 uint32_t mark:1; /**< Mark is present in the flow. */ 439 uint32_t count:1; /**< Count is present in the flow. */ 440 uint32_t mark_id; /**< Mark identifier. */ 441 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ 442 uint16_t queues_n; /**< Number of entries in queue[]. */ 443 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ 444 uint8_t rss_key[40]; /**< copy of the RSS key. */ 445 enum hash_rxq_type layer; /**< Last pattern layer detected. */ 446 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */ 447 union { 448 struct { 449 struct ibv_flow_attr *ibv_attr; 450 /**< Pointer to Verbs attributes. */ 451 unsigned int offset; 452 /**< Current position or total size of the attribute. */ 453 } queue[RTE_DIM(hash_rxq_init)]; 454 struct { 455 struct ibv_flow_attr *ibv_attr; 456 /**< Pointer to Verbs attributes. */ 457 unsigned int offset; 458 /**< Current position or total size of the attribute. */ 459 } drop_q; 460 }; 461 }; 462 463 static const struct rte_flow_ops mlx5_flow_ops = { 464 .validate = mlx5_flow_validate, 465 .create = mlx5_flow_create, 466 .destroy = mlx5_flow_destroy, 467 .flush = mlx5_flow_flush, 468 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 469 .query = mlx5_flow_query, 470 #else 471 .query = NULL, 472 #endif 473 .isolate = mlx5_flow_isolate, 474 }; 475 476 /* Convert FDIR request to Generic flow. */ 477 struct mlx5_fdir { 478 struct rte_flow_attr attr; 479 struct rte_flow_action actions[2]; 480 struct rte_flow_item items[4]; 481 struct rte_flow_item_eth l2; 482 struct rte_flow_item_eth l2_mask; 483 union { 484 struct rte_flow_item_ipv4 ipv4; 485 struct rte_flow_item_ipv6 ipv6; 486 } l3; 487 union { 488 struct rte_flow_item_udp udp; 489 struct rte_flow_item_tcp tcp; 490 } l4; 491 struct rte_flow_action_queue queue; 492 }; 493 494 /* Verbs specification header. */ 495 struct ibv_spec_header { 496 enum ibv_flow_spec_type type; 497 uint16_t size; 498 }; 499 500 /** 501 * Check support for a given item. 502 * 503 * @param item[in] 504 * Item specification. 505 * @param mask[in] 506 * Bit-masks covering supported fields to compare with spec, last and mask in 507 * \item. 508 * @param size 509 * Bit-Mask size in bytes. 510 * 511 * @return 512 * 0 on success. 513 */ 514 static int 515 mlx5_flow_item_validate(const struct rte_flow_item *item, 516 const uint8_t *mask, unsigned int size) 517 { 518 int ret = 0; 519 520 if (!item->spec && (item->mask || item->last)) 521 return -1; 522 if (item->spec && !item->mask) { 523 unsigned int i; 524 const uint8_t *spec = item->spec; 525 526 for (i = 0; i < size; ++i) 527 if ((spec[i] | mask[i]) != mask[i]) 528 return -1; 529 } 530 if (item->last && !item->mask) { 531 unsigned int i; 532 const uint8_t *spec = item->last; 533 534 for (i = 0; i < size; ++i) 535 if ((spec[i] | mask[i]) != mask[i]) 536 return -1; 537 } 538 if (item->mask) { 539 unsigned int i; 540 const uint8_t *spec = item->mask; 541 542 for (i = 0; i < size; ++i) 543 if ((spec[i] | mask[i]) != mask[i]) 544 return -1; 545 } 546 if (item->spec && item->last) { 547 uint8_t spec[size]; 548 uint8_t last[size]; 549 const uint8_t *apply = mask; 550 unsigned int i; 551 552 if (item->mask) 553 apply = item->mask; 554 for (i = 0; i < size; ++i) { 555 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; 556 last[i] = ((const uint8_t *)item->last)[i] & apply[i]; 557 } 558 ret = memcmp(spec, last, size); 559 } 560 return ret; 561 } 562 563 /** 564 * Copy the RSS configuration from the user ones. 565 * 566 * @param priv 567 * Pointer to private structure. 568 * @param parser 569 * Internal parser structure. 570 * @param rss_conf 571 * User RSS configuration to save. 572 * 573 * @return 574 * 0 on success, errno value on failure. 575 */ 576 static int 577 priv_flow_convert_rss_conf(struct priv *priv, 578 struct mlx5_flow_parse *parser, 579 const struct rte_eth_rss_conf *rss_conf) 580 { 581 const struct rte_eth_rss_conf *rss = 582 rss_conf ? rss_conf : &priv->rss_conf; 583 584 if (rss->rss_key_len > 40) 585 return EINVAL; 586 parser->rss_conf.rss_key_len = rss->rss_key_len; 587 parser->rss_conf.rss_hf = rss->rss_hf; 588 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len); 589 parser->rss_conf.rss_key = parser->rss_key; 590 return 0; 591 } 592 593 /** 594 * Extract attribute to the parser. 595 * 596 * @param priv 597 * Pointer to private structure. 598 * @param[in] attr 599 * Flow rule attributes. 600 * @param[out] error 601 * Perform verbose error reporting if not NULL. 602 * @param[in, out] parser 603 * Internal parser structure. 604 * 605 * @return 606 * 0 on success, a negative errno value otherwise and rte_errno is set. 607 */ 608 static int 609 priv_flow_convert_attributes(struct priv *priv, 610 const struct rte_flow_attr *attr, 611 struct rte_flow_error *error, 612 struct mlx5_flow_parse *parser) 613 { 614 (void)priv; 615 (void)parser; 616 if (attr->group) { 617 rte_flow_error_set(error, ENOTSUP, 618 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 619 NULL, 620 "groups are not supported"); 621 return -rte_errno; 622 } 623 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) { 624 rte_flow_error_set(error, ENOTSUP, 625 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 626 NULL, 627 "priorities are not supported"); 628 return -rte_errno; 629 } 630 if (attr->egress) { 631 rte_flow_error_set(error, ENOTSUP, 632 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 633 NULL, 634 "egress is not supported"); 635 return -rte_errno; 636 } 637 if (!attr->ingress) { 638 rte_flow_error_set(error, ENOTSUP, 639 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 640 NULL, 641 "only ingress is supported"); 642 return -rte_errno; 643 } 644 return 0; 645 } 646 647 /** 648 * Extract actions request to the parser. 649 * 650 * @param priv 651 * Pointer to private structure. 652 * @param[in] actions 653 * Associated actions (list terminated by the END action). 654 * @param[out] error 655 * Perform verbose error reporting if not NULL. 656 * @param[in, out] parser 657 * Internal parser structure. 658 * 659 * @return 660 * 0 on success, a negative errno value otherwise and rte_errno is set. 661 */ 662 static int 663 priv_flow_convert_actions(struct priv *priv, 664 const struct rte_flow_action actions[], 665 struct rte_flow_error *error, 666 struct mlx5_flow_parse *parser) 667 { 668 /* 669 * Add default RSS configuration necessary for Verbs to create QP even 670 * if no RSS is necessary. 671 */ 672 priv_flow_convert_rss_conf(priv, parser, 673 (const struct rte_eth_rss_conf *) 674 &priv->rss_conf); 675 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { 676 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { 677 continue; 678 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { 679 parser->drop = 1; 680 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 681 const struct rte_flow_action_queue *queue = 682 (const struct rte_flow_action_queue *) 683 actions->conf; 684 uint16_t n; 685 uint16_t found = 0; 686 687 if (!queue || (queue->index > (priv->rxqs_n - 1))) 688 goto exit_action_not_supported; 689 for (n = 0; n < parser->queues_n; ++n) { 690 if (parser->queues[n] == queue->index) { 691 found = 1; 692 break; 693 } 694 } 695 if (parser->queues_n > 1 && !found) { 696 rte_flow_error_set(error, ENOTSUP, 697 RTE_FLOW_ERROR_TYPE_ACTION, 698 actions, 699 "queue action not in RSS queues"); 700 return -rte_errno; 701 } 702 if (!found) { 703 parser->queues_n = 1; 704 parser->queues[0] = queue->index; 705 } 706 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { 707 const struct rte_flow_action_rss *rss = 708 (const struct rte_flow_action_rss *) 709 actions->conf; 710 uint16_t n; 711 712 if (!rss || !rss->num) { 713 rte_flow_error_set(error, EINVAL, 714 RTE_FLOW_ERROR_TYPE_ACTION, 715 actions, 716 "no valid queues"); 717 return -rte_errno; 718 } 719 if (parser->queues_n == 1) { 720 uint16_t found = 0; 721 722 assert(parser->queues_n); 723 for (n = 0; n < rss->num; ++n) { 724 if (parser->queues[0] == 725 rss->queue[n]) { 726 found = 1; 727 break; 728 } 729 } 730 if (!found) { 731 rte_flow_error_set(error, ENOTSUP, 732 RTE_FLOW_ERROR_TYPE_ACTION, 733 actions, 734 "queue action not in RSS" 735 " queues"); 736 return -rte_errno; 737 } 738 } 739 for (n = 0; n < rss->num; ++n) { 740 if (rss->queue[n] >= priv->rxqs_n) { 741 rte_flow_error_set(error, EINVAL, 742 RTE_FLOW_ERROR_TYPE_ACTION, 743 actions, 744 "queue id > number of" 745 " queues"); 746 return -rte_errno; 747 } 748 } 749 for (n = 0; n < rss->num; ++n) 750 parser->queues[n] = rss->queue[n]; 751 parser->queues_n = rss->num; 752 if (priv_flow_convert_rss_conf(priv, parser, 753 rss->rss_conf)) { 754 rte_flow_error_set(error, EINVAL, 755 RTE_FLOW_ERROR_TYPE_ACTION, 756 actions, 757 "wrong RSS configuration"); 758 return -rte_errno; 759 } 760 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { 761 const struct rte_flow_action_mark *mark = 762 (const struct rte_flow_action_mark *) 763 actions->conf; 764 765 if (!mark) { 766 rte_flow_error_set(error, EINVAL, 767 RTE_FLOW_ERROR_TYPE_ACTION, 768 actions, 769 "mark must be defined"); 770 return -rte_errno; 771 } else if (mark->id >= MLX5_FLOW_MARK_MAX) { 772 rte_flow_error_set(error, ENOTSUP, 773 RTE_FLOW_ERROR_TYPE_ACTION, 774 actions, 775 "mark must be between 0" 776 " and 16777199"); 777 return -rte_errno; 778 } 779 parser->mark = 1; 780 parser->mark_id = mark->id; 781 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { 782 parser->mark = 1; 783 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && 784 priv->counter_set_supported) { 785 parser->count = 1; 786 } else { 787 goto exit_action_not_supported; 788 } 789 } 790 if (parser->drop && parser->mark) 791 parser->mark = 0; 792 if (!parser->queues_n && !parser->drop) { 793 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, 794 NULL, "no valid action"); 795 return -rte_errno; 796 } 797 return 0; 798 exit_action_not_supported: 799 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 800 actions, "action not supported"); 801 return -rte_errno; 802 } 803 804 /** 805 * Validate items. 806 * 807 * @param priv 808 * Pointer to private structure. 809 * @param[in] items 810 * Pattern specification (list terminated by the END pattern item). 811 * @param[out] error 812 * Perform verbose error reporting if not NULL. 813 * @param[in, out] parser 814 * Internal parser structure. 815 * 816 * @return 817 * 0 on success, a negative errno value otherwise and rte_errno is set. 818 */ 819 static int 820 priv_flow_convert_items_validate(struct priv *priv, 821 const struct rte_flow_item items[], 822 struct rte_flow_error *error, 823 struct mlx5_flow_parse *parser) 824 { 825 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 826 unsigned int i; 827 828 (void)priv; 829 /* Initialise the offsets to start after verbs attribute. */ 830 if (parser->drop) { 831 parser->drop_q.offset = sizeof(struct ibv_flow_attr); 832 } else { 833 for (i = 0; i != hash_rxq_init_n; ++i) 834 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 835 } 836 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 837 const struct mlx5_flow_items *token = NULL; 838 unsigned int n; 839 int err; 840 841 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 842 continue; 843 for (i = 0; 844 cur_item->items && 845 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; 846 ++i) { 847 if (cur_item->items[i] == items->type) { 848 token = &mlx5_flow_items[items->type]; 849 break; 850 } 851 } 852 if (!token) 853 goto exit_item_not_supported; 854 cur_item = token; 855 err = mlx5_flow_item_validate(items, 856 (const uint8_t *)cur_item->mask, 857 cur_item->mask_sz); 858 if (err) 859 goto exit_item_not_supported; 860 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 861 if (parser->inner) { 862 rte_flow_error_set(error, ENOTSUP, 863 RTE_FLOW_ERROR_TYPE_ITEM, 864 items, 865 "cannot recognize multiple" 866 " VXLAN encapsulations"); 867 return -rte_errno; 868 } 869 parser->inner = IBV_FLOW_SPEC_INNER; 870 } 871 if (parser->drop) { 872 parser->drop_q.offset += cur_item->dst_sz; 873 } else if (parser->queues_n == 1) { 874 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; 875 } else { 876 for (n = 0; n != hash_rxq_init_n; ++n) 877 parser->queue[n].offset += cur_item->dst_sz; 878 } 879 } 880 if (parser->mark) { 881 for (i = 0; i != hash_rxq_init_n; ++i) 882 parser->queue[i].offset += 883 sizeof(struct ibv_flow_spec_action_tag); 884 } 885 if (parser->count) { 886 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 887 888 if (parser->drop) { 889 parser->drop_q.offset += size; 890 } else { 891 for (i = 0; i != hash_rxq_init_n; ++i) 892 parser->queue[i].offset += size; 893 } 894 } 895 return 0; 896 exit_item_not_supported: 897 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 898 items, "item not supported"); 899 return -rte_errno; 900 } 901 902 /** 903 * Allocate memory space to store verbs flow attributes. 904 * 905 * @param priv 906 * Pointer to private structure. 907 * @param[in] priority 908 * Flow priority. 909 * @param[in] size 910 * Amount of byte to allocate. 911 * @param[out] error 912 * Perform verbose error reporting if not NULL. 913 * 914 * @return 915 * A verbs flow attribute on success, NULL otherwise. 916 */ 917 static struct ibv_flow_attr* 918 priv_flow_convert_allocate(struct priv *priv, 919 unsigned int priority, 920 unsigned int size, 921 struct rte_flow_error *error) 922 { 923 struct ibv_flow_attr *ibv_attr; 924 925 (void)priv; 926 ibv_attr = rte_calloc(__func__, 1, size, 0); 927 if (!ibv_attr) { 928 rte_flow_error_set(error, ENOMEM, 929 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 930 NULL, 931 "cannot allocate verbs spec attributes."); 932 return NULL; 933 } 934 ibv_attr->priority = priority; 935 return ibv_attr; 936 } 937 938 /** 939 * Finalise verbs flow attributes. 940 * 941 * @param priv 942 * Pointer to private structure. 943 * @param[in, out] parser 944 * Internal parser structure. 945 */ 946 static void 947 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) 948 { 949 const unsigned int ipv4 = 950 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; 951 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6; 952 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 953 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4; 954 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4; 955 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; 956 unsigned int i; 957 958 (void)priv; 959 if (parser->layer == HASH_RXQ_ETH) { 960 goto fill; 961 } else { 962 /* 963 * This layer becomes useless as the pattern define under 964 * layers. 965 */ 966 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr); 967 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; 968 } 969 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */ 970 for (i = ohmin; i != (ohmax + 1); ++i) { 971 if (!parser->queue[i].ibv_attr) 972 continue; 973 rte_free(parser->queue[i].ibv_attr); 974 parser->queue[i].ibv_attr = NULL; 975 } 976 /* Remove impossible flow according to the RSS configuration. */ 977 if (hash_rxq_init[parser->layer].dpdk_rss_hf & 978 parser->rss_conf.rss_hf) { 979 /* Remove any other flow. */ 980 for (i = hmin; i != (hmax + 1); ++i) { 981 if ((i == parser->layer) || 982 (!parser->queue[i].ibv_attr)) 983 continue; 984 rte_free(parser->queue[i].ibv_attr); 985 parser->queue[i].ibv_attr = NULL; 986 } 987 } else if (!parser->queue[ip].ibv_attr) { 988 /* no RSS possible with the current configuration. */ 989 parser->queues_n = 1; 990 return; 991 } 992 fill: 993 /* 994 * Fill missing layers in verbs specifications, or compute the correct 995 * offset to allocate the memory space for the attributes and 996 * specifications. 997 */ 998 for (i = 0; i != hash_rxq_init_n - 1; ++i) { 999 union { 1000 struct ibv_flow_spec_ipv4_ext ipv4; 1001 struct ibv_flow_spec_ipv6 ipv6; 1002 struct ibv_flow_spec_tcp_udp udp_tcp; 1003 } specs; 1004 void *dst; 1005 uint16_t size; 1006 1007 if (i == parser->layer) 1008 continue; 1009 if (parser->layer == HASH_RXQ_ETH) { 1010 if (hash_rxq_init[i].ip_version == MLX5_IPV4) { 1011 size = sizeof(struct ibv_flow_spec_ipv4_ext); 1012 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){ 1013 .type = IBV_FLOW_SPEC_IPV4_EXT, 1014 .size = size, 1015 }; 1016 } else { 1017 size = sizeof(struct ibv_flow_spec_ipv6); 1018 specs.ipv6 = (struct ibv_flow_spec_ipv6){ 1019 .type = IBV_FLOW_SPEC_IPV6, 1020 .size = size, 1021 }; 1022 } 1023 if (parser->queue[i].ibv_attr) { 1024 dst = (void *)((uintptr_t) 1025 parser->queue[i].ibv_attr + 1026 parser->queue[i].offset); 1027 memcpy(dst, &specs, size); 1028 ++parser->queue[i].ibv_attr->num_of_specs; 1029 } 1030 parser->queue[i].offset += size; 1031 } 1032 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) || 1033 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) { 1034 size = sizeof(struct ibv_flow_spec_tcp_udp); 1035 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) { 1036 .type = ((i == HASH_RXQ_UDPV4 || 1037 i == HASH_RXQ_UDPV6) ? 1038 IBV_FLOW_SPEC_UDP : 1039 IBV_FLOW_SPEC_TCP), 1040 .size = size, 1041 }; 1042 if (parser->queue[i].ibv_attr) { 1043 dst = (void *)((uintptr_t) 1044 parser->queue[i].ibv_attr + 1045 parser->queue[i].offset); 1046 memcpy(dst, &specs, size); 1047 ++parser->queue[i].ibv_attr->num_of_specs; 1048 } 1049 parser->queue[i].offset += size; 1050 } 1051 } 1052 } 1053 1054 /** 1055 * Validate and convert a flow supported by the NIC. 1056 * 1057 * @param priv 1058 * Pointer to private structure. 1059 * @param[in] attr 1060 * Flow rule attributes. 1061 * @param[in] pattern 1062 * Pattern specification (list terminated by the END pattern item). 1063 * @param[in] actions 1064 * Associated actions (list terminated by the END action). 1065 * @param[out] error 1066 * Perform verbose error reporting if not NULL. 1067 * @param[in, out] parser 1068 * Internal parser structure. 1069 * 1070 * @return 1071 * 0 on success, a negative errno value otherwise and rte_errno is set. 1072 */ 1073 static int 1074 priv_flow_convert(struct priv *priv, 1075 const struct rte_flow_attr *attr, 1076 const struct rte_flow_item items[], 1077 const struct rte_flow_action actions[], 1078 struct rte_flow_error *error, 1079 struct mlx5_flow_parse *parser) 1080 { 1081 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 1082 unsigned int i; 1083 int ret; 1084 1085 /* First step. Validate the attributes, items and actions. */ 1086 *parser = (struct mlx5_flow_parse){ 1087 .create = parser->create, 1088 .layer = HASH_RXQ_ETH, 1089 .mark_id = MLX5_FLOW_MARK_DEFAULT, 1090 }; 1091 ret = priv_flow_convert_attributes(priv, attr, error, parser); 1092 if (ret) 1093 return ret; 1094 ret = priv_flow_convert_actions(priv, actions, error, parser); 1095 if (ret) 1096 return ret; 1097 ret = priv_flow_convert_items_validate(priv, items, error, parser); 1098 if (ret) 1099 return ret; 1100 priv_flow_convert_finalise(priv, parser); 1101 /* 1102 * Second step. 1103 * Allocate the memory space to store verbs specifications. 1104 */ 1105 if (parser->drop) { 1106 parser->drop_q.ibv_attr = 1107 priv_flow_convert_allocate(priv, attr->priority, 1108 parser->drop_q.offset, 1109 error); 1110 if (!parser->drop_q.ibv_attr) 1111 return ENOMEM; 1112 parser->drop_q.offset = sizeof(struct ibv_flow_attr); 1113 } else if (parser->queues_n == 1) { 1114 unsigned int priority = 1115 attr->priority + 1116 hash_rxq_init[HASH_RXQ_ETH].flow_priority; 1117 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; 1118 1119 parser->queue[HASH_RXQ_ETH].ibv_attr = 1120 priv_flow_convert_allocate(priv, priority, 1121 offset, error); 1122 if (!parser->queue[HASH_RXQ_ETH].ibv_attr) 1123 return ENOMEM; 1124 parser->queue[HASH_RXQ_ETH].offset = 1125 sizeof(struct ibv_flow_attr); 1126 } else { 1127 for (i = 0; i != hash_rxq_init_n; ++i) { 1128 unsigned int priority = 1129 attr->priority + 1130 hash_rxq_init[i].flow_priority; 1131 unsigned int offset; 1132 1133 if (!(parser->rss_conf.rss_hf & 1134 hash_rxq_init[i].dpdk_rss_hf) && 1135 (i != HASH_RXQ_ETH)) 1136 continue; 1137 offset = parser->queue[i].offset; 1138 parser->queue[i].ibv_attr = 1139 priv_flow_convert_allocate(priv, priority, 1140 offset, error); 1141 if (!parser->queue[i].ibv_attr) 1142 goto exit_enomem; 1143 parser->queue[i].offset = sizeof(struct ibv_flow_attr); 1144 } 1145 } 1146 /* Third step. Conversion parse, fill the specifications. */ 1147 parser->inner = 0; 1148 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 1149 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 1150 continue; 1151 cur_item = &mlx5_flow_items[items->type]; 1152 ret = cur_item->convert(items, 1153 (cur_item->default_mask ? 1154 cur_item->default_mask : 1155 cur_item->mask), 1156 parser); 1157 if (ret) { 1158 rte_flow_error_set(error, ret, 1159 RTE_FLOW_ERROR_TYPE_ITEM, 1160 items, "item not supported"); 1161 goto exit_free; 1162 } 1163 } 1164 if (parser->mark) 1165 mlx5_flow_create_flag_mark(parser, parser->mark_id); 1166 if (parser->count && parser->create) { 1167 mlx5_flow_create_count(priv, parser); 1168 if (!parser->cs) 1169 goto exit_count_error; 1170 } 1171 /* 1172 * Last step. Complete missing specification to reach the RSS 1173 * configuration. 1174 */ 1175 if (parser->drop) { 1176 /* 1177 * Drop queue priority needs to be adjusted to 1178 * their most specific layer priority. 1179 */ 1180 parser->drop_q.ibv_attr->priority = 1181 attr->priority + 1182 hash_rxq_init[parser->layer].flow_priority; 1183 } else if (parser->queues_n > 1) { 1184 priv_flow_convert_finalise(priv, parser); 1185 } else { 1186 /* 1187 * Action queue have their priority overridden with 1188 * Ethernet priority, this priority needs to be adjusted to 1189 * their most specific layer priority. 1190 */ 1191 parser->queue[HASH_RXQ_ETH].ibv_attr->priority = 1192 attr->priority + 1193 hash_rxq_init[parser->layer].flow_priority; 1194 } 1195 exit_free: 1196 /* Only verification is expected, all resources should be released. */ 1197 if (!parser->create) { 1198 if (parser->drop) { 1199 rte_free(parser->drop_q.ibv_attr); 1200 parser->drop_q.ibv_attr = NULL; 1201 } 1202 for (i = 0; i != hash_rxq_init_n; ++i) { 1203 if (parser->queue[i].ibv_attr) { 1204 rte_free(parser->queue[i].ibv_attr); 1205 parser->queue[i].ibv_attr = NULL; 1206 } 1207 } 1208 } 1209 return ret; 1210 exit_enomem: 1211 for (i = 0; i != hash_rxq_init_n; ++i) { 1212 if (parser->queue[i].ibv_attr) { 1213 rte_free(parser->queue[i].ibv_attr); 1214 parser->queue[i].ibv_attr = NULL; 1215 } 1216 } 1217 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1218 NULL, "cannot allocate verbs spec attributes."); 1219 return ret; 1220 exit_count_error: 1221 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1222 NULL, "cannot create counter."); 1223 return rte_errno; 1224 } 1225 1226 /** 1227 * Copy the specification created into the flow. 1228 * 1229 * @param parser 1230 * Internal parser structure. 1231 * @param src 1232 * Create specification. 1233 * @param size 1234 * Size in bytes of the specification to copy. 1235 */ 1236 static void 1237 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, 1238 unsigned int size) 1239 { 1240 unsigned int i; 1241 void *dst; 1242 1243 if (parser->drop) { 1244 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1245 parser->drop_q.offset); 1246 memcpy(dst, src, size); 1247 ++parser->drop_q.ibv_attr->num_of_specs; 1248 parser->drop_q.offset += size; 1249 return; 1250 } 1251 for (i = 0; i != hash_rxq_init_n; ++i) { 1252 if (!parser->queue[i].ibv_attr) 1253 continue; 1254 /* Specification must be the same l3 type or none. */ 1255 if (parser->layer == HASH_RXQ_ETH || 1256 (hash_rxq_init[parser->layer].ip_version == 1257 hash_rxq_init[i].ip_version) || 1258 (hash_rxq_init[i].ip_version == 0)) { 1259 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1260 parser->queue[i].offset); 1261 memcpy(dst, src, size); 1262 ++parser->queue[i].ibv_attr->num_of_specs; 1263 parser->queue[i].offset += size; 1264 } 1265 } 1266 } 1267 1268 /** 1269 * Convert Ethernet item to Verbs specification. 1270 * 1271 * @param item[in] 1272 * Item specification. 1273 * @param default_mask[in] 1274 * Default bit-masks to use when item->mask is not provided. 1275 * @param data[in, out] 1276 * User structure. 1277 */ 1278 static int 1279 mlx5_flow_create_eth(const struct rte_flow_item *item, 1280 const void *default_mask, 1281 void *data) 1282 { 1283 const struct rte_flow_item_eth *spec = item->spec; 1284 const struct rte_flow_item_eth *mask = item->mask; 1285 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1286 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1287 struct ibv_flow_spec_eth eth = { 1288 .type = parser->inner | IBV_FLOW_SPEC_ETH, 1289 .size = eth_size, 1290 }; 1291 1292 /* Don't update layer for the inner pattern. */ 1293 if (!parser->inner) 1294 parser->layer = HASH_RXQ_ETH; 1295 if (spec) { 1296 unsigned int i; 1297 1298 if (!mask) 1299 mask = default_mask; 1300 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); 1301 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); 1302 eth.val.ether_type = spec->type; 1303 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); 1304 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); 1305 eth.mask.ether_type = mask->type; 1306 /* Remove unwanted bits from values. */ 1307 for (i = 0; i < ETHER_ADDR_LEN; ++i) { 1308 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 1309 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 1310 } 1311 eth.val.ether_type &= eth.mask.ether_type; 1312 } 1313 mlx5_flow_create_copy(parser, ð, eth_size); 1314 return 0; 1315 } 1316 1317 /** 1318 * Convert VLAN item to Verbs specification. 1319 * 1320 * @param item[in] 1321 * Item specification. 1322 * @param default_mask[in] 1323 * Default bit-masks to use when item->mask is not provided. 1324 * @param data[in, out] 1325 * User structure. 1326 */ 1327 static int 1328 mlx5_flow_create_vlan(const struct rte_flow_item *item, 1329 const void *default_mask, 1330 void *data) 1331 { 1332 const struct rte_flow_item_vlan *spec = item->spec; 1333 const struct rte_flow_item_vlan *mask = item->mask; 1334 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1335 struct ibv_flow_spec_eth *eth; 1336 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 1337 1338 if (spec) { 1339 unsigned int i; 1340 if (!mask) 1341 mask = default_mask; 1342 1343 if (parser->drop) { 1344 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1345 parser->drop_q.offset - eth_size); 1346 eth->val.vlan_tag = spec->tci; 1347 eth->mask.vlan_tag = mask->tci; 1348 eth->val.vlan_tag &= eth->mask.vlan_tag; 1349 return 0; 1350 } 1351 for (i = 0; i != hash_rxq_init_n; ++i) { 1352 if (!parser->queue[i].ibv_attr) 1353 continue; 1354 1355 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr + 1356 parser->queue[i].offset - eth_size); 1357 eth->val.vlan_tag = spec->tci; 1358 eth->mask.vlan_tag = mask->tci; 1359 eth->val.vlan_tag &= eth->mask.vlan_tag; 1360 } 1361 } 1362 return 0; 1363 } 1364 1365 /** 1366 * Convert IPv4 item to Verbs specification. 1367 * 1368 * @param item[in] 1369 * Item specification. 1370 * @param default_mask[in] 1371 * Default bit-masks to use when item->mask is not provided. 1372 * @param data[in, out] 1373 * User structure. 1374 */ 1375 static int 1376 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 1377 const void *default_mask, 1378 void *data) 1379 { 1380 const struct rte_flow_item_ipv4 *spec = item->spec; 1381 const struct rte_flow_item_ipv4 *mask = item->mask; 1382 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1383 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); 1384 struct ibv_flow_spec_ipv4_ext ipv4 = { 1385 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, 1386 .size = ipv4_size, 1387 }; 1388 1389 /* Don't update layer for the inner pattern. */ 1390 if (!parser->inner) 1391 parser->layer = HASH_RXQ_IPV4; 1392 if (spec) { 1393 if (!mask) 1394 mask = default_mask; 1395 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 1396 .src_ip = spec->hdr.src_addr, 1397 .dst_ip = spec->hdr.dst_addr, 1398 .proto = spec->hdr.next_proto_id, 1399 .tos = spec->hdr.type_of_service, 1400 }; 1401 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 1402 .src_ip = mask->hdr.src_addr, 1403 .dst_ip = mask->hdr.dst_addr, 1404 .proto = mask->hdr.next_proto_id, 1405 .tos = mask->hdr.type_of_service, 1406 }; 1407 /* Remove unwanted bits from values. */ 1408 ipv4.val.src_ip &= ipv4.mask.src_ip; 1409 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 1410 ipv4.val.proto &= ipv4.mask.proto; 1411 ipv4.val.tos &= ipv4.mask.tos; 1412 } 1413 mlx5_flow_create_copy(parser, &ipv4, ipv4_size); 1414 return 0; 1415 } 1416 1417 /** 1418 * Convert IPv6 item to Verbs specification. 1419 * 1420 * @param item[in] 1421 * Item specification. 1422 * @param default_mask[in] 1423 * Default bit-masks to use when item->mask is not provided. 1424 * @param data[in, out] 1425 * User structure. 1426 */ 1427 static int 1428 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 1429 const void *default_mask, 1430 void *data) 1431 { 1432 const struct rte_flow_item_ipv6 *spec = item->spec; 1433 const struct rte_flow_item_ipv6 *mask = item->mask; 1434 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1435 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); 1436 struct ibv_flow_spec_ipv6 ipv6 = { 1437 .type = parser->inner | IBV_FLOW_SPEC_IPV6, 1438 .size = ipv6_size, 1439 }; 1440 1441 /* Don't update layer for the inner pattern. */ 1442 if (!parser->inner) 1443 parser->layer = HASH_RXQ_IPV6; 1444 if (spec) { 1445 unsigned int i; 1446 1447 if (!mask) 1448 mask = default_mask; 1449 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 1450 RTE_DIM(ipv6.val.src_ip)); 1451 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 1452 RTE_DIM(ipv6.val.dst_ip)); 1453 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 1454 RTE_DIM(ipv6.mask.src_ip)); 1455 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 1456 RTE_DIM(ipv6.mask.dst_ip)); 1457 ipv6.mask.flow_label = mask->hdr.vtc_flow; 1458 ipv6.mask.next_hdr = mask->hdr.proto; 1459 ipv6.mask.hop_limit = mask->hdr.hop_limits; 1460 /* Remove unwanted bits from values. */ 1461 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 1462 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 1463 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 1464 } 1465 ipv6.val.flow_label &= ipv6.mask.flow_label; 1466 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 1467 ipv6.val.hop_limit &= ipv6.mask.hop_limit; 1468 } 1469 mlx5_flow_create_copy(parser, &ipv6, ipv6_size); 1470 return 0; 1471 } 1472 1473 /** 1474 * Convert UDP item to Verbs specification. 1475 * 1476 * @param item[in] 1477 * Item specification. 1478 * @param default_mask[in] 1479 * Default bit-masks to use when item->mask is not provided. 1480 * @param data[in, out] 1481 * User structure. 1482 */ 1483 static int 1484 mlx5_flow_create_udp(const struct rte_flow_item *item, 1485 const void *default_mask, 1486 void *data) 1487 { 1488 const struct rte_flow_item_udp *spec = item->spec; 1489 const struct rte_flow_item_udp *mask = item->mask; 1490 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1491 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1492 struct ibv_flow_spec_tcp_udp udp = { 1493 .type = parser->inner | IBV_FLOW_SPEC_UDP, 1494 .size = udp_size, 1495 }; 1496 1497 /* Don't update layer for the inner pattern. */ 1498 if (!parser->inner) { 1499 if (parser->layer == HASH_RXQ_IPV4) 1500 parser->layer = HASH_RXQ_UDPV4; 1501 else 1502 parser->layer = HASH_RXQ_UDPV6; 1503 } 1504 if (spec) { 1505 if (!mask) 1506 mask = default_mask; 1507 udp.val.dst_port = spec->hdr.dst_port; 1508 udp.val.src_port = spec->hdr.src_port; 1509 udp.mask.dst_port = mask->hdr.dst_port; 1510 udp.mask.src_port = mask->hdr.src_port; 1511 /* Remove unwanted bits from values. */ 1512 udp.val.src_port &= udp.mask.src_port; 1513 udp.val.dst_port &= udp.mask.dst_port; 1514 } 1515 mlx5_flow_create_copy(parser, &udp, udp_size); 1516 return 0; 1517 } 1518 1519 /** 1520 * Convert TCP item to Verbs specification. 1521 * 1522 * @param item[in] 1523 * Item specification. 1524 * @param default_mask[in] 1525 * Default bit-masks to use when item->mask is not provided. 1526 * @param data[in, out] 1527 * User structure. 1528 */ 1529 static int 1530 mlx5_flow_create_tcp(const struct rte_flow_item *item, 1531 const void *default_mask, 1532 void *data) 1533 { 1534 const struct rte_flow_item_tcp *spec = item->spec; 1535 const struct rte_flow_item_tcp *mask = item->mask; 1536 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1537 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); 1538 struct ibv_flow_spec_tcp_udp tcp = { 1539 .type = parser->inner | IBV_FLOW_SPEC_TCP, 1540 .size = tcp_size, 1541 }; 1542 1543 /* Don't update layer for the inner pattern. */ 1544 if (!parser->inner) { 1545 if (parser->layer == HASH_RXQ_IPV4) 1546 parser->layer = HASH_RXQ_TCPV4; 1547 else 1548 parser->layer = HASH_RXQ_TCPV6; 1549 } 1550 if (spec) { 1551 if (!mask) 1552 mask = default_mask; 1553 tcp.val.dst_port = spec->hdr.dst_port; 1554 tcp.val.src_port = spec->hdr.src_port; 1555 tcp.mask.dst_port = mask->hdr.dst_port; 1556 tcp.mask.src_port = mask->hdr.src_port; 1557 /* Remove unwanted bits from values. */ 1558 tcp.val.src_port &= tcp.mask.src_port; 1559 tcp.val.dst_port &= tcp.mask.dst_port; 1560 } 1561 mlx5_flow_create_copy(parser, &tcp, tcp_size); 1562 return 0; 1563 } 1564 1565 /** 1566 * Convert VXLAN item to Verbs specification. 1567 * 1568 * @param item[in] 1569 * Item specification. 1570 * @param default_mask[in] 1571 * Default bit-masks to use when item->mask is not provided. 1572 * @param data[in, out] 1573 * User structure. 1574 */ 1575 static int 1576 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 1577 const void *default_mask, 1578 void *data) 1579 { 1580 const struct rte_flow_item_vxlan *spec = item->spec; 1581 const struct rte_flow_item_vxlan *mask = item->mask; 1582 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; 1583 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 1584 struct ibv_flow_spec_tunnel vxlan = { 1585 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, 1586 .size = size, 1587 }; 1588 union vni { 1589 uint32_t vlan_id; 1590 uint8_t vni[4]; 1591 } id; 1592 1593 id.vni[0] = 0; 1594 parser->inner = IBV_FLOW_SPEC_INNER; 1595 if (spec) { 1596 if (!mask) 1597 mask = default_mask; 1598 memcpy(&id.vni[1], spec->vni, 3); 1599 vxlan.val.tunnel_id = id.vlan_id; 1600 memcpy(&id.vni[1], mask->vni, 3); 1601 vxlan.mask.tunnel_id = id.vlan_id; 1602 /* Remove unwanted bits from values. */ 1603 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 1604 } 1605 /* 1606 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1607 * layer is defined in the Verbs specification it is interpreted as 1608 * wildcard and all packets will match this rule, if it follows a full 1609 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1610 * before will also match this rule. 1611 * To avoid such situation, VNI 0 is currently refused. 1612 */ 1613 if (!vxlan.val.tunnel_id) 1614 return EINVAL; 1615 mlx5_flow_create_copy(parser, &vxlan, size); 1616 return 0; 1617 } 1618 1619 /** 1620 * Convert mark/flag action to Verbs specification. 1621 * 1622 * @param parser 1623 * Internal parser structure. 1624 * @param mark_id 1625 * Mark identifier. 1626 */ 1627 static int 1628 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) 1629 { 1630 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1631 struct ibv_flow_spec_action_tag tag = { 1632 .type = IBV_FLOW_SPEC_ACTION_TAG, 1633 .size = size, 1634 .tag_id = mlx5_flow_mark_set(mark_id), 1635 }; 1636 1637 assert(parser->mark); 1638 mlx5_flow_create_copy(parser, &tag, size); 1639 return 0; 1640 } 1641 1642 /** 1643 * Convert count action to Verbs specification. 1644 * 1645 * @param priv 1646 * Pointer to private structure. 1647 * @param parser 1648 * Pointer to MLX5 flow parser structure. 1649 * 1650 * @return 1651 * 0 on success, errno value on failure. 1652 */ 1653 static int 1654 mlx5_flow_create_count(struct priv *priv __rte_unused, 1655 struct mlx5_flow_parse *parser __rte_unused) 1656 { 1657 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 1658 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1659 struct ibv_counter_set_init_attr init_attr = {0}; 1660 struct ibv_flow_spec_counter_action counter = { 1661 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1662 .size = size, 1663 .counter_set_handle = 0, 1664 }; 1665 1666 init_attr.counter_set_id = 0; 1667 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr); 1668 if (!parser->cs) 1669 return EINVAL; 1670 counter.counter_set_handle = parser->cs->handle; 1671 mlx5_flow_create_copy(parser, &counter, size); 1672 #endif 1673 return 0; 1674 } 1675 1676 /** 1677 * Complete flow rule creation with a drop queue. 1678 * 1679 * @param priv 1680 * Pointer to private structure. 1681 * @param parser 1682 * Internal parser structure. 1683 * @param flow 1684 * Pointer to the rte_flow. 1685 * @param[out] error 1686 * Perform verbose error reporting if not NULL. 1687 * 1688 * @return 1689 * 0 on success, errno value on failure. 1690 */ 1691 static int 1692 priv_flow_create_action_queue_drop(struct priv *priv, 1693 struct mlx5_flow_parse *parser, 1694 struct rte_flow *flow, 1695 struct rte_flow_error *error) 1696 { 1697 struct ibv_flow_spec_action_drop *drop; 1698 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1699 int err = 0; 1700 1701 assert(priv->pd); 1702 assert(priv->ctx); 1703 flow->drop = 1; 1704 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr + 1705 parser->drop_q.offset); 1706 *drop = (struct ibv_flow_spec_action_drop){ 1707 .type = IBV_FLOW_SPEC_ACTION_DROP, 1708 .size = size, 1709 }; 1710 ++parser->drop_q.ibv_attr->num_of_specs; 1711 parser->drop_q.offset += size; 1712 flow->drxq.ibv_attr = parser->drop_q.ibv_attr; 1713 if (parser->count) 1714 flow->cs = parser->cs; 1715 if (!priv->dev->data->dev_started) 1716 return 0; 1717 parser->drop_q.ibv_attr = NULL; 1718 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp, 1719 flow->drxq.ibv_attr); 1720 if (!flow->drxq.ibv_flow) { 1721 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1722 NULL, "flow rule creation failure"); 1723 err = ENOMEM; 1724 goto error; 1725 } 1726 return 0; 1727 error: 1728 assert(flow); 1729 if (flow->drxq.ibv_flow) { 1730 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 1731 flow->drxq.ibv_flow = NULL; 1732 } 1733 if (flow->drxq.ibv_attr) { 1734 rte_free(flow->drxq.ibv_attr); 1735 flow->drxq.ibv_attr = NULL; 1736 } 1737 if (flow->cs) { 1738 claim_zero(ibv_destroy_counter_set(flow->cs)); 1739 flow->cs = NULL; 1740 parser->cs = NULL; 1741 } 1742 return err; 1743 } 1744 1745 /** 1746 * Create hash Rx queues when RSS is enabled. 1747 * 1748 * @param priv 1749 * Pointer to private structure. 1750 * @param parser 1751 * Internal parser structure. 1752 * @param flow 1753 * Pointer to the rte_flow. 1754 * @param[out] error 1755 * Perform verbose error reporting if not NULL. 1756 * 1757 * @return 1758 * 0 on success, a errno value otherwise and rte_errno is set. 1759 */ 1760 static int 1761 priv_flow_create_action_queue_rss(struct priv *priv, 1762 struct mlx5_flow_parse *parser, 1763 struct rte_flow *flow, 1764 struct rte_flow_error *error) 1765 { 1766 unsigned int i; 1767 1768 for (i = 0; i != hash_rxq_init_n; ++i) { 1769 uint64_t hash_fields; 1770 1771 if (!parser->queue[i].ibv_attr) 1772 continue; 1773 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; 1774 parser->queue[i].ibv_attr = NULL; 1775 hash_fields = hash_rxq_init[i].hash_fields; 1776 if (!priv->dev->data->dev_started) 1777 continue; 1778 flow->frxq[i].hrxq = 1779 mlx5_priv_hrxq_get(priv, 1780 parser->rss_conf.rss_key, 1781 parser->rss_conf.rss_key_len, 1782 hash_fields, 1783 parser->queues, 1784 parser->queues_n); 1785 if (flow->frxq[i].hrxq) 1786 continue; 1787 flow->frxq[i].hrxq = 1788 mlx5_priv_hrxq_new(priv, 1789 parser->rss_conf.rss_key, 1790 parser->rss_conf.rss_key_len, 1791 hash_fields, 1792 parser->queues, 1793 parser->queues_n); 1794 if (!flow->frxq[i].hrxq) { 1795 rte_flow_error_set(error, ENOMEM, 1796 RTE_FLOW_ERROR_TYPE_HANDLE, 1797 NULL, "cannot create hash rxq"); 1798 return ENOMEM; 1799 } 1800 } 1801 return 0; 1802 } 1803 1804 /** 1805 * Complete flow rule creation. 1806 * 1807 * @param priv 1808 * Pointer to private structure. 1809 * @param parser 1810 * Internal parser structure. 1811 * @param flow 1812 * Pointer to the rte_flow. 1813 * @param[out] error 1814 * Perform verbose error reporting if not NULL. 1815 * 1816 * @return 1817 * 0 on success, a errno value otherwise and rte_errno is set. 1818 */ 1819 static int 1820 priv_flow_create_action_queue(struct priv *priv, 1821 struct mlx5_flow_parse *parser, 1822 struct rte_flow *flow, 1823 struct rte_flow_error *error) 1824 { 1825 int err = 0; 1826 unsigned int i; 1827 1828 assert(priv->pd); 1829 assert(priv->ctx); 1830 assert(!parser->drop); 1831 err = priv_flow_create_action_queue_rss(priv, parser, flow, error); 1832 if (err) 1833 goto error; 1834 if (parser->count) 1835 flow->cs = parser->cs; 1836 if (!priv->dev->data->dev_started) 1837 return 0; 1838 for (i = 0; i != hash_rxq_init_n; ++i) { 1839 if (!flow->frxq[i].hrxq) 1840 continue; 1841 flow->frxq[i].ibv_flow = 1842 ibv_create_flow(flow->frxq[i].hrxq->qp, 1843 flow->frxq[i].ibv_attr); 1844 if (!flow->frxq[i].ibv_flow) { 1845 rte_flow_error_set(error, ENOMEM, 1846 RTE_FLOW_ERROR_TYPE_HANDLE, 1847 NULL, "flow rule creation failure"); 1848 err = ENOMEM; 1849 goto error; 1850 } 1851 DEBUG("%p type %d QP %p ibv_flow %p", 1852 (void *)flow, i, 1853 (void *)flow->frxq[i].hrxq, 1854 (void *)flow->frxq[i].ibv_flow); 1855 } 1856 for (i = 0; i != parser->queues_n; ++i) { 1857 struct mlx5_rxq_data *q = 1858 (*priv->rxqs)[parser->queues[i]]; 1859 1860 q->mark |= parser->mark; 1861 } 1862 return 0; 1863 error: 1864 assert(flow); 1865 for (i = 0; i != hash_rxq_init_n; ++i) { 1866 if (flow->frxq[i].ibv_flow) { 1867 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; 1868 1869 claim_zero(ibv_destroy_flow(ibv_flow)); 1870 } 1871 if (flow->frxq[i].hrxq) 1872 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); 1873 if (flow->frxq[i].ibv_attr) 1874 rte_free(flow->frxq[i].ibv_attr); 1875 } 1876 if (flow->cs) { 1877 claim_zero(ibv_destroy_counter_set(flow->cs)); 1878 flow->cs = NULL; 1879 parser->cs = NULL; 1880 } 1881 return err; 1882 } 1883 1884 /** 1885 * Convert a flow. 1886 * 1887 * @param priv 1888 * Pointer to private structure. 1889 * @param list 1890 * Pointer to a TAILQ flow list. 1891 * @param[in] attr 1892 * Flow rule attributes. 1893 * @param[in] pattern 1894 * Pattern specification (list terminated by the END pattern item). 1895 * @param[in] actions 1896 * Associated actions (list terminated by the END action). 1897 * @param[out] error 1898 * Perform verbose error reporting if not NULL. 1899 * 1900 * @return 1901 * A flow on success, NULL otherwise. 1902 */ 1903 static struct rte_flow * 1904 priv_flow_create(struct priv *priv, 1905 struct mlx5_flows *list, 1906 const struct rte_flow_attr *attr, 1907 const struct rte_flow_item items[], 1908 const struct rte_flow_action actions[], 1909 struct rte_flow_error *error) 1910 { 1911 struct mlx5_flow_parse parser = { .create = 1, }; 1912 struct rte_flow *flow = NULL; 1913 unsigned int i; 1914 int err; 1915 1916 err = priv_flow_convert(priv, attr, items, actions, error, &parser); 1917 if (err) 1918 goto exit; 1919 flow = rte_calloc(__func__, 1, 1920 sizeof(*flow) + parser.queues_n * sizeof(uint16_t), 1921 0); 1922 if (!flow) { 1923 rte_flow_error_set(error, ENOMEM, 1924 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1925 NULL, 1926 "cannot allocate flow memory"); 1927 return NULL; 1928 } 1929 /* Copy queues configuration. */ 1930 flow->queues = (uint16_t (*)[])(flow + 1); 1931 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t)); 1932 flow->queues_n = parser.queues_n; 1933 flow->mark = parser.mark; 1934 /* Copy RSS configuration. */ 1935 flow->rss_conf = parser.rss_conf; 1936 flow->rss_conf.rss_key = flow->rss_key; 1937 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len); 1938 /* finalise the flow. */ 1939 if (parser.drop) 1940 err = priv_flow_create_action_queue_drop(priv, &parser, flow, 1941 error); 1942 else 1943 err = priv_flow_create_action_queue(priv, &parser, flow, error); 1944 if (err) 1945 goto exit; 1946 TAILQ_INSERT_TAIL(list, flow, next); 1947 DEBUG("Flow created %p", (void *)flow); 1948 return flow; 1949 exit: 1950 if (parser.drop) { 1951 rte_free(parser.drop_q.ibv_attr); 1952 } else { 1953 for (i = 0; i != hash_rxq_init_n; ++i) { 1954 if (parser.queue[i].ibv_attr) 1955 rte_free(parser.queue[i].ibv_attr); 1956 } 1957 } 1958 rte_free(flow); 1959 return NULL; 1960 } 1961 1962 /** 1963 * Validate a flow supported by the NIC. 1964 * 1965 * @see rte_flow_validate() 1966 * @see rte_flow_ops 1967 */ 1968 int 1969 mlx5_flow_validate(struct rte_eth_dev *dev, 1970 const struct rte_flow_attr *attr, 1971 const struct rte_flow_item items[], 1972 const struct rte_flow_action actions[], 1973 struct rte_flow_error *error) 1974 { 1975 struct priv *priv = dev->data->dev_private; 1976 int ret; 1977 struct mlx5_flow_parse parser = { .create = 0, }; 1978 1979 priv_lock(priv); 1980 ret = priv_flow_convert(priv, attr, items, actions, error, &parser); 1981 priv_unlock(priv); 1982 return ret; 1983 } 1984 1985 /** 1986 * Create a flow. 1987 * 1988 * @see rte_flow_create() 1989 * @see rte_flow_ops 1990 */ 1991 struct rte_flow * 1992 mlx5_flow_create(struct rte_eth_dev *dev, 1993 const struct rte_flow_attr *attr, 1994 const struct rte_flow_item items[], 1995 const struct rte_flow_action actions[], 1996 struct rte_flow_error *error) 1997 { 1998 struct priv *priv = dev->data->dev_private; 1999 struct rte_flow *flow; 2000 2001 priv_lock(priv); 2002 flow = priv_flow_create(priv, &priv->flows, attr, items, actions, 2003 error); 2004 priv_unlock(priv); 2005 return flow; 2006 } 2007 2008 /** 2009 * Destroy a flow. 2010 * 2011 * @param priv 2012 * Pointer to private structure. 2013 * @param list 2014 * Pointer to a TAILQ flow list. 2015 * @param[in] flow 2016 * Flow to destroy. 2017 */ 2018 static void 2019 priv_flow_destroy(struct priv *priv, 2020 struct mlx5_flows *list, 2021 struct rte_flow *flow) 2022 { 2023 unsigned int i; 2024 2025 if (flow->drop || !flow->mark) 2026 goto free; 2027 for (i = 0; i != flow->queues_n; ++i) { 2028 struct rte_flow *tmp; 2029 int mark = 0; 2030 2031 /* 2032 * To remove the mark from the queue, the queue must not be 2033 * present in any other marked flow (RSS or not). 2034 */ 2035 TAILQ_FOREACH(tmp, list, next) { 2036 unsigned int j; 2037 uint16_t *tqs = NULL; 2038 uint16_t tq_n = 0; 2039 2040 if (!tmp->mark) 2041 continue; 2042 for (j = 0; j != hash_rxq_init_n; ++j) { 2043 if (!tmp->frxq[j].hrxq) 2044 continue; 2045 tqs = tmp->frxq[j].hrxq->ind_table->queues; 2046 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n; 2047 } 2048 if (!tq_n) 2049 continue; 2050 for (j = 0; (j != tq_n) && !mark; j++) 2051 if (tqs[j] == (*flow->queues)[i]) 2052 mark = 1; 2053 } 2054 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark; 2055 } 2056 free: 2057 if (flow->drop) { 2058 if (flow->drxq.ibv_flow) 2059 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 2060 rte_free(flow->drxq.ibv_attr); 2061 } else { 2062 for (i = 0; i != hash_rxq_init_n; ++i) { 2063 struct mlx5_flow *frxq = &flow->frxq[i]; 2064 2065 if (frxq->ibv_flow) 2066 claim_zero(ibv_destroy_flow(frxq->ibv_flow)); 2067 if (frxq->hrxq) 2068 mlx5_priv_hrxq_release(priv, frxq->hrxq); 2069 if (frxq->ibv_attr) 2070 rte_free(frxq->ibv_attr); 2071 } 2072 } 2073 if (flow->cs) { 2074 claim_zero(ibv_destroy_counter_set(flow->cs)); 2075 flow->cs = NULL; 2076 } 2077 TAILQ_REMOVE(list, flow, next); 2078 DEBUG("Flow destroyed %p", (void *)flow); 2079 rte_free(flow); 2080 } 2081 2082 /** 2083 * Destroy all flows. 2084 * 2085 * @param priv 2086 * Pointer to private structure. 2087 * @param list 2088 * Pointer to a TAILQ flow list. 2089 */ 2090 void 2091 priv_flow_flush(struct priv *priv, struct mlx5_flows *list) 2092 { 2093 while (!TAILQ_EMPTY(list)) { 2094 struct rte_flow *flow; 2095 2096 flow = TAILQ_FIRST(list); 2097 priv_flow_destroy(priv, list, flow); 2098 } 2099 } 2100 2101 /** 2102 * Create drop queue. 2103 * 2104 * @param priv 2105 * Pointer to private structure. 2106 * 2107 * @return 2108 * 0 on success. 2109 */ 2110 int 2111 priv_flow_create_drop_queue(struct priv *priv) 2112 { 2113 struct mlx5_hrxq_drop *fdq = NULL; 2114 2115 assert(priv->pd); 2116 assert(priv->ctx); 2117 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); 2118 if (!fdq) { 2119 WARN("cannot allocate memory for drop queue"); 2120 goto error; 2121 } 2122 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); 2123 if (!fdq->cq) { 2124 WARN("cannot allocate CQ for drop queue"); 2125 goto error; 2126 } 2127 fdq->wq = ibv_create_wq(priv->ctx, 2128 &(struct ibv_wq_init_attr){ 2129 .wq_type = IBV_WQT_RQ, 2130 .max_wr = 1, 2131 .max_sge = 1, 2132 .pd = priv->pd, 2133 .cq = fdq->cq, 2134 }); 2135 if (!fdq->wq) { 2136 WARN("cannot allocate WQ for drop queue"); 2137 goto error; 2138 } 2139 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx, 2140 &(struct ibv_rwq_ind_table_init_attr){ 2141 .log_ind_tbl_size = 0, 2142 .ind_tbl = &fdq->wq, 2143 .comp_mask = 0, 2144 }); 2145 if (!fdq->ind_table) { 2146 WARN("cannot allocate indirection table for drop queue"); 2147 goto error; 2148 } 2149 fdq->qp = ibv_create_qp_ex(priv->ctx, 2150 &(struct ibv_qp_init_attr_ex){ 2151 .qp_type = IBV_QPT_RAW_PACKET, 2152 .comp_mask = 2153 IBV_QP_INIT_ATTR_PD | 2154 IBV_QP_INIT_ATTR_IND_TABLE | 2155 IBV_QP_INIT_ATTR_RX_HASH, 2156 .rx_hash_conf = (struct ibv_rx_hash_conf){ 2157 .rx_hash_function = 2158 IBV_RX_HASH_FUNC_TOEPLITZ, 2159 .rx_hash_key_len = rss_hash_default_key_len, 2160 .rx_hash_key = rss_hash_default_key, 2161 .rx_hash_fields_mask = 0, 2162 }, 2163 .rwq_ind_tbl = fdq->ind_table, 2164 .pd = priv->pd 2165 }); 2166 if (!fdq->qp) { 2167 WARN("cannot allocate QP for drop queue"); 2168 goto error; 2169 } 2170 priv->flow_drop_queue = fdq; 2171 return 0; 2172 error: 2173 if (fdq->qp) 2174 claim_zero(ibv_destroy_qp(fdq->qp)); 2175 if (fdq->ind_table) 2176 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 2177 if (fdq->wq) 2178 claim_zero(ibv_destroy_wq(fdq->wq)); 2179 if (fdq->cq) 2180 claim_zero(ibv_destroy_cq(fdq->cq)); 2181 if (fdq) 2182 rte_free(fdq); 2183 priv->flow_drop_queue = NULL; 2184 return -1; 2185 } 2186 2187 /** 2188 * Delete drop queue. 2189 * 2190 * @param priv 2191 * Pointer to private structure. 2192 */ 2193 void 2194 priv_flow_delete_drop_queue(struct priv *priv) 2195 { 2196 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; 2197 2198 if (!fdq) 2199 return; 2200 if (fdq->qp) 2201 claim_zero(ibv_destroy_qp(fdq->qp)); 2202 if (fdq->ind_table) 2203 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 2204 if (fdq->wq) 2205 claim_zero(ibv_destroy_wq(fdq->wq)); 2206 if (fdq->cq) 2207 claim_zero(ibv_destroy_cq(fdq->cq)); 2208 rte_free(fdq); 2209 priv->flow_drop_queue = NULL; 2210 } 2211 2212 /** 2213 * Remove all flows. 2214 * 2215 * @param priv 2216 * Pointer to private structure. 2217 * @param list 2218 * Pointer to a TAILQ flow list. 2219 */ 2220 void 2221 priv_flow_stop(struct priv *priv, struct mlx5_flows *list) 2222 { 2223 struct rte_flow *flow; 2224 2225 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 2226 unsigned int i; 2227 2228 if (flow->drop) { 2229 if (!flow->drxq.ibv_flow) 2230 continue; 2231 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow)); 2232 flow->drxq.ibv_flow = NULL; 2233 /* Next flow. */ 2234 continue; 2235 } 2236 if (flow->mark) { 2237 struct mlx5_ind_table_ibv *ind_tbl = NULL; 2238 2239 for (i = 0; i != hash_rxq_init_n; ++i) { 2240 if (!flow->frxq[i].hrxq) 2241 continue; 2242 ind_tbl = flow->frxq[i].hrxq->ind_table; 2243 } 2244 assert(ind_tbl); 2245 for (i = 0; i != ind_tbl->queues_n; ++i) 2246 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; 2247 } 2248 for (i = 0; i != hash_rxq_init_n; ++i) { 2249 if (!flow->frxq[i].ibv_flow) 2250 continue; 2251 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow)); 2252 flow->frxq[i].ibv_flow = NULL; 2253 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); 2254 flow->frxq[i].hrxq = NULL; 2255 } 2256 DEBUG("Flow %p removed", (void *)flow); 2257 } 2258 } 2259 2260 /** 2261 * Add all flows. 2262 * 2263 * @param priv 2264 * Pointer to private structure. 2265 * @param list 2266 * Pointer to a TAILQ flow list. 2267 * 2268 * @return 2269 * 0 on success, a errno value otherwise and rte_errno is set. 2270 */ 2271 int 2272 priv_flow_start(struct priv *priv, struct mlx5_flows *list) 2273 { 2274 struct rte_flow *flow; 2275 2276 TAILQ_FOREACH(flow, list, next) { 2277 unsigned int i; 2278 2279 if (flow->drop) { 2280 flow->drxq.ibv_flow = 2281 ibv_create_flow(priv->flow_drop_queue->qp, 2282 flow->drxq.ibv_attr); 2283 if (!flow->drxq.ibv_flow) { 2284 DEBUG("Flow %p cannot be applied", 2285 (void *)flow); 2286 rte_errno = EINVAL; 2287 return rte_errno; 2288 } 2289 DEBUG("Flow %p applied", (void *)flow); 2290 /* Next flow. */ 2291 continue; 2292 } 2293 for (i = 0; i != hash_rxq_init_n; ++i) { 2294 if (!flow->frxq[i].ibv_attr) 2295 continue; 2296 flow->frxq[i].hrxq = 2297 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key, 2298 flow->rss_conf.rss_key_len, 2299 hash_rxq_init[i].hash_fields, 2300 (*flow->queues), 2301 flow->queues_n); 2302 if (flow->frxq[i].hrxq) 2303 goto flow_create; 2304 flow->frxq[i].hrxq = 2305 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key, 2306 flow->rss_conf.rss_key_len, 2307 hash_rxq_init[i].hash_fields, 2308 (*flow->queues), 2309 flow->queues_n); 2310 if (!flow->frxq[i].hrxq) { 2311 DEBUG("Flow %p cannot be applied", 2312 (void *)flow); 2313 rte_errno = EINVAL; 2314 return rte_errno; 2315 } 2316 flow_create: 2317 flow->frxq[i].ibv_flow = 2318 ibv_create_flow(flow->frxq[i].hrxq->qp, 2319 flow->frxq[i].ibv_attr); 2320 if (!flow->frxq[i].ibv_flow) { 2321 DEBUG("Flow %p cannot be applied", 2322 (void *)flow); 2323 rte_errno = EINVAL; 2324 return rte_errno; 2325 } 2326 DEBUG("Flow %p applied", (void *)flow); 2327 } 2328 if (!flow->mark) 2329 continue; 2330 for (i = 0; i != flow->queues_n; ++i) 2331 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1; 2332 } 2333 return 0; 2334 } 2335 2336 /** 2337 * Verify the flow list is empty 2338 * 2339 * @param priv 2340 * Pointer to private structure. 2341 * 2342 * @return the number of flows not released. 2343 */ 2344 int 2345 priv_flow_verify(struct priv *priv) 2346 { 2347 struct rte_flow *flow; 2348 int ret = 0; 2349 2350 TAILQ_FOREACH(flow, &priv->flows, next) { 2351 DEBUG("%p: flow %p still referenced", (void *)priv, 2352 (void *)flow); 2353 ++ret; 2354 } 2355 return ret; 2356 } 2357 2358 /** 2359 * Enable a control flow configured from the control plane. 2360 * 2361 * @param dev 2362 * Pointer to Ethernet device. 2363 * @param eth_spec 2364 * An Ethernet flow spec to apply. 2365 * @param eth_mask 2366 * An Ethernet flow mask to apply. 2367 * @param vlan_spec 2368 * A VLAN flow spec to apply. 2369 * @param vlan_mask 2370 * A VLAN flow mask to apply. 2371 * 2372 * @return 2373 * 0 on success. 2374 */ 2375 int 2376 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 2377 struct rte_flow_item_eth *eth_spec, 2378 struct rte_flow_item_eth *eth_mask, 2379 struct rte_flow_item_vlan *vlan_spec, 2380 struct rte_flow_item_vlan *vlan_mask) 2381 { 2382 struct priv *priv = dev->data->dev_private; 2383 const struct rte_flow_attr attr = { 2384 .ingress = 1, 2385 .priority = MLX5_CTRL_FLOW_PRIORITY, 2386 }; 2387 struct rte_flow_item items[] = { 2388 { 2389 .type = RTE_FLOW_ITEM_TYPE_ETH, 2390 .spec = eth_spec, 2391 .last = NULL, 2392 .mask = eth_mask, 2393 }, 2394 { 2395 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 2396 RTE_FLOW_ITEM_TYPE_END, 2397 .spec = vlan_spec, 2398 .last = NULL, 2399 .mask = vlan_mask, 2400 }, 2401 { 2402 .type = RTE_FLOW_ITEM_TYPE_END, 2403 }, 2404 }; 2405 struct rte_flow_action actions[] = { 2406 { 2407 .type = RTE_FLOW_ACTION_TYPE_RSS, 2408 }, 2409 { 2410 .type = RTE_FLOW_ACTION_TYPE_END, 2411 }, 2412 }; 2413 struct rte_flow *flow; 2414 struct rte_flow_error error; 2415 unsigned int i; 2416 union { 2417 struct rte_flow_action_rss rss; 2418 struct { 2419 const struct rte_eth_rss_conf *rss_conf; 2420 uint16_t num; 2421 uint16_t queue[RTE_MAX_QUEUES_PER_PORT]; 2422 } local; 2423 } action_rss; 2424 2425 if (!priv->reta_idx_n) 2426 return EINVAL; 2427 for (i = 0; i != priv->reta_idx_n; ++i) 2428 action_rss.local.queue[i] = (*priv->reta_idx)[i]; 2429 action_rss.local.rss_conf = &priv->rss_conf; 2430 action_rss.local.num = priv->reta_idx_n; 2431 actions[0].conf = (const void *)&action_rss.rss; 2432 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions, 2433 &error); 2434 if (!flow) 2435 return rte_errno; 2436 return 0; 2437 } 2438 2439 /** 2440 * Enable a flow control configured from the control plane. 2441 * 2442 * @param dev 2443 * Pointer to Ethernet device. 2444 * @param eth_spec 2445 * An Ethernet flow spec to apply. 2446 * @param eth_mask 2447 * An Ethernet flow mask to apply. 2448 * 2449 * @return 2450 * 0 on success. 2451 */ 2452 int 2453 mlx5_ctrl_flow(struct rte_eth_dev *dev, 2454 struct rte_flow_item_eth *eth_spec, 2455 struct rte_flow_item_eth *eth_mask) 2456 { 2457 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 2458 } 2459 2460 /** 2461 * Destroy a flow. 2462 * 2463 * @see rte_flow_destroy() 2464 * @see rte_flow_ops 2465 */ 2466 int 2467 mlx5_flow_destroy(struct rte_eth_dev *dev, 2468 struct rte_flow *flow, 2469 struct rte_flow_error *error) 2470 { 2471 struct priv *priv = dev->data->dev_private; 2472 2473 (void)error; 2474 priv_lock(priv); 2475 priv_flow_destroy(priv, &priv->flows, flow); 2476 priv_unlock(priv); 2477 return 0; 2478 } 2479 2480 /** 2481 * Destroy all flows. 2482 * 2483 * @see rte_flow_flush() 2484 * @see rte_flow_ops 2485 */ 2486 int 2487 mlx5_flow_flush(struct rte_eth_dev *dev, 2488 struct rte_flow_error *error) 2489 { 2490 struct priv *priv = dev->data->dev_private; 2491 2492 (void)error; 2493 priv_lock(priv); 2494 priv_flow_flush(priv, &priv->flows); 2495 priv_unlock(priv); 2496 return 0; 2497 } 2498 2499 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT 2500 /** 2501 * Query flow counter. 2502 * 2503 * @param cs 2504 * the counter set. 2505 * @param counter_value 2506 * returned data from the counter. 2507 * 2508 * @return 2509 * 0 on success, a errno value otherwise and rte_errno is set. 2510 */ 2511 static int 2512 priv_flow_query_count(struct ibv_counter_set *cs, 2513 struct mlx5_flow_counter_stats *counter_stats, 2514 struct rte_flow_query_count *query_count, 2515 struct rte_flow_error *error) 2516 { 2517 uint64_t counters[2]; 2518 struct ibv_query_counter_set_attr query_cs_attr = { 2519 .cs = cs, 2520 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 2521 }; 2522 struct ibv_counter_set_data query_out = { 2523 .out = counters, 2524 .outlen = 2 * sizeof(uint64_t), 2525 }; 2526 int res = ibv_query_counter_set(&query_cs_attr, &query_out); 2527 2528 if (res) { 2529 rte_flow_error_set(error, -res, 2530 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2531 NULL, 2532 "cannot read counter"); 2533 return -res; 2534 } 2535 query_count->hits_set = 1; 2536 query_count->bytes_set = 1; 2537 query_count->hits = counters[0] - counter_stats->hits; 2538 query_count->bytes = counters[1] - counter_stats->bytes; 2539 if (query_count->reset) { 2540 counter_stats->hits = counters[0]; 2541 counter_stats->bytes = counters[1]; 2542 } 2543 return 0; 2544 } 2545 2546 /** 2547 * Query a flows. 2548 * 2549 * @see rte_flow_query() 2550 * @see rte_flow_ops 2551 */ 2552 int 2553 mlx5_flow_query(struct rte_eth_dev *dev, 2554 struct rte_flow *flow, 2555 enum rte_flow_action_type action __rte_unused, 2556 void *data, 2557 struct rte_flow_error *error) 2558 { 2559 struct priv *priv = dev->data->dev_private; 2560 int res = EINVAL; 2561 2562 priv_lock(priv); 2563 if (flow->cs) { 2564 res = priv_flow_query_count(flow->cs, 2565 &flow->counter_stats, 2566 (struct rte_flow_query_count *)data, 2567 error); 2568 } else { 2569 rte_flow_error_set(error, res, 2570 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2571 NULL, 2572 "no counter found for flow"); 2573 } 2574 priv_unlock(priv); 2575 return -res; 2576 } 2577 #endif 2578 2579 /** 2580 * Isolated mode. 2581 * 2582 * @see rte_flow_isolate() 2583 * @see rte_flow_ops 2584 */ 2585 int 2586 mlx5_flow_isolate(struct rte_eth_dev *dev, 2587 int enable, 2588 struct rte_flow_error *error) 2589 { 2590 struct priv *priv = dev->data->dev_private; 2591 2592 priv_lock(priv); 2593 if (dev->data->dev_started) { 2594 rte_flow_error_set(error, EBUSY, 2595 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2596 NULL, 2597 "port must be stopped first"); 2598 priv_unlock(priv); 2599 return -rte_errno; 2600 } 2601 priv->isolated = !!enable; 2602 if (enable) 2603 priv->dev->dev_ops = &mlx5_dev_ops_isolate; 2604 else 2605 priv->dev->dev_ops = &mlx5_dev_ops; 2606 priv_unlock(priv); 2607 return 0; 2608 } 2609 2610 /** 2611 * Convert a flow director filter to a generic flow. 2612 * 2613 * @param priv 2614 * Private structure. 2615 * @param fdir_filter 2616 * Flow director filter to add. 2617 * @param attributes 2618 * Generic flow parameters structure. 2619 * 2620 * @return 2621 * 0 on success, errno value on error. 2622 */ 2623 static int 2624 priv_fdir_filter_convert(struct priv *priv, 2625 const struct rte_eth_fdir_filter *fdir_filter, 2626 struct mlx5_fdir *attributes) 2627 { 2628 const struct rte_eth_fdir_input *input = &fdir_filter->input; 2629 2630 /* Validate queue number. */ 2631 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 2632 ERROR("invalid queue number %d", fdir_filter->action.rx_queue); 2633 return EINVAL; 2634 } 2635 attributes->attr.ingress = 1; 2636 attributes->items[0] = (struct rte_flow_item) { 2637 .type = RTE_FLOW_ITEM_TYPE_ETH, 2638 .spec = &attributes->l2, 2639 .mask = &attributes->l2_mask, 2640 }; 2641 switch (fdir_filter->action.behavior) { 2642 case RTE_ETH_FDIR_ACCEPT: 2643 attributes->actions[0] = (struct rte_flow_action){ 2644 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 2645 .conf = &attributes->queue, 2646 }; 2647 break; 2648 case RTE_ETH_FDIR_REJECT: 2649 attributes->actions[0] = (struct rte_flow_action){ 2650 .type = RTE_FLOW_ACTION_TYPE_DROP, 2651 }; 2652 break; 2653 default: 2654 ERROR("invalid behavior %d", fdir_filter->action.behavior); 2655 return ENOTSUP; 2656 } 2657 attributes->queue.index = fdir_filter->action.rx_queue; 2658 switch (fdir_filter->input.flow_type) { 2659 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 2660 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2661 .src_addr = input->flow.udp4_flow.ip.src_ip, 2662 .dst_addr = input->flow.udp4_flow.ip.dst_ip, 2663 .time_to_live = input->flow.udp4_flow.ip.ttl, 2664 .type_of_service = input->flow.udp4_flow.ip.tos, 2665 .next_proto_id = input->flow.udp4_flow.ip.proto, 2666 }; 2667 attributes->l4.udp.hdr = (struct udp_hdr){ 2668 .src_port = input->flow.udp4_flow.src_port, 2669 .dst_port = input->flow.udp4_flow.dst_port, 2670 }; 2671 attributes->items[1] = (struct rte_flow_item){ 2672 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2673 .spec = &attributes->l3, 2674 }; 2675 attributes->items[2] = (struct rte_flow_item){ 2676 .type = RTE_FLOW_ITEM_TYPE_UDP, 2677 .spec = &attributes->l4, 2678 }; 2679 break; 2680 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 2681 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2682 .src_addr = input->flow.tcp4_flow.ip.src_ip, 2683 .dst_addr = input->flow.tcp4_flow.ip.dst_ip, 2684 .time_to_live = input->flow.tcp4_flow.ip.ttl, 2685 .type_of_service = input->flow.tcp4_flow.ip.tos, 2686 .next_proto_id = input->flow.tcp4_flow.ip.proto, 2687 }; 2688 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2689 .src_port = input->flow.tcp4_flow.src_port, 2690 .dst_port = input->flow.tcp4_flow.dst_port, 2691 }; 2692 attributes->items[1] = (struct rte_flow_item){ 2693 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2694 .spec = &attributes->l3, 2695 }; 2696 attributes->items[2] = (struct rte_flow_item){ 2697 .type = RTE_FLOW_ITEM_TYPE_TCP, 2698 .spec = &attributes->l4, 2699 }; 2700 break; 2701 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 2702 attributes->l3.ipv4.hdr = (struct ipv4_hdr){ 2703 .src_addr = input->flow.ip4_flow.src_ip, 2704 .dst_addr = input->flow.ip4_flow.dst_ip, 2705 .time_to_live = input->flow.ip4_flow.ttl, 2706 .type_of_service = input->flow.ip4_flow.tos, 2707 .next_proto_id = input->flow.ip4_flow.proto, 2708 }; 2709 attributes->items[1] = (struct rte_flow_item){ 2710 .type = RTE_FLOW_ITEM_TYPE_IPV4, 2711 .spec = &attributes->l3, 2712 }; 2713 break; 2714 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 2715 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2716 .hop_limits = input->flow.udp6_flow.ip.hop_limits, 2717 .proto = input->flow.udp6_flow.ip.proto, 2718 }; 2719 memcpy(attributes->l3.ipv6.hdr.src_addr, 2720 input->flow.udp6_flow.ip.src_ip, 2721 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2722 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2723 input->flow.udp6_flow.ip.dst_ip, 2724 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2725 attributes->l4.udp.hdr = (struct udp_hdr){ 2726 .src_port = input->flow.udp6_flow.src_port, 2727 .dst_port = input->flow.udp6_flow.dst_port, 2728 }; 2729 attributes->items[1] = (struct rte_flow_item){ 2730 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2731 .spec = &attributes->l3, 2732 }; 2733 attributes->items[2] = (struct rte_flow_item){ 2734 .type = RTE_FLOW_ITEM_TYPE_UDP, 2735 .spec = &attributes->l4, 2736 }; 2737 break; 2738 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 2739 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2740 .hop_limits = input->flow.tcp6_flow.ip.hop_limits, 2741 .proto = input->flow.tcp6_flow.ip.proto, 2742 }; 2743 memcpy(attributes->l3.ipv6.hdr.src_addr, 2744 input->flow.tcp6_flow.ip.src_ip, 2745 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2746 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2747 input->flow.tcp6_flow.ip.dst_ip, 2748 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2749 attributes->l4.tcp.hdr = (struct tcp_hdr){ 2750 .src_port = input->flow.tcp6_flow.src_port, 2751 .dst_port = input->flow.tcp6_flow.dst_port, 2752 }; 2753 attributes->items[1] = (struct rte_flow_item){ 2754 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2755 .spec = &attributes->l3, 2756 }; 2757 attributes->items[2] = (struct rte_flow_item){ 2758 .type = RTE_FLOW_ITEM_TYPE_TCP, 2759 .spec = &attributes->l4, 2760 }; 2761 break; 2762 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 2763 attributes->l3.ipv6.hdr = (struct ipv6_hdr){ 2764 .hop_limits = input->flow.ipv6_flow.hop_limits, 2765 .proto = input->flow.ipv6_flow.proto, 2766 }; 2767 memcpy(attributes->l3.ipv6.hdr.src_addr, 2768 input->flow.ipv6_flow.src_ip, 2769 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2770 memcpy(attributes->l3.ipv6.hdr.dst_addr, 2771 input->flow.ipv6_flow.dst_ip, 2772 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 2773 attributes->items[1] = (struct rte_flow_item){ 2774 .type = RTE_FLOW_ITEM_TYPE_IPV6, 2775 .spec = &attributes->l3, 2776 }; 2777 break; 2778 default: 2779 ERROR("invalid flow type%d", 2780 fdir_filter->input.flow_type); 2781 return ENOTSUP; 2782 } 2783 return 0; 2784 } 2785 2786 /** 2787 * Add new flow director filter and store it in list. 2788 * 2789 * @param priv 2790 * Private structure. 2791 * @param fdir_filter 2792 * Flow director filter to add. 2793 * 2794 * @return 2795 * 0 on success, errno value on failure. 2796 */ 2797 static int 2798 priv_fdir_filter_add(struct priv *priv, 2799 const struct rte_eth_fdir_filter *fdir_filter) 2800 { 2801 struct mlx5_fdir attributes = { 2802 .attr.group = 0, 2803 .l2_mask = { 2804 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2805 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 2806 .type = 0, 2807 }, 2808 }; 2809 struct mlx5_flow_parse parser = { 2810 .layer = HASH_RXQ_ETH, 2811 }; 2812 struct rte_flow_error error; 2813 struct rte_flow *flow; 2814 int ret; 2815 2816 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); 2817 if (ret) 2818 return -ret; 2819 ret = priv_flow_convert(priv, &attributes.attr, attributes.items, 2820 attributes.actions, &error, &parser); 2821 if (ret) 2822 return -ret; 2823 flow = priv_flow_create(priv, 2824 &priv->flows, 2825 &attributes.attr, 2826 attributes.items, 2827 attributes.actions, 2828 &error); 2829 if (flow) { 2830 DEBUG("FDIR created %p", (void *)flow); 2831 return 0; 2832 } 2833 return ENOTSUP; 2834 } 2835 2836 /** 2837 * Delete specific filter. 2838 * 2839 * @param priv 2840 * Private structure. 2841 * @param fdir_filter 2842 * Filter to be deleted. 2843 * 2844 * @return 2845 * 0 on success, errno value on failure. 2846 */ 2847 static int 2848 priv_fdir_filter_delete(struct priv *priv, 2849 const struct rte_eth_fdir_filter *fdir_filter) 2850 { 2851 struct mlx5_fdir attributes = { 2852 .attr.group = 0, 2853 }; 2854 struct mlx5_flow_parse parser = { 2855 .create = 1, 2856 .layer = HASH_RXQ_ETH, 2857 }; 2858 struct rte_flow_error error; 2859 struct rte_flow *flow; 2860 unsigned int i; 2861 int ret; 2862 2863 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); 2864 if (ret) 2865 return -ret; 2866 ret = priv_flow_convert(priv, &attributes.attr, attributes.items, 2867 attributes.actions, &error, &parser); 2868 if (ret) 2869 goto exit; 2870 /* 2871 * Special case for drop action which is only set in the 2872 * specifications when the flow is created. In this situation the 2873 * drop specification is missing. 2874 */ 2875 if (parser.drop) { 2876 struct ibv_flow_spec_action_drop *drop; 2877 2878 drop = (void *)((uintptr_t)parser.drop_q.ibv_attr + 2879 parser.drop_q.offset); 2880 *drop = (struct ibv_flow_spec_action_drop){ 2881 .type = IBV_FLOW_SPEC_ACTION_DROP, 2882 .size = sizeof(struct ibv_flow_spec_action_drop), 2883 }; 2884 parser.drop_q.ibv_attr->num_of_specs++; 2885 } 2886 TAILQ_FOREACH(flow, &priv->flows, next) { 2887 struct ibv_flow_attr *attr; 2888 struct ibv_spec_header *attr_h; 2889 void *spec; 2890 struct ibv_flow_attr *flow_attr; 2891 struct ibv_spec_header *flow_h; 2892 void *flow_spec; 2893 unsigned int specs_n; 2894 2895 if (parser.drop) 2896 attr = parser.drop_q.ibv_attr; 2897 else 2898 attr = parser.queue[HASH_RXQ_ETH].ibv_attr; 2899 if (flow->drop) 2900 flow_attr = flow->drxq.ibv_attr; 2901 else 2902 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; 2903 /* Compare first the attributes. */ 2904 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) 2905 continue; 2906 if (attr->num_of_specs == 0) 2907 continue; 2908 spec = (void *)((uintptr_t)attr + 2909 sizeof(struct ibv_flow_attr)); 2910 flow_spec = (void *)((uintptr_t)flow_attr + 2911 sizeof(struct ibv_flow_attr)); 2912 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs); 2913 for (i = 0; i != specs_n; ++i) { 2914 attr_h = spec; 2915 flow_h = flow_spec; 2916 if (memcmp(spec, flow_spec, 2917 RTE_MIN(attr_h->size, flow_h->size))) 2918 goto wrong_flow; 2919 spec = (void *)((uintptr_t)spec + attr_h->size); 2920 flow_spec = (void *)((uintptr_t)flow_spec + 2921 flow_h->size); 2922 } 2923 /* At this point, the flow match. */ 2924 break; 2925 wrong_flow: 2926 /* The flow does not match. */ 2927 continue; 2928 } 2929 if (flow) 2930 priv_flow_destroy(priv, &priv->flows, flow); 2931 exit: 2932 if (parser.drop) { 2933 rte_free(parser.drop_q.ibv_attr); 2934 } else { 2935 for (i = 0; i != hash_rxq_init_n; ++i) { 2936 if (parser.queue[i].ibv_attr) 2937 rte_free(parser.queue[i].ibv_attr); 2938 } 2939 } 2940 return -ret; 2941 } 2942 2943 /** 2944 * Update queue for specific filter. 2945 * 2946 * @param priv 2947 * Private structure. 2948 * @param fdir_filter 2949 * Filter to be updated. 2950 * 2951 * @return 2952 * 0 on success, errno value on failure. 2953 */ 2954 static int 2955 priv_fdir_filter_update(struct priv *priv, 2956 const struct rte_eth_fdir_filter *fdir_filter) 2957 { 2958 int ret; 2959 2960 ret = priv_fdir_filter_delete(priv, fdir_filter); 2961 if (ret) 2962 return ret; 2963 ret = priv_fdir_filter_add(priv, fdir_filter); 2964 return ret; 2965 } 2966 2967 /** 2968 * Flush all filters. 2969 * 2970 * @param priv 2971 * Private structure. 2972 */ 2973 static void 2974 priv_fdir_filter_flush(struct priv *priv) 2975 { 2976 priv_flow_flush(priv, &priv->flows); 2977 } 2978 2979 /** 2980 * Get flow director information. 2981 * 2982 * @param priv 2983 * Private structure. 2984 * @param[out] fdir_info 2985 * Resulting flow director information. 2986 */ 2987 static void 2988 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) 2989 { 2990 struct rte_eth_fdir_masks *mask = 2991 &priv->dev->data->dev_conf.fdir_conf.mask; 2992 2993 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; 2994 fdir_info->guarant_spc = 0; 2995 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 2996 fdir_info->max_flexpayload = 0; 2997 fdir_info->flow_types_mask[0] = 0; 2998 fdir_info->flex_payload_unit = 0; 2999 fdir_info->max_flex_payload_segment_num = 0; 3000 fdir_info->flex_payload_limit = 0; 3001 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 3002 } 3003 3004 /** 3005 * Deal with flow director operations. 3006 * 3007 * @param priv 3008 * Pointer to private structure. 3009 * @param filter_op 3010 * Operation to perform. 3011 * @param arg 3012 * Pointer to operation-specific structure. 3013 * 3014 * @return 3015 * 0 on success, errno value on failure. 3016 */ 3017 static int 3018 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) 3019 { 3020 enum rte_fdir_mode fdir_mode = 3021 priv->dev->data->dev_conf.fdir_conf.mode; 3022 int ret = 0; 3023 3024 if (filter_op == RTE_ETH_FILTER_NOP) 3025 return 0; 3026 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 3027 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 3028 ERROR("%p: flow director mode %d not supported", 3029 (void *)priv, fdir_mode); 3030 return EINVAL; 3031 } 3032 switch (filter_op) { 3033 case RTE_ETH_FILTER_ADD: 3034 ret = priv_fdir_filter_add(priv, arg); 3035 break; 3036 case RTE_ETH_FILTER_UPDATE: 3037 ret = priv_fdir_filter_update(priv, arg); 3038 break; 3039 case RTE_ETH_FILTER_DELETE: 3040 ret = priv_fdir_filter_delete(priv, arg); 3041 break; 3042 case RTE_ETH_FILTER_FLUSH: 3043 priv_fdir_filter_flush(priv); 3044 break; 3045 case RTE_ETH_FILTER_INFO: 3046 priv_fdir_info_get(priv, arg); 3047 break; 3048 default: 3049 DEBUG("%p: unknown operation %u", (void *)priv, 3050 filter_op); 3051 ret = EINVAL; 3052 break; 3053 } 3054 return ret; 3055 } 3056 3057 /** 3058 * Manage filter operations. 3059 * 3060 * @param dev 3061 * Pointer to Ethernet device structure. 3062 * @param filter_type 3063 * Filter type. 3064 * @param filter_op 3065 * Operation to perform. 3066 * @param arg 3067 * Pointer to operation-specific structure. 3068 * 3069 * @return 3070 * 0 on success, negative errno value on failure. 3071 */ 3072 int 3073 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 3074 enum rte_filter_type filter_type, 3075 enum rte_filter_op filter_op, 3076 void *arg) 3077 { 3078 int ret = EINVAL; 3079 struct priv *priv = dev->data->dev_private; 3080 3081 switch (filter_type) { 3082 case RTE_ETH_FILTER_GENERIC: 3083 if (filter_op != RTE_ETH_FILTER_GET) 3084 return -EINVAL; 3085 *(const void **)arg = &mlx5_flow_ops; 3086 return 0; 3087 case RTE_ETH_FILTER_FDIR: 3088 priv_lock(priv); 3089 ret = priv_fdir_ctrl_func(priv, filter_op, arg); 3090 priv_unlock(priv); 3091 break; 3092 default: 3093 ERROR("%p: filter type (%d) not supported", 3094 (void *)dev, filter_type); 3095 break; 3096 } 3097 return -ret; 3098 } 3099