1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2016 6WIND S.A. 5 * Copyright 2016 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/queue.h> 35 #include <string.h> 36 37 /* Verbs header. */ 38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 39 #ifdef PEDANTIC 40 #pragma GCC diagnostic ignored "-Wpedantic" 41 #endif 42 #include <infiniband/verbs.h> 43 #ifdef PEDANTIC 44 #pragma GCC diagnostic error "-Wpedantic" 45 #endif 46 47 #include <rte_ethdev.h> 48 #include <rte_flow.h> 49 #include <rte_flow_driver.h> 50 #include <rte_malloc.h> 51 52 #include "mlx5.h" 53 #include "mlx5_prm.h" 54 55 static int 56 mlx5_flow_create_eth(const struct rte_flow_item *item, 57 const void *default_mask, 58 void *data); 59 60 static int 61 mlx5_flow_create_vlan(const struct rte_flow_item *item, 62 const void *default_mask, 63 void *data); 64 65 static int 66 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 67 const void *default_mask, 68 void *data); 69 70 static int 71 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 72 const void *default_mask, 73 void *data); 74 75 static int 76 mlx5_flow_create_udp(const struct rte_flow_item *item, 77 const void *default_mask, 78 void *data); 79 80 static int 81 mlx5_flow_create_tcp(const struct rte_flow_item *item, 82 const void *default_mask, 83 void *data); 84 85 static int 86 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 87 const void *default_mask, 88 void *data); 89 90 struct rte_flow { 91 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ 92 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ 93 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 94 struct ibv_qp *qp; /**< Verbs queue pair. */ 95 struct ibv_flow *ibv_flow; /**< Verbs flow. */ 96 struct ibv_wq *wq; /**< Verbs work queue. */ 97 struct ibv_cq *cq; /**< Verbs completion queue. */ 98 uint16_t rxqs_n; /**< Number of queues in this flow, 0 if drop queue. */ 99 uint32_t mark:1; /**< Set if the flow is marked. */ 100 uint32_t drop:1; /**< Drop queue. */ 101 uint64_t hash_fields; /**< Fields that participate in the hash. */ 102 struct rxq *rxqs[]; /**< Pointer to the queues array. */ 103 }; 104 105 /** Static initializer for items. */ 106 #define ITEMS(...) \ 107 (const enum rte_flow_item_type []){ \ 108 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ 109 } 110 111 /** Structure to generate a simple graph of layers supported by the NIC. */ 112 struct mlx5_flow_items { 113 /** List of possible actions for these items. */ 114 const enum rte_flow_action_type *const actions; 115 /** Bit-masks corresponding to the possibilities for the item. */ 116 const void *mask; 117 /** 118 * Default bit-masks to use when item->mask is not provided. When 119 * \default_mask is also NULL, the full supported bit-mask (\mask) is 120 * used instead. 121 */ 122 const void *default_mask; 123 /** Bit-masks size in bytes. */ 124 const unsigned int mask_sz; 125 /** 126 * Conversion function from rte_flow to NIC specific flow. 127 * 128 * @param item 129 * rte_flow item to convert. 130 * @param default_mask 131 * Default bit-masks to use when item->mask is not provided. 132 * @param data 133 * Internal structure to store the conversion. 134 * 135 * @return 136 * 0 on success, negative value otherwise. 137 */ 138 int (*convert)(const struct rte_flow_item *item, 139 const void *default_mask, 140 void *data); 141 /** Size in bytes of the destination structure. */ 142 const unsigned int dst_sz; 143 /** List of possible following items. */ 144 const enum rte_flow_item_type *const items; 145 }; 146 147 /** Valid action for this PMD. */ 148 static const enum rte_flow_action_type valid_actions[] = { 149 RTE_FLOW_ACTION_TYPE_DROP, 150 RTE_FLOW_ACTION_TYPE_QUEUE, 151 RTE_FLOW_ACTION_TYPE_MARK, 152 RTE_FLOW_ACTION_TYPE_FLAG, 153 RTE_FLOW_ACTION_TYPE_END, 154 }; 155 156 /** Graph of supported items and associated actions. */ 157 static const struct mlx5_flow_items mlx5_flow_items[] = { 158 [RTE_FLOW_ITEM_TYPE_END] = { 159 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, 160 RTE_FLOW_ITEM_TYPE_VXLAN), 161 }, 162 [RTE_FLOW_ITEM_TYPE_ETH] = { 163 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, 164 RTE_FLOW_ITEM_TYPE_IPV4, 165 RTE_FLOW_ITEM_TYPE_IPV6), 166 .actions = valid_actions, 167 .mask = &(const struct rte_flow_item_eth){ 168 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 169 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 170 .type = -1, 171 }, 172 .default_mask = &rte_flow_item_eth_mask, 173 .mask_sz = sizeof(struct rte_flow_item_eth), 174 .convert = mlx5_flow_create_eth, 175 .dst_sz = sizeof(struct ibv_flow_spec_eth), 176 }, 177 [RTE_FLOW_ITEM_TYPE_VLAN] = { 178 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, 179 RTE_FLOW_ITEM_TYPE_IPV6), 180 .actions = valid_actions, 181 .mask = &(const struct rte_flow_item_vlan){ 182 .tci = -1, 183 }, 184 .default_mask = &rte_flow_item_vlan_mask, 185 .mask_sz = sizeof(struct rte_flow_item_vlan), 186 .convert = mlx5_flow_create_vlan, 187 .dst_sz = 0, 188 }, 189 [RTE_FLOW_ITEM_TYPE_IPV4] = { 190 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 191 RTE_FLOW_ITEM_TYPE_TCP), 192 .actions = valid_actions, 193 .mask = &(const struct rte_flow_item_ipv4){ 194 .hdr = { 195 .src_addr = -1, 196 .dst_addr = -1, 197 .type_of_service = -1, 198 .next_proto_id = -1, 199 }, 200 }, 201 .default_mask = &rte_flow_item_ipv4_mask, 202 .mask_sz = sizeof(struct rte_flow_item_ipv4), 203 .convert = mlx5_flow_create_ipv4, 204 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), 205 }, 206 [RTE_FLOW_ITEM_TYPE_IPV6] = { 207 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, 208 RTE_FLOW_ITEM_TYPE_TCP), 209 .actions = valid_actions, 210 .mask = &(const struct rte_flow_item_ipv6){ 211 .hdr = { 212 .src_addr = { 213 0xff, 0xff, 0xff, 0xff, 214 0xff, 0xff, 0xff, 0xff, 215 0xff, 0xff, 0xff, 0xff, 216 0xff, 0xff, 0xff, 0xff, 217 }, 218 .dst_addr = { 219 0xff, 0xff, 0xff, 0xff, 220 0xff, 0xff, 0xff, 0xff, 221 0xff, 0xff, 0xff, 0xff, 222 0xff, 0xff, 0xff, 0xff, 223 }, 224 .vtc_flow = -1, 225 .proto = -1, 226 .hop_limits = -1, 227 }, 228 }, 229 .default_mask = &rte_flow_item_ipv6_mask, 230 .mask_sz = sizeof(struct rte_flow_item_ipv6), 231 .convert = mlx5_flow_create_ipv6, 232 .dst_sz = sizeof(struct ibv_flow_spec_ipv6), 233 }, 234 [RTE_FLOW_ITEM_TYPE_UDP] = { 235 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), 236 .actions = valid_actions, 237 .mask = &(const struct rte_flow_item_udp){ 238 .hdr = { 239 .src_port = -1, 240 .dst_port = -1, 241 }, 242 }, 243 .default_mask = &rte_flow_item_udp_mask, 244 .mask_sz = sizeof(struct rte_flow_item_udp), 245 .convert = mlx5_flow_create_udp, 246 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 247 }, 248 [RTE_FLOW_ITEM_TYPE_TCP] = { 249 .actions = valid_actions, 250 .mask = &(const struct rte_flow_item_tcp){ 251 .hdr = { 252 .src_port = -1, 253 .dst_port = -1, 254 }, 255 }, 256 .default_mask = &rte_flow_item_tcp_mask, 257 .mask_sz = sizeof(struct rte_flow_item_tcp), 258 .convert = mlx5_flow_create_tcp, 259 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), 260 }, 261 [RTE_FLOW_ITEM_TYPE_VXLAN] = { 262 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), 263 .actions = valid_actions, 264 .mask = &(const struct rte_flow_item_vxlan){ 265 .vni = "\xff\xff\xff", 266 }, 267 .default_mask = &rte_flow_item_vxlan_mask, 268 .mask_sz = sizeof(struct rte_flow_item_vxlan), 269 .convert = mlx5_flow_create_vxlan, 270 .dst_sz = sizeof(struct ibv_flow_spec_tunnel), 271 }, 272 }; 273 274 /** Structure to pass to the conversion function. */ 275 struct mlx5_flow { 276 struct ibv_flow_attr *ibv_attr; /**< Verbs attribute. */ 277 unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */ 278 uint32_t inner; /**< Set once VXLAN is encountered. */ 279 uint64_t hash_fields; /**< Fields that participate in the hash. */ 280 }; 281 282 /** Structure for Drop queue. */ 283 struct rte_flow_drop { 284 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 285 struct ibv_qp *qp; /**< Verbs queue pair. */ 286 struct ibv_wq *wq; /**< Verbs work queue. */ 287 struct ibv_cq *cq; /**< Verbs completion queue. */ 288 }; 289 290 struct mlx5_flow_action { 291 uint32_t queue:1; /**< Target is a receive queue. */ 292 uint32_t drop:1; /**< Target is a drop queue. */ 293 uint32_t mark:1; /**< Mark is present in the flow. */ 294 uint32_t mark_id; /**< Mark identifier. */ 295 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ 296 uint16_t queues_n; /**< Number of entries in queue[]. */ 297 }; 298 299 /** 300 * Check support for a given item. 301 * 302 * @param item[in] 303 * Item specification. 304 * @param mask[in] 305 * Bit-masks covering supported fields to compare with spec, last and mask in 306 * \item. 307 * @param size 308 * Bit-Mask size in bytes. 309 * 310 * @return 311 * 0 on success. 312 */ 313 static int 314 mlx5_flow_item_validate(const struct rte_flow_item *item, 315 const uint8_t *mask, unsigned int size) 316 { 317 int ret = 0; 318 319 if (!item->spec && (item->mask || item->last)) 320 return -1; 321 if (item->spec && !item->mask) { 322 unsigned int i; 323 const uint8_t *spec = item->spec; 324 325 for (i = 0; i < size; ++i) 326 if ((spec[i] | mask[i]) != mask[i]) 327 return -1; 328 } 329 if (item->last && !item->mask) { 330 unsigned int i; 331 const uint8_t *spec = item->last; 332 333 for (i = 0; i < size; ++i) 334 if ((spec[i] | mask[i]) != mask[i]) 335 return -1; 336 } 337 if (item->mask) { 338 unsigned int i; 339 const uint8_t *spec = item->mask; 340 341 for (i = 0; i < size; ++i) 342 if ((spec[i] | mask[i]) != mask[i]) 343 return -1; 344 } 345 if (item->spec && item->last) { 346 uint8_t spec[size]; 347 uint8_t last[size]; 348 const uint8_t *apply = mask; 349 unsigned int i; 350 351 if (item->mask) 352 apply = item->mask; 353 for (i = 0; i < size; ++i) { 354 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; 355 last[i] = ((const uint8_t *)item->last)[i] & apply[i]; 356 } 357 ret = memcmp(spec, last, size); 358 } 359 return ret; 360 } 361 362 /** 363 * Validate a flow supported by the NIC. 364 * 365 * @param priv 366 * Pointer to private structure. 367 * @param[in] attr 368 * Flow rule attributes. 369 * @param[in] pattern 370 * Pattern specification (list terminated by the END pattern item). 371 * @param[in] actions 372 * Associated actions (list terminated by the END action). 373 * @param[out] error 374 * Perform verbose error reporting if not NULL. 375 * @param[in, out] flow 376 * Flow structure to update. 377 * @param[in, out] action 378 * Action structure to update. 379 * 380 * @return 381 * 0 on success, a negative errno value otherwise and rte_errno is set. 382 */ 383 static int 384 priv_flow_validate(struct priv *priv, 385 const struct rte_flow_attr *attr, 386 const struct rte_flow_item items[], 387 const struct rte_flow_action actions[], 388 struct rte_flow_error *error, 389 struct mlx5_flow *flow, 390 struct mlx5_flow_action *action) 391 { 392 const struct mlx5_flow_items *cur_item = mlx5_flow_items; 393 394 (void)priv; 395 if (attr->group) { 396 rte_flow_error_set(error, ENOTSUP, 397 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 398 NULL, 399 "groups are not supported"); 400 return -rte_errno; 401 } 402 if (attr->priority) { 403 rte_flow_error_set(error, ENOTSUP, 404 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 405 NULL, 406 "priorities are not supported"); 407 return -rte_errno; 408 } 409 if (attr->egress) { 410 rte_flow_error_set(error, ENOTSUP, 411 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 412 NULL, 413 "egress is not supported"); 414 return -rte_errno; 415 } 416 if (!attr->ingress) { 417 rte_flow_error_set(error, ENOTSUP, 418 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 419 NULL, 420 "only ingress is supported"); 421 return -rte_errno; 422 } 423 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { 424 const struct mlx5_flow_items *token = NULL; 425 unsigned int i; 426 int err; 427 428 if (items->type == RTE_FLOW_ITEM_TYPE_VOID) 429 continue; 430 for (i = 0; 431 cur_item->items && 432 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; 433 ++i) { 434 if (cur_item->items[i] == items->type) { 435 token = &mlx5_flow_items[items->type]; 436 break; 437 } 438 } 439 if (!token) 440 goto exit_item_not_supported; 441 cur_item = token; 442 err = mlx5_flow_item_validate(items, 443 (const uint8_t *)cur_item->mask, 444 cur_item->mask_sz); 445 if (err) 446 goto exit_item_not_supported; 447 if (flow->ibv_attr && cur_item->convert) { 448 err = cur_item->convert(items, 449 (cur_item->default_mask ? 450 cur_item->default_mask : 451 cur_item->mask), 452 flow); 453 if (err) 454 goto exit_item_not_supported; 455 } else if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 456 if (flow->inner) { 457 rte_flow_error_set(error, ENOTSUP, 458 RTE_FLOW_ERROR_TYPE_ITEM, 459 items, 460 "cannot recognize multiple" 461 " VXLAN encapsulations"); 462 return -rte_errno; 463 } 464 flow->inner = 1; 465 } 466 flow->offset += cur_item->dst_sz; 467 } 468 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { 469 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { 470 continue; 471 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { 472 action->drop = 1; 473 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 474 const struct rte_flow_action_queue *queue = 475 (const struct rte_flow_action_queue *) 476 actions->conf; 477 uint16_t n; 478 uint16_t found = 0; 479 480 if (!queue || (queue->index > (priv->rxqs_n - 1))) 481 goto exit_action_not_supported; 482 for (n = 0; n < action->queues_n; ++n) { 483 if (action->queues[n] == queue->index) { 484 found = 1; 485 break; 486 } 487 } 488 if (action->queues_n > 1 && !found) { 489 rte_flow_error_set(error, ENOTSUP, 490 RTE_FLOW_ERROR_TYPE_ACTION, 491 actions, 492 "queue action not in RSS queues"); 493 return -rte_errno; 494 } 495 if (!found) { 496 action->queue = 1; 497 action->queues_n = 1; 498 action->queues[0] = queue->index; 499 } 500 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { 501 const struct rte_flow_action_rss *rss = 502 (const struct rte_flow_action_rss *) 503 actions->conf; 504 uint16_t n; 505 506 if (!rss || !rss->num) { 507 rte_flow_error_set(error, EINVAL, 508 RTE_FLOW_ERROR_TYPE_ACTION, 509 actions, 510 "no valid queues"); 511 return -rte_errno; 512 } 513 if (action->queues_n == 1) { 514 uint16_t found = 0; 515 516 assert(action->queues_n); 517 for (n = 0; n < rss->num; ++n) { 518 if (action->queues[0] == 519 rss->queue[n]) { 520 found = 1; 521 break; 522 } 523 } 524 if (!found) { 525 rte_flow_error_set(error, ENOTSUP, 526 RTE_FLOW_ERROR_TYPE_ACTION, 527 actions, 528 "queue action not in RSS" 529 " queues"); 530 return -rte_errno; 531 } 532 } 533 for (n = 0; n < rss->num; ++n) { 534 if (rss->queue[n] >= priv->rxqs_n) { 535 rte_flow_error_set(error, EINVAL, 536 RTE_FLOW_ERROR_TYPE_ACTION, 537 actions, 538 "queue id > number of" 539 " queues"); 540 return -rte_errno; 541 } 542 } 543 action->queue = 1; 544 for (n = 0; n < rss->num; ++n) 545 action->queues[n] = rss->queue[n]; 546 action->queues_n = rss->num; 547 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { 548 const struct rte_flow_action_mark *mark = 549 (const struct rte_flow_action_mark *) 550 actions->conf; 551 552 if (!mark) { 553 rte_flow_error_set(error, EINVAL, 554 RTE_FLOW_ERROR_TYPE_ACTION, 555 actions, 556 "mark must be defined"); 557 return -rte_errno; 558 } else if (mark->id >= MLX5_FLOW_MARK_MAX) { 559 rte_flow_error_set(error, ENOTSUP, 560 RTE_FLOW_ERROR_TYPE_ACTION, 561 actions, 562 "mark must be between 0" 563 " and 16777199"); 564 return -rte_errno; 565 } 566 action->mark = 1; 567 action->mark_id = mark->id; 568 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { 569 action->mark = 1; 570 } else { 571 goto exit_action_not_supported; 572 } 573 } 574 if (action->mark && !flow->ibv_attr && !action->drop) 575 flow->offset += sizeof(struct ibv_flow_spec_action_tag); 576 if (!flow->ibv_attr && action->drop) 577 flow->offset += sizeof(struct ibv_flow_spec_action_drop); 578 if (!action->queue && !action->drop) { 579 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, 580 NULL, "no valid action"); 581 return -rte_errno; 582 } 583 return 0; 584 exit_item_not_supported: 585 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 586 items, "item not supported"); 587 return -rte_errno; 588 exit_action_not_supported: 589 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 590 actions, "action not supported"); 591 return -rte_errno; 592 } 593 594 /** 595 * Validate a flow supported by the NIC. 596 * 597 * @see rte_flow_validate() 598 * @see rte_flow_ops 599 */ 600 int 601 mlx5_flow_validate(struct rte_eth_dev *dev, 602 const struct rte_flow_attr *attr, 603 const struct rte_flow_item items[], 604 const struct rte_flow_action actions[], 605 struct rte_flow_error *error) 606 { 607 struct priv *priv = dev->data->dev_private; 608 int ret; 609 struct mlx5_flow flow = { .offset = sizeof(struct ibv_flow_attr) }; 610 struct mlx5_flow_action action = { 611 .queue = 0, 612 .drop = 0, 613 .mark = 0, 614 .mark_id = MLX5_FLOW_MARK_DEFAULT, 615 .queues_n = 0, 616 }; 617 618 priv_lock(priv); 619 ret = priv_flow_validate(priv, attr, items, actions, error, &flow, 620 &action); 621 priv_unlock(priv); 622 return ret; 623 } 624 625 /** 626 * Convert Ethernet item to Verbs specification. 627 * 628 * @param item[in] 629 * Item specification. 630 * @param default_mask[in] 631 * Default bit-masks to use when item->mask is not provided. 632 * @param data[in, out] 633 * User structure. 634 */ 635 static int 636 mlx5_flow_create_eth(const struct rte_flow_item *item, 637 const void *default_mask, 638 void *data) 639 { 640 const struct rte_flow_item_eth *spec = item->spec; 641 const struct rte_flow_item_eth *mask = item->mask; 642 struct mlx5_flow *flow = (struct mlx5_flow *)data; 643 struct ibv_flow_spec_eth *eth; 644 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 645 unsigned int i; 646 647 ++flow->ibv_attr->num_of_specs; 648 flow->ibv_attr->priority = 2; 649 flow->hash_fields = 0; 650 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset); 651 *eth = (struct ibv_flow_spec_eth) { 652 .type = flow->inner | IBV_FLOW_SPEC_ETH, 653 .size = eth_size, 654 }; 655 if (!spec) 656 return 0; 657 if (!mask) 658 mask = default_mask; 659 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); 660 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); 661 eth->val.ether_type = spec->type; 662 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); 663 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); 664 eth->mask.ether_type = mask->type; 665 /* Remove unwanted bits from values. */ 666 for (i = 0; i < ETHER_ADDR_LEN; ++i) { 667 eth->val.dst_mac[i] &= eth->mask.dst_mac[i]; 668 eth->val.src_mac[i] &= eth->mask.src_mac[i]; 669 } 670 eth->val.ether_type &= eth->mask.ether_type; 671 return 0; 672 } 673 674 /** 675 * Convert VLAN item to Verbs specification. 676 * 677 * @param item[in] 678 * Item specification. 679 * @param default_mask[in] 680 * Default bit-masks to use when item->mask is not provided. 681 * @param data[in, out] 682 * User structure. 683 */ 684 static int 685 mlx5_flow_create_vlan(const struct rte_flow_item *item, 686 const void *default_mask, 687 void *data) 688 { 689 const struct rte_flow_item_vlan *spec = item->spec; 690 const struct rte_flow_item_vlan *mask = item->mask; 691 struct mlx5_flow *flow = (struct mlx5_flow *)data; 692 struct ibv_flow_spec_eth *eth; 693 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); 694 695 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size); 696 if (!spec) 697 return 0; 698 if (!mask) 699 mask = default_mask; 700 eth->val.vlan_tag = spec->tci; 701 eth->mask.vlan_tag = mask->tci; 702 eth->val.vlan_tag &= eth->mask.vlan_tag; 703 return 0; 704 } 705 706 /** 707 * Convert IPv4 item to Verbs specification. 708 * 709 * @param item[in] 710 * Item specification. 711 * @param default_mask[in] 712 * Default bit-masks to use when item->mask is not provided. 713 * @param data[in, out] 714 * User structure. 715 */ 716 static int 717 mlx5_flow_create_ipv4(const struct rte_flow_item *item, 718 const void *default_mask, 719 void *data) 720 { 721 const struct rte_flow_item_ipv4 *spec = item->spec; 722 const struct rte_flow_item_ipv4 *mask = item->mask; 723 struct mlx5_flow *flow = (struct mlx5_flow *)data; 724 struct ibv_flow_spec_ipv4_ext *ipv4; 725 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); 726 727 ++flow->ibv_attr->num_of_specs; 728 flow->ibv_attr->priority = 1; 729 flow->hash_fields = (IBV_RX_HASH_SRC_IPV4 | 730 IBV_RX_HASH_DST_IPV4); 731 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset); 732 *ipv4 = (struct ibv_flow_spec_ipv4_ext) { 733 .type = flow->inner | IBV_FLOW_SPEC_IPV4_EXT, 734 .size = ipv4_size, 735 }; 736 if (!spec) 737 return 0; 738 if (!mask) 739 mask = default_mask; 740 ipv4->val = (struct ibv_flow_ipv4_ext_filter){ 741 .src_ip = spec->hdr.src_addr, 742 .dst_ip = spec->hdr.dst_addr, 743 .proto = spec->hdr.next_proto_id, 744 .tos = spec->hdr.type_of_service, 745 }; 746 ipv4->mask = (struct ibv_flow_ipv4_ext_filter){ 747 .src_ip = mask->hdr.src_addr, 748 .dst_ip = mask->hdr.dst_addr, 749 .proto = mask->hdr.next_proto_id, 750 .tos = mask->hdr.type_of_service, 751 }; 752 /* Remove unwanted bits from values. */ 753 ipv4->val.src_ip &= ipv4->mask.src_ip; 754 ipv4->val.dst_ip &= ipv4->mask.dst_ip; 755 ipv4->val.proto &= ipv4->mask.proto; 756 ipv4->val.tos &= ipv4->mask.tos; 757 return 0; 758 } 759 760 /** 761 * Convert IPv6 item to Verbs specification. 762 * 763 * @param item[in] 764 * Item specification. 765 * @param default_mask[in] 766 * Default bit-masks to use when item->mask is not provided. 767 * @param data[in, out] 768 * User structure. 769 */ 770 static int 771 mlx5_flow_create_ipv6(const struct rte_flow_item *item, 772 const void *default_mask, 773 void *data) 774 { 775 const struct rte_flow_item_ipv6 *spec = item->spec; 776 const struct rte_flow_item_ipv6 *mask = item->mask; 777 struct mlx5_flow *flow = (struct mlx5_flow *)data; 778 struct ibv_flow_spec_ipv6 *ipv6; 779 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); 780 unsigned int i; 781 782 ++flow->ibv_attr->num_of_specs; 783 flow->ibv_attr->priority = 1; 784 flow->hash_fields = (IBV_RX_HASH_SRC_IPV6 | 785 IBV_RX_HASH_DST_IPV6); 786 ipv6 = (void *)((uintptr_t)flow->ibv_attr + flow->offset); 787 *ipv6 = (struct ibv_flow_spec_ipv6) { 788 .type = flow->inner | IBV_FLOW_SPEC_IPV6, 789 .size = ipv6_size, 790 }; 791 if (!spec) 792 return 0; 793 if (!mask) 794 mask = default_mask; 795 memcpy(ipv6->val.src_ip, spec->hdr.src_addr, 796 RTE_DIM(ipv6->val.src_ip)); 797 memcpy(ipv6->val.dst_ip, spec->hdr.dst_addr, 798 RTE_DIM(ipv6->val.dst_ip)); 799 memcpy(ipv6->mask.src_ip, mask->hdr.src_addr, 800 RTE_DIM(ipv6->mask.src_ip)); 801 memcpy(ipv6->mask.dst_ip, mask->hdr.dst_addr, 802 RTE_DIM(ipv6->mask.dst_ip)); 803 ipv6->mask.flow_label = mask->hdr.vtc_flow; 804 ipv6->mask.next_hdr = mask->hdr.proto; 805 ipv6->mask.hop_limit = mask->hdr.hop_limits; 806 /* Remove unwanted bits from values. */ 807 for (i = 0; i < RTE_DIM(ipv6->val.src_ip); ++i) { 808 ipv6->val.src_ip[i] &= ipv6->mask.src_ip[i]; 809 ipv6->val.dst_ip[i] &= ipv6->mask.dst_ip[i]; 810 } 811 ipv6->val.flow_label &= ipv6->mask.flow_label; 812 ipv6->val.next_hdr &= ipv6->mask.next_hdr; 813 ipv6->val.hop_limit &= ipv6->mask.hop_limit; 814 return 0; 815 } 816 817 /** 818 * Convert UDP item to Verbs specification. 819 * 820 * @param item[in] 821 * Item specification. 822 * @param default_mask[in] 823 * Default bit-masks to use when item->mask is not provided. 824 * @param data[in, out] 825 * User structure. 826 */ 827 static int 828 mlx5_flow_create_udp(const struct rte_flow_item *item, 829 const void *default_mask, 830 void *data) 831 { 832 const struct rte_flow_item_udp *spec = item->spec; 833 const struct rte_flow_item_udp *mask = item->mask; 834 struct mlx5_flow *flow = (struct mlx5_flow *)data; 835 struct ibv_flow_spec_tcp_udp *udp; 836 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); 837 838 ++flow->ibv_attr->num_of_specs; 839 flow->ibv_attr->priority = 0; 840 flow->hash_fields |= (IBV_RX_HASH_SRC_PORT_UDP | 841 IBV_RX_HASH_DST_PORT_UDP); 842 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset); 843 *udp = (struct ibv_flow_spec_tcp_udp) { 844 .type = flow->inner | IBV_FLOW_SPEC_UDP, 845 .size = udp_size, 846 }; 847 if (!spec) 848 return 0; 849 if (!mask) 850 mask = default_mask; 851 udp->val.dst_port = spec->hdr.dst_port; 852 udp->val.src_port = spec->hdr.src_port; 853 udp->mask.dst_port = mask->hdr.dst_port; 854 udp->mask.src_port = mask->hdr.src_port; 855 /* Remove unwanted bits from values. */ 856 udp->val.src_port &= udp->mask.src_port; 857 udp->val.dst_port &= udp->mask.dst_port; 858 return 0; 859 } 860 861 /** 862 * Convert TCP item to Verbs specification. 863 * 864 * @param item[in] 865 * Item specification. 866 * @param default_mask[in] 867 * Default bit-masks to use when item->mask is not provided. 868 * @param data[in, out] 869 * User structure. 870 */ 871 static int 872 mlx5_flow_create_tcp(const struct rte_flow_item *item, 873 const void *default_mask, 874 void *data) 875 { 876 const struct rte_flow_item_tcp *spec = item->spec; 877 const struct rte_flow_item_tcp *mask = item->mask; 878 struct mlx5_flow *flow = (struct mlx5_flow *)data; 879 struct ibv_flow_spec_tcp_udp *tcp; 880 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); 881 882 ++flow->ibv_attr->num_of_specs; 883 flow->ibv_attr->priority = 0; 884 flow->hash_fields |= (IBV_RX_HASH_SRC_PORT_TCP | 885 IBV_RX_HASH_DST_PORT_TCP); 886 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset); 887 *tcp = (struct ibv_flow_spec_tcp_udp) { 888 .type = flow->inner | IBV_FLOW_SPEC_TCP, 889 .size = tcp_size, 890 }; 891 if (!spec) 892 return 0; 893 if (!mask) 894 mask = default_mask; 895 tcp->val.dst_port = spec->hdr.dst_port; 896 tcp->val.src_port = spec->hdr.src_port; 897 tcp->mask.dst_port = mask->hdr.dst_port; 898 tcp->mask.src_port = mask->hdr.src_port; 899 /* Remove unwanted bits from values. */ 900 tcp->val.src_port &= tcp->mask.src_port; 901 tcp->val.dst_port &= tcp->mask.dst_port; 902 return 0; 903 } 904 905 /** 906 * Convert VXLAN item to Verbs specification. 907 * 908 * @param item[in] 909 * Item specification. 910 * @param default_mask[in] 911 * Default bit-masks to use when item->mask is not provided. 912 * @param data[in, out] 913 * User structure. 914 */ 915 static int 916 mlx5_flow_create_vxlan(const struct rte_flow_item *item, 917 const void *default_mask, 918 void *data) 919 { 920 const struct rte_flow_item_vxlan *spec = item->spec; 921 const struct rte_flow_item_vxlan *mask = item->mask; 922 struct mlx5_flow *flow = (struct mlx5_flow *)data; 923 struct ibv_flow_spec_tunnel *vxlan; 924 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 925 union vni { 926 uint32_t vlan_id; 927 uint8_t vni[4]; 928 } id; 929 930 ++flow->ibv_attr->num_of_specs; 931 flow->ibv_attr->priority = 0; 932 id.vni[0] = 0; 933 vxlan = (void *)((uintptr_t)flow->ibv_attr + flow->offset); 934 *vxlan = (struct ibv_flow_spec_tunnel) { 935 .type = flow->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, 936 .size = size, 937 }; 938 flow->inner = IBV_FLOW_SPEC_INNER; 939 if (!spec) 940 return 0; 941 if (!mask) 942 mask = default_mask; 943 memcpy(&id.vni[1], spec->vni, 3); 944 vxlan->val.tunnel_id = id.vlan_id; 945 memcpy(&id.vni[1], mask->vni, 3); 946 vxlan->mask.tunnel_id = id.vlan_id; 947 /* Remove unwanted bits from values. */ 948 vxlan->val.tunnel_id &= vxlan->mask.tunnel_id; 949 return 0; 950 } 951 952 /** 953 * Convert mark/flag action to Verbs specification. 954 * 955 * @param flow 956 * Pointer to MLX5 flow structure. 957 * @param mark_id 958 * Mark identifier. 959 */ 960 static int 961 mlx5_flow_create_flag_mark(struct mlx5_flow *flow, uint32_t mark_id) 962 { 963 struct ibv_flow_spec_action_tag *tag; 964 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 965 966 tag = (void *)((uintptr_t)flow->ibv_attr + flow->offset); 967 *tag = (struct ibv_flow_spec_action_tag){ 968 .type = IBV_FLOW_SPEC_ACTION_TAG, 969 .size = size, 970 .tag_id = mlx5_flow_mark_set(mark_id), 971 }; 972 ++flow->ibv_attr->num_of_specs; 973 return 0; 974 } 975 976 /** 977 * Complete flow rule creation with a drop queue. 978 * 979 * @param priv 980 * Pointer to private structure. 981 * @param flow 982 * MLX5 flow attributes (filled by mlx5_flow_validate()). 983 * @param[out] error 984 * Perform verbose error reporting if not NULL. 985 * 986 * @return 987 * A flow if the rule could be created. 988 */ 989 static struct rte_flow * 990 priv_flow_create_action_queue_drop(struct priv *priv, 991 struct mlx5_flow *flow, 992 struct rte_flow_error *error) 993 { 994 struct rte_flow *rte_flow; 995 struct ibv_flow_spec_action_drop *drop; 996 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 997 998 assert(priv->pd); 999 assert(priv->ctx); 1000 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0); 1001 if (!rte_flow) { 1002 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1003 NULL, "cannot allocate flow memory"); 1004 return NULL; 1005 } 1006 rte_flow->drop = 1; 1007 drop = (void *)((uintptr_t)flow->ibv_attr + flow->offset); 1008 *drop = (struct ibv_flow_spec_action_drop){ 1009 .type = IBV_FLOW_SPEC_ACTION_DROP, 1010 .size = size, 1011 }; 1012 ++flow->ibv_attr->num_of_specs; 1013 flow->offset += sizeof(struct ibv_flow_spec_action_drop); 1014 rte_flow->ibv_attr = flow->ibv_attr; 1015 if (!priv->started) 1016 return rte_flow; 1017 rte_flow->qp = priv->flow_drop_queue->qp; 1018 rte_flow->ibv_flow = ibv_create_flow(rte_flow->qp, 1019 rte_flow->ibv_attr); 1020 if (!rte_flow->ibv_flow) { 1021 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1022 NULL, "flow rule creation failure"); 1023 goto error; 1024 } 1025 return rte_flow; 1026 error: 1027 assert(rte_flow); 1028 rte_free(rte_flow); 1029 return NULL; 1030 } 1031 1032 /** 1033 * Complete flow rule creation. 1034 * 1035 * @param priv 1036 * Pointer to private structure. 1037 * @param flow 1038 * MLX5 flow attributes (filled by mlx5_flow_validate()). 1039 * @param action 1040 * Target action structure. 1041 * @param[out] error 1042 * Perform verbose error reporting if not NULL. 1043 * 1044 * @return 1045 * A flow if the rule could be created. 1046 */ 1047 static struct rte_flow * 1048 priv_flow_create_action_queue(struct priv *priv, 1049 struct mlx5_flow *flow, 1050 struct mlx5_flow_action *action, 1051 struct rte_flow_error *error) 1052 { 1053 struct rte_flow *rte_flow; 1054 unsigned int i; 1055 unsigned int j; 1056 const unsigned int wqs_n = 1 << log2above(action->queues_n); 1057 struct ibv_wq *wqs[wqs_n]; 1058 1059 assert(priv->pd); 1060 assert(priv->ctx); 1061 assert(!action->drop); 1062 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow) + 1063 sizeof(*rte_flow->rxqs) * action->queues_n, 0); 1064 if (!rte_flow) { 1065 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1066 NULL, "cannot allocate flow memory"); 1067 return NULL; 1068 } 1069 for (i = 0; i < action->queues_n; ++i) { 1070 struct rxq_ctrl *rxq; 1071 1072 rxq = container_of((*priv->rxqs)[action->queues[i]], 1073 struct rxq_ctrl, rxq); 1074 wqs[i] = rxq->wq; 1075 rte_flow->rxqs[i] = &rxq->rxq; 1076 ++rte_flow->rxqs_n; 1077 rxq->rxq.mark |= action->mark; 1078 } 1079 /* finalise indirection table. */ 1080 for (j = 0; i < wqs_n; ++i, ++j) { 1081 wqs[i] = wqs[j]; 1082 if (j == action->queues_n) 1083 j = 0; 1084 } 1085 rte_flow->mark = action->mark; 1086 rte_flow->ibv_attr = flow->ibv_attr; 1087 rte_flow->hash_fields = flow->hash_fields; 1088 rte_flow->ind_table = ibv_create_rwq_ind_table( 1089 priv->ctx, 1090 &(struct ibv_rwq_ind_table_init_attr){ 1091 .log_ind_tbl_size = log2above(action->queues_n), 1092 .ind_tbl = wqs, 1093 .comp_mask = 0, 1094 }); 1095 if (!rte_flow->ind_table) { 1096 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1097 NULL, "cannot allocate indirection table"); 1098 goto error; 1099 } 1100 rte_flow->qp = ibv_create_qp_ex( 1101 priv->ctx, 1102 &(struct ibv_qp_init_attr_ex){ 1103 .qp_type = IBV_QPT_RAW_PACKET, 1104 .comp_mask = 1105 IBV_QP_INIT_ATTR_PD | 1106 IBV_QP_INIT_ATTR_IND_TABLE | 1107 IBV_QP_INIT_ATTR_RX_HASH, 1108 .rx_hash_conf = (struct ibv_rx_hash_conf){ 1109 .rx_hash_function = 1110 IBV_RX_HASH_FUNC_TOEPLITZ, 1111 .rx_hash_key_len = rss_hash_default_key_len, 1112 .rx_hash_key = rss_hash_default_key, 1113 .rx_hash_fields_mask = rte_flow->hash_fields, 1114 }, 1115 .rwq_ind_tbl = rte_flow->ind_table, 1116 .pd = priv->pd 1117 }); 1118 if (!rte_flow->qp) { 1119 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1120 NULL, "cannot allocate QP"); 1121 goto error; 1122 } 1123 if (!priv->started) 1124 return rte_flow; 1125 rte_flow->ibv_flow = ibv_create_flow(rte_flow->qp, 1126 rte_flow->ibv_attr); 1127 if (!rte_flow->ibv_flow) { 1128 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1129 NULL, "flow rule creation failure"); 1130 goto error; 1131 } 1132 return rte_flow; 1133 error: 1134 assert(rte_flow); 1135 if (rte_flow->qp) 1136 ibv_destroy_qp(rte_flow->qp); 1137 if (rte_flow->ind_table) 1138 ibv_destroy_rwq_ind_table(rte_flow->ind_table); 1139 rte_free(rte_flow); 1140 return NULL; 1141 } 1142 1143 /** 1144 * Convert a flow. 1145 * 1146 * @param priv 1147 * Pointer to private structure. 1148 * @param[in] attr 1149 * Flow rule attributes. 1150 * @param[in] pattern 1151 * Pattern specification (list terminated by the END pattern item). 1152 * @param[in] actions 1153 * Associated actions (list terminated by the END action). 1154 * @param[out] error 1155 * Perform verbose error reporting if not NULL. 1156 * 1157 * @return 1158 * A flow on success, NULL otherwise. 1159 */ 1160 static struct rte_flow * 1161 priv_flow_create(struct priv *priv, 1162 const struct rte_flow_attr *attr, 1163 const struct rte_flow_item items[], 1164 const struct rte_flow_action actions[], 1165 struct rte_flow_error *error) 1166 { 1167 struct rte_flow *rte_flow; 1168 struct mlx5_flow flow = { .offset = sizeof(struct ibv_flow_attr), }; 1169 struct mlx5_flow_action action = { 1170 .queue = 0, 1171 .drop = 0, 1172 .mark = 0, 1173 .mark_id = MLX5_FLOW_MARK_DEFAULT, 1174 .queues_n = 0, 1175 }; 1176 int err; 1177 1178 err = priv_flow_validate(priv, attr, items, actions, error, &flow, 1179 &action); 1180 if (err) 1181 goto exit; 1182 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0); 1183 flow.offset = sizeof(struct ibv_flow_attr); 1184 if (!flow.ibv_attr) { 1185 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 1186 NULL, "cannot allocate ibv_attr memory"); 1187 goto exit; 1188 } 1189 *flow.ibv_attr = (struct ibv_flow_attr){ 1190 .type = IBV_FLOW_ATTR_NORMAL, 1191 .size = sizeof(struct ibv_flow_attr), 1192 .priority = attr->priority, 1193 .num_of_specs = 0, 1194 .port = 0, 1195 .flags = 0, 1196 }; 1197 flow.inner = 0; 1198 flow.hash_fields = 0; 1199 claim_zero(priv_flow_validate(priv, attr, items, actions, 1200 error, &flow, &action)); 1201 if (action.mark && !action.drop) { 1202 mlx5_flow_create_flag_mark(&flow, action.mark_id); 1203 flow.offset += sizeof(struct ibv_flow_spec_action_tag); 1204 } 1205 if (action.drop) 1206 rte_flow = 1207 priv_flow_create_action_queue_drop(priv, &flow, error); 1208 else 1209 rte_flow = priv_flow_create_action_queue(priv, &flow, &action, 1210 error); 1211 if (!rte_flow) 1212 goto exit; 1213 return rte_flow; 1214 exit: 1215 rte_free(flow.ibv_attr); 1216 return NULL; 1217 } 1218 1219 /** 1220 * Create a flow. 1221 * 1222 * @see rte_flow_create() 1223 * @see rte_flow_ops 1224 */ 1225 struct rte_flow * 1226 mlx5_flow_create(struct rte_eth_dev *dev, 1227 const struct rte_flow_attr *attr, 1228 const struct rte_flow_item items[], 1229 const struct rte_flow_action actions[], 1230 struct rte_flow_error *error) 1231 { 1232 struct priv *priv = dev->data->dev_private; 1233 struct rte_flow *flow; 1234 1235 priv_lock(priv); 1236 flow = priv_flow_create(priv, attr, items, actions, error); 1237 if (flow) { 1238 TAILQ_INSERT_TAIL(&priv->flows, flow, next); 1239 DEBUG("Flow created %p", (void *)flow); 1240 } 1241 priv_unlock(priv); 1242 return flow; 1243 } 1244 1245 /** 1246 * Destroy a flow. 1247 * 1248 * @param priv 1249 * Pointer to private structure. 1250 * @param[in] flow 1251 * Flow to destroy. 1252 */ 1253 static void 1254 priv_flow_destroy(struct priv *priv, 1255 struct rte_flow *flow) 1256 { 1257 TAILQ_REMOVE(&priv->flows, flow, next); 1258 if (flow->ibv_flow) 1259 claim_zero(ibv_destroy_flow(flow->ibv_flow)); 1260 if (flow->drop) 1261 goto free; 1262 if (flow->qp) 1263 claim_zero(ibv_destroy_qp(flow->qp)); 1264 if (flow->ind_table) 1265 claim_zero(ibv_destroy_rwq_ind_table(flow->ind_table)); 1266 if (flow->mark) { 1267 struct rte_flow *tmp; 1268 struct rxq *rxq; 1269 uint32_t mark_n = 0; 1270 uint32_t queue_n; 1271 1272 /* 1273 * To remove the mark from the queue, the queue must not be 1274 * present in any other marked flow (RSS or not). 1275 */ 1276 for (queue_n = 0; queue_n < flow->rxqs_n; ++queue_n) { 1277 rxq = flow->rxqs[queue_n]; 1278 for (tmp = TAILQ_FIRST(&priv->flows); 1279 tmp; 1280 tmp = TAILQ_NEXT(tmp, next)) { 1281 uint32_t tqueue_n; 1282 1283 if (tmp->drop) 1284 continue; 1285 for (tqueue_n = 0; 1286 tqueue_n < tmp->rxqs_n; 1287 ++tqueue_n) { 1288 struct rxq *trxq; 1289 1290 trxq = tmp->rxqs[tqueue_n]; 1291 if (rxq == trxq) 1292 ++mark_n; 1293 } 1294 } 1295 rxq->mark = !!mark_n; 1296 } 1297 } 1298 free: 1299 rte_free(flow->ibv_attr); 1300 DEBUG("Flow destroyed %p", (void *)flow); 1301 rte_free(flow); 1302 } 1303 1304 /** 1305 * Destroy a flow. 1306 * 1307 * @see rte_flow_destroy() 1308 * @see rte_flow_ops 1309 */ 1310 int 1311 mlx5_flow_destroy(struct rte_eth_dev *dev, 1312 struct rte_flow *flow, 1313 struct rte_flow_error *error) 1314 { 1315 struct priv *priv = dev->data->dev_private; 1316 1317 (void)error; 1318 priv_lock(priv); 1319 priv_flow_destroy(priv, flow); 1320 priv_unlock(priv); 1321 return 0; 1322 } 1323 1324 /** 1325 * Destroy all flows. 1326 * 1327 * @param priv 1328 * Pointer to private structure. 1329 */ 1330 static void 1331 priv_flow_flush(struct priv *priv) 1332 { 1333 while (!TAILQ_EMPTY(&priv->flows)) { 1334 struct rte_flow *flow; 1335 1336 flow = TAILQ_FIRST(&priv->flows); 1337 priv_flow_destroy(priv, flow); 1338 } 1339 } 1340 1341 /** 1342 * Destroy all flows. 1343 * 1344 * @see rte_flow_flush() 1345 * @see rte_flow_ops 1346 */ 1347 int 1348 mlx5_flow_flush(struct rte_eth_dev *dev, 1349 struct rte_flow_error *error) 1350 { 1351 struct priv *priv = dev->data->dev_private; 1352 1353 (void)error; 1354 priv_lock(priv); 1355 priv_flow_flush(priv); 1356 priv_unlock(priv); 1357 return 0; 1358 } 1359 1360 /** 1361 * Create drop queue. 1362 * 1363 * @param priv 1364 * Pointer to private structure. 1365 * 1366 * @return 1367 * 0 on success. 1368 */ 1369 static int 1370 priv_flow_create_drop_queue(struct priv *priv) 1371 { 1372 struct rte_flow_drop *fdq = NULL; 1373 1374 assert(priv->pd); 1375 assert(priv->ctx); 1376 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); 1377 if (!fdq) { 1378 WARN("cannot allocate memory for drop queue"); 1379 goto error; 1380 } 1381 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); 1382 if (!fdq->cq) { 1383 WARN("cannot allocate CQ for drop queue"); 1384 goto error; 1385 } 1386 fdq->wq = ibv_create_wq(priv->ctx, 1387 &(struct ibv_wq_init_attr){ 1388 .wq_type = IBV_WQT_RQ, 1389 .max_wr = 1, 1390 .max_sge = 1, 1391 .pd = priv->pd, 1392 .cq = fdq->cq, 1393 }); 1394 if (!fdq->wq) { 1395 WARN("cannot allocate WQ for drop queue"); 1396 goto error; 1397 } 1398 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx, 1399 &(struct ibv_rwq_ind_table_init_attr){ 1400 .log_ind_tbl_size = 0, 1401 .ind_tbl = &fdq->wq, 1402 .comp_mask = 0, 1403 }); 1404 if (!fdq->ind_table) { 1405 WARN("cannot allocate indirection table for drop queue"); 1406 goto error; 1407 } 1408 fdq->qp = ibv_create_qp_ex(priv->ctx, 1409 &(struct ibv_qp_init_attr_ex){ 1410 .qp_type = IBV_QPT_RAW_PACKET, 1411 .comp_mask = 1412 IBV_QP_INIT_ATTR_PD | 1413 IBV_QP_INIT_ATTR_IND_TABLE | 1414 IBV_QP_INIT_ATTR_RX_HASH, 1415 .rx_hash_conf = (struct ibv_rx_hash_conf){ 1416 .rx_hash_function = 1417 IBV_RX_HASH_FUNC_TOEPLITZ, 1418 .rx_hash_key_len = rss_hash_default_key_len, 1419 .rx_hash_key = rss_hash_default_key, 1420 .rx_hash_fields_mask = 0, 1421 }, 1422 .rwq_ind_tbl = fdq->ind_table, 1423 .pd = priv->pd 1424 }); 1425 if (!fdq->qp) { 1426 WARN("cannot allocate QP for drop queue"); 1427 goto error; 1428 } 1429 priv->flow_drop_queue = fdq; 1430 return 0; 1431 error: 1432 if (fdq->qp) 1433 claim_zero(ibv_destroy_qp(fdq->qp)); 1434 if (fdq->ind_table) 1435 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 1436 if (fdq->wq) 1437 claim_zero(ibv_destroy_wq(fdq->wq)); 1438 if (fdq->cq) 1439 claim_zero(ibv_destroy_cq(fdq->cq)); 1440 if (fdq) 1441 rte_free(fdq); 1442 priv->flow_drop_queue = NULL; 1443 return -1; 1444 } 1445 1446 /** 1447 * Delete drop queue. 1448 * 1449 * @param priv 1450 * Pointer to private structure. 1451 */ 1452 static void 1453 priv_flow_delete_drop_queue(struct priv *priv) 1454 { 1455 struct rte_flow_drop *fdq = priv->flow_drop_queue; 1456 1457 if (!fdq) 1458 return; 1459 if (fdq->qp) 1460 claim_zero(ibv_destroy_qp(fdq->qp)); 1461 if (fdq->ind_table) 1462 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); 1463 if (fdq->wq) 1464 claim_zero(ibv_destroy_wq(fdq->wq)); 1465 if (fdq->cq) 1466 claim_zero(ibv_destroy_cq(fdq->cq)); 1467 rte_free(fdq); 1468 priv->flow_drop_queue = NULL; 1469 } 1470 1471 /** 1472 * Remove all flows. 1473 * 1474 * Called by dev_stop() to remove all flows. 1475 * 1476 * @param priv 1477 * Pointer to private structure. 1478 */ 1479 void 1480 priv_flow_stop(struct priv *priv) 1481 { 1482 struct rte_flow *flow; 1483 1484 TAILQ_FOREACH_REVERSE(flow, &priv->flows, mlx5_flows, next) { 1485 claim_zero(ibv_destroy_flow(flow->ibv_flow)); 1486 flow->ibv_flow = NULL; 1487 if (flow->mark) { 1488 unsigned int n; 1489 1490 for (n = 0; n < flow->rxqs_n; ++n) 1491 flow->rxqs[n]->mark = 0; 1492 } 1493 DEBUG("Flow %p removed", (void *)flow); 1494 } 1495 priv_flow_delete_drop_queue(priv); 1496 } 1497 1498 /** 1499 * Add all flows. 1500 * 1501 * @param priv 1502 * Pointer to private structure. 1503 * 1504 * @return 1505 * 0 on success, a errno value otherwise and rte_errno is set. 1506 */ 1507 int 1508 priv_flow_start(struct priv *priv) 1509 { 1510 int ret; 1511 struct rte_flow *flow; 1512 1513 ret = priv_flow_create_drop_queue(priv); 1514 if (ret) 1515 return -1; 1516 TAILQ_FOREACH(flow, &priv->flows, next) { 1517 struct ibv_qp *qp; 1518 1519 if (flow->drop) 1520 qp = priv->flow_drop_queue->qp; 1521 else 1522 qp = flow->qp; 1523 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr); 1524 if (!flow->ibv_flow) { 1525 DEBUG("Flow %p cannot be applied", (void *)flow); 1526 rte_errno = EINVAL; 1527 return rte_errno; 1528 } 1529 DEBUG("Flow %p applied", (void *)flow); 1530 if (flow->mark) { 1531 unsigned int n; 1532 1533 for (n = 0; n < flow->rxqs_n; ++n) 1534 flow->rxqs[n]->mark = 1; 1535 } 1536 } 1537 return 0; 1538 } 1539 1540 /** 1541 * Verify if the Rx queue is used in a flow. 1542 * 1543 * @param priv 1544 * Pointer to private structure. 1545 * @param rxq 1546 * Pointer to the queue to search. 1547 * 1548 * @return 1549 * Nonzero if the queue is used by a flow. 1550 */ 1551 int 1552 priv_flow_rxq_in_use(struct priv *priv, struct rxq *rxq) 1553 { 1554 struct rte_flow *flow; 1555 1556 for (flow = TAILQ_FIRST(&priv->flows); 1557 flow; 1558 flow = TAILQ_NEXT(flow, next)) { 1559 unsigned int n; 1560 1561 if (flow->drop) 1562 continue; 1563 for (n = 0; n < flow->rxqs_n; ++n) { 1564 if (flow->rxqs[n] == rxq) 1565 return 1; 1566 } 1567 } 1568 return 0; 1569 } 1570 1571 /** 1572 * Isolated mode. 1573 * 1574 * @see rte_flow_isolate() 1575 * @see rte_flow_ops 1576 */ 1577 int 1578 mlx5_flow_isolate(struct rte_eth_dev *dev, 1579 int enable, 1580 struct rte_flow_error *error) 1581 { 1582 struct priv *priv = dev->data->dev_private; 1583 1584 priv_lock(priv); 1585 if (priv->started) { 1586 rte_flow_error_set(error, EBUSY, 1587 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1588 NULL, 1589 "port must be stopped first"); 1590 priv_unlock(priv); 1591 return -rte_errno; 1592 } 1593 priv->isolated = !!enable; 1594 priv_unlock(priv); 1595 return 0; 1596 } 1597