1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <netinet/in.h> 6 #include <sys/queue.h> 7 #include <stdalign.h> 8 #include <stdint.h> 9 #include <string.h> 10 11 #include <rte_common.h> 12 #include <rte_ether.h> 13 #include <ethdev_driver.h> 14 #include <rte_flow.h> 15 #include <rte_flow_driver.h> 16 #include <rte_malloc.h> 17 #include <rte_ip.h> 18 19 #include <mlx5_glue.h> 20 #include <mlx5_prm.h> 21 #include <mlx5_malloc.h> 22 23 #include "mlx5_defs.h" 24 #include "mlx5.h" 25 #include "mlx5_flow.h" 26 #include "mlx5_rx.h" 27 28 #define VERBS_SPEC_INNER(item_flags) \ 29 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0) 30 31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 33 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 34 }; 35 36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 38 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 39 { 9, 10, 11 }, { 12, 13, 14 }, 40 }; 41 42 /* Verbs specification header. */ 43 struct ibv_spec_header { 44 enum ibv_flow_spec_type type; 45 uint16_t size; 46 }; 47 48 /** 49 * Discover the maximum number of priority available. 50 * 51 * @param[in] dev 52 * Pointer to the Ethernet device structure. 53 * 54 * @return 55 * number of supported flow priority on success, a negative errno 56 * value otherwise and rte_errno is set. 57 */ 58 int 59 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 60 { 61 struct mlx5_priv *priv = dev->data->dev_private; 62 struct { 63 struct ibv_flow_attr attr; 64 struct ibv_flow_spec_eth eth; 65 struct ibv_flow_spec_action_drop drop; 66 } flow_attr = { 67 .attr = { 68 .num_of_specs = 2, 69 .port = (uint8_t)priv->dev_port, 70 }, 71 .eth = { 72 .type = IBV_FLOW_SPEC_ETH, 73 .size = sizeof(struct ibv_flow_spec_eth), 74 }, 75 .drop = { 76 .size = sizeof(struct ibv_flow_spec_action_drop), 77 .type = IBV_FLOW_SPEC_ACTION_DROP, 78 }, 79 }; 80 struct ibv_flow *flow; 81 struct mlx5_hrxq *drop = priv->drop_queue.hrxq; 82 uint16_t vprio[] = { 8, 16 }; 83 int i; 84 int priority = 0; 85 86 if (!drop->qp) { 87 rte_errno = ENOTSUP; 88 return -rte_errno; 89 } 90 for (i = 0; i != RTE_DIM(vprio); i++) { 91 flow_attr.attr.priority = vprio[i] - 1; 92 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 93 if (!flow) 94 break; 95 claim_zero(mlx5_glue->destroy_flow(flow)); 96 priority = vprio[i]; 97 } 98 switch (priority) { 99 case 8: 100 priority = RTE_DIM(priority_map_3); 101 break; 102 case 16: 103 priority = RTE_DIM(priority_map_5); 104 break; 105 default: 106 rte_errno = ENOTSUP; 107 DRV_LOG(ERR, 108 "port %u verbs maximum priority: %d expected 8/16", 109 dev->data->port_id, priority); 110 return -rte_errno; 111 } 112 DRV_LOG(INFO, "port %u supported flow priorities:" 113 " 0-%d for ingress or egress root table," 114 " 0-%d for non-root table or transfer root table.", 115 dev->data->port_id, priority - 2, 116 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 117 return priority; 118 } 119 120 /** 121 * Adjust flow priority based on the highest layer and the request priority. 122 * 123 * @param[in] dev 124 * Pointer to the Ethernet device structure. 125 * @param[in] priority 126 * The rule base priority. 127 * @param[in] subpriority 128 * The priority based on the items. 129 * 130 * @return 131 * The new priority. 132 */ 133 uint32_t 134 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 135 uint32_t subpriority) 136 { 137 uint32_t res = 0; 138 struct mlx5_priv *priv = dev->data->dev_private; 139 140 switch (priv->config.flow_prio) { 141 case RTE_DIM(priority_map_3): 142 res = priority_map_3[priority][subpriority]; 143 break; 144 case RTE_DIM(priority_map_5): 145 res = priority_map_5[priority][subpriority]; 146 break; 147 } 148 return res; 149 } 150 151 /** 152 * Get Verbs flow counter by index. 153 * 154 * @param[in] dev 155 * Pointer to the Ethernet device structure. 156 * @param[in] idx 157 * mlx5 flow counter index in the container. 158 * @param[out] ppool 159 * mlx5 flow counter pool in the container, 160 * 161 * @return 162 * A pointer to the counter, NULL otherwise. 163 */ 164 static struct mlx5_flow_counter * 165 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev, 166 uint32_t idx, 167 struct mlx5_flow_counter_pool **ppool) 168 { 169 struct mlx5_priv *priv = dev->data->dev_private; 170 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 171 struct mlx5_flow_counter_pool *pool; 172 173 idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1); 174 pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL]; 175 MLX5_ASSERT(pool); 176 if (ppool) 177 *ppool = pool; 178 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL); 179 } 180 181 /** 182 * Create Verbs flow counter with Verbs library. 183 * 184 * @param[in] dev 185 * Pointer to the Ethernet device structure. 186 * @param[in, out] counter 187 * mlx5 flow counter object, contains the counter id, 188 * handle of created Verbs flow counter is returned 189 * in cs field (if counters are supported). 190 * 191 * @return 192 * 0 On success else a negative errno value is returned 193 * and rte_errno is set. 194 */ 195 static int 196 flow_verbs_counter_create(struct rte_eth_dev *dev, 197 struct mlx5_flow_counter *counter) 198 { 199 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 200 struct mlx5_priv *priv = dev->data->dev_private; 201 struct ibv_context *ctx = priv->sh->ctx; 202 struct ibv_counter_set_init_attr init = { 203 .counter_set_id = counter->shared_info.id}; 204 205 counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init); 206 if (!counter->dcs_when_free) { 207 rte_errno = ENOTSUP; 208 return -ENOTSUP; 209 } 210 return 0; 211 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 212 struct mlx5_priv *priv = dev->data->dev_private; 213 struct ibv_context *ctx = priv->sh->ctx; 214 struct ibv_counters_init_attr init = {0}; 215 struct ibv_counter_attach_attr attach; 216 int ret; 217 218 memset(&attach, 0, sizeof(attach)); 219 counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init); 220 if (!counter->dcs_when_free) { 221 rte_errno = ENOTSUP; 222 return -ENOTSUP; 223 } 224 attach.counter_desc = IBV_COUNTER_PACKETS; 225 attach.index = 0; 226 ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL); 227 if (!ret) { 228 attach.counter_desc = IBV_COUNTER_BYTES; 229 attach.index = 1; 230 ret = mlx5_glue->attach_counters 231 (counter->dcs_when_free, &attach, NULL); 232 } 233 if (ret) { 234 claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free)); 235 counter->dcs_when_free = NULL; 236 rte_errno = ret; 237 return -ret; 238 } 239 return 0; 240 #else 241 (void)dev; 242 (void)counter; 243 rte_errno = ENOTSUP; 244 return -ENOTSUP; 245 #endif 246 } 247 248 /** 249 * Get a flow counter. 250 * 251 * @param[in] dev 252 * Pointer to the Ethernet device structure. 253 * @param[in] id 254 * Counter identifier. 255 * 256 * @return 257 * Index to the counter, 0 otherwise and rte_errno is set. 258 */ 259 static uint32_t 260 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t id __rte_unused) 261 { 262 struct mlx5_priv *priv = dev->data->dev_private; 263 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 264 struct mlx5_flow_counter_pool *pool = NULL; 265 struct mlx5_flow_counter *cnt = NULL; 266 uint32_t n_valid = cmng->n_valid; 267 uint32_t pool_idx, cnt_idx; 268 uint32_t i; 269 int ret; 270 271 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) { 272 pool = cmng->pools[pool_idx]; 273 if (!pool) 274 continue; 275 cnt = TAILQ_FIRST(&pool->counters[0]); 276 if (cnt) 277 break; 278 } 279 if (!cnt) { 280 struct mlx5_flow_counter_pool **pools; 281 uint32_t size; 282 283 if (n_valid == cmng->n) { 284 /* Resize the container pool array. */ 285 size = sizeof(struct mlx5_flow_counter_pool *) * 286 (n_valid + MLX5_CNT_CONTAINER_RESIZE); 287 pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0, 288 SOCKET_ID_ANY); 289 if (!pools) 290 return 0; 291 if (n_valid) { 292 memcpy(pools, cmng->pools, 293 sizeof(struct mlx5_flow_counter_pool *) * 294 n_valid); 295 mlx5_free(cmng->pools); 296 } 297 cmng->pools = pools; 298 cmng->n += MLX5_CNT_CONTAINER_RESIZE; 299 } 300 /* Allocate memory for new pool*/ 301 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL; 302 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY); 303 if (!pool) 304 return 0; 305 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 306 cnt = MLX5_POOL_GET_CNT(pool, i); 307 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 308 } 309 cnt = MLX5_POOL_GET_CNT(pool, 0); 310 cmng->pools[n_valid] = pool; 311 pool_idx = n_valid; 312 cmng->n_valid++; 313 } 314 TAILQ_REMOVE(&pool->counters[0], cnt, next); 315 i = MLX5_CNT_ARRAY_IDX(pool, cnt); 316 cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i); 317 /* Create counter with Verbs. */ 318 ret = flow_verbs_counter_create(dev, cnt); 319 if (!ret) { 320 cnt->dcs_when_active = cnt->dcs_when_free; 321 cnt->hits = 0; 322 cnt->bytes = 0; 323 return cnt_idx; 324 } 325 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 326 /* Some error occurred in Verbs library. */ 327 rte_errno = -ret; 328 return 0; 329 } 330 331 /** 332 * Release a flow counter. 333 * 334 * @param[in] dev 335 * Pointer to the Ethernet device structure. 336 * @param[in] counter 337 * Index to the counter handler. 338 */ 339 static void 340 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter) 341 { 342 struct mlx5_flow_counter_pool *pool; 343 struct mlx5_flow_counter *cnt; 344 345 cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool); 346 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 347 claim_zero(mlx5_glue->destroy_counter_set 348 ((struct ibv_counter_set *)cnt->dcs_when_active)); 349 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 350 claim_zero(mlx5_glue->destroy_counters 351 ((struct ibv_counters *)cnt->dcs_when_active)); 352 #endif 353 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 354 } 355 356 /** 357 * Query a flow counter via Verbs library call. 358 * 359 * @see rte_flow_query() 360 * @see rte_flow_ops 361 */ 362 static int 363 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, 364 struct rte_flow *flow, void *data, 365 struct rte_flow_error *error) 366 { 367 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 368 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 369 if (flow->counter) { 370 struct mlx5_flow_counter_pool *pool; 371 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx 372 (dev, flow->counter, &pool); 373 struct rte_flow_query_count *qc = data; 374 uint64_t counters[2] = {0, 0}; 375 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 376 struct ibv_query_counter_set_attr query_cs_attr = { 377 .dcs_when_free = (struct ibv_counter_set *) 378 cnt->dcs_when_active, 379 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 380 }; 381 struct ibv_counter_set_data query_out = { 382 .out = counters, 383 .outlen = 2 * sizeof(uint64_t), 384 }; 385 int err = mlx5_glue->query_counter_set(&query_cs_attr, 386 &query_out); 387 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 388 int err = mlx5_glue->query_counters 389 ((struct ibv_counters *)cnt->dcs_when_active, counters, 390 RTE_DIM(counters), 391 IBV_READ_COUNTERS_ATTR_PREFER_CACHED); 392 #endif 393 if (err) 394 return rte_flow_error_set 395 (error, err, 396 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 397 NULL, 398 "cannot read counter"); 399 qc->hits_set = 1; 400 qc->bytes_set = 1; 401 qc->hits = counters[0] - cnt->hits; 402 qc->bytes = counters[1] - cnt->bytes; 403 if (qc->reset) { 404 cnt->hits = counters[0]; 405 cnt->bytes = counters[1]; 406 } 407 return 0; 408 } 409 return rte_flow_error_set(error, EINVAL, 410 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 411 NULL, 412 "flow does not have counter"); 413 #else 414 (void)flow; 415 (void)data; 416 return rte_flow_error_set(error, ENOTSUP, 417 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 418 NULL, 419 "counters are not available"); 420 #endif 421 } 422 423 /** 424 * Add a verbs item specification into @p verbs. 425 * 426 * @param[out] verbs 427 * Pointer to verbs structure. 428 * @param[in] src 429 * Create specification. 430 * @param[in] size 431 * Size in bytes of the specification to copy. 432 */ 433 static void 434 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs, 435 void *src, unsigned int size) 436 { 437 void *dst; 438 439 if (!verbs) 440 return; 441 MLX5_ASSERT(verbs->specs); 442 dst = (void *)(verbs->specs + verbs->size); 443 memcpy(dst, src, size); 444 ++verbs->attr.num_of_specs; 445 verbs->size += size; 446 } 447 448 /** 449 * Convert the @p item into a Verbs specification. This function assumes that 450 * the input is valid and that there is space to insert the requested item 451 * into the flow. 452 * 453 * @param[in, out] dev_flow 454 * Pointer to dev_flow structure. 455 * @param[in] item 456 * Item specification. 457 * @param[in] item_flags 458 * Parsed item flags. 459 */ 460 static void 461 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow, 462 const struct rte_flow_item *item, 463 uint64_t item_flags) 464 { 465 const struct rte_flow_item_eth *spec = item->spec; 466 const struct rte_flow_item_eth *mask = item->mask; 467 const unsigned int size = sizeof(struct ibv_flow_spec_eth); 468 struct ibv_flow_spec_eth eth = { 469 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags), 470 .size = size, 471 }; 472 473 if (!mask) 474 mask = &rte_flow_item_eth_mask; 475 if (spec) { 476 unsigned int i; 477 478 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, 479 RTE_ETHER_ADDR_LEN); 480 memcpy(ð.val.src_mac, spec->src.addr_bytes, 481 RTE_ETHER_ADDR_LEN); 482 eth.val.ether_type = spec->type; 483 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, 484 RTE_ETHER_ADDR_LEN); 485 memcpy(ð.mask.src_mac, mask->src.addr_bytes, 486 RTE_ETHER_ADDR_LEN); 487 eth.mask.ether_type = mask->type; 488 /* Remove unwanted bits from values. */ 489 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) { 490 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 491 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 492 } 493 eth.val.ether_type &= eth.mask.ether_type; 494 } 495 flow_verbs_spec_add(&dev_flow->verbs, ð, size); 496 } 497 498 /** 499 * Update the VLAN tag in the Verbs Ethernet specification. 500 * This function assumes that the input is valid and there is space to add 501 * the requested item. 502 * 503 * @param[in, out] attr 504 * Pointer to Verbs attributes structure. 505 * @param[in] eth 506 * Verbs structure containing the VLAN information to copy. 507 */ 508 static void 509 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr, 510 struct ibv_flow_spec_eth *eth) 511 { 512 unsigned int i; 513 const enum ibv_flow_spec_type search = eth->type; 514 struct ibv_spec_header *hdr = (struct ibv_spec_header *) 515 ((uint8_t *)attr + sizeof(struct ibv_flow_attr)); 516 517 for (i = 0; i != attr->num_of_specs; ++i) { 518 if (hdr->type == search) { 519 struct ibv_flow_spec_eth *e = 520 (struct ibv_flow_spec_eth *)hdr; 521 522 e->val.vlan_tag = eth->val.vlan_tag; 523 e->mask.vlan_tag = eth->mask.vlan_tag; 524 e->val.ether_type = eth->val.ether_type; 525 e->mask.ether_type = eth->mask.ether_type; 526 break; 527 } 528 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size); 529 } 530 } 531 532 /** 533 * Convert the @p item into a Verbs specification. This function assumes that 534 * the input is valid and that there is space to insert the requested item 535 * into the flow. 536 * 537 * @param[in, out] dev_flow 538 * Pointer to dev_flow structure. 539 * @param[in] item 540 * Item specification. 541 * @param[in] item_flags 542 * Parsed item flags. 543 */ 544 static void 545 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow, 546 const struct rte_flow_item *item, 547 uint64_t item_flags) 548 { 549 const struct rte_flow_item_vlan *spec = item->spec; 550 const struct rte_flow_item_vlan *mask = item->mask; 551 unsigned int size = sizeof(struct ibv_flow_spec_eth); 552 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 553 struct ibv_flow_spec_eth eth = { 554 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags), 555 .size = size, 556 }; 557 const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 558 MLX5_FLOW_LAYER_OUTER_L2; 559 560 if (!mask) 561 mask = &rte_flow_item_vlan_mask; 562 if (spec) { 563 eth.val.vlan_tag = spec->tci; 564 eth.mask.vlan_tag = mask->tci; 565 eth.val.vlan_tag &= eth.mask.vlan_tag; 566 eth.val.ether_type = spec->inner_type; 567 eth.mask.ether_type = mask->inner_type; 568 eth.val.ether_type &= eth.mask.ether_type; 569 } 570 if (!(item_flags & l2m)) 571 flow_verbs_spec_add(&dev_flow->verbs, ð, size); 572 else 573 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð); 574 if (!tunnel) 575 dev_flow->handle->vf_vlan.tag = 576 rte_be_to_cpu_16(spec->tci) & 0x0fff; 577 } 578 579 /** 580 * Convert the @p item into a Verbs specification. This function assumes that 581 * the input is valid and that there is space to insert the requested item 582 * into the flow. 583 * 584 * @param[in, out] dev_flow 585 * Pointer to dev_flow structure. 586 * @param[in] item 587 * Item specification. 588 * @param[in] item_flags 589 * Parsed item flags. 590 */ 591 static void 592 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow, 593 const struct rte_flow_item *item, 594 uint64_t item_flags) 595 { 596 const struct rte_flow_item_ipv4 *spec = item->spec; 597 const struct rte_flow_item_ipv4 *mask = item->mask; 598 unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext); 599 struct ibv_flow_spec_ipv4_ext ipv4 = { 600 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags), 601 .size = size, 602 }; 603 604 if (!mask) 605 mask = &rte_flow_item_ipv4_mask; 606 if (spec) { 607 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 608 .src_ip = spec->hdr.src_addr, 609 .dst_ip = spec->hdr.dst_addr, 610 .proto = spec->hdr.next_proto_id, 611 .tos = spec->hdr.type_of_service, 612 }; 613 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 614 .src_ip = mask->hdr.src_addr, 615 .dst_ip = mask->hdr.dst_addr, 616 .proto = mask->hdr.next_proto_id, 617 .tos = mask->hdr.type_of_service, 618 }; 619 /* Remove unwanted bits from values. */ 620 ipv4.val.src_ip &= ipv4.mask.src_ip; 621 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 622 ipv4.val.proto &= ipv4.mask.proto; 623 ipv4.val.tos &= ipv4.mask.tos; 624 } 625 flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size); 626 } 627 628 /** 629 * Convert the @p item into a Verbs specification. This function assumes that 630 * the input is valid and that there is space to insert the requested item 631 * into the flow. 632 * 633 * @param[in, out] dev_flow 634 * Pointer to dev_flow structure. 635 * @param[in] item 636 * Item specification. 637 * @param[in] item_flags 638 * Parsed item flags. 639 */ 640 static void 641 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow, 642 const struct rte_flow_item *item, 643 uint64_t item_flags) 644 { 645 const struct rte_flow_item_ipv6 *spec = item->spec; 646 const struct rte_flow_item_ipv6 *mask = item->mask; 647 unsigned int size = sizeof(struct ibv_flow_spec_ipv6); 648 struct ibv_flow_spec_ipv6 ipv6 = { 649 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags), 650 .size = size, 651 }; 652 653 if (!mask) 654 mask = &rte_flow_item_ipv6_mask; 655 if (spec) { 656 unsigned int i; 657 uint32_t vtc_flow_val; 658 uint32_t vtc_flow_mask; 659 660 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 661 RTE_DIM(ipv6.val.src_ip)); 662 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 663 RTE_DIM(ipv6.val.dst_ip)); 664 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 665 RTE_DIM(ipv6.mask.src_ip)); 666 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 667 RTE_DIM(ipv6.mask.dst_ip)); 668 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); 669 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); 670 ipv6.val.flow_label = 671 rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >> 672 RTE_IPV6_HDR_FL_SHIFT); 673 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >> 674 RTE_IPV6_HDR_TC_SHIFT; 675 ipv6.val.next_hdr = spec->hdr.proto; 676 ipv6.mask.flow_label = 677 rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >> 678 RTE_IPV6_HDR_FL_SHIFT); 679 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >> 680 RTE_IPV6_HDR_TC_SHIFT; 681 ipv6.mask.next_hdr = mask->hdr.proto; 682 /* Remove unwanted bits from values. */ 683 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 684 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 685 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 686 } 687 ipv6.val.flow_label &= ipv6.mask.flow_label; 688 ipv6.val.traffic_class &= ipv6.mask.traffic_class; 689 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 690 } 691 flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size); 692 } 693 694 /** 695 * Convert the @p item into a Verbs specification. This function assumes that 696 * the input is valid and that there is space to insert the requested item 697 * into the flow. 698 * 699 * @param[in, out] dev_flow 700 * Pointer to dev_flow structure. 701 * @param[in] item 702 * Item specification. 703 * @param[in] item_flags 704 * Parsed item flags. 705 */ 706 static void 707 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow, 708 const struct rte_flow_item *item, 709 uint64_t item_flags __rte_unused) 710 { 711 const struct rte_flow_item_tcp *spec = item->spec; 712 const struct rte_flow_item_tcp *mask = item->mask; 713 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp); 714 struct ibv_flow_spec_tcp_udp tcp = { 715 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags), 716 .size = size, 717 }; 718 719 if (!mask) 720 mask = &rte_flow_item_tcp_mask; 721 if (spec) { 722 tcp.val.dst_port = spec->hdr.dst_port; 723 tcp.val.src_port = spec->hdr.src_port; 724 tcp.mask.dst_port = mask->hdr.dst_port; 725 tcp.mask.src_port = mask->hdr.src_port; 726 /* Remove unwanted bits from values. */ 727 tcp.val.src_port &= tcp.mask.src_port; 728 tcp.val.dst_port &= tcp.mask.dst_port; 729 } 730 flow_verbs_spec_add(&dev_flow->verbs, &tcp, size); 731 } 732 733 /** 734 * Convert the @p item into a Verbs specification. This function assumes that 735 * the input is valid and that there is space to insert the requested item 736 * into the flow. 737 * 738 * @param[in, out] dev_flow 739 * Pointer to dev_flow structure. 740 * @param[in] item 741 * Item specification. 742 * @param[in] item_flags 743 * Parsed item flags. 744 */ 745 static void 746 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow, 747 const struct rte_flow_item *item, 748 uint64_t item_flags __rte_unused) 749 { 750 const struct rte_flow_item_udp *spec = item->spec; 751 const struct rte_flow_item_udp *mask = item->mask; 752 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp); 753 struct ibv_flow_spec_tcp_udp udp = { 754 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags), 755 .size = size, 756 }; 757 758 if (!mask) 759 mask = &rte_flow_item_udp_mask; 760 if (spec) { 761 udp.val.dst_port = spec->hdr.dst_port; 762 udp.val.src_port = spec->hdr.src_port; 763 udp.mask.dst_port = mask->hdr.dst_port; 764 udp.mask.src_port = mask->hdr.src_port; 765 /* Remove unwanted bits from values. */ 766 udp.val.src_port &= udp.mask.src_port; 767 udp.val.dst_port &= udp.mask.dst_port; 768 } 769 item++; 770 while (item->type == RTE_FLOW_ITEM_TYPE_VOID) 771 item++; 772 if (!(udp.val.dst_port & udp.mask.dst_port)) { 773 switch ((item)->type) { 774 case RTE_FLOW_ITEM_TYPE_VXLAN: 775 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN); 776 udp.mask.dst_port = 0xffff; 777 break; 778 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 779 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE); 780 udp.mask.dst_port = 0xffff; 781 break; 782 case RTE_FLOW_ITEM_TYPE_MPLS: 783 udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS); 784 udp.mask.dst_port = 0xffff; 785 break; 786 default: 787 break; 788 } 789 } 790 791 flow_verbs_spec_add(&dev_flow->verbs, &udp, size); 792 } 793 794 /** 795 * Convert the @p item into a Verbs specification. This function assumes that 796 * the input is valid and that there is space to insert the requested item 797 * into the flow. 798 * 799 * @param[in, out] dev_flow 800 * Pointer to dev_flow structure. 801 * @param[in] item 802 * Item specification. 803 * @param[in] item_flags 804 * Parsed item flags. 805 */ 806 static void 807 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow, 808 const struct rte_flow_item *item, 809 uint64_t item_flags __rte_unused) 810 { 811 const struct rte_flow_item_vxlan *spec = item->spec; 812 const struct rte_flow_item_vxlan *mask = item->mask; 813 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 814 struct ibv_flow_spec_tunnel vxlan = { 815 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 816 .size = size, 817 }; 818 union vni { 819 uint32_t vlan_id; 820 uint8_t vni[4]; 821 } id = { .vlan_id = 0, }; 822 823 if (!mask) 824 mask = &rte_flow_item_vxlan_mask; 825 if (spec) { 826 memcpy(&id.vni[1], spec->vni, 3); 827 vxlan.val.tunnel_id = id.vlan_id; 828 memcpy(&id.vni[1], mask->vni, 3); 829 vxlan.mask.tunnel_id = id.vlan_id; 830 /* Remove unwanted bits from values. */ 831 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 832 } 833 flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size); 834 } 835 836 /** 837 * Convert the @p item into a Verbs specification. This function assumes that 838 * the input is valid and that there is space to insert the requested item 839 * into the flow. 840 * 841 * @param[in, out] dev_flow 842 * Pointer to dev_flow structure. 843 * @param[in] item 844 * Item specification. 845 * @param[in] item_flags 846 * Parsed item flags. 847 */ 848 static void 849 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow, 850 const struct rte_flow_item *item, 851 uint64_t item_flags __rte_unused) 852 { 853 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 854 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 855 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 856 struct ibv_flow_spec_tunnel vxlan_gpe = { 857 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 858 .size = size, 859 }; 860 union vni { 861 uint32_t vlan_id; 862 uint8_t vni[4]; 863 } id = { .vlan_id = 0, }; 864 865 if (!mask) 866 mask = &rte_flow_item_vxlan_gpe_mask; 867 if (spec) { 868 memcpy(&id.vni[1], spec->vni, 3); 869 vxlan_gpe.val.tunnel_id = id.vlan_id; 870 memcpy(&id.vni[1], mask->vni, 3); 871 vxlan_gpe.mask.tunnel_id = id.vlan_id; 872 /* Remove unwanted bits from values. */ 873 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id; 874 } 875 flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size); 876 } 877 878 /** 879 * Update the protocol in Verbs IPv4/IPv6 spec. 880 * 881 * @param[in, out] attr 882 * Pointer to Verbs attributes structure. 883 * @param[in] search 884 * Specification type to search in order to update the IP protocol. 885 * @param[in] protocol 886 * Protocol value to set if none is present in the specification. 887 */ 888 static void 889 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr, 890 enum ibv_flow_spec_type search, 891 uint8_t protocol) 892 { 893 unsigned int i; 894 struct ibv_spec_header *hdr = (struct ibv_spec_header *) 895 ((uint8_t *)attr + sizeof(struct ibv_flow_attr)); 896 897 if (!attr) 898 return; 899 for (i = 0; i != attr->num_of_specs; ++i) { 900 if (hdr->type == search) { 901 union { 902 struct ibv_flow_spec_ipv4_ext *ipv4; 903 struct ibv_flow_spec_ipv6 *ipv6; 904 } ip; 905 906 switch (search) { 907 case IBV_FLOW_SPEC_IPV4_EXT: 908 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr; 909 if (!ip.ipv4->val.proto) { 910 ip.ipv4->val.proto = protocol; 911 ip.ipv4->mask.proto = 0xff; 912 } 913 break; 914 case IBV_FLOW_SPEC_IPV6: 915 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr; 916 if (!ip.ipv6->val.next_hdr) { 917 ip.ipv6->val.next_hdr = protocol; 918 ip.ipv6->mask.next_hdr = 0xff; 919 } 920 break; 921 default: 922 break; 923 } 924 break; 925 } 926 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size); 927 } 928 } 929 930 /** 931 * Convert the @p item into a Verbs specification. This function assumes that 932 * the input is valid and that there is space to insert the requested item 933 * into the flow. 934 * 935 * @param[in, out] dev_flow 936 * Pointer to dev_flow structure. 937 * @param[in] item 938 * Item specification. 939 * @param[in] item_flags 940 * Parsed item flags. 941 */ 942 static void 943 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow, 944 const struct rte_flow_item *item __rte_unused, 945 uint64_t item_flags) 946 { 947 struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs; 948 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 949 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 950 struct ibv_flow_spec_tunnel tunnel = { 951 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 952 .size = size, 953 }; 954 #else 955 const struct rte_flow_item_gre *spec = item->spec; 956 const struct rte_flow_item_gre *mask = item->mask; 957 unsigned int size = sizeof(struct ibv_flow_spec_gre); 958 struct ibv_flow_spec_gre tunnel = { 959 .type = IBV_FLOW_SPEC_GRE, 960 .size = size, 961 }; 962 963 if (!mask) 964 mask = &rte_flow_item_gre_mask; 965 if (spec) { 966 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver; 967 tunnel.val.protocol = spec->protocol; 968 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver; 969 tunnel.mask.protocol = mask->protocol; 970 /* Remove unwanted bits from values. */ 971 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver; 972 tunnel.val.protocol &= tunnel.mask.protocol; 973 tunnel.val.key &= tunnel.mask.key; 974 } 975 #endif 976 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) 977 flow_verbs_item_gre_ip_protocol_update(&verbs->attr, 978 IBV_FLOW_SPEC_IPV4_EXT, 979 IPPROTO_GRE); 980 else 981 flow_verbs_item_gre_ip_protocol_update(&verbs->attr, 982 IBV_FLOW_SPEC_IPV6, 983 IPPROTO_GRE); 984 flow_verbs_spec_add(verbs, &tunnel, size); 985 } 986 987 /** 988 * Convert the @p action into a Verbs specification. This function assumes that 989 * the input is valid and that there is space to insert the requested action 990 * into the flow. This function also return the action that was added. 991 * 992 * @param[in, out] dev_flow 993 * Pointer to dev_flow structure. 994 * @param[in] item 995 * Item specification. 996 * @param[in] item_flags 997 * Parsed item flags. 998 */ 999 static void 1000 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused, 1001 const struct rte_flow_item *item __rte_unused, 1002 uint64_t item_flags __rte_unused) 1003 { 1004 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1005 const struct rte_flow_item_mpls *spec = item->spec; 1006 const struct rte_flow_item_mpls *mask = item->mask; 1007 unsigned int size = sizeof(struct ibv_flow_spec_mpls); 1008 struct ibv_flow_spec_mpls mpls = { 1009 .type = IBV_FLOW_SPEC_MPLS, 1010 .size = size, 1011 }; 1012 1013 if (!mask) 1014 mask = &rte_flow_item_mpls_mask; 1015 if (spec) { 1016 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label)); 1017 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label)); 1018 /* Remove unwanted bits from values. */ 1019 mpls.val.label &= mpls.mask.label; 1020 } 1021 flow_verbs_spec_add(&dev_flow->verbs, &mpls, size); 1022 #endif 1023 } 1024 1025 /** 1026 * Convert the @p action into a Verbs specification. This function assumes that 1027 * the input is valid and that there is space to insert the requested action 1028 * into the flow. 1029 * 1030 * @param[in] dev_flow 1031 * Pointer to mlx5_flow. 1032 * @param[in] action 1033 * Action configuration. 1034 */ 1035 static void 1036 flow_verbs_translate_action_drop 1037 (struct mlx5_flow *dev_flow, 1038 const struct rte_flow_action *action __rte_unused) 1039 { 1040 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1041 struct ibv_flow_spec_action_drop drop = { 1042 .type = IBV_FLOW_SPEC_ACTION_DROP, 1043 .size = size, 1044 }; 1045 1046 flow_verbs_spec_add(&dev_flow->verbs, &drop, size); 1047 } 1048 1049 /** 1050 * Convert the @p action into a Verbs specification. This function assumes that 1051 * the input is valid and that there is space to insert the requested action 1052 * into the flow. 1053 * 1054 * @param[in] rss_desc 1055 * Pointer to mlx5_flow_rss_desc. 1056 * @param[in] action 1057 * Action configuration. 1058 */ 1059 static void 1060 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc, 1061 const struct rte_flow_action *action) 1062 { 1063 const struct rte_flow_action_queue *queue = action->conf; 1064 1065 rss_desc->queue[0] = queue->index; 1066 rss_desc->queue_num = 1; 1067 } 1068 1069 /** 1070 * Convert the @p action into a Verbs specification. This function assumes that 1071 * the input is valid and that there is space to insert the requested action 1072 * into the flow. 1073 * 1074 * @param[in] rss_desc 1075 * Pointer to mlx5_flow_rss_desc. 1076 * @param[in] action 1077 * Action configuration. 1078 */ 1079 static void 1080 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc, 1081 const struct rte_flow_action *action) 1082 { 1083 const struct rte_flow_action_rss *rss = action->conf; 1084 const uint8_t *rss_key; 1085 1086 memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t)); 1087 rss_desc->queue_num = rss->queue_num; 1088 /* NULL RSS key indicates default RSS key. */ 1089 rss_key = !rss->key ? rss_hash_default_key : rss->key; 1090 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 1091 /* 1092 * rss->level and rss.types should be set in advance when expanding 1093 * items for RSS. 1094 */ 1095 } 1096 1097 /** 1098 * Convert the @p action into a Verbs specification. This function assumes that 1099 * the input is valid and that there is space to insert the requested action 1100 * into the flow. 1101 * 1102 * @param[in] dev_flow 1103 * Pointer to mlx5_flow. 1104 * @param[in] action 1105 * Action configuration. 1106 */ 1107 static void 1108 flow_verbs_translate_action_flag 1109 (struct mlx5_flow *dev_flow, 1110 const struct rte_flow_action *action __rte_unused) 1111 { 1112 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1113 struct ibv_flow_spec_action_tag tag = { 1114 .type = IBV_FLOW_SPEC_ACTION_TAG, 1115 .size = size, 1116 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT), 1117 }; 1118 1119 flow_verbs_spec_add(&dev_flow->verbs, &tag, size); 1120 } 1121 1122 /** 1123 * Convert the @p action into a Verbs specification. This function assumes that 1124 * the input is valid and that there is space to insert the requested action 1125 * into the flow. 1126 * 1127 * @param[in] dev_flow 1128 * Pointer to mlx5_flow. 1129 * @param[in] action 1130 * Action configuration. 1131 */ 1132 static void 1133 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow, 1134 const struct rte_flow_action *action) 1135 { 1136 const struct rte_flow_action_mark *mark = action->conf; 1137 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1138 struct ibv_flow_spec_action_tag tag = { 1139 .type = IBV_FLOW_SPEC_ACTION_TAG, 1140 .size = size, 1141 .tag_id = mlx5_flow_mark_set(mark->id), 1142 }; 1143 1144 flow_verbs_spec_add(&dev_flow->verbs, &tag, size); 1145 } 1146 1147 /** 1148 * Convert the @p action into a Verbs specification. This function assumes that 1149 * the input is valid and that there is space to insert the requested action 1150 * into the flow. 1151 * 1152 * @param[in] dev 1153 * Pointer to the Ethernet device structure. 1154 * @param[in] action 1155 * Action configuration. 1156 * @param[in] dev_flow 1157 * Pointer to mlx5_flow. 1158 * @param[out] error 1159 * Pointer to error structure. 1160 * 1161 * @return 1162 * 0 On success else a negative errno value is returned and rte_errno is set. 1163 */ 1164 static int 1165 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, 1166 const struct rte_flow_action *action, 1167 struct rte_eth_dev *dev, 1168 struct rte_flow_error *error) 1169 { 1170 const struct rte_flow_action_count *count = action->conf; 1171 struct rte_flow *flow = dev_flow->flow; 1172 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 1173 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1174 struct mlx5_flow_counter_pool *pool; 1175 struct mlx5_flow_counter *cnt = NULL; 1176 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1177 struct ibv_flow_spec_counter_action counter = { 1178 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1179 .size = size, 1180 }; 1181 #endif 1182 1183 if (!flow->counter) { 1184 flow->counter = flow_verbs_counter_new(dev, count->id); 1185 if (!flow->counter) 1186 return rte_flow_error_set(error, rte_errno, 1187 RTE_FLOW_ERROR_TYPE_ACTION, 1188 action, 1189 "cannot get counter" 1190 " context."); 1191 } 1192 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 1193 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); 1194 counter.counter_set_handle = 1195 ((struct ibv_counter_set *)cnt->dcs_when_active)->handle; 1196 flow_verbs_spec_add(&dev_flow->verbs, &counter, size); 1197 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1198 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); 1199 counter.counters = (struct ibv_counters *)cnt->dcs_when_active; 1200 flow_verbs_spec_add(&dev_flow->verbs, &counter, size); 1201 #endif 1202 return 0; 1203 } 1204 1205 /** 1206 * Internal validation function. For validating both actions and items. 1207 * 1208 * @param[in] dev 1209 * Pointer to the Ethernet device structure. 1210 * @param[in] attr 1211 * Pointer to the flow attributes. 1212 * @param[in] items 1213 * Pointer to the list of items. 1214 * @param[in] actions 1215 * Pointer to the list of actions. 1216 * @param[in] external 1217 * This flow rule is created by request external to PMD. 1218 * @param[in] hairpin 1219 * Number of hairpin TX actions, 0 means classic flow. 1220 * @param[out] error 1221 * Pointer to the error structure. 1222 * 1223 * @return 1224 * 0 on success, a negative errno value otherwise and rte_errno is set. 1225 */ 1226 static int 1227 flow_verbs_validate(struct rte_eth_dev *dev, 1228 const struct rte_flow_attr *attr, 1229 const struct rte_flow_item items[], 1230 const struct rte_flow_action actions[], 1231 bool external __rte_unused, 1232 int hairpin __rte_unused, 1233 struct rte_flow_error *error) 1234 { 1235 int ret; 1236 uint64_t action_flags = 0; 1237 uint64_t item_flags = 0; 1238 uint64_t last_item = 0; 1239 uint8_t next_protocol = 0xff; 1240 uint16_t ether_type = 0; 1241 bool is_empty_vlan = false; 1242 uint16_t udp_dport = 0; 1243 1244 if (items == NULL) 1245 return -1; 1246 ret = mlx5_flow_validate_attributes(dev, attr, error); 1247 if (ret < 0) 1248 return ret; 1249 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1250 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1251 int ret = 0; 1252 1253 switch (items->type) { 1254 case RTE_FLOW_ITEM_TYPE_VOID: 1255 break; 1256 case RTE_FLOW_ITEM_TYPE_ETH: 1257 ret = mlx5_flow_validate_item_eth(items, item_flags, 1258 false, error); 1259 if (ret < 0) 1260 return ret; 1261 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1262 MLX5_FLOW_LAYER_OUTER_L2; 1263 if (items->mask != NULL && items->spec != NULL) { 1264 ether_type = 1265 ((const struct rte_flow_item_eth *) 1266 items->spec)->type; 1267 ether_type &= 1268 ((const struct rte_flow_item_eth *) 1269 items->mask)->type; 1270 if (ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN)) 1271 is_empty_vlan = true; 1272 ether_type = rte_be_to_cpu_16(ether_type); 1273 } else { 1274 ether_type = 0; 1275 } 1276 break; 1277 case RTE_FLOW_ITEM_TYPE_VLAN: 1278 ret = mlx5_flow_validate_item_vlan(items, item_flags, 1279 dev, error); 1280 if (ret < 0) 1281 return ret; 1282 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 1283 MLX5_FLOW_LAYER_INNER_VLAN) : 1284 (MLX5_FLOW_LAYER_OUTER_L2 | 1285 MLX5_FLOW_LAYER_OUTER_VLAN); 1286 if (items->mask != NULL && items->spec != NULL) { 1287 ether_type = 1288 ((const struct rte_flow_item_vlan *) 1289 items->spec)->inner_type; 1290 ether_type &= 1291 ((const struct rte_flow_item_vlan *) 1292 items->mask)->inner_type; 1293 ether_type = rte_be_to_cpu_16(ether_type); 1294 } else { 1295 ether_type = 0; 1296 } 1297 is_empty_vlan = false; 1298 break; 1299 case RTE_FLOW_ITEM_TYPE_IPV4: 1300 ret = mlx5_flow_validate_item_ipv4 1301 (items, item_flags, 1302 last_item, ether_type, NULL, 1303 MLX5_ITEM_RANGE_NOT_ACCEPTED, 1304 error); 1305 if (ret < 0) 1306 return ret; 1307 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1308 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1309 if (items->mask != NULL && 1310 ((const struct rte_flow_item_ipv4 *) 1311 items->mask)->hdr.next_proto_id) { 1312 next_protocol = 1313 ((const struct rte_flow_item_ipv4 *) 1314 (items->spec))->hdr.next_proto_id; 1315 next_protocol &= 1316 ((const struct rte_flow_item_ipv4 *) 1317 (items->mask))->hdr.next_proto_id; 1318 } else { 1319 /* Reset for inner layer. */ 1320 next_protocol = 0xff; 1321 } 1322 break; 1323 case RTE_FLOW_ITEM_TYPE_IPV6: 1324 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 1325 last_item, 1326 ether_type, NULL, 1327 error); 1328 if (ret < 0) 1329 return ret; 1330 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1331 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1332 if (items->mask != NULL && 1333 ((const struct rte_flow_item_ipv6 *) 1334 items->mask)->hdr.proto) { 1335 next_protocol = 1336 ((const struct rte_flow_item_ipv6 *) 1337 items->spec)->hdr.proto; 1338 next_protocol &= 1339 ((const struct rte_flow_item_ipv6 *) 1340 items->mask)->hdr.proto; 1341 } else { 1342 /* Reset for inner layer. */ 1343 next_protocol = 0xff; 1344 } 1345 break; 1346 case RTE_FLOW_ITEM_TYPE_UDP: 1347 ret = mlx5_flow_validate_item_udp(items, item_flags, 1348 next_protocol, 1349 error); 1350 const struct rte_flow_item_udp *spec = items->spec; 1351 const struct rte_flow_item_udp *mask = items->mask; 1352 if (!mask) 1353 mask = &rte_flow_item_udp_mask; 1354 if (spec != NULL) 1355 udp_dport = rte_be_to_cpu_16 1356 (spec->hdr.dst_port & 1357 mask->hdr.dst_port); 1358 1359 if (ret < 0) 1360 return ret; 1361 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 1362 MLX5_FLOW_LAYER_OUTER_L4_UDP; 1363 break; 1364 case RTE_FLOW_ITEM_TYPE_TCP: 1365 ret = mlx5_flow_validate_item_tcp 1366 (items, item_flags, 1367 next_protocol, 1368 &rte_flow_item_tcp_mask, 1369 error); 1370 if (ret < 0) 1371 return ret; 1372 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 1373 MLX5_FLOW_LAYER_OUTER_L4_TCP; 1374 break; 1375 case RTE_FLOW_ITEM_TYPE_VXLAN: 1376 ret = mlx5_flow_validate_item_vxlan(dev, udp_dport, 1377 items, item_flags, 1378 attr, error); 1379 if (ret < 0) 1380 return ret; 1381 last_item = MLX5_FLOW_LAYER_VXLAN; 1382 break; 1383 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1384 ret = mlx5_flow_validate_item_vxlan_gpe(items, 1385 item_flags, 1386 dev, error); 1387 if (ret < 0) 1388 return ret; 1389 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 1390 break; 1391 case RTE_FLOW_ITEM_TYPE_GRE: 1392 ret = mlx5_flow_validate_item_gre(items, item_flags, 1393 next_protocol, error); 1394 if (ret < 0) 1395 return ret; 1396 last_item = MLX5_FLOW_LAYER_GRE; 1397 break; 1398 case RTE_FLOW_ITEM_TYPE_MPLS: 1399 ret = mlx5_flow_validate_item_mpls(dev, items, 1400 item_flags, 1401 last_item, error); 1402 if (ret < 0) 1403 return ret; 1404 last_item = MLX5_FLOW_LAYER_MPLS; 1405 break; 1406 case RTE_FLOW_ITEM_TYPE_ICMP: 1407 case RTE_FLOW_ITEM_TYPE_ICMP6: 1408 return rte_flow_error_set(error, ENOTSUP, 1409 RTE_FLOW_ERROR_TYPE_ITEM, 1410 NULL, "ICMP/ICMP6 " 1411 "item not supported"); 1412 default: 1413 return rte_flow_error_set(error, ENOTSUP, 1414 RTE_FLOW_ERROR_TYPE_ITEM, 1415 NULL, "item not supported"); 1416 } 1417 item_flags |= last_item; 1418 } 1419 if (is_empty_vlan) 1420 return rte_flow_error_set(error, ENOTSUP, 1421 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1422 "VLAN matching without vid specification is not supported"); 1423 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1424 switch (actions->type) { 1425 case RTE_FLOW_ACTION_TYPE_VOID: 1426 break; 1427 case RTE_FLOW_ACTION_TYPE_FLAG: 1428 ret = mlx5_flow_validate_action_flag(action_flags, 1429 attr, 1430 error); 1431 if (ret < 0) 1432 return ret; 1433 action_flags |= MLX5_FLOW_ACTION_FLAG; 1434 break; 1435 case RTE_FLOW_ACTION_TYPE_MARK: 1436 ret = mlx5_flow_validate_action_mark(actions, 1437 action_flags, 1438 attr, 1439 error); 1440 if (ret < 0) 1441 return ret; 1442 action_flags |= MLX5_FLOW_ACTION_MARK; 1443 break; 1444 case RTE_FLOW_ACTION_TYPE_DROP: 1445 ret = mlx5_flow_validate_action_drop(action_flags, 1446 attr, 1447 error); 1448 if (ret < 0) 1449 return ret; 1450 action_flags |= MLX5_FLOW_ACTION_DROP; 1451 break; 1452 case RTE_FLOW_ACTION_TYPE_QUEUE: 1453 ret = mlx5_flow_validate_action_queue(actions, 1454 action_flags, dev, 1455 attr, 1456 error); 1457 if (ret < 0) 1458 return ret; 1459 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1460 break; 1461 case RTE_FLOW_ACTION_TYPE_RSS: 1462 ret = mlx5_flow_validate_action_rss(actions, 1463 action_flags, dev, 1464 attr, item_flags, 1465 error); 1466 if (ret < 0) 1467 return ret; 1468 action_flags |= MLX5_FLOW_ACTION_RSS; 1469 break; 1470 case RTE_FLOW_ACTION_TYPE_COUNT: 1471 ret = mlx5_flow_validate_action_count(dev, attr, error); 1472 if (ret < 0) 1473 return ret; 1474 action_flags |= MLX5_FLOW_ACTION_COUNT; 1475 break; 1476 default: 1477 return rte_flow_error_set(error, ENOTSUP, 1478 RTE_FLOW_ERROR_TYPE_ACTION, 1479 actions, 1480 "action not supported"); 1481 } 1482 } 1483 /* 1484 * Validate the drop action mutual exclusion with other actions. 1485 * Drop action is mutually-exclusive with any other action, except for 1486 * Count action. 1487 */ 1488 if ((action_flags & MLX5_FLOW_ACTION_DROP) && 1489 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1492 "Drop action is mutually-exclusive " 1493 "with any other action, except for " 1494 "Count action"); 1495 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS)) 1496 return rte_flow_error_set(error, EINVAL, 1497 RTE_FLOW_ERROR_TYPE_ACTION, actions, 1498 "no fate action is found"); 1499 return 0; 1500 } 1501 1502 /** 1503 * Calculate the required bytes that are needed for the action part of the verbs 1504 * flow. 1505 * 1506 * @param[in] actions 1507 * Pointer to the list of actions. 1508 * 1509 * @return 1510 * The size of the memory needed for all actions. 1511 */ 1512 static int 1513 flow_verbs_get_actions_size(const struct rte_flow_action actions[]) 1514 { 1515 int size = 0; 1516 1517 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1518 switch (actions->type) { 1519 case RTE_FLOW_ACTION_TYPE_VOID: 1520 break; 1521 case RTE_FLOW_ACTION_TYPE_FLAG: 1522 size += sizeof(struct ibv_flow_spec_action_tag); 1523 break; 1524 case RTE_FLOW_ACTION_TYPE_MARK: 1525 size += sizeof(struct ibv_flow_spec_action_tag); 1526 break; 1527 case RTE_FLOW_ACTION_TYPE_DROP: 1528 size += sizeof(struct ibv_flow_spec_action_drop); 1529 break; 1530 case RTE_FLOW_ACTION_TYPE_QUEUE: 1531 break; 1532 case RTE_FLOW_ACTION_TYPE_RSS: 1533 break; 1534 case RTE_FLOW_ACTION_TYPE_COUNT: 1535 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 1536 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1537 size += sizeof(struct ibv_flow_spec_counter_action); 1538 #endif 1539 break; 1540 default: 1541 break; 1542 } 1543 } 1544 return size; 1545 } 1546 1547 /** 1548 * Calculate the required bytes that are needed for the item part of the verbs 1549 * flow. 1550 * 1551 * @param[in] items 1552 * Pointer to the list of items. 1553 * 1554 * @return 1555 * The size of the memory needed for all items. 1556 */ 1557 static int 1558 flow_verbs_get_items_size(const struct rte_flow_item items[]) 1559 { 1560 int size = 0; 1561 1562 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1563 switch (items->type) { 1564 case RTE_FLOW_ITEM_TYPE_VOID: 1565 break; 1566 case RTE_FLOW_ITEM_TYPE_ETH: 1567 size += sizeof(struct ibv_flow_spec_eth); 1568 break; 1569 case RTE_FLOW_ITEM_TYPE_VLAN: 1570 size += sizeof(struct ibv_flow_spec_eth); 1571 break; 1572 case RTE_FLOW_ITEM_TYPE_IPV4: 1573 size += sizeof(struct ibv_flow_spec_ipv4_ext); 1574 break; 1575 case RTE_FLOW_ITEM_TYPE_IPV6: 1576 size += sizeof(struct ibv_flow_spec_ipv6); 1577 break; 1578 case RTE_FLOW_ITEM_TYPE_UDP: 1579 size += sizeof(struct ibv_flow_spec_tcp_udp); 1580 break; 1581 case RTE_FLOW_ITEM_TYPE_TCP: 1582 size += sizeof(struct ibv_flow_spec_tcp_udp); 1583 break; 1584 case RTE_FLOW_ITEM_TYPE_VXLAN: 1585 size += sizeof(struct ibv_flow_spec_tunnel); 1586 break; 1587 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1588 size += sizeof(struct ibv_flow_spec_tunnel); 1589 break; 1590 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1591 case RTE_FLOW_ITEM_TYPE_GRE: 1592 size += sizeof(struct ibv_flow_spec_gre); 1593 break; 1594 case RTE_FLOW_ITEM_TYPE_MPLS: 1595 size += sizeof(struct ibv_flow_spec_mpls); 1596 break; 1597 #else 1598 case RTE_FLOW_ITEM_TYPE_GRE: 1599 size += sizeof(struct ibv_flow_spec_tunnel); 1600 break; 1601 #endif 1602 default: 1603 break; 1604 } 1605 } 1606 return size; 1607 } 1608 1609 /** 1610 * Internal preparation function. Allocate mlx5_flow with the required size. 1611 * The required size is calculate based on the actions and items. This function 1612 * also returns the detected actions and items for later use. 1613 * 1614 * @param[in] dev 1615 * Pointer to Ethernet device. 1616 * @param[in] attr 1617 * Pointer to the flow attributes. 1618 * @param[in] items 1619 * Pointer to the list of items. 1620 * @param[in] actions 1621 * Pointer to the list of actions. 1622 * @param[out] error 1623 * Pointer to the error structure. 1624 * 1625 * @return 1626 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno 1627 * is set. 1628 */ 1629 static struct mlx5_flow * 1630 flow_verbs_prepare(struct rte_eth_dev *dev, 1631 const struct rte_flow_attr *attr __rte_unused, 1632 const struct rte_flow_item items[], 1633 const struct rte_flow_action actions[], 1634 struct rte_flow_error *error) 1635 { 1636 size_t size = 0; 1637 uint32_t handle_idx = 0; 1638 struct mlx5_flow *dev_flow; 1639 struct mlx5_flow_handle *dev_handle; 1640 struct mlx5_priv *priv = dev->data->dev_private; 1641 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1642 1643 MLX5_ASSERT(wks); 1644 size += flow_verbs_get_actions_size(actions); 1645 size += flow_verbs_get_items_size(items); 1646 if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) { 1647 rte_flow_error_set(error, E2BIG, 1648 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1649 "Verbs spec/action size too large"); 1650 return NULL; 1651 } 1652 /* In case of corrupting the memory. */ 1653 if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { 1654 rte_flow_error_set(error, ENOSPC, 1655 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1656 "not free temporary device flow"); 1657 return NULL; 1658 } 1659 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1660 &handle_idx); 1661 if (!dev_handle) { 1662 rte_flow_error_set(error, ENOMEM, 1663 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1664 "not enough memory to create flow handle"); 1665 return NULL; 1666 } 1667 MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows)); 1668 dev_flow = &wks->flows[wks->flow_idx++]; 1669 dev_flow->handle = dev_handle; 1670 dev_flow->handle_idx = handle_idx; 1671 /* Memcpy is used, only size needs to be cleared to 0. */ 1672 dev_flow->verbs.size = 0; 1673 dev_flow->verbs.attr.num_of_specs = 0; 1674 dev_flow->ingress = attr->ingress; 1675 dev_flow->hash_fields = 0; 1676 /* Need to set transfer attribute: not supported in Verbs mode. */ 1677 return dev_flow; 1678 } 1679 1680 /** 1681 * Fill the flow with verb spec. 1682 * 1683 * @param[in] dev 1684 * Pointer to Ethernet device. 1685 * @param[in, out] dev_flow 1686 * Pointer to the mlx5 flow. 1687 * @param[in] attr 1688 * Pointer to the flow attributes. 1689 * @param[in] items 1690 * Pointer to the list of items. 1691 * @param[in] actions 1692 * Pointer to the list of actions. 1693 * @param[out] error 1694 * Pointer to the error structure. 1695 * 1696 * @return 1697 * 0 on success, else a negative errno value otherwise and rte_errno is set. 1698 */ 1699 static int 1700 flow_verbs_translate(struct rte_eth_dev *dev, 1701 struct mlx5_flow *dev_flow, 1702 const struct rte_flow_attr *attr, 1703 const struct rte_flow_item items[], 1704 const struct rte_flow_action actions[], 1705 struct rte_flow_error *error) 1706 { 1707 uint64_t item_flags = 0; 1708 uint64_t action_flags = 0; 1709 uint64_t priority = attr->priority; 1710 uint32_t subpriority = 0; 1711 struct mlx5_priv *priv = dev->data->dev_private; 1712 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1713 struct mlx5_flow_rss_desc *rss_desc; 1714 1715 MLX5_ASSERT(wks); 1716 rss_desc = &wks->rss_desc; 1717 if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1718 priority = priv->config.flow_prio - 1; 1719 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1720 int ret; 1721 1722 switch (actions->type) { 1723 case RTE_FLOW_ACTION_TYPE_VOID: 1724 break; 1725 case RTE_FLOW_ACTION_TYPE_FLAG: 1726 flow_verbs_translate_action_flag(dev_flow, actions); 1727 action_flags |= MLX5_FLOW_ACTION_FLAG; 1728 dev_flow->handle->mark = 1; 1729 break; 1730 case RTE_FLOW_ACTION_TYPE_MARK: 1731 flow_verbs_translate_action_mark(dev_flow, actions); 1732 action_flags |= MLX5_FLOW_ACTION_MARK; 1733 dev_flow->handle->mark = 1; 1734 break; 1735 case RTE_FLOW_ACTION_TYPE_DROP: 1736 flow_verbs_translate_action_drop(dev_flow, actions); 1737 action_flags |= MLX5_FLOW_ACTION_DROP; 1738 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; 1739 break; 1740 case RTE_FLOW_ACTION_TYPE_QUEUE: 1741 flow_verbs_translate_action_queue(rss_desc, actions); 1742 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1743 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 1744 break; 1745 case RTE_FLOW_ACTION_TYPE_RSS: 1746 flow_verbs_translate_action_rss(rss_desc, actions); 1747 action_flags |= MLX5_FLOW_ACTION_RSS; 1748 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 1749 break; 1750 case RTE_FLOW_ACTION_TYPE_COUNT: 1751 ret = flow_verbs_translate_action_count(dev_flow, 1752 actions, 1753 dev, error); 1754 if (ret < 0) 1755 return ret; 1756 action_flags |= MLX5_FLOW_ACTION_COUNT; 1757 break; 1758 default: 1759 return rte_flow_error_set(error, ENOTSUP, 1760 RTE_FLOW_ERROR_TYPE_ACTION, 1761 actions, 1762 "action not supported"); 1763 } 1764 } 1765 dev_flow->act_flags = action_flags; 1766 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1767 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1768 1769 switch (items->type) { 1770 case RTE_FLOW_ITEM_TYPE_VOID: 1771 break; 1772 case RTE_FLOW_ITEM_TYPE_ETH: 1773 flow_verbs_translate_item_eth(dev_flow, items, 1774 item_flags); 1775 subpriority = MLX5_PRIORITY_MAP_L2; 1776 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1777 MLX5_FLOW_LAYER_OUTER_L2; 1778 break; 1779 case RTE_FLOW_ITEM_TYPE_VLAN: 1780 flow_verbs_translate_item_vlan(dev_flow, items, 1781 item_flags); 1782 subpriority = MLX5_PRIORITY_MAP_L2; 1783 item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 1784 MLX5_FLOW_LAYER_INNER_VLAN) : 1785 (MLX5_FLOW_LAYER_OUTER_L2 | 1786 MLX5_FLOW_LAYER_OUTER_VLAN); 1787 break; 1788 case RTE_FLOW_ITEM_TYPE_IPV4: 1789 flow_verbs_translate_item_ipv4(dev_flow, items, 1790 item_flags); 1791 subpriority = MLX5_PRIORITY_MAP_L3; 1792 dev_flow->hash_fields |= 1793 mlx5_flow_hashfields_adjust 1794 (rss_desc, tunnel, 1795 MLX5_IPV4_LAYER_TYPES, 1796 MLX5_IPV4_IBV_RX_HASH); 1797 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1798 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1799 break; 1800 case RTE_FLOW_ITEM_TYPE_IPV6: 1801 flow_verbs_translate_item_ipv6(dev_flow, items, 1802 item_flags); 1803 subpriority = MLX5_PRIORITY_MAP_L3; 1804 dev_flow->hash_fields |= 1805 mlx5_flow_hashfields_adjust 1806 (rss_desc, tunnel, 1807 MLX5_IPV6_LAYER_TYPES, 1808 MLX5_IPV6_IBV_RX_HASH); 1809 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1810 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1811 break; 1812 case RTE_FLOW_ITEM_TYPE_TCP: 1813 flow_verbs_translate_item_tcp(dev_flow, items, 1814 item_flags); 1815 subpriority = MLX5_PRIORITY_MAP_L4; 1816 if (dev_flow->hash_fields != 0) 1817 dev_flow->hash_fields |= 1818 mlx5_flow_hashfields_adjust 1819 (rss_desc, tunnel, ETH_RSS_TCP, 1820 (IBV_RX_HASH_SRC_PORT_TCP | 1821 IBV_RX_HASH_DST_PORT_TCP)); 1822 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 1823 MLX5_FLOW_LAYER_OUTER_L4_TCP; 1824 break; 1825 case RTE_FLOW_ITEM_TYPE_UDP: 1826 flow_verbs_translate_item_udp(dev_flow, items, 1827 item_flags); 1828 subpriority = MLX5_PRIORITY_MAP_L4; 1829 if (dev_flow->hash_fields != 0) 1830 dev_flow->hash_fields |= 1831 mlx5_flow_hashfields_adjust 1832 (rss_desc, tunnel, ETH_RSS_UDP, 1833 (IBV_RX_HASH_SRC_PORT_UDP | 1834 IBV_RX_HASH_DST_PORT_UDP)); 1835 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 1836 MLX5_FLOW_LAYER_OUTER_L4_UDP; 1837 break; 1838 case RTE_FLOW_ITEM_TYPE_VXLAN: 1839 flow_verbs_translate_item_vxlan(dev_flow, items, 1840 item_flags); 1841 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1842 item_flags |= MLX5_FLOW_LAYER_VXLAN; 1843 break; 1844 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1845 flow_verbs_translate_item_vxlan_gpe(dev_flow, items, 1846 item_flags); 1847 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1848 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE; 1849 break; 1850 case RTE_FLOW_ITEM_TYPE_GRE: 1851 flow_verbs_translate_item_gre(dev_flow, items, 1852 item_flags); 1853 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1854 item_flags |= MLX5_FLOW_LAYER_GRE; 1855 break; 1856 case RTE_FLOW_ITEM_TYPE_MPLS: 1857 flow_verbs_translate_item_mpls(dev_flow, items, 1858 item_flags); 1859 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1860 item_flags |= MLX5_FLOW_LAYER_MPLS; 1861 break; 1862 default: 1863 return rte_flow_error_set(error, ENOTSUP, 1864 RTE_FLOW_ERROR_TYPE_ITEM, 1865 NULL, "item not supported"); 1866 } 1867 } 1868 dev_flow->handle->layers = item_flags; 1869 /* Other members of attr will be ignored. */ 1870 dev_flow->verbs.attr.priority = 1871 mlx5_flow_adjust_priority(dev, priority, subpriority); 1872 dev_flow->verbs.attr.port = (uint8_t)priv->dev_port; 1873 return 0; 1874 } 1875 1876 /** 1877 * Remove the flow from the NIC but keeps it in memory. 1878 * 1879 * @param[in] dev 1880 * Pointer to the Ethernet device structure. 1881 * @param[in, out] flow 1882 * Pointer to flow structure. 1883 */ 1884 static void 1885 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 1886 { 1887 struct mlx5_priv *priv = dev->data->dev_private; 1888 struct mlx5_flow_handle *handle; 1889 uint32_t handle_idx; 1890 1891 if (!flow) 1892 return; 1893 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1894 handle_idx, handle, next) { 1895 if (handle->drv_flow) { 1896 claim_zero(mlx5_glue->destroy_flow(handle->drv_flow)); 1897 handle->drv_flow = NULL; 1898 } 1899 /* hrxq is union, don't touch it only the flag is set. */ 1900 if (handle->rix_hrxq && 1901 handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1902 mlx5_hrxq_release(dev, handle->rix_hrxq); 1903 handle->rix_hrxq = 0; 1904 } 1905 if (handle->vf_vlan.tag && handle->vf_vlan.created) 1906 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); 1907 } 1908 } 1909 1910 /** 1911 * Remove the flow from the NIC and the memory. 1912 * 1913 * @param[in] dev 1914 * Pointer to the Ethernet device structure. 1915 * @param[in, out] flow 1916 * Pointer to flow structure. 1917 */ 1918 static void 1919 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 1920 { 1921 struct mlx5_priv *priv = dev->data->dev_private; 1922 struct mlx5_flow_handle *handle; 1923 1924 if (!flow) 1925 return; 1926 flow_verbs_remove(dev, flow); 1927 while (flow->dev_handles) { 1928 uint32_t tmp_idx = flow->dev_handles; 1929 1930 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1931 tmp_idx); 1932 if (!handle) 1933 return; 1934 flow->dev_handles = handle->next.next; 1935 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1936 tmp_idx); 1937 } 1938 if (flow->counter) { 1939 flow_verbs_counter_release(dev, flow->counter); 1940 flow->counter = 0; 1941 } 1942 } 1943 1944 /** 1945 * Apply the flow to the NIC. 1946 * 1947 * @param[in] dev 1948 * Pointer to the Ethernet device structure. 1949 * @param[in, out] flow 1950 * Pointer to flow structure. 1951 * @param[out] error 1952 * Pointer to error structure. 1953 * 1954 * @return 1955 * 0 on success, a negative errno value otherwise and rte_errno is set. 1956 */ 1957 static int 1958 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 1959 struct rte_flow_error *error) 1960 { 1961 struct mlx5_priv *priv = dev->data->dev_private; 1962 struct mlx5_flow_handle *handle; 1963 struct mlx5_flow *dev_flow; 1964 struct mlx5_hrxq *hrxq; 1965 uint32_t dev_handles; 1966 int err; 1967 int idx; 1968 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1969 1970 MLX5_ASSERT(wks); 1971 for (idx = wks->flow_idx - 1; idx >= 0; idx--) { 1972 dev_flow = &wks->flows[idx]; 1973 handle = dev_flow->handle; 1974 if (handle->fate_action == MLX5_FLOW_FATE_DROP) { 1975 MLX5_ASSERT(priv->drop_queue.hrxq); 1976 hrxq = priv->drop_queue.hrxq; 1977 } else { 1978 uint32_t hrxq_idx; 1979 struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc; 1980 1981 MLX5_ASSERT(rss_desc->queue_num); 1982 rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; 1983 rss_desc->hash_fields = dev_flow->hash_fields; 1984 rss_desc->tunnel = !!(handle->layers & 1985 MLX5_FLOW_LAYER_TUNNEL); 1986 rss_desc->shared_rss = 0; 1987 hrxq_idx = mlx5_hrxq_get(dev, rss_desc); 1988 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1989 hrxq_idx); 1990 if (!hrxq) { 1991 rte_flow_error_set 1992 (error, rte_errno, 1993 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1994 "cannot get hash queue"); 1995 goto error; 1996 } 1997 handle->rix_hrxq = hrxq_idx; 1998 } 1999 MLX5_ASSERT(hrxq); 2000 handle->drv_flow = mlx5_glue->create_flow 2001 (hrxq->qp, &dev_flow->verbs.attr); 2002 if (!handle->drv_flow) { 2003 rte_flow_error_set(error, errno, 2004 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2005 NULL, 2006 "hardware refuses to create flow"); 2007 goto error; 2008 } 2009 if (priv->vmwa_context && 2010 handle->vf_vlan.tag && !handle->vf_vlan.created) { 2011 /* 2012 * The rule contains the VLAN pattern. 2013 * For VF we are going to create VLAN 2014 * interface to make hypervisor set correct 2015 * e-Switch vport context. 2016 */ 2017 mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan); 2018 } 2019 } 2020 return 0; 2021 error: 2022 err = rte_errno; /* Save rte_errno before cleanup. */ 2023 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2024 dev_handles, handle, next) { 2025 /* hrxq is union, don't touch it only the flag is set. */ 2026 if (handle->rix_hrxq && 2027 handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 2028 mlx5_hrxq_release(dev, handle->rix_hrxq); 2029 handle->rix_hrxq = 0; 2030 } 2031 if (handle->vf_vlan.tag && handle->vf_vlan.created) 2032 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); 2033 } 2034 rte_errno = err; /* Restore rte_errno. */ 2035 return -rte_errno; 2036 } 2037 2038 /** 2039 * Query a flow. 2040 * 2041 * @see rte_flow_query() 2042 * @see rte_flow_ops 2043 */ 2044 static int 2045 flow_verbs_query(struct rte_eth_dev *dev, 2046 struct rte_flow *flow, 2047 const struct rte_flow_action *actions, 2048 void *data, 2049 struct rte_flow_error *error) 2050 { 2051 int ret = -EINVAL; 2052 2053 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2054 switch (actions->type) { 2055 case RTE_FLOW_ACTION_TYPE_VOID: 2056 break; 2057 case RTE_FLOW_ACTION_TYPE_COUNT: 2058 ret = flow_verbs_counter_query(dev, flow, data, error); 2059 break; 2060 default: 2061 return rte_flow_error_set(error, ENOTSUP, 2062 RTE_FLOW_ERROR_TYPE_ACTION, 2063 actions, 2064 "action not supported"); 2065 } 2066 } 2067 return ret; 2068 } 2069 2070 static int 2071 flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains, 2072 uint32_t flags) 2073 { 2074 RTE_SET_USED(dev); 2075 RTE_SET_USED(domains); 2076 RTE_SET_USED(flags); 2077 2078 return 0; 2079 } 2080 2081 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = { 2082 .validate = flow_verbs_validate, 2083 .prepare = flow_verbs_prepare, 2084 .translate = flow_verbs_translate, 2085 .apply = flow_verbs_apply, 2086 .remove = flow_verbs_remove, 2087 .destroy = flow_verbs_destroy, 2088 .query = flow_verbs_query, 2089 .sync_domain = flow_verbs_sync_domain, 2090 }; 2091