1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <netinet/in.h> 6 #include <sys/queue.h> 7 #include <stdalign.h> 8 #include <stdint.h> 9 #include <string.h> 10 11 #include <rte_common.h> 12 #include <rte_ether.h> 13 #include <ethdev_driver.h> 14 #include <rte_flow.h> 15 #include <rte_flow_driver.h> 16 #include <rte_malloc.h> 17 #include <rte_ip.h> 18 19 #include <mlx5_glue.h> 20 #include <mlx5_prm.h> 21 #include <mlx5_malloc.h> 22 23 #include "mlx5_defs.h" 24 #include "mlx5.h" 25 #include "mlx5_flow.h" 26 #include "mlx5_rx.h" 27 28 #define VERBS_SPEC_INNER(item_flags) \ 29 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0) 30 31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 33 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 34 }; 35 36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 38 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 39 { 9, 10, 11 }, { 12, 13, 14 }, 40 }; 41 42 /* Verbs specification header. */ 43 struct ibv_spec_header { 44 enum ibv_flow_spec_type type; 45 uint16_t size; 46 }; 47 48 /** 49 * Discover the maximum number of priority available. 50 * 51 * @param[in] dev 52 * Pointer to the Ethernet device structure. 53 * 54 * @return 55 * number of supported flow priority on success, a negative errno 56 * value otherwise and rte_errno is set. 57 */ 58 int 59 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 60 { 61 struct mlx5_priv *priv = dev->data->dev_private; 62 struct { 63 struct ibv_flow_attr attr; 64 struct ibv_flow_spec_eth eth; 65 struct ibv_flow_spec_action_drop drop; 66 } flow_attr = { 67 .attr = { 68 .num_of_specs = 2, 69 .port = (uint8_t)priv->dev_port, 70 }, 71 .eth = { 72 .type = IBV_FLOW_SPEC_ETH, 73 .size = sizeof(struct ibv_flow_spec_eth), 74 }, 75 .drop = { 76 .size = sizeof(struct ibv_flow_spec_action_drop), 77 .type = IBV_FLOW_SPEC_ACTION_DROP, 78 }, 79 }; 80 struct ibv_flow *flow; 81 struct mlx5_hrxq *drop = priv->drop_queue.hrxq; 82 uint16_t vprio[] = { 8, 16 }; 83 int i; 84 int priority = 0; 85 86 #if defined(HAVE_MLX5DV_DR_DEVX_PORT) || defined(HAVE_MLX5DV_DR_DEVX_PORT_V35) 87 /* If DevX supported, driver must support 16 verbs flow priorities. */ 88 priority = RTE_DIM(priority_map_5); 89 goto out; 90 #endif 91 if (!drop->qp) { 92 rte_errno = ENOTSUP; 93 return -rte_errno; 94 } 95 for (i = 0; i != RTE_DIM(vprio); i++) { 96 flow_attr.attr.priority = vprio[i] - 1; 97 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 98 if (!flow) 99 break; 100 claim_zero(mlx5_glue->destroy_flow(flow)); 101 priority = vprio[i]; 102 } 103 switch (priority) { 104 case 8: 105 priority = RTE_DIM(priority_map_3); 106 break; 107 case 16: 108 priority = RTE_DIM(priority_map_5); 109 break; 110 default: 111 rte_errno = ENOTSUP; 112 DRV_LOG(ERR, 113 "port %u verbs maximum priority: %d expected 8/16", 114 dev->data->port_id, priority); 115 return -rte_errno; 116 } 117 #if defined(HAVE_MLX5DV_DR_DEVX_PORT) || defined(HAVE_MLX5DV_DR_DEVX_PORT_V35) 118 out: 119 #endif 120 DRV_LOG(INFO, "port %u supported flow priorities:" 121 " 0-%d for ingress or egress root table," 122 " 0-%d for non-root table or transfer root table.", 123 dev->data->port_id, priority - 2, 124 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 125 return priority; 126 } 127 128 /** 129 * Adjust flow priority based on the highest layer and the request priority. 130 * 131 * @param[in] dev 132 * Pointer to the Ethernet device structure. 133 * @param[in] priority 134 * The rule base priority. 135 * @param[in] subpriority 136 * The priority based on the items. 137 * 138 * @return 139 * The new priority. 140 */ 141 uint32_t 142 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 143 uint32_t subpriority) 144 { 145 uint32_t res = 0; 146 struct mlx5_priv *priv = dev->data->dev_private; 147 148 switch (priv->config.flow_prio) { 149 case RTE_DIM(priority_map_3): 150 res = priority_map_3[priority][subpriority]; 151 break; 152 case RTE_DIM(priority_map_5): 153 res = priority_map_5[priority][subpriority]; 154 break; 155 } 156 return res; 157 } 158 159 /** 160 * Get Verbs flow counter by index. 161 * 162 * @param[in] dev 163 * Pointer to the Ethernet device structure. 164 * @param[in] idx 165 * mlx5 flow counter index in the container. 166 * @param[out] ppool 167 * mlx5 flow counter pool in the container, 168 * 169 * @return 170 * A pointer to the counter, NULL otherwise. 171 */ 172 static struct mlx5_flow_counter * 173 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev, 174 uint32_t idx, 175 struct mlx5_flow_counter_pool **ppool) 176 { 177 struct mlx5_priv *priv = dev->data->dev_private; 178 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 179 struct mlx5_flow_counter_pool *pool; 180 181 idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1); 182 pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL]; 183 MLX5_ASSERT(pool); 184 if (ppool) 185 *ppool = pool; 186 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL); 187 } 188 189 /** 190 * Create Verbs flow counter with Verbs library. 191 * 192 * @param[in] dev 193 * Pointer to the Ethernet device structure. 194 * @param[in, out] counter 195 * mlx5 flow counter object, contains the counter id, 196 * handle of created Verbs flow counter is returned 197 * in cs field (if counters are supported). 198 * 199 * @return 200 * 0 On success else a negative errno value is returned 201 * and rte_errno is set. 202 */ 203 static int 204 flow_verbs_counter_create(struct rte_eth_dev *dev, 205 struct mlx5_flow_counter *counter) 206 { 207 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 208 struct mlx5_priv *priv = dev->data->dev_private; 209 struct ibv_context *ctx = priv->sh->cdev->ctx; 210 struct ibv_counter_set_init_attr init = { 211 .counter_set_id = counter->shared_info.id}; 212 213 counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init); 214 if (!counter->dcs_when_free) { 215 rte_errno = ENOTSUP; 216 return -ENOTSUP; 217 } 218 return 0; 219 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 220 struct mlx5_priv *priv = dev->data->dev_private; 221 struct ibv_context *ctx = priv->sh->cdev->ctx; 222 struct ibv_counters_init_attr init = {0}; 223 struct ibv_counter_attach_attr attach; 224 int ret; 225 226 memset(&attach, 0, sizeof(attach)); 227 counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init); 228 if (!counter->dcs_when_free) { 229 rte_errno = ENOTSUP; 230 return -ENOTSUP; 231 } 232 attach.counter_desc = IBV_COUNTER_PACKETS; 233 attach.index = 0; 234 ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL); 235 if (!ret) { 236 attach.counter_desc = IBV_COUNTER_BYTES; 237 attach.index = 1; 238 ret = mlx5_glue->attach_counters 239 (counter->dcs_when_free, &attach, NULL); 240 } 241 if (ret) { 242 claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free)); 243 counter->dcs_when_free = NULL; 244 rte_errno = ret; 245 return -ret; 246 } 247 return 0; 248 #else 249 (void)dev; 250 (void)counter; 251 rte_errno = ENOTSUP; 252 return -ENOTSUP; 253 #endif 254 } 255 256 /** 257 * Get a flow counter. 258 * 259 * @param[in] dev 260 * Pointer to the Ethernet device structure. 261 * @param[in] id 262 * Counter identifier. 263 * 264 * @return 265 * Index to the counter, 0 otherwise and rte_errno is set. 266 */ 267 static uint32_t 268 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t id __rte_unused) 269 { 270 struct mlx5_priv *priv = dev->data->dev_private; 271 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 272 struct mlx5_flow_counter_pool *pool = NULL; 273 struct mlx5_flow_counter *cnt = NULL; 274 uint32_t n_valid = cmng->n_valid; 275 uint32_t pool_idx, cnt_idx; 276 uint32_t i; 277 int ret; 278 279 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) { 280 pool = cmng->pools[pool_idx]; 281 if (!pool) 282 continue; 283 cnt = TAILQ_FIRST(&pool->counters[0]); 284 if (cnt) 285 break; 286 } 287 if (!cnt) { 288 struct mlx5_flow_counter_pool **pools; 289 uint32_t size; 290 291 if (n_valid == cmng->n) { 292 /* Resize the container pool array. */ 293 size = sizeof(struct mlx5_flow_counter_pool *) * 294 (n_valid + MLX5_CNT_CONTAINER_RESIZE); 295 pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0, 296 SOCKET_ID_ANY); 297 if (!pools) 298 return 0; 299 if (n_valid) { 300 memcpy(pools, cmng->pools, 301 sizeof(struct mlx5_flow_counter_pool *) * 302 n_valid); 303 mlx5_free(cmng->pools); 304 } 305 cmng->pools = pools; 306 cmng->n += MLX5_CNT_CONTAINER_RESIZE; 307 } 308 /* Allocate memory for new pool*/ 309 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL; 310 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY); 311 if (!pool) 312 return 0; 313 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 314 cnt = MLX5_POOL_GET_CNT(pool, i); 315 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 316 } 317 cnt = MLX5_POOL_GET_CNT(pool, 0); 318 cmng->pools[n_valid] = pool; 319 pool_idx = n_valid; 320 cmng->n_valid++; 321 } 322 TAILQ_REMOVE(&pool->counters[0], cnt, next); 323 i = MLX5_CNT_ARRAY_IDX(pool, cnt); 324 cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i); 325 /* Create counter with Verbs. */ 326 ret = flow_verbs_counter_create(dev, cnt); 327 if (!ret) { 328 cnt->dcs_when_active = cnt->dcs_when_free; 329 cnt->hits = 0; 330 cnt->bytes = 0; 331 return cnt_idx; 332 } 333 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 334 /* Some error occurred in Verbs library. */ 335 rte_errno = -ret; 336 return 0; 337 } 338 339 /** 340 * Release a flow counter. 341 * 342 * @param[in] dev 343 * Pointer to the Ethernet device structure. 344 * @param[in] counter 345 * Index to the counter handler. 346 */ 347 static void 348 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter) 349 { 350 struct mlx5_flow_counter_pool *pool; 351 struct mlx5_flow_counter *cnt; 352 353 cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool); 354 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 355 claim_zero(mlx5_glue->destroy_counter_set 356 ((struct ibv_counter_set *)cnt->dcs_when_active)); 357 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 358 claim_zero(mlx5_glue->destroy_counters 359 ((struct ibv_counters *)cnt->dcs_when_active)); 360 #endif 361 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 362 } 363 364 /** 365 * Query a flow counter via Verbs library call. 366 * 367 * @see rte_flow_query() 368 * @see rte_flow_ops 369 */ 370 static int 371 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, 372 struct rte_flow *flow, void *data, 373 struct rte_flow_error *error) 374 { 375 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 376 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 377 if (flow->counter) { 378 struct mlx5_flow_counter_pool *pool; 379 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx 380 (dev, flow->counter, &pool); 381 struct rte_flow_query_count *qc = data; 382 uint64_t counters[2] = {0, 0}; 383 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 384 struct ibv_query_counter_set_attr query_cs_attr = { 385 .dcs_when_free = (struct ibv_counter_set *) 386 cnt->dcs_when_active, 387 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 388 }; 389 struct ibv_counter_set_data query_out = { 390 .out = counters, 391 .outlen = 2 * sizeof(uint64_t), 392 }; 393 int err = mlx5_glue->query_counter_set(&query_cs_attr, 394 &query_out); 395 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 396 int err = mlx5_glue->query_counters 397 ((struct ibv_counters *)cnt->dcs_when_active, counters, 398 RTE_DIM(counters), 399 IBV_READ_COUNTERS_ATTR_PREFER_CACHED); 400 #endif 401 if (err) 402 return rte_flow_error_set 403 (error, err, 404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 405 NULL, 406 "cannot read counter"); 407 qc->hits_set = 1; 408 qc->bytes_set = 1; 409 qc->hits = counters[0] - cnt->hits; 410 qc->bytes = counters[1] - cnt->bytes; 411 if (qc->reset) { 412 cnt->hits = counters[0]; 413 cnt->bytes = counters[1]; 414 } 415 return 0; 416 } 417 return rte_flow_error_set(error, EINVAL, 418 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 419 NULL, 420 "flow does not have counter"); 421 #else 422 (void)flow; 423 (void)data; 424 return rte_flow_error_set(error, ENOTSUP, 425 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 426 NULL, 427 "counters are not available"); 428 #endif 429 } 430 431 /** 432 * Add a verbs item specification into @p verbs. 433 * 434 * @param[out] verbs 435 * Pointer to verbs structure. 436 * @param[in] src 437 * Create specification. 438 * @param[in] size 439 * Size in bytes of the specification to copy. 440 */ 441 static void 442 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs, 443 void *src, unsigned int size) 444 { 445 void *dst; 446 447 if (!verbs) 448 return; 449 MLX5_ASSERT(verbs->specs); 450 dst = (void *)(verbs->specs + verbs->size); 451 memcpy(dst, src, size); 452 ++verbs->attr.num_of_specs; 453 verbs->size += size; 454 } 455 456 /** 457 * Convert the @p item into a Verbs specification. This function assumes that 458 * the input is valid and that there is space to insert the requested item 459 * into the flow. 460 * 461 * @param[in, out] dev_flow 462 * Pointer to dev_flow structure. 463 * @param[in] item 464 * Item specification. 465 * @param[in] item_flags 466 * Parsed item flags. 467 */ 468 static void 469 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow, 470 const struct rte_flow_item *item, 471 uint64_t item_flags) 472 { 473 const struct rte_flow_item_eth *spec = item->spec; 474 const struct rte_flow_item_eth *mask = item->mask; 475 const unsigned int size = sizeof(struct ibv_flow_spec_eth); 476 struct ibv_flow_spec_eth eth = { 477 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags), 478 .size = size, 479 }; 480 481 if (!mask) 482 mask = &rte_flow_item_eth_mask; 483 if (spec) { 484 unsigned int i; 485 486 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, 487 RTE_ETHER_ADDR_LEN); 488 memcpy(ð.val.src_mac, spec->src.addr_bytes, 489 RTE_ETHER_ADDR_LEN); 490 eth.val.ether_type = spec->type; 491 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, 492 RTE_ETHER_ADDR_LEN); 493 memcpy(ð.mask.src_mac, mask->src.addr_bytes, 494 RTE_ETHER_ADDR_LEN); 495 eth.mask.ether_type = mask->type; 496 /* Remove unwanted bits from values. */ 497 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) { 498 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 499 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 500 } 501 eth.val.ether_type &= eth.mask.ether_type; 502 } 503 flow_verbs_spec_add(&dev_flow->verbs, ð, size); 504 } 505 506 /** 507 * Update the VLAN tag in the Verbs Ethernet specification. 508 * This function assumes that the input is valid and there is space to add 509 * the requested item. 510 * 511 * @param[in, out] attr 512 * Pointer to Verbs attributes structure. 513 * @param[in] eth 514 * Verbs structure containing the VLAN information to copy. 515 */ 516 static void 517 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr, 518 struct ibv_flow_spec_eth *eth) 519 { 520 unsigned int i; 521 const enum ibv_flow_spec_type search = eth->type; 522 struct ibv_spec_header *hdr = (struct ibv_spec_header *) 523 ((uint8_t *)attr + sizeof(struct ibv_flow_attr)); 524 525 for (i = 0; i != attr->num_of_specs; ++i) { 526 if (hdr->type == search) { 527 struct ibv_flow_spec_eth *e = 528 (struct ibv_flow_spec_eth *)hdr; 529 530 e->val.vlan_tag = eth->val.vlan_tag; 531 e->mask.vlan_tag = eth->mask.vlan_tag; 532 e->val.ether_type = eth->val.ether_type; 533 e->mask.ether_type = eth->mask.ether_type; 534 break; 535 } 536 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size); 537 } 538 } 539 540 /** 541 * Convert the @p item into a Verbs specification. This function assumes that 542 * the input is valid and that there is space to insert the requested item 543 * into the flow. 544 * 545 * @param[in, out] dev_flow 546 * Pointer to dev_flow structure. 547 * @param[in] item 548 * Item specification. 549 * @param[in] item_flags 550 * Parsed item flags. 551 */ 552 static void 553 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow, 554 const struct rte_flow_item *item, 555 uint64_t item_flags) 556 { 557 const struct rte_flow_item_vlan *spec = item->spec; 558 const struct rte_flow_item_vlan *mask = item->mask; 559 unsigned int size = sizeof(struct ibv_flow_spec_eth); 560 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 561 struct ibv_flow_spec_eth eth = { 562 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags), 563 .size = size, 564 }; 565 const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 566 MLX5_FLOW_LAYER_OUTER_L2; 567 568 if (!mask) 569 mask = &rte_flow_item_vlan_mask; 570 if (spec) { 571 eth.val.vlan_tag = spec->tci; 572 eth.mask.vlan_tag = mask->tci; 573 eth.val.vlan_tag &= eth.mask.vlan_tag; 574 eth.val.ether_type = spec->inner_type; 575 eth.mask.ether_type = mask->inner_type; 576 eth.val.ether_type &= eth.mask.ether_type; 577 } 578 if (!(item_flags & l2m)) 579 flow_verbs_spec_add(&dev_flow->verbs, ð, size); 580 else 581 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð); 582 if (!tunnel) 583 dev_flow->handle->vf_vlan.tag = 584 rte_be_to_cpu_16(spec->tci) & 0x0fff; 585 } 586 587 /** 588 * Convert the @p item into a Verbs specification. This function assumes that 589 * the input is valid and that there is space to insert the requested item 590 * into the flow. 591 * 592 * @param[in, out] dev_flow 593 * Pointer to dev_flow structure. 594 * @param[in] item 595 * Item specification. 596 * @param[in] item_flags 597 * Parsed item flags. 598 */ 599 static void 600 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow, 601 const struct rte_flow_item *item, 602 uint64_t item_flags) 603 { 604 const struct rte_flow_item_ipv4 *spec = item->spec; 605 const struct rte_flow_item_ipv4 *mask = item->mask; 606 unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext); 607 struct ibv_flow_spec_ipv4_ext ipv4 = { 608 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags), 609 .size = size, 610 }; 611 612 if (!mask) 613 mask = &rte_flow_item_ipv4_mask; 614 if (spec) { 615 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 616 .src_ip = spec->hdr.src_addr, 617 .dst_ip = spec->hdr.dst_addr, 618 .proto = spec->hdr.next_proto_id, 619 .tos = spec->hdr.type_of_service, 620 }; 621 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 622 .src_ip = mask->hdr.src_addr, 623 .dst_ip = mask->hdr.dst_addr, 624 .proto = mask->hdr.next_proto_id, 625 .tos = mask->hdr.type_of_service, 626 }; 627 /* Remove unwanted bits from values. */ 628 ipv4.val.src_ip &= ipv4.mask.src_ip; 629 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 630 ipv4.val.proto &= ipv4.mask.proto; 631 ipv4.val.tos &= ipv4.mask.tos; 632 } 633 flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size); 634 } 635 636 /** 637 * Convert the @p item into a Verbs specification. This function assumes that 638 * the input is valid and that there is space to insert the requested item 639 * into the flow. 640 * 641 * @param[in, out] dev_flow 642 * Pointer to dev_flow structure. 643 * @param[in] item 644 * Item specification. 645 * @param[in] item_flags 646 * Parsed item flags. 647 */ 648 static void 649 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow, 650 const struct rte_flow_item *item, 651 uint64_t item_flags) 652 { 653 const struct rte_flow_item_ipv6 *spec = item->spec; 654 const struct rte_flow_item_ipv6 *mask = item->mask; 655 unsigned int size = sizeof(struct ibv_flow_spec_ipv6); 656 struct ibv_flow_spec_ipv6 ipv6 = { 657 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags), 658 .size = size, 659 }; 660 661 if (!mask) 662 mask = &rte_flow_item_ipv6_mask; 663 if (spec) { 664 unsigned int i; 665 uint32_t vtc_flow_val; 666 uint32_t vtc_flow_mask; 667 668 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 669 RTE_DIM(ipv6.val.src_ip)); 670 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 671 RTE_DIM(ipv6.val.dst_ip)); 672 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 673 RTE_DIM(ipv6.mask.src_ip)); 674 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 675 RTE_DIM(ipv6.mask.dst_ip)); 676 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); 677 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); 678 ipv6.val.flow_label = 679 rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >> 680 RTE_IPV6_HDR_FL_SHIFT); 681 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >> 682 RTE_IPV6_HDR_TC_SHIFT; 683 ipv6.val.next_hdr = spec->hdr.proto; 684 ipv6.mask.flow_label = 685 rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >> 686 RTE_IPV6_HDR_FL_SHIFT); 687 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >> 688 RTE_IPV6_HDR_TC_SHIFT; 689 ipv6.mask.next_hdr = mask->hdr.proto; 690 /* Remove unwanted bits from values. */ 691 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 692 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 693 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 694 } 695 ipv6.val.flow_label &= ipv6.mask.flow_label; 696 ipv6.val.traffic_class &= ipv6.mask.traffic_class; 697 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 698 } 699 flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size); 700 } 701 702 /** 703 * Convert the @p item into a Verbs specification. This function assumes that 704 * the input is valid and that there is space to insert the requested item 705 * into the flow. 706 * 707 * @param[in, out] dev_flow 708 * Pointer to dev_flow structure. 709 * @param[in] item 710 * Item specification. 711 * @param[in] item_flags 712 * Parsed item flags. 713 */ 714 static void 715 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow, 716 const struct rte_flow_item *item, 717 uint64_t item_flags __rte_unused) 718 { 719 const struct rte_flow_item_tcp *spec = item->spec; 720 const struct rte_flow_item_tcp *mask = item->mask; 721 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp); 722 struct ibv_flow_spec_tcp_udp tcp = { 723 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags), 724 .size = size, 725 }; 726 727 if (!mask) 728 mask = &rte_flow_item_tcp_mask; 729 if (spec) { 730 tcp.val.dst_port = spec->hdr.dst_port; 731 tcp.val.src_port = spec->hdr.src_port; 732 tcp.mask.dst_port = mask->hdr.dst_port; 733 tcp.mask.src_port = mask->hdr.src_port; 734 /* Remove unwanted bits from values. */ 735 tcp.val.src_port &= tcp.mask.src_port; 736 tcp.val.dst_port &= tcp.mask.dst_port; 737 } 738 flow_verbs_spec_add(&dev_flow->verbs, &tcp, size); 739 } 740 741 /** 742 * Convert the @p item into a Verbs specification. This function assumes that 743 * the input is valid and that there is space to insert the requested item 744 * into the flow. 745 * 746 * @param[in, out] dev_flow 747 * Pointer to dev_flow structure. 748 * @param[in] item 749 * Item specification. 750 * @param[in] item_flags 751 * Parsed item flags. 752 */ 753 static void 754 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow, 755 const struct rte_flow_item *item, 756 uint64_t item_flags __rte_unused) 757 { 758 const struct rte_flow_item_udp *spec = item->spec; 759 const struct rte_flow_item_udp *mask = item->mask; 760 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp); 761 struct ibv_flow_spec_tcp_udp udp = { 762 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags), 763 .size = size, 764 }; 765 766 if (!mask) 767 mask = &rte_flow_item_udp_mask; 768 if (spec) { 769 udp.val.dst_port = spec->hdr.dst_port; 770 udp.val.src_port = spec->hdr.src_port; 771 udp.mask.dst_port = mask->hdr.dst_port; 772 udp.mask.src_port = mask->hdr.src_port; 773 /* Remove unwanted bits from values. */ 774 udp.val.src_port &= udp.mask.src_port; 775 udp.val.dst_port &= udp.mask.dst_port; 776 } 777 item++; 778 while (item->type == RTE_FLOW_ITEM_TYPE_VOID) 779 item++; 780 if (!(udp.val.dst_port & udp.mask.dst_port)) { 781 switch ((item)->type) { 782 case RTE_FLOW_ITEM_TYPE_VXLAN: 783 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN); 784 udp.mask.dst_port = 0xffff; 785 break; 786 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 787 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE); 788 udp.mask.dst_port = 0xffff; 789 break; 790 case RTE_FLOW_ITEM_TYPE_MPLS: 791 udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS); 792 udp.mask.dst_port = 0xffff; 793 break; 794 default: 795 break; 796 } 797 } 798 799 flow_verbs_spec_add(&dev_flow->verbs, &udp, size); 800 } 801 802 /** 803 * Convert the @p item into a Verbs specification. This function assumes that 804 * the input is valid and that there is space to insert the requested item 805 * into the flow. 806 * 807 * @param[in, out] dev_flow 808 * Pointer to dev_flow structure. 809 * @param[in] item 810 * Item specification. 811 * @param[in] item_flags 812 * Parsed item flags. 813 */ 814 static void 815 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow, 816 const struct rte_flow_item *item, 817 uint64_t item_flags __rte_unused) 818 { 819 const struct rte_flow_item_vxlan *spec = item->spec; 820 const struct rte_flow_item_vxlan *mask = item->mask; 821 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 822 struct ibv_flow_spec_tunnel vxlan = { 823 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 824 .size = size, 825 }; 826 union vni { 827 uint32_t vlan_id; 828 uint8_t vni[4]; 829 } id = { .vlan_id = 0, }; 830 831 if (!mask) 832 mask = &rte_flow_item_vxlan_mask; 833 if (spec) { 834 memcpy(&id.vni[1], spec->vni, 3); 835 vxlan.val.tunnel_id = id.vlan_id; 836 memcpy(&id.vni[1], mask->vni, 3); 837 vxlan.mask.tunnel_id = id.vlan_id; 838 /* Remove unwanted bits from values. */ 839 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 840 } 841 flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size); 842 } 843 844 /** 845 * Convert the @p item into a Verbs specification. This function assumes that 846 * the input is valid and that there is space to insert the requested item 847 * into the flow. 848 * 849 * @param[in, out] dev_flow 850 * Pointer to dev_flow structure. 851 * @param[in] item 852 * Item specification. 853 * @param[in] item_flags 854 * Parsed item flags. 855 */ 856 static void 857 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow, 858 const struct rte_flow_item *item, 859 uint64_t item_flags __rte_unused) 860 { 861 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 862 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 863 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 864 struct ibv_flow_spec_tunnel vxlan_gpe = { 865 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 866 .size = size, 867 }; 868 union vni { 869 uint32_t vlan_id; 870 uint8_t vni[4]; 871 } id = { .vlan_id = 0, }; 872 873 if (!mask) 874 mask = &rte_flow_item_vxlan_gpe_mask; 875 if (spec) { 876 memcpy(&id.vni[1], spec->vni, 3); 877 vxlan_gpe.val.tunnel_id = id.vlan_id; 878 memcpy(&id.vni[1], mask->vni, 3); 879 vxlan_gpe.mask.tunnel_id = id.vlan_id; 880 /* Remove unwanted bits from values. */ 881 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id; 882 } 883 flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size); 884 } 885 886 /** 887 * Update the protocol in Verbs IPv4/IPv6 spec. 888 * 889 * @param[in, out] attr 890 * Pointer to Verbs attributes structure. 891 * @param[in] search 892 * Specification type to search in order to update the IP protocol. 893 * @param[in] protocol 894 * Protocol value to set if none is present in the specification. 895 */ 896 static void 897 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr, 898 enum ibv_flow_spec_type search, 899 uint8_t protocol) 900 { 901 unsigned int i; 902 struct ibv_spec_header *hdr = (struct ibv_spec_header *) 903 ((uint8_t *)attr + sizeof(struct ibv_flow_attr)); 904 905 if (!attr) 906 return; 907 for (i = 0; i != attr->num_of_specs; ++i) { 908 if (hdr->type == search) { 909 union { 910 struct ibv_flow_spec_ipv4_ext *ipv4; 911 struct ibv_flow_spec_ipv6 *ipv6; 912 } ip; 913 914 switch (search) { 915 case IBV_FLOW_SPEC_IPV4_EXT: 916 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr; 917 if (!ip.ipv4->val.proto) { 918 ip.ipv4->val.proto = protocol; 919 ip.ipv4->mask.proto = 0xff; 920 } 921 break; 922 case IBV_FLOW_SPEC_IPV6: 923 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr; 924 if (!ip.ipv6->val.next_hdr) { 925 ip.ipv6->val.next_hdr = protocol; 926 ip.ipv6->mask.next_hdr = 0xff; 927 } 928 break; 929 default: 930 break; 931 } 932 break; 933 } 934 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size); 935 } 936 } 937 938 /** 939 * Convert the @p item into a Verbs specification. This function assumes that 940 * the input is valid and that there is space to insert the requested item 941 * into the flow. 942 * 943 * @param[in, out] dev_flow 944 * Pointer to dev_flow structure. 945 * @param[in] item 946 * Item specification. 947 * @param[in] item_flags 948 * Parsed item flags. 949 */ 950 static void 951 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow, 952 const struct rte_flow_item *item __rte_unused, 953 uint64_t item_flags) 954 { 955 struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs; 956 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 957 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 958 struct ibv_flow_spec_tunnel tunnel = { 959 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 960 .size = size, 961 }; 962 #else 963 const struct rte_flow_item_gre *spec = item->spec; 964 const struct rte_flow_item_gre *mask = item->mask; 965 unsigned int size = sizeof(struct ibv_flow_spec_gre); 966 struct ibv_flow_spec_gre tunnel = { 967 .type = IBV_FLOW_SPEC_GRE, 968 .size = size, 969 }; 970 971 if (!mask) 972 mask = &rte_flow_item_gre_mask; 973 if (spec) { 974 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver; 975 tunnel.val.protocol = spec->protocol; 976 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver; 977 tunnel.mask.protocol = mask->protocol; 978 /* Remove unwanted bits from values. */ 979 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver; 980 tunnel.val.protocol &= tunnel.mask.protocol; 981 tunnel.val.key &= tunnel.mask.key; 982 } 983 #endif 984 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) 985 flow_verbs_item_gre_ip_protocol_update(&verbs->attr, 986 IBV_FLOW_SPEC_IPV4_EXT, 987 IPPROTO_GRE); 988 else 989 flow_verbs_item_gre_ip_protocol_update(&verbs->attr, 990 IBV_FLOW_SPEC_IPV6, 991 IPPROTO_GRE); 992 flow_verbs_spec_add(verbs, &tunnel, size); 993 } 994 995 /** 996 * Convert the @p action into a Verbs specification. This function assumes that 997 * the input is valid and that there is space to insert the requested action 998 * into the flow. This function also return the action that was added. 999 * 1000 * @param[in, out] dev_flow 1001 * Pointer to dev_flow structure. 1002 * @param[in] item 1003 * Item specification. 1004 * @param[in] item_flags 1005 * Parsed item flags. 1006 */ 1007 static void 1008 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused, 1009 const struct rte_flow_item *item __rte_unused, 1010 uint64_t item_flags __rte_unused) 1011 { 1012 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1013 const struct rte_flow_item_mpls *spec = item->spec; 1014 const struct rte_flow_item_mpls *mask = item->mask; 1015 unsigned int size = sizeof(struct ibv_flow_spec_mpls); 1016 struct ibv_flow_spec_mpls mpls = { 1017 .type = IBV_FLOW_SPEC_MPLS, 1018 .size = size, 1019 }; 1020 1021 if (!mask) 1022 mask = &rte_flow_item_mpls_mask; 1023 if (spec) { 1024 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label)); 1025 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label)); 1026 /* Remove unwanted bits from values. */ 1027 mpls.val.label &= mpls.mask.label; 1028 } 1029 flow_verbs_spec_add(&dev_flow->verbs, &mpls, size); 1030 #endif 1031 } 1032 1033 /** 1034 * Convert the @p action into a Verbs specification. This function assumes that 1035 * the input is valid and that there is space to insert the requested action 1036 * into the flow. 1037 * 1038 * @param[in] dev_flow 1039 * Pointer to mlx5_flow. 1040 * @param[in] action 1041 * Action configuration. 1042 */ 1043 static void 1044 flow_verbs_translate_action_drop 1045 (struct mlx5_flow *dev_flow, 1046 const struct rte_flow_action *action __rte_unused) 1047 { 1048 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1049 struct ibv_flow_spec_action_drop drop = { 1050 .type = IBV_FLOW_SPEC_ACTION_DROP, 1051 .size = size, 1052 }; 1053 1054 flow_verbs_spec_add(&dev_flow->verbs, &drop, size); 1055 } 1056 1057 /** 1058 * Convert the @p action into a Verbs specification. This function assumes that 1059 * the input is valid and that there is space to insert the requested action 1060 * into the flow. 1061 * 1062 * @param[in] rss_desc 1063 * Pointer to mlx5_flow_rss_desc. 1064 * @param[in] action 1065 * Action configuration. 1066 */ 1067 static void 1068 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc, 1069 const struct rte_flow_action *action) 1070 { 1071 const struct rte_flow_action_queue *queue = action->conf; 1072 1073 rss_desc->queue[0] = queue->index; 1074 rss_desc->queue_num = 1; 1075 } 1076 1077 /** 1078 * Convert the @p action into a Verbs specification. This function assumes that 1079 * the input is valid and that there is space to insert the requested action 1080 * into the flow. 1081 * 1082 * @param[in] rss_desc 1083 * Pointer to mlx5_flow_rss_desc. 1084 * @param[in] action 1085 * Action configuration. 1086 */ 1087 static void 1088 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc, 1089 const struct rte_flow_action *action) 1090 { 1091 const struct rte_flow_action_rss *rss = action->conf; 1092 const uint8_t *rss_key; 1093 1094 memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t)); 1095 rss_desc->queue_num = rss->queue_num; 1096 /* NULL RSS key indicates default RSS key. */ 1097 rss_key = !rss->key ? rss_hash_default_key : rss->key; 1098 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 1099 /* 1100 * rss->level and rss.types should be set in advance when expanding 1101 * items for RSS. 1102 */ 1103 } 1104 1105 /** 1106 * Convert the @p action into a Verbs specification. This function assumes that 1107 * the input is valid and that there is space to insert the requested action 1108 * into the flow. 1109 * 1110 * @param[in] dev_flow 1111 * Pointer to mlx5_flow. 1112 * @param[in] action 1113 * Action configuration. 1114 */ 1115 static void 1116 flow_verbs_translate_action_flag 1117 (struct mlx5_flow *dev_flow, 1118 const struct rte_flow_action *action __rte_unused) 1119 { 1120 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1121 struct ibv_flow_spec_action_tag tag = { 1122 .type = IBV_FLOW_SPEC_ACTION_TAG, 1123 .size = size, 1124 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT), 1125 }; 1126 1127 flow_verbs_spec_add(&dev_flow->verbs, &tag, size); 1128 } 1129 1130 /** 1131 * Convert the @p action into a Verbs specification. This function assumes that 1132 * the input is valid and that there is space to insert the requested action 1133 * into the flow. 1134 * 1135 * @param[in] dev_flow 1136 * Pointer to mlx5_flow. 1137 * @param[in] action 1138 * Action configuration. 1139 */ 1140 static void 1141 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow, 1142 const struct rte_flow_action *action) 1143 { 1144 const struct rte_flow_action_mark *mark = action->conf; 1145 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1146 struct ibv_flow_spec_action_tag tag = { 1147 .type = IBV_FLOW_SPEC_ACTION_TAG, 1148 .size = size, 1149 .tag_id = mlx5_flow_mark_set(mark->id), 1150 }; 1151 1152 flow_verbs_spec_add(&dev_flow->verbs, &tag, size); 1153 } 1154 1155 /** 1156 * Convert the @p action into a Verbs specification. This function assumes that 1157 * the input is valid and that there is space to insert the requested action 1158 * into the flow. 1159 * 1160 * @param[in] dev 1161 * Pointer to the Ethernet device structure. 1162 * @param[in] action 1163 * Action configuration. 1164 * @param[in] dev_flow 1165 * Pointer to mlx5_flow. 1166 * @param[out] error 1167 * Pointer to error structure. 1168 * 1169 * @return 1170 * 0 On success else a negative errno value is returned and rte_errno is set. 1171 */ 1172 static int 1173 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, 1174 const struct rte_flow_action *action, 1175 struct rte_eth_dev *dev, 1176 struct rte_flow_error *error) 1177 { 1178 const struct rte_flow_action_count *count = action->conf; 1179 struct rte_flow *flow = dev_flow->flow; 1180 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 1181 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1182 struct mlx5_flow_counter_pool *pool; 1183 struct mlx5_flow_counter *cnt = NULL; 1184 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1185 struct ibv_flow_spec_counter_action counter = { 1186 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1187 .size = size, 1188 }; 1189 #endif 1190 1191 if (!flow->counter) { 1192 flow->counter = flow_verbs_counter_new(dev, count->id); 1193 if (!flow->counter) 1194 return rte_flow_error_set(error, rte_errno, 1195 RTE_FLOW_ERROR_TYPE_ACTION, 1196 action, 1197 "cannot get counter" 1198 " context."); 1199 } 1200 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 1201 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); 1202 counter.counter_set_handle = 1203 ((struct ibv_counter_set *)cnt->dcs_when_active)->handle; 1204 flow_verbs_spec_add(&dev_flow->verbs, &counter, size); 1205 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1206 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); 1207 counter.counters = (struct ibv_counters *)cnt->dcs_when_active; 1208 flow_verbs_spec_add(&dev_flow->verbs, &counter, size); 1209 #endif 1210 return 0; 1211 } 1212 1213 /** 1214 * Internal validation function. For validating both actions and items. 1215 * 1216 * @param[in] dev 1217 * Pointer to the Ethernet device structure. 1218 * @param[in] attr 1219 * Pointer to the flow attributes. 1220 * @param[in] items 1221 * Pointer to the list of items. 1222 * @param[in] actions 1223 * Pointer to the list of actions. 1224 * @param[in] external 1225 * This flow rule is created by request external to PMD. 1226 * @param[in] hairpin 1227 * Number of hairpin TX actions, 0 means classic flow. 1228 * @param[out] error 1229 * Pointer to the error structure. 1230 * 1231 * @return 1232 * 0 on success, a negative errno value otherwise and rte_errno is set. 1233 */ 1234 static int 1235 flow_verbs_validate(struct rte_eth_dev *dev, 1236 const struct rte_flow_attr *attr, 1237 const struct rte_flow_item items[], 1238 const struct rte_flow_action actions[], 1239 bool external __rte_unused, 1240 int hairpin __rte_unused, 1241 struct rte_flow_error *error) 1242 { 1243 int ret; 1244 uint64_t action_flags = 0; 1245 uint64_t item_flags = 0; 1246 uint64_t last_item = 0; 1247 uint8_t next_protocol = 0xff; 1248 uint16_t ether_type = 0; 1249 bool is_empty_vlan = false; 1250 uint16_t udp_dport = 0; 1251 1252 if (items == NULL) 1253 return -1; 1254 ret = mlx5_flow_validate_attributes(dev, attr, error); 1255 if (ret < 0) 1256 return ret; 1257 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1258 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1259 int ret = 0; 1260 1261 switch (items->type) { 1262 case RTE_FLOW_ITEM_TYPE_VOID: 1263 break; 1264 case RTE_FLOW_ITEM_TYPE_ETH: 1265 ret = mlx5_flow_validate_item_eth(items, item_flags, 1266 false, error); 1267 if (ret < 0) 1268 return ret; 1269 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1270 MLX5_FLOW_LAYER_OUTER_L2; 1271 if (items->mask != NULL && items->spec != NULL) { 1272 ether_type = 1273 ((const struct rte_flow_item_eth *) 1274 items->spec)->type; 1275 ether_type &= 1276 ((const struct rte_flow_item_eth *) 1277 items->mask)->type; 1278 if (ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN)) 1279 is_empty_vlan = true; 1280 ether_type = rte_be_to_cpu_16(ether_type); 1281 } else { 1282 ether_type = 0; 1283 } 1284 break; 1285 case RTE_FLOW_ITEM_TYPE_VLAN: 1286 ret = mlx5_flow_validate_item_vlan(items, item_flags, 1287 dev, error); 1288 if (ret < 0) 1289 return ret; 1290 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 1291 MLX5_FLOW_LAYER_INNER_VLAN) : 1292 (MLX5_FLOW_LAYER_OUTER_L2 | 1293 MLX5_FLOW_LAYER_OUTER_VLAN); 1294 if (items->mask != NULL && items->spec != NULL) { 1295 ether_type = 1296 ((const struct rte_flow_item_vlan *) 1297 items->spec)->inner_type; 1298 ether_type &= 1299 ((const struct rte_flow_item_vlan *) 1300 items->mask)->inner_type; 1301 ether_type = rte_be_to_cpu_16(ether_type); 1302 } else { 1303 ether_type = 0; 1304 } 1305 is_empty_vlan = false; 1306 break; 1307 case RTE_FLOW_ITEM_TYPE_IPV4: 1308 ret = mlx5_flow_validate_item_ipv4 1309 (items, item_flags, 1310 last_item, ether_type, NULL, 1311 MLX5_ITEM_RANGE_NOT_ACCEPTED, 1312 error); 1313 if (ret < 0) 1314 return ret; 1315 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1316 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1317 if (items->mask != NULL && 1318 ((const struct rte_flow_item_ipv4 *) 1319 items->mask)->hdr.next_proto_id) { 1320 next_protocol = 1321 ((const struct rte_flow_item_ipv4 *) 1322 (items->spec))->hdr.next_proto_id; 1323 next_protocol &= 1324 ((const struct rte_flow_item_ipv4 *) 1325 (items->mask))->hdr.next_proto_id; 1326 } else { 1327 /* Reset for inner layer. */ 1328 next_protocol = 0xff; 1329 } 1330 break; 1331 case RTE_FLOW_ITEM_TYPE_IPV6: 1332 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 1333 last_item, 1334 ether_type, NULL, 1335 error); 1336 if (ret < 0) 1337 return ret; 1338 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1339 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1340 if (items->mask != NULL && 1341 ((const struct rte_flow_item_ipv6 *) 1342 items->mask)->hdr.proto) { 1343 next_protocol = 1344 ((const struct rte_flow_item_ipv6 *) 1345 items->spec)->hdr.proto; 1346 next_protocol &= 1347 ((const struct rte_flow_item_ipv6 *) 1348 items->mask)->hdr.proto; 1349 } else { 1350 /* Reset for inner layer. */ 1351 next_protocol = 0xff; 1352 } 1353 break; 1354 case RTE_FLOW_ITEM_TYPE_UDP: 1355 ret = mlx5_flow_validate_item_udp(items, item_flags, 1356 next_protocol, 1357 error); 1358 const struct rte_flow_item_udp *spec = items->spec; 1359 const struct rte_flow_item_udp *mask = items->mask; 1360 if (!mask) 1361 mask = &rte_flow_item_udp_mask; 1362 if (spec != NULL) 1363 udp_dport = rte_be_to_cpu_16 1364 (spec->hdr.dst_port & 1365 mask->hdr.dst_port); 1366 1367 if (ret < 0) 1368 return ret; 1369 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 1370 MLX5_FLOW_LAYER_OUTER_L4_UDP; 1371 break; 1372 case RTE_FLOW_ITEM_TYPE_TCP: 1373 ret = mlx5_flow_validate_item_tcp 1374 (items, item_flags, 1375 next_protocol, 1376 &rte_flow_item_tcp_mask, 1377 error); 1378 if (ret < 0) 1379 return ret; 1380 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 1381 MLX5_FLOW_LAYER_OUTER_L4_TCP; 1382 break; 1383 case RTE_FLOW_ITEM_TYPE_VXLAN: 1384 ret = mlx5_flow_validate_item_vxlan(dev, udp_dport, 1385 items, item_flags, 1386 attr, error); 1387 if (ret < 0) 1388 return ret; 1389 last_item = MLX5_FLOW_LAYER_VXLAN; 1390 break; 1391 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1392 ret = mlx5_flow_validate_item_vxlan_gpe(items, 1393 item_flags, 1394 dev, error); 1395 if (ret < 0) 1396 return ret; 1397 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 1398 break; 1399 case RTE_FLOW_ITEM_TYPE_GRE: 1400 ret = mlx5_flow_validate_item_gre(items, item_flags, 1401 next_protocol, error); 1402 if (ret < 0) 1403 return ret; 1404 last_item = MLX5_FLOW_LAYER_GRE; 1405 break; 1406 case RTE_FLOW_ITEM_TYPE_MPLS: 1407 ret = mlx5_flow_validate_item_mpls(dev, items, 1408 item_flags, 1409 last_item, error); 1410 if (ret < 0) 1411 return ret; 1412 last_item = MLX5_FLOW_LAYER_MPLS; 1413 break; 1414 case RTE_FLOW_ITEM_TYPE_ICMP: 1415 case RTE_FLOW_ITEM_TYPE_ICMP6: 1416 return rte_flow_error_set(error, ENOTSUP, 1417 RTE_FLOW_ERROR_TYPE_ITEM, 1418 NULL, "ICMP/ICMP6 " 1419 "item not supported"); 1420 default: 1421 return rte_flow_error_set(error, ENOTSUP, 1422 RTE_FLOW_ERROR_TYPE_ITEM, 1423 NULL, "item not supported"); 1424 } 1425 item_flags |= last_item; 1426 } 1427 if (is_empty_vlan) 1428 return rte_flow_error_set(error, ENOTSUP, 1429 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1430 "VLAN matching without vid specification is not supported"); 1431 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1432 switch (actions->type) { 1433 case RTE_FLOW_ACTION_TYPE_VOID: 1434 break; 1435 case RTE_FLOW_ACTION_TYPE_FLAG: 1436 ret = mlx5_flow_validate_action_flag(action_flags, 1437 attr, 1438 error); 1439 if (ret < 0) 1440 return ret; 1441 action_flags |= MLX5_FLOW_ACTION_FLAG; 1442 break; 1443 case RTE_FLOW_ACTION_TYPE_MARK: 1444 ret = mlx5_flow_validate_action_mark(actions, 1445 action_flags, 1446 attr, 1447 error); 1448 if (ret < 0) 1449 return ret; 1450 action_flags |= MLX5_FLOW_ACTION_MARK; 1451 break; 1452 case RTE_FLOW_ACTION_TYPE_DROP: 1453 ret = mlx5_flow_validate_action_drop(action_flags, 1454 attr, 1455 error); 1456 if (ret < 0) 1457 return ret; 1458 action_flags |= MLX5_FLOW_ACTION_DROP; 1459 break; 1460 case RTE_FLOW_ACTION_TYPE_QUEUE: 1461 ret = mlx5_flow_validate_action_queue(actions, 1462 action_flags, dev, 1463 attr, 1464 error); 1465 if (ret < 0) 1466 return ret; 1467 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1468 break; 1469 case RTE_FLOW_ACTION_TYPE_RSS: 1470 ret = mlx5_flow_validate_action_rss(actions, 1471 action_flags, dev, 1472 attr, item_flags, 1473 error); 1474 if (ret < 0) 1475 return ret; 1476 action_flags |= MLX5_FLOW_ACTION_RSS; 1477 break; 1478 case RTE_FLOW_ACTION_TYPE_COUNT: 1479 ret = mlx5_flow_validate_action_count(dev, attr, error); 1480 if (ret < 0) 1481 return ret; 1482 action_flags |= MLX5_FLOW_ACTION_COUNT; 1483 break; 1484 default: 1485 return rte_flow_error_set(error, ENOTSUP, 1486 RTE_FLOW_ERROR_TYPE_ACTION, 1487 actions, 1488 "action not supported"); 1489 } 1490 } 1491 /* 1492 * Validate the drop action mutual exclusion with other actions. 1493 * Drop action is mutually-exclusive with any other action, except for 1494 * Count action. 1495 */ 1496 if ((action_flags & MLX5_FLOW_ACTION_DROP) && 1497 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) 1498 return rte_flow_error_set(error, EINVAL, 1499 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1500 "Drop action is mutually-exclusive " 1501 "with any other action, except for " 1502 "Count action"); 1503 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS)) 1504 return rte_flow_error_set(error, EINVAL, 1505 RTE_FLOW_ERROR_TYPE_ACTION, actions, 1506 "no fate action is found"); 1507 return 0; 1508 } 1509 1510 /** 1511 * Calculate the required bytes that are needed for the action part of the verbs 1512 * flow. 1513 * 1514 * @param[in] actions 1515 * Pointer to the list of actions. 1516 * 1517 * @return 1518 * The size of the memory needed for all actions. 1519 */ 1520 static int 1521 flow_verbs_get_actions_size(const struct rte_flow_action actions[]) 1522 { 1523 int size = 0; 1524 1525 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1526 switch (actions->type) { 1527 case RTE_FLOW_ACTION_TYPE_VOID: 1528 break; 1529 case RTE_FLOW_ACTION_TYPE_FLAG: 1530 size += sizeof(struct ibv_flow_spec_action_tag); 1531 break; 1532 case RTE_FLOW_ACTION_TYPE_MARK: 1533 size += sizeof(struct ibv_flow_spec_action_tag); 1534 break; 1535 case RTE_FLOW_ACTION_TYPE_DROP: 1536 size += sizeof(struct ibv_flow_spec_action_drop); 1537 break; 1538 case RTE_FLOW_ACTION_TYPE_QUEUE: 1539 break; 1540 case RTE_FLOW_ACTION_TYPE_RSS: 1541 break; 1542 case RTE_FLOW_ACTION_TYPE_COUNT: 1543 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 1544 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1545 size += sizeof(struct ibv_flow_spec_counter_action); 1546 #endif 1547 break; 1548 default: 1549 break; 1550 } 1551 } 1552 return size; 1553 } 1554 1555 /** 1556 * Calculate the required bytes that are needed for the item part of the verbs 1557 * flow. 1558 * 1559 * @param[in] items 1560 * Pointer to the list of items. 1561 * 1562 * @return 1563 * The size of the memory needed for all items. 1564 */ 1565 static int 1566 flow_verbs_get_items_size(const struct rte_flow_item items[]) 1567 { 1568 int size = 0; 1569 1570 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1571 switch (items->type) { 1572 case RTE_FLOW_ITEM_TYPE_VOID: 1573 break; 1574 case RTE_FLOW_ITEM_TYPE_ETH: 1575 size += sizeof(struct ibv_flow_spec_eth); 1576 break; 1577 case RTE_FLOW_ITEM_TYPE_VLAN: 1578 size += sizeof(struct ibv_flow_spec_eth); 1579 break; 1580 case RTE_FLOW_ITEM_TYPE_IPV4: 1581 size += sizeof(struct ibv_flow_spec_ipv4_ext); 1582 break; 1583 case RTE_FLOW_ITEM_TYPE_IPV6: 1584 size += sizeof(struct ibv_flow_spec_ipv6); 1585 break; 1586 case RTE_FLOW_ITEM_TYPE_UDP: 1587 size += sizeof(struct ibv_flow_spec_tcp_udp); 1588 break; 1589 case RTE_FLOW_ITEM_TYPE_TCP: 1590 size += sizeof(struct ibv_flow_spec_tcp_udp); 1591 break; 1592 case RTE_FLOW_ITEM_TYPE_VXLAN: 1593 size += sizeof(struct ibv_flow_spec_tunnel); 1594 break; 1595 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1596 size += sizeof(struct ibv_flow_spec_tunnel); 1597 break; 1598 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1599 case RTE_FLOW_ITEM_TYPE_GRE: 1600 size += sizeof(struct ibv_flow_spec_gre); 1601 break; 1602 case RTE_FLOW_ITEM_TYPE_MPLS: 1603 size += sizeof(struct ibv_flow_spec_mpls); 1604 break; 1605 #else 1606 case RTE_FLOW_ITEM_TYPE_GRE: 1607 size += sizeof(struct ibv_flow_spec_tunnel); 1608 break; 1609 #endif 1610 default: 1611 break; 1612 } 1613 } 1614 return size; 1615 } 1616 1617 /** 1618 * Internal preparation function. Allocate mlx5_flow with the required size. 1619 * The required size is calculate based on the actions and items. This function 1620 * also returns the detected actions and items for later use. 1621 * 1622 * @param[in] dev 1623 * Pointer to Ethernet device. 1624 * @param[in] attr 1625 * Pointer to the flow attributes. 1626 * @param[in] items 1627 * Pointer to the list of items. 1628 * @param[in] actions 1629 * Pointer to the list of actions. 1630 * @param[out] error 1631 * Pointer to the error structure. 1632 * 1633 * @return 1634 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno 1635 * is set. 1636 */ 1637 static struct mlx5_flow * 1638 flow_verbs_prepare(struct rte_eth_dev *dev, 1639 const struct rte_flow_attr *attr __rte_unused, 1640 const struct rte_flow_item items[], 1641 const struct rte_flow_action actions[], 1642 struct rte_flow_error *error) 1643 { 1644 size_t size = 0; 1645 uint32_t handle_idx = 0; 1646 struct mlx5_flow *dev_flow; 1647 struct mlx5_flow_handle *dev_handle; 1648 struct mlx5_priv *priv = dev->data->dev_private; 1649 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1650 1651 MLX5_ASSERT(wks); 1652 size += flow_verbs_get_actions_size(actions); 1653 size += flow_verbs_get_items_size(items); 1654 if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) { 1655 rte_flow_error_set(error, E2BIG, 1656 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1657 "Verbs spec/action size too large"); 1658 return NULL; 1659 } 1660 /* In case of corrupting the memory. */ 1661 if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { 1662 rte_flow_error_set(error, ENOSPC, 1663 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1664 "not free temporary device flow"); 1665 return NULL; 1666 } 1667 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1668 &handle_idx); 1669 if (!dev_handle) { 1670 rte_flow_error_set(error, ENOMEM, 1671 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1672 "not enough memory to create flow handle"); 1673 return NULL; 1674 } 1675 MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows)); 1676 dev_flow = &wks->flows[wks->flow_idx++]; 1677 dev_flow->handle = dev_handle; 1678 dev_flow->handle_idx = handle_idx; 1679 /* Memcpy is used, only size needs to be cleared to 0. */ 1680 dev_flow->verbs.size = 0; 1681 dev_flow->verbs.attr.num_of_specs = 0; 1682 dev_flow->ingress = attr->ingress; 1683 dev_flow->hash_fields = 0; 1684 /* Need to set transfer attribute: not supported in Verbs mode. */ 1685 return dev_flow; 1686 } 1687 1688 /** 1689 * Fill the flow with verb spec. 1690 * 1691 * @param[in] dev 1692 * Pointer to Ethernet device. 1693 * @param[in, out] dev_flow 1694 * Pointer to the mlx5 flow. 1695 * @param[in] attr 1696 * Pointer to the flow attributes. 1697 * @param[in] items 1698 * Pointer to the list of items. 1699 * @param[in] actions 1700 * Pointer to the list of actions. 1701 * @param[out] error 1702 * Pointer to the error structure. 1703 * 1704 * @return 1705 * 0 on success, else a negative errno value otherwise and rte_errno is set. 1706 */ 1707 static int 1708 flow_verbs_translate(struct rte_eth_dev *dev, 1709 struct mlx5_flow *dev_flow, 1710 const struct rte_flow_attr *attr, 1711 const struct rte_flow_item items[], 1712 const struct rte_flow_action actions[], 1713 struct rte_flow_error *error) 1714 { 1715 uint64_t item_flags = 0; 1716 uint64_t action_flags = 0; 1717 uint64_t priority = attr->priority; 1718 uint32_t subpriority = 0; 1719 struct mlx5_priv *priv = dev->data->dev_private; 1720 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1721 struct mlx5_flow_rss_desc *rss_desc; 1722 1723 MLX5_ASSERT(wks); 1724 rss_desc = &wks->rss_desc; 1725 if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1726 priority = priv->config.flow_prio - 1; 1727 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1728 int ret; 1729 1730 switch (actions->type) { 1731 case RTE_FLOW_ACTION_TYPE_VOID: 1732 break; 1733 case RTE_FLOW_ACTION_TYPE_FLAG: 1734 flow_verbs_translate_action_flag(dev_flow, actions); 1735 action_flags |= MLX5_FLOW_ACTION_FLAG; 1736 dev_flow->handle->mark = 1; 1737 break; 1738 case RTE_FLOW_ACTION_TYPE_MARK: 1739 flow_verbs_translate_action_mark(dev_flow, actions); 1740 action_flags |= MLX5_FLOW_ACTION_MARK; 1741 dev_flow->handle->mark = 1; 1742 break; 1743 case RTE_FLOW_ACTION_TYPE_DROP: 1744 flow_verbs_translate_action_drop(dev_flow, actions); 1745 action_flags |= MLX5_FLOW_ACTION_DROP; 1746 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; 1747 break; 1748 case RTE_FLOW_ACTION_TYPE_QUEUE: 1749 flow_verbs_translate_action_queue(rss_desc, actions); 1750 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1751 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 1752 break; 1753 case RTE_FLOW_ACTION_TYPE_RSS: 1754 flow_verbs_translate_action_rss(rss_desc, actions); 1755 action_flags |= MLX5_FLOW_ACTION_RSS; 1756 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 1757 break; 1758 case RTE_FLOW_ACTION_TYPE_COUNT: 1759 ret = flow_verbs_translate_action_count(dev_flow, 1760 actions, 1761 dev, error); 1762 if (ret < 0) 1763 return ret; 1764 action_flags |= MLX5_FLOW_ACTION_COUNT; 1765 break; 1766 default: 1767 return rte_flow_error_set(error, ENOTSUP, 1768 RTE_FLOW_ERROR_TYPE_ACTION, 1769 actions, 1770 "action not supported"); 1771 } 1772 } 1773 dev_flow->act_flags = action_flags; 1774 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1775 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1776 1777 switch (items->type) { 1778 case RTE_FLOW_ITEM_TYPE_VOID: 1779 break; 1780 case RTE_FLOW_ITEM_TYPE_ETH: 1781 flow_verbs_translate_item_eth(dev_flow, items, 1782 item_flags); 1783 subpriority = MLX5_PRIORITY_MAP_L2; 1784 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1785 MLX5_FLOW_LAYER_OUTER_L2; 1786 break; 1787 case RTE_FLOW_ITEM_TYPE_VLAN: 1788 flow_verbs_translate_item_vlan(dev_flow, items, 1789 item_flags); 1790 subpriority = MLX5_PRIORITY_MAP_L2; 1791 item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 1792 MLX5_FLOW_LAYER_INNER_VLAN) : 1793 (MLX5_FLOW_LAYER_OUTER_L2 | 1794 MLX5_FLOW_LAYER_OUTER_VLAN); 1795 break; 1796 case RTE_FLOW_ITEM_TYPE_IPV4: 1797 flow_verbs_translate_item_ipv4(dev_flow, items, 1798 item_flags); 1799 subpriority = MLX5_PRIORITY_MAP_L3; 1800 dev_flow->hash_fields |= 1801 mlx5_flow_hashfields_adjust 1802 (rss_desc, tunnel, 1803 MLX5_IPV4_LAYER_TYPES, 1804 MLX5_IPV4_IBV_RX_HASH); 1805 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1806 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1807 break; 1808 case RTE_FLOW_ITEM_TYPE_IPV6: 1809 flow_verbs_translate_item_ipv6(dev_flow, items, 1810 item_flags); 1811 subpriority = MLX5_PRIORITY_MAP_L3; 1812 dev_flow->hash_fields |= 1813 mlx5_flow_hashfields_adjust 1814 (rss_desc, tunnel, 1815 MLX5_IPV6_LAYER_TYPES, 1816 MLX5_IPV6_IBV_RX_HASH); 1817 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1818 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1819 break; 1820 case RTE_FLOW_ITEM_TYPE_TCP: 1821 flow_verbs_translate_item_tcp(dev_flow, items, 1822 item_flags); 1823 subpriority = MLX5_PRIORITY_MAP_L4; 1824 if (dev_flow->hash_fields != 0) 1825 dev_flow->hash_fields |= 1826 mlx5_flow_hashfields_adjust 1827 (rss_desc, tunnel, RTE_ETH_RSS_TCP, 1828 (IBV_RX_HASH_SRC_PORT_TCP | 1829 IBV_RX_HASH_DST_PORT_TCP)); 1830 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 1831 MLX5_FLOW_LAYER_OUTER_L4_TCP; 1832 break; 1833 case RTE_FLOW_ITEM_TYPE_UDP: 1834 flow_verbs_translate_item_udp(dev_flow, items, 1835 item_flags); 1836 subpriority = MLX5_PRIORITY_MAP_L4; 1837 if (dev_flow->hash_fields != 0) 1838 dev_flow->hash_fields |= 1839 mlx5_flow_hashfields_adjust 1840 (rss_desc, tunnel, RTE_ETH_RSS_UDP, 1841 (IBV_RX_HASH_SRC_PORT_UDP | 1842 IBV_RX_HASH_DST_PORT_UDP)); 1843 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 1844 MLX5_FLOW_LAYER_OUTER_L4_UDP; 1845 break; 1846 case RTE_FLOW_ITEM_TYPE_VXLAN: 1847 flow_verbs_translate_item_vxlan(dev_flow, items, 1848 item_flags); 1849 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1850 item_flags |= MLX5_FLOW_LAYER_VXLAN; 1851 break; 1852 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1853 flow_verbs_translate_item_vxlan_gpe(dev_flow, items, 1854 item_flags); 1855 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1856 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE; 1857 break; 1858 case RTE_FLOW_ITEM_TYPE_GRE: 1859 flow_verbs_translate_item_gre(dev_flow, items, 1860 item_flags); 1861 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1862 item_flags |= MLX5_FLOW_LAYER_GRE; 1863 break; 1864 case RTE_FLOW_ITEM_TYPE_MPLS: 1865 flow_verbs_translate_item_mpls(dev_flow, items, 1866 item_flags); 1867 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1868 item_flags |= MLX5_FLOW_LAYER_MPLS; 1869 break; 1870 default: 1871 return rte_flow_error_set(error, ENOTSUP, 1872 RTE_FLOW_ERROR_TYPE_ITEM, 1873 NULL, "item not supported"); 1874 } 1875 } 1876 dev_flow->handle->layers = item_flags; 1877 /* Other members of attr will be ignored. */ 1878 dev_flow->verbs.attr.priority = 1879 mlx5_flow_adjust_priority(dev, priority, subpriority); 1880 dev_flow->verbs.attr.port = (uint8_t)priv->dev_port; 1881 return 0; 1882 } 1883 1884 /** 1885 * Remove the flow from the NIC but keeps it in memory. 1886 * 1887 * @param[in] dev 1888 * Pointer to the Ethernet device structure. 1889 * @param[in, out] flow 1890 * Pointer to flow structure. 1891 */ 1892 static void 1893 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 1894 { 1895 struct mlx5_priv *priv = dev->data->dev_private; 1896 struct mlx5_flow_handle *handle; 1897 uint32_t handle_idx; 1898 1899 if (!flow) 1900 return; 1901 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1902 handle_idx, handle, next) { 1903 if (handle->drv_flow) { 1904 claim_zero(mlx5_glue->destroy_flow(handle->drv_flow)); 1905 handle->drv_flow = NULL; 1906 } 1907 /* hrxq is union, don't touch it only the flag is set. */ 1908 if (handle->rix_hrxq && 1909 handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1910 mlx5_hrxq_release(dev, handle->rix_hrxq); 1911 handle->rix_hrxq = 0; 1912 } 1913 if (handle->vf_vlan.tag && handle->vf_vlan.created) 1914 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); 1915 } 1916 } 1917 1918 /** 1919 * Remove the flow from the NIC and the memory. 1920 * 1921 * @param[in] dev 1922 * Pointer to the Ethernet device structure. 1923 * @param[in, out] flow 1924 * Pointer to flow structure. 1925 */ 1926 static void 1927 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 1928 { 1929 struct mlx5_priv *priv = dev->data->dev_private; 1930 struct mlx5_flow_handle *handle; 1931 1932 if (!flow) 1933 return; 1934 flow_verbs_remove(dev, flow); 1935 while (flow->dev_handles) { 1936 uint32_t tmp_idx = flow->dev_handles; 1937 1938 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1939 tmp_idx); 1940 if (!handle) 1941 return; 1942 flow->dev_handles = handle->next.next; 1943 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1944 tmp_idx); 1945 } 1946 if (flow->counter) { 1947 flow_verbs_counter_release(dev, flow->counter); 1948 flow->counter = 0; 1949 } 1950 } 1951 1952 /** 1953 * Apply the flow to the NIC. 1954 * 1955 * @param[in] dev 1956 * Pointer to the Ethernet device structure. 1957 * @param[in, out] flow 1958 * Pointer to flow structure. 1959 * @param[out] error 1960 * Pointer to error structure. 1961 * 1962 * @return 1963 * 0 on success, a negative errno value otherwise and rte_errno is set. 1964 */ 1965 static int 1966 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 1967 struct rte_flow_error *error) 1968 { 1969 struct mlx5_priv *priv = dev->data->dev_private; 1970 struct mlx5_flow_handle *handle; 1971 struct mlx5_flow *dev_flow; 1972 struct mlx5_hrxq *hrxq; 1973 uint32_t dev_handles; 1974 int err; 1975 int idx; 1976 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1977 1978 MLX5_ASSERT(wks); 1979 for (idx = wks->flow_idx - 1; idx >= 0; idx--) { 1980 dev_flow = &wks->flows[idx]; 1981 handle = dev_flow->handle; 1982 if (handle->fate_action == MLX5_FLOW_FATE_DROP) { 1983 MLX5_ASSERT(priv->drop_queue.hrxq); 1984 hrxq = priv->drop_queue.hrxq; 1985 } else { 1986 uint32_t hrxq_idx; 1987 struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc; 1988 1989 MLX5_ASSERT(rss_desc->queue_num); 1990 rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; 1991 rss_desc->hash_fields = dev_flow->hash_fields; 1992 rss_desc->tunnel = !!(handle->layers & 1993 MLX5_FLOW_LAYER_TUNNEL); 1994 rss_desc->shared_rss = 0; 1995 hrxq_idx = mlx5_hrxq_get(dev, rss_desc); 1996 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1997 hrxq_idx); 1998 if (!hrxq) { 1999 rte_flow_error_set 2000 (error, rte_errno, 2001 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2002 "cannot get hash queue"); 2003 goto error; 2004 } 2005 handle->rix_hrxq = hrxq_idx; 2006 } 2007 MLX5_ASSERT(hrxq); 2008 handle->drv_flow = mlx5_glue->create_flow 2009 (hrxq->qp, &dev_flow->verbs.attr); 2010 if (!handle->drv_flow) { 2011 rte_flow_error_set(error, errno, 2012 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2013 NULL, 2014 "hardware refuses to create flow"); 2015 goto error; 2016 } 2017 if (priv->vmwa_context && 2018 handle->vf_vlan.tag && !handle->vf_vlan.created) { 2019 /* 2020 * The rule contains the VLAN pattern. 2021 * For VF we are going to create VLAN 2022 * interface to make hypervisor set correct 2023 * e-Switch vport context. 2024 */ 2025 mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan); 2026 } 2027 } 2028 return 0; 2029 error: 2030 err = rte_errno; /* Save rte_errno before cleanup. */ 2031 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2032 dev_handles, handle, next) { 2033 /* hrxq is union, don't touch it only the flag is set. */ 2034 if (handle->rix_hrxq && 2035 handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 2036 mlx5_hrxq_release(dev, handle->rix_hrxq); 2037 handle->rix_hrxq = 0; 2038 } 2039 if (handle->vf_vlan.tag && handle->vf_vlan.created) 2040 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); 2041 } 2042 rte_errno = err; /* Restore rte_errno. */ 2043 return -rte_errno; 2044 } 2045 2046 /** 2047 * Query a flow. 2048 * 2049 * @see rte_flow_query() 2050 * @see rte_flow_ops 2051 */ 2052 static int 2053 flow_verbs_query(struct rte_eth_dev *dev, 2054 struct rte_flow *flow, 2055 const struct rte_flow_action *actions, 2056 void *data, 2057 struct rte_flow_error *error) 2058 { 2059 int ret = -EINVAL; 2060 2061 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2062 switch (actions->type) { 2063 case RTE_FLOW_ACTION_TYPE_VOID: 2064 break; 2065 case RTE_FLOW_ACTION_TYPE_COUNT: 2066 ret = flow_verbs_counter_query(dev, flow, data, error); 2067 break; 2068 default: 2069 return rte_flow_error_set(error, ENOTSUP, 2070 RTE_FLOW_ERROR_TYPE_ACTION, 2071 actions, 2072 "action not supported"); 2073 } 2074 } 2075 return ret; 2076 } 2077 2078 static int 2079 flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains, 2080 uint32_t flags) 2081 { 2082 RTE_SET_USED(dev); 2083 RTE_SET_USED(domains); 2084 RTE_SET_USED(flags); 2085 2086 return 0; 2087 } 2088 2089 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = { 2090 .validate = flow_verbs_validate, 2091 .prepare = flow_verbs_prepare, 2092 .translate = flow_verbs_translate, 2093 .apply = flow_verbs_apply, 2094 .remove = flow_verbs_remove, 2095 .destroy = flow_verbs_destroy, 2096 .query = flow_verbs_query, 2097 .sync_domain = flow_verbs_sync_domain, 2098 }; 2099