1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <netinet/in.h> 6 #include <sys/queue.h> 7 #include <stdalign.h> 8 #include <stdint.h> 9 #include <string.h> 10 11 #include <rte_common.h> 12 #include <rte_ether.h> 13 #include <ethdev_driver.h> 14 #include <rte_flow.h> 15 #include <rte_flow_driver.h> 16 #include <rte_malloc.h> 17 #include <rte_ip.h> 18 19 #include <mlx5_glue.h> 20 #include <mlx5_prm.h> 21 #include <mlx5_malloc.h> 22 23 #include "mlx5_defs.h" 24 #include "mlx5.h" 25 #include "mlx5_flow.h" 26 #include "mlx5_rx.h" 27 28 #define VERBS_SPEC_INNER(item_flags) \ 29 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0) 30 31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 33 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 34 }; 35 36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 38 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 39 { 9, 10, 11 }, { 12, 13, 14 }, 40 }; 41 42 /* Verbs specification header. */ 43 struct ibv_spec_header { 44 enum ibv_flow_spec_type type; 45 uint16_t size; 46 }; 47 48 /** 49 * Discover the maximum number of priority available. 50 * 51 * @param[in] dev 52 * Pointer to the Ethernet device structure. 53 * 54 * @return 55 * number of supported flow priority on success, a negative errno 56 * value otherwise and rte_errno is set. 57 */ 58 int 59 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 60 { 61 struct mlx5_priv *priv = dev->data->dev_private; 62 struct { 63 struct ibv_flow_attr attr; 64 struct ibv_flow_spec_eth eth; 65 struct ibv_flow_spec_action_drop drop; 66 } flow_attr = { 67 .attr = { 68 .num_of_specs = 2, 69 .port = (uint8_t)priv->dev_port, 70 }, 71 .eth = { 72 .type = IBV_FLOW_SPEC_ETH, 73 .size = sizeof(struct ibv_flow_spec_eth), 74 }, 75 .drop = { 76 .size = sizeof(struct ibv_flow_spec_action_drop), 77 .type = IBV_FLOW_SPEC_ACTION_DROP, 78 }, 79 }; 80 struct ibv_flow *flow; 81 struct mlx5_hrxq *drop = priv->drop_queue.hrxq; 82 uint16_t vprio[] = { 8, 16 }; 83 int i; 84 int priority = 0; 85 86 if (!drop->qp) { 87 rte_errno = ENOTSUP; 88 return -rte_errno; 89 } 90 for (i = 0; i != RTE_DIM(vprio); i++) { 91 flow_attr.attr.priority = vprio[i] - 1; 92 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 93 if (!flow) 94 break; 95 claim_zero(mlx5_glue->destroy_flow(flow)); 96 priority = vprio[i]; 97 } 98 switch (priority) { 99 case 8: 100 priority = RTE_DIM(priority_map_3); 101 break; 102 case 16: 103 priority = RTE_DIM(priority_map_5); 104 break; 105 default: 106 rte_errno = ENOTSUP; 107 DRV_LOG(ERR, 108 "port %u verbs maximum priority: %d expected 8/16", 109 dev->data->port_id, priority); 110 return -rte_errno; 111 } 112 DRV_LOG(INFO, "port %u supported flow priorities:" 113 " 0-%d for ingress or egress root table," 114 " 0-%d for non-root table or transfer root table.", 115 dev->data->port_id, priority - 2, 116 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 117 return priority; 118 } 119 120 /** 121 * Adjust flow priority based on the highest layer and the request priority. 122 * 123 * @param[in] dev 124 * Pointer to the Ethernet device structure. 125 * @param[in] priority 126 * The rule base priority. 127 * @param[in] subpriority 128 * The priority based on the items. 129 * 130 * @return 131 * The new priority. 132 */ 133 uint32_t 134 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 135 uint32_t subpriority) 136 { 137 uint32_t res = 0; 138 struct mlx5_priv *priv = dev->data->dev_private; 139 140 switch (priv->config.flow_prio) { 141 case RTE_DIM(priority_map_3): 142 res = priority_map_3[priority][subpriority]; 143 break; 144 case RTE_DIM(priority_map_5): 145 res = priority_map_5[priority][subpriority]; 146 break; 147 } 148 return res; 149 } 150 151 /** 152 * Get Verbs flow counter by index. 153 * 154 * @param[in] dev 155 * Pointer to the Ethernet device structure. 156 * @param[in] idx 157 * mlx5 flow counter index in the container. 158 * @param[out] ppool 159 * mlx5 flow counter pool in the container, 160 * 161 * @return 162 * A pointer to the counter, NULL otherwise. 163 */ 164 static struct mlx5_flow_counter * 165 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev, 166 uint32_t idx, 167 struct mlx5_flow_counter_pool **ppool) 168 { 169 struct mlx5_priv *priv = dev->data->dev_private; 170 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 171 struct mlx5_flow_counter_pool *pool; 172 173 idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1); 174 pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL]; 175 MLX5_ASSERT(pool); 176 if (ppool) 177 *ppool = pool; 178 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL); 179 } 180 181 /** 182 * Create Verbs flow counter with Verbs library. 183 * 184 * @param[in] dev 185 * Pointer to the Ethernet device structure. 186 * @param[in, out] counter 187 * mlx5 flow counter object, contains the counter id, 188 * handle of created Verbs flow counter is returned 189 * in cs field (if counters are supported). 190 * 191 * @return 192 * 0 On success else a negative errno value is returned 193 * and rte_errno is set. 194 */ 195 static int 196 flow_verbs_counter_create(struct rte_eth_dev *dev, 197 struct mlx5_flow_counter *counter) 198 { 199 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 200 struct mlx5_priv *priv = dev->data->dev_private; 201 struct ibv_context *ctx = priv->sh->ctx; 202 struct ibv_counter_set_init_attr init = { 203 .counter_set_id = counter->shared_info.id}; 204 205 counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init); 206 if (!counter->dcs_when_free) { 207 rte_errno = ENOTSUP; 208 return -ENOTSUP; 209 } 210 return 0; 211 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 212 struct mlx5_priv *priv = dev->data->dev_private; 213 struct ibv_context *ctx = priv->sh->ctx; 214 struct ibv_counters_init_attr init = {0}; 215 struct ibv_counter_attach_attr attach; 216 int ret; 217 218 memset(&attach, 0, sizeof(attach)); 219 counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init); 220 if (!counter->dcs_when_free) { 221 rte_errno = ENOTSUP; 222 return -ENOTSUP; 223 } 224 attach.counter_desc = IBV_COUNTER_PACKETS; 225 attach.index = 0; 226 ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL); 227 if (!ret) { 228 attach.counter_desc = IBV_COUNTER_BYTES; 229 attach.index = 1; 230 ret = mlx5_glue->attach_counters 231 (counter->dcs_when_free, &attach, NULL); 232 } 233 if (ret) { 234 claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free)); 235 counter->dcs_when_free = NULL; 236 rte_errno = ret; 237 return -ret; 238 } 239 return 0; 240 #else 241 (void)dev; 242 (void)counter; 243 rte_errno = ENOTSUP; 244 return -ENOTSUP; 245 #endif 246 } 247 248 /** 249 * Get a flow counter. 250 * 251 * @param[in] dev 252 * Pointer to the Ethernet device structure. 253 * @param[in] shared 254 * Indicate if this counter is shared with other flows. 255 * @param[in] id 256 * Counter identifier. 257 * 258 * @return 259 * Index to the counter, 0 otherwise and rte_errno is set. 260 */ 261 static uint32_t 262 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) 263 { 264 struct mlx5_priv *priv = dev->data->dev_private; 265 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 266 struct mlx5_flow_counter_pool *pool = NULL; 267 struct mlx5_flow_counter *cnt = NULL; 268 union mlx5_l3t_data data; 269 uint32_t n_valid = cmng->n_valid; 270 uint32_t pool_idx, cnt_idx; 271 uint32_t i; 272 int ret; 273 274 if (shared && !mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) && 275 data.dword) 276 return data.dword; 277 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) { 278 pool = cmng->pools[pool_idx]; 279 if (!pool) 280 continue; 281 cnt = TAILQ_FIRST(&pool->counters[0]); 282 if (cnt) 283 break; 284 } 285 if (!cnt) { 286 struct mlx5_flow_counter_pool **pools; 287 uint32_t size; 288 289 if (n_valid == cmng->n) { 290 /* Resize the container pool array. */ 291 size = sizeof(struct mlx5_flow_counter_pool *) * 292 (n_valid + MLX5_CNT_CONTAINER_RESIZE); 293 pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0, 294 SOCKET_ID_ANY); 295 if (!pools) 296 return 0; 297 if (n_valid) { 298 memcpy(pools, cmng->pools, 299 sizeof(struct mlx5_flow_counter_pool *) * 300 n_valid); 301 mlx5_free(cmng->pools); 302 } 303 cmng->pools = pools; 304 cmng->n += MLX5_CNT_CONTAINER_RESIZE; 305 } 306 /* Allocate memory for new pool*/ 307 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL; 308 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY); 309 if (!pool) 310 return 0; 311 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 312 cnt = MLX5_POOL_GET_CNT(pool, i); 313 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 314 } 315 cnt = MLX5_POOL_GET_CNT(pool, 0); 316 cmng->pools[n_valid] = pool; 317 pool_idx = n_valid; 318 cmng->n_valid++; 319 } 320 TAILQ_REMOVE(&pool->counters[0], cnt, next); 321 i = MLX5_CNT_ARRAY_IDX(pool, cnt); 322 cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i); 323 if (shared) { 324 data.dword = cnt_idx; 325 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data)) 326 return 0; 327 cnt->shared_info.id = id; 328 cnt_idx |= MLX5_CNT_SHARED_OFFSET; 329 } 330 /* Create counter with Verbs. */ 331 ret = flow_verbs_counter_create(dev, cnt); 332 if (!ret) { 333 cnt->dcs_when_active = cnt->dcs_when_free; 334 cnt->hits = 0; 335 cnt->bytes = 0; 336 return cnt_idx; 337 } 338 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 339 /* Some error occurred in Verbs library. */ 340 rte_errno = -ret; 341 return 0; 342 } 343 344 /** 345 * Release a flow counter. 346 * 347 * @param[in] dev 348 * Pointer to the Ethernet device structure. 349 * @param[in] counter 350 * Index to the counter handler. 351 */ 352 static void 353 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter) 354 { 355 struct mlx5_priv *priv = dev->data->dev_private; 356 struct mlx5_flow_counter_pool *pool; 357 struct mlx5_flow_counter *cnt; 358 359 cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool); 360 if (IS_LEGACY_SHARED_CNT(counter) && 361 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id)) 362 return; 363 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 364 claim_zero(mlx5_glue->destroy_counter_set 365 ((struct ibv_counter_set *)cnt->dcs_when_active)); 366 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 367 claim_zero(mlx5_glue->destroy_counters 368 ((struct ibv_counters *)cnt->dcs_when_active)); 369 #endif 370 TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); 371 } 372 373 /** 374 * Query a flow counter via Verbs library call. 375 * 376 * @see rte_flow_query() 377 * @see rte_flow_ops 378 */ 379 static int 380 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, 381 struct rte_flow *flow, void *data, 382 struct rte_flow_error *error) 383 { 384 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 385 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 386 if (flow->counter) { 387 struct mlx5_flow_counter_pool *pool; 388 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx 389 (dev, flow->counter, &pool); 390 struct rte_flow_query_count *qc = data; 391 uint64_t counters[2] = {0, 0}; 392 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 393 struct ibv_query_counter_set_attr query_cs_attr = { 394 .dcs_when_free = (struct ibv_counter_set *) 395 cnt->dcs_when_active, 396 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 397 }; 398 struct ibv_counter_set_data query_out = { 399 .out = counters, 400 .outlen = 2 * sizeof(uint64_t), 401 }; 402 int err = mlx5_glue->query_counter_set(&query_cs_attr, 403 &query_out); 404 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 405 int err = mlx5_glue->query_counters 406 ((struct ibv_counters *)cnt->dcs_when_active, counters, 407 RTE_DIM(counters), 408 IBV_READ_COUNTERS_ATTR_PREFER_CACHED); 409 #endif 410 if (err) 411 return rte_flow_error_set 412 (error, err, 413 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 414 NULL, 415 "cannot read counter"); 416 qc->hits_set = 1; 417 qc->bytes_set = 1; 418 qc->hits = counters[0] - cnt->hits; 419 qc->bytes = counters[1] - cnt->bytes; 420 if (qc->reset) { 421 cnt->hits = counters[0]; 422 cnt->bytes = counters[1]; 423 } 424 return 0; 425 } 426 return rte_flow_error_set(error, EINVAL, 427 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 428 NULL, 429 "flow does not have counter"); 430 #else 431 (void)flow; 432 (void)data; 433 return rte_flow_error_set(error, ENOTSUP, 434 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 435 NULL, 436 "counters are not available"); 437 #endif 438 } 439 440 /** 441 * Add a verbs item specification into @p verbs. 442 * 443 * @param[out] verbs 444 * Pointer to verbs structure. 445 * @param[in] src 446 * Create specification. 447 * @param[in] size 448 * Size in bytes of the specification to copy. 449 */ 450 static void 451 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs, 452 void *src, unsigned int size) 453 { 454 void *dst; 455 456 if (!verbs) 457 return; 458 MLX5_ASSERT(verbs->specs); 459 dst = (void *)(verbs->specs + verbs->size); 460 memcpy(dst, src, size); 461 ++verbs->attr.num_of_specs; 462 verbs->size += size; 463 } 464 465 /** 466 * Convert the @p item into a Verbs specification. This function assumes that 467 * the input is valid and that there is space to insert the requested item 468 * into the flow. 469 * 470 * @param[in, out] dev_flow 471 * Pointer to dev_flow structure. 472 * @param[in] item 473 * Item specification. 474 * @param[in] item_flags 475 * Parsed item flags. 476 */ 477 static void 478 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow, 479 const struct rte_flow_item *item, 480 uint64_t item_flags) 481 { 482 const struct rte_flow_item_eth *spec = item->spec; 483 const struct rte_flow_item_eth *mask = item->mask; 484 const unsigned int size = sizeof(struct ibv_flow_spec_eth); 485 struct ibv_flow_spec_eth eth = { 486 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags), 487 .size = size, 488 }; 489 490 if (!mask) 491 mask = &rte_flow_item_eth_mask; 492 if (spec) { 493 unsigned int i; 494 495 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, 496 RTE_ETHER_ADDR_LEN); 497 memcpy(ð.val.src_mac, spec->src.addr_bytes, 498 RTE_ETHER_ADDR_LEN); 499 eth.val.ether_type = spec->type; 500 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, 501 RTE_ETHER_ADDR_LEN); 502 memcpy(ð.mask.src_mac, mask->src.addr_bytes, 503 RTE_ETHER_ADDR_LEN); 504 eth.mask.ether_type = mask->type; 505 /* Remove unwanted bits from values. */ 506 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) { 507 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 508 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 509 } 510 eth.val.ether_type &= eth.mask.ether_type; 511 } 512 flow_verbs_spec_add(&dev_flow->verbs, ð, size); 513 } 514 515 /** 516 * Update the VLAN tag in the Verbs Ethernet specification. 517 * This function assumes that the input is valid and there is space to add 518 * the requested item. 519 * 520 * @param[in, out] attr 521 * Pointer to Verbs attributes structure. 522 * @param[in] eth 523 * Verbs structure containing the VLAN information to copy. 524 */ 525 static void 526 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr, 527 struct ibv_flow_spec_eth *eth) 528 { 529 unsigned int i; 530 const enum ibv_flow_spec_type search = eth->type; 531 struct ibv_spec_header *hdr = (struct ibv_spec_header *) 532 ((uint8_t *)attr + sizeof(struct ibv_flow_attr)); 533 534 for (i = 0; i != attr->num_of_specs; ++i) { 535 if (hdr->type == search) { 536 struct ibv_flow_spec_eth *e = 537 (struct ibv_flow_spec_eth *)hdr; 538 539 e->val.vlan_tag = eth->val.vlan_tag; 540 e->mask.vlan_tag = eth->mask.vlan_tag; 541 e->val.ether_type = eth->val.ether_type; 542 e->mask.ether_type = eth->mask.ether_type; 543 break; 544 } 545 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size); 546 } 547 } 548 549 /** 550 * Convert the @p item into a Verbs specification. This function assumes that 551 * the input is valid and that there is space to insert the requested item 552 * into the flow. 553 * 554 * @param[in, out] dev_flow 555 * Pointer to dev_flow structure. 556 * @param[in] item 557 * Item specification. 558 * @param[in] item_flags 559 * Parsed item flags. 560 */ 561 static void 562 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow, 563 const struct rte_flow_item *item, 564 uint64_t item_flags) 565 { 566 const struct rte_flow_item_vlan *spec = item->spec; 567 const struct rte_flow_item_vlan *mask = item->mask; 568 unsigned int size = sizeof(struct ibv_flow_spec_eth); 569 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 570 struct ibv_flow_spec_eth eth = { 571 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags), 572 .size = size, 573 }; 574 const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 575 MLX5_FLOW_LAYER_OUTER_L2; 576 577 if (!mask) 578 mask = &rte_flow_item_vlan_mask; 579 if (spec) { 580 eth.val.vlan_tag = spec->tci; 581 eth.mask.vlan_tag = mask->tci; 582 eth.val.vlan_tag &= eth.mask.vlan_tag; 583 eth.val.ether_type = spec->inner_type; 584 eth.mask.ether_type = mask->inner_type; 585 eth.val.ether_type &= eth.mask.ether_type; 586 } 587 if (!(item_flags & l2m)) 588 flow_verbs_spec_add(&dev_flow->verbs, ð, size); 589 else 590 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð); 591 if (!tunnel) 592 dev_flow->handle->vf_vlan.tag = 593 rte_be_to_cpu_16(spec->tci) & 0x0fff; 594 } 595 596 /** 597 * Convert the @p item into a Verbs specification. This function assumes that 598 * the input is valid and that there is space to insert the requested item 599 * into the flow. 600 * 601 * @param[in, out] dev_flow 602 * Pointer to dev_flow structure. 603 * @param[in] item 604 * Item specification. 605 * @param[in] item_flags 606 * Parsed item flags. 607 */ 608 static void 609 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow, 610 const struct rte_flow_item *item, 611 uint64_t item_flags) 612 { 613 const struct rte_flow_item_ipv4 *spec = item->spec; 614 const struct rte_flow_item_ipv4 *mask = item->mask; 615 unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext); 616 struct ibv_flow_spec_ipv4_ext ipv4 = { 617 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags), 618 .size = size, 619 }; 620 621 if (!mask) 622 mask = &rte_flow_item_ipv4_mask; 623 if (spec) { 624 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 625 .src_ip = spec->hdr.src_addr, 626 .dst_ip = spec->hdr.dst_addr, 627 .proto = spec->hdr.next_proto_id, 628 .tos = spec->hdr.type_of_service, 629 }; 630 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 631 .src_ip = mask->hdr.src_addr, 632 .dst_ip = mask->hdr.dst_addr, 633 .proto = mask->hdr.next_proto_id, 634 .tos = mask->hdr.type_of_service, 635 }; 636 /* Remove unwanted bits from values. */ 637 ipv4.val.src_ip &= ipv4.mask.src_ip; 638 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 639 ipv4.val.proto &= ipv4.mask.proto; 640 ipv4.val.tos &= ipv4.mask.tos; 641 } 642 flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size); 643 } 644 645 /** 646 * Convert the @p item into a Verbs specification. This function assumes that 647 * the input is valid and that there is space to insert the requested item 648 * into the flow. 649 * 650 * @param[in, out] dev_flow 651 * Pointer to dev_flow structure. 652 * @param[in] item 653 * Item specification. 654 * @param[in] item_flags 655 * Parsed item flags. 656 */ 657 static void 658 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow, 659 const struct rte_flow_item *item, 660 uint64_t item_flags) 661 { 662 const struct rte_flow_item_ipv6 *spec = item->spec; 663 const struct rte_flow_item_ipv6 *mask = item->mask; 664 unsigned int size = sizeof(struct ibv_flow_spec_ipv6); 665 struct ibv_flow_spec_ipv6 ipv6 = { 666 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags), 667 .size = size, 668 }; 669 670 if (!mask) 671 mask = &rte_flow_item_ipv6_mask; 672 if (spec) { 673 unsigned int i; 674 uint32_t vtc_flow_val; 675 uint32_t vtc_flow_mask; 676 677 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 678 RTE_DIM(ipv6.val.src_ip)); 679 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 680 RTE_DIM(ipv6.val.dst_ip)); 681 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 682 RTE_DIM(ipv6.mask.src_ip)); 683 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 684 RTE_DIM(ipv6.mask.dst_ip)); 685 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); 686 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); 687 ipv6.val.flow_label = 688 rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >> 689 RTE_IPV6_HDR_FL_SHIFT); 690 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >> 691 RTE_IPV6_HDR_TC_SHIFT; 692 ipv6.val.next_hdr = spec->hdr.proto; 693 ipv6.mask.flow_label = 694 rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >> 695 RTE_IPV6_HDR_FL_SHIFT); 696 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >> 697 RTE_IPV6_HDR_TC_SHIFT; 698 ipv6.mask.next_hdr = mask->hdr.proto; 699 /* Remove unwanted bits from values. */ 700 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 701 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 702 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 703 } 704 ipv6.val.flow_label &= ipv6.mask.flow_label; 705 ipv6.val.traffic_class &= ipv6.mask.traffic_class; 706 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 707 } 708 flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size); 709 } 710 711 /** 712 * Convert the @p item into a Verbs specification. This function assumes that 713 * the input is valid and that there is space to insert the requested item 714 * into the flow. 715 * 716 * @param[in, out] dev_flow 717 * Pointer to dev_flow structure. 718 * @param[in] item 719 * Item specification. 720 * @param[in] item_flags 721 * Parsed item flags. 722 */ 723 static void 724 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow, 725 const struct rte_flow_item *item, 726 uint64_t item_flags __rte_unused) 727 { 728 const struct rte_flow_item_tcp *spec = item->spec; 729 const struct rte_flow_item_tcp *mask = item->mask; 730 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp); 731 struct ibv_flow_spec_tcp_udp tcp = { 732 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags), 733 .size = size, 734 }; 735 736 if (!mask) 737 mask = &rte_flow_item_tcp_mask; 738 if (spec) { 739 tcp.val.dst_port = spec->hdr.dst_port; 740 tcp.val.src_port = spec->hdr.src_port; 741 tcp.mask.dst_port = mask->hdr.dst_port; 742 tcp.mask.src_port = mask->hdr.src_port; 743 /* Remove unwanted bits from values. */ 744 tcp.val.src_port &= tcp.mask.src_port; 745 tcp.val.dst_port &= tcp.mask.dst_port; 746 } 747 flow_verbs_spec_add(&dev_flow->verbs, &tcp, size); 748 } 749 750 /** 751 * Convert the @p item into a Verbs specification. This function assumes that 752 * the input is valid and that there is space to insert the requested item 753 * into the flow. 754 * 755 * @param[in, out] dev_flow 756 * Pointer to dev_flow structure. 757 * @param[in] item 758 * Item specification. 759 * @param[in] item_flags 760 * Parsed item flags. 761 */ 762 static void 763 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow, 764 const struct rte_flow_item *item, 765 uint64_t item_flags __rte_unused) 766 { 767 const struct rte_flow_item_udp *spec = item->spec; 768 const struct rte_flow_item_udp *mask = item->mask; 769 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp); 770 struct ibv_flow_spec_tcp_udp udp = { 771 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags), 772 .size = size, 773 }; 774 775 if (!mask) 776 mask = &rte_flow_item_udp_mask; 777 if (spec) { 778 udp.val.dst_port = spec->hdr.dst_port; 779 udp.val.src_port = spec->hdr.src_port; 780 udp.mask.dst_port = mask->hdr.dst_port; 781 udp.mask.src_port = mask->hdr.src_port; 782 /* Remove unwanted bits from values. */ 783 udp.val.src_port &= udp.mask.src_port; 784 udp.val.dst_port &= udp.mask.dst_port; 785 } 786 item++; 787 while (item->type == RTE_FLOW_ITEM_TYPE_VOID) 788 item++; 789 if (!(udp.val.dst_port & udp.mask.dst_port)) { 790 switch ((item)->type) { 791 case RTE_FLOW_ITEM_TYPE_VXLAN: 792 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN); 793 udp.mask.dst_port = 0xffff; 794 break; 795 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 796 udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE); 797 udp.mask.dst_port = 0xffff; 798 break; 799 case RTE_FLOW_ITEM_TYPE_MPLS: 800 udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS); 801 udp.mask.dst_port = 0xffff; 802 break; 803 default: 804 break; 805 } 806 } 807 808 flow_verbs_spec_add(&dev_flow->verbs, &udp, size); 809 } 810 811 /** 812 * Convert the @p item into a Verbs specification. This function assumes that 813 * the input is valid and that there is space to insert the requested item 814 * into the flow. 815 * 816 * @param[in, out] dev_flow 817 * Pointer to dev_flow structure. 818 * @param[in] item 819 * Item specification. 820 * @param[in] item_flags 821 * Parsed item flags. 822 */ 823 static void 824 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow, 825 const struct rte_flow_item *item, 826 uint64_t item_flags __rte_unused) 827 { 828 const struct rte_flow_item_vxlan *spec = item->spec; 829 const struct rte_flow_item_vxlan *mask = item->mask; 830 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 831 struct ibv_flow_spec_tunnel vxlan = { 832 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 833 .size = size, 834 }; 835 union vni { 836 uint32_t vlan_id; 837 uint8_t vni[4]; 838 } id = { .vlan_id = 0, }; 839 840 if (!mask) 841 mask = &rte_flow_item_vxlan_mask; 842 if (spec) { 843 memcpy(&id.vni[1], spec->vni, 3); 844 vxlan.val.tunnel_id = id.vlan_id; 845 memcpy(&id.vni[1], mask->vni, 3); 846 vxlan.mask.tunnel_id = id.vlan_id; 847 /* Remove unwanted bits from values. */ 848 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 849 } 850 flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size); 851 } 852 853 /** 854 * Convert the @p item into a Verbs specification. This function assumes that 855 * the input is valid and that there is space to insert the requested item 856 * into the flow. 857 * 858 * @param[in, out] dev_flow 859 * Pointer to dev_flow structure. 860 * @param[in] item 861 * Item specification. 862 * @param[in] item_flags 863 * Parsed item flags. 864 */ 865 static void 866 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow, 867 const struct rte_flow_item *item, 868 uint64_t item_flags __rte_unused) 869 { 870 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 871 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 872 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 873 struct ibv_flow_spec_tunnel vxlan_gpe = { 874 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 875 .size = size, 876 }; 877 union vni { 878 uint32_t vlan_id; 879 uint8_t vni[4]; 880 } id = { .vlan_id = 0, }; 881 882 if (!mask) 883 mask = &rte_flow_item_vxlan_gpe_mask; 884 if (spec) { 885 memcpy(&id.vni[1], spec->vni, 3); 886 vxlan_gpe.val.tunnel_id = id.vlan_id; 887 memcpy(&id.vni[1], mask->vni, 3); 888 vxlan_gpe.mask.tunnel_id = id.vlan_id; 889 /* Remove unwanted bits from values. */ 890 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id; 891 } 892 flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size); 893 } 894 895 /** 896 * Update the protocol in Verbs IPv4/IPv6 spec. 897 * 898 * @param[in, out] attr 899 * Pointer to Verbs attributes structure. 900 * @param[in] search 901 * Specification type to search in order to update the IP protocol. 902 * @param[in] protocol 903 * Protocol value to set if none is present in the specification. 904 */ 905 static void 906 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr, 907 enum ibv_flow_spec_type search, 908 uint8_t protocol) 909 { 910 unsigned int i; 911 struct ibv_spec_header *hdr = (struct ibv_spec_header *) 912 ((uint8_t *)attr + sizeof(struct ibv_flow_attr)); 913 914 if (!attr) 915 return; 916 for (i = 0; i != attr->num_of_specs; ++i) { 917 if (hdr->type == search) { 918 union { 919 struct ibv_flow_spec_ipv4_ext *ipv4; 920 struct ibv_flow_spec_ipv6 *ipv6; 921 } ip; 922 923 switch (search) { 924 case IBV_FLOW_SPEC_IPV4_EXT: 925 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr; 926 if (!ip.ipv4->val.proto) { 927 ip.ipv4->val.proto = protocol; 928 ip.ipv4->mask.proto = 0xff; 929 } 930 break; 931 case IBV_FLOW_SPEC_IPV6: 932 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr; 933 if (!ip.ipv6->val.next_hdr) { 934 ip.ipv6->val.next_hdr = protocol; 935 ip.ipv6->mask.next_hdr = 0xff; 936 } 937 break; 938 default: 939 break; 940 } 941 break; 942 } 943 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size); 944 } 945 } 946 947 /** 948 * Convert the @p item into a Verbs specification. This function assumes that 949 * the input is valid and that there is space to insert the requested item 950 * into the flow. 951 * 952 * @param[in, out] dev_flow 953 * Pointer to dev_flow structure. 954 * @param[in] item 955 * Item specification. 956 * @param[in] item_flags 957 * Parsed item flags. 958 */ 959 static void 960 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow, 961 const struct rte_flow_item *item __rte_unused, 962 uint64_t item_flags) 963 { 964 struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs; 965 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 966 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 967 struct ibv_flow_spec_tunnel tunnel = { 968 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 969 .size = size, 970 }; 971 #else 972 const struct rte_flow_item_gre *spec = item->spec; 973 const struct rte_flow_item_gre *mask = item->mask; 974 unsigned int size = sizeof(struct ibv_flow_spec_gre); 975 struct ibv_flow_spec_gre tunnel = { 976 .type = IBV_FLOW_SPEC_GRE, 977 .size = size, 978 }; 979 980 if (!mask) 981 mask = &rte_flow_item_gre_mask; 982 if (spec) { 983 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver; 984 tunnel.val.protocol = spec->protocol; 985 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver; 986 tunnel.mask.protocol = mask->protocol; 987 /* Remove unwanted bits from values. */ 988 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver; 989 tunnel.val.protocol &= tunnel.mask.protocol; 990 tunnel.val.key &= tunnel.mask.key; 991 } 992 #endif 993 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) 994 flow_verbs_item_gre_ip_protocol_update(&verbs->attr, 995 IBV_FLOW_SPEC_IPV4_EXT, 996 IPPROTO_GRE); 997 else 998 flow_verbs_item_gre_ip_protocol_update(&verbs->attr, 999 IBV_FLOW_SPEC_IPV6, 1000 IPPROTO_GRE); 1001 flow_verbs_spec_add(verbs, &tunnel, size); 1002 } 1003 1004 /** 1005 * Convert the @p action into a Verbs specification. This function assumes that 1006 * the input is valid and that there is space to insert the requested action 1007 * into the flow. This function also return the action that was added. 1008 * 1009 * @param[in, out] dev_flow 1010 * Pointer to dev_flow structure. 1011 * @param[in] item 1012 * Item specification. 1013 * @param[in] item_flags 1014 * Parsed item flags. 1015 */ 1016 static void 1017 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused, 1018 const struct rte_flow_item *item __rte_unused, 1019 uint64_t item_flags __rte_unused) 1020 { 1021 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1022 const struct rte_flow_item_mpls *spec = item->spec; 1023 const struct rte_flow_item_mpls *mask = item->mask; 1024 unsigned int size = sizeof(struct ibv_flow_spec_mpls); 1025 struct ibv_flow_spec_mpls mpls = { 1026 .type = IBV_FLOW_SPEC_MPLS, 1027 .size = size, 1028 }; 1029 1030 if (!mask) 1031 mask = &rte_flow_item_mpls_mask; 1032 if (spec) { 1033 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label)); 1034 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label)); 1035 /* Remove unwanted bits from values. */ 1036 mpls.val.label &= mpls.mask.label; 1037 } 1038 flow_verbs_spec_add(&dev_flow->verbs, &mpls, size); 1039 #endif 1040 } 1041 1042 /** 1043 * Convert the @p action into a Verbs specification. This function assumes that 1044 * the input is valid and that there is space to insert the requested action 1045 * into the flow. 1046 * 1047 * @param[in] dev_flow 1048 * Pointer to mlx5_flow. 1049 * @param[in] action 1050 * Action configuration. 1051 */ 1052 static void 1053 flow_verbs_translate_action_drop 1054 (struct mlx5_flow *dev_flow, 1055 const struct rte_flow_action *action __rte_unused) 1056 { 1057 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 1058 struct ibv_flow_spec_action_drop drop = { 1059 .type = IBV_FLOW_SPEC_ACTION_DROP, 1060 .size = size, 1061 }; 1062 1063 flow_verbs_spec_add(&dev_flow->verbs, &drop, size); 1064 } 1065 1066 /** 1067 * Convert the @p action into a Verbs specification. This function assumes that 1068 * the input is valid and that there is space to insert the requested action 1069 * into the flow. 1070 * 1071 * @param[in] rss_desc 1072 * Pointer to mlx5_flow_rss_desc. 1073 * @param[in] action 1074 * Action configuration. 1075 */ 1076 static void 1077 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc, 1078 const struct rte_flow_action *action) 1079 { 1080 const struct rte_flow_action_queue *queue = action->conf; 1081 1082 rss_desc->queue[0] = queue->index; 1083 rss_desc->queue_num = 1; 1084 } 1085 1086 /** 1087 * Convert the @p action into a Verbs specification. This function assumes that 1088 * the input is valid and that there is space to insert the requested action 1089 * into the flow. 1090 * 1091 * @param[in] rss_desc 1092 * Pointer to mlx5_flow_rss_desc. 1093 * @param[in] action 1094 * Action configuration. 1095 */ 1096 static void 1097 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc, 1098 const struct rte_flow_action *action) 1099 { 1100 const struct rte_flow_action_rss *rss = action->conf; 1101 const uint8_t *rss_key; 1102 1103 memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t)); 1104 rss_desc->queue_num = rss->queue_num; 1105 /* NULL RSS key indicates default RSS key. */ 1106 rss_key = !rss->key ? rss_hash_default_key : rss->key; 1107 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 1108 /* 1109 * rss->level and rss.types should be set in advance when expanding 1110 * items for RSS. 1111 */ 1112 } 1113 1114 /** 1115 * Convert the @p action into a Verbs specification. This function assumes that 1116 * the input is valid and that there is space to insert the requested action 1117 * into the flow. 1118 * 1119 * @param[in] dev_flow 1120 * Pointer to mlx5_flow. 1121 * @param[in] action 1122 * Action configuration. 1123 */ 1124 static void 1125 flow_verbs_translate_action_flag 1126 (struct mlx5_flow *dev_flow, 1127 const struct rte_flow_action *action __rte_unused) 1128 { 1129 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1130 struct ibv_flow_spec_action_tag tag = { 1131 .type = IBV_FLOW_SPEC_ACTION_TAG, 1132 .size = size, 1133 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT), 1134 }; 1135 1136 flow_verbs_spec_add(&dev_flow->verbs, &tag, size); 1137 } 1138 1139 /** 1140 * Convert the @p action into a Verbs specification. This function assumes that 1141 * the input is valid and that there is space to insert the requested action 1142 * into the flow. 1143 * 1144 * @param[in] dev_flow 1145 * Pointer to mlx5_flow. 1146 * @param[in] action 1147 * Action configuration. 1148 */ 1149 static void 1150 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow, 1151 const struct rte_flow_action *action) 1152 { 1153 const struct rte_flow_action_mark *mark = action->conf; 1154 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 1155 struct ibv_flow_spec_action_tag tag = { 1156 .type = IBV_FLOW_SPEC_ACTION_TAG, 1157 .size = size, 1158 .tag_id = mlx5_flow_mark_set(mark->id), 1159 }; 1160 1161 flow_verbs_spec_add(&dev_flow->verbs, &tag, size); 1162 } 1163 1164 /** 1165 * Convert the @p action into a Verbs specification. This function assumes that 1166 * the input is valid and that there is space to insert the requested action 1167 * into the flow. 1168 * 1169 * @param[in] dev 1170 * Pointer to the Ethernet device structure. 1171 * @param[in] action 1172 * Action configuration. 1173 * @param[in] dev_flow 1174 * Pointer to mlx5_flow. 1175 * @param[out] error 1176 * Pointer to error structure. 1177 * 1178 * @return 1179 * 0 On success else a negative errno value is returned and rte_errno is set. 1180 */ 1181 static int 1182 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, 1183 const struct rte_flow_action *action, 1184 struct rte_eth_dev *dev, 1185 struct rte_flow_error *error) 1186 { 1187 const struct rte_flow_action_count *count = action->conf; 1188 struct rte_flow *flow = dev_flow->flow; 1189 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 1190 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1191 struct mlx5_flow_counter_pool *pool; 1192 struct mlx5_flow_counter *cnt = NULL; 1193 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 1194 struct ibv_flow_spec_counter_action counter = { 1195 .type = IBV_FLOW_SPEC_ACTION_COUNT, 1196 .size = size, 1197 }; 1198 #endif 1199 1200 if (!flow->counter) { 1201 flow->counter = flow_verbs_counter_new(dev, count->shared, 1202 count->id); 1203 if (!flow->counter) 1204 return rte_flow_error_set(error, rte_errno, 1205 RTE_FLOW_ERROR_TYPE_ACTION, 1206 action, 1207 "cannot get counter" 1208 " context."); 1209 } 1210 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 1211 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); 1212 counter.counter_set_handle = 1213 ((struct ibv_counter_set *)cnt->dcs_when_active)->handle; 1214 flow_verbs_spec_add(&dev_flow->verbs, &counter, size); 1215 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1216 cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); 1217 counter.counters = (struct ibv_counters *)cnt->dcs_when_active; 1218 flow_verbs_spec_add(&dev_flow->verbs, &counter, size); 1219 #endif 1220 return 0; 1221 } 1222 1223 /** 1224 * Internal validation function. For validating both actions and items. 1225 * 1226 * @param[in] dev 1227 * Pointer to the Ethernet device structure. 1228 * @param[in] attr 1229 * Pointer to the flow attributes. 1230 * @param[in] items 1231 * Pointer to the list of items. 1232 * @param[in] actions 1233 * Pointer to the list of actions. 1234 * @param[in] external 1235 * This flow rule is created by request external to PMD. 1236 * @param[in] hairpin 1237 * Number of hairpin TX actions, 0 means classic flow. 1238 * @param[out] error 1239 * Pointer to the error structure. 1240 * 1241 * @return 1242 * 0 on success, a negative errno value otherwise and rte_errno is set. 1243 */ 1244 static int 1245 flow_verbs_validate(struct rte_eth_dev *dev, 1246 const struct rte_flow_attr *attr, 1247 const struct rte_flow_item items[], 1248 const struct rte_flow_action actions[], 1249 bool external __rte_unused, 1250 int hairpin __rte_unused, 1251 struct rte_flow_error *error) 1252 { 1253 int ret; 1254 uint64_t action_flags = 0; 1255 uint64_t item_flags = 0; 1256 uint64_t last_item = 0; 1257 uint8_t next_protocol = 0xff; 1258 uint16_t ether_type = 0; 1259 bool is_empty_vlan = false; 1260 uint16_t udp_dport = 0; 1261 1262 if (items == NULL) 1263 return -1; 1264 ret = mlx5_flow_validate_attributes(dev, attr, error); 1265 if (ret < 0) 1266 return ret; 1267 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1268 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1269 int ret = 0; 1270 1271 switch (items->type) { 1272 case RTE_FLOW_ITEM_TYPE_VOID: 1273 break; 1274 case RTE_FLOW_ITEM_TYPE_ETH: 1275 ret = mlx5_flow_validate_item_eth(items, item_flags, 1276 false, error); 1277 if (ret < 0) 1278 return ret; 1279 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1280 MLX5_FLOW_LAYER_OUTER_L2; 1281 if (items->mask != NULL && items->spec != NULL) { 1282 ether_type = 1283 ((const struct rte_flow_item_eth *) 1284 items->spec)->type; 1285 ether_type &= 1286 ((const struct rte_flow_item_eth *) 1287 items->mask)->type; 1288 if (ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN)) 1289 is_empty_vlan = true; 1290 ether_type = rte_be_to_cpu_16(ether_type); 1291 } else { 1292 ether_type = 0; 1293 } 1294 break; 1295 case RTE_FLOW_ITEM_TYPE_VLAN: 1296 ret = mlx5_flow_validate_item_vlan(items, item_flags, 1297 dev, error); 1298 if (ret < 0) 1299 return ret; 1300 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 1301 MLX5_FLOW_LAYER_INNER_VLAN) : 1302 (MLX5_FLOW_LAYER_OUTER_L2 | 1303 MLX5_FLOW_LAYER_OUTER_VLAN); 1304 if (items->mask != NULL && items->spec != NULL) { 1305 ether_type = 1306 ((const struct rte_flow_item_vlan *) 1307 items->spec)->inner_type; 1308 ether_type &= 1309 ((const struct rte_flow_item_vlan *) 1310 items->mask)->inner_type; 1311 ether_type = rte_be_to_cpu_16(ether_type); 1312 } else { 1313 ether_type = 0; 1314 } 1315 is_empty_vlan = false; 1316 break; 1317 case RTE_FLOW_ITEM_TYPE_IPV4: 1318 ret = mlx5_flow_validate_item_ipv4 1319 (items, item_flags, 1320 last_item, ether_type, NULL, 1321 MLX5_ITEM_RANGE_NOT_ACCEPTED, 1322 error); 1323 if (ret < 0) 1324 return ret; 1325 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1326 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1327 if (items->mask != NULL && 1328 ((const struct rte_flow_item_ipv4 *) 1329 items->mask)->hdr.next_proto_id) { 1330 next_protocol = 1331 ((const struct rte_flow_item_ipv4 *) 1332 (items->spec))->hdr.next_proto_id; 1333 next_protocol &= 1334 ((const struct rte_flow_item_ipv4 *) 1335 (items->mask))->hdr.next_proto_id; 1336 } else { 1337 /* Reset for inner layer. */ 1338 next_protocol = 0xff; 1339 } 1340 break; 1341 case RTE_FLOW_ITEM_TYPE_IPV6: 1342 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 1343 last_item, 1344 ether_type, NULL, 1345 error); 1346 if (ret < 0) 1347 return ret; 1348 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1349 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1350 if (items->mask != NULL && 1351 ((const struct rte_flow_item_ipv6 *) 1352 items->mask)->hdr.proto) { 1353 next_protocol = 1354 ((const struct rte_flow_item_ipv6 *) 1355 items->spec)->hdr.proto; 1356 next_protocol &= 1357 ((const struct rte_flow_item_ipv6 *) 1358 items->mask)->hdr.proto; 1359 } else { 1360 /* Reset for inner layer. */ 1361 next_protocol = 0xff; 1362 } 1363 break; 1364 case RTE_FLOW_ITEM_TYPE_UDP: 1365 ret = mlx5_flow_validate_item_udp(items, item_flags, 1366 next_protocol, 1367 error); 1368 const struct rte_flow_item_udp *spec = items->spec; 1369 const struct rte_flow_item_udp *mask = items->mask; 1370 if (!mask) 1371 mask = &rte_flow_item_udp_mask; 1372 if (spec != NULL) 1373 udp_dport = rte_be_to_cpu_16 1374 (spec->hdr.dst_port & 1375 mask->hdr.dst_port); 1376 1377 if (ret < 0) 1378 return ret; 1379 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 1380 MLX5_FLOW_LAYER_OUTER_L4_UDP; 1381 break; 1382 case RTE_FLOW_ITEM_TYPE_TCP: 1383 ret = mlx5_flow_validate_item_tcp 1384 (items, item_flags, 1385 next_protocol, 1386 &rte_flow_item_tcp_mask, 1387 error); 1388 if (ret < 0) 1389 return ret; 1390 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 1391 MLX5_FLOW_LAYER_OUTER_L4_TCP; 1392 break; 1393 case RTE_FLOW_ITEM_TYPE_VXLAN: 1394 ret = mlx5_flow_validate_item_vxlan(dev, udp_dport, 1395 items, item_flags, 1396 attr, error); 1397 if (ret < 0) 1398 return ret; 1399 last_item = MLX5_FLOW_LAYER_VXLAN; 1400 break; 1401 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1402 ret = mlx5_flow_validate_item_vxlan_gpe(items, 1403 item_flags, 1404 dev, error); 1405 if (ret < 0) 1406 return ret; 1407 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 1408 break; 1409 case RTE_FLOW_ITEM_TYPE_GRE: 1410 ret = mlx5_flow_validate_item_gre(items, item_flags, 1411 next_protocol, error); 1412 if (ret < 0) 1413 return ret; 1414 last_item = MLX5_FLOW_LAYER_GRE; 1415 break; 1416 case RTE_FLOW_ITEM_TYPE_MPLS: 1417 ret = mlx5_flow_validate_item_mpls(dev, items, 1418 item_flags, 1419 last_item, error); 1420 if (ret < 0) 1421 return ret; 1422 last_item = MLX5_FLOW_LAYER_MPLS; 1423 break; 1424 case RTE_FLOW_ITEM_TYPE_ICMP: 1425 case RTE_FLOW_ITEM_TYPE_ICMP6: 1426 return rte_flow_error_set(error, ENOTSUP, 1427 RTE_FLOW_ERROR_TYPE_ITEM, 1428 NULL, "ICMP/ICMP6 " 1429 "item not supported"); 1430 default: 1431 return rte_flow_error_set(error, ENOTSUP, 1432 RTE_FLOW_ERROR_TYPE_ITEM, 1433 NULL, "item not supported"); 1434 } 1435 item_flags |= last_item; 1436 } 1437 if (is_empty_vlan) 1438 return rte_flow_error_set(error, ENOTSUP, 1439 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1440 "VLAN matching without vid specification is not supported"); 1441 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1442 switch (actions->type) { 1443 case RTE_FLOW_ACTION_TYPE_VOID: 1444 break; 1445 case RTE_FLOW_ACTION_TYPE_FLAG: 1446 ret = mlx5_flow_validate_action_flag(action_flags, 1447 attr, 1448 error); 1449 if (ret < 0) 1450 return ret; 1451 action_flags |= MLX5_FLOW_ACTION_FLAG; 1452 break; 1453 case RTE_FLOW_ACTION_TYPE_MARK: 1454 ret = mlx5_flow_validate_action_mark(actions, 1455 action_flags, 1456 attr, 1457 error); 1458 if (ret < 0) 1459 return ret; 1460 action_flags |= MLX5_FLOW_ACTION_MARK; 1461 break; 1462 case RTE_FLOW_ACTION_TYPE_DROP: 1463 ret = mlx5_flow_validate_action_drop(action_flags, 1464 attr, 1465 error); 1466 if (ret < 0) 1467 return ret; 1468 action_flags |= MLX5_FLOW_ACTION_DROP; 1469 break; 1470 case RTE_FLOW_ACTION_TYPE_QUEUE: 1471 ret = mlx5_flow_validate_action_queue(actions, 1472 action_flags, dev, 1473 attr, 1474 error); 1475 if (ret < 0) 1476 return ret; 1477 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1478 break; 1479 case RTE_FLOW_ACTION_TYPE_RSS: 1480 ret = mlx5_flow_validate_action_rss(actions, 1481 action_flags, dev, 1482 attr, item_flags, 1483 error); 1484 if (ret < 0) 1485 return ret; 1486 action_flags |= MLX5_FLOW_ACTION_RSS; 1487 break; 1488 case RTE_FLOW_ACTION_TYPE_COUNT: 1489 ret = mlx5_flow_validate_action_count(dev, attr, error); 1490 if (ret < 0) 1491 return ret; 1492 action_flags |= MLX5_FLOW_ACTION_COUNT; 1493 break; 1494 default: 1495 return rte_flow_error_set(error, ENOTSUP, 1496 RTE_FLOW_ERROR_TYPE_ACTION, 1497 actions, 1498 "action not supported"); 1499 } 1500 } 1501 /* 1502 * Validate the drop action mutual exclusion with other actions. 1503 * Drop action is mutually-exclusive with any other action, except for 1504 * Count action. 1505 */ 1506 if ((action_flags & MLX5_FLOW_ACTION_DROP) && 1507 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) 1508 return rte_flow_error_set(error, EINVAL, 1509 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1510 "Drop action is mutually-exclusive " 1511 "with any other action, except for " 1512 "Count action"); 1513 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS)) 1514 return rte_flow_error_set(error, EINVAL, 1515 RTE_FLOW_ERROR_TYPE_ACTION, actions, 1516 "no fate action is found"); 1517 return 0; 1518 } 1519 1520 /** 1521 * Calculate the required bytes that are needed for the action part of the verbs 1522 * flow. 1523 * 1524 * @param[in] actions 1525 * Pointer to the list of actions. 1526 * 1527 * @return 1528 * The size of the memory needed for all actions. 1529 */ 1530 static int 1531 flow_verbs_get_actions_size(const struct rte_flow_action actions[]) 1532 { 1533 int size = 0; 1534 1535 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1536 switch (actions->type) { 1537 case RTE_FLOW_ACTION_TYPE_VOID: 1538 break; 1539 case RTE_FLOW_ACTION_TYPE_FLAG: 1540 size += sizeof(struct ibv_flow_spec_action_tag); 1541 break; 1542 case RTE_FLOW_ACTION_TYPE_MARK: 1543 size += sizeof(struct ibv_flow_spec_action_tag); 1544 break; 1545 case RTE_FLOW_ACTION_TYPE_DROP: 1546 size += sizeof(struct ibv_flow_spec_action_drop); 1547 break; 1548 case RTE_FLOW_ACTION_TYPE_QUEUE: 1549 break; 1550 case RTE_FLOW_ACTION_TYPE_RSS: 1551 break; 1552 case RTE_FLOW_ACTION_TYPE_COUNT: 1553 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 1554 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1555 size += sizeof(struct ibv_flow_spec_counter_action); 1556 #endif 1557 break; 1558 default: 1559 break; 1560 } 1561 } 1562 return size; 1563 } 1564 1565 /** 1566 * Calculate the required bytes that are needed for the item part of the verbs 1567 * flow. 1568 * 1569 * @param[in] items 1570 * Pointer to the list of items. 1571 * 1572 * @return 1573 * The size of the memory needed for all items. 1574 */ 1575 static int 1576 flow_verbs_get_items_size(const struct rte_flow_item items[]) 1577 { 1578 int size = 0; 1579 1580 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1581 switch (items->type) { 1582 case RTE_FLOW_ITEM_TYPE_VOID: 1583 break; 1584 case RTE_FLOW_ITEM_TYPE_ETH: 1585 size += sizeof(struct ibv_flow_spec_eth); 1586 break; 1587 case RTE_FLOW_ITEM_TYPE_VLAN: 1588 size += sizeof(struct ibv_flow_spec_eth); 1589 break; 1590 case RTE_FLOW_ITEM_TYPE_IPV4: 1591 size += sizeof(struct ibv_flow_spec_ipv4_ext); 1592 break; 1593 case RTE_FLOW_ITEM_TYPE_IPV6: 1594 size += sizeof(struct ibv_flow_spec_ipv6); 1595 break; 1596 case RTE_FLOW_ITEM_TYPE_UDP: 1597 size += sizeof(struct ibv_flow_spec_tcp_udp); 1598 break; 1599 case RTE_FLOW_ITEM_TYPE_TCP: 1600 size += sizeof(struct ibv_flow_spec_tcp_udp); 1601 break; 1602 case RTE_FLOW_ITEM_TYPE_VXLAN: 1603 size += sizeof(struct ibv_flow_spec_tunnel); 1604 break; 1605 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1606 size += sizeof(struct ibv_flow_spec_tunnel); 1607 break; 1608 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1609 case RTE_FLOW_ITEM_TYPE_GRE: 1610 size += sizeof(struct ibv_flow_spec_gre); 1611 break; 1612 case RTE_FLOW_ITEM_TYPE_MPLS: 1613 size += sizeof(struct ibv_flow_spec_mpls); 1614 break; 1615 #else 1616 case RTE_FLOW_ITEM_TYPE_GRE: 1617 size += sizeof(struct ibv_flow_spec_tunnel); 1618 break; 1619 #endif 1620 default: 1621 break; 1622 } 1623 } 1624 return size; 1625 } 1626 1627 /** 1628 * Internal preparation function. Allocate mlx5_flow with the required size. 1629 * The required size is calculate based on the actions and items. This function 1630 * also returns the detected actions and items for later use. 1631 * 1632 * @param[in] dev 1633 * Pointer to Ethernet device. 1634 * @param[in] attr 1635 * Pointer to the flow attributes. 1636 * @param[in] items 1637 * Pointer to the list of items. 1638 * @param[in] actions 1639 * Pointer to the list of actions. 1640 * @param[out] error 1641 * Pointer to the error structure. 1642 * 1643 * @return 1644 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno 1645 * is set. 1646 */ 1647 static struct mlx5_flow * 1648 flow_verbs_prepare(struct rte_eth_dev *dev, 1649 const struct rte_flow_attr *attr __rte_unused, 1650 const struct rte_flow_item items[], 1651 const struct rte_flow_action actions[], 1652 struct rte_flow_error *error) 1653 { 1654 size_t size = 0; 1655 uint32_t handle_idx = 0; 1656 struct mlx5_flow *dev_flow; 1657 struct mlx5_flow_handle *dev_handle; 1658 struct mlx5_priv *priv = dev->data->dev_private; 1659 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1660 1661 MLX5_ASSERT(wks); 1662 size += flow_verbs_get_actions_size(actions); 1663 size += flow_verbs_get_items_size(items); 1664 if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) { 1665 rte_flow_error_set(error, E2BIG, 1666 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1667 "Verbs spec/action size too large"); 1668 return NULL; 1669 } 1670 /* In case of corrupting the memory. */ 1671 if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { 1672 rte_flow_error_set(error, ENOSPC, 1673 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1674 "not free temporary device flow"); 1675 return NULL; 1676 } 1677 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1678 &handle_idx); 1679 if (!dev_handle) { 1680 rte_flow_error_set(error, ENOMEM, 1681 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1682 "not enough memory to create flow handle"); 1683 return NULL; 1684 } 1685 MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows)); 1686 dev_flow = &wks->flows[wks->flow_idx++]; 1687 dev_flow->handle = dev_handle; 1688 dev_flow->handle_idx = handle_idx; 1689 /* Memcpy is used, only size needs to be cleared to 0. */ 1690 dev_flow->verbs.size = 0; 1691 dev_flow->verbs.attr.num_of_specs = 0; 1692 dev_flow->ingress = attr->ingress; 1693 dev_flow->hash_fields = 0; 1694 /* Need to set transfer attribute: not supported in Verbs mode. */ 1695 return dev_flow; 1696 } 1697 1698 /** 1699 * Fill the flow with verb spec. 1700 * 1701 * @param[in] dev 1702 * Pointer to Ethernet device. 1703 * @param[in, out] dev_flow 1704 * Pointer to the mlx5 flow. 1705 * @param[in] attr 1706 * Pointer to the flow attributes. 1707 * @param[in] items 1708 * Pointer to the list of items. 1709 * @param[in] actions 1710 * Pointer to the list of actions. 1711 * @param[out] error 1712 * Pointer to the error structure. 1713 * 1714 * @return 1715 * 0 on success, else a negative errno value otherwise and rte_errno is set. 1716 */ 1717 static int 1718 flow_verbs_translate(struct rte_eth_dev *dev, 1719 struct mlx5_flow *dev_flow, 1720 const struct rte_flow_attr *attr, 1721 const struct rte_flow_item items[], 1722 const struct rte_flow_action actions[], 1723 struct rte_flow_error *error) 1724 { 1725 uint64_t item_flags = 0; 1726 uint64_t action_flags = 0; 1727 uint64_t priority = attr->priority; 1728 uint32_t subpriority = 0; 1729 struct mlx5_priv *priv = dev->data->dev_private; 1730 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1731 struct mlx5_flow_rss_desc *rss_desc; 1732 1733 MLX5_ASSERT(wks); 1734 rss_desc = &wks->rss_desc; 1735 if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1736 priority = priv->config.flow_prio - 1; 1737 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1738 int ret; 1739 1740 switch (actions->type) { 1741 case RTE_FLOW_ACTION_TYPE_VOID: 1742 break; 1743 case RTE_FLOW_ACTION_TYPE_FLAG: 1744 flow_verbs_translate_action_flag(dev_flow, actions); 1745 action_flags |= MLX5_FLOW_ACTION_FLAG; 1746 dev_flow->handle->mark = 1; 1747 break; 1748 case RTE_FLOW_ACTION_TYPE_MARK: 1749 flow_verbs_translate_action_mark(dev_flow, actions); 1750 action_flags |= MLX5_FLOW_ACTION_MARK; 1751 dev_flow->handle->mark = 1; 1752 break; 1753 case RTE_FLOW_ACTION_TYPE_DROP: 1754 flow_verbs_translate_action_drop(dev_flow, actions); 1755 action_flags |= MLX5_FLOW_ACTION_DROP; 1756 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; 1757 break; 1758 case RTE_FLOW_ACTION_TYPE_QUEUE: 1759 flow_verbs_translate_action_queue(rss_desc, actions); 1760 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1761 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 1762 break; 1763 case RTE_FLOW_ACTION_TYPE_RSS: 1764 flow_verbs_translate_action_rss(rss_desc, actions); 1765 action_flags |= MLX5_FLOW_ACTION_RSS; 1766 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 1767 break; 1768 case RTE_FLOW_ACTION_TYPE_COUNT: 1769 ret = flow_verbs_translate_action_count(dev_flow, 1770 actions, 1771 dev, error); 1772 if (ret < 0) 1773 return ret; 1774 action_flags |= MLX5_FLOW_ACTION_COUNT; 1775 break; 1776 default: 1777 return rte_flow_error_set(error, ENOTSUP, 1778 RTE_FLOW_ERROR_TYPE_ACTION, 1779 actions, 1780 "action not supported"); 1781 } 1782 } 1783 dev_flow->act_flags = action_flags; 1784 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1785 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1786 1787 switch (items->type) { 1788 case RTE_FLOW_ITEM_TYPE_VOID: 1789 break; 1790 case RTE_FLOW_ITEM_TYPE_ETH: 1791 flow_verbs_translate_item_eth(dev_flow, items, 1792 item_flags); 1793 subpriority = MLX5_PRIORITY_MAP_L2; 1794 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1795 MLX5_FLOW_LAYER_OUTER_L2; 1796 break; 1797 case RTE_FLOW_ITEM_TYPE_VLAN: 1798 flow_verbs_translate_item_vlan(dev_flow, items, 1799 item_flags); 1800 subpriority = MLX5_PRIORITY_MAP_L2; 1801 item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 1802 MLX5_FLOW_LAYER_INNER_VLAN) : 1803 (MLX5_FLOW_LAYER_OUTER_L2 | 1804 MLX5_FLOW_LAYER_OUTER_VLAN); 1805 break; 1806 case RTE_FLOW_ITEM_TYPE_IPV4: 1807 flow_verbs_translate_item_ipv4(dev_flow, items, 1808 item_flags); 1809 subpriority = MLX5_PRIORITY_MAP_L3; 1810 dev_flow->hash_fields |= 1811 mlx5_flow_hashfields_adjust 1812 (rss_desc, tunnel, 1813 MLX5_IPV4_LAYER_TYPES, 1814 MLX5_IPV4_IBV_RX_HASH); 1815 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1816 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1817 break; 1818 case RTE_FLOW_ITEM_TYPE_IPV6: 1819 flow_verbs_translate_item_ipv6(dev_flow, items, 1820 item_flags); 1821 subpriority = MLX5_PRIORITY_MAP_L3; 1822 dev_flow->hash_fields |= 1823 mlx5_flow_hashfields_adjust 1824 (rss_desc, tunnel, 1825 MLX5_IPV6_LAYER_TYPES, 1826 MLX5_IPV6_IBV_RX_HASH); 1827 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1828 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1829 break; 1830 case RTE_FLOW_ITEM_TYPE_TCP: 1831 flow_verbs_translate_item_tcp(dev_flow, items, 1832 item_flags); 1833 subpriority = MLX5_PRIORITY_MAP_L4; 1834 if (dev_flow->hash_fields != 0) 1835 dev_flow->hash_fields |= 1836 mlx5_flow_hashfields_adjust 1837 (rss_desc, tunnel, ETH_RSS_TCP, 1838 (IBV_RX_HASH_SRC_PORT_TCP | 1839 IBV_RX_HASH_DST_PORT_TCP)); 1840 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 1841 MLX5_FLOW_LAYER_OUTER_L4_TCP; 1842 break; 1843 case RTE_FLOW_ITEM_TYPE_UDP: 1844 flow_verbs_translate_item_udp(dev_flow, items, 1845 item_flags); 1846 subpriority = MLX5_PRIORITY_MAP_L4; 1847 if (dev_flow->hash_fields != 0) 1848 dev_flow->hash_fields |= 1849 mlx5_flow_hashfields_adjust 1850 (rss_desc, tunnel, ETH_RSS_UDP, 1851 (IBV_RX_HASH_SRC_PORT_UDP | 1852 IBV_RX_HASH_DST_PORT_UDP)); 1853 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 1854 MLX5_FLOW_LAYER_OUTER_L4_UDP; 1855 break; 1856 case RTE_FLOW_ITEM_TYPE_VXLAN: 1857 flow_verbs_translate_item_vxlan(dev_flow, items, 1858 item_flags); 1859 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1860 item_flags |= MLX5_FLOW_LAYER_VXLAN; 1861 break; 1862 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1863 flow_verbs_translate_item_vxlan_gpe(dev_flow, items, 1864 item_flags); 1865 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1866 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE; 1867 break; 1868 case RTE_FLOW_ITEM_TYPE_GRE: 1869 flow_verbs_translate_item_gre(dev_flow, items, 1870 item_flags); 1871 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1872 item_flags |= MLX5_FLOW_LAYER_GRE; 1873 break; 1874 case RTE_FLOW_ITEM_TYPE_MPLS: 1875 flow_verbs_translate_item_mpls(dev_flow, items, 1876 item_flags); 1877 subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); 1878 item_flags |= MLX5_FLOW_LAYER_MPLS; 1879 break; 1880 default: 1881 return rte_flow_error_set(error, ENOTSUP, 1882 RTE_FLOW_ERROR_TYPE_ITEM, 1883 NULL, "item not supported"); 1884 } 1885 } 1886 dev_flow->handle->layers = item_flags; 1887 /* Other members of attr will be ignored. */ 1888 dev_flow->verbs.attr.priority = 1889 mlx5_flow_adjust_priority(dev, priority, subpriority); 1890 dev_flow->verbs.attr.port = (uint8_t)priv->dev_port; 1891 return 0; 1892 } 1893 1894 /** 1895 * Remove the flow from the NIC but keeps it in memory. 1896 * 1897 * @param[in] dev 1898 * Pointer to the Ethernet device structure. 1899 * @param[in, out] flow 1900 * Pointer to flow structure. 1901 */ 1902 static void 1903 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 1904 { 1905 struct mlx5_priv *priv = dev->data->dev_private; 1906 struct mlx5_flow_handle *handle; 1907 uint32_t handle_idx; 1908 1909 if (!flow) 1910 return; 1911 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1912 handle_idx, handle, next) { 1913 if (handle->drv_flow) { 1914 claim_zero(mlx5_glue->destroy_flow(handle->drv_flow)); 1915 handle->drv_flow = NULL; 1916 } 1917 /* hrxq is union, don't touch it only the flag is set. */ 1918 if (handle->rix_hrxq && 1919 handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1920 mlx5_hrxq_release(dev, handle->rix_hrxq); 1921 handle->rix_hrxq = 0; 1922 } 1923 if (handle->vf_vlan.tag && handle->vf_vlan.created) 1924 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); 1925 } 1926 } 1927 1928 /** 1929 * Remove the flow from the NIC and the memory. 1930 * 1931 * @param[in] dev 1932 * Pointer to the Ethernet device structure. 1933 * @param[in, out] flow 1934 * Pointer to flow structure. 1935 */ 1936 static void 1937 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 1938 { 1939 struct mlx5_priv *priv = dev->data->dev_private; 1940 struct mlx5_flow_handle *handle; 1941 1942 if (!flow) 1943 return; 1944 flow_verbs_remove(dev, flow); 1945 while (flow->dev_handles) { 1946 uint32_t tmp_idx = flow->dev_handles; 1947 1948 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1949 tmp_idx); 1950 if (!handle) 1951 return; 1952 flow->dev_handles = handle->next.next; 1953 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 1954 tmp_idx); 1955 } 1956 if (flow->counter) { 1957 flow_verbs_counter_release(dev, flow->counter); 1958 flow->counter = 0; 1959 } 1960 } 1961 1962 /** 1963 * Apply the flow to the NIC. 1964 * 1965 * @param[in] dev 1966 * Pointer to the Ethernet device structure. 1967 * @param[in, out] flow 1968 * Pointer to flow structure. 1969 * @param[out] error 1970 * Pointer to error structure. 1971 * 1972 * @return 1973 * 0 on success, a negative errno value otherwise and rte_errno is set. 1974 */ 1975 static int 1976 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 1977 struct rte_flow_error *error) 1978 { 1979 struct mlx5_priv *priv = dev->data->dev_private; 1980 struct mlx5_flow_handle *handle; 1981 struct mlx5_flow *dev_flow; 1982 struct mlx5_hrxq *hrxq; 1983 uint32_t dev_handles; 1984 int err; 1985 int idx; 1986 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1987 1988 MLX5_ASSERT(wks); 1989 for (idx = wks->flow_idx - 1; idx >= 0; idx--) { 1990 dev_flow = &wks->flows[idx]; 1991 handle = dev_flow->handle; 1992 if (handle->fate_action == MLX5_FLOW_FATE_DROP) { 1993 MLX5_ASSERT(priv->drop_queue.hrxq); 1994 hrxq = priv->drop_queue.hrxq; 1995 } else { 1996 uint32_t hrxq_idx; 1997 struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc; 1998 1999 MLX5_ASSERT(rss_desc->queue_num); 2000 rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; 2001 rss_desc->hash_fields = dev_flow->hash_fields; 2002 rss_desc->tunnel = !!(handle->layers & 2003 MLX5_FLOW_LAYER_TUNNEL); 2004 rss_desc->shared_rss = 0; 2005 hrxq_idx = mlx5_hrxq_get(dev, rss_desc); 2006 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 2007 hrxq_idx); 2008 if (!hrxq) { 2009 rte_flow_error_set 2010 (error, rte_errno, 2011 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2012 "cannot get hash queue"); 2013 goto error; 2014 } 2015 handle->rix_hrxq = hrxq_idx; 2016 } 2017 MLX5_ASSERT(hrxq); 2018 handle->drv_flow = mlx5_glue->create_flow 2019 (hrxq->qp, &dev_flow->verbs.attr); 2020 if (!handle->drv_flow) { 2021 rte_flow_error_set(error, errno, 2022 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2023 NULL, 2024 "hardware refuses to create flow"); 2025 goto error; 2026 } 2027 if (priv->vmwa_context && 2028 handle->vf_vlan.tag && !handle->vf_vlan.created) { 2029 /* 2030 * The rule contains the VLAN pattern. 2031 * For VF we are going to create VLAN 2032 * interface to make hypervisor set correct 2033 * e-Switch vport context. 2034 */ 2035 mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan); 2036 } 2037 } 2038 return 0; 2039 error: 2040 err = rte_errno; /* Save rte_errno before cleanup. */ 2041 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2042 dev_handles, handle, next) { 2043 /* hrxq is union, don't touch it only the flag is set. */ 2044 if (handle->rix_hrxq && 2045 handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 2046 mlx5_hrxq_release(dev, handle->rix_hrxq); 2047 handle->rix_hrxq = 0; 2048 } 2049 if (handle->vf_vlan.tag && handle->vf_vlan.created) 2050 mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); 2051 } 2052 rte_errno = err; /* Restore rte_errno. */ 2053 return -rte_errno; 2054 } 2055 2056 /** 2057 * Query a flow. 2058 * 2059 * @see rte_flow_query() 2060 * @see rte_flow_ops 2061 */ 2062 static int 2063 flow_verbs_query(struct rte_eth_dev *dev, 2064 struct rte_flow *flow, 2065 const struct rte_flow_action *actions, 2066 void *data, 2067 struct rte_flow_error *error) 2068 { 2069 int ret = -EINVAL; 2070 2071 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2072 switch (actions->type) { 2073 case RTE_FLOW_ACTION_TYPE_VOID: 2074 break; 2075 case RTE_FLOW_ACTION_TYPE_COUNT: 2076 ret = flow_verbs_counter_query(dev, flow, data, error); 2077 break; 2078 default: 2079 return rte_flow_error_set(error, ENOTSUP, 2080 RTE_FLOW_ERROR_TYPE_ACTION, 2081 actions, 2082 "action not supported"); 2083 } 2084 } 2085 return ret; 2086 } 2087 2088 static int 2089 flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains, 2090 uint32_t flags) 2091 { 2092 RTE_SET_USED(dev); 2093 RTE_SET_USED(domains); 2094 RTE_SET_USED(flags); 2095 2096 return 0; 2097 } 2098 2099 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = { 2100 .validate = flow_verbs_validate, 2101 .prepare = flow_verbs_prepare, 2102 .translate = flow_verbs_translate, 2103 .apply = flow_verbs_apply, 2104 .remove = flow_verbs_remove, 2105 .destroy = flow_verbs_destroy, 2106 .query = flow_verbs_query, 2107 .sync_domain = flow_verbs_sync_domain, 2108 }; 2109