1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <netinet/in.h> 6 #include <sys/queue.h> 7 #include <stdalign.h> 8 #include <stdint.h> 9 #include <string.h> 10 11 /* Verbs header. */ 12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 13 #ifdef PEDANTIC 14 #pragma GCC diagnostic ignored "-Wpedantic" 15 #endif 16 #include <infiniband/verbs.h> 17 #ifdef PEDANTIC 18 #pragma GCC diagnostic error "-Wpedantic" 19 #endif 20 21 #include <rte_common.h> 22 #include <rte_ether.h> 23 #include <rte_ethdev_driver.h> 24 #include <rte_flow.h> 25 #include <rte_flow_driver.h> 26 #include <rte_malloc.h> 27 #include <rte_ip.h> 28 29 #include <mlx5_glue.h> 30 #include <mlx5_prm.h> 31 32 #include "mlx5_defs.h" 33 #include "mlx5.h" 34 #include "mlx5_flow.h" 35 #include "mlx5_rxtx.h" 36 37 #define VERBS_SPEC_INNER(item_flags) \ 38 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0) 39 40 /** 41 * Create Verbs flow counter with Verbs library. 42 * 43 * @param[in] dev 44 * Pointer to the Ethernet device structure. 45 * @param[in, out] counter 46 * mlx5 flow counter object, contains the counter id, 47 * handle of created Verbs flow counter is returned 48 * in cs field (if counters are supported). 49 * 50 * @return 51 * 0 On success else a negative errno value is returned 52 * and rte_errno is set. 53 */ 54 static int 55 flow_verbs_counter_create(struct rte_eth_dev *dev, 56 struct mlx5_flow_counter *counter) 57 { 58 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 59 struct mlx5_priv *priv = dev->data->dev_private; 60 struct ibv_context *ctx = priv->sh->ctx; 61 struct ibv_counter_set_init_attr init = { 62 .counter_set_id = counter->id}; 63 64 counter->cs = mlx5_glue->create_counter_set(ctx, &init); 65 if (!counter->cs) { 66 rte_errno = ENOTSUP; 67 return -ENOTSUP; 68 } 69 return 0; 70 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 71 struct mlx5_priv *priv = dev->data->dev_private; 72 struct ibv_context *ctx = priv->sh->ctx; 73 struct ibv_counters_init_attr init = {0}; 74 struct ibv_counter_attach_attr attach; 75 int ret; 76 77 memset(&attach, 0, sizeof(attach)); 78 counter->cs = mlx5_glue->create_counters(ctx, &init); 79 if (!counter->cs) { 80 rte_errno = ENOTSUP; 81 return -ENOTSUP; 82 } 83 attach.counter_desc = IBV_COUNTER_PACKETS; 84 attach.index = 0; 85 ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL); 86 if (!ret) { 87 attach.counter_desc = IBV_COUNTER_BYTES; 88 attach.index = 1; 89 ret = mlx5_glue->attach_counters 90 (counter->cs, &attach, NULL); 91 } 92 if (ret) { 93 claim_zero(mlx5_glue->destroy_counters(counter->cs)); 94 counter->cs = NULL; 95 rte_errno = ret; 96 return -ret; 97 } 98 return 0; 99 #else 100 (void)dev; 101 (void)counter; 102 rte_errno = ENOTSUP; 103 return -ENOTSUP; 104 #endif 105 } 106 107 /** 108 * Get a flow counter. 109 * 110 * @param[in] dev 111 * Pointer to the Ethernet device structure. 112 * @param[in] shared 113 * Indicate if this counter is shared with other flows. 114 * @param[in] id 115 * Counter identifier. 116 * 117 * @return 118 * A pointer to the counter, NULL otherwise and rte_errno is set. 119 */ 120 static struct mlx5_flow_counter * 121 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) 122 { 123 struct mlx5_priv *priv = dev->data->dev_private; 124 struct mlx5_flow_counter *cnt; 125 int ret; 126 127 if (shared) { 128 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) { 129 if (cnt->shared && cnt->id == id) { 130 cnt->ref_cnt++; 131 return cnt; 132 } 133 } 134 } 135 cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0); 136 if (!cnt) { 137 rte_errno = ENOMEM; 138 return NULL; 139 } 140 cnt->id = id; 141 cnt->shared = shared; 142 cnt->ref_cnt = 1; 143 cnt->hits = 0; 144 cnt->bytes = 0; 145 /* Create counter with Verbs. */ 146 ret = flow_verbs_counter_create(dev, cnt); 147 if (!ret) { 148 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next); 149 return cnt; 150 } 151 /* Some error occurred in Verbs library. */ 152 rte_free(cnt); 153 rte_errno = -ret; 154 return NULL; 155 } 156 157 /** 158 * Release a flow counter. 159 * 160 * @param[in] dev 161 * Pointer to the Ethernet device structure. 162 * @param[in] counter 163 * Pointer to the counter handler. 164 */ 165 static void 166 flow_verbs_counter_release(struct rte_eth_dev *dev, 167 struct mlx5_flow_counter *counter) 168 { 169 struct mlx5_priv *priv = dev->data->dev_private; 170 171 if (--counter->ref_cnt == 0) { 172 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 173 claim_zero(mlx5_glue->destroy_counter_set(counter->cs)); 174 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 175 claim_zero(mlx5_glue->destroy_counters(counter->cs)); 176 #endif 177 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next); 178 rte_free(counter); 179 } 180 } 181 182 /** 183 * Query a flow counter via Verbs library call. 184 * 185 * @see rte_flow_query() 186 * @see rte_flow_ops 187 */ 188 static int 189 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, 190 struct rte_flow *flow, void *data, 191 struct rte_flow_error *error) 192 { 193 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 194 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 195 if (flow->counter && flow->counter->cs) { 196 struct rte_flow_query_count *qc = data; 197 uint64_t counters[2] = {0, 0}; 198 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 199 struct ibv_query_counter_set_attr query_cs_attr = { 200 .cs = flow->counter->cs, 201 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, 202 }; 203 struct ibv_counter_set_data query_out = { 204 .out = counters, 205 .outlen = 2 * sizeof(uint64_t), 206 }; 207 int err = mlx5_glue->query_counter_set(&query_cs_attr, 208 &query_out); 209 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 210 int err = mlx5_glue->query_counters 211 (flow->counter->cs, counters, 212 RTE_DIM(counters), 213 IBV_READ_COUNTERS_ATTR_PREFER_CACHED); 214 #endif 215 if (err) 216 return rte_flow_error_set 217 (error, err, 218 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 219 NULL, 220 "cannot read counter"); 221 qc->hits_set = 1; 222 qc->bytes_set = 1; 223 qc->hits = counters[0] - flow->counter->hits; 224 qc->bytes = counters[1] - flow->counter->bytes; 225 if (qc->reset) { 226 flow->counter->hits = counters[0]; 227 flow->counter->bytes = counters[1]; 228 } 229 return 0; 230 } 231 return rte_flow_error_set(error, EINVAL, 232 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 233 NULL, 234 "flow does not have counter"); 235 #else 236 (void)flow; 237 (void)data; 238 return rte_flow_error_set(error, ENOTSUP, 239 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 240 NULL, 241 "counters are not available"); 242 #endif 243 } 244 245 /** 246 * Add a verbs item specification into @p verbs. 247 * 248 * @param[out] verbs 249 * Pointer to verbs structure. 250 * @param[in] src 251 * Create specification. 252 * @param[in] size 253 * Size in bytes of the specification to copy. 254 */ 255 static void 256 flow_verbs_spec_add(struct mlx5_flow_verbs *verbs, void *src, unsigned int size) 257 { 258 void *dst; 259 260 if (!verbs) 261 return; 262 MLX5_ASSERT(verbs->specs); 263 dst = (void *)(verbs->specs + verbs->size); 264 memcpy(dst, src, size); 265 ++verbs->attr->num_of_specs; 266 verbs->size += size; 267 } 268 269 /** 270 * Convert the @p item into a Verbs specification. This function assumes that 271 * the input is valid and that there is space to insert the requested item 272 * into the flow. 273 * 274 * @param[in, out] dev_flow 275 * Pointer to dev_flow structure. 276 * @param[in] item 277 * Item specification. 278 * @param[in] item_flags 279 * Parsed item flags. 280 */ 281 static void 282 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow, 283 const struct rte_flow_item *item, 284 uint64_t item_flags) 285 { 286 const struct rte_flow_item_eth *spec = item->spec; 287 const struct rte_flow_item_eth *mask = item->mask; 288 const unsigned int size = sizeof(struct ibv_flow_spec_eth); 289 struct ibv_flow_spec_eth eth = { 290 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags), 291 .size = size, 292 }; 293 294 if (!mask) 295 mask = &rte_flow_item_eth_mask; 296 if (spec) { 297 unsigned int i; 298 299 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, 300 RTE_ETHER_ADDR_LEN); 301 memcpy(ð.val.src_mac, spec->src.addr_bytes, 302 RTE_ETHER_ADDR_LEN); 303 eth.val.ether_type = spec->type; 304 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, 305 RTE_ETHER_ADDR_LEN); 306 memcpy(ð.mask.src_mac, mask->src.addr_bytes, 307 RTE_ETHER_ADDR_LEN); 308 eth.mask.ether_type = mask->type; 309 /* Remove unwanted bits from values. */ 310 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) { 311 eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; 312 eth.val.src_mac[i] &= eth.mask.src_mac[i]; 313 } 314 eth.val.ether_type &= eth.mask.ether_type; 315 } 316 flow_verbs_spec_add(&dev_flow->verbs, ð, size); 317 } 318 319 /** 320 * Update the VLAN tag in the Verbs Ethernet specification. 321 * This function assumes that the input is valid and there is space to add 322 * the requested item. 323 * 324 * @param[in, out] attr 325 * Pointer to Verbs attributes structure. 326 * @param[in] eth 327 * Verbs structure containing the VLAN information to copy. 328 */ 329 static void 330 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr, 331 struct ibv_flow_spec_eth *eth) 332 { 333 unsigned int i; 334 const enum ibv_flow_spec_type search = eth->type; 335 struct ibv_spec_header *hdr = (struct ibv_spec_header *) 336 ((uint8_t *)attr + sizeof(struct ibv_flow_attr)); 337 338 for (i = 0; i != attr->num_of_specs; ++i) { 339 if (hdr->type == search) { 340 struct ibv_flow_spec_eth *e = 341 (struct ibv_flow_spec_eth *)hdr; 342 343 e->val.vlan_tag = eth->val.vlan_tag; 344 e->mask.vlan_tag = eth->mask.vlan_tag; 345 e->val.ether_type = eth->val.ether_type; 346 e->mask.ether_type = eth->mask.ether_type; 347 break; 348 } 349 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size); 350 } 351 } 352 353 /** 354 * Convert the @p item into a Verbs specification. This function assumes that 355 * the input is valid and that there is space to insert the requested item 356 * into the flow. 357 * 358 * @param[in, out] dev_flow 359 * Pointer to dev_flow structure. 360 * @param[in] item 361 * Item specification. 362 * @param[in] item_flags 363 * Parsed item flags. 364 */ 365 static void 366 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow, 367 const struct rte_flow_item *item, 368 uint64_t item_flags) 369 { 370 const struct rte_flow_item_vlan *spec = item->spec; 371 const struct rte_flow_item_vlan *mask = item->mask; 372 unsigned int size = sizeof(struct ibv_flow_spec_eth); 373 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 374 struct ibv_flow_spec_eth eth = { 375 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags), 376 .size = size, 377 }; 378 const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 379 MLX5_FLOW_LAYER_OUTER_L2; 380 381 if (!mask) 382 mask = &rte_flow_item_vlan_mask; 383 if (spec) { 384 eth.val.vlan_tag = spec->tci; 385 eth.mask.vlan_tag = mask->tci; 386 eth.val.vlan_tag &= eth.mask.vlan_tag; 387 eth.val.ether_type = spec->inner_type; 388 eth.mask.ether_type = mask->inner_type; 389 eth.val.ether_type &= eth.mask.ether_type; 390 } 391 if (!(item_flags & l2m)) 392 flow_verbs_spec_add(&dev_flow->verbs, ð, size); 393 else 394 flow_verbs_item_vlan_update(dev_flow->verbs.attr, ð); 395 if (!tunnel) 396 dev_flow->verbs.vf_vlan.tag = 397 rte_be_to_cpu_16(spec->tci) & 0x0fff; 398 } 399 400 /** 401 * Convert the @p item into a Verbs specification. This function assumes that 402 * the input is valid and that there is space to insert the requested item 403 * into the flow. 404 * 405 * @param[in, out] dev_flow 406 * Pointer to dev_flow structure. 407 * @param[in] item 408 * Item specification. 409 * @param[in] item_flags 410 * Parsed item flags. 411 */ 412 static void 413 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow, 414 const struct rte_flow_item *item, 415 uint64_t item_flags) 416 { 417 const struct rte_flow_item_ipv4 *spec = item->spec; 418 const struct rte_flow_item_ipv4 *mask = item->mask; 419 unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext); 420 struct ibv_flow_spec_ipv4_ext ipv4 = { 421 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags), 422 .size = size, 423 }; 424 425 if (!mask) 426 mask = &rte_flow_item_ipv4_mask; 427 if (spec) { 428 ipv4.val = (struct ibv_flow_ipv4_ext_filter){ 429 .src_ip = spec->hdr.src_addr, 430 .dst_ip = spec->hdr.dst_addr, 431 .proto = spec->hdr.next_proto_id, 432 .tos = spec->hdr.type_of_service, 433 }; 434 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ 435 .src_ip = mask->hdr.src_addr, 436 .dst_ip = mask->hdr.dst_addr, 437 .proto = mask->hdr.next_proto_id, 438 .tos = mask->hdr.type_of_service, 439 }; 440 /* Remove unwanted bits from values. */ 441 ipv4.val.src_ip &= ipv4.mask.src_ip; 442 ipv4.val.dst_ip &= ipv4.mask.dst_ip; 443 ipv4.val.proto &= ipv4.mask.proto; 444 ipv4.val.tos &= ipv4.mask.tos; 445 } 446 flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size); 447 } 448 449 /** 450 * Convert the @p item into a Verbs specification. This function assumes that 451 * the input is valid and that there is space to insert the requested item 452 * into the flow. 453 * 454 * @param[in, out] dev_flow 455 * Pointer to dev_flow structure. 456 * @param[in] item 457 * Item specification. 458 * @param[in] item_flags 459 * Parsed item flags. 460 */ 461 static void 462 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow, 463 const struct rte_flow_item *item, 464 uint64_t item_flags) 465 { 466 const struct rte_flow_item_ipv6 *spec = item->spec; 467 const struct rte_flow_item_ipv6 *mask = item->mask; 468 unsigned int size = sizeof(struct ibv_flow_spec_ipv6); 469 struct ibv_flow_spec_ipv6 ipv6 = { 470 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags), 471 .size = size, 472 }; 473 474 if (!mask) 475 mask = &rte_flow_item_ipv6_mask; 476 if (spec) { 477 unsigned int i; 478 uint32_t vtc_flow_val; 479 uint32_t vtc_flow_mask; 480 481 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, 482 RTE_DIM(ipv6.val.src_ip)); 483 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, 484 RTE_DIM(ipv6.val.dst_ip)); 485 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, 486 RTE_DIM(ipv6.mask.src_ip)); 487 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, 488 RTE_DIM(ipv6.mask.dst_ip)); 489 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); 490 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); 491 ipv6.val.flow_label = 492 rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >> 493 RTE_IPV6_HDR_FL_SHIFT); 494 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >> 495 RTE_IPV6_HDR_TC_SHIFT; 496 ipv6.val.next_hdr = spec->hdr.proto; 497 ipv6.mask.flow_label = 498 rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >> 499 RTE_IPV6_HDR_FL_SHIFT); 500 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >> 501 RTE_IPV6_HDR_TC_SHIFT; 502 ipv6.mask.next_hdr = mask->hdr.proto; 503 /* Remove unwanted bits from values. */ 504 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { 505 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; 506 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; 507 } 508 ipv6.val.flow_label &= ipv6.mask.flow_label; 509 ipv6.val.traffic_class &= ipv6.mask.traffic_class; 510 ipv6.val.next_hdr &= ipv6.mask.next_hdr; 511 } 512 flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size); 513 } 514 515 /** 516 * Convert the @p item into a Verbs specification. This function assumes that 517 * the input is valid and that there is space to insert the requested item 518 * into the flow. 519 * 520 * @param[in, out] dev_flow 521 * Pointer to dev_flow structure. 522 * @param[in] item 523 * Item specification. 524 * @param[in] item_flags 525 * Parsed item flags. 526 */ 527 static void 528 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow, 529 const struct rte_flow_item *item, 530 uint64_t item_flags __rte_unused) 531 { 532 const struct rte_flow_item_tcp *spec = item->spec; 533 const struct rte_flow_item_tcp *mask = item->mask; 534 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp); 535 struct ibv_flow_spec_tcp_udp tcp = { 536 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags), 537 .size = size, 538 }; 539 540 if (!mask) 541 mask = &rte_flow_item_tcp_mask; 542 if (spec) { 543 tcp.val.dst_port = spec->hdr.dst_port; 544 tcp.val.src_port = spec->hdr.src_port; 545 tcp.mask.dst_port = mask->hdr.dst_port; 546 tcp.mask.src_port = mask->hdr.src_port; 547 /* Remove unwanted bits from values. */ 548 tcp.val.src_port &= tcp.mask.src_port; 549 tcp.val.dst_port &= tcp.mask.dst_port; 550 } 551 flow_verbs_spec_add(&dev_flow->verbs, &tcp, size); 552 } 553 554 /** 555 * Convert the @p item into a Verbs specification. This function assumes that 556 * the input is valid and that there is space to insert the requested item 557 * into the flow. 558 * 559 * @param[in, out] dev_flow 560 * Pointer to dev_flow structure. 561 * @param[in] item 562 * Item specification. 563 * @param[in] item_flags 564 * Parsed item flags. 565 */ 566 static void 567 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow, 568 const struct rte_flow_item *item, 569 uint64_t item_flags __rte_unused) 570 { 571 const struct rte_flow_item_udp *spec = item->spec; 572 const struct rte_flow_item_udp *mask = item->mask; 573 unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp); 574 struct ibv_flow_spec_tcp_udp udp = { 575 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags), 576 .size = size, 577 }; 578 579 if (!mask) 580 mask = &rte_flow_item_udp_mask; 581 if (spec) { 582 udp.val.dst_port = spec->hdr.dst_port; 583 udp.val.src_port = spec->hdr.src_port; 584 udp.mask.dst_port = mask->hdr.dst_port; 585 udp.mask.src_port = mask->hdr.src_port; 586 /* Remove unwanted bits from values. */ 587 udp.val.src_port &= udp.mask.src_port; 588 udp.val.dst_port &= udp.mask.dst_port; 589 } 590 flow_verbs_spec_add(&dev_flow->verbs, &udp, size); 591 } 592 593 /** 594 * Convert the @p item into a Verbs specification. This function assumes that 595 * the input is valid and that there is space to insert the requested item 596 * into the flow. 597 * 598 * @param[in, out] dev_flow 599 * Pointer to dev_flow structure. 600 * @param[in] item 601 * Item specification. 602 * @param[in] item_flags 603 * Parsed item flags. 604 */ 605 static void 606 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow, 607 const struct rte_flow_item *item, 608 uint64_t item_flags __rte_unused) 609 { 610 const struct rte_flow_item_vxlan *spec = item->spec; 611 const struct rte_flow_item_vxlan *mask = item->mask; 612 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 613 struct ibv_flow_spec_tunnel vxlan = { 614 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 615 .size = size, 616 }; 617 union vni { 618 uint32_t vlan_id; 619 uint8_t vni[4]; 620 } id = { .vlan_id = 0, }; 621 622 if (!mask) 623 mask = &rte_flow_item_vxlan_mask; 624 if (spec) { 625 memcpy(&id.vni[1], spec->vni, 3); 626 vxlan.val.tunnel_id = id.vlan_id; 627 memcpy(&id.vni[1], mask->vni, 3); 628 vxlan.mask.tunnel_id = id.vlan_id; 629 /* Remove unwanted bits from values. */ 630 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; 631 } 632 flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size); 633 } 634 635 /** 636 * Convert the @p item into a Verbs specification. This function assumes that 637 * the input is valid and that there is space to insert the requested item 638 * into the flow. 639 * 640 * @param[in, out] dev_flow 641 * Pointer to dev_flow structure. 642 * @param[in] item 643 * Item specification. 644 * @param[in] item_flags 645 * Parsed item flags. 646 */ 647 static void 648 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow, 649 const struct rte_flow_item *item, 650 uint64_t item_flags __rte_unused) 651 { 652 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 653 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 654 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 655 struct ibv_flow_spec_tunnel vxlan_gpe = { 656 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 657 .size = size, 658 }; 659 union vni { 660 uint32_t vlan_id; 661 uint8_t vni[4]; 662 } id = { .vlan_id = 0, }; 663 664 if (!mask) 665 mask = &rte_flow_item_vxlan_gpe_mask; 666 if (spec) { 667 memcpy(&id.vni[1], spec->vni, 3); 668 vxlan_gpe.val.tunnel_id = id.vlan_id; 669 memcpy(&id.vni[1], mask->vni, 3); 670 vxlan_gpe.mask.tunnel_id = id.vlan_id; 671 /* Remove unwanted bits from values. */ 672 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id; 673 } 674 flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size); 675 } 676 677 /** 678 * Update the protocol in Verbs IPv4/IPv6 spec. 679 * 680 * @param[in, out] attr 681 * Pointer to Verbs attributes structure. 682 * @param[in] search 683 * Specification type to search in order to update the IP protocol. 684 * @param[in] protocol 685 * Protocol value to set if none is present in the specification. 686 */ 687 static void 688 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr, 689 enum ibv_flow_spec_type search, 690 uint8_t protocol) 691 { 692 unsigned int i; 693 struct ibv_spec_header *hdr = (struct ibv_spec_header *) 694 ((uint8_t *)attr + sizeof(struct ibv_flow_attr)); 695 696 if (!attr) 697 return; 698 for (i = 0; i != attr->num_of_specs; ++i) { 699 if (hdr->type == search) { 700 union { 701 struct ibv_flow_spec_ipv4_ext *ipv4; 702 struct ibv_flow_spec_ipv6 *ipv6; 703 } ip; 704 705 switch (search) { 706 case IBV_FLOW_SPEC_IPV4_EXT: 707 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr; 708 if (!ip.ipv4->val.proto) { 709 ip.ipv4->val.proto = protocol; 710 ip.ipv4->mask.proto = 0xff; 711 } 712 break; 713 case IBV_FLOW_SPEC_IPV6: 714 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr; 715 if (!ip.ipv6->val.next_hdr) { 716 ip.ipv6->val.next_hdr = protocol; 717 ip.ipv6->mask.next_hdr = 0xff; 718 } 719 break; 720 default: 721 break; 722 } 723 break; 724 } 725 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size); 726 } 727 } 728 729 /** 730 * Convert the @p item into a Verbs specification. This function assumes that 731 * the input is valid and that there is space to insert the requested item 732 * into the flow. 733 * 734 * @param[in, out] dev_flow 735 * Pointer to dev_flow structure. 736 * @param[in] item 737 * Item specification. 738 * @param[in] item_flags 739 * Parsed item flags. 740 */ 741 static void 742 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow, 743 const struct rte_flow_item *item __rte_unused, 744 uint64_t item_flags) 745 { 746 struct mlx5_flow_verbs *verbs = &dev_flow->verbs; 747 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 748 unsigned int size = sizeof(struct ibv_flow_spec_tunnel); 749 struct ibv_flow_spec_tunnel tunnel = { 750 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL, 751 .size = size, 752 }; 753 #else 754 const struct rte_flow_item_gre *spec = item->spec; 755 const struct rte_flow_item_gre *mask = item->mask; 756 unsigned int size = sizeof(struct ibv_flow_spec_gre); 757 struct ibv_flow_spec_gre tunnel = { 758 .type = IBV_FLOW_SPEC_GRE, 759 .size = size, 760 }; 761 762 if (!mask) 763 mask = &rte_flow_item_gre_mask; 764 if (spec) { 765 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver; 766 tunnel.val.protocol = spec->protocol; 767 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver; 768 tunnel.mask.protocol = mask->protocol; 769 /* Remove unwanted bits from values. */ 770 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver; 771 tunnel.val.protocol &= tunnel.mask.protocol; 772 tunnel.val.key &= tunnel.mask.key; 773 } 774 #endif 775 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) 776 flow_verbs_item_gre_ip_protocol_update(verbs->attr, 777 IBV_FLOW_SPEC_IPV4_EXT, 778 IPPROTO_GRE); 779 else 780 flow_verbs_item_gre_ip_protocol_update(verbs->attr, 781 IBV_FLOW_SPEC_IPV6, 782 IPPROTO_GRE); 783 flow_verbs_spec_add(verbs, &tunnel, size); 784 } 785 786 /** 787 * Convert the @p action into a Verbs specification. This function assumes that 788 * the input is valid and that there is space to insert the requested action 789 * into the flow. This function also return the action that was added. 790 * 791 * @param[in, out] dev_flow 792 * Pointer to dev_flow structure. 793 * @param[in] item 794 * Item specification. 795 * @param[in] item_flags 796 * Parsed item flags. 797 */ 798 static void 799 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused, 800 const struct rte_flow_item *item __rte_unused, 801 uint64_t item_flags __rte_unused) 802 { 803 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 804 const struct rte_flow_item_mpls *spec = item->spec; 805 const struct rte_flow_item_mpls *mask = item->mask; 806 unsigned int size = sizeof(struct ibv_flow_spec_mpls); 807 struct ibv_flow_spec_mpls mpls = { 808 .type = IBV_FLOW_SPEC_MPLS, 809 .size = size, 810 }; 811 812 if (!mask) 813 mask = &rte_flow_item_mpls_mask; 814 if (spec) { 815 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label)); 816 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label)); 817 /* Remove unwanted bits from values. */ 818 mpls.val.label &= mpls.mask.label; 819 } 820 flow_verbs_spec_add(&dev_flow->verbs, &mpls, size); 821 #endif 822 } 823 824 /** 825 * Convert the @p action into a Verbs specification. This function assumes that 826 * the input is valid and that there is space to insert the requested action 827 * into the flow. 828 * 829 * @param[in] dev_flow 830 * Pointer to mlx5_flow. 831 * @param[in] action 832 * Action configuration. 833 */ 834 static void 835 flow_verbs_translate_action_drop 836 (struct mlx5_flow *dev_flow, 837 const struct rte_flow_action *action __rte_unused) 838 { 839 unsigned int size = sizeof(struct ibv_flow_spec_action_drop); 840 struct ibv_flow_spec_action_drop drop = { 841 .type = IBV_FLOW_SPEC_ACTION_DROP, 842 .size = size, 843 }; 844 845 flow_verbs_spec_add(&dev_flow->verbs, &drop, size); 846 } 847 848 /** 849 * Convert the @p action into a Verbs specification. This function assumes that 850 * the input is valid and that there is space to insert the requested action 851 * into the flow. 852 * 853 * @param[in] dev_flow 854 * Pointer to mlx5_flow. 855 * @param[in] action 856 * Action configuration. 857 */ 858 static void 859 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow, 860 const struct rte_flow_action *action) 861 { 862 const struct rte_flow_action_queue *queue = action->conf; 863 struct rte_flow *flow = dev_flow->flow; 864 865 if (flow->rss.queue) 866 (*flow->rss.queue)[0] = queue->index; 867 flow->rss.queue_num = 1; 868 } 869 870 /** 871 * Convert the @p action into a Verbs specification. This function assumes that 872 * the input is valid and that there is space to insert the requested action 873 * into the flow. 874 * 875 * @param[in] action 876 * Action configuration. 877 * @param[in, out] action_flags 878 * Pointer to the detected actions. 879 * @param[in] dev_flow 880 * Pointer to mlx5_flow. 881 */ 882 static void 883 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow, 884 const struct rte_flow_action *action) 885 { 886 const struct rte_flow_action_rss *rss = action->conf; 887 const uint8_t *rss_key; 888 struct rte_flow *flow = dev_flow->flow; 889 890 if (flow->rss.queue) 891 memcpy((*flow->rss.queue), rss->queue, 892 rss->queue_num * sizeof(uint16_t)); 893 flow->rss.queue_num = rss->queue_num; 894 /* NULL RSS key indicates default RSS key. */ 895 rss_key = !rss->key ? rss_hash_default_key : rss->key; 896 memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN); 897 /* 898 * rss->level and rss.types should be set in advance when expanding 899 * items for RSS. 900 */ 901 } 902 903 /** 904 * Convert the @p action into a Verbs specification. This function assumes that 905 * the input is valid and that there is space to insert the requested action 906 * into the flow. 907 * 908 * @param[in] dev_flow 909 * Pointer to mlx5_flow. 910 * @param[in] action 911 * Action configuration. 912 */ 913 static void 914 flow_verbs_translate_action_flag 915 (struct mlx5_flow *dev_flow, 916 const struct rte_flow_action *action __rte_unused) 917 { 918 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 919 struct ibv_flow_spec_action_tag tag = { 920 .type = IBV_FLOW_SPEC_ACTION_TAG, 921 .size = size, 922 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT), 923 }; 924 925 flow_verbs_spec_add(&dev_flow->verbs, &tag, size); 926 } 927 928 /** 929 * Convert the @p action into a Verbs specification. This function assumes that 930 * the input is valid and that there is space to insert the requested action 931 * into the flow. 932 * 933 * @param[in] dev_flow 934 * Pointer to mlx5_flow. 935 * @param[in] action 936 * Action configuration. 937 */ 938 static void 939 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow, 940 const struct rte_flow_action *action) 941 { 942 const struct rte_flow_action_mark *mark = action->conf; 943 unsigned int size = sizeof(struct ibv_flow_spec_action_tag); 944 struct ibv_flow_spec_action_tag tag = { 945 .type = IBV_FLOW_SPEC_ACTION_TAG, 946 .size = size, 947 .tag_id = mlx5_flow_mark_set(mark->id), 948 }; 949 950 flow_verbs_spec_add(&dev_flow->verbs, &tag, size); 951 } 952 953 /** 954 * Convert the @p action into a Verbs specification. This function assumes that 955 * the input is valid and that there is space to insert the requested action 956 * into the flow. 957 * 958 * @param[in] dev 959 * Pointer to the Ethernet device structure. 960 * @param[in] action 961 * Action configuration. 962 * @param[in] dev_flow 963 * Pointer to mlx5_flow. 964 * @param[out] error 965 * Pointer to error structure. 966 * 967 * @return 968 * 0 On success else a negative errno value is returned and rte_errno is set. 969 */ 970 static int 971 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, 972 const struct rte_flow_action *action, 973 struct rte_eth_dev *dev, 974 struct rte_flow_error *error) 975 { 976 const struct rte_flow_action_count *count = action->conf; 977 struct rte_flow *flow = dev_flow->flow; 978 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 979 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 980 unsigned int size = sizeof(struct ibv_flow_spec_counter_action); 981 struct ibv_flow_spec_counter_action counter = { 982 .type = IBV_FLOW_SPEC_ACTION_COUNT, 983 .size = size, 984 }; 985 #endif 986 987 if (!flow->counter) { 988 flow->counter = flow_verbs_counter_new(dev, count->shared, 989 count->id); 990 if (!flow->counter) 991 return rte_flow_error_set(error, rte_errno, 992 RTE_FLOW_ERROR_TYPE_ACTION, 993 action, 994 "cannot get counter" 995 " context."); 996 } 997 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) 998 counter.counter_set_handle = flow->counter->cs->handle; 999 flow_verbs_spec_add(&dev_flow->verbs, &counter, size); 1000 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1001 counter.counters = flow->counter->cs; 1002 flow_verbs_spec_add(&dev_flow->verbs, &counter, size); 1003 #endif 1004 return 0; 1005 } 1006 1007 /** 1008 * Internal validation function. For validating both actions and items. 1009 * 1010 * @param[in] dev 1011 * Pointer to the Ethernet device structure. 1012 * @param[in] attr 1013 * Pointer to the flow attributes. 1014 * @param[in] items 1015 * Pointer to the list of items. 1016 * @param[in] actions 1017 * Pointer to the list of actions. 1018 * @param[in] external 1019 * This flow rule is created by request external to PMD. 1020 * @param[out] error 1021 * Pointer to the error structure. 1022 * 1023 * @return 1024 * 0 on success, a negative errno value otherwise and rte_errno is set. 1025 */ 1026 static int 1027 flow_verbs_validate(struct rte_eth_dev *dev, 1028 const struct rte_flow_attr *attr, 1029 const struct rte_flow_item items[], 1030 const struct rte_flow_action actions[], 1031 bool external __rte_unused, 1032 struct rte_flow_error *error) 1033 { 1034 int ret; 1035 uint64_t action_flags = 0; 1036 uint64_t item_flags = 0; 1037 uint64_t last_item = 0; 1038 uint8_t next_protocol = 0xff; 1039 uint16_t ether_type = 0; 1040 1041 if (items == NULL) 1042 return -1; 1043 ret = mlx5_flow_validate_attributes(dev, attr, error); 1044 if (ret < 0) 1045 return ret; 1046 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1047 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1048 int ret = 0; 1049 1050 switch (items->type) { 1051 case RTE_FLOW_ITEM_TYPE_VOID: 1052 break; 1053 case RTE_FLOW_ITEM_TYPE_ETH: 1054 ret = mlx5_flow_validate_item_eth(items, item_flags, 1055 error); 1056 if (ret < 0) 1057 return ret; 1058 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1059 MLX5_FLOW_LAYER_OUTER_L2; 1060 if (items->mask != NULL && items->spec != NULL) { 1061 ether_type = 1062 ((const struct rte_flow_item_eth *) 1063 items->spec)->type; 1064 ether_type &= 1065 ((const struct rte_flow_item_eth *) 1066 items->mask)->type; 1067 ether_type = rte_be_to_cpu_16(ether_type); 1068 } else { 1069 ether_type = 0; 1070 } 1071 break; 1072 case RTE_FLOW_ITEM_TYPE_VLAN: 1073 ret = mlx5_flow_validate_item_vlan(items, item_flags, 1074 dev, error); 1075 if (ret < 0) 1076 return ret; 1077 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 1078 MLX5_FLOW_LAYER_INNER_VLAN) : 1079 (MLX5_FLOW_LAYER_OUTER_L2 | 1080 MLX5_FLOW_LAYER_OUTER_VLAN); 1081 if (items->mask != NULL && items->spec != NULL) { 1082 ether_type = 1083 ((const struct rte_flow_item_vlan *) 1084 items->spec)->inner_type; 1085 ether_type &= 1086 ((const struct rte_flow_item_vlan *) 1087 items->mask)->inner_type; 1088 ether_type = rte_be_to_cpu_16(ether_type); 1089 } else { 1090 ether_type = 0; 1091 } 1092 break; 1093 case RTE_FLOW_ITEM_TYPE_IPV4: 1094 ret = mlx5_flow_validate_item_ipv4(items, item_flags, 1095 last_item, 1096 ether_type, NULL, 1097 error); 1098 if (ret < 0) 1099 return ret; 1100 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1101 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1102 if (items->mask != NULL && 1103 ((const struct rte_flow_item_ipv4 *) 1104 items->mask)->hdr.next_proto_id) { 1105 next_protocol = 1106 ((const struct rte_flow_item_ipv4 *) 1107 (items->spec))->hdr.next_proto_id; 1108 next_protocol &= 1109 ((const struct rte_flow_item_ipv4 *) 1110 (items->mask))->hdr.next_proto_id; 1111 } else { 1112 /* Reset for inner layer. */ 1113 next_protocol = 0xff; 1114 } 1115 break; 1116 case RTE_FLOW_ITEM_TYPE_IPV6: 1117 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 1118 last_item, 1119 ether_type, NULL, 1120 error); 1121 if (ret < 0) 1122 return ret; 1123 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1124 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1125 if (items->mask != NULL && 1126 ((const struct rte_flow_item_ipv6 *) 1127 items->mask)->hdr.proto) { 1128 next_protocol = 1129 ((const struct rte_flow_item_ipv6 *) 1130 items->spec)->hdr.proto; 1131 next_protocol &= 1132 ((const struct rte_flow_item_ipv6 *) 1133 items->mask)->hdr.proto; 1134 } else { 1135 /* Reset for inner layer. */ 1136 next_protocol = 0xff; 1137 } 1138 break; 1139 case RTE_FLOW_ITEM_TYPE_UDP: 1140 ret = mlx5_flow_validate_item_udp(items, item_flags, 1141 next_protocol, 1142 error); 1143 if (ret < 0) 1144 return ret; 1145 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 1146 MLX5_FLOW_LAYER_OUTER_L4_UDP; 1147 break; 1148 case RTE_FLOW_ITEM_TYPE_TCP: 1149 ret = mlx5_flow_validate_item_tcp 1150 (items, item_flags, 1151 next_protocol, 1152 &rte_flow_item_tcp_mask, 1153 error); 1154 if (ret < 0) 1155 return ret; 1156 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 1157 MLX5_FLOW_LAYER_OUTER_L4_TCP; 1158 break; 1159 case RTE_FLOW_ITEM_TYPE_VXLAN: 1160 ret = mlx5_flow_validate_item_vxlan(items, item_flags, 1161 error); 1162 if (ret < 0) 1163 return ret; 1164 last_item = MLX5_FLOW_LAYER_VXLAN; 1165 break; 1166 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1167 ret = mlx5_flow_validate_item_vxlan_gpe(items, 1168 item_flags, 1169 dev, error); 1170 if (ret < 0) 1171 return ret; 1172 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 1173 break; 1174 case RTE_FLOW_ITEM_TYPE_GRE: 1175 ret = mlx5_flow_validate_item_gre(items, item_flags, 1176 next_protocol, error); 1177 if (ret < 0) 1178 return ret; 1179 last_item = MLX5_FLOW_LAYER_GRE; 1180 break; 1181 case RTE_FLOW_ITEM_TYPE_MPLS: 1182 ret = mlx5_flow_validate_item_mpls(dev, items, 1183 item_flags, 1184 last_item, error); 1185 if (ret < 0) 1186 return ret; 1187 last_item = MLX5_FLOW_LAYER_MPLS; 1188 break; 1189 default: 1190 return rte_flow_error_set(error, ENOTSUP, 1191 RTE_FLOW_ERROR_TYPE_ITEM, 1192 NULL, "item not supported"); 1193 } 1194 item_flags |= last_item; 1195 } 1196 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1197 switch (actions->type) { 1198 case RTE_FLOW_ACTION_TYPE_VOID: 1199 break; 1200 case RTE_FLOW_ACTION_TYPE_FLAG: 1201 ret = mlx5_flow_validate_action_flag(action_flags, 1202 attr, 1203 error); 1204 if (ret < 0) 1205 return ret; 1206 action_flags |= MLX5_FLOW_ACTION_FLAG; 1207 break; 1208 case RTE_FLOW_ACTION_TYPE_MARK: 1209 ret = mlx5_flow_validate_action_mark(actions, 1210 action_flags, 1211 attr, 1212 error); 1213 if (ret < 0) 1214 return ret; 1215 action_flags |= MLX5_FLOW_ACTION_MARK; 1216 break; 1217 case RTE_FLOW_ACTION_TYPE_DROP: 1218 ret = mlx5_flow_validate_action_drop(action_flags, 1219 attr, 1220 error); 1221 if (ret < 0) 1222 return ret; 1223 action_flags |= MLX5_FLOW_ACTION_DROP; 1224 break; 1225 case RTE_FLOW_ACTION_TYPE_QUEUE: 1226 ret = mlx5_flow_validate_action_queue(actions, 1227 action_flags, dev, 1228 attr, 1229 error); 1230 if (ret < 0) 1231 return ret; 1232 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1233 break; 1234 case RTE_FLOW_ACTION_TYPE_RSS: 1235 ret = mlx5_flow_validate_action_rss(actions, 1236 action_flags, dev, 1237 attr, item_flags, 1238 error); 1239 if (ret < 0) 1240 return ret; 1241 action_flags |= MLX5_FLOW_ACTION_RSS; 1242 break; 1243 case RTE_FLOW_ACTION_TYPE_COUNT: 1244 ret = mlx5_flow_validate_action_count(dev, attr, error); 1245 if (ret < 0) 1246 return ret; 1247 action_flags |= MLX5_FLOW_ACTION_COUNT; 1248 break; 1249 default: 1250 return rte_flow_error_set(error, ENOTSUP, 1251 RTE_FLOW_ERROR_TYPE_ACTION, 1252 actions, 1253 "action not supported"); 1254 } 1255 } 1256 /* 1257 * Validate the drop action mutual exclusion with other actions. 1258 * Drop action is mutually-exclusive with any other action, except for 1259 * Count action. 1260 */ 1261 if ((action_flags & MLX5_FLOW_ACTION_DROP) && 1262 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) 1263 return rte_flow_error_set(error, EINVAL, 1264 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1265 "Drop action is mutually-exclusive " 1266 "with any other action, except for " 1267 "Count action"); 1268 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS)) 1269 return rte_flow_error_set(error, EINVAL, 1270 RTE_FLOW_ERROR_TYPE_ACTION, actions, 1271 "no fate action is found"); 1272 return 0; 1273 } 1274 1275 /** 1276 * Calculate the required bytes that are needed for the action part of the verbs 1277 * flow. 1278 * 1279 * @param[in] actions 1280 * Pointer to the list of actions. 1281 * 1282 * @return 1283 * The size of the memory needed for all actions. 1284 */ 1285 static int 1286 flow_verbs_get_actions_size(const struct rte_flow_action actions[]) 1287 { 1288 int size = 0; 1289 1290 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1291 switch (actions->type) { 1292 case RTE_FLOW_ACTION_TYPE_VOID: 1293 break; 1294 case RTE_FLOW_ACTION_TYPE_FLAG: 1295 size += sizeof(struct ibv_flow_spec_action_tag); 1296 break; 1297 case RTE_FLOW_ACTION_TYPE_MARK: 1298 size += sizeof(struct ibv_flow_spec_action_tag); 1299 break; 1300 case RTE_FLOW_ACTION_TYPE_DROP: 1301 size += sizeof(struct ibv_flow_spec_action_drop); 1302 break; 1303 case RTE_FLOW_ACTION_TYPE_QUEUE: 1304 break; 1305 case RTE_FLOW_ACTION_TYPE_RSS: 1306 break; 1307 case RTE_FLOW_ACTION_TYPE_COUNT: 1308 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ 1309 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) 1310 size += sizeof(struct ibv_flow_spec_counter_action); 1311 #endif 1312 break; 1313 default: 1314 break; 1315 } 1316 } 1317 return size; 1318 } 1319 1320 /** 1321 * Calculate the required bytes that are needed for the item part of the verbs 1322 * flow. 1323 * 1324 * @param[in] items 1325 * Pointer to the list of items. 1326 * 1327 * @return 1328 * The size of the memory needed for all items. 1329 */ 1330 static int 1331 flow_verbs_get_items_size(const struct rte_flow_item items[]) 1332 { 1333 int size = 0; 1334 1335 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1336 switch (items->type) { 1337 case RTE_FLOW_ITEM_TYPE_VOID: 1338 break; 1339 case RTE_FLOW_ITEM_TYPE_ETH: 1340 size += sizeof(struct ibv_flow_spec_eth); 1341 break; 1342 case RTE_FLOW_ITEM_TYPE_VLAN: 1343 size += sizeof(struct ibv_flow_spec_eth); 1344 break; 1345 case RTE_FLOW_ITEM_TYPE_IPV4: 1346 size += sizeof(struct ibv_flow_spec_ipv4_ext); 1347 break; 1348 case RTE_FLOW_ITEM_TYPE_IPV6: 1349 size += sizeof(struct ibv_flow_spec_ipv6); 1350 break; 1351 case RTE_FLOW_ITEM_TYPE_UDP: 1352 size += sizeof(struct ibv_flow_spec_tcp_udp); 1353 break; 1354 case RTE_FLOW_ITEM_TYPE_TCP: 1355 size += sizeof(struct ibv_flow_spec_tcp_udp); 1356 break; 1357 case RTE_FLOW_ITEM_TYPE_VXLAN: 1358 size += sizeof(struct ibv_flow_spec_tunnel); 1359 break; 1360 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1361 size += sizeof(struct ibv_flow_spec_tunnel); 1362 break; 1363 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 1364 case RTE_FLOW_ITEM_TYPE_GRE: 1365 size += sizeof(struct ibv_flow_spec_gre); 1366 break; 1367 case RTE_FLOW_ITEM_TYPE_MPLS: 1368 size += sizeof(struct ibv_flow_spec_mpls); 1369 break; 1370 #else 1371 case RTE_FLOW_ITEM_TYPE_GRE: 1372 size += sizeof(struct ibv_flow_spec_tunnel); 1373 break; 1374 #endif 1375 default: 1376 break; 1377 } 1378 } 1379 return size; 1380 } 1381 1382 /** 1383 * Internal preparation function. Allocate mlx5_flow with the required size. 1384 * The required size is calculate based on the actions and items. This function 1385 * also returns the detected actions and items for later use. 1386 * 1387 * @param[in] attr 1388 * Pointer to the flow attributes. 1389 * @param[in] items 1390 * Pointer to the list of items. 1391 * @param[in] actions 1392 * Pointer to the list of actions. 1393 * @param[out] error 1394 * Pointer to the error structure. 1395 * 1396 * @return 1397 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno 1398 * is set. 1399 */ 1400 static struct mlx5_flow * 1401 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused, 1402 const struct rte_flow_item items[], 1403 const struct rte_flow_action actions[], 1404 struct rte_flow_error *error) 1405 { 1406 size_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr); 1407 struct mlx5_flow *dev_flow; 1408 1409 size += flow_verbs_get_actions_size(actions); 1410 size += flow_verbs_get_items_size(items); 1411 dev_flow = rte_calloc(__func__, 1, size, 0); 1412 if (!dev_flow) { 1413 rte_flow_error_set(error, ENOMEM, 1414 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1415 "not enough memory to create flow"); 1416 return NULL; 1417 } 1418 dev_flow->verbs.attr = (void *)(dev_flow + 1); 1419 dev_flow->verbs.specs = (void *)(dev_flow->verbs.attr + 1); 1420 dev_flow->ingress = attr->ingress; 1421 dev_flow->transfer = attr->transfer; 1422 return dev_flow; 1423 } 1424 1425 /** 1426 * Fill the flow with verb spec. 1427 * 1428 * @param[in] dev 1429 * Pointer to Ethernet device. 1430 * @param[in, out] dev_flow 1431 * Pointer to the mlx5 flow. 1432 * @param[in] attr 1433 * Pointer to the flow attributes. 1434 * @param[in] items 1435 * Pointer to the list of items. 1436 * @param[in] actions 1437 * Pointer to the list of actions. 1438 * @param[out] error 1439 * Pointer to the error structure. 1440 * 1441 * @return 1442 * 0 on success, else a negative errno value otherwise and rte_errno is set. 1443 */ 1444 static int 1445 flow_verbs_translate(struct rte_eth_dev *dev, 1446 struct mlx5_flow *dev_flow, 1447 const struct rte_flow_attr *attr, 1448 const struct rte_flow_item items[], 1449 const struct rte_flow_action actions[], 1450 struct rte_flow_error *error) 1451 { 1452 uint64_t item_flags = 0; 1453 uint64_t action_flags = 0; 1454 uint64_t priority = attr->priority; 1455 uint32_t subpriority = 0; 1456 struct mlx5_priv *priv = dev->data->dev_private; 1457 1458 if (priority == MLX5_FLOW_PRIO_RSVD) 1459 priority = priv->config.flow_prio - 1; 1460 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1461 int ret; 1462 1463 switch (actions->type) { 1464 case RTE_FLOW_ACTION_TYPE_VOID: 1465 break; 1466 case RTE_FLOW_ACTION_TYPE_FLAG: 1467 flow_verbs_translate_action_flag(dev_flow, actions); 1468 action_flags |= MLX5_FLOW_ACTION_FLAG; 1469 break; 1470 case RTE_FLOW_ACTION_TYPE_MARK: 1471 flow_verbs_translate_action_mark(dev_flow, actions); 1472 action_flags |= MLX5_FLOW_ACTION_MARK; 1473 break; 1474 case RTE_FLOW_ACTION_TYPE_DROP: 1475 flow_verbs_translate_action_drop(dev_flow, actions); 1476 action_flags |= MLX5_FLOW_ACTION_DROP; 1477 break; 1478 case RTE_FLOW_ACTION_TYPE_QUEUE: 1479 flow_verbs_translate_action_queue(dev_flow, actions); 1480 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1481 break; 1482 case RTE_FLOW_ACTION_TYPE_RSS: 1483 flow_verbs_translate_action_rss(dev_flow, actions); 1484 action_flags |= MLX5_FLOW_ACTION_RSS; 1485 break; 1486 case RTE_FLOW_ACTION_TYPE_COUNT: 1487 ret = flow_verbs_translate_action_count(dev_flow, 1488 actions, 1489 dev, error); 1490 if (ret < 0) 1491 return ret; 1492 action_flags |= MLX5_FLOW_ACTION_COUNT; 1493 break; 1494 default: 1495 return rte_flow_error_set(error, ENOTSUP, 1496 RTE_FLOW_ERROR_TYPE_ACTION, 1497 actions, 1498 "action not supported"); 1499 } 1500 } 1501 dev_flow->actions = action_flags; 1502 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1503 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1504 1505 switch (items->type) { 1506 case RTE_FLOW_ITEM_TYPE_VOID: 1507 break; 1508 case RTE_FLOW_ITEM_TYPE_ETH: 1509 flow_verbs_translate_item_eth(dev_flow, items, 1510 item_flags); 1511 subpriority = MLX5_PRIORITY_MAP_L2; 1512 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1513 MLX5_FLOW_LAYER_OUTER_L2; 1514 break; 1515 case RTE_FLOW_ITEM_TYPE_VLAN: 1516 flow_verbs_translate_item_vlan(dev_flow, items, 1517 item_flags); 1518 subpriority = MLX5_PRIORITY_MAP_L2; 1519 item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 1520 MLX5_FLOW_LAYER_INNER_VLAN) : 1521 (MLX5_FLOW_LAYER_OUTER_L2 | 1522 MLX5_FLOW_LAYER_OUTER_VLAN); 1523 break; 1524 case RTE_FLOW_ITEM_TYPE_IPV4: 1525 flow_verbs_translate_item_ipv4(dev_flow, items, 1526 item_flags); 1527 subpriority = MLX5_PRIORITY_MAP_L3; 1528 dev_flow->hash_fields |= 1529 mlx5_flow_hashfields_adjust 1530 (dev_flow, tunnel, 1531 MLX5_IPV4_LAYER_TYPES, 1532 MLX5_IPV4_IBV_RX_HASH); 1533 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1534 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1535 break; 1536 case RTE_FLOW_ITEM_TYPE_IPV6: 1537 flow_verbs_translate_item_ipv6(dev_flow, items, 1538 item_flags); 1539 subpriority = MLX5_PRIORITY_MAP_L3; 1540 dev_flow->hash_fields |= 1541 mlx5_flow_hashfields_adjust 1542 (dev_flow, tunnel, 1543 MLX5_IPV6_LAYER_TYPES, 1544 MLX5_IPV6_IBV_RX_HASH); 1545 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1546 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1547 break; 1548 case RTE_FLOW_ITEM_TYPE_TCP: 1549 flow_verbs_translate_item_tcp(dev_flow, items, 1550 item_flags); 1551 subpriority = MLX5_PRIORITY_MAP_L4; 1552 dev_flow->hash_fields |= 1553 mlx5_flow_hashfields_adjust 1554 (dev_flow, tunnel, ETH_RSS_TCP, 1555 (IBV_RX_HASH_SRC_PORT_TCP | 1556 IBV_RX_HASH_DST_PORT_TCP)); 1557 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 1558 MLX5_FLOW_LAYER_OUTER_L4_TCP; 1559 break; 1560 case RTE_FLOW_ITEM_TYPE_UDP: 1561 flow_verbs_translate_item_udp(dev_flow, items, 1562 item_flags); 1563 subpriority = MLX5_PRIORITY_MAP_L4; 1564 dev_flow->hash_fields |= 1565 mlx5_flow_hashfields_adjust 1566 (dev_flow, tunnel, ETH_RSS_UDP, 1567 (IBV_RX_HASH_SRC_PORT_UDP | 1568 IBV_RX_HASH_DST_PORT_UDP)); 1569 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 1570 MLX5_FLOW_LAYER_OUTER_L4_UDP; 1571 break; 1572 case RTE_FLOW_ITEM_TYPE_VXLAN: 1573 flow_verbs_translate_item_vxlan(dev_flow, items, 1574 item_flags); 1575 subpriority = MLX5_PRIORITY_MAP_L2; 1576 item_flags |= MLX5_FLOW_LAYER_VXLAN; 1577 break; 1578 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1579 flow_verbs_translate_item_vxlan_gpe(dev_flow, items, 1580 item_flags); 1581 subpriority = MLX5_PRIORITY_MAP_L2; 1582 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE; 1583 break; 1584 case RTE_FLOW_ITEM_TYPE_GRE: 1585 flow_verbs_translate_item_gre(dev_flow, items, 1586 item_flags); 1587 subpriority = MLX5_PRIORITY_MAP_L2; 1588 item_flags |= MLX5_FLOW_LAYER_GRE; 1589 break; 1590 case RTE_FLOW_ITEM_TYPE_MPLS: 1591 flow_verbs_translate_item_mpls(dev_flow, items, 1592 item_flags); 1593 subpriority = MLX5_PRIORITY_MAP_L2; 1594 item_flags |= MLX5_FLOW_LAYER_MPLS; 1595 break; 1596 default: 1597 return rte_flow_error_set(error, ENOTSUP, 1598 RTE_FLOW_ERROR_TYPE_ITEM, 1599 NULL, 1600 "item not supported"); 1601 } 1602 } 1603 dev_flow->layers = item_flags; 1604 dev_flow->verbs.attr->priority = 1605 mlx5_flow_adjust_priority(dev, priority, subpriority); 1606 dev_flow->verbs.attr->port = (uint8_t)priv->ibv_port; 1607 return 0; 1608 } 1609 1610 /** 1611 * Remove the flow from the NIC but keeps it in memory. 1612 * 1613 * @param[in] dev 1614 * Pointer to the Ethernet device structure. 1615 * @param[in, out] flow 1616 * Pointer to flow structure. 1617 */ 1618 static void 1619 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 1620 { 1621 struct mlx5_flow_verbs *verbs; 1622 struct mlx5_flow *dev_flow; 1623 1624 if (!flow) 1625 return; 1626 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 1627 verbs = &dev_flow->verbs; 1628 if (verbs->flow) { 1629 claim_zero(mlx5_glue->destroy_flow(verbs->flow)); 1630 verbs->flow = NULL; 1631 } 1632 if (verbs->hrxq) { 1633 if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) 1634 mlx5_hrxq_drop_release(dev); 1635 else 1636 mlx5_hrxq_release(dev, verbs->hrxq); 1637 verbs->hrxq = NULL; 1638 } 1639 if (dev_flow->verbs.vf_vlan.tag && 1640 dev_flow->verbs.vf_vlan.created) { 1641 mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan); 1642 } 1643 } 1644 } 1645 1646 /** 1647 * Remove the flow from the NIC and the memory. 1648 * 1649 * @param[in] dev 1650 * Pointer to the Ethernet device structure. 1651 * @param[in, out] flow 1652 * Pointer to flow structure. 1653 */ 1654 static void 1655 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 1656 { 1657 struct mlx5_flow *dev_flow; 1658 1659 if (!flow) 1660 return; 1661 flow_verbs_remove(dev, flow); 1662 while (!LIST_EMPTY(&flow->dev_flows)) { 1663 dev_flow = LIST_FIRST(&flow->dev_flows); 1664 LIST_REMOVE(dev_flow, next); 1665 rte_free(dev_flow); 1666 } 1667 if (flow->counter) { 1668 flow_verbs_counter_release(dev, flow->counter); 1669 flow->counter = NULL; 1670 } 1671 } 1672 1673 /** 1674 * Apply the flow to the NIC. 1675 * 1676 * @param[in] dev 1677 * Pointer to the Ethernet device structure. 1678 * @param[in, out] flow 1679 * Pointer to flow structure. 1680 * @param[out] error 1681 * Pointer to error structure. 1682 * 1683 * @return 1684 * 0 on success, a negative errno value otherwise and rte_errno is set. 1685 */ 1686 static int 1687 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 1688 struct rte_flow_error *error) 1689 { 1690 struct mlx5_priv *priv = dev->data->dev_private; 1691 struct mlx5_flow_verbs *verbs; 1692 struct mlx5_flow *dev_flow; 1693 int err; 1694 1695 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 1696 verbs = &dev_flow->verbs; 1697 if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) { 1698 verbs->hrxq = mlx5_hrxq_drop_new(dev); 1699 if (!verbs->hrxq) { 1700 rte_flow_error_set 1701 (error, errno, 1702 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1703 "cannot get drop hash queue"); 1704 goto error; 1705 } 1706 } else { 1707 struct mlx5_hrxq *hrxq; 1708 1709 MLX5_ASSERT(flow->rss.queue); 1710 hrxq = mlx5_hrxq_get(dev, flow->rss.key, 1711 MLX5_RSS_HASH_KEY_LEN, 1712 dev_flow->hash_fields, 1713 (*flow->rss.queue), 1714 flow->rss.queue_num); 1715 if (!hrxq) 1716 hrxq = mlx5_hrxq_new(dev, flow->rss.key, 1717 MLX5_RSS_HASH_KEY_LEN, 1718 dev_flow->hash_fields, 1719 (*flow->rss.queue), 1720 flow->rss.queue_num, 1721 !!(dev_flow->layers & 1722 MLX5_FLOW_LAYER_TUNNEL)); 1723 if (!hrxq) { 1724 rte_flow_error_set 1725 (error, rte_errno, 1726 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1727 "cannot get hash queue"); 1728 goto error; 1729 } 1730 verbs->hrxq = hrxq; 1731 } 1732 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp, 1733 verbs->attr); 1734 if (!verbs->flow) { 1735 rte_flow_error_set(error, errno, 1736 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1737 NULL, 1738 "hardware refuses to create flow"); 1739 goto error; 1740 } 1741 if (priv->vmwa_context && 1742 dev_flow->verbs.vf_vlan.tag && 1743 !dev_flow->verbs.vf_vlan.created) { 1744 /* 1745 * The rule contains the VLAN pattern. 1746 * For VF we are going to create VLAN 1747 * interface to make hypervisor set correct 1748 * e-Switch vport context. 1749 */ 1750 mlx5_vlan_vmwa_acquire(dev, &dev_flow->verbs.vf_vlan); 1751 } 1752 } 1753 return 0; 1754 error: 1755 err = rte_errno; /* Save rte_errno before cleanup. */ 1756 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 1757 verbs = &dev_flow->verbs; 1758 if (verbs->hrxq) { 1759 if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) 1760 mlx5_hrxq_drop_release(dev); 1761 else 1762 mlx5_hrxq_release(dev, verbs->hrxq); 1763 verbs->hrxq = NULL; 1764 } 1765 if (dev_flow->verbs.vf_vlan.tag && 1766 dev_flow->verbs.vf_vlan.created) { 1767 mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan); 1768 } 1769 } 1770 rte_errno = err; /* Restore rte_errno. */ 1771 return -rte_errno; 1772 } 1773 1774 /** 1775 * Query a flow. 1776 * 1777 * @see rte_flow_query() 1778 * @see rte_flow_ops 1779 */ 1780 static int 1781 flow_verbs_query(struct rte_eth_dev *dev, 1782 struct rte_flow *flow, 1783 const struct rte_flow_action *actions, 1784 void *data, 1785 struct rte_flow_error *error) 1786 { 1787 int ret = -EINVAL; 1788 1789 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1790 switch (actions->type) { 1791 case RTE_FLOW_ACTION_TYPE_VOID: 1792 break; 1793 case RTE_FLOW_ACTION_TYPE_COUNT: 1794 ret = flow_verbs_counter_query(dev, flow, data, error); 1795 break; 1796 default: 1797 return rte_flow_error_set(error, ENOTSUP, 1798 RTE_FLOW_ERROR_TYPE_ACTION, 1799 actions, 1800 "action not supported"); 1801 } 1802 } 1803 return ret; 1804 } 1805 1806 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = { 1807 .validate = flow_verbs_validate, 1808 .prepare = flow_verbs_prepare, 1809 .translate = flow_verbs_translate, 1810 .apply = flow_verbs_apply, 1811 .remove = flow_verbs_remove, 1812 .destroy = flow_verbs_destroy, 1813 .query = flow_verbs_query, 1814 }; 1815