1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <unistd.h> 10 11 #include <rte_common.h> 12 #include <rte_ether.h> 13 #include <rte_ethdev_driver.h> 14 #include <rte_flow.h> 15 #include <rte_flow_driver.h> 16 #include <rte_malloc.h> 17 #include <rte_cycles.h> 18 #include <rte_ip.h> 19 #include <rte_gre.h> 20 #include <rte_vxlan.h> 21 #include <rte_gtp.h> 22 #include <rte_eal_paging.h> 23 #include <rte_mpls.h> 24 25 #include <mlx5_glue.h> 26 #include <mlx5_devx_cmds.h> 27 #include <mlx5_prm.h> 28 #include <mlx5_malloc.h> 29 30 #include "mlx5_defs.h" 31 #include "mlx5.h" 32 #include "mlx5_common_os.h" 33 #include "mlx5_flow.h" 34 #include "mlx5_flow_os.h" 35 #include "mlx5_rxtx.h" 36 37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 38 39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS 40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 41 #endif 42 43 #ifndef HAVE_MLX5DV_DR_ESWITCH 44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB 45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0 46 #endif 47 #endif 48 49 #ifndef HAVE_MLX5DV_DR 50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1 51 #endif 52 53 /* VLAN header definitions */ 54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13 55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT) 56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff 57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK) 58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK) 59 60 union flow_dv_attr { 61 struct { 62 uint32_t valid:1; 63 uint32_t ipv4:1; 64 uint32_t ipv6:1; 65 uint32_t tcp:1; 66 uint32_t udp:1; 67 uint32_t reserved:27; 68 }; 69 uint32_t attr; 70 }; 71 72 static int 73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev, 74 struct mlx5_flow_tbl_resource *tbl); 75 76 static int 77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev); 78 79 /** 80 * Initialize flow attributes structure according to flow items' types. 81 * 82 * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel 83 * mode. For tunnel mode, the items to be modified are the outermost ones. 84 * 85 * @param[in] item 86 * Pointer to item specification. 87 * @param[out] attr 88 * Pointer to flow attributes structure. 89 * @param[in] dev_flow 90 * Pointer to the sub flow. 91 * @param[in] tunnel_decap 92 * Whether action is after tunnel decapsulation. 93 */ 94 static void 95 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr, 96 struct mlx5_flow *dev_flow, bool tunnel_decap) 97 { 98 uint64_t layers = dev_flow->handle->layers; 99 100 /* 101 * If layers is already initialized, it means this dev_flow is the 102 * suffix flow, the layers flags is set by the prefix flow. Need to 103 * use the layer flags from prefix flow as the suffix flow may not 104 * have the user defined items as the flow is split. 105 */ 106 if (layers) { 107 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4) 108 attr->ipv4 = 1; 109 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6) 110 attr->ipv6 = 1; 111 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP) 112 attr->tcp = 1; 113 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP) 114 attr->udp = 1; 115 attr->valid = 1; 116 return; 117 } 118 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 119 uint8_t next_protocol = 0xff; 120 switch (item->type) { 121 case RTE_FLOW_ITEM_TYPE_GRE: 122 case RTE_FLOW_ITEM_TYPE_NVGRE: 123 case RTE_FLOW_ITEM_TYPE_VXLAN: 124 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 125 case RTE_FLOW_ITEM_TYPE_GENEVE: 126 case RTE_FLOW_ITEM_TYPE_MPLS: 127 if (tunnel_decap) 128 attr->attr = 0; 129 break; 130 case RTE_FLOW_ITEM_TYPE_IPV4: 131 if (!attr->ipv6) 132 attr->ipv4 = 1; 133 if (item->mask != NULL && 134 ((const struct rte_flow_item_ipv4 *) 135 item->mask)->hdr.next_proto_id) 136 next_protocol = 137 ((const struct rte_flow_item_ipv4 *) 138 (item->spec))->hdr.next_proto_id & 139 ((const struct rte_flow_item_ipv4 *) 140 (item->mask))->hdr.next_proto_id; 141 if ((next_protocol == IPPROTO_IPIP || 142 next_protocol == IPPROTO_IPV6) && tunnel_decap) 143 attr->attr = 0; 144 break; 145 case RTE_FLOW_ITEM_TYPE_IPV6: 146 if (!attr->ipv4) 147 attr->ipv6 = 1; 148 if (item->mask != NULL && 149 ((const struct rte_flow_item_ipv6 *) 150 item->mask)->hdr.proto) 151 next_protocol = 152 ((const struct rte_flow_item_ipv6 *) 153 (item->spec))->hdr.proto & 154 ((const struct rte_flow_item_ipv6 *) 155 (item->mask))->hdr.proto; 156 if ((next_protocol == IPPROTO_IPIP || 157 next_protocol == IPPROTO_IPV6) && tunnel_decap) 158 attr->attr = 0; 159 break; 160 case RTE_FLOW_ITEM_TYPE_UDP: 161 if (!attr->tcp) 162 attr->udp = 1; 163 break; 164 case RTE_FLOW_ITEM_TYPE_TCP: 165 if (!attr->udp) 166 attr->tcp = 1; 167 break; 168 default: 169 break; 170 } 171 } 172 attr->valid = 1; 173 } 174 175 /** 176 * Convert rte_mtr_color to mlx5 color. 177 * 178 * @param[in] rcol 179 * rte_mtr_color. 180 * 181 * @return 182 * mlx5 color. 183 */ 184 static int 185 rte_col_2_mlx5_col(enum rte_color rcol) 186 { 187 switch (rcol) { 188 case RTE_COLOR_GREEN: 189 return MLX5_FLOW_COLOR_GREEN; 190 case RTE_COLOR_YELLOW: 191 return MLX5_FLOW_COLOR_YELLOW; 192 case RTE_COLOR_RED: 193 return MLX5_FLOW_COLOR_RED; 194 default: 195 break; 196 } 197 return MLX5_FLOW_COLOR_UNDEFINED; 198 } 199 200 struct field_modify_info { 201 uint32_t size; /* Size of field in protocol header, in bytes. */ 202 uint32_t offset; /* Offset of field in protocol header, in bytes. */ 203 enum mlx5_modification_field id; 204 }; 205 206 struct field_modify_info modify_eth[] = { 207 {4, 0, MLX5_MODI_OUT_DMAC_47_16}, 208 {2, 4, MLX5_MODI_OUT_DMAC_15_0}, 209 {4, 6, MLX5_MODI_OUT_SMAC_47_16}, 210 {2, 10, MLX5_MODI_OUT_SMAC_15_0}, 211 {0, 0, 0}, 212 }; 213 214 struct field_modify_info modify_vlan_out_first_vid[] = { 215 /* Size in bits !!! */ 216 {12, 0, MLX5_MODI_OUT_FIRST_VID}, 217 {0, 0, 0}, 218 }; 219 220 struct field_modify_info modify_ipv4[] = { 221 {1, 1, MLX5_MODI_OUT_IP_DSCP}, 222 {1, 8, MLX5_MODI_OUT_IPV4_TTL}, 223 {4, 12, MLX5_MODI_OUT_SIPV4}, 224 {4, 16, MLX5_MODI_OUT_DIPV4}, 225 {0, 0, 0}, 226 }; 227 228 struct field_modify_info modify_ipv6[] = { 229 {1, 0, MLX5_MODI_OUT_IP_DSCP}, 230 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT}, 231 {4, 8, MLX5_MODI_OUT_SIPV6_127_96}, 232 {4, 12, MLX5_MODI_OUT_SIPV6_95_64}, 233 {4, 16, MLX5_MODI_OUT_SIPV6_63_32}, 234 {4, 20, MLX5_MODI_OUT_SIPV6_31_0}, 235 {4, 24, MLX5_MODI_OUT_DIPV6_127_96}, 236 {4, 28, MLX5_MODI_OUT_DIPV6_95_64}, 237 {4, 32, MLX5_MODI_OUT_DIPV6_63_32}, 238 {4, 36, MLX5_MODI_OUT_DIPV6_31_0}, 239 {0, 0, 0}, 240 }; 241 242 struct field_modify_info modify_udp[] = { 243 {2, 0, MLX5_MODI_OUT_UDP_SPORT}, 244 {2, 2, MLX5_MODI_OUT_UDP_DPORT}, 245 {0, 0, 0}, 246 }; 247 248 struct field_modify_info modify_tcp[] = { 249 {2, 0, MLX5_MODI_OUT_TCP_SPORT}, 250 {2, 2, MLX5_MODI_OUT_TCP_DPORT}, 251 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM}, 252 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM}, 253 {0, 0, 0}, 254 }; 255 256 static void 257 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, 258 uint8_t next_protocol, uint64_t *item_flags, 259 int *tunnel) 260 { 261 MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 || 262 item->type == RTE_FLOW_ITEM_TYPE_IPV6); 263 if (next_protocol == IPPROTO_IPIP) { 264 *item_flags |= MLX5_FLOW_LAYER_IPIP; 265 *tunnel = 1; 266 } 267 if (next_protocol == IPPROTO_IPV6) { 268 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP; 269 *tunnel = 1; 270 } 271 } 272 273 /** 274 * Acquire the synchronizing object to protect multithreaded access 275 * to shared dv context. Lock occurs only if context is actually 276 * shared, i.e. we have multiport IB device and representors are 277 * created. 278 * 279 * @param[in] dev 280 * Pointer to the rte_eth_dev structure. 281 */ 282 static void 283 flow_dv_shared_lock(struct rte_eth_dev *dev) 284 { 285 struct mlx5_priv *priv = dev->data->dev_private; 286 struct mlx5_dev_ctx_shared *sh = priv->sh; 287 288 if (sh->dv_refcnt > 1) { 289 int ret; 290 291 ret = pthread_mutex_lock(&sh->dv_mutex); 292 MLX5_ASSERT(!ret); 293 (void)ret; 294 } 295 } 296 297 static void 298 flow_dv_shared_unlock(struct rte_eth_dev *dev) 299 { 300 struct mlx5_priv *priv = dev->data->dev_private; 301 struct mlx5_dev_ctx_shared *sh = priv->sh; 302 303 if (sh->dv_refcnt > 1) { 304 int ret; 305 306 ret = pthread_mutex_unlock(&sh->dv_mutex); 307 MLX5_ASSERT(!ret); 308 (void)ret; 309 } 310 } 311 312 /* Update VLAN's VID/PCP based on input rte_flow_action. 313 * 314 * @param[in] action 315 * Pointer to struct rte_flow_action. 316 * @param[out] vlan 317 * Pointer to struct rte_vlan_hdr. 318 */ 319 static void 320 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action, 321 struct rte_vlan_hdr *vlan) 322 { 323 uint16_t vlan_tci; 324 if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) { 325 vlan_tci = 326 ((const struct rte_flow_action_of_set_vlan_pcp *) 327 action->conf)->vlan_pcp; 328 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT; 329 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; 330 vlan->vlan_tci |= vlan_tci; 331 } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) { 332 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK; 333 vlan->vlan_tci |= rte_be_to_cpu_16 334 (((const struct rte_flow_action_of_set_vlan_vid *) 335 action->conf)->vlan_vid); 336 } 337 } 338 339 /** 340 * Fetch 1, 2, 3 or 4 byte field from the byte array 341 * and return as unsigned integer in host-endian format. 342 * 343 * @param[in] data 344 * Pointer to data array. 345 * @param[in] size 346 * Size of field to extract. 347 * 348 * @return 349 * converted field in host endian format. 350 */ 351 static inline uint32_t 352 flow_dv_fetch_field(const uint8_t *data, uint32_t size) 353 { 354 uint32_t ret; 355 356 switch (size) { 357 case 1: 358 ret = *data; 359 break; 360 case 2: 361 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); 362 break; 363 case 3: 364 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); 365 ret = (ret << 8) | *(data + sizeof(uint16_t)); 366 break; 367 case 4: 368 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data); 369 break; 370 default: 371 MLX5_ASSERT(false); 372 ret = 0; 373 break; 374 } 375 return ret; 376 } 377 378 /** 379 * Convert modify-header action to DV specification. 380 * 381 * Data length of each action is determined by provided field description 382 * and the item mask. Data bit offset and width of each action is determined 383 * by provided item mask. 384 * 385 * @param[in] item 386 * Pointer to item specification. 387 * @param[in] field 388 * Pointer to field modification information. 389 * For MLX5_MODIFICATION_TYPE_SET specifies destination field. 390 * For MLX5_MODIFICATION_TYPE_ADD specifies destination field. 391 * For MLX5_MODIFICATION_TYPE_COPY specifies source field. 392 * @param[in] dcopy 393 * Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type. 394 * Negative offset value sets the same offset as source offset. 395 * size field is ignored, value is taken from source field. 396 * @param[in,out] resource 397 * Pointer to the modify-header resource. 398 * @param[in] type 399 * Type of modification. 400 * @param[out] error 401 * Pointer to the error structure. 402 * 403 * @return 404 * 0 on success, a negative errno value otherwise and rte_errno is set. 405 */ 406 static int 407 flow_dv_convert_modify_action(struct rte_flow_item *item, 408 struct field_modify_info *field, 409 struct field_modify_info *dcopy, 410 struct mlx5_flow_dv_modify_hdr_resource *resource, 411 uint32_t type, struct rte_flow_error *error) 412 { 413 uint32_t i = resource->actions_num; 414 struct mlx5_modification_cmd *actions = resource->actions; 415 416 /* 417 * The item and mask are provided in big-endian format. 418 * The fields should be presented as in big-endian format either. 419 * Mask must be always present, it defines the actual field width. 420 */ 421 MLX5_ASSERT(item->mask); 422 MLX5_ASSERT(field->size); 423 do { 424 unsigned int size_b; 425 unsigned int off_b; 426 uint32_t mask; 427 uint32_t data; 428 429 if (i >= MLX5_MAX_MODIFY_NUM) 430 return rte_flow_error_set(error, EINVAL, 431 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 432 "too many items to modify"); 433 /* Fetch variable byte size mask from the array. */ 434 mask = flow_dv_fetch_field((const uint8_t *)item->mask + 435 field->offset, field->size); 436 if (!mask) { 437 ++field; 438 continue; 439 } 440 /* Deduce actual data width in bits from mask value. */ 441 off_b = rte_bsf32(mask); 442 size_b = sizeof(uint32_t) * CHAR_BIT - 443 off_b - __builtin_clz(mask); 444 MLX5_ASSERT(size_b); 445 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b; 446 actions[i] = (struct mlx5_modification_cmd) { 447 .action_type = type, 448 .field = field->id, 449 .offset = off_b, 450 .length = size_b, 451 }; 452 /* Convert entire record to expected big-endian format. */ 453 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 454 if (type == MLX5_MODIFICATION_TYPE_COPY) { 455 MLX5_ASSERT(dcopy); 456 actions[i].dst_field = dcopy->id; 457 actions[i].dst_offset = 458 (int)dcopy->offset < 0 ? off_b : dcopy->offset; 459 /* Convert entire record to big-endian format. */ 460 actions[i].data1 = rte_cpu_to_be_32(actions[i].data1); 461 } else { 462 MLX5_ASSERT(item->spec); 463 data = flow_dv_fetch_field((const uint8_t *)item->spec + 464 field->offset, field->size); 465 /* Shift out the trailing masked bits from data. */ 466 data = (data & mask) >> off_b; 467 actions[i].data1 = rte_cpu_to_be_32(data); 468 } 469 ++i; 470 ++field; 471 } while (field->size); 472 if (resource->actions_num == i) 473 return rte_flow_error_set(error, EINVAL, 474 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 475 "invalid modification flow item"); 476 resource->actions_num = i; 477 return 0; 478 } 479 480 /** 481 * Convert modify-header set IPv4 address action to DV specification. 482 * 483 * @param[in,out] resource 484 * Pointer to the modify-header resource. 485 * @param[in] action 486 * Pointer to action specification. 487 * @param[out] error 488 * Pointer to the error structure. 489 * 490 * @return 491 * 0 on success, a negative errno value otherwise and rte_errno is set. 492 */ 493 static int 494 flow_dv_convert_action_modify_ipv4 495 (struct mlx5_flow_dv_modify_hdr_resource *resource, 496 const struct rte_flow_action *action, 497 struct rte_flow_error *error) 498 { 499 const struct rte_flow_action_set_ipv4 *conf = 500 (const struct rte_flow_action_set_ipv4 *)(action->conf); 501 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 502 struct rte_flow_item_ipv4 ipv4; 503 struct rte_flow_item_ipv4 ipv4_mask; 504 505 memset(&ipv4, 0, sizeof(ipv4)); 506 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 507 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) { 508 ipv4.hdr.src_addr = conf->ipv4_addr; 509 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr; 510 } else { 511 ipv4.hdr.dst_addr = conf->ipv4_addr; 512 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr; 513 } 514 item.spec = &ipv4; 515 item.mask = &ipv4_mask; 516 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, 517 MLX5_MODIFICATION_TYPE_SET, error); 518 } 519 520 /** 521 * Convert modify-header set IPv6 address action to DV specification. 522 * 523 * @param[in,out] resource 524 * Pointer to the modify-header resource. 525 * @param[in] action 526 * Pointer to action specification. 527 * @param[out] error 528 * Pointer to the error structure. 529 * 530 * @return 531 * 0 on success, a negative errno value otherwise and rte_errno is set. 532 */ 533 static int 534 flow_dv_convert_action_modify_ipv6 535 (struct mlx5_flow_dv_modify_hdr_resource *resource, 536 const struct rte_flow_action *action, 537 struct rte_flow_error *error) 538 { 539 const struct rte_flow_action_set_ipv6 *conf = 540 (const struct rte_flow_action_set_ipv6 *)(action->conf); 541 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 542 struct rte_flow_item_ipv6 ipv6; 543 struct rte_flow_item_ipv6 ipv6_mask; 544 545 memset(&ipv6, 0, sizeof(ipv6)); 546 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 547 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) { 548 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr, 549 sizeof(ipv6.hdr.src_addr)); 550 memcpy(&ipv6_mask.hdr.src_addr, 551 &rte_flow_item_ipv6_mask.hdr.src_addr, 552 sizeof(ipv6.hdr.src_addr)); 553 } else { 554 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr, 555 sizeof(ipv6.hdr.dst_addr)); 556 memcpy(&ipv6_mask.hdr.dst_addr, 557 &rte_flow_item_ipv6_mask.hdr.dst_addr, 558 sizeof(ipv6.hdr.dst_addr)); 559 } 560 item.spec = &ipv6; 561 item.mask = &ipv6_mask; 562 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, 563 MLX5_MODIFICATION_TYPE_SET, error); 564 } 565 566 /** 567 * Convert modify-header set MAC address action to DV specification. 568 * 569 * @param[in,out] resource 570 * Pointer to the modify-header resource. 571 * @param[in] action 572 * Pointer to action specification. 573 * @param[out] error 574 * Pointer to the error structure. 575 * 576 * @return 577 * 0 on success, a negative errno value otherwise and rte_errno is set. 578 */ 579 static int 580 flow_dv_convert_action_modify_mac 581 (struct mlx5_flow_dv_modify_hdr_resource *resource, 582 const struct rte_flow_action *action, 583 struct rte_flow_error *error) 584 { 585 const struct rte_flow_action_set_mac *conf = 586 (const struct rte_flow_action_set_mac *)(action->conf); 587 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH }; 588 struct rte_flow_item_eth eth; 589 struct rte_flow_item_eth eth_mask; 590 591 memset(ð, 0, sizeof(eth)); 592 memset(ð_mask, 0, sizeof(eth_mask)); 593 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) { 594 memcpy(ð.src.addr_bytes, &conf->mac_addr, 595 sizeof(eth.src.addr_bytes)); 596 memcpy(ð_mask.src.addr_bytes, 597 &rte_flow_item_eth_mask.src.addr_bytes, 598 sizeof(eth_mask.src.addr_bytes)); 599 } else { 600 memcpy(ð.dst.addr_bytes, &conf->mac_addr, 601 sizeof(eth.dst.addr_bytes)); 602 memcpy(ð_mask.dst.addr_bytes, 603 &rte_flow_item_eth_mask.dst.addr_bytes, 604 sizeof(eth_mask.dst.addr_bytes)); 605 } 606 item.spec = ð 607 item.mask = ð_mask; 608 return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource, 609 MLX5_MODIFICATION_TYPE_SET, error); 610 } 611 612 /** 613 * Convert modify-header set VLAN VID action to DV specification. 614 * 615 * @param[in,out] resource 616 * Pointer to the modify-header resource. 617 * @param[in] action 618 * Pointer to action specification. 619 * @param[out] error 620 * Pointer to the error structure. 621 * 622 * @return 623 * 0 on success, a negative errno value otherwise and rte_errno is set. 624 */ 625 static int 626 flow_dv_convert_action_modify_vlan_vid 627 (struct mlx5_flow_dv_modify_hdr_resource *resource, 628 const struct rte_flow_action *action, 629 struct rte_flow_error *error) 630 { 631 const struct rte_flow_action_of_set_vlan_vid *conf = 632 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf); 633 int i = resource->actions_num; 634 struct mlx5_modification_cmd *actions = resource->actions; 635 struct field_modify_info *field = modify_vlan_out_first_vid; 636 637 if (i >= MLX5_MAX_MODIFY_NUM) 638 return rte_flow_error_set(error, EINVAL, 639 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 640 "too many items to modify"); 641 actions[i] = (struct mlx5_modification_cmd) { 642 .action_type = MLX5_MODIFICATION_TYPE_SET, 643 .field = field->id, 644 .length = field->size, 645 .offset = field->offset, 646 }; 647 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 648 actions[i].data1 = conf->vlan_vid; 649 actions[i].data1 = actions[i].data1 << 16; 650 resource->actions_num = ++i; 651 return 0; 652 } 653 654 /** 655 * Convert modify-header set TP action to DV specification. 656 * 657 * @param[in,out] resource 658 * Pointer to the modify-header resource. 659 * @param[in] action 660 * Pointer to action specification. 661 * @param[in] items 662 * Pointer to rte_flow_item objects list. 663 * @param[in] attr 664 * Pointer to flow attributes structure. 665 * @param[in] dev_flow 666 * Pointer to the sub flow. 667 * @param[in] tunnel_decap 668 * Whether action is after tunnel decapsulation. 669 * @param[out] error 670 * Pointer to the error structure. 671 * 672 * @return 673 * 0 on success, a negative errno value otherwise and rte_errno is set. 674 */ 675 static int 676 flow_dv_convert_action_modify_tp 677 (struct mlx5_flow_dv_modify_hdr_resource *resource, 678 const struct rte_flow_action *action, 679 const struct rte_flow_item *items, 680 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 681 bool tunnel_decap, struct rte_flow_error *error) 682 { 683 const struct rte_flow_action_set_tp *conf = 684 (const struct rte_flow_action_set_tp *)(action->conf); 685 struct rte_flow_item item; 686 struct rte_flow_item_udp udp; 687 struct rte_flow_item_udp udp_mask; 688 struct rte_flow_item_tcp tcp; 689 struct rte_flow_item_tcp tcp_mask; 690 struct field_modify_info *field; 691 692 if (!attr->valid) 693 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 694 if (attr->udp) { 695 memset(&udp, 0, sizeof(udp)); 696 memset(&udp_mask, 0, sizeof(udp_mask)); 697 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 698 udp.hdr.src_port = conf->port; 699 udp_mask.hdr.src_port = 700 rte_flow_item_udp_mask.hdr.src_port; 701 } else { 702 udp.hdr.dst_port = conf->port; 703 udp_mask.hdr.dst_port = 704 rte_flow_item_udp_mask.hdr.dst_port; 705 } 706 item.type = RTE_FLOW_ITEM_TYPE_UDP; 707 item.spec = &udp; 708 item.mask = &udp_mask; 709 field = modify_udp; 710 } else { 711 MLX5_ASSERT(attr->tcp); 712 memset(&tcp, 0, sizeof(tcp)); 713 memset(&tcp_mask, 0, sizeof(tcp_mask)); 714 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 715 tcp.hdr.src_port = conf->port; 716 tcp_mask.hdr.src_port = 717 rte_flow_item_tcp_mask.hdr.src_port; 718 } else { 719 tcp.hdr.dst_port = conf->port; 720 tcp_mask.hdr.dst_port = 721 rte_flow_item_tcp_mask.hdr.dst_port; 722 } 723 item.type = RTE_FLOW_ITEM_TYPE_TCP; 724 item.spec = &tcp; 725 item.mask = &tcp_mask; 726 field = modify_tcp; 727 } 728 return flow_dv_convert_modify_action(&item, field, NULL, resource, 729 MLX5_MODIFICATION_TYPE_SET, error); 730 } 731 732 /** 733 * Convert modify-header set TTL action to DV specification. 734 * 735 * @param[in,out] resource 736 * Pointer to the modify-header resource. 737 * @param[in] action 738 * Pointer to action specification. 739 * @param[in] items 740 * Pointer to rte_flow_item objects list. 741 * @param[in] attr 742 * Pointer to flow attributes structure. 743 * @param[in] dev_flow 744 * Pointer to the sub flow. 745 * @param[in] tunnel_decap 746 * Whether action is after tunnel decapsulation. 747 * @param[out] error 748 * Pointer to the error structure. 749 * 750 * @return 751 * 0 on success, a negative errno value otherwise and rte_errno is set. 752 */ 753 static int 754 flow_dv_convert_action_modify_ttl 755 (struct mlx5_flow_dv_modify_hdr_resource *resource, 756 const struct rte_flow_action *action, 757 const struct rte_flow_item *items, 758 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 759 bool tunnel_decap, struct rte_flow_error *error) 760 { 761 const struct rte_flow_action_set_ttl *conf = 762 (const struct rte_flow_action_set_ttl *)(action->conf); 763 struct rte_flow_item item; 764 struct rte_flow_item_ipv4 ipv4; 765 struct rte_flow_item_ipv4 ipv4_mask; 766 struct rte_flow_item_ipv6 ipv6; 767 struct rte_flow_item_ipv6 ipv6_mask; 768 struct field_modify_info *field; 769 770 if (!attr->valid) 771 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 772 if (attr->ipv4) { 773 memset(&ipv4, 0, sizeof(ipv4)); 774 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 775 ipv4.hdr.time_to_live = conf->ttl_value; 776 ipv4_mask.hdr.time_to_live = 0xFF; 777 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 778 item.spec = &ipv4; 779 item.mask = &ipv4_mask; 780 field = modify_ipv4; 781 } else { 782 MLX5_ASSERT(attr->ipv6); 783 memset(&ipv6, 0, sizeof(ipv6)); 784 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 785 ipv6.hdr.hop_limits = conf->ttl_value; 786 ipv6_mask.hdr.hop_limits = 0xFF; 787 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 788 item.spec = &ipv6; 789 item.mask = &ipv6_mask; 790 field = modify_ipv6; 791 } 792 return flow_dv_convert_modify_action(&item, field, NULL, resource, 793 MLX5_MODIFICATION_TYPE_SET, error); 794 } 795 796 /** 797 * Convert modify-header decrement TTL action to DV specification. 798 * 799 * @param[in,out] resource 800 * Pointer to the modify-header resource. 801 * @param[in] action 802 * Pointer to action specification. 803 * @param[in] items 804 * Pointer to rte_flow_item objects list. 805 * @param[in] attr 806 * Pointer to flow attributes structure. 807 * @param[in] dev_flow 808 * Pointer to the sub flow. 809 * @param[in] tunnel_decap 810 * Whether action is after tunnel decapsulation. 811 * @param[out] error 812 * Pointer to the error structure. 813 * 814 * @return 815 * 0 on success, a negative errno value otherwise and rte_errno is set. 816 */ 817 static int 818 flow_dv_convert_action_modify_dec_ttl 819 (struct mlx5_flow_dv_modify_hdr_resource *resource, 820 const struct rte_flow_item *items, 821 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 822 bool tunnel_decap, struct rte_flow_error *error) 823 { 824 struct rte_flow_item item; 825 struct rte_flow_item_ipv4 ipv4; 826 struct rte_flow_item_ipv4 ipv4_mask; 827 struct rte_flow_item_ipv6 ipv6; 828 struct rte_flow_item_ipv6 ipv6_mask; 829 struct field_modify_info *field; 830 831 if (!attr->valid) 832 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 833 if (attr->ipv4) { 834 memset(&ipv4, 0, sizeof(ipv4)); 835 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 836 ipv4.hdr.time_to_live = 0xFF; 837 ipv4_mask.hdr.time_to_live = 0xFF; 838 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 839 item.spec = &ipv4; 840 item.mask = &ipv4_mask; 841 field = modify_ipv4; 842 } else { 843 MLX5_ASSERT(attr->ipv6); 844 memset(&ipv6, 0, sizeof(ipv6)); 845 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 846 ipv6.hdr.hop_limits = 0xFF; 847 ipv6_mask.hdr.hop_limits = 0xFF; 848 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 849 item.spec = &ipv6; 850 item.mask = &ipv6_mask; 851 field = modify_ipv6; 852 } 853 return flow_dv_convert_modify_action(&item, field, NULL, resource, 854 MLX5_MODIFICATION_TYPE_ADD, error); 855 } 856 857 /** 858 * Convert modify-header increment/decrement TCP Sequence number 859 * to DV specification. 860 * 861 * @param[in,out] resource 862 * Pointer to the modify-header resource. 863 * @param[in] action 864 * Pointer to action specification. 865 * @param[out] error 866 * Pointer to the error structure. 867 * 868 * @return 869 * 0 on success, a negative errno value otherwise and rte_errno is set. 870 */ 871 static int 872 flow_dv_convert_action_modify_tcp_seq 873 (struct mlx5_flow_dv_modify_hdr_resource *resource, 874 const struct rte_flow_action *action, 875 struct rte_flow_error *error) 876 { 877 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 878 uint64_t value = rte_be_to_cpu_32(*conf); 879 struct rte_flow_item item; 880 struct rte_flow_item_tcp tcp; 881 struct rte_flow_item_tcp tcp_mask; 882 883 memset(&tcp, 0, sizeof(tcp)); 884 memset(&tcp_mask, 0, sizeof(tcp_mask)); 885 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ) 886 /* 887 * The HW has no decrement operation, only increment operation. 888 * To simulate decrement X from Y using increment operation 889 * we need to add UINT32_MAX X times to Y. 890 * Each adding of UINT32_MAX decrements Y by 1. 891 */ 892 value *= UINT32_MAX; 893 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value); 894 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX); 895 item.type = RTE_FLOW_ITEM_TYPE_TCP; 896 item.spec = &tcp; 897 item.mask = &tcp_mask; 898 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, 899 MLX5_MODIFICATION_TYPE_ADD, error); 900 } 901 902 /** 903 * Convert modify-header increment/decrement TCP Acknowledgment number 904 * to DV specification. 905 * 906 * @param[in,out] resource 907 * Pointer to the modify-header resource. 908 * @param[in] action 909 * Pointer to action specification. 910 * @param[out] error 911 * Pointer to the error structure. 912 * 913 * @return 914 * 0 on success, a negative errno value otherwise and rte_errno is set. 915 */ 916 static int 917 flow_dv_convert_action_modify_tcp_ack 918 (struct mlx5_flow_dv_modify_hdr_resource *resource, 919 const struct rte_flow_action *action, 920 struct rte_flow_error *error) 921 { 922 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 923 uint64_t value = rte_be_to_cpu_32(*conf); 924 struct rte_flow_item item; 925 struct rte_flow_item_tcp tcp; 926 struct rte_flow_item_tcp tcp_mask; 927 928 memset(&tcp, 0, sizeof(tcp)); 929 memset(&tcp_mask, 0, sizeof(tcp_mask)); 930 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK) 931 /* 932 * The HW has no decrement operation, only increment operation. 933 * To simulate decrement X from Y using increment operation 934 * we need to add UINT32_MAX X times to Y. 935 * Each adding of UINT32_MAX decrements Y by 1. 936 */ 937 value *= UINT32_MAX; 938 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value); 939 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX); 940 item.type = RTE_FLOW_ITEM_TYPE_TCP; 941 item.spec = &tcp; 942 item.mask = &tcp_mask; 943 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, 944 MLX5_MODIFICATION_TYPE_ADD, error); 945 } 946 947 static enum mlx5_modification_field reg_to_field[] = { 948 [REG_NONE] = MLX5_MODI_OUT_NONE, 949 [REG_A] = MLX5_MODI_META_DATA_REG_A, 950 [REG_B] = MLX5_MODI_META_DATA_REG_B, 951 [REG_C_0] = MLX5_MODI_META_REG_C_0, 952 [REG_C_1] = MLX5_MODI_META_REG_C_1, 953 [REG_C_2] = MLX5_MODI_META_REG_C_2, 954 [REG_C_3] = MLX5_MODI_META_REG_C_3, 955 [REG_C_4] = MLX5_MODI_META_REG_C_4, 956 [REG_C_5] = MLX5_MODI_META_REG_C_5, 957 [REG_C_6] = MLX5_MODI_META_REG_C_6, 958 [REG_C_7] = MLX5_MODI_META_REG_C_7, 959 }; 960 961 /** 962 * Convert register set to DV specification. 963 * 964 * @param[in,out] resource 965 * Pointer to the modify-header resource. 966 * @param[in] action 967 * Pointer to action specification. 968 * @param[out] error 969 * Pointer to the error structure. 970 * 971 * @return 972 * 0 on success, a negative errno value otherwise and rte_errno is set. 973 */ 974 static int 975 flow_dv_convert_action_set_reg 976 (struct mlx5_flow_dv_modify_hdr_resource *resource, 977 const struct rte_flow_action *action, 978 struct rte_flow_error *error) 979 { 980 const struct mlx5_rte_flow_action_set_tag *conf = action->conf; 981 struct mlx5_modification_cmd *actions = resource->actions; 982 uint32_t i = resource->actions_num; 983 984 if (i >= MLX5_MAX_MODIFY_NUM) 985 return rte_flow_error_set(error, EINVAL, 986 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 987 "too many items to modify"); 988 MLX5_ASSERT(conf->id != REG_NONE); 989 MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field)); 990 actions[i] = (struct mlx5_modification_cmd) { 991 .action_type = MLX5_MODIFICATION_TYPE_SET, 992 .field = reg_to_field[conf->id], 993 }; 994 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 995 actions[i].data1 = rte_cpu_to_be_32(conf->data); 996 ++i; 997 resource->actions_num = i; 998 return 0; 999 } 1000 1001 /** 1002 * Convert SET_TAG action to DV specification. 1003 * 1004 * @param[in] dev 1005 * Pointer to the rte_eth_dev structure. 1006 * @param[in,out] resource 1007 * Pointer to the modify-header resource. 1008 * @param[in] conf 1009 * Pointer to action specification. 1010 * @param[out] error 1011 * Pointer to the error structure. 1012 * 1013 * @return 1014 * 0 on success, a negative errno value otherwise and rte_errno is set. 1015 */ 1016 static int 1017 flow_dv_convert_action_set_tag 1018 (struct rte_eth_dev *dev, 1019 struct mlx5_flow_dv_modify_hdr_resource *resource, 1020 const struct rte_flow_action_set_tag *conf, 1021 struct rte_flow_error *error) 1022 { 1023 rte_be32_t data = rte_cpu_to_be_32(conf->data); 1024 rte_be32_t mask = rte_cpu_to_be_32(conf->mask); 1025 struct rte_flow_item item = { 1026 .spec = &data, 1027 .mask = &mask, 1028 }; 1029 struct field_modify_info reg_c_x[] = { 1030 [1] = {0, 0, 0}, 1031 }; 1032 enum mlx5_modification_field reg_type; 1033 int ret; 1034 1035 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); 1036 if (ret < 0) 1037 return ret; 1038 MLX5_ASSERT(ret != REG_NONE); 1039 MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field)); 1040 reg_type = reg_to_field[ret]; 1041 MLX5_ASSERT(reg_type > 0); 1042 reg_c_x[0] = (struct field_modify_info){4, 0, reg_type}; 1043 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1044 MLX5_MODIFICATION_TYPE_SET, error); 1045 } 1046 1047 /** 1048 * Convert internal COPY_REG action to DV specification. 1049 * 1050 * @param[in] dev 1051 * Pointer to the rte_eth_dev structure. 1052 * @param[in,out] res 1053 * Pointer to the modify-header resource. 1054 * @param[in] action 1055 * Pointer to action specification. 1056 * @param[out] error 1057 * Pointer to the error structure. 1058 * 1059 * @return 1060 * 0 on success, a negative errno value otherwise and rte_errno is set. 1061 */ 1062 static int 1063 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev, 1064 struct mlx5_flow_dv_modify_hdr_resource *res, 1065 const struct rte_flow_action *action, 1066 struct rte_flow_error *error) 1067 { 1068 const struct mlx5_flow_action_copy_mreg *conf = action->conf; 1069 rte_be32_t mask = RTE_BE32(UINT32_MAX); 1070 struct rte_flow_item item = { 1071 .spec = NULL, 1072 .mask = &mask, 1073 }; 1074 struct field_modify_info reg_src[] = { 1075 {4, 0, reg_to_field[conf->src]}, 1076 {0, 0, 0}, 1077 }; 1078 struct field_modify_info reg_dst = { 1079 .offset = 0, 1080 .id = reg_to_field[conf->dst], 1081 }; 1082 /* Adjust reg_c[0] usage according to reported mask. */ 1083 if (conf->dst == REG_C_0 || conf->src == REG_C_0) { 1084 struct mlx5_priv *priv = dev->data->dev_private; 1085 uint32_t reg_c0 = priv->sh->dv_regc0_mask; 1086 1087 MLX5_ASSERT(reg_c0); 1088 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY); 1089 if (conf->dst == REG_C_0) { 1090 /* Copy to reg_c[0], within mask only. */ 1091 reg_dst.offset = rte_bsf32(reg_c0); 1092 /* 1093 * Mask is ignoring the enianness, because 1094 * there is no conversion in datapath. 1095 */ 1096 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1097 /* Copy from destination lower bits to reg_c[0]. */ 1098 mask = reg_c0 >> reg_dst.offset; 1099 #else 1100 /* Copy from destination upper bits to reg_c[0]. */ 1101 mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT - 1102 rte_fls_u32(reg_c0)); 1103 #endif 1104 } else { 1105 mask = rte_cpu_to_be_32(reg_c0); 1106 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1107 /* Copy from reg_c[0] to destination lower bits. */ 1108 reg_dst.offset = 0; 1109 #else 1110 /* Copy from reg_c[0] to destination upper bits. */ 1111 reg_dst.offset = sizeof(reg_c0) * CHAR_BIT - 1112 (rte_fls_u32(reg_c0) - 1113 rte_bsf32(reg_c0)); 1114 #endif 1115 } 1116 } 1117 return flow_dv_convert_modify_action(&item, 1118 reg_src, ®_dst, res, 1119 MLX5_MODIFICATION_TYPE_COPY, 1120 error); 1121 } 1122 1123 /** 1124 * Convert MARK action to DV specification. This routine is used 1125 * in extensive metadata only and requires metadata register to be 1126 * handled. In legacy mode hardware tag resource is engaged. 1127 * 1128 * @param[in] dev 1129 * Pointer to the rte_eth_dev structure. 1130 * @param[in] conf 1131 * Pointer to MARK action specification. 1132 * @param[in,out] resource 1133 * Pointer to the modify-header resource. 1134 * @param[out] error 1135 * Pointer to the error structure. 1136 * 1137 * @return 1138 * 0 on success, a negative errno value otherwise and rte_errno is set. 1139 */ 1140 static int 1141 flow_dv_convert_action_mark(struct rte_eth_dev *dev, 1142 const struct rte_flow_action_mark *conf, 1143 struct mlx5_flow_dv_modify_hdr_resource *resource, 1144 struct rte_flow_error *error) 1145 { 1146 struct mlx5_priv *priv = dev->data->dev_private; 1147 rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK & 1148 priv->sh->dv_mark_mask); 1149 rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask; 1150 struct rte_flow_item item = { 1151 .spec = &data, 1152 .mask = &mask, 1153 }; 1154 struct field_modify_info reg_c_x[] = { 1155 [1] = {0, 0, 0}, 1156 }; 1157 int reg; 1158 1159 if (!mask) 1160 return rte_flow_error_set(error, EINVAL, 1161 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1162 NULL, "zero mark action mask"); 1163 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 1164 if (reg < 0) 1165 return reg; 1166 MLX5_ASSERT(reg > 0); 1167 if (reg == REG_C_0) { 1168 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 1169 uint32_t shl_c0 = rte_bsf32(msk_c0); 1170 1171 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0); 1172 mask = rte_cpu_to_be_32(mask) & msk_c0; 1173 mask = rte_cpu_to_be_32(mask << shl_c0); 1174 } 1175 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; 1176 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1177 MLX5_MODIFICATION_TYPE_SET, error); 1178 } 1179 1180 /** 1181 * Get metadata register index for specified steering domain. 1182 * 1183 * @param[in] dev 1184 * Pointer to the rte_eth_dev structure. 1185 * @param[in] attr 1186 * Attributes of flow to determine steering domain. 1187 * @param[out] error 1188 * Pointer to the error structure. 1189 * 1190 * @return 1191 * positive index on success, a negative errno value otherwise 1192 * and rte_errno is set. 1193 */ 1194 static enum modify_reg 1195 flow_dv_get_metadata_reg(struct rte_eth_dev *dev, 1196 const struct rte_flow_attr *attr, 1197 struct rte_flow_error *error) 1198 { 1199 int reg = 1200 mlx5_flow_get_reg_id(dev, attr->transfer ? 1201 MLX5_METADATA_FDB : 1202 attr->egress ? 1203 MLX5_METADATA_TX : 1204 MLX5_METADATA_RX, 0, error); 1205 if (reg < 0) 1206 return rte_flow_error_set(error, 1207 ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 1208 NULL, "unavailable " 1209 "metadata register"); 1210 return reg; 1211 } 1212 1213 /** 1214 * Convert SET_META action to DV specification. 1215 * 1216 * @param[in] dev 1217 * Pointer to the rte_eth_dev structure. 1218 * @param[in,out] resource 1219 * Pointer to the modify-header resource. 1220 * @param[in] attr 1221 * Attributes of flow that includes this item. 1222 * @param[in] conf 1223 * Pointer to action specification. 1224 * @param[out] error 1225 * Pointer to the error structure. 1226 * 1227 * @return 1228 * 0 on success, a negative errno value otherwise and rte_errno is set. 1229 */ 1230 static int 1231 flow_dv_convert_action_set_meta 1232 (struct rte_eth_dev *dev, 1233 struct mlx5_flow_dv_modify_hdr_resource *resource, 1234 const struct rte_flow_attr *attr, 1235 const struct rte_flow_action_set_meta *conf, 1236 struct rte_flow_error *error) 1237 { 1238 uint32_t data = conf->data; 1239 uint32_t mask = conf->mask; 1240 struct rte_flow_item item = { 1241 .spec = &data, 1242 .mask = &mask, 1243 }; 1244 struct field_modify_info reg_c_x[] = { 1245 [1] = {0, 0, 0}, 1246 }; 1247 int reg = flow_dv_get_metadata_reg(dev, attr, error); 1248 1249 if (reg < 0) 1250 return reg; 1251 /* 1252 * In datapath code there is no endianness 1253 * coversions for perfromance reasons, all 1254 * pattern conversions are done in rte_flow. 1255 */ 1256 if (reg == REG_C_0) { 1257 struct mlx5_priv *priv = dev->data->dev_private; 1258 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 1259 uint32_t shl_c0; 1260 1261 MLX5_ASSERT(msk_c0); 1262 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1263 shl_c0 = rte_bsf32(msk_c0); 1264 #else 1265 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0); 1266 #endif 1267 mask <<= shl_c0; 1268 data <<= shl_c0; 1269 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask))); 1270 } 1271 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; 1272 /* The routine expects parameters in memory as big-endian ones. */ 1273 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1274 MLX5_MODIFICATION_TYPE_SET, error); 1275 } 1276 1277 /** 1278 * Convert modify-header set IPv4 DSCP action to DV specification. 1279 * 1280 * @param[in,out] resource 1281 * Pointer to the modify-header resource. 1282 * @param[in] action 1283 * Pointer to action specification. 1284 * @param[out] error 1285 * Pointer to the error structure. 1286 * 1287 * @return 1288 * 0 on success, a negative errno value otherwise and rte_errno is set. 1289 */ 1290 static int 1291 flow_dv_convert_action_modify_ipv4_dscp 1292 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1293 const struct rte_flow_action *action, 1294 struct rte_flow_error *error) 1295 { 1296 const struct rte_flow_action_set_dscp *conf = 1297 (const struct rte_flow_action_set_dscp *)(action->conf); 1298 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 1299 struct rte_flow_item_ipv4 ipv4; 1300 struct rte_flow_item_ipv4 ipv4_mask; 1301 1302 memset(&ipv4, 0, sizeof(ipv4)); 1303 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 1304 ipv4.hdr.type_of_service = conf->dscp; 1305 ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2; 1306 item.spec = &ipv4; 1307 item.mask = &ipv4_mask; 1308 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, 1309 MLX5_MODIFICATION_TYPE_SET, error); 1310 } 1311 1312 /** 1313 * Convert modify-header set IPv6 DSCP action to DV specification. 1314 * 1315 * @param[in,out] resource 1316 * Pointer to the modify-header resource. 1317 * @param[in] action 1318 * Pointer to action specification. 1319 * @param[out] error 1320 * Pointer to the error structure. 1321 * 1322 * @return 1323 * 0 on success, a negative errno value otherwise and rte_errno is set. 1324 */ 1325 static int 1326 flow_dv_convert_action_modify_ipv6_dscp 1327 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1328 const struct rte_flow_action *action, 1329 struct rte_flow_error *error) 1330 { 1331 const struct rte_flow_action_set_dscp *conf = 1332 (const struct rte_flow_action_set_dscp *)(action->conf); 1333 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 1334 struct rte_flow_item_ipv6 ipv6; 1335 struct rte_flow_item_ipv6 ipv6_mask; 1336 1337 memset(&ipv6, 0, sizeof(ipv6)); 1338 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 1339 /* 1340 * Even though the DSCP bits offset of IPv6 is not byte aligned, 1341 * rdma-core only accept the DSCP bits byte aligned start from 1342 * bit 0 to 5 as to be compatible with IPv4. No need to shift the 1343 * bits in IPv6 case as rdma-core requires byte aligned value. 1344 */ 1345 ipv6.hdr.vtc_flow = conf->dscp; 1346 ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22; 1347 item.spec = &ipv6; 1348 item.mask = &ipv6_mask; 1349 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, 1350 MLX5_MODIFICATION_TYPE_SET, error); 1351 } 1352 1353 /** 1354 * Validate MARK item. 1355 * 1356 * @param[in] dev 1357 * Pointer to the rte_eth_dev structure. 1358 * @param[in] item 1359 * Item specification. 1360 * @param[in] attr 1361 * Attributes of flow that includes this item. 1362 * @param[out] error 1363 * Pointer to error structure. 1364 * 1365 * @return 1366 * 0 on success, a negative errno value otherwise and rte_errno is set. 1367 */ 1368 static int 1369 flow_dv_validate_item_mark(struct rte_eth_dev *dev, 1370 const struct rte_flow_item *item, 1371 const struct rte_flow_attr *attr __rte_unused, 1372 struct rte_flow_error *error) 1373 { 1374 struct mlx5_priv *priv = dev->data->dev_private; 1375 struct mlx5_dev_config *config = &priv->config; 1376 const struct rte_flow_item_mark *spec = item->spec; 1377 const struct rte_flow_item_mark *mask = item->mask; 1378 const struct rte_flow_item_mark nic_mask = { 1379 .id = priv->sh->dv_mark_mask, 1380 }; 1381 int ret; 1382 1383 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 1384 return rte_flow_error_set(error, ENOTSUP, 1385 RTE_FLOW_ERROR_TYPE_ITEM, item, 1386 "extended metadata feature" 1387 " isn't enabled"); 1388 if (!mlx5_flow_ext_mreg_supported(dev)) 1389 return rte_flow_error_set(error, ENOTSUP, 1390 RTE_FLOW_ERROR_TYPE_ITEM, item, 1391 "extended metadata register" 1392 " isn't supported"); 1393 if (!nic_mask.id) 1394 return rte_flow_error_set(error, ENOTSUP, 1395 RTE_FLOW_ERROR_TYPE_ITEM, item, 1396 "extended metadata register" 1397 " isn't available"); 1398 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 1399 if (ret < 0) 1400 return ret; 1401 if (!spec) 1402 return rte_flow_error_set(error, EINVAL, 1403 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1404 item->spec, 1405 "data cannot be empty"); 1406 if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id)) 1407 return rte_flow_error_set(error, EINVAL, 1408 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1409 &spec->id, 1410 "mark id exceeds the limit"); 1411 if (!mask) 1412 mask = &nic_mask; 1413 if (!mask->id) 1414 return rte_flow_error_set(error, EINVAL, 1415 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1416 "mask cannot be zero"); 1417 1418 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1419 (const uint8_t *)&nic_mask, 1420 sizeof(struct rte_flow_item_mark), 1421 error); 1422 if (ret < 0) 1423 return ret; 1424 return 0; 1425 } 1426 1427 /** 1428 * Validate META item. 1429 * 1430 * @param[in] dev 1431 * Pointer to the rte_eth_dev structure. 1432 * @param[in] item 1433 * Item specification. 1434 * @param[in] attr 1435 * Attributes of flow that includes this item. 1436 * @param[out] error 1437 * Pointer to error structure. 1438 * 1439 * @return 1440 * 0 on success, a negative errno value otherwise and rte_errno is set. 1441 */ 1442 static int 1443 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, 1444 const struct rte_flow_item *item, 1445 const struct rte_flow_attr *attr, 1446 struct rte_flow_error *error) 1447 { 1448 struct mlx5_priv *priv = dev->data->dev_private; 1449 struct mlx5_dev_config *config = &priv->config; 1450 const struct rte_flow_item_meta *spec = item->spec; 1451 const struct rte_flow_item_meta *mask = item->mask; 1452 struct rte_flow_item_meta nic_mask = { 1453 .data = UINT32_MAX 1454 }; 1455 int reg; 1456 int ret; 1457 1458 if (!spec) 1459 return rte_flow_error_set(error, EINVAL, 1460 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1461 item->spec, 1462 "data cannot be empty"); 1463 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 1464 if (!mlx5_flow_ext_mreg_supported(dev)) 1465 return rte_flow_error_set(error, ENOTSUP, 1466 RTE_FLOW_ERROR_TYPE_ITEM, item, 1467 "extended metadata register" 1468 " isn't supported"); 1469 reg = flow_dv_get_metadata_reg(dev, attr, error); 1470 if (reg < 0) 1471 return reg; 1472 if (reg == REG_B) 1473 return rte_flow_error_set(error, ENOTSUP, 1474 RTE_FLOW_ERROR_TYPE_ITEM, item, 1475 "match on reg_b " 1476 "isn't supported"); 1477 if (reg != REG_A) 1478 nic_mask.data = priv->sh->dv_meta_mask; 1479 } else if (attr->transfer) { 1480 return rte_flow_error_set(error, ENOTSUP, 1481 RTE_FLOW_ERROR_TYPE_ITEM, item, 1482 "extended metadata feature " 1483 "should be enabled when " 1484 "meta item is requested " 1485 "with e-switch mode "); 1486 } 1487 if (!mask) 1488 mask = &rte_flow_item_meta_mask; 1489 if (!mask->data) 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1492 "mask cannot be zero"); 1493 1494 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1495 (const uint8_t *)&nic_mask, 1496 sizeof(struct rte_flow_item_meta), 1497 error); 1498 return ret; 1499 } 1500 1501 /** 1502 * Validate TAG item. 1503 * 1504 * @param[in] dev 1505 * Pointer to the rte_eth_dev structure. 1506 * @param[in] item 1507 * Item specification. 1508 * @param[in] attr 1509 * Attributes of flow that includes this item. 1510 * @param[out] error 1511 * Pointer to error structure. 1512 * 1513 * @return 1514 * 0 on success, a negative errno value otherwise and rte_errno is set. 1515 */ 1516 static int 1517 flow_dv_validate_item_tag(struct rte_eth_dev *dev, 1518 const struct rte_flow_item *item, 1519 const struct rte_flow_attr *attr __rte_unused, 1520 struct rte_flow_error *error) 1521 { 1522 const struct rte_flow_item_tag *spec = item->spec; 1523 const struct rte_flow_item_tag *mask = item->mask; 1524 const struct rte_flow_item_tag nic_mask = { 1525 .data = RTE_BE32(UINT32_MAX), 1526 .index = 0xff, 1527 }; 1528 int ret; 1529 1530 if (!mlx5_flow_ext_mreg_supported(dev)) 1531 return rte_flow_error_set(error, ENOTSUP, 1532 RTE_FLOW_ERROR_TYPE_ITEM, item, 1533 "extensive metadata register" 1534 " isn't supported"); 1535 if (!spec) 1536 return rte_flow_error_set(error, EINVAL, 1537 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1538 item->spec, 1539 "data cannot be empty"); 1540 if (!mask) 1541 mask = &rte_flow_item_tag_mask; 1542 if (!mask->data) 1543 return rte_flow_error_set(error, EINVAL, 1544 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1545 "mask cannot be zero"); 1546 1547 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1548 (const uint8_t *)&nic_mask, 1549 sizeof(struct rte_flow_item_tag), 1550 error); 1551 if (ret < 0) 1552 return ret; 1553 if (mask->index != 0xff) 1554 return rte_flow_error_set(error, EINVAL, 1555 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1556 "partial mask for tag index" 1557 " is not supported"); 1558 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error); 1559 if (ret < 0) 1560 return ret; 1561 MLX5_ASSERT(ret != REG_NONE); 1562 return 0; 1563 } 1564 1565 /** 1566 * Validate vport item. 1567 * 1568 * @param[in] dev 1569 * Pointer to the rte_eth_dev structure. 1570 * @param[in] item 1571 * Item specification. 1572 * @param[in] attr 1573 * Attributes of flow that includes this item. 1574 * @param[in] item_flags 1575 * Bit-fields that holds the items detected until now. 1576 * @param[out] error 1577 * Pointer to error structure. 1578 * 1579 * @return 1580 * 0 on success, a negative errno value otherwise and rte_errno is set. 1581 */ 1582 static int 1583 flow_dv_validate_item_port_id(struct rte_eth_dev *dev, 1584 const struct rte_flow_item *item, 1585 const struct rte_flow_attr *attr, 1586 uint64_t item_flags, 1587 struct rte_flow_error *error) 1588 { 1589 const struct rte_flow_item_port_id *spec = item->spec; 1590 const struct rte_flow_item_port_id *mask = item->mask; 1591 const struct rte_flow_item_port_id switch_mask = { 1592 .id = 0xffffffff, 1593 }; 1594 struct mlx5_priv *esw_priv; 1595 struct mlx5_priv *dev_priv; 1596 int ret; 1597 1598 if (!attr->transfer) 1599 return rte_flow_error_set(error, EINVAL, 1600 RTE_FLOW_ERROR_TYPE_ITEM, 1601 NULL, 1602 "match on port id is valid only" 1603 " when transfer flag is enabled"); 1604 if (item_flags & MLX5_FLOW_ITEM_PORT_ID) 1605 return rte_flow_error_set(error, ENOTSUP, 1606 RTE_FLOW_ERROR_TYPE_ITEM, item, 1607 "multiple source ports are not" 1608 " supported"); 1609 if (!mask) 1610 mask = &switch_mask; 1611 if (mask->id != 0xffffffff) 1612 return rte_flow_error_set(error, ENOTSUP, 1613 RTE_FLOW_ERROR_TYPE_ITEM_MASK, 1614 mask, 1615 "no support for partial mask on" 1616 " \"id\" field"); 1617 ret = mlx5_flow_item_acceptable 1618 (item, (const uint8_t *)mask, 1619 (const uint8_t *)&rte_flow_item_port_id_mask, 1620 sizeof(struct rte_flow_item_port_id), 1621 error); 1622 if (ret) 1623 return ret; 1624 if (!spec) 1625 return 0; 1626 esw_priv = mlx5_port_to_eswitch_info(spec->id, false); 1627 if (!esw_priv) 1628 return rte_flow_error_set(error, rte_errno, 1629 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 1630 "failed to obtain E-Switch info for" 1631 " port"); 1632 dev_priv = mlx5_dev_to_eswitch_info(dev); 1633 if (!dev_priv) 1634 return rte_flow_error_set(error, rte_errno, 1635 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1636 NULL, 1637 "failed to obtain E-Switch info"); 1638 if (esw_priv->domain_id != dev_priv->domain_id) 1639 return rte_flow_error_set(error, EINVAL, 1640 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 1641 "cannot match on a port from a" 1642 " different E-Switch"); 1643 return 0; 1644 } 1645 1646 /** 1647 * Validate VLAN item. 1648 * 1649 * @param[in] item 1650 * Item specification. 1651 * @param[in] item_flags 1652 * Bit-fields that holds the items detected until now. 1653 * @param[in] dev 1654 * Ethernet device flow is being created on. 1655 * @param[out] error 1656 * Pointer to error structure. 1657 * 1658 * @return 1659 * 0 on success, a negative errno value otherwise and rte_errno is set. 1660 */ 1661 static int 1662 flow_dv_validate_item_vlan(const struct rte_flow_item *item, 1663 uint64_t item_flags, 1664 struct rte_eth_dev *dev, 1665 struct rte_flow_error *error) 1666 { 1667 const struct rte_flow_item_vlan *mask = item->mask; 1668 const struct rte_flow_item_vlan nic_mask = { 1669 .tci = RTE_BE16(UINT16_MAX), 1670 .inner_type = RTE_BE16(UINT16_MAX), 1671 }; 1672 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1673 int ret; 1674 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1675 MLX5_FLOW_LAYER_INNER_L4) : 1676 (MLX5_FLOW_LAYER_OUTER_L3 | 1677 MLX5_FLOW_LAYER_OUTER_L4); 1678 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1679 MLX5_FLOW_LAYER_OUTER_VLAN; 1680 1681 if (item_flags & vlanm) 1682 return rte_flow_error_set(error, EINVAL, 1683 RTE_FLOW_ERROR_TYPE_ITEM, item, 1684 "multiple VLAN layers not supported"); 1685 else if ((item_flags & l34m) != 0) 1686 return rte_flow_error_set(error, EINVAL, 1687 RTE_FLOW_ERROR_TYPE_ITEM, item, 1688 "VLAN cannot follow L3/L4 layer"); 1689 if (!mask) 1690 mask = &rte_flow_item_vlan_mask; 1691 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1692 (const uint8_t *)&nic_mask, 1693 sizeof(struct rte_flow_item_vlan), 1694 error); 1695 if (ret) 1696 return ret; 1697 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1698 struct mlx5_priv *priv = dev->data->dev_private; 1699 1700 if (priv->vmwa_context) { 1701 /* 1702 * Non-NULL context means we have a virtual machine 1703 * and SR-IOV enabled, we have to create VLAN interface 1704 * to make hypervisor to setup E-Switch vport 1705 * context correctly. We avoid creating the multiple 1706 * VLAN interfaces, so we cannot support VLAN tag mask. 1707 */ 1708 return rte_flow_error_set(error, EINVAL, 1709 RTE_FLOW_ERROR_TYPE_ITEM, 1710 item, 1711 "VLAN tag mask is not" 1712 " supported in virtual" 1713 " environment"); 1714 } 1715 } 1716 return 0; 1717 } 1718 1719 /* 1720 * GTP flags are contained in 1 byte of the format: 1721 * ------------------------------------------- 1722 * | bit | 0 - 2 | 3 | 4 | 5 | 6 | 7 | 1723 * |-----------------------------------------| 1724 * | value | Version | PT | Res | E | S | PN | 1725 * ------------------------------------------- 1726 * 1727 * Matching is supported only for GTP flags E, S, PN. 1728 */ 1729 #define MLX5_GTP_FLAGS_MASK 0x07 1730 1731 /** 1732 * Validate GTP item. 1733 * 1734 * @param[in] dev 1735 * Pointer to the rte_eth_dev structure. 1736 * @param[in] item 1737 * Item specification. 1738 * @param[in] item_flags 1739 * Bit-fields that holds the items detected until now. 1740 * @param[out] error 1741 * Pointer to error structure. 1742 * 1743 * @return 1744 * 0 on success, a negative errno value otherwise and rte_errno is set. 1745 */ 1746 static int 1747 flow_dv_validate_item_gtp(struct rte_eth_dev *dev, 1748 const struct rte_flow_item *item, 1749 uint64_t item_flags, 1750 struct rte_flow_error *error) 1751 { 1752 struct mlx5_priv *priv = dev->data->dev_private; 1753 const struct rte_flow_item_gtp *spec = item->spec; 1754 const struct rte_flow_item_gtp *mask = item->mask; 1755 const struct rte_flow_item_gtp nic_mask = { 1756 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK, 1757 .msg_type = 0xff, 1758 .teid = RTE_BE32(0xffffffff), 1759 }; 1760 1761 if (!priv->config.hca_attr.tunnel_stateless_gtp) 1762 return rte_flow_error_set(error, ENOTSUP, 1763 RTE_FLOW_ERROR_TYPE_ITEM, item, 1764 "GTP support is not enabled"); 1765 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1766 return rte_flow_error_set(error, ENOTSUP, 1767 RTE_FLOW_ERROR_TYPE_ITEM, item, 1768 "multiple tunnel layers not" 1769 " supported"); 1770 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1771 return rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_ITEM, item, 1773 "no outer UDP layer found"); 1774 if (!mask) 1775 mask = &rte_flow_item_gtp_mask; 1776 if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK) 1777 return rte_flow_error_set(error, ENOTSUP, 1778 RTE_FLOW_ERROR_TYPE_ITEM, item, 1779 "Match is supported for GTP" 1780 " flags only"); 1781 return mlx5_flow_item_acceptable 1782 (item, (const uint8_t *)mask, 1783 (const uint8_t *)&nic_mask, 1784 sizeof(struct rte_flow_item_gtp), 1785 error); 1786 } 1787 1788 /** 1789 * Validate the pop VLAN action. 1790 * 1791 * @param[in] dev 1792 * Pointer to the rte_eth_dev structure. 1793 * @param[in] action_flags 1794 * Holds the actions detected until now. 1795 * @param[in] action 1796 * Pointer to the pop vlan action. 1797 * @param[in] item_flags 1798 * The items found in this flow rule. 1799 * @param[in] attr 1800 * Pointer to flow attributes. 1801 * @param[out] error 1802 * Pointer to error structure. 1803 * 1804 * @return 1805 * 0 on success, a negative errno value otherwise and rte_errno is set. 1806 */ 1807 static int 1808 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, 1809 uint64_t action_flags, 1810 const struct rte_flow_action *action, 1811 uint64_t item_flags, 1812 const struct rte_flow_attr *attr, 1813 struct rte_flow_error *error) 1814 { 1815 const struct mlx5_priv *priv = dev->data->dev_private; 1816 1817 (void)action; 1818 (void)attr; 1819 if (!priv->sh->pop_vlan_action) 1820 return rte_flow_error_set(error, ENOTSUP, 1821 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1822 NULL, 1823 "pop vlan action is not supported"); 1824 if (attr->egress) 1825 return rte_flow_error_set(error, ENOTSUP, 1826 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 1827 NULL, 1828 "pop vlan action not supported for " 1829 "egress"); 1830 if (action_flags & MLX5_FLOW_VLAN_ACTIONS) 1831 return rte_flow_error_set(error, ENOTSUP, 1832 RTE_FLOW_ERROR_TYPE_ACTION, action, 1833 "no support for multiple VLAN " 1834 "actions"); 1835 /* Pop VLAN with preceding Decap requires inner header with VLAN. */ 1836 if ((action_flags & MLX5_FLOW_ACTION_DECAP) && 1837 !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN)) 1838 return rte_flow_error_set(error, ENOTSUP, 1839 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1840 NULL, 1841 "cannot pop vlan after decap without " 1842 "match on inner vlan in the flow"); 1843 /* Pop VLAN without preceding Decap requires outer header with VLAN. */ 1844 if (!(action_flags & MLX5_FLOW_ACTION_DECAP) && 1845 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) 1846 return rte_flow_error_set(error, ENOTSUP, 1847 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1848 NULL, 1849 "cannot pop vlan without a " 1850 "match on (outer) vlan in the flow"); 1851 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 1852 return rte_flow_error_set(error, EINVAL, 1853 RTE_FLOW_ERROR_TYPE_ACTION, action, 1854 "wrong action order, port_id should " 1855 "be after pop VLAN action"); 1856 if (!attr->transfer && priv->representor) 1857 return rte_flow_error_set(error, ENOTSUP, 1858 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1859 "pop vlan action for VF representor " 1860 "not supported on NIC table"); 1861 return 0; 1862 } 1863 1864 /** 1865 * Get VLAN default info from vlan match info. 1866 * 1867 * @param[in] items 1868 * the list of item specifications. 1869 * @param[out] vlan 1870 * pointer VLAN info to fill to. 1871 * 1872 * @return 1873 * 0 on success, a negative errno value otherwise and rte_errno is set. 1874 */ 1875 static void 1876 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, 1877 struct rte_vlan_hdr *vlan) 1878 { 1879 const struct rte_flow_item_vlan nic_mask = { 1880 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK | 1881 MLX5DV_FLOW_VLAN_VID_MASK), 1882 .inner_type = RTE_BE16(0xffff), 1883 }; 1884 1885 if (items == NULL) 1886 return; 1887 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1888 int type = items->type; 1889 1890 if (type == RTE_FLOW_ITEM_TYPE_VLAN || 1891 type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN) 1892 break; 1893 } 1894 if (items->type != RTE_FLOW_ITEM_TYPE_END) { 1895 const struct rte_flow_item_vlan *vlan_m = items->mask; 1896 const struct rte_flow_item_vlan *vlan_v = items->spec; 1897 1898 /* If VLAN item in pattern doesn't contain data, return here. */ 1899 if (!vlan_v) 1900 return; 1901 if (!vlan_m) 1902 vlan_m = &nic_mask; 1903 /* Only full match values are accepted */ 1904 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) == 1905 MLX5DV_FLOW_VLAN_PCP_MASK_BE) { 1906 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; 1907 vlan->vlan_tci |= 1908 rte_be_to_cpu_16(vlan_v->tci & 1909 MLX5DV_FLOW_VLAN_PCP_MASK_BE); 1910 } 1911 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) == 1912 MLX5DV_FLOW_VLAN_VID_MASK_BE) { 1913 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK; 1914 vlan->vlan_tci |= 1915 rte_be_to_cpu_16(vlan_v->tci & 1916 MLX5DV_FLOW_VLAN_VID_MASK_BE); 1917 } 1918 if (vlan_m->inner_type == nic_mask.inner_type) 1919 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type & 1920 vlan_m->inner_type); 1921 } 1922 } 1923 1924 /** 1925 * Validate the push VLAN action. 1926 * 1927 * @param[in] dev 1928 * Pointer to the rte_eth_dev structure. 1929 * @param[in] action_flags 1930 * Holds the actions detected until now. 1931 * @param[in] item_flags 1932 * The items found in this flow rule. 1933 * @param[in] action 1934 * Pointer to the action structure. 1935 * @param[in] attr 1936 * Pointer to flow attributes 1937 * @param[out] error 1938 * Pointer to error structure. 1939 * 1940 * @return 1941 * 0 on success, a negative errno value otherwise and rte_errno is set. 1942 */ 1943 static int 1944 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev, 1945 uint64_t action_flags, 1946 const struct rte_flow_item_vlan *vlan_m, 1947 const struct rte_flow_action *action, 1948 const struct rte_flow_attr *attr, 1949 struct rte_flow_error *error) 1950 { 1951 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf; 1952 const struct mlx5_priv *priv = dev->data->dev_private; 1953 1954 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) && 1955 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ)) 1956 return rte_flow_error_set(error, EINVAL, 1957 RTE_FLOW_ERROR_TYPE_ACTION, action, 1958 "invalid vlan ethertype"); 1959 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 1960 return rte_flow_error_set(error, EINVAL, 1961 RTE_FLOW_ERROR_TYPE_ACTION, action, 1962 "wrong action order, port_id should " 1963 "be after push VLAN"); 1964 if (!attr->transfer && priv->representor) 1965 return rte_flow_error_set(error, ENOTSUP, 1966 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1967 "push vlan action for VF representor " 1968 "not supported on NIC table"); 1969 if (vlan_m && 1970 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) && 1971 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) != 1972 MLX5DV_FLOW_VLAN_PCP_MASK_BE && 1973 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) && 1974 !(mlx5_flow_find_action 1975 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP))) 1976 return rte_flow_error_set(error, EINVAL, 1977 RTE_FLOW_ERROR_TYPE_ACTION, action, 1978 "not full match mask on VLAN PCP and " 1979 "there is no of_set_vlan_pcp action, " 1980 "push VLAN action cannot figure out " 1981 "PCP value"); 1982 if (vlan_m && 1983 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) && 1984 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) != 1985 MLX5DV_FLOW_VLAN_VID_MASK_BE && 1986 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) && 1987 !(mlx5_flow_find_action 1988 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID))) 1989 return rte_flow_error_set(error, EINVAL, 1990 RTE_FLOW_ERROR_TYPE_ACTION, action, 1991 "not full match mask on VLAN VID and " 1992 "there is no of_set_vlan_vid action, " 1993 "push VLAN action cannot figure out " 1994 "VID value"); 1995 (void)attr; 1996 return 0; 1997 } 1998 1999 /** 2000 * Validate the set VLAN PCP. 2001 * 2002 * @param[in] action_flags 2003 * Holds the actions detected until now. 2004 * @param[in] actions 2005 * Pointer to the list of actions remaining in the flow rule. 2006 * @param[out] error 2007 * Pointer to error structure. 2008 * 2009 * @return 2010 * 0 on success, a negative errno value otherwise and rte_errno is set. 2011 */ 2012 static int 2013 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags, 2014 const struct rte_flow_action actions[], 2015 struct rte_flow_error *error) 2016 { 2017 const struct rte_flow_action *action = actions; 2018 const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf; 2019 2020 if (conf->vlan_pcp > 7) 2021 return rte_flow_error_set(error, EINVAL, 2022 RTE_FLOW_ERROR_TYPE_ACTION, action, 2023 "VLAN PCP value is too big"); 2024 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)) 2025 return rte_flow_error_set(error, ENOTSUP, 2026 RTE_FLOW_ERROR_TYPE_ACTION, action, 2027 "set VLAN PCP action must follow " 2028 "the push VLAN action"); 2029 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) 2030 return rte_flow_error_set(error, ENOTSUP, 2031 RTE_FLOW_ERROR_TYPE_ACTION, action, 2032 "Multiple VLAN PCP modification are " 2033 "not supported"); 2034 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2035 return rte_flow_error_set(error, EINVAL, 2036 RTE_FLOW_ERROR_TYPE_ACTION, action, 2037 "wrong action order, port_id should " 2038 "be after set VLAN PCP"); 2039 return 0; 2040 } 2041 2042 /** 2043 * Validate the set VLAN VID. 2044 * 2045 * @param[in] item_flags 2046 * Holds the items detected in this rule. 2047 * @param[in] action_flags 2048 * Holds the actions detected until now. 2049 * @param[in] actions 2050 * Pointer to the list of actions remaining in the flow rule. 2051 * @param[out] error 2052 * Pointer to error structure. 2053 * 2054 * @return 2055 * 0 on success, a negative errno value otherwise and rte_errno is set. 2056 */ 2057 static int 2058 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags, 2059 uint64_t action_flags, 2060 const struct rte_flow_action actions[], 2061 struct rte_flow_error *error) 2062 { 2063 const struct rte_flow_action *action = actions; 2064 const struct rte_flow_action_of_set_vlan_vid *conf = action->conf; 2065 2066 if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE) 2067 return rte_flow_error_set(error, EINVAL, 2068 RTE_FLOW_ERROR_TYPE_ACTION, action, 2069 "VLAN VID value is too big"); 2070 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) && 2071 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) 2072 return rte_flow_error_set(error, ENOTSUP, 2073 RTE_FLOW_ERROR_TYPE_ACTION, action, 2074 "set VLAN VID action must follow push" 2075 " VLAN action or match on VLAN item"); 2076 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) 2077 return rte_flow_error_set(error, ENOTSUP, 2078 RTE_FLOW_ERROR_TYPE_ACTION, action, 2079 "Multiple VLAN VID modifications are " 2080 "not supported"); 2081 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2082 return rte_flow_error_set(error, EINVAL, 2083 RTE_FLOW_ERROR_TYPE_ACTION, action, 2084 "wrong action order, port_id should " 2085 "be after set VLAN VID"); 2086 return 0; 2087 } 2088 2089 /* 2090 * Validate the FLAG action. 2091 * 2092 * @param[in] dev 2093 * Pointer to the rte_eth_dev structure. 2094 * @param[in] action_flags 2095 * Holds the actions detected until now. 2096 * @param[in] attr 2097 * Pointer to flow attributes 2098 * @param[out] error 2099 * Pointer to error structure. 2100 * 2101 * @return 2102 * 0 on success, a negative errno value otherwise and rte_errno is set. 2103 */ 2104 static int 2105 flow_dv_validate_action_flag(struct rte_eth_dev *dev, 2106 uint64_t action_flags, 2107 const struct rte_flow_attr *attr, 2108 struct rte_flow_error *error) 2109 { 2110 struct mlx5_priv *priv = dev->data->dev_private; 2111 struct mlx5_dev_config *config = &priv->config; 2112 int ret; 2113 2114 /* Fall back if no extended metadata register support. */ 2115 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 2116 return mlx5_flow_validate_action_flag(action_flags, attr, 2117 error); 2118 /* Extensive metadata mode requires registers. */ 2119 if (!mlx5_flow_ext_mreg_supported(dev)) 2120 return rte_flow_error_set(error, ENOTSUP, 2121 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2122 "no metadata registers " 2123 "to support flag action"); 2124 if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT)) 2125 return rte_flow_error_set(error, ENOTSUP, 2126 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2127 "extended metadata register" 2128 " isn't available"); 2129 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2130 if (ret < 0) 2131 return ret; 2132 MLX5_ASSERT(ret > 0); 2133 if (action_flags & MLX5_FLOW_ACTION_MARK) 2134 return rte_flow_error_set(error, EINVAL, 2135 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2136 "can't mark and flag in same flow"); 2137 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2138 return rte_flow_error_set(error, EINVAL, 2139 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2140 "can't have 2 flag" 2141 " actions in same flow"); 2142 return 0; 2143 } 2144 2145 /** 2146 * Validate MARK action. 2147 * 2148 * @param[in] dev 2149 * Pointer to the rte_eth_dev structure. 2150 * @param[in] action 2151 * Pointer to action. 2152 * @param[in] action_flags 2153 * Holds the actions detected until now. 2154 * @param[in] attr 2155 * Pointer to flow attributes 2156 * @param[out] error 2157 * Pointer to error structure. 2158 * 2159 * @return 2160 * 0 on success, a negative errno value otherwise and rte_errno is set. 2161 */ 2162 static int 2163 flow_dv_validate_action_mark(struct rte_eth_dev *dev, 2164 const struct rte_flow_action *action, 2165 uint64_t action_flags, 2166 const struct rte_flow_attr *attr, 2167 struct rte_flow_error *error) 2168 { 2169 struct mlx5_priv *priv = dev->data->dev_private; 2170 struct mlx5_dev_config *config = &priv->config; 2171 const struct rte_flow_action_mark *mark = action->conf; 2172 int ret; 2173 2174 /* Fall back if no extended metadata register support. */ 2175 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 2176 return mlx5_flow_validate_action_mark(action, action_flags, 2177 attr, error); 2178 /* Extensive metadata mode requires registers. */ 2179 if (!mlx5_flow_ext_mreg_supported(dev)) 2180 return rte_flow_error_set(error, ENOTSUP, 2181 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2182 "no metadata registers " 2183 "to support mark action"); 2184 if (!priv->sh->dv_mark_mask) 2185 return rte_flow_error_set(error, ENOTSUP, 2186 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2187 "extended metadata register" 2188 " isn't available"); 2189 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2190 if (ret < 0) 2191 return ret; 2192 MLX5_ASSERT(ret > 0); 2193 if (!mark) 2194 return rte_flow_error_set(error, EINVAL, 2195 RTE_FLOW_ERROR_TYPE_ACTION, action, 2196 "configuration cannot be null"); 2197 if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask)) 2198 return rte_flow_error_set(error, EINVAL, 2199 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2200 &mark->id, 2201 "mark id exceeds the limit"); 2202 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2203 return rte_flow_error_set(error, EINVAL, 2204 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2205 "can't flag and mark in same flow"); 2206 if (action_flags & MLX5_FLOW_ACTION_MARK) 2207 return rte_flow_error_set(error, EINVAL, 2208 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2209 "can't have 2 mark actions in same" 2210 " flow"); 2211 return 0; 2212 } 2213 2214 /** 2215 * Validate SET_META action. 2216 * 2217 * @param[in] dev 2218 * Pointer to the rte_eth_dev structure. 2219 * @param[in] action 2220 * Pointer to the action structure. 2221 * @param[in] action_flags 2222 * Holds the actions detected until now. 2223 * @param[in] attr 2224 * Pointer to flow attributes 2225 * @param[out] error 2226 * Pointer to error structure. 2227 * 2228 * @return 2229 * 0 on success, a negative errno value otherwise and rte_errno is set. 2230 */ 2231 static int 2232 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, 2233 const struct rte_flow_action *action, 2234 uint64_t action_flags __rte_unused, 2235 const struct rte_flow_attr *attr, 2236 struct rte_flow_error *error) 2237 { 2238 const struct rte_flow_action_set_meta *conf; 2239 uint32_t nic_mask = UINT32_MAX; 2240 int reg; 2241 2242 if (!mlx5_flow_ext_mreg_supported(dev)) 2243 return rte_flow_error_set(error, ENOTSUP, 2244 RTE_FLOW_ERROR_TYPE_ACTION, action, 2245 "extended metadata register" 2246 " isn't supported"); 2247 reg = flow_dv_get_metadata_reg(dev, attr, error); 2248 if (reg < 0) 2249 return reg; 2250 if (reg != REG_A && reg != REG_B) { 2251 struct mlx5_priv *priv = dev->data->dev_private; 2252 2253 nic_mask = priv->sh->dv_meta_mask; 2254 } 2255 if (!(action->conf)) 2256 return rte_flow_error_set(error, EINVAL, 2257 RTE_FLOW_ERROR_TYPE_ACTION, action, 2258 "configuration cannot be null"); 2259 conf = (const struct rte_flow_action_set_meta *)action->conf; 2260 if (!conf->mask) 2261 return rte_flow_error_set(error, EINVAL, 2262 RTE_FLOW_ERROR_TYPE_ACTION, action, 2263 "zero mask doesn't have any effect"); 2264 if (conf->mask & ~nic_mask) 2265 return rte_flow_error_set(error, EINVAL, 2266 RTE_FLOW_ERROR_TYPE_ACTION, action, 2267 "meta data must be within reg C0"); 2268 return 0; 2269 } 2270 2271 /** 2272 * Validate SET_TAG action. 2273 * 2274 * @param[in] dev 2275 * Pointer to the rte_eth_dev structure. 2276 * @param[in] action 2277 * Pointer to the action structure. 2278 * @param[in] action_flags 2279 * Holds the actions detected until now. 2280 * @param[in] attr 2281 * Pointer to flow attributes 2282 * @param[out] error 2283 * Pointer to error structure. 2284 * 2285 * @return 2286 * 0 on success, a negative errno value otherwise and rte_errno is set. 2287 */ 2288 static int 2289 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, 2290 const struct rte_flow_action *action, 2291 uint64_t action_flags, 2292 const struct rte_flow_attr *attr, 2293 struct rte_flow_error *error) 2294 { 2295 const struct rte_flow_action_set_tag *conf; 2296 const uint64_t terminal_action_flags = 2297 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE | 2298 MLX5_FLOW_ACTION_RSS; 2299 int ret; 2300 2301 if (!mlx5_flow_ext_mreg_supported(dev)) 2302 return rte_flow_error_set(error, ENOTSUP, 2303 RTE_FLOW_ERROR_TYPE_ACTION, action, 2304 "extensive metadata register" 2305 " isn't supported"); 2306 if (!(action->conf)) 2307 return rte_flow_error_set(error, EINVAL, 2308 RTE_FLOW_ERROR_TYPE_ACTION, action, 2309 "configuration cannot be null"); 2310 conf = (const struct rte_flow_action_set_tag *)action->conf; 2311 if (!conf->mask) 2312 return rte_flow_error_set(error, EINVAL, 2313 RTE_FLOW_ERROR_TYPE_ACTION, action, 2314 "zero mask doesn't have any effect"); 2315 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); 2316 if (ret < 0) 2317 return ret; 2318 if (!attr->transfer && attr->ingress && 2319 (action_flags & terminal_action_flags)) 2320 return rte_flow_error_set(error, EINVAL, 2321 RTE_FLOW_ERROR_TYPE_ACTION, action, 2322 "set_tag has no effect" 2323 " with terminal actions"); 2324 return 0; 2325 } 2326 2327 /** 2328 * Validate count action. 2329 * 2330 * @param[in] dev 2331 * Pointer to rte_eth_dev structure. 2332 * @param[out] error 2333 * Pointer to error structure. 2334 * 2335 * @return 2336 * 0 on success, a negative errno value otherwise and rte_errno is set. 2337 */ 2338 static int 2339 flow_dv_validate_action_count(struct rte_eth_dev *dev, 2340 struct rte_flow_error *error) 2341 { 2342 struct mlx5_priv *priv = dev->data->dev_private; 2343 2344 if (!priv->config.devx) 2345 goto notsup_err; 2346 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS 2347 return 0; 2348 #endif 2349 notsup_err: 2350 return rte_flow_error_set 2351 (error, ENOTSUP, 2352 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2353 NULL, 2354 "count action not supported"); 2355 } 2356 2357 /** 2358 * Validate the L2 encap action. 2359 * 2360 * @param[in] dev 2361 * Pointer to the rte_eth_dev structure. 2362 * @param[in] action_flags 2363 * Holds the actions detected until now. 2364 * @param[in] action 2365 * Pointer to the action structure. 2366 * @param[in] attr 2367 * Pointer to flow attributes. 2368 * @param[out] error 2369 * Pointer to error structure. 2370 * 2371 * @return 2372 * 0 on success, a negative errno value otherwise and rte_errno is set. 2373 */ 2374 static int 2375 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, 2376 uint64_t action_flags, 2377 const struct rte_flow_action *action, 2378 const struct rte_flow_attr *attr, 2379 struct rte_flow_error *error) 2380 { 2381 const struct mlx5_priv *priv = dev->data->dev_private; 2382 2383 if (!(action->conf)) 2384 return rte_flow_error_set(error, EINVAL, 2385 RTE_FLOW_ERROR_TYPE_ACTION, action, 2386 "configuration cannot be null"); 2387 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 2388 return rte_flow_error_set(error, EINVAL, 2389 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2390 "can only have a single encap action " 2391 "in a flow"); 2392 if (!attr->transfer && priv->representor) 2393 return rte_flow_error_set(error, ENOTSUP, 2394 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2395 "encap action for VF representor " 2396 "not supported on NIC table"); 2397 return 0; 2398 } 2399 2400 /** 2401 * Validate a decap action. 2402 * 2403 * @param[in] dev 2404 * Pointer to the rte_eth_dev structure. 2405 * @param[in] action_flags 2406 * Holds the actions detected until now. 2407 * @param[in] attr 2408 * Pointer to flow attributes 2409 * @param[out] error 2410 * Pointer to error structure. 2411 * 2412 * @return 2413 * 0 on success, a negative errno value otherwise and rte_errno is set. 2414 */ 2415 static int 2416 flow_dv_validate_action_decap(struct rte_eth_dev *dev, 2417 uint64_t action_flags, 2418 const struct rte_flow_attr *attr, 2419 struct rte_flow_error *error) 2420 { 2421 const struct mlx5_priv *priv = dev->data->dev_private; 2422 2423 if (priv->config.hca_attr.scatter_fcs_w_decap_disable && 2424 !priv->config.decap_en) 2425 return rte_flow_error_set(error, ENOTSUP, 2426 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2427 "decap is not enabled"); 2428 if (action_flags & MLX5_FLOW_XCAP_ACTIONS) 2429 return rte_flow_error_set(error, ENOTSUP, 2430 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2431 action_flags & 2432 MLX5_FLOW_ACTION_DECAP ? "can only " 2433 "have a single decap action" : "decap " 2434 "after encap is not supported"); 2435 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) 2436 return rte_flow_error_set(error, EINVAL, 2437 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2438 "can't have decap action after" 2439 " modify action"); 2440 if (attr->egress) 2441 return rte_flow_error_set(error, ENOTSUP, 2442 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 2443 NULL, 2444 "decap action not supported for " 2445 "egress"); 2446 if (!attr->transfer && priv->representor) 2447 return rte_flow_error_set(error, ENOTSUP, 2448 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2449 "decap action for VF representor " 2450 "not supported on NIC table"); 2451 return 0; 2452 } 2453 2454 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,}; 2455 2456 /** 2457 * Validate the raw encap and decap actions. 2458 * 2459 * @param[in] dev 2460 * Pointer to the rte_eth_dev structure. 2461 * @param[in] decap 2462 * Pointer to the decap action. 2463 * @param[in] encap 2464 * Pointer to the encap action. 2465 * @param[in] attr 2466 * Pointer to flow attributes 2467 * @param[in/out] action_flags 2468 * Holds the actions detected until now. 2469 * @param[out] actions_n 2470 * pointer to the number of actions counter. 2471 * @param[out] error 2472 * Pointer to error structure. 2473 * 2474 * @return 2475 * 0 on success, a negative errno value otherwise and rte_errno is set. 2476 */ 2477 static int 2478 flow_dv_validate_action_raw_encap_decap 2479 (struct rte_eth_dev *dev, 2480 const struct rte_flow_action_raw_decap *decap, 2481 const struct rte_flow_action_raw_encap *encap, 2482 const struct rte_flow_attr *attr, uint64_t *action_flags, 2483 int *actions_n, struct rte_flow_error *error) 2484 { 2485 const struct mlx5_priv *priv = dev->data->dev_private; 2486 int ret; 2487 2488 if (encap && (!encap->size || !encap->data)) 2489 return rte_flow_error_set(error, EINVAL, 2490 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2491 "raw encap data cannot be empty"); 2492 if (decap && encap) { 2493 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE && 2494 encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2495 /* L3 encap. */ 2496 decap = NULL; 2497 else if (encap->size <= 2498 MLX5_ENCAPSULATION_DECISION_SIZE && 2499 decap->size > 2500 MLX5_ENCAPSULATION_DECISION_SIZE) 2501 /* L3 decap. */ 2502 encap = NULL; 2503 else if (encap->size > 2504 MLX5_ENCAPSULATION_DECISION_SIZE && 2505 decap->size > 2506 MLX5_ENCAPSULATION_DECISION_SIZE) 2507 /* 2 L2 actions: encap and decap. */ 2508 ; 2509 else 2510 return rte_flow_error_set(error, 2511 ENOTSUP, 2512 RTE_FLOW_ERROR_TYPE_ACTION, 2513 NULL, "unsupported too small " 2514 "raw decap and too small raw " 2515 "encap combination"); 2516 } 2517 if (decap) { 2518 ret = flow_dv_validate_action_decap(dev, *action_flags, attr, 2519 error); 2520 if (ret < 0) 2521 return ret; 2522 *action_flags |= MLX5_FLOW_ACTION_DECAP; 2523 ++(*actions_n); 2524 } 2525 if (encap) { 2526 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE) 2527 return rte_flow_error_set(error, ENOTSUP, 2528 RTE_FLOW_ERROR_TYPE_ACTION, 2529 NULL, 2530 "small raw encap size"); 2531 if (*action_flags & MLX5_FLOW_ACTION_ENCAP) 2532 return rte_flow_error_set(error, EINVAL, 2533 RTE_FLOW_ERROR_TYPE_ACTION, 2534 NULL, 2535 "more than one encap action"); 2536 if (!attr->transfer && priv->representor) 2537 return rte_flow_error_set 2538 (error, ENOTSUP, 2539 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2540 "encap action for VF representor " 2541 "not supported on NIC table"); 2542 *action_flags |= MLX5_FLOW_ACTION_ENCAP; 2543 ++(*actions_n); 2544 } 2545 return 0; 2546 } 2547 2548 /** 2549 * Find existing encap/decap resource or create and register a new one. 2550 * 2551 * @param[in, out] dev 2552 * Pointer to rte_eth_dev structure. 2553 * @param[in, out] resource 2554 * Pointer to encap/decap resource. 2555 * @parm[in, out] dev_flow 2556 * Pointer to the dev_flow. 2557 * @param[out] error 2558 * pointer to error structure. 2559 * 2560 * @return 2561 * 0 on success otherwise -errno and errno is set. 2562 */ 2563 static int 2564 flow_dv_encap_decap_resource_register 2565 (struct rte_eth_dev *dev, 2566 struct mlx5_flow_dv_encap_decap_resource *resource, 2567 struct mlx5_flow *dev_flow, 2568 struct rte_flow_error *error) 2569 { 2570 struct mlx5_priv *priv = dev->data->dev_private; 2571 struct mlx5_dev_ctx_shared *sh = priv->sh; 2572 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 2573 struct mlx5dv_dr_domain *domain; 2574 uint32_t idx = 0; 2575 int ret; 2576 2577 resource->flags = dev_flow->dv.group ? 0 : 1; 2578 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 2579 domain = sh->fdb_domain; 2580 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 2581 domain = sh->rx_domain; 2582 else 2583 domain = sh->tx_domain; 2584 /* Lookup a matching resource from cache. */ 2585 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx, 2586 cache_resource, next) { 2587 if (resource->reformat_type == cache_resource->reformat_type && 2588 resource->ft_type == cache_resource->ft_type && 2589 resource->flags == cache_resource->flags && 2590 resource->size == cache_resource->size && 2591 !memcmp((const void *)resource->buf, 2592 (const void *)cache_resource->buf, 2593 resource->size)) { 2594 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++", 2595 (void *)cache_resource, 2596 rte_atomic32_read(&cache_resource->refcnt)); 2597 rte_atomic32_inc(&cache_resource->refcnt); 2598 dev_flow->handle->dvh.rix_encap_decap = idx; 2599 dev_flow->dv.encap_decap = cache_resource; 2600 return 0; 2601 } 2602 } 2603 /* Register new encap/decap resource. */ 2604 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 2605 &dev_flow->handle->dvh.rix_encap_decap); 2606 if (!cache_resource) 2607 return rte_flow_error_set(error, ENOMEM, 2608 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2609 "cannot allocate resource memory"); 2610 *cache_resource = *resource; 2611 ret = mlx5_flow_os_create_flow_action_packet_reformat 2612 (sh->ctx, domain, cache_resource, 2613 &cache_resource->action); 2614 if (ret) { 2615 mlx5_free(cache_resource); 2616 return rte_flow_error_set(error, ENOMEM, 2617 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2618 NULL, "cannot create action"); 2619 } 2620 rte_atomic32_init(&cache_resource->refcnt); 2621 rte_atomic32_inc(&cache_resource->refcnt); 2622 ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps, 2623 dev_flow->handle->dvh.rix_encap_decap, cache_resource, 2624 next); 2625 dev_flow->dv.encap_decap = cache_resource; 2626 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", 2627 (void *)cache_resource, 2628 rte_atomic32_read(&cache_resource->refcnt)); 2629 return 0; 2630 } 2631 2632 /** 2633 * Find existing table jump resource or create and register a new one. 2634 * 2635 * @param[in, out] dev 2636 * Pointer to rte_eth_dev structure. 2637 * @param[in, out] tbl 2638 * Pointer to flow table resource. 2639 * @parm[in, out] dev_flow 2640 * Pointer to the dev_flow. 2641 * @param[out] error 2642 * pointer to error structure. 2643 * 2644 * @return 2645 * 0 on success otherwise -errno and errno is set. 2646 */ 2647 static int 2648 flow_dv_jump_tbl_resource_register 2649 (struct rte_eth_dev *dev __rte_unused, 2650 struct mlx5_flow_tbl_resource *tbl, 2651 struct mlx5_flow *dev_flow, 2652 struct rte_flow_error *error) 2653 { 2654 struct mlx5_flow_tbl_data_entry *tbl_data = 2655 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 2656 int cnt, ret; 2657 2658 MLX5_ASSERT(tbl); 2659 cnt = rte_atomic32_read(&tbl_data->jump.refcnt); 2660 if (!cnt) { 2661 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl 2662 (tbl->obj, &tbl_data->jump.action); 2663 if (ret) 2664 return rte_flow_error_set(error, ENOMEM, 2665 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2666 NULL, "cannot create jump action"); 2667 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", 2668 (void *)&tbl_data->jump, cnt); 2669 } else { 2670 /* old jump should not make the table ref++. */ 2671 flow_dv_tbl_resource_release(dev, &tbl_data->tbl); 2672 MLX5_ASSERT(tbl_data->jump.action); 2673 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++", 2674 (void *)&tbl_data->jump, cnt); 2675 } 2676 rte_atomic32_inc(&tbl_data->jump.refcnt); 2677 dev_flow->handle->rix_jump = tbl_data->idx; 2678 dev_flow->dv.jump = &tbl_data->jump; 2679 return 0; 2680 } 2681 2682 /** 2683 * Find existing default miss resource or create and register a new one. 2684 * 2685 * @param[in, out] dev 2686 * Pointer to rte_eth_dev structure. 2687 * @param[out] error 2688 * pointer to error structure. 2689 * 2690 * @return 2691 * 0 on success otherwise -errno and errno is set. 2692 */ 2693 static int 2694 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev, 2695 struct rte_flow_error *error) 2696 { 2697 struct mlx5_priv *priv = dev->data->dev_private; 2698 struct mlx5_dev_ctx_shared *sh = priv->sh; 2699 struct mlx5_flow_default_miss_resource *cache_resource = 2700 &sh->default_miss; 2701 int cnt = rte_atomic32_read(&cache_resource->refcnt); 2702 2703 if (!cnt) { 2704 MLX5_ASSERT(cache_resource->action); 2705 cache_resource->action = 2706 mlx5_glue->dr_create_flow_action_default_miss(); 2707 if (!cache_resource->action) 2708 return rte_flow_error_set(error, ENOMEM, 2709 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2710 "cannot create default miss action"); 2711 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++", 2712 (void *)cache_resource->action, cnt); 2713 } 2714 rte_atomic32_inc(&cache_resource->refcnt); 2715 return 0; 2716 } 2717 2718 /** 2719 * Find existing table port ID resource or create and register a new one. 2720 * 2721 * @param[in, out] dev 2722 * Pointer to rte_eth_dev structure. 2723 * @param[in, out] resource 2724 * Pointer to port ID action resource. 2725 * @parm[in, out] dev_flow 2726 * Pointer to the dev_flow. 2727 * @param[out] error 2728 * pointer to error structure. 2729 * 2730 * @return 2731 * 0 on success otherwise -errno and errno is set. 2732 */ 2733 static int 2734 flow_dv_port_id_action_resource_register 2735 (struct rte_eth_dev *dev, 2736 struct mlx5_flow_dv_port_id_action_resource *resource, 2737 struct mlx5_flow *dev_flow, 2738 struct rte_flow_error *error) 2739 { 2740 struct mlx5_priv *priv = dev->data->dev_private; 2741 struct mlx5_dev_ctx_shared *sh = priv->sh; 2742 struct mlx5_flow_dv_port_id_action_resource *cache_resource; 2743 uint32_t idx = 0; 2744 int ret; 2745 2746 /* Lookup a matching resource from cache. */ 2747 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list, 2748 idx, cache_resource, next) { 2749 if (resource->port_id == cache_resource->port_id) { 2750 DRV_LOG(DEBUG, "port id action resource resource %p: " 2751 "refcnt %d++", 2752 (void *)cache_resource, 2753 rte_atomic32_read(&cache_resource->refcnt)); 2754 rte_atomic32_inc(&cache_resource->refcnt); 2755 dev_flow->handle->rix_port_id_action = idx; 2756 dev_flow->dv.port_id_action = cache_resource; 2757 return 0; 2758 } 2759 } 2760 /* Register new port id action resource. */ 2761 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], 2762 &dev_flow->handle->rix_port_id_action); 2763 if (!cache_resource) 2764 return rte_flow_error_set(error, ENOMEM, 2765 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2766 "cannot allocate resource memory"); 2767 *cache_resource = *resource; 2768 ret = mlx5_flow_os_create_flow_action_dest_port 2769 (priv->sh->fdb_domain, resource->port_id, 2770 &cache_resource->action); 2771 if (ret) { 2772 mlx5_free(cache_resource); 2773 return rte_flow_error_set(error, ENOMEM, 2774 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2775 NULL, "cannot create action"); 2776 } 2777 rte_atomic32_init(&cache_resource->refcnt); 2778 rte_atomic32_inc(&cache_resource->refcnt); 2779 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list, 2780 dev_flow->handle->rix_port_id_action, cache_resource, 2781 next); 2782 dev_flow->dv.port_id_action = cache_resource; 2783 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", 2784 (void *)cache_resource, 2785 rte_atomic32_read(&cache_resource->refcnt)); 2786 return 0; 2787 } 2788 2789 /** 2790 * Find existing push vlan resource or create and register a new one. 2791 * 2792 * @param [in, out] dev 2793 * Pointer to rte_eth_dev structure. 2794 * @param[in, out] resource 2795 * Pointer to port ID action resource. 2796 * @parm[in, out] dev_flow 2797 * Pointer to the dev_flow. 2798 * @param[out] error 2799 * pointer to error structure. 2800 * 2801 * @return 2802 * 0 on success otherwise -errno and errno is set. 2803 */ 2804 static int 2805 flow_dv_push_vlan_action_resource_register 2806 (struct rte_eth_dev *dev, 2807 struct mlx5_flow_dv_push_vlan_action_resource *resource, 2808 struct mlx5_flow *dev_flow, 2809 struct rte_flow_error *error) 2810 { 2811 struct mlx5_priv *priv = dev->data->dev_private; 2812 struct mlx5_dev_ctx_shared *sh = priv->sh; 2813 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; 2814 struct mlx5dv_dr_domain *domain; 2815 uint32_t idx = 0; 2816 int ret; 2817 2818 /* Lookup a matching resource from cache. */ 2819 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2820 sh->push_vlan_action_list, idx, cache_resource, next) { 2821 if (resource->vlan_tag == cache_resource->vlan_tag && 2822 resource->ft_type == cache_resource->ft_type) { 2823 DRV_LOG(DEBUG, "push-VLAN action resource resource %p: " 2824 "refcnt %d++", 2825 (void *)cache_resource, 2826 rte_atomic32_read(&cache_resource->refcnt)); 2827 rte_atomic32_inc(&cache_resource->refcnt); 2828 dev_flow->handle->dvh.rix_push_vlan = idx; 2829 dev_flow->dv.push_vlan_res = cache_resource; 2830 return 0; 2831 } 2832 } 2833 /* Register new push_vlan action resource. */ 2834 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2835 &dev_flow->handle->dvh.rix_push_vlan); 2836 if (!cache_resource) 2837 return rte_flow_error_set(error, ENOMEM, 2838 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2839 "cannot allocate resource memory"); 2840 *cache_resource = *resource; 2841 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 2842 domain = sh->fdb_domain; 2843 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 2844 domain = sh->rx_domain; 2845 else 2846 domain = sh->tx_domain; 2847 ret = mlx5_flow_os_create_flow_action_push_vlan 2848 (domain, resource->vlan_tag, 2849 &cache_resource->action); 2850 if (ret) { 2851 mlx5_free(cache_resource); 2852 return rte_flow_error_set(error, ENOMEM, 2853 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2854 NULL, "cannot create action"); 2855 } 2856 rte_atomic32_init(&cache_resource->refcnt); 2857 rte_atomic32_inc(&cache_resource->refcnt); 2858 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2859 &sh->push_vlan_action_list, 2860 dev_flow->handle->dvh.rix_push_vlan, 2861 cache_resource, next); 2862 dev_flow->dv.push_vlan_res = cache_resource; 2863 DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++", 2864 (void *)cache_resource, 2865 rte_atomic32_read(&cache_resource->refcnt)); 2866 return 0; 2867 } 2868 /** 2869 * Get the size of specific rte_flow_item_type hdr size 2870 * 2871 * @param[in] item_type 2872 * Tested rte_flow_item_type. 2873 * 2874 * @return 2875 * sizeof struct item_type, 0 if void or irrelevant. 2876 */ 2877 static size_t 2878 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type) 2879 { 2880 size_t retval; 2881 2882 switch (item_type) { 2883 case RTE_FLOW_ITEM_TYPE_ETH: 2884 retval = sizeof(struct rte_ether_hdr); 2885 break; 2886 case RTE_FLOW_ITEM_TYPE_VLAN: 2887 retval = sizeof(struct rte_vlan_hdr); 2888 break; 2889 case RTE_FLOW_ITEM_TYPE_IPV4: 2890 retval = sizeof(struct rte_ipv4_hdr); 2891 break; 2892 case RTE_FLOW_ITEM_TYPE_IPV6: 2893 retval = sizeof(struct rte_ipv6_hdr); 2894 break; 2895 case RTE_FLOW_ITEM_TYPE_UDP: 2896 retval = sizeof(struct rte_udp_hdr); 2897 break; 2898 case RTE_FLOW_ITEM_TYPE_TCP: 2899 retval = sizeof(struct rte_tcp_hdr); 2900 break; 2901 case RTE_FLOW_ITEM_TYPE_VXLAN: 2902 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 2903 retval = sizeof(struct rte_vxlan_hdr); 2904 break; 2905 case RTE_FLOW_ITEM_TYPE_GRE: 2906 case RTE_FLOW_ITEM_TYPE_NVGRE: 2907 retval = sizeof(struct rte_gre_hdr); 2908 break; 2909 case RTE_FLOW_ITEM_TYPE_MPLS: 2910 retval = sizeof(struct rte_mpls_hdr); 2911 break; 2912 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */ 2913 default: 2914 retval = 0; 2915 break; 2916 } 2917 return retval; 2918 } 2919 2920 #define MLX5_ENCAP_IPV4_VERSION 0x40 2921 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05 2922 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40 2923 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000 2924 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff 2925 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000 2926 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04 2927 2928 /** 2929 * Convert the encap action data from list of rte_flow_item to raw buffer 2930 * 2931 * @param[in] items 2932 * Pointer to rte_flow_item objects list. 2933 * @param[out] buf 2934 * Pointer to the output buffer. 2935 * @param[out] size 2936 * Pointer to the output buffer size. 2937 * @param[out] error 2938 * Pointer to the error structure. 2939 * 2940 * @return 2941 * 0 on success, a negative errno value otherwise and rte_errno is set. 2942 */ 2943 static int 2944 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, 2945 size_t *size, struct rte_flow_error *error) 2946 { 2947 struct rte_ether_hdr *eth = NULL; 2948 struct rte_vlan_hdr *vlan = NULL; 2949 struct rte_ipv4_hdr *ipv4 = NULL; 2950 struct rte_ipv6_hdr *ipv6 = NULL; 2951 struct rte_udp_hdr *udp = NULL; 2952 struct rte_vxlan_hdr *vxlan = NULL; 2953 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL; 2954 struct rte_gre_hdr *gre = NULL; 2955 size_t len; 2956 size_t temp_size = 0; 2957 2958 if (!items) 2959 return rte_flow_error_set(error, EINVAL, 2960 RTE_FLOW_ERROR_TYPE_ACTION, 2961 NULL, "invalid empty data"); 2962 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 2963 len = flow_dv_get_item_hdr_len(items->type); 2964 if (len + temp_size > MLX5_ENCAP_MAX_LEN) 2965 return rte_flow_error_set(error, EINVAL, 2966 RTE_FLOW_ERROR_TYPE_ACTION, 2967 (void *)items->type, 2968 "items total size is too big" 2969 " for encap action"); 2970 rte_memcpy((void *)&buf[temp_size], items->spec, len); 2971 switch (items->type) { 2972 case RTE_FLOW_ITEM_TYPE_ETH: 2973 eth = (struct rte_ether_hdr *)&buf[temp_size]; 2974 break; 2975 case RTE_FLOW_ITEM_TYPE_VLAN: 2976 vlan = (struct rte_vlan_hdr *)&buf[temp_size]; 2977 if (!eth) 2978 return rte_flow_error_set(error, EINVAL, 2979 RTE_FLOW_ERROR_TYPE_ACTION, 2980 (void *)items->type, 2981 "eth header not found"); 2982 if (!eth->ether_type) 2983 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN); 2984 break; 2985 case RTE_FLOW_ITEM_TYPE_IPV4: 2986 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size]; 2987 if (!vlan && !eth) 2988 return rte_flow_error_set(error, EINVAL, 2989 RTE_FLOW_ERROR_TYPE_ACTION, 2990 (void *)items->type, 2991 "neither eth nor vlan" 2992 " header found"); 2993 if (vlan && !vlan->eth_proto) 2994 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4); 2995 else if (eth && !eth->ether_type) 2996 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4); 2997 if (!ipv4->version_ihl) 2998 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION | 2999 MLX5_ENCAP_IPV4_IHL_MIN; 3000 if (!ipv4->time_to_live) 3001 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF; 3002 break; 3003 case RTE_FLOW_ITEM_TYPE_IPV6: 3004 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size]; 3005 if (!vlan && !eth) 3006 return rte_flow_error_set(error, EINVAL, 3007 RTE_FLOW_ERROR_TYPE_ACTION, 3008 (void *)items->type, 3009 "neither eth nor vlan" 3010 " header found"); 3011 if (vlan && !vlan->eth_proto) 3012 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6); 3013 else if (eth && !eth->ether_type) 3014 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6); 3015 if (!ipv6->vtc_flow) 3016 ipv6->vtc_flow = 3017 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW); 3018 if (!ipv6->hop_limits) 3019 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT; 3020 break; 3021 case RTE_FLOW_ITEM_TYPE_UDP: 3022 udp = (struct rte_udp_hdr *)&buf[temp_size]; 3023 if (!ipv4 && !ipv6) 3024 return rte_flow_error_set(error, EINVAL, 3025 RTE_FLOW_ERROR_TYPE_ACTION, 3026 (void *)items->type, 3027 "ip header not found"); 3028 if (ipv4 && !ipv4->next_proto_id) 3029 ipv4->next_proto_id = IPPROTO_UDP; 3030 else if (ipv6 && !ipv6->proto) 3031 ipv6->proto = IPPROTO_UDP; 3032 break; 3033 case RTE_FLOW_ITEM_TYPE_VXLAN: 3034 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size]; 3035 if (!udp) 3036 return rte_flow_error_set(error, EINVAL, 3037 RTE_FLOW_ERROR_TYPE_ACTION, 3038 (void *)items->type, 3039 "udp header not found"); 3040 if (!udp->dst_port) 3041 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN); 3042 if (!vxlan->vx_flags) 3043 vxlan->vx_flags = 3044 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS); 3045 break; 3046 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 3047 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size]; 3048 if (!udp) 3049 return rte_flow_error_set(error, EINVAL, 3050 RTE_FLOW_ERROR_TYPE_ACTION, 3051 (void *)items->type, 3052 "udp header not found"); 3053 if (!vxlan_gpe->proto) 3054 return rte_flow_error_set(error, EINVAL, 3055 RTE_FLOW_ERROR_TYPE_ACTION, 3056 (void *)items->type, 3057 "next protocol not found"); 3058 if (!udp->dst_port) 3059 udp->dst_port = 3060 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE); 3061 if (!vxlan_gpe->vx_flags) 3062 vxlan_gpe->vx_flags = 3063 MLX5_ENCAP_VXLAN_GPE_FLAGS; 3064 break; 3065 case RTE_FLOW_ITEM_TYPE_GRE: 3066 case RTE_FLOW_ITEM_TYPE_NVGRE: 3067 gre = (struct rte_gre_hdr *)&buf[temp_size]; 3068 if (!gre->proto) 3069 return rte_flow_error_set(error, EINVAL, 3070 RTE_FLOW_ERROR_TYPE_ACTION, 3071 (void *)items->type, 3072 "next protocol not found"); 3073 if (!ipv4 && !ipv6) 3074 return rte_flow_error_set(error, EINVAL, 3075 RTE_FLOW_ERROR_TYPE_ACTION, 3076 (void *)items->type, 3077 "ip header not found"); 3078 if (ipv4 && !ipv4->next_proto_id) 3079 ipv4->next_proto_id = IPPROTO_GRE; 3080 else if (ipv6 && !ipv6->proto) 3081 ipv6->proto = IPPROTO_GRE; 3082 break; 3083 case RTE_FLOW_ITEM_TYPE_VOID: 3084 break; 3085 default: 3086 return rte_flow_error_set(error, EINVAL, 3087 RTE_FLOW_ERROR_TYPE_ACTION, 3088 (void *)items->type, 3089 "unsupported item type"); 3090 break; 3091 } 3092 temp_size += len; 3093 } 3094 *size = temp_size; 3095 return 0; 3096 } 3097 3098 static int 3099 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error) 3100 { 3101 struct rte_ether_hdr *eth = NULL; 3102 struct rte_vlan_hdr *vlan = NULL; 3103 struct rte_ipv6_hdr *ipv6 = NULL; 3104 struct rte_udp_hdr *udp = NULL; 3105 char *next_hdr; 3106 uint16_t proto; 3107 3108 eth = (struct rte_ether_hdr *)data; 3109 next_hdr = (char *)(eth + 1); 3110 proto = RTE_BE16(eth->ether_type); 3111 3112 /* VLAN skipping */ 3113 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) { 3114 vlan = (struct rte_vlan_hdr *)next_hdr; 3115 proto = RTE_BE16(vlan->eth_proto); 3116 next_hdr += sizeof(struct rte_vlan_hdr); 3117 } 3118 3119 /* HW calculates IPv4 csum. no need to proceed */ 3120 if (proto == RTE_ETHER_TYPE_IPV4) 3121 return 0; 3122 3123 /* non IPv4/IPv6 header. not supported */ 3124 if (proto != RTE_ETHER_TYPE_IPV6) { 3125 return rte_flow_error_set(error, ENOTSUP, 3126 RTE_FLOW_ERROR_TYPE_ACTION, 3127 NULL, "Cannot offload non IPv4/IPv6"); 3128 } 3129 3130 ipv6 = (struct rte_ipv6_hdr *)next_hdr; 3131 3132 /* ignore non UDP */ 3133 if (ipv6->proto != IPPROTO_UDP) 3134 return 0; 3135 3136 udp = (struct rte_udp_hdr *)(ipv6 + 1); 3137 udp->dgram_cksum = 0; 3138 3139 return 0; 3140 } 3141 3142 /** 3143 * Convert L2 encap action to DV specification. 3144 * 3145 * @param[in] dev 3146 * Pointer to rte_eth_dev structure. 3147 * @param[in] action 3148 * Pointer to action structure. 3149 * @param[in, out] dev_flow 3150 * Pointer to the mlx5_flow. 3151 * @param[in] transfer 3152 * Mark if the flow is E-Switch flow. 3153 * @param[out] error 3154 * Pointer to the error structure. 3155 * 3156 * @return 3157 * 0 on success, a negative errno value otherwise and rte_errno is set. 3158 */ 3159 static int 3160 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, 3161 const struct rte_flow_action *action, 3162 struct mlx5_flow *dev_flow, 3163 uint8_t transfer, 3164 struct rte_flow_error *error) 3165 { 3166 const struct rte_flow_item *encap_data; 3167 const struct rte_flow_action_raw_encap *raw_encap_data; 3168 struct mlx5_flow_dv_encap_decap_resource res = { 3169 .reformat_type = 3170 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL, 3171 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 3172 MLX5DV_FLOW_TABLE_TYPE_NIC_TX, 3173 }; 3174 3175 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 3176 raw_encap_data = 3177 (const struct rte_flow_action_raw_encap *)action->conf; 3178 res.size = raw_encap_data->size; 3179 memcpy(res.buf, raw_encap_data->data, res.size); 3180 } else { 3181 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) 3182 encap_data = 3183 ((const struct rte_flow_action_vxlan_encap *) 3184 action->conf)->definition; 3185 else 3186 encap_data = 3187 ((const struct rte_flow_action_nvgre_encap *) 3188 action->conf)->definition; 3189 if (flow_dv_convert_encap_data(encap_data, res.buf, 3190 &res.size, error)) 3191 return -rte_errno; 3192 } 3193 if (flow_dv_zero_encap_udp_csum(res.buf, error)) 3194 return -rte_errno; 3195 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3196 return rte_flow_error_set(error, EINVAL, 3197 RTE_FLOW_ERROR_TYPE_ACTION, 3198 NULL, "can't create L2 encap action"); 3199 return 0; 3200 } 3201 3202 /** 3203 * Convert L2 decap action to DV specification. 3204 * 3205 * @param[in] dev 3206 * Pointer to rte_eth_dev structure. 3207 * @param[in, out] dev_flow 3208 * Pointer to the mlx5_flow. 3209 * @param[in] transfer 3210 * Mark if the flow is E-Switch flow. 3211 * @param[out] error 3212 * Pointer to the error structure. 3213 * 3214 * @return 3215 * 0 on success, a negative errno value otherwise and rte_errno is set. 3216 */ 3217 static int 3218 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev, 3219 struct mlx5_flow *dev_flow, 3220 uint8_t transfer, 3221 struct rte_flow_error *error) 3222 { 3223 struct mlx5_flow_dv_encap_decap_resource res = { 3224 .size = 0, 3225 .reformat_type = 3226 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2, 3227 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 3228 MLX5DV_FLOW_TABLE_TYPE_NIC_RX, 3229 }; 3230 3231 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3232 return rte_flow_error_set(error, EINVAL, 3233 RTE_FLOW_ERROR_TYPE_ACTION, 3234 NULL, "can't create L2 decap action"); 3235 return 0; 3236 } 3237 3238 /** 3239 * Convert raw decap/encap (L3 tunnel) action to DV specification. 3240 * 3241 * @param[in] dev 3242 * Pointer to rte_eth_dev structure. 3243 * @param[in] action 3244 * Pointer to action structure. 3245 * @param[in, out] dev_flow 3246 * Pointer to the mlx5_flow. 3247 * @param[in] attr 3248 * Pointer to the flow attributes. 3249 * @param[out] error 3250 * Pointer to the error structure. 3251 * 3252 * @return 3253 * 0 on success, a negative errno value otherwise and rte_errno is set. 3254 */ 3255 static int 3256 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, 3257 const struct rte_flow_action *action, 3258 struct mlx5_flow *dev_flow, 3259 const struct rte_flow_attr *attr, 3260 struct rte_flow_error *error) 3261 { 3262 const struct rte_flow_action_raw_encap *encap_data; 3263 struct mlx5_flow_dv_encap_decap_resource res; 3264 3265 memset(&res, 0, sizeof(res)); 3266 encap_data = (const struct rte_flow_action_raw_encap *)action->conf; 3267 res.size = encap_data->size; 3268 memcpy(res.buf, encap_data->data, res.size); 3269 res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ? 3270 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 : 3271 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 3272 if (attr->transfer) 3273 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 3274 else 3275 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 3276 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 3277 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3278 return rte_flow_error_set(error, EINVAL, 3279 RTE_FLOW_ERROR_TYPE_ACTION, 3280 NULL, "can't create encap action"); 3281 return 0; 3282 } 3283 3284 /** 3285 * Create action push VLAN. 3286 * 3287 * @param[in] dev 3288 * Pointer to rte_eth_dev structure. 3289 * @param[in] attr 3290 * Pointer to the flow attributes. 3291 * @param[in] vlan 3292 * Pointer to the vlan to push to the Ethernet header. 3293 * @param[in, out] dev_flow 3294 * Pointer to the mlx5_flow. 3295 * @param[out] error 3296 * Pointer to the error structure. 3297 * 3298 * @return 3299 * 0 on success, a negative errno value otherwise and rte_errno is set. 3300 */ 3301 static int 3302 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev, 3303 const struct rte_flow_attr *attr, 3304 const struct rte_vlan_hdr *vlan, 3305 struct mlx5_flow *dev_flow, 3306 struct rte_flow_error *error) 3307 { 3308 struct mlx5_flow_dv_push_vlan_action_resource res; 3309 3310 memset(&res, 0, sizeof(res)); 3311 res.vlan_tag = 3312 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 | 3313 vlan->vlan_tci); 3314 if (attr->transfer) 3315 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 3316 else 3317 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 3318 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 3319 return flow_dv_push_vlan_action_resource_register 3320 (dev, &res, dev_flow, error); 3321 } 3322 3323 /** 3324 * Validate the modify-header actions. 3325 * 3326 * @param[in] action_flags 3327 * Holds the actions detected until now. 3328 * @param[in] action 3329 * Pointer to the modify action. 3330 * @param[out] error 3331 * Pointer to error structure. 3332 * 3333 * @return 3334 * 0 on success, a negative errno value otherwise and rte_errno is set. 3335 */ 3336 static int 3337 flow_dv_validate_action_modify_hdr(const uint64_t action_flags, 3338 const struct rte_flow_action *action, 3339 struct rte_flow_error *error) 3340 { 3341 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf) 3342 return rte_flow_error_set(error, EINVAL, 3343 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3344 NULL, "action configuration not set"); 3345 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 3346 return rte_flow_error_set(error, EINVAL, 3347 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3348 "can't have encap action before" 3349 " modify action"); 3350 return 0; 3351 } 3352 3353 /** 3354 * Validate the modify-header MAC address actions. 3355 * 3356 * @param[in] action_flags 3357 * Holds the actions detected until now. 3358 * @param[in] action 3359 * Pointer to the modify action. 3360 * @param[in] item_flags 3361 * Holds the items detected. 3362 * @param[out] error 3363 * Pointer to error structure. 3364 * 3365 * @return 3366 * 0 on success, a negative errno value otherwise and rte_errno is set. 3367 */ 3368 static int 3369 flow_dv_validate_action_modify_mac(const uint64_t action_flags, 3370 const struct rte_flow_action *action, 3371 const uint64_t item_flags, 3372 struct rte_flow_error *error) 3373 { 3374 int ret = 0; 3375 3376 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3377 if (!ret) { 3378 if (!(item_flags & MLX5_FLOW_LAYER_L2)) 3379 return rte_flow_error_set(error, EINVAL, 3380 RTE_FLOW_ERROR_TYPE_ACTION, 3381 NULL, 3382 "no L2 item in pattern"); 3383 } 3384 return ret; 3385 } 3386 3387 /** 3388 * Validate the modify-header IPv4 address actions. 3389 * 3390 * @param[in] action_flags 3391 * Holds the actions detected until now. 3392 * @param[in] action 3393 * Pointer to the modify action. 3394 * @param[in] item_flags 3395 * Holds the items detected. 3396 * @param[out] error 3397 * Pointer to error structure. 3398 * 3399 * @return 3400 * 0 on success, a negative errno value otherwise and rte_errno is set. 3401 */ 3402 static int 3403 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags, 3404 const struct rte_flow_action *action, 3405 const uint64_t item_flags, 3406 struct rte_flow_error *error) 3407 { 3408 int ret = 0; 3409 uint64_t layer; 3410 3411 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3412 if (!ret) { 3413 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3414 MLX5_FLOW_LAYER_INNER_L3_IPV4 : 3415 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3416 if (!(item_flags & layer)) 3417 return rte_flow_error_set(error, EINVAL, 3418 RTE_FLOW_ERROR_TYPE_ACTION, 3419 NULL, 3420 "no ipv4 item in pattern"); 3421 } 3422 return ret; 3423 } 3424 3425 /** 3426 * Validate the modify-header IPv6 address actions. 3427 * 3428 * @param[in] action_flags 3429 * Holds the actions detected until now. 3430 * @param[in] action 3431 * Pointer to the modify action. 3432 * @param[in] item_flags 3433 * Holds the items detected. 3434 * @param[out] error 3435 * Pointer to error structure. 3436 * 3437 * @return 3438 * 0 on success, a negative errno value otherwise and rte_errno is set. 3439 */ 3440 static int 3441 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags, 3442 const struct rte_flow_action *action, 3443 const uint64_t item_flags, 3444 struct rte_flow_error *error) 3445 { 3446 int ret = 0; 3447 uint64_t layer; 3448 3449 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3450 if (!ret) { 3451 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3452 MLX5_FLOW_LAYER_INNER_L3_IPV6 : 3453 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3454 if (!(item_flags & layer)) 3455 return rte_flow_error_set(error, EINVAL, 3456 RTE_FLOW_ERROR_TYPE_ACTION, 3457 NULL, 3458 "no ipv6 item in pattern"); 3459 } 3460 return ret; 3461 } 3462 3463 /** 3464 * Validate the modify-header TP actions. 3465 * 3466 * @param[in] action_flags 3467 * Holds the actions detected until now. 3468 * @param[in] action 3469 * Pointer to the modify action. 3470 * @param[in] item_flags 3471 * Holds the items detected. 3472 * @param[out] error 3473 * Pointer to error structure. 3474 * 3475 * @return 3476 * 0 on success, a negative errno value otherwise and rte_errno is set. 3477 */ 3478 static int 3479 flow_dv_validate_action_modify_tp(const uint64_t action_flags, 3480 const struct rte_flow_action *action, 3481 const uint64_t item_flags, 3482 struct rte_flow_error *error) 3483 { 3484 int ret = 0; 3485 uint64_t layer; 3486 3487 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3488 if (!ret) { 3489 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3490 MLX5_FLOW_LAYER_INNER_L4 : 3491 MLX5_FLOW_LAYER_OUTER_L4; 3492 if (!(item_flags & layer)) 3493 return rte_flow_error_set(error, EINVAL, 3494 RTE_FLOW_ERROR_TYPE_ACTION, 3495 NULL, "no transport layer " 3496 "in pattern"); 3497 } 3498 return ret; 3499 } 3500 3501 /** 3502 * Validate the modify-header actions of increment/decrement 3503 * TCP Sequence-number. 3504 * 3505 * @param[in] action_flags 3506 * Holds the actions detected until now. 3507 * @param[in] action 3508 * Pointer to the modify action. 3509 * @param[in] item_flags 3510 * Holds the items detected. 3511 * @param[out] error 3512 * Pointer to error structure. 3513 * 3514 * @return 3515 * 0 on success, a negative errno value otherwise and rte_errno is set. 3516 */ 3517 static int 3518 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags, 3519 const struct rte_flow_action *action, 3520 const uint64_t item_flags, 3521 struct rte_flow_error *error) 3522 { 3523 int ret = 0; 3524 uint64_t layer; 3525 3526 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3527 if (!ret) { 3528 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3529 MLX5_FLOW_LAYER_INNER_L4_TCP : 3530 MLX5_FLOW_LAYER_OUTER_L4_TCP; 3531 if (!(item_flags & layer)) 3532 return rte_flow_error_set(error, EINVAL, 3533 RTE_FLOW_ERROR_TYPE_ACTION, 3534 NULL, "no TCP item in" 3535 " pattern"); 3536 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ && 3537 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) || 3538 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ && 3539 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ))) 3540 return rte_flow_error_set(error, EINVAL, 3541 RTE_FLOW_ERROR_TYPE_ACTION, 3542 NULL, 3543 "cannot decrease and increase" 3544 " TCP sequence number" 3545 " at the same time"); 3546 } 3547 return ret; 3548 } 3549 3550 /** 3551 * Validate the modify-header actions of increment/decrement 3552 * TCP Acknowledgment number. 3553 * 3554 * @param[in] action_flags 3555 * Holds the actions detected until now. 3556 * @param[in] action 3557 * Pointer to the modify action. 3558 * @param[in] item_flags 3559 * Holds the items detected. 3560 * @param[out] error 3561 * Pointer to error structure. 3562 * 3563 * @return 3564 * 0 on success, a negative errno value otherwise and rte_errno is set. 3565 */ 3566 static int 3567 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags, 3568 const struct rte_flow_action *action, 3569 const uint64_t item_flags, 3570 struct rte_flow_error *error) 3571 { 3572 int ret = 0; 3573 uint64_t layer; 3574 3575 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3576 if (!ret) { 3577 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3578 MLX5_FLOW_LAYER_INNER_L4_TCP : 3579 MLX5_FLOW_LAYER_OUTER_L4_TCP; 3580 if (!(item_flags & layer)) 3581 return rte_flow_error_set(error, EINVAL, 3582 RTE_FLOW_ERROR_TYPE_ACTION, 3583 NULL, "no TCP item in" 3584 " pattern"); 3585 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK && 3586 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) || 3587 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK && 3588 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK))) 3589 return rte_flow_error_set(error, EINVAL, 3590 RTE_FLOW_ERROR_TYPE_ACTION, 3591 NULL, 3592 "cannot decrease and increase" 3593 " TCP acknowledgment number" 3594 " at the same time"); 3595 } 3596 return ret; 3597 } 3598 3599 /** 3600 * Validate the modify-header TTL actions. 3601 * 3602 * @param[in] action_flags 3603 * Holds the actions detected until now. 3604 * @param[in] action 3605 * Pointer to the modify action. 3606 * @param[in] item_flags 3607 * Holds the items detected. 3608 * @param[out] error 3609 * Pointer to error structure. 3610 * 3611 * @return 3612 * 0 on success, a negative errno value otherwise and rte_errno is set. 3613 */ 3614 static int 3615 flow_dv_validate_action_modify_ttl(const uint64_t action_flags, 3616 const struct rte_flow_action *action, 3617 const uint64_t item_flags, 3618 struct rte_flow_error *error) 3619 { 3620 int ret = 0; 3621 uint64_t layer; 3622 3623 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3624 if (!ret) { 3625 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3626 MLX5_FLOW_LAYER_INNER_L3 : 3627 MLX5_FLOW_LAYER_OUTER_L3; 3628 if (!(item_flags & layer)) 3629 return rte_flow_error_set(error, EINVAL, 3630 RTE_FLOW_ERROR_TYPE_ACTION, 3631 NULL, 3632 "no IP protocol in pattern"); 3633 } 3634 return ret; 3635 } 3636 3637 /** 3638 * Validate jump action. 3639 * 3640 * @param[in] action 3641 * Pointer to the jump action. 3642 * @param[in] action_flags 3643 * Holds the actions detected until now. 3644 * @param[in] attributes 3645 * Pointer to flow attributes 3646 * @param[in] external 3647 * Action belongs to flow rule created by request external to PMD. 3648 * @param[out] error 3649 * Pointer to error structure. 3650 * 3651 * @return 3652 * 0 on success, a negative errno value otherwise and rte_errno is set. 3653 */ 3654 static int 3655 flow_dv_validate_action_jump(const struct rte_flow_action *action, 3656 uint64_t action_flags, 3657 const struct rte_flow_attr *attributes, 3658 bool external, struct rte_flow_error *error) 3659 { 3660 uint32_t target_group, table; 3661 int ret = 0; 3662 3663 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 3664 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 3665 return rte_flow_error_set(error, EINVAL, 3666 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3667 "can't have 2 fate actions in" 3668 " same flow"); 3669 if (action_flags & MLX5_FLOW_ACTION_METER) 3670 return rte_flow_error_set(error, ENOTSUP, 3671 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3672 "jump with meter not support"); 3673 if (!action->conf) 3674 return rte_flow_error_set(error, EINVAL, 3675 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3676 NULL, "action configuration not set"); 3677 target_group = 3678 ((const struct rte_flow_action_jump *)action->conf)->group; 3679 ret = mlx5_flow_group_to_table(attributes, external, target_group, 3680 true, &table, error); 3681 if (ret) 3682 return ret; 3683 if (attributes->group == target_group) 3684 return rte_flow_error_set(error, EINVAL, 3685 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3686 "target group must be other than" 3687 " the current flow group"); 3688 return 0; 3689 } 3690 3691 /* 3692 * Validate the port_id action. 3693 * 3694 * @param[in] dev 3695 * Pointer to rte_eth_dev structure. 3696 * @param[in] action_flags 3697 * Bit-fields that holds the actions detected until now. 3698 * @param[in] action 3699 * Port_id RTE action structure. 3700 * @param[in] attr 3701 * Attributes of flow that includes this action. 3702 * @param[out] error 3703 * Pointer to error structure. 3704 * 3705 * @return 3706 * 0 on success, a negative errno value otherwise and rte_errno is set. 3707 */ 3708 static int 3709 flow_dv_validate_action_port_id(struct rte_eth_dev *dev, 3710 uint64_t action_flags, 3711 const struct rte_flow_action *action, 3712 const struct rte_flow_attr *attr, 3713 struct rte_flow_error *error) 3714 { 3715 const struct rte_flow_action_port_id *port_id; 3716 struct mlx5_priv *act_priv; 3717 struct mlx5_priv *dev_priv; 3718 uint16_t port; 3719 3720 if (!attr->transfer) 3721 return rte_flow_error_set(error, ENOTSUP, 3722 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3723 NULL, 3724 "port id action is valid in transfer" 3725 " mode only"); 3726 if (!action || !action->conf) 3727 return rte_flow_error_set(error, ENOTSUP, 3728 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3729 NULL, 3730 "port id action parameters must be" 3731 " specified"); 3732 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 3733 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 3734 return rte_flow_error_set(error, EINVAL, 3735 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3736 "can have only one fate actions in" 3737 " a flow"); 3738 dev_priv = mlx5_dev_to_eswitch_info(dev); 3739 if (!dev_priv) 3740 return rte_flow_error_set(error, rte_errno, 3741 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3742 NULL, 3743 "failed to obtain E-Switch info"); 3744 port_id = action->conf; 3745 port = port_id->original ? dev->data->port_id : port_id->id; 3746 act_priv = mlx5_port_to_eswitch_info(port, false); 3747 if (!act_priv) 3748 return rte_flow_error_set 3749 (error, rte_errno, 3750 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id, 3751 "failed to obtain E-Switch port id for port"); 3752 if (act_priv->domain_id != dev_priv->domain_id) 3753 return rte_flow_error_set 3754 (error, EINVAL, 3755 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3756 "port does not belong to" 3757 " E-Switch being configured"); 3758 return 0; 3759 } 3760 3761 /** 3762 * Get the maximum number of modify header actions. 3763 * 3764 * @param dev 3765 * Pointer to rte_eth_dev structure. 3766 * @param flags 3767 * Flags bits to check if root level. 3768 * 3769 * @return 3770 * Max number of modify header actions device can support. 3771 */ 3772 static inline unsigned int 3773 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused, 3774 uint64_t flags) 3775 { 3776 /* 3777 * There's no way to directly query the max capacity from FW. 3778 * The maximal value on root table should be assumed to be supported. 3779 */ 3780 if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL)) 3781 return MLX5_MAX_MODIFY_NUM; 3782 else 3783 return MLX5_ROOT_TBL_MODIFY_NUM; 3784 } 3785 3786 /** 3787 * Validate the meter action. 3788 * 3789 * @param[in] dev 3790 * Pointer to rte_eth_dev structure. 3791 * @param[in] action_flags 3792 * Bit-fields that holds the actions detected until now. 3793 * @param[in] action 3794 * Pointer to the meter action. 3795 * @param[in] attr 3796 * Attributes of flow that includes this action. 3797 * @param[out] error 3798 * Pointer to error structure. 3799 * 3800 * @return 3801 * 0 on success, a negative errno value otherwise and rte_ernno is set. 3802 */ 3803 static int 3804 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, 3805 uint64_t action_flags, 3806 const struct rte_flow_action *action, 3807 const struct rte_flow_attr *attr, 3808 struct rte_flow_error *error) 3809 { 3810 struct mlx5_priv *priv = dev->data->dev_private; 3811 const struct rte_flow_action_meter *am = action->conf; 3812 struct mlx5_flow_meter *fm; 3813 3814 if (!am) 3815 return rte_flow_error_set(error, EINVAL, 3816 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3817 "meter action conf is NULL"); 3818 3819 if (action_flags & MLX5_FLOW_ACTION_METER) 3820 return rte_flow_error_set(error, ENOTSUP, 3821 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3822 "meter chaining not support"); 3823 if (action_flags & MLX5_FLOW_ACTION_JUMP) 3824 return rte_flow_error_set(error, ENOTSUP, 3825 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3826 "meter with jump not support"); 3827 if (!priv->mtr_en) 3828 return rte_flow_error_set(error, ENOTSUP, 3829 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3830 NULL, 3831 "meter action not supported"); 3832 fm = mlx5_flow_meter_find(priv, am->mtr_id); 3833 if (!fm) 3834 return rte_flow_error_set(error, EINVAL, 3835 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3836 "Meter not found"); 3837 if (fm->ref_cnt && (!(fm->transfer == attr->transfer || 3838 (!fm->ingress && !attr->ingress && attr->egress) || 3839 (!fm->egress && !attr->egress && attr->ingress)))) 3840 return rte_flow_error_set(error, EINVAL, 3841 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3842 "Flow attributes are either invalid " 3843 "or have a conflict with current " 3844 "meter attributes"); 3845 return 0; 3846 } 3847 3848 /** 3849 * Validate the age action. 3850 * 3851 * @param[in] action_flags 3852 * Holds the actions detected until now. 3853 * @param[in] action 3854 * Pointer to the age action. 3855 * @param[in] dev 3856 * Pointer to the Ethernet device structure. 3857 * @param[out] error 3858 * Pointer to error structure. 3859 * 3860 * @return 3861 * 0 on success, a negative errno value otherwise and rte_errno is set. 3862 */ 3863 static int 3864 flow_dv_validate_action_age(uint64_t action_flags, 3865 const struct rte_flow_action *action, 3866 struct rte_eth_dev *dev, 3867 struct rte_flow_error *error) 3868 { 3869 struct mlx5_priv *priv = dev->data->dev_private; 3870 const struct rte_flow_action_age *age = action->conf; 3871 3872 if (!priv->config.devx || priv->counter_fallback) 3873 return rte_flow_error_set(error, ENOTSUP, 3874 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3875 NULL, 3876 "age action not supported"); 3877 if (!(action->conf)) 3878 return rte_flow_error_set(error, EINVAL, 3879 RTE_FLOW_ERROR_TYPE_ACTION, action, 3880 "configuration cannot be null"); 3881 if (age->timeout >= UINT16_MAX / 2 / 10) 3882 return rte_flow_error_set(error, ENOTSUP, 3883 RTE_FLOW_ERROR_TYPE_ACTION, action, 3884 "Max age time: 3275 seconds"); 3885 if (action_flags & MLX5_FLOW_ACTION_AGE) 3886 return rte_flow_error_set(error, EINVAL, 3887 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3888 "Duplicate age ctions set"); 3889 return 0; 3890 } 3891 3892 /** 3893 * Validate the modify-header IPv4 DSCP actions. 3894 * 3895 * @param[in] action_flags 3896 * Holds the actions detected until now. 3897 * @param[in] action 3898 * Pointer to the modify action. 3899 * @param[in] item_flags 3900 * Holds the items detected. 3901 * @param[out] error 3902 * Pointer to error structure. 3903 * 3904 * @return 3905 * 0 on success, a negative errno value otherwise and rte_errno is set. 3906 */ 3907 static int 3908 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags, 3909 const struct rte_flow_action *action, 3910 const uint64_t item_flags, 3911 struct rte_flow_error *error) 3912 { 3913 int ret = 0; 3914 3915 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3916 if (!ret) { 3917 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) 3918 return rte_flow_error_set(error, EINVAL, 3919 RTE_FLOW_ERROR_TYPE_ACTION, 3920 NULL, 3921 "no ipv4 item in pattern"); 3922 } 3923 return ret; 3924 } 3925 3926 /** 3927 * Validate the modify-header IPv6 DSCP actions. 3928 * 3929 * @param[in] action_flags 3930 * Holds the actions detected until now. 3931 * @param[in] action 3932 * Pointer to the modify action. 3933 * @param[in] item_flags 3934 * Holds the items detected. 3935 * @param[out] error 3936 * Pointer to error structure. 3937 * 3938 * @return 3939 * 0 on success, a negative errno value otherwise and rte_errno is set. 3940 */ 3941 static int 3942 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags, 3943 const struct rte_flow_action *action, 3944 const uint64_t item_flags, 3945 struct rte_flow_error *error) 3946 { 3947 int ret = 0; 3948 3949 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3950 if (!ret) { 3951 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) 3952 return rte_flow_error_set(error, EINVAL, 3953 RTE_FLOW_ERROR_TYPE_ACTION, 3954 NULL, 3955 "no ipv6 item in pattern"); 3956 } 3957 return ret; 3958 } 3959 3960 /** 3961 * Find existing modify-header resource or create and register a new one. 3962 * 3963 * @param dev[in, out] 3964 * Pointer to rte_eth_dev structure. 3965 * @param[in, out] resource 3966 * Pointer to modify-header resource. 3967 * @parm[in, out] dev_flow 3968 * Pointer to the dev_flow. 3969 * @param[out] error 3970 * pointer to error structure. 3971 * 3972 * @return 3973 * 0 on success otherwise -errno and errno is set. 3974 */ 3975 static int 3976 flow_dv_modify_hdr_resource_register 3977 (struct rte_eth_dev *dev, 3978 struct mlx5_flow_dv_modify_hdr_resource *resource, 3979 struct mlx5_flow *dev_flow, 3980 struct rte_flow_error *error) 3981 { 3982 struct mlx5_priv *priv = dev->data->dev_private; 3983 struct mlx5_dev_ctx_shared *sh = priv->sh; 3984 struct mlx5_flow_dv_modify_hdr_resource *cache_resource; 3985 struct mlx5dv_dr_domain *ns; 3986 uint32_t actions_len; 3987 int ret; 3988 3989 resource->flags = dev_flow->dv.group ? 0 : 3990 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; 3991 if (resource->actions_num > flow_dv_modify_hdr_action_max(dev, 3992 resource->flags)) 3993 return rte_flow_error_set(error, EOVERFLOW, 3994 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3995 "too many modify header items"); 3996 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 3997 ns = sh->fdb_domain; 3998 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) 3999 ns = sh->tx_domain; 4000 else 4001 ns = sh->rx_domain; 4002 /* Lookup a matching resource from cache. */ 4003 actions_len = resource->actions_num * sizeof(resource->actions[0]); 4004 LIST_FOREACH(cache_resource, &sh->modify_cmds, next) { 4005 if (resource->ft_type == cache_resource->ft_type && 4006 resource->actions_num == cache_resource->actions_num && 4007 resource->flags == cache_resource->flags && 4008 !memcmp((const void *)resource->actions, 4009 (const void *)cache_resource->actions, 4010 actions_len)) { 4011 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++", 4012 (void *)cache_resource, 4013 rte_atomic32_read(&cache_resource->refcnt)); 4014 rte_atomic32_inc(&cache_resource->refcnt); 4015 dev_flow->handle->dvh.modify_hdr = cache_resource; 4016 return 0; 4017 } 4018 } 4019 /* Register new modify-header resource. */ 4020 cache_resource = mlx5_malloc(MLX5_MEM_ZERO, 4021 sizeof(*cache_resource) + actions_len, 0, 4022 SOCKET_ID_ANY); 4023 if (!cache_resource) 4024 return rte_flow_error_set(error, ENOMEM, 4025 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4026 "cannot allocate resource memory"); 4027 *cache_resource = *resource; 4028 rte_memcpy(cache_resource->actions, resource->actions, actions_len); 4029 ret = mlx5_flow_os_create_flow_action_modify_header 4030 (sh->ctx, ns, cache_resource, 4031 actions_len, &cache_resource->action); 4032 if (ret) { 4033 mlx5_free(cache_resource); 4034 return rte_flow_error_set(error, ENOMEM, 4035 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4036 NULL, "cannot create action"); 4037 } 4038 rte_atomic32_init(&cache_resource->refcnt); 4039 rte_atomic32_inc(&cache_resource->refcnt); 4040 LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next); 4041 dev_flow->handle->dvh.modify_hdr = cache_resource; 4042 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", 4043 (void *)cache_resource, 4044 rte_atomic32_read(&cache_resource->refcnt)); 4045 return 0; 4046 } 4047 4048 /** 4049 * Get DV flow counter by index. 4050 * 4051 * @param[in] dev 4052 * Pointer to the Ethernet device structure. 4053 * @param[in] idx 4054 * mlx5 flow counter index in the container. 4055 * @param[out] ppool 4056 * mlx5 flow counter pool in the container, 4057 * 4058 * @return 4059 * Pointer to the counter, NULL otherwise. 4060 */ 4061 static struct mlx5_flow_counter * 4062 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev, 4063 uint32_t idx, 4064 struct mlx5_flow_counter_pool **ppool) 4065 { 4066 struct mlx5_priv *priv = dev->data->dev_private; 4067 struct mlx5_pools_container *cont; 4068 struct mlx5_flow_counter_pool *pool; 4069 uint32_t batch = 0, age = 0; 4070 4071 idx--; 4072 age = MLX_CNT_IS_AGE(idx); 4073 idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx; 4074 if (idx >= MLX5_CNT_BATCH_OFFSET) { 4075 idx -= MLX5_CNT_BATCH_OFFSET; 4076 batch = 1; 4077 } 4078 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age); 4079 MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n); 4080 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL]; 4081 MLX5_ASSERT(pool); 4082 if (ppool) 4083 *ppool = pool; 4084 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL); 4085 } 4086 4087 /** 4088 * Check the devx counter belongs to the pool. 4089 * 4090 * @param[in] pool 4091 * Pointer to the counter pool. 4092 * @param[in] id 4093 * The counter devx ID. 4094 * 4095 * @return 4096 * True if counter belongs to the pool, false otherwise. 4097 */ 4098 static bool 4099 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id) 4100 { 4101 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) * 4102 MLX5_COUNTERS_PER_POOL; 4103 4104 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) 4105 return true; 4106 return false; 4107 } 4108 4109 /** 4110 * Get a pool by devx counter ID. 4111 * 4112 * @param[in] cont 4113 * Pointer to the counter container. 4114 * @param[in] id 4115 * The counter devx ID. 4116 * 4117 * @return 4118 * The counter pool pointer if exists, NULL otherwise, 4119 */ 4120 static struct mlx5_flow_counter_pool * 4121 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id) 4122 { 4123 uint32_t i; 4124 4125 /* Check last used pool. */ 4126 if (cont->last_pool_idx != POOL_IDX_INVALID && 4127 flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id)) 4128 return cont->pools[cont->last_pool_idx]; 4129 /* ID out of range means no suitable pool in the container. */ 4130 if (id > cont->max_id || id < cont->min_id) 4131 return NULL; 4132 /* 4133 * Find the pool from the end of the container, since mostly counter 4134 * ID is sequence increasing, and the last pool should be the needed 4135 * one. 4136 */ 4137 i = rte_atomic16_read(&cont->n_valid); 4138 while (i--) { 4139 struct mlx5_flow_counter_pool *pool = cont->pools[i]; 4140 4141 if (flow_dv_is_counter_in_pool(pool, id)) 4142 return pool; 4143 } 4144 return NULL; 4145 } 4146 4147 /** 4148 * Allocate a new memory for the counter values wrapped by all the needed 4149 * management. 4150 * 4151 * @param[in] dev 4152 * Pointer to the Ethernet device structure. 4153 * @param[in] raws_n 4154 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters. 4155 * 4156 * @return 4157 * The new memory management pointer on success, otherwise NULL and rte_errno 4158 * is set. 4159 */ 4160 static struct mlx5_counter_stats_mem_mng * 4161 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n) 4162 { 4163 struct mlx5_priv *priv = dev->data->dev_private; 4164 struct mlx5_dev_ctx_shared *sh = priv->sh; 4165 struct mlx5_devx_mkey_attr mkey_attr; 4166 struct mlx5_counter_stats_mem_mng *mem_mng; 4167 volatile struct flow_counter_stats *raw_data; 4168 int size = (sizeof(struct flow_counter_stats) * 4169 MLX5_COUNTERS_PER_POOL + 4170 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 4171 sizeof(struct mlx5_counter_stats_mem_mng); 4172 size_t pgsize = rte_mem_page_size(); 4173 if (pgsize == (size_t)-1) { 4174 DRV_LOG(ERR, "Failed to get mem page size"); 4175 rte_errno = ENOMEM; 4176 return NULL; 4177 } 4178 uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, 4179 SOCKET_ID_ANY); 4180 int i; 4181 4182 if (!mem) { 4183 rte_errno = ENOMEM; 4184 return NULL; 4185 } 4186 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 4187 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 4188 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 4189 IBV_ACCESS_LOCAL_WRITE); 4190 if (!mem_mng->umem) { 4191 rte_errno = errno; 4192 mlx5_free(mem); 4193 return NULL; 4194 } 4195 mkey_attr.addr = (uintptr_t)mem; 4196 mkey_attr.size = size; 4197 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 4198 mkey_attr.pd = sh->pdn; 4199 mkey_attr.log_entity_size = 0; 4200 mkey_attr.pg_access = 0; 4201 mkey_attr.klm_array = NULL; 4202 mkey_attr.klm_num = 0; 4203 if (priv->config.hca_attr.relaxed_ordering_write && 4204 priv->config.hca_attr.relaxed_ordering_read && 4205 !haswell_broadwell_cpu) 4206 mkey_attr.relaxed_ordering = 1; 4207 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 4208 if (!mem_mng->dm) { 4209 mlx5_glue->devx_umem_dereg(mem_mng->umem); 4210 rte_errno = errno; 4211 mlx5_free(mem); 4212 return NULL; 4213 } 4214 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 4215 raw_data = (volatile struct flow_counter_stats *)mem; 4216 for (i = 0; i < raws_n; ++i) { 4217 mem_mng->raws[i].mem_mng = mem_mng; 4218 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 4219 } 4220 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 4221 return mem_mng; 4222 } 4223 4224 /** 4225 * Resize a counter container. 4226 * 4227 * @param[in] dev 4228 * Pointer to the Ethernet device structure. 4229 * @param[in] batch 4230 * Whether the pool is for counter that was allocated by batch command. 4231 * @param[in] age 4232 * Whether the pool is for Aging counter. 4233 * 4234 * @return 4235 * 0 on success, otherwise negative errno value and rte_errno is set. 4236 */ 4237 static int 4238 flow_dv_container_resize(struct rte_eth_dev *dev, 4239 uint32_t batch, uint32_t age) 4240 { 4241 struct mlx5_priv *priv = dev->data->dev_private; 4242 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4243 age); 4244 struct mlx5_counter_stats_mem_mng *mem_mng = NULL; 4245 void *old_pools = cont->pools; 4246 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE; 4247 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize; 4248 void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); 4249 4250 if (!pools) { 4251 rte_errno = ENOMEM; 4252 return -ENOMEM; 4253 } 4254 if (old_pools) 4255 memcpy(pools, old_pools, cont->n * 4256 sizeof(struct mlx5_flow_counter_pool *)); 4257 /* 4258 * Fallback mode query the counter directly, no background query 4259 * resources are needed. 4260 */ 4261 if (!priv->counter_fallback) { 4262 int i; 4263 4264 mem_mng = flow_dv_create_counter_stat_mem_mng(dev, 4265 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES); 4266 if (!mem_mng) { 4267 mlx5_free(pools); 4268 return -ENOMEM; 4269 } 4270 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 4271 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws, 4272 mem_mng->raws + 4273 MLX5_CNT_CONTAINER_RESIZE + 4274 i, next); 4275 } 4276 rte_spinlock_lock(&cont->resize_sl); 4277 cont->n = resize; 4278 cont->mem_mng = mem_mng; 4279 cont->pools = pools; 4280 rte_spinlock_unlock(&cont->resize_sl); 4281 if (old_pools) 4282 mlx5_free(old_pools); 4283 return 0; 4284 } 4285 4286 /** 4287 * Query a devx flow counter. 4288 * 4289 * @param[in] dev 4290 * Pointer to the Ethernet device structure. 4291 * @param[in] cnt 4292 * Index to the flow counter. 4293 * @param[out] pkts 4294 * The statistics value of packets. 4295 * @param[out] bytes 4296 * The statistics value of bytes. 4297 * 4298 * @return 4299 * 0 on success, otherwise a negative errno value and rte_errno is set. 4300 */ 4301 static inline int 4302 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts, 4303 uint64_t *bytes) 4304 { 4305 struct mlx5_priv *priv = dev->data->dev_private; 4306 struct mlx5_flow_counter_pool *pool = NULL; 4307 struct mlx5_flow_counter *cnt; 4308 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4309 int offset; 4310 4311 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); 4312 MLX5_ASSERT(pool); 4313 if (counter < MLX5_CNT_BATCH_OFFSET) { 4314 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); 4315 if (priv->counter_fallback) 4316 return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0, 4317 0, pkts, bytes, 0, NULL, NULL, 0); 4318 } 4319 4320 rte_spinlock_lock(&pool->sl); 4321 /* 4322 * The single counters allocation may allocate smaller ID than the 4323 * current allocated in parallel to the host reading. 4324 * In this case the new counter values must be reported as 0. 4325 */ 4326 if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) { 4327 *pkts = 0; 4328 *bytes = 0; 4329 } else { 4330 offset = MLX5_CNT_ARRAY_IDX(pool, cnt); 4331 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits); 4332 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes); 4333 } 4334 rte_spinlock_unlock(&pool->sl); 4335 return 0; 4336 } 4337 4338 /** 4339 * Create and initialize a new counter pool. 4340 * 4341 * @param[in] dev 4342 * Pointer to the Ethernet device structure. 4343 * @param[out] dcs 4344 * The devX counter handle. 4345 * @param[in] batch 4346 * Whether the pool is for counter that was allocated by batch command. 4347 * @param[in] age 4348 * Whether the pool is for counter that was allocated for aging. 4349 * @param[in/out] cont_cur 4350 * Pointer to the container pointer, it will be update in pool resize. 4351 * 4352 * @return 4353 * The pool container pointer on success, NULL otherwise and rte_errno is set. 4354 */ 4355 static struct mlx5_flow_counter_pool * 4356 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, 4357 uint32_t batch, uint32_t age) 4358 { 4359 struct mlx5_priv *priv = dev->data->dev_private; 4360 struct mlx5_flow_counter_pool *pool; 4361 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4362 age); 4363 int16_t n_valid = rte_atomic16_read(&cont->n_valid); 4364 uint32_t size = sizeof(*pool); 4365 4366 if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age)) 4367 return NULL; 4368 size += MLX5_COUNTERS_PER_POOL * CNT_SIZE; 4369 size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE); 4370 size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE); 4371 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY); 4372 if (!pool) { 4373 rte_errno = ENOMEM; 4374 return NULL; 4375 } 4376 pool->min_dcs = dcs; 4377 if (!priv->counter_fallback) 4378 pool->raw = cont->mem_mng->raws + n_valid % 4379 MLX5_CNT_CONTAINER_RESIZE; 4380 pool->raw_hw = NULL; 4381 pool->type = 0; 4382 pool->type |= (batch ? 0 : CNT_POOL_TYPE_EXT); 4383 pool->type |= (!age ? 0 : CNT_POOL_TYPE_AGE); 4384 pool->query_gen = 0; 4385 rte_spinlock_init(&pool->sl); 4386 TAILQ_INIT(&pool->counters[0]); 4387 TAILQ_INIT(&pool->counters[1]); 4388 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next); 4389 pool->index = n_valid; 4390 cont->pools[n_valid] = pool; 4391 if (!batch) { 4392 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL); 4393 4394 if (base < cont->min_id) 4395 cont->min_id = base; 4396 if (base > cont->max_id) 4397 cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1; 4398 cont->last_pool_idx = pool->index; 4399 } 4400 /* Pool initialization must be updated before host thread access. */ 4401 rte_cio_wmb(); 4402 rte_atomic16_add(&cont->n_valid, 1); 4403 return pool; 4404 } 4405 4406 /** 4407 * Restore skipped counters in the pool. 4408 * 4409 * As counter pool query requires the first counter dcs 4410 * ID start with 4 alinged, if the pool counters with 4411 * min_dcs ID are not aligned with 4, the counters will 4412 * be skipped. 4413 * Once other min_dcs ID less than these skipped counter 4414 * dcs ID appears, the skipped counters will be safe to 4415 * use. 4416 * Should be called when min_dcs is updated. 4417 * 4418 * @param[in] pool 4419 * Current counter pool. 4420 * @param[in] last_min_dcs 4421 * Last min_dcs. 4422 */ 4423 static void 4424 flow_dv_counter_restore(struct mlx5_flow_counter_pool *pool, 4425 struct mlx5_devx_obj *last_min_dcs) 4426 { 4427 struct mlx5_flow_counter_ext *cnt_ext; 4428 uint32_t offset, new_offset; 4429 uint32_t skip_cnt = 0; 4430 uint32_t i; 4431 4432 if (!pool->skip_cnt) 4433 return; 4434 /* 4435 * If last min_dcs is not valid. The skipped counter may even after 4436 * last min_dcs, set the offset to the whole pool. 4437 */ 4438 if (last_min_dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) 4439 offset = MLX5_COUNTERS_PER_POOL; 4440 else 4441 offset = last_min_dcs->id % MLX5_COUNTERS_PER_POOL; 4442 new_offset = pool->min_dcs->id % MLX5_COUNTERS_PER_POOL; 4443 /* 4444 * Check the counters from 1 to the last_min_dcs range. Counters 4445 * before new min_dcs indicates pool still has skipped counters. 4446 * Counters be skipped after new min_dcs will be ready to use. 4447 * Offset 0 counter must be empty or min_dcs, start from 1. 4448 */ 4449 for (i = 1; i < offset; i++) { 4450 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i); 4451 if (cnt_ext->skipped) { 4452 if (i > new_offset) { 4453 cnt_ext->skipped = 0; 4454 TAILQ_INSERT_TAIL 4455 (&pool->counters[pool->query_gen], 4456 MLX5_POOL_GET_CNT(pool, i), next); 4457 } else { 4458 skip_cnt++; 4459 } 4460 } 4461 } 4462 if (!skip_cnt) 4463 pool->skip_cnt = 0; 4464 } 4465 4466 /** 4467 * Prepare a new counter and/or a new counter pool. 4468 * 4469 * @param[in] dev 4470 * Pointer to the Ethernet device structure. 4471 * @param[out] cnt_free 4472 * Where to put the pointer of a new counter. 4473 * @param[in] batch 4474 * Whether the pool is for counter that was allocated by batch command. 4475 * @param[in] age 4476 * Whether the pool is for counter that was allocated for aging. 4477 * 4478 * @return 4479 * The counter pool pointer and @p cnt_free is set on success, 4480 * NULL otherwise and rte_errno is set. 4481 */ 4482 static struct mlx5_flow_counter_pool * 4483 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, 4484 struct mlx5_flow_counter **cnt_free, 4485 uint32_t batch, uint32_t age) 4486 { 4487 struct mlx5_priv *priv = dev->data->dev_private; 4488 struct mlx5_pools_container *cont; 4489 struct mlx5_flow_counter_pool *pool; 4490 struct mlx5_counters tmp_tq; 4491 struct mlx5_devx_obj *last_min_dcs; 4492 struct mlx5_devx_obj *dcs = NULL; 4493 struct mlx5_flow_counter *cnt; 4494 uint32_t add2other; 4495 uint32_t i; 4496 4497 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age); 4498 if (!batch) { 4499 retry: 4500 add2other = 0; 4501 /* bulk_bitmap must be 0 for single counter allocation. */ 4502 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); 4503 if (!dcs) 4504 return NULL; 4505 pool = flow_dv_find_pool_by_id(cont, dcs->id); 4506 /* Check if counter belongs to exist pool ID range. */ 4507 if (!pool) { 4508 pool = flow_dv_find_pool_by_id 4509 (MLX5_CNT_CONTAINER 4510 (priv->sh, batch, (age ^ 0x1)), dcs->id); 4511 /* 4512 * Pool eixsts, counter will be added to the other 4513 * container, need to reallocate it later. 4514 */ 4515 if (pool) { 4516 add2other = 1; 4517 } else { 4518 pool = flow_dv_pool_create(dev, dcs, batch, 4519 age); 4520 if (!pool) { 4521 mlx5_devx_cmd_destroy(dcs); 4522 return NULL; 4523 } 4524 } 4525 } 4526 if ((dcs->id < pool->min_dcs->id || 4527 pool->min_dcs->id & 4528 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) && 4529 !(dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))) { 4530 /* 4531 * Update the pool min_dcs only if current dcs is 4532 * valid and exist min_dcs is not valid or greater 4533 * than new dcs. 4534 */ 4535 last_min_dcs = pool->min_dcs; 4536 rte_atomic64_set(&pool->a64_dcs, 4537 (int64_t)(uintptr_t)dcs); 4538 /* 4539 * Restore any skipped counters if the new min_dcs 4540 * ID is smaller or min_dcs is not valid. 4541 */ 4542 if (dcs->id < last_min_dcs->id || 4543 last_min_dcs->id & 4544 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) 4545 flow_dv_counter_restore(pool, last_min_dcs); 4546 } 4547 i = dcs->id % MLX5_COUNTERS_PER_POOL; 4548 cnt = MLX5_POOL_GET_CNT(pool, i); 4549 cnt->pool = pool; 4550 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs; 4551 /* 4552 * If min_dcs is not valid, it means the new allocated dcs 4553 * also fail to become the valid min_dcs, just skip it. 4554 * Or if min_dcs is valid, and new dcs ID is smaller than 4555 * min_dcs, but not become the min_dcs, also skip it. 4556 */ 4557 if (pool->min_dcs->id & 4558 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1) || 4559 dcs->id < pool->min_dcs->id) { 4560 MLX5_GET_POOL_CNT_EXT(pool, i)->skipped = 1; 4561 pool->skip_cnt = 1; 4562 goto retry; 4563 } 4564 if (add2other) { 4565 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], 4566 cnt, next); 4567 goto retry; 4568 } 4569 *cnt_free = cnt; 4570 return pool; 4571 } 4572 /* bulk_bitmap is in 128 counters units. */ 4573 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4) 4574 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); 4575 if (!dcs) { 4576 rte_errno = ENODATA; 4577 return NULL; 4578 } 4579 pool = flow_dv_pool_create(dev, dcs, batch, age); 4580 if (!pool) { 4581 mlx5_devx_cmd_destroy(dcs); 4582 return NULL; 4583 } 4584 TAILQ_INIT(&tmp_tq); 4585 for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) { 4586 cnt = MLX5_POOL_GET_CNT(pool, i); 4587 cnt->pool = pool; 4588 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next); 4589 } 4590 rte_spinlock_lock(&cont->csl); 4591 TAILQ_CONCAT(&cont->counters, &tmp_tq, next); 4592 rte_spinlock_unlock(&cont->csl); 4593 *cnt_free = MLX5_POOL_GET_CNT(pool, 0); 4594 (*cnt_free)->pool = pool; 4595 return pool; 4596 } 4597 4598 /** 4599 * Search for existed shared counter. 4600 * 4601 * @param[in] dev 4602 * Pointer to the Ethernet device structure. 4603 * @param[in] id 4604 * The shared counter ID to search. 4605 * @param[out] ppool 4606 * mlx5 flow counter pool in the container, 4607 * 4608 * @return 4609 * NULL if not existed, otherwise pointer to the shared extend counter. 4610 */ 4611 static struct mlx5_flow_counter_ext * 4612 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id, 4613 struct mlx5_flow_counter_pool **ppool) 4614 { 4615 struct mlx5_priv *priv = dev->data->dev_private; 4616 union mlx5_l3t_data data; 4617 uint32_t cnt_idx; 4618 4619 if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword) 4620 return NULL; 4621 cnt_idx = data.dword; 4622 /* 4623 * Shared counters don't have age info. The counter extend is after 4624 * the counter datat structure. 4625 */ 4626 return (struct mlx5_flow_counter_ext *) 4627 ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1); 4628 } 4629 4630 /** 4631 * Allocate a flow counter. 4632 * 4633 * @param[in] dev 4634 * Pointer to the Ethernet device structure. 4635 * @param[in] shared 4636 * Indicate if this counter is shared with other flows. 4637 * @param[in] id 4638 * Counter identifier. 4639 * @param[in] group 4640 * Counter flow group. 4641 * @param[in] age 4642 * Whether the counter was allocated for aging. 4643 * 4644 * @return 4645 * Index to flow counter on success, 0 otherwise and rte_errno is set. 4646 */ 4647 static uint32_t 4648 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, 4649 uint16_t group, uint32_t age) 4650 { 4651 struct mlx5_priv *priv = dev->data->dev_private; 4652 struct mlx5_flow_counter_pool *pool = NULL; 4653 struct mlx5_flow_counter *cnt_free = NULL; 4654 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4655 /* 4656 * Currently group 0 flow counter cannot be assigned to a flow if it is 4657 * not the first one in the batch counter allocation, so it is better 4658 * to allocate counters one by one for these flows in a separate 4659 * container. 4660 * A counter can be shared between different groups so need to take 4661 * shared counters from the single container. 4662 */ 4663 uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0; 4664 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4665 age); 4666 uint32_t cnt_idx; 4667 4668 if (!priv->config.devx) { 4669 rte_errno = ENOTSUP; 4670 return 0; 4671 } 4672 if (shared) { 4673 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool); 4674 if (cnt_ext) { 4675 if (cnt_ext->ref_cnt + 1 == 0) { 4676 rte_errno = E2BIG; 4677 return 0; 4678 } 4679 cnt_ext->ref_cnt++; 4680 cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL + 4681 (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL) 4682 + 1; 4683 return cnt_idx; 4684 } 4685 } 4686 /* Get free counters from container. */ 4687 rte_spinlock_lock(&cont->csl); 4688 cnt_free = TAILQ_FIRST(&cont->counters); 4689 if (cnt_free) 4690 TAILQ_REMOVE(&cont->counters, cnt_free, next); 4691 rte_spinlock_unlock(&cont->csl); 4692 if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, 4693 batch, age)) 4694 goto err; 4695 pool = cnt_free->pool; 4696 if (!batch) 4697 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free); 4698 /* Create a DV counter action only in the first time usage. */ 4699 if (!cnt_free->action) { 4700 uint16_t offset; 4701 struct mlx5_devx_obj *dcs; 4702 int ret; 4703 4704 if (batch) { 4705 offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free); 4706 dcs = pool->min_dcs; 4707 } else { 4708 offset = 0; 4709 dcs = cnt_ext->dcs; 4710 } 4711 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset, 4712 &cnt_free->action); 4713 if (ret) { 4714 rte_errno = errno; 4715 goto err; 4716 } 4717 } 4718 cnt_idx = MLX5_MAKE_CNT_IDX(pool->index, 4719 MLX5_CNT_ARRAY_IDX(pool, cnt_free)); 4720 cnt_idx += batch * MLX5_CNT_BATCH_OFFSET; 4721 cnt_idx += age * MLX5_CNT_AGE_OFFSET; 4722 /* Update the counter reset values. */ 4723 if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits, 4724 &cnt_free->bytes)) 4725 goto err; 4726 if (cnt_ext) { 4727 cnt_ext->shared = shared; 4728 cnt_ext->ref_cnt = 1; 4729 cnt_ext->id = id; 4730 if (shared) { 4731 union mlx5_l3t_data data; 4732 4733 data.dword = cnt_idx; 4734 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data)) 4735 return 0; 4736 } 4737 } 4738 if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on) 4739 /* Start the asynchronous batch query by the host thread. */ 4740 mlx5_set_query_alarm(priv->sh); 4741 return cnt_idx; 4742 err: 4743 if (cnt_free) { 4744 cnt_free->pool = pool; 4745 rte_spinlock_lock(&cont->csl); 4746 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next); 4747 rte_spinlock_unlock(&cont->csl); 4748 } 4749 return 0; 4750 } 4751 4752 /** 4753 * Get age param from counter index. 4754 * 4755 * @param[in] dev 4756 * Pointer to the Ethernet device structure. 4757 * @param[in] counter 4758 * Index to the counter handler. 4759 * 4760 * @return 4761 * The aging parameter specified for the counter index. 4762 */ 4763 static struct mlx5_age_param* 4764 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev, 4765 uint32_t counter) 4766 { 4767 struct mlx5_flow_counter *cnt; 4768 struct mlx5_flow_counter_pool *pool = NULL; 4769 4770 flow_dv_counter_get_by_idx(dev, counter, &pool); 4771 counter = (counter - 1) % MLX5_COUNTERS_PER_POOL; 4772 cnt = MLX5_POOL_GET_CNT(pool, counter); 4773 return MLX5_CNT_TO_AGE(cnt); 4774 } 4775 4776 /** 4777 * Remove a flow counter from aged counter list. 4778 * 4779 * @param[in] dev 4780 * Pointer to the Ethernet device structure. 4781 * @param[in] counter 4782 * Index to the counter handler. 4783 * @param[in] cnt 4784 * Pointer to the counter handler. 4785 */ 4786 static void 4787 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev, 4788 uint32_t counter, struct mlx5_flow_counter *cnt) 4789 { 4790 struct mlx5_age_info *age_info; 4791 struct mlx5_age_param *age_param; 4792 struct mlx5_priv *priv = dev->data->dev_private; 4793 4794 age_info = GET_PORT_AGE_INFO(priv); 4795 age_param = flow_dv_counter_idx_get_age(dev, counter); 4796 if (rte_atomic16_cmpset((volatile uint16_t *) 4797 &age_param->state, 4798 AGE_CANDIDATE, AGE_FREE) 4799 != AGE_CANDIDATE) { 4800 /** 4801 * We need the lock even it is age timeout, 4802 * since counter may still in process. 4803 */ 4804 rte_spinlock_lock(&age_info->aged_sl); 4805 TAILQ_REMOVE(&age_info->aged_counters, cnt, next); 4806 rte_spinlock_unlock(&age_info->aged_sl); 4807 } 4808 rte_atomic16_set(&age_param->state, AGE_FREE); 4809 } 4810 /** 4811 * Release a flow counter. 4812 * 4813 * @param[in] dev 4814 * Pointer to the Ethernet device structure. 4815 * @param[in] counter 4816 * Index to the counter handler. 4817 */ 4818 static void 4819 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter) 4820 { 4821 struct mlx5_priv *priv = dev->data->dev_private; 4822 struct mlx5_flow_counter_pool *pool = NULL; 4823 struct mlx5_flow_counter *cnt; 4824 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4825 4826 if (!counter) 4827 return; 4828 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); 4829 MLX5_ASSERT(pool); 4830 if (counter < MLX5_CNT_BATCH_OFFSET) { 4831 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); 4832 if (cnt_ext) { 4833 if (--cnt_ext->ref_cnt) 4834 return; 4835 if (cnt_ext->shared) 4836 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, 4837 cnt_ext->id); 4838 } 4839 } 4840 if (IS_AGE_POOL(pool)) 4841 flow_dv_counter_remove_from_age(dev, counter, cnt); 4842 cnt->pool = pool; 4843 /* 4844 * Put the counter back to list to be updated in none fallback mode. 4845 * Currently, we are using two list alternately, while one is in query, 4846 * add the freed counter to the other list based on the pool query_gen 4847 * value. After query finishes, add counter the list to the global 4848 * container counter list. The list changes while query starts. In 4849 * this case, lock will not be needed as query callback and release 4850 * function both operate with the different list. 4851 * 4852 */ 4853 if (!priv->counter_fallback) 4854 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next); 4855 else 4856 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER 4857 (priv->sh, 0, 0))->counters), 4858 cnt, next); 4859 } 4860 4861 /** 4862 * Verify the @p attributes will be correctly understood by the NIC and store 4863 * them in the @p flow if everything is correct. 4864 * 4865 * @param[in] dev 4866 * Pointer to dev struct. 4867 * @param[in] attributes 4868 * Pointer to flow attributes 4869 * @param[in] external 4870 * This flow rule is created by request external to PMD. 4871 * @param[out] error 4872 * Pointer to error structure. 4873 * 4874 * @return 4875 * - 0 on success and non root table. 4876 * - 1 on success and root table. 4877 * - a negative errno value otherwise and rte_errno is set. 4878 */ 4879 static int 4880 flow_dv_validate_attributes(struct rte_eth_dev *dev, 4881 const struct rte_flow_attr *attributes, 4882 bool external __rte_unused, 4883 struct rte_flow_error *error) 4884 { 4885 struct mlx5_priv *priv = dev->data->dev_private; 4886 uint32_t priority_max = priv->config.flow_prio - 1; 4887 int ret = 0; 4888 4889 #ifndef HAVE_MLX5DV_DR 4890 if (attributes->group) 4891 return rte_flow_error_set(error, ENOTSUP, 4892 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 4893 NULL, 4894 "groups are not supported"); 4895 #else 4896 uint32_t table = 0; 4897 4898 ret = mlx5_flow_group_to_table(attributes, external, 4899 attributes->group, !!priv->fdb_def_rule, 4900 &table, error); 4901 if (ret) 4902 return ret; 4903 if (!table) 4904 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; 4905 #endif 4906 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 4907 attributes->priority >= priority_max) 4908 return rte_flow_error_set(error, ENOTSUP, 4909 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 4910 NULL, 4911 "priority out of range"); 4912 if (attributes->transfer) { 4913 if (!priv->config.dv_esw_en) 4914 return rte_flow_error_set 4915 (error, ENOTSUP, 4916 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4917 "E-Switch dr is not supported"); 4918 if (!(priv->representor || priv->master)) 4919 return rte_flow_error_set 4920 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4921 NULL, "E-Switch configuration can only be" 4922 " done by a master or a representor device"); 4923 if (attributes->egress) 4924 return rte_flow_error_set 4925 (error, ENOTSUP, 4926 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes, 4927 "egress is not supported"); 4928 } 4929 if (!(attributes->egress ^ attributes->ingress)) 4930 return rte_flow_error_set(error, ENOTSUP, 4931 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 4932 "must specify exactly one of " 4933 "ingress or egress"); 4934 return ret; 4935 } 4936 4937 /** 4938 * Internal validation function. For validating both actions and items. 4939 * 4940 * @param[in] dev 4941 * Pointer to the rte_eth_dev structure. 4942 * @param[in] attr 4943 * Pointer to the flow attributes. 4944 * @param[in] items 4945 * Pointer to the list of items. 4946 * @param[in] actions 4947 * Pointer to the list of actions. 4948 * @param[in] external 4949 * This flow rule is created by request external to PMD. 4950 * @param[in] hairpin 4951 * Number of hairpin TX actions, 0 means classic flow. 4952 * @param[out] error 4953 * Pointer to the error structure. 4954 * 4955 * @return 4956 * 0 on success, a negative errno value otherwise and rte_errno is set. 4957 */ 4958 static int 4959 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, 4960 const struct rte_flow_item items[], 4961 const struct rte_flow_action actions[], 4962 bool external, int hairpin, struct rte_flow_error *error) 4963 { 4964 int ret; 4965 uint64_t action_flags = 0; 4966 uint64_t item_flags = 0; 4967 uint64_t last_item = 0; 4968 uint8_t next_protocol = 0xff; 4969 uint16_t ether_type = 0; 4970 int actions_n = 0; 4971 uint8_t item_ipv6_proto = 0; 4972 const struct rte_flow_item *gre_item = NULL; 4973 const struct rte_flow_action_raw_decap *decap; 4974 const struct rte_flow_action_raw_encap *encap; 4975 const struct rte_flow_action_rss *rss; 4976 const struct rte_flow_item_tcp nic_tcp_mask = { 4977 .hdr = { 4978 .tcp_flags = 0xFF, 4979 .src_port = RTE_BE16(UINT16_MAX), 4980 .dst_port = RTE_BE16(UINT16_MAX), 4981 } 4982 }; 4983 const struct rte_flow_item_ipv4 nic_ipv4_mask = { 4984 .hdr = { 4985 .src_addr = RTE_BE32(0xffffffff), 4986 .dst_addr = RTE_BE32(0xffffffff), 4987 .type_of_service = 0xff, 4988 .next_proto_id = 0xff, 4989 .time_to_live = 0xff, 4990 }, 4991 }; 4992 const struct rte_flow_item_ipv6 nic_ipv6_mask = { 4993 .hdr = { 4994 .src_addr = 4995 "\xff\xff\xff\xff\xff\xff\xff\xff" 4996 "\xff\xff\xff\xff\xff\xff\xff\xff", 4997 .dst_addr = 4998 "\xff\xff\xff\xff\xff\xff\xff\xff" 4999 "\xff\xff\xff\xff\xff\xff\xff\xff", 5000 .vtc_flow = RTE_BE32(0xffffffff), 5001 .proto = 0xff, 5002 .hop_limits = 0xff, 5003 }, 5004 }; 5005 const struct rte_flow_item_ecpri nic_ecpri_mask = { 5006 .hdr = { 5007 .common = { 5008 .u32 = 5009 RTE_BE32(((const struct rte_ecpri_common_hdr) { 5010 .type = 0xFF, 5011 }).u32), 5012 }, 5013 .dummy[0] = 0xffffffff, 5014 }, 5015 }; 5016 struct mlx5_priv *priv = dev->data->dev_private; 5017 struct mlx5_dev_config *dev_conf = &priv->config; 5018 uint16_t queue_index = 0xFFFF; 5019 const struct rte_flow_item_vlan *vlan_m = NULL; 5020 int16_t rw_act_num = 0; 5021 uint64_t is_root; 5022 5023 if (items == NULL) 5024 return -1; 5025 ret = flow_dv_validate_attributes(dev, attr, external, error); 5026 if (ret < 0) 5027 return ret; 5028 is_root = (uint64_t)ret; 5029 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5030 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 5031 int type = items->type; 5032 5033 if (!mlx5_flow_os_item_supported(type)) 5034 return rte_flow_error_set(error, ENOTSUP, 5035 RTE_FLOW_ERROR_TYPE_ITEM, 5036 NULL, "item not supported"); 5037 switch (type) { 5038 case RTE_FLOW_ITEM_TYPE_VOID: 5039 break; 5040 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5041 ret = flow_dv_validate_item_port_id 5042 (dev, items, attr, item_flags, error); 5043 if (ret < 0) 5044 return ret; 5045 last_item = MLX5_FLOW_ITEM_PORT_ID; 5046 break; 5047 case RTE_FLOW_ITEM_TYPE_ETH: 5048 ret = mlx5_flow_validate_item_eth(items, item_flags, 5049 error); 5050 if (ret < 0) 5051 return ret; 5052 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 5053 MLX5_FLOW_LAYER_OUTER_L2; 5054 if (items->mask != NULL && items->spec != NULL) { 5055 ether_type = 5056 ((const struct rte_flow_item_eth *) 5057 items->spec)->type; 5058 ether_type &= 5059 ((const struct rte_flow_item_eth *) 5060 items->mask)->type; 5061 ether_type = rte_be_to_cpu_16(ether_type); 5062 } else { 5063 ether_type = 0; 5064 } 5065 break; 5066 case RTE_FLOW_ITEM_TYPE_VLAN: 5067 ret = flow_dv_validate_item_vlan(items, item_flags, 5068 dev, error); 5069 if (ret < 0) 5070 return ret; 5071 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 5072 MLX5_FLOW_LAYER_OUTER_VLAN; 5073 if (items->mask != NULL && items->spec != NULL) { 5074 ether_type = 5075 ((const struct rte_flow_item_vlan *) 5076 items->spec)->inner_type; 5077 ether_type &= 5078 ((const struct rte_flow_item_vlan *) 5079 items->mask)->inner_type; 5080 ether_type = rte_be_to_cpu_16(ether_type); 5081 } else { 5082 ether_type = 0; 5083 } 5084 /* Store outer VLAN mask for of_push_vlan action. */ 5085 if (!tunnel) 5086 vlan_m = items->mask; 5087 break; 5088 case RTE_FLOW_ITEM_TYPE_IPV4: 5089 mlx5_flow_tunnel_ip_check(items, next_protocol, 5090 &item_flags, &tunnel); 5091 ret = mlx5_flow_validate_item_ipv4(items, item_flags, 5092 last_item, 5093 ether_type, 5094 &nic_ipv4_mask, 5095 error); 5096 if (ret < 0) 5097 return ret; 5098 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 5099 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 5100 if (items->mask != NULL && 5101 ((const struct rte_flow_item_ipv4 *) 5102 items->mask)->hdr.next_proto_id) { 5103 next_protocol = 5104 ((const struct rte_flow_item_ipv4 *) 5105 (items->spec))->hdr.next_proto_id; 5106 next_protocol &= 5107 ((const struct rte_flow_item_ipv4 *) 5108 (items->mask))->hdr.next_proto_id; 5109 } else { 5110 /* Reset for inner layer. */ 5111 next_protocol = 0xff; 5112 } 5113 break; 5114 case RTE_FLOW_ITEM_TYPE_IPV6: 5115 mlx5_flow_tunnel_ip_check(items, next_protocol, 5116 &item_flags, &tunnel); 5117 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 5118 last_item, 5119 ether_type, 5120 &nic_ipv6_mask, 5121 error); 5122 if (ret < 0) 5123 return ret; 5124 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 5125 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 5126 if (items->mask != NULL && 5127 ((const struct rte_flow_item_ipv6 *) 5128 items->mask)->hdr.proto) { 5129 item_ipv6_proto = 5130 ((const struct rte_flow_item_ipv6 *) 5131 items->spec)->hdr.proto; 5132 next_protocol = 5133 ((const struct rte_flow_item_ipv6 *) 5134 items->spec)->hdr.proto; 5135 next_protocol &= 5136 ((const struct rte_flow_item_ipv6 *) 5137 items->mask)->hdr.proto; 5138 } else { 5139 /* Reset for inner layer. */ 5140 next_protocol = 0xff; 5141 } 5142 break; 5143 case RTE_FLOW_ITEM_TYPE_TCP: 5144 ret = mlx5_flow_validate_item_tcp 5145 (items, item_flags, 5146 next_protocol, 5147 &nic_tcp_mask, 5148 error); 5149 if (ret < 0) 5150 return ret; 5151 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 5152 MLX5_FLOW_LAYER_OUTER_L4_TCP; 5153 break; 5154 case RTE_FLOW_ITEM_TYPE_UDP: 5155 ret = mlx5_flow_validate_item_udp(items, item_flags, 5156 next_protocol, 5157 error); 5158 if (ret < 0) 5159 return ret; 5160 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 5161 MLX5_FLOW_LAYER_OUTER_L4_UDP; 5162 break; 5163 case RTE_FLOW_ITEM_TYPE_GRE: 5164 ret = mlx5_flow_validate_item_gre(items, item_flags, 5165 next_protocol, error); 5166 if (ret < 0) 5167 return ret; 5168 gre_item = items; 5169 last_item = MLX5_FLOW_LAYER_GRE; 5170 break; 5171 case RTE_FLOW_ITEM_TYPE_NVGRE: 5172 ret = mlx5_flow_validate_item_nvgre(items, item_flags, 5173 next_protocol, 5174 error); 5175 if (ret < 0) 5176 return ret; 5177 last_item = MLX5_FLOW_LAYER_NVGRE; 5178 break; 5179 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 5180 ret = mlx5_flow_validate_item_gre_key 5181 (items, item_flags, gre_item, error); 5182 if (ret < 0) 5183 return ret; 5184 last_item = MLX5_FLOW_LAYER_GRE_KEY; 5185 break; 5186 case RTE_FLOW_ITEM_TYPE_VXLAN: 5187 ret = mlx5_flow_validate_item_vxlan(items, item_flags, 5188 error); 5189 if (ret < 0) 5190 return ret; 5191 last_item = MLX5_FLOW_LAYER_VXLAN; 5192 break; 5193 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 5194 ret = mlx5_flow_validate_item_vxlan_gpe(items, 5195 item_flags, dev, 5196 error); 5197 if (ret < 0) 5198 return ret; 5199 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 5200 break; 5201 case RTE_FLOW_ITEM_TYPE_GENEVE: 5202 ret = mlx5_flow_validate_item_geneve(items, 5203 item_flags, dev, 5204 error); 5205 if (ret < 0) 5206 return ret; 5207 last_item = MLX5_FLOW_LAYER_GENEVE; 5208 break; 5209 case RTE_FLOW_ITEM_TYPE_MPLS: 5210 ret = mlx5_flow_validate_item_mpls(dev, items, 5211 item_flags, 5212 last_item, error); 5213 if (ret < 0) 5214 return ret; 5215 last_item = MLX5_FLOW_LAYER_MPLS; 5216 break; 5217 5218 case RTE_FLOW_ITEM_TYPE_MARK: 5219 ret = flow_dv_validate_item_mark(dev, items, attr, 5220 error); 5221 if (ret < 0) 5222 return ret; 5223 last_item = MLX5_FLOW_ITEM_MARK; 5224 break; 5225 case RTE_FLOW_ITEM_TYPE_META: 5226 ret = flow_dv_validate_item_meta(dev, items, attr, 5227 error); 5228 if (ret < 0) 5229 return ret; 5230 last_item = MLX5_FLOW_ITEM_METADATA; 5231 break; 5232 case RTE_FLOW_ITEM_TYPE_ICMP: 5233 ret = mlx5_flow_validate_item_icmp(items, item_flags, 5234 next_protocol, 5235 error); 5236 if (ret < 0) 5237 return ret; 5238 last_item = MLX5_FLOW_LAYER_ICMP; 5239 break; 5240 case RTE_FLOW_ITEM_TYPE_ICMP6: 5241 ret = mlx5_flow_validate_item_icmp6(items, item_flags, 5242 next_protocol, 5243 error); 5244 if (ret < 0) 5245 return ret; 5246 item_ipv6_proto = IPPROTO_ICMPV6; 5247 last_item = MLX5_FLOW_LAYER_ICMP6; 5248 break; 5249 case RTE_FLOW_ITEM_TYPE_TAG: 5250 ret = flow_dv_validate_item_tag(dev, items, 5251 attr, error); 5252 if (ret < 0) 5253 return ret; 5254 last_item = MLX5_FLOW_ITEM_TAG; 5255 break; 5256 case MLX5_RTE_FLOW_ITEM_TYPE_TAG: 5257 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: 5258 break; 5259 case RTE_FLOW_ITEM_TYPE_GTP: 5260 ret = flow_dv_validate_item_gtp(dev, items, item_flags, 5261 error); 5262 if (ret < 0) 5263 return ret; 5264 last_item = MLX5_FLOW_LAYER_GTP; 5265 break; 5266 case RTE_FLOW_ITEM_TYPE_ECPRI: 5267 /* Capacity will be checked in the translate stage. */ 5268 ret = mlx5_flow_validate_item_ecpri(items, item_flags, 5269 last_item, 5270 ether_type, 5271 &nic_ecpri_mask, 5272 error); 5273 if (ret < 0) 5274 return ret; 5275 last_item = MLX5_FLOW_LAYER_ECPRI; 5276 break; 5277 default: 5278 return rte_flow_error_set(error, ENOTSUP, 5279 RTE_FLOW_ERROR_TYPE_ITEM, 5280 NULL, "item not supported"); 5281 } 5282 item_flags |= last_item; 5283 } 5284 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5285 int type = actions->type; 5286 5287 if (!mlx5_flow_os_action_supported(type)) 5288 return rte_flow_error_set(error, ENOTSUP, 5289 RTE_FLOW_ERROR_TYPE_ACTION, 5290 actions, 5291 "action not supported"); 5292 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) 5293 return rte_flow_error_set(error, ENOTSUP, 5294 RTE_FLOW_ERROR_TYPE_ACTION, 5295 actions, "too many actions"); 5296 switch (type) { 5297 case RTE_FLOW_ACTION_TYPE_VOID: 5298 break; 5299 case RTE_FLOW_ACTION_TYPE_PORT_ID: 5300 ret = flow_dv_validate_action_port_id(dev, 5301 action_flags, 5302 actions, 5303 attr, 5304 error); 5305 if (ret) 5306 return ret; 5307 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 5308 ++actions_n; 5309 break; 5310 case RTE_FLOW_ACTION_TYPE_FLAG: 5311 ret = flow_dv_validate_action_flag(dev, action_flags, 5312 attr, error); 5313 if (ret < 0) 5314 return ret; 5315 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 5316 /* Count all modify-header actions as one. */ 5317 if (!(action_flags & 5318 MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5319 ++actions_n; 5320 action_flags |= MLX5_FLOW_ACTION_FLAG | 5321 MLX5_FLOW_ACTION_MARK_EXT; 5322 } else { 5323 action_flags |= MLX5_FLOW_ACTION_FLAG; 5324 ++actions_n; 5325 } 5326 rw_act_num += MLX5_ACT_NUM_SET_MARK; 5327 break; 5328 case RTE_FLOW_ACTION_TYPE_MARK: 5329 ret = flow_dv_validate_action_mark(dev, actions, 5330 action_flags, 5331 attr, error); 5332 if (ret < 0) 5333 return ret; 5334 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 5335 /* Count all modify-header actions as one. */ 5336 if (!(action_flags & 5337 MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5338 ++actions_n; 5339 action_flags |= MLX5_FLOW_ACTION_MARK | 5340 MLX5_FLOW_ACTION_MARK_EXT; 5341 } else { 5342 action_flags |= MLX5_FLOW_ACTION_MARK; 5343 ++actions_n; 5344 } 5345 rw_act_num += MLX5_ACT_NUM_SET_MARK; 5346 break; 5347 case RTE_FLOW_ACTION_TYPE_SET_META: 5348 ret = flow_dv_validate_action_set_meta(dev, actions, 5349 action_flags, 5350 attr, error); 5351 if (ret < 0) 5352 return ret; 5353 /* Count all modify-header actions as one action. */ 5354 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5355 ++actions_n; 5356 action_flags |= MLX5_FLOW_ACTION_SET_META; 5357 rw_act_num += MLX5_ACT_NUM_SET_META; 5358 break; 5359 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5360 ret = flow_dv_validate_action_set_tag(dev, actions, 5361 action_flags, 5362 attr, error); 5363 if (ret < 0) 5364 return ret; 5365 /* Count all modify-header actions as one action. */ 5366 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5367 ++actions_n; 5368 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 5369 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5370 break; 5371 case RTE_FLOW_ACTION_TYPE_DROP: 5372 ret = mlx5_flow_validate_action_drop(action_flags, 5373 attr, error); 5374 if (ret < 0) 5375 return ret; 5376 action_flags |= MLX5_FLOW_ACTION_DROP; 5377 ++actions_n; 5378 break; 5379 case RTE_FLOW_ACTION_TYPE_QUEUE: 5380 ret = mlx5_flow_validate_action_queue(actions, 5381 action_flags, dev, 5382 attr, error); 5383 if (ret < 0) 5384 return ret; 5385 queue_index = ((const struct rte_flow_action_queue *) 5386 (actions->conf))->index; 5387 action_flags |= MLX5_FLOW_ACTION_QUEUE; 5388 ++actions_n; 5389 break; 5390 case RTE_FLOW_ACTION_TYPE_RSS: 5391 rss = actions->conf; 5392 ret = mlx5_flow_validate_action_rss(actions, 5393 action_flags, dev, 5394 attr, item_flags, 5395 error); 5396 if (ret < 0) 5397 return ret; 5398 if (rss != NULL && rss->queue_num) 5399 queue_index = rss->queue[0]; 5400 action_flags |= MLX5_FLOW_ACTION_RSS; 5401 ++actions_n; 5402 break; 5403 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: 5404 ret = 5405 mlx5_flow_validate_action_default_miss(action_flags, 5406 attr, error); 5407 if (ret < 0) 5408 return ret; 5409 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; 5410 ++actions_n; 5411 break; 5412 case RTE_FLOW_ACTION_TYPE_COUNT: 5413 ret = flow_dv_validate_action_count(dev, error); 5414 if (ret < 0) 5415 return ret; 5416 action_flags |= MLX5_FLOW_ACTION_COUNT; 5417 ++actions_n; 5418 break; 5419 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5420 if (flow_dv_validate_action_pop_vlan(dev, 5421 action_flags, 5422 actions, 5423 item_flags, attr, 5424 error)) 5425 return -rte_errno; 5426 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; 5427 ++actions_n; 5428 break; 5429 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5430 ret = flow_dv_validate_action_push_vlan(dev, 5431 action_flags, 5432 vlan_m, 5433 actions, attr, 5434 error); 5435 if (ret < 0) 5436 return ret; 5437 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; 5438 ++actions_n; 5439 break; 5440 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5441 ret = flow_dv_validate_action_set_vlan_pcp 5442 (action_flags, actions, error); 5443 if (ret < 0) 5444 return ret; 5445 /* Count PCP with push_vlan command. */ 5446 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP; 5447 break; 5448 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5449 ret = flow_dv_validate_action_set_vlan_vid 5450 (item_flags, action_flags, 5451 actions, error); 5452 if (ret < 0) 5453 return ret; 5454 /* Count VID with push_vlan command. */ 5455 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; 5456 rw_act_num += MLX5_ACT_NUM_MDF_VID; 5457 break; 5458 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5459 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5460 ret = flow_dv_validate_action_l2_encap(dev, 5461 action_flags, 5462 actions, attr, 5463 error); 5464 if (ret < 0) 5465 return ret; 5466 action_flags |= MLX5_FLOW_ACTION_ENCAP; 5467 ++actions_n; 5468 break; 5469 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5470 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5471 ret = flow_dv_validate_action_decap(dev, action_flags, 5472 attr, error); 5473 if (ret < 0) 5474 return ret; 5475 action_flags |= MLX5_FLOW_ACTION_DECAP; 5476 ++actions_n; 5477 break; 5478 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5479 ret = flow_dv_validate_action_raw_encap_decap 5480 (dev, NULL, actions->conf, attr, &action_flags, 5481 &actions_n, error); 5482 if (ret < 0) 5483 return ret; 5484 break; 5485 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5486 decap = actions->conf; 5487 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5488 ; 5489 if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5490 encap = NULL; 5491 actions--; 5492 } else { 5493 encap = actions->conf; 5494 } 5495 ret = flow_dv_validate_action_raw_encap_decap 5496 (dev, 5497 decap ? decap : &empty_decap, encap, 5498 attr, &action_flags, &actions_n, 5499 error); 5500 if (ret < 0) 5501 return ret; 5502 break; 5503 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5504 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5505 ret = flow_dv_validate_action_modify_mac(action_flags, 5506 actions, 5507 item_flags, 5508 error); 5509 if (ret < 0) 5510 return ret; 5511 /* Count all modify-header actions as one action. */ 5512 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5513 ++actions_n; 5514 action_flags |= actions->type == 5515 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 5516 MLX5_FLOW_ACTION_SET_MAC_SRC : 5517 MLX5_FLOW_ACTION_SET_MAC_DST; 5518 /* 5519 * Even if the source and destination MAC addresses have 5520 * overlap in the header with 4B alignment, the convert 5521 * function will handle them separately and 4 SW actions 5522 * will be created. And 2 actions will be added each 5523 * time no matter how many bytes of address will be set. 5524 */ 5525 rw_act_num += MLX5_ACT_NUM_MDF_MAC; 5526 break; 5527 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5528 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5529 ret = flow_dv_validate_action_modify_ipv4(action_flags, 5530 actions, 5531 item_flags, 5532 error); 5533 if (ret < 0) 5534 return ret; 5535 /* Count all modify-header actions as one action. */ 5536 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5537 ++actions_n; 5538 action_flags |= actions->type == 5539 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 5540 MLX5_FLOW_ACTION_SET_IPV4_SRC : 5541 MLX5_FLOW_ACTION_SET_IPV4_DST; 5542 rw_act_num += MLX5_ACT_NUM_MDF_IPV4; 5543 break; 5544 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5545 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5546 ret = flow_dv_validate_action_modify_ipv6(action_flags, 5547 actions, 5548 item_flags, 5549 error); 5550 if (ret < 0) 5551 return ret; 5552 if (item_ipv6_proto == IPPROTO_ICMPV6) 5553 return rte_flow_error_set(error, ENOTSUP, 5554 RTE_FLOW_ERROR_TYPE_ACTION, 5555 actions, 5556 "Can't change header " 5557 "with ICMPv6 proto"); 5558 /* Count all modify-header actions as one action. */ 5559 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5560 ++actions_n; 5561 action_flags |= actions->type == 5562 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 5563 MLX5_FLOW_ACTION_SET_IPV6_SRC : 5564 MLX5_FLOW_ACTION_SET_IPV6_DST; 5565 rw_act_num += MLX5_ACT_NUM_MDF_IPV6; 5566 break; 5567 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5568 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5569 ret = flow_dv_validate_action_modify_tp(action_flags, 5570 actions, 5571 item_flags, 5572 error); 5573 if (ret < 0) 5574 return ret; 5575 /* Count all modify-header actions as one action. */ 5576 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5577 ++actions_n; 5578 action_flags |= actions->type == 5579 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 5580 MLX5_FLOW_ACTION_SET_TP_SRC : 5581 MLX5_FLOW_ACTION_SET_TP_DST; 5582 rw_act_num += MLX5_ACT_NUM_MDF_PORT; 5583 break; 5584 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5585 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5586 ret = flow_dv_validate_action_modify_ttl(action_flags, 5587 actions, 5588 item_flags, 5589 error); 5590 if (ret < 0) 5591 return ret; 5592 /* Count all modify-header actions as one action. */ 5593 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5594 ++actions_n; 5595 action_flags |= actions->type == 5596 RTE_FLOW_ACTION_TYPE_SET_TTL ? 5597 MLX5_FLOW_ACTION_SET_TTL : 5598 MLX5_FLOW_ACTION_DEC_TTL; 5599 rw_act_num += MLX5_ACT_NUM_MDF_TTL; 5600 break; 5601 case RTE_FLOW_ACTION_TYPE_JUMP: 5602 ret = flow_dv_validate_action_jump(actions, 5603 action_flags, 5604 attr, external, 5605 error); 5606 if (ret) 5607 return ret; 5608 ++actions_n; 5609 action_flags |= MLX5_FLOW_ACTION_JUMP; 5610 break; 5611 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5612 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5613 ret = flow_dv_validate_action_modify_tcp_seq 5614 (action_flags, 5615 actions, 5616 item_flags, 5617 error); 5618 if (ret < 0) 5619 return ret; 5620 /* Count all modify-header actions as one action. */ 5621 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5622 ++actions_n; 5623 action_flags |= actions->type == 5624 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 5625 MLX5_FLOW_ACTION_INC_TCP_SEQ : 5626 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 5627 rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ; 5628 break; 5629 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5630 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5631 ret = flow_dv_validate_action_modify_tcp_ack 5632 (action_flags, 5633 actions, 5634 item_flags, 5635 error); 5636 if (ret < 0) 5637 return ret; 5638 /* Count all modify-header actions as one action. */ 5639 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5640 ++actions_n; 5641 action_flags |= actions->type == 5642 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 5643 MLX5_FLOW_ACTION_INC_TCP_ACK : 5644 MLX5_FLOW_ACTION_DEC_TCP_ACK; 5645 rw_act_num += MLX5_ACT_NUM_MDF_TCPACK; 5646 break; 5647 case MLX5_RTE_FLOW_ACTION_TYPE_MARK: 5648 break; 5649 case MLX5_RTE_FLOW_ACTION_TYPE_TAG: 5650 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: 5651 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5652 break; 5653 case RTE_FLOW_ACTION_TYPE_METER: 5654 ret = mlx5_flow_validate_action_meter(dev, 5655 action_flags, 5656 actions, attr, 5657 error); 5658 if (ret < 0) 5659 return ret; 5660 action_flags |= MLX5_FLOW_ACTION_METER; 5661 ++actions_n; 5662 /* Meter action will add one more TAG action. */ 5663 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5664 break; 5665 case RTE_FLOW_ACTION_TYPE_AGE: 5666 ret = flow_dv_validate_action_age(action_flags, 5667 actions, dev, 5668 error); 5669 if (ret < 0) 5670 return ret; 5671 action_flags |= MLX5_FLOW_ACTION_AGE; 5672 ++actions_n; 5673 break; 5674 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5675 ret = flow_dv_validate_action_modify_ipv4_dscp 5676 (action_flags, 5677 actions, 5678 item_flags, 5679 error); 5680 if (ret < 0) 5681 return ret; 5682 /* Count all modify-header actions as one action. */ 5683 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5684 ++actions_n; 5685 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; 5686 rw_act_num += MLX5_ACT_NUM_SET_DSCP; 5687 break; 5688 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5689 ret = flow_dv_validate_action_modify_ipv6_dscp 5690 (action_flags, 5691 actions, 5692 item_flags, 5693 error); 5694 if (ret < 0) 5695 return ret; 5696 /* Count all modify-header actions as one action. */ 5697 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5698 ++actions_n; 5699 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; 5700 rw_act_num += MLX5_ACT_NUM_SET_DSCP; 5701 break; 5702 default: 5703 return rte_flow_error_set(error, ENOTSUP, 5704 RTE_FLOW_ERROR_TYPE_ACTION, 5705 actions, 5706 "action not supported"); 5707 } 5708 } 5709 /* 5710 * Validate the drop action mutual exclusion with other actions. 5711 * Drop action is mutually-exclusive with any other action, except for 5712 * Count action. 5713 */ 5714 if ((action_flags & MLX5_FLOW_ACTION_DROP) && 5715 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) 5716 return rte_flow_error_set(error, EINVAL, 5717 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5718 "Drop action is mutually-exclusive " 5719 "with any other action, except for " 5720 "Count action"); 5721 /* Eswitch has few restrictions on using items and actions */ 5722 if (attr->transfer) { 5723 if (!mlx5_flow_ext_mreg_supported(dev) && 5724 action_flags & MLX5_FLOW_ACTION_FLAG) 5725 return rte_flow_error_set(error, ENOTSUP, 5726 RTE_FLOW_ERROR_TYPE_ACTION, 5727 NULL, 5728 "unsupported action FLAG"); 5729 if (!mlx5_flow_ext_mreg_supported(dev) && 5730 action_flags & MLX5_FLOW_ACTION_MARK) 5731 return rte_flow_error_set(error, ENOTSUP, 5732 RTE_FLOW_ERROR_TYPE_ACTION, 5733 NULL, 5734 "unsupported action MARK"); 5735 if (action_flags & MLX5_FLOW_ACTION_QUEUE) 5736 return rte_flow_error_set(error, ENOTSUP, 5737 RTE_FLOW_ERROR_TYPE_ACTION, 5738 NULL, 5739 "unsupported action QUEUE"); 5740 if (action_flags & MLX5_FLOW_ACTION_RSS) 5741 return rte_flow_error_set(error, ENOTSUP, 5742 RTE_FLOW_ERROR_TYPE_ACTION, 5743 NULL, 5744 "unsupported action RSS"); 5745 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 5746 return rte_flow_error_set(error, EINVAL, 5747 RTE_FLOW_ERROR_TYPE_ACTION, 5748 actions, 5749 "no fate action is found"); 5750 } else { 5751 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress) 5752 return rte_flow_error_set(error, EINVAL, 5753 RTE_FLOW_ERROR_TYPE_ACTION, 5754 actions, 5755 "no fate action is found"); 5756 } 5757 /* Continue validation for Xcap and VLAN actions.*/ 5758 if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS | 5759 MLX5_FLOW_VLAN_ACTIONS)) && 5760 (queue_index == 0xFFFF || 5761 mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) { 5762 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) == 5763 MLX5_FLOW_XCAP_ACTIONS) 5764 return rte_flow_error_set(error, ENOTSUP, 5765 RTE_FLOW_ERROR_TYPE_ACTION, 5766 NULL, "encap and decap " 5767 "combination aren't supported"); 5768 if (!attr->transfer && attr->ingress) { 5769 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 5770 return rte_flow_error_set 5771 (error, ENOTSUP, 5772 RTE_FLOW_ERROR_TYPE_ACTION, 5773 NULL, "encap is not supported" 5774 " for ingress traffic"); 5775 else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) 5776 return rte_flow_error_set 5777 (error, ENOTSUP, 5778 RTE_FLOW_ERROR_TYPE_ACTION, 5779 NULL, "push VLAN action not " 5780 "supported for ingress"); 5781 else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) == 5782 MLX5_FLOW_VLAN_ACTIONS) 5783 return rte_flow_error_set 5784 (error, ENOTSUP, 5785 RTE_FLOW_ERROR_TYPE_ACTION, 5786 NULL, "no support for " 5787 "multiple VLAN actions"); 5788 } 5789 } 5790 /* Hairpin flow will add one more TAG action. */ 5791 if (hairpin > 0) 5792 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5793 /* extra metadata enabled: one more TAG action will be add. */ 5794 if (dev_conf->dv_flow_en && 5795 dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 5796 mlx5_flow_ext_mreg_supported(dev)) 5797 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5798 if ((uint32_t)rw_act_num > 5799 flow_dv_modify_hdr_action_max(dev, is_root)) { 5800 return rte_flow_error_set(error, ENOTSUP, 5801 RTE_FLOW_ERROR_TYPE_ACTION, 5802 NULL, "too many header modify" 5803 " actions to support"); 5804 } 5805 return 0; 5806 } 5807 5808 /** 5809 * Internal preparation function. Allocates the DV flow size, 5810 * this size is constant. 5811 * 5812 * @param[in] dev 5813 * Pointer to the rte_eth_dev structure. 5814 * @param[in] attr 5815 * Pointer to the flow attributes. 5816 * @param[in] items 5817 * Pointer to the list of items. 5818 * @param[in] actions 5819 * Pointer to the list of actions. 5820 * @param[out] error 5821 * Pointer to the error structure. 5822 * 5823 * @return 5824 * Pointer to mlx5_flow object on success, 5825 * otherwise NULL and rte_errno is set. 5826 */ 5827 static struct mlx5_flow * 5828 flow_dv_prepare(struct rte_eth_dev *dev, 5829 const struct rte_flow_attr *attr __rte_unused, 5830 const struct rte_flow_item items[] __rte_unused, 5831 const struct rte_flow_action actions[] __rte_unused, 5832 struct rte_flow_error *error) 5833 { 5834 uint32_t handle_idx = 0; 5835 struct mlx5_flow *dev_flow; 5836 struct mlx5_flow_handle *dev_handle; 5837 struct mlx5_priv *priv = dev->data->dev_private; 5838 5839 /* In case of corrupting the memory. */ 5840 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { 5841 rte_flow_error_set(error, ENOSPC, 5842 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5843 "not free temporary device flow"); 5844 return NULL; 5845 } 5846 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 5847 &handle_idx); 5848 if (!dev_handle) { 5849 rte_flow_error_set(error, ENOMEM, 5850 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5851 "not enough memory to create flow handle"); 5852 return NULL; 5853 } 5854 /* No multi-thread supporting. */ 5855 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; 5856 dev_flow->handle = dev_handle; 5857 dev_flow->handle_idx = handle_idx; 5858 /* 5859 * In some old rdma-core releases, before continuing, a check of the 5860 * length of matching parameter will be done at first. It needs to use 5861 * the length without misc4 param. If the flow has misc4 support, then 5862 * the length needs to be adjusted accordingly. Each param member is 5863 * aligned with a 64B boundary naturally. 5864 */ 5865 dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) - 5866 MLX5_ST_SZ_BYTES(fte_match_set_misc4); 5867 /* 5868 * The matching value needs to be cleared to 0 before using. In the 5869 * past, it will be automatically cleared when using rte_*alloc 5870 * API. The time consumption will be almost the same as before. 5871 */ 5872 memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param)); 5873 dev_flow->ingress = attr->ingress; 5874 dev_flow->dv.transfer = attr->transfer; 5875 return dev_flow; 5876 } 5877 5878 #ifdef RTE_LIBRTE_MLX5_DEBUG 5879 /** 5880 * Sanity check for match mask and value. Similar to check_valid_spec() in 5881 * kernel driver. If unmasked bit is present in value, it returns failure. 5882 * 5883 * @param match_mask 5884 * pointer to match mask buffer. 5885 * @param match_value 5886 * pointer to match value buffer. 5887 * 5888 * @return 5889 * 0 if valid, -EINVAL otherwise. 5890 */ 5891 static int 5892 flow_dv_check_valid_spec(void *match_mask, void *match_value) 5893 { 5894 uint8_t *m = match_mask; 5895 uint8_t *v = match_value; 5896 unsigned int i; 5897 5898 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) { 5899 if (v[i] & ~m[i]) { 5900 DRV_LOG(ERR, 5901 "match_value differs from match_criteria" 5902 " %p[%u] != %p[%u]", 5903 match_value, i, match_mask, i); 5904 return -EINVAL; 5905 } 5906 } 5907 return 0; 5908 } 5909 #endif 5910 5911 /** 5912 * Add match of ip_version. 5913 * 5914 * @param[in] group 5915 * Flow group. 5916 * @param[in] headers_v 5917 * Values header pointer. 5918 * @param[in] headers_m 5919 * Masks header pointer. 5920 * @param[in] ip_version 5921 * The IP version to set. 5922 */ 5923 static inline void 5924 flow_dv_set_match_ip_version(uint32_t group, 5925 void *headers_v, 5926 void *headers_m, 5927 uint8_t ip_version) 5928 { 5929 if (group == 0) 5930 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); 5931 else 5932 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 5933 ip_version); 5934 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version); 5935 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0); 5936 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0); 5937 } 5938 5939 /** 5940 * Add Ethernet item to matcher and to the value. 5941 * 5942 * @param[in, out] matcher 5943 * Flow matcher. 5944 * @param[in, out] key 5945 * Flow matcher value. 5946 * @param[in] item 5947 * Flow pattern to translate. 5948 * @param[in] inner 5949 * Item is inner pattern. 5950 */ 5951 static void 5952 flow_dv_translate_item_eth(void *matcher, void *key, 5953 const struct rte_flow_item *item, int inner, 5954 uint32_t group) 5955 { 5956 const struct rte_flow_item_eth *eth_m = item->mask; 5957 const struct rte_flow_item_eth *eth_v = item->spec; 5958 const struct rte_flow_item_eth nic_mask = { 5959 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 5960 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 5961 .type = RTE_BE16(0xffff), 5962 }; 5963 void *headers_m; 5964 void *headers_v; 5965 char *l24_v; 5966 unsigned int i; 5967 5968 if (!eth_v) 5969 return; 5970 if (!eth_m) 5971 eth_m = &nic_mask; 5972 if (inner) { 5973 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 5974 inner_headers); 5975 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 5976 } else { 5977 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 5978 outer_headers); 5979 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 5980 } 5981 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16), 5982 ð_m->dst, sizeof(eth_m->dst)); 5983 /* The value must be in the range of the mask. */ 5984 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16); 5985 for (i = 0; i < sizeof(eth_m->dst); ++i) 5986 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i]; 5987 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16), 5988 ð_m->src, sizeof(eth_m->src)); 5989 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16); 5990 /* The value must be in the range of the mask. */ 5991 for (i = 0; i < sizeof(eth_m->dst); ++i) 5992 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i]; 5993 if (eth_v->type) { 5994 /* When ethertype is present set mask for tagged VLAN. */ 5995 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 5996 /* Set value for tagged VLAN if ethertype is 802.1Q. */ 5997 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) || 5998 eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) { 5999 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 6000 1); 6001 /* Return here to avoid setting match on ethertype. */ 6002 return; 6003 } 6004 } 6005 /* 6006 * HW supports match on one Ethertype, the Ethertype following the last 6007 * VLAN tag of the packet (see PRM). 6008 * Set match on ethertype only if ETH header is not followed by VLAN. 6009 * HW is optimized for IPv4/IPv6. In such cases, avoid setting 6010 * ethertype, and use ip_version field instead. 6011 * eCPRI over Ether layer will use type value 0xAEFE. 6012 */ 6013 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) && 6014 eth_m->type == 0xFFFF) { 6015 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 6016 } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) && 6017 eth_m->type == 0xFFFF) { 6018 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 6019 } else { 6020 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 6021 rte_be_to_cpu_16(eth_m->type)); 6022 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6023 ethertype); 6024 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type; 6025 } 6026 } 6027 6028 /** 6029 * Add VLAN item to matcher and to the value. 6030 * 6031 * @param[in, out] dev_flow 6032 * Flow descriptor. 6033 * @param[in, out] matcher 6034 * Flow matcher. 6035 * @param[in, out] key 6036 * Flow matcher value. 6037 * @param[in] item 6038 * Flow pattern to translate. 6039 * @param[in] inner 6040 * Item is inner pattern. 6041 */ 6042 static void 6043 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, 6044 void *matcher, void *key, 6045 const struct rte_flow_item *item, 6046 int inner, uint32_t group) 6047 { 6048 const struct rte_flow_item_vlan *vlan_m = item->mask; 6049 const struct rte_flow_item_vlan *vlan_v = item->spec; 6050 void *headers_m; 6051 void *headers_v; 6052 uint16_t tci_m; 6053 uint16_t tci_v; 6054 6055 if (inner) { 6056 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6057 inner_headers); 6058 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6059 } else { 6060 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6061 outer_headers); 6062 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6063 /* 6064 * This is workaround, masks are not supported, 6065 * and pre-validated. 6066 */ 6067 if (vlan_v) 6068 dev_flow->handle->vf_vlan.tag = 6069 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff; 6070 } 6071 /* 6072 * When VLAN item exists in flow, mark packet as tagged, 6073 * even if TCI is not specified. 6074 */ 6075 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6076 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 6077 if (!vlan_v) 6078 return; 6079 if (!vlan_m) 6080 vlan_m = &rte_flow_item_vlan_mask; 6081 tci_m = rte_be_to_cpu_16(vlan_m->tci); 6082 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci); 6083 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m); 6084 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v); 6085 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12); 6086 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12); 6087 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13); 6088 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13); 6089 /* 6090 * HW is optimized for IPv4/IPv6. In such cases, avoid setting 6091 * ethertype, and use ip_version field instead. 6092 */ 6093 if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) && 6094 vlan_m->inner_type == 0xFFFF) { 6095 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 6096 } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) && 6097 vlan_m->inner_type == 0xFFFF) { 6098 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 6099 } else { 6100 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 6101 rte_be_to_cpu_16(vlan_m->inner_type)); 6102 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 6103 rte_be_to_cpu_16(vlan_m->inner_type & 6104 vlan_v->inner_type)); 6105 } 6106 } 6107 6108 /** 6109 * Add IPV4 item to matcher and to the value. 6110 * 6111 * @param[in, out] matcher 6112 * Flow matcher. 6113 * @param[in, out] key 6114 * Flow matcher value. 6115 * @param[in] item 6116 * Flow pattern to translate. 6117 * @param[in] item_flags 6118 * Bit-fields that holds the items detected until now. 6119 * @param[in] inner 6120 * Item is inner pattern. 6121 * @param[in] group 6122 * The group to insert the rule. 6123 */ 6124 static void 6125 flow_dv_translate_item_ipv4(void *matcher, void *key, 6126 const struct rte_flow_item *item, 6127 const uint64_t item_flags, 6128 int inner, uint32_t group) 6129 { 6130 const struct rte_flow_item_ipv4 *ipv4_m = item->mask; 6131 const struct rte_flow_item_ipv4 *ipv4_v = item->spec; 6132 const struct rte_flow_item_ipv4 nic_mask = { 6133 .hdr = { 6134 .src_addr = RTE_BE32(0xffffffff), 6135 .dst_addr = RTE_BE32(0xffffffff), 6136 .type_of_service = 0xff, 6137 .next_proto_id = 0xff, 6138 .time_to_live = 0xff, 6139 }, 6140 }; 6141 void *headers_m; 6142 void *headers_v; 6143 char *l24_m; 6144 char *l24_v; 6145 uint8_t tos; 6146 6147 if (inner) { 6148 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6149 inner_headers); 6150 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6151 } else { 6152 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6153 outer_headers); 6154 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6155 } 6156 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 6157 /* 6158 * On outer header (which must contains L2), or inner header with L2, 6159 * set cvlan_tag mask bit to mark this packet as untagged. 6160 * This should be done even if item->spec is empty. 6161 */ 6162 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) 6163 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6164 if (!ipv4_v) 6165 return; 6166 if (!ipv4_m) 6167 ipv4_m = &nic_mask; 6168 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6169 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 6170 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6171 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 6172 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr; 6173 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr; 6174 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6175 src_ipv4_src_ipv6.ipv4_layout.ipv4); 6176 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6177 src_ipv4_src_ipv6.ipv4_layout.ipv4); 6178 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr; 6179 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr; 6180 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service; 6181 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, 6182 ipv4_m->hdr.type_of_service); 6183 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos); 6184 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, 6185 ipv4_m->hdr.type_of_service >> 2); 6186 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2); 6187 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 6188 ipv4_m->hdr.next_proto_id); 6189 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6190 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id); 6191 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, 6192 ipv4_m->hdr.time_to_live); 6193 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, 6194 ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live); 6195 } 6196 6197 /** 6198 * Add IPV6 item to matcher and to the value. 6199 * 6200 * @param[in, out] matcher 6201 * Flow matcher. 6202 * @param[in, out] key 6203 * Flow matcher value. 6204 * @param[in] item 6205 * Flow pattern to translate. 6206 * @param[in] item_flags 6207 * Bit-fields that holds the items detected until now. 6208 * @param[in] inner 6209 * Item is inner pattern. 6210 * @param[in] group 6211 * The group to insert the rule. 6212 */ 6213 static void 6214 flow_dv_translate_item_ipv6(void *matcher, void *key, 6215 const struct rte_flow_item *item, 6216 const uint64_t item_flags, 6217 int inner, uint32_t group) 6218 { 6219 const struct rte_flow_item_ipv6 *ipv6_m = item->mask; 6220 const struct rte_flow_item_ipv6 *ipv6_v = item->spec; 6221 const struct rte_flow_item_ipv6 nic_mask = { 6222 .hdr = { 6223 .src_addr = 6224 "\xff\xff\xff\xff\xff\xff\xff\xff" 6225 "\xff\xff\xff\xff\xff\xff\xff\xff", 6226 .dst_addr = 6227 "\xff\xff\xff\xff\xff\xff\xff\xff" 6228 "\xff\xff\xff\xff\xff\xff\xff\xff", 6229 .vtc_flow = RTE_BE32(0xffffffff), 6230 .proto = 0xff, 6231 .hop_limits = 0xff, 6232 }, 6233 }; 6234 void *headers_m; 6235 void *headers_v; 6236 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6237 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6238 char *l24_m; 6239 char *l24_v; 6240 uint32_t vtc_m; 6241 uint32_t vtc_v; 6242 int i; 6243 int size; 6244 6245 if (inner) { 6246 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6247 inner_headers); 6248 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6249 } else { 6250 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6251 outer_headers); 6252 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6253 } 6254 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 6255 /* 6256 * On outer header (which must contains L2), or inner header with L2, 6257 * set cvlan_tag mask bit to mark this packet as untagged. 6258 * This should be done even if item->spec is empty. 6259 */ 6260 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) 6261 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6262 if (!ipv6_v) 6263 return; 6264 if (!ipv6_m) 6265 ipv6_m = &nic_mask; 6266 size = sizeof(ipv6_m->hdr.dst_addr); 6267 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6268 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 6269 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6270 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 6271 memcpy(l24_m, ipv6_m->hdr.dst_addr, size); 6272 for (i = 0; i < size; ++i) 6273 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i]; 6274 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6275 src_ipv4_src_ipv6.ipv6_layout.ipv6); 6276 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6277 src_ipv4_src_ipv6.ipv6_layout.ipv6); 6278 memcpy(l24_m, ipv6_m->hdr.src_addr, size); 6279 for (i = 0; i < size; ++i) 6280 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i]; 6281 /* TOS. */ 6282 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow); 6283 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow); 6284 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20); 6285 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20); 6286 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22); 6287 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22); 6288 /* Label. */ 6289 if (inner) { 6290 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label, 6291 vtc_m); 6292 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label, 6293 vtc_v); 6294 } else { 6295 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label, 6296 vtc_m); 6297 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label, 6298 vtc_v); 6299 } 6300 /* Protocol. */ 6301 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 6302 ipv6_m->hdr.proto); 6303 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6304 ipv6_v->hdr.proto & ipv6_m->hdr.proto); 6305 /* Hop limit. */ 6306 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, 6307 ipv6_m->hdr.hop_limits); 6308 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, 6309 ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits); 6310 } 6311 6312 /** 6313 * Add TCP item to matcher and to the value. 6314 * 6315 * @param[in, out] matcher 6316 * Flow matcher. 6317 * @param[in, out] key 6318 * Flow matcher value. 6319 * @param[in] item 6320 * Flow pattern to translate. 6321 * @param[in] inner 6322 * Item is inner pattern. 6323 */ 6324 static void 6325 flow_dv_translate_item_tcp(void *matcher, void *key, 6326 const struct rte_flow_item *item, 6327 int inner) 6328 { 6329 const struct rte_flow_item_tcp *tcp_m = item->mask; 6330 const struct rte_flow_item_tcp *tcp_v = item->spec; 6331 void *headers_m; 6332 void *headers_v; 6333 6334 if (inner) { 6335 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6336 inner_headers); 6337 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6338 } else { 6339 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6340 outer_headers); 6341 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6342 } 6343 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6344 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP); 6345 if (!tcp_v) 6346 return; 6347 if (!tcp_m) 6348 tcp_m = &rte_flow_item_tcp_mask; 6349 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport, 6350 rte_be_to_cpu_16(tcp_m->hdr.src_port)); 6351 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 6352 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port)); 6353 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport, 6354 rte_be_to_cpu_16(tcp_m->hdr.dst_port)); 6355 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 6356 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port)); 6357 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags, 6358 tcp_m->hdr.tcp_flags); 6359 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags, 6360 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags)); 6361 } 6362 6363 /** 6364 * Add UDP item to matcher and to the value. 6365 * 6366 * @param[in, out] matcher 6367 * Flow matcher. 6368 * @param[in, out] key 6369 * Flow matcher value. 6370 * @param[in] item 6371 * Flow pattern to translate. 6372 * @param[in] inner 6373 * Item is inner pattern. 6374 */ 6375 static void 6376 flow_dv_translate_item_udp(void *matcher, void *key, 6377 const struct rte_flow_item *item, 6378 int inner) 6379 { 6380 const struct rte_flow_item_udp *udp_m = item->mask; 6381 const struct rte_flow_item_udp *udp_v = item->spec; 6382 void *headers_m; 6383 void *headers_v; 6384 6385 if (inner) { 6386 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6387 inner_headers); 6388 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6389 } else { 6390 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6391 outer_headers); 6392 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6393 } 6394 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6395 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 6396 if (!udp_v) 6397 return; 6398 if (!udp_m) 6399 udp_m = &rte_flow_item_udp_mask; 6400 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport, 6401 rte_be_to_cpu_16(udp_m->hdr.src_port)); 6402 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 6403 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port)); 6404 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 6405 rte_be_to_cpu_16(udp_m->hdr.dst_port)); 6406 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 6407 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port)); 6408 } 6409 6410 /** 6411 * Add GRE optional Key item to matcher and to the value. 6412 * 6413 * @param[in, out] matcher 6414 * Flow matcher. 6415 * @param[in, out] key 6416 * Flow matcher value. 6417 * @param[in] item 6418 * Flow pattern to translate. 6419 * @param[in] inner 6420 * Item is inner pattern. 6421 */ 6422 static void 6423 flow_dv_translate_item_gre_key(void *matcher, void *key, 6424 const struct rte_flow_item *item) 6425 { 6426 const rte_be32_t *key_m = item->mask; 6427 const rte_be32_t *key_v = item->spec; 6428 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6429 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6430 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 6431 6432 /* GRE K bit must be on and should already be validated */ 6433 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1); 6434 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1); 6435 if (!key_v) 6436 return; 6437 if (!key_m) 6438 key_m = &gre_key_default_mask; 6439 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h, 6440 rte_be_to_cpu_32(*key_m) >> 8); 6441 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h, 6442 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8); 6443 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l, 6444 rte_be_to_cpu_32(*key_m) & 0xFF); 6445 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l, 6446 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF); 6447 } 6448 6449 /** 6450 * Add GRE item to matcher and to the value. 6451 * 6452 * @param[in, out] matcher 6453 * Flow matcher. 6454 * @param[in, out] key 6455 * Flow matcher value. 6456 * @param[in] item 6457 * Flow pattern to translate. 6458 * @param[in] inner 6459 * Item is inner pattern. 6460 */ 6461 static void 6462 flow_dv_translate_item_gre(void *matcher, void *key, 6463 const struct rte_flow_item *item, 6464 int inner) 6465 { 6466 const struct rte_flow_item_gre *gre_m = item->mask; 6467 const struct rte_flow_item_gre *gre_v = item->spec; 6468 void *headers_m; 6469 void *headers_v; 6470 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6471 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6472 struct { 6473 union { 6474 __extension__ 6475 struct { 6476 uint16_t version:3; 6477 uint16_t rsvd0:9; 6478 uint16_t s_present:1; 6479 uint16_t k_present:1; 6480 uint16_t rsvd_bit1:1; 6481 uint16_t c_present:1; 6482 }; 6483 uint16_t value; 6484 }; 6485 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v; 6486 6487 if (inner) { 6488 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6489 inner_headers); 6490 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6491 } else { 6492 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6493 outer_headers); 6494 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6495 } 6496 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6497 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE); 6498 if (!gre_v) 6499 return; 6500 if (!gre_m) 6501 gre_m = &rte_flow_item_gre_mask; 6502 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 6503 rte_be_to_cpu_16(gre_m->protocol)); 6504 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 6505 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol)); 6506 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver); 6507 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver); 6508 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present, 6509 gre_crks_rsvd0_ver_m.c_present); 6510 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present, 6511 gre_crks_rsvd0_ver_v.c_present & 6512 gre_crks_rsvd0_ver_m.c_present); 6513 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 6514 gre_crks_rsvd0_ver_m.k_present); 6515 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 6516 gre_crks_rsvd0_ver_v.k_present & 6517 gre_crks_rsvd0_ver_m.k_present); 6518 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present, 6519 gre_crks_rsvd0_ver_m.s_present); 6520 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present, 6521 gre_crks_rsvd0_ver_v.s_present & 6522 gre_crks_rsvd0_ver_m.s_present); 6523 } 6524 6525 /** 6526 * Add NVGRE item to matcher and to the value. 6527 * 6528 * @param[in, out] matcher 6529 * Flow matcher. 6530 * @param[in, out] key 6531 * Flow matcher value. 6532 * @param[in] item 6533 * Flow pattern to translate. 6534 * @param[in] inner 6535 * Item is inner pattern. 6536 */ 6537 static void 6538 flow_dv_translate_item_nvgre(void *matcher, void *key, 6539 const struct rte_flow_item *item, 6540 int inner) 6541 { 6542 const struct rte_flow_item_nvgre *nvgre_m = item->mask; 6543 const struct rte_flow_item_nvgre *nvgre_v = item->spec; 6544 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6545 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6546 const char *tni_flow_id_m; 6547 const char *tni_flow_id_v; 6548 char *gre_key_m; 6549 char *gre_key_v; 6550 int size; 6551 int i; 6552 6553 /* For NVGRE, GRE header fields must be set with defined values. */ 6554 const struct rte_flow_item_gre gre_spec = { 6555 .c_rsvd0_ver = RTE_BE16(0x2000), 6556 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB) 6557 }; 6558 const struct rte_flow_item_gre gre_mask = { 6559 .c_rsvd0_ver = RTE_BE16(0xB000), 6560 .protocol = RTE_BE16(UINT16_MAX), 6561 }; 6562 const struct rte_flow_item gre_item = { 6563 .spec = &gre_spec, 6564 .mask = &gre_mask, 6565 .last = NULL, 6566 }; 6567 flow_dv_translate_item_gre(matcher, key, &gre_item, inner); 6568 if (!nvgre_v) 6569 return; 6570 if (!nvgre_m) 6571 nvgre_m = &rte_flow_item_nvgre_mask; 6572 tni_flow_id_m = (const char *)nvgre_m->tni; 6573 tni_flow_id_v = (const char *)nvgre_v->tni; 6574 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id); 6575 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h); 6576 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h); 6577 memcpy(gre_key_m, tni_flow_id_m, size); 6578 for (i = 0; i < size; ++i) 6579 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i]; 6580 } 6581 6582 /** 6583 * Add VXLAN item to matcher and to the value. 6584 * 6585 * @param[in, out] matcher 6586 * Flow matcher. 6587 * @param[in, out] key 6588 * Flow matcher value. 6589 * @param[in] item 6590 * Flow pattern to translate. 6591 * @param[in] inner 6592 * Item is inner pattern. 6593 */ 6594 static void 6595 flow_dv_translate_item_vxlan(void *matcher, void *key, 6596 const struct rte_flow_item *item, 6597 int inner) 6598 { 6599 const struct rte_flow_item_vxlan *vxlan_m = item->mask; 6600 const struct rte_flow_item_vxlan *vxlan_v = item->spec; 6601 void *headers_m; 6602 void *headers_v; 6603 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6604 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6605 char *vni_m; 6606 char *vni_v; 6607 uint16_t dport; 6608 int size; 6609 int i; 6610 6611 if (inner) { 6612 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6613 inner_headers); 6614 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6615 } else { 6616 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6617 outer_headers); 6618 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6619 } 6620 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 6621 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 6622 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6623 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6624 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6625 } 6626 if (!vxlan_v) 6627 return; 6628 if (!vxlan_m) 6629 vxlan_m = &rte_flow_item_vxlan_mask; 6630 size = sizeof(vxlan_m->vni); 6631 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni); 6632 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni); 6633 memcpy(vni_m, vxlan_m->vni, size); 6634 for (i = 0; i < size; ++i) 6635 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 6636 } 6637 6638 /** 6639 * Add VXLAN-GPE item to matcher and to the value. 6640 * 6641 * @param[in, out] matcher 6642 * Flow matcher. 6643 * @param[in, out] key 6644 * Flow matcher value. 6645 * @param[in] item 6646 * Flow pattern to translate. 6647 * @param[in] inner 6648 * Item is inner pattern. 6649 */ 6650 6651 static void 6652 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key, 6653 const struct rte_flow_item *item, int inner) 6654 { 6655 const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask; 6656 const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec; 6657 void *headers_m; 6658 void *headers_v; 6659 void *misc_m = 6660 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3); 6661 void *misc_v = 6662 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 6663 char *vni_m; 6664 char *vni_v; 6665 uint16_t dport; 6666 int size; 6667 int i; 6668 uint8_t flags_m = 0xff; 6669 uint8_t flags_v = 0xc; 6670 6671 if (inner) { 6672 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6673 inner_headers); 6674 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6675 } else { 6676 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6677 outer_headers); 6678 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6679 } 6680 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 6681 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 6682 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6683 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6684 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6685 } 6686 if (!vxlan_v) 6687 return; 6688 if (!vxlan_m) 6689 vxlan_m = &rte_flow_item_vxlan_gpe_mask; 6690 size = sizeof(vxlan_m->vni); 6691 vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni); 6692 vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni); 6693 memcpy(vni_m, vxlan_m->vni, size); 6694 for (i = 0; i < size; ++i) 6695 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 6696 if (vxlan_m->flags) { 6697 flags_m = vxlan_m->flags; 6698 flags_v = vxlan_v->flags; 6699 } 6700 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m); 6701 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v); 6702 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol, 6703 vxlan_m->protocol); 6704 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol, 6705 vxlan_v->protocol); 6706 } 6707 6708 /** 6709 * Add Geneve item to matcher and to the value. 6710 * 6711 * @param[in, out] matcher 6712 * Flow matcher. 6713 * @param[in, out] key 6714 * Flow matcher value. 6715 * @param[in] item 6716 * Flow pattern to translate. 6717 * @param[in] inner 6718 * Item is inner pattern. 6719 */ 6720 6721 static void 6722 flow_dv_translate_item_geneve(void *matcher, void *key, 6723 const struct rte_flow_item *item, int inner) 6724 { 6725 const struct rte_flow_item_geneve *geneve_m = item->mask; 6726 const struct rte_flow_item_geneve *geneve_v = item->spec; 6727 void *headers_m; 6728 void *headers_v; 6729 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6730 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6731 uint16_t dport; 6732 uint16_t gbhdr_m; 6733 uint16_t gbhdr_v; 6734 char *vni_m; 6735 char *vni_v; 6736 size_t size, i; 6737 6738 if (inner) { 6739 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6740 inner_headers); 6741 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6742 } else { 6743 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6744 outer_headers); 6745 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6746 } 6747 dport = MLX5_UDP_PORT_GENEVE; 6748 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6749 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6750 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6751 } 6752 if (!geneve_v) 6753 return; 6754 if (!geneve_m) 6755 geneve_m = &rte_flow_item_geneve_mask; 6756 size = sizeof(geneve_m->vni); 6757 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni); 6758 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni); 6759 memcpy(vni_m, geneve_m->vni, size); 6760 for (i = 0; i < size; ++i) 6761 vni_v[i] = vni_m[i] & geneve_v->vni[i]; 6762 MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type, 6763 rte_be_to_cpu_16(geneve_m->protocol)); 6764 MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type, 6765 rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol)); 6766 gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0); 6767 gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0); 6768 MLX5_SET(fte_match_set_misc, misc_m, geneve_oam, 6769 MLX5_GENEVE_OAMF_VAL(gbhdr_m)); 6770 MLX5_SET(fte_match_set_misc, misc_v, geneve_oam, 6771 MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m)); 6772 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len, 6773 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); 6774 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, 6775 MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) & 6776 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); 6777 } 6778 6779 /** 6780 * Add MPLS item to matcher and to the value. 6781 * 6782 * @param[in, out] matcher 6783 * Flow matcher. 6784 * @param[in, out] key 6785 * Flow matcher value. 6786 * @param[in] item 6787 * Flow pattern to translate. 6788 * @param[in] prev_layer 6789 * The protocol layer indicated in previous item. 6790 * @param[in] inner 6791 * Item is inner pattern. 6792 */ 6793 static void 6794 flow_dv_translate_item_mpls(void *matcher, void *key, 6795 const struct rte_flow_item *item, 6796 uint64_t prev_layer, 6797 int inner) 6798 { 6799 const uint32_t *in_mpls_m = item->mask; 6800 const uint32_t *in_mpls_v = item->spec; 6801 uint32_t *out_mpls_m = 0; 6802 uint32_t *out_mpls_v = 0; 6803 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6804 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6805 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher, 6806 misc_parameters_2); 6807 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 6808 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 6809 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6810 6811 switch (prev_layer) { 6812 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 6813 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff); 6814 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 6815 MLX5_UDP_PORT_MPLS); 6816 break; 6817 case MLX5_FLOW_LAYER_GRE: 6818 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff); 6819 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 6820 RTE_ETHER_TYPE_MPLS); 6821 break; 6822 default: 6823 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6824 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6825 IPPROTO_MPLS); 6826 break; 6827 } 6828 if (!in_mpls_v) 6829 return; 6830 if (!in_mpls_m) 6831 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask; 6832 switch (prev_layer) { 6833 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 6834 out_mpls_m = 6835 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 6836 outer_first_mpls_over_udp); 6837 out_mpls_v = 6838 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 6839 outer_first_mpls_over_udp); 6840 break; 6841 case MLX5_FLOW_LAYER_GRE: 6842 out_mpls_m = 6843 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 6844 outer_first_mpls_over_gre); 6845 out_mpls_v = 6846 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 6847 outer_first_mpls_over_gre); 6848 break; 6849 default: 6850 /* Inner MPLS not over GRE is not supported. */ 6851 if (!inner) { 6852 out_mpls_m = 6853 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 6854 misc2_m, 6855 outer_first_mpls); 6856 out_mpls_v = 6857 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 6858 misc2_v, 6859 outer_first_mpls); 6860 } 6861 break; 6862 } 6863 if (out_mpls_m && out_mpls_v) { 6864 *out_mpls_m = *in_mpls_m; 6865 *out_mpls_v = *in_mpls_v & *in_mpls_m; 6866 } 6867 } 6868 6869 /** 6870 * Add metadata register item to matcher 6871 * 6872 * @param[in, out] matcher 6873 * Flow matcher. 6874 * @param[in, out] key 6875 * Flow matcher value. 6876 * @param[in] reg_type 6877 * Type of device metadata register 6878 * @param[in] value 6879 * Register value 6880 * @param[in] mask 6881 * Register mask 6882 */ 6883 static void 6884 flow_dv_match_meta_reg(void *matcher, void *key, 6885 enum modify_reg reg_type, 6886 uint32_t data, uint32_t mask) 6887 { 6888 void *misc2_m = 6889 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); 6890 void *misc2_v = 6891 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 6892 uint32_t temp; 6893 6894 data &= mask; 6895 switch (reg_type) { 6896 case REG_A: 6897 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask); 6898 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data); 6899 break; 6900 case REG_B: 6901 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask); 6902 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data); 6903 break; 6904 case REG_C_0: 6905 /* 6906 * The metadata register C0 field might be divided into 6907 * source vport index and META item value, we should set 6908 * this field according to specified mask, not as whole one. 6909 */ 6910 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0); 6911 temp |= mask; 6912 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp); 6913 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0); 6914 temp &= ~mask; 6915 temp |= data; 6916 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp); 6917 break; 6918 case REG_C_1: 6919 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask); 6920 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data); 6921 break; 6922 case REG_C_2: 6923 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask); 6924 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data); 6925 break; 6926 case REG_C_3: 6927 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask); 6928 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data); 6929 break; 6930 case REG_C_4: 6931 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask); 6932 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data); 6933 break; 6934 case REG_C_5: 6935 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask); 6936 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data); 6937 break; 6938 case REG_C_6: 6939 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask); 6940 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data); 6941 break; 6942 case REG_C_7: 6943 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask); 6944 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data); 6945 break; 6946 default: 6947 MLX5_ASSERT(false); 6948 break; 6949 } 6950 } 6951 6952 /** 6953 * Add MARK item to matcher 6954 * 6955 * @param[in] dev 6956 * The device to configure through. 6957 * @param[in, out] matcher 6958 * Flow matcher. 6959 * @param[in, out] key 6960 * Flow matcher value. 6961 * @param[in] item 6962 * Flow pattern to translate. 6963 */ 6964 static void 6965 flow_dv_translate_item_mark(struct rte_eth_dev *dev, 6966 void *matcher, void *key, 6967 const struct rte_flow_item *item) 6968 { 6969 struct mlx5_priv *priv = dev->data->dev_private; 6970 const struct rte_flow_item_mark *mark; 6971 uint32_t value; 6972 uint32_t mask; 6973 6974 mark = item->mask ? (const void *)item->mask : 6975 &rte_flow_item_mark_mask; 6976 mask = mark->id & priv->sh->dv_mark_mask; 6977 mark = (const void *)item->spec; 6978 MLX5_ASSERT(mark); 6979 value = mark->id & priv->sh->dv_mark_mask & mask; 6980 if (mask) { 6981 enum modify_reg reg; 6982 6983 /* Get the metadata register index for the mark. */ 6984 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL); 6985 MLX5_ASSERT(reg > 0); 6986 if (reg == REG_C_0) { 6987 struct mlx5_priv *priv = dev->data->dev_private; 6988 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 6989 uint32_t shl_c0 = rte_bsf32(msk_c0); 6990 6991 mask &= msk_c0; 6992 mask <<= shl_c0; 6993 value <<= shl_c0; 6994 } 6995 flow_dv_match_meta_reg(matcher, key, reg, value, mask); 6996 } 6997 } 6998 6999 /** 7000 * Add META item to matcher 7001 * 7002 * @param[in] dev 7003 * The devich to configure through. 7004 * @param[in, out] matcher 7005 * Flow matcher. 7006 * @param[in, out] key 7007 * Flow matcher value. 7008 * @param[in] attr 7009 * Attributes of flow that includes this item. 7010 * @param[in] item 7011 * Flow pattern to translate. 7012 */ 7013 static void 7014 flow_dv_translate_item_meta(struct rte_eth_dev *dev, 7015 void *matcher, void *key, 7016 const struct rte_flow_attr *attr, 7017 const struct rte_flow_item *item) 7018 { 7019 const struct rte_flow_item_meta *meta_m; 7020 const struct rte_flow_item_meta *meta_v; 7021 7022 meta_m = (const void *)item->mask; 7023 if (!meta_m) 7024 meta_m = &rte_flow_item_meta_mask; 7025 meta_v = (const void *)item->spec; 7026 if (meta_v) { 7027 int reg; 7028 uint32_t value = meta_v->data; 7029 uint32_t mask = meta_m->data; 7030 7031 reg = flow_dv_get_metadata_reg(dev, attr, NULL); 7032 if (reg < 0) 7033 return; 7034 /* 7035 * In datapath code there is no endianness 7036 * coversions for perfromance reasons, all 7037 * pattern conversions are done in rte_flow. 7038 */ 7039 value = rte_cpu_to_be_32(value); 7040 mask = rte_cpu_to_be_32(mask); 7041 if (reg == REG_C_0) { 7042 struct mlx5_priv *priv = dev->data->dev_private; 7043 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 7044 uint32_t shl_c0 = rte_bsf32(msk_c0); 7045 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 7046 uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask); 7047 7048 value >>= shr_c0; 7049 mask >>= shr_c0; 7050 #endif 7051 value <<= shl_c0; 7052 mask <<= shl_c0; 7053 MLX5_ASSERT(msk_c0); 7054 MLX5_ASSERT(!(~msk_c0 & mask)); 7055 } 7056 flow_dv_match_meta_reg(matcher, key, reg, value, mask); 7057 } 7058 } 7059 7060 /** 7061 * Add vport metadata Reg C0 item to matcher 7062 * 7063 * @param[in, out] matcher 7064 * Flow matcher. 7065 * @param[in, out] key 7066 * Flow matcher value. 7067 * @param[in] reg 7068 * Flow pattern to translate. 7069 */ 7070 static void 7071 flow_dv_translate_item_meta_vport(void *matcher, void *key, 7072 uint32_t value, uint32_t mask) 7073 { 7074 flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask); 7075 } 7076 7077 /** 7078 * Add tag item to matcher 7079 * 7080 * @param[in] dev 7081 * The devich to configure through. 7082 * @param[in, out] matcher 7083 * Flow matcher. 7084 * @param[in, out] key 7085 * Flow matcher value. 7086 * @param[in] item 7087 * Flow pattern to translate. 7088 */ 7089 static void 7090 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev, 7091 void *matcher, void *key, 7092 const struct rte_flow_item *item) 7093 { 7094 const struct mlx5_rte_flow_item_tag *tag_v = item->spec; 7095 const struct mlx5_rte_flow_item_tag *tag_m = item->mask; 7096 uint32_t mask, value; 7097 7098 MLX5_ASSERT(tag_v); 7099 value = tag_v->data; 7100 mask = tag_m ? tag_m->data : UINT32_MAX; 7101 if (tag_v->id == REG_C_0) { 7102 struct mlx5_priv *priv = dev->data->dev_private; 7103 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 7104 uint32_t shl_c0 = rte_bsf32(msk_c0); 7105 7106 mask &= msk_c0; 7107 mask <<= shl_c0; 7108 value <<= shl_c0; 7109 } 7110 flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask); 7111 } 7112 7113 /** 7114 * Add TAG item to matcher 7115 * 7116 * @param[in] dev 7117 * The devich to configure through. 7118 * @param[in, out] matcher 7119 * Flow matcher. 7120 * @param[in, out] key 7121 * Flow matcher value. 7122 * @param[in] item 7123 * Flow pattern to translate. 7124 */ 7125 static void 7126 flow_dv_translate_item_tag(struct rte_eth_dev *dev, 7127 void *matcher, void *key, 7128 const struct rte_flow_item *item) 7129 { 7130 const struct rte_flow_item_tag *tag_v = item->spec; 7131 const struct rte_flow_item_tag *tag_m = item->mask; 7132 enum modify_reg reg; 7133 7134 MLX5_ASSERT(tag_v); 7135 tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask; 7136 /* Get the metadata register index for the tag. */ 7137 reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL); 7138 MLX5_ASSERT(reg > 0); 7139 flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data); 7140 } 7141 7142 /** 7143 * Add source vport match to the specified matcher. 7144 * 7145 * @param[in, out] matcher 7146 * Flow matcher. 7147 * @param[in, out] key 7148 * Flow matcher value. 7149 * @param[in] port 7150 * Source vport value to match 7151 * @param[in] mask 7152 * Mask 7153 */ 7154 static void 7155 flow_dv_translate_item_source_vport(void *matcher, void *key, 7156 int16_t port, uint16_t mask) 7157 { 7158 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 7159 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 7160 7161 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); 7162 MLX5_SET(fte_match_set_misc, misc_v, source_port, port); 7163 } 7164 7165 /** 7166 * Translate port-id item to eswitch match on port-id. 7167 * 7168 * @param[in] dev 7169 * The devich to configure through. 7170 * @param[in, out] matcher 7171 * Flow matcher. 7172 * @param[in, out] key 7173 * Flow matcher value. 7174 * @param[in] item 7175 * Flow pattern to translate. 7176 * 7177 * @return 7178 * 0 on success, a negative errno value otherwise. 7179 */ 7180 static int 7181 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, 7182 void *key, const struct rte_flow_item *item) 7183 { 7184 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; 7185 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; 7186 struct mlx5_priv *priv; 7187 uint16_t mask, id; 7188 7189 mask = pid_m ? pid_m->id : 0xffff; 7190 id = pid_v ? pid_v->id : dev->data->port_id; 7191 priv = mlx5_port_to_eswitch_info(id, item == NULL); 7192 if (!priv) 7193 return -rte_errno; 7194 /* Translate to vport field or to metadata, depending on mode. */ 7195 if (priv->vport_meta_mask) 7196 flow_dv_translate_item_meta_vport(matcher, key, 7197 priv->vport_meta_tag, 7198 priv->vport_meta_mask); 7199 else 7200 flow_dv_translate_item_source_vport(matcher, key, 7201 priv->vport_id, mask); 7202 return 0; 7203 } 7204 7205 /** 7206 * Add ICMP6 item to matcher and to the value. 7207 * 7208 * @param[in, out] matcher 7209 * Flow matcher. 7210 * @param[in, out] key 7211 * Flow matcher value. 7212 * @param[in] item 7213 * Flow pattern to translate. 7214 * @param[in] inner 7215 * Item is inner pattern. 7216 */ 7217 static void 7218 flow_dv_translate_item_icmp6(void *matcher, void *key, 7219 const struct rte_flow_item *item, 7220 int inner) 7221 { 7222 const struct rte_flow_item_icmp6 *icmp6_m = item->mask; 7223 const struct rte_flow_item_icmp6 *icmp6_v = item->spec; 7224 void *headers_m; 7225 void *headers_v; 7226 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7227 misc_parameters_3); 7228 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7229 if (inner) { 7230 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7231 inner_headers); 7232 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7233 } else { 7234 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7235 outer_headers); 7236 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7237 } 7238 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 7239 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6); 7240 if (!icmp6_v) 7241 return; 7242 if (!icmp6_m) 7243 icmp6_m = &rte_flow_item_icmp6_mask; 7244 /* 7245 * Force flow only to match the non-fragmented IPv6 ICMPv6 packets. 7246 * If only the protocol is specified, no need to match the frag. 7247 */ 7248 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); 7249 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 7250 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type); 7251 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type, 7252 icmp6_v->type & icmp6_m->type); 7253 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code); 7254 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code, 7255 icmp6_v->code & icmp6_m->code); 7256 } 7257 7258 /** 7259 * Add ICMP item to matcher and to the value. 7260 * 7261 * @param[in, out] matcher 7262 * Flow matcher. 7263 * @param[in, out] key 7264 * Flow matcher value. 7265 * @param[in] item 7266 * Flow pattern to translate. 7267 * @param[in] inner 7268 * Item is inner pattern. 7269 */ 7270 static void 7271 flow_dv_translate_item_icmp(void *matcher, void *key, 7272 const struct rte_flow_item *item, 7273 int inner) 7274 { 7275 const struct rte_flow_item_icmp *icmp_m = item->mask; 7276 const struct rte_flow_item_icmp *icmp_v = item->spec; 7277 void *headers_m; 7278 void *headers_v; 7279 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7280 misc_parameters_3); 7281 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7282 if (inner) { 7283 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7284 inner_headers); 7285 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7286 } else { 7287 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7288 outer_headers); 7289 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7290 } 7291 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 7292 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP); 7293 if (!icmp_v) 7294 return; 7295 if (!icmp_m) 7296 icmp_m = &rte_flow_item_icmp_mask; 7297 /* 7298 * Force flow only to match the non-fragmented IPv4 ICMP packets. 7299 * If only the protocol is specified, no need to match the frag. 7300 */ 7301 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); 7302 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 7303 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type, 7304 icmp_m->hdr.icmp_type); 7305 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type, 7306 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type); 7307 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code, 7308 icmp_m->hdr.icmp_code); 7309 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code, 7310 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code); 7311 } 7312 7313 /** 7314 * Add GTP item to matcher and to the value. 7315 * 7316 * @param[in, out] matcher 7317 * Flow matcher. 7318 * @param[in, out] key 7319 * Flow matcher value. 7320 * @param[in] item 7321 * Flow pattern to translate. 7322 * @param[in] inner 7323 * Item is inner pattern. 7324 */ 7325 static void 7326 flow_dv_translate_item_gtp(void *matcher, void *key, 7327 const struct rte_flow_item *item, int inner) 7328 { 7329 const struct rte_flow_item_gtp *gtp_m = item->mask; 7330 const struct rte_flow_item_gtp *gtp_v = item->spec; 7331 void *headers_m; 7332 void *headers_v; 7333 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7334 misc_parameters_3); 7335 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7336 uint16_t dport = RTE_GTPU_UDP_PORT; 7337 7338 if (inner) { 7339 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7340 inner_headers); 7341 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7342 } else { 7343 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7344 outer_headers); 7345 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7346 } 7347 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 7348 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 7349 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 7350 } 7351 if (!gtp_v) 7352 return; 7353 if (!gtp_m) 7354 gtp_m = &rte_flow_item_gtp_mask; 7355 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, 7356 gtp_m->v_pt_rsv_flags); 7357 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, 7358 gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags); 7359 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type); 7360 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type, 7361 gtp_v->msg_type & gtp_m->msg_type); 7362 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid, 7363 rte_be_to_cpu_32(gtp_m->teid)); 7364 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid, 7365 rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid)); 7366 } 7367 7368 /** 7369 * Add eCPRI item to matcher and to the value. 7370 * 7371 * @param[in] dev 7372 * The devich to configure through. 7373 * @param[in, out] matcher 7374 * Flow matcher. 7375 * @param[in, out] key 7376 * Flow matcher value. 7377 * @param[in] item 7378 * Flow pattern to translate. 7379 * @param[in] samples 7380 * Sample IDs to be used in the matching. 7381 */ 7382 static void 7383 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, 7384 void *key, const struct rte_flow_item *item) 7385 { 7386 struct mlx5_priv *priv = dev->data->dev_private; 7387 const struct rte_flow_item_ecpri *ecpri_m = item->mask; 7388 const struct rte_flow_item_ecpri *ecpri_v = item->spec; 7389 void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher, 7390 misc_parameters_4); 7391 void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4); 7392 uint32_t *samples; 7393 void *dw_m; 7394 void *dw_v; 7395 7396 if (!ecpri_v) 7397 return; 7398 if (!ecpri_m) 7399 ecpri_m = &rte_flow_item_ecpri_mask; 7400 /* 7401 * Maximal four DW samples are supported in a single matching now. 7402 * Two are used now for a eCPRI matching: 7403 * 1. Type: one byte, mask should be 0x00ff0000 in network order 7404 * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000 7405 * if any. 7406 */ 7407 if (!ecpri_m->hdr.common.u32) 7408 return; 7409 samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids; 7410 /* Need to take the whole DW as the mask to fill the entry. */ 7411 dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m, 7412 prog_sample_field_value_0); 7413 dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v, 7414 prog_sample_field_value_0); 7415 /* Already big endian (network order) in the header. */ 7416 *(uint32_t *)dw_m = ecpri_m->hdr.common.u32; 7417 *(uint32_t *)dw_v = ecpri_v->hdr.common.u32; 7418 /* Sample#0, used for matching type, offset 0. */ 7419 MLX5_SET(fte_match_set_misc4, misc4_m, 7420 prog_sample_field_id_0, samples[0]); 7421 /* It makes no sense to set the sample ID in the mask field. */ 7422 MLX5_SET(fte_match_set_misc4, misc4_v, 7423 prog_sample_field_id_0, samples[0]); 7424 /* 7425 * Checking if message body part needs to be matched. 7426 * Some wildcard rules only matching type field should be supported. 7427 */ 7428 if (ecpri_m->hdr.dummy[0]) { 7429 switch (ecpri_v->hdr.common.type) { 7430 case RTE_ECPRI_MSG_TYPE_IQ_DATA: 7431 case RTE_ECPRI_MSG_TYPE_RTC_CTRL: 7432 case RTE_ECPRI_MSG_TYPE_DLY_MSR: 7433 dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m, 7434 prog_sample_field_value_1); 7435 dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v, 7436 prog_sample_field_value_1); 7437 *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0]; 7438 *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0]; 7439 /* Sample#1, to match message body, offset 4. */ 7440 MLX5_SET(fte_match_set_misc4, misc4_m, 7441 prog_sample_field_id_1, samples[1]); 7442 MLX5_SET(fte_match_set_misc4, misc4_v, 7443 prog_sample_field_id_1, samples[1]); 7444 break; 7445 default: 7446 /* Others, do not match any sample ID. */ 7447 break; 7448 } 7449 } 7450 } 7451 7452 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; 7453 7454 #define HEADER_IS_ZERO(match_criteria, headers) \ 7455 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 7456 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 7457 7458 /** 7459 * Calculate flow matcher enable bitmap. 7460 * 7461 * @param match_criteria 7462 * Pointer to flow matcher criteria. 7463 * 7464 * @return 7465 * Bitmap of enabled fields. 7466 */ 7467 static uint8_t 7468 flow_dv_matcher_enable(uint32_t *match_criteria) 7469 { 7470 uint8_t match_criteria_enable; 7471 7472 match_criteria_enable = 7473 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 7474 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT; 7475 match_criteria_enable |= 7476 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 7477 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT; 7478 match_criteria_enable |= 7479 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 7480 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT; 7481 match_criteria_enable |= 7482 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 7483 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 7484 match_criteria_enable |= 7485 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) << 7486 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT; 7487 match_criteria_enable |= 7488 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) << 7489 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT; 7490 return match_criteria_enable; 7491 } 7492 7493 7494 /** 7495 * Get a flow table. 7496 * 7497 * @param[in, out] dev 7498 * Pointer to rte_eth_dev structure. 7499 * @param[in] table_id 7500 * Table id to use. 7501 * @param[in] egress 7502 * Direction of the table. 7503 * @param[in] transfer 7504 * E-Switch or NIC flow. 7505 * @param[out] error 7506 * pointer to error structure. 7507 * 7508 * @return 7509 * Returns tables resource based on the index, NULL in case of failed. 7510 */ 7511 static struct mlx5_flow_tbl_resource * 7512 flow_dv_tbl_resource_get(struct rte_eth_dev *dev, 7513 uint32_t table_id, uint8_t egress, 7514 uint8_t transfer, 7515 struct rte_flow_error *error) 7516 { 7517 struct mlx5_priv *priv = dev->data->dev_private; 7518 struct mlx5_dev_ctx_shared *sh = priv->sh; 7519 struct mlx5_flow_tbl_resource *tbl; 7520 union mlx5_flow_tbl_key table_key = { 7521 { 7522 .table_id = table_id, 7523 .reserved = 0, 7524 .domain = !!transfer, 7525 .direction = !!egress, 7526 } 7527 }; 7528 struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, 7529 table_key.v64); 7530 struct mlx5_flow_tbl_data_entry *tbl_data; 7531 uint32_t idx = 0; 7532 int ret; 7533 void *domain; 7534 7535 if (pos) { 7536 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, 7537 entry); 7538 tbl = &tbl_data->tbl; 7539 rte_atomic32_inc(&tbl->refcnt); 7540 return tbl; 7541 } 7542 tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); 7543 if (!tbl_data) { 7544 rte_flow_error_set(error, ENOMEM, 7545 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7546 NULL, 7547 "cannot allocate flow table data entry"); 7548 return NULL; 7549 } 7550 tbl_data->idx = idx; 7551 tbl = &tbl_data->tbl; 7552 pos = &tbl_data->entry; 7553 if (transfer) 7554 domain = sh->fdb_domain; 7555 else if (egress) 7556 domain = sh->tx_domain; 7557 else 7558 domain = sh->rx_domain; 7559 ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj); 7560 if (ret) { 7561 rte_flow_error_set(error, ENOMEM, 7562 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7563 NULL, "cannot create flow table object"); 7564 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 7565 return NULL; 7566 } 7567 /* 7568 * No multi-threads now, but still better to initialize the reference 7569 * count before insert it into the hash list. 7570 */ 7571 rte_atomic32_init(&tbl->refcnt); 7572 /* Jump action reference count is initialized here. */ 7573 rte_atomic32_init(&tbl_data->jump.refcnt); 7574 pos->key = table_key.v64; 7575 ret = mlx5_hlist_insert(sh->flow_tbls, pos); 7576 if (ret < 0) { 7577 rte_flow_error_set(error, -ret, 7578 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7579 "cannot insert flow table data entry"); 7580 mlx5_flow_os_destroy_flow_tbl(tbl->obj); 7581 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 7582 } 7583 rte_atomic32_inc(&tbl->refcnt); 7584 return tbl; 7585 } 7586 7587 /** 7588 * Release a flow table. 7589 * 7590 * @param[in] dev 7591 * Pointer to rte_eth_dev structure. 7592 * @param[in] tbl 7593 * Table resource to be released. 7594 * 7595 * @return 7596 * Returns 0 if table was released, else return 1; 7597 */ 7598 static int 7599 flow_dv_tbl_resource_release(struct rte_eth_dev *dev, 7600 struct mlx5_flow_tbl_resource *tbl) 7601 { 7602 struct mlx5_priv *priv = dev->data->dev_private; 7603 struct mlx5_dev_ctx_shared *sh = priv->sh; 7604 struct mlx5_flow_tbl_data_entry *tbl_data = 7605 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 7606 7607 if (!tbl) 7608 return 0; 7609 if (rte_atomic32_dec_and_test(&tbl->refcnt)) { 7610 struct mlx5_hlist_entry *pos = &tbl_data->entry; 7611 7612 mlx5_flow_os_destroy_flow_tbl(tbl->obj); 7613 tbl->obj = NULL; 7614 /* remove the entry from the hash list and free memory. */ 7615 mlx5_hlist_remove(sh->flow_tbls, pos); 7616 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP], 7617 tbl_data->idx); 7618 return 0; 7619 } 7620 return 1; 7621 } 7622 7623 /** 7624 * Register the flow matcher. 7625 * 7626 * @param[in, out] dev 7627 * Pointer to rte_eth_dev structure. 7628 * @param[in, out] matcher 7629 * Pointer to flow matcher. 7630 * @param[in, out] key 7631 * Pointer to flow table key. 7632 * @parm[in, out] dev_flow 7633 * Pointer to the dev_flow. 7634 * @param[out] error 7635 * pointer to error structure. 7636 * 7637 * @return 7638 * 0 on success otherwise -errno and errno is set. 7639 */ 7640 static int 7641 flow_dv_matcher_register(struct rte_eth_dev *dev, 7642 struct mlx5_flow_dv_matcher *matcher, 7643 union mlx5_flow_tbl_key *key, 7644 struct mlx5_flow *dev_flow, 7645 struct rte_flow_error *error) 7646 { 7647 struct mlx5_priv *priv = dev->data->dev_private; 7648 struct mlx5_dev_ctx_shared *sh = priv->sh; 7649 struct mlx5_flow_dv_matcher *cache_matcher; 7650 struct mlx5dv_flow_matcher_attr dv_attr = { 7651 .type = IBV_FLOW_ATTR_NORMAL, 7652 .match_mask = (void *)&matcher->mask, 7653 }; 7654 struct mlx5_flow_tbl_resource *tbl; 7655 struct mlx5_flow_tbl_data_entry *tbl_data; 7656 int ret; 7657 7658 tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, 7659 key->domain, error); 7660 if (!tbl) 7661 return -rte_errno; /* No need to refill the error info */ 7662 tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 7663 /* Lookup from cache. */ 7664 LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) { 7665 if (matcher->crc == cache_matcher->crc && 7666 matcher->priority == cache_matcher->priority && 7667 !memcmp((const void *)matcher->mask.buf, 7668 (const void *)cache_matcher->mask.buf, 7669 cache_matcher->mask.size)) { 7670 DRV_LOG(DEBUG, 7671 "%s group %u priority %hd use %s " 7672 "matcher %p: refcnt %d++", 7673 key->domain ? "FDB" : "NIC", key->table_id, 7674 cache_matcher->priority, 7675 key->direction ? "tx" : "rx", 7676 (void *)cache_matcher, 7677 rte_atomic32_read(&cache_matcher->refcnt)); 7678 rte_atomic32_inc(&cache_matcher->refcnt); 7679 dev_flow->handle->dvh.matcher = cache_matcher; 7680 /* old matcher should not make the table ref++. */ 7681 flow_dv_tbl_resource_release(dev, tbl); 7682 return 0; 7683 } 7684 } 7685 /* Register new matcher. */ 7686 cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0, 7687 SOCKET_ID_ANY); 7688 if (!cache_matcher) { 7689 flow_dv_tbl_resource_release(dev, tbl); 7690 return rte_flow_error_set(error, ENOMEM, 7691 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7692 "cannot allocate matcher memory"); 7693 } 7694 *cache_matcher = *matcher; 7695 dv_attr.match_criteria_enable = 7696 flow_dv_matcher_enable(cache_matcher->mask.buf); 7697 dv_attr.priority = matcher->priority; 7698 if (key->direction) 7699 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; 7700 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj, 7701 &cache_matcher->matcher_object); 7702 if (ret) { 7703 mlx5_free(cache_matcher); 7704 #ifdef HAVE_MLX5DV_DR 7705 flow_dv_tbl_resource_release(dev, tbl); 7706 #endif 7707 return rte_flow_error_set(error, ENOMEM, 7708 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7709 NULL, "cannot create matcher"); 7710 } 7711 /* Save the table information */ 7712 cache_matcher->tbl = tbl; 7713 rte_atomic32_init(&cache_matcher->refcnt); 7714 /* only matcher ref++, table ref++ already done above in get API. */ 7715 rte_atomic32_inc(&cache_matcher->refcnt); 7716 LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); 7717 dev_flow->handle->dvh.matcher = cache_matcher; 7718 DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", 7719 key->domain ? "FDB" : "NIC", key->table_id, 7720 cache_matcher->priority, 7721 key->direction ? "tx" : "rx", (void *)cache_matcher, 7722 rte_atomic32_read(&cache_matcher->refcnt)); 7723 return 0; 7724 } 7725 7726 /** 7727 * Find existing tag resource or create and register a new one. 7728 * 7729 * @param dev[in, out] 7730 * Pointer to rte_eth_dev structure. 7731 * @param[in, out] tag_be24 7732 * Tag value in big endian then R-shift 8. 7733 * @parm[in, out] dev_flow 7734 * Pointer to the dev_flow. 7735 * @param[out] error 7736 * pointer to error structure. 7737 * 7738 * @return 7739 * 0 on success otherwise -errno and errno is set. 7740 */ 7741 static int 7742 flow_dv_tag_resource_register 7743 (struct rte_eth_dev *dev, 7744 uint32_t tag_be24, 7745 struct mlx5_flow *dev_flow, 7746 struct rte_flow_error *error) 7747 { 7748 struct mlx5_priv *priv = dev->data->dev_private; 7749 struct mlx5_dev_ctx_shared *sh = priv->sh; 7750 struct mlx5_flow_dv_tag_resource *cache_resource; 7751 struct mlx5_hlist_entry *entry; 7752 int ret; 7753 7754 /* Lookup a matching resource from cache. */ 7755 entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24); 7756 if (entry) { 7757 cache_resource = container_of 7758 (entry, struct mlx5_flow_dv_tag_resource, entry); 7759 rte_atomic32_inc(&cache_resource->refcnt); 7760 dev_flow->handle->dvh.rix_tag = cache_resource->idx; 7761 dev_flow->dv.tag_resource = cache_resource; 7762 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", 7763 (void *)cache_resource, 7764 rte_atomic32_read(&cache_resource->refcnt)); 7765 return 0; 7766 } 7767 /* Register new resource. */ 7768 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], 7769 &dev_flow->handle->dvh.rix_tag); 7770 if (!cache_resource) 7771 return rte_flow_error_set(error, ENOMEM, 7772 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7773 "cannot allocate resource memory"); 7774 cache_resource->entry.key = (uint64_t)tag_be24; 7775 ret = mlx5_flow_os_create_flow_action_tag(tag_be24, 7776 &cache_resource->action); 7777 if (ret) { 7778 mlx5_free(cache_resource); 7779 return rte_flow_error_set(error, ENOMEM, 7780 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7781 NULL, "cannot create action"); 7782 } 7783 rte_atomic32_init(&cache_resource->refcnt); 7784 rte_atomic32_inc(&cache_resource->refcnt); 7785 if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) { 7786 mlx5_flow_os_destroy_flow_action(cache_resource->action); 7787 mlx5_free(cache_resource); 7788 return rte_flow_error_set(error, EEXIST, 7789 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7790 NULL, "cannot insert tag"); 7791 } 7792 dev_flow->dv.tag_resource = cache_resource; 7793 DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", 7794 (void *)cache_resource, 7795 rte_atomic32_read(&cache_resource->refcnt)); 7796 return 0; 7797 } 7798 7799 /** 7800 * Release the tag. 7801 * 7802 * @param dev 7803 * Pointer to Ethernet device. 7804 * @param tag_idx 7805 * Tag index. 7806 * 7807 * @return 7808 * 1 while a reference on it exists, 0 when freed. 7809 */ 7810 static int 7811 flow_dv_tag_release(struct rte_eth_dev *dev, 7812 uint32_t tag_idx) 7813 { 7814 struct mlx5_priv *priv = dev->data->dev_private; 7815 struct mlx5_dev_ctx_shared *sh = priv->sh; 7816 struct mlx5_flow_dv_tag_resource *tag; 7817 7818 tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); 7819 if (!tag) 7820 return 0; 7821 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", 7822 dev->data->port_id, (void *)tag, 7823 rte_atomic32_read(&tag->refcnt)); 7824 if (rte_atomic32_dec_and_test(&tag->refcnt)) { 7825 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action)); 7826 mlx5_hlist_remove(sh->tag_table, &tag->entry); 7827 DRV_LOG(DEBUG, "port %u tag %p: removed", 7828 dev->data->port_id, (void *)tag); 7829 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); 7830 return 0; 7831 } 7832 return 1; 7833 } 7834 7835 /** 7836 * Translate port ID action to vport. 7837 * 7838 * @param[in] dev 7839 * Pointer to rte_eth_dev structure. 7840 * @param[in] action 7841 * Pointer to the port ID action. 7842 * @param[out] dst_port_id 7843 * The target port ID. 7844 * @param[out] error 7845 * Pointer to the error structure. 7846 * 7847 * @return 7848 * 0 on success, a negative errno value otherwise and rte_errno is set. 7849 */ 7850 static int 7851 flow_dv_translate_action_port_id(struct rte_eth_dev *dev, 7852 const struct rte_flow_action *action, 7853 uint32_t *dst_port_id, 7854 struct rte_flow_error *error) 7855 { 7856 uint32_t port; 7857 struct mlx5_priv *priv; 7858 const struct rte_flow_action_port_id *conf = 7859 (const struct rte_flow_action_port_id *)action->conf; 7860 7861 port = conf->original ? dev->data->port_id : conf->id; 7862 priv = mlx5_port_to_eswitch_info(port, false); 7863 if (!priv) 7864 return rte_flow_error_set(error, -rte_errno, 7865 RTE_FLOW_ERROR_TYPE_ACTION, 7866 NULL, 7867 "No eswitch info was found for port"); 7868 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 7869 /* 7870 * This parameter is transferred to 7871 * mlx5dv_dr_action_create_dest_ib_port(). 7872 */ 7873 *dst_port_id = priv->dev_port; 7874 #else 7875 /* 7876 * Legacy mode, no LAG configurations is supported. 7877 * This parameter is transferred to 7878 * mlx5dv_dr_action_create_dest_vport(). 7879 */ 7880 *dst_port_id = priv->vport_id; 7881 #endif 7882 return 0; 7883 } 7884 7885 /** 7886 * Create a counter with aging configuration. 7887 * 7888 * @param[in] dev 7889 * Pointer to rte_eth_dev structure. 7890 * @param[out] count 7891 * Pointer to the counter action configuration. 7892 * @param[in] age 7893 * Pointer to the aging action configuration. 7894 * 7895 * @return 7896 * Index to flow counter on success, 0 otherwise. 7897 */ 7898 static uint32_t 7899 flow_dv_translate_create_counter(struct rte_eth_dev *dev, 7900 struct mlx5_flow *dev_flow, 7901 const struct rte_flow_action_count *count, 7902 const struct rte_flow_action_age *age) 7903 { 7904 uint32_t counter; 7905 struct mlx5_age_param *age_param; 7906 7907 counter = flow_dv_counter_alloc(dev, 7908 count ? count->shared : 0, 7909 count ? count->id : 0, 7910 dev_flow->dv.group, !!age); 7911 if (!counter || age == NULL) 7912 return counter; 7913 age_param = flow_dv_counter_idx_get_age(dev, counter); 7914 /* 7915 * The counter age accuracy may have a bit delay. Have 3/4 7916 * second bias on the timeount in order to let it age in time. 7917 */ 7918 age_param->context = age->context ? age->context : 7919 (void *)(uintptr_t)(dev_flow->flow_idx); 7920 /* 7921 * The counter age accuracy may have a bit delay. Have 3/4 7922 * second bias on the timeount in order to let it age in time. 7923 */ 7924 age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY; 7925 /* Set expire time in unit of 0.1 sec. */ 7926 age_param->port_id = dev->data->port_id; 7927 age_param->expire = age_param->timeout + 7928 rte_rdtsc() / (rte_get_tsc_hz() / 10); 7929 rte_atomic16_set(&age_param->state, AGE_CANDIDATE); 7930 return counter; 7931 } 7932 /** 7933 * Add Tx queue matcher 7934 * 7935 * @param[in] dev 7936 * Pointer to the dev struct. 7937 * @param[in, out] matcher 7938 * Flow matcher. 7939 * @param[in, out] key 7940 * Flow matcher value. 7941 * @param[in] item 7942 * Flow pattern to translate. 7943 * @param[in] inner 7944 * Item is inner pattern. 7945 */ 7946 static void 7947 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev, 7948 void *matcher, void *key, 7949 const struct rte_flow_item *item) 7950 { 7951 const struct mlx5_rte_flow_item_tx_queue *queue_m; 7952 const struct mlx5_rte_flow_item_tx_queue *queue_v; 7953 void *misc_m = 7954 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 7955 void *misc_v = 7956 MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 7957 struct mlx5_txq_ctrl *txq; 7958 uint32_t queue; 7959 7960 7961 queue_m = (const void *)item->mask; 7962 if (!queue_m) 7963 return; 7964 queue_v = (const void *)item->spec; 7965 if (!queue_v) 7966 return; 7967 txq = mlx5_txq_get(dev, queue_v->queue); 7968 if (!txq) 7969 return; 7970 queue = txq->obj->sq->id; 7971 MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue); 7972 MLX5_SET(fte_match_set_misc, misc_v, source_sqn, 7973 queue & queue_m->queue); 7974 mlx5_txq_release(dev, queue_v->queue); 7975 } 7976 7977 /** 7978 * Set the hash fields according to the @p flow information. 7979 * 7980 * @param[in] dev_flow 7981 * Pointer to the mlx5_flow. 7982 * @param[in] rss_desc 7983 * Pointer to the mlx5_flow_rss_desc. 7984 */ 7985 static void 7986 flow_dv_hashfields_set(struct mlx5_flow *dev_flow, 7987 struct mlx5_flow_rss_desc *rss_desc) 7988 { 7989 uint64_t items = dev_flow->handle->layers; 7990 int rss_inner = 0; 7991 uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types); 7992 7993 dev_flow->hash_fields = 0; 7994 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 7995 if (rss_desc->level >= 2) { 7996 dev_flow->hash_fields |= IBV_RX_HASH_INNER; 7997 rss_inner = 1; 7998 } 7999 #endif 8000 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) || 8001 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) { 8002 if (rss_types & MLX5_IPV4_LAYER_TYPES) { 8003 if (rss_types & ETH_RSS_L3_SRC_ONLY) 8004 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4; 8005 else if (rss_types & ETH_RSS_L3_DST_ONLY) 8006 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4; 8007 else 8008 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH; 8009 } 8010 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) || 8011 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) { 8012 if (rss_types & MLX5_IPV6_LAYER_TYPES) { 8013 if (rss_types & ETH_RSS_L3_SRC_ONLY) 8014 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6; 8015 else if (rss_types & ETH_RSS_L3_DST_ONLY) 8016 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6; 8017 else 8018 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH; 8019 } 8020 } 8021 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) || 8022 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) { 8023 if (rss_types & ETH_RSS_UDP) { 8024 if (rss_types & ETH_RSS_L4_SRC_ONLY) 8025 dev_flow->hash_fields |= 8026 IBV_RX_HASH_SRC_PORT_UDP; 8027 else if (rss_types & ETH_RSS_L4_DST_ONLY) 8028 dev_flow->hash_fields |= 8029 IBV_RX_HASH_DST_PORT_UDP; 8030 else 8031 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH; 8032 } 8033 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) || 8034 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) { 8035 if (rss_types & ETH_RSS_TCP) { 8036 if (rss_types & ETH_RSS_L4_SRC_ONLY) 8037 dev_flow->hash_fields |= 8038 IBV_RX_HASH_SRC_PORT_TCP; 8039 else if (rss_types & ETH_RSS_L4_DST_ONLY) 8040 dev_flow->hash_fields |= 8041 IBV_RX_HASH_DST_PORT_TCP; 8042 else 8043 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH; 8044 } 8045 } 8046 } 8047 8048 /** 8049 * Fill the flow with DV spec, lock free 8050 * (mutex should be acquired by caller). 8051 * 8052 * @param[in] dev 8053 * Pointer to rte_eth_dev structure. 8054 * @param[in, out] dev_flow 8055 * Pointer to the sub flow. 8056 * @param[in] attr 8057 * Pointer to the flow attributes. 8058 * @param[in] items 8059 * Pointer to the list of items. 8060 * @param[in] actions 8061 * Pointer to the list of actions. 8062 * @param[out] error 8063 * Pointer to the error structure. 8064 * 8065 * @return 8066 * 0 on success, a negative errno value otherwise and rte_errno is set. 8067 */ 8068 static int 8069 __flow_dv_translate(struct rte_eth_dev *dev, 8070 struct mlx5_flow *dev_flow, 8071 const struct rte_flow_attr *attr, 8072 const struct rte_flow_item items[], 8073 const struct rte_flow_action actions[], 8074 struct rte_flow_error *error) 8075 { 8076 struct mlx5_priv *priv = dev->data->dev_private; 8077 struct mlx5_dev_config *dev_conf = &priv->config; 8078 struct rte_flow *flow = dev_flow->flow; 8079 struct mlx5_flow_handle *handle = dev_flow->handle; 8080 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 8081 priv->rss_desc) 8082 [!!priv->flow_nested_idx]; 8083 uint64_t item_flags = 0; 8084 uint64_t last_item = 0; 8085 uint64_t action_flags = 0; 8086 uint64_t priority = attr->priority; 8087 struct mlx5_flow_dv_matcher matcher = { 8088 .mask = { 8089 .size = sizeof(matcher.mask.buf) - 8090 MLX5_ST_SZ_BYTES(fte_match_set_misc4), 8091 }, 8092 }; 8093 int actions_n = 0; 8094 bool actions_end = false; 8095 union { 8096 struct mlx5_flow_dv_modify_hdr_resource res; 8097 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) + 8098 sizeof(struct mlx5_modification_cmd) * 8099 (MLX5_MAX_MODIFY_NUM + 1)]; 8100 } mhdr_dummy; 8101 struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; 8102 const struct rte_flow_action_count *count = NULL; 8103 const struct rte_flow_action_age *age = NULL; 8104 union flow_dv_attr flow_attr = { .attr = 0 }; 8105 uint32_t tag_be; 8106 union mlx5_flow_tbl_key tbl_key; 8107 uint32_t modify_action_position = UINT32_MAX; 8108 void *match_mask = matcher.mask.buf; 8109 void *match_value = dev_flow->dv.value.buf; 8110 uint8_t next_protocol = 0xff; 8111 struct rte_vlan_hdr vlan = { 0 }; 8112 uint32_t table; 8113 int ret = 0; 8114 8115 mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 8116 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 8117 ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group, 8118 !!priv->fdb_def_rule, &table, error); 8119 if (ret) 8120 return ret; 8121 dev_flow->dv.group = table; 8122 if (attr->transfer) 8123 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 8124 if (priority == MLX5_FLOW_PRIO_RSVD) 8125 priority = dev_conf->flow_prio - 1; 8126 /* number of actions must be set to 0 in case of dirty stack. */ 8127 mhdr_res->actions_num = 0; 8128 for (; !actions_end ; actions++) { 8129 const struct rte_flow_action_queue *queue; 8130 const struct rte_flow_action_rss *rss; 8131 const struct rte_flow_action *action = actions; 8132 const uint8_t *rss_key; 8133 const struct rte_flow_action_jump *jump_data; 8134 const struct rte_flow_action_meter *mtr; 8135 struct mlx5_flow_tbl_resource *tbl; 8136 uint32_t port_id = 0; 8137 struct mlx5_flow_dv_port_id_action_resource port_id_resource; 8138 int action_type = actions->type; 8139 const struct rte_flow_action *found_action = NULL; 8140 struct mlx5_flow_meter *fm = NULL; 8141 8142 if (!mlx5_flow_os_action_supported(action_type)) 8143 return rte_flow_error_set(error, ENOTSUP, 8144 RTE_FLOW_ERROR_TYPE_ACTION, 8145 actions, 8146 "action not supported"); 8147 switch (action_type) { 8148 case RTE_FLOW_ACTION_TYPE_VOID: 8149 break; 8150 case RTE_FLOW_ACTION_TYPE_PORT_ID: 8151 if (flow_dv_translate_action_port_id(dev, action, 8152 &port_id, error)) 8153 return -rte_errno; 8154 port_id_resource.port_id = port_id; 8155 MLX5_ASSERT(!handle->rix_port_id_action); 8156 if (flow_dv_port_id_action_resource_register 8157 (dev, &port_id_resource, dev_flow, error)) 8158 return -rte_errno; 8159 dev_flow->dv.actions[actions_n++] = 8160 dev_flow->dv.port_id_action->action; 8161 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 8162 dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID; 8163 break; 8164 case RTE_FLOW_ACTION_TYPE_FLAG: 8165 action_flags |= MLX5_FLOW_ACTION_FLAG; 8166 dev_flow->handle->mark = 1; 8167 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 8168 struct rte_flow_action_mark mark = { 8169 .id = MLX5_FLOW_MARK_DEFAULT, 8170 }; 8171 8172 if (flow_dv_convert_action_mark(dev, &mark, 8173 mhdr_res, 8174 error)) 8175 return -rte_errno; 8176 action_flags |= MLX5_FLOW_ACTION_MARK_EXT; 8177 break; 8178 } 8179 tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); 8180 /* 8181 * Only one FLAG or MARK is supported per device flow 8182 * right now. So the pointer to the tag resource must be 8183 * zero before the register process. 8184 */ 8185 MLX5_ASSERT(!handle->dvh.rix_tag); 8186 if (flow_dv_tag_resource_register(dev, tag_be, 8187 dev_flow, error)) 8188 return -rte_errno; 8189 MLX5_ASSERT(dev_flow->dv.tag_resource); 8190 dev_flow->dv.actions[actions_n++] = 8191 dev_flow->dv.tag_resource->action; 8192 break; 8193 case RTE_FLOW_ACTION_TYPE_MARK: 8194 action_flags |= MLX5_FLOW_ACTION_MARK; 8195 dev_flow->handle->mark = 1; 8196 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 8197 const struct rte_flow_action_mark *mark = 8198 (const struct rte_flow_action_mark *) 8199 actions->conf; 8200 8201 if (flow_dv_convert_action_mark(dev, mark, 8202 mhdr_res, 8203 error)) 8204 return -rte_errno; 8205 action_flags |= MLX5_FLOW_ACTION_MARK_EXT; 8206 break; 8207 } 8208 /* Fall-through */ 8209 case MLX5_RTE_FLOW_ACTION_TYPE_MARK: 8210 /* Legacy (non-extensive) MARK action. */ 8211 tag_be = mlx5_flow_mark_set 8212 (((const struct rte_flow_action_mark *) 8213 (actions->conf))->id); 8214 MLX5_ASSERT(!handle->dvh.rix_tag); 8215 if (flow_dv_tag_resource_register(dev, tag_be, 8216 dev_flow, error)) 8217 return -rte_errno; 8218 MLX5_ASSERT(dev_flow->dv.tag_resource); 8219 dev_flow->dv.actions[actions_n++] = 8220 dev_flow->dv.tag_resource->action; 8221 break; 8222 case RTE_FLOW_ACTION_TYPE_SET_META: 8223 if (flow_dv_convert_action_set_meta 8224 (dev, mhdr_res, attr, 8225 (const struct rte_flow_action_set_meta *) 8226 actions->conf, error)) 8227 return -rte_errno; 8228 action_flags |= MLX5_FLOW_ACTION_SET_META; 8229 break; 8230 case RTE_FLOW_ACTION_TYPE_SET_TAG: 8231 if (flow_dv_convert_action_set_tag 8232 (dev, mhdr_res, 8233 (const struct rte_flow_action_set_tag *) 8234 actions->conf, error)) 8235 return -rte_errno; 8236 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8237 break; 8238 case RTE_FLOW_ACTION_TYPE_DROP: 8239 action_flags |= MLX5_FLOW_ACTION_DROP; 8240 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; 8241 break; 8242 case RTE_FLOW_ACTION_TYPE_QUEUE: 8243 queue = actions->conf; 8244 rss_desc->queue_num = 1; 8245 rss_desc->queue[0] = queue->index; 8246 action_flags |= MLX5_FLOW_ACTION_QUEUE; 8247 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 8248 break; 8249 case RTE_FLOW_ACTION_TYPE_RSS: 8250 rss = actions->conf; 8251 memcpy(rss_desc->queue, rss->queue, 8252 rss->queue_num * sizeof(uint16_t)); 8253 rss_desc->queue_num = rss->queue_num; 8254 /* NULL RSS key indicates default RSS key. */ 8255 rss_key = !rss->key ? rss_hash_default_key : rss->key; 8256 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 8257 /* 8258 * rss->level and rss.types should be set in advance 8259 * when expanding items for RSS. 8260 */ 8261 action_flags |= MLX5_FLOW_ACTION_RSS; 8262 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 8263 break; 8264 case RTE_FLOW_ACTION_TYPE_AGE: 8265 case RTE_FLOW_ACTION_TYPE_COUNT: 8266 if (!dev_conf->devx) { 8267 return rte_flow_error_set 8268 (error, ENOTSUP, 8269 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8270 NULL, 8271 "count action not supported"); 8272 } 8273 /* Save information first, will apply later. */ 8274 if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT) 8275 count = action->conf; 8276 else 8277 age = action->conf; 8278 action_flags |= MLX5_FLOW_ACTION_COUNT; 8279 break; 8280 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 8281 dev_flow->dv.actions[actions_n++] = 8282 priv->sh->pop_vlan_action; 8283 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; 8284 break; 8285 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 8286 if (!(action_flags & 8287 MLX5_FLOW_ACTION_OF_SET_VLAN_VID)) 8288 flow_dev_get_vlan_info_from_items(items, &vlan); 8289 vlan.eth_proto = rte_be_to_cpu_16 8290 ((((const struct rte_flow_action_of_push_vlan *) 8291 actions->conf)->ethertype)); 8292 found_action = mlx5_flow_find_action 8293 (actions + 1, 8294 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID); 8295 if (found_action) 8296 mlx5_update_vlan_vid_pcp(found_action, &vlan); 8297 found_action = mlx5_flow_find_action 8298 (actions + 1, 8299 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP); 8300 if (found_action) 8301 mlx5_update_vlan_vid_pcp(found_action, &vlan); 8302 if (flow_dv_create_action_push_vlan 8303 (dev, attr, &vlan, dev_flow, error)) 8304 return -rte_errno; 8305 dev_flow->dv.actions[actions_n++] = 8306 dev_flow->dv.push_vlan_res->action; 8307 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; 8308 break; 8309 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 8310 /* of_vlan_push action handled this action */ 8311 MLX5_ASSERT(action_flags & 8312 MLX5_FLOW_ACTION_OF_PUSH_VLAN); 8313 break; 8314 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 8315 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) 8316 break; 8317 flow_dev_get_vlan_info_from_items(items, &vlan); 8318 mlx5_update_vlan_vid_pcp(actions, &vlan); 8319 /* If no VLAN push - this is a modify header action */ 8320 if (flow_dv_convert_action_modify_vlan_vid 8321 (mhdr_res, actions, error)) 8322 return -rte_errno; 8323 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; 8324 break; 8325 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 8326 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 8327 if (flow_dv_create_action_l2_encap(dev, actions, 8328 dev_flow, 8329 attr->transfer, 8330 error)) 8331 return -rte_errno; 8332 dev_flow->dv.actions[actions_n++] = 8333 dev_flow->dv.encap_decap->action; 8334 action_flags |= MLX5_FLOW_ACTION_ENCAP; 8335 break; 8336 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 8337 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 8338 if (flow_dv_create_action_l2_decap(dev, dev_flow, 8339 attr->transfer, 8340 error)) 8341 return -rte_errno; 8342 dev_flow->dv.actions[actions_n++] = 8343 dev_flow->dv.encap_decap->action; 8344 action_flags |= MLX5_FLOW_ACTION_DECAP; 8345 break; 8346 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 8347 /* Handle encap with preceding decap. */ 8348 if (action_flags & MLX5_FLOW_ACTION_DECAP) { 8349 if (flow_dv_create_action_raw_encap 8350 (dev, actions, dev_flow, attr, error)) 8351 return -rte_errno; 8352 dev_flow->dv.actions[actions_n++] = 8353 dev_flow->dv.encap_decap->action; 8354 } else { 8355 /* Handle encap without preceding decap. */ 8356 if (flow_dv_create_action_l2_encap 8357 (dev, actions, dev_flow, attr->transfer, 8358 error)) 8359 return -rte_errno; 8360 dev_flow->dv.actions[actions_n++] = 8361 dev_flow->dv.encap_decap->action; 8362 } 8363 action_flags |= MLX5_FLOW_ACTION_ENCAP; 8364 break; 8365 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 8366 while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID) 8367 ; 8368 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 8369 if (flow_dv_create_action_l2_decap 8370 (dev, dev_flow, attr->transfer, error)) 8371 return -rte_errno; 8372 dev_flow->dv.actions[actions_n++] = 8373 dev_flow->dv.encap_decap->action; 8374 } 8375 /* If decap is followed by encap, handle it at encap. */ 8376 action_flags |= MLX5_FLOW_ACTION_DECAP; 8377 break; 8378 case RTE_FLOW_ACTION_TYPE_JUMP: 8379 jump_data = action->conf; 8380 ret = mlx5_flow_group_to_table(attr, dev_flow->external, 8381 jump_data->group, 8382 !!priv->fdb_def_rule, 8383 &table, error); 8384 if (ret) 8385 return ret; 8386 tbl = flow_dv_tbl_resource_get(dev, table, 8387 attr->egress, 8388 attr->transfer, error); 8389 if (!tbl) 8390 return rte_flow_error_set 8391 (error, errno, 8392 RTE_FLOW_ERROR_TYPE_ACTION, 8393 NULL, 8394 "cannot create jump action."); 8395 if (flow_dv_jump_tbl_resource_register 8396 (dev, tbl, dev_flow, error)) { 8397 flow_dv_tbl_resource_release(dev, tbl); 8398 return rte_flow_error_set 8399 (error, errno, 8400 RTE_FLOW_ERROR_TYPE_ACTION, 8401 NULL, 8402 "cannot create jump action."); 8403 } 8404 dev_flow->dv.actions[actions_n++] = 8405 dev_flow->dv.jump->action; 8406 action_flags |= MLX5_FLOW_ACTION_JUMP; 8407 dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP; 8408 break; 8409 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 8410 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 8411 if (flow_dv_convert_action_modify_mac 8412 (mhdr_res, actions, error)) 8413 return -rte_errno; 8414 action_flags |= actions->type == 8415 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 8416 MLX5_FLOW_ACTION_SET_MAC_SRC : 8417 MLX5_FLOW_ACTION_SET_MAC_DST; 8418 break; 8419 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 8420 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 8421 if (flow_dv_convert_action_modify_ipv4 8422 (mhdr_res, actions, error)) 8423 return -rte_errno; 8424 action_flags |= actions->type == 8425 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 8426 MLX5_FLOW_ACTION_SET_IPV4_SRC : 8427 MLX5_FLOW_ACTION_SET_IPV4_DST; 8428 break; 8429 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 8430 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 8431 if (flow_dv_convert_action_modify_ipv6 8432 (mhdr_res, actions, error)) 8433 return -rte_errno; 8434 action_flags |= actions->type == 8435 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 8436 MLX5_FLOW_ACTION_SET_IPV6_SRC : 8437 MLX5_FLOW_ACTION_SET_IPV6_DST; 8438 break; 8439 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 8440 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 8441 if (flow_dv_convert_action_modify_tp 8442 (mhdr_res, actions, items, 8443 &flow_attr, dev_flow, !!(action_flags & 8444 MLX5_FLOW_ACTION_DECAP), error)) 8445 return -rte_errno; 8446 action_flags |= actions->type == 8447 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 8448 MLX5_FLOW_ACTION_SET_TP_SRC : 8449 MLX5_FLOW_ACTION_SET_TP_DST; 8450 break; 8451 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 8452 if (flow_dv_convert_action_modify_dec_ttl 8453 (mhdr_res, items, &flow_attr, dev_flow, 8454 !!(action_flags & 8455 MLX5_FLOW_ACTION_DECAP), error)) 8456 return -rte_errno; 8457 action_flags |= MLX5_FLOW_ACTION_DEC_TTL; 8458 break; 8459 case RTE_FLOW_ACTION_TYPE_SET_TTL: 8460 if (flow_dv_convert_action_modify_ttl 8461 (mhdr_res, actions, items, &flow_attr, 8462 dev_flow, !!(action_flags & 8463 MLX5_FLOW_ACTION_DECAP), error)) 8464 return -rte_errno; 8465 action_flags |= MLX5_FLOW_ACTION_SET_TTL; 8466 break; 8467 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 8468 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 8469 if (flow_dv_convert_action_modify_tcp_seq 8470 (mhdr_res, actions, error)) 8471 return -rte_errno; 8472 action_flags |= actions->type == 8473 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 8474 MLX5_FLOW_ACTION_INC_TCP_SEQ : 8475 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 8476 break; 8477 8478 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 8479 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 8480 if (flow_dv_convert_action_modify_tcp_ack 8481 (mhdr_res, actions, error)) 8482 return -rte_errno; 8483 action_flags |= actions->type == 8484 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 8485 MLX5_FLOW_ACTION_INC_TCP_ACK : 8486 MLX5_FLOW_ACTION_DEC_TCP_ACK; 8487 break; 8488 case MLX5_RTE_FLOW_ACTION_TYPE_TAG: 8489 if (flow_dv_convert_action_set_reg 8490 (mhdr_res, actions, error)) 8491 return -rte_errno; 8492 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8493 break; 8494 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: 8495 if (flow_dv_convert_action_copy_mreg 8496 (dev, mhdr_res, actions, error)) 8497 return -rte_errno; 8498 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8499 break; 8500 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: 8501 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; 8502 dev_flow->handle->fate_action = 8503 MLX5_FLOW_FATE_DEFAULT_MISS; 8504 break; 8505 case RTE_FLOW_ACTION_TYPE_METER: 8506 mtr = actions->conf; 8507 if (!flow->meter) { 8508 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, 8509 attr, error); 8510 if (!fm) 8511 return rte_flow_error_set(error, 8512 rte_errno, 8513 RTE_FLOW_ERROR_TYPE_ACTION, 8514 NULL, 8515 "meter not found " 8516 "or invalid parameters"); 8517 flow->meter = fm->idx; 8518 } 8519 /* Set the meter action. */ 8520 if (!fm) { 8521 fm = mlx5_ipool_get(priv->sh->ipool 8522 [MLX5_IPOOL_MTR], flow->meter); 8523 if (!fm) 8524 return rte_flow_error_set(error, 8525 rte_errno, 8526 RTE_FLOW_ERROR_TYPE_ACTION, 8527 NULL, 8528 "meter not found " 8529 "or invalid parameters"); 8530 } 8531 dev_flow->dv.actions[actions_n++] = 8532 fm->mfts->meter_action; 8533 action_flags |= MLX5_FLOW_ACTION_METER; 8534 break; 8535 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 8536 if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res, 8537 actions, error)) 8538 return -rte_errno; 8539 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; 8540 break; 8541 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 8542 if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res, 8543 actions, error)) 8544 return -rte_errno; 8545 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; 8546 break; 8547 case RTE_FLOW_ACTION_TYPE_END: 8548 actions_end = true; 8549 if (mhdr_res->actions_num) { 8550 /* create modify action if needed. */ 8551 if (flow_dv_modify_hdr_resource_register 8552 (dev, mhdr_res, dev_flow, error)) 8553 return -rte_errno; 8554 dev_flow->dv.actions[modify_action_position] = 8555 handle->dvh.modify_hdr->action; 8556 } 8557 if (action_flags & MLX5_FLOW_ACTION_COUNT) { 8558 flow->counter = 8559 flow_dv_translate_create_counter(dev, 8560 dev_flow, count, age); 8561 8562 if (!flow->counter) 8563 return rte_flow_error_set 8564 (error, rte_errno, 8565 RTE_FLOW_ERROR_TYPE_ACTION, 8566 NULL, 8567 "cannot create counter" 8568 " object."); 8569 dev_flow->dv.actions[actions_n++] = 8570 (flow_dv_counter_get_by_idx(dev, 8571 flow->counter, NULL))->action; 8572 } 8573 break; 8574 default: 8575 break; 8576 } 8577 if (mhdr_res->actions_num && 8578 modify_action_position == UINT32_MAX) 8579 modify_action_position = actions_n++; 8580 } 8581 dev_flow->dv.actions_n = actions_n; 8582 dev_flow->act_flags = action_flags; 8583 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 8584 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 8585 int item_type = items->type; 8586 8587 if (!mlx5_flow_os_item_supported(item_type)) 8588 return rte_flow_error_set(error, ENOTSUP, 8589 RTE_FLOW_ERROR_TYPE_ITEM, 8590 NULL, "item not supported"); 8591 switch (item_type) { 8592 case RTE_FLOW_ITEM_TYPE_PORT_ID: 8593 flow_dv_translate_item_port_id(dev, match_mask, 8594 match_value, items); 8595 last_item = MLX5_FLOW_ITEM_PORT_ID; 8596 break; 8597 case RTE_FLOW_ITEM_TYPE_ETH: 8598 flow_dv_translate_item_eth(match_mask, match_value, 8599 items, tunnel, 8600 dev_flow->dv.group); 8601 matcher.priority = action_flags & 8602 MLX5_FLOW_ACTION_DEFAULT_MISS && 8603 !dev_flow->external ? 8604 MLX5_PRIORITY_MAP_L3 : 8605 MLX5_PRIORITY_MAP_L2; 8606 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 8607 MLX5_FLOW_LAYER_OUTER_L2; 8608 break; 8609 case RTE_FLOW_ITEM_TYPE_VLAN: 8610 flow_dv_translate_item_vlan(dev_flow, 8611 match_mask, match_value, 8612 items, tunnel, 8613 dev_flow->dv.group); 8614 matcher.priority = MLX5_PRIORITY_MAP_L2; 8615 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 8616 MLX5_FLOW_LAYER_INNER_VLAN) : 8617 (MLX5_FLOW_LAYER_OUTER_L2 | 8618 MLX5_FLOW_LAYER_OUTER_VLAN); 8619 break; 8620 case RTE_FLOW_ITEM_TYPE_IPV4: 8621 mlx5_flow_tunnel_ip_check(items, next_protocol, 8622 &item_flags, &tunnel); 8623 flow_dv_translate_item_ipv4(match_mask, match_value, 8624 items, item_flags, tunnel, 8625 dev_flow->dv.group); 8626 matcher.priority = MLX5_PRIORITY_MAP_L3; 8627 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 8628 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 8629 if (items->mask != NULL && 8630 ((const struct rte_flow_item_ipv4 *) 8631 items->mask)->hdr.next_proto_id) { 8632 next_protocol = 8633 ((const struct rte_flow_item_ipv4 *) 8634 (items->spec))->hdr.next_proto_id; 8635 next_protocol &= 8636 ((const struct rte_flow_item_ipv4 *) 8637 (items->mask))->hdr.next_proto_id; 8638 } else { 8639 /* Reset for inner layer. */ 8640 next_protocol = 0xff; 8641 } 8642 break; 8643 case RTE_FLOW_ITEM_TYPE_IPV6: 8644 mlx5_flow_tunnel_ip_check(items, next_protocol, 8645 &item_flags, &tunnel); 8646 flow_dv_translate_item_ipv6(match_mask, match_value, 8647 items, item_flags, tunnel, 8648 dev_flow->dv.group); 8649 matcher.priority = MLX5_PRIORITY_MAP_L3; 8650 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 8651 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 8652 if (items->mask != NULL && 8653 ((const struct rte_flow_item_ipv6 *) 8654 items->mask)->hdr.proto) { 8655 next_protocol = 8656 ((const struct rte_flow_item_ipv6 *) 8657 items->spec)->hdr.proto; 8658 next_protocol &= 8659 ((const struct rte_flow_item_ipv6 *) 8660 items->mask)->hdr.proto; 8661 } else { 8662 /* Reset for inner layer. */ 8663 next_protocol = 0xff; 8664 } 8665 break; 8666 case RTE_FLOW_ITEM_TYPE_TCP: 8667 flow_dv_translate_item_tcp(match_mask, match_value, 8668 items, tunnel); 8669 matcher.priority = MLX5_PRIORITY_MAP_L4; 8670 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 8671 MLX5_FLOW_LAYER_OUTER_L4_TCP; 8672 break; 8673 case RTE_FLOW_ITEM_TYPE_UDP: 8674 flow_dv_translate_item_udp(match_mask, match_value, 8675 items, tunnel); 8676 matcher.priority = MLX5_PRIORITY_MAP_L4; 8677 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 8678 MLX5_FLOW_LAYER_OUTER_L4_UDP; 8679 break; 8680 case RTE_FLOW_ITEM_TYPE_GRE: 8681 flow_dv_translate_item_gre(match_mask, match_value, 8682 items, tunnel); 8683 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8684 last_item = MLX5_FLOW_LAYER_GRE; 8685 break; 8686 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 8687 flow_dv_translate_item_gre_key(match_mask, 8688 match_value, items); 8689 last_item = MLX5_FLOW_LAYER_GRE_KEY; 8690 break; 8691 case RTE_FLOW_ITEM_TYPE_NVGRE: 8692 flow_dv_translate_item_nvgre(match_mask, match_value, 8693 items, tunnel); 8694 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8695 last_item = MLX5_FLOW_LAYER_GRE; 8696 break; 8697 case RTE_FLOW_ITEM_TYPE_VXLAN: 8698 flow_dv_translate_item_vxlan(match_mask, match_value, 8699 items, tunnel); 8700 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8701 last_item = MLX5_FLOW_LAYER_VXLAN; 8702 break; 8703 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 8704 flow_dv_translate_item_vxlan_gpe(match_mask, 8705 match_value, items, 8706 tunnel); 8707 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8708 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 8709 break; 8710 case RTE_FLOW_ITEM_TYPE_GENEVE: 8711 flow_dv_translate_item_geneve(match_mask, match_value, 8712 items, tunnel); 8713 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8714 last_item = MLX5_FLOW_LAYER_GENEVE; 8715 break; 8716 case RTE_FLOW_ITEM_TYPE_MPLS: 8717 flow_dv_translate_item_mpls(match_mask, match_value, 8718 items, last_item, tunnel); 8719 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8720 last_item = MLX5_FLOW_LAYER_MPLS; 8721 break; 8722 case RTE_FLOW_ITEM_TYPE_MARK: 8723 flow_dv_translate_item_mark(dev, match_mask, 8724 match_value, items); 8725 last_item = MLX5_FLOW_ITEM_MARK; 8726 break; 8727 case RTE_FLOW_ITEM_TYPE_META: 8728 flow_dv_translate_item_meta(dev, match_mask, 8729 match_value, attr, items); 8730 last_item = MLX5_FLOW_ITEM_METADATA; 8731 break; 8732 case RTE_FLOW_ITEM_TYPE_ICMP: 8733 flow_dv_translate_item_icmp(match_mask, match_value, 8734 items, tunnel); 8735 last_item = MLX5_FLOW_LAYER_ICMP; 8736 break; 8737 case RTE_FLOW_ITEM_TYPE_ICMP6: 8738 flow_dv_translate_item_icmp6(match_mask, match_value, 8739 items, tunnel); 8740 last_item = MLX5_FLOW_LAYER_ICMP6; 8741 break; 8742 case RTE_FLOW_ITEM_TYPE_TAG: 8743 flow_dv_translate_item_tag(dev, match_mask, 8744 match_value, items); 8745 last_item = MLX5_FLOW_ITEM_TAG; 8746 break; 8747 case MLX5_RTE_FLOW_ITEM_TYPE_TAG: 8748 flow_dv_translate_mlx5_item_tag(dev, match_mask, 8749 match_value, items); 8750 last_item = MLX5_FLOW_ITEM_TAG; 8751 break; 8752 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: 8753 flow_dv_translate_item_tx_queue(dev, match_mask, 8754 match_value, 8755 items); 8756 last_item = MLX5_FLOW_ITEM_TX_QUEUE; 8757 break; 8758 case RTE_FLOW_ITEM_TYPE_GTP: 8759 flow_dv_translate_item_gtp(match_mask, match_value, 8760 items, tunnel); 8761 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8762 last_item = MLX5_FLOW_LAYER_GTP; 8763 break; 8764 case RTE_FLOW_ITEM_TYPE_ECPRI: 8765 if (!mlx5_flex_parser_ecpri_exist(dev)) { 8766 /* Create it only the first time to be used. */ 8767 ret = mlx5_flex_parser_ecpri_alloc(dev); 8768 if (ret) 8769 return rte_flow_error_set 8770 (error, -ret, 8771 RTE_FLOW_ERROR_TYPE_ITEM, 8772 NULL, 8773 "cannot create eCPRI parser"); 8774 } 8775 /* Adjust the length matcher and device flow value. */ 8776 matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param); 8777 dev_flow->dv.value.size = 8778 MLX5_ST_SZ_BYTES(fte_match_param); 8779 flow_dv_translate_item_ecpri(dev, match_mask, 8780 match_value, items); 8781 /* No other protocol should follow eCPRI layer. */ 8782 last_item = MLX5_FLOW_LAYER_ECPRI; 8783 break; 8784 default: 8785 break; 8786 } 8787 item_flags |= last_item; 8788 } 8789 /* 8790 * When E-Switch mode is enabled, we have two cases where we need to 8791 * set the source port manually. 8792 * The first one, is in case of Nic steering rule, and the second is 8793 * E-Switch rule where no port_id item was found. In both cases 8794 * the source port is set according the current port in use. 8795 */ 8796 if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && 8797 (priv->representor || priv->master)) { 8798 if (flow_dv_translate_item_port_id(dev, match_mask, 8799 match_value, NULL)) 8800 return -rte_errno; 8801 } 8802 #ifdef RTE_LIBRTE_MLX5_DEBUG 8803 MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf, 8804 dev_flow->dv.value.buf)); 8805 #endif 8806 /* 8807 * Layers may be already initialized from prefix flow if this dev_flow 8808 * is the suffix flow. 8809 */ 8810 handle->layers |= item_flags; 8811 if (action_flags & MLX5_FLOW_ACTION_RSS) 8812 flow_dv_hashfields_set(dev_flow, rss_desc); 8813 /* Register matcher. */ 8814 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, 8815 matcher.mask.size); 8816 matcher.priority = mlx5_flow_adjust_priority(dev, priority, 8817 matcher.priority); 8818 /* reserved field no needs to be set to 0 here. */ 8819 tbl_key.domain = attr->transfer; 8820 tbl_key.direction = attr->egress; 8821 tbl_key.table_id = dev_flow->dv.group; 8822 if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error)) 8823 return -rte_errno; 8824 return 0; 8825 } 8826 8827 /** 8828 * Apply the flow to the NIC, lock free, 8829 * (mutex should be acquired by caller). 8830 * 8831 * @param[in] dev 8832 * Pointer to the Ethernet device structure. 8833 * @param[in, out] flow 8834 * Pointer to flow structure. 8835 * @param[out] error 8836 * Pointer to error structure. 8837 * 8838 * @return 8839 * 0 on success, a negative errno value otherwise and rte_errno is set. 8840 */ 8841 static int 8842 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 8843 struct rte_flow_error *error) 8844 { 8845 struct mlx5_flow_dv_workspace *dv; 8846 struct mlx5_flow_handle *dh; 8847 struct mlx5_flow_handle_dv *dv_h; 8848 struct mlx5_flow *dev_flow; 8849 struct mlx5_priv *priv = dev->data->dev_private; 8850 uint32_t handle_idx; 8851 int n; 8852 int err; 8853 int idx; 8854 8855 for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) { 8856 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; 8857 dv = &dev_flow->dv; 8858 dh = dev_flow->handle; 8859 dv_h = &dh->dvh; 8860 n = dv->actions_n; 8861 if (dh->fate_action == MLX5_FLOW_FATE_DROP) { 8862 if (dv->transfer) { 8863 dv->actions[n++] = priv->sh->esw_drop_action; 8864 } else { 8865 struct mlx5_hrxq *drop_hrxq; 8866 drop_hrxq = mlx5_hrxq_drop_new(dev); 8867 if (!drop_hrxq) { 8868 rte_flow_error_set 8869 (error, errno, 8870 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8871 NULL, 8872 "cannot get drop hash queue"); 8873 goto error; 8874 } 8875 /* 8876 * Drop queues will be released by the specify 8877 * mlx5_hrxq_drop_release() function. Assign 8878 * the special index to hrxq to mark the queue 8879 * has been allocated. 8880 */ 8881 dh->rix_hrxq = UINT32_MAX; 8882 dv->actions[n++] = drop_hrxq->action; 8883 } 8884 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { 8885 struct mlx5_hrxq *hrxq; 8886 uint32_t hrxq_idx; 8887 struct mlx5_flow_rss_desc *rss_desc = 8888 &((struct mlx5_flow_rss_desc *)priv->rss_desc) 8889 [!!priv->flow_nested_idx]; 8890 8891 MLX5_ASSERT(rss_desc->queue_num); 8892 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, 8893 MLX5_RSS_HASH_KEY_LEN, 8894 dev_flow->hash_fields, 8895 rss_desc->queue, 8896 rss_desc->queue_num); 8897 if (!hrxq_idx) { 8898 hrxq_idx = mlx5_hrxq_new 8899 (dev, rss_desc->key, 8900 MLX5_RSS_HASH_KEY_LEN, 8901 dev_flow->hash_fields, 8902 rss_desc->queue, 8903 rss_desc->queue_num, 8904 !!(dh->layers & 8905 MLX5_FLOW_LAYER_TUNNEL)); 8906 } 8907 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 8908 hrxq_idx); 8909 if (!hrxq) { 8910 rte_flow_error_set 8911 (error, rte_errno, 8912 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8913 "cannot get hash queue"); 8914 goto error; 8915 } 8916 dh->rix_hrxq = hrxq_idx; 8917 dv->actions[n++] = hrxq->action; 8918 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) { 8919 if (flow_dv_default_miss_resource_register 8920 (dev, error)) { 8921 rte_flow_error_set 8922 (error, rte_errno, 8923 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8924 "cannot create default miss resource"); 8925 goto error_default_miss; 8926 } 8927 dh->rix_default_fate = MLX5_FLOW_FATE_DEFAULT_MISS; 8928 dv->actions[n++] = priv->sh->default_miss.action; 8929 } 8930 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object, 8931 (void *)&dv->value, n, 8932 dv->actions, &dh->drv_flow); 8933 if (err) { 8934 rte_flow_error_set(error, errno, 8935 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8936 NULL, 8937 "hardware refuses to create flow"); 8938 goto error; 8939 } 8940 if (priv->vmwa_context && 8941 dh->vf_vlan.tag && !dh->vf_vlan.created) { 8942 /* 8943 * The rule contains the VLAN pattern. 8944 * For VF we are going to create VLAN 8945 * interface to make hypervisor set correct 8946 * e-Switch vport context. 8947 */ 8948 mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan); 8949 } 8950 } 8951 return 0; 8952 error: 8953 if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) 8954 flow_dv_default_miss_resource_release(dev); 8955 error_default_miss: 8956 err = rte_errno; /* Save rte_errno before cleanup. */ 8957 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 8958 handle_idx, dh, next) { 8959 /* hrxq is union, don't clear it if the flag is not set. */ 8960 if (dh->rix_hrxq) { 8961 if (dh->fate_action == MLX5_FLOW_FATE_DROP) { 8962 mlx5_hrxq_drop_release(dev); 8963 dh->rix_hrxq = 0; 8964 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { 8965 mlx5_hrxq_release(dev, dh->rix_hrxq); 8966 dh->rix_hrxq = 0; 8967 } 8968 } 8969 if (dh->vf_vlan.tag && dh->vf_vlan.created) 8970 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); 8971 } 8972 rte_errno = err; /* Restore rte_errno. */ 8973 return -rte_errno; 8974 } 8975 8976 /** 8977 * Release the flow matcher. 8978 * 8979 * @param dev 8980 * Pointer to Ethernet device. 8981 * @param handle 8982 * Pointer to mlx5_flow_handle. 8983 * 8984 * @return 8985 * 1 while a reference on it exists, 0 when freed. 8986 */ 8987 static int 8988 flow_dv_matcher_release(struct rte_eth_dev *dev, 8989 struct mlx5_flow_handle *handle) 8990 { 8991 struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher; 8992 8993 MLX5_ASSERT(matcher->matcher_object); 8994 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", 8995 dev->data->port_id, (void *)matcher, 8996 rte_atomic32_read(&matcher->refcnt)); 8997 if (rte_atomic32_dec_and_test(&matcher->refcnt)) { 8998 claim_zero(mlx5_flow_os_destroy_flow_matcher 8999 (matcher->matcher_object)); 9000 LIST_REMOVE(matcher, next); 9001 /* table ref-- in release interface. */ 9002 flow_dv_tbl_resource_release(dev, matcher->tbl); 9003 mlx5_free(matcher); 9004 DRV_LOG(DEBUG, "port %u matcher %p: removed", 9005 dev->data->port_id, (void *)matcher); 9006 return 0; 9007 } 9008 return 1; 9009 } 9010 9011 /** 9012 * Release an encap/decap resource. 9013 * 9014 * @param dev 9015 * Pointer to Ethernet device. 9016 * @param handle 9017 * Pointer to mlx5_flow_handle. 9018 * 9019 * @return 9020 * 1 while a reference on it exists, 0 when freed. 9021 */ 9022 static int 9023 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, 9024 struct mlx5_flow_handle *handle) 9025 { 9026 struct mlx5_priv *priv = dev->data->dev_private; 9027 uint32_t idx = handle->dvh.rix_encap_decap; 9028 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 9029 9030 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 9031 idx); 9032 if (!cache_resource) 9033 return 0; 9034 MLX5_ASSERT(cache_resource->action); 9035 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", 9036 (void *)cache_resource, 9037 rte_atomic32_read(&cache_resource->refcnt)); 9038 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9039 claim_zero(mlx5_flow_os_destroy_flow_action 9040 (cache_resource->action)); 9041 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 9042 &priv->sh->encaps_decaps, idx, 9043 cache_resource, next); 9044 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx); 9045 DRV_LOG(DEBUG, "encap/decap resource %p: removed", 9046 (void *)cache_resource); 9047 return 0; 9048 } 9049 return 1; 9050 } 9051 9052 /** 9053 * Release an jump to table action resource. 9054 * 9055 * @param dev 9056 * Pointer to Ethernet device. 9057 * @param handle 9058 * Pointer to mlx5_flow_handle. 9059 * 9060 * @return 9061 * 1 while a reference on it exists, 0 when freed. 9062 */ 9063 static int 9064 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, 9065 struct mlx5_flow_handle *handle) 9066 { 9067 struct mlx5_priv *priv = dev->data->dev_private; 9068 struct mlx5_flow_dv_jump_tbl_resource *cache_resource; 9069 struct mlx5_flow_tbl_data_entry *tbl_data; 9070 9071 tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP], 9072 handle->rix_jump); 9073 if (!tbl_data) 9074 return 0; 9075 cache_resource = &tbl_data->jump; 9076 MLX5_ASSERT(cache_resource->action); 9077 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", 9078 (void *)cache_resource, 9079 rte_atomic32_read(&cache_resource->refcnt)); 9080 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9081 claim_zero(mlx5_flow_os_destroy_flow_action 9082 (cache_resource->action)); 9083 /* jump action memory free is inside the table release. */ 9084 flow_dv_tbl_resource_release(dev, &tbl_data->tbl); 9085 DRV_LOG(DEBUG, "jump table resource %p: removed", 9086 (void *)cache_resource); 9087 return 0; 9088 } 9089 return 1; 9090 } 9091 9092 /** 9093 * Release a default miss resource. 9094 * 9095 * @param dev 9096 * Pointer to Ethernet device. 9097 * @return 9098 * 1 while a reference on it exists, 0 when freed. 9099 */ 9100 static int 9101 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev) 9102 { 9103 struct mlx5_priv *priv = dev->data->dev_private; 9104 struct mlx5_dev_ctx_shared *sh = priv->sh; 9105 struct mlx5_flow_default_miss_resource *cache_resource = 9106 &sh->default_miss; 9107 9108 MLX5_ASSERT(cache_resource->action); 9109 DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--", 9110 (void *)cache_resource->action, 9111 rte_atomic32_read(&cache_resource->refcnt)); 9112 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9113 claim_zero(mlx5_glue->destroy_flow_action 9114 (cache_resource->action)); 9115 DRV_LOG(DEBUG, "default miss resource %p: removed", 9116 (void *)cache_resource->action); 9117 return 0; 9118 } 9119 return 1; 9120 } 9121 9122 /** 9123 * Release a modify-header resource. 9124 * 9125 * @param handle 9126 * Pointer to mlx5_flow_handle. 9127 * 9128 * @return 9129 * 1 while a reference on it exists, 0 when freed. 9130 */ 9131 static int 9132 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle) 9133 { 9134 struct mlx5_flow_dv_modify_hdr_resource *cache_resource = 9135 handle->dvh.modify_hdr; 9136 9137 MLX5_ASSERT(cache_resource->action); 9138 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--", 9139 (void *)cache_resource, 9140 rte_atomic32_read(&cache_resource->refcnt)); 9141 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9142 claim_zero(mlx5_flow_os_destroy_flow_action 9143 (cache_resource->action)); 9144 LIST_REMOVE(cache_resource, next); 9145 mlx5_free(cache_resource); 9146 DRV_LOG(DEBUG, "modify-header resource %p: removed", 9147 (void *)cache_resource); 9148 return 0; 9149 } 9150 return 1; 9151 } 9152 9153 /** 9154 * Release port ID action resource. 9155 * 9156 * @param dev 9157 * Pointer to Ethernet device. 9158 * @param handle 9159 * Pointer to mlx5_flow_handle. 9160 * 9161 * @return 9162 * 1 while a reference on it exists, 0 when freed. 9163 */ 9164 static int 9165 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, 9166 struct mlx5_flow_handle *handle) 9167 { 9168 struct mlx5_priv *priv = dev->data->dev_private; 9169 struct mlx5_flow_dv_port_id_action_resource *cache_resource; 9170 uint32_t idx = handle->rix_port_id_action; 9171 9172 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], 9173 idx); 9174 if (!cache_resource) 9175 return 0; 9176 MLX5_ASSERT(cache_resource->action); 9177 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", 9178 (void *)cache_resource, 9179 rte_atomic32_read(&cache_resource->refcnt)); 9180 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9181 claim_zero(mlx5_flow_os_destroy_flow_action 9182 (cache_resource->action)); 9183 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID], 9184 &priv->sh->port_id_action_list, idx, 9185 cache_resource, next); 9186 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx); 9187 DRV_LOG(DEBUG, "port id action resource %p: removed", 9188 (void *)cache_resource); 9189 return 0; 9190 } 9191 return 1; 9192 } 9193 9194 /** 9195 * Release push vlan action resource. 9196 * 9197 * @param dev 9198 * Pointer to Ethernet device. 9199 * @param handle 9200 * Pointer to mlx5_flow_handle. 9201 * 9202 * @return 9203 * 1 while a reference on it exists, 0 when freed. 9204 */ 9205 static int 9206 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, 9207 struct mlx5_flow_handle *handle) 9208 { 9209 struct mlx5_priv *priv = dev->data->dev_private; 9210 uint32_t idx = handle->dvh.rix_push_vlan; 9211 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; 9212 9213 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], 9214 idx); 9215 if (!cache_resource) 9216 return 0; 9217 MLX5_ASSERT(cache_resource->action); 9218 DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--", 9219 (void *)cache_resource, 9220 rte_atomic32_read(&cache_resource->refcnt)); 9221 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9222 claim_zero(mlx5_flow_os_destroy_flow_action 9223 (cache_resource->action)); 9224 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], 9225 &priv->sh->push_vlan_action_list, idx, 9226 cache_resource, next); 9227 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); 9228 DRV_LOG(DEBUG, "push vlan action resource %p: removed", 9229 (void *)cache_resource); 9230 return 0; 9231 } 9232 return 1; 9233 } 9234 9235 /** 9236 * Release the fate resource. 9237 * 9238 * @param dev 9239 * Pointer to Ethernet device. 9240 * @param handle 9241 * Pointer to mlx5_flow_handle. 9242 */ 9243 static void 9244 flow_dv_fate_resource_release(struct rte_eth_dev *dev, 9245 struct mlx5_flow_handle *handle) 9246 { 9247 if (!handle->rix_fate) 9248 return; 9249 switch (handle->fate_action) { 9250 case MLX5_FLOW_FATE_DROP: 9251 mlx5_hrxq_drop_release(dev); 9252 break; 9253 case MLX5_FLOW_FATE_QUEUE: 9254 mlx5_hrxq_release(dev, handle->rix_hrxq); 9255 break; 9256 case MLX5_FLOW_FATE_JUMP: 9257 flow_dv_jump_tbl_resource_release(dev, handle); 9258 break; 9259 case MLX5_FLOW_FATE_PORT_ID: 9260 flow_dv_port_id_action_resource_release(dev, handle); 9261 break; 9262 case MLX5_FLOW_FATE_DEFAULT_MISS: 9263 flow_dv_default_miss_resource_release(dev); 9264 break; 9265 default: 9266 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action); 9267 break; 9268 } 9269 handle->rix_fate = 0; 9270 } 9271 9272 /** 9273 * Remove the flow from the NIC but keeps it in memory. 9274 * Lock free, (mutex should be acquired by caller). 9275 * 9276 * @param[in] dev 9277 * Pointer to Ethernet device. 9278 * @param[in, out] flow 9279 * Pointer to flow structure. 9280 */ 9281 static void 9282 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 9283 { 9284 struct mlx5_flow_handle *dh; 9285 uint32_t handle_idx; 9286 struct mlx5_priv *priv = dev->data->dev_private; 9287 9288 if (!flow) 9289 return; 9290 handle_idx = flow->dev_handles; 9291 while (handle_idx) { 9292 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9293 handle_idx); 9294 if (!dh) 9295 return; 9296 if (dh->drv_flow) { 9297 claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow)); 9298 dh->drv_flow = NULL; 9299 } 9300 if (dh->fate_action == MLX5_FLOW_FATE_DROP || 9301 dh->fate_action == MLX5_FLOW_FATE_QUEUE || 9302 dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) 9303 flow_dv_fate_resource_release(dev, dh); 9304 if (dh->vf_vlan.tag && dh->vf_vlan.created) 9305 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); 9306 handle_idx = dh->next.next; 9307 } 9308 } 9309 9310 /** 9311 * Remove the flow from the NIC and the memory. 9312 * Lock free, (mutex should be acquired by caller). 9313 * 9314 * @param[in] dev 9315 * Pointer to the Ethernet device structure. 9316 * @param[in, out] flow 9317 * Pointer to flow structure. 9318 */ 9319 static void 9320 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 9321 { 9322 struct mlx5_flow_handle *dev_handle; 9323 struct mlx5_priv *priv = dev->data->dev_private; 9324 9325 if (!flow) 9326 return; 9327 __flow_dv_remove(dev, flow); 9328 if (flow->counter) { 9329 flow_dv_counter_release(dev, flow->counter); 9330 flow->counter = 0; 9331 } 9332 if (flow->meter) { 9333 struct mlx5_flow_meter *fm; 9334 9335 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], 9336 flow->meter); 9337 if (fm) 9338 mlx5_flow_meter_detach(fm); 9339 flow->meter = 0; 9340 } 9341 while (flow->dev_handles) { 9342 uint32_t tmp_idx = flow->dev_handles; 9343 9344 dev_handle = mlx5_ipool_get(priv->sh->ipool 9345 [MLX5_IPOOL_MLX5_FLOW], tmp_idx); 9346 if (!dev_handle) 9347 return; 9348 flow->dev_handles = dev_handle->next.next; 9349 if (dev_handle->dvh.matcher) 9350 flow_dv_matcher_release(dev, dev_handle); 9351 if (dev_handle->dvh.rix_encap_decap) 9352 flow_dv_encap_decap_resource_release(dev, dev_handle); 9353 if (dev_handle->dvh.modify_hdr) 9354 flow_dv_modify_hdr_resource_release(dev_handle); 9355 if (dev_handle->dvh.rix_push_vlan) 9356 flow_dv_push_vlan_action_resource_release(dev, 9357 dev_handle); 9358 if (dev_handle->dvh.rix_tag) 9359 flow_dv_tag_release(dev, 9360 dev_handle->dvh.rix_tag); 9361 flow_dv_fate_resource_release(dev, dev_handle); 9362 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9363 tmp_idx); 9364 } 9365 } 9366 9367 /** 9368 * Query a dv flow rule for its statistics via devx. 9369 * 9370 * @param[in] dev 9371 * Pointer to Ethernet device. 9372 * @param[in] flow 9373 * Pointer to the sub flow. 9374 * @param[out] data 9375 * data retrieved by the query. 9376 * @param[out] error 9377 * Perform verbose error reporting if not NULL. 9378 * 9379 * @return 9380 * 0 on success, a negative errno value otherwise and rte_errno is set. 9381 */ 9382 static int 9383 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, 9384 void *data, struct rte_flow_error *error) 9385 { 9386 struct mlx5_priv *priv = dev->data->dev_private; 9387 struct rte_flow_query_count *qc = data; 9388 9389 if (!priv->config.devx) 9390 return rte_flow_error_set(error, ENOTSUP, 9391 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9392 NULL, 9393 "counters are not supported"); 9394 if (flow->counter) { 9395 uint64_t pkts, bytes; 9396 struct mlx5_flow_counter *cnt; 9397 9398 cnt = flow_dv_counter_get_by_idx(dev, flow->counter, 9399 NULL); 9400 int err = _flow_dv_query_count(dev, flow->counter, &pkts, 9401 &bytes); 9402 9403 if (err) 9404 return rte_flow_error_set(error, -err, 9405 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9406 NULL, "cannot read counters"); 9407 qc->hits_set = 1; 9408 qc->bytes_set = 1; 9409 qc->hits = pkts - cnt->hits; 9410 qc->bytes = bytes - cnt->bytes; 9411 if (qc->reset) { 9412 cnt->hits = pkts; 9413 cnt->bytes = bytes; 9414 } 9415 return 0; 9416 } 9417 return rte_flow_error_set(error, EINVAL, 9418 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9419 NULL, 9420 "counters are not available"); 9421 } 9422 9423 /** 9424 * Query a flow. 9425 * 9426 * @see rte_flow_query() 9427 * @see rte_flow_ops 9428 */ 9429 static int 9430 flow_dv_query(struct rte_eth_dev *dev, 9431 struct rte_flow *flow __rte_unused, 9432 const struct rte_flow_action *actions __rte_unused, 9433 void *data __rte_unused, 9434 struct rte_flow_error *error __rte_unused) 9435 { 9436 int ret = -EINVAL; 9437 9438 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 9439 switch (actions->type) { 9440 case RTE_FLOW_ACTION_TYPE_VOID: 9441 break; 9442 case RTE_FLOW_ACTION_TYPE_COUNT: 9443 ret = flow_dv_query_count(dev, flow, data, error); 9444 break; 9445 default: 9446 return rte_flow_error_set(error, ENOTSUP, 9447 RTE_FLOW_ERROR_TYPE_ACTION, 9448 actions, 9449 "action not supported"); 9450 } 9451 } 9452 return ret; 9453 } 9454 9455 /** 9456 * Destroy the meter table set. 9457 * Lock free, (mutex should be acquired by caller). 9458 * 9459 * @param[in] dev 9460 * Pointer to Ethernet device. 9461 * @param[in] tbl 9462 * Pointer to the meter table set. 9463 * 9464 * @return 9465 * Always 0. 9466 */ 9467 static int 9468 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, 9469 struct mlx5_meter_domains_infos *tbl) 9470 { 9471 struct mlx5_priv *priv = dev->data->dev_private; 9472 struct mlx5_meter_domains_infos *mtd = 9473 (struct mlx5_meter_domains_infos *)tbl; 9474 9475 if (!mtd || !priv->config.dv_flow_en) 9476 return 0; 9477 if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) 9478 claim_zero(mlx5_flow_os_destroy_flow 9479 (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); 9480 if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) 9481 claim_zero(mlx5_flow_os_destroy_flow 9482 (mtd->egress.policer_rules[RTE_MTR_DROPPED])); 9483 if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) 9484 claim_zero(mlx5_flow_os_destroy_flow 9485 (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); 9486 if (mtd->egress.color_matcher) 9487 claim_zero(mlx5_flow_os_destroy_flow_matcher 9488 (mtd->egress.color_matcher)); 9489 if (mtd->egress.any_matcher) 9490 claim_zero(mlx5_flow_os_destroy_flow_matcher 9491 (mtd->egress.any_matcher)); 9492 if (mtd->egress.tbl) 9493 flow_dv_tbl_resource_release(dev, mtd->egress.tbl); 9494 if (mtd->egress.sfx_tbl) 9495 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl); 9496 if (mtd->ingress.color_matcher) 9497 claim_zero(mlx5_flow_os_destroy_flow_matcher 9498 (mtd->ingress.color_matcher)); 9499 if (mtd->ingress.any_matcher) 9500 claim_zero(mlx5_flow_os_destroy_flow_matcher 9501 (mtd->ingress.any_matcher)); 9502 if (mtd->ingress.tbl) 9503 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl); 9504 if (mtd->ingress.sfx_tbl) 9505 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl); 9506 if (mtd->transfer.color_matcher) 9507 claim_zero(mlx5_flow_os_destroy_flow_matcher 9508 (mtd->transfer.color_matcher)); 9509 if (mtd->transfer.any_matcher) 9510 claim_zero(mlx5_flow_os_destroy_flow_matcher 9511 (mtd->transfer.any_matcher)); 9512 if (mtd->transfer.tbl) 9513 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl); 9514 if (mtd->transfer.sfx_tbl) 9515 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl); 9516 if (mtd->drop_actn) 9517 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); 9518 mlx5_free(mtd); 9519 return 0; 9520 } 9521 9522 /* Number of meter flow actions, count and jump or count and drop. */ 9523 #define METER_ACTIONS 2 9524 9525 /** 9526 * Create specify domain meter table and suffix table. 9527 * 9528 * @param[in] dev 9529 * Pointer to Ethernet device. 9530 * @param[in,out] mtb 9531 * Pointer to DV meter table set. 9532 * @param[in] egress 9533 * Table attribute. 9534 * @param[in] transfer 9535 * Table attribute. 9536 * @param[in] color_reg_c_idx 9537 * Reg C index for color match. 9538 * 9539 * @return 9540 * 0 on success, -1 otherwise and rte_errno is set. 9541 */ 9542 static int 9543 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, 9544 struct mlx5_meter_domains_infos *mtb, 9545 uint8_t egress, uint8_t transfer, 9546 uint32_t color_reg_c_idx) 9547 { 9548 struct mlx5_priv *priv = dev->data->dev_private; 9549 struct mlx5_dev_ctx_shared *sh = priv->sh; 9550 struct mlx5_flow_dv_match_params mask = { 9551 .size = sizeof(mask.buf), 9552 }; 9553 struct mlx5_flow_dv_match_params value = { 9554 .size = sizeof(value.buf), 9555 }; 9556 struct mlx5dv_flow_matcher_attr dv_attr = { 9557 .type = IBV_FLOW_ATTR_NORMAL, 9558 .priority = 0, 9559 .match_criteria_enable = 0, 9560 .match_mask = (void *)&mask, 9561 }; 9562 void *actions[METER_ACTIONS]; 9563 struct mlx5_meter_domain_info *dtb; 9564 struct rte_flow_error error; 9565 int i = 0; 9566 int ret; 9567 9568 if (transfer) 9569 dtb = &mtb->transfer; 9570 else if (egress) 9571 dtb = &mtb->egress; 9572 else 9573 dtb = &mtb->ingress; 9574 /* Create the meter table with METER level. */ 9575 dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, 9576 egress, transfer, &error); 9577 if (!dtb->tbl) { 9578 DRV_LOG(ERR, "Failed to create meter policer table."); 9579 return -1; 9580 } 9581 /* Create the meter suffix table with SUFFIX level. */ 9582 dtb->sfx_tbl = flow_dv_tbl_resource_get(dev, 9583 MLX5_FLOW_TABLE_LEVEL_SUFFIX, 9584 egress, transfer, &error); 9585 if (!dtb->sfx_tbl) { 9586 DRV_LOG(ERR, "Failed to create meter suffix table."); 9587 return -1; 9588 } 9589 /* Create matchers, Any and Color. */ 9590 dv_attr.priority = 3; 9591 dv_attr.match_criteria_enable = 0; 9592 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, 9593 &dtb->any_matcher); 9594 if (ret) { 9595 DRV_LOG(ERR, "Failed to create meter" 9596 " policer default matcher."); 9597 goto error_exit; 9598 } 9599 dv_attr.priority = 0; 9600 dv_attr.match_criteria_enable = 9601 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 9602 flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, 9603 rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); 9604 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, 9605 &dtb->color_matcher); 9606 if (ret) { 9607 DRV_LOG(ERR, "Failed to create meter policer color matcher."); 9608 goto error_exit; 9609 } 9610 if (mtb->count_actns[RTE_MTR_DROPPED]) 9611 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; 9612 actions[i++] = mtb->drop_actn; 9613 /* Default rule: lowest priority, match any, actions: drop. */ 9614 ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, 9615 actions, 9616 &dtb->policer_rules[RTE_MTR_DROPPED]); 9617 if (ret) { 9618 DRV_LOG(ERR, "Failed to create meter policer drop rule."); 9619 goto error_exit; 9620 } 9621 return 0; 9622 error_exit: 9623 return -1; 9624 } 9625 9626 /** 9627 * Create the needed meter and suffix tables. 9628 * Lock free, (mutex should be acquired by caller). 9629 * 9630 * @param[in] dev 9631 * Pointer to Ethernet device. 9632 * @param[in] fm 9633 * Pointer to the flow meter. 9634 * 9635 * @return 9636 * Pointer to table set on success, NULL otherwise and rte_errno is set. 9637 */ 9638 static struct mlx5_meter_domains_infos * 9639 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, 9640 const struct mlx5_flow_meter *fm) 9641 { 9642 struct mlx5_priv *priv = dev->data->dev_private; 9643 struct mlx5_meter_domains_infos *mtb; 9644 int ret; 9645 int i; 9646 9647 if (!priv->mtr_en) { 9648 rte_errno = ENOTSUP; 9649 return NULL; 9650 } 9651 mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY); 9652 if (!mtb) { 9653 DRV_LOG(ERR, "Failed to allocate memory for meter."); 9654 return NULL; 9655 } 9656 /* Create meter count actions */ 9657 for (i = 0; i <= RTE_MTR_DROPPED; i++) { 9658 struct mlx5_flow_counter *cnt; 9659 if (!fm->policer_stats.cnt[i]) 9660 continue; 9661 cnt = flow_dv_counter_get_by_idx(dev, 9662 fm->policer_stats.cnt[i], NULL); 9663 mtb->count_actns[i] = cnt->action; 9664 } 9665 /* Create drop action. */ 9666 ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); 9667 if (ret) { 9668 DRV_LOG(ERR, "Failed to create drop action."); 9669 goto error_exit; 9670 } 9671 /* Egress meter table. */ 9672 ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); 9673 if (ret) { 9674 DRV_LOG(ERR, "Failed to prepare egress meter table."); 9675 goto error_exit; 9676 } 9677 /* Ingress meter table. */ 9678 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); 9679 if (ret) { 9680 DRV_LOG(ERR, "Failed to prepare ingress meter table."); 9681 goto error_exit; 9682 } 9683 /* FDB meter table. */ 9684 if (priv->config.dv_esw_en) { 9685 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, 9686 priv->mtr_color_reg); 9687 if (ret) { 9688 DRV_LOG(ERR, "Failed to prepare fdb meter table."); 9689 goto error_exit; 9690 } 9691 } 9692 return mtb; 9693 error_exit: 9694 flow_dv_destroy_mtr_tbl(dev, mtb); 9695 return NULL; 9696 } 9697 9698 /** 9699 * Destroy domain policer rule. 9700 * 9701 * @param[in] dt 9702 * Pointer to domain table. 9703 */ 9704 static void 9705 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) 9706 { 9707 int i; 9708 9709 for (i = 0; i < RTE_MTR_DROPPED; i++) { 9710 if (dt->policer_rules[i]) { 9711 claim_zero(mlx5_flow_os_destroy_flow 9712 (dt->policer_rules[i])); 9713 dt->policer_rules[i] = NULL; 9714 } 9715 } 9716 if (dt->jump_actn) { 9717 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); 9718 dt->jump_actn = NULL; 9719 } 9720 } 9721 9722 /** 9723 * Destroy policer rules. 9724 * 9725 * @param[in] dev 9726 * Pointer to Ethernet device. 9727 * @param[in] fm 9728 * Pointer to flow meter structure. 9729 * @param[in] attr 9730 * Pointer to flow attributes. 9731 * 9732 * @return 9733 * Always 0. 9734 */ 9735 static int 9736 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, 9737 const struct mlx5_flow_meter *fm, 9738 const struct rte_flow_attr *attr) 9739 { 9740 struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; 9741 9742 if (!mtb) 9743 return 0; 9744 if (attr->egress) 9745 flow_dv_destroy_domain_policer_rule(&mtb->egress); 9746 if (attr->ingress) 9747 flow_dv_destroy_domain_policer_rule(&mtb->ingress); 9748 if (attr->transfer) 9749 flow_dv_destroy_domain_policer_rule(&mtb->transfer); 9750 return 0; 9751 } 9752 9753 /** 9754 * Create specify domain meter policer rule. 9755 * 9756 * @param[in] fm 9757 * Pointer to flow meter structure. 9758 * @param[in] mtb 9759 * Pointer to DV meter table set. 9760 * @param[in] mtr_reg_c 9761 * Color match REG_C. 9762 * 9763 * @return 9764 * 0 on success, -1 otherwise. 9765 */ 9766 static int 9767 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, 9768 struct mlx5_meter_domain_info *dtb, 9769 uint8_t mtr_reg_c) 9770 { 9771 struct mlx5_flow_dv_match_params matcher = { 9772 .size = sizeof(matcher.buf), 9773 }; 9774 struct mlx5_flow_dv_match_params value = { 9775 .size = sizeof(value.buf), 9776 }; 9777 struct mlx5_meter_domains_infos *mtb = fm->mfts; 9778 void *actions[METER_ACTIONS]; 9779 int i; 9780 int ret = 0; 9781 9782 /* Create jump action. */ 9783 if (!dtb->jump_actn) 9784 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl 9785 (dtb->sfx_tbl->obj, &dtb->jump_actn); 9786 if (ret) { 9787 DRV_LOG(ERR, "Failed to create policer jump action."); 9788 goto error; 9789 } 9790 for (i = 0; i < RTE_MTR_DROPPED; i++) { 9791 int j = 0; 9792 9793 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, 9794 rte_col_2_mlx5_col(i), UINT8_MAX); 9795 if (mtb->count_actns[i]) 9796 actions[j++] = mtb->count_actns[i]; 9797 if (fm->action[i] == MTR_POLICER_ACTION_DROP) 9798 actions[j++] = mtb->drop_actn; 9799 else 9800 actions[j++] = dtb->jump_actn; 9801 ret = mlx5_flow_os_create_flow(dtb->color_matcher, 9802 (void *)&value, j, actions, 9803 &dtb->policer_rules[i]); 9804 if (ret) { 9805 DRV_LOG(ERR, "Failed to create policer rule."); 9806 goto error; 9807 } 9808 } 9809 return 0; 9810 error: 9811 rte_errno = errno; 9812 return -1; 9813 } 9814 9815 /** 9816 * Create policer rules. 9817 * 9818 * @param[in] dev 9819 * Pointer to Ethernet device. 9820 * @param[in] fm 9821 * Pointer to flow meter structure. 9822 * @param[in] attr 9823 * Pointer to flow attributes. 9824 * 9825 * @return 9826 * 0 on success, -1 otherwise. 9827 */ 9828 static int 9829 flow_dv_create_policer_rules(struct rte_eth_dev *dev, 9830 struct mlx5_flow_meter *fm, 9831 const struct rte_flow_attr *attr) 9832 { 9833 struct mlx5_priv *priv = dev->data->dev_private; 9834 struct mlx5_meter_domains_infos *mtb = fm->mfts; 9835 int ret; 9836 9837 if (attr->egress) { 9838 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, 9839 priv->mtr_color_reg); 9840 if (ret) { 9841 DRV_LOG(ERR, "Failed to create egress policer."); 9842 goto error; 9843 } 9844 } 9845 if (attr->ingress) { 9846 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, 9847 priv->mtr_color_reg); 9848 if (ret) { 9849 DRV_LOG(ERR, "Failed to create ingress policer."); 9850 goto error; 9851 } 9852 } 9853 if (attr->transfer) { 9854 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, 9855 priv->mtr_color_reg); 9856 if (ret) { 9857 DRV_LOG(ERR, "Failed to create transfer policer."); 9858 goto error; 9859 } 9860 } 9861 return 0; 9862 error: 9863 flow_dv_destroy_policer_rules(dev, fm, attr); 9864 return -1; 9865 } 9866 9867 /** 9868 * Query a devx counter. 9869 * 9870 * @param[in] dev 9871 * Pointer to the Ethernet device structure. 9872 * @param[in] cnt 9873 * Index to the flow counter. 9874 * @param[in] clear 9875 * Set to clear the counter statistics. 9876 * @param[out] pkts 9877 * The statistics value of packets. 9878 * @param[out] bytes 9879 * The statistics value of bytes. 9880 * 9881 * @return 9882 * 0 on success, otherwise return -1. 9883 */ 9884 static int 9885 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear, 9886 uint64_t *pkts, uint64_t *bytes) 9887 { 9888 struct mlx5_priv *priv = dev->data->dev_private; 9889 struct mlx5_flow_counter *cnt; 9890 uint64_t inn_pkts, inn_bytes; 9891 int ret; 9892 9893 if (!priv->config.devx) 9894 return -1; 9895 9896 ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes); 9897 if (ret) 9898 return -1; 9899 cnt = flow_dv_counter_get_by_idx(dev, counter, NULL); 9900 *pkts = inn_pkts - cnt->hits; 9901 *bytes = inn_bytes - cnt->bytes; 9902 if (clear) { 9903 cnt->hits = inn_pkts; 9904 cnt->bytes = inn_bytes; 9905 } 9906 return 0; 9907 } 9908 9909 /** 9910 * Get aged-out flows. 9911 * 9912 * @param[in] dev 9913 * Pointer to the Ethernet device structure. 9914 * @param[in] context 9915 * The address of an array of pointers to the aged-out flows contexts. 9916 * @param[in] nb_contexts 9917 * The length of context array pointers. 9918 * @param[out] error 9919 * Perform verbose error reporting if not NULL. Initialized in case of 9920 * error only. 9921 * 9922 * @return 9923 * how many contexts get in success, otherwise negative errno value. 9924 * if nb_contexts is 0, return the amount of all aged contexts. 9925 * if nb_contexts is not 0 , return the amount of aged flows reported 9926 * in the context array. 9927 * @note: only stub for now 9928 */ 9929 static int 9930 flow_get_aged_flows(struct rte_eth_dev *dev, 9931 void **context, 9932 uint32_t nb_contexts, 9933 struct rte_flow_error *error) 9934 { 9935 struct mlx5_priv *priv = dev->data->dev_private; 9936 struct mlx5_age_info *age_info; 9937 struct mlx5_age_param *age_param; 9938 struct mlx5_flow_counter *counter; 9939 int nb_flows = 0; 9940 9941 if (nb_contexts && !context) 9942 return rte_flow_error_set(error, EINVAL, 9943 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9944 NULL, 9945 "Should assign at least one flow or" 9946 " context to get if nb_contexts != 0"); 9947 age_info = GET_PORT_AGE_INFO(priv); 9948 rte_spinlock_lock(&age_info->aged_sl); 9949 TAILQ_FOREACH(counter, &age_info->aged_counters, next) { 9950 nb_flows++; 9951 if (nb_contexts) { 9952 age_param = MLX5_CNT_TO_AGE(counter); 9953 context[nb_flows - 1] = age_param->context; 9954 if (!(--nb_contexts)) 9955 break; 9956 } 9957 } 9958 rte_spinlock_unlock(&age_info->aged_sl); 9959 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 9960 return nb_flows; 9961 } 9962 9963 /* 9964 * Mutex-protected thunk to lock-free __flow_dv_translate(). 9965 */ 9966 static int 9967 flow_dv_translate(struct rte_eth_dev *dev, 9968 struct mlx5_flow *dev_flow, 9969 const struct rte_flow_attr *attr, 9970 const struct rte_flow_item items[], 9971 const struct rte_flow_action actions[], 9972 struct rte_flow_error *error) 9973 { 9974 int ret; 9975 9976 flow_dv_shared_lock(dev); 9977 ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error); 9978 flow_dv_shared_unlock(dev); 9979 return ret; 9980 } 9981 9982 /* 9983 * Mutex-protected thunk to lock-free __flow_dv_apply(). 9984 */ 9985 static int 9986 flow_dv_apply(struct rte_eth_dev *dev, 9987 struct rte_flow *flow, 9988 struct rte_flow_error *error) 9989 { 9990 int ret; 9991 9992 flow_dv_shared_lock(dev); 9993 ret = __flow_dv_apply(dev, flow, error); 9994 flow_dv_shared_unlock(dev); 9995 return ret; 9996 } 9997 9998 /* 9999 * Mutex-protected thunk to lock-free __flow_dv_remove(). 10000 */ 10001 static void 10002 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 10003 { 10004 flow_dv_shared_lock(dev); 10005 __flow_dv_remove(dev, flow); 10006 flow_dv_shared_unlock(dev); 10007 } 10008 10009 /* 10010 * Mutex-protected thunk to lock-free __flow_dv_destroy(). 10011 */ 10012 static void 10013 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 10014 { 10015 flow_dv_shared_lock(dev); 10016 __flow_dv_destroy(dev, flow); 10017 flow_dv_shared_unlock(dev); 10018 } 10019 10020 /* 10021 * Mutex-protected thunk to lock-free flow_dv_counter_alloc(). 10022 */ 10023 static uint32_t 10024 flow_dv_counter_allocate(struct rte_eth_dev *dev) 10025 { 10026 uint32_t cnt; 10027 10028 flow_dv_shared_lock(dev); 10029 cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0); 10030 flow_dv_shared_unlock(dev); 10031 return cnt; 10032 } 10033 10034 /* 10035 * Mutex-protected thunk to lock-free flow_dv_counter_release(). 10036 */ 10037 static void 10038 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 10039 { 10040 flow_dv_shared_lock(dev); 10041 flow_dv_counter_release(dev, cnt); 10042 flow_dv_shared_unlock(dev); 10043 } 10044 10045 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { 10046 .validate = flow_dv_validate, 10047 .prepare = flow_dv_prepare, 10048 .translate = flow_dv_translate, 10049 .apply = flow_dv_apply, 10050 .remove = flow_dv_remove, 10051 .destroy = flow_dv_destroy, 10052 .query = flow_dv_query, 10053 .create_mtr_tbls = flow_dv_create_mtr_tbl, 10054 .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, 10055 .create_policer_rules = flow_dv_create_policer_rules, 10056 .destroy_policer_rules = flow_dv_destroy_policer_rules, 10057 .counter_alloc = flow_dv_counter_allocate, 10058 .counter_free = flow_dv_counter_free, 10059 .counter_query = flow_dv_counter_query, 10060 .get_aged_flows = flow_get_aged_flows, 10061 }; 10062 10063 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10064 10065