1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <unistd.h> 10 11 #include <rte_common.h> 12 #include <rte_ether.h> 13 #include <ethdev_driver.h> 14 #include <rte_flow.h> 15 #include <rte_flow_driver.h> 16 #include <rte_malloc.h> 17 #include <rte_cycles.h> 18 #include <rte_bus_pci.h> 19 #include <rte_ip.h> 20 #include <rte_gre.h> 21 #include <rte_vxlan.h> 22 #include <rte_gtp.h> 23 #include <rte_eal_paging.h> 24 #include <rte_mpls.h> 25 #include <rte_mtr.h> 26 #include <rte_mtr_driver.h> 27 #include <rte_tailq.h> 28 29 #include <mlx5_glue.h> 30 #include <mlx5_devx_cmds.h> 31 #include <mlx5_prm.h> 32 #include <mlx5_malloc.h> 33 34 #include "mlx5_defs.h" 35 #include "mlx5.h" 36 #include "mlx5_common_os.h" 37 #include "mlx5_flow.h" 38 #include "mlx5_flow_os.h" 39 #include "mlx5_rx.h" 40 #include "mlx5_tx.h" 41 #include "rte_pmd_mlx5.h" 42 43 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 44 45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS 46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 47 #endif 48 49 #ifndef HAVE_MLX5DV_DR_ESWITCH 50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB 51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0 52 #endif 53 #endif 54 55 #ifndef HAVE_MLX5DV_DR 56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1 57 #endif 58 59 /* VLAN header definitions */ 60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13 61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT) 62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff 63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK) 64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK) 65 66 union flow_dv_attr { 67 struct { 68 uint32_t valid:1; 69 uint32_t ipv4:1; 70 uint32_t ipv6:1; 71 uint32_t tcp:1; 72 uint32_t udp:1; 73 uint32_t reserved:27; 74 }; 75 uint32_t attr; 76 }; 77 78 static int 79 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh, 80 struct mlx5_flow_tbl_resource *tbl); 81 82 static int 83 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, 84 uint32_t encap_decap_idx); 85 86 static int 87 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, 88 uint32_t port_id); 89 static void 90 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss); 91 92 static int 93 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, 94 uint32_t rix_jump); 95 96 static int16_t 97 flow_dv_get_esw_manager_vport_id(struct rte_eth_dev *dev) 98 { 99 struct mlx5_priv *priv = dev->data->dev_private; 100 struct mlx5_common_device *cdev = priv->sh->cdev; 101 102 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 103 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 104 105 if (priv->pci_dev == NULL) 106 return 0; 107 switch (priv->pci_dev->id.device_id) { 108 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF: 109 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 110 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 111 return (int16_t)0xfffe; 112 default: 113 return 0; 114 } 115 } 116 117 /** 118 * Initialize flow attributes structure according to flow items' types. 119 * 120 * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel 121 * mode. For tunnel mode, the items to be modified are the outermost ones. 122 * 123 * @param[in] item 124 * Pointer to item specification. 125 * @param[out] attr 126 * Pointer to flow attributes structure. 127 * @param[in] dev_flow 128 * Pointer to the sub flow. 129 * @param[in] tunnel_decap 130 * Whether action is after tunnel decapsulation. 131 */ 132 static void 133 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr, 134 struct mlx5_flow *dev_flow, bool tunnel_decap) 135 { 136 uint64_t layers = dev_flow->handle->layers; 137 138 /* 139 * If layers is already initialized, it means this dev_flow is the 140 * suffix flow, the layers flags is set by the prefix flow. Need to 141 * use the layer flags from prefix flow as the suffix flow may not 142 * have the user defined items as the flow is split. 143 */ 144 if (layers) { 145 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4) 146 attr->ipv4 = 1; 147 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6) 148 attr->ipv6 = 1; 149 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP) 150 attr->tcp = 1; 151 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP) 152 attr->udp = 1; 153 attr->valid = 1; 154 return; 155 } 156 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 157 uint8_t next_protocol = 0xff; 158 switch (item->type) { 159 case RTE_FLOW_ITEM_TYPE_GRE: 160 case RTE_FLOW_ITEM_TYPE_NVGRE: 161 case RTE_FLOW_ITEM_TYPE_VXLAN: 162 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 163 case RTE_FLOW_ITEM_TYPE_GENEVE: 164 case RTE_FLOW_ITEM_TYPE_MPLS: 165 case RTE_FLOW_ITEM_TYPE_GTP: 166 if (tunnel_decap) 167 attr->attr = 0; 168 break; 169 case RTE_FLOW_ITEM_TYPE_IPV4: 170 if (!attr->ipv6) 171 attr->ipv4 = 1; 172 if (item->mask != NULL && 173 ((const struct rte_flow_item_ipv4 *) 174 item->mask)->hdr.next_proto_id) 175 next_protocol = 176 ((const struct rte_flow_item_ipv4 *) 177 (item->spec))->hdr.next_proto_id & 178 ((const struct rte_flow_item_ipv4 *) 179 (item->mask))->hdr.next_proto_id; 180 if ((next_protocol == IPPROTO_IPIP || 181 next_protocol == IPPROTO_IPV6) && tunnel_decap) 182 attr->attr = 0; 183 break; 184 case RTE_FLOW_ITEM_TYPE_IPV6: 185 if (!attr->ipv4) 186 attr->ipv6 = 1; 187 if (item->mask != NULL && 188 ((const struct rte_flow_item_ipv6 *) 189 item->mask)->hdr.proto) 190 next_protocol = 191 ((const struct rte_flow_item_ipv6 *) 192 (item->spec))->hdr.proto & 193 ((const struct rte_flow_item_ipv6 *) 194 (item->mask))->hdr.proto; 195 if ((next_protocol == IPPROTO_IPIP || 196 next_protocol == IPPROTO_IPV6) && tunnel_decap) 197 attr->attr = 0; 198 break; 199 case RTE_FLOW_ITEM_TYPE_UDP: 200 if (!attr->tcp) 201 attr->udp = 1; 202 break; 203 case RTE_FLOW_ITEM_TYPE_TCP: 204 if (!attr->udp) 205 attr->tcp = 1; 206 break; 207 default: 208 break; 209 } 210 } 211 attr->valid = 1; 212 } 213 214 /* 215 * Convert rte_mtr_color to mlx5 color. 216 * 217 * @param[in] rcol 218 * rte_mtr_color. 219 * 220 * @return 221 * mlx5 color. 222 */ 223 static inline int 224 rte_col_2_mlx5_col(enum rte_color rcol) 225 { 226 switch (rcol) { 227 case RTE_COLOR_GREEN: 228 return MLX5_FLOW_COLOR_GREEN; 229 case RTE_COLOR_YELLOW: 230 return MLX5_FLOW_COLOR_YELLOW; 231 case RTE_COLOR_RED: 232 return MLX5_FLOW_COLOR_RED; 233 default: 234 break; 235 } 236 return MLX5_FLOW_COLOR_UNDEFINED; 237 } 238 239 struct field_modify_info { 240 uint32_t size; /* Size of field in protocol header, in bytes. */ 241 uint32_t offset; /* Offset of field in protocol header, in bytes. */ 242 enum mlx5_modification_field id; 243 }; 244 245 struct field_modify_info modify_eth[] = { 246 {4, 0, MLX5_MODI_OUT_DMAC_47_16}, 247 {2, 4, MLX5_MODI_OUT_DMAC_15_0}, 248 {4, 6, MLX5_MODI_OUT_SMAC_47_16}, 249 {2, 10, MLX5_MODI_OUT_SMAC_15_0}, 250 {0, 0, 0}, 251 }; 252 253 struct field_modify_info modify_vlan_out_first_vid[] = { 254 /* Size in bits !!! */ 255 {12, 0, MLX5_MODI_OUT_FIRST_VID}, 256 {0, 0, 0}, 257 }; 258 259 struct field_modify_info modify_ipv4[] = { 260 {1, 1, MLX5_MODI_OUT_IP_DSCP}, 261 {1, 8, MLX5_MODI_OUT_IPV4_TTL}, 262 {4, 12, MLX5_MODI_OUT_SIPV4}, 263 {4, 16, MLX5_MODI_OUT_DIPV4}, 264 {0, 0, 0}, 265 }; 266 267 struct field_modify_info modify_ipv6[] = { 268 {1, 0, MLX5_MODI_OUT_IP_DSCP}, 269 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT}, 270 {4, 8, MLX5_MODI_OUT_SIPV6_127_96}, 271 {4, 12, MLX5_MODI_OUT_SIPV6_95_64}, 272 {4, 16, MLX5_MODI_OUT_SIPV6_63_32}, 273 {4, 20, MLX5_MODI_OUT_SIPV6_31_0}, 274 {4, 24, MLX5_MODI_OUT_DIPV6_127_96}, 275 {4, 28, MLX5_MODI_OUT_DIPV6_95_64}, 276 {4, 32, MLX5_MODI_OUT_DIPV6_63_32}, 277 {4, 36, MLX5_MODI_OUT_DIPV6_31_0}, 278 {0, 0, 0}, 279 }; 280 281 struct field_modify_info modify_udp[] = { 282 {2, 0, MLX5_MODI_OUT_UDP_SPORT}, 283 {2, 2, MLX5_MODI_OUT_UDP_DPORT}, 284 {0, 0, 0}, 285 }; 286 287 struct field_modify_info modify_tcp[] = { 288 {2, 0, MLX5_MODI_OUT_TCP_SPORT}, 289 {2, 2, MLX5_MODI_OUT_TCP_DPORT}, 290 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM}, 291 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM}, 292 {0, 0, 0}, 293 }; 294 295 static void 296 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, 297 uint8_t next_protocol, uint64_t *item_flags, 298 int *tunnel) 299 { 300 MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 || 301 item->type == RTE_FLOW_ITEM_TYPE_IPV6); 302 if (next_protocol == IPPROTO_IPIP) { 303 *item_flags |= MLX5_FLOW_LAYER_IPIP; 304 *tunnel = 1; 305 } 306 if (next_protocol == IPPROTO_IPV6) { 307 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP; 308 *tunnel = 1; 309 } 310 } 311 312 static inline struct mlx5_hlist * 313 flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl, 314 const char *name, uint32_t size, bool direct_key, 315 bool lcores_share, void *ctx, 316 mlx5_list_create_cb cb_create, 317 mlx5_list_match_cb cb_match, 318 mlx5_list_remove_cb cb_remove, 319 mlx5_list_clone_cb cb_clone, 320 mlx5_list_clone_free_cb cb_clone_free, 321 struct rte_flow_error *error) 322 { 323 struct mlx5_hlist *hl; 324 struct mlx5_hlist *expected = NULL; 325 char s[MLX5_NAME_SIZE]; 326 327 hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST); 328 if (likely(hl)) 329 return hl; 330 snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name); 331 hl = mlx5_hlist_create(s, size, direct_key, lcores_share, 332 ctx, cb_create, cb_match, cb_remove, cb_clone, 333 cb_clone_free); 334 if (!hl) { 335 DRV_LOG(ERR, "%s hash creation failed", name); 336 rte_flow_error_set(error, ENOMEM, 337 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 338 "cannot allocate resource memory"); 339 return NULL; 340 } 341 if (!__atomic_compare_exchange_n(phl, &expected, hl, false, 342 __ATOMIC_SEQ_CST, 343 __ATOMIC_SEQ_CST)) { 344 mlx5_hlist_destroy(hl); 345 hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST); 346 } 347 return hl; 348 } 349 350 /* Update VLAN's VID/PCP based on input rte_flow_action. 351 * 352 * @param[in] action 353 * Pointer to struct rte_flow_action. 354 * @param[out] vlan 355 * Pointer to struct rte_vlan_hdr. 356 */ 357 static void 358 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action, 359 struct rte_vlan_hdr *vlan) 360 { 361 uint16_t vlan_tci; 362 if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) { 363 vlan_tci = 364 ((const struct rte_flow_action_of_set_vlan_pcp *) 365 action->conf)->vlan_pcp; 366 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT; 367 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; 368 vlan->vlan_tci |= vlan_tci; 369 } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) { 370 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK; 371 vlan->vlan_tci |= rte_be_to_cpu_16 372 (((const struct rte_flow_action_of_set_vlan_vid *) 373 action->conf)->vlan_vid); 374 } 375 } 376 377 /** 378 * Fetch 1, 2, 3 or 4 byte field from the byte array 379 * and return as unsigned integer in host-endian format. 380 * 381 * @param[in] data 382 * Pointer to data array. 383 * @param[in] size 384 * Size of field to extract. 385 * 386 * @return 387 * converted field in host endian format. 388 */ 389 static inline uint32_t 390 flow_dv_fetch_field(const uint8_t *data, uint32_t size) 391 { 392 uint32_t ret; 393 394 switch (size) { 395 case 1: 396 ret = *data; 397 break; 398 case 2: 399 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); 400 break; 401 case 3: 402 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); 403 ret = (ret << 8) | *(data + sizeof(uint16_t)); 404 break; 405 case 4: 406 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data); 407 break; 408 default: 409 MLX5_ASSERT(false); 410 ret = 0; 411 break; 412 } 413 return ret; 414 } 415 416 /** 417 * Convert modify-header action to DV specification. 418 * 419 * Data length of each action is determined by provided field description 420 * and the item mask. Data bit offset and width of each action is determined 421 * by provided item mask. 422 * 423 * @param[in] item 424 * Pointer to item specification. 425 * @param[in] field 426 * Pointer to field modification information. 427 * For MLX5_MODIFICATION_TYPE_SET specifies destination field. 428 * For MLX5_MODIFICATION_TYPE_ADD specifies destination field. 429 * For MLX5_MODIFICATION_TYPE_COPY specifies source field. 430 * @param[in] dcopy 431 * Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type. 432 * Negative offset value sets the same offset as source offset. 433 * size field is ignored, value is taken from source field. 434 * @param[in,out] resource 435 * Pointer to the modify-header resource. 436 * @param[in] type 437 * Type of modification. 438 * @param[out] error 439 * Pointer to the error structure. 440 * 441 * @return 442 * 0 on success, a negative errno value otherwise and rte_errno is set. 443 */ 444 static int 445 flow_dv_convert_modify_action(struct rte_flow_item *item, 446 struct field_modify_info *field, 447 struct field_modify_info *dcopy, 448 struct mlx5_flow_dv_modify_hdr_resource *resource, 449 uint32_t type, struct rte_flow_error *error) 450 { 451 uint32_t i = resource->actions_num; 452 struct mlx5_modification_cmd *actions = resource->actions; 453 uint32_t carry_b = 0; 454 455 /* 456 * The item and mask are provided in big-endian format. 457 * The fields should be presented as in big-endian format either. 458 * Mask must be always present, it defines the actual field width. 459 */ 460 MLX5_ASSERT(item->mask); 461 MLX5_ASSERT(field->size); 462 do { 463 uint32_t size_b; 464 uint32_t off_b; 465 uint32_t mask; 466 uint32_t data; 467 bool next_field = true; 468 bool next_dcopy = true; 469 470 if (i >= MLX5_MAX_MODIFY_NUM) 471 return rte_flow_error_set(error, EINVAL, 472 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 473 "too many items to modify"); 474 /* Fetch variable byte size mask from the array. */ 475 mask = flow_dv_fetch_field((const uint8_t *)item->mask + 476 field->offset, field->size); 477 if (!mask) { 478 ++field; 479 continue; 480 } 481 /* Deduce actual data width in bits from mask value. */ 482 off_b = rte_bsf32(mask) + carry_b; 483 size_b = sizeof(uint32_t) * CHAR_BIT - 484 off_b - __builtin_clz(mask); 485 MLX5_ASSERT(size_b); 486 actions[i] = (struct mlx5_modification_cmd) { 487 .action_type = type, 488 .field = field->id, 489 .offset = off_b, 490 .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ? 491 0 : size_b, 492 }; 493 if (type == MLX5_MODIFICATION_TYPE_COPY) { 494 MLX5_ASSERT(dcopy); 495 actions[i].dst_field = dcopy->id; 496 actions[i].dst_offset = 497 (int)dcopy->offset < 0 ? off_b : dcopy->offset; 498 /* Convert entire record to big-endian format. */ 499 actions[i].data1 = rte_cpu_to_be_32(actions[i].data1); 500 /* 501 * Destination field overflow. Copy leftovers of 502 * a source field to the next destination field. 503 */ 504 carry_b = 0; 505 if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) && 506 dcopy->size != 0) { 507 actions[i].length = 508 dcopy->size * CHAR_BIT - dcopy->offset; 509 carry_b = actions[i].length; 510 next_field = false; 511 } 512 /* 513 * Not enough bits in a source filed to fill a 514 * destination field. Switch to the next source. 515 */ 516 if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) && 517 (size_b == field->size * CHAR_BIT - off_b)) { 518 actions[i].length = 519 field->size * CHAR_BIT - off_b; 520 dcopy->offset += actions[i].length; 521 next_dcopy = false; 522 } 523 if (next_dcopy) 524 ++dcopy; 525 } else { 526 MLX5_ASSERT(item->spec); 527 data = flow_dv_fetch_field((const uint8_t *)item->spec + 528 field->offset, field->size); 529 /* Shift out the trailing masked bits from data. */ 530 data = (data & mask) >> off_b; 531 actions[i].data1 = rte_cpu_to_be_32(data); 532 } 533 /* Convert entire record to expected big-endian format. */ 534 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 535 if (next_field) 536 ++field; 537 ++i; 538 } while (field->size); 539 if (resource->actions_num == i) 540 return rte_flow_error_set(error, EINVAL, 541 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 542 "invalid modification flow item"); 543 resource->actions_num = i; 544 return 0; 545 } 546 547 /** 548 * Convert modify-header set IPv4 address action to DV specification. 549 * 550 * @param[in,out] resource 551 * Pointer to the modify-header resource. 552 * @param[in] action 553 * Pointer to action specification. 554 * @param[out] error 555 * Pointer to the error structure. 556 * 557 * @return 558 * 0 on success, a negative errno value otherwise and rte_errno is set. 559 */ 560 static int 561 flow_dv_convert_action_modify_ipv4 562 (struct mlx5_flow_dv_modify_hdr_resource *resource, 563 const struct rte_flow_action *action, 564 struct rte_flow_error *error) 565 { 566 const struct rte_flow_action_set_ipv4 *conf = 567 (const struct rte_flow_action_set_ipv4 *)(action->conf); 568 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 569 struct rte_flow_item_ipv4 ipv4; 570 struct rte_flow_item_ipv4 ipv4_mask; 571 572 memset(&ipv4, 0, sizeof(ipv4)); 573 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 574 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) { 575 ipv4.hdr.src_addr = conf->ipv4_addr; 576 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr; 577 } else { 578 ipv4.hdr.dst_addr = conf->ipv4_addr; 579 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr; 580 } 581 item.spec = &ipv4; 582 item.mask = &ipv4_mask; 583 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, 584 MLX5_MODIFICATION_TYPE_SET, error); 585 } 586 587 /** 588 * Convert modify-header set IPv6 address action to DV specification. 589 * 590 * @param[in,out] resource 591 * Pointer to the modify-header resource. 592 * @param[in] action 593 * Pointer to action specification. 594 * @param[out] error 595 * Pointer to the error structure. 596 * 597 * @return 598 * 0 on success, a negative errno value otherwise and rte_errno is set. 599 */ 600 static int 601 flow_dv_convert_action_modify_ipv6 602 (struct mlx5_flow_dv_modify_hdr_resource *resource, 603 const struct rte_flow_action *action, 604 struct rte_flow_error *error) 605 { 606 const struct rte_flow_action_set_ipv6 *conf = 607 (const struct rte_flow_action_set_ipv6 *)(action->conf); 608 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 609 struct rte_flow_item_ipv6 ipv6; 610 struct rte_flow_item_ipv6 ipv6_mask; 611 612 memset(&ipv6, 0, sizeof(ipv6)); 613 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 614 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) { 615 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr, 616 sizeof(ipv6.hdr.src_addr)); 617 memcpy(&ipv6_mask.hdr.src_addr, 618 &rte_flow_item_ipv6_mask.hdr.src_addr, 619 sizeof(ipv6.hdr.src_addr)); 620 } else { 621 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr, 622 sizeof(ipv6.hdr.dst_addr)); 623 memcpy(&ipv6_mask.hdr.dst_addr, 624 &rte_flow_item_ipv6_mask.hdr.dst_addr, 625 sizeof(ipv6.hdr.dst_addr)); 626 } 627 item.spec = &ipv6; 628 item.mask = &ipv6_mask; 629 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, 630 MLX5_MODIFICATION_TYPE_SET, error); 631 } 632 633 /** 634 * Convert modify-header set MAC address action to DV specification. 635 * 636 * @param[in,out] resource 637 * Pointer to the modify-header resource. 638 * @param[in] action 639 * Pointer to action specification. 640 * @param[out] error 641 * Pointer to the error structure. 642 * 643 * @return 644 * 0 on success, a negative errno value otherwise and rte_errno is set. 645 */ 646 static int 647 flow_dv_convert_action_modify_mac 648 (struct mlx5_flow_dv_modify_hdr_resource *resource, 649 const struct rte_flow_action *action, 650 struct rte_flow_error *error) 651 { 652 const struct rte_flow_action_set_mac *conf = 653 (const struct rte_flow_action_set_mac *)(action->conf); 654 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH }; 655 struct rte_flow_item_eth eth; 656 struct rte_flow_item_eth eth_mask; 657 658 memset(ð, 0, sizeof(eth)); 659 memset(ð_mask, 0, sizeof(eth_mask)); 660 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) { 661 memcpy(ð.src.addr_bytes, &conf->mac_addr, 662 sizeof(eth.src.addr_bytes)); 663 memcpy(ð_mask.src.addr_bytes, 664 &rte_flow_item_eth_mask.src.addr_bytes, 665 sizeof(eth_mask.src.addr_bytes)); 666 } else { 667 memcpy(ð.dst.addr_bytes, &conf->mac_addr, 668 sizeof(eth.dst.addr_bytes)); 669 memcpy(ð_mask.dst.addr_bytes, 670 &rte_flow_item_eth_mask.dst.addr_bytes, 671 sizeof(eth_mask.dst.addr_bytes)); 672 } 673 item.spec = ð 674 item.mask = ð_mask; 675 return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource, 676 MLX5_MODIFICATION_TYPE_SET, error); 677 } 678 679 /** 680 * Convert modify-header set VLAN VID action to DV specification. 681 * 682 * @param[in,out] resource 683 * Pointer to the modify-header resource. 684 * @param[in] action 685 * Pointer to action specification. 686 * @param[out] error 687 * Pointer to the error structure. 688 * 689 * @return 690 * 0 on success, a negative errno value otherwise and rte_errno is set. 691 */ 692 static int 693 flow_dv_convert_action_modify_vlan_vid 694 (struct mlx5_flow_dv_modify_hdr_resource *resource, 695 const struct rte_flow_action *action, 696 struct rte_flow_error *error) 697 { 698 const struct rte_flow_action_of_set_vlan_vid *conf = 699 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf); 700 int i = resource->actions_num; 701 struct mlx5_modification_cmd *actions = resource->actions; 702 struct field_modify_info *field = modify_vlan_out_first_vid; 703 704 if (i >= MLX5_MAX_MODIFY_NUM) 705 return rte_flow_error_set(error, EINVAL, 706 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 707 "too many items to modify"); 708 actions[i] = (struct mlx5_modification_cmd) { 709 .action_type = MLX5_MODIFICATION_TYPE_SET, 710 .field = field->id, 711 .length = field->size, 712 .offset = field->offset, 713 }; 714 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 715 actions[i].data1 = conf->vlan_vid; 716 actions[i].data1 = actions[i].data1 << 16; 717 resource->actions_num = ++i; 718 return 0; 719 } 720 721 /** 722 * Convert modify-header set TP action to DV specification. 723 * 724 * @param[in,out] resource 725 * Pointer to the modify-header resource. 726 * @param[in] action 727 * Pointer to action specification. 728 * @param[in] items 729 * Pointer to rte_flow_item objects list. 730 * @param[in] attr 731 * Pointer to flow attributes structure. 732 * @param[in] dev_flow 733 * Pointer to the sub flow. 734 * @param[in] tunnel_decap 735 * Whether action is after tunnel decapsulation. 736 * @param[out] error 737 * Pointer to the error structure. 738 * 739 * @return 740 * 0 on success, a negative errno value otherwise and rte_errno is set. 741 */ 742 static int 743 flow_dv_convert_action_modify_tp 744 (struct mlx5_flow_dv_modify_hdr_resource *resource, 745 const struct rte_flow_action *action, 746 const struct rte_flow_item *items, 747 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 748 bool tunnel_decap, struct rte_flow_error *error) 749 { 750 const struct rte_flow_action_set_tp *conf = 751 (const struct rte_flow_action_set_tp *)(action->conf); 752 struct rte_flow_item item; 753 struct rte_flow_item_udp udp; 754 struct rte_flow_item_udp udp_mask; 755 struct rte_flow_item_tcp tcp; 756 struct rte_flow_item_tcp tcp_mask; 757 struct field_modify_info *field; 758 759 if (!attr->valid) 760 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 761 if (attr->udp) { 762 memset(&udp, 0, sizeof(udp)); 763 memset(&udp_mask, 0, sizeof(udp_mask)); 764 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 765 udp.hdr.src_port = conf->port; 766 udp_mask.hdr.src_port = 767 rte_flow_item_udp_mask.hdr.src_port; 768 } else { 769 udp.hdr.dst_port = conf->port; 770 udp_mask.hdr.dst_port = 771 rte_flow_item_udp_mask.hdr.dst_port; 772 } 773 item.type = RTE_FLOW_ITEM_TYPE_UDP; 774 item.spec = &udp; 775 item.mask = &udp_mask; 776 field = modify_udp; 777 } else { 778 MLX5_ASSERT(attr->tcp); 779 memset(&tcp, 0, sizeof(tcp)); 780 memset(&tcp_mask, 0, sizeof(tcp_mask)); 781 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 782 tcp.hdr.src_port = conf->port; 783 tcp_mask.hdr.src_port = 784 rte_flow_item_tcp_mask.hdr.src_port; 785 } else { 786 tcp.hdr.dst_port = conf->port; 787 tcp_mask.hdr.dst_port = 788 rte_flow_item_tcp_mask.hdr.dst_port; 789 } 790 item.type = RTE_FLOW_ITEM_TYPE_TCP; 791 item.spec = &tcp; 792 item.mask = &tcp_mask; 793 field = modify_tcp; 794 } 795 return flow_dv_convert_modify_action(&item, field, NULL, resource, 796 MLX5_MODIFICATION_TYPE_SET, error); 797 } 798 799 /** 800 * Convert modify-header set TTL action to DV specification. 801 * 802 * @param[in,out] resource 803 * Pointer to the modify-header resource. 804 * @param[in] action 805 * Pointer to action specification. 806 * @param[in] items 807 * Pointer to rte_flow_item objects list. 808 * @param[in] attr 809 * Pointer to flow attributes structure. 810 * @param[in] dev_flow 811 * Pointer to the sub flow. 812 * @param[in] tunnel_decap 813 * Whether action is after tunnel decapsulation. 814 * @param[out] error 815 * Pointer to the error structure. 816 * 817 * @return 818 * 0 on success, a negative errno value otherwise and rte_errno is set. 819 */ 820 static int 821 flow_dv_convert_action_modify_ttl 822 (struct mlx5_flow_dv_modify_hdr_resource *resource, 823 const struct rte_flow_action *action, 824 const struct rte_flow_item *items, 825 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 826 bool tunnel_decap, struct rte_flow_error *error) 827 { 828 const struct rte_flow_action_set_ttl *conf = 829 (const struct rte_flow_action_set_ttl *)(action->conf); 830 struct rte_flow_item item; 831 struct rte_flow_item_ipv4 ipv4; 832 struct rte_flow_item_ipv4 ipv4_mask; 833 struct rte_flow_item_ipv6 ipv6; 834 struct rte_flow_item_ipv6 ipv6_mask; 835 struct field_modify_info *field; 836 837 if (!attr->valid) 838 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 839 if (attr->ipv4) { 840 memset(&ipv4, 0, sizeof(ipv4)); 841 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 842 ipv4.hdr.time_to_live = conf->ttl_value; 843 ipv4_mask.hdr.time_to_live = 0xFF; 844 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 845 item.spec = &ipv4; 846 item.mask = &ipv4_mask; 847 field = modify_ipv4; 848 } else { 849 MLX5_ASSERT(attr->ipv6); 850 memset(&ipv6, 0, sizeof(ipv6)); 851 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 852 ipv6.hdr.hop_limits = conf->ttl_value; 853 ipv6_mask.hdr.hop_limits = 0xFF; 854 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 855 item.spec = &ipv6; 856 item.mask = &ipv6_mask; 857 field = modify_ipv6; 858 } 859 return flow_dv_convert_modify_action(&item, field, NULL, resource, 860 MLX5_MODIFICATION_TYPE_SET, error); 861 } 862 863 /** 864 * Convert modify-header decrement TTL action to DV specification. 865 * 866 * @param[in,out] resource 867 * Pointer to the modify-header resource. 868 * @param[in] action 869 * Pointer to action specification. 870 * @param[in] items 871 * Pointer to rte_flow_item objects list. 872 * @param[in] attr 873 * Pointer to flow attributes structure. 874 * @param[in] dev_flow 875 * Pointer to the sub flow. 876 * @param[in] tunnel_decap 877 * Whether action is after tunnel decapsulation. 878 * @param[out] error 879 * Pointer to the error structure. 880 * 881 * @return 882 * 0 on success, a negative errno value otherwise and rte_errno is set. 883 */ 884 static int 885 flow_dv_convert_action_modify_dec_ttl 886 (struct mlx5_flow_dv_modify_hdr_resource *resource, 887 const struct rte_flow_item *items, 888 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 889 bool tunnel_decap, struct rte_flow_error *error) 890 { 891 struct rte_flow_item item; 892 struct rte_flow_item_ipv4 ipv4; 893 struct rte_flow_item_ipv4 ipv4_mask; 894 struct rte_flow_item_ipv6 ipv6; 895 struct rte_flow_item_ipv6 ipv6_mask; 896 struct field_modify_info *field; 897 898 if (!attr->valid) 899 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 900 if (attr->ipv4) { 901 memset(&ipv4, 0, sizeof(ipv4)); 902 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 903 ipv4.hdr.time_to_live = 0xFF; 904 ipv4_mask.hdr.time_to_live = 0xFF; 905 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 906 item.spec = &ipv4; 907 item.mask = &ipv4_mask; 908 field = modify_ipv4; 909 } else { 910 MLX5_ASSERT(attr->ipv6); 911 memset(&ipv6, 0, sizeof(ipv6)); 912 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 913 ipv6.hdr.hop_limits = 0xFF; 914 ipv6_mask.hdr.hop_limits = 0xFF; 915 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 916 item.spec = &ipv6; 917 item.mask = &ipv6_mask; 918 field = modify_ipv6; 919 } 920 return flow_dv_convert_modify_action(&item, field, NULL, resource, 921 MLX5_MODIFICATION_TYPE_ADD, error); 922 } 923 924 /** 925 * Convert modify-header increment/decrement TCP Sequence number 926 * to DV specification. 927 * 928 * @param[in,out] resource 929 * Pointer to the modify-header resource. 930 * @param[in] action 931 * Pointer to action specification. 932 * @param[out] error 933 * Pointer to the error structure. 934 * 935 * @return 936 * 0 on success, a negative errno value otherwise and rte_errno is set. 937 */ 938 static int 939 flow_dv_convert_action_modify_tcp_seq 940 (struct mlx5_flow_dv_modify_hdr_resource *resource, 941 const struct rte_flow_action *action, 942 struct rte_flow_error *error) 943 { 944 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 945 uint64_t value = rte_be_to_cpu_32(*conf); 946 struct rte_flow_item item; 947 struct rte_flow_item_tcp tcp; 948 struct rte_flow_item_tcp tcp_mask; 949 950 memset(&tcp, 0, sizeof(tcp)); 951 memset(&tcp_mask, 0, sizeof(tcp_mask)); 952 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ) 953 /* 954 * The HW has no decrement operation, only increment operation. 955 * To simulate decrement X from Y using increment operation 956 * we need to add UINT32_MAX X times to Y. 957 * Each adding of UINT32_MAX decrements Y by 1. 958 */ 959 value *= UINT32_MAX; 960 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value); 961 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX); 962 item.type = RTE_FLOW_ITEM_TYPE_TCP; 963 item.spec = &tcp; 964 item.mask = &tcp_mask; 965 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, 966 MLX5_MODIFICATION_TYPE_ADD, error); 967 } 968 969 /** 970 * Convert modify-header increment/decrement TCP Acknowledgment number 971 * to DV specification. 972 * 973 * @param[in,out] resource 974 * Pointer to the modify-header resource. 975 * @param[in] action 976 * Pointer to action specification. 977 * @param[out] error 978 * Pointer to the error structure. 979 * 980 * @return 981 * 0 on success, a negative errno value otherwise and rte_errno is set. 982 */ 983 static int 984 flow_dv_convert_action_modify_tcp_ack 985 (struct mlx5_flow_dv_modify_hdr_resource *resource, 986 const struct rte_flow_action *action, 987 struct rte_flow_error *error) 988 { 989 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 990 uint64_t value = rte_be_to_cpu_32(*conf); 991 struct rte_flow_item item; 992 struct rte_flow_item_tcp tcp; 993 struct rte_flow_item_tcp tcp_mask; 994 995 memset(&tcp, 0, sizeof(tcp)); 996 memset(&tcp_mask, 0, sizeof(tcp_mask)); 997 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK) 998 /* 999 * The HW has no decrement operation, only increment operation. 1000 * To simulate decrement X from Y using increment operation 1001 * we need to add UINT32_MAX X times to Y. 1002 * Each adding of UINT32_MAX decrements Y by 1. 1003 */ 1004 value *= UINT32_MAX; 1005 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value); 1006 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX); 1007 item.type = RTE_FLOW_ITEM_TYPE_TCP; 1008 item.spec = &tcp; 1009 item.mask = &tcp_mask; 1010 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, 1011 MLX5_MODIFICATION_TYPE_ADD, error); 1012 } 1013 1014 static enum mlx5_modification_field reg_to_field[] = { 1015 [REG_NON] = MLX5_MODI_OUT_NONE, 1016 [REG_A] = MLX5_MODI_META_DATA_REG_A, 1017 [REG_B] = MLX5_MODI_META_DATA_REG_B, 1018 [REG_C_0] = MLX5_MODI_META_REG_C_0, 1019 [REG_C_1] = MLX5_MODI_META_REG_C_1, 1020 [REG_C_2] = MLX5_MODI_META_REG_C_2, 1021 [REG_C_3] = MLX5_MODI_META_REG_C_3, 1022 [REG_C_4] = MLX5_MODI_META_REG_C_4, 1023 [REG_C_5] = MLX5_MODI_META_REG_C_5, 1024 [REG_C_6] = MLX5_MODI_META_REG_C_6, 1025 [REG_C_7] = MLX5_MODI_META_REG_C_7, 1026 }; 1027 1028 /** 1029 * Convert register set to DV specification. 1030 * 1031 * @param[in,out] resource 1032 * Pointer to the modify-header resource. 1033 * @param[in] action 1034 * Pointer to action specification. 1035 * @param[out] error 1036 * Pointer to the error structure. 1037 * 1038 * @return 1039 * 0 on success, a negative errno value otherwise and rte_errno is set. 1040 */ 1041 static int 1042 flow_dv_convert_action_set_reg 1043 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1044 const struct rte_flow_action *action, 1045 struct rte_flow_error *error) 1046 { 1047 const struct mlx5_rte_flow_action_set_tag *conf = action->conf; 1048 struct mlx5_modification_cmd *actions = resource->actions; 1049 uint32_t i = resource->actions_num; 1050 1051 if (i >= MLX5_MAX_MODIFY_NUM) 1052 return rte_flow_error_set(error, EINVAL, 1053 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1054 "too many items to modify"); 1055 MLX5_ASSERT(conf->id != REG_NON); 1056 MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field)); 1057 actions[i] = (struct mlx5_modification_cmd) { 1058 .action_type = MLX5_MODIFICATION_TYPE_SET, 1059 .field = reg_to_field[conf->id], 1060 .offset = conf->offset, 1061 .length = conf->length, 1062 }; 1063 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 1064 actions[i].data1 = rte_cpu_to_be_32(conf->data); 1065 ++i; 1066 resource->actions_num = i; 1067 return 0; 1068 } 1069 1070 /** 1071 * Convert SET_TAG action to DV specification. 1072 * 1073 * @param[in] dev 1074 * Pointer to the rte_eth_dev structure. 1075 * @param[in,out] resource 1076 * Pointer to the modify-header resource. 1077 * @param[in] conf 1078 * Pointer to action specification. 1079 * @param[out] error 1080 * Pointer to the error structure. 1081 * 1082 * @return 1083 * 0 on success, a negative errno value otherwise and rte_errno is set. 1084 */ 1085 static int 1086 flow_dv_convert_action_set_tag 1087 (struct rte_eth_dev *dev, 1088 struct mlx5_flow_dv_modify_hdr_resource *resource, 1089 const struct rte_flow_action_set_tag *conf, 1090 struct rte_flow_error *error) 1091 { 1092 rte_be32_t data = rte_cpu_to_be_32(conf->data); 1093 rte_be32_t mask = rte_cpu_to_be_32(conf->mask); 1094 struct rte_flow_item item = { 1095 .spec = &data, 1096 .mask = &mask, 1097 }; 1098 struct field_modify_info reg_c_x[] = { 1099 [1] = {0, 0, 0}, 1100 }; 1101 enum mlx5_modification_field reg_type; 1102 int ret; 1103 1104 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); 1105 if (ret < 0) 1106 return ret; 1107 MLX5_ASSERT(ret != REG_NON); 1108 MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field)); 1109 reg_type = reg_to_field[ret]; 1110 MLX5_ASSERT(reg_type > 0); 1111 reg_c_x[0] = (struct field_modify_info){4, 0, reg_type}; 1112 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1113 MLX5_MODIFICATION_TYPE_SET, error); 1114 } 1115 1116 /** 1117 * Convert internal COPY_REG action to DV specification. 1118 * 1119 * @param[in] dev 1120 * Pointer to the rte_eth_dev structure. 1121 * @param[in,out] res 1122 * Pointer to the modify-header resource. 1123 * @param[in] action 1124 * Pointer to action specification. 1125 * @param[out] error 1126 * Pointer to the error structure. 1127 * 1128 * @return 1129 * 0 on success, a negative errno value otherwise and rte_errno is set. 1130 */ 1131 static int 1132 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev, 1133 struct mlx5_flow_dv_modify_hdr_resource *res, 1134 const struct rte_flow_action *action, 1135 struct rte_flow_error *error) 1136 { 1137 const struct mlx5_flow_action_copy_mreg *conf = action->conf; 1138 rte_be32_t mask = RTE_BE32(UINT32_MAX); 1139 struct rte_flow_item item = { 1140 .spec = NULL, 1141 .mask = &mask, 1142 }; 1143 struct field_modify_info reg_src[] = { 1144 {4, 0, reg_to_field[conf->src]}, 1145 {0, 0, 0}, 1146 }; 1147 struct field_modify_info reg_dst = { 1148 .offset = 0, 1149 .id = reg_to_field[conf->dst], 1150 }; 1151 /* Adjust reg_c[0] usage according to reported mask. */ 1152 if (conf->dst == REG_C_0 || conf->src == REG_C_0) { 1153 struct mlx5_priv *priv = dev->data->dev_private; 1154 uint32_t reg_c0 = priv->sh->dv_regc0_mask; 1155 1156 MLX5_ASSERT(reg_c0); 1157 MLX5_ASSERT(priv->sh->config.dv_xmeta_en != 1158 MLX5_XMETA_MODE_LEGACY); 1159 if (conf->dst == REG_C_0) { 1160 /* Copy to reg_c[0], within mask only. */ 1161 reg_dst.offset = rte_bsf32(reg_c0); 1162 mask = rte_cpu_to_be_32(reg_c0 >> reg_dst.offset); 1163 } else { 1164 reg_dst.offset = 0; 1165 mask = rte_cpu_to_be_32(reg_c0); 1166 } 1167 } 1168 return flow_dv_convert_modify_action(&item, 1169 reg_src, ®_dst, res, 1170 MLX5_MODIFICATION_TYPE_COPY, 1171 error); 1172 } 1173 1174 /** 1175 * Convert MARK action to DV specification. This routine is used 1176 * in extensive metadata only and requires metadata register to be 1177 * handled. In legacy mode hardware tag resource is engaged. 1178 * 1179 * @param[in] dev 1180 * Pointer to the rte_eth_dev structure. 1181 * @param[in] conf 1182 * Pointer to MARK action specification. 1183 * @param[in,out] resource 1184 * Pointer to the modify-header resource. 1185 * @param[out] error 1186 * Pointer to the error structure. 1187 * 1188 * @return 1189 * 0 on success, a negative errno value otherwise and rte_errno is set. 1190 */ 1191 static int 1192 flow_dv_convert_action_mark(struct rte_eth_dev *dev, 1193 const struct rte_flow_action_mark *conf, 1194 struct mlx5_flow_dv_modify_hdr_resource *resource, 1195 struct rte_flow_error *error) 1196 { 1197 struct mlx5_priv *priv = dev->data->dev_private; 1198 rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK & 1199 priv->sh->dv_mark_mask); 1200 rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask; 1201 struct rte_flow_item item = { 1202 .spec = &data, 1203 .mask = &mask, 1204 }; 1205 struct field_modify_info reg_c_x[] = { 1206 [1] = {0, 0, 0}, 1207 }; 1208 int reg; 1209 1210 if (!mask) 1211 return rte_flow_error_set(error, EINVAL, 1212 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1213 NULL, "zero mark action mask"); 1214 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 1215 if (reg < 0) 1216 return reg; 1217 MLX5_ASSERT(reg > 0); 1218 if (reg == REG_C_0) { 1219 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 1220 uint32_t shl_c0 = rte_bsf32(msk_c0); 1221 1222 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0); 1223 mask = rte_cpu_to_be_32(mask) & msk_c0; 1224 mask = rte_cpu_to_be_32(mask << shl_c0); 1225 } 1226 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; 1227 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1228 MLX5_MODIFICATION_TYPE_SET, error); 1229 } 1230 1231 /** 1232 * Get metadata register index for specified steering domain. 1233 * 1234 * @param[in] dev 1235 * Pointer to the rte_eth_dev structure. 1236 * @param[in] attr 1237 * Attributes of flow to determine steering domain. 1238 * @param[out] error 1239 * Pointer to the error structure. 1240 * 1241 * @return 1242 * positive index on success, a negative errno value otherwise 1243 * and rte_errno is set. 1244 */ 1245 static enum modify_reg 1246 flow_dv_get_metadata_reg(struct rte_eth_dev *dev, 1247 const struct rte_flow_attr *attr, 1248 struct rte_flow_error *error) 1249 { 1250 int reg = 1251 mlx5_flow_get_reg_id(dev, attr->transfer ? 1252 MLX5_METADATA_FDB : 1253 attr->egress ? 1254 MLX5_METADATA_TX : 1255 MLX5_METADATA_RX, 0, error); 1256 if (reg < 0) 1257 return rte_flow_error_set(error, 1258 ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 1259 NULL, "unavailable " 1260 "metadata register"); 1261 return reg; 1262 } 1263 1264 /** 1265 * Convert SET_META action to DV specification. 1266 * 1267 * @param[in] dev 1268 * Pointer to the rte_eth_dev structure. 1269 * @param[in,out] resource 1270 * Pointer to the modify-header resource. 1271 * @param[in] attr 1272 * Attributes of flow that includes this item. 1273 * @param[in] conf 1274 * Pointer to action specification. 1275 * @param[out] error 1276 * Pointer to the error structure. 1277 * 1278 * @return 1279 * 0 on success, a negative errno value otherwise and rte_errno is set. 1280 */ 1281 static int 1282 flow_dv_convert_action_set_meta 1283 (struct rte_eth_dev *dev, 1284 struct mlx5_flow_dv_modify_hdr_resource *resource, 1285 const struct rte_flow_attr *attr, 1286 const struct rte_flow_action_set_meta *conf, 1287 struct rte_flow_error *error) 1288 { 1289 uint32_t mask = rte_cpu_to_be_32(conf->mask); 1290 uint32_t data = rte_cpu_to_be_32(conf->data) & mask; 1291 struct rte_flow_item item = { 1292 .spec = &data, 1293 .mask = &mask, 1294 }; 1295 struct field_modify_info reg_c_x[] = { 1296 [1] = {0, 0, 0}, 1297 }; 1298 int reg = flow_dv_get_metadata_reg(dev, attr, error); 1299 1300 if (reg < 0) 1301 return reg; 1302 MLX5_ASSERT(reg != REG_NON); 1303 if (reg == REG_C_0) { 1304 struct mlx5_priv *priv = dev->data->dev_private; 1305 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 1306 uint32_t shl_c0 = rte_bsf32(msk_c0); 1307 1308 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0); 1309 mask = rte_cpu_to_be_32(mask) & msk_c0; 1310 mask = rte_cpu_to_be_32(mask << shl_c0); 1311 } 1312 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; 1313 /* The routine expects parameters in memory as big-endian ones. */ 1314 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1315 MLX5_MODIFICATION_TYPE_SET, error); 1316 } 1317 1318 /** 1319 * Convert modify-header set IPv4 DSCP action to DV specification. 1320 * 1321 * @param[in,out] resource 1322 * Pointer to the modify-header resource. 1323 * @param[in] action 1324 * Pointer to action specification. 1325 * @param[out] error 1326 * Pointer to the error structure. 1327 * 1328 * @return 1329 * 0 on success, a negative errno value otherwise and rte_errno is set. 1330 */ 1331 static int 1332 flow_dv_convert_action_modify_ipv4_dscp 1333 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1334 const struct rte_flow_action *action, 1335 struct rte_flow_error *error) 1336 { 1337 const struct rte_flow_action_set_dscp *conf = 1338 (const struct rte_flow_action_set_dscp *)(action->conf); 1339 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 1340 struct rte_flow_item_ipv4 ipv4; 1341 struct rte_flow_item_ipv4 ipv4_mask; 1342 1343 memset(&ipv4, 0, sizeof(ipv4)); 1344 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 1345 ipv4.hdr.type_of_service = conf->dscp; 1346 ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2; 1347 item.spec = &ipv4; 1348 item.mask = &ipv4_mask; 1349 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, 1350 MLX5_MODIFICATION_TYPE_SET, error); 1351 } 1352 1353 /** 1354 * Convert modify-header set IPv6 DSCP action to DV specification. 1355 * 1356 * @param[in,out] resource 1357 * Pointer to the modify-header resource. 1358 * @param[in] action 1359 * Pointer to action specification. 1360 * @param[out] error 1361 * Pointer to the error structure. 1362 * 1363 * @return 1364 * 0 on success, a negative errno value otherwise and rte_errno is set. 1365 */ 1366 static int 1367 flow_dv_convert_action_modify_ipv6_dscp 1368 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1369 const struct rte_flow_action *action, 1370 struct rte_flow_error *error) 1371 { 1372 const struct rte_flow_action_set_dscp *conf = 1373 (const struct rte_flow_action_set_dscp *)(action->conf); 1374 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 1375 struct rte_flow_item_ipv6 ipv6; 1376 struct rte_flow_item_ipv6 ipv6_mask; 1377 1378 memset(&ipv6, 0, sizeof(ipv6)); 1379 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 1380 /* 1381 * Even though the DSCP bits offset of IPv6 is not byte aligned, 1382 * rdma-core only accept the DSCP bits byte aligned start from 1383 * bit 0 to 5 as to be compatible with IPv4. No need to shift the 1384 * bits in IPv6 case as rdma-core requires byte aligned value. 1385 */ 1386 ipv6.hdr.vtc_flow = conf->dscp; 1387 ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22; 1388 item.spec = &ipv6; 1389 item.mask = &ipv6_mask; 1390 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, 1391 MLX5_MODIFICATION_TYPE_SET, error); 1392 } 1393 1394 static int 1395 mlx5_flow_item_field_width(struct rte_eth_dev *dev, 1396 enum rte_flow_field_id field, int inherit, 1397 const struct rte_flow_attr *attr, 1398 struct rte_flow_error *error) 1399 { 1400 struct mlx5_priv *priv = dev->data->dev_private; 1401 1402 switch (field) { 1403 case RTE_FLOW_FIELD_START: 1404 return 32; 1405 case RTE_FLOW_FIELD_MAC_DST: 1406 case RTE_FLOW_FIELD_MAC_SRC: 1407 return 48; 1408 case RTE_FLOW_FIELD_VLAN_TYPE: 1409 return 16; 1410 case RTE_FLOW_FIELD_VLAN_ID: 1411 return 12; 1412 case RTE_FLOW_FIELD_MAC_TYPE: 1413 return 16; 1414 case RTE_FLOW_FIELD_IPV4_DSCP: 1415 return 6; 1416 case RTE_FLOW_FIELD_IPV4_TTL: 1417 return 8; 1418 case RTE_FLOW_FIELD_IPV4_SRC: 1419 case RTE_FLOW_FIELD_IPV4_DST: 1420 return 32; 1421 case RTE_FLOW_FIELD_IPV6_DSCP: 1422 return 6; 1423 case RTE_FLOW_FIELD_IPV6_HOPLIMIT: 1424 return 8; 1425 case RTE_FLOW_FIELD_IPV6_SRC: 1426 case RTE_FLOW_FIELD_IPV6_DST: 1427 return 128; 1428 case RTE_FLOW_FIELD_TCP_PORT_SRC: 1429 case RTE_FLOW_FIELD_TCP_PORT_DST: 1430 return 16; 1431 case RTE_FLOW_FIELD_TCP_SEQ_NUM: 1432 case RTE_FLOW_FIELD_TCP_ACK_NUM: 1433 return 32; 1434 case RTE_FLOW_FIELD_TCP_FLAGS: 1435 return 9; 1436 case RTE_FLOW_FIELD_UDP_PORT_SRC: 1437 case RTE_FLOW_FIELD_UDP_PORT_DST: 1438 return 16; 1439 case RTE_FLOW_FIELD_VXLAN_VNI: 1440 case RTE_FLOW_FIELD_GENEVE_VNI: 1441 return 24; 1442 case RTE_FLOW_FIELD_GTP_TEID: 1443 case RTE_FLOW_FIELD_TAG: 1444 return 32; 1445 case RTE_FLOW_FIELD_MARK: 1446 return __builtin_popcount(priv->sh->dv_mark_mask); 1447 case RTE_FLOW_FIELD_META: 1448 return (flow_dv_get_metadata_reg(dev, attr, error) == REG_C_0) ? 1449 __builtin_popcount(priv->sh->dv_meta_mask) : 32; 1450 case RTE_FLOW_FIELD_POINTER: 1451 case RTE_FLOW_FIELD_VALUE: 1452 return inherit < 0 ? 0 : inherit; 1453 default: 1454 MLX5_ASSERT(false); 1455 } 1456 return 0; 1457 } 1458 1459 static void 1460 mlx5_flow_field_id_to_modify_info 1461 (const struct rte_flow_action_modify_data *data, 1462 struct field_modify_info *info, uint32_t *mask, 1463 uint32_t width, struct rte_eth_dev *dev, 1464 const struct rte_flow_attr *attr, struct rte_flow_error *error) 1465 { 1466 struct mlx5_priv *priv = dev->data->dev_private; 1467 uint32_t idx = 0; 1468 uint32_t off = 0; 1469 1470 switch (data->field) { 1471 case RTE_FLOW_FIELD_START: 1472 /* not supported yet */ 1473 MLX5_ASSERT(false); 1474 break; 1475 case RTE_FLOW_FIELD_MAC_DST: 1476 off = data->offset > 16 ? data->offset - 16 : 0; 1477 if (mask) { 1478 if (data->offset < 16) { 1479 info[idx] = (struct field_modify_info){2, 4, 1480 MLX5_MODI_OUT_DMAC_15_0}; 1481 if (width < 16) { 1482 mask[1] = rte_cpu_to_be_16(0xffff >> 1483 (16 - width)); 1484 width = 0; 1485 } else { 1486 mask[1] = RTE_BE16(0xffff); 1487 width -= 16; 1488 } 1489 if (!width) 1490 break; 1491 ++idx; 1492 } 1493 info[idx] = (struct field_modify_info){4, 0, 1494 MLX5_MODI_OUT_DMAC_47_16}; 1495 mask[0] = rte_cpu_to_be_32((0xffffffff >> 1496 (32 - width)) << off); 1497 } else { 1498 if (data->offset < 16) 1499 info[idx++] = (struct field_modify_info){2, 0, 1500 MLX5_MODI_OUT_DMAC_15_0}; 1501 info[idx] = (struct field_modify_info){4, off, 1502 MLX5_MODI_OUT_DMAC_47_16}; 1503 } 1504 break; 1505 case RTE_FLOW_FIELD_MAC_SRC: 1506 off = data->offset > 16 ? data->offset - 16 : 0; 1507 if (mask) { 1508 if (data->offset < 16) { 1509 info[idx] = (struct field_modify_info){2, 4, 1510 MLX5_MODI_OUT_SMAC_15_0}; 1511 if (width < 16) { 1512 mask[1] = rte_cpu_to_be_16(0xffff >> 1513 (16 - width)); 1514 width = 0; 1515 } else { 1516 mask[1] = RTE_BE16(0xffff); 1517 width -= 16; 1518 } 1519 if (!width) 1520 break; 1521 ++idx; 1522 } 1523 info[idx] = (struct field_modify_info){4, 0, 1524 MLX5_MODI_OUT_SMAC_47_16}; 1525 mask[0] = rte_cpu_to_be_32((0xffffffff >> 1526 (32 - width)) << off); 1527 } else { 1528 if (data->offset < 16) 1529 info[idx++] = (struct field_modify_info){2, 0, 1530 MLX5_MODI_OUT_SMAC_15_0}; 1531 info[idx] = (struct field_modify_info){4, off, 1532 MLX5_MODI_OUT_SMAC_47_16}; 1533 } 1534 break; 1535 case RTE_FLOW_FIELD_VLAN_TYPE: 1536 /* not supported yet */ 1537 break; 1538 case RTE_FLOW_FIELD_VLAN_ID: 1539 info[idx] = (struct field_modify_info){2, 0, 1540 MLX5_MODI_OUT_FIRST_VID}; 1541 if (mask) 1542 mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width)); 1543 break; 1544 case RTE_FLOW_FIELD_MAC_TYPE: 1545 info[idx] = (struct field_modify_info){2, 0, 1546 MLX5_MODI_OUT_ETHERTYPE}; 1547 if (mask) 1548 mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); 1549 break; 1550 case RTE_FLOW_FIELD_IPV4_DSCP: 1551 info[idx] = (struct field_modify_info){1, 0, 1552 MLX5_MODI_OUT_IP_DSCP}; 1553 if (mask) 1554 mask[idx] = 0x3f >> (6 - width); 1555 break; 1556 case RTE_FLOW_FIELD_IPV4_TTL: 1557 info[idx] = (struct field_modify_info){1, 0, 1558 MLX5_MODI_OUT_IPV4_TTL}; 1559 if (mask) 1560 mask[idx] = 0xff >> (8 - width); 1561 break; 1562 case RTE_FLOW_FIELD_IPV4_SRC: 1563 info[idx] = (struct field_modify_info){4, 0, 1564 MLX5_MODI_OUT_SIPV4}; 1565 if (mask) 1566 mask[idx] = rte_cpu_to_be_32(0xffffffff >> 1567 (32 - width)); 1568 break; 1569 case RTE_FLOW_FIELD_IPV4_DST: 1570 info[idx] = (struct field_modify_info){4, 0, 1571 MLX5_MODI_OUT_DIPV4}; 1572 if (mask) 1573 mask[idx] = rte_cpu_to_be_32(0xffffffff >> 1574 (32 - width)); 1575 break; 1576 case RTE_FLOW_FIELD_IPV6_DSCP: 1577 info[idx] = (struct field_modify_info){1, 0, 1578 MLX5_MODI_OUT_IP_DSCP}; 1579 if (mask) 1580 mask[idx] = 0x3f >> (6 - width); 1581 break; 1582 case RTE_FLOW_FIELD_IPV6_HOPLIMIT: 1583 info[idx] = (struct field_modify_info){1, 0, 1584 MLX5_MODI_OUT_IPV6_HOPLIMIT}; 1585 if (mask) 1586 mask[idx] = 0xff >> (8 - width); 1587 break; 1588 case RTE_FLOW_FIELD_IPV6_SRC: 1589 if (mask) { 1590 if (data->offset < 32) { 1591 info[idx] = (struct field_modify_info){4, 12, 1592 MLX5_MODI_OUT_SIPV6_31_0}; 1593 if (width < 32) { 1594 mask[3] = 1595 rte_cpu_to_be_32(0xffffffff >> 1596 (32 - width)); 1597 width = 0; 1598 } else { 1599 mask[3] = RTE_BE32(0xffffffff); 1600 width -= 32; 1601 } 1602 if (!width) 1603 break; 1604 ++idx; 1605 } 1606 if (data->offset < 64) { 1607 info[idx] = (struct field_modify_info){4, 8, 1608 MLX5_MODI_OUT_SIPV6_63_32}; 1609 if (width < 32) { 1610 mask[2] = 1611 rte_cpu_to_be_32(0xffffffff >> 1612 (32 - width)); 1613 width = 0; 1614 } else { 1615 mask[2] = RTE_BE32(0xffffffff); 1616 width -= 32; 1617 } 1618 if (!width) 1619 break; 1620 ++idx; 1621 } 1622 if (data->offset < 96) { 1623 info[idx] = (struct field_modify_info){4, 4, 1624 MLX5_MODI_OUT_SIPV6_95_64}; 1625 if (width < 32) { 1626 mask[1] = 1627 rte_cpu_to_be_32(0xffffffff >> 1628 (32 - width)); 1629 width = 0; 1630 } else { 1631 mask[1] = RTE_BE32(0xffffffff); 1632 width -= 32; 1633 } 1634 if (!width) 1635 break; 1636 ++idx; 1637 } 1638 info[idx] = (struct field_modify_info){4, 0, 1639 MLX5_MODI_OUT_SIPV6_127_96}; 1640 mask[0] = rte_cpu_to_be_32(0xffffffff >> (32 - width)); 1641 } else { 1642 if (data->offset < 32) 1643 info[idx++] = (struct field_modify_info){4, 0, 1644 MLX5_MODI_OUT_SIPV6_31_0}; 1645 if (data->offset < 64) 1646 info[idx++] = (struct field_modify_info){4, 0, 1647 MLX5_MODI_OUT_SIPV6_63_32}; 1648 if (data->offset < 96) 1649 info[idx++] = (struct field_modify_info){4, 0, 1650 MLX5_MODI_OUT_SIPV6_95_64}; 1651 if (data->offset < 128) 1652 info[idx++] = (struct field_modify_info){4, 0, 1653 MLX5_MODI_OUT_SIPV6_127_96}; 1654 } 1655 break; 1656 case RTE_FLOW_FIELD_IPV6_DST: 1657 if (mask) { 1658 if (data->offset < 32) { 1659 info[idx] = (struct field_modify_info){4, 12, 1660 MLX5_MODI_OUT_DIPV6_31_0}; 1661 if (width < 32) { 1662 mask[3] = 1663 rte_cpu_to_be_32(0xffffffff >> 1664 (32 - width)); 1665 width = 0; 1666 } else { 1667 mask[3] = RTE_BE32(0xffffffff); 1668 width -= 32; 1669 } 1670 if (!width) 1671 break; 1672 ++idx; 1673 } 1674 if (data->offset < 64) { 1675 info[idx] = (struct field_modify_info){4, 8, 1676 MLX5_MODI_OUT_DIPV6_63_32}; 1677 if (width < 32) { 1678 mask[2] = 1679 rte_cpu_to_be_32(0xffffffff >> 1680 (32 - width)); 1681 width = 0; 1682 } else { 1683 mask[2] = RTE_BE32(0xffffffff); 1684 width -= 32; 1685 } 1686 if (!width) 1687 break; 1688 ++idx; 1689 } 1690 if (data->offset < 96) { 1691 info[idx] = (struct field_modify_info){4, 4, 1692 MLX5_MODI_OUT_DIPV6_95_64}; 1693 if (width < 32) { 1694 mask[1] = 1695 rte_cpu_to_be_32(0xffffffff >> 1696 (32 - width)); 1697 width = 0; 1698 } else { 1699 mask[1] = RTE_BE32(0xffffffff); 1700 width -= 32; 1701 } 1702 if (!width) 1703 break; 1704 ++idx; 1705 } 1706 info[idx] = (struct field_modify_info){4, 0, 1707 MLX5_MODI_OUT_DIPV6_127_96}; 1708 mask[0] = rte_cpu_to_be_32(0xffffffff >> (32 - width)); 1709 } else { 1710 if (data->offset < 32) 1711 info[idx++] = (struct field_modify_info){4, 0, 1712 MLX5_MODI_OUT_DIPV6_31_0}; 1713 if (data->offset < 64) 1714 info[idx++] = (struct field_modify_info){4, 0, 1715 MLX5_MODI_OUT_DIPV6_63_32}; 1716 if (data->offset < 96) 1717 info[idx++] = (struct field_modify_info){4, 0, 1718 MLX5_MODI_OUT_DIPV6_95_64}; 1719 if (data->offset < 128) 1720 info[idx++] = (struct field_modify_info){4, 0, 1721 MLX5_MODI_OUT_DIPV6_127_96}; 1722 } 1723 break; 1724 case RTE_FLOW_FIELD_TCP_PORT_SRC: 1725 info[idx] = (struct field_modify_info){2, 0, 1726 MLX5_MODI_OUT_TCP_SPORT}; 1727 if (mask) 1728 mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); 1729 break; 1730 case RTE_FLOW_FIELD_TCP_PORT_DST: 1731 info[idx] = (struct field_modify_info){2, 0, 1732 MLX5_MODI_OUT_TCP_DPORT}; 1733 if (mask) 1734 mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); 1735 break; 1736 case RTE_FLOW_FIELD_TCP_SEQ_NUM: 1737 info[idx] = (struct field_modify_info){4, 0, 1738 MLX5_MODI_OUT_TCP_SEQ_NUM}; 1739 if (mask) 1740 mask[idx] = rte_cpu_to_be_32(0xffffffff >> 1741 (32 - width)); 1742 break; 1743 case RTE_FLOW_FIELD_TCP_ACK_NUM: 1744 info[idx] = (struct field_modify_info){4, 0, 1745 MLX5_MODI_OUT_TCP_ACK_NUM}; 1746 if (mask) 1747 mask[idx] = rte_cpu_to_be_32(0xffffffff >> 1748 (32 - width)); 1749 break; 1750 case RTE_FLOW_FIELD_TCP_FLAGS: 1751 info[idx] = (struct field_modify_info){2, 0, 1752 MLX5_MODI_OUT_TCP_FLAGS}; 1753 if (mask) 1754 mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width)); 1755 break; 1756 case RTE_FLOW_FIELD_UDP_PORT_SRC: 1757 info[idx] = (struct field_modify_info){2, 0, 1758 MLX5_MODI_OUT_UDP_SPORT}; 1759 if (mask) 1760 mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); 1761 break; 1762 case RTE_FLOW_FIELD_UDP_PORT_DST: 1763 info[idx] = (struct field_modify_info){2, 0, 1764 MLX5_MODI_OUT_UDP_DPORT}; 1765 if (mask) 1766 mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); 1767 break; 1768 case RTE_FLOW_FIELD_VXLAN_VNI: 1769 /* not supported yet */ 1770 break; 1771 case RTE_FLOW_FIELD_GENEVE_VNI: 1772 /* not supported yet*/ 1773 break; 1774 case RTE_FLOW_FIELD_GTP_TEID: 1775 info[idx] = (struct field_modify_info){4, 0, 1776 MLX5_MODI_GTP_TEID}; 1777 if (mask) 1778 mask[idx] = rte_cpu_to_be_32(0xffffffff >> 1779 (32 - width)); 1780 break; 1781 case RTE_FLOW_FIELD_TAG: 1782 { 1783 int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 1784 data->level, error); 1785 if (reg < 0) 1786 return; 1787 MLX5_ASSERT(reg != REG_NON); 1788 MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); 1789 info[idx] = (struct field_modify_info){4, 0, 1790 reg_to_field[reg]}; 1791 if (mask) 1792 mask[idx] = 1793 rte_cpu_to_be_32(0xffffffff >> 1794 (32 - width)); 1795 } 1796 break; 1797 case RTE_FLOW_FIELD_MARK: 1798 { 1799 uint32_t mark_mask = priv->sh->dv_mark_mask; 1800 uint32_t mark_count = __builtin_popcount(mark_mask); 1801 int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 1802 0, error); 1803 if (reg < 0) 1804 return; 1805 MLX5_ASSERT(reg != REG_NON); 1806 MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); 1807 info[idx] = (struct field_modify_info){4, 0, 1808 reg_to_field[reg]}; 1809 if (mask) 1810 mask[idx] = rte_cpu_to_be_32((mark_mask >> 1811 (mark_count - width)) & mark_mask); 1812 } 1813 break; 1814 case RTE_FLOW_FIELD_META: 1815 { 1816 uint32_t meta_mask = priv->sh->dv_meta_mask; 1817 uint32_t meta_count = __builtin_popcount(meta_mask); 1818 int reg = flow_dv_get_metadata_reg(dev, attr, error); 1819 if (reg < 0) 1820 return; 1821 MLX5_ASSERT(reg != REG_NON); 1822 MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); 1823 info[idx] = (struct field_modify_info){4, 0, 1824 reg_to_field[reg]}; 1825 if (mask) 1826 mask[idx] = rte_cpu_to_be_32((meta_mask >> 1827 (meta_count - width)) & meta_mask); 1828 } 1829 break; 1830 case RTE_FLOW_FIELD_POINTER: 1831 case RTE_FLOW_FIELD_VALUE: 1832 default: 1833 MLX5_ASSERT(false); 1834 break; 1835 } 1836 } 1837 1838 /** 1839 * Convert modify_field action to DV specification. 1840 * 1841 * @param[in] dev 1842 * Pointer to the rte_eth_dev structure. 1843 * @param[in,out] resource 1844 * Pointer to the modify-header resource. 1845 * @param[in] action 1846 * Pointer to action specification. 1847 * @param[in] attr 1848 * Attributes of flow that includes this item. 1849 * @param[out] error 1850 * Pointer to the error structure. 1851 * 1852 * @return 1853 * 0 on success, a negative errno value otherwise and rte_errno is set. 1854 */ 1855 static int 1856 flow_dv_convert_action_modify_field 1857 (struct rte_eth_dev *dev, 1858 struct mlx5_flow_dv_modify_hdr_resource *resource, 1859 const struct rte_flow_action *action, 1860 const struct rte_flow_attr *attr, 1861 struct rte_flow_error *error) 1862 { 1863 const struct rte_flow_action_modify_field *conf = 1864 (const struct rte_flow_action_modify_field *)(action->conf); 1865 struct rte_flow_item item = { 1866 .spec = NULL, 1867 .mask = NULL 1868 }; 1869 struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = { 1870 {0, 0, 0} }; 1871 struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = { 1872 {0, 0, 0} }; 1873 uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; 1874 uint32_t type, meta = 0; 1875 1876 if (conf->src.field == RTE_FLOW_FIELD_POINTER || 1877 conf->src.field == RTE_FLOW_FIELD_VALUE) { 1878 type = MLX5_MODIFICATION_TYPE_SET; 1879 /** For SET fill the destination field (field) first. */ 1880 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask, 1881 conf->width, dev, 1882 attr, error); 1883 item.spec = conf->src.field == RTE_FLOW_FIELD_POINTER ? 1884 (void *)(uintptr_t)conf->src.pvalue : 1885 (void *)(uintptr_t)&conf->src.value; 1886 if (conf->dst.field == RTE_FLOW_FIELD_META) { 1887 meta = *(const unaligned_uint32_t *)item.spec; 1888 meta = rte_cpu_to_be_32(meta); 1889 item.spec = &meta; 1890 } 1891 } else { 1892 type = MLX5_MODIFICATION_TYPE_COPY; 1893 /** For COPY fill the destination field (dcopy) without mask. */ 1894 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL, 1895 conf->width, dev, 1896 attr, error); 1897 /** Then construct the source field (field) with mask. */ 1898 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask, 1899 conf->width, dev, 1900 attr, error); 1901 } 1902 item.mask = &mask; 1903 return flow_dv_convert_modify_action(&item, 1904 field, dcopy, resource, type, error); 1905 } 1906 1907 /** 1908 * Validate MARK item. 1909 * 1910 * @param[in] dev 1911 * Pointer to the rte_eth_dev structure. 1912 * @param[in] item 1913 * Item specification. 1914 * @param[in] attr 1915 * Attributes of flow that includes this item. 1916 * @param[out] error 1917 * Pointer to error structure. 1918 * 1919 * @return 1920 * 0 on success, a negative errno value otherwise and rte_errno is set. 1921 */ 1922 static int 1923 flow_dv_validate_item_mark(struct rte_eth_dev *dev, 1924 const struct rte_flow_item *item, 1925 const struct rte_flow_attr *attr __rte_unused, 1926 struct rte_flow_error *error) 1927 { 1928 struct mlx5_priv *priv = dev->data->dev_private; 1929 struct mlx5_sh_config *config = &priv->sh->config; 1930 const struct rte_flow_item_mark *spec = item->spec; 1931 const struct rte_flow_item_mark *mask = item->mask; 1932 const struct rte_flow_item_mark nic_mask = { 1933 .id = priv->sh->dv_mark_mask, 1934 }; 1935 int ret; 1936 1937 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 1938 return rte_flow_error_set(error, ENOTSUP, 1939 RTE_FLOW_ERROR_TYPE_ITEM, item, 1940 "extended metadata feature" 1941 " isn't enabled"); 1942 if (!mlx5_flow_ext_mreg_supported(dev)) 1943 return rte_flow_error_set(error, ENOTSUP, 1944 RTE_FLOW_ERROR_TYPE_ITEM, item, 1945 "extended metadata register" 1946 " isn't supported"); 1947 if (!nic_mask.id) 1948 return rte_flow_error_set(error, ENOTSUP, 1949 RTE_FLOW_ERROR_TYPE_ITEM, item, 1950 "extended metadata register" 1951 " isn't available"); 1952 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 1953 if (ret < 0) 1954 return ret; 1955 if (!spec) 1956 return rte_flow_error_set(error, EINVAL, 1957 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1958 item->spec, 1959 "data cannot be empty"); 1960 if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id)) 1961 return rte_flow_error_set(error, EINVAL, 1962 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1963 &spec->id, 1964 "mark id exceeds the limit"); 1965 if (!mask) 1966 mask = &nic_mask; 1967 if (!mask->id) 1968 return rte_flow_error_set(error, EINVAL, 1969 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1970 "mask cannot be zero"); 1971 1972 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1973 (const uint8_t *)&nic_mask, 1974 sizeof(struct rte_flow_item_mark), 1975 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1976 if (ret < 0) 1977 return ret; 1978 return 0; 1979 } 1980 1981 /** 1982 * Validate META item. 1983 * 1984 * @param[in] dev 1985 * Pointer to the rte_eth_dev structure. 1986 * @param[in] item 1987 * Item specification. 1988 * @param[in] attr 1989 * Attributes of flow that includes this item. 1990 * @param[out] error 1991 * Pointer to error structure. 1992 * 1993 * @return 1994 * 0 on success, a negative errno value otherwise and rte_errno is set. 1995 */ 1996 static int 1997 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, 1998 const struct rte_flow_item *item, 1999 const struct rte_flow_attr *attr, 2000 struct rte_flow_error *error) 2001 { 2002 struct mlx5_priv *priv = dev->data->dev_private; 2003 struct mlx5_sh_config *config = &priv->sh->config; 2004 const struct rte_flow_item_meta *spec = item->spec; 2005 const struct rte_flow_item_meta *mask = item->mask; 2006 struct rte_flow_item_meta nic_mask = { 2007 .data = UINT32_MAX 2008 }; 2009 int reg; 2010 int ret; 2011 2012 if (!spec) 2013 return rte_flow_error_set(error, EINVAL, 2014 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2015 item->spec, 2016 "data cannot be empty"); 2017 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 2018 if (!mlx5_flow_ext_mreg_supported(dev)) 2019 return rte_flow_error_set(error, ENOTSUP, 2020 RTE_FLOW_ERROR_TYPE_ITEM, item, 2021 "extended metadata register" 2022 " isn't supported"); 2023 reg = flow_dv_get_metadata_reg(dev, attr, error); 2024 if (reg < 0) 2025 return reg; 2026 if (reg == REG_NON) 2027 return rte_flow_error_set(error, ENOTSUP, 2028 RTE_FLOW_ERROR_TYPE_ITEM, item, 2029 "unavailable extended metadata register"); 2030 if (reg == REG_B) 2031 return rte_flow_error_set(error, ENOTSUP, 2032 RTE_FLOW_ERROR_TYPE_ITEM, item, 2033 "match on reg_b " 2034 "isn't supported"); 2035 if (reg != REG_A) 2036 nic_mask.data = priv->sh->dv_meta_mask; 2037 } else { 2038 if (attr->transfer) 2039 return rte_flow_error_set(error, ENOTSUP, 2040 RTE_FLOW_ERROR_TYPE_ITEM, item, 2041 "extended metadata feature " 2042 "should be enabled when " 2043 "meta item is requested " 2044 "with e-switch mode "); 2045 if (attr->ingress) 2046 return rte_flow_error_set(error, ENOTSUP, 2047 RTE_FLOW_ERROR_TYPE_ITEM, item, 2048 "match on metadata for ingress " 2049 "is not supported in legacy " 2050 "metadata mode"); 2051 } 2052 if (!mask) 2053 mask = &rte_flow_item_meta_mask; 2054 if (!mask->data) 2055 return rte_flow_error_set(error, EINVAL, 2056 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 2057 "mask cannot be zero"); 2058 2059 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2060 (const uint8_t *)&nic_mask, 2061 sizeof(struct rte_flow_item_meta), 2062 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2063 return ret; 2064 } 2065 2066 /** 2067 * Validate TAG item. 2068 * 2069 * @param[in] dev 2070 * Pointer to the rte_eth_dev structure. 2071 * @param[in] item 2072 * Item specification. 2073 * @param[in] attr 2074 * Attributes of flow that includes this item. 2075 * @param[out] error 2076 * Pointer to error structure. 2077 * 2078 * @return 2079 * 0 on success, a negative errno value otherwise and rte_errno is set. 2080 */ 2081 static int 2082 flow_dv_validate_item_tag(struct rte_eth_dev *dev, 2083 const struct rte_flow_item *item, 2084 const struct rte_flow_attr *attr __rte_unused, 2085 struct rte_flow_error *error) 2086 { 2087 const struct rte_flow_item_tag *spec = item->spec; 2088 const struct rte_flow_item_tag *mask = item->mask; 2089 const struct rte_flow_item_tag nic_mask = { 2090 .data = RTE_BE32(UINT32_MAX), 2091 .index = 0xff, 2092 }; 2093 int ret; 2094 2095 if (!mlx5_flow_ext_mreg_supported(dev)) 2096 return rte_flow_error_set(error, ENOTSUP, 2097 RTE_FLOW_ERROR_TYPE_ITEM, item, 2098 "extensive metadata register" 2099 " isn't supported"); 2100 if (!spec) 2101 return rte_flow_error_set(error, EINVAL, 2102 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2103 item->spec, 2104 "data cannot be empty"); 2105 if (!mask) 2106 mask = &rte_flow_item_tag_mask; 2107 if (!mask->data) 2108 return rte_flow_error_set(error, EINVAL, 2109 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 2110 "mask cannot be zero"); 2111 2112 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2113 (const uint8_t *)&nic_mask, 2114 sizeof(struct rte_flow_item_tag), 2115 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2116 if (ret < 0) 2117 return ret; 2118 if (mask->index != 0xff) 2119 return rte_flow_error_set(error, EINVAL, 2120 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 2121 "partial mask for tag index" 2122 " is not supported"); 2123 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error); 2124 if (ret < 0) 2125 return ret; 2126 MLX5_ASSERT(ret != REG_NON); 2127 return 0; 2128 } 2129 2130 /** 2131 * Validate vport item. 2132 * 2133 * @param[in] dev 2134 * Pointer to the rte_eth_dev structure. 2135 * @param[in] item 2136 * Item specification. 2137 * @param[in] attr 2138 * Attributes of flow that includes this item. 2139 * @param[in] item_flags 2140 * Bit-fields that holds the items detected until now. 2141 * @param[out] error 2142 * Pointer to error structure. 2143 * 2144 * @return 2145 * 0 on success, a negative errno value otherwise and rte_errno is set. 2146 */ 2147 static int 2148 flow_dv_validate_item_port_id(struct rte_eth_dev *dev, 2149 const struct rte_flow_item *item, 2150 const struct rte_flow_attr *attr, 2151 uint64_t item_flags, 2152 struct rte_flow_error *error) 2153 { 2154 const struct rte_flow_item_port_id *spec = item->spec; 2155 const struct rte_flow_item_port_id *mask = item->mask; 2156 const struct rte_flow_item_port_id switch_mask = { 2157 .id = 0xffffffff, 2158 }; 2159 struct mlx5_priv *esw_priv; 2160 struct mlx5_priv *dev_priv; 2161 int ret; 2162 2163 if (!attr->transfer) 2164 return rte_flow_error_set(error, EINVAL, 2165 RTE_FLOW_ERROR_TYPE_ITEM, 2166 NULL, 2167 "match on port id is valid only" 2168 " when transfer flag is enabled"); 2169 if (item_flags & MLX5_FLOW_ITEM_PORT_ID) 2170 return rte_flow_error_set(error, ENOTSUP, 2171 RTE_FLOW_ERROR_TYPE_ITEM, item, 2172 "multiple source ports are not" 2173 " supported"); 2174 if (!mask) 2175 mask = &switch_mask; 2176 if (mask->id != 0xffffffff) 2177 return rte_flow_error_set(error, ENOTSUP, 2178 RTE_FLOW_ERROR_TYPE_ITEM_MASK, 2179 mask, 2180 "no support for partial mask on" 2181 " \"id\" field"); 2182 ret = mlx5_flow_item_acceptable 2183 (item, (const uint8_t *)mask, 2184 (const uint8_t *)&rte_flow_item_port_id_mask, 2185 sizeof(struct rte_flow_item_port_id), 2186 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2187 if (ret) 2188 return ret; 2189 if (!spec) 2190 return 0; 2191 if (spec->id == MLX5_PORT_ESW_MGR) 2192 return 0; 2193 esw_priv = mlx5_port_to_eswitch_info(spec->id, false); 2194 if (!esw_priv) 2195 return rte_flow_error_set(error, rte_errno, 2196 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 2197 "failed to obtain E-Switch info for" 2198 " port"); 2199 dev_priv = mlx5_dev_to_eswitch_info(dev); 2200 if (!dev_priv) 2201 return rte_flow_error_set(error, rte_errno, 2202 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2203 NULL, 2204 "failed to obtain E-Switch info"); 2205 if (esw_priv->domain_id != dev_priv->domain_id) 2206 return rte_flow_error_set(error, EINVAL, 2207 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 2208 "cannot match on a port from a" 2209 " different E-Switch"); 2210 return 0; 2211 } 2212 2213 /** 2214 * Validate VLAN item. 2215 * 2216 * @param[in] item 2217 * Item specification. 2218 * @param[in] item_flags 2219 * Bit-fields that holds the items detected until now. 2220 * @param[in] dev 2221 * Ethernet device flow is being created on. 2222 * @param[out] error 2223 * Pointer to error structure. 2224 * 2225 * @return 2226 * 0 on success, a negative errno value otherwise and rte_errno is set. 2227 */ 2228 static int 2229 flow_dv_validate_item_vlan(const struct rte_flow_item *item, 2230 uint64_t item_flags, 2231 struct rte_eth_dev *dev, 2232 struct rte_flow_error *error) 2233 { 2234 const struct rte_flow_item_vlan *mask = item->mask; 2235 const struct rte_flow_item_vlan nic_mask = { 2236 .tci = RTE_BE16(UINT16_MAX), 2237 .inner_type = RTE_BE16(UINT16_MAX), 2238 .has_more_vlan = 1, 2239 }; 2240 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2241 int ret; 2242 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2243 MLX5_FLOW_LAYER_INNER_L4) : 2244 (MLX5_FLOW_LAYER_OUTER_L3 | 2245 MLX5_FLOW_LAYER_OUTER_L4); 2246 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2247 MLX5_FLOW_LAYER_OUTER_VLAN; 2248 2249 if (item_flags & vlanm) 2250 return rte_flow_error_set(error, EINVAL, 2251 RTE_FLOW_ERROR_TYPE_ITEM, item, 2252 "multiple VLAN layers not supported"); 2253 else if ((item_flags & l34m) != 0) 2254 return rte_flow_error_set(error, EINVAL, 2255 RTE_FLOW_ERROR_TYPE_ITEM, item, 2256 "VLAN cannot follow L3/L4 layer"); 2257 if (!mask) 2258 mask = &rte_flow_item_vlan_mask; 2259 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2260 (const uint8_t *)&nic_mask, 2261 sizeof(struct rte_flow_item_vlan), 2262 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2263 if (ret) 2264 return ret; 2265 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2266 struct mlx5_priv *priv = dev->data->dev_private; 2267 2268 if (priv->vmwa_context) { 2269 /* 2270 * Non-NULL context means we have a virtual machine 2271 * and SR-IOV enabled, we have to create VLAN interface 2272 * to make hypervisor to setup E-Switch vport 2273 * context correctly. We avoid creating the multiple 2274 * VLAN interfaces, so we cannot support VLAN tag mask. 2275 */ 2276 return rte_flow_error_set(error, EINVAL, 2277 RTE_FLOW_ERROR_TYPE_ITEM, 2278 item, 2279 "VLAN tag mask is not" 2280 " supported in virtual" 2281 " environment"); 2282 } 2283 } 2284 return 0; 2285 } 2286 2287 /* 2288 * GTP flags are contained in 1 byte of the format: 2289 * ------------------------------------------- 2290 * | bit | 0 - 2 | 3 | 4 | 5 | 6 | 7 | 2291 * |-----------------------------------------| 2292 * | value | Version | PT | Res | E | S | PN | 2293 * ------------------------------------------- 2294 * 2295 * Matching is supported only for GTP flags E, S, PN. 2296 */ 2297 #define MLX5_GTP_FLAGS_MASK 0x07 2298 2299 /** 2300 * Validate GTP item. 2301 * 2302 * @param[in] dev 2303 * Pointer to the rte_eth_dev structure. 2304 * @param[in] item 2305 * Item specification. 2306 * @param[in] item_flags 2307 * Bit-fields that holds the items detected until now. 2308 * @param[out] error 2309 * Pointer to error structure. 2310 * 2311 * @return 2312 * 0 on success, a negative errno value otherwise and rte_errno is set. 2313 */ 2314 static int 2315 flow_dv_validate_item_gtp(struct rte_eth_dev *dev, 2316 const struct rte_flow_item *item, 2317 uint64_t item_flags, 2318 struct rte_flow_error *error) 2319 { 2320 struct mlx5_priv *priv = dev->data->dev_private; 2321 const struct rte_flow_item_gtp *spec = item->spec; 2322 const struct rte_flow_item_gtp *mask = item->mask; 2323 const struct rte_flow_item_gtp nic_mask = { 2324 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK, 2325 .msg_type = 0xff, 2326 .teid = RTE_BE32(0xffffffff), 2327 }; 2328 2329 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_gtp) 2330 return rte_flow_error_set(error, ENOTSUP, 2331 RTE_FLOW_ERROR_TYPE_ITEM, item, 2332 "GTP support is not enabled"); 2333 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2334 return rte_flow_error_set(error, ENOTSUP, 2335 RTE_FLOW_ERROR_TYPE_ITEM, item, 2336 "multiple tunnel layers not" 2337 " supported"); 2338 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2339 return rte_flow_error_set(error, EINVAL, 2340 RTE_FLOW_ERROR_TYPE_ITEM, item, 2341 "no outer UDP layer found"); 2342 if (!mask) 2343 mask = &rte_flow_item_gtp_mask; 2344 if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK) 2345 return rte_flow_error_set(error, ENOTSUP, 2346 RTE_FLOW_ERROR_TYPE_ITEM, item, 2347 "Match is supported for GTP" 2348 " flags only"); 2349 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2350 (const uint8_t *)&nic_mask, 2351 sizeof(struct rte_flow_item_gtp), 2352 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2353 } 2354 2355 /** 2356 * Validate GTP PSC item. 2357 * 2358 * @param[in] item 2359 * Item specification. 2360 * @param[in] last_item 2361 * Previous validated item in the pattern items. 2362 * @param[in] gtp_item 2363 * Previous GTP item specification. 2364 * @param[in] attr 2365 * Pointer to flow attributes. 2366 * @param[out] error 2367 * Pointer to error structure. 2368 * 2369 * @return 2370 * 0 on success, a negative errno value otherwise and rte_errno is set. 2371 */ 2372 static int 2373 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item, 2374 uint64_t last_item, 2375 const struct rte_flow_item *gtp_item, 2376 const struct rte_flow_attr *attr, 2377 struct rte_flow_error *error) 2378 { 2379 const struct rte_flow_item_gtp *gtp_spec; 2380 const struct rte_flow_item_gtp *gtp_mask; 2381 const struct rte_flow_item_gtp_psc *mask; 2382 const struct rte_flow_item_gtp_psc nic_mask = { 2383 .hdr.type = 0xF, 2384 .hdr.qfi = 0x3F, 2385 }; 2386 2387 if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP)) 2388 return rte_flow_error_set 2389 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2390 "GTP PSC item must be preceded with GTP item"); 2391 gtp_spec = gtp_item->spec; 2392 gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask; 2393 /* GTP spec and E flag is requested to match zero. */ 2394 if (gtp_spec && 2395 (gtp_mask->v_pt_rsv_flags & 2396 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG)) 2397 return rte_flow_error_set 2398 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2399 "GTP E flag must be 1 to match GTP PSC"); 2400 /* Check the flow is not created in group zero. */ 2401 if (!attr->transfer && !attr->group) 2402 return rte_flow_error_set 2403 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2404 "GTP PSC is not supported for group 0"); 2405 /* GTP spec is here and E flag is requested to match zero. */ 2406 if (!item->spec) 2407 return 0; 2408 mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask; 2409 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2410 (const uint8_t *)&nic_mask, 2411 sizeof(struct rte_flow_item_gtp_psc), 2412 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2413 } 2414 2415 /** 2416 * Validate IPV4 item. 2417 * Use existing validation function mlx5_flow_validate_item_ipv4(), and 2418 * add specific validation of fragment_offset field, 2419 * 2420 * @param[in] item 2421 * Item specification. 2422 * @param[in] item_flags 2423 * Bit-fields that holds the items detected until now. 2424 * @param[out] error 2425 * Pointer to error structure. 2426 * 2427 * @return 2428 * 0 on success, a negative errno value otherwise and rte_errno is set. 2429 */ 2430 static int 2431 flow_dv_validate_item_ipv4(struct rte_eth_dev *dev, 2432 const struct rte_flow_item *item, 2433 uint64_t item_flags, uint64_t last_item, 2434 uint16_t ether_type, struct rte_flow_error *error) 2435 { 2436 int ret; 2437 struct mlx5_priv *priv = dev->data->dev_private; 2438 struct mlx5_hca_attr *attr = &priv->sh->cdev->config.hca_attr; 2439 const struct rte_flow_item_ipv4 *spec = item->spec; 2440 const struct rte_flow_item_ipv4 *last = item->last; 2441 const struct rte_flow_item_ipv4 *mask = item->mask; 2442 rte_be16_t fragment_offset_spec = 0; 2443 rte_be16_t fragment_offset_last = 0; 2444 struct rte_flow_item_ipv4 nic_ipv4_mask = { 2445 .hdr = { 2446 .src_addr = RTE_BE32(0xffffffff), 2447 .dst_addr = RTE_BE32(0xffffffff), 2448 .type_of_service = 0xff, 2449 .fragment_offset = RTE_BE16(0xffff), 2450 .next_proto_id = 0xff, 2451 .time_to_live = 0xff, 2452 }, 2453 }; 2454 2455 if (mask && (mask->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK)) { 2456 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2457 bool ihl_cap = !tunnel ? 2458 attr->outer_ipv4_ihl : attr->inner_ipv4_ihl; 2459 if (!ihl_cap) 2460 return rte_flow_error_set(error, ENOTSUP, 2461 RTE_FLOW_ERROR_TYPE_ITEM, 2462 item, 2463 "IPV4 ihl offload not supported"); 2464 nic_ipv4_mask.hdr.version_ihl = mask->hdr.version_ihl; 2465 } 2466 ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item, 2467 ether_type, &nic_ipv4_mask, 2468 MLX5_ITEM_RANGE_ACCEPTED, error); 2469 if (ret < 0) 2470 return ret; 2471 if (spec && mask) 2472 fragment_offset_spec = spec->hdr.fragment_offset & 2473 mask->hdr.fragment_offset; 2474 if (!fragment_offset_spec) 2475 return 0; 2476 /* 2477 * spec and mask are valid, enforce using full mask to make sure the 2478 * complete value is used correctly. 2479 */ 2480 if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)) 2481 != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)) 2482 return rte_flow_error_set(error, EINVAL, 2483 RTE_FLOW_ERROR_TYPE_ITEM_MASK, 2484 item, "must use full mask for" 2485 " fragment_offset"); 2486 /* 2487 * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0, 2488 * indicating this is 1st fragment of fragmented packet. 2489 * This is not yet supported in MLX5, return appropriate error message. 2490 */ 2491 if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG)) 2492 return rte_flow_error_set(error, ENOTSUP, 2493 RTE_FLOW_ERROR_TYPE_ITEM, item, 2494 "match on first fragment not " 2495 "supported"); 2496 if (fragment_offset_spec && !last) 2497 return rte_flow_error_set(error, ENOTSUP, 2498 RTE_FLOW_ERROR_TYPE_ITEM, item, 2499 "specified value not supported"); 2500 /* spec and last are valid, validate the specified range. */ 2501 fragment_offset_last = last->hdr.fragment_offset & 2502 mask->hdr.fragment_offset; 2503 /* 2504 * Match on fragment_offset spec 0x2001 and last 0x3fff 2505 * means MF is 1 and frag-offset is > 0. 2506 * This packet is fragment 2nd and onward, excluding last. 2507 * This is not yet supported in MLX5, return appropriate 2508 * error message. 2509 */ 2510 if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) && 2511 fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)) 2512 return rte_flow_error_set(error, ENOTSUP, 2513 RTE_FLOW_ERROR_TYPE_ITEM_LAST, 2514 last, "match on following " 2515 "fragments not supported"); 2516 /* 2517 * Match on fragment_offset spec 0x0001 and last 0x1fff 2518 * means MF is 0 and frag-offset is > 0. 2519 * This packet is last fragment of fragmented packet. 2520 * This is not yet supported in MLX5, return appropriate 2521 * error message. 2522 */ 2523 if (fragment_offset_spec == RTE_BE16(1) && 2524 fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK)) 2525 return rte_flow_error_set(error, ENOTSUP, 2526 RTE_FLOW_ERROR_TYPE_ITEM_LAST, 2527 last, "match on last " 2528 "fragment not supported"); 2529 /* 2530 * Match on fragment_offset spec 0x0001 and last 0x3fff 2531 * means MF and/or frag-offset is not 0. 2532 * This is a fragmented packet. 2533 * Other range values are invalid and rejected. 2534 */ 2535 if (!(fragment_offset_spec == RTE_BE16(1) && 2536 fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))) 2537 return rte_flow_error_set(error, ENOTSUP, 2538 RTE_FLOW_ERROR_TYPE_ITEM_LAST, last, 2539 "specified range not supported"); 2540 return 0; 2541 } 2542 2543 /** 2544 * Validate IPV6 fragment extension item. 2545 * 2546 * @param[in] item 2547 * Item specification. 2548 * @param[in] item_flags 2549 * Bit-fields that holds the items detected until now. 2550 * @param[out] error 2551 * Pointer to error structure. 2552 * 2553 * @return 2554 * 0 on success, a negative errno value otherwise and rte_errno is set. 2555 */ 2556 static int 2557 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item, 2558 uint64_t item_flags, 2559 struct rte_flow_error *error) 2560 { 2561 const struct rte_flow_item_ipv6_frag_ext *spec = item->spec; 2562 const struct rte_flow_item_ipv6_frag_ext *last = item->last; 2563 const struct rte_flow_item_ipv6_frag_ext *mask = item->mask; 2564 rte_be16_t frag_data_spec = 0; 2565 rte_be16_t frag_data_last = 0; 2566 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2567 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2568 MLX5_FLOW_LAYER_OUTER_L4; 2569 int ret = 0; 2570 struct rte_flow_item_ipv6_frag_ext nic_mask = { 2571 .hdr = { 2572 .next_header = 0xff, 2573 .frag_data = RTE_BE16(0xffff), 2574 }, 2575 }; 2576 2577 if (item_flags & l4m) 2578 return rte_flow_error_set(error, EINVAL, 2579 RTE_FLOW_ERROR_TYPE_ITEM, item, 2580 "ipv6 fragment extension item cannot " 2581 "follow L4 item."); 2582 if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) || 2583 (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) 2584 return rte_flow_error_set(error, EINVAL, 2585 RTE_FLOW_ERROR_TYPE_ITEM, item, 2586 "ipv6 fragment extension item must " 2587 "follow ipv6 item"); 2588 if (spec && mask) 2589 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data; 2590 if (!frag_data_spec) 2591 return 0; 2592 /* 2593 * spec and mask are valid, enforce using full mask to make sure the 2594 * complete value is used correctly. 2595 */ 2596 if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) != 2597 RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) 2598 return rte_flow_error_set(error, EINVAL, 2599 RTE_FLOW_ERROR_TYPE_ITEM_MASK, 2600 item, "must use full mask for" 2601 " frag_data"); 2602 /* 2603 * Match on frag_data 0x00001 means M is 1 and frag-offset is 0. 2604 * This is 1st fragment of fragmented packet. 2605 */ 2606 if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK)) 2607 return rte_flow_error_set(error, ENOTSUP, 2608 RTE_FLOW_ERROR_TYPE_ITEM, item, 2609 "match on first fragment not " 2610 "supported"); 2611 if (frag_data_spec && !last) 2612 return rte_flow_error_set(error, EINVAL, 2613 RTE_FLOW_ERROR_TYPE_ITEM, item, 2614 "specified value not supported"); 2615 ret = mlx5_flow_item_acceptable 2616 (item, (const uint8_t *)mask, 2617 (const uint8_t *)&nic_mask, 2618 sizeof(struct rte_flow_item_ipv6_frag_ext), 2619 MLX5_ITEM_RANGE_ACCEPTED, error); 2620 if (ret) 2621 return ret; 2622 /* spec and last are valid, validate the specified range. */ 2623 frag_data_last = last->hdr.frag_data & mask->hdr.frag_data; 2624 /* 2625 * Match on frag_data spec 0x0009 and last 0xfff9 2626 * means M is 1 and frag-offset is > 0. 2627 * This packet is fragment 2nd and onward, excluding last. 2628 * This is not yet supported in MLX5, return appropriate 2629 * error message. 2630 */ 2631 if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN | 2632 RTE_IPV6_EHDR_MF_MASK) && 2633 frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) 2634 return rte_flow_error_set(error, ENOTSUP, 2635 RTE_FLOW_ERROR_TYPE_ITEM_LAST, 2636 last, "match on following " 2637 "fragments not supported"); 2638 /* 2639 * Match on frag_data spec 0x0008 and last 0xfff8 2640 * means M is 0 and frag-offset is > 0. 2641 * This packet is last fragment of fragmented packet. 2642 * This is not yet supported in MLX5, return appropriate 2643 * error message. 2644 */ 2645 if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) && 2646 frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK)) 2647 return rte_flow_error_set(error, ENOTSUP, 2648 RTE_FLOW_ERROR_TYPE_ITEM_LAST, 2649 last, "match on last " 2650 "fragment not supported"); 2651 /* Other range values are invalid and rejected. */ 2652 return rte_flow_error_set(error, EINVAL, 2653 RTE_FLOW_ERROR_TYPE_ITEM_LAST, last, 2654 "specified range not supported"); 2655 } 2656 2657 /* 2658 * Validate ASO CT item. 2659 * 2660 * @param[in] dev 2661 * Pointer to the rte_eth_dev structure. 2662 * @param[in] item 2663 * Item specification. 2664 * @param[in] item_flags 2665 * Pointer to bit-fields that holds the items detected until now. 2666 * @param[out] error 2667 * Pointer to error structure. 2668 * 2669 * @return 2670 * 0 on success, a negative errno value otherwise and rte_errno is set. 2671 */ 2672 static int 2673 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev, 2674 const struct rte_flow_item *item, 2675 uint64_t *item_flags, 2676 struct rte_flow_error *error) 2677 { 2678 const struct rte_flow_item_conntrack *spec = item->spec; 2679 const struct rte_flow_item_conntrack *mask = item->mask; 2680 RTE_SET_USED(dev); 2681 uint32_t flags; 2682 2683 if (*item_flags & MLX5_FLOW_LAYER_ASO_CT) 2684 return rte_flow_error_set(error, EINVAL, 2685 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2686 "Only one CT is supported"); 2687 if (!mask) 2688 mask = &rte_flow_item_conntrack_mask; 2689 flags = spec->flags & mask->flags; 2690 if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) && 2691 ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) || 2692 (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) || 2693 (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))) 2694 return rte_flow_error_set(error, EINVAL, 2695 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2696 "Conflict status bits"); 2697 /* State change also needs to be considered. */ 2698 *item_flags |= MLX5_FLOW_LAYER_ASO_CT; 2699 return 0; 2700 } 2701 2702 /** 2703 * Validate the pop VLAN action. 2704 * 2705 * @param[in] dev 2706 * Pointer to the rte_eth_dev structure. 2707 * @param[in] action_flags 2708 * Holds the actions detected until now. 2709 * @param[in] action 2710 * Pointer to the pop vlan action. 2711 * @param[in] item_flags 2712 * The items found in this flow rule. 2713 * @param[in] attr 2714 * Pointer to flow attributes. 2715 * @param[out] error 2716 * Pointer to error structure. 2717 * 2718 * @return 2719 * 0 on success, a negative errno value otherwise and rte_errno is set. 2720 */ 2721 static int 2722 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, 2723 uint64_t action_flags, 2724 const struct rte_flow_action *action, 2725 uint64_t item_flags, 2726 const struct rte_flow_attr *attr, 2727 struct rte_flow_error *error) 2728 { 2729 const struct mlx5_priv *priv = dev->data->dev_private; 2730 struct mlx5_dev_ctx_shared *sh = priv->sh; 2731 bool direction_error = false; 2732 2733 if (!priv->sh->pop_vlan_action) 2734 return rte_flow_error_set(error, ENOTSUP, 2735 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2736 NULL, 2737 "pop vlan action is not supported"); 2738 /* Pop VLAN is not supported in egress except for CX6 FDB mode. */ 2739 if (attr->transfer) { 2740 bool fdb_tx = priv->representor_id != UINT16_MAX; 2741 bool is_cx5 = sh->steering_format_version == 2742 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5; 2743 2744 if (fdb_tx && is_cx5) 2745 direction_error = true; 2746 } else if (attr->egress) { 2747 direction_error = true; 2748 } 2749 if (direction_error) 2750 return rte_flow_error_set(error, ENOTSUP, 2751 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 2752 NULL, 2753 "pop vlan action not supported for egress"); 2754 if (action_flags & MLX5_FLOW_VLAN_ACTIONS) 2755 return rte_flow_error_set(error, ENOTSUP, 2756 RTE_FLOW_ERROR_TYPE_ACTION, action, 2757 "no support for multiple VLAN " 2758 "actions"); 2759 /* Pop VLAN with preceding Decap requires inner header with VLAN. */ 2760 if ((action_flags & MLX5_FLOW_ACTION_DECAP) && 2761 !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN)) 2762 return rte_flow_error_set(error, ENOTSUP, 2763 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2764 NULL, 2765 "cannot pop vlan after decap without " 2766 "match on inner vlan in the flow"); 2767 /* Pop VLAN without preceding Decap requires outer header with VLAN. */ 2768 if (!(action_flags & MLX5_FLOW_ACTION_DECAP) && 2769 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) 2770 return rte_flow_error_set(error, ENOTSUP, 2771 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2772 NULL, 2773 "cannot pop vlan without a " 2774 "match on (outer) vlan in the flow"); 2775 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2776 return rte_flow_error_set(error, EINVAL, 2777 RTE_FLOW_ERROR_TYPE_ACTION, action, 2778 "wrong action order, port_id should " 2779 "be after pop VLAN action"); 2780 if (!attr->transfer && priv->representor) 2781 return rte_flow_error_set(error, ENOTSUP, 2782 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2783 "pop vlan action for VF representor " 2784 "not supported on NIC table"); 2785 return 0; 2786 } 2787 2788 /** 2789 * Get VLAN default info from vlan match info. 2790 * 2791 * @param[in] items 2792 * the list of item specifications. 2793 * @param[out] vlan 2794 * pointer VLAN info to fill to. 2795 * 2796 * @return 2797 * 0 on success, a negative errno value otherwise and rte_errno is set. 2798 */ 2799 static void 2800 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, 2801 struct rte_vlan_hdr *vlan) 2802 { 2803 const struct rte_flow_item_vlan nic_mask = { 2804 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK | 2805 MLX5DV_FLOW_VLAN_VID_MASK), 2806 .inner_type = RTE_BE16(0xffff), 2807 }; 2808 2809 if (items == NULL) 2810 return; 2811 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 2812 int type = items->type; 2813 2814 if (type == RTE_FLOW_ITEM_TYPE_VLAN || 2815 type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN) 2816 break; 2817 } 2818 if (items->type != RTE_FLOW_ITEM_TYPE_END) { 2819 const struct rte_flow_item_vlan *vlan_m = items->mask; 2820 const struct rte_flow_item_vlan *vlan_v = items->spec; 2821 2822 /* If VLAN item in pattern doesn't contain data, return here. */ 2823 if (!vlan_v) 2824 return; 2825 if (!vlan_m) 2826 vlan_m = &nic_mask; 2827 /* Only full match values are accepted */ 2828 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) == 2829 MLX5DV_FLOW_VLAN_PCP_MASK_BE) { 2830 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; 2831 vlan->vlan_tci |= 2832 rte_be_to_cpu_16(vlan_v->tci & 2833 MLX5DV_FLOW_VLAN_PCP_MASK_BE); 2834 } 2835 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) == 2836 MLX5DV_FLOW_VLAN_VID_MASK_BE) { 2837 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK; 2838 vlan->vlan_tci |= 2839 rte_be_to_cpu_16(vlan_v->tci & 2840 MLX5DV_FLOW_VLAN_VID_MASK_BE); 2841 } 2842 if (vlan_m->inner_type == nic_mask.inner_type) 2843 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type & 2844 vlan_m->inner_type); 2845 } 2846 } 2847 2848 /** 2849 * Validate the push VLAN action. 2850 * 2851 * @param[in] dev 2852 * Pointer to the rte_eth_dev structure. 2853 * @param[in] action_flags 2854 * Holds the actions detected until now. 2855 * @param[in] item_flags 2856 * The items found in this flow rule. 2857 * @param[in] action 2858 * Pointer to the action structure. 2859 * @param[in] attr 2860 * Pointer to flow attributes 2861 * @param[out] error 2862 * Pointer to error structure. 2863 * 2864 * @return 2865 * 0 on success, a negative errno value otherwise and rte_errno is set. 2866 */ 2867 static int 2868 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev, 2869 uint64_t action_flags, 2870 const struct rte_flow_item_vlan *vlan_m, 2871 const struct rte_flow_action *action, 2872 const struct rte_flow_attr *attr, 2873 struct rte_flow_error *error) 2874 { 2875 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf; 2876 const struct mlx5_priv *priv = dev->data->dev_private; 2877 2878 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) && 2879 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ)) 2880 return rte_flow_error_set(error, EINVAL, 2881 RTE_FLOW_ERROR_TYPE_ACTION, action, 2882 "invalid vlan ethertype"); 2883 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2884 return rte_flow_error_set(error, EINVAL, 2885 RTE_FLOW_ERROR_TYPE_ACTION, action, 2886 "wrong action order, port_id should " 2887 "be after push VLAN"); 2888 if (!attr->transfer && priv->representor) 2889 return rte_flow_error_set(error, ENOTSUP, 2890 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2891 "push vlan action for VF representor " 2892 "not supported on NIC table"); 2893 if (vlan_m && 2894 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) && 2895 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) != 2896 MLX5DV_FLOW_VLAN_PCP_MASK_BE && 2897 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) && 2898 !(mlx5_flow_find_action 2899 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP))) 2900 return rte_flow_error_set(error, EINVAL, 2901 RTE_FLOW_ERROR_TYPE_ACTION, action, 2902 "not full match mask on VLAN PCP and " 2903 "there is no of_set_vlan_pcp action, " 2904 "push VLAN action cannot figure out " 2905 "PCP value"); 2906 if (vlan_m && 2907 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) && 2908 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) != 2909 MLX5DV_FLOW_VLAN_VID_MASK_BE && 2910 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) && 2911 !(mlx5_flow_find_action 2912 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID))) 2913 return rte_flow_error_set(error, EINVAL, 2914 RTE_FLOW_ERROR_TYPE_ACTION, action, 2915 "not full match mask on VLAN VID and " 2916 "there is no of_set_vlan_vid action, " 2917 "push VLAN action cannot figure out " 2918 "VID value"); 2919 (void)attr; 2920 return 0; 2921 } 2922 2923 /** 2924 * Validate the set VLAN PCP. 2925 * 2926 * @param[in] action_flags 2927 * Holds the actions detected until now. 2928 * @param[in] actions 2929 * Pointer to the list of actions remaining in the flow rule. 2930 * @param[out] error 2931 * Pointer to error structure. 2932 * 2933 * @return 2934 * 0 on success, a negative errno value otherwise and rte_errno is set. 2935 */ 2936 static int 2937 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags, 2938 const struct rte_flow_action actions[], 2939 struct rte_flow_error *error) 2940 { 2941 const struct rte_flow_action *action = actions; 2942 const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf; 2943 2944 if (conf->vlan_pcp > 7) 2945 return rte_flow_error_set(error, EINVAL, 2946 RTE_FLOW_ERROR_TYPE_ACTION, action, 2947 "VLAN PCP value is too big"); 2948 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)) 2949 return rte_flow_error_set(error, ENOTSUP, 2950 RTE_FLOW_ERROR_TYPE_ACTION, action, 2951 "set VLAN PCP action must follow " 2952 "the push VLAN action"); 2953 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) 2954 return rte_flow_error_set(error, ENOTSUP, 2955 RTE_FLOW_ERROR_TYPE_ACTION, action, 2956 "Multiple VLAN PCP modification are " 2957 "not supported"); 2958 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2959 return rte_flow_error_set(error, EINVAL, 2960 RTE_FLOW_ERROR_TYPE_ACTION, action, 2961 "wrong action order, port_id should " 2962 "be after set VLAN PCP"); 2963 return 0; 2964 } 2965 2966 /** 2967 * Validate the set VLAN VID. 2968 * 2969 * @param[in] item_flags 2970 * Holds the items detected in this rule. 2971 * @param[in] action_flags 2972 * Holds the actions detected until now. 2973 * @param[in] actions 2974 * Pointer to the list of actions remaining in the flow rule. 2975 * @param[out] error 2976 * Pointer to error structure. 2977 * 2978 * @return 2979 * 0 on success, a negative errno value otherwise and rte_errno is set. 2980 */ 2981 static int 2982 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags, 2983 uint64_t action_flags, 2984 const struct rte_flow_action actions[], 2985 struct rte_flow_error *error) 2986 { 2987 const struct rte_flow_action *action = actions; 2988 const struct rte_flow_action_of_set_vlan_vid *conf = action->conf; 2989 2990 if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE) 2991 return rte_flow_error_set(error, EINVAL, 2992 RTE_FLOW_ERROR_TYPE_ACTION, action, 2993 "VLAN VID value is too big"); 2994 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) && 2995 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) 2996 return rte_flow_error_set(error, ENOTSUP, 2997 RTE_FLOW_ERROR_TYPE_ACTION, action, 2998 "set VLAN VID action must follow push" 2999 " VLAN action or match on VLAN item"); 3000 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) 3001 return rte_flow_error_set(error, ENOTSUP, 3002 RTE_FLOW_ERROR_TYPE_ACTION, action, 3003 "Multiple VLAN VID modifications are " 3004 "not supported"); 3005 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 3006 return rte_flow_error_set(error, EINVAL, 3007 RTE_FLOW_ERROR_TYPE_ACTION, action, 3008 "wrong action order, port_id should " 3009 "be after set VLAN VID"); 3010 return 0; 3011 } 3012 3013 /* 3014 * Validate the FLAG action. 3015 * 3016 * @param[in] dev 3017 * Pointer to the rte_eth_dev structure. 3018 * @param[in] action_flags 3019 * Holds the actions detected until now. 3020 * @param[in] attr 3021 * Pointer to flow attributes 3022 * @param[out] error 3023 * Pointer to error structure. 3024 * 3025 * @return 3026 * 0 on success, a negative errno value otherwise and rte_errno is set. 3027 */ 3028 static int 3029 flow_dv_validate_action_flag(struct rte_eth_dev *dev, 3030 uint64_t action_flags, 3031 const struct rte_flow_attr *attr, 3032 struct rte_flow_error *error) 3033 { 3034 struct mlx5_priv *priv = dev->data->dev_private; 3035 struct mlx5_sh_config *config = &priv->sh->config; 3036 int ret; 3037 3038 /* Fall back if no extended metadata register support. */ 3039 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 3040 return mlx5_flow_validate_action_flag(action_flags, attr, 3041 error); 3042 /* Extensive metadata mode requires registers. */ 3043 if (!mlx5_flow_ext_mreg_supported(dev)) 3044 return rte_flow_error_set(error, ENOTSUP, 3045 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3046 "no metadata registers " 3047 "to support flag action"); 3048 if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT)) 3049 return rte_flow_error_set(error, ENOTSUP, 3050 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3051 "extended metadata register" 3052 " isn't available"); 3053 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3054 if (ret < 0) 3055 return ret; 3056 MLX5_ASSERT(ret > 0); 3057 if (action_flags & MLX5_FLOW_ACTION_MARK) 3058 return rte_flow_error_set(error, EINVAL, 3059 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3060 "can't mark and flag in same flow"); 3061 if (action_flags & MLX5_FLOW_ACTION_FLAG) 3062 return rte_flow_error_set(error, EINVAL, 3063 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3064 "can't have 2 flag" 3065 " actions in same flow"); 3066 return 0; 3067 } 3068 3069 /** 3070 * Validate MARK action. 3071 * 3072 * @param[in] dev 3073 * Pointer to the rte_eth_dev structure. 3074 * @param[in] action 3075 * Pointer to action. 3076 * @param[in] action_flags 3077 * Holds the actions detected until now. 3078 * @param[in] attr 3079 * Pointer to flow attributes 3080 * @param[out] error 3081 * Pointer to error structure. 3082 * 3083 * @return 3084 * 0 on success, a negative errno value otherwise and rte_errno is set. 3085 */ 3086 static int 3087 flow_dv_validate_action_mark(struct rte_eth_dev *dev, 3088 const struct rte_flow_action *action, 3089 uint64_t action_flags, 3090 const struct rte_flow_attr *attr, 3091 struct rte_flow_error *error) 3092 { 3093 struct mlx5_priv *priv = dev->data->dev_private; 3094 struct mlx5_sh_config *config = &priv->sh->config; 3095 const struct rte_flow_action_mark *mark = action->conf; 3096 int ret; 3097 3098 if (is_tunnel_offload_active(dev)) 3099 return rte_flow_error_set(error, ENOTSUP, 3100 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3101 "no mark action " 3102 "if tunnel offload active"); 3103 /* Fall back if no extended metadata register support. */ 3104 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 3105 return mlx5_flow_validate_action_mark(action, action_flags, 3106 attr, error); 3107 /* Extensive metadata mode requires registers. */ 3108 if (!mlx5_flow_ext_mreg_supported(dev)) 3109 return rte_flow_error_set(error, ENOTSUP, 3110 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3111 "no metadata registers " 3112 "to support mark action"); 3113 if (!priv->sh->dv_mark_mask) 3114 return rte_flow_error_set(error, ENOTSUP, 3115 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3116 "extended metadata register" 3117 " isn't available"); 3118 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3119 if (ret < 0) 3120 return ret; 3121 MLX5_ASSERT(ret > 0); 3122 if (!mark) 3123 return rte_flow_error_set(error, EINVAL, 3124 RTE_FLOW_ERROR_TYPE_ACTION, action, 3125 "configuration cannot be null"); 3126 if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask)) 3127 return rte_flow_error_set(error, EINVAL, 3128 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3129 &mark->id, 3130 "mark id exceeds the limit"); 3131 if (action_flags & MLX5_FLOW_ACTION_FLAG) 3132 return rte_flow_error_set(error, EINVAL, 3133 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3134 "can't flag and mark in same flow"); 3135 if (action_flags & MLX5_FLOW_ACTION_MARK) 3136 return rte_flow_error_set(error, EINVAL, 3137 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3138 "can't have 2 mark actions in same" 3139 " flow"); 3140 return 0; 3141 } 3142 3143 /** 3144 * Validate SET_META action. 3145 * 3146 * @param[in] dev 3147 * Pointer to the rte_eth_dev structure. 3148 * @param[in] action 3149 * Pointer to the action structure. 3150 * @param[in] action_flags 3151 * Holds the actions detected until now. 3152 * @param[in] attr 3153 * Pointer to flow attributes 3154 * @param[out] error 3155 * Pointer to error structure. 3156 * 3157 * @return 3158 * 0 on success, a negative errno value otherwise and rte_errno is set. 3159 */ 3160 static int 3161 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, 3162 const struct rte_flow_action *action, 3163 uint64_t action_flags __rte_unused, 3164 const struct rte_flow_attr *attr, 3165 struct rte_flow_error *error) 3166 { 3167 struct mlx5_priv *priv = dev->data->dev_private; 3168 struct mlx5_sh_config *config = &priv->sh->config; 3169 const struct rte_flow_action_set_meta *conf; 3170 uint32_t nic_mask = UINT32_MAX; 3171 int reg; 3172 3173 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 3174 !mlx5_flow_ext_mreg_supported(dev)) 3175 return rte_flow_error_set(error, ENOTSUP, 3176 RTE_FLOW_ERROR_TYPE_ACTION, action, 3177 "extended metadata register" 3178 " isn't supported"); 3179 reg = flow_dv_get_metadata_reg(dev, attr, error); 3180 if (reg < 0) 3181 return reg; 3182 if (reg == REG_NON) 3183 return rte_flow_error_set(error, ENOTSUP, 3184 RTE_FLOW_ERROR_TYPE_ACTION, action, 3185 "unavailable extended metadata register"); 3186 if (reg != REG_A && reg != REG_B) { 3187 struct mlx5_priv *priv = dev->data->dev_private; 3188 3189 nic_mask = priv->sh->dv_meta_mask; 3190 } 3191 if (!(action->conf)) 3192 return rte_flow_error_set(error, EINVAL, 3193 RTE_FLOW_ERROR_TYPE_ACTION, action, 3194 "configuration cannot be null"); 3195 conf = (const struct rte_flow_action_set_meta *)action->conf; 3196 if (!conf->mask) 3197 return rte_flow_error_set(error, EINVAL, 3198 RTE_FLOW_ERROR_TYPE_ACTION, action, 3199 "zero mask doesn't have any effect"); 3200 if (conf->mask & ~nic_mask) 3201 return rte_flow_error_set(error, EINVAL, 3202 RTE_FLOW_ERROR_TYPE_ACTION, action, 3203 "meta data must be within reg C0"); 3204 return 0; 3205 } 3206 3207 /** 3208 * Validate SET_TAG action. 3209 * 3210 * @param[in] dev 3211 * Pointer to the rte_eth_dev structure. 3212 * @param[in] action 3213 * Pointer to the action structure. 3214 * @param[in] action_flags 3215 * Holds the actions detected until now. 3216 * @param[in] attr 3217 * Pointer to flow attributes 3218 * @param[out] error 3219 * Pointer to error structure. 3220 * 3221 * @return 3222 * 0 on success, a negative errno value otherwise and rte_errno is set. 3223 */ 3224 static int 3225 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, 3226 const struct rte_flow_action *action, 3227 uint64_t action_flags, 3228 const struct rte_flow_attr *attr, 3229 struct rte_flow_error *error) 3230 { 3231 const struct rte_flow_action_set_tag *conf; 3232 const uint64_t terminal_action_flags = 3233 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE | 3234 MLX5_FLOW_ACTION_RSS; 3235 int ret; 3236 3237 if (!mlx5_flow_ext_mreg_supported(dev)) 3238 return rte_flow_error_set(error, ENOTSUP, 3239 RTE_FLOW_ERROR_TYPE_ACTION, action, 3240 "extensive metadata register" 3241 " isn't supported"); 3242 if (!(action->conf)) 3243 return rte_flow_error_set(error, EINVAL, 3244 RTE_FLOW_ERROR_TYPE_ACTION, action, 3245 "configuration cannot be null"); 3246 conf = (const struct rte_flow_action_set_tag *)action->conf; 3247 if (!conf->mask) 3248 return rte_flow_error_set(error, EINVAL, 3249 RTE_FLOW_ERROR_TYPE_ACTION, action, 3250 "zero mask doesn't have any effect"); 3251 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); 3252 if (ret < 0) 3253 return ret; 3254 if (!attr->transfer && attr->ingress && 3255 (action_flags & terminal_action_flags)) 3256 return rte_flow_error_set(error, EINVAL, 3257 RTE_FLOW_ERROR_TYPE_ACTION, action, 3258 "set_tag has no effect" 3259 " with terminal actions"); 3260 return 0; 3261 } 3262 3263 /** 3264 * Indicates whether ASO aging is supported. 3265 * 3266 * @param[in] sh 3267 * Pointer to shared device context structure. 3268 * @param[in] attr 3269 * Attributes of flow that includes AGE action. 3270 * 3271 * @return 3272 * True when ASO aging is supported, false otherwise. 3273 */ 3274 static inline bool 3275 flow_hit_aso_supported(const struct mlx5_dev_ctx_shared *sh, 3276 const struct rte_flow_attr *attr) 3277 { 3278 MLX5_ASSERT(sh && attr); 3279 return (sh->flow_hit_aso_en && (attr->transfer || attr->group)); 3280 } 3281 3282 /** 3283 * Validate count action. 3284 * 3285 * @param[in] dev 3286 * Pointer to rte_eth_dev structure. 3287 * @param[in] shared 3288 * Indicator if action is shared. 3289 * @param[in] action_flags 3290 * Holds the actions detected until now. 3291 * @param[in] attr 3292 * Attributes of flow that includes this action. 3293 * @param[out] error 3294 * Pointer to error structure. 3295 * 3296 * @return 3297 * 0 on success, a negative errno value otherwise and rte_errno is set. 3298 */ 3299 static int 3300 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared, 3301 uint64_t action_flags, 3302 const struct rte_flow_attr *attr, 3303 struct rte_flow_error *error) 3304 { 3305 struct mlx5_priv *priv = dev->data->dev_private; 3306 3307 if (!priv->sh->cdev->config.devx) 3308 goto notsup_err; 3309 if (action_flags & MLX5_FLOW_ACTION_COUNT) 3310 return rte_flow_error_set(error, EINVAL, 3311 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3312 "duplicate count actions set"); 3313 if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) && 3314 !flow_hit_aso_supported(priv->sh, attr)) 3315 return rte_flow_error_set(error, EINVAL, 3316 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3317 "old age and indirect count combination is not supported"); 3318 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS 3319 return 0; 3320 #endif 3321 notsup_err: 3322 return rte_flow_error_set 3323 (error, ENOTSUP, 3324 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3325 NULL, 3326 "count action not supported"); 3327 } 3328 3329 /** 3330 * Validate the L2 encap action. 3331 * 3332 * @param[in] dev 3333 * Pointer to the rte_eth_dev structure. 3334 * @param[in] action_flags 3335 * Holds the actions detected until now. 3336 * @param[in] action 3337 * Pointer to the action structure. 3338 * @param[in] attr 3339 * Pointer to flow attributes. 3340 * @param[out] error 3341 * Pointer to error structure. 3342 * 3343 * @return 3344 * 0 on success, a negative errno value otherwise and rte_errno is set. 3345 */ 3346 static int 3347 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, 3348 uint64_t action_flags, 3349 const struct rte_flow_action *action, 3350 const struct rte_flow_attr *attr, 3351 struct rte_flow_error *error) 3352 { 3353 const struct mlx5_priv *priv = dev->data->dev_private; 3354 3355 if (!(action->conf)) 3356 return rte_flow_error_set(error, EINVAL, 3357 RTE_FLOW_ERROR_TYPE_ACTION, action, 3358 "configuration cannot be null"); 3359 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 3360 return rte_flow_error_set(error, EINVAL, 3361 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3362 "can only have a single encap action " 3363 "in a flow"); 3364 if (!attr->transfer && priv->representor) 3365 return rte_flow_error_set(error, ENOTSUP, 3366 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3367 "encap action for VF representor " 3368 "not supported on NIC table"); 3369 return 0; 3370 } 3371 3372 /** 3373 * Validate a decap action. 3374 * 3375 * @param[in] dev 3376 * Pointer to the rte_eth_dev structure. 3377 * @param[in] action_flags 3378 * Holds the actions detected until now. 3379 * @param[in] action 3380 * Pointer to the action structure. 3381 * @param[in] item_flags 3382 * Holds the items detected. 3383 * @param[in] attr 3384 * Pointer to flow attributes 3385 * @param[out] error 3386 * Pointer to error structure. 3387 * 3388 * @return 3389 * 0 on success, a negative errno value otherwise and rte_errno is set. 3390 */ 3391 static int 3392 flow_dv_validate_action_decap(struct rte_eth_dev *dev, 3393 uint64_t action_flags, 3394 const struct rte_flow_action *action, 3395 const uint64_t item_flags, 3396 const struct rte_flow_attr *attr, 3397 struct rte_flow_error *error) 3398 { 3399 const struct mlx5_priv *priv = dev->data->dev_private; 3400 3401 if (priv->sh->cdev->config.hca_attr.scatter_fcs_w_decap_disable && 3402 !priv->sh->config.decap_en) 3403 return rte_flow_error_set(error, ENOTSUP, 3404 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3405 "decap is not enabled"); 3406 if (action_flags & MLX5_FLOW_XCAP_ACTIONS) 3407 return rte_flow_error_set(error, ENOTSUP, 3408 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3409 action_flags & 3410 MLX5_FLOW_ACTION_DECAP ? "can only " 3411 "have a single decap action" : "decap " 3412 "after encap is not supported"); 3413 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) 3414 return rte_flow_error_set(error, EINVAL, 3415 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3416 "can't have decap action after" 3417 " modify action"); 3418 if (attr->egress) 3419 return rte_flow_error_set(error, ENOTSUP, 3420 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 3421 NULL, 3422 "decap action not supported for " 3423 "egress"); 3424 if (!attr->transfer && priv->representor) 3425 return rte_flow_error_set(error, ENOTSUP, 3426 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3427 "decap action for VF representor " 3428 "not supported on NIC table"); 3429 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP && 3430 !(item_flags & MLX5_FLOW_LAYER_VXLAN)) 3431 return rte_flow_error_set(error, ENOTSUP, 3432 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3433 "VXLAN item should be present for VXLAN decap"); 3434 return 0; 3435 } 3436 3437 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,}; 3438 3439 /** 3440 * Validate the raw encap and decap actions. 3441 * 3442 * @param[in] dev 3443 * Pointer to the rte_eth_dev structure. 3444 * @param[in] decap 3445 * Pointer to the decap action. 3446 * @param[in] encap 3447 * Pointer to the encap action. 3448 * @param[in] attr 3449 * Pointer to flow attributes 3450 * @param[in/out] action_flags 3451 * Holds the actions detected until now. 3452 * @param[out] actions_n 3453 * pointer to the number of actions counter. 3454 * @param[in] action 3455 * Pointer to the action structure. 3456 * @param[in] item_flags 3457 * Holds the items detected. 3458 * @param[out] error 3459 * Pointer to error structure. 3460 * 3461 * @return 3462 * 0 on success, a negative errno value otherwise and rte_errno is set. 3463 */ 3464 static int 3465 flow_dv_validate_action_raw_encap_decap 3466 (struct rte_eth_dev *dev, 3467 const struct rte_flow_action_raw_decap *decap, 3468 const struct rte_flow_action_raw_encap *encap, 3469 const struct rte_flow_attr *attr, uint64_t *action_flags, 3470 int *actions_n, const struct rte_flow_action *action, 3471 uint64_t item_flags, struct rte_flow_error *error) 3472 { 3473 const struct mlx5_priv *priv = dev->data->dev_private; 3474 int ret; 3475 3476 if (encap && (!encap->size || !encap->data)) 3477 return rte_flow_error_set(error, EINVAL, 3478 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3479 "raw encap data cannot be empty"); 3480 if (decap && encap) { 3481 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE && 3482 encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3483 /* L3 encap. */ 3484 decap = NULL; 3485 else if (encap->size <= 3486 MLX5_ENCAPSULATION_DECISION_SIZE && 3487 decap->size > 3488 MLX5_ENCAPSULATION_DECISION_SIZE) 3489 /* L3 decap. */ 3490 encap = NULL; 3491 else if (encap->size > 3492 MLX5_ENCAPSULATION_DECISION_SIZE && 3493 decap->size > 3494 MLX5_ENCAPSULATION_DECISION_SIZE) 3495 /* 2 L2 actions: encap and decap. */ 3496 ; 3497 else 3498 return rte_flow_error_set(error, 3499 ENOTSUP, 3500 RTE_FLOW_ERROR_TYPE_ACTION, 3501 NULL, "unsupported too small " 3502 "raw decap and too small raw " 3503 "encap combination"); 3504 } 3505 if (decap) { 3506 ret = flow_dv_validate_action_decap(dev, *action_flags, action, 3507 item_flags, attr, error); 3508 if (ret < 0) 3509 return ret; 3510 *action_flags |= MLX5_FLOW_ACTION_DECAP; 3511 ++(*actions_n); 3512 } 3513 if (encap) { 3514 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE) 3515 return rte_flow_error_set(error, ENOTSUP, 3516 RTE_FLOW_ERROR_TYPE_ACTION, 3517 NULL, 3518 "small raw encap size"); 3519 if (*action_flags & MLX5_FLOW_ACTION_ENCAP) 3520 return rte_flow_error_set(error, EINVAL, 3521 RTE_FLOW_ERROR_TYPE_ACTION, 3522 NULL, 3523 "more than one encap action"); 3524 if (!attr->transfer && priv->representor) 3525 return rte_flow_error_set 3526 (error, ENOTSUP, 3527 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3528 "encap action for VF representor " 3529 "not supported on NIC table"); 3530 *action_flags |= MLX5_FLOW_ACTION_ENCAP; 3531 ++(*actions_n); 3532 } 3533 return 0; 3534 } 3535 3536 /* 3537 * Validate the ASO CT action. 3538 * 3539 * @param[in] dev 3540 * Pointer to the rte_eth_dev structure. 3541 * @param[in] action_flags 3542 * Holds the actions detected until now. 3543 * @param[in] item_flags 3544 * The items found in this flow rule. 3545 * @param[in] attr 3546 * Pointer to flow attributes. 3547 * @param[out] error 3548 * Pointer to error structure. 3549 * 3550 * @return 3551 * 0 on success, a negative errno value otherwise and rte_errno is set. 3552 */ 3553 static int 3554 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev, 3555 uint64_t action_flags, 3556 uint64_t item_flags, 3557 const struct rte_flow_attr *attr, 3558 struct rte_flow_error *error) 3559 { 3560 RTE_SET_USED(dev); 3561 3562 if (attr->group == 0 && !attr->transfer) 3563 return rte_flow_error_set(error, ENOTSUP, 3564 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3565 NULL, 3566 "Only support non-root table"); 3567 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 3568 return rte_flow_error_set(error, ENOTSUP, 3569 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3570 "CT cannot follow a fate action"); 3571 if ((action_flags & MLX5_FLOW_ACTION_METER) || 3572 (action_flags & MLX5_FLOW_ACTION_AGE)) 3573 return rte_flow_error_set(error, EINVAL, 3574 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3575 "Only one ASO action is supported"); 3576 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 3577 return rte_flow_error_set(error, EINVAL, 3578 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3579 "Encap cannot exist before CT"); 3580 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)) 3581 return rte_flow_error_set(error, EINVAL, 3582 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3583 "Not a outer TCP packet"); 3584 return 0; 3585 } 3586 3587 int 3588 flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused, 3589 struct mlx5_list_entry *entry, void *cb_ctx) 3590 { 3591 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3592 struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data; 3593 struct mlx5_flow_dv_encap_decap_resource *resource; 3594 3595 resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource, 3596 entry); 3597 if (resource->reformat_type == ctx_resource->reformat_type && 3598 resource->ft_type == ctx_resource->ft_type && 3599 resource->flags == ctx_resource->flags && 3600 resource->size == ctx_resource->size && 3601 !memcmp((const void *)resource->buf, 3602 (const void *)ctx_resource->buf, 3603 resource->size)) 3604 return 0; 3605 return -1; 3606 } 3607 3608 struct mlx5_list_entry * 3609 flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx) 3610 { 3611 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3612 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3613 struct mlx5dv_dr_domain *domain; 3614 struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data; 3615 struct mlx5_flow_dv_encap_decap_resource *resource; 3616 uint32_t idx; 3617 int ret; 3618 3619 if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 3620 domain = sh->fdb_domain; 3621 else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 3622 domain = sh->rx_domain; 3623 else 3624 domain = sh->tx_domain; 3625 /* Register new encap/decap resource. */ 3626 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx); 3627 if (!resource) { 3628 rte_flow_error_set(ctx->error, ENOMEM, 3629 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3630 "cannot allocate resource memory"); 3631 return NULL; 3632 } 3633 *resource = *ctx_resource; 3634 resource->idx = idx; 3635 ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->cdev->ctx, 3636 domain, resource, 3637 &resource->action); 3638 if (ret) { 3639 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx); 3640 rte_flow_error_set(ctx->error, ENOMEM, 3641 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3642 NULL, "cannot create action"); 3643 return NULL; 3644 } 3645 3646 return &resource->entry; 3647 } 3648 3649 struct mlx5_list_entry * 3650 flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 3651 void *cb_ctx) 3652 { 3653 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3654 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3655 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 3656 uint32_t idx; 3657 3658 cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 3659 &idx); 3660 if (!cache_resource) { 3661 rte_flow_error_set(ctx->error, ENOMEM, 3662 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3663 "cannot allocate resource memory"); 3664 return NULL; 3665 } 3666 memcpy(cache_resource, oentry, sizeof(*cache_resource)); 3667 cache_resource->idx = idx; 3668 return &cache_resource->entry; 3669 } 3670 3671 void 3672 flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 3673 { 3674 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3675 struct mlx5_flow_dv_encap_decap_resource *res = 3676 container_of(entry, typeof(*res), entry); 3677 3678 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx); 3679 } 3680 3681 /** 3682 * Find existing encap/decap resource or create and register a new one. 3683 * 3684 * @param[in, out] dev 3685 * Pointer to rte_eth_dev structure. 3686 * @param[in, out] resource 3687 * Pointer to encap/decap resource. 3688 * @parm[in, out] dev_flow 3689 * Pointer to the dev_flow. 3690 * @param[out] error 3691 * pointer to error structure. 3692 * 3693 * @return 3694 * 0 on success otherwise -errno and errno is set. 3695 */ 3696 static int 3697 flow_dv_encap_decap_resource_register 3698 (struct rte_eth_dev *dev, 3699 struct mlx5_flow_dv_encap_decap_resource *resource, 3700 struct mlx5_flow *dev_flow, 3701 struct rte_flow_error *error) 3702 { 3703 struct mlx5_priv *priv = dev->data->dev_private; 3704 struct mlx5_dev_ctx_shared *sh = priv->sh; 3705 struct mlx5_list_entry *entry; 3706 union { 3707 struct { 3708 uint32_t ft_type:8; 3709 uint32_t refmt_type:8; 3710 /* 3711 * Header reformat actions can be shared between 3712 * non-root tables. One bit to indicate non-root 3713 * table or not. 3714 */ 3715 uint32_t is_root:1; 3716 uint32_t reserve:15; 3717 }; 3718 uint32_t v32; 3719 } encap_decap_key = { 3720 { 3721 .ft_type = resource->ft_type, 3722 .refmt_type = resource->reformat_type, 3723 .is_root = !!dev_flow->dv.group, 3724 .reserve = 0, 3725 } 3726 }; 3727 struct mlx5_flow_cb_ctx ctx = { 3728 .error = error, 3729 .data = resource, 3730 }; 3731 struct mlx5_hlist *encaps_decaps; 3732 uint64_t key64; 3733 3734 encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps, 3735 "encaps_decaps", 3736 MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ, 3737 true, true, sh, 3738 flow_dv_encap_decap_create_cb, 3739 flow_dv_encap_decap_match_cb, 3740 flow_dv_encap_decap_remove_cb, 3741 flow_dv_encap_decap_clone_cb, 3742 flow_dv_encap_decap_clone_free_cb, 3743 error); 3744 if (unlikely(!encaps_decaps)) 3745 return -rte_errno; 3746 resource->flags = dev_flow->dv.group ? 0 : 1; 3747 key64 = __rte_raw_cksum(&encap_decap_key.v32, 3748 sizeof(encap_decap_key.v32), 0); 3749 if (resource->reformat_type != 3750 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 && 3751 resource->size) 3752 key64 = __rte_raw_cksum(resource->buf, resource->size, key64); 3753 entry = mlx5_hlist_register(encaps_decaps, key64, &ctx); 3754 if (!entry) 3755 return -rte_errno; 3756 resource = container_of(entry, typeof(*resource), entry); 3757 dev_flow->dv.encap_decap = resource; 3758 dev_flow->handle->dvh.rix_encap_decap = resource->idx; 3759 return 0; 3760 } 3761 3762 /** 3763 * Find existing table jump resource or create and register a new one. 3764 * 3765 * @param[in, out] dev 3766 * Pointer to rte_eth_dev structure. 3767 * @param[in, out] tbl 3768 * Pointer to flow table resource. 3769 * @parm[in, out] dev_flow 3770 * Pointer to the dev_flow. 3771 * @param[out] error 3772 * pointer to error structure. 3773 * 3774 * @return 3775 * 0 on success otherwise -errno and errno is set. 3776 */ 3777 static int 3778 flow_dv_jump_tbl_resource_register 3779 (struct rte_eth_dev *dev __rte_unused, 3780 struct mlx5_flow_tbl_resource *tbl, 3781 struct mlx5_flow *dev_flow, 3782 struct rte_flow_error *error __rte_unused) 3783 { 3784 struct mlx5_flow_tbl_data_entry *tbl_data = 3785 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 3786 3787 MLX5_ASSERT(tbl); 3788 MLX5_ASSERT(tbl_data->jump.action); 3789 dev_flow->handle->rix_jump = tbl_data->idx; 3790 dev_flow->dv.jump = &tbl_data->jump; 3791 return 0; 3792 } 3793 3794 int 3795 flow_dv_port_id_match_cb(void *tool_ctx __rte_unused, 3796 struct mlx5_list_entry *entry, void *cb_ctx) 3797 { 3798 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3799 struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data; 3800 struct mlx5_flow_dv_port_id_action_resource *res = 3801 container_of(entry, typeof(*res), entry); 3802 3803 return ref->port_id != res->port_id; 3804 } 3805 3806 struct mlx5_list_entry * 3807 flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx) 3808 { 3809 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3810 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3811 struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data; 3812 struct mlx5_flow_dv_port_id_action_resource *resource; 3813 uint32_t idx; 3814 int ret; 3815 3816 /* Register new port id action resource. */ 3817 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx); 3818 if (!resource) { 3819 rte_flow_error_set(ctx->error, ENOMEM, 3820 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3821 "cannot allocate port_id action memory"); 3822 return NULL; 3823 } 3824 *resource = *ref; 3825 ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain, 3826 ref->port_id, 3827 &resource->action); 3828 if (ret) { 3829 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx); 3830 rte_flow_error_set(ctx->error, ENOMEM, 3831 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3832 "cannot create action"); 3833 return NULL; 3834 } 3835 resource->idx = idx; 3836 return &resource->entry; 3837 } 3838 3839 struct mlx5_list_entry * 3840 flow_dv_port_id_clone_cb(void *tool_ctx, 3841 struct mlx5_list_entry *entry __rte_unused, 3842 void *cb_ctx) 3843 { 3844 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3845 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3846 struct mlx5_flow_dv_port_id_action_resource *resource; 3847 uint32_t idx; 3848 3849 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx); 3850 if (!resource) { 3851 rte_flow_error_set(ctx->error, ENOMEM, 3852 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3853 "cannot allocate port_id action memory"); 3854 return NULL; 3855 } 3856 memcpy(resource, entry, sizeof(*resource)); 3857 resource->idx = idx; 3858 return &resource->entry; 3859 } 3860 3861 void 3862 flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 3863 { 3864 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3865 struct mlx5_flow_dv_port_id_action_resource *resource = 3866 container_of(entry, typeof(*resource), entry); 3867 3868 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx); 3869 } 3870 3871 /** 3872 * Find existing table port ID resource or create and register a new one. 3873 * 3874 * @param[in, out] dev 3875 * Pointer to rte_eth_dev structure. 3876 * @param[in, out] ref 3877 * Pointer to port ID action resource reference. 3878 * @parm[in, out] dev_flow 3879 * Pointer to the dev_flow. 3880 * @param[out] error 3881 * pointer to error structure. 3882 * 3883 * @return 3884 * 0 on success otherwise -errno and errno is set. 3885 */ 3886 static int 3887 flow_dv_port_id_action_resource_register 3888 (struct rte_eth_dev *dev, 3889 struct mlx5_flow_dv_port_id_action_resource *ref, 3890 struct mlx5_flow *dev_flow, 3891 struct rte_flow_error *error) 3892 { 3893 struct mlx5_priv *priv = dev->data->dev_private; 3894 struct mlx5_list_entry *entry; 3895 struct mlx5_flow_dv_port_id_action_resource *resource; 3896 struct mlx5_flow_cb_ctx ctx = { 3897 .error = error, 3898 .data = ref, 3899 }; 3900 3901 entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx); 3902 if (!entry) 3903 return -rte_errno; 3904 resource = container_of(entry, typeof(*resource), entry); 3905 dev_flow->dv.port_id_action = resource; 3906 dev_flow->handle->rix_port_id_action = resource->idx; 3907 return 0; 3908 } 3909 3910 int 3911 flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused, 3912 struct mlx5_list_entry *entry, void *cb_ctx) 3913 { 3914 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3915 struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data; 3916 struct mlx5_flow_dv_push_vlan_action_resource *res = 3917 container_of(entry, typeof(*res), entry); 3918 3919 return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type; 3920 } 3921 3922 struct mlx5_list_entry * 3923 flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx) 3924 { 3925 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3926 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3927 struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data; 3928 struct mlx5_flow_dv_push_vlan_action_resource *resource; 3929 struct mlx5dv_dr_domain *domain; 3930 uint32_t idx; 3931 int ret; 3932 3933 /* Register new port id action resource. */ 3934 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx); 3935 if (!resource) { 3936 rte_flow_error_set(ctx->error, ENOMEM, 3937 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3938 "cannot allocate push_vlan action memory"); 3939 return NULL; 3940 } 3941 *resource = *ref; 3942 if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 3943 domain = sh->fdb_domain; 3944 else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 3945 domain = sh->rx_domain; 3946 else 3947 domain = sh->tx_domain; 3948 ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag, 3949 &resource->action); 3950 if (ret) { 3951 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); 3952 rte_flow_error_set(ctx->error, ENOMEM, 3953 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3954 "cannot create push vlan action"); 3955 return NULL; 3956 } 3957 resource->idx = idx; 3958 return &resource->entry; 3959 } 3960 3961 struct mlx5_list_entry * 3962 flow_dv_push_vlan_clone_cb(void *tool_ctx, 3963 struct mlx5_list_entry *entry __rte_unused, 3964 void *cb_ctx) 3965 { 3966 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3967 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3968 struct mlx5_flow_dv_push_vlan_action_resource *resource; 3969 uint32_t idx; 3970 3971 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx); 3972 if (!resource) { 3973 rte_flow_error_set(ctx->error, ENOMEM, 3974 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3975 "cannot allocate push_vlan action memory"); 3976 return NULL; 3977 } 3978 memcpy(resource, entry, sizeof(*resource)); 3979 resource->idx = idx; 3980 return &resource->entry; 3981 } 3982 3983 void 3984 flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 3985 { 3986 struct mlx5_dev_ctx_shared *sh = tool_ctx; 3987 struct mlx5_flow_dv_push_vlan_action_resource *resource = 3988 container_of(entry, typeof(*resource), entry); 3989 3990 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx); 3991 } 3992 3993 /** 3994 * Find existing push vlan resource or create and register a new one. 3995 * 3996 * @param [in, out] dev 3997 * Pointer to rte_eth_dev structure. 3998 * @param[in, out] ref 3999 * Pointer to port ID action resource reference. 4000 * @parm[in, out] dev_flow 4001 * Pointer to the dev_flow. 4002 * @param[out] error 4003 * pointer to error structure. 4004 * 4005 * @return 4006 * 0 on success otherwise -errno and errno is set. 4007 */ 4008 static int 4009 flow_dv_push_vlan_action_resource_register 4010 (struct rte_eth_dev *dev, 4011 struct mlx5_flow_dv_push_vlan_action_resource *ref, 4012 struct mlx5_flow *dev_flow, 4013 struct rte_flow_error *error) 4014 { 4015 struct mlx5_priv *priv = dev->data->dev_private; 4016 struct mlx5_flow_dv_push_vlan_action_resource *resource; 4017 struct mlx5_list_entry *entry; 4018 struct mlx5_flow_cb_ctx ctx = { 4019 .error = error, 4020 .data = ref, 4021 }; 4022 4023 entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx); 4024 if (!entry) 4025 return -rte_errno; 4026 resource = container_of(entry, typeof(*resource), entry); 4027 4028 dev_flow->handle->dvh.rix_push_vlan = resource->idx; 4029 dev_flow->dv.push_vlan_res = resource; 4030 return 0; 4031 } 4032 4033 /** 4034 * Get the size of specific rte_flow_item_type hdr size 4035 * 4036 * @param[in] item_type 4037 * Tested rte_flow_item_type. 4038 * 4039 * @return 4040 * sizeof struct item_type, 0 if void or irrelevant. 4041 */ 4042 size_t 4043 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type) 4044 { 4045 size_t retval; 4046 4047 switch (item_type) { 4048 case RTE_FLOW_ITEM_TYPE_ETH: 4049 retval = sizeof(struct rte_ether_hdr); 4050 break; 4051 case RTE_FLOW_ITEM_TYPE_VLAN: 4052 retval = sizeof(struct rte_vlan_hdr); 4053 break; 4054 case RTE_FLOW_ITEM_TYPE_IPV4: 4055 retval = sizeof(struct rte_ipv4_hdr); 4056 break; 4057 case RTE_FLOW_ITEM_TYPE_IPV6: 4058 retval = sizeof(struct rte_ipv6_hdr); 4059 break; 4060 case RTE_FLOW_ITEM_TYPE_UDP: 4061 retval = sizeof(struct rte_udp_hdr); 4062 break; 4063 case RTE_FLOW_ITEM_TYPE_TCP: 4064 retval = sizeof(struct rte_tcp_hdr); 4065 break; 4066 case RTE_FLOW_ITEM_TYPE_VXLAN: 4067 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 4068 retval = sizeof(struct rte_vxlan_hdr); 4069 break; 4070 case RTE_FLOW_ITEM_TYPE_GRE: 4071 case RTE_FLOW_ITEM_TYPE_NVGRE: 4072 retval = sizeof(struct rte_gre_hdr); 4073 break; 4074 case RTE_FLOW_ITEM_TYPE_MPLS: 4075 retval = sizeof(struct rte_mpls_hdr); 4076 break; 4077 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */ 4078 default: 4079 retval = 0; 4080 break; 4081 } 4082 return retval; 4083 } 4084 4085 #define MLX5_ENCAP_IPV4_VERSION 0x40 4086 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05 4087 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40 4088 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000 4089 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff 4090 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000 4091 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04 4092 4093 /** 4094 * Convert the encap action data from list of rte_flow_item to raw buffer 4095 * 4096 * @param[in] items 4097 * Pointer to rte_flow_item objects list. 4098 * @param[out] buf 4099 * Pointer to the output buffer. 4100 * @param[out] size 4101 * Pointer to the output buffer size. 4102 * @param[out] error 4103 * Pointer to the error structure. 4104 * 4105 * @return 4106 * 0 on success, a negative errno value otherwise and rte_errno is set. 4107 */ 4108 int 4109 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, 4110 size_t *size, struct rte_flow_error *error) 4111 { 4112 struct rte_ether_hdr *eth = NULL; 4113 struct rte_vlan_hdr *vlan = NULL; 4114 struct rte_ipv4_hdr *ipv4 = NULL; 4115 struct rte_ipv6_hdr *ipv6 = NULL; 4116 struct rte_udp_hdr *udp = NULL; 4117 struct rte_vxlan_hdr *vxlan = NULL; 4118 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL; 4119 struct rte_gre_hdr *gre = NULL; 4120 size_t len; 4121 size_t temp_size = 0; 4122 4123 if (!items) 4124 return rte_flow_error_set(error, EINVAL, 4125 RTE_FLOW_ERROR_TYPE_ACTION, 4126 NULL, "invalid empty data"); 4127 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4128 len = flow_dv_get_item_hdr_len(items->type); 4129 if (len + temp_size > MLX5_ENCAP_MAX_LEN) 4130 return rte_flow_error_set(error, EINVAL, 4131 RTE_FLOW_ERROR_TYPE_ACTION, 4132 (void *)items->type, 4133 "items total size is too big" 4134 " for encap action"); 4135 rte_memcpy((void *)&buf[temp_size], items->spec, len); 4136 switch (items->type) { 4137 case RTE_FLOW_ITEM_TYPE_ETH: 4138 eth = (struct rte_ether_hdr *)&buf[temp_size]; 4139 break; 4140 case RTE_FLOW_ITEM_TYPE_VLAN: 4141 vlan = (struct rte_vlan_hdr *)&buf[temp_size]; 4142 if (!eth) 4143 return rte_flow_error_set(error, EINVAL, 4144 RTE_FLOW_ERROR_TYPE_ACTION, 4145 (void *)items->type, 4146 "eth header not found"); 4147 if (!eth->ether_type) 4148 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN); 4149 break; 4150 case RTE_FLOW_ITEM_TYPE_IPV4: 4151 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size]; 4152 if (!vlan && !eth) 4153 return rte_flow_error_set(error, EINVAL, 4154 RTE_FLOW_ERROR_TYPE_ACTION, 4155 (void *)items->type, 4156 "neither eth nor vlan" 4157 " header found"); 4158 if (vlan && !vlan->eth_proto) 4159 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4); 4160 else if (eth && !eth->ether_type) 4161 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4); 4162 if (!ipv4->version_ihl) 4163 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION | 4164 MLX5_ENCAP_IPV4_IHL_MIN; 4165 if (!ipv4->time_to_live) 4166 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF; 4167 break; 4168 case RTE_FLOW_ITEM_TYPE_IPV6: 4169 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size]; 4170 if (!vlan && !eth) 4171 return rte_flow_error_set(error, EINVAL, 4172 RTE_FLOW_ERROR_TYPE_ACTION, 4173 (void *)items->type, 4174 "neither eth nor vlan" 4175 " header found"); 4176 if (vlan && !vlan->eth_proto) 4177 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6); 4178 else if (eth && !eth->ether_type) 4179 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6); 4180 if (!ipv6->vtc_flow) 4181 ipv6->vtc_flow = 4182 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW); 4183 if (!ipv6->hop_limits) 4184 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT; 4185 break; 4186 case RTE_FLOW_ITEM_TYPE_UDP: 4187 udp = (struct rte_udp_hdr *)&buf[temp_size]; 4188 if (!ipv4 && !ipv6) 4189 return rte_flow_error_set(error, EINVAL, 4190 RTE_FLOW_ERROR_TYPE_ACTION, 4191 (void *)items->type, 4192 "ip header not found"); 4193 if (ipv4 && !ipv4->next_proto_id) 4194 ipv4->next_proto_id = IPPROTO_UDP; 4195 else if (ipv6 && !ipv6->proto) 4196 ipv6->proto = IPPROTO_UDP; 4197 break; 4198 case RTE_FLOW_ITEM_TYPE_VXLAN: 4199 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size]; 4200 if (!udp) 4201 return rte_flow_error_set(error, EINVAL, 4202 RTE_FLOW_ERROR_TYPE_ACTION, 4203 (void *)items->type, 4204 "udp header not found"); 4205 if (!udp->dst_port) 4206 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN); 4207 if (!vxlan->vx_flags) 4208 vxlan->vx_flags = 4209 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS); 4210 break; 4211 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 4212 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size]; 4213 if (!udp) 4214 return rte_flow_error_set(error, EINVAL, 4215 RTE_FLOW_ERROR_TYPE_ACTION, 4216 (void *)items->type, 4217 "udp header not found"); 4218 if (!vxlan_gpe->proto) 4219 return rte_flow_error_set(error, EINVAL, 4220 RTE_FLOW_ERROR_TYPE_ACTION, 4221 (void *)items->type, 4222 "next protocol not found"); 4223 if (!udp->dst_port) 4224 udp->dst_port = 4225 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE); 4226 if (!vxlan_gpe->vx_flags) 4227 vxlan_gpe->vx_flags = 4228 MLX5_ENCAP_VXLAN_GPE_FLAGS; 4229 break; 4230 case RTE_FLOW_ITEM_TYPE_GRE: 4231 case RTE_FLOW_ITEM_TYPE_NVGRE: 4232 gre = (struct rte_gre_hdr *)&buf[temp_size]; 4233 if (!gre->proto) 4234 return rte_flow_error_set(error, EINVAL, 4235 RTE_FLOW_ERROR_TYPE_ACTION, 4236 (void *)items->type, 4237 "next protocol not found"); 4238 if (!ipv4 && !ipv6) 4239 return rte_flow_error_set(error, EINVAL, 4240 RTE_FLOW_ERROR_TYPE_ACTION, 4241 (void *)items->type, 4242 "ip header not found"); 4243 if (ipv4 && !ipv4->next_proto_id) 4244 ipv4->next_proto_id = IPPROTO_GRE; 4245 else if (ipv6 && !ipv6->proto) 4246 ipv6->proto = IPPROTO_GRE; 4247 break; 4248 case RTE_FLOW_ITEM_TYPE_VOID: 4249 break; 4250 default: 4251 return rte_flow_error_set(error, EINVAL, 4252 RTE_FLOW_ERROR_TYPE_ACTION, 4253 (void *)items->type, 4254 "unsupported item type"); 4255 break; 4256 } 4257 temp_size += len; 4258 } 4259 *size = temp_size; 4260 return 0; 4261 } 4262 4263 static int 4264 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error) 4265 { 4266 struct rte_ether_hdr *eth = NULL; 4267 struct rte_vlan_hdr *vlan = NULL; 4268 struct rte_ipv6_hdr *ipv6 = NULL; 4269 struct rte_udp_hdr *udp = NULL; 4270 char *next_hdr; 4271 uint16_t proto; 4272 4273 eth = (struct rte_ether_hdr *)data; 4274 next_hdr = (char *)(eth + 1); 4275 proto = RTE_BE16(eth->ether_type); 4276 4277 /* VLAN skipping */ 4278 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) { 4279 vlan = (struct rte_vlan_hdr *)next_hdr; 4280 proto = RTE_BE16(vlan->eth_proto); 4281 next_hdr += sizeof(struct rte_vlan_hdr); 4282 } 4283 4284 /* HW calculates IPv4 csum. no need to proceed */ 4285 if (proto == RTE_ETHER_TYPE_IPV4) 4286 return 0; 4287 4288 /* non IPv4/IPv6 header. not supported */ 4289 if (proto != RTE_ETHER_TYPE_IPV6) { 4290 return rte_flow_error_set(error, ENOTSUP, 4291 RTE_FLOW_ERROR_TYPE_ACTION, 4292 NULL, "Cannot offload non IPv4/IPv6"); 4293 } 4294 4295 ipv6 = (struct rte_ipv6_hdr *)next_hdr; 4296 4297 /* ignore non UDP */ 4298 if (ipv6->proto != IPPROTO_UDP) 4299 return 0; 4300 4301 udp = (struct rte_udp_hdr *)(ipv6 + 1); 4302 udp->dgram_cksum = 0; 4303 4304 return 0; 4305 } 4306 4307 /** 4308 * Convert L2 encap action to DV specification. 4309 * 4310 * @param[in] dev 4311 * Pointer to rte_eth_dev structure. 4312 * @param[in] action 4313 * Pointer to action structure. 4314 * @param[in, out] dev_flow 4315 * Pointer to the mlx5_flow. 4316 * @param[in] transfer 4317 * Mark if the flow is E-Switch flow. 4318 * @param[out] error 4319 * Pointer to the error structure. 4320 * 4321 * @return 4322 * 0 on success, a negative errno value otherwise and rte_errno is set. 4323 */ 4324 static int 4325 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, 4326 const struct rte_flow_action *action, 4327 struct mlx5_flow *dev_flow, 4328 uint8_t transfer, 4329 struct rte_flow_error *error) 4330 { 4331 const struct rte_flow_item *encap_data; 4332 const struct rte_flow_action_raw_encap *raw_encap_data; 4333 struct mlx5_flow_dv_encap_decap_resource res = { 4334 .reformat_type = 4335 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL, 4336 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 4337 MLX5DV_FLOW_TABLE_TYPE_NIC_TX, 4338 }; 4339 4340 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 4341 raw_encap_data = 4342 (const struct rte_flow_action_raw_encap *)action->conf; 4343 res.size = raw_encap_data->size; 4344 memcpy(res.buf, raw_encap_data->data, res.size); 4345 } else { 4346 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) 4347 encap_data = 4348 ((const struct rte_flow_action_vxlan_encap *) 4349 action->conf)->definition; 4350 else 4351 encap_data = 4352 ((const struct rte_flow_action_nvgre_encap *) 4353 action->conf)->definition; 4354 if (flow_dv_convert_encap_data(encap_data, res.buf, 4355 &res.size, error)) 4356 return -rte_errno; 4357 } 4358 if (flow_dv_zero_encap_udp_csum(res.buf, error)) 4359 return -rte_errno; 4360 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 4361 return rte_flow_error_set(error, EINVAL, 4362 RTE_FLOW_ERROR_TYPE_ACTION, 4363 NULL, "can't create L2 encap action"); 4364 return 0; 4365 } 4366 4367 /** 4368 * Convert L2 decap action to DV specification. 4369 * 4370 * @param[in] dev 4371 * Pointer to rte_eth_dev structure. 4372 * @param[in, out] dev_flow 4373 * Pointer to the mlx5_flow. 4374 * @param[in] transfer 4375 * Mark if the flow is E-Switch flow. 4376 * @param[out] error 4377 * Pointer to the error structure. 4378 * 4379 * @return 4380 * 0 on success, a negative errno value otherwise and rte_errno is set. 4381 */ 4382 static int 4383 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev, 4384 struct mlx5_flow *dev_flow, 4385 uint8_t transfer, 4386 struct rte_flow_error *error) 4387 { 4388 struct mlx5_flow_dv_encap_decap_resource res = { 4389 .size = 0, 4390 .reformat_type = 4391 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2, 4392 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 4393 MLX5DV_FLOW_TABLE_TYPE_NIC_RX, 4394 }; 4395 4396 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 4397 return rte_flow_error_set(error, EINVAL, 4398 RTE_FLOW_ERROR_TYPE_ACTION, 4399 NULL, "can't create L2 decap action"); 4400 return 0; 4401 } 4402 4403 /** 4404 * Convert raw decap/encap (L3 tunnel) action to DV specification. 4405 * 4406 * @param[in] dev 4407 * Pointer to rte_eth_dev structure. 4408 * @param[in] action 4409 * Pointer to action structure. 4410 * @param[in, out] dev_flow 4411 * Pointer to the mlx5_flow. 4412 * @param[in] attr 4413 * Pointer to the flow attributes. 4414 * @param[out] error 4415 * Pointer to the error structure. 4416 * 4417 * @return 4418 * 0 on success, a negative errno value otherwise and rte_errno is set. 4419 */ 4420 static int 4421 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, 4422 const struct rte_flow_action *action, 4423 struct mlx5_flow *dev_flow, 4424 const struct rte_flow_attr *attr, 4425 struct rte_flow_error *error) 4426 { 4427 const struct rte_flow_action_raw_encap *encap_data; 4428 struct mlx5_flow_dv_encap_decap_resource res; 4429 4430 memset(&res, 0, sizeof(res)); 4431 encap_data = (const struct rte_flow_action_raw_encap *)action->conf; 4432 res.size = encap_data->size; 4433 memcpy(res.buf, encap_data->data, res.size); 4434 res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ? 4435 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 : 4436 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 4437 if (attr->transfer) 4438 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 4439 else 4440 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 4441 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 4442 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 4443 return rte_flow_error_set(error, EINVAL, 4444 RTE_FLOW_ERROR_TYPE_ACTION, 4445 NULL, "can't create encap action"); 4446 return 0; 4447 } 4448 4449 /** 4450 * Create action push VLAN. 4451 * 4452 * @param[in] dev 4453 * Pointer to rte_eth_dev structure. 4454 * @param[in] attr 4455 * Pointer to the flow attributes. 4456 * @param[in] vlan 4457 * Pointer to the vlan to push to the Ethernet header. 4458 * @param[in, out] dev_flow 4459 * Pointer to the mlx5_flow. 4460 * @param[out] error 4461 * Pointer to the error structure. 4462 * 4463 * @return 4464 * 0 on success, a negative errno value otherwise and rte_errno is set. 4465 */ 4466 static int 4467 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev, 4468 const struct rte_flow_attr *attr, 4469 const struct rte_vlan_hdr *vlan, 4470 struct mlx5_flow *dev_flow, 4471 struct rte_flow_error *error) 4472 { 4473 struct mlx5_flow_dv_push_vlan_action_resource res; 4474 4475 memset(&res, 0, sizeof(res)); 4476 res.vlan_tag = 4477 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 | 4478 vlan->vlan_tci); 4479 if (attr->transfer) 4480 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 4481 else 4482 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 4483 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 4484 return flow_dv_push_vlan_action_resource_register 4485 (dev, &res, dev_flow, error); 4486 } 4487 4488 /** 4489 * Validate the modify-header actions. 4490 * 4491 * @param[in] action_flags 4492 * Holds the actions detected until now. 4493 * @param[in] action 4494 * Pointer to the modify action. 4495 * @param[out] error 4496 * Pointer to error structure. 4497 * 4498 * @return 4499 * 0 on success, a negative errno value otherwise and rte_errno is set. 4500 */ 4501 static int 4502 flow_dv_validate_action_modify_hdr(const uint64_t action_flags, 4503 const struct rte_flow_action *action, 4504 struct rte_flow_error *error) 4505 { 4506 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf) 4507 return rte_flow_error_set(error, EINVAL, 4508 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 4509 NULL, "action configuration not set"); 4510 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 4511 return rte_flow_error_set(error, EINVAL, 4512 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 4513 "can't have encap action before" 4514 " modify action"); 4515 return 0; 4516 } 4517 4518 /** 4519 * Validate the modify-header MAC address actions. 4520 * 4521 * @param[in] action_flags 4522 * Holds the actions detected until now. 4523 * @param[in] action 4524 * Pointer to the modify action. 4525 * @param[in] item_flags 4526 * Holds the items detected. 4527 * @param[out] error 4528 * Pointer to error structure. 4529 * 4530 * @return 4531 * 0 on success, a negative errno value otherwise and rte_errno is set. 4532 */ 4533 static int 4534 flow_dv_validate_action_modify_mac(const uint64_t action_flags, 4535 const struct rte_flow_action *action, 4536 const uint64_t item_flags, 4537 struct rte_flow_error *error) 4538 { 4539 int ret = 0; 4540 4541 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 4542 if (!ret) { 4543 if (!(item_flags & MLX5_FLOW_LAYER_L2)) 4544 return rte_flow_error_set(error, EINVAL, 4545 RTE_FLOW_ERROR_TYPE_ACTION, 4546 NULL, 4547 "no L2 item in pattern"); 4548 } 4549 return ret; 4550 } 4551 4552 /** 4553 * Validate the modify-header IPv4 address actions. 4554 * 4555 * @param[in] action_flags 4556 * Holds the actions detected until now. 4557 * @param[in] action 4558 * Pointer to the modify action. 4559 * @param[in] item_flags 4560 * Holds the items detected. 4561 * @param[out] error 4562 * Pointer to error structure. 4563 * 4564 * @return 4565 * 0 on success, a negative errno value otherwise and rte_errno is set. 4566 */ 4567 static int 4568 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags, 4569 const struct rte_flow_action *action, 4570 const uint64_t item_flags, 4571 struct rte_flow_error *error) 4572 { 4573 int ret = 0; 4574 uint64_t layer; 4575 4576 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 4577 if (!ret) { 4578 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 4579 MLX5_FLOW_LAYER_INNER_L3_IPV4 : 4580 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4581 if (!(item_flags & layer)) 4582 return rte_flow_error_set(error, EINVAL, 4583 RTE_FLOW_ERROR_TYPE_ACTION, 4584 NULL, 4585 "no ipv4 item in pattern"); 4586 } 4587 return ret; 4588 } 4589 4590 /** 4591 * Validate the modify-header IPv6 address actions. 4592 * 4593 * @param[in] action_flags 4594 * Holds the actions detected until now. 4595 * @param[in] action 4596 * Pointer to the modify action. 4597 * @param[in] item_flags 4598 * Holds the items detected. 4599 * @param[out] error 4600 * Pointer to error structure. 4601 * 4602 * @return 4603 * 0 on success, a negative errno value otherwise and rte_errno is set. 4604 */ 4605 static int 4606 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags, 4607 const struct rte_flow_action *action, 4608 const uint64_t item_flags, 4609 struct rte_flow_error *error) 4610 { 4611 int ret = 0; 4612 uint64_t layer; 4613 4614 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 4615 if (!ret) { 4616 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 4617 MLX5_FLOW_LAYER_INNER_L3_IPV6 : 4618 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4619 if (!(item_flags & layer)) 4620 return rte_flow_error_set(error, EINVAL, 4621 RTE_FLOW_ERROR_TYPE_ACTION, 4622 NULL, 4623 "no ipv6 item in pattern"); 4624 } 4625 return ret; 4626 } 4627 4628 /** 4629 * Validate the modify-header TP actions. 4630 * 4631 * @param[in] action_flags 4632 * Holds the actions detected until now. 4633 * @param[in] action 4634 * Pointer to the modify action. 4635 * @param[in] item_flags 4636 * Holds the items detected. 4637 * @param[out] error 4638 * Pointer to error structure. 4639 * 4640 * @return 4641 * 0 on success, a negative errno value otherwise and rte_errno is set. 4642 */ 4643 static int 4644 flow_dv_validate_action_modify_tp(const uint64_t action_flags, 4645 const struct rte_flow_action *action, 4646 const uint64_t item_flags, 4647 struct rte_flow_error *error) 4648 { 4649 int ret = 0; 4650 uint64_t layer; 4651 4652 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 4653 if (!ret) { 4654 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 4655 MLX5_FLOW_LAYER_INNER_L4 : 4656 MLX5_FLOW_LAYER_OUTER_L4; 4657 if (!(item_flags & layer)) 4658 return rte_flow_error_set(error, EINVAL, 4659 RTE_FLOW_ERROR_TYPE_ACTION, 4660 NULL, "no transport layer " 4661 "in pattern"); 4662 } 4663 return ret; 4664 } 4665 4666 /** 4667 * Validate the modify-header actions of increment/decrement 4668 * TCP Sequence-number. 4669 * 4670 * @param[in] action_flags 4671 * Holds the actions detected until now. 4672 * @param[in] action 4673 * Pointer to the modify action. 4674 * @param[in] item_flags 4675 * Holds the items detected. 4676 * @param[out] error 4677 * Pointer to error structure. 4678 * 4679 * @return 4680 * 0 on success, a negative errno value otherwise and rte_errno is set. 4681 */ 4682 static int 4683 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags, 4684 const struct rte_flow_action *action, 4685 const uint64_t item_flags, 4686 struct rte_flow_error *error) 4687 { 4688 int ret = 0; 4689 uint64_t layer; 4690 4691 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 4692 if (!ret) { 4693 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 4694 MLX5_FLOW_LAYER_INNER_L4_TCP : 4695 MLX5_FLOW_LAYER_OUTER_L4_TCP; 4696 if (!(item_flags & layer)) 4697 return rte_flow_error_set(error, EINVAL, 4698 RTE_FLOW_ERROR_TYPE_ACTION, 4699 NULL, "no TCP item in" 4700 " pattern"); 4701 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ && 4702 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) || 4703 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ && 4704 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ))) 4705 return rte_flow_error_set(error, EINVAL, 4706 RTE_FLOW_ERROR_TYPE_ACTION, 4707 NULL, 4708 "cannot decrease and increase" 4709 " TCP sequence number" 4710 " at the same time"); 4711 } 4712 return ret; 4713 } 4714 4715 /** 4716 * Validate the modify-header actions of increment/decrement 4717 * TCP Acknowledgment number. 4718 * 4719 * @param[in] action_flags 4720 * Holds the actions detected until now. 4721 * @param[in] action 4722 * Pointer to the modify action. 4723 * @param[in] item_flags 4724 * Holds the items detected. 4725 * @param[out] error 4726 * Pointer to error structure. 4727 * 4728 * @return 4729 * 0 on success, a negative errno value otherwise and rte_errno is set. 4730 */ 4731 static int 4732 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags, 4733 const struct rte_flow_action *action, 4734 const uint64_t item_flags, 4735 struct rte_flow_error *error) 4736 { 4737 int ret = 0; 4738 uint64_t layer; 4739 4740 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 4741 if (!ret) { 4742 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 4743 MLX5_FLOW_LAYER_INNER_L4_TCP : 4744 MLX5_FLOW_LAYER_OUTER_L4_TCP; 4745 if (!(item_flags & layer)) 4746 return rte_flow_error_set(error, EINVAL, 4747 RTE_FLOW_ERROR_TYPE_ACTION, 4748 NULL, "no TCP item in" 4749 " pattern"); 4750 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK && 4751 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) || 4752 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK && 4753 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK))) 4754 return rte_flow_error_set(error, EINVAL, 4755 RTE_FLOW_ERROR_TYPE_ACTION, 4756 NULL, 4757 "cannot decrease and increase" 4758 " TCP acknowledgment number" 4759 " at the same time"); 4760 } 4761 return ret; 4762 } 4763 4764 /** 4765 * Validate the modify-header TTL actions. 4766 * 4767 * @param[in] action_flags 4768 * Holds the actions detected until now. 4769 * @param[in] action 4770 * Pointer to the modify action. 4771 * @param[in] item_flags 4772 * Holds the items detected. 4773 * @param[out] error 4774 * Pointer to error structure. 4775 * 4776 * @return 4777 * 0 on success, a negative errno value otherwise and rte_errno is set. 4778 */ 4779 static int 4780 flow_dv_validate_action_modify_ttl(const uint64_t action_flags, 4781 const struct rte_flow_action *action, 4782 const uint64_t item_flags, 4783 struct rte_flow_error *error) 4784 { 4785 int ret = 0; 4786 uint64_t layer; 4787 4788 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 4789 if (!ret) { 4790 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 4791 MLX5_FLOW_LAYER_INNER_L3 : 4792 MLX5_FLOW_LAYER_OUTER_L3; 4793 if (!(item_flags & layer)) 4794 return rte_flow_error_set(error, EINVAL, 4795 RTE_FLOW_ERROR_TYPE_ACTION, 4796 NULL, 4797 "no IP protocol in pattern"); 4798 } 4799 return ret; 4800 } 4801 4802 /** 4803 * Validate the generic modify field actions. 4804 * @param[in] dev 4805 * Pointer to the rte_eth_dev structure. 4806 * @param[in] action_flags 4807 * Holds the actions detected until now. 4808 * @param[in] action 4809 * Pointer to the modify action. 4810 * @param[in] attr 4811 * Pointer to the flow attributes. 4812 * @param[out] error 4813 * Pointer to error structure. 4814 * 4815 * @return 4816 * Number of header fields to modify (0 or more) on success, 4817 * a negative errno value otherwise and rte_errno is set. 4818 */ 4819 static int 4820 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev, 4821 const uint64_t action_flags, 4822 const struct rte_flow_action *action, 4823 const struct rte_flow_attr *attr, 4824 struct rte_flow_error *error) 4825 { 4826 int ret = 0; 4827 struct mlx5_priv *priv = dev->data->dev_private; 4828 struct mlx5_sh_config *config = &priv->sh->config; 4829 const struct rte_flow_action_modify_field *action_modify_field = 4830 action->conf; 4831 uint32_t dst_width = mlx5_flow_item_field_width(dev, 4832 action_modify_field->dst.field, 4833 -1, attr, error); 4834 uint32_t src_width = mlx5_flow_item_field_width(dev, 4835 action_modify_field->src.field, 4836 dst_width, attr, error); 4837 4838 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 4839 if (ret) 4840 return ret; 4841 4842 if (action_modify_field->width == 0) 4843 return rte_flow_error_set(error, EINVAL, 4844 RTE_FLOW_ERROR_TYPE_ACTION, action, 4845 "no bits are requested to be modified"); 4846 else if (action_modify_field->width > dst_width || 4847 action_modify_field->width > src_width) 4848 return rte_flow_error_set(error, EINVAL, 4849 RTE_FLOW_ERROR_TYPE_ACTION, action, 4850 "cannot modify more bits than" 4851 " the width of a field"); 4852 if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE && 4853 action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) { 4854 if ((action_modify_field->dst.offset + 4855 action_modify_field->width > dst_width) || 4856 (action_modify_field->dst.offset % 32)) 4857 return rte_flow_error_set(error, EINVAL, 4858 RTE_FLOW_ERROR_TYPE_ACTION, action, 4859 "destination offset is too big" 4860 " or not aligned to 4 bytes"); 4861 if (action_modify_field->dst.level && 4862 action_modify_field->dst.field != RTE_FLOW_FIELD_TAG) 4863 return rte_flow_error_set(error, ENOTSUP, 4864 RTE_FLOW_ERROR_TYPE_ACTION, action, 4865 "inner header fields modification" 4866 " is not supported"); 4867 } 4868 if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE && 4869 action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) { 4870 if (!attr->transfer && !attr->group) 4871 return rte_flow_error_set(error, ENOTSUP, 4872 RTE_FLOW_ERROR_TYPE_ACTION, action, 4873 "modify field action is not" 4874 " supported for group 0"); 4875 if ((action_modify_field->src.offset + 4876 action_modify_field->width > src_width) || 4877 (action_modify_field->src.offset % 32)) 4878 return rte_flow_error_set(error, EINVAL, 4879 RTE_FLOW_ERROR_TYPE_ACTION, action, 4880 "source offset is too big" 4881 " or not aligned to 4 bytes"); 4882 if (action_modify_field->src.level && 4883 action_modify_field->src.field != RTE_FLOW_FIELD_TAG) 4884 return rte_flow_error_set(error, ENOTSUP, 4885 RTE_FLOW_ERROR_TYPE_ACTION, action, 4886 "inner header fields modification" 4887 " is not supported"); 4888 } 4889 if ((action_modify_field->dst.field == 4890 action_modify_field->src.field) && 4891 (action_modify_field->dst.level == 4892 action_modify_field->src.level)) 4893 return rte_flow_error_set(error, EINVAL, 4894 RTE_FLOW_ERROR_TYPE_ACTION, action, 4895 "source and destination fields" 4896 " cannot be the same"); 4897 if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE || 4898 action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER || 4899 action_modify_field->dst.field == RTE_FLOW_FIELD_MARK) 4900 return rte_flow_error_set(error, EINVAL, 4901 RTE_FLOW_ERROR_TYPE_ACTION, action, 4902 "mark, immediate value or a pointer to it" 4903 " cannot be used as a destination"); 4904 if (action_modify_field->dst.field == RTE_FLOW_FIELD_START || 4905 action_modify_field->src.field == RTE_FLOW_FIELD_START) 4906 return rte_flow_error_set(error, ENOTSUP, 4907 RTE_FLOW_ERROR_TYPE_ACTION, action, 4908 "modifications of an arbitrary" 4909 " place in a packet is not supported"); 4910 if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE || 4911 action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE) 4912 return rte_flow_error_set(error, ENOTSUP, 4913 RTE_FLOW_ERROR_TYPE_ACTION, action, 4914 "modifications of the 802.1Q Tag" 4915 " Identifier is not supported"); 4916 if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI || 4917 action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI) 4918 return rte_flow_error_set(error, ENOTSUP, 4919 RTE_FLOW_ERROR_TYPE_ACTION, action, 4920 "modifications of the VXLAN Network" 4921 " Identifier is not supported"); 4922 if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI || 4923 action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI) 4924 return rte_flow_error_set(error, ENOTSUP, 4925 RTE_FLOW_ERROR_TYPE_ACTION, action, 4926 "modifications of the GENEVE Network" 4927 " Identifier is not supported"); 4928 if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK || 4929 action_modify_field->src.field == RTE_FLOW_FIELD_MARK) 4930 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4931 !mlx5_flow_ext_mreg_supported(dev)) 4932 return rte_flow_error_set(error, ENOTSUP, 4933 RTE_FLOW_ERROR_TYPE_ACTION, action, 4934 "cannot modify mark in legacy mode" 4935 " or without extensive registers"); 4936 if (action_modify_field->dst.field == RTE_FLOW_FIELD_META || 4937 action_modify_field->src.field == RTE_FLOW_FIELD_META) { 4938 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4939 !mlx5_flow_ext_mreg_supported(dev)) 4940 return rte_flow_error_set(error, ENOTSUP, 4941 RTE_FLOW_ERROR_TYPE_ACTION, action, 4942 "cannot modify meta without" 4943 " extensive registers support"); 4944 ret = flow_dv_get_metadata_reg(dev, attr, error); 4945 if (ret < 0 || ret == REG_NON) 4946 return rte_flow_error_set(error, ENOTSUP, 4947 RTE_FLOW_ERROR_TYPE_ACTION, action, 4948 "cannot modify meta without" 4949 " extensive registers available"); 4950 } 4951 if (action_modify_field->operation != RTE_FLOW_MODIFY_SET) 4952 return rte_flow_error_set(error, ENOTSUP, 4953 RTE_FLOW_ERROR_TYPE_ACTION, action, 4954 "add and sub operations" 4955 " are not supported"); 4956 return (action_modify_field->width / 32) + 4957 !!(action_modify_field->width % 32); 4958 } 4959 4960 /** 4961 * Validate jump action. 4962 * 4963 * @param[in] action 4964 * Pointer to the jump action. 4965 * @param[in] action_flags 4966 * Holds the actions detected until now. 4967 * @param[in] attributes 4968 * Pointer to flow attributes 4969 * @param[in] external 4970 * Action belongs to flow rule created by request external to PMD. 4971 * @param[out] error 4972 * Pointer to error structure. 4973 * 4974 * @return 4975 * 0 on success, a negative errno value otherwise and rte_errno is set. 4976 */ 4977 static int 4978 flow_dv_validate_action_jump(struct rte_eth_dev *dev, 4979 const struct mlx5_flow_tunnel *tunnel, 4980 const struct rte_flow_action *action, 4981 uint64_t action_flags, 4982 const struct rte_flow_attr *attributes, 4983 bool external, struct rte_flow_error *error) 4984 { 4985 uint32_t target_group, table = 0; 4986 int ret = 0; 4987 struct flow_grp_info grp_info = { 4988 .external = !!external, 4989 .transfer = !!attributes->transfer, 4990 .fdb_def_rule = 1, 4991 .std_tbl_fix = 0 4992 }; 4993 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 4994 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 4995 return rte_flow_error_set(error, EINVAL, 4996 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 4997 "can't have 2 fate actions in" 4998 " same flow"); 4999 if (!action->conf) 5000 return rte_flow_error_set(error, EINVAL, 5001 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 5002 NULL, "action configuration not set"); 5003 target_group = 5004 ((const struct rte_flow_action_jump *)action->conf)->group; 5005 ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table, 5006 &grp_info, error); 5007 if (ret) 5008 return ret; 5009 if (attributes->group == target_group && 5010 !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET | 5011 MLX5_FLOW_ACTION_TUNNEL_MATCH))) 5012 return rte_flow_error_set(error, EINVAL, 5013 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5014 "target group must be other than" 5015 " the current flow group"); 5016 if (table == 0) 5017 return rte_flow_error_set(error, EINVAL, 5018 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 5019 NULL, "root table shouldn't be destination"); 5020 return 0; 5021 } 5022 5023 /* 5024 * Validate action PORT_ID / REPRESENTED_PORT. 5025 * 5026 * @param[in] dev 5027 * Pointer to rte_eth_dev structure. 5028 * @param[in] action_flags 5029 * Bit-fields that holds the actions detected until now. 5030 * @param[in] action 5031 * PORT_ID / REPRESENTED_PORT action structure. 5032 * @param[in] attr 5033 * Attributes of flow that includes this action. 5034 * @param[out] error 5035 * Pointer to error structure. 5036 * 5037 * @return 5038 * 0 on success, a negative errno value otherwise and rte_errno is set. 5039 */ 5040 static int 5041 flow_dv_validate_action_port_id(struct rte_eth_dev *dev, 5042 uint64_t action_flags, 5043 const struct rte_flow_action *action, 5044 const struct rte_flow_attr *attr, 5045 struct rte_flow_error *error) 5046 { 5047 const struct rte_flow_action_port_id *port_id; 5048 const struct rte_flow_action_ethdev *ethdev; 5049 struct mlx5_priv *act_priv; 5050 struct mlx5_priv *dev_priv; 5051 uint16_t port; 5052 5053 if (!attr->transfer) 5054 return rte_flow_error_set(error, ENOTSUP, 5055 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5056 NULL, 5057 "port action is valid in transfer" 5058 " mode only"); 5059 if (!action || !action->conf) 5060 return rte_flow_error_set(error, ENOTSUP, 5061 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 5062 NULL, 5063 "port action parameters must be" 5064 " specified"); 5065 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 5066 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 5067 return rte_flow_error_set(error, EINVAL, 5068 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5069 "can have only one fate actions in" 5070 " a flow"); 5071 dev_priv = mlx5_dev_to_eswitch_info(dev); 5072 if (!dev_priv) 5073 return rte_flow_error_set(error, rte_errno, 5074 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5075 NULL, 5076 "failed to obtain E-Switch info"); 5077 switch (action->type) { 5078 case RTE_FLOW_ACTION_TYPE_PORT_ID: 5079 port_id = action->conf; 5080 port = port_id->original ? dev->data->port_id : port_id->id; 5081 break; 5082 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 5083 ethdev = action->conf; 5084 port = ethdev->port_id; 5085 break; 5086 default: 5087 MLX5_ASSERT(false); 5088 return rte_flow_error_set 5089 (error, EINVAL, 5090 RTE_FLOW_ERROR_TYPE_ACTION, action, 5091 "unknown E-Switch action"); 5092 } 5093 act_priv = mlx5_port_to_eswitch_info(port, false); 5094 if (!act_priv) 5095 return rte_flow_error_set 5096 (error, rte_errno, 5097 RTE_FLOW_ERROR_TYPE_ACTION_CONF, action->conf, 5098 "failed to obtain E-Switch port id for port"); 5099 if (act_priv->domain_id != dev_priv->domain_id) 5100 return rte_flow_error_set 5101 (error, EINVAL, 5102 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5103 "port does not belong to" 5104 " E-Switch being configured"); 5105 return 0; 5106 } 5107 5108 /** 5109 * Get the maximum number of modify header actions. 5110 * 5111 * @param dev 5112 * Pointer to rte_eth_dev structure. 5113 * @param root 5114 * Whether action is on root table. 5115 * 5116 * @return 5117 * Max number of modify header actions device can support. 5118 */ 5119 static inline unsigned int 5120 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused, 5121 bool root) 5122 { 5123 /* 5124 * There's no way to directly query the max capacity from FW. 5125 * The maximal value on root table should be assumed to be supported. 5126 */ 5127 if (!root) 5128 return MLX5_MAX_MODIFY_NUM; 5129 else 5130 return MLX5_ROOT_TBL_MODIFY_NUM; 5131 } 5132 5133 /** 5134 * Validate the meter action. 5135 * 5136 * @param[in] dev 5137 * Pointer to rte_eth_dev structure. 5138 * @param[in] action_flags 5139 * Bit-fields that holds the actions detected until now. 5140 * @param[in] item_flags 5141 * Holds the items detected. 5142 * @param[in] action 5143 * Pointer to the meter action. 5144 * @param[in] attr 5145 * Attributes of flow that includes this action. 5146 * @param[in] port_id_item 5147 * Pointer to item indicating port id. 5148 * @param[out] error 5149 * Pointer to error structure. 5150 * 5151 * @return 5152 * 0 on success, a negative errno value otherwise and rte_errno is set. 5153 */ 5154 static int 5155 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, 5156 uint64_t action_flags, uint64_t item_flags, 5157 const struct rte_flow_action *action, 5158 const struct rte_flow_attr *attr, 5159 const struct rte_flow_item *port_id_item, 5160 bool *def_policy, 5161 struct rte_flow_error *error) 5162 { 5163 struct mlx5_priv *priv = dev->data->dev_private; 5164 const struct rte_flow_action_meter *am = action->conf; 5165 struct mlx5_flow_meter_info *fm; 5166 struct mlx5_flow_meter_policy *mtr_policy; 5167 struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng; 5168 5169 if (!am) 5170 return rte_flow_error_set(error, EINVAL, 5171 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5172 "meter action conf is NULL"); 5173 5174 if (action_flags & MLX5_FLOW_ACTION_METER) 5175 return rte_flow_error_set(error, ENOTSUP, 5176 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5177 "meter chaining not support"); 5178 if (action_flags & MLX5_FLOW_ACTION_JUMP) 5179 return rte_flow_error_set(error, ENOTSUP, 5180 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5181 "meter with jump not support"); 5182 if (!priv->mtr_en) 5183 return rte_flow_error_set(error, ENOTSUP, 5184 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5185 NULL, 5186 "meter action not supported"); 5187 fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL); 5188 if (!fm) 5189 return rte_flow_error_set(error, EINVAL, 5190 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5191 "Meter not found"); 5192 /* aso meter can always be shared by different domains */ 5193 if (fm->ref_cnt && !priv->sh->meter_aso_en && 5194 !(fm->transfer == attr->transfer || 5195 (!fm->ingress && !attr->ingress && attr->egress) || 5196 (!fm->egress && !attr->egress && attr->ingress))) 5197 return rte_flow_error_set(error, EINVAL, 5198 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5199 "Flow attributes domain are either invalid " 5200 "or have a domain conflict with current " 5201 "meter attributes"); 5202 if (fm->def_policy) { 5203 if (!((attr->transfer && 5204 mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) || 5205 (attr->egress && 5206 mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) || 5207 (attr->ingress && 5208 mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS]))) 5209 return rte_flow_error_set(error, EINVAL, 5210 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5211 "Flow attributes domain " 5212 "have a conflict with current " 5213 "meter domain attributes"); 5214 *def_policy = true; 5215 } else { 5216 mtr_policy = mlx5_flow_meter_policy_find(dev, 5217 fm->policy_id, NULL); 5218 if (!mtr_policy) 5219 return rte_flow_error_set(error, EINVAL, 5220 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5221 "Invalid policy id for meter "); 5222 if (!((attr->transfer && mtr_policy->transfer) || 5223 (attr->egress && mtr_policy->egress) || 5224 (attr->ingress && mtr_policy->ingress))) 5225 return rte_flow_error_set(error, EINVAL, 5226 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5227 "Flow attributes domain " 5228 "have a conflict with current " 5229 "meter domain attributes"); 5230 if (attr->transfer && mtr_policy->dev) { 5231 /** 5232 * When policy has fate action of port_id, 5233 * the flow should have the same src port as policy. 5234 */ 5235 struct mlx5_priv *policy_port_priv = 5236 mtr_policy->dev->data->dev_private; 5237 int32_t flow_src_port = priv->representor_id; 5238 5239 if (port_id_item) { 5240 const struct rte_flow_item_port_id *spec = 5241 port_id_item->spec; 5242 struct mlx5_priv *port_priv = 5243 mlx5_port_to_eswitch_info(spec->id, 5244 false); 5245 if (!port_priv) 5246 return rte_flow_error_set(error, 5247 rte_errno, 5248 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5249 spec, 5250 "Failed to get port info."); 5251 flow_src_port = port_priv->representor_id; 5252 } 5253 if (flow_src_port != policy_port_priv->representor_id) 5254 return rte_flow_error_set(error, 5255 rte_errno, 5256 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5257 NULL, 5258 "Flow and meter policy " 5259 "have different src port."); 5260 } else if (mtr_policy->is_rss) { 5261 struct mlx5_flow_meter_policy *fp; 5262 struct mlx5_meter_policy_action_container *acg; 5263 struct mlx5_meter_policy_action_container *acy; 5264 const struct rte_flow_action *rss_act; 5265 int ret; 5266 5267 fp = mlx5_flow_meter_hierarchy_get_final_policy(dev, 5268 mtr_policy); 5269 if (fp == NULL) 5270 return rte_flow_error_set(error, EINVAL, 5271 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5272 "Unable to get the final " 5273 "policy in the hierarchy"); 5274 acg = &fp->act_cnt[RTE_COLOR_GREEN]; 5275 acy = &fp->act_cnt[RTE_COLOR_YELLOW]; 5276 MLX5_ASSERT(acg->fate_action == 5277 MLX5_FLOW_FATE_SHARED_RSS || 5278 acy->fate_action == 5279 MLX5_FLOW_FATE_SHARED_RSS); 5280 if (acg->fate_action == MLX5_FLOW_FATE_SHARED_RSS) 5281 rss_act = acg->rss; 5282 else 5283 rss_act = acy->rss; 5284 ret = mlx5_flow_validate_action_rss(rss_act, 5285 action_flags, dev, attr, 5286 item_flags, error); 5287 if (ret) 5288 return ret; 5289 } 5290 *def_policy = false; 5291 } 5292 return 0; 5293 } 5294 5295 /** 5296 * Validate the age action. 5297 * 5298 * @param[in] action_flags 5299 * Holds the actions detected until now. 5300 * @param[in] action 5301 * Pointer to the age action. 5302 * @param[in] dev 5303 * Pointer to the Ethernet device structure. 5304 * @param[out] error 5305 * Pointer to error structure. 5306 * 5307 * @return 5308 * 0 on success, a negative errno value otherwise and rte_errno is set. 5309 */ 5310 static int 5311 flow_dv_validate_action_age(uint64_t action_flags, 5312 const struct rte_flow_action *action, 5313 struct rte_eth_dev *dev, 5314 struct rte_flow_error *error) 5315 { 5316 struct mlx5_priv *priv = dev->data->dev_private; 5317 const struct rte_flow_action_age *age = action->conf; 5318 5319 if (!priv->sh->cdev->config.devx || 5320 (priv->sh->cmng.counter_fallback && !priv->sh->aso_age_mng)) 5321 return rte_flow_error_set(error, ENOTSUP, 5322 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5323 NULL, 5324 "age action not supported"); 5325 if (!(action->conf)) 5326 return rte_flow_error_set(error, EINVAL, 5327 RTE_FLOW_ERROR_TYPE_ACTION, action, 5328 "configuration cannot be null"); 5329 if (!(age->timeout)) 5330 return rte_flow_error_set(error, EINVAL, 5331 RTE_FLOW_ERROR_TYPE_ACTION, action, 5332 "invalid timeout value 0"); 5333 if (action_flags & MLX5_FLOW_ACTION_AGE) 5334 return rte_flow_error_set(error, EINVAL, 5335 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5336 "duplicate age actions set"); 5337 return 0; 5338 } 5339 5340 /** 5341 * Validate the modify-header IPv4 DSCP actions. 5342 * 5343 * @param[in] action_flags 5344 * Holds the actions detected until now. 5345 * @param[in] action 5346 * Pointer to the modify action. 5347 * @param[in] item_flags 5348 * Holds the items detected. 5349 * @param[out] error 5350 * Pointer to error structure. 5351 * 5352 * @return 5353 * 0 on success, a negative errno value otherwise and rte_errno is set. 5354 */ 5355 static int 5356 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags, 5357 const struct rte_flow_action *action, 5358 const uint64_t item_flags, 5359 struct rte_flow_error *error) 5360 { 5361 int ret = 0; 5362 5363 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 5364 if (!ret) { 5365 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) 5366 return rte_flow_error_set(error, EINVAL, 5367 RTE_FLOW_ERROR_TYPE_ACTION, 5368 NULL, 5369 "no ipv4 item in pattern"); 5370 } 5371 return ret; 5372 } 5373 5374 /** 5375 * Validate the modify-header IPv6 DSCP actions. 5376 * 5377 * @param[in] action_flags 5378 * Holds the actions detected until now. 5379 * @param[in] action 5380 * Pointer to the modify action. 5381 * @param[in] item_flags 5382 * Holds the items detected. 5383 * @param[out] error 5384 * Pointer to error structure. 5385 * 5386 * @return 5387 * 0 on success, a negative errno value otherwise and rte_errno is set. 5388 */ 5389 static int 5390 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags, 5391 const struct rte_flow_action *action, 5392 const uint64_t item_flags, 5393 struct rte_flow_error *error) 5394 { 5395 int ret = 0; 5396 5397 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 5398 if (!ret) { 5399 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) 5400 return rte_flow_error_set(error, EINVAL, 5401 RTE_FLOW_ERROR_TYPE_ACTION, 5402 NULL, 5403 "no ipv6 item in pattern"); 5404 } 5405 return ret; 5406 } 5407 5408 int 5409 flow_dv_modify_match_cb(void *tool_ctx __rte_unused, 5410 struct mlx5_list_entry *entry, void *cb_ctx) 5411 { 5412 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 5413 struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data; 5414 struct mlx5_flow_dv_modify_hdr_resource *resource = 5415 container_of(entry, typeof(*resource), entry); 5416 uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type); 5417 5418 key_len += ref->actions_num * sizeof(ref->actions[0]); 5419 return ref->actions_num != resource->actions_num || 5420 memcmp(&ref->ft_type, &resource->ft_type, key_len); 5421 } 5422 5423 static struct mlx5_indexed_pool * 5424 flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index) 5425 { 5426 struct mlx5_indexed_pool *ipool = __atomic_load_n 5427 (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST); 5428 5429 if (!ipool) { 5430 struct mlx5_indexed_pool *expected = NULL; 5431 struct mlx5_indexed_pool_config cfg = 5432 (struct mlx5_indexed_pool_config) { 5433 .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) + 5434 (index + 1) * 5435 sizeof(struct mlx5_modification_cmd), 5436 .trunk_size = 64, 5437 .grow_trunk = 3, 5438 .grow_shift = 2, 5439 .need_lock = 1, 5440 .release_mem_en = !!sh->config.reclaim_mode, 5441 .per_core_cache = 5442 sh->config.reclaim_mode ? 0 : (1 << 16), 5443 .malloc = mlx5_malloc, 5444 .free = mlx5_free, 5445 .type = "mlx5_modify_action_resource", 5446 }; 5447 5448 cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool)); 5449 ipool = mlx5_ipool_create(&cfg); 5450 if (!ipool) 5451 return NULL; 5452 if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index], 5453 &expected, ipool, false, 5454 __ATOMIC_SEQ_CST, 5455 __ATOMIC_SEQ_CST)) { 5456 mlx5_ipool_destroy(ipool); 5457 ipool = __atomic_load_n(&sh->mdh_ipools[index], 5458 __ATOMIC_SEQ_CST); 5459 } 5460 } 5461 return ipool; 5462 } 5463 5464 struct mlx5_list_entry * 5465 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx) 5466 { 5467 struct mlx5_dev_ctx_shared *sh = tool_ctx; 5468 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 5469 struct mlx5dv_dr_domain *ns; 5470 struct mlx5_flow_dv_modify_hdr_resource *entry; 5471 struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data; 5472 struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh, 5473 ref->actions_num - 1); 5474 int ret; 5475 uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]); 5476 uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type); 5477 uint32_t idx; 5478 5479 if (unlikely(!ipool)) { 5480 rte_flow_error_set(ctx->error, ENOMEM, 5481 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5482 NULL, "cannot allocate modify ipool"); 5483 return NULL; 5484 } 5485 entry = mlx5_ipool_zmalloc(ipool, &idx); 5486 if (!entry) { 5487 rte_flow_error_set(ctx->error, ENOMEM, 5488 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5489 "cannot allocate resource memory"); 5490 return NULL; 5491 } 5492 rte_memcpy(&entry->ft_type, 5493 RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)), 5494 key_len + data_len); 5495 if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 5496 ns = sh->fdb_domain; 5497 else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) 5498 ns = sh->tx_domain; 5499 else 5500 ns = sh->rx_domain; 5501 ret = mlx5_flow_os_create_flow_action_modify_header 5502 (sh->cdev->ctx, ns, entry, 5503 data_len, &entry->action); 5504 if (ret) { 5505 mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx); 5506 rte_flow_error_set(ctx->error, ENOMEM, 5507 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5508 NULL, "cannot create modification action"); 5509 return NULL; 5510 } 5511 entry->idx = idx; 5512 return &entry->entry; 5513 } 5514 5515 struct mlx5_list_entry * 5516 flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 5517 void *cb_ctx) 5518 { 5519 struct mlx5_dev_ctx_shared *sh = tool_ctx; 5520 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 5521 struct mlx5_flow_dv_modify_hdr_resource *entry; 5522 struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data; 5523 uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]); 5524 uint32_t idx; 5525 5526 entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1], 5527 &idx); 5528 if (!entry) { 5529 rte_flow_error_set(ctx->error, ENOMEM, 5530 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5531 "cannot allocate resource memory"); 5532 return NULL; 5533 } 5534 memcpy(entry, oentry, sizeof(*entry) + data_len); 5535 entry->idx = idx; 5536 return &entry->entry; 5537 } 5538 5539 void 5540 flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5541 { 5542 struct mlx5_dev_ctx_shared *sh = tool_ctx; 5543 struct mlx5_flow_dv_modify_hdr_resource *res = 5544 container_of(entry, typeof(*res), entry); 5545 5546 mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx); 5547 } 5548 5549 /** 5550 * Validate the sample action. 5551 * 5552 * @param[in, out] action_flags 5553 * Holds the actions detected until now. 5554 * @param[in] action 5555 * Pointer to the sample action. 5556 * @param[in] dev 5557 * Pointer to the Ethernet device structure. 5558 * @param[in] attr 5559 * Attributes of flow that includes this action. 5560 * @param[in] item_flags 5561 * Holds the items detected. 5562 * @param[in] rss 5563 * Pointer to the RSS action. 5564 * @param[out] sample_rss 5565 * Pointer to the RSS action in sample action list. 5566 * @param[out] count 5567 * Pointer to the COUNT action in sample action list. 5568 * @param[out] fdb_mirror_limit 5569 * Pointer to the FDB mirror limitation flag. 5570 * @param[out] error 5571 * Pointer to error structure. 5572 * 5573 * @return 5574 * 0 on success, a negative errno value otherwise and rte_errno is set. 5575 */ 5576 static int 5577 flow_dv_validate_action_sample(uint64_t *action_flags, 5578 const struct rte_flow_action *action, 5579 struct rte_eth_dev *dev, 5580 const struct rte_flow_attr *attr, 5581 uint64_t item_flags, 5582 const struct rte_flow_action_rss *rss, 5583 const struct rte_flow_action_rss **sample_rss, 5584 const struct rte_flow_action_count **count, 5585 int *fdb_mirror_limit, 5586 struct rte_flow_error *error) 5587 { 5588 struct mlx5_priv *priv = dev->data->dev_private; 5589 struct mlx5_sh_config *dev_conf = &priv->sh->config; 5590 const struct rte_flow_action_sample *sample = action->conf; 5591 const struct rte_flow_action *act; 5592 uint64_t sub_action_flags = 0; 5593 uint16_t queue_index = 0xFFFF; 5594 int actions_n = 0; 5595 int ret; 5596 5597 if (!sample) 5598 return rte_flow_error_set(error, EINVAL, 5599 RTE_FLOW_ERROR_TYPE_ACTION, action, 5600 "configuration cannot be NULL"); 5601 if (sample->ratio == 0) 5602 return rte_flow_error_set(error, EINVAL, 5603 RTE_FLOW_ERROR_TYPE_ACTION, action, 5604 "ratio value starts from 1"); 5605 if (!priv->sh->cdev->config.devx || 5606 (sample->ratio > 0 && !priv->sampler_en)) 5607 return rte_flow_error_set(error, ENOTSUP, 5608 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5609 NULL, 5610 "sample action not supported"); 5611 if (*action_flags & MLX5_FLOW_ACTION_SAMPLE) 5612 return rte_flow_error_set(error, EINVAL, 5613 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5614 "Multiple sample actions not " 5615 "supported"); 5616 if (*action_flags & MLX5_FLOW_ACTION_METER) 5617 return rte_flow_error_set(error, EINVAL, 5618 RTE_FLOW_ERROR_TYPE_ACTION, action, 5619 "wrong action order, meter should " 5620 "be after sample action"); 5621 if (*action_flags & MLX5_FLOW_ACTION_JUMP) 5622 return rte_flow_error_set(error, EINVAL, 5623 RTE_FLOW_ERROR_TYPE_ACTION, action, 5624 "wrong action order, jump should " 5625 "be after sample action"); 5626 if (*action_flags & MLX5_FLOW_ACTION_CT) 5627 return rte_flow_error_set(error, EINVAL, 5628 RTE_FLOW_ERROR_TYPE_ACTION, action, 5629 "Sample after CT not supported"); 5630 act = sample->actions; 5631 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) { 5632 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) 5633 return rte_flow_error_set(error, ENOTSUP, 5634 RTE_FLOW_ERROR_TYPE_ACTION, 5635 act, "too many actions"); 5636 switch (act->type) { 5637 case RTE_FLOW_ACTION_TYPE_QUEUE: 5638 ret = mlx5_flow_validate_action_queue(act, 5639 sub_action_flags, 5640 dev, 5641 attr, error); 5642 if (ret < 0) 5643 return ret; 5644 queue_index = ((const struct rte_flow_action_queue *) 5645 (act->conf))->index; 5646 sub_action_flags |= MLX5_FLOW_ACTION_QUEUE; 5647 ++actions_n; 5648 break; 5649 case RTE_FLOW_ACTION_TYPE_RSS: 5650 *sample_rss = act->conf; 5651 ret = mlx5_flow_validate_action_rss(act, 5652 sub_action_flags, 5653 dev, attr, 5654 item_flags, 5655 error); 5656 if (ret < 0) 5657 return ret; 5658 if (rss && *sample_rss && 5659 ((*sample_rss)->level != rss->level || 5660 (*sample_rss)->types != rss->types)) 5661 return rte_flow_error_set(error, ENOTSUP, 5662 RTE_FLOW_ERROR_TYPE_ACTION, 5663 NULL, 5664 "Can't use the different RSS types " 5665 "or level in the same flow"); 5666 if (*sample_rss != NULL && (*sample_rss)->queue_num) 5667 queue_index = (*sample_rss)->queue[0]; 5668 sub_action_flags |= MLX5_FLOW_ACTION_RSS; 5669 ++actions_n; 5670 break; 5671 case RTE_FLOW_ACTION_TYPE_MARK: 5672 ret = flow_dv_validate_action_mark(dev, act, 5673 sub_action_flags, 5674 attr, error); 5675 if (ret < 0) 5676 return ret; 5677 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 5678 sub_action_flags |= MLX5_FLOW_ACTION_MARK | 5679 MLX5_FLOW_ACTION_MARK_EXT; 5680 else 5681 sub_action_flags |= MLX5_FLOW_ACTION_MARK; 5682 ++actions_n; 5683 break; 5684 case RTE_FLOW_ACTION_TYPE_COUNT: 5685 ret = flow_dv_validate_action_count 5686 (dev, false, *action_flags | sub_action_flags, 5687 attr, error); 5688 if (ret < 0) 5689 return ret; 5690 *count = act->conf; 5691 sub_action_flags |= MLX5_FLOW_ACTION_COUNT; 5692 *action_flags |= MLX5_FLOW_ACTION_COUNT; 5693 ++actions_n; 5694 break; 5695 case RTE_FLOW_ACTION_TYPE_PORT_ID: 5696 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 5697 ret = flow_dv_validate_action_port_id(dev, 5698 sub_action_flags, 5699 act, 5700 attr, 5701 error); 5702 if (ret) 5703 return ret; 5704 sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID; 5705 ++actions_n; 5706 break; 5707 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5708 ret = flow_dv_validate_action_raw_encap_decap 5709 (dev, NULL, act->conf, attr, &sub_action_flags, 5710 &actions_n, action, item_flags, error); 5711 if (ret < 0) 5712 return ret; 5713 ++actions_n; 5714 break; 5715 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5716 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5717 ret = flow_dv_validate_action_l2_encap(dev, 5718 sub_action_flags, 5719 act, attr, 5720 error); 5721 if (ret < 0) 5722 return ret; 5723 sub_action_flags |= MLX5_FLOW_ACTION_ENCAP; 5724 ++actions_n; 5725 break; 5726 default: 5727 return rte_flow_error_set(error, ENOTSUP, 5728 RTE_FLOW_ERROR_TYPE_ACTION, 5729 NULL, 5730 "Doesn't support optional " 5731 "action"); 5732 } 5733 } 5734 if (attr->ingress && !attr->transfer) { 5735 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE | 5736 MLX5_FLOW_ACTION_RSS))) 5737 return rte_flow_error_set(error, EINVAL, 5738 RTE_FLOW_ERROR_TYPE_ACTION, 5739 NULL, 5740 "Ingress must has a dest " 5741 "QUEUE for Sample"); 5742 } else if (attr->egress && !attr->transfer) { 5743 return rte_flow_error_set(error, ENOTSUP, 5744 RTE_FLOW_ERROR_TYPE_ACTION, 5745 NULL, 5746 "Sample Only support Ingress " 5747 "or E-Switch"); 5748 } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) { 5749 MLX5_ASSERT(attr->transfer); 5750 if (sample->ratio > 1) 5751 return rte_flow_error_set(error, ENOTSUP, 5752 RTE_FLOW_ERROR_TYPE_ACTION, 5753 NULL, 5754 "E-Switch doesn't support " 5755 "any optional action " 5756 "for sampling"); 5757 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE) 5758 return rte_flow_error_set(error, ENOTSUP, 5759 RTE_FLOW_ERROR_TYPE_ACTION, 5760 NULL, 5761 "unsupported action QUEUE"); 5762 if (sub_action_flags & MLX5_FLOW_ACTION_RSS) 5763 return rte_flow_error_set(error, ENOTSUP, 5764 RTE_FLOW_ERROR_TYPE_ACTION, 5765 NULL, 5766 "unsupported action QUEUE"); 5767 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID)) 5768 return rte_flow_error_set(error, EINVAL, 5769 RTE_FLOW_ERROR_TYPE_ACTION, 5770 NULL, 5771 "E-Switch must has a dest " 5772 "port for mirroring"); 5773 if (!priv->sh->cdev->config.hca_attr.reg_c_preserve && 5774 priv->representor_id != UINT16_MAX) 5775 *fdb_mirror_limit = 1; 5776 } 5777 /* Continue validation for Xcap actions.*/ 5778 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) && 5779 (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index))) { 5780 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) == 5781 MLX5_FLOW_XCAP_ACTIONS) 5782 return rte_flow_error_set(error, ENOTSUP, 5783 RTE_FLOW_ERROR_TYPE_ACTION, 5784 NULL, "encap and decap " 5785 "combination aren't " 5786 "supported"); 5787 if (!attr->transfer && attr->ingress && (sub_action_flags & 5788 MLX5_FLOW_ACTION_ENCAP)) 5789 return rte_flow_error_set(error, ENOTSUP, 5790 RTE_FLOW_ERROR_TYPE_ACTION, 5791 NULL, "encap is not supported" 5792 " for ingress traffic"); 5793 } 5794 return 0; 5795 } 5796 5797 /** 5798 * Find existing modify-header resource or create and register a new one. 5799 * 5800 * @param dev[in, out] 5801 * Pointer to rte_eth_dev structure. 5802 * @param[in, out] resource 5803 * Pointer to modify-header resource. 5804 * @parm[in, out] dev_flow 5805 * Pointer to the dev_flow. 5806 * @param[out] error 5807 * pointer to error structure. 5808 * 5809 * @return 5810 * 0 on success otherwise -errno and errno is set. 5811 */ 5812 static int 5813 flow_dv_modify_hdr_resource_register 5814 (struct rte_eth_dev *dev, 5815 struct mlx5_flow_dv_modify_hdr_resource *resource, 5816 struct mlx5_flow *dev_flow, 5817 struct rte_flow_error *error) 5818 { 5819 struct mlx5_priv *priv = dev->data->dev_private; 5820 struct mlx5_dev_ctx_shared *sh = priv->sh; 5821 uint32_t key_len = sizeof(*resource) - 5822 offsetof(typeof(*resource), ft_type) + 5823 resource->actions_num * sizeof(resource->actions[0]); 5824 struct mlx5_list_entry *entry; 5825 struct mlx5_flow_cb_ctx ctx = { 5826 .error = error, 5827 .data = resource, 5828 }; 5829 struct mlx5_hlist *modify_cmds; 5830 uint64_t key64; 5831 5832 modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds, 5833 "hdr_modify", 5834 MLX5_FLOW_HDR_MODIFY_HTABLE_SZ, 5835 true, false, sh, 5836 flow_dv_modify_create_cb, 5837 flow_dv_modify_match_cb, 5838 flow_dv_modify_remove_cb, 5839 flow_dv_modify_clone_cb, 5840 flow_dv_modify_clone_free_cb, 5841 error); 5842 if (unlikely(!modify_cmds)) 5843 return -rte_errno; 5844 resource->root = !dev_flow->dv.group; 5845 if (resource->actions_num > flow_dv_modify_hdr_action_max(dev, 5846 resource->root)) 5847 return rte_flow_error_set(error, EOVERFLOW, 5848 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5849 "too many modify header items"); 5850 key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0); 5851 entry = mlx5_hlist_register(modify_cmds, key64, &ctx); 5852 if (!entry) 5853 return -rte_errno; 5854 resource = container_of(entry, typeof(*resource), entry); 5855 dev_flow->handle->dvh.modify_hdr = resource; 5856 return 0; 5857 } 5858 5859 /** 5860 * Get DV flow counter by index. 5861 * 5862 * @param[in] dev 5863 * Pointer to the Ethernet device structure. 5864 * @param[in] idx 5865 * mlx5 flow counter index in the container. 5866 * @param[out] ppool 5867 * mlx5 flow counter pool in the container. 5868 * 5869 * @return 5870 * Pointer to the counter, NULL otherwise. 5871 */ 5872 static struct mlx5_flow_counter * 5873 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev, 5874 uint32_t idx, 5875 struct mlx5_flow_counter_pool **ppool) 5876 { 5877 struct mlx5_priv *priv = dev->data->dev_private; 5878 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 5879 struct mlx5_flow_counter_pool *pool; 5880 5881 /* Decrease to original index and clear shared bit. */ 5882 idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1); 5883 MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n); 5884 pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL]; 5885 MLX5_ASSERT(pool); 5886 if (ppool) 5887 *ppool = pool; 5888 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL); 5889 } 5890 5891 /** 5892 * Check the devx counter belongs to the pool. 5893 * 5894 * @param[in] pool 5895 * Pointer to the counter pool. 5896 * @param[in] id 5897 * The counter devx ID. 5898 * 5899 * @return 5900 * True if counter belongs to the pool, false otherwise. 5901 */ 5902 static bool 5903 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id) 5904 { 5905 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) * 5906 MLX5_COUNTERS_PER_POOL; 5907 5908 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) 5909 return true; 5910 return false; 5911 } 5912 5913 /** 5914 * Get a pool by devx counter ID. 5915 * 5916 * @param[in] cmng 5917 * Pointer to the counter management. 5918 * @param[in] id 5919 * The counter devx ID. 5920 * 5921 * @return 5922 * The counter pool pointer if exists, NULL otherwise, 5923 */ 5924 static struct mlx5_flow_counter_pool * 5925 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id) 5926 { 5927 uint32_t i; 5928 struct mlx5_flow_counter_pool *pool = NULL; 5929 5930 rte_spinlock_lock(&cmng->pool_update_sl); 5931 /* Check last used pool. */ 5932 if (cmng->last_pool_idx != POOL_IDX_INVALID && 5933 flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) { 5934 pool = cmng->pools[cmng->last_pool_idx]; 5935 goto out; 5936 } 5937 /* ID out of range means no suitable pool in the container. */ 5938 if (id > cmng->max_id || id < cmng->min_id) 5939 goto out; 5940 /* 5941 * Find the pool from the end of the container, since mostly counter 5942 * ID is sequence increasing, and the last pool should be the needed 5943 * one. 5944 */ 5945 i = cmng->n_valid; 5946 while (i--) { 5947 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i]; 5948 5949 if (flow_dv_is_counter_in_pool(pool_tmp, id)) { 5950 pool = pool_tmp; 5951 break; 5952 } 5953 } 5954 out: 5955 rte_spinlock_unlock(&cmng->pool_update_sl); 5956 return pool; 5957 } 5958 5959 /** 5960 * Resize a counter container. 5961 * 5962 * @param[in] dev 5963 * Pointer to the Ethernet device structure. 5964 * 5965 * @return 5966 * 0 on success, otherwise negative errno value and rte_errno is set. 5967 */ 5968 static int 5969 flow_dv_container_resize(struct rte_eth_dev *dev) 5970 { 5971 struct mlx5_priv *priv = dev->data->dev_private; 5972 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 5973 void *old_pools = cmng->pools; 5974 uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE; 5975 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize; 5976 void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); 5977 5978 if (!pools) { 5979 rte_errno = ENOMEM; 5980 return -ENOMEM; 5981 } 5982 if (old_pools) 5983 memcpy(pools, old_pools, cmng->n * 5984 sizeof(struct mlx5_flow_counter_pool *)); 5985 cmng->n = resize; 5986 cmng->pools = pools; 5987 if (old_pools) 5988 mlx5_free(old_pools); 5989 return 0; 5990 } 5991 5992 /** 5993 * Query a devx flow counter. 5994 * 5995 * @param[in] dev 5996 * Pointer to the Ethernet device structure. 5997 * @param[in] counter 5998 * Index to the flow counter. 5999 * @param[out] pkts 6000 * The statistics value of packets. 6001 * @param[out] bytes 6002 * The statistics value of bytes. 6003 * 6004 * @return 6005 * 0 on success, otherwise a negative errno value and rte_errno is set. 6006 */ 6007 static inline int 6008 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts, 6009 uint64_t *bytes) 6010 { 6011 struct mlx5_priv *priv = dev->data->dev_private; 6012 struct mlx5_flow_counter_pool *pool = NULL; 6013 struct mlx5_flow_counter *cnt; 6014 int offset; 6015 6016 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); 6017 MLX5_ASSERT(pool); 6018 if (priv->sh->cmng.counter_fallback) 6019 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0, 6020 0, pkts, bytes, 0, NULL, NULL, 0); 6021 rte_spinlock_lock(&pool->sl); 6022 if (!pool->raw) { 6023 *pkts = 0; 6024 *bytes = 0; 6025 } else { 6026 offset = MLX5_CNT_ARRAY_IDX(pool, cnt); 6027 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits); 6028 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes); 6029 } 6030 rte_spinlock_unlock(&pool->sl); 6031 return 0; 6032 } 6033 6034 /** 6035 * Create and initialize a new counter pool. 6036 * 6037 * @param[in] dev 6038 * Pointer to the Ethernet device structure. 6039 * @param[out] dcs 6040 * The devX counter handle. 6041 * @param[in] age 6042 * Whether the pool is for counter that was allocated for aging. 6043 * @param[in/out] cont_cur 6044 * Pointer to the container pointer, it will be update in pool resize. 6045 * 6046 * @return 6047 * The pool container pointer on success, NULL otherwise and rte_errno is set. 6048 */ 6049 static struct mlx5_flow_counter_pool * 6050 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, 6051 uint32_t age) 6052 { 6053 struct mlx5_priv *priv = dev->data->dev_private; 6054 struct mlx5_flow_counter_pool *pool; 6055 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 6056 bool fallback = priv->sh->cmng.counter_fallback; 6057 uint32_t size = sizeof(*pool); 6058 6059 size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE; 6060 size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE); 6061 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY); 6062 if (!pool) { 6063 rte_errno = ENOMEM; 6064 return NULL; 6065 } 6066 pool->raw = NULL; 6067 pool->is_aged = !!age; 6068 pool->query_gen = 0; 6069 pool->min_dcs = dcs; 6070 rte_spinlock_init(&pool->sl); 6071 rte_spinlock_init(&pool->csl); 6072 TAILQ_INIT(&pool->counters[0]); 6073 TAILQ_INIT(&pool->counters[1]); 6074 pool->time_of_last_age_check = MLX5_CURR_TIME_SEC; 6075 rte_spinlock_lock(&cmng->pool_update_sl); 6076 pool->index = cmng->n_valid; 6077 if (pool->index == cmng->n && flow_dv_container_resize(dev)) { 6078 mlx5_free(pool); 6079 rte_spinlock_unlock(&cmng->pool_update_sl); 6080 return NULL; 6081 } 6082 cmng->pools[pool->index] = pool; 6083 cmng->n_valid++; 6084 if (unlikely(fallback)) { 6085 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL); 6086 6087 if (base < cmng->min_id) 6088 cmng->min_id = base; 6089 if (base > cmng->max_id) 6090 cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1; 6091 cmng->last_pool_idx = pool->index; 6092 } 6093 rte_spinlock_unlock(&cmng->pool_update_sl); 6094 return pool; 6095 } 6096 6097 /** 6098 * Prepare a new counter and/or a new counter pool. 6099 * 6100 * @param[in] dev 6101 * Pointer to the Ethernet device structure. 6102 * @param[out] cnt_free 6103 * Where to put the pointer of a new counter. 6104 * @param[in] age 6105 * Whether the pool is for counter that was allocated for aging. 6106 * 6107 * @return 6108 * The counter pool pointer and @p cnt_free is set on success, 6109 * NULL otherwise and rte_errno is set. 6110 */ 6111 static struct mlx5_flow_counter_pool * 6112 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, 6113 struct mlx5_flow_counter **cnt_free, 6114 uint32_t age) 6115 { 6116 struct mlx5_priv *priv = dev->data->dev_private; 6117 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 6118 struct mlx5_flow_counter_pool *pool; 6119 struct mlx5_counters tmp_tq; 6120 struct mlx5_devx_obj *dcs = NULL; 6121 struct mlx5_flow_counter *cnt; 6122 enum mlx5_counter_type cnt_type = 6123 age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN; 6124 bool fallback = priv->sh->cmng.counter_fallback; 6125 uint32_t i; 6126 6127 if (fallback) { 6128 /* bulk_bitmap must be 0 for single counter allocation. */ 6129 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0); 6130 if (!dcs) 6131 return NULL; 6132 pool = flow_dv_find_pool_by_id(cmng, dcs->id); 6133 if (!pool) { 6134 pool = flow_dv_pool_create(dev, dcs, age); 6135 if (!pool) { 6136 mlx5_devx_cmd_destroy(dcs); 6137 return NULL; 6138 } 6139 } 6140 i = dcs->id % MLX5_COUNTERS_PER_POOL; 6141 cnt = MLX5_POOL_GET_CNT(pool, i); 6142 cnt->pool = pool; 6143 cnt->dcs_when_free = dcs; 6144 *cnt_free = cnt; 6145 return pool; 6146 } 6147 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0x4); 6148 if (!dcs) { 6149 rte_errno = ENODATA; 6150 return NULL; 6151 } 6152 pool = flow_dv_pool_create(dev, dcs, age); 6153 if (!pool) { 6154 mlx5_devx_cmd_destroy(dcs); 6155 return NULL; 6156 } 6157 TAILQ_INIT(&tmp_tq); 6158 for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) { 6159 cnt = MLX5_POOL_GET_CNT(pool, i); 6160 cnt->pool = pool; 6161 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next); 6162 } 6163 rte_spinlock_lock(&cmng->csl[cnt_type]); 6164 TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next); 6165 rte_spinlock_unlock(&cmng->csl[cnt_type]); 6166 *cnt_free = MLX5_POOL_GET_CNT(pool, 0); 6167 (*cnt_free)->pool = pool; 6168 return pool; 6169 } 6170 6171 /** 6172 * Allocate a flow counter. 6173 * 6174 * @param[in] dev 6175 * Pointer to the Ethernet device structure. 6176 * @param[in] age 6177 * Whether the counter was allocated for aging. 6178 * 6179 * @return 6180 * Index to flow counter on success, 0 otherwise and rte_errno is set. 6181 */ 6182 static uint32_t 6183 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age) 6184 { 6185 struct mlx5_priv *priv = dev->data->dev_private; 6186 struct mlx5_flow_counter_pool *pool = NULL; 6187 struct mlx5_flow_counter *cnt_free = NULL; 6188 bool fallback = priv->sh->cmng.counter_fallback; 6189 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 6190 enum mlx5_counter_type cnt_type = 6191 age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN; 6192 uint32_t cnt_idx; 6193 6194 if (!priv->sh->cdev->config.devx) { 6195 rte_errno = ENOTSUP; 6196 return 0; 6197 } 6198 /* Get free counters from container. */ 6199 rte_spinlock_lock(&cmng->csl[cnt_type]); 6200 cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]); 6201 if (cnt_free) 6202 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next); 6203 rte_spinlock_unlock(&cmng->csl[cnt_type]); 6204 if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age)) 6205 goto err; 6206 pool = cnt_free->pool; 6207 if (fallback) 6208 cnt_free->dcs_when_active = cnt_free->dcs_when_free; 6209 /* Create a DV counter action only in the first time usage. */ 6210 if (!cnt_free->action) { 6211 uint16_t offset; 6212 struct mlx5_devx_obj *dcs; 6213 int ret; 6214 6215 if (!fallback) { 6216 offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free); 6217 dcs = pool->min_dcs; 6218 } else { 6219 offset = 0; 6220 dcs = cnt_free->dcs_when_free; 6221 } 6222 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset, 6223 &cnt_free->action); 6224 if (ret) { 6225 rte_errno = errno; 6226 goto err; 6227 } 6228 } 6229 cnt_idx = MLX5_MAKE_CNT_IDX(pool->index, 6230 MLX5_CNT_ARRAY_IDX(pool, cnt_free)); 6231 /* Update the counter reset values. */ 6232 if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits, 6233 &cnt_free->bytes)) 6234 goto err; 6235 if (!fallback && !priv->sh->cmng.query_thread_on) 6236 /* Start the asynchronous batch query by the host thread. */ 6237 mlx5_set_query_alarm(priv->sh); 6238 /* 6239 * When the count action isn't shared (by ID), shared_info field is 6240 * used for indirect action API's refcnt. 6241 * When the counter action is not shared neither by ID nor by indirect 6242 * action API, shared info must be 1. 6243 */ 6244 cnt_free->shared_info.refcnt = 1; 6245 return cnt_idx; 6246 err: 6247 if (cnt_free) { 6248 cnt_free->pool = pool; 6249 if (fallback) 6250 cnt_free->dcs_when_free = cnt_free->dcs_when_active; 6251 rte_spinlock_lock(&cmng->csl[cnt_type]); 6252 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next); 6253 rte_spinlock_unlock(&cmng->csl[cnt_type]); 6254 } 6255 return 0; 6256 } 6257 6258 /** 6259 * Get age param from counter index. 6260 * 6261 * @param[in] dev 6262 * Pointer to the Ethernet device structure. 6263 * @param[in] counter 6264 * Index to the counter handler. 6265 * 6266 * @return 6267 * The aging parameter specified for the counter index. 6268 */ 6269 static struct mlx5_age_param* 6270 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev, 6271 uint32_t counter) 6272 { 6273 struct mlx5_flow_counter *cnt; 6274 struct mlx5_flow_counter_pool *pool = NULL; 6275 6276 flow_dv_counter_get_by_idx(dev, counter, &pool); 6277 counter = (counter - 1) % MLX5_COUNTERS_PER_POOL; 6278 cnt = MLX5_POOL_GET_CNT(pool, counter); 6279 return MLX5_CNT_TO_AGE(cnt); 6280 } 6281 6282 /** 6283 * Remove a flow counter from aged counter list. 6284 * 6285 * @param[in] dev 6286 * Pointer to the Ethernet device structure. 6287 * @param[in] counter 6288 * Index to the counter handler. 6289 * @param[in] cnt 6290 * Pointer to the counter handler. 6291 */ 6292 static void 6293 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev, 6294 uint32_t counter, struct mlx5_flow_counter *cnt) 6295 { 6296 struct mlx5_age_info *age_info; 6297 struct mlx5_age_param *age_param; 6298 struct mlx5_priv *priv = dev->data->dev_private; 6299 uint16_t expected = AGE_CANDIDATE; 6300 6301 age_info = GET_PORT_AGE_INFO(priv); 6302 age_param = flow_dv_counter_idx_get_age(dev, counter); 6303 if (!__atomic_compare_exchange_n(&age_param->state, &expected, 6304 AGE_FREE, false, __ATOMIC_RELAXED, 6305 __ATOMIC_RELAXED)) { 6306 /** 6307 * We need the lock even it is age timeout, 6308 * since counter may still in process. 6309 */ 6310 rte_spinlock_lock(&age_info->aged_sl); 6311 TAILQ_REMOVE(&age_info->aged_counters, cnt, next); 6312 rte_spinlock_unlock(&age_info->aged_sl); 6313 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED); 6314 } 6315 } 6316 6317 /** 6318 * Release a flow counter. 6319 * 6320 * @param[in] dev 6321 * Pointer to the Ethernet device structure. 6322 * @param[in] counter 6323 * Index to the counter handler. 6324 */ 6325 static void 6326 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter) 6327 { 6328 struct mlx5_priv *priv = dev->data->dev_private; 6329 struct mlx5_flow_counter_pool *pool = NULL; 6330 struct mlx5_flow_counter *cnt; 6331 enum mlx5_counter_type cnt_type; 6332 6333 if (!counter) 6334 return; 6335 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); 6336 MLX5_ASSERT(pool); 6337 if (pool->is_aged) { 6338 flow_dv_counter_remove_from_age(dev, counter, cnt); 6339 } else { 6340 /* 6341 * If the counter action is shared by indirect action API, 6342 * the atomic function reduces its references counter. 6343 * If after the reduction the action is still referenced, the 6344 * function returns here and does not release it. 6345 * When the counter action is not shared by 6346 * indirect action API, shared info is 1 before the reduction, 6347 * so this condition is failed and function doesn't return here. 6348 */ 6349 if (__atomic_sub_fetch(&cnt->shared_info.refcnt, 1, 6350 __ATOMIC_RELAXED)) 6351 return; 6352 } 6353 cnt->pool = pool; 6354 /* 6355 * Put the counter back to list to be updated in none fallback mode. 6356 * Currently, we are using two list alternately, while one is in query, 6357 * add the freed counter to the other list based on the pool query_gen 6358 * value. After query finishes, add counter the list to the global 6359 * container counter list. The list changes while query starts. In 6360 * this case, lock will not be needed as query callback and release 6361 * function both operate with the different list. 6362 */ 6363 if (!priv->sh->cmng.counter_fallback) { 6364 rte_spinlock_lock(&pool->csl); 6365 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next); 6366 rte_spinlock_unlock(&pool->csl); 6367 } else { 6368 cnt->dcs_when_free = cnt->dcs_when_active; 6369 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 6370 MLX5_COUNTER_TYPE_ORIGIN; 6371 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]); 6372 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type], 6373 cnt, next); 6374 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]); 6375 } 6376 } 6377 6378 /** 6379 * Resize a meter id container. 6380 * 6381 * @param[in] dev 6382 * Pointer to the Ethernet device structure. 6383 * 6384 * @return 6385 * 0 on success, otherwise negative errno value and rte_errno is set. 6386 */ 6387 static int 6388 flow_dv_mtr_container_resize(struct rte_eth_dev *dev) 6389 { 6390 struct mlx5_priv *priv = dev->data->dev_private; 6391 struct mlx5_aso_mtr_pools_mng *pools_mng = 6392 &priv->sh->mtrmng->pools_mng; 6393 void *old_pools = pools_mng->pools; 6394 uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE; 6395 uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize; 6396 void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); 6397 6398 if (!pools) { 6399 rte_errno = ENOMEM; 6400 return -ENOMEM; 6401 } 6402 if (!pools_mng->n) 6403 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) { 6404 mlx5_free(pools); 6405 return -ENOMEM; 6406 } 6407 if (old_pools) 6408 memcpy(pools, old_pools, pools_mng->n * 6409 sizeof(struct mlx5_aso_mtr_pool *)); 6410 pools_mng->n = resize; 6411 pools_mng->pools = pools; 6412 if (old_pools) 6413 mlx5_free(old_pools); 6414 return 0; 6415 } 6416 6417 /** 6418 * Prepare a new meter and/or a new meter pool. 6419 * 6420 * @param[in] dev 6421 * Pointer to the Ethernet device structure. 6422 * @param[out] mtr_free 6423 * Where to put the pointer of a new meter.g. 6424 * 6425 * @return 6426 * The meter pool pointer and @mtr_free is set on success, 6427 * NULL otherwise and rte_errno is set. 6428 */ 6429 static struct mlx5_aso_mtr_pool * 6430 flow_dv_mtr_pool_create(struct rte_eth_dev *dev, struct mlx5_aso_mtr **mtr_free) 6431 { 6432 struct mlx5_priv *priv = dev->data->dev_private; 6433 struct mlx5_aso_mtr_pools_mng *pools_mng = &priv->sh->mtrmng->pools_mng; 6434 struct mlx5_aso_mtr_pool *pool = NULL; 6435 struct mlx5_devx_obj *dcs = NULL; 6436 uint32_t i; 6437 uint32_t log_obj_size; 6438 6439 log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); 6440 dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->cdev->ctx, 6441 priv->sh->cdev->pdn, 6442 log_obj_size); 6443 if (!dcs) { 6444 rte_errno = ENODATA; 6445 return NULL; 6446 } 6447 pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); 6448 if (!pool) { 6449 rte_errno = ENOMEM; 6450 claim_zero(mlx5_devx_cmd_destroy(dcs)); 6451 return NULL; 6452 } 6453 pool->devx_obj = dcs; 6454 rte_rwlock_write_lock(&pools_mng->resize_mtrwl); 6455 pool->index = pools_mng->n_valid; 6456 if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) { 6457 mlx5_free(pool); 6458 claim_zero(mlx5_devx_cmd_destroy(dcs)); 6459 rte_rwlock_write_unlock(&pools_mng->resize_mtrwl); 6460 return NULL; 6461 } 6462 pools_mng->pools[pool->index] = pool; 6463 pools_mng->n_valid++; 6464 rte_rwlock_write_unlock(&pools_mng->resize_mtrwl); 6465 for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) { 6466 pool->mtrs[i].offset = i; 6467 LIST_INSERT_HEAD(&pools_mng->meters, &pool->mtrs[i], next); 6468 } 6469 pool->mtrs[0].offset = 0; 6470 *mtr_free = &pool->mtrs[0]; 6471 return pool; 6472 } 6473 6474 /** 6475 * Release a flow meter into pool. 6476 * 6477 * @param[in] dev 6478 * Pointer to the Ethernet device structure. 6479 * @param[in] mtr_idx 6480 * Index to aso flow meter. 6481 */ 6482 static void 6483 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx) 6484 { 6485 struct mlx5_priv *priv = dev->data->dev_private; 6486 struct mlx5_aso_mtr_pools_mng *pools_mng = 6487 &priv->sh->mtrmng->pools_mng; 6488 struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 6489 6490 MLX5_ASSERT(aso_mtr); 6491 rte_spinlock_lock(&pools_mng->mtrsl); 6492 memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info)); 6493 aso_mtr->state = ASO_METER_FREE; 6494 LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next); 6495 rte_spinlock_unlock(&pools_mng->mtrsl); 6496 } 6497 6498 /** 6499 * Allocate a aso flow meter. 6500 * 6501 * @param[in] dev 6502 * Pointer to the Ethernet device structure. 6503 * 6504 * @return 6505 * Index to aso flow meter on success, 0 otherwise and rte_errno is set. 6506 */ 6507 static uint32_t 6508 flow_dv_mtr_alloc(struct rte_eth_dev *dev) 6509 { 6510 struct mlx5_priv *priv = dev->data->dev_private; 6511 struct mlx5_aso_mtr *mtr_free = NULL; 6512 struct mlx5_aso_mtr_pools_mng *pools_mng = 6513 &priv->sh->mtrmng->pools_mng; 6514 struct mlx5_aso_mtr_pool *pool; 6515 uint32_t mtr_idx = 0; 6516 6517 if (!priv->sh->cdev->config.devx) { 6518 rte_errno = ENOTSUP; 6519 return 0; 6520 } 6521 /* Allocate the flow meter memory. */ 6522 /* Get free meters from management. */ 6523 rte_spinlock_lock(&pools_mng->mtrsl); 6524 mtr_free = LIST_FIRST(&pools_mng->meters); 6525 if (mtr_free) 6526 LIST_REMOVE(mtr_free, next); 6527 if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) { 6528 rte_spinlock_unlock(&pools_mng->mtrsl); 6529 return 0; 6530 } 6531 mtr_free->state = ASO_METER_WAIT; 6532 rte_spinlock_unlock(&pools_mng->mtrsl); 6533 pool = container_of(mtr_free, 6534 struct mlx5_aso_mtr_pool, 6535 mtrs[mtr_free->offset]); 6536 mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset); 6537 if (!mtr_free->fm.meter_action_g) { 6538 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 6539 struct rte_flow_error error; 6540 uint8_t reg_id; 6541 6542 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error); 6543 mtr_free->fm.meter_action_g = 6544 mlx5_glue->dv_create_flow_action_aso 6545 (priv->sh->rx_domain, 6546 pool->devx_obj->obj, 6547 mtr_free->offset, 6548 (1 << MLX5_FLOW_COLOR_GREEN), 6549 reg_id - REG_C_0); 6550 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 6551 if (!mtr_free->fm.meter_action_g) { 6552 flow_dv_aso_mtr_release_to_pool(dev, mtr_idx); 6553 return 0; 6554 } 6555 } 6556 return mtr_idx; 6557 } 6558 6559 /** 6560 * Verify the @p attributes will be correctly understood by the NIC and store 6561 * them in the @p flow if everything is correct. 6562 * 6563 * @param[in] dev 6564 * Pointer to dev struct. 6565 * @param[in] attributes 6566 * Pointer to flow attributes 6567 * @param[in] external 6568 * This flow rule is created by request external to PMD. 6569 * @param[out] error 6570 * Pointer to error structure. 6571 * 6572 * @return 6573 * - 0 on success and non root table. 6574 * - 1 on success and root table. 6575 * - a negative errno value otherwise and rte_errno is set. 6576 */ 6577 static int 6578 flow_dv_validate_attributes(struct rte_eth_dev *dev, 6579 const struct mlx5_flow_tunnel *tunnel, 6580 const struct rte_flow_attr *attributes, 6581 const struct flow_grp_info *grp_info, 6582 struct rte_flow_error *error) 6583 { 6584 struct mlx5_priv *priv = dev->data->dev_private; 6585 uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes); 6586 int ret = 0; 6587 6588 #ifndef HAVE_MLX5DV_DR 6589 RTE_SET_USED(tunnel); 6590 RTE_SET_USED(grp_info); 6591 if (attributes->group) 6592 return rte_flow_error_set(error, ENOTSUP, 6593 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6594 NULL, 6595 "groups are not supported"); 6596 #else 6597 uint32_t table = 0; 6598 6599 ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table, 6600 grp_info, error); 6601 if (ret) 6602 return ret; 6603 if (!table) 6604 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; 6605 #endif 6606 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 6607 attributes->priority > lowest_priority) 6608 return rte_flow_error_set(error, ENOTSUP, 6609 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 6610 NULL, 6611 "priority out of range"); 6612 if (attributes->transfer) { 6613 if (!priv->sh->config.dv_esw_en) 6614 return rte_flow_error_set 6615 (error, ENOTSUP, 6616 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 6617 "E-Switch dr is not supported"); 6618 if (attributes->egress) 6619 return rte_flow_error_set 6620 (error, ENOTSUP, 6621 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes, 6622 "egress is not supported"); 6623 } 6624 if (!(attributes->egress ^ attributes->ingress)) 6625 return rte_flow_error_set(error, ENOTSUP, 6626 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 6627 "must specify exactly one of " 6628 "ingress or egress"); 6629 return ret; 6630 } 6631 6632 static int 6633 validate_integrity_bits(const struct rte_flow_item_integrity *mask, 6634 int64_t pattern_flags, uint64_t l3_flags, 6635 uint64_t l4_flags, uint64_t ip4_flag, 6636 struct rte_flow_error *error) 6637 { 6638 if (mask->l3_ok && !(pattern_flags & l3_flags)) 6639 return rte_flow_error_set(error, EINVAL, 6640 RTE_FLOW_ERROR_TYPE_ITEM, 6641 NULL, "missing L3 protocol"); 6642 6643 if (mask->ipv4_csum_ok && !(pattern_flags & ip4_flag)) 6644 return rte_flow_error_set(error, EINVAL, 6645 RTE_FLOW_ERROR_TYPE_ITEM, 6646 NULL, "missing IPv4 protocol"); 6647 6648 if ((mask->l4_ok || mask->l4_csum_ok) && !(pattern_flags & l4_flags)) 6649 return rte_flow_error_set(error, EINVAL, 6650 RTE_FLOW_ERROR_TYPE_ITEM, 6651 NULL, "missing L4 protocol"); 6652 6653 return 0; 6654 } 6655 6656 static int 6657 flow_dv_validate_item_integrity_post(const struct 6658 rte_flow_item *integrity_items[2], 6659 int64_t pattern_flags, 6660 struct rte_flow_error *error) 6661 { 6662 const struct rte_flow_item_integrity *mask; 6663 int ret; 6664 6665 if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY) { 6666 mask = (typeof(mask))integrity_items[0]->mask; 6667 ret = validate_integrity_bits(mask, pattern_flags, 6668 MLX5_FLOW_LAYER_OUTER_L3, 6669 MLX5_FLOW_LAYER_OUTER_L4, 6670 MLX5_FLOW_LAYER_OUTER_L3_IPV4, 6671 error); 6672 if (ret) 6673 return ret; 6674 } 6675 if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY) { 6676 mask = (typeof(mask))integrity_items[1]->mask; 6677 ret = validate_integrity_bits(mask, pattern_flags, 6678 MLX5_FLOW_LAYER_INNER_L3, 6679 MLX5_FLOW_LAYER_INNER_L4, 6680 MLX5_FLOW_LAYER_INNER_L3_IPV4, 6681 error); 6682 if (ret) 6683 return ret; 6684 } 6685 return 0; 6686 } 6687 6688 static int 6689 flow_dv_validate_item_integrity(struct rte_eth_dev *dev, 6690 const struct rte_flow_item *integrity_item, 6691 uint64_t pattern_flags, uint64_t *last_item, 6692 const struct rte_flow_item *integrity_items[2], 6693 struct rte_flow_error *error) 6694 { 6695 struct mlx5_priv *priv = dev->data->dev_private; 6696 const struct rte_flow_item_integrity *mask = (typeof(mask)) 6697 integrity_item->mask; 6698 const struct rte_flow_item_integrity *spec = (typeof(spec)) 6699 integrity_item->spec; 6700 6701 if (!priv->sh->cdev->config.hca_attr.pkt_integrity_match) 6702 return rte_flow_error_set(error, ENOTSUP, 6703 RTE_FLOW_ERROR_TYPE_ITEM, 6704 integrity_item, 6705 "packet integrity integrity_item not supported"); 6706 if (!spec) 6707 return rte_flow_error_set(error, ENOTSUP, 6708 RTE_FLOW_ERROR_TYPE_ITEM, 6709 integrity_item, 6710 "no spec for integrity item"); 6711 if (!mask) 6712 mask = &rte_flow_item_integrity_mask; 6713 if (!mlx5_validate_integrity_item(mask)) 6714 return rte_flow_error_set(error, ENOTSUP, 6715 RTE_FLOW_ERROR_TYPE_ITEM, 6716 integrity_item, 6717 "unsupported integrity filter"); 6718 if (spec->level > 1) { 6719 if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY) 6720 return rte_flow_error_set 6721 (error, ENOTSUP, 6722 RTE_FLOW_ERROR_TYPE_ITEM, 6723 NULL, "multiple inner integrity items not supported"); 6724 integrity_items[1] = integrity_item; 6725 *last_item |= MLX5_FLOW_ITEM_INNER_INTEGRITY; 6726 } else { 6727 if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY) 6728 return rte_flow_error_set 6729 (error, ENOTSUP, 6730 RTE_FLOW_ERROR_TYPE_ITEM, 6731 NULL, "multiple outer integrity items not supported"); 6732 integrity_items[0] = integrity_item; 6733 *last_item |= MLX5_FLOW_ITEM_OUTER_INTEGRITY; 6734 } 6735 return 0; 6736 } 6737 6738 static int 6739 flow_dv_validate_item_flex(struct rte_eth_dev *dev, 6740 const struct rte_flow_item *item, 6741 uint64_t item_flags, 6742 uint64_t *last_item, 6743 bool is_inner, 6744 struct rte_flow_error *error) 6745 { 6746 const struct rte_flow_item_flex *flow_spec = item->spec; 6747 const struct rte_flow_item_flex *flow_mask = item->mask; 6748 struct mlx5_flex_item *flex; 6749 6750 if (!flow_spec) 6751 return rte_flow_error_set(error, EINVAL, 6752 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 6753 "flex flow item spec cannot be NULL"); 6754 if (!flow_mask) 6755 return rte_flow_error_set(error, EINVAL, 6756 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 6757 "flex flow item mask cannot be NULL"); 6758 if (item->last) 6759 return rte_flow_error_set(error, ENOTSUP, 6760 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 6761 "flex flow item last not supported"); 6762 if (mlx5_flex_acquire_index(dev, flow_spec->handle, false) < 0) 6763 return rte_flow_error_set(error, EINVAL, 6764 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 6765 "invalid flex flow item handle"); 6766 flex = (struct mlx5_flex_item *)flow_spec->handle; 6767 switch (flex->tunnel_mode) { 6768 case FLEX_TUNNEL_MODE_SINGLE: 6769 if (item_flags & 6770 (MLX5_FLOW_ITEM_OUTER_FLEX | MLX5_FLOW_ITEM_INNER_FLEX)) 6771 rte_flow_error_set(error, EINVAL, 6772 RTE_FLOW_ERROR_TYPE_ITEM, 6773 NULL, "multiple flex items not supported"); 6774 break; 6775 case FLEX_TUNNEL_MODE_OUTER: 6776 if (is_inner) 6777 rte_flow_error_set(error, EINVAL, 6778 RTE_FLOW_ERROR_TYPE_ITEM, 6779 NULL, "inner flex item was not configured"); 6780 if (item_flags & MLX5_FLOW_ITEM_OUTER_FLEX) 6781 rte_flow_error_set(error, ENOTSUP, 6782 RTE_FLOW_ERROR_TYPE_ITEM, 6783 NULL, "multiple flex items not supported"); 6784 break; 6785 case FLEX_TUNNEL_MODE_INNER: 6786 if (!is_inner) 6787 rte_flow_error_set(error, EINVAL, 6788 RTE_FLOW_ERROR_TYPE_ITEM, 6789 NULL, "outer flex item was not configured"); 6790 if (item_flags & MLX5_FLOW_ITEM_INNER_FLEX) 6791 rte_flow_error_set(error, EINVAL, 6792 RTE_FLOW_ERROR_TYPE_ITEM, 6793 NULL, "multiple flex items not supported"); 6794 break; 6795 case FLEX_TUNNEL_MODE_MULTI: 6796 if ((is_inner && (item_flags & MLX5_FLOW_ITEM_INNER_FLEX)) || 6797 (!is_inner && (item_flags & MLX5_FLOW_ITEM_OUTER_FLEX))) { 6798 rte_flow_error_set(error, EINVAL, 6799 RTE_FLOW_ERROR_TYPE_ITEM, 6800 NULL, "multiple flex items not supported"); 6801 } 6802 break; 6803 case FLEX_TUNNEL_MODE_TUNNEL: 6804 if (is_inner || (item_flags & MLX5_FLOW_ITEM_FLEX_TUNNEL)) 6805 rte_flow_error_set(error, EINVAL, 6806 RTE_FLOW_ERROR_TYPE_ITEM, 6807 NULL, "multiple flex tunnel items not supported"); 6808 break; 6809 default: 6810 rte_flow_error_set(error, EINVAL, 6811 RTE_FLOW_ERROR_TYPE_ITEM, 6812 NULL, "invalid flex item configuration"); 6813 } 6814 *last_item = flex->tunnel_mode == FLEX_TUNNEL_MODE_TUNNEL ? 6815 MLX5_FLOW_ITEM_FLEX_TUNNEL : is_inner ? 6816 MLX5_FLOW_ITEM_INNER_FLEX : MLX5_FLOW_ITEM_OUTER_FLEX; 6817 return 0; 6818 } 6819 6820 /** 6821 * Internal validation function. For validating both actions and items. 6822 * 6823 * @param[in] dev 6824 * Pointer to the rte_eth_dev structure. 6825 * @param[in] attr 6826 * Pointer to the flow attributes. 6827 * @param[in] items 6828 * Pointer to the list of items. 6829 * @param[in] actions 6830 * Pointer to the list of actions. 6831 * @param[in] external 6832 * This flow rule is created by request external to PMD. 6833 * @param[in] hairpin 6834 * Number of hairpin TX actions, 0 means classic flow. 6835 * @param[out] error 6836 * Pointer to the error structure. 6837 * 6838 * @return 6839 * 0 on success, a negative errno value otherwise and rte_errno is set. 6840 */ 6841 static int 6842 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, 6843 const struct rte_flow_item items[], 6844 const struct rte_flow_action actions[], 6845 bool external, int hairpin, struct rte_flow_error *error) 6846 { 6847 int ret; 6848 uint64_t aso_mask, action_flags = 0; 6849 uint64_t item_flags = 0; 6850 uint64_t last_item = 0; 6851 uint8_t next_protocol = 0xff; 6852 uint16_t ether_type = 0; 6853 int actions_n = 0; 6854 uint8_t item_ipv6_proto = 0; 6855 int fdb_mirror_limit = 0; 6856 int modify_after_mirror = 0; 6857 const struct rte_flow_item *geneve_item = NULL; 6858 const struct rte_flow_item *gre_item = NULL; 6859 const struct rte_flow_item *gtp_item = NULL; 6860 const struct rte_flow_action_raw_decap *decap; 6861 const struct rte_flow_action_raw_encap *encap; 6862 const struct rte_flow_action_rss *rss = NULL; 6863 const struct rte_flow_action_rss *sample_rss = NULL; 6864 const struct rte_flow_action_count *sample_count = NULL; 6865 const struct rte_flow_item_tcp nic_tcp_mask = { 6866 .hdr = { 6867 .tcp_flags = 0xFF, 6868 .src_port = RTE_BE16(UINT16_MAX), 6869 .dst_port = RTE_BE16(UINT16_MAX), 6870 } 6871 }; 6872 const struct rte_flow_item_ipv6 nic_ipv6_mask = { 6873 .hdr = { 6874 .src_addr = 6875 "\xff\xff\xff\xff\xff\xff\xff\xff" 6876 "\xff\xff\xff\xff\xff\xff\xff\xff", 6877 .dst_addr = 6878 "\xff\xff\xff\xff\xff\xff\xff\xff" 6879 "\xff\xff\xff\xff\xff\xff\xff\xff", 6880 .vtc_flow = RTE_BE32(0xffffffff), 6881 .proto = 0xff, 6882 .hop_limits = 0xff, 6883 }, 6884 .has_frag_ext = 1, 6885 }; 6886 const struct rte_flow_item_ecpri nic_ecpri_mask = { 6887 .hdr = { 6888 .common = { 6889 .u32 = 6890 RTE_BE32(((const struct rte_ecpri_common_hdr) { 6891 .type = 0xFF, 6892 }).u32), 6893 }, 6894 .dummy[0] = 0xffffffff, 6895 }, 6896 }; 6897 struct mlx5_priv *priv = dev->data->dev_private; 6898 struct mlx5_sh_config *dev_conf = &priv->sh->config; 6899 uint16_t queue_index = 0xFFFF; 6900 const struct rte_flow_item_vlan *vlan_m = NULL; 6901 uint32_t rw_act_num = 0; 6902 uint64_t is_root; 6903 const struct mlx5_flow_tunnel *tunnel; 6904 enum mlx5_tof_rule_type tof_rule_type; 6905 struct flow_grp_info grp_info = { 6906 .external = !!external, 6907 .transfer = !!attr->transfer, 6908 .fdb_def_rule = !!priv->fdb_def_rule, 6909 .std_tbl_fix = true, 6910 }; 6911 const struct rte_eth_hairpin_conf *conf; 6912 const struct rte_flow_item *integrity_items[2] = {NULL, NULL}; 6913 const struct rte_flow_item *port_id_item = NULL; 6914 bool def_policy = false; 6915 bool shared_count = false; 6916 uint16_t udp_dport = 0; 6917 uint32_t tag_id = 0; 6918 const struct rte_flow_action_age *non_shared_age = NULL; 6919 const struct rte_flow_action_count *count = NULL; 6920 6921 if (items == NULL) 6922 return -1; 6923 tunnel = is_tunnel_offload_active(dev) ? 6924 mlx5_get_tof(items, actions, &tof_rule_type) : NULL; 6925 if (tunnel) { 6926 if (!dev_conf->dv_flow_en) 6927 return rte_flow_error_set 6928 (error, ENOTSUP, 6929 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6930 NULL, "tunnel offload requires DV flow interface"); 6931 if (priv->representor) 6932 return rte_flow_error_set 6933 (error, ENOTSUP, 6934 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6935 NULL, "decap not supported for VF representor"); 6936 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE) 6937 action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET; 6938 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE) 6939 action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH | 6940 MLX5_FLOW_ACTION_DECAP; 6941 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate 6942 (dev, attr, tunnel, tof_rule_type); 6943 } 6944 ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error); 6945 if (ret < 0) 6946 return ret; 6947 is_root = (uint64_t)ret; 6948 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6949 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 6950 int type = items->type; 6951 6952 if (!mlx5_flow_os_item_supported(type)) 6953 return rte_flow_error_set(error, ENOTSUP, 6954 RTE_FLOW_ERROR_TYPE_ITEM, 6955 NULL, "item not supported"); 6956 switch (type) { 6957 case RTE_FLOW_ITEM_TYPE_VOID: 6958 break; 6959 case RTE_FLOW_ITEM_TYPE_ESP: 6960 ret = mlx5_flow_os_validate_item_esp(items, item_flags, 6961 next_protocol, 6962 error); 6963 if (ret < 0) 6964 return ret; 6965 last_item = MLX5_FLOW_ITEM_ESP; 6966 break; 6967 case RTE_FLOW_ITEM_TYPE_PORT_ID: 6968 ret = flow_dv_validate_item_port_id 6969 (dev, items, attr, item_flags, error); 6970 if (ret < 0) 6971 return ret; 6972 last_item = MLX5_FLOW_ITEM_PORT_ID; 6973 port_id_item = items; 6974 break; 6975 case RTE_FLOW_ITEM_TYPE_ETH: 6976 ret = mlx5_flow_validate_item_eth(items, item_flags, 6977 true, error); 6978 if (ret < 0) 6979 return ret; 6980 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 6981 MLX5_FLOW_LAYER_OUTER_L2; 6982 if (items->mask != NULL && items->spec != NULL) { 6983 ether_type = 6984 ((const struct rte_flow_item_eth *) 6985 items->spec)->type; 6986 ether_type &= 6987 ((const struct rte_flow_item_eth *) 6988 items->mask)->type; 6989 ether_type = rte_be_to_cpu_16(ether_type); 6990 } else { 6991 ether_type = 0; 6992 } 6993 break; 6994 case RTE_FLOW_ITEM_TYPE_VLAN: 6995 ret = flow_dv_validate_item_vlan(items, item_flags, 6996 dev, error); 6997 if (ret < 0) 6998 return ret; 6999 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 7000 MLX5_FLOW_LAYER_OUTER_VLAN; 7001 if (items->mask != NULL && items->spec != NULL) { 7002 ether_type = 7003 ((const struct rte_flow_item_vlan *) 7004 items->spec)->inner_type; 7005 ether_type &= 7006 ((const struct rte_flow_item_vlan *) 7007 items->mask)->inner_type; 7008 ether_type = rte_be_to_cpu_16(ether_type); 7009 } else { 7010 ether_type = 0; 7011 } 7012 /* Store outer VLAN mask for of_push_vlan action. */ 7013 if (!tunnel) 7014 vlan_m = items->mask; 7015 break; 7016 case RTE_FLOW_ITEM_TYPE_IPV4: 7017 mlx5_flow_tunnel_ip_check(items, next_protocol, 7018 &item_flags, &tunnel); 7019 ret = flow_dv_validate_item_ipv4(dev, items, item_flags, 7020 last_item, ether_type, 7021 error); 7022 if (ret < 0) 7023 return ret; 7024 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 7025 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 7026 if (items->mask != NULL && 7027 ((const struct rte_flow_item_ipv4 *) 7028 items->mask)->hdr.next_proto_id) { 7029 next_protocol = 7030 ((const struct rte_flow_item_ipv4 *) 7031 (items->spec))->hdr.next_proto_id; 7032 next_protocol &= 7033 ((const struct rte_flow_item_ipv4 *) 7034 (items->mask))->hdr.next_proto_id; 7035 } else { 7036 /* Reset for inner layer. */ 7037 next_protocol = 0xff; 7038 } 7039 break; 7040 case RTE_FLOW_ITEM_TYPE_IPV6: 7041 mlx5_flow_tunnel_ip_check(items, next_protocol, 7042 &item_flags, &tunnel); 7043 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 7044 last_item, 7045 ether_type, 7046 &nic_ipv6_mask, 7047 error); 7048 if (ret < 0) 7049 return ret; 7050 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 7051 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 7052 if (items->mask != NULL && 7053 ((const struct rte_flow_item_ipv6 *) 7054 items->mask)->hdr.proto) { 7055 item_ipv6_proto = 7056 ((const struct rte_flow_item_ipv6 *) 7057 items->spec)->hdr.proto; 7058 next_protocol = 7059 ((const struct rte_flow_item_ipv6 *) 7060 items->spec)->hdr.proto; 7061 next_protocol &= 7062 ((const struct rte_flow_item_ipv6 *) 7063 items->mask)->hdr.proto; 7064 } else { 7065 /* Reset for inner layer. */ 7066 next_protocol = 0xff; 7067 } 7068 break; 7069 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 7070 ret = flow_dv_validate_item_ipv6_frag_ext(items, 7071 item_flags, 7072 error); 7073 if (ret < 0) 7074 return ret; 7075 last_item = tunnel ? 7076 MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT : 7077 MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT; 7078 if (items->mask != NULL && 7079 ((const struct rte_flow_item_ipv6_frag_ext *) 7080 items->mask)->hdr.next_header) { 7081 next_protocol = 7082 ((const struct rte_flow_item_ipv6_frag_ext *) 7083 items->spec)->hdr.next_header; 7084 next_protocol &= 7085 ((const struct rte_flow_item_ipv6_frag_ext *) 7086 items->mask)->hdr.next_header; 7087 } else { 7088 /* Reset for inner layer. */ 7089 next_protocol = 0xff; 7090 } 7091 break; 7092 case RTE_FLOW_ITEM_TYPE_TCP: 7093 ret = mlx5_flow_validate_item_tcp 7094 (items, item_flags, 7095 next_protocol, 7096 &nic_tcp_mask, 7097 error); 7098 if (ret < 0) 7099 return ret; 7100 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 7101 MLX5_FLOW_LAYER_OUTER_L4_TCP; 7102 break; 7103 case RTE_FLOW_ITEM_TYPE_UDP: 7104 ret = mlx5_flow_validate_item_udp(items, item_flags, 7105 next_protocol, 7106 error); 7107 const struct rte_flow_item_udp *spec = items->spec; 7108 const struct rte_flow_item_udp *mask = items->mask; 7109 if (!mask) 7110 mask = &rte_flow_item_udp_mask; 7111 if (spec != NULL) 7112 udp_dport = rte_be_to_cpu_16 7113 (spec->hdr.dst_port & 7114 mask->hdr.dst_port); 7115 if (ret < 0) 7116 return ret; 7117 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 7118 MLX5_FLOW_LAYER_OUTER_L4_UDP; 7119 break; 7120 case RTE_FLOW_ITEM_TYPE_GRE: 7121 ret = mlx5_flow_validate_item_gre(items, item_flags, 7122 next_protocol, error); 7123 if (ret < 0) 7124 return ret; 7125 gre_item = items; 7126 last_item = MLX5_FLOW_LAYER_GRE; 7127 break; 7128 case RTE_FLOW_ITEM_TYPE_GRE_OPTION: 7129 ret = mlx5_flow_validate_item_gre_option(dev, items, item_flags, 7130 attr, gre_item, error); 7131 if (ret < 0) 7132 return ret; 7133 last_item = MLX5_FLOW_LAYER_GRE; 7134 break; 7135 case RTE_FLOW_ITEM_TYPE_NVGRE: 7136 ret = mlx5_flow_validate_item_nvgre(items, item_flags, 7137 next_protocol, 7138 error); 7139 if (ret < 0) 7140 return ret; 7141 last_item = MLX5_FLOW_LAYER_NVGRE; 7142 break; 7143 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 7144 ret = mlx5_flow_validate_item_gre_key 7145 (items, item_flags, gre_item, error); 7146 if (ret < 0) 7147 return ret; 7148 last_item = MLX5_FLOW_LAYER_GRE_KEY; 7149 break; 7150 case RTE_FLOW_ITEM_TYPE_VXLAN: 7151 ret = mlx5_flow_validate_item_vxlan(dev, udp_dport, 7152 items, item_flags, 7153 attr, error); 7154 if (ret < 0) 7155 return ret; 7156 last_item = MLX5_FLOW_LAYER_VXLAN; 7157 break; 7158 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 7159 ret = mlx5_flow_validate_item_vxlan_gpe(items, 7160 item_flags, dev, 7161 error); 7162 if (ret < 0) 7163 return ret; 7164 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 7165 break; 7166 case RTE_FLOW_ITEM_TYPE_GENEVE: 7167 ret = mlx5_flow_validate_item_geneve(items, 7168 item_flags, dev, 7169 error); 7170 if (ret < 0) 7171 return ret; 7172 geneve_item = items; 7173 last_item = MLX5_FLOW_LAYER_GENEVE; 7174 break; 7175 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: 7176 ret = mlx5_flow_validate_item_geneve_opt(items, 7177 last_item, 7178 geneve_item, 7179 dev, 7180 error); 7181 if (ret < 0) 7182 return ret; 7183 last_item = MLX5_FLOW_LAYER_GENEVE_OPT; 7184 break; 7185 case RTE_FLOW_ITEM_TYPE_MPLS: 7186 ret = mlx5_flow_validate_item_mpls(dev, items, 7187 item_flags, 7188 last_item, error); 7189 if (ret < 0) 7190 return ret; 7191 last_item = MLX5_FLOW_LAYER_MPLS; 7192 break; 7193 7194 case RTE_FLOW_ITEM_TYPE_MARK: 7195 ret = flow_dv_validate_item_mark(dev, items, attr, 7196 error); 7197 if (ret < 0) 7198 return ret; 7199 last_item = MLX5_FLOW_ITEM_MARK; 7200 break; 7201 case RTE_FLOW_ITEM_TYPE_META: 7202 ret = flow_dv_validate_item_meta(dev, items, attr, 7203 error); 7204 if (ret < 0) 7205 return ret; 7206 last_item = MLX5_FLOW_ITEM_METADATA; 7207 break; 7208 case RTE_FLOW_ITEM_TYPE_ICMP: 7209 ret = mlx5_flow_validate_item_icmp(items, item_flags, 7210 next_protocol, 7211 error); 7212 if (ret < 0) 7213 return ret; 7214 last_item = MLX5_FLOW_LAYER_ICMP; 7215 break; 7216 case RTE_FLOW_ITEM_TYPE_ICMP6: 7217 ret = mlx5_flow_validate_item_icmp6(items, item_flags, 7218 next_protocol, 7219 error); 7220 if (ret < 0) 7221 return ret; 7222 item_ipv6_proto = IPPROTO_ICMPV6; 7223 last_item = MLX5_FLOW_LAYER_ICMP6; 7224 break; 7225 case RTE_FLOW_ITEM_TYPE_TAG: 7226 ret = flow_dv_validate_item_tag(dev, items, 7227 attr, error); 7228 if (ret < 0) 7229 return ret; 7230 last_item = MLX5_FLOW_ITEM_TAG; 7231 break; 7232 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: 7233 last_item = MLX5_FLOW_ITEM_TX_QUEUE; 7234 break; 7235 case MLX5_RTE_FLOW_ITEM_TYPE_TAG: 7236 break; 7237 case RTE_FLOW_ITEM_TYPE_GTP: 7238 ret = flow_dv_validate_item_gtp(dev, items, item_flags, 7239 error); 7240 if (ret < 0) 7241 return ret; 7242 gtp_item = items; 7243 last_item = MLX5_FLOW_LAYER_GTP; 7244 break; 7245 case RTE_FLOW_ITEM_TYPE_GTP_PSC: 7246 ret = flow_dv_validate_item_gtp_psc(items, last_item, 7247 gtp_item, attr, 7248 error); 7249 if (ret < 0) 7250 return ret; 7251 last_item = MLX5_FLOW_LAYER_GTP_PSC; 7252 break; 7253 case RTE_FLOW_ITEM_TYPE_ECPRI: 7254 /* Capacity will be checked in the translate stage. */ 7255 ret = mlx5_flow_validate_item_ecpri(items, item_flags, 7256 last_item, 7257 ether_type, 7258 &nic_ecpri_mask, 7259 error); 7260 if (ret < 0) 7261 return ret; 7262 last_item = MLX5_FLOW_LAYER_ECPRI; 7263 break; 7264 case RTE_FLOW_ITEM_TYPE_INTEGRITY: 7265 ret = flow_dv_validate_item_integrity(dev, items, 7266 item_flags, 7267 &last_item, 7268 integrity_items, 7269 error); 7270 if (ret < 0) 7271 return ret; 7272 break; 7273 case RTE_FLOW_ITEM_TYPE_CONNTRACK: 7274 ret = flow_dv_validate_item_aso_ct(dev, items, 7275 &item_flags, error); 7276 if (ret < 0) 7277 return ret; 7278 break; 7279 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL: 7280 /* tunnel offload item was processed before 7281 * list it here as a supported type 7282 */ 7283 break; 7284 case RTE_FLOW_ITEM_TYPE_FLEX: 7285 ret = flow_dv_validate_item_flex(dev, items, item_flags, 7286 &last_item, 7287 tunnel != 0, error); 7288 if (ret < 0) 7289 return ret; 7290 break; 7291 default: 7292 return rte_flow_error_set(error, ENOTSUP, 7293 RTE_FLOW_ERROR_TYPE_ITEM, 7294 NULL, "item not supported"); 7295 } 7296 item_flags |= last_item; 7297 } 7298 if (item_flags & MLX5_FLOW_ITEM_INTEGRITY) { 7299 ret = flow_dv_validate_item_integrity_post(integrity_items, 7300 item_flags, error); 7301 if (ret) 7302 return ret; 7303 } 7304 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 7305 int type = actions->type; 7306 7307 if (!mlx5_flow_os_action_supported(type)) 7308 return rte_flow_error_set(error, ENOTSUP, 7309 RTE_FLOW_ERROR_TYPE_ACTION, 7310 actions, 7311 "action not supported"); 7312 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) 7313 return rte_flow_error_set(error, ENOTSUP, 7314 RTE_FLOW_ERROR_TYPE_ACTION, 7315 actions, "too many actions"); 7316 if (action_flags & 7317 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) 7318 return rte_flow_error_set(error, ENOTSUP, 7319 RTE_FLOW_ERROR_TYPE_ACTION, 7320 NULL, "meter action with policy " 7321 "must be the last action"); 7322 switch (type) { 7323 case RTE_FLOW_ACTION_TYPE_VOID: 7324 break; 7325 case RTE_FLOW_ACTION_TYPE_PORT_ID: 7326 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 7327 ret = flow_dv_validate_action_port_id(dev, 7328 action_flags, 7329 actions, 7330 attr, 7331 error); 7332 if (ret) 7333 return ret; 7334 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 7335 ++actions_n; 7336 break; 7337 case RTE_FLOW_ACTION_TYPE_FLAG: 7338 ret = flow_dv_validate_action_flag(dev, action_flags, 7339 attr, error); 7340 if (ret < 0) 7341 return ret; 7342 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 7343 /* Count all modify-header actions as one. */ 7344 if (!(action_flags & 7345 MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7346 ++actions_n; 7347 action_flags |= MLX5_FLOW_ACTION_FLAG | 7348 MLX5_FLOW_ACTION_MARK_EXT; 7349 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7350 modify_after_mirror = 1; 7351 7352 } else { 7353 action_flags |= MLX5_FLOW_ACTION_FLAG; 7354 ++actions_n; 7355 } 7356 rw_act_num += MLX5_ACT_NUM_SET_MARK; 7357 break; 7358 case RTE_FLOW_ACTION_TYPE_MARK: 7359 ret = flow_dv_validate_action_mark(dev, actions, 7360 action_flags, 7361 attr, error); 7362 if (ret < 0) 7363 return ret; 7364 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 7365 /* Count all modify-header actions as one. */ 7366 if (!(action_flags & 7367 MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7368 ++actions_n; 7369 action_flags |= MLX5_FLOW_ACTION_MARK | 7370 MLX5_FLOW_ACTION_MARK_EXT; 7371 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7372 modify_after_mirror = 1; 7373 } else { 7374 action_flags |= MLX5_FLOW_ACTION_MARK; 7375 ++actions_n; 7376 } 7377 rw_act_num += MLX5_ACT_NUM_SET_MARK; 7378 break; 7379 case RTE_FLOW_ACTION_TYPE_SET_META: 7380 ret = flow_dv_validate_action_set_meta(dev, actions, 7381 action_flags, 7382 attr, error); 7383 if (ret < 0) 7384 return ret; 7385 /* Count all modify-header actions as one action. */ 7386 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7387 ++actions_n; 7388 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7389 modify_after_mirror = 1; 7390 action_flags |= MLX5_FLOW_ACTION_SET_META; 7391 rw_act_num += MLX5_ACT_NUM_SET_META; 7392 break; 7393 case RTE_FLOW_ACTION_TYPE_SET_TAG: 7394 ret = flow_dv_validate_action_set_tag(dev, actions, 7395 action_flags, 7396 attr, error); 7397 if (ret < 0) 7398 return ret; 7399 /* Count all modify-header actions as one action. */ 7400 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7401 ++actions_n; 7402 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7403 modify_after_mirror = 1; 7404 tag_id = ((const struct rte_flow_action_set_tag *) 7405 actions->conf)->index; 7406 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 7407 rw_act_num += MLX5_ACT_NUM_SET_TAG; 7408 break; 7409 case RTE_FLOW_ACTION_TYPE_DROP: 7410 ret = mlx5_flow_validate_action_drop(action_flags, 7411 attr, error); 7412 if (ret < 0) 7413 return ret; 7414 action_flags |= MLX5_FLOW_ACTION_DROP; 7415 ++actions_n; 7416 break; 7417 case RTE_FLOW_ACTION_TYPE_QUEUE: 7418 ret = mlx5_flow_validate_action_queue(actions, 7419 action_flags, dev, 7420 attr, error); 7421 if (ret < 0) 7422 return ret; 7423 queue_index = ((const struct rte_flow_action_queue *) 7424 (actions->conf))->index; 7425 action_flags |= MLX5_FLOW_ACTION_QUEUE; 7426 ++actions_n; 7427 break; 7428 case RTE_FLOW_ACTION_TYPE_RSS: 7429 rss = actions->conf; 7430 ret = mlx5_flow_validate_action_rss(actions, 7431 action_flags, dev, 7432 attr, item_flags, 7433 error); 7434 if (ret < 0) 7435 return ret; 7436 if (rss && sample_rss && 7437 (sample_rss->level != rss->level || 7438 sample_rss->types != rss->types)) 7439 return rte_flow_error_set(error, ENOTSUP, 7440 RTE_FLOW_ERROR_TYPE_ACTION, 7441 NULL, 7442 "Can't use the different RSS types " 7443 "or level in the same flow"); 7444 if (rss != NULL && rss->queue_num) 7445 queue_index = rss->queue[0]; 7446 action_flags |= MLX5_FLOW_ACTION_RSS; 7447 ++actions_n; 7448 break; 7449 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: 7450 ret = 7451 mlx5_flow_validate_action_default_miss(action_flags, 7452 attr, error); 7453 if (ret < 0) 7454 return ret; 7455 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; 7456 ++actions_n; 7457 break; 7458 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT: 7459 shared_count = true; 7460 /* fall-through. */ 7461 case RTE_FLOW_ACTION_TYPE_COUNT: 7462 ret = flow_dv_validate_action_count(dev, shared_count, 7463 action_flags, 7464 attr, error); 7465 if (ret < 0) 7466 return ret; 7467 count = actions->conf; 7468 action_flags |= MLX5_FLOW_ACTION_COUNT; 7469 ++actions_n; 7470 break; 7471 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 7472 if (flow_dv_validate_action_pop_vlan(dev, 7473 action_flags, 7474 actions, 7475 item_flags, attr, 7476 error)) 7477 return -rte_errno; 7478 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7479 modify_after_mirror = 1; 7480 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; 7481 ++actions_n; 7482 break; 7483 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 7484 ret = flow_dv_validate_action_push_vlan(dev, 7485 action_flags, 7486 vlan_m, 7487 actions, attr, 7488 error); 7489 if (ret < 0) 7490 return ret; 7491 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7492 modify_after_mirror = 1; 7493 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; 7494 ++actions_n; 7495 break; 7496 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 7497 ret = flow_dv_validate_action_set_vlan_pcp 7498 (action_flags, actions, error); 7499 if (ret < 0) 7500 return ret; 7501 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7502 modify_after_mirror = 1; 7503 /* Count PCP with push_vlan command. */ 7504 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP; 7505 break; 7506 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 7507 ret = flow_dv_validate_action_set_vlan_vid 7508 (item_flags, action_flags, 7509 actions, error); 7510 if (ret < 0) 7511 return ret; 7512 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7513 modify_after_mirror = 1; 7514 /* Count VID with push_vlan command. */ 7515 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; 7516 rw_act_num += MLX5_ACT_NUM_MDF_VID; 7517 break; 7518 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 7519 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 7520 ret = flow_dv_validate_action_l2_encap(dev, 7521 action_flags, 7522 actions, attr, 7523 error); 7524 if (ret < 0) 7525 return ret; 7526 action_flags |= MLX5_FLOW_ACTION_ENCAP; 7527 ++actions_n; 7528 break; 7529 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 7530 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 7531 ret = flow_dv_validate_action_decap(dev, action_flags, 7532 actions, item_flags, 7533 attr, error); 7534 if (ret < 0) 7535 return ret; 7536 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7537 modify_after_mirror = 1; 7538 action_flags |= MLX5_FLOW_ACTION_DECAP; 7539 ++actions_n; 7540 break; 7541 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 7542 ret = flow_dv_validate_action_raw_encap_decap 7543 (dev, NULL, actions->conf, attr, &action_flags, 7544 &actions_n, actions, item_flags, error); 7545 if (ret < 0) 7546 return ret; 7547 break; 7548 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 7549 decap = actions->conf; 7550 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 7551 ; 7552 if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 7553 encap = NULL; 7554 actions--; 7555 } else { 7556 encap = actions->conf; 7557 } 7558 ret = flow_dv_validate_action_raw_encap_decap 7559 (dev, 7560 decap ? decap : &empty_decap, encap, 7561 attr, &action_flags, &actions_n, 7562 actions, item_flags, error); 7563 if (ret < 0) 7564 return ret; 7565 if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && 7566 (action_flags & MLX5_FLOW_ACTION_DECAP)) 7567 modify_after_mirror = 1; 7568 break; 7569 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 7570 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 7571 ret = flow_dv_validate_action_modify_mac(action_flags, 7572 actions, 7573 item_flags, 7574 error); 7575 if (ret < 0) 7576 return ret; 7577 /* Count all modify-header actions as one action. */ 7578 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7579 ++actions_n; 7580 action_flags |= actions->type == 7581 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 7582 MLX5_FLOW_ACTION_SET_MAC_SRC : 7583 MLX5_FLOW_ACTION_SET_MAC_DST; 7584 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7585 modify_after_mirror = 1; 7586 /* 7587 * Even if the source and destination MAC addresses have 7588 * overlap in the header with 4B alignment, the convert 7589 * function will handle them separately and 4 SW actions 7590 * will be created. And 2 actions will be added each 7591 * time no matter how many bytes of address will be set. 7592 */ 7593 rw_act_num += MLX5_ACT_NUM_MDF_MAC; 7594 break; 7595 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 7596 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 7597 ret = flow_dv_validate_action_modify_ipv4(action_flags, 7598 actions, 7599 item_flags, 7600 error); 7601 if (ret < 0) 7602 return ret; 7603 /* Count all modify-header actions as one action. */ 7604 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7605 ++actions_n; 7606 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7607 modify_after_mirror = 1; 7608 action_flags |= actions->type == 7609 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 7610 MLX5_FLOW_ACTION_SET_IPV4_SRC : 7611 MLX5_FLOW_ACTION_SET_IPV4_DST; 7612 rw_act_num += MLX5_ACT_NUM_MDF_IPV4; 7613 break; 7614 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 7615 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 7616 ret = flow_dv_validate_action_modify_ipv6(action_flags, 7617 actions, 7618 item_flags, 7619 error); 7620 if (ret < 0) 7621 return ret; 7622 if (item_ipv6_proto == IPPROTO_ICMPV6) 7623 return rte_flow_error_set(error, ENOTSUP, 7624 RTE_FLOW_ERROR_TYPE_ACTION, 7625 actions, 7626 "Can't change header " 7627 "with ICMPv6 proto"); 7628 /* Count all modify-header actions as one action. */ 7629 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7630 ++actions_n; 7631 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7632 modify_after_mirror = 1; 7633 action_flags |= actions->type == 7634 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 7635 MLX5_FLOW_ACTION_SET_IPV6_SRC : 7636 MLX5_FLOW_ACTION_SET_IPV6_DST; 7637 rw_act_num += MLX5_ACT_NUM_MDF_IPV6; 7638 break; 7639 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 7640 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 7641 ret = flow_dv_validate_action_modify_tp(action_flags, 7642 actions, 7643 item_flags, 7644 error); 7645 if (ret < 0) 7646 return ret; 7647 /* Count all modify-header actions as one action. */ 7648 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7649 ++actions_n; 7650 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7651 modify_after_mirror = 1; 7652 action_flags |= actions->type == 7653 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 7654 MLX5_FLOW_ACTION_SET_TP_SRC : 7655 MLX5_FLOW_ACTION_SET_TP_DST; 7656 rw_act_num += MLX5_ACT_NUM_MDF_PORT; 7657 break; 7658 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 7659 case RTE_FLOW_ACTION_TYPE_SET_TTL: 7660 ret = flow_dv_validate_action_modify_ttl(action_flags, 7661 actions, 7662 item_flags, 7663 error); 7664 if (ret < 0) 7665 return ret; 7666 /* Count all modify-header actions as one action. */ 7667 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7668 ++actions_n; 7669 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7670 modify_after_mirror = 1; 7671 action_flags |= actions->type == 7672 RTE_FLOW_ACTION_TYPE_SET_TTL ? 7673 MLX5_FLOW_ACTION_SET_TTL : 7674 MLX5_FLOW_ACTION_DEC_TTL; 7675 rw_act_num += MLX5_ACT_NUM_MDF_TTL; 7676 break; 7677 case RTE_FLOW_ACTION_TYPE_JUMP: 7678 ret = flow_dv_validate_action_jump(dev, tunnel, actions, 7679 action_flags, 7680 attr, external, 7681 error); 7682 if (ret) 7683 return ret; 7684 if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && 7685 fdb_mirror_limit) 7686 return rte_flow_error_set(error, EINVAL, 7687 RTE_FLOW_ERROR_TYPE_ACTION, 7688 NULL, 7689 "sample and jump action combination is not supported"); 7690 ++actions_n; 7691 action_flags |= MLX5_FLOW_ACTION_JUMP; 7692 break; 7693 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 7694 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 7695 ret = flow_dv_validate_action_modify_tcp_seq 7696 (action_flags, 7697 actions, 7698 item_flags, 7699 error); 7700 if (ret < 0) 7701 return ret; 7702 /* Count all modify-header actions as one action. */ 7703 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7704 ++actions_n; 7705 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7706 modify_after_mirror = 1; 7707 action_flags |= actions->type == 7708 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 7709 MLX5_FLOW_ACTION_INC_TCP_SEQ : 7710 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 7711 rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ; 7712 break; 7713 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 7714 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 7715 ret = flow_dv_validate_action_modify_tcp_ack 7716 (action_flags, 7717 actions, 7718 item_flags, 7719 error); 7720 if (ret < 0) 7721 return ret; 7722 /* Count all modify-header actions as one action. */ 7723 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7724 ++actions_n; 7725 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7726 modify_after_mirror = 1; 7727 action_flags |= actions->type == 7728 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 7729 MLX5_FLOW_ACTION_INC_TCP_ACK : 7730 MLX5_FLOW_ACTION_DEC_TCP_ACK; 7731 rw_act_num += MLX5_ACT_NUM_MDF_TCPACK; 7732 break; 7733 case MLX5_RTE_FLOW_ACTION_TYPE_MARK: 7734 break; 7735 case MLX5_RTE_FLOW_ACTION_TYPE_TAG: 7736 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: 7737 rw_act_num += MLX5_ACT_NUM_SET_TAG; 7738 break; 7739 case RTE_FLOW_ACTION_TYPE_METER: 7740 ret = mlx5_flow_validate_action_meter(dev, 7741 action_flags, 7742 item_flags, 7743 actions, attr, 7744 port_id_item, 7745 &def_policy, 7746 error); 7747 if (ret < 0) 7748 return ret; 7749 action_flags |= MLX5_FLOW_ACTION_METER; 7750 if (!def_policy) 7751 action_flags |= 7752 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY; 7753 ++actions_n; 7754 /* Meter action will add one more TAG action. */ 7755 rw_act_num += MLX5_ACT_NUM_SET_TAG; 7756 break; 7757 case MLX5_RTE_FLOW_ACTION_TYPE_AGE: 7758 if (!attr->transfer && !attr->group) 7759 return rte_flow_error_set(error, ENOTSUP, 7760 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7761 NULL, 7762 "Shared ASO age action is not supported for group 0"); 7763 if (action_flags & MLX5_FLOW_ACTION_AGE) 7764 return rte_flow_error_set 7765 (error, EINVAL, 7766 RTE_FLOW_ERROR_TYPE_ACTION, 7767 NULL, 7768 "duplicate age actions set"); 7769 action_flags |= MLX5_FLOW_ACTION_AGE; 7770 ++actions_n; 7771 break; 7772 case RTE_FLOW_ACTION_TYPE_AGE: 7773 non_shared_age = actions->conf; 7774 ret = flow_dv_validate_action_age(action_flags, 7775 actions, dev, 7776 error); 7777 if (ret < 0) 7778 return ret; 7779 /* 7780 * Validate the regular AGE action (using counter) 7781 * mutual exclusion with indirect counter actions. 7782 */ 7783 if (!flow_hit_aso_supported(priv->sh, attr)) { 7784 if (shared_count) 7785 return rte_flow_error_set 7786 (error, EINVAL, 7787 RTE_FLOW_ERROR_TYPE_ACTION, 7788 NULL, 7789 "old age and indirect count combination is not supported"); 7790 if (sample_count) 7791 return rte_flow_error_set 7792 (error, EINVAL, 7793 RTE_FLOW_ERROR_TYPE_ACTION, 7794 NULL, 7795 "old age action and count must be in the same sub flow"); 7796 } 7797 action_flags |= MLX5_FLOW_ACTION_AGE; 7798 ++actions_n; 7799 break; 7800 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 7801 ret = flow_dv_validate_action_modify_ipv4_dscp 7802 (action_flags, 7803 actions, 7804 item_flags, 7805 error); 7806 if (ret < 0) 7807 return ret; 7808 /* Count all modify-header actions as one action. */ 7809 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7810 ++actions_n; 7811 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7812 modify_after_mirror = 1; 7813 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; 7814 rw_act_num += MLX5_ACT_NUM_SET_DSCP; 7815 break; 7816 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 7817 ret = flow_dv_validate_action_modify_ipv6_dscp 7818 (action_flags, 7819 actions, 7820 item_flags, 7821 error); 7822 if (ret < 0) 7823 return ret; 7824 /* Count all modify-header actions as one action. */ 7825 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7826 ++actions_n; 7827 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7828 modify_after_mirror = 1; 7829 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; 7830 rw_act_num += MLX5_ACT_NUM_SET_DSCP; 7831 break; 7832 case RTE_FLOW_ACTION_TYPE_SAMPLE: 7833 ret = flow_dv_validate_action_sample(&action_flags, 7834 actions, dev, 7835 attr, item_flags, 7836 rss, &sample_rss, 7837 &sample_count, 7838 &fdb_mirror_limit, 7839 error); 7840 if (ret < 0) 7841 return ret; 7842 if ((action_flags & MLX5_FLOW_ACTION_SET_TAG) && 7843 tag_id == 0 && priv->mtr_color_reg == REG_NON) 7844 return rte_flow_error_set(error, EINVAL, 7845 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 7846 "sample after tag action causes metadata tag index 0 corruption"); 7847 action_flags |= MLX5_FLOW_ACTION_SAMPLE; 7848 ++actions_n; 7849 break; 7850 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 7851 ret = flow_dv_validate_action_modify_field(dev, 7852 action_flags, 7853 actions, 7854 attr, 7855 error); 7856 if (ret < 0) 7857 return ret; 7858 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 7859 modify_after_mirror = 1; 7860 /* Count all modify-header actions as one action. */ 7861 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 7862 ++actions_n; 7863 action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD; 7864 rw_act_num += ret; 7865 break; 7866 case RTE_FLOW_ACTION_TYPE_CONNTRACK: 7867 ret = flow_dv_validate_action_aso_ct(dev, action_flags, 7868 item_flags, attr, 7869 error); 7870 if (ret < 0) 7871 return ret; 7872 action_flags |= MLX5_FLOW_ACTION_CT; 7873 break; 7874 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET: 7875 /* tunnel offload action was processed before 7876 * list it here as a supported type 7877 */ 7878 break; 7879 default: 7880 return rte_flow_error_set(error, ENOTSUP, 7881 RTE_FLOW_ERROR_TYPE_ACTION, 7882 actions, 7883 "action not supported"); 7884 } 7885 } 7886 /* 7887 * Validate actions in flow rules 7888 * - Explicit decap action is prohibited by the tunnel offload API. 7889 * - Drop action in tunnel steer rule is prohibited by the API. 7890 * - Application cannot use MARK action because it's value can mask 7891 * tunnel default miss notification. 7892 * - JUMP in tunnel match rule has no support in current PMD 7893 * implementation. 7894 * - TAG & META are reserved for future uses. 7895 */ 7896 if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) { 7897 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP | 7898 MLX5_FLOW_ACTION_MARK | 7899 MLX5_FLOW_ACTION_SET_TAG | 7900 MLX5_FLOW_ACTION_SET_META | 7901 MLX5_FLOW_ACTION_DROP; 7902 7903 if (action_flags & bad_actions_mask) 7904 return rte_flow_error_set 7905 (error, EINVAL, 7906 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 7907 "Invalid RTE action in tunnel " 7908 "set decap rule"); 7909 if (!(action_flags & MLX5_FLOW_ACTION_JUMP)) 7910 return rte_flow_error_set 7911 (error, EINVAL, 7912 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 7913 "tunnel set decap rule must terminate " 7914 "with JUMP"); 7915 if (!attr->ingress) 7916 return rte_flow_error_set 7917 (error, EINVAL, 7918 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 7919 "tunnel flows for ingress traffic only"); 7920 } 7921 if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) { 7922 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP | 7923 MLX5_FLOW_ACTION_MARK | 7924 MLX5_FLOW_ACTION_SET_TAG | 7925 MLX5_FLOW_ACTION_SET_META; 7926 7927 if (action_flags & bad_actions_mask) 7928 return rte_flow_error_set 7929 (error, EINVAL, 7930 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 7931 "Invalid RTE action in tunnel " 7932 "set match rule"); 7933 } 7934 /* 7935 * Validate the drop action mutual exclusion with other actions. 7936 * Drop action is mutually-exclusive with any other action, except for 7937 * Count action. 7938 * Drop action compatibility with tunnel offload was already validated. 7939 */ 7940 if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH | 7941 MLX5_FLOW_ACTION_TUNNEL_MATCH)); 7942 else if ((action_flags & MLX5_FLOW_ACTION_DROP) && 7943 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) 7944 return rte_flow_error_set(error, EINVAL, 7945 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 7946 "Drop action is mutually-exclusive " 7947 "with any other action, except for " 7948 "Count action"); 7949 /* Eswitch has few restrictions on using items and actions */ 7950 if (attr->transfer) { 7951 if (!mlx5_flow_ext_mreg_supported(dev) && 7952 action_flags & MLX5_FLOW_ACTION_FLAG) 7953 return rte_flow_error_set(error, ENOTSUP, 7954 RTE_FLOW_ERROR_TYPE_ACTION, 7955 NULL, 7956 "unsupported action FLAG"); 7957 if (!mlx5_flow_ext_mreg_supported(dev) && 7958 action_flags & MLX5_FLOW_ACTION_MARK) 7959 return rte_flow_error_set(error, ENOTSUP, 7960 RTE_FLOW_ERROR_TYPE_ACTION, 7961 NULL, 7962 "unsupported action MARK"); 7963 if (action_flags & MLX5_FLOW_ACTION_QUEUE) 7964 return rte_flow_error_set(error, ENOTSUP, 7965 RTE_FLOW_ERROR_TYPE_ACTION, 7966 NULL, 7967 "unsupported action QUEUE"); 7968 if (action_flags & MLX5_FLOW_ACTION_RSS) 7969 return rte_flow_error_set(error, ENOTSUP, 7970 RTE_FLOW_ERROR_TYPE_ACTION, 7971 NULL, 7972 "unsupported action RSS"); 7973 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 7974 return rte_flow_error_set(error, EINVAL, 7975 RTE_FLOW_ERROR_TYPE_ACTION, 7976 actions, 7977 "no fate action is found"); 7978 } else { 7979 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress) 7980 return rte_flow_error_set(error, EINVAL, 7981 RTE_FLOW_ERROR_TYPE_ACTION, 7982 actions, 7983 "no fate action is found"); 7984 } 7985 /* 7986 * Continue validation for Xcap and VLAN actions. 7987 * If hairpin is working in explicit TX rule mode, there is no actions 7988 * splitting and the validation of hairpin ingress flow should be the 7989 * same as other standard flows. 7990 */ 7991 if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS | 7992 MLX5_FLOW_VLAN_ACTIONS)) && 7993 (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index) || 7994 ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL && 7995 conf->tx_explicit != 0))) { 7996 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) == 7997 MLX5_FLOW_XCAP_ACTIONS) 7998 return rte_flow_error_set(error, ENOTSUP, 7999 RTE_FLOW_ERROR_TYPE_ACTION, 8000 NULL, "encap and decap " 8001 "combination aren't supported"); 8002 /* Push VLAN is not supported in ingress except for NICs newer than CX5. */ 8003 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) { 8004 struct mlx5_dev_ctx_shared *sh = priv->sh; 8005 bool direction_error = false; 8006 8007 if (attr->transfer) { 8008 bool fdb_tx = priv->representor_id != UINT16_MAX; 8009 bool is_cx5 = sh->steering_format_version == 8010 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5; 8011 8012 if (!fdb_tx && is_cx5) 8013 direction_error = true; 8014 } else if (attr->ingress) { 8015 direction_error = true; 8016 } 8017 if (direction_error) 8018 return rte_flow_error_set(error, ENOTSUP, 8019 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 8020 NULL, 8021 "push VLAN action not supported " 8022 "for ingress"); 8023 } 8024 if (!attr->transfer && attr->ingress) { 8025 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 8026 return rte_flow_error_set 8027 (error, ENOTSUP, 8028 RTE_FLOW_ERROR_TYPE_ACTION, 8029 NULL, "encap is not supported" 8030 " for ingress traffic"); 8031 else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) == 8032 MLX5_FLOW_VLAN_ACTIONS) 8033 return rte_flow_error_set 8034 (error, ENOTSUP, 8035 RTE_FLOW_ERROR_TYPE_ACTION, 8036 NULL, "no support for " 8037 "multiple VLAN actions"); 8038 } 8039 } 8040 if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) { 8041 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS & 8042 ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) && 8043 attr->ingress) 8044 return rte_flow_error_set 8045 (error, ENOTSUP, 8046 RTE_FLOW_ERROR_TYPE_ACTION, 8047 NULL, "fate action not supported for " 8048 "meter with policy"); 8049 if (attr->egress) { 8050 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) 8051 return rte_flow_error_set 8052 (error, ENOTSUP, 8053 RTE_FLOW_ERROR_TYPE_ACTION, 8054 NULL, "modify header action in egress " 8055 "cannot be done before meter action"); 8056 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 8057 return rte_flow_error_set 8058 (error, ENOTSUP, 8059 RTE_FLOW_ERROR_TYPE_ACTION, 8060 NULL, "encap action in egress " 8061 "cannot be done before meter action"); 8062 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) 8063 return rte_flow_error_set 8064 (error, ENOTSUP, 8065 RTE_FLOW_ERROR_TYPE_ACTION, 8066 NULL, "push vlan action in egress " 8067 "cannot be done before meter action"); 8068 } 8069 } 8070 /* 8071 * Only support one ASO action in a single flow rule. 8072 * non-shared AGE + counter will fallback to use HW counter, no ASO hit object. 8073 * Group 0 uses HW counter for AGE too even if no counter action. 8074 */ 8075 aso_mask = (action_flags & MLX5_FLOW_ACTION_METER && priv->sh->meter_aso_en) << 2 | 8076 (action_flags & MLX5_FLOW_ACTION_CT && priv->sh->ct_aso_en) << 1 | 8077 (action_flags & MLX5_FLOW_ACTION_AGE && 8078 !(non_shared_age && count) && 8079 (attr->group || (attr->transfer && priv->fdb_def_rule)) && 8080 priv->sh->flow_hit_aso_en); 8081 if (__builtin_popcountl(aso_mask) > 1) 8082 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8083 NULL, "unsupported combining AGE, METER, CT ASO actions in a single rule"); 8084 /* 8085 * Hairpin flow will add one more TAG action in TX implicit mode. 8086 * In TX explicit mode, there will be no hairpin flow ID. 8087 */ 8088 if (hairpin > 0) 8089 rw_act_num += MLX5_ACT_NUM_SET_TAG; 8090 /* extra metadata enabled: one more TAG action will be add. */ 8091 if (dev_conf->dv_flow_en && 8092 dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 8093 mlx5_flow_ext_mreg_supported(dev)) 8094 rw_act_num += MLX5_ACT_NUM_SET_TAG; 8095 if (rw_act_num > 8096 flow_dv_modify_hdr_action_max(dev, is_root)) { 8097 return rte_flow_error_set(error, ENOTSUP, 8098 RTE_FLOW_ERROR_TYPE_ACTION, 8099 NULL, "too many header modify" 8100 " actions to support"); 8101 } 8102 /* Eswitch egress mirror and modify flow has limitation on CX5 */ 8103 if (fdb_mirror_limit && modify_after_mirror) 8104 return rte_flow_error_set(error, EINVAL, 8105 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 8106 "sample before modify action is not supported"); 8107 /* 8108 * Validation the NIC Egress flow on representor, except implicit 8109 * hairpin default egress flow with TX_QUEUE item, other flows not 8110 * work due to metadata regC0 mismatch. 8111 */ 8112 if ((!attr->transfer && attr->egress) && priv->representor && 8113 !(item_flags & MLX5_FLOW_ITEM_TX_QUEUE)) 8114 return rte_flow_error_set(error, EINVAL, 8115 RTE_FLOW_ERROR_TYPE_ITEM, 8116 NULL, 8117 "NIC egress rules on representors" 8118 " is not supported"); 8119 return 0; 8120 } 8121 8122 /** 8123 * Internal preparation function. Allocates the DV flow size, 8124 * this size is constant. 8125 * 8126 * @param[in] dev 8127 * Pointer to the rte_eth_dev structure. 8128 * @param[in] attr 8129 * Pointer to the flow attributes. 8130 * @param[in] items 8131 * Pointer to the list of items. 8132 * @param[in] actions 8133 * Pointer to the list of actions. 8134 * @param[out] error 8135 * Pointer to the error structure. 8136 * 8137 * @return 8138 * Pointer to mlx5_flow object on success, 8139 * otherwise NULL and rte_errno is set. 8140 */ 8141 static struct mlx5_flow * 8142 flow_dv_prepare(struct rte_eth_dev *dev, 8143 const struct rte_flow_attr *attr __rte_unused, 8144 const struct rte_flow_item items[] __rte_unused, 8145 const struct rte_flow_action actions[] __rte_unused, 8146 struct rte_flow_error *error) 8147 { 8148 uint32_t handle_idx = 0; 8149 struct mlx5_flow *dev_flow; 8150 struct mlx5_flow_handle *dev_handle; 8151 struct mlx5_priv *priv = dev->data->dev_private; 8152 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 8153 8154 MLX5_ASSERT(wks); 8155 wks->skip_matcher_reg = 0; 8156 wks->policy = NULL; 8157 wks->final_policy = NULL; 8158 /* In case of corrupting the memory. */ 8159 if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { 8160 rte_flow_error_set(error, ENOSPC, 8161 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8162 "not free temporary device flow"); 8163 return NULL; 8164 } 8165 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8166 &handle_idx); 8167 if (!dev_handle) { 8168 rte_flow_error_set(error, ENOMEM, 8169 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8170 "not enough memory to create flow handle"); 8171 return NULL; 8172 } 8173 MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows)); 8174 dev_flow = &wks->flows[wks->flow_idx++]; 8175 memset(dev_flow, 0, sizeof(*dev_flow)); 8176 dev_flow->handle = dev_handle; 8177 dev_flow->handle_idx = handle_idx; 8178 dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); 8179 dev_flow->ingress = attr->ingress; 8180 dev_flow->dv.transfer = attr->transfer; 8181 return dev_flow; 8182 } 8183 8184 #ifdef RTE_LIBRTE_MLX5_DEBUG 8185 /** 8186 * Sanity check for match mask and value. Similar to check_valid_spec() in 8187 * kernel driver. If unmasked bit is present in value, it returns failure. 8188 * 8189 * @param match_mask 8190 * pointer to match mask buffer. 8191 * @param match_value 8192 * pointer to match value buffer. 8193 * 8194 * @return 8195 * 0 if valid, -EINVAL otherwise. 8196 */ 8197 static int 8198 flow_dv_check_valid_spec(void *match_mask, void *match_value) 8199 { 8200 uint8_t *m = match_mask; 8201 uint8_t *v = match_value; 8202 unsigned int i; 8203 8204 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) { 8205 if (v[i] & ~m[i]) { 8206 DRV_LOG(ERR, 8207 "match_value differs from match_criteria" 8208 " %p[%u] != %p[%u]", 8209 match_value, i, match_mask, i); 8210 return -EINVAL; 8211 } 8212 } 8213 return 0; 8214 } 8215 #endif 8216 8217 /** 8218 * Add match of ip_version. 8219 * 8220 * @param[in] group 8221 * Flow group. 8222 * @param[in] headers_v 8223 * Values header pointer. 8224 * @param[in] headers_m 8225 * Masks header pointer. 8226 * @param[in] ip_version 8227 * The IP version to set. 8228 */ 8229 static inline void 8230 flow_dv_set_match_ip_version(uint32_t group, 8231 void *headers_v, 8232 void *headers_m, 8233 uint8_t ip_version) 8234 { 8235 if (group == 0) 8236 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); 8237 else 8238 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 8239 ip_version); 8240 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version); 8241 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0); 8242 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0); 8243 } 8244 8245 /** 8246 * Add Ethernet item to matcher and to the value. 8247 * 8248 * @param[in, out] matcher 8249 * Flow matcher. 8250 * @param[in, out] key 8251 * Flow matcher value. 8252 * @param[in] item 8253 * Flow pattern to translate. 8254 * @param[in] inner 8255 * Item is inner pattern. 8256 */ 8257 static void 8258 flow_dv_translate_item_eth(void *matcher, void *key, 8259 const struct rte_flow_item *item, int inner, 8260 uint32_t group) 8261 { 8262 const struct rte_flow_item_eth *eth_m = item->mask; 8263 const struct rte_flow_item_eth *eth_v = item->spec; 8264 const struct rte_flow_item_eth nic_mask = { 8265 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 8266 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 8267 .type = RTE_BE16(0xffff), 8268 .has_vlan = 0, 8269 }; 8270 void *hdrs_m; 8271 void *hdrs_v; 8272 char *l24_v; 8273 unsigned int i; 8274 8275 if (!eth_v) 8276 return; 8277 if (!eth_m) 8278 eth_m = &nic_mask; 8279 if (inner) { 8280 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, 8281 inner_headers); 8282 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 8283 } else { 8284 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, 8285 outer_headers); 8286 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8287 } 8288 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16), 8289 ð_m->dst, sizeof(eth_m->dst)); 8290 /* The value must be in the range of the mask. */ 8291 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16); 8292 for (i = 0; i < sizeof(eth_m->dst); ++i) 8293 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i]; 8294 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16), 8295 ð_m->src, sizeof(eth_m->src)); 8296 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16); 8297 /* The value must be in the range of the mask. */ 8298 for (i = 0; i < sizeof(eth_m->dst); ++i) 8299 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i]; 8300 /* 8301 * HW supports match on one Ethertype, the Ethertype following the last 8302 * VLAN tag of the packet (see PRM). 8303 * Set match on ethertype only if ETH header is not followed by VLAN. 8304 * HW is optimized for IPv4/IPv6. In such cases, avoid setting 8305 * ethertype, and use ip_version field instead. 8306 * eCPRI over Ether layer will use type value 0xAEFE. 8307 */ 8308 if (eth_m->type == 0xFFFF) { 8309 /* Set cvlan_tag mask for any single\multi\un-tagged case. */ 8310 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1); 8311 switch (eth_v->type) { 8312 case RTE_BE16(RTE_ETHER_TYPE_VLAN): 8313 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1); 8314 return; 8315 case RTE_BE16(RTE_ETHER_TYPE_QINQ): 8316 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1); 8317 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1); 8318 return; 8319 case RTE_BE16(RTE_ETHER_TYPE_IPV4): 8320 flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4); 8321 return; 8322 case RTE_BE16(RTE_ETHER_TYPE_IPV6): 8323 flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6); 8324 return; 8325 default: 8326 break; 8327 } 8328 } 8329 if (eth_m->has_vlan) { 8330 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1); 8331 if (eth_v->has_vlan) { 8332 /* 8333 * Here, when also has_more_vlan field in VLAN item is 8334 * not set, only single-tagged packets will be matched. 8335 */ 8336 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1); 8337 return; 8338 } 8339 } 8340 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype, 8341 rte_be_to_cpu_16(eth_m->type)); 8342 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype); 8343 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type; 8344 } 8345 8346 /** 8347 * Add VLAN item to matcher and to the value. 8348 * 8349 * @param[in, out] dev_flow 8350 * Flow descriptor. 8351 * @param[in, out] matcher 8352 * Flow matcher. 8353 * @param[in, out] key 8354 * Flow matcher value. 8355 * @param[in] item 8356 * Flow pattern to translate. 8357 * @param[in] inner 8358 * Item is inner pattern. 8359 */ 8360 static void 8361 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, 8362 void *matcher, void *key, 8363 const struct rte_flow_item *item, 8364 int inner, uint32_t group) 8365 { 8366 const struct rte_flow_item_vlan *vlan_m = item->mask; 8367 const struct rte_flow_item_vlan *vlan_v = item->spec; 8368 void *hdrs_m; 8369 void *hdrs_v; 8370 uint16_t tci_m; 8371 uint16_t tci_v; 8372 8373 if (inner) { 8374 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, 8375 inner_headers); 8376 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 8377 } else { 8378 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, 8379 outer_headers); 8380 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8381 /* 8382 * This is workaround, masks are not supported, 8383 * and pre-validated. 8384 */ 8385 if (vlan_v) 8386 dev_flow->handle->vf_vlan.tag = 8387 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff; 8388 } 8389 /* 8390 * When VLAN item exists in flow, mark packet as tagged, 8391 * even if TCI is not specified. 8392 */ 8393 if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) { 8394 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1); 8395 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1); 8396 } 8397 if (!vlan_v) 8398 return; 8399 if (!vlan_m) 8400 vlan_m = &rte_flow_item_vlan_mask; 8401 tci_m = rte_be_to_cpu_16(vlan_m->tci); 8402 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci); 8403 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m); 8404 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v); 8405 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12); 8406 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12); 8407 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13); 8408 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13); 8409 /* 8410 * HW is optimized for IPv4/IPv6. In such cases, avoid setting 8411 * ethertype, and use ip_version field instead. 8412 */ 8413 if (vlan_m->inner_type == 0xFFFF) { 8414 switch (vlan_v->inner_type) { 8415 case RTE_BE16(RTE_ETHER_TYPE_VLAN): 8416 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1); 8417 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1); 8418 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0); 8419 return; 8420 case RTE_BE16(RTE_ETHER_TYPE_IPV4): 8421 flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4); 8422 return; 8423 case RTE_BE16(RTE_ETHER_TYPE_IPV6): 8424 flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6); 8425 return; 8426 default: 8427 break; 8428 } 8429 } 8430 if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) { 8431 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1); 8432 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1); 8433 /* Only one vlan_tag bit can be set. */ 8434 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0); 8435 return; 8436 } 8437 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype, 8438 rte_be_to_cpu_16(vlan_m->inner_type)); 8439 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype, 8440 rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type)); 8441 } 8442 8443 /** 8444 * Add IPV4 item to matcher and to the value. 8445 * 8446 * @param[in, out] matcher 8447 * Flow matcher. 8448 * @param[in, out] key 8449 * Flow matcher value. 8450 * @param[in] item 8451 * Flow pattern to translate. 8452 * @param[in] inner 8453 * Item is inner pattern. 8454 * @param[in] group 8455 * The group to insert the rule. 8456 */ 8457 static void 8458 flow_dv_translate_item_ipv4(void *matcher, void *key, 8459 const struct rte_flow_item *item, 8460 int inner, uint32_t group) 8461 { 8462 const struct rte_flow_item_ipv4 *ipv4_m = item->mask; 8463 const struct rte_flow_item_ipv4 *ipv4_v = item->spec; 8464 const struct rte_flow_item_ipv4 nic_mask = { 8465 .hdr = { 8466 .src_addr = RTE_BE32(0xffffffff), 8467 .dst_addr = RTE_BE32(0xffffffff), 8468 .type_of_service = 0xff, 8469 .next_proto_id = 0xff, 8470 .time_to_live = 0xff, 8471 }, 8472 }; 8473 void *headers_m; 8474 void *headers_v; 8475 char *l24_m; 8476 char *l24_v; 8477 uint8_t tos, ihl_m, ihl_v; 8478 8479 if (inner) { 8480 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8481 inner_headers); 8482 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 8483 } else { 8484 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8485 outer_headers); 8486 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8487 } 8488 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 8489 if (!ipv4_v) 8490 return; 8491 if (!ipv4_m) 8492 ipv4_m = &nic_mask; 8493 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 8494 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 8495 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 8496 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 8497 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr; 8498 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr; 8499 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 8500 src_ipv4_src_ipv6.ipv4_layout.ipv4); 8501 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 8502 src_ipv4_src_ipv6.ipv4_layout.ipv4); 8503 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr; 8504 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr; 8505 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service; 8506 ihl_m = ipv4_m->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK; 8507 ihl_v = ipv4_v->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK; 8508 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_ihl, ihl_m); 8509 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_ihl, ihl_m & ihl_v); 8510 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, 8511 ipv4_m->hdr.type_of_service); 8512 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos); 8513 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, 8514 ipv4_m->hdr.type_of_service >> 2); 8515 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2); 8516 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 8517 ipv4_m->hdr.next_proto_id); 8518 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 8519 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id); 8520 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, 8521 ipv4_m->hdr.time_to_live); 8522 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, 8523 ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live); 8524 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 8525 !!(ipv4_m->hdr.fragment_offset)); 8526 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 8527 !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset)); 8528 } 8529 8530 /** 8531 * Add IPV6 item to matcher and to the value. 8532 * 8533 * @param[in, out] matcher 8534 * Flow matcher. 8535 * @param[in, out] key 8536 * Flow matcher value. 8537 * @param[in] item 8538 * Flow pattern to translate. 8539 * @param[in] inner 8540 * Item is inner pattern. 8541 * @param[in] group 8542 * The group to insert the rule. 8543 */ 8544 static void 8545 flow_dv_translate_item_ipv6(void *matcher, void *key, 8546 const struct rte_flow_item *item, 8547 int inner, uint32_t group) 8548 { 8549 const struct rte_flow_item_ipv6 *ipv6_m = item->mask; 8550 const struct rte_flow_item_ipv6 *ipv6_v = item->spec; 8551 const struct rte_flow_item_ipv6 nic_mask = { 8552 .hdr = { 8553 .src_addr = 8554 "\xff\xff\xff\xff\xff\xff\xff\xff" 8555 "\xff\xff\xff\xff\xff\xff\xff\xff", 8556 .dst_addr = 8557 "\xff\xff\xff\xff\xff\xff\xff\xff" 8558 "\xff\xff\xff\xff\xff\xff\xff\xff", 8559 .vtc_flow = RTE_BE32(0xffffffff), 8560 .proto = 0xff, 8561 .hop_limits = 0xff, 8562 }, 8563 }; 8564 void *headers_m; 8565 void *headers_v; 8566 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 8567 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 8568 char *l24_m; 8569 char *l24_v; 8570 uint32_t vtc_m; 8571 uint32_t vtc_v; 8572 int i; 8573 int size; 8574 8575 if (inner) { 8576 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8577 inner_headers); 8578 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 8579 } else { 8580 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8581 outer_headers); 8582 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8583 } 8584 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 8585 if (!ipv6_v) 8586 return; 8587 if (!ipv6_m) 8588 ipv6_m = &nic_mask; 8589 size = sizeof(ipv6_m->hdr.dst_addr); 8590 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 8591 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 8592 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 8593 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 8594 memcpy(l24_m, ipv6_m->hdr.dst_addr, size); 8595 for (i = 0; i < size; ++i) 8596 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i]; 8597 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 8598 src_ipv4_src_ipv6.ipv6_layout.ipv6); 8599 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 8600 src_ipv4_src_ipv6.ipv6_layout.ipv6); 8601 memcpy(l24_m, ipv6_m->hdr.src_addr, size); 8602 for (i = 0; i < size; ++i) 8603 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i]; 8604 /* TOS. */ 8605 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow); 8606 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow); 8607 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20); 8608 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20); 8609 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22); 8610 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22); 8611 /* Label. */ 8612 if (inner) { 8613 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label, 8614 vtc_m); 8615 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label, 8616 vtc_v); 8617 } else { 8618 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label, 8619 vtc_m); 8620 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label, 8621 vtc_v); 8622 } 8623 /* Protocol. */ 8624 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 8625 ipv6_m->hdr.proto); 8626 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 8627 ipv6_v->hdr.proto & ipv6_m->hdr.proto); 8628 /* Hop limit. */ 8629 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, 8630 ipv6_m->hdr.hop_limits); 8631 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, 8632 ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits); 8633 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 8634 !!(ipv6_m->has_frag_ext)); 8635 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 8636 !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext)); 8637 } 8638 8639 /** 8640 * Add IPV6 fragment extension item to matcher and to the value. 8641 * 8642 * @param[in, out] matcher 8643 * Flow matcher. 8644 * @param[in, out] key 8645 * Flow matcher value. 8646 * @param[in] item 8647 * Flow pattern to translate. 8648 * @param[in] inner 8649 * Item is inner pattern. 8650 */ 8651 static void 8652 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key, 8653 const struct rte_flow_item *item, 8654 int inner) 8655 { 8656 const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask; 8657 const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec; 8658 const struct rte_flow_item_ipv6_frag_ext nic_mask = { 8659 .hdr = { 8660 .next_header = 0xff, 8661 .frag_data = RTE_BE16(0xffff), 8662 }, 8663 }; 8664 void *headers_m; 8665 void *headers_v; 8666 8667 if (inner) { 8668 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8669 inner_headers); 8670 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 8671 } else { 8672 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8673 outer_headers); 8674 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8675 } 8676 /* IPv6 fragment extension item exists, so packet is IP fragment. */ 8677 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); 8678 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1); 8679 if (!ipv6_frag_ext_v) 8680 return; 8681 if (!ipv6_frag_ext_m) 8682 ipv6_frag_ext_m = &nic_mask; 8683 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 8684 ipv6_frag_ext_m->hdr.next_header); 8685 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 8686 ipv6_frag_ext_v->hdr.next_header & 8687 ipv6_frag_ext_m->hdr.next_header); 8688 } 8689 8690 /** 8691 * Add TCP item to matcher and to the value. 8692 * 8693 * @param[in, out] matcher 8694 * Flow matcher. 8695 * @param[in, out] key 8696 * Flow matcher value. 8697 * @param[in] item 8698 * Flow pattern to translate. 8699 * @param[in] inner 8700 * Item is inner pattern. 8701 */ 8702 static void 8703 flow_dv_translate_item_tcp(void *matcher, void *key, 8704 const struct rte_flow_item *item, 8705 int inner) 8706 { 8707 const struct rte_flow_item_tcp *tcp_m = item->mask; 8708 const struct rte_flow_item_tcp *tcp_v = item->spec; 8709 void *headers_m; 8710 void *headers_v; 8711 8712 if (inner) { 8713 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8714 inner_headers); 8715 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 8716 } else { 8717 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8718 outer_headers); 8719 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8720 } 8721 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 8722 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP); 8723 if (!tcp_v) 8724 return; 8725 if (!tcp_m) 8726 tcp_m = &rte_flow_item_tcp_mask; 8727 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport, 8728 rte_be_to_cpu_16(tcp_m->hdr.src_port)); 8729 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 8730 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port)); 8731 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport, 8732 rte_be_to_cpu_16(tcp_m->hdr.dst_port)); 8733 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 8734 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port)); 8735 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags, 8736 tcp_m->hdr.tcp_flags); 8737 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags, 8738 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags)); 8739 } 8740 8741 /** 8742 * Add ESP item to matcher and to the value. 8743 * 8744 * @param[in, out] matcher 8745 * Flow matcher. 8746 * @param[in, out] key 8747 * Flow matcher value. 8748 * @param[in] item 8749 * Flow pattern to translate. 8750 * @param[in] inner 8751 * Item is inner pattern. 8752 */ 8753 static void 8754 flow_dv_translate_item_esp(void *matcher, void *key, 8755 const struct rte_flow_item *item, 8756 int inner) 8757 { 8758 const struct rte_flow_item_esp *esp_m = item->mask; 8759 const struct rte_flow_item_esp *esp_v = item->spec; 8760 void *headers_m; 8761 void *headers_v; 8762 char *spi_m; 8763 char *spi_v; 8764 8765 if (inner) { 8766 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8767 inner_headers); 8768 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 8769 } else { 8770 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8771 outer_headers); 8772 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8773 } 8774 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 8775 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ESP); 8776 if (!esp_v) 8777 return; 8778 if (!esp_m) 8779 esp_m = &rte_flow_item_esp_mask; 8780 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 8781 headers_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 8782 if (inner) { 8783 spi_m = MLX5_ADDR_OF(fte_match_set_misc, headers_m, inner_esp_spi); 8784 spi_v = MLX5_ADDR_OF(fte_match_set_misc, headers_v, inner_esp_spi); 8785 } else { 8786 spi_m = MLX5_ADDR_OF(fte_match_set_misc, headers_m, outer_esp_spi); 8787 spi_v = MLX5_ADDR_OF(fte_match_set_misc, headers_v, outer_esp_spi); 8788 } 8789 *(uint32_t *)spi_m = esp_m->hdr.spi; 8790 *(uint32_t *)spi_v = esp_m->hdr.spi & esp_v->hdr.spi; 8791 } 8792 8793 /** 8794 * Add UDP item to matcher and to the value. 8795 * 8796 * @param[in, out] matcher 8797 * Flow matcher. 8798 * @param[in, out] key 8799 * Flow matcher value. 8800 * @param[in] item 8801 * Flow pattern to translate. 8802 * @param[in] inner 8803 * Item is inner pattern. 8804 */ 8805 static void 8806 flow_dv_translate_item_udp(void *matcher, void *key, 8807 const struct rte_flow_item *item, 8808 int inner) 8809 { 8810 const struct rte_flow_item_udp *udp_m = item->mask; 8811 const struct rte_flow_item_udp *udp_v = item->spec; 8812 void *headers_m; 8813 void *headers_v; 8814 8815 if (inner) { 8816 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8817 inner_headers); 8818 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 8819 } else { 8820 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 8821 outer_headers); 8822 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8823 } 8824 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 8825 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 8826 if (!udp_v) 8827 return; 8828 if (!udp_m) 8829 udp_m = &rte_flow_item_udp_mask; 8830 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport, 8831 rte_be_to_cpu_16(udp_m->hdr.src_port)); 8832 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 8833 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port)); 8834 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 8835 rte_be_to_cpu_16(udp_m->hdr.dst_port)); 8836 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 8837 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port)); 8838 } 8839 8840 /** 8841 * Add GRE optional Key item to matcher and to the value. 8842 * 8843 * @param[in, out] matcher 8844 * Flow matcher. 8845 * @param[in, out] key 8846 * Flow matcher value. 8847 * @param[in] item 8848 * Flow pattern to translate. 8849 * @param[in] inner 8850 * Item is inner pattern. 8851 */ 8852 static void 8853 flow_dv_translate_item_gre_key(void *matcher, void *key, 8854 const struct rte_flow_item *item) 8855 { 8856 const rte_be32_t *key_m = item->mask; 8857 const rte_be32_t *key_v = item->spec; 8858 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 8859 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 8860 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 8861 8862 /* GRE K bit must be on and should already be validated */ 8863 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1); 8864 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1); 8865 if (!key_v) 8866 return; 8867 if (!key_m) 8868 key_m = &gre_key_default_mask; 8869 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h, 8870 rte_be_to_cpu_32(*key_m) >> 8); 8871 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h, 8872 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8); 8873 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l, 8874 rte_be_to_cpu_32(*key_m) & 0xFF); 8875 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l, 8876 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF); 8877 } 8878 8879 /** 8880 * Add GRE item to matcher and to the value. 8881 * 8882 * @param[in, out] matcher 8883 * Flow matcher. 8884 * @param[in, out] key 8885 * Flow matcher value. 8886 * @param[in] item 8887 * Flow pattern to translate. 8888 * @param[in] pattern_flags 8889 * Accumulated pattern flags. 8890 */ 8891 static void 8892 flow_dv_translate_item_gre(void *matcher, void *key, 8893 const struct rte_flow_item *item, 8894 uint64_t pattern_flags) 8895 { 8896 static const struct rte_flow_item_gre empty_gre = {0,}; 8897 const struct rte_flow_item_gre *gre_m = item->mask; 8898 const struct rte_flow_item_gre *gre_v = item->spec; 8899 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 8900 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 8901 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 8902 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 8903 struct { 8904 union { 8905 __extension__ 8906 struct { 8907 uint16_t version:3; 8908 uint16_t rsvd0:9; 8909 uint16_t s_present:1; 8910 uint16_t k_present:1; 8911 uint16_t rsvd_bit1:1; 8912 uint16_t c_present:1; 8913 }; 8914 uint16_t value; 8915 }; 8916 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v; 8917 uint16_t protocol_m, protocol_v; 8918 8919 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 8920 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE); 8921 if (!gre_v) { 8922 gre_v = &empty_gre; 8923 gre_m = &empty_gre; 8924 } else { 8925 if (!gre_m) 8926 gre_m = &rte_flow_item_gre_mask; 8927 } 8928 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver); 8929 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver); 8930 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present, 8931 gre_crks_rsvd0_ver_m.c_present); 8932 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present, 8933 gre_crks_rsvd0_ver_v.c_present & 8934 gre_crks_rsvd0_ver_m.c_present); 8935 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 8936 gre_crks_rsvd0_ver_m.k_present); 8937 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 8938 gre_crks_rsvd0_ver_v.k_present & 8939 gre_crks_rsvd0_ver_m.k_present); 8940 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present, 8941 gre_crks_rsvd0_ver_m.s_present); 8942 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present, 8943 gre_crks_rsvd0_ver_v.s_present & 8944 gre_crks_rsvd0_ver_m.s_present); 8945 protocol_m = rte_be_to_cpu_16(gre_m->protocol); 8946 protocol_v = rte_be_to_cpu_16(gre_v->protocol); 8947 if (!protocol_m) { 8948 /* Force next protocol to prevent matchers duplication */ 8949 protocol_v = mlx5_translate_tunnel_etypes(pattern_flags); 8950 if (protocol_v) 8951 protocol_m = 0xFFFF; 8952 } 8953 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, protocol_m); 8954 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 8955 protocol_m & protocol_v); 8956 } 8957 8958 /** 8959 * Add GRE optional items to matcher and to the value. 8960 * 8961 * @param[in, out] matcher 8962 * Flow matcher. 8963 * @param[in, out] key 8964 * Flow matcher value. 8965 * @param[in] item 8966 * Flow pattern to translate. 8967 * @param[in] gre_item 8968 * Pointer to gre_item. 8969 * @param[in] pattern_flags 8970 * Accumulated pattern flags. 8971 */ 8972 static void 8973 flow_dv_translate_item_gre_option(void *matcher, void *key, 8974 const struct rte_flow_item *item, 8975 const struct rte_flow_item *gre_item, 8976 uint64_t pattern_flags) 8977 { 8978 const struct rte_flow_item_gre_opt *option_m = item->mask; 8979 const struct rte_flow_item_gre_opt *option_v = item->spec; 8980 const struct rte_flow_item_gre *gre_m = gre_item->mask; 8981 const struct rte_flow_item_gre *gre_v = gre_item->spec; 8982 static const struct rte_flow_item_gre empty_gre = {0}; 8983 struct rte_flow_item gre_key_item; 8984 uint16_t c_rsvd0_ver_m, c_rsvd0_ver_v; 8985 uint16_t protocol_m, protocol_v; 8986 void *misc5_m; 8987 void *misc5_v; 8988 8989 /* 8990 * If only match key field, keep using misc for matching. 8991 * If need to match checksum or sequence, using misc5 and do 8992 * not need using misc. 8993 */ 8994 if (!(option_m->sequence.sequence || 8995 option_m->checksum_rsvd.checksum)) { 8996 flow_dv_translate_item_gre(matcher, key, gre_item, 8997 pattern_flags); 8998 gre_key_item.spec = &option_v->key.key; 8999 gre_key_item.mask = &option_m->key.key; 9000 flow_dv_translate_item_gre_key(matcher, key, &gre_key_item); 9001 return; 9002 } 9003 if (!gre_v) { 9004 gre_v = &empty_gre; 9005 gre_m = &empty_gre; 9006 } else { 9007 if (!gre_m) 9008 gre_m = &rte_flow_item_gre_mask; 9009 } 9010 protocol_v = gre_v->protocol; 9011 protocol_m = gre_m->protocol; 9012 if (!protocol_m) { 9013 /* Force next protocol to prevent matchers duplication */ 9014 uint16_t ether_type = 9015 mlx5_translate_tunnel_etypes(pattern_flags); 9016 if (ether_type) { 9017 protocol_v = rte_be_to_cpu_16(ether_type); 9018 protocol_m = UINT16_MAX; 9019 } 9020 } 9021 c_rsvd0_ver_v = gre_v->c_rsvd0_ver; 9022 c_rsvd0_ver_m = gre_m->c_rsvd0_ver; 9023 if (option_m->sequence.sequence) { 9024 c_rsvd0_ver_v |= RTE_BE16(0x1000); 9025 c_rsvd0_ver_m |= RTE_BE16(0x1000); 9026 } 9027 if (option_m->key.key) { 9028 c_rsvd0_ver_v |= RTE_BE16(0x2000); 9029 c_rsvd0_ver_m |= RTE_BE16(0x2000); 9030 } 9031 if (option_m->checksum_rsvd.checksum) { 9032 c_rsvd0_ver_v |= RTE_BE16(0x8000); 9033 c_rsvd0_ver_m |= RTE_BE16(0x8000); 9034 } 9035 /* 9036 * Hardware parses GRE optional field into the fixed location, 9037 * do not need to adjust the tunnel dword indices. 9038 */ 9039 misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5); 9040 misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5); 9041 MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_0, 9042 rte_be_to_cpu_32((c_rsvd0_ver_v | protocol_v << 16) & 9043 (c_rsvd0_ver_m | protocol_m << 16))); 9044 MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_0, 9045 rte_be_to_cpu_32(c_rsvd0_ver_m | protocol_m << 16)); 9046 MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_1, 9047 rte_be_to_cpu_32(option_v->checksum_rsvd.checksum & 9048 option_m->checksum_rsvd.checksum)); 9049 MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_1, 9050 rte_be_to_cpu_32(option_m->checksum_rsvd.checksum)); 9051 MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_2, 9052 rte_be_to_cpu_32(option_v->key.key & option_m->key.key)); 9053 MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_2, 9054 rte_be_to_cpu_32(option_m->key.key)); 9055 MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_3, 9056 rte_be_to_cpu_32(option_v->sequence.sequence & 9057 option_m->sequence.sequence)); 9058 MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_3, 9059 rte_be_to_cpu_32(option_m->sequence.sequence)); 9060 } 9061 9062 /** 9063 * Add NVGRE item to matcher and to the value. 9064 * 9065 * @param[in, out] matcher 9066 * Flow matcher. 9067 * @param[in, out] key 9068 * Flow matcher value. 9069 * @param[in] item 9070 * Flow pattern to translate. 9071 * @param[in] pattern_flags 9072 * Accumulated pattern flags. 9073 */ 9074 static void 9075 flow_dv_translate_item_nvgre(void *matcher, void *key, 9076 const struct rte_flow_item *item, 9077 unsigned long pattern_flags) 9078 { 9079 const struct rte_flow_item_nvgre *nvgre_m = item->mask; 9080 const struct rte_flow_item_nvgre *nvgre_v = item->spec; 9081 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 9082 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 9083 const char *tni_flow_id_m; 9084 const char *tni_flow_id_v; 9085 char *gre_key_m; 9086 char *gre_key_v; 9087 int size; 9088 int i; 9089 9090 /* For NVGRE, GRE header fields must be set with defined values. */ 9091 const struct rte_flow_item_gre gre_spec = { 9092 .c_rsvd0_ver = RTE_BE16(0x2000), 9093 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB) 9094 }; 9095 const struct rte_flow_item_gre gre_mask = { 9096 .c_rsvd0_ver = RTE_BE16(0xB000), 9097 .protocol = RTE_BE16(UINT16_MAX), 9098 }; 9099 const struct rte_flow_item gre_item = { 9100 .spec = &gre_spec, 9101 .mask = &gre_mask, 9102 .last = NULL, 9103 }; 9104 flow_dv_translate_item_gre(matcher, key, &gre_item, pattern_flags); 9105 if (!nvgre_v) 9106 return; 9107 if (!nvgre_m) 9108 nvgre_m = &rte_flow_item_nvgre_mask; 9109 tni_flow_id_m = (const char *)nvgre_m->tni; 9110 tni_flow_id_v = (const char *)nvgre_v->tni; 9111 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id); 9112 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h); 9113 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h); 9114 memcpy(gre_key_m, tni_flow_id_m, size); 9115 for (i = 0; i < size; ++i) 9116 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i]; 9117 } 9118 9119 /** 9120 * Add VXLAN item to matcher and to the value. 9121 * 9122 * @param[in] dev 9123 * Pointer to the Ethernet device structure. 9124 * @param[in] attr 9125 * Flow rule attributes. 9126 * @param[in, out] matcher 9127 * Flow matcher. 9128 * @param[in, out] key 9129 * Flow matcher value. 9130 * @param[in] item 9131 * Flow pattern to translate. 9132 * @param[in] inner 9133 * Item is inner pattern. 9134 */ 9135 static void 9136 flow_dv_translate_item_vxlan(struct rte_eth_dev *dev, 9137 const struct rte_flow_attr *attr, 9138 void *matcher, void *key, 9139 const struct rte_flow_item *item, 9140 int inner) 9141 { 9142 const struct rte_flow_item_vxlan *vxlan_m = item->mask; 9143 const struct rte_flow_item_vxlan *vxlan_v = item->spec; 9144 void *headers_m; 9145 void *headers_v; 9146 void *misc5_m; 9147 void *misc5_v; 9148 uint32_t *tunnel_header_v; 9149 uint32_t *tunnel_header_m; 9150 uint16_t dport; 9151 struct mlx5_priv *priv = dev->data->dev_private; 9152 const struct rte_flow_item_vxlan nic_mask = { 9153 .vni = "\xff\xff\xff", 9154 .rsvd1 = 0xff, 9155 }; 9156 9157 if (inner) { 9158 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 9159 inner_headers); 9160 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 9161 } else { 9162 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 9163 outer_headers); 9164 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 9165 } 9166 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 9167 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 9168 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 9169 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 9170 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 9171 } 9172 dport = MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport); 9173 if (!vxlan_v) 9174 return; 9175 if (!vxlan_m) { 9176 if ((!attr->group && !priv->sh->tunnel_header_0_1) || 9177 (attr->group && !priv->sh->misc5_cap)) 9178 vxlan_m = &rte_flow_item_vxlan_mask; 9179 else 9180 vxlan_m = &nic_mask; 9181 } 9182 if ((priv->sh->steering_format_version == 9183 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 && 9184 dport != MLX5_UDP_PORT_VXLAN) || 9185 (!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) || 9186 ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) { 9187 void *misc_m; 9188 void *misc_v; 9189 char *vni_m; 9190 char *vni_v; 9191 int size; 9192 int i; 9193 misc_m = MLX5_ADDR_OF(fte_match_param, 9194 matcher, misc_parameters); 9195 misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 9196 size = sizeof(vxlan_m->vni); 9197 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni); 9198 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni); 9199 memcpy(vni_m, vxlan_m->vni, size); 9200 for (i = 0; i < size; ++i) 9201 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 9202 return; 9203 } 9204 misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5); 9205 misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5); 9206 tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5, 9207 misc5_v, 9208 tunnel_header_1); 9209 tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5, 9210 misc5_m, 9211 tunnel_header_1); 9212 *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) | 9213 (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 | 9214 (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16; 9215 if (*tunnel_header_v) 9216 *tunnel_header_m = vxlan_m->vni[0] | 9217 vxlan_m->vni[1] << 8 | 9218 vxlan_m->vni[2] << 16; 9219 else 9220 *tunnel_header_m = 0x0; 9221 *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24; 9222 if (vxlan_v->rsvd1 & vxlan_m->rsvd1) 9223 *tunnel_header_m |= vxlan_m->rsvd1 << 24; 9224 } 9225 9226 /** 9227 * Add VXLAN-GPE item to matcher and to the value. 9228 * 9229 * @param[in, out] matcher 9230 * Flow matcher. 9231 * @param[in, out] key 9232 * Flow matcher value. 9233 * @param[in] item 9234 * Flow pattern to translate. 9235 * @param[in] inner 9236 * Item is inner pattern. 9237 */ 9238 9239 static void 9240 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key, 9241 const struct rte_flow_item *item, 9242 const uint64_t pattern_flags) 9243 { 9244 static const struct rte_flow_item_vxlan_gpe dummy_vxlan_gpe_hdr = {0, }; 9245 const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask; 9246 const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec; 9247 /* The item was validated to be on the outer side */ 9248 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 9249 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 9250 void *misc_m = 9251 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3); 9252 void *misc_v = 9253 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 9254 char *vni_m = 9255 MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni); 9256 char *vni_v = 9257 MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni); 9258 int i, size = sizeof(vxlan_m->vni); 9259 uint8_t flags_m = 0xff; 9260 uint8_t flags_v = 0xc; 9261 uint8_t m_protocol, v_protocol; 9262 9263 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 9264 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 9265 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 9266 MLX5_UDP_PORT_VXLAN_GPE); 9267 } 9268 if (!vxlan_v) { 9269 vxlan_v = &dummy_vxlan_gpe_hdr; 9270 vxlan_m = &dummy_vxlan_gpe_hdr; 9271 } else { 9272 if (!vxlan_m) 9273 vxlan_m = &rte_flow_item_vxlan_gpe_mask; 9274 } 9275 memcpy(vni_m, vxlan_m->vni, size); 9276 for (i = 0; i < size; ++i) 9277 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 9278 if (vxlan_m->flags) { 9279 flags_m = vxlan_m->flags; 9280 flags_v = vxlan_v->flags; 9281 } 9282 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m); 9283 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v); 9284 m_protocol = vxlan_m->protocol; 9285 v_protocol = vxlan_v->protocol; 9286 if (!m_protocol) { 9287 /* Force next protocol to ensure next headers parsing. */ 9288 if (pattern_flags & MLX5_FLOW_LAYER_INNER_L2) 9289 v_protocol = RTE_VXLAN_GPE_TYPE_ETH; 9290 else if (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV4) 9291 v_protocol = RTE_VXLAN_GPE_TYPE_IPV4; 9292 else if (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6) 9293 v_protocol = RTE_VXLAN_GPE_TYPE_IPV6; 9294 if (v_protocol) 9295 m_protocol = 0xFF; 9296 } 9297 MLX5_SET(fte_match_set_misc3, misc_m, 9298 outer_vxlan_gpe_next_protocol, m_protocol); 9299 MLX5_SET(fte_match_set_misc3, misc_v, 9300 outer_vxlan_gpe_next_protocol, m_protocol & v_protocol); 9301 } 9302 9303 /** 9304 * Add Geneve item to matcher and to the value. 9305 * 9306 * @param[in, out] matcher 9307 * Flow matcher. 9308 * @param[in, out] key 9309 * Flow matcher value. 9310 * @param[in] item 9311 * Flow pattern to translate. 9312 * @param[in] inner 9313 * Item is inner pattern. 9314 */ 9315 9316 static void 9317 flow_dv_translate_item_geneve(void *matcher, void *key, 9318 const struct rte_flow_item *item, 9319 uint64_t pattern_flags) 9320 { 9321 static const struct rte_flow_item_geneve empty_geneve = {0,}; 9322 const struct rte_flow_item_geneve *geneve_m = item->mask; 9323 const struct rte_flow_item_geneve *geneve_v = item->spec; 9324 /* GENEVE flow item validation allows single tunnel item */ 9325 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 9326 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 9327 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 9328 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 9329 uint16_t gbhdr_m; 9330 uint16_t gbhdr_v; 9331 char *vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni); 9332 char *vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni); 9333 size_t size = sizeof(geneve_m->vni), i; 9334 uint16_t protocol_m, protocol_v; 9335 9336 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 9337 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 9338 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 9339 MLX5_UDP_PORT_GENEVE); 9340 } 9341 if (!geneve_v) { 9342 geneve_v = &empty_geneve; 9343 geneve_m = &empty_geneve; 9344 } else { 9345 if (!geneve_m) 9346 geneve_m = &rte_flow_item_geneve_mask; 9347 } 9348 memcpy(vni_m, geneve_m->vni, size); 9349 for (i = 0; i < size; ++i) 9350 vni_v[i] = vni_m[i] & geneve_v->vni[i]; 9351 gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0); 9352 gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0); 9353 MLX5_SET(fte_match_set_misc, misc_m, geneve_oam, 9354 MLX5_GENEVE_OAMF_VAL(gbhdr_m)); 9355 MLX5_SET(fte_match_set_misc, misc_v, geneve_oam, 9356 MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m)); 9357 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len, 9358 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); 9359 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, 9360 MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) & 9361 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); 9362 protocol_m = rte_be_to_cpu_16(geneve_m->protocol); 9363 protocol_v = rte_be_to_cpu_16(geneve_v->protocol); 9364 if (!protocol_m) { 9365 /* Force next protocol to prevent matchers duplication */ 9366 protocol_v = mlx5_translate_tunnel_etypes(pattern_flags); 9367 if (protocol_v) 9368 protocol_m = 0xFFFF; 9369 } 9370 MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type, protocol_m); 9371 MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type, 9372 protocol_m & protocol_v); 9373 } 9374 9375 /** 9376 * Create Geneve TLV option resource. 9377 * 9378 * @param dev[in, out] 9379 * Pointer to rte_eth_dev structure. 9380 * @param[in, out] tag_be24 9381 * Tag value in big endian then R-shift 8. 9382 * @parm[in, out] dev_flow 9383 * Pointer to the dev_flow. 9384 * @param[out] error 9385 * pointer to error structure. 9386 * 9387 * @return 9388 * 0 on success otherwise -errno and errno is set. 9389 */ 9390 9391 int 9392 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev, 9393 const struct rte_flow_item *item, 9394 struct rte_flow_error *error) 9395 { 9396 struct mlx5_priv *priv = dev->data->dev_private; 9397 struct mlx5_dev_ctx_shared *sh = priv->sh; 9398 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource = 9399 sh->geneve_tlv_option_resource; 9400 struct mlx5_devx_obj *obj; 9401 const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec; 9402 int ret = 0; 9403 9404 if (!geneve_opt_v) 9405 return -1; 9406 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 9407 if (geneve_opt_resource != NULL) { 9408 if (geneve_opt_resource->option_class == 9409 geneve_opt_v->option_class && 9410 geneve_opt_resource->option_type == 9411 geneve_opt_v->option_type && 9412 geneve_opt_resource->length == 9413 geneve_opt_v->option_len) { 9414 /* We already have GENEVE TLV option obj allocated. */ 9415 __atomic_fetch_add(&geneve_opt_resource->refcnt, 1, 9416 __ATOMIC_RELAXED); 9417 } else { 9418 ret = rte_flow_error_set(error, ENOMEM, 9419 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9420 "Only one GENEVE TLV option supported"); 9421 goto exit; 9422 } 9423 } else { 9424 /* Create a GENEVE TLV object and resource. */ 9425 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->cdev->ctx, 9426 geneve_opt_v->option_class, 9427 geneve_opt_v->option_type, 9428 geneve_opt_v->option_len); 9429 if (!obj) { 9430 ret = rte_flow_error_set(error, ENODATA, 9431 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9432 "Failed to create GENEVE TLV Devx object"); 9433 goto exit; 9434 } 9435 sh->geneve_tlv_option_resource = 9436 mlx5_malloc(MLX5_MEM_ZERO, 9437 sizeof(*geneve_opt_resource), 9438 0, SOCKET_ID_ANY); 9439 if (!sh->geneve_tlv_option_resource) { 9440 claim_zero(mlx5_devx_cmd_destroy(obj)); 9441 ret = rte_flow_error_set(error, ENOMEM, 9442 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9443 "GENEVE TLV object memory allocation failed"); 9444 goto exit; 9445 } 9446 geneve_opt_resource = sh->geneve_tlv_option_resource; 9447 geneve_opt_resource->obj = obj; 9448 geneve_opt_resource->option_class = geneve_opt_v->option_class; 9449 geneve_opt_resource->option_type = geneve_opt_v->option_type; 9450 geneve_opt_resource->length = geneve_opt_v->option_len; 9451 __atomic_store_n(&geneve_opt_resource->refcnt, 1, 9452 __ATOMIC_RELAXED); 9453 } 9454 exit: 9455 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 9456 return ret; 9457 } 9458 9459 /** 9460 * Add Geneve TLV option item to matcher. 9461 * 9462 * @param[in, out] dev 9463 * Pointer to rte_eth_dev structure. 9464 * @param[in, out] matcher 9465 * Flow matcher. 9466 * @param[in, out] key 9467 * Flow matcher value. 9468 * @param[in] item 9469 * Flow pattern to translate. 9470 * @param[out] error 9471 * Pointer to error structure. 9472 */ 9473 static int 9474 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher, 9475 void *key, const struct rte_flow_item *item, 9476 struct rte_flow_error *error) 9477 { 9478 const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask; 9479 const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec; 9480 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 9481 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 9482 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 9483 misc_parameters_3); 9484 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 9485 rte_be32_t opt_data_key = 0, opt_data_mask = 0; 9486 int ret = 0; 9487 9488 if (!geneve_opt_v) 9489 return -1; 9490 if (!geneve_opt_m) 9491 geneve_opt_m = &rte_flow_item_geneve_opt_mask; 9492 ret = flow_dev_geneve_tlv_option_resource_register(dev, item, 9493 error); 9494 if (ret) { 9495 DRV_LOG(ERR, "Failed to create geneve_tlv_obj"); 9496 return ret; 9497 } 9498 /* 9499 * Set the option length in GENEVE header if not requested. 9500 * The GENEVE TLV option length is expressed by the option length field 9501 * in the GENEVE header. 9502 * If the option length was not requested but the GENEVE TLV option item 9503 * is present we set the option length field implicitly. 9504 */ 9505 if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) { 9506 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len, 9507 MLX5_GENEVE_OPTLEN_MASK); 9508 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, 9509 geneve_opt_v->option_len + 1); 9510 } 9511 MLX5_SET(fte_match_set_misc, misc_m, geneve_tlv_option_0_exist, 1); 9512 MLX5_SET(fte_match_set_misc, misc_v, geneve_tlv_option_0_exist, 1); 9513 /* Set the data. */ 9514 if (geneve_opt_v->data) { 9515 memcpy(&opt_data_key, geneve_opt_v->data, 9516 RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4), 9517 sizeof(opt_data_key))); 9518 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <= 9519 sizeof(opt_data_key)); 9520 memcpy(&opt_data_mask, geneve_opt_m->data, 9521 RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4), 9522 sizeof(opt_data_mask))); 9523 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <= 9524 sizeof(opt_data_mask)); 9525 MLX5_SET(fte_match_set_misc3, misc3_m, 9526 geneve_tlv_option_0_data, 9527 rte_be_to_cpu_32(opt_data_mask)); 9528 MLX5_SET(fte_match_set_misc3, misc3_v, 9529 geneve_tlv_option_0_data, 9530 rte_be_to_cpu_32(opt_data_key & opt_data_mask)); 9531 } 9532 return ret; 9533 } 9534 9535 /** 9536 * Add MPLS item to matcher and to the value. 9537 * 9538 * @param[in, out] matcher 9539 * Flow matcher. 9540 * @param[in, out] key 9541 * Flow matcher value. 9542 * @param[in] item 9543 * Flow pattern to translate. 9544 * @param[in] prev_layer 9545 * The protocol layer indicated in previous item. 9546 * @param[in] inner 9547 * Item is inner pattern. 9548 */ 9549 static void 9550 flow_dv_translate_item_mpls(void *matcher, void *key, 9551 const struct rte_flow_item *item, 9552 uint64_t prev_layer, 9553 int inner) 9554 { 9555 const uint32_t *in_mpls_m = item->mask; 9556 const uint32_t *in_mpls_v = item->spec; 9557 uint32_t *out_mpls_m = 0; 9558 uint32_t *out_mpls_v = 0; 9559 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 9560 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 9561 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher, 9562 misc_parameters_2); 9563 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 9564 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 9565 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 9566 9567 switch (prev_layer) { 9568 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 9569 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 9570 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 9571 0xffff); 9572 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 9573 MLX5_UDP_PORT_MPLS); 9574 } 9575 break; 9576 case MLX5_FLOW_LAYER_GRE: 9577 /* Fall-through. */ 9578 case MLX5_FLOW_LAYER_GRE_KEY: 9579 if (!MLX5_GET16(fte_match_set_misc, misc_v, gre_protocol)) { 9580 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 9581 0xffff); 9582 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 9583 RTE_ETHER_TYPE_MPLS); 9584 } 9585 break; 9586 default: 9587 break; 9588 } 9589 if (!in_mpls_v) 9590 return; 9591 if (!in_mpls_m) 9592 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask; 9593 switch (prev_layer) { 9594 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 9595 out_mpls_m = 9596 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 9597 outer_first_mpls_over_udp); 9598 out_mpls_v = 9599 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 9600 outer_first_mpls_over_udp); 9601 break; 9602 case MLX5_FLOW_LAYER_GRE: 9603 out_mpls_m = 9604 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 9605 outer_first_mpls_over_gre); 9606 out_mpls_v = 9607 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 9608 outer_first_mpls_over_gre); 9609 break; 9610 default: 9611 /* Inner MPLS not over GRE is not supported. */ 9612 if (!inner) { 9613 out_mpls_m = 9614 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 9615 misc2_m, 9616 outer_first_mpls); 9617 out_mpls_v = 9618 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 9619 misc2_v, 9620 outer_first_mpls); 9621 } 9622 break; 9623 } 9624 if (out_mpls_m && out_mpls_v) { 9625 *out_mpls_m = *in_mpls_m; 9626 *out_mpls_v = *in_mpls_v & *in_mpls_m; 9627 } 9628 } 9629 9630 /** 9631 * Add metadata register item to matcher 9632 * 9633 * @param[in, out] matcher 9634 * Flow matcher. 9635 * @param[in, out] key 9636 * Flow matcher value. 9637 * @param[in] reg_type 9638 * Type of device metadata register 9639 * @param[in] value 9640 * Register value 9641 * @param[in] mask 9642 * Register mask 9643 */ 9644 static void 9645 flow_dv_match_meta_reg(void *matcher, void *key, 9646 enum modify_reg reg_type, 9647 uint32_t data, uint32_t mask) 9648 { 9649 void *misc2_m = 9650 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); 9651 void *misc2_v = 9652 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 9653 uint32_t temp; 9654 9655 data &= mask; 9656 switch (reg_type) { 9657 case REG_A: 9658 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask); 9659 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data); 9660 break; 9661 case REG_B: 9662 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask); 9663 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data); 9664 break; 9665 case REG_C_0: 9666 /* 9667 * The metadata register C0 field might be divided into 9668 * source vport index and META item value, we should set 9669 * this field according to specified mask, not as whole one. 9670 */ 9671 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0); 9672 temp |= mask; 9673 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp); 9674 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0); 9675 temp &= ~mask; 9676 temp |= data; 9677 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp); 9678 break; 9679 case REG_C_1: 9680 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask); 9681 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data); 9682 break; 9683 case REG_C_2: 9684 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask); 9685 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data); 9686 break; 9687 case REG_C_3: 9688 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask); 9689 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data); 9690 break; 9691 case REG_C_4: 9692 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask); 9693 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data); 9694 break; 9695 case REG_C_5: 9696 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask); 9697 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data); 9698 break; 9699 case REG_C_6: 9700 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask); 9701 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data); 9702 break; 9703 case REG_C_7: 9704 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask); 9705 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data); 9706 break; 9707 default: 9708 MLX5_ASSERT(false); 9709 break; 9710 } 9711 } 9712 9713 /** 9714 * Add MARK item to matcher 9715 * 9716 * @param[in] dev 9717 * The device to configure through. 9718 * @param[in, out] matcher 9719 * Flow matcher. 9720 * @param[in, out] key 9721 * Flow matcher value. 9722 * @param[in] item 9723 * Flow pattern to translate. 9724 */ 9725 static void 9726 flow_dv_translate_item_mark(struct rte_eth_dev *dev, 9727 void *matcher, void *key, 9728 const struct rte_flow_item *item) 9729 { 9730 struct mlx5_priv *priv = dev->data->dev_private; 9731 const struct rte_flow_item_mark *mark; 9732 uint32_t value; 9733 uint32_t mask; 9734 9735 mark = item->mask ? (const void *)item->mask : 9736 &rte_flow_item_mark_mask; 9737 mask = mark->id & priv->sh->dv_mark_mask; 9738 mark = (const void *)item->spec; 9739 MLX5_ASSERT(mark); 9740 value = mark->id & priv->sh->dv_mark_mask & mask; 9741 if (mask) { 9742 enum modify_reg reg; 9743 9744 /* Get the metadata register index for the mark. */ 9745 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL); 9746 MLX5_ASSERT(reg > 0); 9747 if (reg == REG_C_0) { 9748 struct mlx5_priv *priv = dev->data->dev_private; 9749 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 9750 uint32_t shl_c0 = rte_bsf32(msk_c0); 9751 9752 mask &= msk_c0; 9753 mask <<= shl_c0; 9754 value <<= shl_c0; 9755 } 9756 flow_dv_match_meta_reg(matcher, key, reg, value, mask); 9757 } 9758 } 9759 9760 /** 9761 * Add META item to matcher 9762 * 9763 * @param[in] dev 9764 * The devich to configure through. 9765 * @param[in, out] matcher 9766 * Flow matcher. 9767 * @param[in, out] key 9768 * Flow matcher value. 9769 * @param[in] attr 9770 * Attributes of flow that includes this item. 9771 * @param[in] item 9772 * Flow pattern to translate. 9773 */ 9774 static void 9775 flow_dv_translate_item_meta(struct rte_eth_dev *dev, 9776 void *matcher, void *key, 9777 const struct rte_flow_attr *attr, 9778 const struct rte_flow_item *item) 9779 { 9780 const struct rte_flow_item_meta *meta_m; 9781 const struct rte_flow_item_meta *meta_v; 9782 9783 meta_m = (const void *)item->mask; 9784 if (!meta_m) 9785 meta_m = &rte_flow_item_meta_mask; 9786 meta_v = (const void *)item->spec; 9787 if (meta_v) { 9788 int reg; 9789 uint32_t value = meta_v->data; 9790 uint32_t mask = meta_m->data; 9791 9792 reg = flow_dv_get_metadata_reg(dev, attr, NULL); 9793 if (reg < 0) 9794 return; 9795 MLX5_ASSERT(reg != REG_NON); 9796 if (reg == REG_C_0) { 9797 struct mlx5_priv *priv = dev->data->dev_private; 9798 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 9799 uint32_t shl_c0 = rte_bsf32(msk_c0); 9800 9801 mask &= msk_c0; 9802 mask <<= shl_c0; 9803 value <<= shl_c0; 9804 } 9805 flow_dv_match_meta_reg(matcher, key, reg, value, mask); 9806 } 9807 } 9808 9809 /** 9810 * Add vport metadata Reg C0 item to matcher 9811 * 9812 * @param[in, out] matcher 9813 * Flow matcher. 9814 * @param[in, out] key 9815 * Flow matcher value. 9816 * @param[in] reg 9817 * Flow pattern to translate. 9818 */ 9819 static void 9820 flow_dv_translate_item_meta_vport(void *matcher, void *key, 9821 uint32_t value, uint32_t mask) 9822 { 9823 flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask); 9824 } 9825 9826 /** 9827 * Add tag item to matcher 9828 * 9829 * @param[in] dev 9830 * The devich to configure through. 9831 * @param[in, out] matcher 9832 * Flow matcher. 9833 * @param[in, out] key 9834 * Flow matcher value. 9835 * @param[in] item 9836 * Flow pattern to translate. 9837 */ 9838 static void 9839 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev, 9840 void *matcher, void *key, 9841 const struct rte_flow_item *item) 9842 { 9843 const struct mlx5_rte_flow_item_tag *tag_v = item->spec; 9844 const struct mlx5_rte_flow_item_tag *tag_m = item->mask; 9845 uint32_t mask, value; 9846 9847 MLX5_ASSERT(tag_v); 9848 value = tag_v->data; 9849 mask = tag_m ? tag_m->data : UINT32_MAX; 9850 if (tag_v->id == REG_C_0) { 9851 struct mlx5_priv *priv = dev->data->dev_private; 9852 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 9853 uint32_t shl_c0 = rte_bsf32(msk_c0); 9854 9855 mask &= msk_c0; 9856 mask <<= shl_c0; 9857 value <<= shl_c0; 9858 } 9859 flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask); 9860 } 9861 9862 /** 9863 * Add TAG item to matcher 9864 * 9865 * @param[in] dev 9866 * The devich to configure through. 9867 * @param[in, out] matcher 9868 * Flow matcher. 9869 * @param[in, out] key 9870 * Flow matcher value. 9871 * @param[in] item 9872 * Flow pattern to translate. 9873 */ 9874 static void 9875 flow_dv_translate_item_tag(struct rte_eth_dev *dev, 9876 void *matcher, void *key, 9877 const struct rte_flow_item *item) 9878 { 9879 const struct rte_flow_item_tag *tag_v = item->spec; 9880 const struct rte_flow_item_tag *tag_m = item->mask; 9881 enum modify_reg reg; 9882 9883 MLX5_ASSERT(tag_v); 9884 tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask; 9885 /* Get the metadata register index for the tag. */ 9886 reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL); 9887 MLX5_ASSERT(reg > 0); 9888 flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data); 9889 } 9890 9891 /** 9892 * Add source vport match to the specified matcher. 9893 * 9894 * @param[in, out] matcher 9895 * Flow matcher. 9896 * @param[in, out] key 9897 * Flow matcher value. 9898 * @param[in] port 9899 * Source vport value to match 9900 * @param[in] mask 9901 * Mask 9902 */ 9903 static void 9904 flow_dv_translate_item_source_vport(void *matcher, void *key, 9905 int16_t port, uint16_t mask) 9906 { 9907 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 9908 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 9909 9910 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); 9911 MLX5_SET(fte_match_set_misc, misc_v, source_port, port); 9912 } 9913 9914 /** 9915 * Translate port-id item to eswitch match on port-id. 9916 * 9917 * @param[in] dev 9918 * The devich to configure through. 9919 * @param[in, out] matcher 9920 * Flow matcher. 9921 * @param[in, out] key 9922 * Flow matcher value. 9923 * @param[in] item 9924 * Flow pattern to translate. 9925 * @param[in] 9926 * Flow attributes. 9927 * 9928 * @return 9929 * 0 on success, a negative errno value otherwise. 9930 */ 9931 static int 9932 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, 9933 void *key, const struct rte_flow_item *item, 9934 const struct rte_flow_attr *attr) 9935 { 9936 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; 9937 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; 9938 struct mlx5_priv *priv; 9939 uint16_t mask, id; 9940 9941 if (pid_v && pid_v->id == MLX5_PORT_ESW_MGR) { 9942 flow_dv_translate_item_source_vport(matcher, key, 9943 flow_dv_get_esw_manager_vport_id(dev), 0xffff); 9944 return 0; 9945 } 9946 mask = pid_m ? pid_m->id : 0xffff; 9947 id = pid_v ? pid_v->id : dev->data->port_id; 9948 priv = mlx5_port_to_eswitch_info(id, item == NULL); 9949 if (!priv) 9950 return -rte_errno; 9951 /* 9952 * Translate to vport field or to metadata, depending on mode. 9953 * Kernel can use either misc.source_port or half of C0 metadata 9954 * register. 9955 */ 9956 if (priv->vport_meta_mask) { 9957 /* 9958 * Provide the hint for SW steering library 9959 * to insert the flow into ingress domain and 9960 * save the extra vport match. 9961 */ 9962 if (mask == 0xffff && priv->vport_id == 0xffff && 9963 priv->pf_bond < 0 && attr->transfer) 9964 flow_dv_translate_item_source_vport 9965 (matcher, key, priv->vport_id, mask); 9966 /* 9967 * We should always set the vport metadata register, 9968 * otherwise the SW steering library can drop 9969 * the rule if wire vport metadata value is not zero, 9970 * it depends on kernel configuration. 9971 */ 9972 flow_dv_translate_item_meta_vport(matcher, key, 9973 priv->vport_meta_tag, 9974 priv->vport_meta_mask); 9975 } else { 9976 flow_dv_translate_item_source_vport(matcher, key, 9977 priv->vport_id, mask); 9978 } 9979 return 0; 9980 } 9981 9982 /** 9983 * Add ICMP6 item to matcher and to the value. 9984 * 9985 * @param[in, out] matcher 9986 * Flow matcher. 9987 * @param[in, out] key 9988 * Flow matcher value. 9989 * @param[in] item 9990 * Flow pattern to translate. 9991 * @param[in] inner 9992 * Item is inner pattern. 9993 */ 9994 static void 9995 flow_dv_translate_item_icmp6(void *matcher, void *key, 9996 const struct rte_flow_item *item, 9997 int inner) 9998 { 9999 const struct rte_flow_item_icmp6 *icmp6_m = item->mask; 10000 const struct rte_flow_item_icmp6 *icmp6_v = item->spec; 10001 void *headers_m; 10002 void *headers_v; 10003 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 10004 misc_parameters_3); 10005 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 10006 if (inner) { 10007 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 10008 inner_headers); 10009 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 10010 } else { 10011 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 10012 outer_headers); 10013 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 10014 } 10015 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 10016 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6); 10017 if (!icmp6_v) 10018 return; 10019 if (!icmp6_m) 10020 icmp6_m = &rte_flow_item_icmp6_mask; 10021 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type); 10022 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type, 10023 icmp6_v->type & icmp6_m->type); 10024 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code); 10025 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code, 10026 icmp6_v->code & icmp6_m->code); 10027 } 10028 10029 /** 10030 * Add ICMP item to matcher and to the value. 10031 * 10032 * @param[in, out] matcher 10033 * Flow matcher. 10034 * @param[in, out] key 10035 * Flow matcher value. 10036 * @param[in] item 10037 * Flow pattern to translate. 10038 * @param[in] inner 10039 * Item is inner pattern. 10040 */ 10041 static void 10042 flow_dv_translate_item_icmp(void *matcher, void *key, 10043 const struct rte_flow_item *item, 10044 int inner) 10045 { 10046 const struct rte_flow_item_icmp *icmp_m = item->mask; 10047 const struct rte_flow_item_icmp *icmp_v = item->spec; 10048 uint32_t icmp_header_data_m = 0; 10049 uint32_t icmp_header_data_v = 0; 10050 void *headers_m; 10051 void *headers_v; 10052 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 10053 misc_parameters_3); 10054 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 10055 if (inner) { 10056 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 10057 inner_headers); 10058 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 10059 } else { 10060 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 10061 outer_headers); 10062 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 10063 } 10064 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 10065 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP); 10066 if (!icmp_v) 10067 return; 10068 if (!icmp_m) 10069 icmp_m = &rte_flow_item_icmp_mask; 10070 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type, 10071 icmp_m->hdr.icmp_type); 10072 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type, 10073 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type); 10074 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code, 10075 icmp_m->hdr.icmp_code); 10076 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code, 10077 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code); 10078 icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb); 10079 icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16; 10080 if (icmp_header_data_m) { 10081 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb); 10082 icmp_header_data_v |= 10083 rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16; 10084 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data, 10085 icmp_header_data_m); 10086 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data, 10087 icmp_header_data_v & icmp_header_data_m); 10088 } 10089 } 10090 10091 /** 10092 * Add GTP item to matcher and to the value. 10093 * 10094 * @param[in, out] matcher 10095 * Flow matcher. 10096 * @param[in, out] key 10097 * Flow matcher value. 10098 * @param[in] item 10099 * Flow pattern to translate. 10100 * @param[in] inner 10101 * Item is inner pattern. 10102 */ 10103 static void 10104 flow_dv_translate_item_gtp(void *matcher, void *key, 10105 const struct rte_flow_item *item, int inner) 10106 { 10107 const struct rte_flow_item_gtp *gtp_m = item->mask; 10108 const struct rte_flow_item_gtp *gtp_v = item->spec; 10109 void *headers_m; 10110 void *headers_v; 10111 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 10112 misc_parameters_3); 10113 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 10114 uint16_t dport = RTE_GTPU_UDP_PORT; 10115 10116 if (inner) { 10117 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 10118 inner_headers); 10119 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 10120 } else { 10121 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 10122 outer_headers); 10123 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 10124 } 10125 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 10126 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 10127 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 10128 } 10129 if (!gtp_v) 10130 return; 10131 if (!gtp_m) 10132 gtp_m = &rte_flow_item_gtp_mask; 10133 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, 10134 gtp_m->v_pt_rsv_flags); 10135 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, 10136 gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags); 10137 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type); 10138 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type, 10139 gtp_v->msg_type & gtp_m->msg_type); 10140 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid, 10141 rte_be_to_cpu_32(gtp_m->teid)); 10142 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid, 10143 rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid)); 10144 } 10145 10146 /** 10147 * Add GTP PSC item to matcher. 10148 * 10149 * @param[in, out] matcher 10150 * Flow matcher. 10151 * @param[in, out] key 10152 * Flow matcher value. 10153 * @param[in] item 10154 * Flow pattern to translate. 10155 */ 10156 static int 10157 flow_dv_translate_item_gtp_psc(void *matcher, void *key, 10158 const struct rte_flow_item *item) 10159 { 10160 const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask; 10161 const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec; 10162 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 10163 misc_parameters_3); 10164 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 10165 union { 10166 uint32_t w32; 10167 struct { 10168 uint16_t seq_num; 10169 uint8_t npdu_num; 10170 uint8_t next_ext_header_type; 10171 }; 10172 } dw_2; 10173 uint8_t gtp_flags; 10174 10175 /* Always set E-flag match on one, regardless of GTP item settings. */ 10176 gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags); 10177 gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG; 10178 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags); 10179 gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags); 10180 gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG; 10181 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags); 10182 /*Set next extension header type. */ 10183 dw_2.seq_num = 0; 10184 dw_2.npdu_num = 0; 10185 dw_2.next_ext_header_type = 0xff; 10186 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2, 10187 rte_cpu_to_be_32(dw_2.w32)); 10188 dw_2.seq_num = 0; 10189 dw_2.npdu_num = 0; 10190 dw_2.next_ext_header_type = 0x85; 10191 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2, 10192 rte_cpu_to_be_32(dw_2.w32)); 10193 if (gtp_psc_v) { 10194 union { 10195 uint32_t w32; 10196 struct { 10197 uint8_t len; 10198 uint8_t type_flags; 10199 uint8_t qfi; 10200 uint8_t reserved; 10201 }; 10202 } dw_0; 10203 10204 /*Set extension header PDU type and Qos. */ 10205 if (!gtp_psc_m) 10206 gtp_psc_m = &rte_flow_item_gtp_psc_mask; 10207 dw_0.w32 = 0; 10208 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->hdr.type); 10209 dw_0.qfi = gtp_psc_m->hdr.qfi; 10210 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0, 10211 rte_cpu_to_be_32(dw_0.w32)); 10212 dw_0.w32 = 0; 10213 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->hdr.type & 10214 gtp_psc_m->hdr.type); 10215 dw_0.qfi = gtp_psc_v->hdr.qfi & gtp_psc_m->hdr.qfi; 10216 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0, 10217 rte_cpu_to_be_32(dw_0.w32)); 10218 } 10219 return 0; 10220 } 10221 10222 /** 10223 * Add eCPRI item to matcher and to the value. 10224 * 10225 * @param[in] dev 10226 * The devich to configure through. 10227 * @param[in, out] matcher 10228 * Flow matcher. 10229 * @param[in, out] key 10230 * Flow matcher value. 10231 * @param[in] item 10232 * Flow pattern to translate. 10233 * @param[in] last_item 10234 * Last item flags. 10235 */ 10236 static void 10237 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, 10238 void *key, const struct rte_flow_item *item, 10239 uint64_t last_item) 10240 { 10241 struct mlx5_priv *priv = dev->data->dev_private; 10242 const struct rte_flow_item_ecpri *ecpri_m = item->mask; 10243 const struct rte_flow_item_ecpri *ecpri_v = item->spec; 10244 struct rte_ecpri_common_hdr common; 10245 void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher, 10246 misc_parameters_4); 10247 void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4); 10248 uint32_t *samples; 10249 void *dw_m; 10250 void *dw_v; 10251 10252 /* 10253 * In case of eCPRI over Ethernet, if EtherType is not specified, 10254 * match on eCPRI EtherType implicitly. 10255 */ 10256 if (last_item & MLX5_FLOW_LAYER_OUTER_L2) { 10257 void *hdrs_m, *hdrs_v, *l2m, *l2v; 10258 10259 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 10260 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 10261 l2m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, ethertype); 10262 l2v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype); 10263 if (*(uint16_t *)l2m == 0 && *(uint16_t *)l2v == 0) { 10264 *(uint16_t *)l2m = UINT16_MAX; 10265 *(uint16_t *)l2v = RTE_BE16(RTE_ETHER_TYPE_ECPRI); 10266 } 10267 } 10268 if (!ecpri_v) 10269 return; 10270 if (!ecpri_m) 10271 ecpri_m = &rte_flow_item_ecpri_mask; 10272 /* 10273 * Maximal four DW samples are supported in a single matching now. 10274 * Two are used now for a eCPRI matching: 10275 * 1. Type: one byte, mask should be 0x00ff0000 in network order 10276 * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000 10277 * if any. 10278 */ 10279 if (!ecpri_m->hdr.common.u32) 10280 return; 10281 samples = priv->sh->ecpri_parser.ids; 10282 /* Need to take the whole DW as the mask to fill the entry. */ 10283 dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m, 10284 prog_sample_field_value_0); 10285 dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v, 10286 prog_sample_field_value_0); 10287 /* Already big endian (network order) in the header. */ 10288 *(uint32_t *)dw_m = ecpri_m->hdr.common.u32; 10289 *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32; 10290 /* Sample#0, used for matching type, offset 0. */ 10291 MLX5_SET(fte_match_set_misc4, misc4_m, 10292 prog_sample_field_id_0, samples[0]); 10293 /* It makes no sense to set the sample ID in the mask field. */ 10294 MLX5_SET(fte_match_set_misc4, misc4_v, 10295 prog_sample_field_id_0, samples[0]); 10296 /* 10297 * Checking if message body part needs to be matched. 10298 * Some wildcard rules only matching type field should be supported. 10299 */ 10300 if (ecpri_m->hdr.dummy[0]) { 10301 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32); 10302 switch (common.type) { 10303 case RTE_ECPRI_MSG_TYPE_IQ_DATA: 10304 case RTE_ECPRI_MSG_TYPE_RTC_CTRL: 10305 case RTE_ECPRI_MSG_TYPE_DLY_MSR: 10306 dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m, 10307 prog_sample_field_value_1); 10308 dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v, 10309 prog_sample_field_value_1); 10310 *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0]; 10311 *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] & 10312 ecpri_m->hdr.dummy[0]; 10313 /* Sample#1, to match message body, offset 4. */ 10314 MLX5_SET(fte_match_set_misc4, misc4_m, 10315 prog_sample_field_id_1, samples[1]); 10316 MLX5_SET(fte_match_set_misc4, misc4_v, 10317 prog_sample_field_id_1, samples[1]); 10318 break; 10319 default: 10320 /* Others, do not match any sample ID. */ 10321 break; 10322 } 10323 } 10324 } 10325 10326 /* 10327 * Add connection tracking status item to matcher 10328 * 10329 * @param[in] dev 10330 * The devich to configure through. 10331 * @param[in, out] matcher 10332 * Flow matcher. 10333 * @param[in, out] key 10334 * Flow matcher value. 10335 * @param[in] item 10336 * Flow pattern to translate. 10337 */ 10338 static void 10339 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev, 10340 void *matcher, void *key, 10341 const struct rte_flow_item *item) 10342 { 10343 uint32_t reg_value = 0; 10344 int reg_id; 10345 /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */ 10346 uint32_t reg_mask = 0; 10347 const struct rte_flow_item_conntrack *spec = item->spec; 10348 const struct rte_flow_item_conntrack *mask = item->mask; 10349 uint32_t flags; 10350 struct rte_flow_error error; 10351 10352 if (!mask) 10353 mask = &rte_flow_item_conntrack_mask; 10354 if (!spec || !mask->flags) 10355 return; 10356 flags = spec->flags & mask->flags; 10357 /* The conflict should be checked in the validation. */ 10358 if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) 10359 reg_value |= MLX5_CT_SYNDROME_VALID; 10360 if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED) 10361 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE; 10362 if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) 10363 reg_value |= MLX5_CT_SYNDROME_INVALID; 10364 if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED) 10365 reg_value |= MLX5_CT_SYNDROME_TRAP; 10366 if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) 10367 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET; 10368 if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID | 10369 RTE_FLOW_CONNTRACK_PKT_STATE_INVALID | 10370 RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)) 10371 reg_mask |= 0xc0; 10372 if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED) 10373 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE; 10374 if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) 10375 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET; 10376 /* The REG_C_x value could be saved during startup. */ 10377 reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error); 10378 if (reg_id == REG_NON) 10379 return; 10380 flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id, 10381 reg_value, reg_mask); 10382 } 10383 10384 static void 10385 flow_dv_translate_item_flex(struct rte_eth_dev *dev, void *matcher, void *key, 10386 const struct rte_flow_item *item, 10387 struct mlx5_flow *dev_flow, bool is_inner) 10388 { 10389 const struct rte_flow_item_flex *spec = 10390 (const struct rte_flow_item_flex *)item->spec; 10391 int index = mlx5_flex_acquire_index(dev, spec->handle, false); 10392 10393 MLX5_ASSERT(index >= 0 && index <= (int)(sizeof(uint32_t) * CHAR_BIT)); 10394 if (index < 0) 10395 return; 10396 if (!(dev_flow->handle->flex_item & RTE_BIT32(index))) { 10397 /* Don't count both inner and outer flex items in one rule. */ 10398 if (mlx5_flex_acquire_index(dev, spec->handle, true) != index) 10399 MLX5_ASSERT(false); 10400 dev_flow->handle->flex_item |= (uint8_t)RTE_BIT32(index); 10401 } 10402 mlx5_flex_flow_translate_item(dev, matcher, key, item, is_inner); 10403 } 10404 10405 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; 10406 10407 #define HEADER_IS_ZERO(match_criteria, headers) \ 10408 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 10409 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 10410 10411 /** 10412 * Calculate flow matcher enable bitmap. 10413 * 10414 * @param match_criteria 10415 * Pointer to flow matcher criteria. 10416 * 10417 * @return 10418 * Bitmap of enabled fields. 10419 */ 10420 static uint8_t 10421 flow_dv_matcher_enable(uint32_t *match_criteria) 10422 { 10423 uint8_t match_criteria_enable; 10424 10425 match_criteria_enable = 10426 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 10427 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT; 10428 match_criteria_enable |= 10429 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 10430 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT; 10431 match_criteria_enable |= 10432 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 10433 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT; 10434 match_criteria_enable |= 10435 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 10436 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 10437 match_criteria_enable |= 10438 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) << 10439 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT; 10440 match_criteria_enable |= 10441 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) << 10442 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT; 10443 match_criteria_enable |= 10444 (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) << 10445 MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT; 10446 return match_criteria_enable; 10447 } 10448 10449 static void 10450 __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria) 10451 { 10452 /* 10453 * Check flow matching criteria first, subtract misc5/4 length if flow 10454 * doesn't own misc5/4 parameters. In some old rdma-core releases, 10455 * misc5/4 are not supported, and matcher creation failure is expected 10456 * w/o subtraction. If misc5 is provided, misc4 must be counted in since 10457 * misc5 is right after misc4. 10458 */ 10459 if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) { 10460 *size = MLX5_ST_SZ_BYTES(fte_match_param) - 10461 MLX5_ST_SZ_BYTES(fte_match_set_misc5); 10462 if (!(match_criteria & (1 << 10463 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) { 10464 *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4); 10465 } 10466 } 10467 } 10468 10469 static struct mlx5_list_entry * 10470 flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused, 10471 struct mlx5_list_entry *entry, void *cb_ctx) 10472 { 10473 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10474 struct mlx5_flow_dv_matcher *ref = ctx->data; 10475 struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl, 10476 typeof(*tbl), tbl); 10477 struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY, 10478 sizeof(*resource), 10479 0, SOCKET_ID_ANY); 10480 10481 if (!resource) { 10482 rte_flow_error_set(ctx->error, ENOMEM, 10483 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10484 "cannot create matcher"); 10485 return NULL; 10486 } 10487 memcpy(resource, entry, sizeof(*resource)); 10488 resource->tbl = &tbl->tbl; 10489 return &resource->entry; 10490 } 10491 10492 static void 10493 flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused, 10494 struct mlx5_list_entry *entry) 10495 { 10496 mlx5_free(entry); 10497 } 10498 10499 struct mlx5_list_entry * 10500 flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx) 10501 { 10502 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10503 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10504 struct rte_eth_dev *dev = ctx->dev; 10505 struct mlx5_flow_tbl_data_entry *tbl_data; 10506 struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2; 10507 struct rte_flow_error *error = ctx->error; 10508 union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) }; 10509 struct mlx5_flow_tbl_resource *tbl; 10510 void *domain; 10511 uint32_t idx = 0; 10512 int ret; 10513 10514 tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); 10515 if (!tbl_data) { 10516 rte_flow_error_set(error, ENOMEM, 10517 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10518 NULL, 10519 "cannot allocate flow table data entry"); 10520 return NULL; 10521 } 10522 tbl_data->idx = idx; 10523 tbl_data->tunnel = tt_prm->tunnel; 10524 tbl_data->group_id = tt_prm->group_id; 10525 tbl_data->external = !!tt_prm->external; 10526 tbl_data->tunnel_offload = is_tunnel_offload_active(dev); 10527 tbl_data->is_egress = !!key.is_egress; 10528 tbl_data->is_transfer = !!key.is_fdb; 10529 tbl_data->dummy = !!key.dummy; 10530 tbl_data->level = key.level; 10531 tbl_data->id = key.id; 10532 tbl = &tbl_data->tbl; 10533 if (key.dummy) 10534 return &tbl_data->entry; 10535 if (key.is_fdb) 10536 domain = sh->fdb_domain; 10537 else if (key.is_egress) 10538 domain = sh->tx_domain; 10539 else 10540 domain = sh->rx_domain; 10541 ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj); 10542 if (ret) { 10543 rte_flow_error_set(error, ENOMEM, 10544 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10545 NULL, "cannot create flow table object"); 10546 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 10547 return NULL; 10548 } 10549 if (key.level != 0) { 10550 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl 10551 (tbl->obj, &tbl_data->jump.action); 10552 if (ret) { 10553 rte_flow_error_set(error, ENOMEM, 10554 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10555 NULL, 10556 "cannot create flow jump action"); 10557 mlx5_flow_os_destroy_flow_tbl(tbl->obj); 10558 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 10559 return NULL; 10560 } 10561 } 10562 MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list", 10563 key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress", 10564 key.level, key.id); 10565 tbl_data->matchers = mlx5_list_create(matcher_name, sh, true, 10566 flow_dv_matcher_create_cb, 10567 flow_dv_matcher_match_cb, 10568 flow_dv_matcher_remove_cb, 10569 flow_dv_matcher_clone_cb, 10570 flow_dv_matcher_clone_free_cb); 10571 if (!tbl_data->matchers) { 10572 rte_flow_error_set(error, ENOMEM, 10573 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10574 NULL, 10575 "cannot create tbl matcher list"); 10576 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action); 10577 mlx5_flow_os_destroy_flow_tbl(tbl->obj); 10578 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 10579 return NULL; 10580 } 10581 return &tbl_data->entry; 10582 } 10583 10584 int 10585 flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry, 10586 void *cb_ctx) 10587 { 10588 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10589 struct mlx5_flow_tbl_data_entry *tbl_data = 10590 container_of(entry, struct mlx5_flow_tbl_data_entry, entry); 10591 union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) }; 10592 10593 return tbl_data->level != key.level || 10594 tbl_data->id != key.id || 10595 tbl_data->dummy != key.dummy || 10596 tbl_data->is_transfer != !!key.is_fdb || 10597 tbl_data->is_egress != !!key.is_egress; 10598 } 10599 10600 struct mlx5_list_entry * 10601 flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 10602 void *cb_ctx) 10603 { 10604 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10605 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10606 struct mlx5_flow_tbl_data_entry *tbl_data; 10607 struct rte_flow_error *error = ctx->error; 10608 uint32_t idx = 0; 10609 10610 tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); 10611 if (!tbl_data) { 10612 rte_flow_error_set(error, ENOMEM, 10613 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10614 NULL, 10615 "cannot allocate flow table data entry"); 10616 return NULL; 10617 } 10618 memcpy(tbl_data, oentry, sizeof(*tbl_data)); 10619 tbl_data->idx = idx; 10620 return &tbl_data->entry; 10621 } 10622 10623 void 10624 flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 10625 { 10626 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10627 struct mlx5_flow_tbl_data_entry *tbl_data = 10628 container_of(entry, struct mlx5_flow_tbl_data_entry, entry); 10629 10630 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx); 10631 } 10632 10633 /** 10634 * Get a flow table. 10635 * 10636 * @param[in, out] dev 10637 * Pointer to rte_eth_dev structure. 10638 * @param[in] table_level 10639 * Table level to use. 10640 * @param[in] egress 10641 * Direction of the table. 10642 * @param[in] transfer 10643 * E-Switch or NIC flow. 10644 * @param[in] dummy 10645 * Dummy entry for dv API. 10646 * @param[in] table_id 10647 * Table id to use. 10648 * @param[out] error 10649 * pointer to error structure. 10650 * 10651 * @return 10652 * Returns tables resource based on the index, NULL in case of failed. 10653 */ 10654 struct mlx5_flow_tbl_resource * 10655 flow_dv_tbl_resource_get(struct rte_eth_dev *dev, 10656 uint32_t table_level, uint8_t egress, 10657 uint8_t transfer, 10658 bool external, 10659 const struct mlx5_flow_tunnel *tunnel, 10660 uint32_t group_id, uint8_t dummy, 10661 uint32_t table_id, 10662 struct rte_flow_error *error) 10663 { 10664 struct mlx5_priv *priv = dev->data->dev_private; 10665 union mlx5_flow_tbl_key table_key = { 10666 { 10667 .level = table_level, 10668 .id = table_id, 10669 .reserved = 0, 10670 .dummy = !!dummy, 10671 .is_fdb = !!transfer, 10672 .is_egress = !!egress, 10673 } 10674 }; 10675 struct mlx5_flow_tbl_tunnel_prm tt_prm = { 10676 .tunnel = tunnel, 10677 .group_id = group_id, 10678 .external = external, 10679 }; 10680 struct mlx5_flow_cb_ctx ctx = { 10681 .dev = dev, 10682 .error = error, 10683 .data = &table_key.v64, 10684 .data2 = &tt_prm, 10685 }; 10686 struct mlx5_list_entry *entry; 10687 struct mlx5_flow_tbl_data_entry *tbl_data; 10688 10689 entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx); 10690 if (!entry) { 10691 rte_flow_error_set(error, ENOMEM, 10692 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10693 "cannot get table"); 10694 return NULL; 10695 } 10696 DRV_LOG(DEBUG, "table_level %u table_id %u " 10697 "tunnel %u group %u registered.", 10698 table_level, table_id, 10699 tunnel ? tunnel->tunnel_id : 0, group_id); 10700 tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry); 10701 return &tbl_data->tbl; 10702 } 10703 10704 void 10705 flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 10706 { 10707 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10708 struct mlx5_flow_tbl_data_entry *tbl_data = 10709 container_of(entry, struct mlx5_flow_tbl_data_entry, entry); 10710 10711 MLX5_ASSERT(entry && sh); 10712 if (tbl_data->jump.action) 10713 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action); 10714 if (tbl_data->tbl.obj) 10715 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj); 10716 if (tbl_data->tunnel_offload && tbl_data->external) { 10717 struct mlx5_list_entry *he; 10718 struct mlx5_hlist *tunnel_grp_hash; 10719 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10720 union tunnel_tbl_key tunnel_key = { 10721 .tunnel_id = tbl_data->tunnel ? 10722 tbl_data->tunnel->tunnel_id : 0, 10723 .group = tbl_data->group_id 10724 }; 10725 uint32_t table_level = tbl_data->level; 10726 struct mlx5_flow_cb_ctx ctx = { 10727 .data = (void *)&tunnel_key.val, 10728 }; 10729 10730 tunnel_grp_hash = tbl_data->tunnel ? 10731 tbl_data->tunnel->groups : 10732 thub->groups; 10733 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx); 10734 if (he) 10735 mlx5_hlist_unregister(tunnel_grp_hash, he); 10736 DRV_LOG(DEBUG, 10737 "table_level %u id %u tunnel %u group %u released.", 10738 table_level, 10739 tbl_data->id, 10740 tbl_data->tunnel ? 10741 tbl_data->tunnel->tunnel_id : 0, 10742 tbl_data->group_id); 10743 } 10744 if (tbl_data->matchers) 10745 mlx5_list_destroy(tbl_data->matchers); 10746 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx); 10747 } 10748 10749 /** 10750 * Release a flow table. 10751 * 10752 * @param[in] sh 10753 * Pointer to device shared structure. 10754 * @param[in] tbl 10755 * Table resource to be released. 10756 * 10757 * @return 10758 * Returns 0 if table was released, else return 1; 10759 */ 10760 static int 10761 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh, 10762 struct mlx5_flow_tbl_resource *tbl) 10763 { 10764 struct mlx5_flow_tbl_data_entry *tbl_data = 10765 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 10766 10767 if (!tbl) 10768 return 0; 10769 return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry); 10770 } 10771 10772 int 10773 flow_dv_matcher_match_cb(void *tool_ctx __rte_unused, 10774 struct mlx5_list_entry *entry, void *cb_ctx) 10775 { 10776 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10777 struct mlx5_flow_dv_matcher *ref = ctx->data; 10778 struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur), 10779 entry); 10780 10781 return cur->crc != ref->crc || 10782 cur->priority != ref->priority || 10783 memcmp((const void *)cur->mask.buf, 10784 (const void *)ref->mask.buf, ref->mask.size); 10785 } 10786 10787 struct mlx5_list_entry * 10788 flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx) 10789 { 10790 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10791 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10792 struct mlx5_flow_dv_matcher *ref = ctx->data; 10793 struct mlx5_flow_dv_matcher *resource; 10794 struct mlx5dv_flow_matcher_attr dv_attr = { 10795 .type = IBV_FLOW_ATTR_NORMAL, 10796 .match_mask = (void *)&ref->mask, 10797 }; 10798 struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl, 10799 typeof(*tbl), tbl); 10800 int ret; 10801 10802 resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0, 10803 SOCKET_ID_ANY); 10804 if (!resource) { 10805 rte_flow_error_set(ctx->error, ENOMEM, 10806 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10807 "cannot create matcher"); 10808 return NULL; 10809 } 10810 *resource = *ref; 10811 dv_attr.match_criteria_enable = 10812 flow_dv_matcher_enable(resource->mask.buf); 10813 __flow_dv_adjust_buf_size(&ref->mask.size, 10814 dv_attr.match_criteria_enable); 10815 dv_attr.priority = ref->priority; 10816 if (tbl->is_egress) 10817 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; 10818 ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr, 10819 tbl->tbl.obj, 10820 &resource->matcher_object); 10821 if (ret) { 10822 mlx5_free(resource); 10823 rte_flow_error_set(ctx->error, ENOMEM, 10824 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10825 "cannot create matcher"); 10826 return NULL; 10827 } 10828 return &resource->entry; 10829 } 10830 10831 /** 10832 * Register the flow matcher. 10833 * 10834 * @param[in, out] dev 10835 * Pointer to rte_eth_dev structure. 10836 * @param[in, out] matcher 10837 * Pointer to flow matcher. 10838 * @param[in, out] key 10839 * Pointer to flow table key. 10840 * @parm[in, out] dev_flow 10841 * Pointer to the dev_flow. 10842 * @param[out] error 10843 * pointer to error structure. 10844 * 10845 * @return 10846 * 0 on success otherwise -errno and errno is set. 10847 */ 10848 static int 10849 flow_dv_matcher_register(struct rte_eth_dev *dev, 10850 struct mlx5_flow_dv_matcher *ref, 10851 union mlx5_flow_tbl_key *key, 10852 struct mlx5_flow *dev_flow, 10853 const struct mlx5_flow_tunnel *tunnel, 10854 uint32_t group_id, 10855 struct rte_flow_error *error) 10856 { 10857 struct mlx5_list_entry *entry; 10858 struct mlx5_flow_dv_matcher *resource; 10859 struct mlx5_flow_tbl_resource *tbl; 10860 struct mlx5_flow_tbl_data_entry *tbl_data; 10861 struct mlx5_flow_cb_ctx ctx = { 10862 .error = error, 10863 .data = ref, 10864 }; 10865 /** 10866 * tunnel offload API requires this registration for cases when 10867 * tunnel match rule was inserted before tunnel set rule. 10868 */ 10869 tbl = flow_dv_tbl_resource_get(dev, key->level, 10870 key->is_egress, key->is_fdb, 10871 dev_flow->external, tunnel, 10872 group_id, 0, key->id, error); 10873 if (!tbl) 10874 return -rte_errno; /* No need to refill the error info */ 10875 tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 10876 ref->tbl = tbl; 10877 entry = mlx5_list_register(tbl_data->matchers, &ctx); 10878 if (!entry) { 10879 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); 10880 return rte_flow_error_set(error, ENOMEM, 10881 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10882 "cannot allocate ref memory"); 10883 } 10884 resource = container_of(entry, typeof(*resource), entry); 10885 dev_flow->handle->dvh.matcher = resource; 10886 return 0; 10887 } 10888 10889 struct mlx5_list_entry * 10890 flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx) 10891 { 10892 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10893 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10894 struct mlx5_flow_dv_tag_resource *entry; 10895 uint32_t idx = 0; 10896 int ret; 10897 10898 entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx); 10899 if (!entry) { 10900 rte_flow_error_set(ctx->error, ENOMEM, 10901 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10902 "cannot allocate resource memory"); 10903 return NULL; 10904 } 10905 entry->idx = idx; 10906 entry->tag_id = *(uint32_t *)(ctx->data); 10907 ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id, 10908 &entry->action); 10909 if (ret) { 10910 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx); 10911 rte_flow_error_set(ctx->error, ENOMEM, 10912 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10913 NULL, "cannot create action"); 10914 return NULL; 10915 } 10916 return &entry->entry; 10917 } 10918 10919 int 10920 flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry, 10921 void *cb_ctx) 10922 { 10923 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10924 struct mlx5_flow_dv_tag_resource *tag = 10925 container_of(entry, struct mlx5_flow_dv_tag_resource, entry); 10926 10927 return *(uint32_t *)(ctx->data) != tag->tag_id; 10928 } 10929 10930 struct mlx5_list_entry * 10931 flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 10932 void *cb_ctx) 10933 { 10934 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10935 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10936 struct mlx5_flow_dv_tag_resource *entry; 10937 uint32_t idx = 0; 10938 10939 entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx); 10940 if (!entry) { 10941 rte_flow_error_set(ctx->error, ENOMEM, 10942 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10943 "cannot allocate tag resource memory"); 10944 return NULL; 10945 } 10946 memcpy(entry, oentry, sizeof(*entry)); 10947 entry->idx = idx; 10948 return &entry->entry; 10949 } 10950 10951 void 10952 flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 10953 { 10954 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10955 struct mlx5_flow_dv_tag_resource *tag = 10956 container_of(entry, struct mlx5_flow_dv_tag_resource, entry); 10957 10958 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx); 10959 } 10960 10961 /** 10962 * Find existing tag resource or create and register a new one. 10963 * 10964 * @param dev[in, out] 10965 * Pointer to rte_eth_dev structure. 10966 * @param[in, out] tag_be24 10967 * Tag value in big endian then R-shift 8. 10968 * @parm[in, out] dev_flow 10969 * Pointer to the dev_flow. 10970 * @param[out] error 10971 * pointer to error structure. 10972 * 10973 * @return 10974 * 0 on success otherwise -errno and errno is set. 10975 */ 10976 static int 10977 flow_dv_tag_resource_register 10978 (struct rte_eth_dev *dev, 10979 uint32_t tag_be24, 10980 struct mlx5_flow *dev_flow, 10981 struct rte_flow_error *error) 10982 { 10983 struct mlx5_priv *priv = dev->data->dev_private; 10984 struct mlx5_flow_dv_tag_resource *resource; 10985 struct mlx5_list_entry *entry; 10986 struct mlx5_flow_cb_ctx ctx = { 10987 .error = error, 10988 .data = &tag_be24, 10989 }; 10990 struct mlx5_hlist *tag_table; 10991 10992 tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table, 10993 "tags", 10994 MLX5_TAGS_HLIST_ARRAY_SIZE, 10995 false, false, priv->sh, 10996 flow_dv_tag_create_cb, 10997 flow_dv_tag_match_cb, 10998 flow_dv_tag_remove_cb, 10999 flow_dv_tag_clone_cb, 11000 flow_dv_tag_clone_free_cb, 11001 error); 11002 if (unlikely(!tag_table)) 11003 return -rte_errno; 11004 entry = mlx5_hlist_register(tag_table, tag_be24, &ctx); 11005 if (entry) { 11006 resource = container_of(entry, struct mlx5_flow_dv_tag_resource, 11007 entry); 11008 dev_flow->handle->dvh.rix_tag = resource->idx; 11009 dev_flow->dv.tag_resource = resource; 11010 return 0; 11011 } 11012 return -rte_errno; 11013 } 11014 11015 void 11016 flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 11017 { 11018 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11019 struct mlx5_flow_dv_tag_resource *tag = 11020 container_of(entry, struct mlx5_flow_dv_tag_resource, entry); 11021 11022 MLX5_ASSERT(tag && sh && tag->action); 11023 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action)); 11024 DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag); 11025 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx); 11026 } 11027 11028 /** 11029 * Release the tag. 11030 * 11031 * @param dev 11032 * Pointer to Ethernet device. 11033 * @param tag_idx 11034 * Tag index. 11035 * 11036 * @return 11037 * 1 while a reference on it exists, 0 when freed. 11038 */ 11039 static int 11040 flow_dv_tag_release(struct rte_eth_dev *dev, 11041 uint32_t tag_idx) 11042 { 11043 struct mlx5_priv *priv = dev->data->dev_private; 11044 struct mlx5_flow_dv_tag_resource *tag; 11045 11046 tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); 11047 if (!tag) 11048 return 0; 11049 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", 11050 dev->data->port_id, (void *)tag, tag->entry.ref_cnt); 11051 return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry); 11052 } 11053 11054 /** 11055 * Translate action PORT_ID / REPRESENTED_PORT to vport. 11056 * 11057 * @param[in] dev 11058 * Pointer to rte_eth_dev structure. 11059 * @param[in] action 11060 * Pointer to action PORT_ID / REPRESENTED_PORT. 11061 * @param[out] dst_port_id 11062 * The target port ID. 11063 * @param[out] error 11064 * Pointer to the error structure. 11065 * 11066 * @return 11067 * 0 on success, a negative errno value otherwise and rte_errno is set. 11068 */ 11069 static int 11070 flow_dv_translate_action_port_id(struct rte_eth_dev *dev, 11071 const struct rte_flow_action *action, 11072 uint32_t *dst_port_id, 11073 struct rte_flow_error *error) 11074 { 11075 uint32_t port; 11076 struct mlx5_priv *priv; 11077 11078 switch (action->type) { 11079 case RTE_FLOW_ACTION_TYPE_PORT_ID: { 11080 const struct rte_flow_action_port_id *conf; 11081 11082 conf = (const struct rte_flow_action_port_id *)action->conf; 11083 port = conf->original ? dev->data->port_id : conf->id; 11084 break; 11085 } 11086 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: { 11087 const struct rte_flow_action_ethdev *ethdev; 11088 11089 ethdev = (const struct rte_flow_action_ethdev *)action->conf; 11090 port = ethdev->port_id; 11091 break; 11092 } 11093 default: 11094 MLX5_ASSERT(false); 11095 return rte_flow_error_set(error, EINVAL, 11096 RTE_FLOW_ERROR_TYPE_ACTION, action, 11097 "unknown E-Switch action"); 11098 } 11099 11100 priv = mlx5_port_to_eswitch_info(port, false); 11101 if (!priv) 11102 return rte_flow_error_set(error, -rte_errno, 11103 RTE_FLOW_ERROR_TYPE_ACTION, 11104 NULL, 11105 "No eswitch info was found for port"); 11106 #ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT 11107 /* 11108 * This parameter is transferred to 11109 * mlx5dv_dr_action_create_dest_ib_port(). 11110 */ 11111 *dst_port_id = priv->dev_port; 11112 #else 11113 /* 11114 * Legacy mode, no LAG configurations is supported. 11115 * This parameter is transferred to 11116 * mlx5dv_dr_action_create_dest_vport(). 11117 */ 11118 *dst_port_id = priv->vport_id; 11119 #endif 11120 return 0; 11121 } 11122 11123 /** 11124 * Create a counter with aging configuration. 11125 * 11126 * @param[in] dev 11127 * Pointer to rte_eth_dev structure. 11128 * @param[in] dev_flow 11129 * Pointer to the mlx5_flow. 11130 * @param[out] count 11131 * Pointer to the counter action configuration. 11132 * @param[in] age 11133 * Pointer to the aging action configuration. 11134 * 11135 * @return 11136 * Index to flow counter on success, 0 otherwise. 11137 */ 11138 static uint32_t 11139 flow_dv_translate_create_counter(struct rte_eth_dev *dev, 11140 struct mlx5_flow *dev_flow, 11141 const struct rte_flow_action_count *count 11142 __rte_unused, 11143 const struct rte_flow_action_age *age) 11144 { 11145 uint32_t counter; 11146 struct mlx5_age_param *age_param; 11147 11148 counter = flow_dv_counter_alloc(dev, !!age); 11149 if (!counter || age == NULL) 11150 return counter; 11151 age_param = flow_dv_counter_idx_get_age(dev, counter); 11152 age_param->context = age->context ? age->context : 11153 (void *)(uintptr_t)(dev_flow->flow_idx); 11154 age_param->timeout = age->timeout; 11155 age_param->port_id = dev->data->port_id; 11156 __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED); 11157 __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED); 11158 return counter; 11159 } 11160 11161 /** 11162 * Add Tx queue matcher 11163 * 11164 * @param[in] dev 11165 * Pointer to the dev struct. 11166 * @param[in, out] matcher 11167 * Flow matcher. 11168 * @param[in, out] key 11169 * Flow matcher value. 11170 * @param[in] item 11171 * Flow pattern to translate. 11172 * @param[in] inner 11173 * Item is inner pattern. 11174 */ 11175 static void 11176 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev, 11177 void *matcher, void *key, 11178 const struct rte_flow_item *item) 11179 { 11180 const struct mlx5_rte_flow_item_tx_queue *queue_m; 11181 const struct mlx5_rte_flow_item_tx_queue *queue_v; 11182 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 11183 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 11184 struct mlx5_txq_ctrl *txq; 11185 uint32_t queue, mask; 11186 11187 queue_m = (const void *)item->mask; 11188 queue_v = (const void *)item->spec; 11189 if (!queue_v) 11190 return; 11191 txq = mlx5_txq_get(dev, queue_v->queue); 11192 if (!txq) 11193 return; 11194 if (txq->is_hairpin) 11195 queue = txq->obj->sq->id; 11196 else 11197 queue = txq->obj->sq_obj.sq->id; 11198 mask = queue_m == NULL ? UINT32_MAX : queue_m->queue; 11199 MLX5_SET(fte_match_set_misc, misc_m, source_sqn, mask); 11200 MLX5_SET(fte_match_set_misc, misc_v, source_sqn, queue & mask); 11201 mlx5_txq_release(dev, queue_v->queue); 11202 } 11203 11204 /** 11205 * Set the hash fields according to the @p flow information. 11206 * 11207 * @param[in] item_flags 11208 * The match pattern item flags. 11209 * @param[in] rss_desc 11210 * Pointer to the mlx5_flow_rss_desc. 11211 * @param[out] hash_fields 11212 * Pointer to the RSS hash fields. 11213 */ 11214 void 11215 flow_dv_hashfields_set(uint64_t item_flags, 11216 struct mlx5_flow_rss_desc *rss_desc, 11217 uint64_t *hash_fields) 11218 { 11219 uint64_t items = item_flags; 11220 uint64_t fields = 0; 11221 int rss_inner = 0; 11222 uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types); 11223 11224 *hash_fields = 0; 11225 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 11226 if (rss_desc->level >= 2) 11227 rss_inner = 1; 11228 #endif 11229 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) || 11230 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4)) || 11231 !items) { 11232 if (rss_types & MLX5_IPV4_LAYER_TYPES) { 11233 if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY) 11234 fields |= IBV_RX_HASH_SRC_IPV4; 11235 else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY) 11236 fields |= IBV_RX_HASH_DST_IPV4; 11237 else 11238 fields |= MLX5_IPV4_IBV_RX_HASH; 11239 } 11240 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) || 11241 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6)) || 11242 !items) { 11243 if (rss_types & MLX5_IPV6_LAYER_TYPES) { 11244 if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY) 11245 fields |= IBV_RX_HASH_SRC_IPV6; 11246 else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY) 11247 fields |= IBV_RX_HASH_DST_IPV6; 11248 else 11249 fields |= MLX5_IPV6_IBV_RX_HASH; 11250 } 11251 } 11252 if (items & MLX5_FLOW_ITEM_ESP) { 11253 if (rss_types & RTE_ETH_RSS_ESP) 11254 fields |= IBV_RX_HASH_IPSEC_SPI; 11255 } 11256 if ((fields & ~IBV_RX_HASH_IPSEC_SPI) == 0) { 11257 *hash_fields = fields; 11258 /* 11259 * There is no match between the RSS types and the 11260 * L3 protocol (IPv4/IPv6) defined in the flow rule. 11261 */ 11262 return; 11263 } 11264 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) || 11265 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP)) || 11266 !items) { 11267 if (rss_types & RTE_ETH_RSS_UDP) { 11268 if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY) 11269 fields |= IBV_RX_HASH_SRC_PORT_UDP; 11270 else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY) 11271 fields |= IBV_RX_HASH_DST_PORT_UDP; 11272 else 11273 fields |= MLX5_UDP_IBV_RX_HASH; 11274 } 11275 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) || 11276 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP)) || 11277 !items) { 11278 if (rss_types & RTE_ETH_RSS_TCP) { 11279 if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY) 11280 fields |= IBV_RX_HASH_SRC_PORT_TCP; 11281 else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY) 11282 fields |= IBV_RX_HASH_DST_PORT_TCP; 11283 else 11284 fields |= MLX5_TCP_IBV_RX_HASH; 11285 } 11286 } 11287 if (rss_inner) 11288 fields |= IBV_RX_HASH_INNER; 11289 *hash_fields = fields; 11290 } 11291 11292 /** 11293 * Prepare an Rx Hash queue. 11294 * 11295 * @param dev 11296 * Pointer to Ethernet device. 11297 * @param[in] dev_flow 11298 * Pointer to the mlx5_flow. 11299 * @param[in] rss_desc 11300 * Pointer to the mlx5_flow_rss_desc. 11301 * @param[out] hrxq_idx 11302 * Hash Rx queue index. 11303 * 11304 * @return 11305 * The Verbs/DevX object initialised, NULL otherwise and rte_errno is set. 11306 */ 11307 static struct mlx5_hrxq * 11308 flow_dv_hrxq_prepare(struct rte_eth_dev *dev, 11309 struct mlx5_flow *dev_flow, 11310 struct mlx5_flow_rss_desc *rss_desc, 11311 uint32_t *hrxq_idx) 11312 { 11313 struct mlx5_flow_handle *dh = dev_flow->handle; 11314 uint32_t shared_rss = rss_desc->shared_rss; 11315 struct mlx5_hrxq *hrxq; 11316 11317 MLX5_ASSERT(rss_desc->queue_num); 11318 rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; 11319 rss_desc->hash_fields = dev_flow->hash_fields; 11320 rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL); 11321 rss_desc->shared_rss = 0; 11322 if (rss_desc->hash_fields == 0) 11323 rss_desc->queue_num = 1; 11324 hrxq = mlx5_hrxq_get(dev, rss_desc); 11325 *hrxq_idx = hrxq ? hrxq->idx : 0; 11326 rss_desc->shared_rss = shared_rss; 11327 return hrxq; 11328 } 11329 11330 /** 11331 * Release sample sub action resource. 11332 * 11333 * @param[in, out] dev 11334 * Pointer to rte_eth_dev structure. 11335 * @param[in] act_res 11336 * Pointer to sample sub action resource. 11337 */ 11338 static void 11339 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev, 11340 struct mlx5_flow_sub_actions_idx *act_res) 11341 { 11342 if (act_res->rix_hrxq) { 11343 mlx5_hrxq_release(dev, act_res->rix_hrxq); 11344 act_res->rix_hrxq = 0; 11345 } 11346 if (act_res->rix_encap_decap) { 11347 flow_dv_encap_decap_resource_release(dev, 11348 act_res->rix_encap_decap); 11349 act_res->rix_encap_decap = 0; 11350 } 11351 if (act_res->rix_port_id_action) { 11352 flow_dv_port_id_action_resource_release(dev, 11353 act_res->rix_port_id_action); 11354 act_res->rix_port_id_action = 0; 11355 } 11356 if (act_res->rix_tag) { 11357 flow_dv_tag_release(dev, act_res->rix_tag); 11358 act_res->rix_tag = 0; 11359 } 11360 if (act_res->rix_jump) { 11361 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump); 11362 act_res->rix_jump = 0; 11363 } 11364 } 11365 11366 int 11367 flow_dv_sample_match_cb(void *tool_ctx __rte_unused, 11368 struct mlx5_list_entry *entry, void *cb_ctx) 11369 { 11370 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11371 struct rte_eth_dev *dev = ctx->dev; 11372 struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data; 11373 struct mlx5_flow_dv_sample_resource *resource = container_of(entry, 11374 typeof(*resource), 11375 entry); 11376 11377 if (ctx_resource->ratio == resource->ratio && 11378 ctx_resource->ft_type == resource->ft_type && 11379 ctx_resource->ft_id == resource->ft_id && 11380 ctx_resource->set_action == resource->set_action && 11381 !memcmp((void *)&ctx_resource->sample_act, 11382 (void *)&resource->sample_act, 11383 sizeof(struct mlx5_flow_sub_actions_list))) { 11384 /* 11385 * Existing sample action should release the prepared 11386 * sub-actions reference counter. 11387 */ 11388 flow_dv_sample_sub_actions_release(dev, 11389 &ctx_resource->sample_idx); 11390 return 0; 11391 } 11392 return 1; 11393 } 11394 11395 struct mlx5_list_entry * 11396 flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx) 11397 { 11398 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11399 struct rte_eth_dev *dev = ctx->dev; 11400 struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data; 11401 void **sample_dv_actions = ctx_resource->sub_actions; 11402 struct mlx5_flow_dv_sample_resource *resource; 11403 struct mlx5dv_dr_flow_sampler_attr sampler_attr; 11404 struct mlx5_priv *priv = dev->data->dev_private; 11405 struct mlx5_dev_ctx_shared *sh = priv->sh; 11406 struct mlx5_flow_tbl_resource *tbl; 11407 uint32_t idx = 0; 11408 const uint32_t next_ft_step = 1; 11409 uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step; 11410 uint8_t is_egress = 0; 11411 uint8_t is_transfer = 0; 11412 struct rte_flow_error *error = ctx->error; 11413 11414 /* Register new sample resource. */ 11415 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx); 11416 if (!resource) { 11417 rte_flow_error_set(error, ENOMEM, 11418 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11419 NULL, 11420 "cannot allocate resource memory"); 11421 return NULL; 11422 } 11423 *resource = *ctx_resource; 11424 /* Create normal path table level */ 11425 if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 11426 is_transfer = 1; 11427 else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) 11428 is_egress = 1; 11429 tbl = flow_dv_tbl_resource_get(dev, next_ft_id, 11430 is_egress, is_transfer, 11431 true, NULL, 0, 0, 0, error); 11432 if (!tbl) { 11433 rte_flow_error_set(error, ENOMEM, 11434 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11435 NULL, 11436 "fail to create normal path table " 11437 "for sample"); 11438 goto error; 11439 } 11440 resource->normal_path_tbl = tbl; 11441 if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) { 11442 if (!sh->default_miss_action) { 11443 rte_flow_error_set(error, ENOMEM, 11444 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11445 NULL, 11446 "default miss action was not " 11447 "created"); 11448 goto error; 11449 } 11450 sample_dv_actions[ctx_resource->sample_act.actions_num++] = 11451 sh->default_miss_action; 11452 } 11453 /* Create a DR sample action */ 11454 sampler_attr.sample_ratio = resource->ratio; 11455 sampler_attr.default_next_table = tbl->obj; 11456 sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num; 11457 sampler_attr.sample_actions = (struct mlx5dv_dr_action **) 11458 &sample_dv_actions[0]; 11459 sampler_attr.action = resource->set_action; 11460 if (mlx5_os_flow_dr_create_flow_action_sampler 11461 (&sampler_attr, &resource->verbs_action)) { 11462 rte_flow_error_set(error, ENOMEM, 11463 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11464 NULL, "cannot create sample action"); 11465 goto error; 11466 } 11467 resource->idx = idx; 11468 resource->dev = dev; 11469 return &resource->entry; 11470 error: 11471 if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB) 11472 flow_dv_sample_sub_actions_release(dev, 11473 &resource->sample_idx); 11474 if (resource->normal_path_tbl) 11475 flow_dv_tbl_resource_release(MLX5_SH(dev), 11476 resource->normal_path_tbl); 11477 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx); 11478 return NULL; 11479 11480 } 11481 11482 struct mlx5_list_entry * 11483 flow_dv_sample_clone_cb(void *tool_ctx __rte_unused, 11484 struct mlx5_list_entry *entry __rte_unused, 11485 void *cb_ctx) 11486 { 11487 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11488 struct rte_eth_dev *dev = ctx->dev; 11489 struct mlx5_flow_dv_sample_resource *resource; 11490 struct mlx5_priv *priv = dev->data->dev_private; 11491 struct mlx5_dev_ctx_shared *sh = priv->sh; 11492 uint32_t idx = 0; 11493 11494 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx); 11495 if (!resource) { 11496 rte_flow_error_set(ctx->error, ENOMEM, 11497 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11498 NULL, 11499 "cannot allocate resource memory"); 11500 return NULL; 11501 } 11502 memcpy(resource, entry, sizeof(*resource)); 11503 resource->idx = idx; 11504 resource->dev = dev; 11505 return &resource->entry; 11506 } 11507 11508 void 11509 flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused, 11510 struct mlx5_list_entry *entry) 11511 { 11512 struct mlx5_flow_dv_sample_resource *resource = 11513 container_of(entry, typeof(*resource), entry); 11514 struct rte_eth_dev *dev = resource->dev; 11515 struct mlx5_priv *priv = dev->data->dev_private; 11516 11517 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx); 11518 } 11519 11520 /** 11521 * Find existing sample resource or create and register a new one. 11522 * 11523 * @param[in, out] dev 11524 * Pointer to rte_eth_dev structure. 11525 * @param[in] ref 11526 * Pointer to sample resource reference. 11527 * @parm[in, out] dev_flow 11528 * Pointer to the dev_flow. 11529 * @param[out] error 11530 * pointer to error structure. 11531 * 11532 * @return 11533 * 0 on success otherwise -errno and errno is set. 11534 */ 11535 static int 11536 flow_dv_sample_resource_register(struct rte_eth_dev *dev, 11537 struct mlx5_flow_dv_sample_resource *ref, 11538 struct mlx5_flow *dev_flow, 11539 struct rte_flow_error *error) 11540 { 11541 struct mlx5_flow_dv_sample_resource *resource; 11542 struct mlx5_list_entry *entry; 11543 struct mlx5_priv *priv = dev->data->dev_private; 11544 struct mlx5_flow_cb_ctx ctx = { 11545 .dev = dev, 11546 .error = error, 11547 .data = ref, 11548 }; 11549 11550 entry = mlx5_list_register(priv->sh->sample_action_list, &ctx); 11551 if (!entry) 11552 return -rte_errno; 11553 resource = container_of(entry, typeof(*resource), entry); 11554 dev_flow->handle->dvh.rix_sample = resource->idx; 11555 dev_flow->dv.sample_res = resource; 11556 return 0; 11557 } 11558 11559 int 11560 flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused, 11561 struct mlx5_list_entry *entry, void *cb_ctx) 11562 { 11563 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11564 struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data; 11565 struct rte_eth_dev *dev = ctx->dev; 11566 struct mlx5_flow_dv_dest_array_resource *resource = 11567 container_of(entry, typeof(*resource), entry); 11568 uint32_t idx = 0; 11569 11570 if (ctx_resource->num_of_dest == resource->num_of_dest && 11571 ctx_resource->ft_type == resource->ft_type && 11572 !memcmp((void *)resource->sample_act, 11573 (void *)ctx_resource->sample_act, 11574 (ctx_resource->num_of_dest * 11575 sizeof(struct mlx5_flow_sub_actions_list)))) { 11576 /* 11577 * Existing sample action should release the prepared 11578 * sub-actions reference counter. 11579 */ 11580 for (idx = 0; idx < ctx_resource->num_of_dest; idx++) 11581 flow_dv_sample_sub_actions_release(dev, 11582 &ctx_resource->sample_idx[idx]); 11583 return 0; 11584 } 11585 return 1; 11586 } 11587 11588 struct mlx5_list_entry * 11589 flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx) 11590 { 11591 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11592 struct rte_eth_dev *dev = ctx->dev; 11593 struct mlx5_flow_dv_dest_array_resource *resource; 11594 struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data; 11595 struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 }; 11596 struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM]; 11597 struct mlx5_priv *priv = dev->data->dev_private; 11598 struct mlx5_dev_ctx_shared *sh = priv->sh; 11599 struct mlx5_flow_sub_actions_list *sample_act; 11600 struct mlx5dv_dr_domain *domain; 11601 uint32_t idx = 0, res_idx = 0; 11602 struct rte_flow_error *error = ctx->error; 11603 uint64_t action_flags; 11604 int ret; 11605 11606 /* Register new destination array resource. */ 11607 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY], 11608 &res_idx); 11609 if (!resource) { 11610 rte_flow_error_set(error, ENOMEM, 11611 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11612 NULL, 11613 "cannot allocate resource memory"); 11614 return NULL; 11615 } 11616 *resource = *ctx_resource; 11617 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 11618 domain = sh->fdb_domain; 11619 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 11620 domain = sh->rx_domain; 11621 else 11622 domain = sh->tx_domain; 11623 for (idx = 0; idx < ctx_resource->num_of_dest; idx++) { 11624 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *) 11625 mlx5_malloc(MLX5_MEM_ZERO, 11626 sizeof(struct mlx5dv_dr_action_dest_attr), 11627 0, SOCKET_ID_ANY); 11628 if (!dest_attr[idx]) { 11629 rte_flow_error_set(error, ENOMEM, 11630 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11631 NULL, 11632 "cannot allocate resource memory"); 11633 goto error; 11634 } 11635 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST; 11636 sample_act = &ctx_resource->sample_act[idx]; 11637 action_flags = sample_act->action_flags; 11638 switch (action_flags) { 11639 case MLX5_FLOW_ACTION_QUEUE: 11640 dest_attr[idx]->dest = sample_act->dr_queue_action; 11641 break; 11642 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP): 11643 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT; 11644 dest_attr[idx]->dest_reformat = &dest_reformat[idx]; 11645 dest_attr[idx]->dest_reformat->reformat = 11646 sample_act->dr_encap_action; 11647 dest_attr[idx]->dest_reformat->dest = 11648 sample_act->dr_port_id_action; 11649 break; 11650 case MLX5_FLOW_ACTION_PORT_ID: 11651 dest_attr[idx]->dest = sample_act->dr_port_id_action; 11652 break; 11653 case MLX5_FLOW_ACTION_JUMP: 11654 dest_attr[idx]->dest = sample_act->dr_jump_action; 11655 break; 11656 default: 11657 rte_flow_error_set(error, EINVAL, 11658 RTE_FLOW_ERROR_TYPE_ACTION, 11659 NULL, 11660 "unsupported actions type"); 11661 goto error; 11662 } 11663 } 11664 /* create a dest array action */ 11665 ret = mlx5_os_flow_dr_create_flow_action_dest_array 11666 (domain, 11667 resource->num_of_dest, 11668 dest_attr, 11669 &resource->action); 11670 if (ret) { 11671 rte_flow_error_set(error, ENOMEM, 11672 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11673 NULL, 11674 "cannot create destination array action"); 11675 goto error; 11676 } 11677 resource->idx = res_idx; 11678 resource->dev = dev; 11679 for (idx = 0; idx < ctx_resource->num_of_dest; idx++) 11680 mlx5_free(dest_attr[idx]); 11681 return &resource->entry; 11682 error: 11683 for (idx = 0; idx < ctx_resource->num_of_dest; idx++) { 11684 flow_dv_sample_sub_actions_release(dev, 11685 &resource->sample_idx[idx]); 11686 if (dest_attr[idx]) 11687 mlx5_free(dest_attr[idx]); 11688 } 11689 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx); 11690 return NULL; 11691 } 11692 11693 struct mlx5_list_entry * 11694 flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused, 11695 struct mlx5_list_entry *entry __rte_unused, 11696 void *cb_ctx) 11697 { 11698 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11699 struct rte_eth_dev *dev = ctx->dev; 11700 struct mlx5_flow_dv_dest_array_resource *resource; 11701 struct mlx5_priv *priv = dev->data->dev_private; 11702 struct mlx5_dev_ctx_shared *sh = priv->sh; 11703 uint32_t res_idx = 0; 11704 struct rte_flow_error *error = ctx->error; 11705 11706 resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY], 11707 &res_idx); 11708 if (!resource) { 11709 rte_flow_error_set(error, ENOMEM, 11710 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11711 NULL, 11712 "cannot allocate dest-array memory"); 11713 return NULL; 11714 } 11715 memcpy(resource, entry, sizeof(*resource)); 11716 resource->idx = res_idx; 11717 resource->dev = dev; 11718 return &resource->entry; 11719 } 11720 11721 void 11722 flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused, 11723 struct mlx5_list_entry *entry) 11724 { 11725 struct mlx5_flow_dv_dest_array_resource *resource = 11726 container_of(entry, typeof(*resource), entry); 11727 struct rte_eth_dev *dev = resource->dev; 11728 struct mlx5_priv *priv = dev->data->dev_private; 11729 11730 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx); 11731 } 11732 11733 /** 11734 * Find existing destination array resource or create and register a new one. 11735 * 11736 * @param[in, out] dev 11737 * Pointer to rte_eth_dev structure. 11738 * @param[in] ref 11739 * Pointer to destination array resource reference. 11740 * @parm[in, out] dev_flow 11741 * Pointer to the dev_flow. 11742 * @param[out] error 11743 * pointer to error structure. 11744 * 11745 * @return 11746 * 0 on success otherwise -errno and errno is set. 11747 */ 11748 static int 11749 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev, 11750 struct mlx5_flow_dv_dest_array_resource *ref, 11751 struct mlx5_flow *dev_flow, 11752 struct rte_flow_error *error) 11753 { 11754 struct mlx5_flow_dv_dest_array_resource *resource; 11755 struct mlx5_priv *priv = dev->data->dev_private; 11756 struct mlx5_list_entry *entry; 11757 struct mlx5_flow_cb_ctx ctx = { 11758 .dev = dev, 11759 .error = error, 11760 .data = ref, 11761 }; 11762 11763 entry = mlx5_list_register(priv->sh->dest_array_list, &ctx); 11764 if (!entry) 11765 return -rte_errno; 11766 resource = container_of(entry, typeof(*resource), entry); 11767 dev_flow->handle->dvh.rix_dest_array = resource->idx; 11768 dev_flow->dv.dest_array_res = resource; 11769 return 0; 11770 } 11771 11772 /** 11773 * Convert Sample action to DV specification. 11774 * 11775 * @param[in] dev 11776 * Pointer to rte_eth_dev structure. 11777 * @param[in] action 11778 * Pointer to sample action structure. 11779 * @param[in, out] dev_flow 11780 * Pointer to the mlx5_flow. 11781 * @param[in] attr 11782 * Pointer to the flow attributes. 11783 * @param[in, out] num_of_dest 11784 * Pointer to the num of destination. 11785 * @param[in, out] sample_actions 11786 * Pointer to sample actions list. 11787 * @param[in, out] res 11788 * Pointer to sample resource. 11789 * @param[out] error 11790 * Pointer to the error structure. 11791 * 11792 * @return 11793 * 0 on success, a negative errno value otherwise and rte_errno is set. 11794 */ 11795 static int 11796 flow_dv_translate_action_sample(struct rte_eth_dev *dev, 11797 const struct rte_flow_action_sample *action, 11798 struct mlx5_flow *dev_flow, 11799 const struct rte_flow_attr *attr, 11800 uint32_t *num_of_dest, 11801 void **sample_actions, 11802 struct mlx5_flow_dv_sample_resource *res, 11803 struct rte_flow_error *error) 11804 { 11805 struct mlx5_priv *priv = dev->data->dev_private; 11806 const struct rte_flow_action *sub_actions; 11807 struct mlx5_flow_sub_actions_list *sample_act; 11808 struct mlx5_flow_sub_actions_idx *sample_idx; 11809 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 11810 struct rte_flow *flow = dev_flow->flow; 11811 struct mlx5_flow_rss_desc *rss_desc; 11812 uint64_t action_flags = 0; 11813 11814 MLX5_ASSERT(wks); 11815 rss_desc = &wks->rss_desc; 11816 sample_act = &res->sample_act; 11817 sample_idx = &res->sample_idx; 11818 res->ratio = action->ratio; 11819 sub_actions = action->actions; 11820 for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) { 11821 int type = sub_actions->type; 11822 uint32_t pre_rix = 0; 11823 void *pre_r; 11824 switch (type) { 11825 case RTE_FLOW_ACTION_TYPE_QUEUE: 11826 { 11827 const struct rte_flow_action_queue *queue; 11828 struct mlx5_hrxq *hrxq; 11829 uint32_t hrxq_idx; 11830 11831 queue = sub_actions->conf; 11832 rss_desc->queue_num = 1; 11833 rss_desc->queue[0] = queue->index; 11834 hrxq = flow_dv_hrxq_prepare(dev, dev_flow, 11835 rss_desc, &hrxq_idx); 11836 if (!hrxq) 11837 return rte_flow_error_set 11838 (error, rte_errno, 11839 RTE_FLOW_ERROR_TYPE_ACTION, 11840 NULL, 11841 "cannot create fate queue"); 11842 sample_act->dr_queue_action = hrxq->action; 11843 sample_idx->rix_hrxq = hrxq_idx; 11844 sample_actions[sample_act->actions_num++] = 11845 hrxq->action; 11846 (*num_of_dest)++; 11847 action_flags |= MLX5_FLOW_ACTION_QUEUE; 11848 if (action_flags & MLX5_FLOW_ACTION_MARK) 11849 dev_flow->handle->rix_hrxq = hrxq_idx; 11850 dev_flow->handle->fate_action = 11851 MLX5_FLOW_FATE_QUEUE; 11852 break; 11853 } 11854 case RTE_FLOW_ACTION_TYPE_RSS: 11855 { 11856 struct mlx5_hrxq *hrxq; 11857 uint32_t hrxq_idx; 11858 const struct rte_flow_action_rss *rss; 11859 const uint8_t *rss_key; 11860 11861 rss = sub_actions->conf; 11862 memcpy(rss_desc->queue, rss->queue, 11863 rss->queue_num * sizeof(uint16_t)); 11864 rss_desc->queue_num = rss->queue_num; 11865 /* NULL RSS key indicates default RSS key. */ 11866 rss_key = !rss->key ? rss_hash_default_key : rss->key; 11867 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 11868 /* 11869 * rss->level and rss.types should be set in advance 11870 * when expanding items for RSS. 11871 */ 11872 flow_dv_hashfields_set(dev_flow->handle->layers, 11873 rss_desc, 11874 &dev_flow->hash_fields); 11875 hrxq = flow_dv_hrxq_prepare(dev, dev_flow, 11876 rss_desc, &hrxq_idx); 11877 if (!hrxq) 11878 return rte_flow_error_set 11879 (error, rte_errno, 11880 RTE_FLOW_ERROR_TYPE_ACTION, 11881 NULL, 11882 "cannot create fate queue"); 11883 sample_act->dr_queue_action = hrxq->action; 11884 sample_idx->rix_hrxq = hrxq_idx; 11885 sample_actions[sample_act->actions_num++] = 11886 hrxq->action; 11887 (*num_of_dest)++; 11888 action_flags |= MLX5_FLOW_ACTION_RSS; 11889 if (action_flags & MLX5_FLOW_ACTION_MARK) 11890 dev_flow->handle->rix_hrxq = hrxq_idx; 11891 dev_flow->handle->fate_action = 11892 MLX5_FLOW_FATE_QUEUE; 11893 break; 11894 } 11895 case RTE_FLOW_ACTION_TYPE_MARK: 11896 { 11897 uint32_t tag_be = mlx5_flow_mark_set 11898 (((const struct rte_flow_action_mark *) 11899 (sub_actions->conf))->id); 11900 11901 wks->mark = 1; 11902 pre_rix = dev_flow->handle->dvh.rix_tag; 11903 /* Save the mark resource before sample */ 11904 pre_r = dev_flow->dv.tag_resource; 11905 if (flow_dv_tag_resource_register(dev, tag_be, 11906 dev_flow, error)) 11907 return -rte_errno; 11908 MLX5_ASSERT(dev_flow->dv.tag_resource); 11909 sample_act->dr_tag_action = 11910 dev_flow->dv.tag_resource->action; 11911 sample_idx->rix_tag = 11912 dev_flow->handle->dvh.rix_tag; 11913 sample_actions[sample_act->actions_num++] = 11914 sample_act->dr_tag_action; 11915 /* Recover the mark resource after sample */ 11916 dev_flow->dv.tag_resource = pre_r; 11917 dev_flow->handle->dvh.rix_tag = pre_rix; 11918 action_flags |= MLX5_FLOW_ACTION_MARK; 11919 break; 11920 } 11921 case RTE_FLOW_ACTION_TYPE_COUNT: 11922 { 11923 if (!flow->counter) { 11924 flow->counter = 11925 flow_dv_translate_create_counter(dev, 11926 dev_flow, sub_actions->conf, 11927 0); 11928 if (!flow->counter) 11929 return rte_flow_error_set 11930 (error, rte_errno, 11931 RTE_FLOW_ERROR_TYPE_ACTION, 11932 NULL, 11933 "cannot create counter" 11934 " object."); 11935 } 11936 sample_act->dr_cnt_action = 11937 (flow_dv_counter_get_by_idx(dev, 11938 flow->counter, NULL))->action; 11939 sample_actions[sample_act->actions_num++] = 11940 sample_act->dr_cnt_action; 11941 action_flags |= MLX5_FLOW_ACTION_COUNT; 11942 break; 11943 } 11944 case RTE_FLOW_ACTION_TYPE_PORT_ID: 11945 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 11946 { 11947 struct mlx5_flow_dv_port_id_action_resource 11948 port_id_resource; 11949 uint32_t port_id = 0; 11950 11951 memset(&port_id_resource, 0, sizeof(port_id_resource)); 11952 /* Save the port id resource before sample */ 11953 pre_rix = dev_flow->handle->rix_port_id_action; 11954 pre_r = dev_flow->dv.port_id_action; 11955 if (flow_dv_translate_action_port_id(dev, sub_actions, 11956 &port_id, error)) 11957 return -rte_errno; 11958 port_id_resource.port_id = port_id; 11959 if (flow_dv_port_id_action_resource_register 11960 (dev, &port_id_resource, dev_flow, error)) 11961 return -rte_errno; 11962 sample_act->dr_port_id_action = 11963 dev_flow->dv.port_id_action->action; 11964 sample_idx->rix_port_id_action = 11965 dev_flow->handle->rix_port_id_action; 11966 sample_actions[sample_act->actions_num++] = 11967 sample_act->dr_port_id_action; 11968 /* Recover the port id resource after sample */ 11969 dev_flow->dv.port_id_action = pre_r; 11970 dev_flow->handle->rix_port_id_action = pre_rix; 11971 (*num_of_dest)++; 11972 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 11973 break; 11974 } 11975 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 11976 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 11977 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 11978 /* Save the encap resource before sample */ 11979 pre_rix = dev_flow->handle->dvh.rix_encap_decap; 11980 pre_r = dev_flow->dv.encap_decap; 11981 if (flow_dv_create_action_l2_encap(dev, sub_actions, 11982 dev_flow, 11983 attr->transfer, 11984 error)) 11985 return -rte_errno; 11986 sample_act->dr_encap_action = 11987 dev_flow->dv.encap_decap->action; 11988 sample_idx->rix_encap_decap = 11989 dev_flow->handle->dvh.rix_encap_decap; 11990 sample_actions[sample_act->actions_num++] = 11991 sample_act->dr_encap_action; 11992 /* Recover the encap resource after sample */ 11993 dev_flow->dv.encap_decap = pre_r; 11994 dev_flow->handle->dvh.rix_encap_decap = pre_rix; 11995 action_flags |= MLX5_FLOW_ACTION_ENCAP; 11996 break; 11997 default: 11998 return rte_flow_error_set(error, EINVAL, 11999 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12000 NULL, 12001 "Not support for sampler action"); 12002 } 12003 } 12004 sample_act->action_flags = action_flags; 12005 res->ft_id = dev_flow->dv.group; 12006 if (attr->transfer) { 12007 union { 12008 uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)]; 12009 uint64_t set_action; 12010 } action_ctx = { .set_action = 0 }; 12011 12012 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 12013 MLX5_SET(set_action_in, action_ctx.action_in, action_type, 12014 MLX5_MODIFICATION_TYPE_SET); 12015 MLX5_SET(set_action_in, action_ctx.action_in, field, 12016 MLX5_MODI_META_REG_C_0); 12017 MLX5_SET(set_action_in, action_ctx.action_in, data, 12018 priv->vport_meta_tag); 12019 res->set_action = action_ctx.set_action; 12020 } else if (attr->ingress) { 12021 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 12022 } else { 12023 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX; 12024 } 12025 return 0; 12026 } 12027 12028 /** 12029 * Convert Sample action to DV specification. 12030 * 12031 * @param[in] dev 12032 * Pointer to rte_eth_dev structure. 12033 * @param[in, out] dev_flow 12034 * Pointer to the mlx5_flow. 12035 * @param[in] num_of_dest 12036 * The num of destination. 12037 * @param[in, out] res 12038 * Pointer to sample resource. 12039 * @param[in, out] mdest_res 12040 * Pointer to destination array resource. 12041 * @param[in] sample_actions 12042 * Pointer to sample path actions list. 12043 * @param[in] action_flags 12044 * Holds the actions detected until now. 12045 * @param[out] error 12046 * Pointer to the error structure. 12047 * 12048 * @return 12049 * 0 on success, a negative errno value otherwise and rte_errno is set. 12050 */ 12051 static int 12052 flow_dv_create_action_sample(struct rte_eth_dev *dev, 12053 struct mlx5_flow *dev_flow, 12054 uint32_t num_of_dest, 12055 struct mlx5_flow_dv_sample_resource *res, 12056 struct mlx5_flow_dv_dest_array_resource *mdest_res, 12057 void **sample_actions, 12058 uint64_t action_flags, 12059 struct rte_flow_error *error) 12060 { 12061 /* update normal path action resource into last index of array */ 12062 uint32_t dest_index = MLX5_MAX_DEST_NUM - 1; 12063 struct mlx5_flow_sub_actions_list *sample_act = 12064 &mdest_res->sample_act[dest_index]; 12065 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 12066 struct mlx5_flow_rss_desc *rss_desc; 12067 uint32_t normal_idx = 0; 12068 struct mlx5_hrxq *hrxq; 12069 uint32_t hrxq_idx; 12070 12071 MLX5_ASSERT(wks); 12072 rss_desc = &wks->rss_desc; 12073 if (num_of_dest > 1) { 12074 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) { 12075 /* Handle QP action for mirroring */ 12076 hrxq = flow_dv_hrxq_prepare(dev, dev_flow, 12077 rss_desc, &hrxq_idx); 12078 if (!hrxq) 12079 return rte_flow_error_set 12080 (error, rte_errno, 12081 RTE_FLOW_ERROR_TYPE_ACTION, 12082 NULL, 12083 "cannot create rx queue"); 12084 normal_idx++; 12085 mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx; 12086 sample_act->dr_queue_action = hrxq->action; 12087 if (action_flags & MLX5_FLOW_ACTION_MARK) 12088 dev_flow->handle->rix_hrxq = hrxq_idx; 12089 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 12090 } 12091 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) { 12092 normal_idx++; 12093 mdest_res->sample_idx[dest_index].rix_encap_decap = 12094 dev_flow->handle->dvh.rix_encap_decap; 12095 sample_act->dr_encap_action = 12096 dev_flow->dv.encap_decap->action; 12097 dev_flow->handle->dvh.rix_encap_decap = 0; 12098 } 12099 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) { 12100 normal_idx++; 12101 mdest_res->sample_idx[dest_index].rix_port_id_action = 12102 dev_flow->handle->rix_port_id_action; 12103 sample_act->dr_port_id_action = 12104 dev_flow->dv.port_id_action->action; 12105 dev_flow->handle->rix_port_id_action = 0; 12106 } 12107 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) { 12108 normal_idx++; 12109 mdest_res->sample_idx[dest_index].rix_jump = 12110 dev_flow->handle->rix_jump; 12111 sample_act->dr_jump_action = 12112 dev_flow->dv.jump->action; 12113 dev_flow->handle->rix_jump = 0; 12114 } 12115 sample_act->actions_num = normal_idx; 12116 /* update sample action resource into first index of array */ 12117 mdest_res->ft_type = res->ft_type; 12118 memcpy(&mdest_res->sample_idx[0], &res->sample_idx, 12119 sizeof(struct mlx5_flow_sub_actions_idx)); 12120 memcpy(&mdest_res->sample_act[0], &res->sample_act, 12121 sizeof(struct mlx5_flow_sub_actions_list)); 12122 mdest_res->num_of_dest = num_of_dest; 12123 if (flow_dv_dest_array_resource_register(dev, mdest_res, 12124 dev_flow, error)) 12125 return rte_flow_error_set(error, EINVAL, 12126 RTE_FLOW_ERROR_TYPE_ACTION, 12127 NULL, "can't create sample " 12128 "action"); 12129 } else { 12130 res->sub_actions = sample_actions; 12131 if (flow_dv_sample_resource_register(dev, res, dev_flow, error)) 12132 return rte_flow_error_set(error, EINVAL, 12133 RTE_FLOW_ERROR_TYPE_ACTION, 12134 NULL, 12135 "can't create sample action"); 12136 } 12137 return 0; 12138 } 12139 12140 /** 12141 * Remove an ASO age action from age actions list. 12142 * 12143 * @param[in] dev 12144 * Pointer to the Ethernet device structure. 12145 * @param[in] age 12146 * Pointer to the aso age action handler. 12147 */ 12148 static void 12149 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev, 12150 struct mlx5_aso_age_action *age) 12151 { 12152 struct mlx5_age_info *age_info; 12153 struct mlx5_age_param *age_param = &age->age_params; 12154 struct mlx5_priv *priv = dev->data->dev_private; 12155 uint16_t expected = AGE_CANDIDATE; 12156 12157 age_info = GET_PORT_AGE_INFO(priv); 12158 if (!__atomic_compare_exchange_n(&age_param->state, &expected, 12159 AGE_FREE, false, __ATOMIC_RELAXED, 12160 __ATOMIC_RELAXED)) { 12161 /** 12162 * We need the lock even it is age timeout, 12163 * since age action may still in process. 12164 */ 12165 rte_spinlock_lock(&age_info->aged_sl); 12166 LIST_REMOVE(age, next); 12167 rte_spinlock_unlock(&age_info->aged_sl); 12168 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED); 12169 } 12170 } 12171 12172 /** 12173 * Release an ASO age action. 12174 * 12175 * @param[in] dev 12176 * Pointer to the Ethernet device structure. 12177 * @param[in] age_idx 12178 * Index of ASO age action to release. 12179 * @param[in] flow 12180 * True if the release operation is during flow destroy operation. 12181 * False if the release operation is during action destroy operation. 12182 * 12183 * @return 12184 * 0 when age action was removed, otherwise the number of references. 12185 */ 12186 static int 12187 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx) 12188 { 12189 struct mlx5_priv *priv = dev->data->dev_private; 12190 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 12191 struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx); 12192 uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED); 12193 12194 if (!ret) { 12195 flow_dv_aso_age_remove_from_age(dev, age); 12196 rte_spinlock_lock(&mng->free_sl); 12197 LIST_INSERT_HEAD(&mng->free, age, next); 12198 rte_spinlock_unlock(&mng->free_sl); 12199 } 12200 return ret; 12201 } 12202 12203 /** 12204 * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools. 12205 * 12206 * @param[in] dev 12207 * Pointer to the Ethernet device structure. 12208 * 12209 * @return 12210 * 0 on success, otherwise negative errno value and rte_errno is set. 12211 */ 12212 static int 12213 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev) 12214 { 12215 struct mlx5_priv *priv = dev->data->dev_private; 12216 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 12217 void *old_pools = mng->pools; 12218 uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE; 12219 uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize; 12220 void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); 12221 12222 if (!pools) { 12223 rte_errno = ENOMEM; 12224 return -ENOMEM; 12225 } 12226 if (old_pools) { 12227 memcpy(pools, old_pools, 12228 mng->n * sizeof(struct mlx5_flow_counter_pool *)); 12229 mlx5_free(old_pools); 12230 } else { 12231 /* First ASO flow hit allocation - starting ASO data-path. */ 12232 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh); 12233 12234 if (ret) { 12235 mlx5_free(pools); 12236 return ret; 12237 } 12238 } 12239 mng->n = resize; 12240 mng->pools = pools; 12241 return 0; 12242 } 12243 12244 /** 12245 * Create and initialize a new ASO aging pool. 12246 * 12247 * @param[in] dev 12248 * Pointer to the Ethernet device structure. 12249 * @param[out] age_free 12250 * Where to put the pointer of a new age action. 12251 * 12252 * @return 12253 * The age actions pool pointer and @p age_free is set on success, 12254 * NULL otherwise and rte_errno is set. 12255 */ 12256 static struct mlx5_aso_age_pool * 12257 flow_dv_age_pool_create(struct rte_eth_dev *dev, 12258 struct mlx5_aso_age_action **age_free) 12259 { 12260 struct mlx5_priv *priv = dev->data->dev_private; 12261 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 12262 struct mlx5_aso_age_pool *pool = NULL; 12263 struct mlx5_devx_obj *obj = NULL; 12264 uint32_t i; 12265 12266 obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->cdev->ctx, 12267 priv->sh->cdev->pdn); 12268 if (!obj) { 12269 rte_errno = ENODATA; 12270 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX."); 12271 return NULL; 12272 } 12273 pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); 12274 if (!pool) { 12275 claim_zero(mlx5_devx_cmd_destroy(obj)); 12276 rte_errno = ENOMEM; 12277 return NULL; 12278 } 12279 pool->flow_hit_aso_obj = obj; 12280 pool->time_of_last_age_check = MLX5_CURR_TIME_SEC; 12281 rte_rwlock_write_lock(&mng->resize_rwl); 12282 pool->index = mng->next; 12283 /* Resize pools array if there is no room for the new pool in it. */ 12284 if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) { 12285 claim_zero(mlx5_devx_cmd_destroy(obj)); 12286 mlx5_free(pool); 12287 rte_rwlock_write_unlock(&mng->resize_rwl); 12288 return NULL; 12289 } 12290 mng->pools[pool->index] = pool; 12291 mng->next++; 12292 rte_rwlock_write_unlock(&mng->resize_rwl); 12293 /* Assign the first action in the new pool, the rest go to free list. */ 12294 *age_free = &pool->actions[0]; 12295 for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) { 12296 pool->actions[i].offset = i; 12297 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next); 12298 } 12299 return pool; 12300 } 12301 12302 /** 12303 * Allocate a ASO aging bit. 12304 * 12305 * @param[in] dev 12306 * Pointer to the Ethernet device structure. 12307 * @param[out] error 12308 * Pointer to the error structure. 12309 * 12310 * @return 12311 * Index to ASO age action on success, 0 otherwise and rte_errno is set. 12312 */ 12313 static uint32_t 12314 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error) 12315 { 12316 struct mlx5_priv *priv = dev->data->dev_private; 12317 const struct mlx5_aso_age_pool *pool; 12318 struct mlx5_aso_age_action *age_free = NULL; 12319 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 12320 12321 MLX5_ASSERT(mng); 12322 /* Try to get the next free age action bit. */ 12323 rte_spinlock_lock(&mng->free_sl); 12324 age_free = LIST_FIRST(&mng->free); 12325 if (age_free) { 12326 LIST_REMOVE(age_free, next); 12327 } else if (!flow_dv_age_pool_create(dev, &age_free)) { 12328 rte_spinlock_unlock(&mng->free_sl); 12329 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION, 12330 NULL, "failed to create ASO age pool"); 12331 return 0; /* 0 is an error. */ 12332 } 12333 rte_spinlock_unlock(&mng->free_sl); 12334 pool = container_of 12335 ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL]) 12336 (age_free - age_free->offset), const struct mlx5_aso_age_pool, 12337 actions); 12338 if (!age_free->dr_action) { 12339 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0, 12340 error); 12341 12342 if (reg_c < 0) { 12343 rte_flow_error_set(error, rte_errno, 12344 RTE_FLOW_ERROR_TYPE_ACTION, 12345 NULL, "failed to get reg_c " 12346 "for ASO flow hit"); 12347 return 0; /* 0 is an error. */ 12348 } 12349 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 12350 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso 12351 (priv->sh->rx_domain, 12352 pool->flow_hit_aso_obj->obj, age_free->offset, 12353 MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET, 12354 (reg_c - REG_C_0)); 12355 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ 12356 if (!age_free->dr_action) { 12357 rte_errno = errno; 12358 rte_spinlock_lock(&mng->free_sl); 12359 LIST_INSERT_HEAD(&mng->free, age_free, next); 12360 rte_spinlock_unlock(&mng->free_sl); 12361 rte_flow_error_set(error, rte_errno, 12362 RTE_FLOW_ERROR_TYPE_ACTION, 12363 NULL, "failed to create ASO " 12364 "flow hit action"); 12365 return 0; /* 0 is an error. */ 12366 } 12367 } 12368 __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED); 12369 return pool->index | ((age_free->offset + 1) << 16); 12370 } 12371 12372 /** 12373 * Initialize flow ASO age parameters. 12374 * 12375 * @param[in] dev 12376 * Pointer to rte_eth_dev structure. 12377 * @param[in] age_idx 12378 * Index of ASO age action. 12379 * @param[in] context 12380 * Pointer to flow counter age context. 12381 * @param[in] timeout 12382 * Aging timeout in seconds. 12383 * 12384 */ 12385 static void 12386 flow_dv_aso_age_params_init(struct rte_eth_dev *dev, 12387 uint32_t age_idx, 12388 void *context, 12389 uint32_t timeout) 12390 { 12391 struct mlx5_aso_age_action *aso_age; 12392 12393 aso_age = flow_aso_age_get_by_idx(dev, age_idx); 12394 MLX5_ASSERT(aso_age); 12395 aso_age->age_params.context = context; 12396 aso_age->age_params.timeout = timeout; 12397 aso_age->age_params.port_id = dev->data->port_id; 12398 __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0, 12399 __ATOMIC_RELAXED); 12400 __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE, 12401 __ATOMIC_RELAXED); 12402 } 12403 12404 static void 12405 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask, 12406 const struct rte_flow_item_integrity *value, 12407 void *headers_m, void *headers_v) 12408 { 12409 if (mask->l4_ok) { 12410 /* RTE l4_ok filter aggregates hardware l4_ok and 12411 * l4_checksum_ok filters. 12412 * Positive RTE l4_ok match requires hardware match on both L4 12413 * hardware integrity bits. 12414 * For negative match, check hardware l4_checksum_ok bit only, 12415 * because hardware sets that bit to 0 for all packets 12416 * with bad L4. 12417 */ 12418 if (value->l4_ok) { 12419 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok, 1); 12420 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok, 1); 12421 } 12422 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok, 1); 12423 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok, 12424 !!value->l4_ok); 12425 } 12426 if (mask->l4_csum_ok) { 12427 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok, 1); 12428 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok, 12429 value->l4_csum_ok); 12430 } 12431 } 12432 12433 static void 12434 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask, 12435 const struct rte_flow_item_integrity *value, 12436 void *headers_m, void *headers_v, bool is_ipv4) 12437 { 12438 if (mask->l3_ok) { 12439 /* RTE l3_ok filter aggregates for IPv4 hardware l3_ok and 12440 * ipv4_csum_ok filters. 12441 * Positive RTE l3_ok match requires hardware match on both L3 12442 * hardware integrity bits. 12443 * For negative match, check hardware l3_csum_ok bit only, 12444 * because hardware sets that bit to 0 for all packets 12445 * with bad L3. 12446 */ 12447 if (is_ipv4) { 12448 if (value->l3_ok) { 12449 MLX5_SET(fte_match_set_lyr_2_4, headers_m, 12450 l3_ok, 1); 12451 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 12452 l3_ok, 1); 12453 } 12454 MLX5_SET(fte_match_set_lyr_2_4, headers_m, 12455 ipv4_checksum_ok, 1); 12456 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 12457 ipv4_checksum_ok, !!value->l3_ok); 12458 } else { 12459 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok, 1); 12460 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok, 12461 value->l3_ok); 12462 } 12463 } 12464 if (mask->ipv4_csum_ok) { 12465 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok, 1); 12466 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok, 12467 value->ipv4_csum_ok); 12468 } 12469 } 12470 12471 static void 12472 set_integrity_bits(void *headers_m, void *headers_v, 12473 const struct rte_flow_item *integrity_item, bool is_l3_ip4) 12474 { 12475 const struct rte_flow_item_integrity *spec = integrity_item->spec; 12476 const struct rte_flow_item_integrity *mask = integrity_item->mask; 12477 12478 /* Integrity bits validation cleared spec pointer */ 12479 MLX5_ASSERT(spec != NULL); 12480 if (!mask) 12481 mask = &rte_flow_item_integrity_mask; 12482 flow_dv_translate_integrity_l3(mask, spec, headers_m, headers_v, 12483 is_l3_ip4); 12484 flow_dv_translate_integrity_l4(mask, spec, headers_m, headers_v); 12485 } 12486 12487 static void 12488 flow_dv_translate_item_integrity_post(void *matcher, void *key, 12489 const 12490 struct rte_flow_item *integrity_items[2], 12491 uint64_t pattern_flags) 12492 { 12493 void *headers_m, *headers_v; 12494 bool is_l3_ip4; 12495 12496 if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY) { 12497 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 12498 inner_headers); 12499 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 12500 is_l3_ip4 = (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV4) != 12501 0; 12502 set_integrity_bits(headers_m, headers_v, 12503 integrity_items[1], is_l3_ip4); 12504 } 12505 if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY) { 12506 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 12507 outer_headers); 12508 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 12509 is_l3_ip4 = (pattern_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) != 12510 0; 12511 set_integrity_bits(headers_m, headers_v, 12512 integrity_items[0], is_l3_ip4); 12513 } 12514 } 12515 12516 static void 12517 flow_dv_translate_item_integrity(const struct rte_flow_item *item, 12518 const struct rte_flow_item *integrity_items[2], 12519 uint64_t *last_item) 12520 { 12521 const struct rte_flow_item_integrity *spec = (typeof(spec))item->spec; 12522 12523 /* integrity bits validation cleared spec pointer */ 12524 MLX5_ASSERT(spec != NULL); 12525 if (spec->level > 1) { 12526 integrity_items[1] = item; 12527 *last_item |= MLX5_FLOW_ITEM_INNER_INTEGRITY; 12528 } else { 12529 integrity_items[0] = item; 12530 *last_item |= MLX5_FLOW_ITEM_OUTER_INTEGRITY; 12531 } 12532 } 12533 12534 /** 12535 * Prepares DV flow counter with aging configuration. 12536 * Gets it by index when exists, creates a new one when doesn't. 12537 * 12538 * @param[in] dev 12539 * Pointer to rte_eth_dev structure. 12540 * @param[in] dev_flow 12541 * Pointer to the mlx5_flow. 12542 * @param[in, out] flow 12543 * Pointer to the sub flow. 12544 * @param[in] count 12545 * Pointer to the counter action configuration. 12546 * @param[in] age 12547 * Pointer to the aging action configuration. 12548 * @param[out] error 12549 * Pointer to the error structure. 12550 * 12551 * @return 12552 * Pointer to the counter, NULL otherwise. 12553 */ 12554 static struct mlx5_flow_counter * 12555 flow_dv_prepare_counter(struct rte_eth_dev *dev, 12556 struct mlx5_flow *dev_flow, 12557 struct rte_flow *flow, 12558 const struct rte_flow_action_count *count, 12559 const struct rte_flow_action_age *age, 12560 struct rte_flow_error *error) 12561 { 12562 if (!flow->counter) { 12563 flow->counter = flow_dv_translate_create_counter(dev, dev_flow, 12564 count, age); 12565 if (!flow->counter) { 12566 rte_flow_error_set(error, rte_errno, 12567 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 12568 "cannot create counter object."); 12569 return NULL; 12570 } 12571 } 12572 return flow_dv_counter_get_by_idx(dev, flow->counter, NULL); 12573 } 12574 12575 /* 12576 * Release an ASO CT action by its own device. 12577 * 12578 * @param[in] dev 12579 * Pointer to the Ethernet device structure. 12580 * @param[in] idx 12581 * Index of ASO CT action to release. 12582 * 12583 * @return 12584 * 0 when CT action was removed, otherwise the number of references. 12585 */ 12586 static inline int 12587 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx) 12588 { 12589 struct mlx5_priv *priv = dev->data->dev_private; 12590 struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng; 12591 uint32_t ret; 12592 struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx); 12593 enum mlx5_aso_ct_state state = 12594 __atomic_load_n(&ct->state, __ATOMIC_RELAXED); 12595 12596 /* Cannot release when CT is in the ASO SQ. */ 12597 if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY) 12598 return -1; 12599 ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED); 12600 if (!ret) { 12601 if (ct->dr_action_orig) { 12602 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 12603 claim_zero(mlx5_glue->destroy_flow_action 12604 (ct->dr_action_orig)); 12605 #endif 12606 ct->dr_action_orig = NULL; 12607 } 12608 if (ct->dr_action_rply) { 12609 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 12610 claim_zero(mlx5_glue->destroy_flow_action 12611 (ct->dr_action_rply)); 12612 #endif 12613 ct->dr_action_rply = NULL; 12614 } 12615 /* Clear the state to free, no need in 1st allocation. */ 12616 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE); 12617 rte_spinlock_lock(&mng->ct_sl); 12618 LIST_INSERT_HEAD(&mng->free_cts, ct, next); 12619 rte_spinlock_unlock(&mng->ct_sl); 12620 } 12621 return (int)ret; 12622 } 12623 12624 static inline int 12625 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx, 12626 struct rte_flow_error *error) 12627 { 12628 uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx); 12629 uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx); 12630 struct rte_eth_dev *owndev = &rte_eth_devices[owner]; 12631 int ret; 12632 12633 MLX5_ASSERT(owner < RTE_MAX_ETHPORTS); 12634 if (dev->data->dev_started != 1) 12635 return rte_flow_error_set(error, EAGAIN, 12636 RTE_FLOW_ERROR_TYPE_ACTION, 12637 NULL, 12638 "Indirect CT action cannot be destroyed when the port is stopped"); 12639 ret = flow_dv_aso_ct_dev_release(owndev, idx); 12640 if (ret < 0) 12641 return rte_flow_error_set(error, EAGAIN, 12642 RTE_FLOW_ERROR_TYPE_ACTION, 12643 NULL, 12644 "Current state prevents indirect CT action from being destroyed"); 12645 return ret; 12646 } 12647 12648 /* 12649 * Resize the ASO CT pools array by 64 pools. 12650 * 12651 * @param[in] dev 12652 * Pointer to the Ethernet device structure. 12653 * 12654 * @return 12655 * 0 on success, otherwise negative errno value and rte_errno is set. 12656 */ 12657 static int 12658 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev) 12659 { 12660 struct mlx5_priv *priv = dev->data->dev_private; 12661 struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng; 12662 void *old_pools = mng->pools; 12663 /* Magic number now, need a macro. */ 12664 uint32_t resize = mng->n + 64; 12665 uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize; 12666 void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); 12667 12668 if (!pools) { 12669 rte_errno = ENOMEM; 12670 return -rte_errno; 12671 } 12672 rte_rwlock_write_lock(&mng->resize_rwl); 12673 /* ASO SQ/QP was already initialized in the startup. */ 12674 if (old_pools) { 12675 /* Realloc could be an alternative choice. */ 12676 rte_memcpy(pools, old_pools, 12677 mng->n * sizeof(struct mlx5_aso_ct_pool *)); 12678 mlx5_free(old_pools); 12679 } 12680 mng->n = resize; 12681 mng->pools = pools; 12682 rte_rwlock_write_unlock(&mng->resize_rwl); 12683 return 0; 12684 } 12685 12686 /* 12687 * Create and initialize a new ASO CT pool. 12688 * 12689 * @param[in] dev 12690 * Pointer to the Ethernet device structure. 12691 * @param[out] ct_free 12692 * Where to put the pointer of a new CT action. 12693 * 12694 * @return 12695 * The CT actions pool pointer and @p ct_free is set on success, 12696 * NULL otherwise and rte_errno is set. 12697 */ 12698 static struct mlx5_aso_ct_pool * 12699 flow_dv_ct_pool_create(struct rte_eth_dev *dev, 12700 struct mlx5_aso_ct_action **ct_free) 12701 { 12702 struct mlx5_priv *priv = dev->data->dev_private; 12703 struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng; 12704 struct mlx5_aso_ct_pool *pool = NULL; 12705 struct mlx5_devx_obj *obj = NULL; 12706 uint32_t i; 12707 uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL); 12708 12709 obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->cdev->ctx, 12710 priv->sh->cdev->pdn, 12711 log_obj_size); 12712 if (!obj) { 12713 rte_errno = ENODATA; 12714 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX."); 12715 return NULL; 12716 } 12717 pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); 12718 if (!pool) { 12719 rte_errno = ENOMEM; 12720 claim_zero(mlx5_devx_cmd_destroy(obj)); 12721 return NULL; 12722 } 12723 pool->devx_obj = obj; 12724 pool->index = mng->next; 12725 /* Resize pools array if there is no room for the new pool in it. */ 12726 if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) { 12727 claim_zero(mlx5_devx_cmd_destroy(obj)); 12728 mlx5_free(pool); 12729 return NULL; 12730 } 12731 mng->pools[pool->index] = pool; 12732 mng->next++; 12733 /* Assign the first action in the new pool, the rest go to free list. */ 12734 *ct_free = &pool->actions[0]; 12735 /* Lock outside, the list operation is safe here. */ 12736 for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { 12737 /* refcnt is 0 when allocating the memory. */ 12738 pool->actions[i].offset = i; 12739 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next); 12740 } 12741 return pool; 12742 } 12743 12744 /* 12745 * Allocate a ASO CT action from free list. 12746 * 12747 * @param[in] dev 12748 * Pointer to the Ethernet device structure. 12749 * @param[out] error 12750 * Pointer to the error structure. 12751 * 12752 * @return 12753 * Index to ASO CT action on success, 0 otherwise and rte_errno is set. 12754 */ 12755 static uint32_t 12756 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error) 12757 { 12758 struct mlx5_priv *priv = dev->data->dev_private; 12759 struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng; 12760 struct mlx5_aso_ct_action *ct = NULL; 12761 struct mlx5_aso_ct_pool *pool; 12762 uint8_t reg_c; 12763 uint32_t ct_idx; 12764 12765 MLX5_ASSERT(mng); 12766 if (!priv->sh->cdev->config.devx) { 12767 rte_errno = ENOTSUP; 12768 return 0; 12769 } 12770 /* Get a free CT action, if no, a new pool will be created. */ 12771 rte_spinlock_lock(&mng->ct_sl); 12772 ct = LIST_FIRST(&mng->free_cts); 12773 if (ct) { 12774 LIST_REMOVE(ct, next); 12775 } else if (!flow_dv_ct_pool_create(dev, &ct)) { 12776 rte_spinlock_unlock(&mng->ct_sl); 12777 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION, 12778 NULL, "failed to create ASO CT pool"); 12779 return 0; 12780 } 12781 rte_spinlock_unlock(&mng->ct_sl); 12782 pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]); 12783 ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset); 12784 /* 0: inactive, 1: created, 2+: used by flows. */ 12785 __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED); 12786 reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error); 12787 if (!ct->dr_action_orig) { 12788 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 12789 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso 12790 (priv->sh->rx_domain, pool->devx_obj->obj, 12791 ct->offset, 12792 MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR, 12793 reg_c - REG_C_0); 12794 #else 12795 RTE_SET_USED(reg_c); 12796 #endif 12797 if (!ct->dr_action_orig) { 12798 flow_dv_aso_ct_dev_release(dev, ct_idx); 12799 rte_flow_error_set(error, rte_errno, 12800 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 12801 "failed to create ASO CT action"); 12802 return 0; 12803 } 12804 } 12805 if (!ct->dr_action_rply) { 12806 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT 12807 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso 12808 (priv->sh->rx_domain, pool->devx_obj->obj, 12809 ct->offset, 12810 MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER, 12811 reg_c - REG_C_0); 12812 #endif 12813 if (!ct->dr_action_rply) { 12814 flow_dv_aso_ct_dev_release(dev, ct_idx); 12815 rte_flow_error_set(error, rte_errno, 12816 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 12817 "failed to create ASO CT action"); 12818 return 0; 12819 } 12820 } 12821 return ct_idx; 12822 } 12823 12824 /* 12825 * Create a conntrack object with context and actions by using ASO mechanism. 12826 * 12827 * @param[in] dev 12828 * Pointer to rte_eth_dev structure. 12829 * @param[in] pro 12830 * Pointer to conntrack information profile. 12831 * @param[out] error 12832 * Pointer to the error structure. 12833 * 12834 * @return 12835 * Index to conntrack object on success, 0 otherwise. 12836 */ 12837 static uint32_t 12838 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev, 12839 const struct rte_flow_action_conntrack *pro, 12840 struct rte_flow_error *error) 12841 { 12842 struct mlx5_priv *priv = dev->data->dev_private; 12843 struct mlx5_dev_ctx_shared *sh = priv->sh; 12844 struct mlx5_aso_ct_action *ct; 12845 uint32_t idx; 12846 12847 if (!sh->ct_aso_en) 12848 return rte_flow_error_set(error, ENOTSUP, 12849 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 12850 "Connection is not supported"); 12851 idx = flow_dv_aso_ct_alloc(dev, error); 12852 if (!idx) 12853 return rte_flow_error_set(error, rte_errno, 12854 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 12855 "Failed to allocate CT object"); 12856 ct = flow_aso_ct_get_by_dev_idx(dev, idx); 12857 if (mlx5_aso_ct_update_by_wqe(sh, ct, pro)) 12858 return rte_flow_error_set(error, EBUSY, 12859 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 12860 "Failed to update CT"); 12861 ct->is_original = !!pro->is_original_dir; 12862 ct->peer = pro->peer_port; 12863 return idx; 12864 } 12865 12866 /** 12867 * Fill the flow with DV spec, lock free 12868 * (mutex should be acquired by caller). 12869 * 12870 * @param[in] dev 12871 * Pointer to rte_eth_dev structure. 12872 * @param[in, out] dev_flow 12873 * Pointer to the sub flow. 12874 * @param[in] attr 12875 * Pointer to the flow attributes. 12876 * @param[in] items 12877 * Pointer to the list of items. 12878 * @param[in] actions 12879 * Pointer to the list of actions. 12880 * @param[out] error 12881 * Pointer to the error structure. 12882 * 12883 * @return 12884 * 0 on success, a negative errno value otherwise and rte_errno is set. 12885 */ 12886 static int 12887 flow_dv_translate(struct rte_eth_dev *dev, 12888 struct mlx5_flow *dev_flow, 12889 const struct rte_flow_attr *attr, 12890 const struct rte_flow_item items[], 12891 const struct rte_flow_action actions[], 12892 struct rte_flow_error *error) 12893 { 12894 struct mlx5_priv *priv = dev->data->dev_private; 12895 struct mlx5_sh_config *dev_conf = &priv->sh->config; 12896 struct rte_flow *flow = dev_flow->flow; 12897 struct mlx5_flow_handle *handle = dev_flow->handle; 12898 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 12899 struct mlx5_flow_rss_desc *rss_desc; 12900 uint64_t item_flags = 0; 12901 uint64_t last_item = 0; 12902 uint64_t action_flags = 0; 12903 struct mlx5_flow_dv_matcher matcher = { 12904 .mask = { 12905 .size = sizeof(matcher.mask.buf), 12906 }, 12907 }; 12908 int actions_n = 0; 12909 bool actions_end = false; 12910 union { 12911 struct mlx5_flow_dv_modify_hdr_resource res; 12912 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) + 12913 sizeof(struct mlx5_modification_cmd) * 12914 (MLX5_MAX_MODIFY_NUM + 1)]; 12915 } mhdr_dummy; 12916 struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; 12917 const struct rte_flow_action_count *count = NULL; 12918 const struct rte_flow_action_age *non_shared_age = NULL; 12919 union flow_dv_attr flow_attr = { .attr = 0 }; 12920 uint32_t tag_be; 12921 union mlx5_flow_tbl_key tbl_key; 12922 uint32_t modify_action_position = UINT32_MAX; 12923 void *match_mask = matcher.mask.buf; 12924 void *match_value = dev_flow->dv.value.buf; 12925 uint8_t next_protocol = 0xff; 12926 struct rte_vlan_hdr vlan = { 0 }; 12927 struct mlx5_flow_dv_dest_array_resource mdest_res; 12928 struct mlx5_flow_dv_sample_resource sample_res; 12929 void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0}; 12930 const struct rte_flow_action_sample *sample = NULL; 12931 struct mlx5_flow_sub_actions_list *sample_act; 12932 uint32_t sample_act_pos = UINT32_MAX; 12933 uint32_t age_act_pos = UINT32_MAX; 12934 uint32_t num_of_dest = 0; 12935 int tmp_actions_n = 0; 12936 uint32_t table; 12937 int ret = 0; 12938 const struct mlx5_flow_tunnel *tunnel = NULL; 12939 struct flow_grp_info grp_info = { 12940 .external = !!dev_flow->external, 12941 .transfer = !!attr->transfer, 12942 .fdb_def_rule = !!priv->fdb_def_rule, 12943 .skip_scale = dev_flow->skip_scale & 12944 (1 << MLX5_SCALE_FLOW_GROUP_BIT), 12945 .std_tbl_fix = true, 12946 }; 12947 const struct rte_flow_item *integrity_items[2] = {NULL, NULL}; 12948 const struct rte_flow_item *tunnel_item = NULL; 12949 const struct rte_flow_item *gre_item = NULL; 12950 12951 if (!wks) 12952 return rte_flow_error_set(error, ENOMEM, 12953 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12954 NULL, 12955 "failed to push flow workspace"); 12956 rss_desc = &wks->rss_desc; 12957 memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource)); 12958 memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource)); 12959 mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 12960 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 12961 /* update normal path action resource into last index of array */ 12962 sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1]; 12963 if (is_tunnel_offload_active(dev)) { 12964 if (dev_flow->tunnel) { 12965 RTE_VERIFY(dev_flow->tof_type == 12966 MLX5_TUNNEL_OFFLOAD_MISS_RULE); 12967 tunnel = dev_flow->tunnel; 12968 } else { 12969 tunnel = mlx5_get_tof(items, actions, 12970 &dev_flow->tof_type); 12971 dev_flow->tunnel = tunnel; 12972 } 12973 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate 12974 (dev, attr, tunnel, dev_flow->tof_type); 12975 } 12976 mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 12977 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 12978 ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table, 12979 &grp_info, error); 12980 if (ret) 12981 return ret; 12982 dev_flow->dv.group = table; 12983 if (attr->transfer) 12984 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 12985 /* number of actions must be set to 0 in case of dirty stack. */ 12986 mhdr_res->actions_num = 0; 12987 if (is_flow_tunnel_match_rule(dev_flow->tof_type)) { 12988 /* 12989 * do not add decap action if match rule drops packet 12990 * HW rejects rules with decap & drop 12991 * 12992 * if tunnel match rule was inserted before matching tunnel set 12993 * rule flow table used in the match rule must be registered. 12994 * current implementation handles that in the 12995 * flow_dv_match_register() at the function end. 12996 */ 12997 bool add_decap = true; 12998 const struct rte_flow_action *ptr = actions; 12999 13000 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) { 13001 if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) { 13002 add_decap = false; 13003 break; 13004 } 13005 } 13006 if (add_decap) { 13007 if (flow_dv_create_action_l2_decap(dev, dev_flow, 13008 attr->transfer, 13009 error)) 13010 return -rte_errno; 13011 dev_flow->dv.actions[actions_n++] = 13012 dev_flow->dv.encap_decap->action; 13013 action_flags |= MLX5_FLOW_ACTION_DECAP; 13014 } 13015 } 13016 for (; !actions_end ; actions++) { 13017 const struct rte_flow_action_queue *queue; 13018 const struct rte_flow_action_rss *rss; 13019 const struct rte_flow_action *action = actions; 13020 const uint8_t *rss_key; 13021 struct mlx5_flow_tbl_resource *tbl; 13022 struct mlx5_aso_age_action *age_act; 13023 struct mlx5_flow_counter *cnt_act; 13024 uint32_t port_id = 0; 13025 struct mlx5_flow_dv_port_id_action_resource port_id_resource; 13026 int action_type = actions->type; 13027 const struct rte_flow_action *found_action = NULL; 13028 uint32_t jump_group = 0; 13029 uint32_t owner_idx; 13030 struct mlx5_aso_ct_action *ct; 13031 13032 if (!mlx5_flow_os_action_supported(action_type)) 13033 return rte_flow_error_set(error, ENOTSUP, 13034 RTE_FLOW_ERROR_TYPE_ACTION, 13035 actions, 13036 "action not supported"); 13037 switch (action_type) { 13038 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET: 13039 action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET; 13040 break; 13041 case RTE_FLOW_ACTION_TYPE_VOID: 13042 break; 13043 case RTE_FLOW_ACTION_TYPE_PORT_ID: 13044 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 13045 if (flow_dv_translate_action_port_id(dev, action, 13046 &port_id, error)) 13047 return -rte_errno; 13048 port_id_resource.port_id = port_id; 13049 MLX5_ASSERT(!handle->rix_port_id_action); 13050 if (flow_dv_port_id_action_resource_register 13051 (dev, &port_id_resource, dev_flow, error)) 13052 return -rte_errno; 13053 dev_flow->dv.actions[actions_n++] = 13054 dev_flow->dv.port_id_action->action; 13055 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 13056 dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID; 13057 sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID; 13058 num_of_dest++; 13059 break; 13060 case RTE_FLOW_ACTION_TYPE_FLAG: 13061 action_flags |= MLX5_FLOW_ACTION_FLAG; 13062 wks->mark = 1; 13063 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 13064 struct rte_flow_action_mark mark = { 13065 .id = MLX5_FLOW_MARK_DEFAULT, 13066 }; 13067 13068 if (flow_dv_convert_action_mark(dev, &mark, 13069 mhdr_res, 13070 error)) 13071 return -rte_errno; 13072 action_flags |= MLX5_FLOW_ACTION_MARK_EXT; 13073 break; 13074 } 13075 tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); 13076 /* 13077 * Only one FLAG or MARK is supported per device flow 13078 * right now. So the pointer to the tag resource must be 13079 * zero before the register process. 13080 */ 13081 MLX5_ASSERT(!handle->dvh.rix_tag); 13082 if (flow_dv_tag_resource_register(dev, tag_be, 13083 dev_flow, error)) 13084 return -rte_errno; 13085 MLX5_ASSERT(dev_flow->dv.tag_resource); 13086 dev_flow->dv.actions[actions_n++] = 13087 dev_flow->dv.tag_resource->action; 13088 break; 13089 case RTE_FLOW_ACTION_TYPE_MARK: 13090 action_flags |= MLX5_FLOW_ACTION_MARK; 13091 wks->mark = 1; 13092 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 13093 const struct rte_flow_action_mark *mark = 13094 (const struct rte_flow_action_mark *) 13095 actions->conf; 13096 13097 if (flow_dv_convert_action_mark(dev, mark, 13098 mhdr_res, 13099 error)) 13100 return -rte_errno; 13101 action_flags |= MLX5_FLOW_ACTION_MARK_EXT; 13102 break; 13103 } 13104 /* Fall-through */ 13105 case MLX5_RTE_FLOW_ACTION_TYPE_MARK: 13106 /* Legacy (non-extensive) MARK action. */ 13107 tag_be = mlx5_flow_mark_set 13108 (((const struct rte_flow_action_mark *) 13109 (actions->conf))->id); 13110 MLX5_ASSERT(!handle->dvh.rix_tag); 13111 if (flow_dv_tag_resource_register(dev, tag_be, 13112 dev_flow, error)) 13113 return -rte_errno; 13114 MLX5_ASSERT(dev_flow->dv.tag_resource); 13115 dev_flow->dv.actions[actions_n++] = 13116 dev_flow->dv.tag_resource->action; 13117 break; 13118 case RTE_FLOW_ACTION_TYPE_SET_META: 13119 if (flow_dv_convert_action_set_meta 13120 (dev, mhdr_res, attr, 13121 (const struct rte_flow_action_set_meta *) 13122 actions->conf, error)) 13123 return -rte_errno; 13124 action_flags |= MLX5_FLOW_ACTION_SET_META; 13125 break; 13126 case RTE_FLOW_ACTION_TYPE_SET_TAG: 13127 if (flow_dv_convert_action_set_tag 13128 (dev, mhdr_res, 13129 (const struct rte_flow_action_set_tag *) 13130 actions->conf, error)) 13131 return -rte_errno; 13132 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 13133 break; 13134 case RTE_FLOW_ACTION_TYPE_DROP: 13135 action_flags |= MLX5_FLOW_ACTION_DROP; 13136 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; 13137 break; 13138 case RTE_FLOW_ACTION_TYPE_QUEUE: 13139 queue = actions->conf; 13140 rss_desc->queue_num = 1; 13141 rss_desc->queue[0] = queue->index; 13142 action_flags |= MLX5_FLOW_ACTION_QUEUE; 13143 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 13144 sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE; 13145 num_of_dest++; 13146 break; 13147 case RTE_FLOW_ACTION_TYPE_RSS: 13148 rss = actions->conf; 13149 memcpy(rss_desc->queue, rss->queue, 13150 rss->queue_num * sizeof(uint16_t)); 13151 rss_desc->queue_num = rss->queue_num; 13152 /* NULL RSS key indicates default RSS key. */ 13153 rss_key = !rss->key ? rss_hash_default_key : rss->key; 13154 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 13155 /* 13156 * rss->level and rss.types should be set in advance 13157 * when expanding items for RSS. 13158 */ 13159 action_flags |= MLX5_FLOW_ACTION_RSS; 13160 dev_flow->handle->fate_action = rss_desc->shared_rss ? 13161 MLX5_FLOW_FATE_SHARED_RSS : 13162 MLX5_FLOW_FATE_QUEUE; 13163 break; 13164 case MLX5_RTE_FLOW_ACTION_TYPE_AGE: 13165 owner_idx = (uint32_t)(uintptr_t)action->conf; 13166 age_act = flow_aso_age_get_by_idx(dev, owner_idx); 13167 if (flow->age == 0) { 13168 flow->age = owner_idx; 13169 __atomic_fetch_add(&age_act->refcnt, 1, 13170 __ATOMIC_RELAXED); 13171 } 13172 age_act_pos = actions_n++; 13173 action_flags |= MLX5_FLOW_ACTION_AGE; 13174 break; 13175 case RTE_FLOW_ACTION_TYPE_AGE: 13176 non_shared_age = action->conf; 13177 age_act_pos = actions_n++; 13178 action_flags |= MLX5_FLOW_ACTION_AGE; 13179 break; 13180 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT: 13181 owner_idx = (uint32_t)(uintptr_t)action->conf; 13182 cnt_act = flow_dv_counter_get_by_idx(dev, owner_idx, 13183 NULL); 13184 MLX5_ASSERT(cnt_act != NULL); 13185 /** 13186 * When creating meter drop flow in drop table, the 13187 * counter should not overwrite the rte flow counter. 13188 */ 13189 if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER && 13190 dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP) { 13191 dev_flow->dv.actions[actions_n++] = 13192 cnt_act->action; 13193 } else { 13194 if (flow->counter == 0) { 13195 flow->counter = owner_idx; 13196 __atomic_fetch_add 13197 (&cnt_act->shared_info.refcnt, 13198 1, __ATOMIC_RELAXED); 13199 } 13200 /* Save information first, will apply later. */ 13201 action_flags |= MLX5_FLOW_ACTION_COUNT; 13202 } 13203 break; 13204 case RTE_FLOW_ACTION_TYPE_COUNT: 13205 if (!priv->sh->cdev->config.devx) { 13206 return rte_flow_error_set 13207 (error, ENOTSUP, 13208 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 13209 NULL, 13210 "count action not supported"); 13211 } 13212 /* Save information first, will apply later. */ 13213 count = action->conf; 13214 action_flags |= MLX5_FLOW_ACTION_COUNT; 13215 break; 13216 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 13217 dev_flow->dv.actions[actions_n++] = 13218 priv->sh->pop_vlan_action; 13219 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; 13220 break; 13221 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 13222 if (!(action_flags & 13223 MLX5_FLOW_ACTION_OF_SET_VLAN_VID)) 13224 flow_dev_get_vlan_info_from_items(items, &vlan); 13225 vlan.eth_proto = rte_be_to_cpu_16 13226 ((((const struct rte_flow_action_of_push_vlan *) 13227 actions->conf)->ethertype)); 13228 found_action = mlx5_flow_find_action 13229 (actions + 1, 13230 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID); 13231 if (found_action) 13232 mlx5_update_vlan_vid_pcp(found_action, &vlan); 13233 found_action = mlx5_flow_find_action 13234 (actions + 1, 13235 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP); 13236 if (found_action) 13237 mlx5_update_vlan_vid_pcp(found_action, &vlan); 13238 if (flow_dv_create_action_push_vlan 13239 (dev, attr, &vlan, dev_flow, error)) 13240 return -rte_errno; 13241 dev_flow->dv.actions[actions_n++] = 13242 dev_flow->dv.push_vlan_res->action; 13243 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; 13244 break; 13245 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 13246 /* of_vlan_push action handled this action */ 13247 MLX5_ASSERT(action_flags & 13248 MLX5_FLOW_ACTION_OF_PUSH_VLAN); 13249 break; 13250 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 13251 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) 13252 break; 13253 flow_dev_get_vlan_info_from_items(items, &vlan); 13254 mlx5_update_vlan_vid_pcp(actions, &vlan); 13255 /* If no VLAN push - this is a modify header action */ 13256 if (flow_dv_convert_action_modify_vlan_vid 13257 (mhdr_res, actions, error)) 13258 return -rte_errno; 13259 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; 13260 break; 13261 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 13262 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 13263 if (flow_dv_create_action_l2_encap(dev, actions, 13264 dev_flow, 13265 attr->transfer, 13266 error)) 13267 return -rte_errno; 13268 dev_flow->dv.actions[actions_n++] = 13269 dev_flow->dv.encap_decap->action; 13270 action_flags |= MLX5_FLOW_ACTION_ENCAP; 13271 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 13272 sample_act->action_flags |= 13273 MLX5_FLOW_ACTION_ENCAP; 13274 break; 13275 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 13276 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 13277 if (flow_dv_create_action_l2_decap(dev, dev_flow, 13278 attr->transfer, 13279 error)) 13280 return -rte_errno; 13281 dev_flow->dv.actions[actions_n++] = 13282 dev_flow->dv.encap_decap->action; 13283 action_flags |= MLX5_FLOW_ACTION_DECAP; 13284 break; 13285 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 13286 /* Handle encap with preceding decap. */ 13287 if (action_flags & MLX5_FLOW_ACTION_DECAP) { 13288 if (flow_dv_create_action_raw_encap 13289 (dev, actions, dev_flow, attr, error)) 13290 return -rte_errno; 13291 dev_flow->dv.actions[actions_n++] = 13292 dev_flow->dv.encap_decap->action; 13293 } else { 13294 /* Handle encap without preceding decap. */ 13295 if (flow_dv_create_action_l2_encap 13296 (dev, actions, dev_flow, attr->transfer, 13297 error)) 13298 return -rte_errno; 13299 dev_flow->dv.actions[actions_n++] = 13300 dev_flow->dv.encap_decap->action; 13301 } 13302 action_flags |= MLX5_FLOW_ACTION_ENCAP; 13303 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) 13304 sample_act->action_flags |= 13305 MLX5_FLOW_ACTION_ENCAP; 13306 break; 13307 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 13308 while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID) 13309 ; 13310 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 13311 if (flow_dv_create_action_l2_decap 13312 (dev, dev_flow, attr->transfer, error)) 13313 return -rte_errno; 13314 dev_flow->dv.actions[actions_n++] = 13315 dev_flow->dv.encap_decap->action; 13316 } 13317 /* If decap is followed by encap, handle it at encap. */ 13318 action_flags |= MLX5_FLOW_ACTION_DECAP; 13319 break; 13320 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP: 13321 dev_flow->dv.actions[actions_n++] = 13322 (void *)(uintptr_t)action->conf; 13323 action_flags |= MLX5_FLOW_ACTION_JUMP; 13324 break; 13325 case RTE_FLOW_ACTION_TYPE_JUMP: 13326 jump_group = ((const struct rte_flow_action_jump *) 13327 action->conf)->group; 13328 grp_info.std_tbl_fix = 0; 13329 if (dev_flow->skip_scale & 13330 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT)) 13331 grp_info.skip_scale = 1; 13332 else 13333 grp_info.skip_scale = 0; 13334 ret = mlx5_flow_group_to_table(dev, tunnel, 13335 jump_group, 13336 &table, 13337 &grp_info, error); 13338 if (ret) 13339 return ret; 13340 tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, 13341 attr->transfer, 13342 !!dev_flow->external, 13343 tunnel, jump_group, 0, 13344 0, error); 13345 if (!tbl) 13346 return rte_flow_error_set 13347 (error, errno, 13348 RTE_FLOW_ERROR_TYPE_ACTION, 13349 NULL, 13350 "cannot create jump action."); 13351 if (flow_dv_jump_tbl_resource_register 13352 (dev, tbl, dev_flow, error)) { 13353 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); 13354 return rte_flow_error_set 13355 (error, errno, 13356 RTE_FLOW_ERROR_TYPE_ACTION, 13357 NULL, 13358 "cannot create jump action."); 13359 } 13360 dev_flow->dv.actions[actions_n++] = 13361 dev_flow->dv.jump->action; 13362 action_flags |= MLX5_FLOW_ACTION_JUMP; 13363 dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP; 13364 sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP; 13365 num_of_dest++; 13366 break; 13367 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 13368 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 13369 if (flow_dv_convert_action_modify_mac 13370 (mhdr_res, actions, error)) 13371 return -rte_errno; 13372 action_flags |= actions->type == 13373 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 13374 MLX5_FLOW_ACTION_SET_MAC_SRC : 13375 MLX5_FLOW_ACTION_SET_MAC_DST; 13376 break; 13377 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 13378 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 13379 if (flow_dv_convert_action_modify_ipv4 13380 (mhdr_res, actions, error)) 13381 return -rte_errno; 13382 action_flags |= actions->type == 13383 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 13384 MLX5_FLOW_ACTION_SET_IPV4_SRC : 13385 MLX5_FLOW_ACTION_SET_IPV4_DST; 13386 break; 13387 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 13388 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 13389 if (flow_dv_convert_action_modify_ipv6 13390 (mhdr_res, actions, error)) 13391 return -rte_errno; 13392 action_flags |= actions->type == 13393 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 13394 MLX5_FLOW_ACTION_SET_IPV6_SRC : 13395 MLX5_FLOW_ACTION_SET_IPV6_DST; 13396 break; 13397 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 13398 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 13399 if (flow_dv_convert_action_modify_tp 13400 (mhdr_res, actions, items, 13401 &flow_attr, dev_flow, !!(action_flags & 13402 MLX5_FLOW_ACTION_DECAP), error)) 13403 return -rte_errno; 13404 action_flags |= actions->type == 13405 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 13406 MLX5_FLOW_ACTION_SET_TP_SRC : 13407 MLX5_FLOW_ACTION_SET_TP_DST; 13408 break; 13409 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 13410 if (flow_dv_convert_action_modify_dec_ttl 13411 (mhdr_res, items, &flow_attr, dev_flow, 13412 !!(action_flags & 13413 MLX5_FLOW_ACTION_DECAP), error)) 13414 return -rte_errno; 13415 action_flags |= MLX5_FLOW_ACTION_DEC_TTL; 13416 break; 13417 case RTE_FLOW_ACTION_TYPE_SET_TTL: 13418 if (flow_dv_convert_action_modify_ttl 13419 (mhdr_res, actions, items, &flow_attr, 13420 dev_flow, !!(action_flags & 13421 MLX5_FLOW_ACTION_DECAP), error)) 13422 return -rte_errno; 13423 action_flags |= MLX5_FLOW_ACTION_SET_TTL; 13424 break; 13425 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 13426 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 13427 if (flow_dv_convert_action_modify_tcp_seq 13428 (mhdr_res, actions, error)) 13429 return -rte_errno; 13430 action_flags |= actions->type == 13431 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 13432 MLX5_FLOW_ACTION_INC_TCP_SEQ : 13433 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 13434 break; 13435 13436 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 13437 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 13438 if (flow_dv_convert_action_modify_tcp_ack 13439 (mhdr_res, actions, error)) 13440 return -rte_errno; 13441 action_flags |= actions->type == 13442 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 13443 MLX5_FLOW_ACTION_INC_TCP_ACK : 13444 MLX5_FLOW_ACTION_DEC_TCP_ACK; 13445 break; 13446 case MLX5_RTE_FLOW_ACTION_TYPE_TAG: 13447 if (flow_dv_convert_action_set_reg 13448 (mhdr_res, actions, error)) 13449 return -rte_errno; 13450 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 13451 break; 13452 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: 13453 if (flow_dv_convert_action_copy_mreg 13454 (dev, mhdr_res, actions, error)) 13455 return -rte_errno; 13456 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 13457 break; 13458 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: 13459 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; 13460 dev_flow->handle->fate_action = 13461 MLX5_FLOW_FATE_DEFAULT_MISS; 13462 break; 13463 case RTE_FLOW_ACTION_TYPE_METER: 13464 if (!wks->fm) 13465 return rte_flow_error_set(error, rte_errno, 13466 RTE_FLOW_ERROR_TYPE_ACTION, 13467 NULL, "Failed to get meter in flow."); 13468 /* Set the meter action. */ 13469 dev_flow->dv.actions[actions_n++] = 13470 wks->fm->meter_action_g; 13471 action_flags |= MLX5_FLOW_ACTION_METER; 13472 break; 13473 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 13474 if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res, 13475 actions, error)) 13476 return -rte_errno; 13477 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; 13478 break; 13479 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 13480 if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res, 13481 actions, error)) 13482 return -rte_errno; 13483 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; 13484 break; 13485 case RTE_FLOW_ACTION_TYPE_SAMPLE: 13486 sample_act_pos = actions_n; 13487 sample = (const struct rte_flow_action_sample *) 13488 action->conf; 13489 actions_n++; 13490 action_flags |= MLX5_FLOW_ACTION_SAMPLE; 13491 /* put encap action into group if work with port id */ 13492 if ((action_flags & MLX5_FLOW_ACTION_ENCAP) && 13493 (action_flags & MLX5_FLOW_ACTION_PORT_ID)) 13494 sample_act->action_flags |= 13495 MLX5_FLOW_ACTION_ENCAP; 13496 break; 13497 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 13498 if (flow_dv_convert_action_modify_field 13499 (dev, mhdr_res, actions, attr, error)) 13500 return -rte_errno; 13501 action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD; 13502 break; 13503 case RTE_FLOW_ACTION_TYPE_CONNTRACK: 13504 owner_idx = (uint32_t)(uintptr_t)action->conf; 13505 ct = flow_aso_ct_get_by_idx(dev, owner_idx); 13506 if (!ct) 13507 return rte_flow_error_set(error, EINVAL, 13508 RTE_FLOW_ERROR_TYPE_ACTION, 13509 NULL, 13510 "Failed to get CT object."); 13511 if (mlx5_aso_ct_available(priv->sh, ct)) 13512 return rte_flow_error_set(error, rte_errno, 13513 RTE_FLOW_ERROR_TYPE_ACTION, 13514 NULL, 13515 "CT is unavailable."); 13516 if (ct->is_original) 13517 dev_flow->dv.actions[actions_n] = 13518 ct->dr_action_orig; 13519 else 13520 dev_flow->dv.actions[actions_n] = 13521 ct->dr_action_rply; 13522 if (flow->ct == 0) { 13523 flow->indirect_type = 13524 MLX5_INDIRECT_ACTION_TYPE_CT; 13525 flow->ct = owner_idx; 13526 __atomic_fetch_add(&ct->refcnt, 1, 13527 __ATOMIC_RELAXED); 13528 } 13529 actions_n++; 13530 action_flags |= MLX5_FLOW_ACTION_CT; 13531 break; 13532 case RTE_FLOW_ACTION_TYPE_END: 13533 actions_end = true; 13534 if (mhdr_res->actions_num) { 13535 /* create modify action if needed. */ 13536 if (flow_dv_modify_hdr_resource_register 13537 (dev, mhdr_res, dev_flow, error)) 13538 return -rte_errno; 13539 dev_flow->dv.actions[modify_action_position] = 13540 handle->dvh.modify_hdr->action; 13541 } 13542 /* 13543 * Handle AGE and COUNT action by single HW counter 13544 * when they are not shared. 13545 */ 13546 if (action_flags & MLX5_FLOW_ACTION_AGE) { 13547 if ((non_shared_age && count) || 13548 !flow_hit_aso_supported(priv->sh, attr)) { 13549 /* Creates age by counters. */ 13550 cnt_act = flow_dv_prepare_counter 13551 (dev, dev_flow, 13552 flow, count, 13553 non_shared_age, 13554 error); 13555 if (!cnt_act) 13556 return -rte_errno; 13557 dev_flow->dv.actions[age_act_pos] = 13558 cnt_act->action; 13559 break; 13560 } 13561 if (!flow->age && non_shared_age) { 13562 flow->age = flow_dv_aso_age_alloc 13563 (dev, error); 13564 if (!flow->age) 13565 return -rte_errno; 13566 flow_dv_aso_age_params_init 13567 (dev, flow->age, 13568 non_shared_age->context ? 13569 non_shared_age->context : 13570 (void *)(uintptr_t) 13571 (dev_flow->flow_idx), 13572 non_shared_age->timeout); 13573 } 13574 age_act = flow_aso_age_get_by_idx(dev, 13575 flow->age); 13576 dev_flow->dv.actions[age_act_pos] = 13577 age_act->dr_action; 13578 } 13579 if (action_flags & MLX5_FLOW_ACTION_COUNT) { 13580 /* 13581 * Create one count action, to be used 13582 * by all sub-flows. 13583 */ 13584 cnt_act = flow_dv_prepare_counter(dev, dev_flow, 13585 flow, count, 13586 NULL, error); 13587 if (!cnt_act) 13588 return -rte_errno; 13589 dev_flow->dv.actions[actions_n++] = 13590 cnt_act->action; 13591 } 13592 default: 13593 break; 13594 } 13595 if (mhdr_res->actions_num && 13596 modify_action_position == UINT32_MAX) 13597 modify_action_position = actions_n++; 13598 } 13599 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 13600 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 13601 int item_type = items->type; 13602 13603 if (!mlx5_flow_os_item_supported(item_type)) 13604 return rte_flow_error_set(error, ENOTSUP, 13605 RTE_FLOW_ERROR_TYPE_ITEM, 13606 NULL, "item not supported"); 13607 switch (item_type) { 13608 case RTE_FLOW_ITEM_TYPE_ESP: 13609 flow_dv_translate_item_esp(match_mask, match_value, 13610 items, tunnel); 13611 last_item = MLX5_FLOW_ITEM_ESP; 13612 break; 13613 case RTE_FLOW_ITEM_TYPE_PORT_ID: 13614 flow_dv_translate_item_port_id 13615 (dev, match_mask, match_value, items, attr); 13616 last_item = MLX5_FLOW_ITEM_PORT_ID; 13617 break; 13618 case RTE_FLOW_ITEM_TYPE_ETH: 13619 flow_dv_translate_item_eth(match_mask, match_value, 13620 items, tunnel, 13621 dev_flow->dv.group); 13622 matcher.priority = action_flags & 13623 MLX5_FLOW_ACTION_DEFAULT_MISS && 13624 !dev_flow->external ? 13625 MLX5_PRIORITY_MAP_L3 : 13626 MLX5_PRIORITY_MAP_L2; 13627 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 13628 MLX5_FLOW_LAYER_OUTER_L2; 13629 break; 13630 case RTE_FLOW_ITEM_TYPE_VLAN: 13631 flow_dv_translate_item_vlan(dev_flow, 13632 match_mask, match_value, 13633 items, tunnel, 13634 dev_flow->dv.group); 13635 matcher.priority = MLX5_PRIORITY_MAP_L2; 13636 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 13637 MLX5_FLOW_LAYER_INNER_VLAN) : 13638 (MLX5_FLOW_LAYER_OUTER_L2 | 13639 MLX5_FLOW_LAYER_OUTER_VLAN); 13640 break; 13641 case RTE_FLOW_ITEM_TYPE_IPV4: 13642 mlx5_flow_tunnel_ip_check(items, next_protocol, 13643 &item_flags, &tunnel); 13644 flow_dv_translate_item_ipv4(match_mask, match_value, 13645 items, tunnel, 13646 dev_flow->dv.group); 13647 matcher.priority = MLX5_PRIORITY_MAP_L3; 13648 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 13649 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 13650 if (items->mask != NULL && 13651 ((const struct rte_flow_item_ipv4 *) 13652 items->mask)->hdr.next_proto_id) { 13653 next_protocol = 13654 ((const struct rte_flow_item_ipv4 *) 13655 (items->spec))->hdr.next_proto_id; 13656 next_protocol &= 13657 ((const struct rte_flow_item_ipv4 *) 13658 (items->mask))->hdr.next_proto_id; 13659 } else { 13660 /* Reset for inner layer. */ 13661 next_protocol = 0xff; 13662 } 13663 break; 13664 case RTE_FLOW_ITEM_TYPE_IPV6: 13665 mlx5_flow_tunnel_ip_check(items, next_protocol, 13666 &item_flags, &tunnel); 13667 flow_dv_translate_item_ipv6(match_mask, match_value, 13668 items, tunnel, 13669 dev_flow->dv.group); 13670 matcher.priority = MLX5_PRIORITY_MAP_L3; 13671 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 13672 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 13673 if (items->mask != NULL && 13674 ((const struct rte_flow_item_ipv6 *) 13675 items->mask)->hdr.proto) { 13676 next_protocol = 13677 ((const struct rte_flow_item_ipv6 *) 13678 items->spec)->hdr.proto; 13679 next_protocol &= 13680 ((const struct rte_flow_item_ipv6 *) 13681 items->mask)->hdr.proto; 13682 } else { 13683 /* Reset for inner layer. */ 13684 next_protocol = 0xff; 13685 } 13686 break; 13687 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 13688 flow_dv_translate_item_ipv6_frag_ext(match_mask, 13689 match_value, 13690 items, tunnel); 13691 last_item = tunnel ? 13692 MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT : 13693 MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT; 13694 if (items->mask != NULL && 13695 ((const struct rte_flow_item_ipv6_frag_ext *) 13696 items->mask)->hdr.next_header) { 13697 next_protocol = 13698 ((const struct rte_flow_item_ipv6_frag_ext *) 13699 items->spec)->hdr.next_header; 13700 next_protocol &= 13701 ((const struct rte_flow_item_ipv6_frag_ext *) 13702 items->mask)->hdr.next_header; 13703 } else { 13704 /* Reset for inner layer. */ 13705 next_protocol = 0xff; 13706 } 13707 break; 13708 case RTE_FLOW_ITEM_TYPE_TCP: 13709 flow_dv_translate_item_tcp(match_mask, match_value, 13710 items, tunnel); 13711 matcher.priority = MLX5_PRIORITY_MAP_L4; 13712 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 13713 MLX5_FLOW_LAYER_OUTER_L4_TCP; 13714 break; 13715 case RTE_FLOW_ITEM_TYPE_UDP: 13716 flow_dv_translate_item_udp(match_mask, match_value, 13717 items, tunnel); 13718 matcher.priority = MLX5_PRIORITY_MAP_L4; 13719 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 13720 MLX5_FLOW_LAYER_OUTER_L4_UDP; 13721 break; 13722 case RTE_FLOW_ITEM_TYPE_GRE: 13723 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 13724 last_item = MLX5_FLOW_LAYER_GRE; 13725 tunnel_item = items; 13726 gre_item = items; 13727 break; 13728 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 13729 flow_dv_translate_item_gre_key(match_mask, 13730 match_value, items); 13731 last_item = MLX5_FLOW_LAYER_GRE_KEY; 13732 break; 13733 case RTE_FLOW_ITEM_TYPE_GRE_OPTION: 13734 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 13735 last_item = MLX5_FLOW_LAYER_GRE; 13736 tunnel_item = items; 13737 break; 13738 case RTE_FLOW_ITEM_TYPE_NVGRE: 13739 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 13740 last_item = MLX5_FLOW_LAYER_GRE; 13741 tunnel_item = items; 13742 break; 13743 case RTE_FLOW_ITEM_TYPE_VXLAN: 13744 flow_dv_translate_item_vxlan(dev, attr, 13745 match_mask, match_value, 13746 items, tunnel); 13747 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 13748 last_item = MLX5_FLOW_LAYER_VXLAN; 13749 break; 13750 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 13751 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 13752 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 13753 tunnel_item = items; 13754 break; 13755 case RTE_FLOW_ITEM_TYPE_GENEVE: 13756 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 13757 last_item = MLX5_FLOW_LAYER_GENEVE; 13758 tunnel_item = items; 13759 break; 13760 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: 13761 ret = flow_dv_translate_item_geneve_opt(dev, match_mask, 13762 match_value, 13763 items, error); 13764 if (ret) 13765 return rte_flow_error_set(error, -ret, 13766 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 13767 "cannot create GENEVE TLV option"); 13768 flow->geneve_tlv_option = 1; 13769 last_item = MLX5_FLOW_LAYER_GENEVE_OPT; 13770 break; 13771 case RTE_FLOW_ITEM_TYPE_MPLS: 13772 flow_dv_translate_item_mpls(match_mask, match_value, 13773 items, last_item, tunnel); 13774 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 13775 last_item = MLX5_FLOW_LAYER_MPLS; 13776 break; 13777 case RTE_FLOW_ITEM_TYPE_MARK: 13778 flow_dv_translate_item_mark(dev, match_mask, 13779 match_value, items); 13780 last_item = MLX5_FLOW_ITEM_MARK; 13781 break; 13782 case RTE_FLOW_ITEM_TYPE_META: 13783 flow_dv_translate_item_meta(dev, match_mask, 13784 match_value, attr, items); 13785 last_item = MLX5_FLOW_ITEM_METADATA; 13786 break; 13787 case RTE_FLOW_ITEM_TYPE_ICMP: 13788 flow_dv_translate_item_icmp(match_mask, match_value, 13789 items, tunnel); 13790 matcher.priority = MLX5_PRIORITY_MAP_L4; 13791 last_item = MLX5_FLOW_LAYER_ICMP; 13792 break; 13793 case RTE_FLOW_ITEM_TYPE_ICMP6: 13794 flow_dv_translate_item_icmp6(match_mask, match_value, 13795 items, tunnel); 13796 matcher.priority = MLX5_PRIORITY_MAP_L4; 13797 last_item = MLX5_FLOW_LAYER_ICMP6; 13798 break; 13799 case RTE_FLOW_ITEM_TYPE_TAG: 13800 flow_dv_translate_item_tag(dev, match_mask, 13801 match_value, items); 13802 last_item = MLX5_FLOW_ITEM_TAG; 13803 break; 13804 case MLX5_RTE_FLOW_ITEM_TYPE_TAG: 13805 flow_dv_translate_mlx5_item_tag(dev, match_mask, 13806 match_value, items); 13807 last_item = MLX5_FLOW_ITEM_TAG; 13808 break; 13809 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: 13810 flow_dv_translate_item_tx_queue(dev, match_mask, 13811 match_value, 13812 items); 13813 last_item = MLX5_FLOW_ITEM_TX_QUEUE; 13814 break; 13815 case RTE_FLOW_ITEM_TYPE_GTP: 13816 flow_dv_translate_item_gtp(match_mask, match_value, 13817 items, tunnel); 13818 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 13819 last_item = MLX5_FLOW_LAYER_GTP; 13820 break; 13821 case RTE_FLOW_ITEM_TYPE_GTP_PSC: 13822 ret = flow_dv_translate_item_gtp_psc(match_mask, 13823 match_value, 13824 items); 13825 if (ret) 13826 return rte_flow_error_set(error, -ret, 13827 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 13828 "cannot create GTP PSC item"); 13829 last_item = MLX5_FLOW_LAYER_GTP_PSC; 13830 break; 13831 case RTE_FLOW_ITEM_TYPE_ECPRI: 13832 if (!mlx5_flex_parser_ecpri_exist(dev)) { 13833 /* Create it only the first time to be used. */ 13834 ret = mlx5_flex_parser_ecpri_alloc(dev); 13835 if (ret) 13836 return rte_flow_error_set 13837 (error, -ret, 13838 RTE_FLOW_ERROR_TYPE_ITEM, 13839 NULL, 13840 "cannot create eCPRI parser"); 13841 } 13842 flow_dv_translate_item_ecpri(dev, match_mask, 13843 match_value, items, 13844 last_item); 13845 /* No other protocol should follow eCPRI layer. */ 13846 last_item = MLX5_FLOW_LAYER_ECPRI; 13847 break; 13848 case RTE_FLOW_ITEM_TYPE_INTEGRITY: 13849 flow_dv_translate_item_integrity(items, integrity_items, 13850 &last_item); 13851 break; 13852 case RTE_FLOW_ITEM_TYPE_CONNTRACK: 13853 flow_dv_translate_item_aso_ct(dev, match_mask, 13854 match_value, items); 13855 break; 13856 case RTE_FLOW_ITEM_TYPE_FLEX: 13857 flow_dv_translate_item_flex(dev, match_mask, 13858 match_value, items, 13859 dev_flow, tunnel != 0); 13860 last_item = tunnel ? MLX5_FLOW_ITEM_INNER_FLEX : 13861 MLX5_FLOW_ITEM_OUTER_FLEX; 13862 break; 13863 default: 13864 break; 13865 } 13866 item_flags |= last_item; 13867 } 13868 /* 13869 * When E-Switch mode is enabled, we have two cases where we need to 13870 * set the source port manually. 13871 * The first one, is in case of NIC ingress steering rule, and the 13872 * second is E-Switch rule where no port_id item was found. 13873 * In both cases the source port is set according the current port 13874 * in use. 13875 */ 13876 if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && priv->sh->esw_mode && 13877 !(attr->egress && !attr->transfer)) { 13878 if (flow_dv_translate_item_port_id(dev, match_mask, 13879 match_value, NULL, attr)) 13880 return -rte_errno; 13881 } 13882 if (item_flags & MLX5_FLOW_ITEM_INTEGRITY) { 13883 flow_dv_translate_item_integrity_post(match_mask, match_value, 13884 integrity_items, 13885 item_flags); 13886 } 13887 if (item_flags & MLX5_FLOW_LAYER_VXLAN_GPE) 13888 flow_dv_translate_item_vxlan_gpe(match_mask, match_value, 13889 tunnel_item, item_flags); 13890 else if (item_flags & MLX5_FLOW_LAYER_GENEVE) 13891 flow_dv_translate_item_geneve(match_mask, match_value, 13892 tunnel_item, item_flags); 13893 else if (item_flags & MLX5_FLOW_LAYER_GRE) { 13894 if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_GRE) 13895 flow_dv_translate_item_gre(match_mask, match_value, 13896 tunnel_item, item_flags); 13897 else if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_NVGRE) 13898 flow_dv_translate_item_nvgre(match_mask, match_value, 13899 tunnel_item, item_flags); 13900 else if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_GRE_OPTION) 13901 flow_dv_translate_item_gre_option(match_mask, match_value, 13902 tunnel_item, gre_item, item_flags); 13903 else 13904 MLX5_ASSERT(false); 13905 } 13906 #ifdef RTE_LIBRTE_MLX5_DEBUG 13907 MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf, 13908 dev_flow->dv.value.buf)); 13909 #endif 13910 /* 13911 * Layers may be already initialized from prefix flow if this dev_flow 13912 * is the suffix flow. 13913 */ 13914 handle->layers |= item_flags; 13915 if (action_flags & MLX5_FLOW_ACTION_RSS) 13916 flow_dv_hashfields_set(dev_flow->handle->layers, 13917 rss_desc, 13918 &dev_flow->hash_fields); 13919 /* If has RSS action in the sample action, the Sample/Mirror resource 13920 * should be registered after the hash filed be update. 13921 */ 13922 if (action_flags & MLX5_FLOW_ACTION_SAMPLE) { 13923 ret = flow_dv_translate_action_sample(dev, 13924 sample, 13925 dev_flow, attr, 13926 &num_of_dest, 13927 sample_actions, 13928 &sample_res, 13929 error); 13930 if (ret < 0) 13931 return ret; 13932 ret = flow_dv_create_action_sample(dev, 13933 dev_flow, 13934 num_of_dest, 13935 &sample_res, 13936 &mdest_res, 13937 sample_actions, 13938 action_flags, 13939 error); 13940 if (ret < 0) 13941 return rte_flow_error_set 13942 (error, rte_errno, 13943 RTE_FLOW_ERROR_TYPE_ACTION, 13944 NULL, 13945 "cannot create sample action"); 13946 if (num_of_dest > 1) { 13947 dev_flow->dv.actions[sample_act_pos] = 13948 dev_flow->dv.dest_array_res->action; 13949 } else { 13950 dev_flow->dv.actions[sample_act_pos] = 13951 dev_flow->dv.sample_res->verbs_action; 13952 } 13953 } 13954 /* 13955 * For multiple destination (sample action with ratio=1), the encap 13956 * action and port id action will be combined into group action. 13957 * So need remove the original these actions in the flow and only 13958 * use the sample action instead of. 13959 */ 13960 if (num_of_dest > 1 && 13961 (sample_act->dr_port_id_action || sample_act->dr_jump_action)) { 13962 int i; 13963 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0}; 13964 13965 for (i = 0; i < actions_n; i++) { 13966 if ((sample_act->dr_encap_action && 13967 sample_act->dr_encap_action == 13968 dev_flow->dv.actions[i]) || 13969 (sample_act->dr_port_id_action && 13970 sample_act->dr_port_id_action == 13971 dev_flow->dv.actions[i]) || 13972 (sample_act->dr_jump_action && 13973 sample_act->dr_jump_action == 13974 dev_flow->dv.actions[i])) 13975 continue; 13976 temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i]; 13977 } 13978 memcpy((void *)dev_flow->dv.actions, 13979 (void *)temp_actions, 13980 tmp_actions_n * sizeof(void *)); 13981 actions_n = tmp_actions_n; 13982 } 13983 dev_flow->dv.actions_n = actions_n; 13984 dev_flow->act_flags = action_flags; 13985 if (wks->skip_matcher_reg) 13986 return 0; 13987 /* Register matcher. */ 13988 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, 13989 matcher.mask.size); 13990 matcher.priority = mlx5_get_matcher_priority(dev, attr, 13991 matcher.priority, 13992 dev_flow->external); 13993 /** 13994 * When creating meter drop flow in drop table, using original 13995 * 5-tuple match, the matcher priority should be lower than 13996 * mtr_id matcher. 13997 */ 13998 if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER && 13999 dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP && 14000 matcher.priority <= MLX5_REG_BITS) 14001 matcher.priority += MLX5_REG_BITS; 14002 /* reserved field no needs to be set to 0 here. */ 14003 tbl_key.is_fdb = attr->transfer; 14004 tbl_key.is_egress = attr->egress; 14005 tbl_key.level = dev_flow->dv.group; 14006 tbl_key.id = dev_flow->dv.table_id; 14007 if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, 14008 tunnel, attr->group, error)) 14009 return -rte_errno; 14010 return 0; 14011 } 14012 14013 /** 14014 * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields) 14015 * and tunnel. 14016 * 14017 * @param[in, out] action 14018 * Shred RSS action holding hash RX queue objects. 14019 * @param[in] hash_fields 14020 * Defines combination of packet fields to participate in RX hash. 14021 * @param[in] tunnel 14022 * Tunnel type 14023 * @param[in] hrxq_idx 14024 * Hash RX queue index to set. 14025 * 14026 * @return 14027 * 0 on success, otherwise negative errno value. 14028 */ 14029 static int 14030 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, 14031 const uint64_t hash_fields, 14032 uint32_t hrxq_idx) 14033 { 14034 uint32_t *hrxqs = action->hrxq; 14035 14036 switch (hash_fields & ~IBV_RX_HASH_INNER) { 14037 case MLX5_RSS_HASH_IPV4: 14038 /* fall-through. */ 14039 case MLX5_RSS_HASH_IPV4_DST_ONLY: 14040 /* fall-through. */ 14041 case MLX5_RSS_HASH_IPV4_SRC_ONLY: 14042 hrxqs[0] = hrxq_idx; 14043 return 0; 14044 case MLX5_RSS_HASH_IPV4_TCP: 14045 /* fall-through. */ 14046 case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY: 14047 /* fall-through. */ 14048 case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY: 14049 hrxqs[1] = hrxq_idx; 14050 return 0; 14051 case MLX5_RSS_HASH_IPV4_UDP: 14052 /* fall-through. */ 14053 case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY: 14054 /* fall-through. */ 14055 case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY: 14056 hrxqs[2] = hrxq_idx; 14057 return 0; 14058 case MLX5_RSS_HASH_IPV6: 14059 /* fall-through. */ 14060 case MLX5_RSS_HASH_IPV6_DST_ONLY: 14061 /* fall-through. */ 14062 case MLX5_RSS_HASH_IPV6_SRC_ONLY: 14063 hrxqs[3] = hrxq_idx; 14064 return 0; 14065 case MLX5_RSS_HASH_IPV6_TCP: 14066 /* fall-through. */ 14067 case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY: 14068 /* fall-through. */ 14069 case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY: 14070 hrxqs[4] = hrxq_idx; 14071 return 0; 14072 case MLX5_RSS_HASH_IPV6_UDP: 14073 /* fall-through. */ 14074 case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY: 14075 /* fall-through. */ 14076 case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY: 14077 hrxqs[5] = hrxq_idx; 14078 return 0; 14079 case MLX5_RSS_HASH_NONE: 14080 hrxqs[6] = hrxq_idx; 14081 return 0; 14082 case MLX5_RSS_HASH_IPV4_ESP: 14083 hrxqs[7] = hrxq_idx; 14084 return 0; 14085 case MLX5_RSS_HASH_IPV6_ESP: 14086 hrxqs[8] = hrxq_idx; 14087 return 0; 14088 case MLX5_RSS_HASH_ESP_SPI: 14089 hrxqs[9] = hrxq_idx; 14090 return 0; 14091 default: 14092 return -1; 14093 } 14094 } 14095 14096 /** 14097 * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields) 14098 * and tunnel. 14099 * 14100 * @param[in] dev 14101 * Pointer to the Ethernet device structure. 14102 * @param[in] idx 14103 * Shared RSS action ID holding hash RX queue objects. 14104 * @param[in] hash_fields 14105 * Defines combination of packet fields to participate in RX hash. 14106 * @param[in] tunnel 14107 * Tunnel type 14108 * 14109 * @return 14110 * Valid hash RX queue index, otherwise 0. 14111 */ 14112 uint32_t 14113 flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx, 14114 const uint64_t hash_fields) 14115 { 14116 struct mlx5_priv *priv = dev->data->dev_private; 14117 struct mlx5_shared_action_rss *shared_rss = 14118 mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 14119 const uint32_t *hrxqs = shared_rss->hrxq; 14120 14121 switch (hash_fields & ~IBV_RX_HASH_INNER) { 14122 case MLX5_RSS_HASH_IPV4: 14123 /* fall-through. */ 14124 case MLX5_RSS_HASH_IPV4_DST_ONLY: 14125 /* fall-through. */ 14126 case MLX5_RSS_HASH_IPV4_SRC_ONLY: 14127 return hrxqs[0]; 14128 case MLX5_RSS_HASH_IPV4_TCP: 14129 /* fall-through. */ 14130 case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY: 14131 /* fall-through. */ 14132 case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY: 14133 return hrxqs[1]; 14134 case MLX5_RSS_HASH_IPV4_UDP: 14135 /* fall-through. */ 14136 case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY: 14137 /* fall-through. */ 14138 case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY: 14139 return hrxqs[2]; 14140 case MLX5_RSS_HASH_IPV6: 14141 /* fall-through. */ 14142 case MLX5_RSS_HASH_IPV6_DST_ONLY: 14143 /* fall-through. */ 14144 case MLX5_RSS_HASH_IPV6_SRC_ONLY: 14145 return hrxqs[3]; 14146 case MLX5_RSS_HASH_IPV6_TCP: 14147 /* fall-through. */ 14148 case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY: 14149 /* fall-through. */ 14150 case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY: 14151 return hrxqs[4]; 14152 case MLX5_RSS_HASH_IPV6_UDP: 14153 /* fall-through. */ 14154 case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY: 14155 /* fall-through. */ 14156 case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY: 14157 return hrxqs[5]; 14158 case MLX5_RSS_HASH_NONE: 14159 return hrxqs[6]; 14160 case MLX5_RSS_HASH_IPV4_ESP: 14161 return hrxqs[7]; 14162 case MLX5_RSS_HASH_IPV6_ESP: 14163 return hrxqs[8]; 14164 case MLX5_RSS_HASH_ESP_SPI: 14165 return hrxqs[9]; 14166 default: 14167 return 0; 14168 } 14169 14170 } 14171 14172 /** 14173 * Apply the flow to the NIC, lock free, 14174 * (mutex should be acquired by caller). 14175 * 14176 * @param[in] dev 14177 * Pointer to the Ethernet device structure. 14178 * @param[in, out] flow 14179 * Pointer to flow structure. 14180 * @param[out] error 14181 * Pointer to error structure. 14182 * 14183 * @return 14184 * 0 on success, a negative errno value otherwise and rte_errno is set. 14185 */ 14186 static int 14187 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 14188 struct rte_flow_error *error) 14189 { 14190 struct mlx5_flow_dv_workspace *dv; 14191 struct mlx5_flow_handle *dh; 14192 struct mlx5_flow_handle_dv *dv_h; 14193 struct mlx5_flow *dev_flow; 14194 struct mlx5_priv *priv = dev->data->dev_private; 14195 uint32_t handle_idx; 14196 int n; 14197 int err; 14198 int idx; 14199 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 14200 struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc; 14201 uint8_t misc_mask; 14202 14203 MLX5_ASSERT(wks); 14204 for (idx = wks->flow_idx - 1; idx >= 0; idx--) { 14205 dev_flow = &wks->flows[idx]; 14206 dv = &dev_flow->dv; 14207 dh = dev_flow->handle; 14208 dv_h = &dh->dvh; 14209 n = dv->actions_n; 14210 if (dh->fate_action == MLX5_FLOW_FATE_DROP) { 14211 if (dv->transfer) { 14212 MLX5_ASSERT(priv->sh->dr_drop_action); 14213 dv->actions[n++] = priv->sh->dr_drop_action; 14214 } else { 14215 #ifdef HAVE_MLX5DV_DR 14216 /* DR supports drop action placeholder. */ 14217 MLX5_ASSERT(priv->sh->dr_drop_action); 14218 dv->actions[n++] = dv->group ? 14219 priv->sh->dr_drop_action : 14220 priv->root_drop_action; 14221 #else 14222 /* For DV we use the explicit drop queue. */ 14223 MLX5_ASSERT(priv->drop_queue.hrxq); 14224 dv->actions[n++] = 14225 priv->drop_queue.hrxq->action; 14226 #endif 14227 } 14228 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE && 14229 !dv_h->rix_sample && !dv_h->rix_dest_array)) { 14230 struct mlx5_hrxq *hrxq; 14231 uint32_t hrxq_idx; 14232 14233 hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc, 14234 &hrxq_idx); 14235 if (!hrxq) { 14236 rte_flow_error_set 14237 (error, rte_errno, 14238 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 14239 "cannot get hash queue"); 14240 goto error; 14241 } 14242 dh->rix_hrxq = hrxq_idx; 14243 dv->actions[n++] = hrxq->action; 14244 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 14245 struct mlx5_hrxq *hrxq = NULL; 14246 uint32_t hrxq_idx; 14247 14248 hrxq_idx = flow_dv_action_rss_hrxq_lookup(dev, 14249 rss_desc->shared_rss, 14250 dev_flow->hash_fields); 14251 if (hrxq_idx) 14252 hrxq = mlx5_ipool_get 14253 (priv->sh->ipool[MLX5_IPOOL_HRXQ], 14254 hrxq_idx); 14255 if (!hrxq) { 14256 rte_flow_error_set 14257 (error, rte_errno, 14258 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 14259 "cannot get hash queue"); 14260 goto error; 14261 } 14262 dh->rix_srss = rss_desc->shared_rss; 14263 dv->actions[n++] = hrxq->action; 14264 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) { 14265 if (!priv->sh->default_miss_action) { 14266 rte_flow_error_set 14267 (error, rte_errno, 14268 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 14269 "default miss action not be created."); 14270 goto error; 14271 } 14272 dv->actions[n++] = priv->sh->default_miss_action; 14273 } 14274 misc_mask = flow_dv_matcher_enable(dv->value.buf); 14275 __flow_dv_adjust_buf_size(&dv->value.size, misc_mask); 14276 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object, 14277 (void *)&dv->value, n, 14278 dv->actions, &dh->drv_flow); 14279 if (err) { 14280 rte_flow_error_set 14281 (error, errno, 14282 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 14283 NULL, 14284 (!priv->sh->config.allow_duplicate_pattern && 14285 errno == EEXIST) ? 14286 "duplicating pattern is not allowed" : 14287 "hardware refuses to create flow"); 14288 goto error; 14289 } 14290 if (priv->vmwa_context && 14291 dh->vf_vlan.tag && !dh->vf_vlan.created) { 14292 /* 14293 * The rule contains the VLAN pattern. 14294 * For VF we are going to create VLAN 14295 * interface to make hypervisor set correct 14296 * e-Switch vport context. 14297 */ 14298 mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan); 14299 } 14300 } 14301 return 0; 14302 error: 14303 err = rte_errno; /* Save rte_errno before cleanup. */ 14304 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 14305 handle_idx, dh, next) { 14306 /* hrxq is union, don't clear it if the flag is not set. */ 14307 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) { 14308 mlx5_hrxq_release(dev, dh->rix_hrxq); 14309 dh->rix_hrxq = 0; 14310 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 14311 dh->rix_srss = 0; 14312 } 14313 if (dh->vf_vlan.tag && dh->vf_vlan.created) 14314 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); 14315 } 14316 rte_errno = err; /* Restore rte_errno. */ 14317 return -rte_errno; 14318 } 14319 14320 void 14321 flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused, 14322 struct mlx5_list_entry *entry) 14323 { 14324 struct mlx5_flow_dv_matcher *resource = container_of(entry, 14325 typeof(*resource), 14326 entry); 14327 14328 claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object)); 14329 mlx5_free(resource); 14330 } 14331 14332 /** 14333 * Release the flow matcher. 14334 * 14335 * @param dev 14336 * Pointer to Ethernet device. 14337 * @param port_id 14338 * Index to port ID action resource. 14339 * 14340 * @return 14341 * 1 while a reference on it exists, 0 when freed. 14342 */ 14343 static int 14344 flow_dv_matcher_release(struct rte_eth_dev *dev, 14345 struct mlx5_flow_handle *handle) 14346 { 14347 struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher; 14348 struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl, 14349 typeof(*tbl), tbl); 14350 int ret; 14351 14352 MLX5_ASSERT(matcher->matcher_object); 14353 ret = mlx5_list_unregister(tbl->matchers, &matcher->entry); 14354 flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl); 14355 return ret; 14356 } 14357 14358 void 14359 flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 14360 { 14361 struct mlx5_dev_ctx_shared *sh = tool_ctx; 14362 struct mlx5_flow_dv_encap_decap_resource *res = 14363 container_of(entry, typeof(*res), entry); 14364 14365 claim_zero(mlx5_flow_os_destroy_flow_action(res->action)); 14366 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx); 14367 } 14368 14369 /** 14370 * Release an encap/decap resource. 14371 * 14372 * @param dev 14373 * Pointer to Ethernet device. 14374 * @param encap_decap_idx 14375 * Index of encap decap resource. 14376 * 14377 * @return 14378 * 1 while a reference on it exists, 0 when freed. 14379 */ 14380 static int 14381 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, 14382 uint32_t encap_decap_idx) 14383 { 14384 struct mlx5_priv *priv = dev->data->dev_private; 14385 struct mlx5_flow_dv_encap_decap_resource *resource; 14386 14387 resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 14388 encap_decap_idx); 14389 if (!resource) 14390 return 0; 14391 MLX5_ASSERT(resource->action); 14392 return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry); 14393 } 14394 14395 /** 14396 * Release an jump to table action resource. 14397 * 14398 * @param dev 14399 * Pointer to Ethernet device. 14400 * @param rix_jump 14401 * Index to the jump action resource. 14402 * 14403 * @return 14404 * 1 while a reference on it exists, 0 when freed. 14405 */ 14406 static int 14407 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, 14408 uint32_t rix_jump) 14409 { 14410 struct mlx5_priv *priv = dev->data->dev_private; 14411 struct mlx5_flow_tbl_data_entry *tbl_data; 14412 14413 tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP], 14414 rix_jump); 14415 if (!tbl_data) 14416 return 0; 14417 return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl); 14418 } 14419 14420 void 14421 flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 14422 { 14423 struct mlx5_flow_dv_modify_hdr_resource *res = 14424 container_of(entry, typeof(*res), entry); 14425 struct mlx5_dev_ctx_shared *sh = tool_ctx; 14426 14427 claim_zero(mlx5_flow_os_destroy_flow_action(res->action)); 14428 mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx); 14429 } 14430 14431 /** 14432 * Release a modify-header resource. 14433 * 14434 * @param dev 14435 * Pointer to Ethernet device. 14436 * @param handle 14437 * Pointer to mlx5_flow_handle. 14438 * 14439 * @return 14440 * 1 while a reference on it exists, 0 when freed. 14441 */ 14442 static int 14443 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev, 14444 struct mlx5_flow_handle *handle) 14445 { 14446 struct mlx5_priv *priv = dev->data->dev_private; 14447 struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr; 14448 14449 MLX5_ASSERT(entry->action); 14450 return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry); 14451 } 14452 14453 void 14454 flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 14455 { 14456 struct mlx5_dev_ctx_shared *sh = tool_ctx; 14457 struct mlx5_flow_dv_port_id_action_resource *resource = 14458 container_of(entry, typeof(*resource), entry); 14459 14460 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action)); 14461 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx); 14462 } 14463 14464 /** 14465 * Release port ID action resource. 14466 * 14467 * @param dev 14468 * Pointer to Ethernet device. 14469 * @param handle 14470 * Pointer to mlx5_flow_handle. 14471 * 14472 * @return 14473 * 1 while a reference on it exists, 0 when freed. 14474 */ 14475 static int 14476 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, 14477 uint32_t port_id) 14478 { 14479 struct mlx5_priv *priv = dev->data->dev_private; 14480 struct mlx5_flow_dv_port_id_action_resource *resource; 14481 14482 resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id); 14483 if (!resource) 14484 return 0; 14485 MLX5_ASSERT(resource->action); 14486 return mlx5_list_unregister(priv->sh->port_id_action_list, 14487 &resource->entry); 14488 } 14489 14490 /** 14491 * Release shared RSS action resource. 14492 * 14493 * @param dev 14494 * Pointer to Ethernet device. 14495 * @param srss 14496 * Shared RSS action index. 14497 */ 14498 static void 14499 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss) 14500 { 14501 struct mlx5_priv *priv = dev->data->dev_private; 14502 struct mlx5_shared_action_rss *shared_rss; 14503 14504 shared_rss = mlx5_ipool_get 14505 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss); 14506 __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED); 14507 } 14508 14509 void 14510 flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 14511 { 14512 struct mlx5_dev_ctx_shared *sh = tool_ctx; 14513 struct mlx5_flow_dv_push_vlan_action_resource *resource = 14514 container_of(entry, typeof(*resource), entry); 14515 14516 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action)); 14517 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx); 14518 } 14519 14520 /** 14521 * Release push vlan action resource. 14522 * 14523 * @param dev 14524 * Pointer to Ethernet device. 14525 * @param handle 14526 * Pointer to mlx5_flow_handle. 14527 * 14528 * @return 14529 * 1 while a reference on it exists, 0 when freed. 14530 */ 14531 static int 14532 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, 14533 struct mlx5_flow_handle *handle) 14534 { 14535 struct mlx5_priv *priv = dev->data->dev_private; 14536 struct mlx5_flow_dv_push_vlan_action_resource *resource; 14537 uint32_t idx = handle->dvh.rix_push_vlan; 14538 14539 resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); 14540 if (!resource) 14541 return 0; 14542 MLX5_ASSERT(resource->action); 14543 return mlx5_list_unregister(priv->sh->push_vlan_action_list, 14544 &resource->entry); 14545 } 14546 14547 /** 14548 * Release the fate resource. 14549 * 14550 * @param dev 14551 * Pointer to Ethernet device. 14552 * @param handle 14553 * Pointer to mlx5_flow_handle. 14554 */ 14555 static void 14556 flow_dv_fate_resource_release(struct rte_eth_dev *dev, 14557 struct mlx5_flow_handle *handle) 14558 { 14559 if (!handle->rix_fate) 14560 return; 14561 switch (handle->fate_action) { 14562 case MLX5_FLOW_FATE_QUEUE: 14563 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array) 14564 mlx5_hrxq_release(dev, handle->rix_hrxq); 14565 break; 14566 case MLX5_FLOW_FATE_JUMP: 14567 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump); 14568 break; 14569 case MLX5_FLOW_FATE_PORT_ID: 14570 flow_dv_port_id_action_resource_release(dev, 14571 handle->rix_port_id_action); 14572 break; 14573 default: 14574 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action); 14575 break; 14576 } 14577 handle->rix_fate = 0; 14578 } 14579 14580 void 14581 flow_dv_sample_remove_cb(void *tool_ctx __rte_unused, 14582 struct mlx5_list_entry *entry) 14583 { 14584 struct mlx5_flow_dv_sample_resource *resource = container_of(entry, 14585 typeof(*resource), 14586 entry); 14587 struct rte_eth_dev *dev = resource->dev; 14588 struct mlx5_priv *priv = dev->data->dev_private; 14589 14590 if (resource->verbs_action) 14591 claim_zero(mlx5_flow_os_destroy_flow_action 14592 (resource->verbs_action)); 14593 if (resource->normal_path_tbl) 14594 flow_dv_tbl_resource_release(MLX5_SH(dev), 14595 resource->normal_path_tbl); 14596 flow_dv_sample_sub_actions_release(dev, &resource->sample_idx); 14597 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx); 14598 DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource); 14599 } 14600 14601 /** 14602 * Release an sample resource. 14603 * 14604 * @param dev 14605 * Pointer to Ethernet device. 14606 * @param handle 14607 * Pointer to mlx5_flow_handle. 14608 * 14609 * @return 14610 * 1 while a reference on it exists, 0 when freed. 14611 */ 14612 static int 14613 flow_dv_sample_resource_release(struct rte_eth_dev *dev, 14614 struct mlx5_flow_handle *handle) 14615 { 14616 struct mlx5_priv *priv = dev->data->dev_private; 14617 struct mlx5_flow_dv_sample_resource *resource; 14618 14619 resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE], 14620 handle->dvh.rix_sample); 14621 if (!resource) 14622 return 0; 14623 MLX5_ASSERT(resource->verbs_action); 14624 return mlx5_list_unregister(priv->sh->sample_action_list, 14625 &resource->entry); 14626 } 14627 14628 void 14629 flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused, 14630 struct mlx5_list_entry *entry) 14631 { 14632 struct mlx5_flow_dv_dest_array_resource *resource = 14633 container_of(entry, typeof(*resource), entry); 14634 struct rte_eth_dev *dev = resource->dev; 14635 struct mlx5_priv *priv = dev->data->dev_private; 14636 uint32_t i = 0; 14637 14638 MLX5_ASSERT(resource->action); 14639 if (resource->action) 14640 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action)); 14641 for (; i < resource->num_of_dest; i++) 14642 flow_dv_sample_sub_actions_release(dev, 14643 &resource->sample_idx[i]); 14644 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx); 14645 DRV_LOG(DEBUG, "destination array resource %p: removed", 14646 (void *)resource); 14647 } 14648 14649 /** 14650 * Release an destination array resource. 14651 * 14652 * @param dev 14653 * Pointer to Ethernet device. 14654 * @param handle 14655 * Pointer to mlx5_flow_handle. 14656 * 14657 * @return 14658 * 1 while a reference on it exists, 0 when freed. 14659 */ 14660 static int 14661 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev, 14662 struct mlx5_flow_handle *handle) 14663 { 14664 struct mlx5_priv *priv = dev->data->dev_private; 14665 struct mlx5_flow_dv_dest_array_resource *resource; 14666 14667 resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], 14668 handle->dvh.rix_dest_array); 14669 if (!resource) 14670 return 0; 14671 MLX5_ASSERT(resource->action); 14672 return mlx5_list_unregister(priv->sh->dest_array_list, 14673 &resource->entry); 14674 } 14675 14676 static void 14677 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev) 14678 { 14679 struct mlx5_priv *priv = dev->data->dev_private; 14680 struct mlx5_dev_ctx_shared *sh = priv->sh; 14681 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource = 14682 sh->geneve_tlv_option_resource; 14683 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 14684 if (geneve_opt_resource) { 14685 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1, 14686 __ATOMIC_RELAXED))) { 14687 claim_zero(mlx5_devx_cmd_destroy 14688 (geneve_opt_resource->obj)); 14689 mlx5_free(sh->geneve_tlv_option_resource); 14690 sh->geneve_tlv_option_resource = NULL; 14691 } 14692 } 14693 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 14694 } 14695 14696 /** 14697 * Remove the flow from the NIC but keeps it in memory. 14698 * Lock free, (mutex should be acquired by caller). 14699 * 14700 * @param[in] dev 14701 * Pointer to Ethernet device. 14702 * @param[in, out] flow 14703 * Pointer to flow structure. 14704 */ 14705 static void 14706 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 14707 { 14708 struct mlx5_flow_handle *dh; 14709 uint32_t handle_idx; 14710 struct mlx5_priv *priv = dev->data->dev_private; 14711 14712 if (!flow) 14713 return; 14714 handle_idx = flow->dev_handles; 14715 while (handle_idx) { 14716 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 14717 handle_idx); 14718 if (!dh) 14719 return; 14720 if (dh->drv_flow) { 14721 claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow)); 14722 dh->drv_flow = NULL; 14723 } 14724 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) 14725 flow_dv_fate_resource_release(dev, dh); 14726 if (dh->vf_vlan.tag && dh->vf_vlan.created) 14727 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); 14728 handle_idx = dh->next.next; 14729 } 14730 } 14731 14732 /** 14733 * Remove the flow from the NIC and the memory. 14734 * Lock free, (mutex should be acquired by caller). 14735 * 14736 * @param[in] dev 14737 * Pointer to the Ethernet device structure. 14738 * @param[in, out] flow 14739 * Pointer to flow structure. 14740 */ 14741 static void 14742 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 14743 { 14744 struct mlx5_flow_handle *dev_handle; 14745 struct mlx5_priv *priv = dev->data->dev_private; 14746 struct mlx5_flow_meter_info *fm = NULL; 14747 uint32_t srss = 0; 14748 14749 if (!flow) 14750 return; 14751 flow_dv_remove(dev, flow); 14752 if (flow->counter) { 14753 flow_dv_counter_free(dev, flow->counter); 14754 flow->counter = 0; 14755 } 14756 if (flow->meter) { 14757 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 14758 if (fm) 14759 mlx5_flow_meter_detach(priv, fm); 14760 flow->meter = 0; 14761 } 14762 /* Keep the current age handling by default. */ 14763 if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct) 14764 flow_dv_aso_ct_release(dev, flow->ct, NULL); 14765 else if (flow->age) 14766 flow_dv_aso_age_release(dev, flow->age); 14767 if (flow->geneve_tlv_option) { 14768 flow_dv_geneve_tlv_option_resource_release(dev); 14769 flow->geneve_tlv_option = 0; 14770 } 14771 while (flow->dev_handles) { 14772 uint32_t tmp_idx = flow->dev_handles; 14773 14774 dev_handle = mlx5_ipool_get(priv->sh->ipool 14775 [MLX5_IPOOL_MLX5_FLOW], tmp_idx); 14776 if (!dev_handle) 14777 return; 14778 flow->dev_handles = dev_handle->next.next; 14779 while (dev_handle->flex_item) { 14780 int index = rte_bsf32(dev_handle->flex_item); 14781 14782 mlx5_flex_release_index(dev, index); 14783 dev_handle->flex_item &= ~(uint8_t)RTE_BIT32(index); 14784 } 14785 if (dev_handle->dvh.matcher) 14786 flow_dv_matcher_release(dev, dev_handle); 14787 if (dev_handle->dvh.rix_sample) 14788 flow_dv_sample_resource_release(dev, dev_handle); 14789 if (dev_handle->dvh.rix_dest_array) 14790 flow_dv_dest_array_resource_release(dev, dev_handle); 14791 if (dev_handle->dvh.rix_encap_decap) 14792 flow_dv_encap_decap_resource_release(dev, 14793 dev_handle->dvh.rix_encap_decap); 14794 if (dev_handle->dvh.modify_hdr) 14795 flow_dv_modify_hdr_resource_release(dev, dev_handle); 14796 if (dev_handle->dvh.rix_push_vlan) 14797 flow_dv_push_vlan_action_resource_release(dev, 14798 dev_handle); 14799 if (dev_handle->dvh.rix_tag) 14800 flow_dv_tag_release(dev, 14801 dev_handle->dvh.rix_tag); 14802 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS) 14803 flow_dv_fate_resource_release(dev, dev_handle); 14804 else if (!srss) 14805 srss = dev_handle->rix_srss; 14806 if (fm && dev_handle->is_meter_flow_id && 14807 dev_handle->split_flow_id) 14808 mlx5_ipool_free(fm->flow_ipool, 14809 dev_handle->split_flow_id); 14810 else if (dev_handle->split_flow_id && 14811 !dev_handle->is_meter_flow_id) 14812 mlx5_ipool_free(priv->sh->ipool 14813 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 14814 dev_handle->split_flow_id); 14815 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 14816 tmp_idx); 14817 } 14818 if (srss) 14819 flow_dv_shared_rss_action_release(dev, srss); 14820 } 14821 14822 /** 14823 * Release array of hash RX queue objects. 14824 * Helper function. 14825 * 14826 * @param[in] dev 14827 * Pointer to the Ethernet device structure. 14828 * @param[in, out] hrxqs 14829 * Array of hash RX queue objects. 14830 * 14831 * @return 14832 * Total number of references to hash RX queue objects in *hrxqs* array 14833 * after this operation. 14834 */ 14835 static int 14836 __flow_dv_hrxqs_release(struct rte_eth_dev *dev, 14837 uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN]) 14838 { 14839 size_t i; 14840 int remaining = 0; 14841 14842 for (i = 0; i < RTE_DIM(*hrxqs); i++) { 14843 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]); 14844 14845 if (!ret) 14846 (*hrxqs)[i] = 0; 14847 remaining += ret; 14848 } 14849 return remaining; 14850 } 14851 14852 /** 14853 * Release all hash RX queue objects representing shared RSS action. 14854 * 14855 * @param[in] dev 14856 * Pointer to the Ethernet device structure. 14857 * @param[in, out] action 14858 * Shared RSS action to remove hash RX queue objects from. 14859 * 14860 * @return 14861 * Total number of references to hash RX queue objects stored in *action* 14862 * after this operation. 14863 * Expected to be 0 if no external references held. 14864 */ 14865 static int 14866 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, 14867 struct mlx5_shared_action_rss *shared_rss) 14868 { 14869 return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq); 14870 } 14871 14872 /** 14873 * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to 14874 * user input. 14875 * 14876 * Only one hash value is available for one L3+L4 combination: 14877 * for example: 14878 * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and 14879 * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share 14880 * same slot in mlx5_rss_hash_fields. 14881 * 14882 * @param[in] orig_rss_types 14883 * RSS type as provided in shared RSS action. 14884 * @param[in, out] hash_field 14885 * hash_field variable needed to be adjusted. 14886 * 14887 * @return 14888 * void 14889 */ 14890 void 14891 flow_dv_action_rss_l34_hash_adjust(uint64_t orig_rss_types, 14892 uint64_t *hash_field) 14893 { 14894 uint64_t rss_types = rte_eth_rss_hf_refine(orig_rss_types); 14895 14896 switch (*hash_field & ~IBV_RX_HASH_INNER) { 14897 case MLX5_RSS_HASH_IPV4: 14898 if (rss_types & MLX5_IPV4_LAYER_TYPES) { 14899 *hash_field &= ~MLX5_RSS_HASH_IPV4; 14900 if (rss_types & RTE_ETH_RSS_L3_DST_ONLY) 14901 *hash_field |= IBV_RX_HASH_DST_IPV4; 14902 else if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY) 14903 *hash_field |= IBV_RX_HASH_SRC_IPV4; 14904 else 14905 *hash_field |= MLX5_RSS_HASH_IPV4; 14906 } 14907 return; 14908 case MLX5_RSS_HASH_IPV6: 14909 if (rss_types & MLX5_IPV6_LAYER_TYPES) { 14910 *hash_field &= ~MLX5_RSS_HASH_IPV6; 14911 if (rss_types & RTE_ETH_RSS_L3_DST_ONLY) 14912 *hash_field |= IBV_RX_HASH_DST_IPV6; 14913 else if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY) 14914 *hash_field |= IBV_RX_HASH_SRC_IPV6; 14915 else 14916 *hash_field |= MLX5_RSS_HASH_IPV6; 14917 } 14918 return; 14919 case MLX5_RSS_HASH_IPV4_UDP: 14920 /* fall-through. */ 14921 case MLX5_RSS_HASH_IPV6_UDP: 14922 if (rss_types & RTE_ETH_RSS_UDP) { 14923 *hash_field &= ~MLX5_UDP_IBV_RX_HASH; 14924 if (rss_types & RTE_ETH_RSS_L4_DST_ONLY) 14925 *hash_field |= IBV_RX_HASH_DST_PORT_UDP; 14926 else if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY) 14927 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP; 14928 else 14929 *hash_field |= MLX5_UDP_IBV_RX_HASH; 14930 } 14931 return; 14932 case MLX5_RSS_HASH_IPV4_TCP: 14933 /* fall-through. */ 14934 case MLX5_RSS_HASH_IPV6_TCP: 14935 if (rss_types & RTE_ETH_RSS_TCP) { 14936 *hash_field &= ~MLX5_TCP_IBV_RX_HASH; 14937 if (rss_types & RTE_ETH_RSS_L4_DST_ONLY) 14938 *hash_field |= IBV_RX_HASH_DST_PORT_TCP; 14939 else if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY) 14940 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP; 14941 else 14942 *hash_field |= MLX5_TCP_IBV_RX_HASH; 14943 } 14944 return; 14945 default: 14946 return; 14947 } 14948 } 14949 14950 /** 14951 * Setup shared RSS action. 14952 * Prepare set of hash RX queue objects sufficient to handle all valid 14953 * hash_fields combinations (see enum ibv_rx_hash_fields). 14954 * 14955 * @param[in] dev 14956 * Pointer to the Ethernet device structure. 14957 * @param[in] action_idx 14958 * Shared RSS action ipool index. 14959 * @param[in, out] action 14960 * Partially initialized shared RSS action. 14961 * @param[out] error 14962 * Perform verbose error reporting if not NULL. Initialized in case of 14963 * error only. 14964 * 14965 * @return 14966 * 0 on success, otherwise negative errno value. 14967 */ 14968 static int 14969 __flow_dv_action_rss_setup(struct rte_eth_dev *dev, 14970 uint32_t action_idx, 14971 struct mlx5_shared_action_rss *shared_rss, 14972 struct rte_flow_error *error) 14973 { 14974 struct mlx5_priv *priv = dev->data->dev_private; 14975 struct mlx5_flow_rss_desc rss_desc = { 0 }; 14976 size_t i; 14977 int err; 14978 14979 shared_rss->ind_tbl = mlx5_ind_table_obj_new 14980 (dev, shared_rss->origin.queue, 14981 shared_rss->origin.queue_num, 14982 true, 14983 !!dev->data->dev_started); 14984 if (!shared_rss->ind_tbl) 14985 return rte_flow_error_set(error, rte_errno, 14986 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 14987 "cannot setup indirection table"); 14988 memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN); 14989 rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN; 14990 rss_desc.const_q = shared_rss->origin.queue; 14991 rss_desc.queue_num = shared_rss->origin.queue_num; 14992 /* Set non-zero value to indicate a shared RSS. */ 14993 rss_desc.shared_rss = action_idx; 14994 rss_desc.ind_tbl = shared_rss->ind_tbl; 14995 if (priv->sh->config.dv_flow_en == 2) 14996 rss_desc.hws_flags = MLX5DR_ACTION_FLAG_HWS_RX; 14997 for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) { 14998 struct mlx5_hrxq *hrxq; 14999 uint64_t hash_fields = mlx5_rss_hash_fields[i]; 15000 int tunnel = 0; 15001 15002 flow_dv_action_rss_l34_hash_adjust(shared_rss->origin.types, 15003 &hash_fields); 15004 if (shared_rss->origin.level > 1) { 15005 hash_fields |= IBV_RX_HASH_INNER; 15006 tunnel = 1; 15007 } 15008 rss_desc.tunnel = tunnel; 15009 rss_desc.hash_fields = hash_fields; 15010 hrxq = mlx5_hrxq_get(dev, &rss_desc); 15011 if (!hrxq) { 15012 rte_flow_error_set 15013 (error, rte_errno, 15014 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 15015 "cannot get hash queue"); 15016 goto error_hrxq_new; 15017 } 15018 err = __flow_dv_action_rss_hrxq_set 15019 (shared_rss, hash_fields, hrxq->idx); 15020 MLX5_ASSERT(!err); 15021 } 15022 return 0; 15023 error_hrxq_new: 15024 err = rte_errno; 15025 __flow_dv_action_rss_hrxqs_release(dev, shared_rss); 15026 if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true)) 15027 shared_rss->ind_tbl = NULL; 15028 rte_errno = err; 15029 return -rte_errno; 15030 } 15031 15032 /** 15033 * Create shared RSS action. 15034 * 15035 * @param[in] dev 15036 * Pointer to the Ethernet device structure. 15037 * @param[in] conf 15038 * Shared action configuration. 15039 * @param[in] rss 15040 * RSS action specification used to create shared action. 15041 * @param[out] error 15042 * Perform verbose error reporting if not NULL. Initialized in case of 15043 * error only. 15044 * 15045 * @return 15046 * A valid shared action ID in case of success, 0 otherwise and 15047 * rte_errno is set. 15048 */ 15049 static uint32_t 15050 __flow_dv_action_rss_create(struct rte_eth_dev *dev, 15051 const struct rte_flow_indir_action_conf *conf, 15052 const struct rte_flow_action_rss *rss, 15053 struct rte_flow_error *error) 15054 { 15055 struct mlx5_priv *priv = dev->data->dev_private; 15056 struct mlx5_shared_action_rss *shared_rss = NULL; 15057 struct rte_flow_action_rss *origin; 15058 const uint8_t *rss_key; 15059 uint32_t idx; 15060 15061 RTE_SET_USED(conf); 15062 shared_rss = mlx5_ipool_zmalloc 15063 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx); 15064 if (!shared_rss) { 15065 rte_flow_error_set(error, ENOMEM, 15066 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 15067 "cannot allocate resource memory"); 15068 goto error_rss_init; 15069 } 15070 if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) { 15071 rte_flow_error_set(error, E2BIG, 15072 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 15073 "rss action number out of range"); 15074 goto error_rss_init; 15075 } 15076 origin = &shared_rss->origin; 15077 origin->func = rss->func; 15078 origin->level = rss->level; 15079 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 15080 origin->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 15081 /* NULL RSS key indicates default RSS key. */ 15082 rss_key = !rss->key ? rss_hash_default_key : rss->key; 15083 memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 15084 origin->key = &shared_rss->key[0]; 15085 origin->key_len = MLX5_RSS_HASH_KEY_LEN; 15086 origin->queue = rss->queue; 15087 origin->queue_num = rss->queue_num; 15088 if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error)) 15089 goto error_rss_init; 15090 /* Update queue with indirect table queue memoyr. */ 15091 origin->queue = shared_rss->ind_tbl->queues; 15092 rte_spinlock_init(&shared_rss->action_rss_sl); 15093 __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED); 15094 rte_spinlock_lock(&priv->shared_act_sl); 15095 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 15096 &priv->rss_shared_actions, idx, shared_rss, next); 15097 rte_spinlock_unlock(&priv->shared_act_sl); 15098 return idx; 15099 error_rss_init: 15100 if (shared_rss) { 15101 if (shared_rss->ind_tbl) 15102 mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, 15103 !!dev->data->dev_started); 15104 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 15105 idx); 15106 } 15107 return 0; 15108 } 15109 15110 /** 15111 * Destroy the shared RSS action. 15112 * Release related hash RX queue objects. 15113 * 15114 * @param[in] dev 15115 * Pointer to the Ethernet device structure. 15116 * @param[in] idx 15117 * The shared RSS action object ID to be removed. 15118 * @param[out] error 15119 * Perform verbose error reporting if not NULL. Initialized in case of 15120 * error only. 15121 * 15122 * @return 15123 * 0 on success, otherwise negative errno value. 15124 */ 15125 static int 15126 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, 15127 struct rte_flow_error *error) 15128 { 15129 struct mlx5_priv *priv = dev->data->dev_private; 15130 struct mlx5_shared_action_rss *shared_rss = 15131 mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 15132 uint32_t old_refcnt = 1; 15133 int remaining; 15134 15135 if (!shared_rss) 15136 return rte_flow_error_set(error, EINVAL, 15137 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 15138 "invalid shared action"); 15139 if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt, 15140 0, 0, __ATOMIC_ACQUIRE, 15141 __ATOMIC_RELAXED)) 15142 return rte_flow_error_set(error, EBUSY, 15143 RTE_FLOW_ERROR_TYPE_ACTION, 15144 NULL, 15145 "shared rss has references"); 15146 remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss); 15147 if (remaining) 15148 return rte_flow_error_set(error, EBUSY, 15149 RTE_FLOW_ERROR_TYPE_ACTION, 15150 NULL, 15151 "shared rss hrxq has references"); 15152 remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, 15153 !!dev->data->dev_started); 15154 if (remaining) 15155 return rte_flow_error_set(error, EBUSY, 15156 RTE_FLOW_ERROR_TYPE_ACTION, 15157 NULL, 15158 "shared rss indirection table has" 15159 " references"); 15160 rte_spinlock_lock(&priv->shared_act_sl); 15161 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 15162 &priv->rss_shared_actions, idx, shared_rss, next); 15163 rte_spinlock_unlock(&priv->shared_act_sl); 15164 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 15165 idx); 15166 return 0; 15167 } 15168 15169 /** 15170 * Create indirect action, lock free, 15171 * (mutex should be acquired by caller). 15172 * Dispatcher for action type specific call. 15173 * 15174 * @param[in] dev 15175 * Pointer to the Ethernet device structure. 15176 * @param[in] conf 15177 * Shared action configuration. 15178 * @param[in] action 15179 * Action specification used to create indirect action. 15180 * @param[out] error 15181 * Perform verbose error reporting if not NULL. Initialized in case of 15182 * error only. 15183 * 15184 * @return 15185 * A valid shared action handle in case of success, NULL otherwise and 15186 * rte_errno is set. 15187 */ 15188 struct rte_flow_action_handle * 15189 flow_dv_action_create(struct rte_eth_dev *dev, 15190 const struct rte_flow_indir_action_conf *conf, 15191 const struct rte_flow_action *action, 15192 struct rte_flow_error *err) 15193 { 15194 struct mlx5_priv *priv = dev->data->dev_private; 15195 uint32_t age_idx = 0; 15196 uint32_t idx = 0; 15197 uint32_t ret = 0; 15198 15199 switch (action->type) { 15200 case RTE_FLOW_ACTION_TYPE_RSS: 15201 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err); 15202 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS << 15203 MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret; 15204 break; 15205 case RTE_FLOW_ACTION_TYPE_AGE: 15206 age_idx = flow_dv_aso_age_alloc(dev, err); 15207 if (!age_idx) { 15208 ret = -rte_errno; 15209 break; 15210 } 15211 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE << 15212 MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx; 15213 flow_dv_aso_age_params_init(dev, age_idx, 15214 ((const struct rte_flow_action_age *) 15215 action->conf)->context ? 15216 ((const struct rte_flow_action_age *) 15217 action->conf)->context : 15218 (void *)(uintptr_t)idx, 15219 ((const struct rte_flow_action_age *) 15220 action->conf)->timeout); 15221 ret = age_idx; 15222 break; 15223 case RTE_FLOW_ACTION_TYPE_COUNT: 15224 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL); 15225 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT << 15226 MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret; 15227 break; 15228 case RTE_FLOW_ACTION_TYPE_CONNTRACK: 15229 ret = flow_dv_translate_create_conntrack(dev, action->conf, 15230 err); 15231 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret); 15232 break; 15233 default: 15234 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 15235 NULL, "action type not supported"); 15236 break; 15237 } 15238 return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL; 15239 } 15240 15241 /** 15242 * Destroy the indirect action. 15243 * Release action related resources on the NIC and the memory. 15244 * Lock free, (mutex should be acquired by caller). 15245 * Dispatcher for action type specific call. 15246 * 15247 * @param[in] dev 15248 * Pointer to the Ethernet device structure. 15249 * @param[in] handle 15250 * The indirect action object handle to be removed. 15251 * @param[out] error 15252 * Perform verbose error reporting if not NULL. Initialized in case of 15253 * error only. 15254 * 15255 * @return 15256 * 0 on success, otherwise negative errno value. 15257 */ 15258 int 15259 flow_dv_action_destroy(struct rte_eth_dev *dev, 15260 struct rte_flow_action_handle *handle, 15261 struct rte_flow_error *error) 15262 { 15263 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 15264 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 15265 uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 15266 struct mlx5_flow_counter *cnt; 15267 uint32_t no_flow_refcnt = 1; 15268 int ret; 15269 15270 switch (type) { 15271 case MLX5_INDIRECT_ACTION_TYPE_RSS: 15272 return __flow_dv_action_rss_release(dev, idx, error); 15273 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 15274 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL); 15275 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt, 15276 &no_flow_refcnt, 1, false, 15277 __ATOMIC_ACQUIRE, 15278 __ATOMIC_RELAXED)) 15279 return rte_flow_error_set(error, EBUSY, 15280 RTE_FLOW_ERROR_TYPE_ACTION, 15281 NULL, 15282 "Indirect count action has references"); 15283 flow_dv_counter_free(dev, idx); 15284 return 0; 15285 case MLX5_INDIRECT_ACTION_TYPE_AGE: 15286 ret = flow_dv_aso_age_release(dev, idx); 15287 if (ret) 15288 /* 15289 * In this case, the last flow has a reference will 15290 * actually release the age action. 15291 */ 15292 DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was" 15293 " released with references %d.", idx, ret); 15294 return 0; 15295 case MLX5_INDIRECT_ACTION_TYPE_CT: 15296 ret = flow_dv_aso_ct_release(dev, idx, error); 15297 if (ret < 0) 15298 return ret; 15299 if (ret > 0) 15300 DRV_LOG(DEBUG, "Connection tracking object %u still " 15301 "has references %d.", idx, ret); 15302 return 0; 15303 default: 15304 return rte_flow_error_set(error, ENOTSUP, 15305 RTE_FLOW_ERROR_TYPE_ACTION, 15306 NULL, 15307 "action type not supported"); 15308 } 15309 } 15310 15311 /** 15312 * Updates in place shared RSS action configuration. 15313 * 15314 * @param[in] dev 15315 * Pointer to the Ethernet device structure. 15316 * @param[in] idx 15317 * The shared RSS action object ID to be updated. 15318 * @param[in] action_conf 15319 * RSS action specification used to modify *shared_rss*. 15320 * @param[out] error 15321 * Perform verbose error reporting if not NULL. Initialized in case of 15322 * error only. 15323 * 15324 * @return 15325 * 0 on success, otherwise negative errno value. 15326 * @note: currently only support update of RSS queues. 15327 */ 15328 static int 15329 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx, 15330 const struct rte_flow_action_rss *action_conf, 15331 struct rte_flow_error *error) 15332 { 15333 struct mlx5_priv *priv = dev->data->dev_private; 15334 struct mlx5_shared_action_rss *shared_rss = 15335 mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 15336 int ret = 0; 15337 void *queue = NULL; 15338 void *queue_i = NULL; 15339 uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t); 15340 bool dev_started = !!dev->data->dev_started; 15341 15342 if (!shared_rss) 15343 return rte_flow_error_set(error, EINVAL, 15344 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 15345 "invalid shared action to update"); 15346 if (priv->obj_ops.ind_table_modify == NULL) 15347 return rte_flow_error_set(error, ENOTSUP, 15348 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 15349 "cannot modify indirection table"); 15350 queue = mlx5_malloc(MLX5_MEM_ZERO, 15351 RTE_ALIGN_CEIL(queue_size, sizeof(void *)), 15352 0, SOCKET_ID_ANY); 15353 if (!queue) 15354 return rte_flow_error_set(error, ENOMEM, 15355 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 15356 NULL, 15357 "cannot allocate resource memory"); 15358 memcpy(queue, action_conf->queue, queue_size); 15359 MLX5_ASSERT(shared_rss->ind_tbl); 15360 rte_spinlock_lock(&shared_rss->action_rss_sl); 15361 queue_i = shared_rss->ind_tbl->queues; 15362 ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl, 15363 queue, action_conf->queue_num, 15364 true /* standalone */, 15365 dev_started /* ref_new_qs */, 15366 dev_started /* deref_old_qs */); 15367 if (ret) { 15368 ret = rte_flow_error_set(error, rte_errno, 15369 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 15370 "cannot update indirection table"); 15371 } else { 15372 /* Restore the queue to indirect table internal queue. */ 15373 memcpy(queue_i, queue, queue_size); 15374 shared_rss->ind_tbl->queues = queue_i; 15375 shared_rss->origin.queue_num = action_conf->queue_num; 15376 } 15377 mlx5_free(queue); 15378 rte_spinlock_unlock(&shared_rss->action_rss_sl); 15379 return ret; 15380 } 15381 15382 /* 15383 * Updates in place conntrack context or direction. 15384 * Context update should be synchronized. 15385 * 15386 * @param[in] dev 15387 * Pointer to the Ethernet device structure. 15388 * @param[in] idx 15389 * The conntrack object ID to be updated. 15390 * @param[in] update 15391 * Pointer to the structure of information to update. 15392 * @param[out] error 15393 * Perform verbose error reporting if not NULL. Initialized in case of 15394 * error only. 15395 * 15396 * @return 15397 * 0 on success, otherwise negative errno value. 15398 */ 15399 static int 15400 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx, 15401 const struct rte_flow_modify_conntrack *update, 15402 struct rte_flow_error *error) 15403 { 15404 struct mlx5_priv *priv = dev->data->dev_private; 15405 struct mlx5_aso_ct_action *ct; 15406 const struct rte_flow_action_conntrack *new_prf; 15407 int ret = 0; 15408 uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx); 15409 uint32_t dev_idx; 15410 15411 if (PORT_ID(priv) != owner) 15412 return rte_flow_error_set(error, EACCES, 15413 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 15414 NULL, 15415 "CT object owned by another port"); 15416 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx); 15417 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx); 15418 if (!ct->refcnt) 15419 return rte_flow_error_set(error, ENOMEM, 15420 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 15421 NULL, 15422 "CT object is inactive"); 15423 new_prf = &update->new_ct; 15424 if (update->direction) 15425 ct->is_original = !!new_prf->is_original_dir; 15426 if (update->state) { 15427 /* Only validate the profile when it needs to be updated. */ 15428 ret = mlx5_validate_action_ct(dev, new_prf, error); 15429 if (ret) 15430 return ret; 15431 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf); 15432 if (ret) 15433 return rte_flow_error_set(error, EIO, 15434 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 15435 NULL, 15436 "Failed to send CT context update WQE"); 15437 /* Block until ready or a failure. */ 15438 ret = mlx5_aso_ct_available(priv->sh, ct); 15439 if (ret) 15440 rte_flow_error_set(error, rte_errno, 15441 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 15442 NULL, 15443 "Timeout to get the CT update"); 15444 } 15445 return ret; 15446 } 15447 15448 /** 15449 * Updates in place shared action configuration, lock free, 15450 * (mutex should be acquired by caller). 15451 * 15452 * @param[in] dev 15453 * Pointer to the Ethernet device structure. 15454 * @param[in] handle 15455 * The indirect action object handle to be updated. 15456 * @param[in] update 15457 * Action specification used to modify the action pointed by *handle*. 15458 * *update* could be of same type with the action pointed by the *handle* 15459 * handle argument, or some other structures like a wrapper, depending on 15460 * the indirect action type. 15461 * @param[out] error 15462 * Perform verbose error reporting if not NULL. Initialized in case of 15463 * error only. 15464 * 15465 * @return 15466 * 0 on success, otherwise negative errno value. 15467 */ 15468 int 15469 flow_dv_action_update(struct rte_eth_dev *dev, 15470 struct rte_flow_action_handle *handle, 15471 const void *update, 15472 struct rte_flow_error *err) 15473 { 15474 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 15475 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 15476 uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 15477 const void *action_conf; 15478 15479 switch (type) { 15480 case MLX5_INDIRECT_ACTION_TYPE_RSS: 15481 action_conf = ((const struct rte_flow_action *)update)->conf; 15482 return __flow_dv_action_rss_update(dev, idx, action_conf, err); 15483 case MLX5_INDIRECT_ACTION_TYPE_CT: 15484 return __flow_dv_action_ct_update(dev, idx, update, err); 15485 default: 15486 return rte_flow_error_set(err, ENOTSUP, 15487 RTE_FLOW_ERROR_TYPE_ACTION, 15488 NULL, 15489 "action type update not supported"); 15490 } 15491 } 15492 15493 /** 15494 * Destroy the meter sub policy table rules. 15495 * Lock free, (mutex should be acquired by caller). 15496 * 15497 * @param[in] dev 15498 * Pointer to Ethernet device. 15499 * @param[in] sub_policy 15500 * Pointer to meter sub policy table. 15501 */ 15502 static void 15503 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev, 15504 struct mlx5_flow_meter_sub_policy *sub_policy) 15505 { 15506 struct mlx5_priv *priv = dev->data->dev_private; 15507 struct mlx5_flow_tbl_data_entry *tbl; 15508 struct mlx5_flow_meter_policy *policy = sub_policy->main_policy; 15509 struct mlx5_flow_meter_info *next_fm; 15510 struct mlx5_sub_policy_color_rule *color_rule; 15511 void *tmp; 15512 uint32_t i; 15513 15514 for (i = 0; i < RTE_COLORS; i++) { 15515 next_fm = NULL; 15516 if (i <= RTE_COLOR_YELLOW && policy && 15517 policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) 15518 next_fm = mlx5_flow_meter_find(priv, 15519 policy->act_cnt[i].next_mtr_id, NULL); 15520 RTE_TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i], 15521 next_port, tmp) { 15522 claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule)); 15523 tbl = container_of(color_rule->matcher->tbl, 15524 typeof(*tbl), tbl); 15525 mlx5_list_unregister(tbl->matchers, 15526 &color_rule->matcher->entry); 15527 TAILQ_REMOVE(&sub_policy->color_rules[i], 15528 color_rule, next_port); 15529 mlx5_free(color_rule); 15530 if (next_fm) 15531 mlx5_flow_meter_detach(priv, next_fm); 15532 } 15533 } 15534 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 15535 if (sub_policy->rix_hrxq[i]) { 15536 if (policy && !policy->is_hierarchy) 15537 mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]); 15538 sub_policy->rix_hrxq[i] = 0; 15539 } 15540 if (sub_policy->jump_tbl[i]) { 15541 flow_dv_tbl_resource_release(MLX5_SH(dev), 15542 sub_policy->jump_tbl[i]); 15543 sub_policy->jump_tbl[i] = NULL; 15544 } 15545 } 15546 if (sub_policy->tbl_rsc) { 15547 flow_dv_tbl_resource_release(MLX5_SH(dev), 15548 sub_policy->tbl_rsc); 15549 sub_policy->tbl_rsc = NULL; 15550 } 15551 } 15552 15553 /** 15554 * Destroy policy rules, lock free, 15555 * (mutex should be acquired by caller). 15556 * Dispatcher for action type specific call. 15557 * 15558 * @param[in] dev 15559 * Pointer to the Ethernet device structure. 15560 * @param[in] mtr_policy 15561 * Meter policy struct. 15562 */ 15563 static void 15564 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev, 15565 struct mlx5_flow_meter_policy *mtr_policy) 15566 { 15567 uint32_t i, j; 15568 struct mlx5_flow_meter_sub_policy *sub_policy; 15569 uint16_t sub_policy_num; 15570 15571 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 15572 sub_policy_num = (mtr_policy->sub_policy_num >> 15573 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 15574 MLX5_MTR_SUB_POLICY_NUM_MASK; 15575 for (j = 0; j < sub_policy_num; j++) { 15576 sub_policy = mtr_policy->sub_policys[i][j]; 15577 if (sub_policy) 15578 __flow_dv_destroy_sub_policy_rules(dev, 15579 sub_policy); 15580 } 15581 } 15582 } 15583 15584 /** 15585 * Destroy policy action, lock free, 15586 * (mutex should be acquired by caller). 15587 * Dispatcher for action type specific call. 15588 * 15589 * @param[in] dev 15590 * Pointer to the Ethernet device structure. 15591 * @param[in] mtr_policy 15592 * Meter policy struct. 15593 */ 15594 static void 15595 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev, 15596 struct mlx5_flow_meter_policy *mtr_policy) 15597 { 15598 struct rte_flow_action *rss_action; 15599 struct mlx5_flow_handle dev_handle; 15600 uint32_t i, j; 15601 15602 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 15603 if (mtr_policy->act_cnt[i].rix_mark) { 15604 flow_dv_tag_release(dev, 15605 mtr_policy->act_cnt[i].rix_mark); 15606 mtr_policy->act_cnt[i].rix_mark = 0; 15607 } 15608 if (mtr_policy->act_cnt[i].modify_hdr) { 15609 dev_handle.dvh.modify_hdr = 15610 mtr_policy->act_cnt[i].modify_hdr; 15611 flow_dv_modify_hdr_resource_release(dev, &dev_handle); 15612 } 15613 switch (mtr_policy->act_cnt[i].fate_action) { 15614 case MLX5_FLOW_FATE_SHARED_RSS: 15615 rss_action = mtr_policy->act_cnt[i].rss; 15616 mlx5_free(rss_action); 15617 break; 15618 case MLX5_FLOW_FATE_PORT_ID: 15619 if (mtr_policy->act_cnt[i].rix_port_id_action) { 15620 flow_dv_port_id_action_resource_release(dev, 15621 mtr_policy->act_cnt[i].rix_port_id_action); 15622 mtr_policy->act_cnt[i].rix_port_id_action = 0; 15623 } 15624 break; 15625 case MLX5_FLOW_FATE_DROP: 15626 case MLX5_FLOW_FATE_JUMP: 15627 for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++) 15628 mtr_policy->act_cnt[i].dr_jump_action[j] = 15629 NULL; 15630 break; 15631 default: 15632 /*Queue action do nothing*/ 15633 break; 15634 } 15635 } 15636 for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++) 15637 mtr_policy->dr_drop_action[j] = NULL; 15638 } 15639 15640 /** 15641 * Create yellow action for color aware meter. 15642 * 15643 * @param[in] dev 15644 * Pointer to the Ethernet device structure. 15645 * @param[in] fm 15646 * Meter information table. 15647 * @param[out] error 15648 * Perform verbose error reporting if not NULL. Initialized in case of 15649 * error only. 15650 * 15651 * @return 15652 * 0 on success, a negative errno value otherwise and rte_errno is set. 15653 */ 15654 static int 15655 __flow_dv_create_mtr_yellow_action(struct rte_eth_dev *dev, 15656 struct mlx5_flow_meter_info *fm, 15657 struct rte_mtr_error *error) 15658 { 15659 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO 15660 struct mlx5_priv *priv = dev->data->dev_private; 15661 struct rte_flow_error flow_err; 15662 struct mlx5_aso_mtr *aso_mtr; 15663 struct mlx5_aso_mtr_pool *pool; 15664 uint8_t reg_id; 15665 15666 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 15667 pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool, mtrs[aso_mtr->offset]); 15668 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err); 15669 fm->meter_action_y = 15670 mlx5_glue->dv_create_flow_action_aso(priv->sh->rx_domain, 15671 pool->devx_obj->obj, 15672 aso_mtr->offset, 15673 (1 << MLX5_FLOW_COLOR_YELLOW), 15674 reg_id - REG_C_0); 15675 #else 15676 RTE_SET_USED(dev); 15677 #endif 15678 if (!fm->meter_action_y) { 15679 return -rte_mtr_error_set(error, EINVAL, RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 15680 "Fail to create yellow meter action."); 15681 } 15682 return 0; 15683 } 15684 15685 /** 15686 * Create policy action per domain, lock free, 15687 * (mutex should be acquired by caller). 15688 * Dispatcher for action type specific call. 15689 * 15690 * @param[in] dev 15691 * Pointer to the Ethernet device structure. 15692 * @param[in] mtr_policy 15693 * Meter policy struct. 15694 * @param[in] action 15695 * Action specification used to create meter actions. 15696 * @param[out] error 15697 * Perform verbose error reporting if not NULL. Initialized in case of 15698 * error only. 15699 * 15700 * @return 15701 * 0 on success, otherwise negative errno value. 15702 */ 15703 static int 15704 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev, 15705 struct mlx5_flow_meter_policy *mtr_policy, 15706 const struct rte_flow_action *actions[RTE_COLORS], 15707 enum mlx5_meter_domain domain, 15708 struct rte_mtr_error *error) 15709 { 15710 struct mlx5_priv *priv = dev->data->dev_private; 15711 struct rte_flow_error flow_err; 15712 const struct rte_flow_action *act; 15713 uint64_t action_flags; 15714 struct mlx5_flow_handle dh; 15715 struct mlx5_flow dev_flow; 15716 struct mlx5_flow_dv_port_id_action_resource port_id_action; 15717 int i, ret; 15718 uint8_t egress, transfer; 15719 struct mlx5_meter_policy_action_container *act_cnt = NULL; 15720 union { 15721 struct mlx5_flow_dv_modify_hdr_resource res; 15722 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) + 15723 sizeof(struct mlx5_modification_cmd) * 15724 (MLX5_MAX_MODIFY_NUM + 1)]; 15725 } mhdr_dummy; 15726 struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; 15727 15728 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; 15729 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; 15730 memset(&dh, 0, sizeof(struct mlx5_flow_handle)); 15731 memset(&dev_flow, 0, sizeof(struct mlx5_flow)); 15732 memset(&port_id_action, 0, 15733 sizeof(struct mlx5_flow_dv_port_id_action_resource)); 15734 memset(mhdr_res, 0, sizeof(*mhdr_res)); 15735 mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 15736 (egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 15737 MLX5DV_FLOW_TABLE_TYPE_NIC_RX); 15738 dev_flow.handle = &dh; 15739 dev_flow.dv.port_id_action = &port_id_action; 15740 dev_flow.external = true; 15741 for (i = 0; i < RTE_COLORS; i++) { 15742 if (i < MLX5_MTR_RTE_COLORS) 15743 act_cnt = &mtr_policy->act_cnt[i]; 15744 /* Skip the color policy actions creation. */ 15745 if ((i == RTE_COLOR_YELLOW && mtr_policy->skip_y) || 15746 (i == RTE_COLOR_GREEN && mtr_policy->skip_g)) 15747 continue; 15748 action_flags = 0; 15749 for (act = actions[i]; 15750 act && act->type != RTE_FLOW_ACTION_TYPE_END; act++) { 15751 switch (act->type) { 15752 case RTE_FLOW_ACTION_TYPE_MARK: 15753 { 15754 uint32_t tag_be = mlx5_flow_mark_set 15755 (((const struct rte_flow_action_mark *) 15756 (act->conf))->id); 15757 15758 if (i >= MLX5_MTR_RTE_COLORS) 15759 return -rte_mtr_error_set(error, 15760 ENOTSUP, 15761 RTE_MTR_ERROR_TYPE_METER_POLICY, 15762 NULL, 15763 "cannot create policy " 15764 "mark action for this color"); 15765 if (flow_dv_tag_resource_register(dev, tag_be, 15766 &dev_flow, &flow_err)) 15767 return -rte_mtr_error_set(error, 15768 ENOTSUP, 15769 RTE_MTR_ERROR_TYPE_METER_POLICY, 15770 NULL, 15771 "cannot setup policy mark action"); 15772 MLX5_ASSERT(dev_flow.dv.tag_resource); 15773 act_cnt->rix_mark = 15774 dev_flow.handle->dvh.rix_tag; 15775 action_flags |= MLX5_FLOW_ACTION_MARK; 15776 mtr_policy->mark = 1; 15777 break; 15778 } 15779 case RTE_FLOW_ACTION_TYPE_SET_TAG: 15780 if (i >= MLX5_MTR_RTE_COLORS) 15781 return -rte_mtr_error_set(error, 15782 ENOTSUP, 15783 RTE_MTR_ERROR_TYPE_METER_POLICY, 15784 NULL, 15785 "cannot create policy " 15786 "set tag action for this color"); 15787 if (flow_dv_convert_action_set_tag 15788 (dev, mhdr_res, 15789 (const struct rte_flow_action_set_tag *) 15790 act->conf, &flow_err)) 15791 return -rte_mtr_error_set(error, 15792 ENOTSUP, 15793 RTE_MTR_ERROR_TYPE_METER_POLICY, 15794 NULL, "cannot convert policy " 15795 "set tag action"); 15796 if (!mhdr_res->actions_num) 15797 return -rte_mtr_error_set(error, 15798 ENOTSUP, 15799 RTE_MTR_ERROR_TYPE_METER_POLICY, 15800 NULL, "cannot find policy " 15801 "set tag action"); 15802 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 15803 break; 15804 case RTE_FLOW_ACTION_TYPE_DROP: 15805 { 15806 struct mlx5_flow_mtr_mng *mtrmng = 15807 priv->sh->mtrmng; 15808 struct mlx5_flow_tbl_data_entry *tbl_data; 15809 15810 /* 15811 * Create the drop table with 15812 * METER DROP level. 15813 */ 15814 if (!mtrmng->drop_tbl[domain]) { 15815 mtrmng->drop_tbl[domain] = 15816 flow_dv_tbl_resource_get(dev, 15817 MLX5_FLOW_TABLE_LEVEL_METER, 15818 egress, transfer, false, NULL, 0, 15819 0, MLX5_MTR_TABLE_ID_DROP, &flow_err); 15820 if (!mtrmng->drop_tbl[domain]) 15821 return -rte_mtr_error_set 15822 (error, ENOTSUP, 15823 RTE_MTR_ERROR_TYPE_METER_POLICY, 15824 NULL, 15825 "Failed to create meter drop table"); 15826 } 15827 tbl_data = container_of 15828 (mtrmng->drop_tbl[domain], 15829 struct mlx5_flow_tbl_data_entry, tbl); 15830 if (i < MLX5_MTR_RTE_COLORS) { 15831 act_cnt->dr_jump_action[domain] = 15832 tbl_data->jump.action; 15833 act_cnt->fate_action = 15834 MLX5_FLOW_FATE_DROP; 15835 } 15836 if (i == RTE_COLOR_RED) 15837 mtr_policy->dr_drop_action[domain] = 15838 tbl_data->jump.action; 15839 action_flags |= MLX5_FLOW_ACTION_DROP; 15840 break; 15841 } 15842 case RTE_FLOW_ACTION_TYPE_QUEUE: 15843 { 15844 if (i >= MLX5_MTR_RTE_COLORS) 15845 return -rte_mtr_error_set(error, 15846 ENOTSUP, 15847 RTE_MTR_ERROR_TYPE_METER_POLICY, 15848 NULL, "cannot create policy " 15849 "fate queue for this color"); 15850 act_cnt->queue = 15851 ((const struct rte_flow_action_queue *) 15852 (act->conf))->index; 15853 act_cnt->fate_action = 15854 MLX5_FLOW_FATE_QUEUE; 15855 dev_flow.handle->fate_action = 15856 MLX5_FLOW_FATE_QUEUE; 15857 mtr_policy->is_queue = 1; 15858 action_flags |= MLX5_FLOW_ACTION_QUEUE; 15859 break; 15860 } 15861 case RTE_FLOW_ACTION_TYPE_RSS: 15862 { 15863 int rss_size; 15864 15865 if (i >= MLX5_MTR_RTE_COLORS) 15866 return -rte_mtr_error_set(error, 15867 ENOTSUP, 15868 RTE_MTR_ERROR_TYPE_METER_POLICY, 15869 NULL, 15870 "cannot create policy " 15871 "rss action for this color"); 15872 /* 15873 * Save RSS conf into policy struct 15874 * for translate stage. 15875 */ 15876 rss_size = (int)rte_flow_conv 15877 (RTE_FLOW_CONV_OP_ACTION, 15878 NULL, 0, act, &flow_err); 15879 if (rss_size <= 0) 15880 return -rte_mtr_error_set(error, 15881 ENOTSUP, 15882 RTE_MTR_ERROR_TYPE_METER_POLICY, 15883 NULL, "Get the wrong " 15884 "rss action struct size"); 15885 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO, 15886 rss_size, 0, SOCKET_ID_ANY); 15887 if (!act_cnt->rss) 15888 return -rte_mtr_error_set(error, 15889 ENOTSUP, 15890 RTE_MTR_ERROR_TYPE_METER_POLICY, 15891 NULL, 15892 "Fail to malloc rss action memory"); 15893 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION, 15894 act_cnt->rss, rss_size, 15895 act, &flow_err); 15896 if (ret < 0) 15897 return -rte_mtr_error_set(error, 15898 ENOTSUP, 15899 RTE_MTR_ERROR_TYPE_METER_POLICY, 15900 NULL, "Fail to save " 15901 "rss action into policy struct"); 15902 act_cnt->fate_action = 15903 MLX5_FLOW_FATE_SHARED_RSS; 15904 action_flags |= MLX5_FLOW_ACTION_RSS; 15905 break; 15906 } 15907 case RTE_FLOW_ACTION_TYPE_PORT_ID: 15908 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 15909 { 15910 struct mlx5_flow_dv_port_id_action_resource 15911 port_id_resource; 15912 uint32_t port_id = 0; 15913 15914 if (i >= MLX5_MTR_RTE_COLORS) 15915 return -rte_mtr_error_set(error, 15916 ENOTSUP, 15917 RTE_MTR_ERROR_TYPE_METER_POLICY, 15918 NULL, "cannot create policy " 15919 "port action for this color"); 15920 memset(&port_id_resource, 0, 15921 sizeof(port_id_resource)); 15922 if (flow_dv_translate_action_port_id(dev, act, 15923 &port_id, &flow_err)) 15924 return -rte_mtr_error_set(error, 15925 ENOTSUP, 15926 RTE_MTR_ERROR_TYPE_METER_POLICY, 15927 NULL, "cannot translate " 15928 "policy port action"); 15929 port_id_resource.port_id = port_id; 15930 if (flow_dv_port_id_action_resource_register 15931 (dev, &port_id_resource, 15932 &dev_flow, &flow_err)) 15933 return -rte_mtr_error_set(error, 15934 ENOTSUP, 15935 RTE_MTR_ERROR_TYPE_METER_POLICY, 15936 NULL, "cannot setup " 15937 "policy port action"); 15938 act_cnt->rix_port_id_action = 15939 dev_flow.handle->rix_port_id_action; 15940 act_cnt->fate_action = 15941 MLX5_FLOW_FATE_PORT_ID; 15942 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 15943 break; 15944 } 15945 case RTE_FLOW_ACTION_TYPE_JUMP: 15946 { 15947 uint32_t jump_group = 0; 15948 uint32_t table = 0; 15949 struct mlx5_flow_tbl_data_entry *tbl_data; 15950 struct flow_grp_info grp_info = { 15951 .external = !!dev_flow.external, 15952 .transfer = !!transfer, 15953 .fdb_def_rule = !!priv->fdb_def_rule, 15954 .std_tbl_fix = 0, 15955 .skip_scale = dev_flow.skip_scale & 15956 (1 << MLX5_SCALE_FLOW_GROUP_BIT), 15957 }; 15958 struct mlx5_flow_meter_sub_policy *sub_policy = 15959 mtr_policy->sub_policys[domain][0]; 15960 15961 if (i >= MLX5_MTR_RTE_COLORS) 15962 return -rte_mtr_error_set(error, 15963 ENOTSUP, 15964 RTE_MTR_ERROR_TYPE_METER_POLICY, 15965 NULL, 15966 "cannot create policy " 15967 "jump action for this color"); 15968 jump_group = 15969 ((const struct rte_flow_action_jump *) 15970 act->conf)->group; 15971 if (mlx5_flow_group_to_table(dev, NULL, 15972 jump_group, 15973 &table, 15974 &grp_info, &flow_err)) 15975 return -rte_mtr_error_set(error, 15976 ENOTSUP, 15977 RTE_MTR_ERROR_TYPE_METER_POLICY, 15978 NULL, "cannot setup " 15979 "policy jump action"); 15980 sub_policy->jump_tbl[i] = 15981 flow_dv_tbl_resource_get(dev, 15982 table, egress, 15983 transfer, 15984 !!dev_flow.external, 15985 NULL, jump_group, 0, 15986 0, &flow_err); 15987 if 15988 (!sub_policy->jump_tbl[i]) 15989 return -rte_mtr_error_set(error, 15990 ENOTSUP, 15991 RTE_MTR_ERROR_TYPE_METER_POLICY, 15992 NULL, "cannot create jump action."); 15993 tbl_data = container_of 15994 (sub_policy->jump_tbl[i], 15995 struct mlx5_flow_tbl_data_entry, tbl); 15996 act_cnt->dr_jump_action[domain] = 15997 tbl_data->jump.action; 15998 act_cnt->fate_action = 15999 MLX5_FLOW_FATE_JUMP; 16000 action_flags |= MLX5_FLOW_ACTION_JUMP; 16001 break; 16002 } 16003 /* 16004 * No need to check meter hierarchy for R colors 16005 * here since it is done in the validation stage. 16006 */ 16007 case RTE_FLOW_ACTION_TYPE_METER: 16008 { 16009 const struct rte_flow_action_meter *mtr; 16010 struct mlx5_flow_meter_info *next_fm; 16011 struct mlx5_flow_meter_policy *next_policy; 16012 struct rte_flow_action tag_action; 16013 struct mlx5_rte_flow_action_set_tag set_tag; 16014 uint32_t next_mtr_idx = 0; 16015 16016 mtr = act->conf; 16017 next_fm = mlx5_flow_meter_find(priv, 16018 mtr->mtr_id, 16019 &next_mtr_idx); 16020 if (!next_fm) 16021 return -rte_mtr_error_set(error, EINVAL, 16022 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 16023 "Fail to find next meter."); 16024 if (next_fm->def_policy) 16025 return -rte_mtr_error_set(error, EINVAL, 16026 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 16027 "Hierarchy only supports termination meter."); 16028 next_policy = mlx5_flow_meter_policy_find(dev, 16029 next_fm->policy_id, NULL); 16030 MLX5_ASSERT(next_policy); 16031 if (next_fm->drop_cnt) { 16032 set_tag.id = 16033 (enum modify_reg) 16034 mlx5_flow_get_reg_id(dev, 16035 MLX5_MTR_ID, 16036 0, 16037 (struct rte_flow_error *)error); 16038 set_tag.offset = (priv->mtr_reg_share ? 16039 MLX5_MTR_COLOR_BITS : 0); 16040 set_tag.length = (priv->mtr_reg_share ? 16041 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : 16042 MLX5_REG_BITS); 16043 set_tag.data = next_mtr_idx; 16044 tag_action.type = 16045 (enum rte_flow_action_type) 16046 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 16047 tag_action.conf = &set_tag; 16048 if (flow_dv_convert_action_set_reg 16049 (mhdr_res, &tag_action, 16050 (struct rte_flow_error *)error)) 16051 return -rte_errno; 16052 action_flags |= 16053 MLX5_FLOW_ACTION_SET_TAG; 16054 } 16055 if (i == RTE_COLOR_YELLOW && next_fm->color_aware && 16056 !next_fm->meter_action_y) 16057 if (__flow_dv_create_mtr_yellow_action(dev, next_fm, error)) 16058 return -rte_errno; 16059 act_cnt->fate_action = MLX5_FLOW_FATE_MTR; 16060 act_cnt->next_mtr_id = next_fm->meter_id; 16061 act_cnt->next_sub_policy = NULL; 16062 mtr_policy->is_hierarchy = 1; 16063 mtr_policy->dev = next_policy->dev; 16064 if (next_policy->mark) 16065 mtr_policy->mark = 1; 16066 action_flags |= 16067 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY; 16068 break; 16069 } 16070 default: 16071 return -rte_mtr_error_set(error, ENOTSUP, 16072 RTE_MTR_ERROR_TYPE_METER_POLICY, 16073 NULL, "action type not supported"); 16074 } 16075 if (action_flags & MLX5_FLOW_ACTION_SET_TAG) { 16076 /* create modify action if needed. */ 16077 dev_flow.dv.group = 1; 16078 if (flow_dv_modify_hdr_resource_register 16079 (dev, mhdr_res, &dev_flow, &flow_err)) 16080 return -rte_mtr_error_set(error, 16081 ENOTSUP, 16082 RTE_MTR_ERROR_TYPE_METER_POLICY, 16083 NULL, "cannot register policy " 16084 "set tag action"); 16085 act_cnt->modify_hdr = 16086 dev_flow.handle->dvh.modify_hdr; 16087 } 16088 } 16089 } 16090 return 0; 16091 } 16092 16093 /** 16094 * Create policy action per domain, lock free, 16095 * (mutex should be acquired by caller). 16096 * Dispatcher for action type specific call. 16097 * 16098 * @param[in] dev 16099 * Pointer to the Ethernet device structure. 16100 * @param[in] mtr_policy 16101 * Meter policy struct. 16102 * @param[in] action 16103 * Action specification used to create meter actions. 16104 * @param[out] error 16105 * Perform verbose error reporting if not NULL. Initialized in case of 16106 * error only. 16107 * 16108 * @return 16109 * 0 on success, otherwise negative errno value. 16110 */ 16111 static int 16112 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev, 16113 struct mlx5_flow_meter_policy *mtr_policy, 16114 const struct rte_flow_action *actions[RTE_COLORS], 16115 struct rte_mtr_error *error) 16116 { 16117 int ret, i; 16118 uint16_t sub_policy_num; 16119 16120 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 16121 sub_policy_num = (mtr_policy->sub_policy_num >> 16122 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 16123 MLX5_MTR_SUB_POLICY_NUM_MASK; 16124 if (sub_policy_num) { 16125 ret = __flow_dv_create_domain_policy_acts(dev, 16126 mtr_policy, actions, 16127 (enum mlx5_meter_domain)i, error); 16128 /* Cleaning resource is done in the caller level. */ 16129 if (ret) 16130 return ret; 16131 } 16132 } 16133 return 0; 16134 } 16135 16136 /** 16137 * Query a DV flow rule for its statistics via DevX. 16138 * 16139 * @param[in] dev 16140 * Pointer to Ethernet device. 16141 * @param[in] cnt_idx 16142 * Index to the flow counter. 16143 * @param[out] data 16144 * Data retrieved by the query. 16145 * @param[out] error 16146 * Perform verbose error reporting if not NULL. 16147 * 16148 * @return 16149 * 0 on success, a negative errno value otherwise and rte_errno is set. 16150 */ 16151 static int 16152 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data, 16153 struct rte_flow_error *error) 16154 { 16155 struct mlx5_priv *priv = dev->data->dev_private; 16156 struct rte_flow_query_count *qc = data; 16157 16158 if (!priv->sh->cdev->config.devx) 16159 return rte_flow_error_set(error, ENOTSUP, 16160 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 16161 NULL, 16162 "counters are not supported"); 16163 if (cnt_idx) { 16164 uint64_t pkts, bytes; 16165 struct mlx5_flow_counter *cnt; 16166 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes); 16167 16168 if (err) 16169 return rte_flow_error_set(error, -err, 16170 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 16171 NULL, "cannot read counters"); 16172 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL); 16173 qc->hits_set = 1; 16174 qc->bytes_set = 1; 16175 qc->hits = pkts - cnt->hits; 16176 qc->bytes = bytes - cnt->bytes; 16177 if (qc->reset) { 16178 cnt->hits = pkts; 16179 cnt->bytes = bytes; 16180 } 16181 return 0; 16182 } 16183 return rte_flow_error_set(error, EINVAL, 16184 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 16185 NULL, 16186 "counters are not available"); 16187 } 16188 16189 int 16190 flow_dv_action_query(struct rte_eth_dev *dev, 16191 const struct rte_flow_action_handle *handle, void *data, 16192 struct rte_flow_error *error) 16193 { 16194 struct mlx5_age_param *age_param; 16195 struct rte_flow_query_age *resp; 16196 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 16197 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 16198 uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 16199 struct mlx5_priv *priv = dev->data->dev_private; 16200 struct mlx5_aso_ct_action *ct; 16201 uint16_t owner; 16202 uint32_t dev_idx; 16203 16204 switch (type) { 16205 case MLX5_INDIRECT_ACTION_TYPE_AGE: 16206 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params; 16207 resp = data; 16208 resp->aged = __atomic_load_n(&age_param->state, 16209 __ATOMIC_RELAXED) == AGE_TMOUT ? 16210 1 : 0; 16211 resp->sec_since_last_hit_valid = !resp->aged; 16212 if (resp->sec_since_last_hit_valid) 16213 resp->sec_since_last_hit = __atomic_load_n 16214 (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); 16215 return 0; 16216 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 16217 return flow_dv_query_count(dev, idx, data, error); 16218 case MLX5_INDIRECT_ACTION_TYPE_CT: 16219 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx); 16220 if (owner != PORT_ID(priv)) 16221 return rte_flow_error_set(error, EACCES, 16222 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 16223 NULL, 16224 "CT object owned by another port"); 16225 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx); 16226 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx); 16227 MLX5_ASSERT(ct); 16228 if (!ct->refcnt) 16229 return rte_flow_error_set(error, EFAULT, 16230 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 16231 NULL, 16232 "CT object is inactive"); 16233 ((struct rte_flow_action_conntrack *)data)->peer_port = 16234 ct->peer; 16235 ((struct rte_flow_action_conntrack *)data)->is_original_dir = 16236 ct->is_original; 16237 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data)) 16238 return rte_flow_error_set(error, EIO, 16239 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 16240 NULL, 16241 "Failed to query CT context"); 16242 return 0; 16243 default: 16244 return rte_flow_error_set(error, ENOTSUP, 16245 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 16246 "action type query not supported"); 16247 } 16248 } 16249 16250 /** 16251 * Query a flow rule AGE action for aging information. 16252 * 16253 * @param[in] dev 16254 * Pointer to Ethernet device. 16255 * @param[in] flow 16256 * Pointer to the sub flow. 16257 * @param[out] data 16258 * data retrieved by the query. 16259 * @param[out] error 16260 * Perform verbose error reporting if not NULL. 16261 * 16262 * @return 16263 * 0 on success, a negative errno value otherwise and rte_errno is set. 16264 */ 16265 static int 16266 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow, 16267 void *data, struct rte_flow_error *error) 16268 { 16269 struct rte_flow_query_age *resp = data; 16270 struct mlx5_age_param *age_param; 16271 16272 if (flow->age) { 16273 struct mlx5_aso_age_action *act = 16274 flow_aso_age_get_by_idx(dev, flow->age); 16275 16276 age_param = &act->age_params; 16277 } else if (flow->counter) { 16278 age_param = flow_dv_counter_idx_get_age(dev, flow->counter); 16279 16280 if (!age_param || !age_param->timeout) 16281 return rte_flow_error_set 16282 (error, EINVAL, 16283 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 16284 NULL, "cannot read age data"); 16285 } else { 16286 return rte_flow_error_set(error, EINVAL, 16287 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 16288 NULL, "age data not available"); 16289 } 16290 resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) == 16291 AGE_TMOUT ? 1 : 0; 16292 resp->sec_since_last_hit_valid = !resp->aged; 16293 if (resp->sec_since_last_hit_valid) 16294 resp->sec_since_last_hit = __atomic_load_n 16295 (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); 16296 return 0; 16297 } 16298 16299 /** 16300 * Query a flow. 16301 * 16302 * @see rte_flow_query() 16303 * @see rte_flow_ops 16304 */ 16305 static int 16306 flow_dv_query(struct rte_eth_dev *dev, 16307 struct rte_flow *flow __rte_unused, 16308 const struct rte_flow_action *actions __rte_unused, 16309 void *data __rte_unused, 16310 struct rte_flow_error *error __rte_unused) 16311 { 16312 int ret = -EINVAL; 16313 16314 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 16315 switch (actions->type) { 16316 case RTE_FLOW_ACTION_TYPE_VOID: 16317 break; 16318 case RTE_FLOW_ACTION_TYPE_COUNT: 16319 ret = flow_dv_query_count(dev, flow->counter, data, 16320 error); 16321 break; 16322 case RTE_FLOW_ACTION_TYPE_AGE: 16323 ret = flow_dv_query_age(dev, flow, data, error); 16324 break; 16325 default: 16326 return rte_flow_error_set(error, ENOTSUP, 16327 RTE_FLOW_ERROR_TYPE_ACTION, 16328 actions, 16329 "action not supported"); 16330 } 16331 } 16332 return ret; 16333 } 16334 16335 /** 16336 * Destroy the meter table set. 16337 * Lock free, (mutex should be acquired by caller). 16338 * 16339 * @param[in] dev 16340 * Pointer to Ethernet device. 16341 * @param[in] fm 16342 * Meter information table. 16343 */ 16344 static void 16345 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev, 16346 struct mlx5_flow_meter_info *fm) 16347 { 16348 struct mlx5_priv *priv = dev->data->dev_private; 16349 int i; 16350 16351 if (!fm || !priv->sh->config.dv_flow_en) 16352 return; 16353 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 16354 if (fm->drop_rule[i]) { 16355 claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i])); 16356 fm->drop_rule[i] = NULL; 16357 } 16358 } 16359 } 16360 16361 static void 16362 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 16363 { 16364 struct mlx5_priv *priv = dev->data->dev_private; 16365 struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng; 16366 struct mlx5_flow_tbl_data_entry *tbl; 16367 int i, j; 16368 16369 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 16370 if (mtrmng->def_rule[i]) { 16371 claim_zero(mlx5_flow_os_destroy_flow 16372 (mtrmng->def_rule[i])); 16373 mtrmng->def_rule[i] = NULL; 16374 } 16375 if (mtrmng->def_matcher[i]) { 16376 tbl = container_of(mtrmng->def_matcher[i]->tbl, 16377 struct mlx5_flow_tbl_data_entry, tbl); 16378 mlx5_list_unregister(tbl->matchers, 16379 &mtrmng->def_matcher[i]->entry); 16380 mtrmng->def_matcher[i] = NULL; 16381 } 16382 for (j = 0; j < MLX5_REG_BITS; j++) { 16383 if (mtrmng->drop_matcher[i][j]) { 16384 tbl = 16385 container_of(mtrmng->drop_matcher[i][j]->tbl, 16386 struct mlx5_flow_tbl_data_entry, 16387 tbl); 16388 mlx5_list_unregister(tbl->matchers, 16389 &mtrmng->drop_matcher[i][j]->entry); 16390 mtrmng->drop_matcher[i][j] = NULL; 16391 } 16392 } 16393 if (mtrmng->drop_tbl[i]) { 16394 flow_dv_tbl_resource_release(MLX5_SH(dev), 16395 mtrmng->drop_tbl[i]); 16396 mtrmng->drop_tbl[i] = NULL; 16397 } 16398 } 16399 } 16400 16401 /* Number of meter flow actions, count and jump or count and drop. */ 16402 #define METER_ACTIONS 2 16403 16404 static void 16405 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev, 16406 enum mlx5_meter_domain domain) 16407 { 16408 struct mlx5_priv *priv = dev->data->dev_private; 16409 struct mlx5_flow_meter_def_policy *def_policy = 16410 priv->sh->mtrmng->def_policy[domain]; 16411 16412 __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy); 16413 mlx5_free(def_policy); 16414 priv->sh->mtrmng->def_policy[domain] = NULL; 16415 } 16416 16417 /** 16418 * Destroy the default policy table set. 16419 * 16420 * @param[in] dev 16421 * Pointer to Ethernet device. 16422 */ 16423 static void 16424 flow_dv_destroy_def_policy(struct rte_eth_dev *dev) 16425 { 16426 struct mlx5_priv *priv = dev->data->dev_private; 16427 int i; 16428 16429 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) 16430 if (priv->sh->mtrmng->def_policy[i]) 16431 __flow_dv_destroy_domain_def_policy(dev, 16432 (enum mlx5_meter_domain)i); 16433 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 16434 } 16435 16436 static int 16437 __flow_dv_create_policy_flow(struct rte_eth_dev *dev, 16438 uint32_t color_reg_c_idx, 16439 enum rte_color color, void *matcher_object, 16440 int actions_n, void *actions, 16441 bool match_src_port, const struct rte_flow_item *item, 16442 void **rule, const struct rte_flow_attr *attr) 16443 { 16444 int ret; 16445 struct mlx5_flow_dv_match_params value = { 16446 .size = sizeof(value.buf), 16447 }; 16448 struct mlx5_flow_dv_match_params matcher = { 16449 .size = sizeof(matcher.buf), 16450 }; 16451 struct mlx5_priv *priv = dev->data->dev_private; 16452 uint8_t misc_mask; 16453 16454 if (match_src_port && priv->sh->esw_mode) { 16455 if (flow_dv_translate_item_port_id(dev, matcher.buf, 16456 value.buf, item, attr)) { 16457 DRV_LOG(ERR, "Failed to create meter policy%d flow's" 16458 " value with port.", color); 16459 return -1; 16460 } 16461 } 16462 flow_dv_match_meta_reg(matcher.buf, value.buf, 16463 (enum modify_reg)color_reg_c_idx, 16464 rte_col_2_mlx5_col(color), UINT32_MAX); 16465 misc_mask = flow_dv_matcher_enable(value.buf); 16466 __flow_dv_adjust_buf_size(&value.size, misc_mask); 16467 ret = mlx5_flow_os_create_flow(matcher_object, (void *)&value, 16468 actions_n, actions, rule); 16469 if (ret) { 16470 DRV_LOG(ERR, "Failed to create meter policy%d flow.", color); 16471 return -1; 16472 } 16473 return 0; 16474 } 16475 16476 static int 16477 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev, 16478 uint32_t color_reg_c_idx, 16479 uint16_t priority, 16480 struct mlx5_flow_meter_sub_policy *sub_policy, 16481 const struct rte_flow_attr *attr, 16482 bool match_src_port, 16483 const struct rte_flow_item *item, 16484 struct mlx5_flow_dv_matcher **policy_matcher, 16485 struct rte_flow_error *error) 16486 { 16487 struct mlx5_list_entry *entry; 16488 struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc; 16489 struct mlx5_flow_dv_matcher matcher = { 16490 .mask = { 16491 .size = sizeof(matcher.mask.buf), 16492 }, 16493 .tbl = tbl_rsc, 16494 }; 16495 struct mlx5_flow_dv_match_params value = { 16496 .size = sizeof(value.buf), 16497 }; 16498 struct mlx5_flow_cb_ctx ctx = { 16499 .error = error, 16500 .data = &matcher, 16501 }; 16502 struct mlx5_flow_tbl_data_entry *tbl_data; 16503 struct mlx5_priv *priv = dev->data->dev_private; 16504 const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; 16505 16506 if (match_src_port && priv->sh->esw_mode) { 16507 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf, 16508 value.buf, item, attr)) { 16509 DRV_LOG(ERR, "Failed to register meter policy%d matcher" 16510 " with port.", priority); 16511 return -1; 16512 } 16513 } 16514 tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl); 16515 if (priority < RTE_COLOR_RED) 16516 flow_dv_match_meta_reg(matcher.mask.buf, value.buf, 16517 (enum modify_reg)color_reg_c_idx, 0, color_mask); 16518 matcher.priority = priority; 16519 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, 16520 matcher.mask.size); 16521 entry = mlx5_list_register(tbl_data->matchers, &ctx); 16522 if (!entry) { 16523 DRV_LOG(ERR, "Failed to register meter drop matcher."); 16524 return -1; 16525 } 16526 *policy_matcher = 16527 container_of(entry, struct mlx5_flow_dv_matcher, entry); 16528 return 0; 16529 } 16530 16531 /** 16532 * Create the policy rules per domain. 16533 * 16534 * @param[in] dev 16535 * Pointer to Ethernet device. 16536 * @param[in] sub_policy 16537 * Pointer to sub policy table.. 16538 * @param[in] egress 16539 * Direction of the table. 16540 * @param[in] transfer 16541 * E-Switch or NIC flow. 16542 * @param[in] acts 16543 * Pointer to policy action list per color. 16544 * 16545 * @return 16546 * 0 on success, -1 otherwise. 16547 */ 16548 static int 16549 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev, 16550 struct mlx5_flow_meter_sub_policy *sub_policy, 16551 uint8_t egress, uint8_t transfer, bool match_src_port, 16552 struct mlx5_meter_policy_acts acts[RTE_COLORS]) 16553 { 16554 struct mlx5_priv *priv = dev->data->dev_private; 16555 struct rte_flow_error flow_err; 16556 uint32_t color_reg_c_idx; 16557 struct rte_flow_attr attr = { 16558 .group = MLX5_FLOW_TABLE_LEVEL_POLICY, 16559 .priority = 0, 16560 .ingress = 0, 16561 .egress = !!egress, 16562 .transfer = !!transfer, 16563 .reserved = 0, 16564 }; 16565 int i; 16566 int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err); 16567 struct mlx5_sub_policy_color_rule *color_rule; 16568 bool svport_match; 16569 struct mlx5_sub_policy_color_rule *tmp_rules[RTE_COLORS] = {NULL}; 16570 16571 if (ret < 0) 16572 return -1; 16573 /* Create policy table with POLICY level. */ 16574 if (!sub_policy->tbl_rsc) 16575 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev, 16576 MLX5_FLOW_TABLE_LEVEL_POLICY, 16577 egress, transfer, false, NULL, 0, 0, 16578 sub_policy->idx, &flow_err); 16579 if (!sub_policy->tbl_rsc) { 16580 DRV_LOG(ERR, 16581 "Failed to create meter sub policy table."); 16582 return -1; 16583 } 16584 /* Prepare matchers. */ 16585 color_reg_c_idx = ret; 16586 for (i = 0; i < RTE_COLORS; i++) { 16587 TAILQ_INIT(&sub_policy->color_rules[i]); 16588 if (!acts[i].actions_n) 16589 continue; 16590 color_rule = mlx5_malloc(MLX5_MEM_ZERO, 16591 sizeof(struct mlx5_sub_policy_color_rule), 16592 0, SOCKET_ID_ANY); 16593 if (!color_rule) { 16594 DRV_LOG(ERR, "No memory to create color rule."); 16595 goto err_exit; 16596 } 16597 tmp_rules[i] = color_rule; 16598 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i], 16599 color_rule, next_port); 16600 color_rule->src_port = priv->representor_id; 16601 /* No use. */ 16602 attr.priority = i; 16603 /* Create matchers for colors. */ 16604 svport_match = (i != RTE_COLOR_RED) ? match_src_port : false; 16605 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx, 16606 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy, 16607 &attr, svport_match, NULL, 16608 &color_rule->matcher, &flow_err)) { 16609 DRV_LOG(ERR, "Failed to create color%u matcher.", i); 16610 goto err_exit; 16611 } 16612 /* Create flow, matching color. */ 16613 if (__flow_dv_create_policy_flow(dev, 16614 color_reg_c_idx, (enum rte_color)i, 16615 color_rule->matcher->matcher_object, 16616 acts[i].actions_n, acts[i].dv_actions, 16617 svport_match, NULL, &color_rule->rule, 16618 &attr)) { 16619 DRV_LOG(ERR, "Failed to create color%u rule.", i); 16620 goto err_exit; 16621 } 16622 } 16623 return 0; 16624 err_exit: 16625 /* All the policy rules will be cleared. */ 16626 do { 16627 color_rule = tmp_rules[i]; 16628 if (color_rule) { 16629 if (color_rule->rule) 16630 mlx5_flow_os_destroy_flow(color_rule->rule); 16631 if (color_rule->matcher) { 16632 struct mlx5_flow_tbl_data_entry *tbl = 16633 container_of(color_rule->matcher->tbl, 16634 typeof(*tbl), tbl); 16635 mlx5_list_unregister(tbl->matchers, 16636 &color_rule->matcher->entry); 16637 } 16638 TAILQ_REMOVE(&sub_policy->color_rules[i], 16639 color_rule, next_port); 16640 mlx5_free(color_rule); 16641 } 16642 } while (i--); 16643 return -1; 16644 } 16645 16646 static int 16647 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev, 16648 struct mlx5_flow_meter_policy *mtr_policy, 16649 struct mlx5_flow_meter_sub_policy *sub_policy, 16650 uint32_t domain) 16651 { 16652 struct mlx5_priv *priv = dev->data->dev_private; 16653 struct mlx5_meter_policy_acts acts[RTE_COLORS]; 16654 struct mlx5_flow_dv_tag_resource *tag; 16655 struct mlx5_flow_dv_port_id_action_resource *port_action; 16656 struct mlx5_hrxq *hrxq; 16657 struct mlx5_flow_meter_info *next_fm[RTE_COLORS] = {NULL}; 16658 struct mlx5_flow_meter_policy *next_policy; 16659 struct mlx5_flow_meter_sub_policy *next_sub_policy; 16660 struct mlx5_flow_tbl_data_entry *tbl_data; 16661 struct rte_flow_error error; 16662 uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; 16663 uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; 16664 bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX); 16665 bool match_src_port = false; 16666 int i; 16667 16668 /* If RSS or Queue, no previous actions / rules is created. */ 16669 for (i = 0; i < RTE_COLORS; i++) { 16670 acts[i].actions_n = 0; 16671 if (i == RTE_COLOR_RED) { 16672 /* Only support drop on red. */ 16673 acts[i].dv_actions[0] = 16674 mtr_policy->dr_drop_action[domain]; 16675 acts[i].actions_n = 1; 16676 continue; 16677 } 16678 if (mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) { 16679 struct rte_flow_attr attr = { 16680 .transfer = transfer 16681 }; 16682 16683 next_fm[i] = mlx5_flow_meter_find(priv, 16684 mtr_policy->act_cnt[i].next_mtr_id, 16685 NULL); 16686 if (!next_fm[i]) { 16687 DRV_LOG(ERR, 16688 "Failed to get next hierarchy meter."); 16689 goto err_exit; 16690 } 16691 if (mlx5_flow_meter_attach(priv, next_fm[i], 16692 &attr, &error)) { 16693 DRV_LOG(ERR, "%s", error.message); 16694 next_fm[i] = NULL; 16695 goto err_exit; 16696 } 16697 /* Meter action must be the first for TX. */ 16698 if (mtr_first) { 16699 acts[i].dv_actions[acts[i].actions_n] = 16700 (next_fm[i]->color_aware && i == RTE_COLOR_YELLOW) ? 16701 next_fm[i]->meter_action_y : 16702 next_fm[i]->meter_action_g; 16703 acts[i].actions_n++; 16704 } 16705 } 16706 if (mtr_policy->act_cnt[i].rix_mark) { 16707 tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], 16708 mtr_policy->act_cnt[i].rix_mark); 16709 if (!tag) { 16710 DRV_LOG(ERR, "Failed to find " 16711 "mark action for policy."); 16712 goto err_exit; 16713 } 16714 acts[i].dv_actions[acts[i].actions_n] = tag->action; 16715 acts[i].actions_n++; 16716 } 16717 if (mtr_policy->act_cnt[i].modify_hdr) { 16718 acts[i].dv_actions[acts[i].actions_n] = 16719 mtr_policy->act_cnt[i].modify_hdr->action; 16720 acts[i].actions_n++; 16721 } 16722 if (mtr_policy->act_cnt[i].fate_action) { 16723 switch (mtr_policy->act_cnt[i].fate_action) { 16724 case MLX5_FLOW_FATE_PORT_ID: 16725 port_action = mlx5_ipool_get 16726 (priv->sh->ipool[MLX5_IPOOL_PORT_ID], 16727 mtr_policy->act_cnt[i].rix_port_id_action); 16728 if (!port_action) { 16729 DRV_LOG(ERR, "Failed to find " 16730 "port action for policy."); 16731 goto err_exit; 16732 } 16733 acts[i].dv_actions[acts[i].actions_n] = 16734 port_action->action; 16735 acts[i].actions_n++; 16736 mtr_policy->dev = dev; 16737 match_src_port = true; 16738 break; 16739 case MLX5_FLOW_FATE_DROP: 16740 case MLX5_FLOW_FATE_JUMP: 16741 acts[i].dv_actions[acts[i].actions_n] = 16742 mtr_policy->act_cnt[i].dr_jump_action[domain]; 16743 acts[i].actions_n++; 16744 break; 16745 case MLX5_FLOW_FATE_SHARED_RSS: 16746 case MLX5_FLOW_FATE_QUEUE: 16747 hrxq = mlx5_ipool_get 16748 (priv->sh->ipool[MLX5_IPOOL_HRXQ], 16749 sub_policy->rix_hrxq[i]); 16750 if (!hrxq) { 16751 DRV_LOG(ERR, "Failed to find " 16752 "queue action for policy."); 16753 goto err_exit; 16754 } 16755 acts[i].dv_actions[acts[i].actions_n] = 16756 hrxq->action; 16757 acts[i].actions_n++; 16758 break; 16759 case MLX5_FLOW_FATE_MTR: 16760 if (!next_fm[i]) { 16761 DRV_LOG(ERR, 16762 "No next hierarchy meter."); 16763 goto err_exit; 16764 } 16765 if (!mtr_first) { 16766 acts[i].dv_actions[acts[i].actions_n] = 16767 (next_fm[i]->color_aware && i == RTE_COLOR_YELLOW) ? 16768 next_fm[i]->meter_action_y : 16769 next_fm[i]->meter_action_g; 16770 acts[i].actions_n++; 16771 } 16772 if (mtr_policy->act_cnt[i].next_sub_policy) { 16773 next_sub_policy = 16774 mtr_policy->act_cnt[i].next_sub_policy; 16775 } else { 16776 next_policy = 16777 mlx5_flow_meter_policy_find(dev, 16778 next_fm[i]->policy_id, NULL); 16779 MLX5_ASSERT(next_policy); 16780 next_sub_policy = 16781 next_policy->sub_policys[domain][0]; 16782 } 16783 tbl_data = 16784 container_of(next_sub_policy->tbl_rsc, 16785 struct mlx5_flow_tbl_data_entry, tbl); 16786 acts[i].dv_actions[acts[i].actions_n++] = 16787 tbl_data->jump.action; 16788 if (mtr_policy->act_cnt[i].modify_hdr) 16789 match_src_port = !!transfer; 16790 break; 16791 default: 16792 /*Queue action do nothing*/ 16793 break; 16794 } 16795 } 16796 } 16797 if (__flow_dv_create_domain_policy_rules(dev, sub_policy, 16798 egress, transfer, match_src_port, acts)) { 16799 DRV_LOG(ERR, 16800 "Failed to create policy rules per domain."); 16801 goto err_exit; 16802 } 16803 return 0; 16804 err_exit: 16805 for (i = 0; i < RTE_COLORS; i++) 16806 if (next_fm[i]) 16807 mlx5_flow_meter_detach(priv, next_fm[i]); 16808 return -1; 16809 } 16810 16811 /** 16812 * Create the policy rules. 16813 * 16814 * @param[in] dev 16815 * Pointer to Ethernet device. 16816 * @param[in,out] mtr_policy 16817 * Pointer to meter policy table. 16818 * 16819 * @return 16820 * 0 on success, -1 otherwise. 16821 */ 16822 static int 16823 flow_dv_create_policy_rules(struct rte_eth_dev *dev, 16824 struct mlx5_flow_meter_policy *mtr_policy) 16825 { 16826 int i; 16827 uint16_t sub_policy_num; 16828 16829 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 16830 sub_policy_num = (mtr_policy->sub_policy_num >> 16831 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 16832 MLX5_MTR_SUB_POLICY_NUM_MASK; 16833 if (!sub_policy_num) 16834 continue; 16835 /* Prepare actions list and create policy rules. */ 16836 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy, 16837 mtr_policy->sub_policys[i][0], i)) { 16838 DRV_LOG(ERR, "Failed to create policy action " 16839 "list per domain."); 16840 return -1; 16841 } 16842 } 16843 return 0; 16844 } 16845 16846 static int 16847 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain) 16848 { 16849 struct mlx5_priv *priv = dev->data->dev_private; 16850 struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng; 16851 struct mlx5_flow_meter_def_policy *def_policy; 16852 struct mlx5_flow_tbl_resource *jump_tbl; 16853 struct mlx5_flow_tbl_data_entry *tbl_data; 16854 uint8_t egress, transfer; 16855 struct rte_flow_error error; 16856 struct mlx5_meter_policy_acts acts[RTE_COLORS]; 16857 int ret; 16858 16859 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; 16860 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; 16861 def_policy = mtrmng->def_policy[domain]; 16862 if (!def_policy) { 16863 def_policy = mlx5_malloc(MLX5_MEM_ZERO, 16864 sizeof(struct mlx5_flow_meter_def_policy), 16865 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 16866 if (!def_policy) { 16867 DRV_LOG(ERR, "Failed to alloc default policy table."); 16868 goto def_policy_error; 16869 } 16870 mtrmng->def_policy[domain] = def_policy; 16871 /* Create the meter suffix table with SUFFIX level. */ 16872 jump_tbl = flow_dv_tbl_resource_get(dev, 16873 MLX5_FLOW_TABLE_LEVEL_METER, 16874 egress, transfer, false, NULL, 0, 16875 0, MLX5_MTR_TABLE_ID_SUFFIX, &error); 16876 if (!jump_tbl) { 16877 DRV_LOG(ERR, 16878 "Failed to create meter suffix table."); 16879 goto def_policy_error; 16880 } 16881 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl; 16882 tbl_data = container_of(jump_tbl, 16883 struct mlx5_flow_tbl_data_entry, tbl); 16884 def_policy->dr_jump_action[RTE_COLOR_GREEN] = 16885 tbl_data->jump.action; 16886 acts[RTE_COLOR_GREEN].dv_actions[0] = tbl_data->jump.action; 16887 acts[RTE_COLOR_GREEN].actions_n = 1; 16888 /* 16889 * YELLOW has the same default policy as GREEN does. 16890 * G & Y share the same table and action. The 2nd time of table 16891 * resource getting is just to update the reference count for 16892 * the releasing stage. 16893 */ 16894 jump_tbl = flow_dv_tbl_resource_get(dev, 16895 MLX5_FLOW_TABLE_LEVEL_METER, 16896 egress, transfer, false, NULL, 0, 16897 0, MLX5_MTR_TABLE_ID_SUFFIX, &error); 16898 if (!jump_tbl) { 16899 DRV_LOG(ERR, 16900 "Failed to get meter suffix table."); 16901 goto def_policy_error; 16902 } 16903 def_policy->sub_policy.jump_tbl[RTE_COLOR_YELLOW] = jump_tbl; 16904 tbl_data = container_of(jump_tbl, 16905 struct mlx5_flow_tbl_data_entry, tbl); 16906 def_policy->dr_jump_action[RTE_COLOR_YELLOW] = 16907 tbl_data->jump.action; 16908 acts[RTE_COLOR_YELLOW].dv_actions[0] = tbl_data->jump.action; 16909 acts[RTE_COLOR_YELLOW].actions_n = 1; 16910 /* Create jump action to the drop table. */ 16911 if (!mtrmng->drop_tbl[domain]) { 16912 mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get 16913 (dev, MLX5_FLOW_TABLE_LEVEL_METER, 16914 egress, transfer, false, NULL, 0, 16915 0, MLX5_MTR_TABLE_ID_DROP, &error); 16916 if (!mtrmng->drop_tbl[domain]) { 16917 DRV_LOG(ERR, "Failed to create meter " 16918 "drop table for default policy."); 16919 goto def_policy_error; 16920 } 16921 } 16922 /* all RED: unique Drop table for jump action. */ 16923 tbl_data = container_of(mtrmng->drop_tbl[domain], 16924 struct mlx5_flow_tbl_data_entry, tbl); 16925 def_policy->dr_jump_action[RTE_COLOR_RED] = 16926 tbl_data->jump.action; 16927 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action; 16928 acts[RTE_COLOR_RED].actions_n = 1; 16929 /* Create default policy rules. */ 16930 ret = __flow_dv_create_domain_policy_rules(dev, 16931 &def_policy->sub_policy, 16932 egress, transfer, false, acts); 16933 if (ret) { 16934 DRV_LOG(ERR, "Failed to create default policy rules."); 16935 goto def_policy_error; 16936 } 16937 } 16938 return 0; 16939 def_policy_error: 16940 __flow_dv_destroy_domain_def_policy(dev, 16941 (enum mlx5_meter_domain)domain); 16942 return -1; 16943 } 16944 16945 /** 16946 * Create the default policy table set. 16947 * 16948 * @param[in] dev 16949 * Pointer to Ethernet device. 16950 * @return 16951 * 0 on success, -1 otherwise. 16952 */ 16953 static int 16954 flow_dv_create_def_policy(struct rte_eth_dev *dev) 16955 { 16956 struct mlx5_priv *priv = dev->data->dev_private; 16957 int i; 16958 16959 /* Non-termination policy table. */ 16960 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 16961 if (!priv->sh->config.dv_esw_en && 16962 i == MLX5_MTR_DOMAIN_TRANSFER) 16963 continue; 16964 if (__flow_dv_create_domain_def_policy(dev, i)) { 16965 DRV_LOG(ERR, "Failed to create default policy"); 16966 /* Rollback the created default policies for others. */ 16967 flow_dv_destroy_def_policy(dev); 16968 return -1; 16969 } 16970 } 16971 return 0; 16972 } 16973 16974 /** 16975 * Create the needed meter tables. 16976 * Lock free, (mutex should be acquired by caller). 16977 * 16978 * @param[in] dev 16979 * Pointer to Ethernet device. 16980 * @param[in] fm 16981 * Meter information table. 16982 * @param[in] mtr_idx 16983 * Meter index. 16984 * @param[in] domain_bitmap 16985 * Domain bitmap. 16986 * @return 16987 * 0 on success, -1 otherwise. 16988 */ 16989 static int 16990 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev, 16991 struct mlx5_flow_meter_info *fm, 16992 uint32_t mtr_idx, 16993 uint8_t domain_bitmap) 16994 { 16995 struct mlx5_priv *priv = dev->data->dev_private; 16996 struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng; 16997 struct rte_flow_error error; 16998 struct mlx5_flow_tbl_data_entry *tbl_data; 16999 uint8_t egress, transfer; 17000 void *actions[METER_ACTIONS]; 17001 int domain, ret, i; 17002 struct mlx5_flow_counter *cnt; 17003 struct mlx5_flow_dv_match_params value = { 17004 .size = sizeof(value.buf), 17005 }; 17006 struct mlx5_flow_dv_match_params matcher_para = { 17007 .size = sizeof(matcher_para.buf), 17008 }; 17009 int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 17010 0, &error); 17011 uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1; 17012 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 17013 struct mlx5_list_entry *entry; 17014 struct mlx5_flow_dv_matcher matcher = { 17015 .mask = { 17016 .size = sizeof(matcher.mask.buf), 17017 }, 17018 }; 17019 struct mlx5_flow_dv_matcher *drop_matcher; 17020 struct mlx5_flow_cb_ctx ctx = { 17021 .error = &error, 17022 .data = &matcher, 17023 }; 17024 uint8_t misc_mask; 17025 17026 if (!priv->mtr_en || mtr_id_reg_c < 0) { 17027 rte_errno = ENOTSUP; 17028 return -1; 17029 } 17030 for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) { 17031 if (!(domain_bitmap & (1 << domain)) || 17032 (mtrmng->def_rule[domain] && !fm->drop_cnt)) 17033 continue; 17034 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; 17035 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; 17036 /* Create the drop table with METER DROP level. */ 17037 if (!mtrmng->drop_tbl[domain]) { 17038 mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev, 17039 MLX5_FLOW_TABLE_LEVEL_METER, 17040 egress, transfer, false, NULL, 0, 17041 0, MLX5_MTR_TABLE_ID_DROP, &error); 17042 if (!mtrmng->drop_tbl[domain]) { 17043 DRV_LOG(ERR, "Failed to create meter drop table."); 17044 goto policy_error; 17045 } 17046 } 17047 /* Create default matcher in drop table. */ 17048 matcher.tbl = mtrmng->drop_tbl[domain], 17049 tbl_data = container_of(mtrmng->drop_tbl[domain], 17050 struct mlx5_flow_tbl_data_entry, tbl); 17051 if (!mtrmng->def_matcher[domain]) { 17052 flow_dv_match_meta_reg(matcher.mask.buf, value.buf, 17053 (enum modify_reg)mtr_id_reg_c, 17054 0, 0); 17055 matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY; 17056 matcher.crc = rte_raw_cksum 17057 ((const void *)matcher.mask.buf, 17058 matcher.mask.size); 17059 entry = mlx5_list_register(tbl_data->matchers, &ctx); 17060 if (!entry) { 17061 DRV_LOG(ERR, "Failed to register meter " 17062 "drop default matcher."); 17063 goto policy_error; 17064 } 17065 mtrmng->def_matcher[domain] = container_of(entry, 17066 struct mlx5_flow_dv_matcher, entry); 17067 } 17068 /* Create default rule in drop table. */ 17069 if (!mtrmng->def_rule[domain]) { 17070 i = 0; 17071 actions[i++] = priv->sh->dr_drop_action; 17072 flow_dv_match_meta_reg(matcher_para.buf, value.buf, 17073 (enum modify_reg)mtr_id_reg_c, 0, 0); 17074 misc_mask = flow_dv_matcher_enable(value.buf); 17075 __flow_dv_adjust_buf_size(&value.size, misc_mask); 17076 ret = mlx5_flow_os_create_flow 17077 (mtrmng->def_matcher[domain]->matcher_object, 17078 (void *)&value, i, actions, 17079 &mtrmng->def_rule[domain]); 17080 if (ret) { 17081 DRV_LOG(ERR, "Failed to create meter " 17082 "default drop rule for drop table."); 17083 goto policy_error; 17084 } 17085 } 17086 if (!fm->drop_cnt) 17087 continue; 17088 MLX5_ASSERT(mtrmng->max_mtr_bits); 17089 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) { 17090 /* Create matchers for Drop. */ 17091 flow_dv_match_meta_reg(matcher.mask.buf, value.buf, 17092 (enum modify_reg)mtr_id_reg_c, 0, 17093 (mtr_id_mask << mtr_id_offset)); 17094 matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits; 17095 matcher.crc = rte_raw_cksum 17096 ((const void *)matcher.mask.buf, 17097 matcher.mask.size); 17098 entry = mlx5_list_register(tbl_data->matchers, &ctx); 17099 if (!entry) { 17100 DRV_LOG(ERR, 17101 "Failed to register meter drop matcher."); 17102 goto policy_error; 17103 } 17104 mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] = 17105 container_of(entry, struct mlx5_flow_dv_matcher, 17106 entry); 17107 } 17108 drop_matcher = 17109 mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]; 17110 /* Create drop rule, matching meter_id only. */ 17111 flow_dv_match_meta_reg(matcher_para.buf, value.buf, 17112 (enum modify_reg)mtr_id_reg_c, 17113 (mtr_idx << mtr_id_offset), UINT32_MAX); 17114 i = 0; 17115 cnt = flow_dv_counter_get_by_idx(dev, 17116 fm->drop_cnt, NULL); 17117 actions[i++] = cnt->action; 17118 actions[i++] = priv->sh->dr_drop_action; 17119 misc_mask = flow_dv_matcher_enable(value.buf); 17120 __flow_dv_adjust_buf_size(&value.size, misc_mask); 17121 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object, 17122 (void *)&value, i, actions, 17123 &fm->drop_rule[domain]); 17124 if (ret) { 17125 DRV_LOG(ERR, "Failed to create meter " 17126 "drop rule for drop table."); 17127 goto policy_error; 17128 } 17129 } 17130 return 0; 17131 policy_error: 17132 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 17133 if (fm->drop_rule[i]) { 17134 claim_zero(mlx5_flow_os_destroy_flow 17135 (fm->drop_rule[i])); 17136 fm->drop_rule[i] = NULL; 17137 } 17138 } 17139 return -1; 17140 } 17141 17142 static struct mlx5_flow_meter_sub_policy * 17143 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev, 17144 struct mlx5_flow_meter_policy *mtr_policy, 17145 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS], 17146 struct mlx5_flow_meter_sub_policy *next_sub_policy, 17147 bool *is_reuse) 17148 { 17149 struct mlx5_priv *priv = dev->data->dev_private; 17150 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 17151 uint32_t sub_policy_idx = 0; 17152 uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0}; 17153 uint32_t i, j; 17154 struct mlx5_hrxq *hrxq; 17155 struct mlx5_flow_handle dh; 17156 struct mlx5_meter_policy_action_container *act_cnt; 17157 uint32_t domain = MLX5_MTR_DOMAIN_INGRESS; 17158 uint16_t sub_policy_num; 17159 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 17160 17161 MLX5_ASSERT(wks); 17162 rte_spinlock_lock(&mtr_policy->sl); 17163 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 17164 if (!rss_desc[i]) 17165 continue; 17166 hrxq = mlx5_hrxq_get(dev, rss_desc[i]); 17167 if (!hrxq) { 17168 rte_spinlock_unlock(&mtr_policy->sl); 17169 return NULL; 17170 } 17171 hrxq_idx[i] = hrxq->idx; 17172 } 17173 sub_policy_num = (mtr_policy->sub_policy_num >> 17174 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & 17175 MLX5_MTR_SUB_POLICY_NUM_MASK; 17176 for (j = 0; j < sub_policy_num; j++) { 17177 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 17178 if (rss_desc[i] && 17179 hrxq_idx[i] != 17180 mtr_policy->sub_policys[domain][j]->rix_hrxq[i]) 17181 break; 17182 } 17183 if (i >= MLX5_MTR_RTE_COLORS) { 17184 /* 17185 * Found the sub policy table with 17186 * the same queue per color. 17187 */ 17188 rte_spinlock_unlock(&mtr_policy->sl); 17189 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) 17190 mlx5_hrxq_release(dev, hrxq_idx[i]); 17191 *is_reuse = true; 17192 return mtr_policy->sub_policys[domain][j]; 17193 } 17194 } 17195 /* Create sub policy. */ 17196 if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[RTE_COLOR_GREEN] && 17197 !mtr_policy->sub_policys[domain][0]->rix_hrxq[RTE_COLOR_YELLOW]) { 17198 /* Reuse the first pre-allocated sub_policy. */ 17199 sub_policy = mtr_policy->sub_policys[domain][0]; 17200 sub_policy_idx = sub_policy->idx; 17201 } else { 17202 sub_policy = mlx5_ipool_zmalloc 17203 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 17204 &sub_policy_idx); 17205 if (!sub_policy || 17206 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) { 17207 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) 17208 mlx5_hrxq_release(dev, hrxq_idx[i]); 17209 goto rss_sub_policy_error; 17210 } 17211 sub_policy->idx = sub_policy_idx; 17212 sub_policy->main_policy = mtr_policy; 17213 } 17214 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 17215 if (!rss_desc[i]) 17216 continue; 17217 sub_policy->rix_hrxq[i] = hrxq_idx[i]; 17218 if (mtr_policy->is_hierarchy) { 17219 act_cnt = &mtr_policy->act_cnt[i]; 17220 act_cnt->next_sub_policy = next_sub_policy; 17221 mlx5_hrxq_release(dev, hrxq_idx[i]); 17222 } else { 17223 /* 17224 * Overwrite the last action from 17225 * RSS action to Queue action. 17226 */ 17227 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 17228 hrxq_idx[i]); 17229 if (!hrxq) { 17230 DRV_LOG(ERR, "Failed to get policy hrxq"); 17231 goto rss_sub_policy_error; 17232 } 17233 act_cnt = &mtr_policy->act_cnt[i]; 17234 if (act_cnt->rix_mark || act_cnt->modify_hdr) { 17235 memset(&dh, 0, sizeof(struct mlx5_flow_handle)); 17236 if (act_cnt->rix_mark) 17237 wks->mark = 1; 17238 dh.fate_action = MLX5_FLOW_FATE_QUEUE; 17239 dh.rix_hrxq = hrxq_idx[i]; 17240 flow_drv_rxq_flags_set(dev, &dh); 17241 } 17242 } 17243 } 17244 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy, 17245 sub_policy, domain)) { 17246 DRV_LOG(ERR, "Failed to create policy " 17247 "rules for ingress domain."); 17248 goto rss_sub_policy_error; 17249 } 17250 if (sub_policy != mtr_policy->sub_policys[domain][0]) { 17251 i = (mtr_policy->sub_policy_num >> 17252 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & 17253 MLX5_MTR_SUB_POLICY_NUM_MASK; 17254 if (i >= MLX5_MTR_RSS_MAX_SUB_POLICY) { 17255 DRV_LOG(ERR, "No free sub-policy slot."); 17256 goto rss_sub_policy_error; 17257 } 17258 mtr_policy->sub_policys[domain][i] = sub_policy; 17259 i++; 17260 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 17261 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)); 17262 mtr_policy->sub_policy_num |= 17263 (i & MLX5_MTR_SUB_POLICY_NUM_MASK) << 17264 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain); 17265 } 17266 rte_spinlock_unlock(&mtr_policy->sl); 17267 *is_reuse = false; 17268 return sub_policy; 17269 rss_sub_policy_error: 17270 if (sub_policy) { 17271 __flow_dv_destroy_sub_policy_rules(dev, sub_policy); 17272 if (sub_policy != mtr_policy->sub_policys[domain][0]) { 17273 i = (mtr_policy->sub_policy_num >> 17274 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & 17275 MLX5_MTR_SUB_POLICY_NUM_MASK; 17276 mtr_policy->sub_policys[domain][i] = NULL; 17277 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 17278 sub_policy->idx); 17279 } 17280 } 17281 rte_spinlock_unlock(&mtr_policy->sl); 17282 return NULL; 17283 } 17284 17285 /** 17286 * Find the policy table for prefix table with RSS. 17287 * 17288 * @param[in] dev 17289 * Pointer to Ethernet device. 17290 * @param[in] mtr_policy 17291 * Pointer to meter policy table. 17292 * @param[in] rss_desc 17293 * Pointer to rss_desc 17294 * @return 17295 * Pointer to table set on success, NULL otherwise and rte_errno is set. 17296 */ 17297 static struct mlx5_flow_meter_sub_policy * 17298 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 17299 struct mlx5_flow_meter_policy *mtr_policy, 17300 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 17301 { 17302 struct mlx5_priv *priv = dev->data->dev_private; 17303 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 17304 struct mlx5_flow_meter_info *next_fm; 17305 struct mlx5_flow_meter_policy *next_policy; 17306 struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL; 17307 struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM]; 17308 struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM]; 17309 uint32_t domain = MLX5_MTR_DOMAIN_INGRESS; 17310 bool reuse_sub_policy; 17311 uint32_t i = 0; 17312 uint32_t j = 0; 17313 17314 while (true) { 17315 /* Iterate hierarchy to get all policies in this hierarchy. */ 17316 policies[i++] = mtr_policy; 17317 if (!mtr_policy->is_hierarchy) 17318 break; 17319 if (i >= MLX5_MTR_CHAIN_MAX_NUM) { 17320 DRV_LOG(ERR, "Exceed max meter number in hierarchy."); 17321 return NULL; 17322 } 17323 rte_spinlock_lock(&mtr_policy->sl); 17324 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, mtr_policy, NULL); 17325 rte_spinlock_unlock(&mtr_policy->sl); 17326 if (!next_fm) { 17327 DRV_LOG(ERR, "Failed to get next meter in hierarchy."); 17328 return NULL; 17329 } 17330 next_policy = 17331 mlx5_flow_meter_policy_find(dev, next_fm->policy_id, 17332 NULL); 17333 MLX5_ASSERT(next_policy); 17334 mtr_policy = next_policy; 17335 } 17336 while (i) { 17337 /** 17338 * From last policy to the first one in hierarchy, 17339 * create / get the sub policy for each of them. 17340 */ 17341 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev, 17342 policies[--i], 17343 rss_desc, 17344 next_sub_policy, 17345 &reuse_sub_policy); 17346 if (!sub_policy) { 17347 DRV_LOG(ERR, "Failed to get the sub policy."); 17348 goto err_exit; 17349 } 17350 if (!reuse_sub_policy) 17351 sub_policies[j++] = sub_policy; 17352 next_sub_policy = sub_policy; 17353 } 17354 return sub_policy; 17355 err_exit: 17356 while (j) { 17357 uint16_t sub_policy_num; 17358 17359 sub_policy = sub_policies[--j]; 17360 mtr_policy = sub_policy->main_policy; 17361 __flow_dv_destroy_sub_policy_rules(dev, sub_policy); 17362 if (sub_policy != mtr_policy->sub_policys[domain][0]) { 17363 sub_policy_num = (mtr_policy->sub_policy_num >> 17364 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & 17365 MLX5_MTR_SUB_POLICY_NUM_MASK; 17366 mtr_policy->sub_policys[domain][sub_policy_num - 1] = 17367 NULL; 17368 sub_policy_num--; 17369 mtr_policy->sub_policy_num &= 17370 ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 17371 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 17372 mtr_policy->sub_policy_num |= 17373 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 17374 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 17375 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 17376 sub_policy->idx); 17377 } 17378 } 17379 return NULL; 17380 } 17381 17382 /** 17383 * Check if need to create hierarchy tag rule. 17384 * 17385 * @param[in] priv 17386 * Pointer to mlx5_priv. 17387 * @param[in] mtr_policy 17388 * Pointer to current meter policy. 17389 * @param[in] src_port 17390 * The src port this extra rule should use. 17391 * @param[out] next_fm 17392 * Pointer to next meter in hierarchy. 17393 * @param[out] skip 17394 * Indicate if skip the tag rule creation. 17395 * @param[out] error 17396 * Perform verbose error reporting if not NULL. 17397 * @return 17398 * 0 on success, a negative errno value otherwise and rte_errno is set. 17399 */ 17400 static int 17401 mlx5_meter_hierarchy_skip_tag_rule(struct mlx5_priv *priv, 17402 struct mlx5_flow_meter_policy *mtr_policy, 17403 int32_t src_port, 17404 struct mlx5_flow_meter_info **next_fm, 17405 bool *skip, 17406 struct rte_flow_error *error) 17407 { 17408 struct mlx5_flow_meter_sub_policy *sub_policy; 17409 struct mlx5_sub_policy_color_rule *color_rule; 17410 uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER; 17411 int ret = 0; 17412 int i; 17413 17414 *next_fm = NULL; 17415 *skip = false; 17416 rte_spinlock_lock(&mtr_policy->sl); 17417 if (!mtr_policy->is_hierarchy) 17418 goto exit; 17419 *next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, mtr_policy, NULL); 17420 if (!*next_fm) { 17421 ret = rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 17422 NULL, "Failed to find next meter in hierarchy."); 17423 goto exit; 17424 } 17425 if (!(*next_fm)->drop_cnt) { 17426 *skip = true; 17427 goto exit; 17428 } 17429 sub_policy = mtr_policy->sub_policys[domain][0]; 17430 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 17431 if (mtr_policy->act_cnt[i].fate_action != MLX5_FLOW_FATE_MTR) 17432 continue; 17433 TAILQ_FOREACH(color_rule, &sub_policy->color_rules[i], next_port) 17434 if (color_rule->src_port == src_port) { 17435 *skip = true; 17436 goto exit; 17437 } 17438 } 17439 exit: 17440 rte_spinlock_unlock(&mtr_policy->sl); 17441 return ret; 17442 } 17443 17444 /** 17445 * Create the sub policy tag rule for all meters in hierarchy. 17446 * 17447 * @param[in] dev 17448 * Pointer to Ethernet device. 17449 * @param[in] fm 17450 * Meter information table. 17451 * @param[in] src_port 17452 * The src port this extra rule should use. 17453 * @param[in] item 17454 * The src port match item. 17455 * @param[out] error 17456 * Perform verbose error reporting if not NULL. 17457 * @return 17458 * 0 on success, a negative errno value otherwise and rte_errno is set. 17459 */ 17460 static int 17461 flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev, 17462 struct mlx5_flow_meter_info *fm, 17463 int32_t src_port, 17464 const struct rte_flow_item *item, 17465 struct rte_flow_error *error) 17466 { 17467 struct mlx5_priv *priv = dev->data->dev_private; 17468 struct mlx5_flow_meter_policy *mtr_policy; 17469 struct mlx5_flow_meter_sub_policy *sub_policy; 17470 struct mlx5_flow_meter_info *next_fm = NULL; 17471 struct mlx5_flow_meter_policy *next_policy; 17472 struct mlx5_flow_meter_sub_policy *next_sub_policy; 17473 struct mlx5_flow_tbl_data_entry *tbl_data; 17474 struct mlx5_sub_policy_color_rule *color_rule; 17475 struct mlx5_meter_policy_acts acts; 17476 uint32_t color_reg_c_idx; 17477 bool mtr_first = (src_port != UINT16_MAX) ? true : false; 17478 struct rte_flow_attr attr = { 17479 .group = MLX5_FLOW_TABLE_LEVEL_POLICY, 17480 .priority = 0, 17481 .ingress = 0, 17482 .egress = 0, 17483 .transfer = 1, 17484 .reserved = 0, 17485 }; 17486 uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER; 17487 struct { 17488 struct mlx5_flow_meter_policy *fm_policy; 17489 struct mlx5_flow_meter_info *next_fm; 17490 struct mlx5_sub_policy_color_rule *tag_rule[MLX5_MTR_RTE_COLORS]; 17491 } fm_info[MLX5_MTR_CHAIN_MAX_NUM] = { {0} }; 17492 uint32_t fm_cnt = 0; 17493 uint32_t i, j; 17494 17495 color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error); 17496 /* Get all fms who need to create the tag color rule. */ 17497 do { 17498 bool skip = false; 17499 17500 mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 17501 MLX5_ASSERT(mtr_policy); 17502 if (mlx5_meter_hierarchy_skip_tag_rule(priv, mtr_policy, src_port, 17503 &next_fm, &skip, error)) 17504 goto err_exit; 17505 if (next_fm && !skip) { 17506 fm_info[fm_cnt].fm_policy = mtr_policy; 17507 fm_info[fm_cnt].next_fm = next_fm; 17508 if (++fm_cnt >= MLX5_MTR_CHAIN_MAX_NUM) { 17509 rte_flow_error_set(error, errno, 17510 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 17511 "Exceed max meter number in hierarchy."); 17512 goto err_exit; 17513 } 17514 } 17515 fm = next_fm; 17516 } while (fm); 17517 /* Create tag color rules for all needed fms. */ 17518 for (i = 0; i < fm_cnt; i++) { 17519 void *mtr_action; 17520 17521 mtr_policy = fm_info[i].fm_policy; 17522 rte_spinlock_lock(&mtr_policy->sl); 17523 sub_policy = mtr_policy->sub_policys[domain][0]; 17524 for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) { 17525 if (mtr_policy->act_cnt[j].fate_action != MLX5_FLOW_FATE_MTR) 17526 continue; 17527 color_rule = mlx5_malloc(MLX5_MEM_ZERO, 17528 sizeof(struct mlx5_sub_policy_color_rule), 17529 0, SOCKET_ID_ANY); 17530 if (!color_rule) { 17531 rte_spinlock_unlock(&mtr_policy->sl); 17532 rte_flow_error_set(error, ENOMEM, 17533 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 17534 "No memory to create tag color rule."); 17535 goto err_exit; 17536 } 17537 color_rule->src_port = src_port; 17538 next_fm = fm_info[i].next_fm; 17539 if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) { 17540 mlx5_free(color_rule); 17541 rte_spinlock_unlock(&mtr_policy->sl); 17542 goto err_exit; 17543 } 17544 fm_info[i].tag_rule[j] = color_rule; 17545 TAILQ_INSERT_TAIL(&sub_policy->color_rules[j], color_rule, next_port); 17546 /* Prepare to create color rule. */ 17547 mtr_action = (next_fm->color_aware && j == RTE_COLOR_YELLOW) ? 17548 next_fm->meter_action_y : 17549 next_fm->meter_action_g; 17550 next_policy = mlx5_flow_meter_policy_find(dev, next_fm->policy_id, NULL); 17551 MLX5_ASSERT(next_policy); 17552 next_sub_policy = next_policy->sub_policys[domain][0]; 17553 tbl_data = container_of(next_sub_policy->tbl_rsc, 17554 struct mlx5_flow_tbl_data_entry, tbl); 17555 if (mtr_first) { 17556 acts.dv_actions[0] = mtr_action; 17557 acts.dv_actions[1] = mtr_policy->act_cnt[j].modify_hdr->action; 17558 } else { 17559 acts.dv_actions[0] = mtr_policy->act_cnt[j].modify_hdr->action; 17560 acts.dv_actions[1] = mtr_action; 17561 } 17562 acts.dv_actions[2] = tbl_data->jump.action; 17563 acts.actions_n = 3; 17564 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx, 17565 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy, 17566 &attr, true, item, &color_rule->matcher, error)) { 17567 rte_spinlock_unlock(&mtr_policy->sl); 17568 rte_flow_error_set(error, errno, 17569 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 17570 "Failed to create hierarchy meter matcher."); 17571 goto err_exit; 17572 } 17573 if (__flow_dv_create_policy_flow(dev, color_reg_c_idx, (enum rte_color)j, 17574 color_rule->matcher->matcher_object, 17575 acts.actions_n, acts.dv_actions, 17576 true, item, &color_rule->rule, &attr)) { 17577 rte_spinlock_unlock(&mtr_policy->sl); 17578 rte_flow_error_set(error, errno, 17579 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 17580 "Failed to create hierarchy meter rule."); 17581 goto err_exit; 17582 } 17583 } 17584 rte_spinlock_unlock(&mtr_policy->sl); 17585 } 17586 return 0; 17587 err_exit: 17588 for (i = 0; i < fm_cnt; i++) { 17589 mtr_policy = fm_info[i].fm_policy; 17590 rte_spinlock_lock(&mtr_policy->sl); 17591 sub_policy = mtr_policy->sub_policys[domain][0]; 17592 for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) { 17593 color_rule = fm_info[i].tag_rule[j]; 17594 if (!color_rule) 17595 continue; 17596 if (color_rule->rule) 17597 mlx5_flow_os_destroy_flow(color_rule->rule); 17598 if (color_rule->matcher) { 17599 struct mlx5_flow_tbl_data_entry *tbl = 17600 container_of(color_rule->matcher->tbl, typeof(*tbl), tbl); 17601 mlx5_list_unregister(tbl->matchers, &color_rule->matcher->entry); 17602 } 17603 if (fm_info[i].next_fm) 17604 mlx5_flow_meter_detach(priv, fm_info[i].next_fm); 17605 TAILQ_REMOVE(&sub_policy->color_rules[j], color_rule, next_port); 17606 mlx5_free(color_rule); 17607 } 17608 rte_spinlock_unlock(&mtr_policy->sl); 17609 } 17610 return -rte_errno; 17611 } 17612 17613 /** 17614 * Destroy the sub policy table with RX queue. 17615 * 17616 * @param[in] dev 17617 * Pointer to Ethernet device. 17618 * @param[in] mtr_policy 17619 * Pointer to meter policy table. 17620 */ 17621 static void 17622 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 17623 struct mlx5_flow_meter_policy *mtr_policy) 17624 { 17625 struct mlx5_priv *priv = dev->data->dev_private; 17626 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 17627 uint32_t domain = MLX5_MTR_DOMAIN_INGRESS; 17628 uint32_t i, j; 17629 uint16_t sub_policy_num, new_policy_num; 17630 17631 rte_spinlock_lock(&mtr_policy->sl); 17632 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 17633 switch (mtr_policy->act_cnt[i].fate_action) { 17634 case MLX5_FLOW_FATE_SHARED_RSS: 17635 sub_policy_num = (mtr_policy->sub_policy_num >> 17636 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & 17637 MLX5_MTR_SUB_POLICY_NUM_MASK; 17638 new_policy_num = sub_policy_num; 17639 for (j = 0; j < sub_policy_num; j++) { 17640 sub_policy = 17641 mtr_policy->sub_policys[domain][j]; 17642 if (sub_policy) { 17643 __flow_dv_destroy_sub_policy_rules(dev, 17644 sub_policy); 17645 if (sub_policy != 17646 mtr_policy->sub_policys[domain][0]) { 17647 mtr_policy->sub_policys[domain][j] = 17648 NULL; 17649 mlx5_ipool_free 17650 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 17651 sub_policy->idx); 17652 new_policy_num--; 17653 } 17654 } 17655 } 17656 if (new_policy_num != sub_policy_num) { 17657 mtr_policy->sub_policy_num &= 17658 ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 17659 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)); 17660 mtr_policy->sub_policy_num |= 17661 (new_policy_num & 17662 MLX5_MTR_SUB_POLICY_NUM_MASK) << 17663 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain); 17664 } 17665 break; 17666 case MLX5_FLOW_FATE_QUEUE: 17667 sub_policy = mtr_policy->sub_policys[domain][0]; 17668 __flow_dv_destroy_sub_policy_rules(dev, 17669 sub_policy); 17670 break; 17671 default: 17672 /*Other actions without queue and do nothing*/ 17673 break; 17674 } 17675 } 17676 rte_spinlock_unlock(&mtr_policy->sl); 17677 } 17678 /** 17679 * Check whether the DR drop action is supported on the root table or not. 17680 * 17681 * Create a simple flow with DR drop action on root table to validate 17682 * if DR drop action on root table is supported or not. 17683 * 17684 * @param[in] dev 17685 * Pointer to rte_eth_dev structure. 17686 * 17687 * @return 17688 * 0 on success, a negative errno value otherwise and rte_errno is set. 17689 */ 17690 int 17691 mlx5_flow_discover_dr_action_support(struct rte_eth_dev *dev) 17692 { 17693 struct mlx5_priv *priv = dev->data->dev_private; 17694 struct mlx5_dev_ctx_shared *sh = priv->sh; 17695 struct mlx5_flow_dv_match_params mask = { 17696 .size = sizeof(mask.buf), 17697 }; 17698 struct mlx5_flow_dv_match_params value = { 17699 .size = sizeof(value.buf), 17700 }; 17701 struct mlx5dv_flow_matcher_attr dv_attr = { 17702 .type = IBV_FLOW_ATTR_NORMAL, 17703 .priority = 0, 17704 .match_criteria_enable = 0, 17705 .match_mask = (void *)&mask, 17706 }; 17707 struct mlx5_flow_tbl_resource *tbl = NULL; 17708 void *matcher = NULL; 17709 void *flow = NULL; 17710 int ret = -1; 17711 17712 tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 17713 0, 0, 0, NULL); 17714 if (!tbl) 17715 goto err; 17716 dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf); 17717 __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable); 17718 ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr, 17719 tbl->obj, &matcher); 17720 if (ret) 17721 goto err; 17722 __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable); 17723 ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1, 17724 &sh->dr_drop_action, &flow); 17725 err: 17726 /* 17727 * If DR drop action is not supported on root table, flow create will 17728 * be failed with EOPNOTSUPP or EPROTONOSUPPORT. 17729 */ 17730 if (!flow) { 17731 if (matcher && 17732 (errno == EPROTONOSUPPORT || errno == EOPNOTSUPP)) 17733 DRV_LOG(INFO, "DR drop action is not supported in root table."); 17734 else 17735 DRV_LOG(ERR, "Unexpected error in DR drop action support detection"); 17736 ret = -1; 17737 } else { 17738 claim_zero(mlx5_flow_os_destroy_flow(flow)); 17739 } 17740 if (matcher) 17741 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher)); 17742 if (tbl) 17743 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); 17744 return ret; 17745 } 17746 17747 /** 17748 * Validate the batch counter support in root table. 17749 * 17750 * Create a simple flow with invalid counter and drop action on root table to 17751 * validate if batch counter with offset on root table is supported or not. 17752 * 17753 * @param[in] dev 17754 * Pointer to rte_eth_dev structure. 17755 * 17756 * @return 17757 * 0 on success, a negative errno value otherwise and rte_errno is set. 17758 */ 17759 int 17760 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) 17761 { 17762 struct mlx5_priv *priv = dev->data->dev_private; 17763 struct mlx5_dev_ctx_shared *sh = priv->sh; 17764 struct mlx5_flow_dv_match_params mask = { 17765 .size = sizeof(mask.buf), 17766 }; 17767 struct mlx5_flow_dv_match_params value = { 17768 .size = sizeof(value.buf), 17769 }; 17770 struct mlx5dv_flow_matcher_attr dv_attr = { 17771 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS, 17772 .priority = 0, 17773 .match_criteria_enable = 0, 17774 .match_mask = (void *)&mask, 17775 }; 17776 void *actions[2] = { 0 }; 17777 struct mlx5_flow_tbl_resource *tbl = NULL; 17778 struct mlx5_devx_obj *dcs = NULL; 17779 void *matcher = NULL; 17780 void *flow = NULL; 17781 int ret = -1; 17782 17783 tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL, 17784 0, 0, 0, NULL); 17785 if (!tbl) 17786 goto err; 17787 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0x4); 17788 if (!dcs) 17789 goto err; 17790 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX, 17791 &actions[0]); 17792 if (ret) 17793 goto err; 17794 dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf); 17795 __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable); 17796 ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr, 17797 tbl->obj, &matcher); 17798 if (ret) 17799 goto err; 17800 __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable); 17801 ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1, 17802 actions, &flow); 17803 err: 17804 /* 17805 * If batch counter with offset is not supported, the driver will not 17806 * validate the invalid offset value, flow create should success. 17807 * In this case, it means batch counter is not supported in root table. 17808 * 17809 * Otherwise, if flow create is failed, counter offset is supported. 17810 */ 17811 if (flow) { 17812 DRV_LOG(INFO, "Batch counter is not supported in root " 17813 "table. Switch to fallback mode."); 17814 rte_errno = ENOTSUP; 17815 ret = -rte_errno; 17816 claim_zero(mlx5_flow_os_destroy_flow(flow)); 17817 } else { 17818 /* Check matcher to make sure validate fail at flow create. */ 17819 if (!matcher || (matcher && errno != EINVAL)) 17820 DRV_LOG(ERR, "Unexpected error in counter offset " 17821 "support detection"); 17822 ret = 0; 17823 } 17824 if (actions[0]) 17825 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0])); 17826 if (matcher) 17827 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher)); 17828 if (tbl) 17829 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); 17830 if (dcs) 17831 claim_zero(mlx5_devx_cmd_destroy(dcs)); 17832 return ret; 17833 } 17834 17835 /** 17836 * Query a devx counter. 17837 * 17838 * @param[in] dev 17839 * Pointer to the Ethernet device structure. 17840 * @param[in] cnt 17841 * Index to the flow counter. 17842 * @param[in] clear 17843 * Set to clear the counter statistics. 17844 * @param[out] pkts 17845 * The statistics value of packets. 17846 * @param[out] bytes 17847 * The statistics value of bytes. 17848 * 17849 * @return 17850 * 0 on success, otherwise return -1. 17851 */ 17852 static int 17853 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear, 17854 uint64_t *pkts, uint64_t *bytes, void **action) 17855 { 17856 struct mlx5_priv *priv = dev->data->dev_private; 17857 struct mlx5_flow_counter *cnt; 17858 uint64_t inn_pkts, inn_bytes; 17859 int ret; 17860 17861 if (!priv->sh->cdev->config.devx) 17862 return -1; 17863 17864 ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes); 17865 if (ret) 17866 return -1; 17867 cnt = flow_dv_counter_get_by_idx(dev, counter, NULL); 17868 if (cnt && action) 17869 *action = cnt->action; 17870 17871 *pkts = inn_pkts - cnt->hits; 17872 *bytes = inn_bytes - cnt->bytes; 17873 if (clear) { 17874 cnt->hits = inn_pkts; 17875 cnt->bytes = inn_bytes; 17876 } 17877 return 0; 17878 } 17879 17880 /** 17881 * Get aged-out flows. 17882 * 17883 * @param[in] dev 17884 * Pointer to the Ethernet device structure. 17885 * @param[in] context 17886 * The address of an array of pointers to the aged-out flows contexts. 17887 * @param[in] nb_contexts 17888 * The length of context array pointers. 17889 * @param[out] error 17890 * Perform verbose error reporting if not NULL. Initialized in case of 17891 * error only. 17892 * 17893 * @return 17894 * how many contexts get in success, otherwise negative errno value. 17895 * if nb_contexts is 0, return the amount of all aged contexts. 17896 * if nb_contexts is not 0 , return the amount of aged flows reported 17897 * in the context array. 17898 * @note: only stub for now 17899 */ 17900 static int 17901 flow_dv_get_aged_flows(struct rte_eth_dev *dev, 17902 void **context, 17903 uint32_t nb_contexts, 17904 struct rte_flow_error *error) 17905 { 17906 struct mlx5_priv *priv = dev->data->dev_private; 17907 struct mlx5_age_info *age_info; 17908 struct mlx5_age_param *age_param; 17909 struct mlx5_flow_counter *counter; 17910 struct mlx5_aso_age_action *act; 17911 int nb_flows = 0; 17912 17913 if (nb_contexts && !context) 17914 return rte_flow_error_set(error, EINVAL, 17915 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 17916 NULL, "empty context"); 17917 age_info = GET_PORT_AGE_INFO(priv); 17918 rte_spinlock_lock(&age_info->aged_sl); 17919 LIST_FOREACH(act, &age_info->aged_aso, next) { 17920 nb_flows++; 17921 if (nb_contexts) { 17922 context[nb_flows - 1] = 17923 act->age_params.context; 17924 if (!(--nb_contexts)) 17925 break; 17926 } 17927 } 17928 TAILQ_FOREACH(counter, &age_info->aged_counters, next) { 17929 nb_flows++; 17930 if (nb_contexts) { 17931 age_param = MLX5_CNT_TO_AGE(counter); 17932 context[nb_flows - 1] = age_param->context; 17933 if (!(--nb_contexts)) 17934 break; 17935 } 17936 } 17937 rte_spinlock_unlock(&age_info->aged_sl); 17938 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 17939 return nb_flows; 17940 } 17941 17942 /* 17943 * Mutex-protected thunk to lock-free flow_dv_counter_alloc(). 17944 */ 17945 static uint32_t 17946 flow_dv_counter_allocate(struct rte_eth_dev *dev) 17947 { 17948 return flow_dv_counter_alloc(dev, 0); 17949 } 17950 17951 /** 17952 * Validate indirect action. 17953 * Dispatcher for action type specific validation. 17954 * 17955 * @param[in] dev 17956 * Pointer to the Ethernet device structure. 17957 * @param[in] conf 17958 * Indirect action configuration. 17959 * @param[in] action 17960 * The indirect action object to validate. 17961 * @param[out] error 17962 * Perform verbose error reporting if not NULL. Initialized in case of 17963 * error only. 17964 * 17965 * @return 17966 * 0 on success, otherwise negative errno value. 17967 */ 17968 int 17969 flow_dv_action_validate(struct rte_eth_dev *dev, 17970 const struct rte_flow_indir_action_conf *conf, 17971 const struct rte_flow_action *action, 17972 struct rte_flow_error *err) 17973 { 17974 struct mlx5_priv *priv = dev->data->dev_private; 17975 17976 RTE_SET_USED(conf); 17977 switch (action->type) { 17978 case RTE_FLOW_ACTION_TYPE_RSS: 17979 /* 17980 * priv->obj_ops is set according to driver capabilities. 17981 * When DevX capabilities are 17982 * sufficient, it is set to devx_obj_ops. 17983 * Otherwise, it is set to ibv_obj_ops. 17984 * ibv_obj_ops doesn't support ind_table_modify operation. 17985 * In this case the indirect RSS action can't be used. 17986 */ 17987 if (priv->obj_ops.ind_table_modify == NULL) 17988 return rte_flow_error_set 17989 (err, ENOTSUP, 17990 RTE_FLOW_ERROR_TYPE_ACTION, 17991 NULL, 17992 "Indirect RSS action not supported"); 17993 return mlx5_validate_action_rss(dev, action, err); 17994 case RTE_FLOW_ACTION_TYPE_AGE: 17995 if (!priv->sh->aso_age_mng) 17996 return rte_flow_error_set(err, ENOTSUP, 17997 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 17998 NULL, 17999 "Indirect age action not supported"); 18000 return flow_dv_validate_action_age(0, action, dev, err); 18001 case RTE_FLOW_ACTION_TYPE_COUNT: 18002 return flow_dv_validate_action_count(dev, true, 0, NULL, err); 18003 case RTE_FLOW_ACTION_TYPE_CONNTRACK: 18004 if (!priv->sh->ct_aso_en) 18005 return rte_flow_error_set(err, ENOTSUP, 18006 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 18007 "ASO CT is not supported"); 18008 return mlx5_validate_action_ct(dev, action->conf, err); 18009 default: 18010 return rte_flow_error_set(err, ENOTSUP, 18011 RTE_FLOW_ERROR_TYPE_ACTION, 18012 NULL, 18013 "action type not supported"); 18014 } 18015 } 18016 18017 /* 18018 * Check if the RSS configurations for colors of a meter policy match 18019 * each other, except the queues. 18020 * 18021 * @param[in] r1 18022 * Pointer to the first RSS flow action. 18023 * @param[in] r2 18024 * Pointer to the second RSS flow action. 18025 * 18026 * @return 18027 * 0 on match, 1 on conflict. 18028 */ 18029 static inline int 18030 flow_dv_mtr_policy_rss_compare(const struct rte_flow_action_rss *r1, 18031 const struct rte_flow_action_rss *r2) 18032 { 18033 if (r1 == NULL || r2 == NULL) 18034 return 0; 18035 if (!(r1->level <= 1 && r2->level <= 1) && 18036 !(r1->level > 1 && r2->level > 1)) 18037 return 1; 18038 if (r1->types != r2->types && 18039 !((r1->types == 0 || r1->types == RTE_ETH_RSS_IP) && 18040 (r2->types == 0 || r2->types == RTE_ETH_RSS_IP))) 18041 return 1; 18042 if (r1->key || r2->key) { 18043 const void *key1 = r1->key ? r1->key : rss_hash_default_key; 18044 const void *key2 = r2->key ? r2->key : rss_hash_default_key; 18045 18046 if (memcmp(key1, key2, MLX5_RSS_HASH_KEY_LEN)) 18047 return 1; 18048 } 18049 return 0; 18050 } 18051 18052 /** 18053 * Validate the meter hierarchy chain for meter policy. 18054 * 18055 * @param[in] dev 18056 * Pointer to the Ethernet device structure. 18057 * @param[in] meter_id 18058 * Meter id. 18059 * @param[in] action_flags 18060 * Holds the actions detected until now. 18061 * @param[out] is_rss 18062 * Is RSS or not. 18063 * @param[out] hierarchy_domain 18064 * The domain bitmap for hierarchy policy. 18065 * @param[out] error 18066 * Perform verbose error reporting if not NULL. Initialized in case of 18067 * error only. 18068 * 18069 * @return 18070 * 0 on success, otherwise negative errno value with error set. 18071 */ 18072 static int 18073 flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev, 18074 uint32_t meter_id, 18075 uint64_t action_flags, 18076 bool *is_rss, 18077 uint8_t *hierarchy_domain, 18078 struct rte_mtr_error *error) 18079 { 18080 struct mlx5_priv *priv = dev->data->dev_private; 18081 struct mlx5_flow_meter_info *fm; 18082 struct mlx5_flow_meter_policy *policy; 18083 uint8_t cnt = 1; 18084 18085 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 18086 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 18087 return -rte_mtr_error_set(error, EINVAL, 18088 RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN, 18089 NULL, 18090 "Multiple fate actions not supported."); 18091 *hierarchy_domain = 0; 18092 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 18093 while (true) { 18094 if (!fm) 18095 return -rte_mtr_error_set(error, EINVAL, 18096 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 18097 "Meter not found in meter hierarchy."); 18098 if (fm->def_policy) 18099 return -rte_mtr_error_set(error, EINVAL, 18100 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 18101 "Non termination meter not supported in hierarchy."); 18102 if (!fm->shared) 18103 return -rte_mtr_error_set(error, EINVAL, 18104 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 18105 "Only shared meter supported in hierarchy."); 18106 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 18107 MLX5_ASSERT(policy); 18108 /** 18109 * Only inherit the supported domains of the first meter in 18110 * hierarchy. 18111 * One meter supports at least one domain. 18112 */ 18113 if (!*hierarchy_domain) { 18114 if (policy->transfer) 18115 *hierarchy_domain |= 18116 MLX5_MTR_DOMAIN_TRANSFER_BIT; 18117 if (policy->ingress) 18118 *hierarchy_domain |= 18119 MLX5_MTR_DOMAIN_INGRESS_BIT; 18120 if (policy->egress) 18121 *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT; 18122 } 18123 if (!policy->is_hierarchy) { 18124 *is_rss = policy->is_rss; 18125 break; 18126 } 18127 rte_spinlock_lock(&policy->sl); 18128 fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, NULL); 18129 rte_spinlock_unlock(&policy->sl); 18130 if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM) 18131 return -rte_mtr_error_set(error, EINVAL, 18132 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 18133 "Exceed max hierarchy meter number."); 18134 } 18135 return 0; 18136 } 18137 18138 /** 18139 * Validate meter policy actions. 18140 * Dispatcher for action type specific validation. 18141 * 18142 * @param[in] dev 18143 * Pointer to the Ethernet device structure. 18144 * @param[in] action 18145 * The meter policy action object to validate. 18146 * @param[in] attr 18147 * Attributes of flow to determine steering domain. 18148 * @param[out] error 18149 * Perform verbose error reporting if not NULL. Initialized in case of 18150 * error only. 18151 * 18152 * @return 18153 * 0 on success, otherwise negative errno value. 18154 */ 18155 static int 18156 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev, 18157 const struct rte_flow_action *actions[RTE_COLORS], 18158 struct rte_flow_attr *attr, 18159 bool *is_rss, 18160 uint8_t *domain_bitmap, 18161 uint8_t *policy_mode, 18162 struct rte_mtr_error *error) 18163 { 18164 struct mlx5_priv *priv = dev->data->dev_private; 18165 struct mlx5_sh_config *dev_conf = &priv->sh->config; 18166 const struct rte_flow_action *act; 18167 uint64_t action_flags[RTE_COLORS] = {0}; 18168 int actions_n; 18169 int i, ret; 18170 struct rte_flow_error flow_err; 18171 uint8_t domain_color[RTE_COLORS] = {0}; 18172 uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT; 18173 uint8_t hierarchy_domain = 0; 18174 const struct rte_flow_action_meter *mtr; 18175 const struct rte_flow_action_meter *next_mtr = NULL; 18176 bool def_green = false; 18177 bool def_yellow = false; 18178 const struct rte_flow_action_rss *rss_color[RTE_COLORS] = {NULL}; 18179 18180 if (!dev_conf->dv_esw_en) 18181 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 18182 *domain_bitmap = def_domain; 18183 /* Red color could only support DROP action. */ 18184 if (!actions[RTE_COLOR_RED] || 18185 actions[RTE_COLOR_RED]->type != RTE_FLOW_ACTION_TYPE_DROP) 18186 return -rte_mtr_error_set(error, ENOTSUP, 18187 RTE_MTR_ERROR_TYPE_METER_POLICY, 18188 NULL, "Red color only supports drop action."); 18189 /* 18190 * Check default policy actions: 18191 * Green / Yellow: no action, Red: drop action 18192 * Either G or Y will trigger default policy actions to be created. 18193 */ 18194 if (!actions[RTE_COLOR_GREEN] || 18195 actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END) 18196 def_green = true; 18197 if (!actions[RTE_COLOR_YELLOW] || 18198 actions[RTE_COLOR_YELLOW]->type == RTE_FLOW_ACTION_TYPE_END) 18199 def_yellow = true; 18200 if (def_green && def_yellow) { 18201 *policy_mode = MLX5_MTR_POLICY_MODE_DEF; 18202 return 0; 18203 } else if (!def_green && def_yellow) { 18204 *policy_mode = MLX5_MTR_POLICY_MODE_OG; 18205 } else if (def_green && !def_yellow) { 18206 *policy_mode = MLX5_MTR_POLICY_MODE_OY; 18207 } else { 18208 *policy_mode = MLX5_MTR_POLICY_MODE_ALL; 18209 } 18210 /* Set to empty string in case of NULL pointer access by user. */ 18211 flow_err.message = ""; 18212 for (i = 0; i < RTE_COLORS; i++) { 18213 act = actions[i]; 18214 for (action_flags[i] = 0, actions_n = 0; 18215 act && act->type != RTE_FLOW_ACTION_TYPE_END; 18216 act++) { 18217 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) 18218 return -rte_mtr_error_set(error, ENOTSUP, 18219 RTE_MTR_ERROR_TYPE_METER_POLICY, 18220 NULL, "too many actions"); 18221 switch (act->type) { 18222 case RTE_FLOW_ACTION_TYPE_PORT_ID: 18223 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 18224 if (!dev_conf->dv_esw_en) 18225 return -rte_mtr_error_set(error, 18226 ENOTSUP, 18227 RTE_MTR_ERROR_TYPE_METER_POLICY, 18228 NULL, "PORT action validate check" 18229 " fail for ESW disable"); 18230 ret = flow_dv_validate_action_port_id(dev, 18231 action_flags[i], 18232 act, attr, &flow_err); 18233 if (ret) 18234 return -rte_mtr_error_set(error, 18235 ENOTSUP, 18236 RTE_MTR_ERROR_TYPE_METER_POLICY, 18237 NULL, flow_err.message ? 18238 flow_err.message : 18239 "PORT action validate check fail"); 18240 ++actions_n; 18241 action_flags[i] |= MLX5_FLOW_ACTION_PORT_ID; 18242 break; 18243 case RTE_FLOW_ACTION_TYPE_MARK: 18244 ret = flow_dv_validate_action_mark(dev, act, 18245 action_flags[i], 18246 attr, &flow_err); 18247 if (ret < 0) 18248 return -rte_mtr_error_set(error, 18249 ENOTSUP, 18250 RTE_MTR_ERROR_TYPE_METER_POLICY, 18251 NULL, flow_err.message ? 18252 flow_err.message : 18253 "Mark action validate check fail"); 18254 if (dev_conf->dv_xmeta_en != 18255 MLX5_XMETA_MODE_LEGACY) 18256 return -rte_mtr_error_set(error, 18257 ENOTSUP, 18258 RTE_MTR_ERROR_TYPE_METER_POLICY, 18259 NULL, "Extend MARK action is " 18260 "not supported. Please try use " 18261 "default policy for meter."); 18262 action_flags[i] |= MLX5_FLOW_ACTION_MARK; 18263 ++actions_n; 18264 break; 18265 case RTE_FLOW_ACTION_TYPE_SET_TAG: 18266 ret = flow_dv_validate_action_set_tag(dev, 18267 act, action_flags[i], 18268 attr, &flow_err); 18269 if (ret) 18270 return -rte_mtr_error_set(error, 18271 ENOTSUP, 18272 RTE_MTR_ERROR_TYPE_METER_POLICY, 18273 NULL, flow_err.message ? 18274 flow_err.message : 18275 "Set tag action validate check fail"); 18276 action_flags[i] |= MLX5_FLOW_ACTION_SET_TAG; 18277 ++actions_n; 18278 break; 18279 case RTE_FLOW_ACTION_TYPE_DROP: 18280 ret = mlx5_flow_validate_action_drop 18281 (action_flags[i], attr, &flow_err); 18282 if (ret < 0) 18283 return -rte_mtr_error_set(error, 18284 ENOTSUP, 18285 RTE_MTR_ERROR_TYPE_METER_POLICY, 18286 NULL, flow_err.message ? 18287 flow_err.message : 18288 "Drop action validate check fail"); 18289 action_flags[i] |= MLX5_FLOW_ACTION_DROP; 18290 ++actions_n; 18291 break; 18292 case RTE_FLOW_ACTION_TYPE_QUEUE: 18293 /* 18294 * Check whether extensive 18295 * metadata feature is engaged. 18296 */ 18297 if (dev_conf->dv_flow_en && 18298 (dev_conf->dv_xmeta_en != 18299 MLX5_XMETA_MODE_LEGACY) && 18300 mlx5_flow_ext_mreg_supported(dev)) 18301 return -rte_mtr_error_set(error, 18302 ENOTSUP, 18303 RTE_MTR_ERROR_TYPE_METER_POLICY, 18304 NULL, "Queue action with meta " 18305 "is not supported. Please try use " 18306 "default policy for meter."); 18307 ret = mlx5_flow_validate_action_queue(act, 18308 action_flags[i], dev, 18309 attr, &flow_err); 18310 if (ret < 0) 18311 return -rte_mtr_error_set(error, 18312 ENOTSUP, 18313 RTE_MTR_ERROR_TYPE_METER_POLICY, 18314 NULL, flow_err.message ? 18315 flow_err.message : 18316 "Queue action validate check fail"); 18317 action_flags[i] |= MLX5_FLOW_ACTION_QUEUE; 18318 ++actions_n; 18319 break; 18320 case RTE_FLOW_ACTION_TYPE_RSS: 18321 if (dev_conf->dv_flow_en && 18322 (dev_conf->dv_xmeta_en != 18323 MLX5_XMETA_MODE_LEGACY) && 18324 mlx5_flow_ext_mreg_supported(dev)) 18325 return -rte_mtr_error_set(error, 18326 ENOTSUP, 18327 RTE_MTR_ERROR_TYPE_METER_POLICY, 18328 NULL, "RSS action with meta " 18329 "is not supported. Please try use " 18330 "default policy for meter."); 18331 ret = mlx5_validate_action_rss(dev, act, 18332 &flow_err); 18333 if (ret < 0) 18334 return -rte_mtr_error_set(error, 18335 ENOTSUP, 18336 RTE_MTR_ERROR_TYPE_METER_POLICY, 18337 NULL, flow_err.message ? 18338 flow_err.message : 18339 "RSS action validate check fail"); 18340 action_flags[i] |= MLX5_FLOW_ACTION_RSS; 18341 ++actions_n; 18342 /* Either G or Y will set the RSS. */ 18343 rss_color[i] = act->conf; 18344 break; 18345 case RTE_FLOW_ACTION_TYPE_JUMP: 18346 ret = flow_dv_validate_action_jump(dev, 18347 NULL, act, action_flags[i], 18348 attr, true, &flow_err); 18349 if (ret) 18350 return -rte_mtr_error_set(error, 18351 ENOTSUP, 18352 RTE_MTR_ERROR_TYPE_METER_POLICY, 18353 NULL, flow_err.message ? 18354 flow_err.message : 18355 "Jump action validate check fail"); 18356 ++actions_n; 18357 action_flags[i] |= MLX5_FLOW_ACTION_JUMP; 18358 break; 18359 case RTE_FLOW_ACTION_TYPE_METER: 18360 mtr = act->conf; 18361 if (next_mtr && next_mtr->mtr_id != mtr->mtr_id) 18362 return -rte_mtr_error_set(error, ENOTSUP, 18363 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 18364 "Green and Yellow must use the same meter."); 18365 ret = flow_dv_validate_policy_mtr_hierarchy(dev, 18366 mtr->mtr_id, 18367 action_flags[i], 18368 is_rss, 18369 &hierarchy_domain, 18370 error); 18371 if (ret) 18372 return ret; 18373 ++actions_n; 18374 action_flags[i] |= 18375 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY; 18376 next_mtr = mtr; 18377 break; 18378 default: 18379 return -rte_mtr_error_set(error, ENOTSUP, 18380 RTE_MTR_ERROR_TYPE_METER_POLICY, 18381 NULL, 18382 "Doesn't support optional action"); 18383 } 18384 } 18385 if (action_flags[i] & MLX5_FLOW_ACTION_PORT_ID) { 18386 domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT; 18387 } else if ((action_flags[i] & 18388 (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) || 18389 (action_flags[i] & MLX5_FLOW_ACTION_MARK)) { 18390 /* 18391 * Only support MLX5_XMETA_MODE_LEGACY 18392 * so MARK action is only in ingress domain. 18393 */ 18394 domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT; 18395 } else { 18396 domain_color[i] = def_domain; 18397 if (action_flags[i] && 18398 !(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 18399 domain_color[i] &= 18400 ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 18401 } 18402 if (action_flags[i] & 18403 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) 18404 domain_color[i] &= hierarchy_domain; 18405 /* 18406 * Non-termination actions only support NIC Tx domain. 18407 * The adjustion should be skipped when there is no 18408 * action or only END is provided. The default domains 18409 * bit-mask is set to find the MIN intersection. 18410 * The action flags checking should also be skipped. 18411 */ 18412 if ((def_green && i == RTE_COLOR_GREEN) || 18413 (def_yellow && i == RTE_COLOR_YELLOW)) 18414 continue; 18415 /* 18416 * Validate the drop action mutual exclusion 18417 * with other actions. Drop action is mutually-exclusive 18418 * with any other action, except for Count action. 18419 */ 18420 if ((action_flags[i] & MLX5_FLOW_ACTION_DROP) && 18421 (action_flags[i] & ~MLX5_FLOW_ACTION_DROP)) { 18422 return -rte_mtr_error_set(error, ENOTSUP, 18423 RTE_MTR_ERROR_TYPE_METER_POLICY, 18424 NULL, "Drop action is mutually-exclusive " 18425 "with any other action"); 18426 } 18427 /* Eswitch has few restrictions on using items and actions */ 18428 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) { 18429 if (!mlx5_flow_ext_mreg_supported(dev) && 18430 action_flags[i] & MLX5_FLOW_ACTION_MARK) 18431 return -rte_mtr_error_set(error, ENOTSUP, 18432 RTE_MTR_ERROR_TYPE_METER_POLICY, 18433 NULL, "unsupported action MARK"); 18434 if (action_flags[i] & MLX5_FLOW_ACTION_QUEUE) 18435 return -rte_mtr_error_set(error, ENOTSUP, 18436 RTE_MTR_ERROR_TYPE_METER_POLICY, 18437 NULL, "unsupported action QUEUE"); 18438 if (action_flags[i] & MLX5_FLOW_ACTION_RSS) 18439 return -rte_mtr_error_set(error, ENOTSUP, 18440 RTE_MTR_ERROR_TYPE_METER_POLICY, 18441 NULL, "unsupported action RSS"); 18442 if (!(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 18443 return -rte_mtr_error_set(error, ENOTSUP, 18444 RTE_MTR_ERROR_TYPE_METER_POLICY, 18445 NULL, "no fate action is found"); 18446 } else { 18447 if (!(action_flags[i] & MLX5_FLOW_FATE_ACTIONS) && 18448 (domain_color[i] & MLX5_MTR_DOMAIN_INGRESS_BIT)) { 18449 if ((domain_color[i] & 18450 MLX5_MTR_DOMAIN_EGRESS_BIT)) 18451 domain_color[i] = 18452 MLX5_MTR_DOMAIN_EGRESS_BIT; 18453 else 18454 return -rte_mtr_error_set(error, 18455 ENOTSUP, 18456 RTE_MTR_ERROR_TYPE_METER_POLICY, 18457 NULL, 18458 "no fate action is found"); 18459 } 18460 } 18461 } 18462 if (next_mtr && *policy_mode == MLX5_MTR_POLICY_MODE_ALL) { 18463 if (!(action_flags[RTE_COLOR_GREEN] & action_flags[RTE_COLOR_YELLOW] & 18464 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) 18465 return -rte_mtr_error_set(error, EINVAL, RTE_MTR_ERROR_TYPE_METER_POLICY, 18466 NULL, 18467 "Meter hierarchy supports meter action only."); 18468 } 18469 /* If both colors have RSS, the attributes should be the same. */ 18470 if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN], 18471 rss_color[RTE_COLOR_YELLOW])) 18472 return -rte_mtr_error_set(error, EINVAL, 18473 RTE_MTR_ERROR_TYPE_METER_POLICY, 18474 NULL, "policy RSS attr conflict"); 18475 if (rss_color[RTE_COLOR_GREEN] || rss_color[RTE_COLOR_YELLOW]) 18476 *is_rss = true; 18477 /* "domain_color[C]" is non-zero for each color, default is ALL. */ 18478 if (!def_green && !def_yellow && 18479 domain_color[RTE_COLOR_GREEN] != domain_color[RTE_COLOR_YELLOW] && 18480 !(action_flags[RTE_COLOR_GREEN] & MLX5_FLOW_ACTION_DROP) && 18481 !(action_flags[RTE_COLOR_YELLOW] & MLX5_FLOW_ACTION_DROP)) 18482 return -rte_mtr_error_set(error, EINVAL, 18483 RTE_MTR_ERROR_TYPE_METER_POLICY, 18484 NULL, "policy domains conflict"); 18485 /* 18486 * At least one color policy is listed in the actions, the domains 18487 * to be supported should be the intersection. 18488 */ 18489 *domain_bitmap = domain_color[RTE_COLOR_GREEN] & 18490 domain_color[RTE_COLOR_YELLOW]; 18491 return 0; 18492 } 18493 18494 static int 18495 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags) 18496 { 18497 struct mlx5_priv *priv = dev->data->dev_private; 18498 int ret = 0; 18499 18500 if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) { 18501 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain, 18502 flags); 18503 if (ret != 0) 18504 return ret; 18505 } 18506 if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) { 18507 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags); 18508 if (ret != 0) 18509 return ret; 18510 } 18511 if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) { 18512 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags); 18513 if (ret != 0) 18514 return ret; 18515 } 18516 return 0; 18517 } 18518 18519 /** 18520 * Discover the number of available flow priorities 18521 * by trying to create a flow with the highest priority value 18522 * for each possible number. 18523 * 18524 * @param[in] dev 18525 * Ethernet device. 18526 * @param[in] vprio 18527 * List of possible number of available priorities. 18528 * @param[in] vprio_n 18529 * Size of @p vprio array. 18530 * @return 18531 * On success, number of available flow priorities. 18532 * On failure, a negative errno-style code and rte_errno is set. 18533 */ 18534 static int 18535 flow_dv_discover_priorities(struct rte_eth_dev *dev, 18536 const uint16_t *vprio, int vprio_n) 18537 { 18538 struct mlx5_priv *priv = dev->data->dev_private; 18539 struct mlx5_indexed_pool *pool = priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW]; 18540 struct rte_flow_item_eth eth; 18541 struct rte_flow_item item = { 18542 .type = RTE_FLOW_ITEM_TYPE_ETH, 18543 .spec = ð, 18544 .mask = ð, 18545 }; 18546 struct mlx5_flow_dv_matcher matcher = { 18547 .mask = { 18548 .size = sizeof(matcher.mask.buf), 18549 }, 18550 }; 18551 union mlx5_flow_tbl_key tbl_key; 18552 struct mlx5_flow flow; 18553 void *action; 18554 struct rte_flow_error error; 18555 uint8_t misc_mask; 18556 int i, err, ret = -ENOTSUP; 18557 18558 /* 18559 * Prepare a flow with a catch-all pattern and a drop action. 18560 * Use drop queue, because shared drop action may be unavailable. 18561 */ 18562 action = priv->drop_queue.hrxq->action; 18563 if (action == NULL) { 18564 DRV_LOG(ERR, "Priority discovery requires a drop action"); 18565 rte_errno = ENOTSUP; 18566 return -rte_errno; 18567 } 18568 memset(&flow, 0, sizeof(flow)); 18569 flow.handle = mlx5_ipool_zmalloc(pool, &flow.handle_idx); 18570 if (flow.handle == NULL) { 18571 DRV_LOG(ERR, "Cannot create flow handle"); 18572 rte_errno = ENOMEM; 18573 return -rte_errno; 18574 } 18575 flow.ingress = true; 18576 flow.dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); 18577 flow.dv.actions[0] = action; 18578 flow.dv.actions_n = 1; 18579 memset(ð, 0, sizeof(eth)); 18580 flow_dv_translate_item_eth(matcher.mask.buf, flow.dv.value.buf, 18581 &item, /* inner */ false, /* group */ 0); 18582 matcher.crc = rte_raw_cksum(matcher.mask.buf, matcher.mask.size); 18583 for (i = 0; i < vprio_n; i++) { 18584 /* Configure the next proposed maximum priority. */ 18585 matcher.priority = vprio[i] - 1; 18586 memset(&tbl_key, 0, sizeof(tbl_key)); 18587 err = flow_dv_matcher_register(dev, &matcher, &tbl_key, &flow, 18588 /* tunnel */ NULL, 18589 /* group */ 0, 18590 &error); 18591 if (err != 0) { 18592 /* This action is pure SW and must always succeed. */ 18593 DRV_LOG(ERR, "Cannot register matcher"); 18594 ret = -rte_errno; 18595 break; 18596 } 18597 /* Try to apply the flow to HW. */ 18598 misc_mask = flow_dv_matcher_enable(flow.dv.value.buf); 18599 __flow_dv_adjust_buf_size(&flow.dv.value.size, misc_mask); 18600 err = mlx5_flow_os_create_flow 18601 (flow.handle->dvh.matcher->matcher_object, 18602 (void *)&flow.dv.value, flow.dv.actions_n, 18603 flow.dv.actions, &flow.handle->drv_flow); 18604 if (err == 0) { 18605 claim_zero(mlx5_flow_os_destroy_flow 18606 (flow.handle->drv_flow)); 18607 flow.handle->drv_flow = NULL; 18608 } 18609 claim_zero(flow_dv_matcher_release(dev, flow.handle)); 18610 if (err != 0) 18611 break; 18612 ret = vprio[i]; 18613 } 18614 mlx5_ipool_free(pool, flow.handle_idx); 18615 /* Set rte_errno if no expected priority value matched. */ 18616 if (ret < 0) 18617 rte_errno = -ret; 18618 return ret; 18619 } 18620 18621 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { 18622 .validate = flow_dv_validate, 18623 .prepare = flow_dv_prepare, 18624 .translate = flow_dv_translate, 18625 .apply = flow_dv_apply, 18626 .remove = flow_dv_remove, 18627 .destroy = flow_dv_destroy, 18628 .query = flow_dv_query, 18629 .create_mtr_tbls = flow_dv_create_mtr_tbls, 18630 .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls, 18631 .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls, 18632 .create_meter = flow_dv_mtr_alloc, 18633 .free_meter = flow_dv_aso_mtr_release_to_pool, 18634 .validate_mtr_acts = flow_dv_validate_mtr_policy_acts, 18635 .create_mtr_acts = flow_dv_create_mtr_policy_acts, 18636 .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts, 18637 .create_policy_rules = flow_dv_create_policy_rules, 18638 .destroy_policy_rules = flow_dv_destroy_policy_rules, 18639 .create_def_policy = flow_dv_create_def_policy, 18640 .destroy_def_policy = flow_dv_destroy_def_policy, 18641 .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare, 18642 .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create, 18643 .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq, 18644 .counter_alloc = flow_dv_counter_allocate, 18645 .counter_free = flow_dv_counter_free, 18646 .counter_query = flow_dv_counter_query, 18647 .get_aged_flows = flow_dv_get_aged_flows, 18648 .action_validate = flow_dv_action_validate, 18649 .action_create = flow_dv_action_create, 18650 .action_destroy = flow_dv_action_destroy, 18651 .action_update = flow_dv_action_update, 18652 .action_query = flow_dv_action_query, 18653 .sync_domain = flow_dv_sync_domain, 18654 .discover_priorities = flow_dv_discover_priorities, 18655 .item_create = flow_dv_item_create, 18656 .item_release = flow_dv_item_release, 18657 }; 18658 18659 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 18660