xref: /dpdk/drivers/net/mlx5/mlx5_flow_hw.c (revision e9fd1ebf981f361844aea9ec94e17f4bda5e1479)

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2022 NVIDIA Corporation & Affiliates
 */

#include <rte_flow.h>
#include <rte_flow_driver.h>

#include <mlx5_malloc.h>

#include "mlx5.h"
#include "mlx5_defs.h"
#include "mlx5_flow.h"
#include "mlx5_rx.h"

#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
#include "mlx5_hws_cnt.h"

/** Fast path async flow API functions. */
static struct rte_flow_fp_ops mlx5_flow_hw_fp_ops;

/* The maximum actions support in the flow. */
#define MLX5_HW_MAX_ACTS 16

/*
 * The default ipool threshold value indicates which per_core_cache
 * value to set.
 */
#define MLX5_HW_IPOOL_SIZE_THRESHOLD (1 << 19)
/* The default min local cache size. */
#define MLX5_HW_IPOOL_CACHE_MIN (1 << 9)

/* Default push burst threshold. */
#define BURST_THR 32u

/* Default queue to flush the flows. */
#define MLX5_DEFAULT_FLUSH_QUEUE 0

/* Maximum number of rules in control flow tables. */
#define MLX5_HW_CTRL_FLOW_NB_RULES (4096)

/* Lowest flow group usable by an application if group translation is done. */
#define MLX5_HW_LOWEST_USABLE_GROUP (1)

/* Maximum group index usable by user applications for transfer flows. */
#define MLX5_HW_MAX_TRANSFER_GROUP (UINT32_MAX - 1)

/* Maximum group index usable by user applications for egress flows. */
#define MLX5_HW_MAX_EGRESS_GROUP (UINT32_MAX - 1)

/* Lowest priority for HW root table. */
#define MLX5_HW_LOWEST_PRIO_ROOT 15

/* Lowest priority for HW non-root table. */
#define MLX5_HW_LOWEST_PRIO_NON_ROOT (UINT32_MAX)

/* Priorities for Rx control flow rules. */
#define MLX5_HW_CTRL_RX_PRIO_L2 (MLX5_HW_LOWEST_PRIO_ROOT)
#define MLX5_HW_CTRL_RX_PRIO_L3 (MLX5_HW_LOWEST_PRIO_ROOT - 1)
#define MLX5_HW_CTRL_RX_PRIO_L4 (MLX5_HW_LOWEST_PRIO_ROOT - 2)

#define MLX5_HW_VLAN_PUSH_TYPE_IDX 0
#define MLX5_HW_VLAN_PUSH_VID_IDX 1
#define MLX5_HW_VLAN_PUSH_PCP_IDX 2

#define MLX5_MIRROR_MAX_CLONES_NUM 3
#define MLX5_MIRROR_MAX_SAMPLE_ACTIONS_LEN 4

#define MLX5_HW_PORT_IS_PROXY(priv) \
	(!!((priv)->sh->esw_mode && (priv)->master))


struct mlx5_indlst_legacy {
	struct mlx5_indirect_list indirect;
	struct rte_flow_action_handle *handle;
	enum rte_flow_action_type legacy_type;
};

#define MLX5_CONST_ENCAP_ITEM(encap_type, ptr) \
(((const struct encap_type *)(ptr))->definition)

struct mlx5_multi_pattern_ctx {
	union {
		struct mlx5dr_action_reformat_header reformat_hdr;
		struct mlx5dr_action_mh_pattern mh_pattern;
	};
	union {
		/* action template auxiliary structures for object destruction */
		struct mlx5_hw_encap_decap_action *encap;
		struct mlx5_hw_modify_header_action *mhdr;
	};
	/* multi pattern action */
	struct mlx5dr_rule_action *rule_action;
};

#define MLX5_MULTIPATTERN_ENCAP_NUM 4

struct mlx5_tbl_multi_pattern_ctx {
	struct {
		uint32_t elements_num;
		struct mlx5_multi_pattern_ctx ctx[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
	} reformat[MLX5_MULTIPATTERN_ENCAP_NUM];

	struct {
		uint32_t elements_num;
		struct mlx5_multi_pattern_ctx ctx[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
	} mh;
};

#define MLX5_EMPTY_MULTI_PATTERN_CTX {{{0,}},}

static int
mlx5_tbl_multi_pattern_process(struct rte_eth_dev *dev,
			       struct rte_flow_template_table *tbl,
			       struct mlx5_tbl_multi_pattern_ctx *mpat,
			       struct rte_flow_error *error);

static __rte_always_inline int
mlx5_multi_pattern_reformat_to_index(enum mlx5dr_action_type type)
{
	switch (type) {
	case MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2:
		return 0;
	case MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2:
		return 1;
	case MLX5DR_ACTION_TYP_REFORMAT_TNL_L3_TO_L2:
		return 2;
	case MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L3:
		return 3;
	default:
		break;
	}
	return -1;
}

static __rte_always_inline enum mlx5dr_action_type
mlx5_multi_pattern_reformat_index_to_type(uint32_t ix)
{
	switch (ix) {
	case 0:
		return MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2;
	case 1:
		return MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2;
	case 2:
		return MLX5DR_ACTION_TYP_REFORMAT_TNL_L3_TO_L2;
	case 3:
		return MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L3;
	default:
		break;
	}
	return MLX5DR_ACTION_TYP_MAX;
}

static inline enum mlx5dr_table_type
get_mlx5dr_table_type(const struct rte_flow_attr *attr)
{
	enum mlx5dr_table_type type;

	if (attr->transfer)
		type = MLX5DR_TABLE_TYPE_FDB;
	else if (attr->egress)
		type = MLX5DR_TABLE_TYPE_NIC_TX;
	else
		type = MLX5DR_TABLE_TYPE_NIC_RX;
	return type;
}

struct mlx5_mirror_clone {
	enum rte_flow_action_type type;
	void *action_ctx;
};

struct mlx5_mirror {
	struct mlx5_indirect_list indirect;
	uint32_t clones_num;
	struct mlx5dr_action *mirror_action;
	struct mlx5_mirror_clone clone[MLX5_MIRROR_MAX_CLONES_NUM];
};

static int flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev);
static int flow_hw_translate_group(struct rte_eth_dev *dev,
				   const struct mlx5_flow_template_table_cfg *cfg,
				   uint32_t group,
				   uint32_t *table_group,
				   struct rte_flow_error *error);
static __rte_always_inline int
flow_hw_set_vlan_vid_construct(struct rte_eth_dev *dev,
			       struct mlx5_hw_q_job *job,
			       struct mlx5_action_construct_data *act_data,
			       const struct mlx5_hw_actions *hw_acts,
			       const struct rte_flow_action *action);
static void
flow_hw_construct_quota(struct mlx5_priv *priv,
			struct mlx5dr_rule_action *rule_act, uint32_t qid);

static __rte_always_inline uint32_t flow_hw_tx_tag_regc_mask(struct rte_eth_dev *dev);
static __rte_always_inline uint32_t flow_hw_tx_tag_regc_value(struct rte_eth_dev *dev);

const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops;

/* DR action flags with different table. */
static uint32_t mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_MAX]
				[MLX5DR_TABLE_TYPE_MAX] = {
	{
		MLX5DR_ACTION_FLAG_ROOT_RX,
		MLX5DR_ACTION_FLAG_ROOT_TX,
		MLX5DR_ACTION_FLAG_ROOT_FDB,
	},
	{
		MLX5DR_ACTION_FLAG_HWS_RX,
		MLX5DR_ACTION_FLAG_HWS_TX,
		MLX5DR_ACTION_FLAG_HWS_FDB,
	},
};

/* Ethernet item spec for promiscuous mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_promisc_spec = {
	.hdr.dst_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};
/* Ethernet item mask for promiscuous mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_promisc_mask = {
	.hdr.dst_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};

/* Ethernet item spec for all multicast mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_mcast_spec = {
	.hdr.dst_addr.addr_bytes = "\x01\x00\x00\x00\x00\x00",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};
/* Ethernet item mask for all multicast mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_mcast_mask = {
	.hdr.dst_addr.addr_bytes = "\x01\x00\x00\x00\x00\x00",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};

/* Ethernet item spec for IPv4 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv4_mcast_spec = {
	.hdr.dst_addr.addr_bytes = "\x01\x00\x5e\x00\x00\x00",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};
/* Ethernet item mask for IPv4 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv4_mcast_mask = {
	.hdr.dst_addr.addr_bytes = "\xff\xff\xff\x00\x00\x00",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};

/* Ethernet item spec for IPv6 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv6_mcast_spec = {
	.hdr.dst_addr.addr_bytes = "\x33\x33\x00\x00\x00\x00",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};
/* Ethernet item mask for IPv6 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv6_mcast_mask = {
	.hdr.dst_addr.addr_bytes = "\xff\xff\x00\x00\x00\x00",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};

/* Ethernet item mask for unicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_dmac_mask = {
	.hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};

/* Ethernet item spec for broadcast. */
static const struct rte_flow_item_eth ctrl_rx_eth_bcast_spec = {
	.hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff",
	.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.hdr.ether_type = 0,
};

static __rte_always_inline struct mlx5_hw_q_job *
flow_hw_job_get(struct mlx5_priv *priv, uint32_t queue)
{
	MLX5_ASSERT(priv->hw_q[queue].job_idx <= priv->hw_q[queue].size);
	return priv->hw_q[queue].job_idx ?
	       priv->hw_q[queue].job[--priv->hw_q[queue].job_idx] : NULL;
}

static __rte_always_inline void
flow_hw_job_put(struct mlx5_priv *priv, struct mlx5_hw_q_job *job, uint32_t queue)
{
	MLX5_ASSERT(priv->hw_q[queue].job_idx < priv->hw_q[queue].size);
	priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job;
}

static inline enum mlx5dr_matcher_insert_mode
flow_hw_matcher_insert_mode_get(enum rte_flow_table_insertion_type insert_type)
{
	if (insert_type == RTE_FLOW_TABLE_INSERTION_TYPE_PATTERN)
		return MLX5DR_MATCHER_INSERT_BY_HASH;
	else
		return MLX5DR_MATCHER_INSERT_BY_INDEX;
}

static inline enum mlx5dr_matcher_distribute_mode
flow_hw_matcher_distribute_mode_get(enum rte_flow_table_hash_func hash_func)
{
	if (hash_func == RTE_FLOW_TABLE_HASH_FUNC_LINEAR)
		return MLX5DR_MATCHER_DISTRIBUTE_BY_LINEAR;
	else
		return MLX5DR_MATCHER_DISTRIBUTE_BY_HASH;
}

/**
 * Set the hash fields according to the @p rss_desc information.
 *
 * @param[in] rss_desc
 *   Pointer to the mlx5_flow_rss_desc.
 * @param[out] hash_fields
 *   Pointer to the RSS hash fields.
 */
static void
flow_hw_hashfields_set(struct mlx5_flow_rss_desc *rss_desc,
		       uint64_t *hash_fields)
{
	uint64_t fields = 0;
	int rss_inner = 0;
	uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);

#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
	if (rss_desc->level >= 2)
		rss_inner = 1;
#endif
	if (rss_types & MLX5_IPV4_LAYER_TYPES) {
		if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
			fields |= IBV_RX_HASH_SRC_IPV4;
		else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
			fields |= IBV_RX_HASH_DST_IPV4;
		else
			fields |= MLX5_IPV4_IBV_RX_HASH;
	} else if (rss_types & MLX5_IPV6_LAYER_TYPES) {
		if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
			fields |= IBV_RX_HASH_SRC_IPV6;
		else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
			fields |= IBV_RX_HASH_DST_IPV6;
		else
			fields |= MLX5_IPV6_IBV_RX_HASH;
	}
	if (rss_types & RTE_ETH_RSS_UDP) {
		if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
			fields |= IBV_RX_HASH_SRC_PORT_UDP;
		else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
			fields |= IBV_RX_HASH_DST_PORT_UDP;
		else
			fields |= MLX5_UDP_IBV_RX_HASH;
	} else if (rss_types & RTE_ETH_RSS_TCP) {
		if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
			fields |= IBV_RX_HASH_SRC_PORT_TCP;
		else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
			fields |= IBV_RX_HASH_DST_PORT_TCP;
		else
			fields |= MLX5_TCP_IBV_RX_HASH;
	}
	if (rss_types & RTE_ETH_RSS_ESP)
		fields |= IBV_RX_HASH_IPSEC_SPI;
	if (rss_inner)
		fields |= IBV_RX_HASH_INNER;
	*hash_fields = fields;
}

/**
 * Generate the matching pattern item flags.
 *
 * @param[in] items
 *   Pointer to the list of items.
 *
 * @return
 *   Matching item flags. RSS hash field function
 *   silently ignores the flags which are unsupported.
 */
static uint64_t
flow_hw_matching_item_flags_get(const struct rte_flow_item items[])
{
	uint64_t item_flags = 0;
	uint64_t last_item = 0;

	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
		int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
		int item_type = items->type;

		switch (item_type) {
		case RTE_FLOW_ITEM_TYPE_IPV4:
			last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
					     MLX5_FLOW_LAYER_OUTER_L3_IPV4;
			break;
		case RTE_FLOW_ITEM_TYPE_IPV6:
			last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
					     MLX5_FLOW_LAYER_OUTER_L3_IPV6;
			break;
		case RTE_FLOW_ITEM_TYPE_TCP:
			last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
					     MLX5_FLOW_LAYER_OUTER_L4_TCP;
			break;
		case RTE_FLOW_ITEM_TYPE_UDP:
			last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
					     MLX5_FLOW_LAYER_OUTER_L4_UDP;
			break;
		case RTE_FLOW_ITEM_TYPE_IPV6_ROUTING_EXT:
			last_item = tunnel ? MLX5_FLOW_ITEM_INNER_IPV6_ROUTING_EXT :
					     MLX5_FLOW_ITEM_OUTER_IPV6_ROUTING_EXT;
			break;
		case RTE_FLOW_ITEM_TYPE_GRE:
			last_item = MLX5_FLOW_LAYER_GRE;
			break;
		case RTE_FLOW_ITEM_TYPE_NVGRE:
			last_item = MLX5_FLOW_LAYER_GRE;
			break;
		case RTE_FLOW_ITEM_TYPE_VXLAN:
			last_item = MLX5_FLOW_LAYER_VXLAN;
			break;
		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
			last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
			break;
		case RTE_FLOW_ITEM_TYPE_GENEVE:
			last_item = MLX5_FLOW_LAYER_GENEVE;
			break;
		case RTE_FLOW_ITEM_TYPE_MPLS:
			last_item = MLX5_FLOW_LAYER_MPLS;
			break;
		case RTE_FLOW_ITEM_TYPE_GTP:
			last_item = MLX5_FLOW_LAYER_GTP;
			break;
		case RTE_FLOW_ITEM_TYPE_COMPARE:
			last_item = MLX5_FLOW_ITEM_COMPARE;
			break;
		default:
			break;
		}
		item_flags |= last_item;
	}
	return item_flags;
}

/**
 * Register destination table DR jump action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] table_attr
 *   Pointer to the flow attributes.
 * @param[in] dest_group
 *   The destination group ID.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Table on success, NULL otherwise and rte_errno is set.
 */
static struct mlx5_hw_jump_action *
flow_hw_jump_action_register(struct rte_eth_dev *dev,
			     const struct mlx5_flow_template_table_cfg *cfg,
			     uint32_t dest_group,
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_attr jattr = cfg->attr.flow_attr;
	struct mlx5_flow_group *grp;
	struct mlx5_flow_cb_ctx ctx = {
		.dev = dev,
		.error = error,
		.data = &jattr,
	};
	struct mlx5_list_entry *ge;
	uint32_t target_group;

	target_group = dest_group;
	if (flow_hw_translate_group(dev, cfg, dest_group, &target_group, error))
		return NULL;
	jattr.group = target_group;
	ge = mlx5_hlist_register(priv->sh->flow_tbls, target_group, &ctx);
	if (!ge)
		return NULL;
	grp = container_of(ge, struct mlx5_flow_group, entry);
	return &grp->jump;
}

/**
 * Release jump action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] jump
 *   Pointer to the jump action.
 */

static void
flow_hw_jump_release(struct rte_eth_dev *dev, struct mlx5_hw_jump_action *jump)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_group *grp;

	grp = container_of
		(jump, struct mlx5_flow_group, jump);
	mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
}

/**
 * Register queue/RSS action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] hws_flags
 *   DR action flags.
 * @param[in] action
 *   rte flow action.
 *
 * @return
 *    Table on success, NULL otherwise and rte_errno is set.
 */
static inline struct mlx5_hrxq*
flow_hw_tir_action_register(struct rte_eth_dev *dev,
			    uint32_t hws_flags,
			    const struct rte_flow_action *action)
{
	struct mlx5_flow_rss_desc rss_desc = {
		.hws_flags = hws_flags,
	};
	struct mlx5_hrxq *hrxq;

	if (action->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
		const struct rte_flow_action_queue *queue = action->conf;

		rss_desc.const_q = &queue->index;
		rss_desc.queue_num = 1;
	} else {
		const struct rte_flow_action_rss *rss = action->conf;

		rss_desc.queue_num = rss->queue_num;
		rss_desc.const_q = rss->queue;
		memcpy(rss_desc.key,
		       !rss->key ? rss_hash_default_key : rss->key,
		       MLX5_RSS_HASH_KEY_LEN);
		rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
		rss_desc.types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
		rss_desc.symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func);
		flow_hw_hashfields_set(&rss_desc, &rss_desc.hash_fields);
		flow_dv_action_rss_l34_hash_adjust(rss->types,
						   &rss_desc.hash_fields);
		if (rss->level > 1) {
			rss_desc.hash_fields |= IBV_RX_HASH_INNER;
			rss_desc.tunnel = 1;
		}
	}
	hrxq = mlx5_hrxq_get(dev, &rss_desc);
	return hrxq;
}

static __rte_always_inline int
flow_hw_ct_compile(struct rte_eth_dev *dev,
		   uint32_t queue, uint32_t idx,
		   struct mlx5dr_rule_action *rule_act)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_action *ct;

	ct = mlx5_ipool_get(priv->hws_ctpool->cts, MLX5_ACTION_CTX_CT_GET_IDX(idx));
	if (!ct || mlx5_aso_ct_available(priv->sh, queue, ct))
		return -1;
	rule_act->action = priv->hws_ctpool->dr_action;
	rule_act->aso_ct.offset = ct->offset;
	rule_act->aso_ct.direction = ct->is_original ?
		MLX5DR_ACTION_ASO_CT_DIRECTION_INITIATOR :
		MLX5DR_ACTION_ASO_CT_DIRECTION_RESPONDER;
	return 0;
}

static void
flow_hw_template_destroy_reformat_action(struct mlx5_hw_encap_decap_action *encap_decap)
{
	if (encap_decap->multi_pattern) {
		uint32_t refcnt = __atomic_sub_fetch(encap_decap->multi_pattern_refcnt,
						     1, __ATOMIC_RELAXED);
		if (refcnt)
			return;
		mlx5_free((void *)(uintptr_t)encap_decap->multi_pattern_refcnt);
	}
	if (encap_decap->action)
		mlx5dr_action_destroy(encap_decap->action);
}

static void
flow_hw_template_destroy_mhdr_action(struct mlx5_hw_modify_header_action *mhdr)
{
	if (mhdr->multi_pattern) {
		uint32_t refcnt = __atomic_sub_fetch(mhdr->multi_pattern_refcnt,
						     1, __ATOMIC_RELAXED);
		if (refcnt)
			return;
		mlx5_free((void *)(uintptr_t)mhdr->multi_pattern_refcnt);
	}
	if (mhdr->action)
		mlx5dr_action_destroy(mhdr->action);
}

/**
 * Destroy DR actions created by action template.
 *
 * For DR actions created during table creation's action translate.
 * Need to destroy the DR action when destroying the table.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 */
static void
__flow_hw_action_template_destroy(struct rte_eth_dev *dev,
				 struct mlx5_hw_actions *acts)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_action_construct_data *data;

	while (!LIST_EMPTY(&acts->act_list)) {
		data = LIST_FIRST(&acts->act_list);
		LIST_REMOVE(data, next);
		mlx5_ipool_free(priv->acts_ipool, data->idx);
	}

	if (acts->mark)
		if (!(__atomic_fetch_sub(&priv->hws_mark_refcnt, 1, __ATOMIC_RELAXED) - 1))
			flow_hw_rxq_flag_set(dev, false);

	if (acts->jump) {
		struct mlx5_flow_group *grp;

		grp = container_of
			(acts->jump, struct mlx5_flow_group, jump);
		mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
		acts->jump = NULL;
	}
	if (acts->tir) {
		mlx5_hrxq_release(dev, acts->tir->idx);
		acts->tir = NULL;
	}
	if (acts->encap_decap) {
		flow_hw_template_destroy_reformat_action(acts->encap_decap);
		mlx5_free(acts->encap_decap);
		acts->encap_decap = NULL;
	}
	if (acts->push_remove) {
		if (acts->push_remove->action)
			mlx5dr_action_destroy(acts->push_remove->action);
		mlx5_free(acts->push_remove);
		acts->push_remove = NULL;
	}
	if (acts->mhdr) {
		flow_hw_template_destroy_mhdr_action(acts->mhdr);
		mlx5_free(acts->mhdr);
		acts->mhdr = NULL;
	}
	if (mlx5_hws_cnt_id_valid(acts->cnt_id)) {
		mlx5_hws_cnt_shared_put(priv->hws_cpool, &acts->cnt_id);
		acts->cnt_id = 0;
	}
	if (acts->mtr_id) {
		mlx5_ipool_free(priv->hws_mpool->idx_pool, acts->mtr_id);
		acts->mtr_id = 0;
	}
}

/**
 * Append dynamic action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline struct mlx5_action_construct_data *
__flow_hw_act_data_alloc(struct mlx5_priv *priv,
			 enum rte_flow_action_type type,
			 uint16_t action_src,
			 uint16_t action_dst)
{
	struct mlx5_action_construct_data *act_data;
	uint32_t idx = 0;

	act_data = mlx5_ipool_zmalloc(priv->acts_ipool, &idx);
	if (!act_data)
		return NULL;
	act_data->idx = idx;
	act_data->type = type;
	act_data->action_src = action_src;
	act_data->action_dst = action_dst;
	return act_data;
}

/**
 * Append dynamic action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_general_append(struct mlx5_priv *priv,
				  struct mlx5_hw_actions *acts,
				  enum rte_flow_action_type type,
				  uint16_t action_src,
				  uint16_t action_dst)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

static __rte_always_inline int
flow_hw_act_data_indirect_list_append(struct mlx5_priv *priv,
				      struct mlx5_hw_actions *acts,
				      enum rte_flow_action_type type,
				      uint16_t action_src, uint16_t action_dst,
				      indirect_list_callback_t cb)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->indirect_list_cb = cb;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}
/**
 * Append dynamic encap action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] len
 *   Length of the data to be updated.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_encap_append(struct mlx5_priv *priv,
				struct mlx5_hw_actions *acts,
				enum rte_flow_action_type type,
				uint16_t action_src,
				uint16_t action_dst,
				uint16_t len)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->encap.len = len;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Append dynamic push action to the dynamic action list.
 *
 * @param[in] dev
 *   Pointer to the port.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] len
 *   Length of the data to be updated.
 *
 * @return
 *    Data pointer on success, NULL otherwise and rte_errno is set.
 */
static __rte_always_inline void *
__flow_hw_act_data_push_append(struct rte_eth_dev *dev,
			       struct mlx5_hw_actions *acts,
			       enum rte_flow_action_type type,
			       uint16_t action_src,
			       uint16_t action_dst,
			       uint16_t len)
{
	struct mlx5_action_construct_data *act_data;
	struct mlx5_priv *priv = dev->data->dev_private;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return NULL;
	act_data->ipv6_ext.len = len;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return act_data;
}

static __rte_always_inline int
__flow_hw_act_data_hdr_modify_append(struct mlx5_priv *priv,
				     struct mlx5_hw_actions *acts,
				     enum rte_flow_action_type type,
				     uint16_t action_src,
				     uint16_t action_dst,
				     uint16_t mhdr_cmds_off,
				     uint16_t mhdr_cmds_end,
				     bool shared,
				     struct field_modify_info *field,
				     struct field_modify_info *dcopy,
				     uint32_t *mask)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->modify_header.mhdr_cmds_off = mhdr_cmds_off;
	act_data->modify_header.mhdr_cmds_end = mhdr_cmds_end;
	act_data->modify_header.shared = shared;
	rte_memcpy(act_data->modify_header.field, field,
		   sizeof(*field) * MLX5_ACT_MAX_MOD_FIELDS);
	rte_memcpy(act_data->modify_header.dcopy, dcopy,
		   sizeof(*dcopy) * MLX5_ACT_MAX_MOD_FIELDS);
	rte_memcpy(act_data->modify_header.mask, mask,
		   sizeof(*mask) * MLX5_ACT_MAX_MOD_FIELDS);
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Append shared RSS action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] idx
 *   Shared RSS index.
 * @param[in] rss
 *   Pointer to the shared RSS info.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_shared_rss_append(struct mlx5_priv *priv,
				     struct mlx5_hw_actions *acts,
				     enum rte_flow_action_type type,
				     uint16_t action_src,
				     uint16_t action_dst,
				     uint32_t idx,
				     struct mlx5_shared_action_rss *rss)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->shared_rss.level = rss->origin.level;
	act_data->shared_rss.types = !rss->origin.types ? RTE_ETH_RSS_IP :
				     rss->origin.types;
	act_data->shared_rss.idx = idx;
	act_data->shared_rss.symmetric_hash_function =
		MLX5_RSS_IS_SYMM(rss->origin.func);
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Append shared counter action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] cnt_id
 *   Shared counter id.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_shared_cnt_append(struct mlx5_priv *priv,
				     struct mlx5_hw_actions *acts,
				     enum rte_flow_action_type type,
				     uint16_t action_src,
				     uint16_t action_dst,
				     cnt_id_t cnt_id)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->type = type;
	act_data->shared_counter.id = cnt_id;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Append shared meter_mark action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] mtr_id
 *   Shared meter id.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_shared_mtr_append(struct mlx5_priv *priv,
				     struct mlx5_hw_actions *acts,
				     enum rte_flow_action_type type,
				     uint16_t action_src,
				     uint16_t action_dst,
				     cnt_id_t mtr_id)
{	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->type = type;
	act_data->shared_meter.id = mtr_id;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Translate shared indirect action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev data structure.
 * @param[in] action
 *   Pointer to the shared indirect rte_flow action.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_shared_action_translate(struct rte_eth_dev *dev,
				const struct rte_flow_action *action,
				struct mlx5_hw_actions *acts,
				uint16_t action_src,
				uint16_t action_dst)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_shared_action_rss *shared_rss;
	uint32_t act_idx = (uint32_t)(uintptr_t)action->conf;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t idx = act_idx &
		       ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);

	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_RSS:
		shared_rss = mlx5_ipool_get
		  (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
		if (!shared_rss || __flow_hw_act_data_shared_rss_append
		    (priv, acts,
		    (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_RSS,
		    action_src, action_dst, idx, shared_rss))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_COUNT:
		if (__flow_hw_act_data_shared_cnt_append(priv, acts,
			(enum rte_flow_action_type)
			MLX5_RTE_FLOW_ACTION_TYPE_COUNT,
			action_src, action_dst, act_idx))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		/* Not supported, prevent by validate function. */
		MLX5_ASSERT(0);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		if (flow_hw_ct_compile(dev, MLX5_HW_INV_QUEUE,
				       idx, &acts->rule_acts[action_dst]))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		if (__flow_hw_act_data_shared_mtr_append(priv, acts,
			(enum rte_flow_action_type)
			MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK,
			action_src, action_dst, idx))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_QUOTA:
		flow_hw_construct_quota(priv, &acts->rule_acts[action_dst], idx);
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type:%d", type);
		break;
	}
	return 0;
}

static __rte_always_inline bool
flow_hw_action_modify_field_is_shared(const struct rte_flow_action *action,
				      const struct rte_flow_action *mask)
{
	const struct rte_flow_action_modify_field *v = action->conf;
	const struct rte_flow_action_modify_field *m = mask->conf;

	if (v->src.field == RTE_FLOW_FIELD_VALUE) {
		uint32_t j;

		for (j = 0; j < RTE_DIM(m->src.value); ++j) {
			/*
			 * Immediate value is considered to be masked
			 * (and thus shared by all flow rules), if mask
			 * is non-zero. Partial mask over immediate value
			 * is not allowed.
			 */
			if (m->src.value[j])
				return true;
		}
		return false;
	}
	if (v->src.field == RTE_FLOW_FIELD_POINTER)
		return m->src.pvalue != NULL;
	/*
	 * Source field types other than VALUE and
	 * POINTER are always shared.
	 */
	return true;
}

static __rte_always_inline bool
flow_hw_should_insert_nop(const struct mlx5_hw_modify_header_action *mhdr,
			  const struct mlx5_modification_cmd *cmd)
{
	struct mlx5_modification_cmd last_cmd = { { 0 } };
	struct mlx5_modification_cmd new_cmd = { { 0 } };
	const uint32_t cmds_num = mhdr->mhdr_cmds_num;
	unsigned int last_type;
	bool should_insert = false;

	if (cmds_num == 0)
		return false;
	last_cmd = *(&mhdr->mhdr_cmds[cmds_num - 1]);
	last_cmd.data0 = rte_be_to_cpu_32(last_cmd.data0);
	last_cmd.data1 = rte_be_to_cpu_32(last_cmd.data1);
	last_type = last_cmd.action_type;
	new_cmd = *cmd;
	new_cmd.data0 = rte_be_to_cpu_32(new_cmd.data0);
	new_cmd.data1 = rte_be_to_cpu_32(new_cmd.data1);
	switch (new_cmd.action_type) {
	case MLX5_MODIFICATION_TYPE_SET:
	case MLX5_MODIFICATION_TYPE_ADD:
		if (last_type == MLX5_MODIFICATION_TYPE_SET ||
		    last_type == MLX5_MODIFICATION_TYPE_ADD)
			should_insert = new_cmd.field == last_cmd.field;
		else if (last_type == MLX5_MODIFICATION_TYPE_COPY ||
			 last_type == MLX5_MODIFICATION_TYPE_ADD_FIELD)
			should_insert = new_cmd.field == last_cmd.dst_field;
		else if (last_type == MLX5_MODIFICATION_TYPE_NOP)
			should_insert = false;
		else
			MLX5_ASSERT(false); /* Other types are not supported. */
		break;
	case MLX5_MODIFICATION_TYPE_COPY:
	case MLX5_MODIFICATION_TYPE_ADD_FIELD:
		if (last_type == MLX5_MODIFICATION_TYPE_SET ||
		    last_type == MLX5_MODIFICATION_TYPE_ADD)
			should_insert = (new_cmd.field == last_cmd.field ||
					 new_cmd.dst_field == last_cmd.field);
		else if (last_type == MLX5_MODIFICATION_TYPE_COPY ||
			 last_type == MLX5_MODIFICATION_TYPE_ADD_FIELD)
			should_insert = (new_cmd.field == last_cmd.dst_field ||
					 new_cmd.dst_field == last_cmd.dst_field);
		else if (last_type == MLX5_MODIFICATION_TYPE_NOP)
			should_insert = false;
		else
			MLX5_ASSERT(false); /* Other types are not supported. */
		break;
	default:
		/* Other action types should be rejected on AT validation. */
		MLX5_ASSERT(false);
		break;
	}
	return should_insert;
}

static __rte_always_inline int
flow_hw_mhdr_cmd_nop_append(struct mlx5_hw_modify_header_action *mhdr)
{
	struct mlx5_modification_cmd *nop;
	uint32_t num = mhdr->mhdr_cmds_num;

	if (num + 1 >= MLX5_MHDR_MAX_CMD)
		return -ENOMEM;
	nop = mhdr->mhdr_cmds + num;
	nop->data0 = 0;
	nop->action_type = MLX5_MODIFICATION_TYPE_NOP;
	nop->data0 = rte_cpu_to_be_32(nop->data0);
	nop->data1 = 0;
	mhdr->mhdr_cmds_num = num + 1;
	return 0;
}

static __rte_always_inline int
flow_hw_mhdr_cmd_append(struct mlx5_hw_modify_header_action *mhdr,
			struct mlx5_modification_cmd *cmd)
{
	uint32_t num = mhdr->mhdr_cmds_num;

	if (num + 1 >= MLX5_MHDR_MAX_CMD)
		return -ENOMEM;
	mhdr->mhdr_cmds[num] = *cmd;
	mhdr->mhdr_cmds_num = num + 1;
	return 0;
}

static __rte_always_inline int
flow_hw_converted_mhdr_cmds_append(struct mlx5_hw_modify_header_action *mhdr,
				   struct mlx5_flow_dv_modify_hdr_resource *resource)
{
	uint32_t idx;
	int ret;

	for (idx = 0; idx < resource->actions_num; ++idx) {
		struct mlx5_modification_cmd *src = &resource->actions[idx];

		if (flow_hw_should_insert_nop(mhdr, src)) {
			ret = flow_hw_mhdr_cmd_nop_append(mhdr);
			if (ret)
				return ret;
		}
		ret = flow_hw_mhdr_cmd_append(mhdr, src);
		if (ret)
			return ret;
	}
	return 0;
}

static __rte_always_inline void
flow_hw_modify_field_init(struct mlx5_hw_modify_header_action *mhdr,
			  struct rte_flow_actions_template *at)
{
	memset(mhdr, 0, sizeof(*mhdr));
	/* Modify header action without any commands is shared by default. */
	mhdr->shared = true;
	mhdr->pos = at->mhdr_off;
}

static __rte_always_inline int
flow_hw_modify_field_compile(struct rte_eth_dev *dev,
			     const struct rte_flow_attr *attr,
			     const struct rte_flow_action *action, /* Current action from AT. */
			     const struct rte_flow_action *action_mask, /* Current mask from AT. */
			     struct mlx5_hw_actions *acts,
			     struct mlx5_hw_modify_header_action *mhdr,
			     uint16_t src_pos,
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_modify_field *conf = action->conf;
	union {
		struct mlx5_flow_dv_modify_hdr_resource resource;
		uint8_t data[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
			     sizeof(struct mlx5_modification_cmd) * MLX5_MHDR_MAX_CMD];
	} dummy;
	struct mlx5_flow_dv_modify_hdr_resource *resource;
	struct rte_flow_item item = {
		.spec = NULL,
		.mask = NULL
	};
	struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
						{0, 0, MLX5_MODI_OUT_NONE} };
	struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
						{0, 0, MLX5_MODI_OUT_NONE} };
	uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = { 0 };
	uint32_t type, value = 0;
	uint16_t cmds_start, cmds_end;
	bool shared;
	int ret;

	/*
	 * Modify header action is shared if previous modify_field actions
	 * are shared and currently compiled action is shared.
	 */
	shared = flow_hw_action_modify_field_is_shared(action, action_mask);
	mhdr->shared &= shared;
	if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
	    conf->src.field == RTE_FLOW_FIELD_VALUE) {
		type = conf->operation == RTE_FLOW_MODIFY_SET ? MLX5_MODIFICATION_TYPE_SET :
								MLX5_MODIFICATION_TYPE_ADD;
		/* For SET/ADD fill the destination field (field) first. */
		mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
						  conf->width, dev,
						  attr, error);
		item.spec = conf->src.field == RTE_FLOW_FIELD_POINTER ?
				(void *)(uintptr_t)conf->src.pvalue :
				(void *)(uintptr_t)&conf->src.value;
		if (conf->dst.field == RTE_FLOW_FIELD_META ||
		    conf->dst.field == RTE_FLOW_FIELD_TAG ||
		    conf->dst.field == RTE_FLOW_FIELD_METER_COLOR ||
		    conf->dst.field == (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
			uint8_t tag_index = flow_tag_index_get(&conf->dst);

			value = *(const unaligned_uint32_t *)item.spec;
			if (conf->dst.field == RTE_FLOW_FIELD_TAG &&
			    tag_index == RTE_PMD_MLX5_LINEAR_HASH_TAG_INDEX)
				value = rte_cpu_to_be_32(value << 16);
			else
				value = rte_cpu_to_be_32(value);
			item.spec = &value;
		} else if (conf->dst.field == RTE_FLOW_FIELD_GTP_PSC_QFI ||
			   conf->dst.field == RTE_FLOW_FIELD_GENEVE_OPT_TYPE) {
			/*
			 * Both QFI and Geneve option type are passed as an uint8_t integer,
			 * but it is accessed through a 2nd least significant byte of a 32-bit
			 * field in modify header command.
			 */
			value = *(const uint8_t *)item.spec;
			value = rte_cpu_to_be_32(value << 8);
			item.spec = &value;
		}
	} else {
		type = conf->operation == RTE_FLOW_MODIFY_SET ?
		       MLX5_MODIFICATION_TYPE_COPY : MLX5_MODIFICATION_TYPE_ADD_FIELD;
		/* For COPY fill the destination field (dcopy) without mask. */
		mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
						  conf->width, dev,
						  attr, error);
		/* Then construct the source field (field) with mask. */
		mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
						  conf->width, dev,
						  attr, error);
	}
	item.mask = &mask;
	memset(&dummy, 0, sizeof(dummy));
	resource = &dummy.resource;
	ret = flow_dv_convert_modify_action(&item, field, dcopy, resource, type, error);
	if (ret)
		return ret;
	MLX5_ASSERT(resource->actions_num > 0);
	/*
	 * If previous modify field action collide with this one, then insert NOP command.
	 * This NOP command will not be a part of action's command range used to update commands
	 * on rule creation.
	 */
	if (flow_hw_should_insert_nop(mhdr, &resource->actions[0])) {
		ret = flow_hw_mhdr_cmd_nop_append(mhdr);
		if (ret)
			return rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL, "too many modify field operations specified");
	}
	cmds_start = mhdr->mhdr_cmds_num;
	ret = flow_hw_converted_mhdr_cmds_append(mhdr, resource);
	if (ret)
		return rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "too many modify field operations specified");

	cmds_end = mhdr->mhdr_cmds_num;
	if (shared)
		return 0;
	ret = __flow_hw_act_data_hdr_modify_append(priv, acts, RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
						   src_pos, mhdr->pos,
						   cmds_start, cmds_end, shared,
						   field, dcopy, mask);
	if (ret)
		return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "not enough memory to store modify field metadata");
	return 0;
}

static uint32_t
flow_hw_count_nop_modify_field(struct mlx5_hw_modify_header_action *mhdr)
{
	uint32_t i;
	uint32_t nops = 0;

	for (i = 0; i < mhdr->mhdr_cmds_num; ++i) {
		struct mlx5_modification_cmd cmd = mhdr->mhdr_cmds[i];

		cmd.data0 = rte_be_to_cpu_32(cmd.data0);
		if (cmd.action_type == MLX5_MODIFICATION_TYPE_NOP)
			++nops;
	}
	return nops;
}

static int
flow_hw_validate_compiled_modify_field(struct rte_eth_dev *dev,
				       const struct mlx5_flow_template_table_cfg *cfg,
				       struct mlx5_hw_modify_header_action *mhdr,
				       struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hca_attr *hca_attr = &priv->sh->cdev->config.hca_attr;

	/*
	 * Header modify pattern length limitation is only valid for HWS groups, i.e. groups > 0.
	 * In group 0, MODIFY_FIELD actions are handled with header modify actions
	 * managed by rdma-core.
	 */
	if (cfg->attr.flow_attr.group != 0 &&
	    mhdr->mhdr_cmds_num > hca_attr->max_header_modify_pattern_length) {
		uint32_t nops = flow_hw_count_nop_modify_field(mhdr);

		DRV_LOG(ERR, "Too many modify header commands generated from "
			     "MODIFY_FIELD actions. "
			     "Generated HW commands = %u (amount of NOP commands = %u). "
			     "Maximum supported = %u.",
			     mhdr->mhdr_cmds_num, nops,
			     hca_attr->max_header_modify_pattern_length);
		return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "Number of MODIFY_FIELD actions exceeds maximum "
					  "supported limit of actions");
	}
	return 0;
}

static int
flow_hw_represented_port_compile(struct rte_eth_dev *dev,
				 const struct rte_flow_attr *attr,
				 const struct rte_flow_action *action,
				 const struct rte_flow_action *action_mask,
				 struct mlx5_hw_actions *acts,
				 uint16_t action_src, uint16_t action_dst,
				 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_ethdev *v = action->conf;
	const struct rte_flow_action_ethdev *m = action_mask->conf;
	int ret;

	if (!attr->group)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
					  "represented_port action cannot"
					  " be used on group 0");
	if (!attr->transfer)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
					  NULL,
					  "represented_port action requires"
					  " transfer attribute");
	if (attr->ingress || attr->egress)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
					  "represented_port action cannot"
					  " be used with direction attributes");
	if (!priv->master)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "represented_port action must"
					  " be used on proxy port");
	if (m && !!m->port_id) {
		struct mlx5_priv *port_priv;

		if (!v)
			return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
						  action, "port index was not provided");
		port_priv = mlx5_port_to_eswitch_info(v->port_id, false);
		if (port_priv == NULL)
			return rte_flow_error_set
					(error, EINVAL,
					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					 "port does not exist or unable to"
					 " obtain E-Switch info for port");
		MLX5_ASSERT(priv->hw_vport != NULL);
		if (priv->hw_vport[v->port_id]) {
			acts->rule_acts[action_dst].action =
					priv->hw_vport[v->port_id];
		} else {
			return rte_flow_error_set
					(error, EINVAL,
					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					 "cannot use represented_port action"
					 " with this port");
		}
	} else {
		ret = __flow_hw_act_data_general_append
				(priv, acts, action->type,
				 action_src, action_dst);
		if (ret)
			return rte_flow_error_set
					(error, ENOMEM,
					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					 "not enough memory to store"
					 " vport action");
	}
	return 0;
}

static __rte_always_inline int
flow_hw_meter_compile(struct rte_eth_dev *dev,
		      const struct mlx5_flow_template_table_cfg *cfg,
		      uint16_t aso_mtr_pos,
		      uint16_t jump_pos,
		      const struct rte_flow_action *action,
		      struct mlx5_hw_actions *acts,
		      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr *aso_mtr;
	const struct rte_flow_action_meter *meter = action->conf;
	uint32_t group = cfg->attr.flow_attr.group;

	aso_mtr = mlx5_aso_meter_by_idx(priv, meter->mtr_id);
	acts->rule_acts[aso_mtr_pos].action = priv->mtr_bulk.action;
	acts->rule_acts[aso_mtr_pos].aso_meter.offset = aso_mtr->offset;
	acts->jump = flow_hw_jump_action_register
		(dev, cfg, aso_mtr->fm.group, error);
	if (!acts->jump)
		return -ENOMEM;
	acts->rule_acts[jump_pos].action = (!!group) ?
				    acts->jump->hws_action :
				    acts->jump->root_action;
	if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr))
		return -ENOMEM;
	return 0;
}

static __rte_always_inline int
flow_hw_cnt_compile(struct rte_eth_dev *dev, uint32_t  start_pos,
		      struct mlx5_hw_actions *acts)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t pos = start_pos;
	cnt_id_t cnt_id;
	int ret;

	ret = mlx5_hws_cnt_shared_get(priv->hws_cpool, &cnt_id, 0);
	if (ret != 0)
		return ret;
	ret = mlx5_hws_cnt_pool_get_action_offset
				(priv->hws_cpool,
				 cnt_id,
				 &acts->rule_acts[pos].action,
				 &acts->rule_acts[pos].counter.offset);
	if (ret != 0)
		return ret;
	acts->cnt_id = cnt_id;
	return 0;
}

static __rte_always_inline bool
is_of_vlan_pcp_present(const struct rte_flow_action *actions)
{
	/*
	 * Order of RTE VLAN push actions is
	 * OF_PUSH_VLAN / OF_SET_VLAN_VID [ / OF_SET_VLAN_PCP ]
	 */
	return actions[MLX5_HW_VLAN_PUSH_PCP_IDX].type ==
		RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP;
}

static __rte_always_inline bool
is_template_masked_push_vlan(const struct rte_flow_action_of_push_vlan *mask)
{
	/*
	 * In masked push VLAN template all RTE push actions are masked.
	 */
	return mask && mask->ethertype != 0;
}

static rte_be32_t vlan_hdr_to_be32(const struct rte_flow_action *actions)
{
/*
 * OpenFlow Switch Specification defines 801.1q VID as 12+1 bits.
 */
	rte_be32_t type, vid, pcp;
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
	rte_be32_t vid_lo, vid_hi;
#endif

	type = ((const struct rte_flow_action_of_push_vlan *)
		actions[MLX5_HW_VLAN_PUSH_TYPE_IDX].conf)->ethertype;
	vid = ((const struct rte_flow_action_of_set_vlan_vid *)
		actions[MLX5_HW_VLAN_PUSH_VID_IDX].conf)->vlan_vid;
	pcp = is_of_vlan_pcp_present(actions) ?
	      ((const struct rte_flow_action_of_set_vlan_pcp *)
		      actions[MLX5_HW_VLAN_PUSH_PCP_IDX].conf)->vlan_pcp : 0;
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
	vid_hi = vid & 0xff;
	vid_lo = vid >> 8;
	return (((vid_lo << 8) | (pcp << 5) | vid_hi) << 16) | type;
#else
	return (type << 16) | (pcp << 13) | vid;
#endif
}

static __rte_always_inline struct mlx5_aso_mtr *
flow_hw_meter_mark_alloc(struct rte_eth_dev *dev, uint32_t queue,
			 const struct rte_flow_action *action,
			 void *user_data, bool push)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	const struct rte_flow_action_meter_mark *meter_mark = action->conf;
	struct mlx5_aso_mtr *aso_mtr;
	struct mlx5_flow_meter_info *fm;
	uint32_t mtr_id;

	if (meter_mark->profile == NULL)
		return NULL;
	aso_mtr = mlx5_ipool_malloc(priv->hws_mpool->idx_pool, &mtr_id);
	if (!aso_mtr)
		return NULL;
	/* Fill the flow meter parameters. */
	aso_mtr->type = ASO_METER_INDIRECT;
	fm = &aso_mtr->fm;
	fm->meter_id = mtr_id;
	fm->profile = (struct mlx5_flow_meter_profile *)(meter_mark->profile);
	fm->is_enable = meter_mark->state;
	fm->color_aware = meter_mark->color_mode;
	aso_mtr->pool = pool;
	aso_mtr->state = (queue == MLX5_HW_INV_QUEUE) ?
			  ASO_METER_WAIT : ASO_METER_WAIT_ASYNC;
	aso_mtr->offset = mtr_id - 1;
	aso_mtr->init_color = fm->color_aware ? RTE_COLORS : RTE_COLOR_GREEN;
	/* Update ASO flow meter by wqe. */
	if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, aso_mtr,
					 &priv->mtr_bulk, user_data, push)) {
		mlx5_ipool_free(pool->idx_pool, mtr_id);
		return NULL;
	}
	/* Wait for ASO object completion. */
	if (queue == MLX5_HW_INV_QUEUE &&
	    mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) {
		mlx5_ipool_free(pool->idx_pool, mtr_id);
		return NULL;
	}
	return aso_mtr;
}

static __rte_always_inline int
flow_hw_meter_mark_compile(struct rte_eth_dev *dev,
			   uint16_t aso_mtr_pos,
			   const struct rte_flow_action *action,
			   struct mlx5dr_rule_action *acts,
			   uint32_t *index,
			   uint32_t queue)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct mlx5_aso_mtr *aso_mtr;

	aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, NULL, true);
	if (!aso_mtr)
		return -1;

	/* Compile METER_MARK action */
	acts[aso_mtr_pos].action = pool->action;
	acts[aso_mtr_pos].aso_meter.offset = aso_mtr->offset;
	*index = aso_mtr->fm.meter_id;
	return 0;
}

static int
flow_hw_translate_indirect_mirror(__rte_unused struct rte_eth_dev *dev,
				  __rte_unused const struct mlx5_action_construct_data *act_data,
				  const struct rte_flow_action *action,
				  struct mlx5dr_rule_action *dr_rule)
{
	const struct rte_flow_action_indirect_list *list_conf = action->conf;
	const struct mlx5_mirror *mirror = (typeof(mirror))list_conf->handle;

	dr_rule->action = mirror->mirror_action;
	return 0;
}

/**
 * HWS mirror implemented as FW island.
 * The action does not support indirect list flow configuration.
 * If template handle was masked, use handle mirror action in flow rules.
 * Otherwise let flow rule specify mirror handle.
 */
static int
hws_table_tmpl_translate_indirect_mirror(struct rte_eth_dev *dev,
					 const struct rte_flow_action *action,
					 const struct rte_flow_action *mask,
					 struct mlx5_hw_actions *acts,
					 uint16_t action_src, uint16_t action_dst)
{
	int ret = 0;
	const struct rte_flow_action_indirect_list *mask_conf = mask->conf;

	if (mask_conf && mask_conf->handle) {
		/**
		 * If mirror handle was masked, assign fixed DR5 mirror action.
		 */
		flow_hw_translate_indirect_mirror(dev, NULL, action,
						  &acts->rule_acts[action_dst]);
	} else {
		struct mlx5_priv *priv = dev->data->dev_private;
		ret = flow_hw_act_data_indirect_list_append
			(priv, acts, RTE_FLOW_ACTION_TYPE_INDIRECT_LIST,
			 action_src, action_dst,
			 flow_hw_translate_indirect_mirror);
	}
	return ret;
}

static int
flow_hw_reformat_action(__rte_unused struct rte_eth_dev *dev,
			__rte_unused const struct mlx5_action_construct_data *data,
			const struct rte_flow_action *action,
			struct mlx5dr_rule_action *dr_rule)
{
	const struct rte_flow_action_indirect_list *indlst_conf = action->conf;

	dr_rule->action = ((struct mlx5_hw_encap_decap_action *)
			   (indlst_conf->handle))->action;
	if (!dr_rule->action)
		return -EINVAL;
	return 0;
}

/**
 * Template conf must not be masked. If handle is masked, use the one in template,
 * otherwise update per flow rule.
 */
static int
hws_table_tmpl_translate_indirect_reformat(struct rte_eth_dev *dev,
					   const struct rte_flow_action *action,
					   const struct rte_flow_action *mask,
					   struct mlx5_hw_actions *acts,
					   uint16_t action_src, uint16_t action_dst)
{
	int ret = -1;
	const struct rte_flow_action_indirect_list *mask_conf = mask->conf;
	struct mlx5_priv *priv = dev->data->dev_private;

	if (mask_conf && mask_conf->handle && !mask_conf->conf)
		/**
		 * If handle was masked, assign fixed DR action.
		 */
		ret = flow_hw_reformat_action(dev, NULL, action,
					      &acts->rule_acts[action_dst]);
	else if (mask_conf && !mask_conf->handle && !mask_conf->conf)
		ret = flow_hw_act_data_indirect_list_append
			(priv, acts, RTE_FLOW_ACTION_TYPE_INDIRECT_LIST,
			 action_src, action_dst, flow_hw_reformat_action);
	return ret;
}

static int
flow_dr_set_meter(struct mlx5_priv *priv,
		  struct mlx5dr_rule_action *dr_rule,
		  const struct rte_flow_action_indirect_list *action_conf)
{
	const struct mlx5_indlst_legacy *legacy_obj =
		(typeof(legacy_obj))action_conf->handle;
	struct mlx5_aso_mtr_pool *mtr_pool = priv->hws_mpool;
	uint32_t act_idx = (uint32_t)(uintptr_t)legacy_obj->handle;
	uint32_t mtr_id = act_idx & (RTE_BIT32(MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
	struct mlx5_aso_mtr *aso_mtr = mlx5_ipool_get(mtr_pool->idx_pool, mtr_id);

	if (!aso_mtr)
		return -EINVAL;
	dr_rule->action = mtr_pool->action;
	dr_rule->aso_meter.offset = aso_mtr->offset;
	return 0;
}

__rte_always_inline static void
flow_dr_mtr_flow_color(struct mlx5dr_rule_action *dr_rule, enum rte_color init_color)
{
	dr_rule->aso_meter.init_color =
		(enum mlx5dr_action_aso_meter_color)rte_col_2_mlx5_col(init_color);
}

static int
flow_hw_translate_indirect_meter(struct rte_eth_dev *dev,
				 const struct mlx5_action_construct_data *act_data,
				 const struct rte_flow_action *action,
				 struct mlx5dr_rule_action *dr_rule)
{
	int ret;
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_indirect_list *action_conf = action->conf;
	const struct rte_flow_indirect_update_flow_meter_mark **flow_conf =
		(typeof(flow_conf))action_conf->conf;

	/*
	 * Masked indirect handle set dr5 action during template table
	 * translation.
	 */
	if (!dr_rule->action) {
		ret = flow_dr_set_meter(priv, dr_rule, action_conf);
		if (ret)
			return ret;
	}
	if (!act_data->shared_meter.conf_masked) {
		if (flow_conf && flow_conf[0] && flow_conf[0]->init_color < RTE_COLORS)
			flow_dr_mtr_flow_color(dr_rule, flow_conf[0]->init_color);
	}
	return 0;
}

static int
hws_table_tmpl_translate_indirect_meter(struct rte_eth_dev *dev,
					const struct rte_flow_action *action,
					const struct rte_flow_action *mask,
					struct mlx5_hw_actions *acts,
					uint16_t action_src, uint16_t action_dst)
{
	int ret;
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_indirect_list *action_conf = action->conf;
	const struct rte_flow_action_indirect_list *mask_conf = mask->conf;
	bool is_handle_masked = mask_conf && mask_conf->handle;
	bool is_conf_masked = mask_conf && mask_conf->conf && mask_conf->conf[0];
	struct mlx5dr_rule_action *dr_rule = &acts->rule_acts[action_dst];

	if (is_handle_masked) {
		ret = flow_dr_set_meter(priv, dr_rule, action->conf);
		if (ret)
			return ret;
	}
	if (is_conf_masked) {
		const struct
			rte_flow_indirect_update_flow_meter_mark **flow_conf =
			(typeof(flow_conf))action_conf->conf;
		flow_dr_mtr_flow_color(dr_rule,
				       flow_conf[0]->init_color);
	}
	if (!is_handle_masked || !is_conf_masked) {
		struct mlx5_action_construct_data *act_data;

		ret = flow_hw_act_data_indirect_list_append
			(priv, acts, RTE_FLOW_ACTION_TYPE_INDIRECT_LIST,
			 action_src, action_dst, flow_hw_translate_indirect_meter);
		if (ret)
			return ret;
		act_data = LIST_FIRST(&acts->act_list);
		act_data->shared_meter.conf_masked = is_conf_masked;
	}
	return 0;
}

static int
hws_table_tmpl_translate_indirect_legacy(struct rte_eth_dev *dev,
					 const struct rte_flow_action *action,
					 const struct rte_flow_action *mask,
					 struct mlx5_hw_actions *acts,
					 uint16_t action_src, uint16_t action_dst)
{
	int ret;
	const struct rte_flow_action_indirect_list *indlst_conf = action->conf;
	struct mlx5_indlst_legacy *indlst_obj = (typeof(indlst_obj))indlst_conf->handle;
	uint32_t act_idx = (uint32_t)(uintptr_t)indlst_obj->handle;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;

	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		ret = hws_table_tmpl_translate_indirect_meter(dev, action, mask,
							      acts, action_src,
							      action_dst);
		break;
	default:
		ret = -EINVAL;
		break;
	}
	return ret;
}

/*
 * template .. indirect_list handle Ht conf Ct ..
 * mask     .. indirect_list handle Hm conf Cm ..
 *
 * PMD requires Ht != 0 to resolve handle type.
 * If Ht was masked (Hm != 0) DR5 action will be set according to Ht and will
 * not change. Otherwise, DR5 action will be resolved during flow rule build.
 * If Ct was masked (Cm != 0), table template processing updates base
 * indirect action configuration with Ct parameters.
 */
static int
table_template_translate_indirect_list(struct rte_eth_dev *dev,
				       const struct rte_flow_action *action,
				       const struct rte_flow_action *mask,
				       struct mlx5_hw_actions *acts,
				       uint16_t action_src, uint16_t action_dst)
{
	int ret = 0;
	enum mlx5_indirect_list_type type;
	const struct rte_flow_action_indirect_list *list_conf = action->conf;

	if (!list_conf || !list_conf->handle)
		return -EINVAL;
	type = mlx5_get_indirect_list_type(list_conf->handle);
	switch (type) {
	case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY:
		ret = hws_table_tmpl_translate_indirect_legacy(dev, action, mask,
							       acts, action_src,
							       action_dst);
		break;
	case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR:
		ret = hws_table_tmpl_translate_indirect_mirror(dev, action, mask,
							       acts, action_src,
							       action_dst);
		break;
	case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT:
		if (list_conf->conf)
			return -EINVAL;
		ret = hws_table_tmpl_translate_indirect_reformat(dev, action, mask,
								 acts, action_src,
								 action_dst);
		break;
	default:
		return -EINVAL;
	}
	return ret;
}

static int
mlx5_tbl_translate_reformat(struct mlx5_priv *priv,
			    const struct rte_flow_template_table_attr *table_attr,
			    struct mlx5_hw_actions *acts,
			    struct rte_flow_actions_template *at,
			    const struct rte_flow_item *enc_item,
			    const struct rte_flow_item *enc_item_m,
			    uint8_t *encap_data, uint8_t *encap_data_m,
			    struct mlx5_tbl_multi_pattern_ctx *mp_ctx,
			    size_t data_size, uint16_t reformat_src,
			    enum mlx5dr_action_type refmt_type,
			    struct rte_flow_error *error)
{
	int mp_reformat_ix = mlx5_multi_pattern_reformat_to_index(refmt_type);
	const struct rte_flow_attr *attr = &table_attr->flow_attr;
	enum mlx5dr_table_type tbl_type = get_mlx5dr_table_type(attr);
	struct mlx5dr_action_reformat_header hdr;
	uint8_t buf[MLX5_ENCAP_MAX_LEN];
	bool shared_rfmt = false;
	int ret;

	MLX5_ASSERT(at->reformat_off != UINT16_MAX);
	if (enc_item) {
		MLX5_ASSERT(!encap_data);
		ret = flow_dv_convert_encap_data(enc_item, buf, &data_size, error);
		if (ret)
			return ret;
		encap_data = buf;
		if (enc_item_m)
			shared_rfmt = true;
	} else if (encap_data && encap_data_m) {
		shared_rfmt = true;
	}
	acts->encap_decap = mlx5_malloc(MLX5_MEM_ZERO,
					sizeof(*acts->encap_decap) + data_size,
					0, SOCKET_ID_ANY);
	if (!acts->encap_decap)
		return rte_flow_error_set(error, ENOMEM,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "no memory for reformat context");
	hdr.sz = data_size;
	hdr.data = encap_data;
	if (shared_rfmt || mp_reformat_ix < 0) {
		uint16_t reformat_ix = at->reformat_off;
		uint32_t flags = mlx5_hw_act_flag[!!attr->group][tbl_type] |
				 MLX5DR_ACTION_FLAG_SHARED;

		acts->encap_decap->action =
			mlx5dr_action_create_reformat(priv->dr_ctx, refmt_type,
						      1, &hdr, 0, flags);
		if (!acts->encap_decap->action)
			return -rte_errno;
		acts->rule_acts[reformat_ix].action = acts->encap_decap->action;
		acts->rule_acts[reformat_ix].reformat.data = acts->encap_decap->data;
		acts->rule_acts[reformat_ix].reformat.offset = 0;
		acts->encap_decap->shared = true;
	} else {
		uint32_t ix;
		typeof(mp_ctx->reformat[0]) *reformat_ctx = mp_ctx->reformat +
							    mp_reformat_ix;

		ix = reformat_ctx->elements_num++;
		reformat_ctx->ctx[ix].reformat_hdr = hdr;
		reformat_ctx->ctx[ix].rule_action = &acts->rule_acts[at->reformat_off];
		reformat_ctx->ctx[ix].encap = acts->encap_decap;
		acts->rule_acts[at->reformat_off].reformat.hdr_idx = ix;
		acts->encap_decap_pos = at->reformat_off;
		acts->encap_decap->data_size = data_size;
		ret = __flow_hw_act_data_encap_append
			(priv, acts, (at->actions + reformat_src)->type,
			 reformat_src, at->reformat_off, data_size);
		if (ret)
			return -rte_errno;
	}
	return 0;
}

static int
mlx5_tbl_translate_modify_header(struct rte_eth_dev *dev,
				 const struct mlx5_flow_template_table_cfg *cfg,
				 struct mlx5_hw_actions *acts,
				 struct mlx5_tbl_multi_pattern_ctx *mp_ctx,
				 struct mlx5_hw_modify_header_action *mhdr,
				 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_template_table_attr *table_attr = &cfg->attr;
	const struct rte_flow_attr *attr = &table_attr->flow_attr;
	enum mlx5dr_table_type tbl_type = get_mlx5dr_table_type(attr);
	uint16_t mhdr_ix = mhdr->pos;
	struct mlx5dr_action_mh_pattern pattern = {
		.sz = sizeof(struct mlx5_modification_cmd) * mhdr->mhdr_cmds_num
	};

	if (flow_hw_validate_compiled_modify_field(dev, cfg, mhdr, error)) {
		__flow_hw_action_template_destroy(dev, acts);
		return -rte_errno;
	}
	acts->mhdr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*acts->mhdr),
				 0, SOCKET_ID_ANY);
	if (!acts->mhdr)
		return rte_flow_error_set(error, ENOMEM,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "translate modify_header: no memory for modify header context");
	rte_memcpy(acts->mhdr, mhdr, sizeof(*mhdr));
	pattern.data = (__be64 *)acts->mhdr->mhdr_cmds;
	if (mhdr->shared) {
		uint32_t flags = mlx5_hw_act_flag[!!attr->group][tbl_type] |
				 MLX5DR_ACTION_FLAG_SHARED;

		acts->mhdr->action = mlx5dr_action_create_modify_header
						(priv->dr_ctx, 1, &pattern, 0,
						 flags);
		if (!acts->mhdr->action)
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL, "translate modify_header: failed to create DR action");
		acts->rule_acts[mhdr_ix].action = acts->mhdr->action;
	} else {
		typeof(mp_ctx->mh) *mh = &mp_ctx->mh;
		uint32_t idx = mh->elements_num;
		struct mlx5_multi_pattern_ctx *mh_ctx = mh->ctx + mh->elements_num++;

		mh_ctx->mh_pattern = pattern;
		mh_ctx->mhdr = acts->mhdr;
		mh_ctx->rule_action = &acts->rule_acts[mhdr_ix];
		acts->rule_acts[mhdr_ix].modify_header.pattern_idx = idx;
	}
	return 0;
}


static int
mlx5_create_ipv6_ext_reformat(struct rte_eth_dev *dev,
			      const struct mlx5_flow_template_table_cfg *cfg,
			      struct mlx5_hw_actions *acts,
			      struct rte_flow_actions_template *at,
			      uint8_t *push_data, uint8_t *push_data_m,
			      size_t push_size, uint16_t recom_src,
			      enum mlx5dr_action_type recom_type)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_template_table_attr *table_attr = &cfg->attr;
	const struct rte_flow_attr *attr = &table_attr->flow_attr;
	enum mlx5dr_table_type type = get_mlx5dr_table_type(attr);
	struct mlx5_action_construct_data *act_data;
	struct mlx5dr_action_reformat_header hdr = {0};
	uint32_t flag, bulk = 0;

	flag = mlx5_hw_act_flag[!!attr->group][type];
	acts->push_remove = mlx5_malloc(MLX5_MEM_ZERO,
					sizeof(*acts->push_remove) + push_size,
					0, SOCKET_ID_ANY);
	if (!acts->push_remove)
		return -ENOMEM;

	switch (recom_type) {
	case MLX5DR_ACTION_TYP_PUSH_IPV6_ROUTE_EXT:
		if (!push_data || !push_size)
			goto err1;
		if (!push_data_m) {
			bulk = rte_log2_u32(table_attr->nb_flows);
		} else {
			flag |= MLX5DR_ACTION_FLAG_SHARED;
			acts->push_remove->shared = 1;
		}
		acts->push_remove->data_size = push_size;
		memcpy(acts->push_remove->data, push_data, push_size);
		hdr.data = push_data;
		hdr.sz = push_size;
		break;
	case MLX5DR_ACTION_TYP_POP_IPV6_ROUTE_EXT:
		flag |= MLX5DR_ACTION_FLAG_SHARED;
		acts->push_remove->shared = 1;
		break;
	default:
		break;
	}

	acts->push_remove->action =
		mlx5dr_action_create_reformat_ipv6_ext(priv->dr_ctx,
				recom_type, &hdr, bulk, flag);
	if (!acts->push_remove->action)
		goto err1;
	acts->rule_acts[at->recom_off].action = acts->push_remove->action;
	acts->rule_acts[at->recom_off].ipv6_ext.header = acts->push_remove->data;
	acts->rule_acts[at->recom_off].ipv6_ext.offset = 0;
	acts->push_remove_pos = at->recom_off;
	if (!acts->push_remove->shared) {
		act_data = __flow_hw_act_data_push_append(dev, acts,
				RTE_FLOW_ACTION_TYPE_IPV6_EXT_PUSH,
				recom_src, at->recom_off, push_size);
		if (!act_data)
			goto err;
	}
	return 0;
err:
	if (acts->push_remove->action)
		mlx5dr_action_destroy(acts->push_remove->action);
err1:
	if (acts->push_remove) {
		mlx5_free(acts->push_remove);
		acts->push_remove = NULL;
	}
	return -EINVAL;
}

/**
 * Translate rte_flow actions to DR action.
 *
 * As the action template has already indicated the actions. Translate
 * the rte_flow actions to DR action if possbile. So in flow create
 * stage we will save cycles from handing the actions' organizing.
 * For the actions with limited information, need to add these to a
 * list.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] cfg
 *   Pointer to the table configuration.
 * @param[in/out] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] at
 *   Action template.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno otherwise and rte_errno is set.
 */
static int
__flow_hw_actions_translate(struct rte_eth_dev *dev,
			    const struct mlx5_flow_template_table_cfg *cfg,
			    struct mlx5_hw_actions *acts,
			    struct rte_flow_actions_template *at,
			    struct mlx5_tbl_multi_pattern_ctx *mp_ctx,
			    struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_template_table_attr *table_attr = &cfg->attr;
	struct mlx5_hca_flex_attr *hca_attr = &priv->sh->cdev->config.hca_attr.flex;
	const struct rte_flow_attr *attr = &table_attr->flow_attr;
	struct rte_flow_action *actions = at->actions;
	struct rte_flow_action *masks = at->masks;
	enum mlx5dr_action_type refmt_type = MLX5DR_ACTION_TYP_LAST;
	enum mlx5dr_action_type recom_type = MLX5DR_ACTION_TYP_LAST;
	const struct rte_flow_action_raw_encap *raw_encap_data;
	const struct rte_flow_action_ipv6_ext_push *ipv6_ext_data;
	const struct rte_flow_item *enc_item = NULL, *enc_item_m = NULL;
	uint16_t reformat_src = 0, recom_src = 0;
	uint8_t *encap_data = NULL, *encap_data_m = NULL;
	uint8_t *push_data = NULL, *push_data_m = NULL;
	size_t data_size = 0, push_size = 0;
	struct mlx5_hw_modify_header_action mhdr = { 0 };
	bool actions_end = false;
	uint32_t type;
	bool reformat_used = false;
	bool recom_used = false;
	unsigned int of_vlan_offset;
	uint16_t jump_pos;
	uint32_t ct_idx;
	int ret, err;
	uint32_t target_grp = 0;
	int table_type;

	flow_hw_modify_field_init(&mhdr, at);
	if (attr->transfer)
		type = MLX5DR_TABLE_TYPE_FDB;
	else if (attr->egress)
		type = MLX5DR_TABLE_TYPE_NIC_TX;
	else
		type = MLX5DR_TABLE_TYPE_NIC_RX;
	for (; !actions_end; actions++, masks++) {
		uint64_t pos = actions - at->actions;
		uint16_t src_pos = pos - at->src_off[pos];
		uint16_t dr_pos = at->dr_off[pos];

		switch ((int)actions->type) {
		case RTE_FLOW_ACTION_TYPE_INDIRECT_LIST:
			if (!attr->group) {
				DRV_LOG(ERR, "Indirect action is not supported in root table.");
				goto err;
			}
			ret = table_template_translate_indirect_list
				(dev, actions, masks, acts, src_pos, dr_pos);
			if (ret)
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			if (!attr->group) {
				DRV_LOG(ERR, "Indirect action is not supported in root table.");
				goto err;
			}
			if (actions->conf && masks->conf) {
				if (flow_hw_shared_action_translate
				(dev, actions, acts, src_pos, dr_pos))
					goto err;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, RTE_FLOW_ACTION_TYPE_INDIRECT,
					 src_pos, dr_pos)){
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_DROP:
			acts->rule_acts[dr_pos].action =
				priv->hw_drop[!!attr->group];
			break;
		case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
			if (!attr->group) {
				DRV_LOG(ERR, "Port representor is not supported in root table.");
				goto err;
			}
			acts->rule_acts[dr_pos].action = priv->hw_def_miss;
			break;
		case RTE_FLOW_ACTION_TYPE_MARK:
			acts->mark = true;
			if (masks->conf &&
			    ((const struct rte_flow_action_mark *)
			     masks->conf)->id)
				acts->rule_acts[dr_pos].tag.value =
					mlx5_flow_mark_set
					(((const struct rte_flow_action_mark *)
					(actions->conf))->id);
			else if (__flow_hw_act_data_general_append(priv, acts,
								   actions->type,
								   src_pos, dr_pos))
				goto err;
			acts->rule_acts[dr_pos].action =
				priv->hw_tag[!!attr->group];
			__atomic_fetch_add(&priv->hws_mark_refcnt, 1, __ATOMIC_RELAXED);
			flow_hw_rxq_flag_set(dev, true);
			break;
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			acts->rule_acts[dr_pos].action =
				priv->hw_push_vlan[type];
			if (is_template_masked_push_vlan(masks->conf))
				acts->rule_acts[dr_pos].push_vlan.vlan_hdr =
					vlan_hdr_to_be32(actions);
			else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 src_pos, dr_pos))
				goto err;
			of_vlan_offset = is_of_vlan_pcp_present(actions) ?
					MLX5_HW_VLAN_PUSH_PCP_IDX :
					MLX5_HW_VLAN_PUSH_VID_IDX;
			actions += of_vlan_offset;
			masks += of_vlan_offset;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
			acts->rule_acts[dr_pos].action =
				priv->hw_pop_vlan[type];
			break;
		case RTE_FLOW_ACTION_TYPE_JUMP:
			if (masks->conf &&
			    ((const struct rte_flow_action_jump *)
			     masks->conf)->group) {
				uint32_t jump_group =
					((const struct rte_flow_action_jump *)
					actions->conf)->group;
				acts->jump = flow_hw_jump_action_register
						(dev, cfg, jump_group, error);
				if (!acts->jump)
					goto err;
				acts->rule_acts[dr_pos].action = (!!attr->group) ?
								 acts->jump->hws_action :
								 acts->jump->root_action;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 src_pos, dr_pos)){
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_QUEUE:
			if (masks->conf &&
			    ((const struct rte_flow_action_queue *)
			     masks->conf)->index) {
				acts->tir = flow_hw_tir_action_register
				(dev,
				 mlx5_hw_act_flag[!!attr->group][type],
				 actions);
				if (!acts->tir)
					goto err;
				acts->rule_acts[dr_pos].action =
					acts->tir->action;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 src_pos, dr_pos)) {
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_RSS:
			if (actions->conf && masks->conf) {
				acts->tir = flow_hw_tir_action_register
				(dev,
				 mlx5_hw_act_flag[!!attr->group][type],
				 actions);
				if (!acts->tir)
					goto err;
				acts->rule_acts[dr_pos].action =
					acts->tir->action;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 src_pos, dr_pos)) {
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
			MLX5_ASSERT(!reformat_used);
			enc_item = MLX5_CONST_ENCAP_ITEM(rte_flow_action_vxlan_encap,
							 actions->conf);
			if (masks->conf)
				enc_item_m = MLX5_CONST_ENCAP_ITEM(rte_flow_action_vxlan_encap,
								   masks->conf);
			reformat_used = true;
			reformat_src = src_pos;
			refmt_type = MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
			MLX5_ASSERT(!reformat_used);
			enc_item = MLX5_CONST_ENCAP_ITEM(rte_flow_action_nvgre_encap,
							 actions->conf);
			if (masks->conf)
				enc_item_m = MLX5_CONST_ENCAP_ITEM(rte_flow_action_nvgre_encap,
								   masks->conf);
			reformat_used = true;
			reformat_src = src_pos;
			refmt_type = MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			raw_encap_data =
				(const struct rte_flow_action_raw_encap *)
				 masks->conf;
			if (raw_encap_data)
				encap_data_m = raw_encap_data->data;
			raw_encap_data =
				(const struct rte_flow_action_raw_encap *)
				 actions->conf;
			encap_data = raw_encap_data->data;
			data_size = raw_encap_data->size;
			if (reformat_used) {
				refmt_type = data_size <
				MLX5_ENCAPSULATION_DECISION_SIZE ?
				MLX5DR_ACTION_TYP_REFORMAT_TNL_L3_TO_L2 :
				MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L3;
			} else {
				reformat_used = true;
				refmt_type =
				MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2;
			}
			reformat_src = src_pos;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
			MLX5_ASSERT(!reformat_used);
			reformat_used = true;
			refmt_type = MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
			reformat_used = true;
			refmt_type = MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_IPV6_EXT_PUSH:
			if (!hca_attr->query_match_sample_info || !hca_attr->parse_graph_anchor ||
			    !priv->sh->srh_flex_parser.flex.mapnum) {
				DRV_LOG(ERR, "SRv6 anchor is not supported.");
				goto err;
			}
			MLX5_ASSERT(!recom_used && !recom_type);
			recom_used = true;
			recom_type = MLX5DR_ACTION_TYP_PUSH_IPV6_ROUTE_EXT;
			ipv6_ext_data =
				(const struct rte_flow_action_ipv6_ext_push *)masks->conf;
			if (ipv6_ext_data)
				push_data_m = ipv6_ext_data->data;
			ipv6_ext_data =
				(const struct rte_flow_action_ipv6_ext_push *)actions->conf;
			if (ipv6_ext_data) {
				push_data = ipv6_ext_data->data;
				push_size = ipv6_ext_data->size;
			}
			recom_src = src_pos;
			break;
		case RTE_FLOW_ACTION_TYPE_IPV6_EXT_REMOVE:
			if (!hca_attr->query_match_sample_info || !hca_attr->parse_graph_anchor ||
			    !priv->sh->srh_flex_parser.flex.mapnum) {
				DRV_LOG(ERR, "SRv6 anchor is not supported.");
				goto err;
			}
			recom_used = true;
			recom_type = MLX5DR_ACTION_TYP_POP_IPV6_ROUTE_EXT;
			break;
		case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
			flow_hw_translate_group(dev, cfg, attr->group,
						&target_grp, error);
			if (target_grp == 0) {
				__flow_hw_action_template_destroy(dev, acts);
				return rte_flow_error_set(error, ENOTSUP,
						RTE_FLOW_ERROR_TYPE_ACTION,
						NULL,
						"Send to kernel action on root table is not supported in HW steering mode");
			}
			table_type = attr->ingress ? MLX5DR_TABLE_TYPE_NIC_RX :
				     ((attr->egress) ? MLX5DR_TABLE_TYPE_NIC_TX :
				      MLX5DR_TABLE_TYPE_FDB);
			acts->rule_acts[dr_pos].action = priv->hw_send_to_kernel[table_type];
			break;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			err = flow_hw_modify_field_compile(dev, attr, actions,
							   masks, acts, &mhdr,
							   src_pos, error);
			if (err)
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
			if (flow_hw_represented_port_compile
					(dev, attr, actions,
					 masks, acts, src_pos, dr_pos, error))
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_METER:
			/*
			 * METER action is compiled to 2 DR actions - ASO_METER and FT.
			 * Calculated DR offset is stored only for ASO_METER and FT
			 * is assumed to be the next action.
			 */
			jump_pos = dr_pos + 1;
			if (actions->conf && masks->conf &&
			    ((const struct rte_flow_action_meter *)
			     masks->conf)->mtr_id) {
				err = flow_hw_meter_compile(dev, cfg,
							    dr_pos, jump_pos, actions, acts, error);
				if (err)
					goto err;
			} else if (__flow_hw_act_data_general_append(priv, acts,
								     actions->type,
								     src_pos,
								     dr_pos))
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
			flow_hw_translate_group(dev, cfg, attr->group,
						&target_grp, error);
			if (target_grp == 0) {
				__flow_hw_action_template_destroy(dev, acts);
				return rte_flow_error_set(error, ENOTSUP,
						RTE_FLOW_ERROR_TYPE_ACTION,
						NULL,
						"Age action on root table is not supported in HW steering mode");
			}
			if (__flow_hw_act_data_general_append(priv, acts,
							      actions->type,
							      src_pos,
							      dr_pos))
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_COUNT:
			flow_hw_translate_group(dev, cfg, attr->group,
						&target_grp, error);
			if (target_grp == 0) {
				__flow_hw_action_template_destroy(dev, acts);
				return rte_flow_error_set(error, ENOTSUP,
						RTE_FLOW_ERROR_TYPE_ACTION,
						NULL,
						"Counter action on root table is not supported in HW steering mode");
			}
			if ((at->action_flags & MLX5_FLOW_ACTION_AGE) ||
			    (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
				/*
				 * When both COUNT and AGE are requested, it is
				 * saved as AGE action which creates also the
				 * counter.
				 */
				break;
			if (masks->conf &&
			    ((const struct rte_flow_action_count *)
			     masks->conf)->id) {
				err = flow_hw_cnt_compile(dev, dr_pos, acts);
				if (err)
					goto err;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 src_pos, dr_pos)) {
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_CONNTRACK:
			if (masks->conf) {
				ct_idx = MLX5_ACTION_CTX_CT_GET_IDX
					 ((uint32_t)(uintptr_t)actions->conf);
				if (flow_hw_ct_compile(dev, MLX5_HW_INV_QUEUE, ct_idx,
						       &acts->rule_acts[dr_pos]))
					goto err;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 src_pos, dr_pos)) {
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_METER_MARK:
			if (actions->conf && masks->conf &&
			    ((const struct rte_flow_action_meter_mark *)
			     masks->conf)->profile) {
				err = flow_hw_meter_mark_compile(dev,
								 dr_pos, actions,
								 acts->rule_acts,
								 &acts->mtr_id,
								 MLX5_HW_INV_QUEUE);
				if (err)
					goto err;
			} else if (__flow_hw_act_data_general_append(priv, acts,
								     actions->type,
								     src_pos,
								     dr_pos))
				goto err;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
			/* Internal, can be skipped. */
			if (!!attr->group) {
				DRV_LOG(ERR, "DEFAULT MISS action is only"
					" supported in root table.");
				goto err;
			}
			acts->rule_acts[dr_pos].action = priv->hw_def_miss;
			break;
		case RTE_FLOW_ACTION_TYPE_END:
			actions_end = true;
			break;
		default:
			break;
		}
	}
	if (mhdr.pos != UINT16_MAX) {
		ret = mlx5_tbl_translate_modify_header(dev, cfg, acts, mp_ctx,
						       &mhdr, error);
		if (ret)
			goto err;
	}
	if (reformat_used) {
		ret = mlx5_tbl_translate_reformat(priv, table_attr, acts, at,
						  enc_item, enc_item_m,
						  encap_data, encap_data_m,
						  mp_ctx, data_size,
						  reformat_src,
						  refmt_type, error);
		if (ret)
			goto err;
	}
	if (recom_used) {
		MLX5_ASSERT(at->recom_off != UINT16_MAX);
		ret = mlx5_create_ipv6_ext_reformat(dev, cfg, acts, at, push_data,
						    push_data_m, push_size, recom_src,
						    recom_type);
		if (ret)
			goto err;
	}
	return 0;
err:
	err = rte_errno;
	__flow_hw_action_template_destroy(dev, acts);
	return rte_flow_error_set(error, err,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				  "fail to create rte table");
}

/**
 * Translate rte_flow actions to DR action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] tbl
 *   Pointer to the flow template table.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_actions_translate(struct rte_eth_dev *dev,
			  struct rte_flow_template_table *tbl,
			  struct rte_flow_error *error)
{
	int ret;
	uint32_t i;
	struct mlx5_tbl_multi_pattern_ctx mpat = MLX5_EMPTY_MULTI_PATTERN_CTX;

	for (i = 0; i < tbl->nb_action_templates; i++) {
		if (__flow_hw_actions_translate(dev, &tbl->cfg,
						&tbl->ats[i].acts,
						tbl->ats[i].action_template,
						&mpat, error))
			goto err;
	}
	ret = mlx5_tbl_multi_pattern_process(dev, tbl, &mpat, error);
	if (ret)
		goto err;
	return 0;
err:
	while (i--)
		__flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
	return -1;
}

/**
 * Get shared indirect action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev data structure.
 * @param[in] act_data
 *   Pointer to the recorded action construct data.
 * @param[in] item_flags
 *   The matcher itme_flags used for RSS lookup.
 * @param[in] rule_act
 *   Pointer to the shared action's destination rule DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_shared_action_get(struct rte_eth_dev *dev,
			  struct mlx5_action_construct_data *act_data,
			  const uint64_t item_flags,
			  struct mlx5dr_rule_action *rule_act)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_rss_desc rss_desc = { 0 };
	uint64_t hash_fields = 0;
	uint32_t hrxq_idx = 0;
	struct mlx5_hrxq *hrxq = NULL;
	int act_type = act_data->type;

	switch (act_type) {
	case MLX5_RTE_FLOW_ACTION_TYPE_RSS:
		rss_desc.level = act_data->shared_rss.level;
		rss_desc.types = act_data->shared_rss.types;
		rss_desc.symmetric_hash_function = act_data->shared_rss.symmetric_hash_function;
		flow_dv_hashfields_set(item_flags, &rss_desc, &hash_fields);
		hrxq_idx = flow_dv_action_rss_hrxq_lookup
			(dev, act_data->shared_rss.idx, hash_fields);
		if (hrxq_idx)
			hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
					      hrxq_idx);
		if (hrxq) {
			rule_act->action = hrxq->action;
			return 0;
		}
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type:%d",
			act_data->type);
		break;
	}
	return -1;
}

static void
flow_hw_construct_quota(struct mlx5_priv *priv,
			struct mlx5dr_rule_action *rule_act, uint32_t qid)
{
	rule_act->action = priv->quota_ctx.dr_action;
	rule_act->aso_meter.offset = qid - 1;
	rule_act->aso_meter.init_color =
		MLX5DR_ACTION_ASO_METER_COLOR_GREEN;
}

/**
 * Construct shared indirect action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev data structure.
 * @param[in] queue
 *   The flow creation queue index.
 * @param[in] action
 *   Pointer to the shared indirect rte_flow action.
 * @param[in] table
 *   Pointer to the flow table.
 * @param[in] it_idx
 *   Item template index the action template refer to.
 * @param[in] action_flags
 *   Actions bit-map detected in this template.
 * @param[in, out] flow
 *   Pointer to the flow containing the counter.
 * @param[in] rule_act
 *   Pointer to the shared action's destination rule DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_shared_action_construct(struct rte_eth_dev *dev, uint32_t queue,
				const struct rte_flow_action *action,
				struct rte_flow_template_table *table,
				const uint8_t it_idx, uint64_t action_flags,
				struct rte_flow_hw *flow,
				struct mlx5dr_rule_action *rule_act)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct mlx5_action_construct_data act_data;
	struct mlx5_shared_action_rss *shared_rss;
	struct mlx5_aso_mtr *aso_mtr;
	struct mlx5_age_info *age_info;
	struct mlx5_hws_age_param *param;
	uint32_t act_idx = (uint32_t)(uintptr_t)action->conf;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t idx = act_idx &
		       ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
	uint64_t item_flags;
	cnt_id_t age_cnt;

	memset(&act_data, 0, sizeof(act_data));
	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_RSS:
		act_data.type = MLX5_RTE_FLOW_ACTION_TYPE_RSS;
		shared_rss = mlx5_ipool_get
			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
		if (!shared_rss)
			return -1;
		act_data.shared_rss.idx = idx;
		act_data.shared_rss.level = shared_rss->origin.level;
		act_data.shared_rss.types = !shared_rss->origin.types ?
					    RTE_ETH_RSS_IP :
					    shared_rss->origin.types;
		act_data.shared_rss.symmetric_hash_function =
			MLX5_RSS_IS_SYMM(shared_rss->origin.func);

		item_flags = table->its[it_idx]->item_flags;
		if (flow_hw_shared_action_get
				(dev, &act_data, item_flags, rule_act))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_COUNT:
		if (mlx5_hws_cnt_pool_get_action_offset(priv->hws_cpool,
				act_idx,
				&rule_act->action,
				&rule_act->counter.offset))
			return -1;
		flow->cnt_id = act_idx;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		/*
		 * Save the index with the indirect type, to recognize
		 * it in flow destroy.
		 */
		flow->age_idx = act_idx;
		if (action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT)
			/*
			 * The mutual update for idirect AGE & COUNT will be
			 * performed later after we have ID for both of them.
			 */
			break;
		age_info = GET_PORT_AGE_INFO(priv);
		param = mlx5_ipool_get(age_info->ages_ipool, idx);
		if (param == NULL)
			return -1;
		if (action_flags & MLX5_FLOW_ACTION_COUNT) {
			if (mlx5_hws_cnt_pool_get(priv->hws_cpool,
						  &param->queue_id, &age_cnt,
						  idx) < 0)
				return -1;
			flow->cnt_id = age_cnt;
			param->nb_cnts++;
		} else {
			/*
			 * Get the counter of this indirect AGE or create one
			 * if doesn't exist.
			 */
			age_cnt = mlx5_hws_age_cnt_get(priv, param, idx);
			if (age_cnt == 0)
				return -1;
		}
		if (mlx5_hws_cnt_pool_get_action_offset(priv->hws_cpool,
						     age_cnt, &rule_act->action,
						     &rule_act->counter.offset))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		if (flow_hw_ct_compile(dev, queue, idx, rule_act))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		/* Find ASO object. */
		aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
		if (!aso_mtr)
			return -1;
		rule_act->action = pool->action;
		rule_act->aso_meter.offset = aso_mtr->offset;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_QUOTA:
		flow_hw_construct_quota(priv, rule_act, idx);
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type:%d", type);
		break;
	}
	return 0;
}

static __rte_always_inline int
flow_hw_mhdr_cmd_is_nop(const struct mlx5_modification_cmd *cmd)
{
	struct mlx5_modification_cmd cmd_he = {
		.data0 = rte_be_to_cpu_32(cmd->data0),
		.data1 = 0,
	};

	return cmd_he.action_type == MLX5_MODIFICATION_TYPE_NOP;
}

/**
 * Construct flow action array.
 *
 * For action template contains dynamic actions, these actions need to
 * be updated according to the rte_flow action during flow creation.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] job
 *   Pointer to job descriptor.
 * @param[in] hw_acts
 *   Pointer to translated actions from template.
 * @param[in] it_idx
 *   Item template index the action template refer to.
 * @param[in] actions
 *   Array of rte_flow action need to be checked.
 * @param[in] rule_acts
 *   Array of DR rule actions to be used during flow creation..
 * @param[in] acts_num
 *   Pointer to the real acts_num flow has.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_modify_field_construct(struct mlx5_hw_q_job *job,
			       struct mlx5_action_construct_data *act_data,
			       const struct mlx5_hw_actions *hw_acts,
			       const struct rte_flow_action *action)
{
	const struct rte_flow_action_modify_field *mhdr_action = action->conf;
	uint8_t values[16] = { 0 };
	unaligned_uint32_t *value_p;
	uint32_t i;
	struct field_modify_info *field;

	if (!hw_acts->mhdr)
		return -1;
	if (hw_acts->mhdr->shared || act_data->modify_header.shared)
		return 0;
	MLX5_ASSERT(mhdr_action->operation == RTE_FLOW_MODIFY_SET ||
		    mhdr_action->operation == RTE_FLOW_MODIFY_ADD);
	if (mhdr_action->src.field != RTE_FLOW_FIELD_VALUE &&
	    mhdr_action->src.field != RTE_FLOW_FIELD_POINTER)
		return 0;
	if (mhdr_action->src.field == RTE_FLOW_FIELD_VALUE)
		rte_memcpy(values, &mhdr_action->src.value, sizeof(values));
	else
		rte_memcpy(values, mhdr_action->src.pvalue, sizeof(values));
	if (mhdr_action->dst.field == RTE_FLOW_FIELD_META ||
	    mhdr_action->dst.field == RTE_FLOW_FIELD_TAG ||
	    mhdr_action->dst.field == RTE_FLOW_FIELD_METER_COLOR ||
	    mhdr_action->dst.field == (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
		uint8_t tag_index = flow_tag_index_get(&mhdr_action->dst);

		value_p = (unaligned_uint32_t *)values;
		if (mhdr_action->dst.field == RTE_FLOW_FIELD_TAG &&
		    tag_index == RTE_PMD_MLX5_LINEAR_HASH_TAG_INDEX)
			*value_p = rte_cpu_to_be_32(*value_p << 16);
		else
			*value_p = rte_cpu_to_be_32(*value_p);
	} else if (mhdr_action->dst.field == RTE_FLOW_FIELD_GTP_PSC_QFI ||
		   mhdr_action->dst.field == RTE_FLOW_FIELD_GENEVE_OPT_TYPE) {
		uint32_t tmp;

		/*
		 * Both QFI and Geneve option type are passed as an uint8_t integer,
		 * but it is accessed through a 2nd least significant byte of a 32-bit
		 * field in modify header command.
		 */
		tmp = values[0];
		value_p = (unaligned_uint32_t *)values;
		*value_p = rte_cpu_to_be_32(tmp << 8);
	}
	i = act_data->modify_header.mhdr_cmds_off;
	field = act_data->modify_header.field;
	do {
		uint32_t off_b;
		uint32_t mask;
		uint32_t data;
		const uint8_t *mask_src;

		if (i >= act_data->modify_header.mhdr_cmds_end)
			return -1;
		if (flow_hw_mhdr_cmd_is_nop(&job->mhdr_cmd[i])) {
			++i;
			continue;
		}
		mask_src = (const uint8_t *)act_data->modify_header.mask;
		mask = flow_dv_fetch_field(mask_src + field->offset, field->size);
		if (!mask) {
			++field;
			continue;
		}
		off_b = rte_bsf32(mask);
		data = flow_dv_fetch_field(values + field->offset, field->size);
		/*
		 * IPv6 DSCP uses OUT_IPV6_TRAFFIC_CLASS as ID but it starts from 2
		 * bits left. Shift the data left for IPv6 DSCP
		 */
		if (field->id == MLX5_MODI_OUT_IPV6_TRAFFIC_CLASS &&
		    !(mask & MLX5_IPV6_HDR_ECN_MASK))
			data <<= MLX5_IPV6_HDR_DSCP_SHIFT;
		data = (data & mask) >> off_b;
		job->mhdr_cmd[i++].data1 = rte_cpu_to_be_32(data);
		++field;
	} while (field->size);
	return 0;
}

/**
 * Construct flow action array.
 *
 * For action template contains dynamic actions, these actions need to
 * be updated according to the rte_flow action during flow creation.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] job
 *   Pointer to job descriptor.
 * @param[in] hw_acts
 *   Pointer to translated actions from template.
 * @param[in] it_idx
 *   Item template index the action template refer to.
 * @param[in] actions
 *   Array of rte_flow action need to be checked.
 * @param[in] rule_acts
 *   Array of DR rule actions to be used during flow creation..
 * @param[in] acts_num
 *   Pointer to the real acts_num flow has.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_actions_construct(struct rte_eth_dev *dev,
			  struct mlx5_hw_q_job *job,
			  const struct mlx5_hw_action_template *hw_at,
			  const uint8_t it_idx,
			  const struct rte_flow_action actions[],
			  struct mlx5dr_rule_action *rule_acts,
			  uint32_t queue,
			  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct rte_flow_template_table *table = job->flow->table;
	struct mlx5_action_construct_data *act_data;
	const struct rte_flow_actions_template *at = hw_at->action_template;
	const struct mlx5_hw_actions *hw_acts = &hw_at->acts;
	const struct rte_flow_action *action;
	const struct rte_flow_action_raw_encap *raw_encap_data;
	const struct rte_flow_action_ipv6_ext_push *ipv6_push;
	const struct rte_flow_item *enc_item = NULL;
	const struct rte_flow_action_ethdev *port_action = NULL;
	const struct rte_flow_action_meter *meter = NULL;
	const struct rte_flow_action_age *age = NULL;
	uint8_t *buf = job->encap_data;
	uint8_t *push_buf = job->push_data;
	struct rte_flow_attr attr = {
			.ingress = 1,
	};
	uint32_t ft_flag;
	size_t encap_len = 0;
	int ret;
	uint32_t age_idx = 0;
	struct mlx5_aso_mtr *aso_mtr;

	rte_memcpy(rule_acts, hw_acts->rule_acts, sizeof(*rule_acts) * at->dr_actions_num);
	attr.group = table->grp->group_id;
	ft_flag = mlx5_hw_act_flag[!!table->grp->group_id][table->type];
	if (table->type == MLX5DR_TABLE_TYPE_FDB) {
		attr.transfer = 1;
		attr.ingress = 1;
	} else if (table->type == MLX5DR_TABLE_TYPE_NIC_TX) {
		attr.egress = 1;
		attr.ingress = 0;
	} else {
		attr.ingress = 1;
	}
	if (hw_acts->mhdr && hw_acts->mhdr->mhdr_cmds_num > 0) {
		uint16_t pos = hw_acts->mhdr->pos;

		if (!hw_acts->mhdr->shared) {
			rule_acts[pos].modify_header.offset =
						job->flow->res_idx - 1;
			rule_acts[pos].modify_header.data =
						(uint8_t *)job->mhdr_cmd;
			rte_memcpy(job->mhdr_cmd, hw_acts->mhdr->mhdr_cmds,
				   sizeof(*job->mhdr_cmd) * hw_acts->mhdr->mhdr_cmds_num);
		}
	}
	LIST_FOREACH(act_data, &hw_acts->act_list, next) {
		uint32_t jump_group;
		uint32_t tag;
		uint64_t item_flags;
		struct mlx5_hw_jump_action *jump;
		struct mlx5_hrxq *hrxq;
		uint32_t ct_idx;
		cnt_id_t cnt_id;
		uint32_t *cnt_queue;
		uint32_t mtr_id;

		action = &actions[act_data->action_src];
		/*
		 * action template construction replaces
		 * OF_SET_VLAN_VID with MODIFY_FIELD
		 */
		if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
			MLX5_ASSERT(act_data->type ==
				    RTE_FLOW_ACTION_TYPE_MODIFY_FIELD);
		else
			MLX5_ASSERT(action->type ==
				    RTE_FLOW_ACTION_TYPE_INDIRECT ||
				    (int)action->type == act_data->type);
		switch ((int)act_data->type) {
		case RTE_FLOW_ACTION_TYPE_INDIRECT_LIST:
			act_data->indirect_list_cb(dev, act_data, actions,
						   &rule_acts[act_data->action_dst]);
			break;
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			if (flow_hw_shared_action_construct
					(dev, queue, action, table, it_idx,
					 at->action_flags, job->flow,
					 &rule_acts[act_data->action_dst]))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_MARK:
			tag = mlx5_flow_mark_set
			      (((const struct rte_flow_action_mark *)
			      (action->conf))->id);
			rule_acts[act_data->action_dst].tag.value = tag;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			rule_acts[act_data->action_dst].push_vlan.vlan_hdr =
				vlan_hdr_to_be32(action);
			break;
		case RTE_FLOW_ACTION_TYPE_JUMP:
			jump_group = ((const struct rte_flow_action_jump *)
						action->conf)->group;
			jump = flow_hw_jump_action_register
				(dev, &table->cfg, jump_group, NULL);
			if (!jump)
				return -1;
			rule_acts[act_data->action_dst].action =
			(!!attr.group) ? jump->hws_action : jump->root_action;
			job->flow->jump = jump;
			job->flow->fate_type = MLX5_FLOW_FATE_JUMP;
			break;
		case RTE_FLOW_ACTION_TYPE_RSS:
		case RTE_FLOW_ACTION_TYPE_QUEUE:
			hrxq = flow_hw_tir_action_register(dev,
					ft_flag,
					action);
			if (!hrxq)
				return -1;
			rule_acts[act_data->action_dst].action = hrxq->action;
			job->flow->hrxq = hrxq;
			job->flow->fate_type = MLX5_FLOW_FATE_QUEUE;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_RSS:
			item_flags = table->its[it_idx]->item_flags;
			if (flow_hw_shared_action_get
				(dev, act_data, item_flags,
				 &rule_acts[act_data->action_dst]))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
			enc_item = ((const struct rte_flow_action_vxlan_encap *)
				   action->conf)->definition;
			if (flow_dv_convert_encap_data(enc_item, buf, &encap_len, NULL))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
			enc_item = ((const struct rte_flow_action_nvgre_encap *)
				   action->conf)->definition;
			if (flow_dv_convert_encap_data(enc_item, buf, &encap_len, NULL))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			raw_encap_data =
				(const struct rte_flow_action_raw_encap *)
				 action->conf;
			rte_memcpy((void *)buf, raw_encap_data->data, act_data->encap.len);
			MLX5_ASSERT(raw_encap_data->size ==
				    act_data->encap.len);
			break;
		case RTE_FLOW_ACTION_TYPE_IPV6_EXT_PUSH:
			ipv6_push =
				(const struct rte_flow_action_ipv6_ext_push *)action->conf;
			rte_memcpy((void *)push_buf, ipv6_push->data,
				   act_data->ipv6_ext.len);
			MLX5_ASSERT(ipv6_push->size == act_data->ipv6_ext.len);
			break;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
				ret = flow_hw_set_vlan_vid_construct(dev, job,
								     act_data,
								     hw_acts,
								     action);
			else
				ret = flow_hw_modify_field_construct(job,
								     act_data,
								     hw_acts,
								     action);
			if (ret)
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
			port_action = action->conf;
			if (!priv->hw_vport[port_action->port_id])
				return -1;
			rule_acts[act_data->action_dst].action =
					priv->hw_vport[port_action->port_id];
			break;
		case RTE_FLOW_ACTION_TYPE_QUOTA:
			flow_hw_construct_quota(priv,
						rule_acts + act_data->action_dst,
						act_data->shared_meter.id);
			break;
		case RTE_FLOW_ACTION_TYPE_METER:
			meter = action->conf;
			mtr_id = meter->mtr_id;
			aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_id);
			rule_acts[act_data->action_dst].action =
				priv->mtr_bulk.action;
			rule_acts[act_data->action_dst].aso_meter.offset =
								aso_mtr->offset;
			jump = flow_hw_jump_action_register
				(dev, &table->cfg, aso_mtr->fm.group, NULL);
			if (!jump)
				return -1;
			MLX5_ASSERT
				(!rule_acts[act_data->action_dst + 1].action);
			rule_acts[act_data->action_dst + 1].action =
					(!!attr.group) ? jump->hws_action :
							 jump->root_action;
			job->flow->jump = jump;
			job->flow->fate_type = MLX5_FLOW_FATE_JUMP;
			if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
			age = action->conf;
			/*
			 * First, create the AGE parameter, then create its
			 * counter later:
			 * Regular counter - in next case.
			 * Indirect counter - update it after the loop.
			 */
			age_idx = mlx5_hws_age_action_create(priv, queue, 0,
							     age,
							     job->flow->res_idx,
							     error);
			if (age_idx == 0)
				return -rte_errno;
			job->flow->age_idx = age_idx;
			if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT)
				/*
				 * When AGE uses indirect counter, no need to
				 * create counter but need to update it with the
				 * AGE parameter, will be done after the loop.
				 */
				break;
			/* Fall-through. */
		case RTE_FLOW_ACTION_TYPE_COUNT:
			/* If the port is engaged in resource sharing, do not use queue cache. */
			cnt_queue = mlx5_hws_cnt_is_pool_shared(priv) ? NULL : &queue;
			ret = mlx5_hws_cnt_pool_get(priv->hws_cpool, cnt_queue, &cnt_id, age_idx);
			if (ret != 0)
				return ret;
			ret = mlx5_hws_cnt_pool_get_action_offset
				(priv->hws_cpool,
				 cnt_id,
				 &rule_acts[act_data->action_dst].action,
				 &rule_acts[act_data->action_dst].counter.offset
				 );
			if (ret != 0)
				return ret;
			job->flow->cnt_id = cnt_id;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
			ret = mlx5_hws_cnt_pool_get_action_offset
				(priv->hws_cpool,
				 act_data->shared_counter.id,
				 &rule_acts[act_data->action_dst].action,
				 &rule_acts[act_data->action_dst].counter.offset
				 );
			if (ret != 0)
				return ret;
			job->flow->cnt_id = act_data->shared_counter.id;
			break;
		case RTE_FLOW_ACTION_TYPE_CONNTRACK:
			ct_idx = MLX5_ACTION_CTX_CT_GET_IDX
				 ((uint32_t)(uintptr_t)action->conf);
			if (flow_hw_ct_compile(dev, queue, ct_idx,
					       &rule_acts[act_data->action_dst]))
				return -1;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK:
			mtr_id = act_data->shared_meter.id &
				((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
			/* Find ASO object. */
			aso_mtr = mlx5_ipool_get(pool->idx_pool, mtr_id);
			if (!aso_mtr)
				return -1;
			rule_acts[act_data->action_dst].action =
							pool->action;
			rule_acts[act_data->action_dst].aso_meter.offset =
							aso_mtr->offset;
			break;
		case RTE_FLOW_ACTION_TYPE_METER_MARK:
			/*
			 * Allocate meter directly will slow down flow
			 * insertion rate.
			 */
			ret = flow_hw_meter_mark_compile(dev,
				act_data->action_dst, action,
				rule_acts, &job->flow->mtr_id, MLX5_HW_INV_QUEUE);
			if (ret != 0)
				return ret;
			break;
		default:
			break;
		}
	}
	if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT) {
		if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE) {
			age_idx = job->flow->age_idx & MLX5_HWS_AGE_IDX_MASK;
			if (mlx5_hws_cnt_age_get(priv->hws_cpool,
						 job->flow->cnt_id) != age_idx)
				/*
				 * This is first use of this indirect counter
				 * for this indirect AGE, need to increase the
				 * number of counters.
				 */
				mlx5_hws_age_nb_cnt_increase(priv, age_idx);
		}
		/*
		 * Update this indirect counter the indirect/direct AGE in which
		 * using it.
		 */
		mlx5_hws_cnt_age_set(priv->hws_cpool, job->flow->cnt_id,
				     age_idx);
	}
	if (hw_acts->encap_decap && !hw_acts->encap_decap->shared) {
		rule_acts[hw_acts->encap_decap_pos].reformat.offset =
				job->flow->res_idx - 1;
		rule_acts[hw_acts->encap_decap_pos].reformat.data = buf;
	}
	if (hw_acts->push_remove && !hw_acts->push_remove->shared) {
		rule_acts[hw_acts->push_remove_pos].ipv6_ext.offset =
				job->flow->res_idx - 1;
		rule_acts[hw_acts->push_remove_pos].ipv6_ext.header = push_buf;
	}
	if (mlx5_hws_cnt_id_valid(hw_acts->cnt_id))
		job->flow->cnt_id = hw_acts->cnt_id;
	return 0;
}

static const struct rte_flow_item *
flow_hw_get_rule_items(struct rte_eth_dev *dev,
		       const struct rte_flow_template_table *table,
		       const struct rte_flow_item items[],
		       uint8_t pattern_template_index,
		       struct mlx5_hw_q_job *job)
{
	struct rte_flow_pattern_template *pt = table->its[pattern_template_index];

	/* Only one implicit item can be added to flow rule pattern. */
	MLX5_ASSERT(!pt->implicit_port || !pt->implicit_tag);
	/* At least one item was allocated in job descriptor for items. */
	MLX5_ASSERT(MLX5_HW_MAX_ITEMS >= 1);
	if (pt->implicit_port) {
		if (pt->orig_item_nb + 1 > MLX5_HW_MAX_ITEMS) {
			rte_errno = ENOMEM;
			return NULL;
		}
		/* Set up represented port item in job descriptor. */
		job->port_spec = (struct rte_flow_item_ethdev){
			.port_id = dev->data->port_id,
		};
		job->items[0] = (struct rte_flow_item){
			.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
			.spec = &job->port_spec,
		};
		rte_memcpy(&job->items[1], items, sizeof(*items) * pt->orig_item_nb);
		return job->items;
	} else if (pt->implicit_tag) {
		if (pt->orig_item_nb + 1 > MLX5_HW_MAX_ITEMS) {
			rte_errno = ENOMEM;
			return NULL;
		}
		/* Set up tag item in job descriptor. */
		job->tag_spec = (struct rte_flow_item_tag){
			.data = flow_hw_tx_tag_regc_value(dev),
		};
		job->items[0] = (struct rte_flow_item){
			.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
			.spec = &job->tag_spec,
		};
		rte_memcpy(&job->items[1], items, sizeof(*items) * pt->orig_item_nb);
		return job->items;
	} else {
		return items;
	}
}

/**
 * Enqueue HW steering flow creation.
 *
 * The flow will be applied to the HW only if the postpone bit is not set or
 * the extra push function is called.
 * The flow creation status should be checked from dequeue result.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to create the flow.
 * @param[in] attr
 *   Pointer to the flow operation attributes.
 * @param[in] items
 *   Items with flow spec value.
 * @param[in] pattern_template_index
 *   The item pattern flow follows from the table.
 * @param[in] actions
 *   Action with flow spec value.
 * @param[in] action_template_index
 *   The action pattern flow follows from the table.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Flow pointer on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow *
flow_hw_async_flow_create(struct rte_eth_dev *dev,
			  uint32_t queue,
			  const struct rte_flow_op_attr *attr,
			  struct rte_flow_template_table *table,
			  const struct rte_flow_item items[],
			  uint8_t pattern_template_index,
			  const struct rte_flow_action actions[],
			  uint8_t action_template_index,
			  void *user_data,
			  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_rule_attr rule_attr = {
		.queue_id = queue,
		.user_data = user_data,
		.burst = attr->postpone,
	};
	struct mlx5dr_rule_action rule_acts[MLX5_HW_MAX_ACTS];
	struct rte_flow_hw *flow = NULL;
	struct mlx5_hw_q_job *job = NULL;
	const struct rte_flow_item *rule_items;
	uint32_t flow_idx = 0;
	uint32_t res_idx = 0;
	int ret;

	if (unlikely((!dev->data->dev_started))) {
		rte_errno = EINVAL;
		goto error;
	}
	job = flow_hw_job_get(priv, queue);
	if (!job) {
		rte_errno = ENOMEM;
		goto error;
	}
	flow = mlx5_ipool_zmalloc(table->flow, &flow_idx);
	if (!flow)
		goto error;
	mlx5_ipool_malloc(table->resource, &res_idx);
	if (!res_idx)
		goto error;
	/*
	 * Set the table here in order to know the destination table
	 * when free the flow afterward.
	 */
	flow->table = table;
	flow->mt_idx = pattern_template_index;
	flow->idx = flow_idx;
	flow->res_idx = res_idx;
	/*
	 * Set the job type here in order to know if the flow memory
	 * should be freed or not when get the result from dequeue.
	 */
	job->type = MLX5_HW_Q_JOB_TYPE_CREATE;
	job->flow = flow;
	job->user_data = user_data;
	rule_attr.user_data = job;
	/*
	 * Indexed pool returns 1-based indices, but mlx5dr expects 0-based indices for rule
	 * insertion hints.
	 */
	MLX5_ASSERT(res_idx > 0);
	flow->rule_idx = res_idx - 1;
	rule_attr.rule_idx = flow->rule_idx;
	/*
	 * Construct the flow actions based on the input actions.
	 * The implicitly appended action is always fixed, like metadata
	 * copy action from FDB to NIC Rx.
	 * No need to copy and contrust a new "actions" list based on the
	 * user's input, in order to save the cost.
	 */
	if (flow_hw_actions_construct(dev, job,
				      &table->ats[action_template_index],
				      pattern_template_index, actions,
				      rule_acts, queue, error)) {
		rte_errno = EINVAL;
		goto error;
	}
	rule_items = flow_hw_get_rule_items(dev, table, items,
					    pattern_template_index, job);
	if (!rule_items)
		goto error;
	ret = mlx5dr_rule_create(table->matcher,
				 pattern_template_index, rule_items,
				 action_template_index, rule_acts,
				 &rule_attr, (struct mlx5dr_rule *)flow->rule);
	if (likely(!ret))
		return (struct rte_flow *)flow;
error:
	if (job)
		flow_hw_job_put(priv, job, queue);
	if (flow_idx)
		mlx5_ipool_free(table->flow, flow_idx);
	if (res_idx)
		mlx5_ipool_free(table->resource, res_idx);
	rte_flow_error_set(error, rte_errno,
			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
			   "fail to create rte flow");
	return NULL;
}

/**
 * Enqueue HW steering flow creation by index.
 *
 * The flow will be applied to the HW only if the postpone bit is not set or
 * the extra push function is called.
 * The flow creation status should be checked from dequeue result.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to create the flow.
 * @param[in] attr
 *   Pointer to the flow operation attributes.
 * @param[in] rule_index
 *   The item pattern flow follows from the table.
 * @param[in] actions
 *   Action with flow spec value.
 * @param[in] action_template_index
 *   The action pattern flow follows from the table.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Flow pointer on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow *
flow_hw_async_flow_create_by_index(struct rte_eth_dev *dev,
			  uint32_t queue,
			  const struct rte_flow_op_attr *attr,
			  struct rte_flow_template_table *table,
			  uint32_t rule_index,
			  const struct rte_flow_action actions[],
			  uint8_t action_template_index,
			  void *user_data,
			  struct rte_flow_error *error)
{
	struct rte_flow_item items[] = {{.type = RTE_FLOW_ITEM_TYPE_END,}};
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_rule_attr rule_attr = {
		.queue_id = queue,
		.user_data = user_data,
		.burst = attr->postpone,
	};
	struct mlx5dr_rule_action rule_acts[MLX5_HW_MAX_ACTS];
	struct rte_flow_hw *flow = NULL;
	struct mlx5_hw_q_job *job = NULL;
	uint32_t flow_idx = 0;
	uint32_t res_idx = 0;
	int ret;

	if (unlikely(rule_index >= table->cfg.attr.nb_flows)) {
		rte_errno = EINVAL;
		goto error;
	}
	job = flow_hw_job_get(priv, queue);
	if (!job) {
		rte_errno = ENOMEM;
		goto error;
	}
	flow = mlx5_ipool_zmalloc(table->flow, &flow_idx);
	if (!flow)
		goto error;
	mlx5_ipool_malloc(table->resource, &res_idx);
	if (!res_idx)
		goto error;
	/*
	 * Set the table here in order to know the destination table
	 * when free the flow afterwards.
	 */
	flow->table = table;
	flow->mt_idx = 0;
	flow->idx = flow_idx;
	flow->res_idx = res_idx;
	/*
	 * Set the job type here in order to know if the flow memory
	 * should be freed or not when get the result from dequeue.
	 */
	job->type = MLX5_HW_Q_JOB_TYPE_CREATE;
	job->flow = flow;
	job->user_data = user_data;
	rule_attr.user_data = job;
	/*
	 * Set the rule index.
	 */
	flow->rule_idx = rule_index;
	rule_attr.rule_idx = flow->rule_idx;
	/*
	 * Construct the flow actions based on the input actions.
	 * The implicitly appended action is always fixed, like metadata
	 * copy action from FDB to NIC Rx.
	 * No need to copy and contrust a new "actions" list based on the
	 * user's input, in order to save the cost.
	 */
	if (flow_hw_actions_construct(dev, job,
				      &table->ats[action_template_index],
				      0, actions, rule_acts, queue, error)) {
		rte_errno = EINVAL;
		goto error;
	}
	ret = mlx5dr_rule_create(table->matcher,
				 0, items, action_template_index, rule_acts,
				 &rule_attr, (struct mlx5dr_rule *)flow->rule);
	if (likely(!ret))
		return (struct rte_flow *)flow;
error:
	if (job)
		flow_hw_job_put(priv, job, queue);
	if (res_idx)
		mlx5_ipool_free(table->resource, res_idx);
	if (flow_idx)
		mlx5_ipool_free(table->flow, flow_idx);
	rte_flow_error_set(error, rte_errno,
			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
			   "fail to create rte flow");
	return NULL;
}

/**
 * Enqueue HW steering flow update.
 *
 * The flow will be applied to the HW only if the postpone bit is not set or
 * the extra push function is called.
 * The flow destruction status should be checked from dequeue result.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to destroy the flow.
 * @param[in] attr
 *   Pointer to the flow operation attributes.
 * @param[in] flow
 *   Pointer to the flow to be destroyed.
 * @param[in] actions
 *   Action with flow spec value.
 * @param[in] action_template_index
 *   The action pattern flow follows from the table.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_async_flow_update(struct rte_eth_dev *dev,
			   uint32_t queue,
			   const struct rte_flow_op_attr *attr,
			   struct rte_flow *flow,
			   const struct rte_flow_action actions[],
			   uint8_t action_template_index,
			   void *user_data,
			   struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_rule_attr rule_attr = {
		.queue_id = queue,
		.user_data = user_data,
		.burst = attr->postpone,
	};
	struct mlx5dr_rule_action rule_acts[MLX5_HW_MAX_ACTS];
	struct rte_flow_hw *of = (struct rte_flow_hw *)flow;
	struct rte_flow_hw *nf;
	struct rte_flow_template_table *table = of->table;
	struct mlx5_hw_q_job *job = NULL;
	uint32_t res_idx = 0;
	int ret;

	job = flow_hw_job_get(priv, queue);
	if (!job) {
		rte_errno = ENOMEM;
		goto error;
	}
	mlx5_ipool_malloc(table->resource, &res_idx);
	if (!res_idx)
		goto error;
	nf = job->upd_flow;
	memset(nf, 0, sizeof(struct rte_flow_hw));
	/*
	 * Set the table here in order to know the destination table
	 * when free the flow afterwards.
	 */
	nf->table = table;
	nf->mt_idx = of->mt_idx;
	nf->idx = of->idx;
	nf->res_idx = res_idx;
	/*
	 * Set the job type here in order to know if the flow memory
	 * should be freed or not when get the result from dequeue.
	 */
	job->type = MLX5_HW_Q_JOB_TYPE_UPDATE;
	job->flow = nf;
	job->user_data = user_data;
	rule_attr.user_data = job;
	/*
	 * Indexed pool returns 1-based indices, but mlx5dr expects 0-based indices for rule
	 * insertion hints.
	 */
	MLX5_ASSERT(res_idx > 0);
	nf->rule_idx = res_idx - 1;
	rule_attr.rule_idx = nf->rule_idx;
	/*
	 * Construct the flow actions based on the input actions.
	 * The implicitly appended action is always fixed, like metadata
	 * copy action from FDB to NIC Rx.
	 * No need to copy and contrust a new "actions" list based on the
	 * user's input, in order to save the cost.
	 */
	if (flow_hw_actions_construct(dev, job,
				      &table->ats[action_template_index],
				      nf->mt_idx, actions,
				      rule_acts, queue, error)) {
		rte_errno = EINVAL;
		goto error;
	}
	/*
	 * Switch the old flow and the new flow.
	 */
	job->flow = of;
	job->upd_flow = nf;
	ret = mlx5dr_rule_action_update((struct mlx5dr_rule *)of->rule,
					action_template_index, rule_acts, &rule_attr);
	if (likely(!ret))
		return 0;
error:
	/* Flow created fail, return the descriptor and flow memory. */
	if (job)
		flow_hw_job_put(priv, job, queue);
	if (res_idx)
		mlx5_ipool_free(table->resource, res_idx);
	return rte_flow_error_set(error, rte_errno,
			RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
			"fail to update rte flow");
}

/**
 * Enqueue HW steering flow destruction.
 *
 * The flow will be applied to the HW only if the postpone bit is not set or
 * the extra push function is called.
 * The flow destruction status should be checked from dequeue result.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to destroy the flow.
 * @param[in] attr
 *   Pointer to the flow operation attributes.
 * @param[in] flow
 *   Pointer to the flow to be destroyed.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_async_flow_destroy(struct rte_eth_dev *dev,
			   uint32_t queue,
			   const struct rte_flow_op_attr *attr,
			   struct rte_flow *flow,
			   void *user_data,
			   struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_rule_attr rule_attr = {
		.queue_id = queue,
		.user_data = user_data,
		.burst = attr->postpone,
	};
	struct rte_flow_hw *fh = (struct rte_flow_hw *)flow;
	struct mlx5_hw_q_job *job;
	int ret;

	job = flow_hw_job_get(priv, queue);
	if (!job)
		return rte_flow_error_set(error, ENOMEM,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "fail to destroy rte flow: flow queue full");
	job->type = MLX5_HW_Q_JOB_TYPE_DESTROY;
	job->user_data = user_data;
	job->flow = fh;
	rule_attr.user_data = job;
	rule_attr.rule_idx = fh->rule_idx;
	ret = mlx5dr_rule_destroy((struct mlx5dr_rule *)fh->rule, &rule_attr);
	if (ret) {
		flow_hw_job_put(priv, job, queue);
		return rte_flow_error_set(error, rte_errno,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "fail to destroy rte flow");
	}
	return 0;
}

/**
 * Release the AGE and counter for given flow.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] queue
 *   The queue to release the counter.
 * @param[in, out] flow
 *   Pointer to the flow containing the counter.
 * @param[out] error
 *   Pointer to error structure.
 */
static void
flow_hw_age_count_release(struct mlx5_priv *priv, uint32_t queue,
			  struct rte_flow_hw *flow,
			  struct rte_flow_error *error)
{
	uint32_t *cnt_queue;

	if (mlx5_hws_cnt_is_shared(priv->hws_cpool, flow->cnt_id)) {
		if (flow->age_idx && !mlx5_hws_age_is_indirect(flow->age_idx)) {
			/* Remove this AGE parameter from indirect counter. */
			mlx5_hws_cnt_age_set(priv->hws_cpool, flow->cnt_id, 0);
			/* Release the AGE parameter. */
			mlx5_hws_age_action_destroy(priv, flow->age_idx, error);
			flow->age_idx = 0;
		}
		return;
	}
	/* If the port is engaged in resource sharing, do not use queue cache. */
	cnt_queue = mlx5_hws_cnt_is_pool_shared(priv) ? NULL : &queue;
	/* Put the counter first to reduce the race risk in BG thread. */
	mlx5_hws_cnt_pool_put(priv->hws_cpool, cnt_queue, &flow->cnt_id);
	flow->cnt_id = 0;
	if (flow->age_idx) {
		if (mlx5_hws_age_is_indirect(flow->age_idx)) {
			uint32_t idx = flow->age_idx & MLX5_HWS_AGE_IDX_MASK;

			mlx5_hws_age_nb_cnt_decrease(priv, idx);
		} else {
			/* Release the AGE parameter. */
			mlx5_hws_age_action_destroy(priv, flow->age_idx, error);
		}
		flow->age_idx = 0;
	}
}

static __rte_always_inline void
flow_hw_pull_legacy_indirect_comp(struct rte_eth_dev *dev, struct mlx5_hw_q_job *job,
				  uint32_t queue)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_action *aso_ct;
	struct mlx5_aso_mtr *aso_mtr;
	uint32_t type, idx;

	if (MLX5_INDIRECT_ACTION_TYPE_GET(job->action) ==
	    MLX5_INDIRECT_ACTION_TYPE_QUOTA) {
		mlx5_quota_async_completion(dev, queue, job);
	} else if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY) {
		type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
		if (type == MLX5_INDIRECT_ACTION_TYPE_METER_MARK) {
			idx = MLX5_INDIRECT_ACTION_IDX_GET(job->action);
			mlx5_ipool_free(priv->hws_mpool->idx_pool, idx);
		}
	} else if (job->type == MLX5_HW_Q_JOB_TYPE_CREATE) {
		type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
		if (type == MLX5_INDIRECT_ACTION_TYPE_METER_MARK) {
			idx = MLX5_INDIRECT_ACTION_IDX_GET(job->action);
			aso_mtr = mlx5_ipool_get(priv->hws_mpool->idx_pool, idx);
			aso_mtr->state = ASO_METER_READY;
		} else if (type == MLX5_INDIRECT_ACTION_TYPE_CT) {
			idx = MLX5_ACTION_CTX_CT_GET_IDX
			((uint32_t)(uintptr_t)job->action);
			aso_ct = mlx5_ipool_get(priv->hws_ctpool->cts, idx);
			aso_ct->state = ASO_CONNTRACK_READY;
		}
	} else if (job->type == MLX5_HW_Q_JOB_TYPE_QUERY) {
		type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
		if (type == MLX5_INDIRECT_ACTION_TYPE_CT) {
			idx = MLX5_ACTION_CTX_CT_GET_IDX
			((uint32_t)(uintptr_t)job->action);
			aso_ct = mlx5_ipool_get(priv->hws_ctpool->cts, idx);
			mlx5_aso_ct_obj_analyze(job->query.user,
						job->query.hw);
			aso_ct->state = ASO_CONNTRACK_READY;
		}
	} else {
		/*
		 * rte_flow_op_result::user data can point to
		 * struct mlx5_aso_mtr object as well
		 */
		if (queue != CTRL_QUEUE_ID(priv))
			MLX5_ASSERT(false);
	}
}

static inline int
__flow_hw_pull_indir_action_comp(struct rte_eth_dev *dev,
				 uint32_t queue,
				 struct rte_flow_op_result res[],
				 uint16_t n_res)

{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_ring *r = priv->hw_q[queue].indir_cq;
	void *user_data = NULL;
	int ret_comp, i;

	ret_comp = (int)rte_ring_count(r);
	if (ret_comp > n_res)
		ret_comp = n_res;
	for (i = 0; i < ret_comp; i++) {
		rte_ring_dequeue(r, &user_data);
		res[i].user_data = user_data;
		res[i].status = RTE_FLOW_OP_SUCCESS;
	}
	if (ret_comp < n_res && priv->hws_mpool)
		ret_comp += mlx5_aso_pull_completion(&priv->hws_mpool->sq[queue],
				&res[ret_comp], n_res - ret_comp);
	if (ret_comp < n_res && priv->hws_ctpool)
		ret_comp += mlx5_aso_pull_completion(&priv->ct_mng->aso_sqs[queue],
				&res[ret_comp], n_res - ret_comp);
	if (ret_comp < n_res && priv->quota_ctx.sq)
		ret_comp += mlx5_aso_pull_completion(&priv->quota_ctx.sq[queue],
						     &res[ret_comp],
						     n_res - ret_comp);
	for (i = 0; i <  ret_comp; i++) {
		struct mlx5_hw_q_job *job = (struct mlx5_hw_q_job *)res[i].user_data;

		/* Restore user data. */
		res[i].user_data = job->user_data;
		if (job->indirect_type == MLX5_HW_INDIRECT_TYPE_LEGACY)
			flow_hw_pull_legacy_indirect_comp(dev, job, queue);
		/*
		 * Current PMD supports 2 indirect action list types - MIRROR and REFORMAT.
		 * These indirect list types do not post WQE to create action.
		 * Future indirect list types that do post WQE will add
		 * completion handlers here.
		 */
		flow_hw_job_put(priv, job, queue);
	}
	return ret_comp;
}

/**
 * Pull the enqueued flows.
 *
 * For flows enqueued from creation/destruction, the status should be
 * checked from the dequeue result.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to pull the result.
 * @param[in/out] res
 *   Array to save the results.
 * @param[in] n_res
 *   Available result with the array.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Result number on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_pull(struct rte_eth_dev *dev,
	     uint32_t queue,
	     struct rte_flow_op_result res[],
	     uint16_t n_res,
	     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct mlx5_hw_q_job *job;
	uint32_t res_idx;
	int ret, i;

	/* 1. Pull the flow completion. */
	ret = mlx5dr_send_queue_poll(priv->dr_ctx, queue, res, n_res);
	if (ret < 0)
		return rte_flow_error_set(error, rte_errno,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				"fail to query flow queue");
	for (i = 0; i <  ret; i++) {
		job = (struct mlx5_hw_q_job *)res[i].user_data;
		/* Release the original resource index in case of update. */
		res_idx = job->flow->res_idx;
		/* Restore user data. */
		res[i].user_data = job->user_data;
		if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY ||
		    job->type == MLX5_HW_Q_JOB_TYPE_UPDATE) {
			if (job->flow->fate_type == MLX5_FLOW_FATE_JUMP)
				flow_hw_jump_release(dev, job->flow->jump);
			else if (job->flow->fate_type == MLX5_FLOW_FATE_QUEUE)
				mlx5_hrxq_obj_release(dev, job->flow->hrxq);
			if (mlx5_hws_cnt_id_valid(job->flow->cnt_id))
				flow_hw_age_count_release(priv, queue,
							  job->flow, error);
			if (job->flow->mtr_id) {
				mlx5_ipool_free(pool->idx_pool,	job->flow->mtr_id);
				job->flow->mtr_id = 0;
			}
			if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY) {
				mlx5_ipool_free(job->flow->table->resource, res_idx);
				mlx5_ipool_free(job->flow->table->flow, job->flow->idx);
			} else {
				rte_memcpy(job->flow, job->upd_flow,
					offsetof(struct rte_flow_hw, rule));
				mlx5_ipool_free(job->flow->table->resource, res_idx);
			}
		}
		flow_hw_job_put(priv, job, queue);
	}
	/* 2. Pull indirect action comp. */
	if (ret < n_res)
		ret += __flow_hw_pull_indir_action_comp(dev, queue, &res[ret],
							n_res - ret);
	return ret;
}

static inline uint32_t
__flow_hw_push_action(struct rte_eth_dev *dev,
		    uint32_t queue)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_ring *iq = priv->hw_q[queue].indir_iq;
	struct rte_ring *cq = priv->hw_q[queue].indir_cq;
	void *job = NULL;
	uint32_t ret, i;

	ret = rte_ring_count(iq);
	for (i = 0; i < ret; i++) {
		rte_ring_dequeue(iq, &job);
		rte_ring_enqueue(cq, job);
	}
	if (!priv->shared_host) {
		if (priv->hws_ctpool)
			mlx5_aso_push_wqe(priv->sh,
					  &priv->ct_mng->aso_sqs[queue]);
		if (priv->hws_mpool)
			mlx5_aso_push_wqe(priv->sh,
					  &priv->hws_mpool->sq[queue]);
	}
	return priv->hw_q[queue].size - priv->hw_q[queue].job_idx;
}

static int
__flow_hw_push(struct rte_eth_dev *dev,
	       uint32_t queue,
	       struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int ret, num;

	num = __flow_hw_push_action(dev, queue);
	ret = mlx5dr_send_queue_action(priv->dr_ctx, queue,
				       MLX5DR_SEND_QUEUE_ACTION_DRAIN_ASYNC);
	if (ret) {
		rte_flow_error_set(error, rte_errno,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				   "fail to push flows");
		return ret;
	}
	return num;
}

/**
 * Push the enqueued flows to HW.
 *
 * Force apply all the enqueued flows to the HW.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to push the flow.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_push(struct rte_eth_dev *dev,
	     uint32_t queue, struct rte_flow_error *error)
{
	int ret = __flow_hw_push(dev, queue, error);

	return ret >= 0 ? 0 : ret;
}

/**
 * Drain the enqueued flows' completion.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to pull the flow.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
__flow_hw_pull_comp(struct rte_eth_dev *dev,
		    uint32_t queue, struct rte_flow_error *error)
{
	struct rte_flow_op_result comp[BURST_THR];
	int ret, i, empty_loop = 0;
	uint32_t pending_rules;

	ret = __flow_hw_push(dev, queue, error);
	if (ret < 0)
		return ret;
	pending_rules = ret;
	while (pending_rules) {
		ret = flow_hw_pull(dev, queue, comp, BURST_THR, error);
		if (ret < 0)
			return -1;
		if (!ret) {
			rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
			if (++empty_loop > 5) {
				DRV_LOG(WARNING, "No available dequeue %u, quit.", pending_rules);
				break;
			}
			continue;
		}
		for (i = 0; i < ret; i++) {
			if (comp[i].status == RTE_FLOW_OP_ERROR)
				DRV_LOG(WARNING, "Flow flush get error CQE.");
		}
		/*
		 * Indirect **SYNC** METER_MARK and CT actions do not
		 * remove completion after WQE post.
		 * That implementation avoids HW timeout.
		 * The completion is removed before the following WQE post.
		 * However, HWS queue updates do not reflect that behaviour.
		 * Therefore, during port destruction sync queue may have
		 * pending completions.
		 */
		pending_rules -= RTE_MIN(pending_rules, (uint32_t)ret);
		empty_loop = 0;
	}
	return 0;
}

/**
 * Flush created flows.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
int
flow_hw_q_flow_flush(struct rte_eth_dev *dev,
		     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_q *hw_q = &priv->hw_q[MLX5_DEFAULT_FLUSH_QUEUE];
	struct rte_flow_template_table *tbl;
	struct rte_flow_hw *flow;
	struct rte_flow_op_attr attr = {
		.postpone = 0,
	};
	uint32_t pending_rules = 0;
	uint32_t queue;
	uint32_t fidx;

	/*
	 * Ensure to push and dequeue all the enqueued flow
	 * creation/destruction jobs in case user forgot to
	 * dequeue. Or the enqueued created flows will be
	 * leaked. The forgotten dequeues would also cause
	 * flow flush get extra CQEs as expected and pending_rules
	 * be minus value.
	 */
	for (queue = 0; queue < priv->nb_queue; queue++) {
		if (__flow_hw_pull_comp(dev, queue, error))
			return -1;
	}
	/* Flush flow per-table from MLX5_DEFAULT_FLUSH_QUEUE. */
	LIST_FOREACH(tbl, &priv->flow_hw_tbl, next) {
		if (!tbl->cfg.external)
			continue;
		MLX5_IPOOL_FOREACH(tbl->flow, fidx, flow) {
			if (flow_hw_async_flow_destroy(dev,
						MLX5_DEFAULT_FLUSH_QUEUE,
						&attr,
						(struct rte_flow *)flow,
						NULL,
						error))
				return -1;
			pending_rules++;
			/* Drain completion with queue size. */
			if (pending_rules >= hw_q->size) {
				if (__flow_hw_pull_comp(dev,
							MLX5_DEFAULT_FLUSH_QUEUE,
							error))
					return -1;
				pending_rules = 0;
			}
		}
	}
	/* Drain left completion. */
	if (pending_rules &&
	    __flow_hw_pull_comp(dev, MLX5_DEFAULT_FLUSH_QUEUE, error))
		return -1;
	return 0;
}

static int
mlx5_tbl_multi_pattern_process(struct rte_eth_dev *dev,
			       struct rte_flow_template_table *tbl,
			       struct mlx5_tbl_multi_pattern_ctx *mpat,
			       struct rte_flow_error *error)
{
	uint32_t i;
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_template_table_attr *table_attr = &tbl->cfg.attr;
	const struct rte_flow_attr *attr = &table_attr->flow_attr;
	enum mlx5dr_table_type type = get_mlx5dr_table_type(attr);
	uint32_t flags = mlx5_hw_act_flag[!!attr->group][type];
	struct mlx5dr_action *dr_action;
	uint32_t bulk_size = rte_log2_u32(table_attr->nb_flows);

	for (i = 0; i < MLX5_MULTIPATTERN_ENCAP_NUM; i++) {
		uint32_t j;
		uint32_t *reformat_refcnt;
		typeof(mpat->reformat[0]) *reformat = mpat->reformat + i;
		struct mlx5dr_action_reformat_header hdr[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
		enum mlx5dr_action_type reformat_type =
			mlx5_multi_pattern_reformat_index_to_type(i);

		if (!reformat->elements_num)
			continue;
		for (j = 0; j < reformat->elements_num; j++)
			hdr[j] = reformat->ctx[j].reformat_hdr;
		reformat_refcnt = mlx5_malloc(MLX5_MEM_ZERO, sizeof(uint32_t), 0,
					      rte_socket_id());
		if (!reformat_refcnt)
			return rte_flow_error_set(error, ENOMEM,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL, "failed to allocate multi-pattern encap counter");
		*reformat_refcnt = reformat->elements_num;
		dr_action = mlx5dr_action_create_reformat
			(priv->dr_ctx, reformat_type, reformat->elements_num, hdr,
			 bulk_size, flags);
		if (!dr_action) {
			mlx5_free(reformat_refcnt);
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL,
						  "failed to create multi-pattern encap action");
		}
		for (j = 0; j < reformat->elements_num; j++) {
			reformat->ctx[j].rule_action->action = dr_action;
			reformat->ctx[j].encap->action = dr_action;
			reformat->ctx[j].encap->multi_pattern = 1;
			reformat->ctx[j].encap->multi_pattern_refcnt = reformat_refcnt;
		}
	}
	if (mpat->mh.elements_num) {
		typeof(mpat->mh) *mh = &mpat->mh;
		struct mlx5dr_action_mh_pattern pattern[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
		uint32_t *mh_refcnt = mlx5_malloc(MLX5_MEM_ZERO, sizeof(uint32_t),
						 0, rte_socket_id());

		if (!mh_refcnt)
			return rte_flow_error_set(error, ENOMEM,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL, "failed to allocate modify header counter");
		*mh_refcnt = mpat->mh.elements_num;
		for (i = 0; i < mpat->mh.elements_num; i++)
			pattern[i] = mh->ctx[i].mh_pattern;
		dr_action = mlx5dr_action_create_modify_header
			(priv->dr_ctx, mpat->mh.elements_num, pattern,
			 bulk_size, flags);
		if (!dr_action) {
			mlx5_free(mh_refcnt);
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL,
						  "failed to create multi-pattern header modify action");
		}
		for (i = 0; i < mpat->mh.elements_num; i++) {
			mh->ctx[i].rule_action->action = dr_action;
			mh->ctx[i].mhdr->action = dr_action;
			mh->ctx[i].mhdr->multi_pattern = 1;
			mh->ctx[i].mhdr->multi_pattern_refcnt = mh_refcnt;
		}
	}

	return 0;
}

static int
mlx5_hw_build_template_table(struct rte_eth_dev *dev,
			     uint8_t nb_action_templates,
			     struct rte_flow_actions_template *action_templates[],
			     struct mlx5dr_action_template *at[],
			     struct rte_flow_template_table *tbl,
			     struct rte_flow_error *error)
{
	int ret;
	uint8_t i;
	struct mlx5_tbl_multi_pattern_ctx mpat = MLX5_EMPTY_MULTI_PATTERN_CTX;

	for (i = 0; i < nb_action_templates; i++) {
		uint32_t refcnt = __atomic_add_fetch(&action_templates[i]->refcnt, 1,
						     __ATOMIC_RELAXED);

		if (refcnt <= 1) {
			rte_flow_error_set(error, EINVAL,
					   RTE_FLOW_ERROR_TYPE_ACTION,
					   &action_templates[i], "invalid AT refcount");
			goto at_error;
		}
		at[i] = action_templates[i]->tmpl;
		tbl->ats[i].action_template = action_templates[i];
		LIST_INIT(&tbl->ats[i].acts.act_list);
		/* do NOT translate table action if `dev` was not started */
		if (!dev->data->dev_started)
			continue;
		ret = __flow_hw_actions_translate(dev, &tbl->cfg,
						  &tbl->ats[i].acts,
						  action_templates[i],
						  &mpat, error);
		if (ret) {
			i++;
			goto at_error;
		}
	}
	tbl->nb_action_templates = nb_action_templates;
	ret = mlx5_tbl_multi_pattern_process(dev, tbl, &mpat, error);
	if (ret)
		goto at_error;
	return 0;

at_error:
	while (i--) {
		__flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
		__atomic_sub_fetch(&action_templates[i]->refcnt,
				   1, __ATOMIC_RELAXED);
	}
	return rte_errno;
}

/**
 * Create flow table.
 *
 * The input item and action templates will be binded to the table.
 * Flow memory will also be allocated. Matcher will be created based
 * on the item template. Action will be translated to the dedicated
 * DR action if possible.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] table_cfg
 *   Pointer to the table configuration.
 * @param[in] item_templates
 *   Item template array to be binded to the table.
 * @param[in] nb_item_templates
 *   Number of item template.
 * @param[in] action_templates
 *   Action template array to be binded to the table.
 * @param[in] nb_action_templates
 *   Number of action template.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Table on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow_template_table *
flow_hw_table_create(struct rte_eth_dev *dev,
		     const struct mlx5_flow_template_table_cfg *table_cfg,
		     struct rte_flow_pattern_template *item_templates[],
		     uint8_t nb_item_templates,
		     struct rte_flow_actions_template *action_templates[],
		     uint8_t nb_action_templates,
		     struct rte_flow_error *error)
{
	struct rte_flow_error sub_error = {
		.type = RTE_FLOW_ERROR_TYPE_NONE,
		.cause = NULL,
		.message = NULL,
	};
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_matcher_attr matcher_attr = {0};
	struct rte_flow_template_table *tbl = NULL;
	struct mlx5_flow_group *grp;
	struct mlx5dr_match_template *mt[MLX5_HW_TBL_MAX_ITEM_TEMPLATE];
	struct mlx5dr_action_template *at[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
	const struct rte_flow_template_table_attr *attr = &table_cfg->attr;
	struct rte_flow_attr flow_attr = attr->flow_attr;
	struct mlx5_flow_cb_ctx ctx = {
		.dev = dev,
		.error = &sub_error,
		.data = &flow_attr,
	};
	struct mlx5_indexed_pool_config cfg = {
		.size = sizeof(struct rte_flow_hw) + mlx5dr_rule_get_handle_size(),
		.trunk_size = 1 << 12,
		.per_core_cache = 1 << 13,
		.need_lock = 1,
		.release_mem_en = !!priv->sh->config.reclaim_mode,
		.malloc = mlx5_malloc,
		.free = mlx5_free,
		.type = "mlx5_hw_table_flow",
	};
	struct mlx5_list_entry *ge;
	uint32_t i = 0, max_tpl = MLX5_HW_TBL_MAX_ITEM_TEMPLATE;
	uint32_t nb_flows = rte_align32pow2(attr->nb_flows);
	bool port_started = !!dev->data->dev_started;
	int err;

	/* HWS layer accepts only 1 item template with root table. */
	if (!attr->flow_attr.group)
		max_tpl = 1;
	cfg.max_idx = nb_flows;
	/* For table has very limited flows, disable cache. */
	if (nb_flows < cfg.trunk_size) {
		cfg.per_core_cache = 0;
		cfg.trunk_size = nb_flows;
	} else if (nb_flows <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
		cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
	}
	/* Check if we requires too many templates. */
	if (nb_item_templates > max_tpl ||
	    nb_action_templates > MLX5_HW_TBL_MAX_ACTION_TEMPLATE) {
		rte_errno = EINVAL;
		goto error;
	}
	/* Allocate the table memory. */
	tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl), 0, rte_socket_id());
	if (!tbl)
		goto error;
	tbl->cfg = *table_cfg;
	/* Allocate flow indexed pool. */
	tbl->flow = mlx5_ipool_create(&cfg);
	if (!tbl->flow)
		goto error;
	/* Allocate rule indexed pool. */
	cfg.size = 0;
	cfg.type = "mlx5_hw_table_rule";
	cfg.max_idx += priv->hw_q[0].size;
	tbl->resource = mlx5_ipool_create(&cfg);
	if (!tbl->resource)
		goto error;
	/* Register the flow group. */
	ge = mlx5_hlist_register(priv->sh->groups, attr->flow_attr.group, &ctx);
	if (!ge)
		goto error;
	grp = container_of(ge, struct mlx5_flow_group, entry);
	tbl->grp = grp;
	/* Prepare matcher information. */
	matcher_attr.optimize_flow_src = MLX5DR_MATCHER_FLOW_SRC_ANY;
	matcher_attr.priority = attr->flow_attr.priority;
	matcher_attr.optimize_using_rule_idx = true;
	matcher_attr.mode = MLX5DR_MATCHER_RESOURCE_MODE_RULE;
	matcher_attr.insert_mode = flow_hw_matcher_insert_mode_get(attr->insertion_type);
	if (attr->hash_func == RTE_FLOW_TABLE_HASH_FUNC_CRC16) {
		DRV_LOG(ERR, "16-bit checksum hash type is not supported");
		rte_errno = ENOTSUP;
		goto it_error;
	}
	matcher_attr.distribute_mode = flow_hw_matcher_distribute_mode_get(attr->hash_func);
	matcher_attr.rule.num_log = rte_log2_u32(nb_flows);
	/* Parse hints information. */
	if (attr->specialize) {
		if (attr->specialize == RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG)
			matcher_attr.optimize_flow_src = MLX5DR_MATCHER_FLOW_SRC_WIRE;
		else if (attr->specialize == RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG)
			matcher_attr.optimize_flow_src = MLX5DR_MATCHER_FLOW_SRC_VPORT;
		else
			DRV_LOG(INFO, "Unsupported hint value %x", attr->specialize);
	}
	/* Build the item template. */
	for (i = 0; i < nb_item_templates; i++) {
		uint32_t ret;

		if ((flow_attr.ingress && !item_templates[i]->attr.ingress) ||
		    (flow_attr.egress && !item_templates[i]->attr.egress) ||
		    (flow_attr.transfer && !item_templates[i]->attr.transfer)) {
			DRV_LOG(ERR, "pattern template and template table attribute mismatch");
			rte_errno = EINVAL;
			goto it_error;
		}
		if (item_templates[i]->item_flags & MLX5_FLOW_ITEM_COMPARE)
			matcher_attr.mode = MLX5DR_MATCHER_RESOURCE_MODE_HTABLE;
		ret = __atomic_fetch_add(&item_templates[i]->refcnt, 1,
					 __ATOMIC_RELAXED) + 1;
		if (ret <= 1) {
			rte_errno = EINVAL;
			goto it_error;
		}
		mt[i] = item_templates[i]->mt;
		tbl->its[i] = item_templates[i];
	}
	tbl->nb_item_templates = nb_item_templates;
	/* Build the action template. */
	err = mlx5_hw_build_template_table(dev, nb_action_templates,
					   action_templates, at, tbl, &sub_error);
	if (err) {
		i = nb_item_templates;
		goto it_error;
	}
	tbl->matcher = mlx5dr_matcher_create
		(tbl->grp->tbl, mt, nb_item_templates, at, nb_action_templates, &matcher_attr);
	if (!tbl->matcher)
		goto at_error;
	tbl->type = attr->flow_attr.transfer ? MLX5DR_TABLE_TYPE_FDB :
		    (attr->flow_attr.egress ? MLX5DR_TABLE_TYPE_NIC_TX :
		    MLX5DR_TABLE_TYPE_NIC_RX);
	if (port_started)
		LIST_INSERT_HEAD(&priv->flow_hw_tbl, tbl, next);
	else
		LIST_INSERT_HEAD(&priv->flow_hw_tbl_ongo, tbl, next);
	return tbl;
at_error:
	for (i = 0; i < nb_action_templates; i++) {
		__flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
		__atomic_fetch_sub(&action_templates[i]->refcnt,
				   1, __ATOMIC_RELAXED);
	}
	i = nb_item_templates;
it_error:
	while (i--)
		__atomic_fetch_sub(&item_templates[i]->refcnt,
				   1, __ATOMIC_RELAXED);
error:
	err = rte_errno;
	if (tbl) {
		if (tbl->grp)
			mlx5_hlist_unregister(priv->sh->groups,
					      &tbl->grp->entry);
		if (tbl->resource)
			mlx5_ipool_destroy(tbl->resource);
		if (tbl->flow)
			mlx5_ipool_destroy(tbl->flow);
		mlx5_free(tbl);
	}
	if (error != NULL) {
		if (sub_error.type == RTE_FLOW_ERROR_TYPE_NONE)
			rte_flow_error_set(error, err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					   "Failed to create template table");
		else
			rte_memcpy(error, &sub_error, sizeof(sub_error));
	}
	return NULL;
}

/**
 * Update flow template table.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
int
flow_hw_table_update(struct rte_eth_dev *dev,
		     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_template_table *tbl;

	while ((tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo)) != NULL) {
		if (flow_hw_actions_translate(dev, tbl, error))
			return -1;
		LIST_REMOVE(tbl, next);
		LIST_INSERT_HEAD(&priv->flow_hw_tbl, tbl, next);
	}
	return 0;
}

/**
 * Translates group index specified by the user in @p attr to internal
 * group index.
 *
 * Translation is done by incrementing group index, so group n becomes n + 1.
 *
 * @param[in] dev
 *   Pointer to Ethernet device.
 * @param[in] cfg
 *   Pointer to the template table configuration.
 * @param[in] group
 *   Currently used group index (table group or jump destination).
 * @param[out] table_group
 *   Pointer to output group index.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success. Otherwise, returns negative error code, rte_errno is set
 *   and error structure is filled.
 */
static int
flow_hw_translate_group(struct rte_eth_dev *dev,
			const struct mlx5_flow_template_table_cfg *cfg,
			uint32_t group,
			uint32_t *table_group,
			struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_sh_config *config = &priv->sh->config;
	const struct rte_flow_attr *flow_attr = &cfg->attr.flow_attr;

	if (config->dv_esw_en &&
	    priv->fdb_def_rule &&
	    cfg->external &&
	    flow_attr->transfer) {
		if (group > MLX5_HW_MAX_TRANSFER_GROUP)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
						  NULL,
						  "group index not supported");
		*table_group = group + 1;
	} else if (config->dv_esw_en &&
		   (config->repr_matching || config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) &&
		   cfg->external &&
		   flow_attr->egress) {
		/*
		 * On E-Switch setups, default egress flow rules are inserted to allow
		 * representor matching and/or preserving metadata across steering domains.
		 * These flow rules are inserted in group 0 and this group is reserved by PMD
		 * for these purposes.
		 *
		 * As a result, if representor matching or extended metadata mode is enabled,
		 * group provided by the user must be incremented to avoid inserting flow rules
		 * in group 0.
		 */
		if (group > MLX5_HW_MAX_EGRESS_GROUP)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
						  NULL,
						  "group index not supported");
		*table_group = group + 1;
	} else {
		*table_group = group;
	}
	return 0;
}

/**
 * Create flow table.
 *
 * This function is a wrapper over @ref flow_hw_table_create(), which translates parameters
 * provided by user to proper internal values.
 *
 * @param[in] dev
 *   Pointer to Ethernet device.
 * @param[in] attr
 *   Pointer to the table attributes.
 * @param[in] item_templates
 *   Item template array to be binded to the table.
 * @param[in] nb_item_templates
 *   Number of item templates.
 * @param[in] action_templates
 *   Action template array to be binded to the table.
 * @param[in] nb_action_templates
 *   Number of action templates.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   Table on success, Otherwise, returns negative error code, rte_errno is set
 *   and error structure is filled.
 */
static struct rte_flow_template_table *
flow_hw_template_table_create(struct rte_eth_dev *dev,
			      const struct rte_flow_template_table_attr *attr,
			      struct rte_flow_pattern_template *item_templates[],
			      uint8_t nb_item_templates,
			      struct rte_flow_actions_template *action_templates[],
			      uint8_t nb_action_templates,
			      struct rte_flow_error *error)
{
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = *attr,
		.external = true,
	};
	uint32_t group = attr->flow_attr.group;

	if (flow_hw_translate_group(dev, &cfg, group, &cfg.attr.flow_attr.group, error))
		return NULL;
	return flow_hw_table_create(dev, &cfg, item_templates, nb_item_templates,
				    action_templates, nb_action_templates, error);
}

/**
 * Destroy flow table.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] table
 *   Pointer to the table to be destroyed.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_table_destroy(struct rte_eth_dev *dev,
		      struct rte_flow_template_table *table,
		      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int i;
	uint32_t fidx = 1;
	uint32_t ridx = 1;

	/* Build ipool allocated object bitmap. */
	mlx5_ipool_flush_cache(table->resource);
	mlx5_ipool_flush_cache(table->flow);
	/* Check if ipool has allocated objects. */
	if (table->refcnt ||
	    mlx5_ipool_get_next(table->flow, &fidx) ||
	    mlx5_ipool_get_next(table->resource, &ridx)) {
		DRV_LOG(WARNING, "Table %p is still in use.", (void *)table);
		return rte_flow_error_set(error, EBUSY,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "table in use");
	}
	LIST_REMOVE(table, next);
	for (i = 0; i < table->nb_item_templates; i++)
		__atomic_fetch_sub(&table->its[i]->refcnt,
				   1, __ATOMIC_RELAXED);
	for (i = 0; i < table->nb_action_templates; i++) {
		__flow_hw_action_template_destroy(dev, &table->ats[i].acts);
		__atomic_fetch_sub(&table->ats[i].action_template->refcnt,
				   1, __ATOMIC_RELAXED);
	}
	mlx5dr_matcher_destroy(table->matcher);
	mlx5_hlist_unregister(priv->sh->groups, &table->grp->entry);
	mlx5_ipool_destroy(table->resource);
	mlx5_ipool_destroy(table->flow);
	mlx5_free(table);
	return 0;
}

/**
 * Parse group's miss actions.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] cfg
 *   Pointer to the table_cfg structure.
 * @param[in] actions
 *   Array of actions to perform on group miss. Supported types:
 *   RTE_FLOW_ACTION_TYPE_JUMP, RTE_FLOW_ACTION_TYPE_VOID, RTE_FLOW_ACTION_TYPE_END.
 * @param[out] dst_group_id
 *   Pointer to destination group id output. will be set to 0 if actions is END,
 *   otherwise will be set to destination group id.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */

static int
flow_hw_group_parse_miss_actions(struct rte_eth_dev *dev,
				 struct mlx5_flow_template_table_cfg *cfg,
				 const struct rte_flow_action actions[],
				 uint32_t *dst_group_id,
				 struct rte_flow_error *error)
{
	const struct rte_flow_action_jump *jump_conf;
	uint32_t temp = 0;
	uint32_t i;

	for (i = 0; actions[i].type != RTE_FLOW_ACTION_TYPE_END; i++) {
		switch (actions[i].type) {
		case RTE_FLOW_ACTION_TYPE_VOID:
			continue;
		case RTE_FLOW_ACTION_TYPE_JUMP:
			if (temp)
				return rte_flow_error_set(error, ENOTSUP,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, actions,
							  "Miss actions can contain only a single JUMP");

			jump_conf = (const struct rte_flow_action_jump *)actions[i].conf;
			if (!jump_conf)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  jump_conf, "Jump conf must not be NULL");

			if (flow_hw_translate_group(dev, cfg, jump_conf->group, &temp, error))
				return -rte_errno;

			if (!temp)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
							  "Failed to set group miss actions - Invalid target group");
			break;
		default:
			return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
						  &actions[i], "Unsupported default miss action type");
		}
	}

	*dst_group_id = temp;
	return 0;
}

/**
 * Set group's miss group.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] cfg
 *   Pointer to the table_cfg structure.
 * @param[in] src_grp
 *   Pointer to source group structure.
 *   if NULL, a new group will be created based on group id from cfg->attr.flow_attr.group.
 * @param[in] dst_grp
 *   Pointer to destination group structure.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */

static int
flow_hw_group_set_miss_group(struct rte_eth_dev *dev,
			     struct mlx5_flow_template_table_cfg *cfg,
			     struct mlx5_flow_group *src_grp,
			     struct mlx5_flow_group *dst_grp,
			     struct rte_flow_error *error)
{
	struct rte_flow_error sub_error = {
		.type = RTE_FLOW_ERROR_TYPE_NONE,
		.cause = NULL,
		.message = NULL,
	};
	struct mlx5_flow_cb_ctx ctx = {
		.dev = dev,
		.error = &sub_error,
		.data = &cfg->attr.flow_attr,
	};
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_list_entry *ge;
	bool ref = false;
	int ret;

	if (!dst_grp)
		return -EINVAL;

	/* If group doesn't exist - needs to be created. */
	if (!src_grp) {
		ge = mlx5_hlist_register(priv->sh->groups, cfg->attr.flow_attr.group, &ctx);
		if (!ge)
			return -rte_errno;

		src_grp = container_of(ge, struct mlx5_flow_group, entry);
		LIST_INSERT_HEAD(&priv->flow_hw_grp, src_grp, next);
		ref = true;
	} else if (!src_grp->miss_group) {
		/* If group exists, but has no miss actions - need to increase ref_cnt. */
		LIST_INSERT_HEAD(&priv->flow_hw_grp, src_grp, next);
		src_grp->entry.ref_cnt++;
		ref = true;
	}

	ret = mlx5dr_table_set_default_miss(src_grp->tbl, dst_grp->tbl);
	if (ret)
		goto mlx5dr_error;

	/* If group existed and had old miss actions - ref_cnt is already correct.
	 * However, need to reduce ref counter for old miss group.
	 */
	if (src_grp->miss_group)
		mlx5_hlist_unregister(priv->sh->groups, &src_grp->miss_group->entry);

	src_grp->miss_group = dst_grp;
	return 0;

mlx5dr_error:
	/* Reduce src_grp ref_cnt back & remove from grp list in case of mlx5dr error */
	if (ref) {
		mlx5_hlist_unregister(priv->sh->groups, &src_grp->entry);
		LIST_REMOVE(src_grp, next);
	}

	return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				  "Failed to set group miss actions");
}

/**
 * Unset group's miss group.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] grp
 *   Pointer to group structure.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */

static int
flow_hw_group_unset_miss_group(struct rte_eth_dev *dev,
			       struct mlx5_flow_group *grp,
			       struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int ret;

	/* If group doesn't exist - no need to change anything. */
	if (!grp)
		return 0;

	/* If group exists, but miss actions is already default behavior -
	 * no need to change anything.
	 */
	if (!grp->miss_group)
		return 0;

	ret = mlx5dr_table_set_default_miss(grp->tbl, NULL);
	if (ret)
		return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "Failed to unset group miss actions");

	mlx5_hlist_unregister(priv->sh->groups, &grp->miss_group->entry);
	grp->miss_group = NULL;

	LIST_REMOVE(grp, next);
	mlx5_hlist_unregister(priv->sh->groups, &grp->entry);

	return 0;
}

/**
 * Set group miss actions.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] group_id
 *   Group id.
 * @param[in] attr
 *   Pointer to group attributes structure.
 * @param[in] actions
 *   Array of actions to perform on group miss. Supported types:
 *   RTE_FLOW_ACTION_TYPE_JUMP, RTE_FLOW_ACTION_TYPE_VOID, RTE_FLOW_ACTION_TYPE_END.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */

static int
flow_hw_group_set_miss_actions(struct rte_eth_dev *dev,
			       uint32_t group_id,
			       const struct rte_flow_group_attr *attr,
			       const struct rte_flow_action actions[],
			       struct rte_flow_error *error)
{
	struct rte_flow_error sub_error = {
		.type = RTE_FLOW_ERROR_TYPE_NONE,
		.cause = NULL,
		.message = NULL,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.external = true,
		.attr = {
			.flow_attr = {
				.group = group_id,
				.ingress = attr->ingress,
				.egress = attr->egress,
				.transfer = attr->transfer,
			},
		},
	};
	struct mlx5_flow_cb_ctx ctx = {
		.dev = dev,
		.error = &sub_error,
		.data = &cfg.attr.flow_attr,
	};
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_group *src_grp = NULL;
	struct mlx5_flow_group *dst_grp = NULL;
	struct mlx5_list_entry *ge;
	uint32_t dst_group_id = 0;
	int ret;

	if (flow_hw_translate_group(dev, &cfg, group_id, &group_id, error))
		return -rte_errno;

	if (!group_id)
		return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "Failed to set group miss actions - invalid group id");

	ret = flow_hw_group_parse_miss_actions(dev, &cfg, actions, &dst_group_id, error);
	if (ret)
		return -rte_errno;

	if (dst_group_id == group_id) {
		return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "Failed to set group miss actions - target group id must differ from group_id");
	}

	cfg.attr.flow_attr.group = group_id;
	ge = mlx5_hlist_lookup(priv->sh->groups, group_id, &ctx);
	if (ge)
		src_grp = container_of(ge, struct mlx5_flow_group, entry);

	if (dst_group_id) {
		/* Increase ref_cnt for new miss group. */
		cfg.attr.flow_attr.group = dst_group_id;
		ge = mlx5_hlist_register(priv->sh->groups, dst_group_id, &ctx);
		if (!ge)
			return -rte_errno;

		dst_grp = container_of(ge, struct mlx5_flow_group, entry);

		cfg.attr.flow_attr.group = group_id;
		ret = flow_hw_group_set_miss_group(dev, &cfg, src_grp, dst_grp, error);
		if (ret)
			goto error;
	} else {
		return flow_hw_group_unset_miss_group(dev, src_grp, error);
	}

	return 0;

error:
	if (dst_grp)
		mlx5_hlist_unregister(priv->sh->groups, &dst_grp->entry);
	return -rte_errno;
}

static bool
flow_hw_modify_field_is_used(const struct rte_flow_action_modify_field *action,
			     enum rte_flow_field_id field)
{
	return action->src.field == field || action->dst.field == field;
}

static bool
flow_hw_modify_field_is_geneve_opt(enum rte_flow_field_id field)
{
	return field == RTE_FLOW_FIELD_GENEVE_OPT_TYPE ||
	       field == RTE_FLOW_FIELD_GENEVE_OPT_CLASS ||
	       field == RTE_FLOW_FIELD_GENEVE_OPT_DATA;
}

static bool
flow_hw_modify_field_is_add_dst_valid(const struct rte_flow_action_modify_field *conf)
{
	if (conf->operation != RTE_FLOW_MODIFY_ADD)
		return true;
	if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
	    conf->src.field == RTE_FLOW_FIELD_VALUE)
		return true;
	switch (conf->dst.field) {
	case RTE_FLOW_FIELD_IPV4_TTL:
	case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
	case RTE_FLOW_FIELD_TCP_SEQ_NUM:
	case RTE_FLOW_FIELD_TCP_ACK_NUM:
	case RTE_FLOW_FIELD_TAG:
	case RTE_FLOW_FIELD_META:
	case RTE_FLOW_FIELD_FLEX_ITEM:
	case RTE_FLOW_FIELD_TCP_DATA_OFFSET:
	case RTE_FLOW_FIELD_IPV4_IHL:
	case RTE_FLOW_FIELD_IPV4_TOTAL_LEN:
	case RTE_FLOW_FIELD_IPV6_PAYLOAD_LEN:
		return true;
	default:
		break;
	}
	return false;
}

static int
flow_hw_validate_action_modify_field(struct rte_eth_dev *dev,
				     const struct rte_flow_action *action,
				     const struct rte_flow_action *mask,
				     struct rte_flow_error *error)
{
	const struct rte_flow_action_modify_field *action_conf = action->conf;
	const struct rte_flow_action_modify_field *mask_conf = mask->conf;
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hca_attr *attr = &priv->sh->cdev->config.hca_attr;
	int ret;

	if (!mask_conf)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "modify_field mask conf is missing");
	if (action_conf->operation != mask_conf->operation)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modify_field operation mask and template are not equal");
	if (action_conf->dst.field != mask_conf->dst.field)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"destination field mask and template are not equal");
	if (action_conf->dst.field == RTE_FLOW_FIELD_POINTER ||
	    action_conf->dst.field == RTE_FLOW_FIELD_VALUE ||
	    action_conf->dst.field == RTE_FLOW_FIELD_HASH_RESULT)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"immediate value, pointer and hash result cannot be used as destination");
	ret = flow_validate_modify_field_level(&action_conf->dst, error);
	if (ret)
		return ret;
	if (action_conf->dst.field != RTE_FLOW_FIELD_FLEX_ITEM &&
	    !flow_hw_modify_field_is_geneve_opt(action_conf->dst.field)) {
		if (action_conf->dst.tag_index &&
		    !flow_modify_field_support_tag_array(action_conf->dst.field))
			return rte_flow_error_set(error, EINVAL,
					RTE_FLOW_ERROR_TYPE_ACTION, action,
					"destination tag index is not supported");
		if (action_conf->dst.class_id)
			return rte_flow_error_set(error, EINVAL,
					RTE_FLOW_ERROR_TYPE_ACTION, action,
					"destination class id is not supported");
	}
	if (mask_conf->dst.level != UINT8_MAX)
		return rte_flow_error_set(error, EINVAL,
			RTE_FLOW_ERROR_TYPE_ACTION, action,
			"destination encapsulation level must be fully masked");
	if (mask_conf->dst.offset != UINT32_MAX)
		return rte_flow_error_set(error, EINVAL,
			RTE_FLOW_ERROR_TYPE_ACTION, action,
			"destination offset level must be fully masked");
	if (action_conf->src.field != mask_conf->src.field)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"destination field mask and template are not equal");
	if (action_conf->src.field != RTE_FLOW_FIELD_POINTER &&
	    action_conf->src.field != RTE_FLOW_FIELD_VALUE) {
		if (action_conf->src.field != RTE_FLOW_FIELD_FLEX_ITEM &&
		    !flow_hw_modify_field_is_geneve_opt(action_conf->src.field)) {
			if (action_conf->src.tag_index &&
			    !flow_modify_field_support_tag_array(action_conf->src.field))
				return rte_flow_error_set(error, EINVAL,
					RTE_FLOW_ERROR_TYPE_ACTION, action,
					"source tag index is not supported");
			if (action_conf->src.class_id)
				return rte_flow_error_set(error, EINVAL,
					RTE_FLOW_ERROR_TYPE_ACTION, action,
					"source class id is not supported");
		}
		if (mask_conf->src.level != UINT8_MAX)
			return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"source encapsulation level must be fully masked");
		if (mask_conf->src.offset != UINT32_MAX)
			return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"source offset level must be fully masked");
		ret = flow_validate_modify_field_level(&action_conf->src, error);
		if (ret)
			return ret;
	}
	if ((action_conf->dst.field == RTE_FLOW_FIELD_TAG &&
	     action_conf->dst.tag_index >= MLX5_FLOW_HW_TAGS_MAX &&
	     action_conf->dst.tag_index != RTE_PMD_MLX5_LINEAR_HASH_TAG_INDEX) ||
	    (action_conf->src.field == RTE_FLOW_FIELD_TAG &&
	     action_conf->src.tag_index >= MLX5_FLOW_HW_TAGS_MAX &&
	     action_conf->src.tag_index != RTE_PMD_MLX5_LINEAR_HASH_TAG_INDEX))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
				 "tag index is out of range");
	if ((action_conf->dst.field == RTE_FLOW_FIELD_TAG &&
	     flow_hw_get_reg_id(dev, RTE_FLOW_ITEM_TYPE_TAG, action_conf->dst.tag_index) == REG_NON) ||
	    (action_conf->src.field == RTE_FLOW_FIELD_TAG &&
	     flow_hw_get_reg_id(dev, RTE_FLOW_ITEM_TYPE_TAG, action_conf->src.tag_index) == REG_NON))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "tag index is out of range");
	if (mask_conf->width != UINT32_MAX)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modify_field width field must be fully masked");
	if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_START))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modifying arbitrary place in a packet is not supported");
	if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_VLAN_TYPE))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modifying vlan_type is not supported");
	if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_RANDOM))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modifying random value is not supported");
	/**
	 * Geneve VNI modification is supported only when Geneve header is
	 * parsed natively. When GENEVE options are supported, they both Geneve
	 * and options headers are parsed as a flex parser.
	 */
	if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_GENEVE_VNI) &&
	    attr->geneve_tlv_opt)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modifying Geneve VNI is not supported when GENEVE opt is supported");
	if (priv->tlv_options == NULL &&
	    (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_GENEVE_OPT_TYPE) ||
	     flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_GENEVE_OPT_CLASS) ||
	     flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_GENEVE_OPT_DATA)))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modifying Geneve TLV option is supported only after parser configuration");
	/* Due to HW bug, tunnel MPLS header is read only. */
	if (action_conf->dst.field == RTE_FLOW_FIELD_MPLS)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"MPLS cannot be used as destination");
	/* ADD_FIELD is not supported for all the fields. */
	if (!flow_hw_modify_field_is_add_dst_valid(action_conf))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"invalid add_field destination");
	return 0;
}
static int
flow_hw_validate_action_port_representor(struct rte_eth_dev *dev __rte_unused,
					 const struct rte_flow_actions_template_attr *attr,
					 const struct rte_flow_action *action,
					 const struct rte_flow_action *mask,
					 struct rte_flow_error *error)
{
	const struct rte_flow_action_ethdev *action_conf = NULL;
	const struct rte_flow_action_ethdev *mask_conf = NULL;

	/* If transfer is set, port has been validated as proxy port. */
	if (!attr->transfer)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "cannot use port_representor actions"
					  " without an E-Switch");
	if (!action || !mask)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "actiona and mask configuration must be set");
	action_conf = action->conf;
	mask_conf = mask->conf;
	if (!mask_conf || mask_conf->port_id != MLX5_REPRESENTED_PORT_ESW_MGR ||
	    !action_conf || action_conf->port_id != MLX5_REPRESENTED_PORT_ESW_MGR)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "only eswitch manager port 0xffff is"
					  " supported");
	return 0;
}

static int
flow_hw_validate_action_represented_port(struct rte_eth_dev *dev,
					 const struct rte_flow_action *action,
					 const struct rte_flow_action *mask,
					 struct rte_flow_error *error)
{
	const struct rte_flow_action_ethdev *action_conf = action->conf;
	const struct rte_flow_action_ethdev *mask_conf = mask->conf;
	struct mlx5_priv *priv = dev->data->dev_private;

	if (!priv->sh->config.dv_esw_en)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "cannot use represented_port actions"
					  " without an E-Switch");
	if (mask_conf && mask_conf->port_id) {
		struct mlx5_priv *port_priv;
		struct mlx5_priv *dev_priv;

		if (!action_conf)
			return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
						  action, "port index was not provided");
		port_priv = mlx5_port_to_eswitch_info(action_conf->port_id, false);
		if (!port_priv)
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "failed to obtain E-Switch"
						  " info for port");
		dev_priv = mlx5_dev_to_eswitch_info(dev);
		if (!dev_priv)
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "failed to obtain E-Switch"
						  " info for transfer proxy");
		if (port_priv->domain_id != dev_priv->domain_id)
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "cannot forward to port from"
						  " a different E-Switch");
	}
	return 0;
}

/**
 * Validate AGE action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] fixed_cnt
 *   Indicator if this list has a fixed COUNT action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_age(struct rte_eth_dev *dev,
			    const struct rte_flow_action *action,
			    uint64_t action_flags, bool fixed_cnt,
			    struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);

	if (!priv->sh->cdev->config.devx)
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "AGE action not supported");
	if (age_info->ages_ipool == NULL)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "aging pool not initialized");
	if ((action_flags & MLX5_FLOW_ACTION_AGE) ||
	    (action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "duplicate AGE actions set");
	if (fixed_cnt)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "AGE and fixed COUNT combination is not supported");
	return 0;
}

/**
 * Validate count action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[in] mask
 *   Pointer to the indirect action mask.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_count(struct rte_eth_dev *dev,
			      const struct rte_flow_action *action,
			      const struct rte_flow_action *mask,
			      uint64_t action_flags,
			      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_count *count = mask->conf;

	if (!priv->sh->cdev->config.devx)
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "count action not supported");
	if (!priv->hws_cpool)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "counters pool not initialized");
	if ((action_flags & MLX5_FLOW_ACTION_COUNT) ||
	    (action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "duplicate count actions set");
	if (count && count->id && (action_flags & MLX5_FLOW_ACTION_AGE))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, mask,
					  "AGE and COUNT action shared by mask combination is not supported");
	return 0;
}

/**
 * Validate meter_mark action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_meter_mark(struct rte_eth_dev *dev,
			      const struct rte_flow_action *action,
			      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	RTE_SET_USED(action);

	if (!priv->sh->cdev->config.devx)
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "meter_mark action not supported");
	if (!priv->hws_mpool)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "meter_mark pool not initialized");
	return 0;
}

/**
 * Validate indirect action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[in] mask
 *   Pointer to the indirect action mask.
 * @param[in, out] action_flags
 *   Holds the actions detected until now.
 * @param[in, out] fixed_cnt
 *   Pointer to indicator if this list has a fixed COUNT action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_indirect(struct rte_eth_dev *dev,
				 const struct rte_flow_action *action,
				 const struct rte_flow_action *mask,
				 uint64_t *action_flags, bool *fixed_cnt,
				 struct rte_flow_error *error)
{
	uint32_t type;
	int ret;

	if (!mask)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "Unable to determine indirect action type without a mask specified");
	type = mask->type;
	switch (type) {
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		ret = flow_hw_validate_action_meter_mark(dev, mask, error);
		if (ret < 0)
			return ret;
		*action_flags |= MLX5_FLOW_ACTION_METER;
		break;
	case RTE_FLOW_ACTION_TYPE_RSS:
		/* TODO: Validation logic (same as flow_hw_actions_validate) */
		*action_flags |= MLX5_FLOW_ACTION_RSS;
		break;
	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
		/* TODO: Validation logic (same as flow_hw_actions_validate) */
		*action_flags |= MLX5_FLOW_ACTION_CT;
		break;
	case RTE_FLOW_ACTION_TYPE_COUNT:
		if (action->conf && mask->conf) {
			if ((*action_flags & MLX5_FLOW_ACTION_AGE) ||
			    (*action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
				/*
				 * AGE cannot use indirect counter which is
				 * shared with enother flow rules.
				 */
				return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  NULL,
						  "AGE and fixed COUNT combination is not supported");
			*fixed_cnt = true;
		}
		ret = flow_hw_validate_action_count(dev, action, mask,
						    *action_flags, error);
		if (ret < 0)
			return ret;
		*action_flags |= MLX5_FLOW_ACTION_INDIRECT_COUNT;
		break;
	case RTE_FLOW_ACTION_TYPE_AGE:
		ret = flow_hw_validate_action_age(dev, action, *action_flags,
						  *fixed_cnt, error);
		if (ret < 0)
			return ret;
		*action_flags |= MLX5_FLOW_ACTION_INDIRECT_AGE;
		break;
	case RTE_FLOW_ACTION_TYPE_QUOTA:
		/* TODO: add proper quota verification */
		*action_flags |= MLX5_FLOW_ACTION_QUOTA;
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type: %d", type);
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, mask,
					  "Unsupported indirect action type");
	}
	return 0;
}

/**
 * Validate ipv6_ext_push action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_ipv6_ext_push(struct rte_eth_dev *dev __rte_unused,
				      const struct rte_flow_action *action,
				      struct rte_flow_error *error)
{
	const struct rte_flow_action_ipv6_ext_push *raw_push_data = action->conf;

	if (!raw_push_data || !raw_push_data->size || !raw_push_data->data)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "invalid ipv6_ext_push data");
	if (raw_push_data->type != IPPROTO_ROUTING ||
	    raw_push_data->size > MLX5_PUSH_MAX_LEN)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "Unsupported ipv6_ext_push type or length");
	return 0;
}

/**
 * Validate raw_encap action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_raw_encap(const struct rte_flow_action *action,
				  const struct rte_flow_action *mask,
				  struct rte_flow_error *error)
{
	const struct rte_flow_action_raw_encap *mask_conf = mask->conf;
	const struct rte_flow_action_raw_encap *action_conf = action->conf;

	if (!mask_conf || !mask_conf->size)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, mask,
					  "raw_encap: size must be masked");
	if (!action_conf || !action_conf->size)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "raw_encap: invalid action configuration");
	if (mask_conf->data && !action_conf->data)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "raw_encap: masked data is missing");
	return 0;
}

/**
 * Process `... / raw_decap / raw_encap / ...` actions sequence.
 * The PMD handles the sequence as a single encap or decap reformat action,
 * depending on the raw_encap configuration.
 *
 * The function assumes that the raw_decap / raw_encap location
 * in actions template list complies with relative HWS actions order:
 * for the required reformat configuration:
 * ENCAP configuration must appear before [JUMP|DROP|PORT]
 * DECAP configuration must appear at the template head.
 */
static uint64_t
mlx5_decap_encap_reformat_type(const struct rte_flow_action *actions,
			       uint32_t encap_ind, uint64_t flags)
{
	const struct rte_flow_action_raw_encap *encap = actions[encap_ind].conf;

	if ((flags & MLX5_FLOW_ACTION_DECAP) == 0)
		return MLX5_FLOW_ACTION_ENCAP;
	if (actions[encap_ind - 1].type != RTE_FLOW_ACTION_TYPE_RAW_DECAP)
		return MLX5_FLOW_ACTION_ENCAP;
	return encap->size >= MLX5_ENCAPSULATION_DECISION_SIZE ?
	       MLX5_FLOW_ACTION_ENCAP : MLX5_FLOW_ACTION_DECAP;
}

static inline uint16_t
flow_hw_template_expand_modify_field(struct rte_flow_action actions[],
				     struct rte_flow_action masks[],
				     const struct rte_flow_action *mf_actions,
				     const struct rte_flow_action *mf_masks,
				     uint64_t flags, uint32_t act_num,
				     uint32_t mf_num)
{
	uint32_t i, tail;

	MLX5_ASSERT(actions && masks);
	MLX5_ASSERT(mf_num > 0);
	if (flags & MLX5_FLOW_ACTION_MODIFY_FIELD) {
		/*
		 * Application action template already has Modify Field.
		 * It's location will be used in DR.
		 * Expanded MF action can be added before the END.
		 */
		i = act_num - 1;
		goto insert;
	}
	/**
	 * Locate the first action positioned BEFORE the new MF.
	 *
	 * Search for a place to insert modify header
	 * from the END action backwards:
	 * 1. END is always present in actions array
	 * 2. END location is always at action[act_num - 1]
	 * 3. END always positioned AFTER modify field location
	 *
	 * Relative actions order is the same for RX, TX and FDB.
	 *
	 * Current actions order (draft-3)
	 * @see action_order_arr[]
	 */
	for (i = act_num - 2; (int)i >= 0; i--) {
		enum rte_flow_action_type type = actions[i].type;
		uint64_t reformat_type;

		if (type == RTE_FLOW_ACTION_TYPE_INDIRECT)
			type = masks[i].type;
		switch (type) {
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
		case RTE_FLOW_ACTION_TYPE_DROP:
		case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
		case RTE_FLOW_ACTION_TYPE_JUMP:
		case RTE_FLOW_ACTION_TYPE_QUEUE:
		case RTE_FLOW_ACTION_TYPE_RSS:
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
		case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
		case RTE_FLOW_ACTION_TYPE_VOID:
		case RTE_FLOW_ACTION_TYPE_END:
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			reformat_type =
				mlx5_decap_encap_reformat_type(actions, i,
							       flags);
			if (reformat_type == MLX5_FLOW_ACTION_DECAP) {
				i++;
				goto insert;
			}
			if (actions[i - 1].type == RTE_FLOW_ACTION_TYPE_RAW_DECAP)
				i--;
			break;
		default:
			i++; /* new MF inserted AFTER actions[i] */
			goto insert;
		}
	}
	i = 0;
insert:
	tail = act_num - i; /* num action to move */
	memmove(actions + i + mf_num, actions + i, sizeof(actions[0]) * tail);
	memcpy(actions + i, mf_actions, sizeof(actions[0]) * mf_num);
	memmove(masks + i + mf_num, masks + i, sizeof(masks[0]) * tail);
	memcpy(masks + i, mf_masks, sizeof(masks[0]) * mf_num);
	return i;
}

static int
flow_hw_validate_action_push_vlan(struct rte_eth_dev *dev,
				  const
				  struct rte_flow_actions_template_attr *attr,
				  const struct rte_flow_action *action,
				  const struct rte_flow_action *mask,
				  struct rte_flow_error *error)
{
#define X_FIELD(ptr, t, f) (((ptr)->conf) && ((t *)((ptr)->conf))->f)

	const bool masked_push =
		X_FIELD(mask + MLX5_HW_VLAN_PUSH_TYPE_IDX,
			const struct rte_flow_action_of_push_vlan, ethertype);
	bool masked_param;

	/*
	 * Mandatory actions order:
	 * OF_PUSH_VLAN / OF_SET_VLAN_VID [ / OF_SET_VLAN_PCP ]
	 */
	RTE_SET_USED(dev);
	RTE_SET_USED(attr);
	/* Check that mark matches OF_PUSH_VLAN */
	if (mask[MLX5_HW_VLAN_PUSH_TYPE_IDX].type !=
	    RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION,
					  action, "OF_PUSH_VLAN: mask does not match");
	/* Check that the second template and mask items are SET_VLAN_VID */
	if (action[MLX5_HW_VLAN_PUSH_VID_IDX].type !=
	    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID ||
	    mask[MLX5_HW_VLAN_PUSH_VID_IDX].type !=
	    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION,
					  action, "OF_PUSH_VLAN: invalid actions order");
	masked_param = X_FIELD(mask + MLX5_HW_VLAN_PUSH_VID_IDX,
			       const struct rte_flow_action_of_set_vlan_vid,
			       vlan_vid);
	/*
	 * PMD requires OF_SET_VLAN_VID mask to must match OF_PUSH_VLAN
	 */
	if (masked_push ^ masked_param)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "OF_SET_VLAN_VID: mask does not match OF_PUSH_VLAN");
	if (is_of_vlan_pcp_present(action)) {
		if (mask[MLX5_HW_VLAN_PUSH_PCP_IDX].type !=
		     RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action, "OF_SET_VLAN_PCP: missing mask configuration");
		masked_param = X_FIELD(mask + MLX5_HW_VLAN_PUSH_PCP_IDX,
				       const struct
				       rte_flow_action_of_set_vlan_pcp,
				       vlan_pcp);
		/*
		 * PMD requires OF_SET_VLAN_PCP mask to must match OF_PUSH_VLAN
		 */
		if (masked_push ^ masked_param)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION, action,
						  "OF_SET_VLAN_PCP: mask does not match OF_PUSH_VLAN");
	}
	return 0;
#undef X_FIELD
}

static int
flow_hw_validate_action_default_miss(struct rte_eth_dev *dev,
				     const struct rte_flow_actions_template_attr *attr,
				     uint64_t action_flags,
				     struct rte_flow_error *error)
{
	/*
	 * The private DEFAULT_MISS action is used internally for LACP in control
	 * flows. So this validation can be ignored. It can be kept right now since
	 * the validation will be done only once.
	 */
	struct mlx5_priv *priv = dev->data->dev_private;

	if (!attr->ingress || attr->egress || attr->transfer)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "DEFAULT MISS is only supported in ingress.");
	if (!priv->hw_def_miss)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "DEFAULT MISS action does not exist.");
	if (action_flags & MLX5_FLOW_FATE_ACTIONS)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "DEFAULT MISS should be the only termination.");
	return 0;
}

static int
mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev,
			      const struct rte_flow_actions_template_attr *attr,
			      const struct rte_flow_action actions[],
			      const struct rte_flow_action masks[],
			      uint64_t *act_flags,
			      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_count *count_mask = NULL;
	bool fixed_cnt = false;
	uint64_t action_flags = 0;
	bool actions_end = false;
#ifdef HAVE_MLX5DV_DR_ACTION_CREATE_DEST_ROOT_TABLE
	int table_type;
#endif
	uint16_t i;
	int ret;
	const struct rte_flow_action_ipv6_ext_remove *remove_data;

	/* FDB actions are only valid to proxy port. */
	if (attr->transfer && (!priv->sh->config.dv_esw_en || !priv->master))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL,
					  "transfer actions are only valid to proxy port");
	for (i = 0; !actions_end; ++i) {
		const struct rte_flow_action *action = &actions[i];
		const struct rte_flow_action *mask = &masks[i];

		MLX5_ASSERT(i < MLX5_HW_MAX_ACTS);
		if (action->type != RTE_FLOW_ACTION_TYPE_INDIRECT &&
		    action->type != mask->type)
			return rte_flow_error_set(error, ENOTSUP,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "mask type does not match action type");
		switch ((int)action->type) {
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_INDIRECT_LIST:
			break;
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			ret = flow_hw_validate_action_indirect(dev, action,
							       mask,
							       &action_flags,
							       &fixed_cnt,
							       error);
			if (ret < 0)
				return ret;
			break;
		case RTE_FLOW_ACTION_TYPE_MARK:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_MARK;
			break;
		case RTE_FLOW_ACTION_TYPE_DROP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_DROP;
			break;
		case RTE_FLOW_ACTION_TYPE_JUMP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_JUMP;
			break;
#ifdef HAVE_MLX5DV_DR_ACTION_CREATE_DEST_ROOT_TABLE
		case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
			if (priv->shared_host)
				return rte_flow_error_set(error, ENOTSUP,
							  RTE_FLOW_ERROR_TYPE_ACTION,
							  action,
							  "action not supported in guest port");
			table_type = attr->ingress ? MLX5DR_TABLE_TYPE_NIC_RX :
				     ((attr->egress) ? MLX5DR_TABLE_TYPE_NIC_TX :
				     MLX5DR_TABLE_TYPE_FDB);
			if (!priv->hw_send_to_kernel[table_type])
				return rte_flow_error_set(error, ENOTSUP,
							  RTE_FLOW_ERROR_TYPE_ACTION,
							  action,
							  "action is not available");
			action_flags |= MLX5_FLOW_ACTION_SEND_TO_KERNEL;
			break;
#endif
		case RTE_FLOW_ACTION_TYPE_QUEUE:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_QUEUE;
			break;
		case RTE_FLOW_ACTION_TYPE_RSS:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_RSS;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_ENCAP;
			break;
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_ENCAP;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_DECAP;
			break;
		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_DECAP;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			ret = flow_hw_validate_action_raw_encap(action, mask, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_ENCAP;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_DECAP;
			break;
		case RTE_FLOW_ACTION_TYPE_IPV6_EXT_PUSH:
			ret = flow_hw_validate_action_ipv6_ext_push(dev, action, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_IPV6_ROUTING_PUSH;
			break;
		case RTE_FLOW_ACTION_TYPE_IPV6_EXT_REMOVE:
			remove_data = action->conf;
			/* Remove action must be shared. */
			if (remove_data->type != IPPROTO_ROUTING || !mask) {
				DRV_LOG(ERR, "Only supports shared IPv6 routing remove");
				return -EINVAL;
			}
			action_flags |= MLX5_FLOW_ACTION_IPV6_ROUTING_REMOVE;
			break;
		case RTE_FLOW_ACTION_TYPE_METER:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_METER;
			break;
		case RTE_FLOW_ACTION_TYPE_METER_MARK:
			ret = flow_hw_validate_action_meter_mark(dev, action,
								 error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_METER;
			break;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			ret = flow_hw_validate_action_modify_field(dev, action, mask,
								   error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
			break;
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
			ret = flow_hw_validate_action_represented_port
					(dev, action, mask, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_PORT_ID;
			break;
		case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
			ret = flow_hw_validate_action_port_representor
					(dev, attr, action, mask, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_PORT_REPRESENTOR;
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
			if (count_mask && count_mask->id)
				fixed_cnt = true;
			ret = flow_hw_validate_action_age(dev, action,
							  action_flags,
							  fixed_cnt, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_AGE;
			break;
		case RTE_FLOW_ACTION_TYPE_COUNT:
			ret = flow_hw_validate_action_count(dev, action, mask,
							    action_flags,
							    error);
			if (ret < 0)
				return ret;
			count_mask = mask->conf;
			action_flags |= MLX5_FLOW_ACTION_COUNT;
			break;
		case RTE_FLOW_ACTION_TYPE_CONNTRACK:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_CT;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
			action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
			action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			ret = flow_hw_validate_action_push_vlan
					(dev, attr, action, mask, error);
			if (ret != 0)
				return ret;
			i += is_of_vlan_pcp_present(action) ?
				MLX5_HW_VLAN_PUSH_PCP_IDX :
				MLX5_HW_VLAN_PUSH_VID_IDX;
			action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
			break;
		case RTE_FLOW_ACTION_TYPE_END:
			actions_end = true;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
			ret = flow_hw_validate_action_default_miss(dev, attr,
								   action_flags, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
			break;
		default:
			return rte_flow_error_set(error, ENOTSUP,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "action not supported in template API");
		}
	}
	if (act_flags != NULL)
		*act_flags = action_flags;
	return 0;
}

static int
flow_hw_actions_validate(struct rte_eth_dev *dev,
			 const struct rte_flow_actions_template_attr *attr,
			 const struct rte_flow_action actions[],
			 const struct rte_flow_action masks[],
			 struct rte_flow_error *error)
{
	return mlx5_flow_hw_actions_validate(dev, attr, actions, masks, NULL, error);
}


static enum mlx5dr_action_type mlx5_hw_dr_action_types[] = {
	[RTE_FLOW_ACTION_TYPE_MARK] = MLX5DR_ACTION_TYP_TAG,
	[RTE_FLOW_ACTION_TYPE_DROP] = MLX5DR_ACTION_TYP_DROP,
	[RTE_FLOW_ACTION_TYPE_JUMP] = MLX5DR_ACTION_TYP_TBL,
	[RTE_FLOW_ACTION_TYPE_QUEUE] = MLX5DR_ACTION_TYP_TIR,
	[RTE_FLOW_ACTION_TYPE_RSS] = MLX5DR_ACTION_TYP_TIR,
	[RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP] = MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2,
	[RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP] = MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2,
	[RTE_FLOW_ACTION_TYPE_VXLAN_DECAP] = MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2,
	[RTE_FLOW_ACTION_TYPE_NVGRE_DECAP] = MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2,
	[RTE_FLOW_ACTION_TYPE_MODIFY_FIELD] = MLX5DR_ACTION_TYP_MODIFY_HDR,
	[RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT] = MLX5DR_ACTION_TYP_VPORT,
	[RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR] = MLX5DR_ACTION_TYP_MISS,
	[RTE_FLOW_ACTION_TYPE_CONNTRACK] = MLX5DR_ACTION_TYP_ASO_CT,
	[RTE_FLOW_ACTION_TYPE_OF_POP_VLAN] = MLX5DR_ACTION_TYP_POP_VLAN,
	[RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN] = MLX5DR_ACTION_TYP_PUSH_VLAN,
	[RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL] = MLX5DR_ACTION_TYP_DEST_ROOT,
	[RTE_FLOW_ACTION_TYPE_IPV6_EXT_PUSH] = MLX5DR_ACTION_TYP_PUSH_IPV6_ROUTE_EXT,
	[RTE_FLOW_ACTION_TYPE_IPV6_EXT_REMOVE] = MLX5DR_ACTION_TYP_POP_IPV6_ROUTE_EXT,
};

static inline void
action_template_set_type(struct rte_flow_actions_template *at,
			 enum mlx5dr_action_type *action_types,
			 unsigned int action_src, uint16_t *curr_off,
			 enum mlx5dr_action_type type)
{
	at->dr_off[action_src] = *curr_off;
	action_types[*curr_off] = type;
	*curr_off = *curr_off + 1;
}

static int
flow_hw_dr_actions_template_handle_shared(int type, uint32_t action_src,
					  enum mlx5dr_action_type *action_types,
					  uint16_t *curr_off, uint16_t *cnt_off,
					  struct rte_flow_actions_template *at)
{
	switch (type) {
	case RTE_FLOW_ACTION_TYPE_RSS:
		action_template_set_type(at, action_types, action_src, curr_off,
					 MLX5DR_ACTION_TYP_TIR);
		break;
	case RTE_FLOW_ACTION_TYPE_AGE:
	case RTE_FLOW_ACTION_TYPE_COUNT:
		/*
		 * Both AGE and COUNT action need counter, the first one fills
		 * the action_types array, and the second only saves the offset.
		 */
		if (*cnt_off == UINT16_MAX) {
			*cnt_off = *curr_off;
			action_template_set_type(at, action_types,
						 action_src, curr_off,
						 MLX5DR_ACTION_TYP_CTR);
		}
		at->dr_off[action_src] = *cnt_off;
		break;
	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
		action_template_set_type(at, action_types, action_src, curr_off,
					 MLX5DR_ACTION_TYP_ASO_CT);
		break;
	case RTE_FLOW_ACTION_TYPE_QUOTA:
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		action_template_set_type(at, action_types, action_src, curr_off,
					 MLX5DR_ACTION_TYP_ASO_METER);
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type: %d", type);
		return -EINVAL;
	}
	return 0;
}


static int
flow_hw_template_actions_list(struct rte_flow_actions_template *at,
			      unsigned int action_src,
			      enum mlx5dr_action_type *action_types,
			      uint16_t *curr_off, uint16_t *cnt_off)
{
	int ret;
	const struct rte_flow_action_indirect_list *indlst_conf = at->actions[action_src].conf;
	enum mlx5_indirect_list_type list_type = mlx5_get_indirect_list_type(indlst_conf->handle);
	const union {
		struct mlx5_indlst_legacy *legacy;
		struct rte_flow_action_list_handle *handle;
	} indlst_obj = { .handle = indlst_conf->handle };
	enum mlx5dr_action_type type;

	switch (list_type) {
	case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY:
		ret = flow_hw_dr_actions_template_handle_shared
			(indlst_obj.legacy->legacy_type, action_src,
			 action_types, curr_off, cnt_off, at);
		if (ret)
			return ret;
		break;
	case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR:
		action_template_set_type(at, action_types, action_src, curr_off,
					 MLX5DR_ACTION_TYP_DEST_ARRAY);
		break;
	case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT:
		type = ((struct mlx5_hw_encap_decap_action *)
			(indlst_conf->handle))->action_type;
		action_template_set_type(at, action_types, action_src, curr_off, type);
		break;
	default:
		DRV_LOG(ERR, "Unsupported indirect list type");
		return -EINVAL;
	}
	return 0;
}

/**
 * Create DR action template based on a provided sequence of flow actions.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] at
 *   Pointer to flow actions template to be updated.
 *
 * @return
 *   DR action template pointer on success and action offsets in @p at are updated.
 *   NULL otherwise.
 */
static struct mlx5dr_action_template *
flow_hw_dr_actions_template_create(struct rte_eth_dev *dev,
				   struct rte_flow_actions_template *at)
{
	struct mlx5dr_action_template *dr_template;
	enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS] = { MLX5DR_ACTION_TYP_LAST };
	unsigned int i;
	uint16_t curr_off;
	enum mlx5dr_action_type reformat_act_type = MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2;
	uint16_t reformat_off = UINT16_MAX;
	uint16_t mhdr_off = UINT16_MAX;
	uint16_t recom_off = UINT16_MAX;
	uint16_t cnt_off = UINT16_MAX;
	enum mlx5dr_action_type recom_type = MLX5DR_ACTION_TYP_LAST;
	int ret;

	for (i = 0, curr_off = 0; at->actions[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
		const struct rte_flow_action_raw_encap *raw_encap_data;
		size_t data_size;
		enum mlx5dr_action_type type;

		if (curr_off >= MLX5_HW_MAX_ACTS)
			goto err_actions_num;
		switch ((int)at->actions[i].type) {
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_INDIRECT_LIST:
			ret = flow_hw_template_actions_list(at, i, action_types,
							    &curr_off, &cnt_off);
			if (ret)
				return NULL;
			break;
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			ret = flow_hw_dr_actions_template_handle_shared
				(at->masks[i].type, i, action_types,
				 &curr_off, &cnt_off, at);
			if (ret)
				return NULL;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
			MLX5_ASSERT(reformat_off == UINT16_MAX);
			reformat_off = curr_off++;
			reformat_act_type = mlx5_hw_dr_action_types[at->actions[i].type];
			break;
		case RTE_FLOW_ACTION_TYPE_IPV6_EXT_PUSH:
			MLX5_ASSERT(recom_off == UINT16_MAX);
			recom_type = MLX5DR_ACTION_TYP_PUSH_IPV6_ROUTE_EXT;
			recom_off = curr_off++;
			break;
		case RTE_FLOW_ACTION_TYPE_IPV6_EXT_REMOVE:
			MLX5_ASSERT(recom_off == UINT16_MAX);
			recom_type = MLX5DR_ACTION_TYP_POP_IPV6_ROUTE_EXT;
			recom_off = curr_off++;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			raw_encap_data = at->actions[i].conf;
			data_size = raw_encap_data->size;
			if (reformat_off != UINT16_MAX) {
				reformat_act_type = data_size < MLX5_ENCAPSULATION_DECISION_SIZE ?
					MLX5DR_ACTION_TYP_REFORMAT_TNL_L3_TO_L2 :
					MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L3;
			} else {
				reformat_off = curr_off++;
				reformat_act_type = MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
			reformat_off = curr_off++;
			reformat_act_type = MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			if (mhdr_off == UINT16_MAX) {
				mhdr_off = curr_off++;
				type = mlx5_hw_dr_action_types[at->actions[i].type];
				action_types[mhdr_off] = type;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_METER:
			at->dr_off[i] = curr_off;
			action_types[curr_off++] = MLX5DR_ACTION_TYP_ASO_METER;
			if (curr_off >= MLX5_HW_MAX_ACTS)
				goto err_actions_num;
			action_types[curr_off++] = MLX5DR_ACTION_TYP_TBL;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			type = mlx5_hw_dr_action_types[at->actions[i].type];
			at->dr_off[i] = curr_off;
			action_types[curr_off++] = type;
			i += is_of_vlan_pcp_present(at->actions + i) ?
				MLX5_HW_VLAN_PUSH_PCP_IDX :
				MLX5_HW_VLAN_PUSH_VID_IDX;
			break;
		case RTE_FLOW_ACTION_TYPE_METER_MARK:
			at->dr_off[i] = curr_off;
			action_types[curr_off++] = MLX5DR_ACTION_TYP_ASO_METER;
			if (curr_off >= MLX5_HW_MAX_ACTS)
				goto err_actions_num;
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
		case RTE_FLOW_ACTION_TYPE_COUNT:
			/*
			 * Both AGE and COUNT action need counter, the first
			 * one fills the action_types array, and the second only
			 * saves the offset.
			 */
			if (cnt_off == UINT16_MAX) {
				cnt_off = curr_off++;
				action_types[cnt_off] = MLX5DR_ACTION_TYP_CTR;
			}
			at->dr_off[i] = cnt_off;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
			at->dr_off[i] = curr_off;
			action_types[curr_off++] = MLX5DR_ACTION_TYP_MISS;
			break;
		default:
			type = mlx5_hw_dr_action_types[at->actions[i].type];
			at->dr_off[i] = curr_off;
			action_types[curr_off++] = type;
			break;
		}
	}
	if (curr_off >= MLX5_HW_MAX_ACTS)
		goto err_actions_num;
	if (mhdr_off != UINT16_MAX)
		at->mhdr_off = mhdr_off;
	if (reformat_off != UINT16_MAX) {
		at->reformat_off = reformat_off;
		action_types[reformat_off] = reformat_act_type;
	}
	if (recom_off != UINT16_MAX) {
		at->recom_off = recom_off;
		action_types[recom_off] = recom_type;
	}
	dr_template = mlx5dr_action_template_create(action_types);
	if (dr_template) {
		at->dr_actions_num = curr_off;
	} else {
		DRV_LOG(ERR, "Failed to create DR action template: %d", rte_errno);
		return NULL;
	}
	/* Create srh flex parser for remove anchor. */
	if ((recom_type == MLX5DR_ACTION_TYP_POP_IPV6_ROUTE_EXT ||
	     recom_type == MLX5DR_ACTION_TYP_PUSH_IPV6_ROUTE_EXT) &&
	    mlx5_alloc_srh_flex_parser(dev)) {
		DRV_LOG(ERR, "Failed to create srv6 flex parser");
		claim_zero(mlx5dr_action_template_destroy(dr_template));
		return NULL;
	}
	return dr_template;
err_actions_num:
	DRV_LOG(ERR, "Number of HW actions (%u) exceeded maximum (%u) allowed in template",
		curr_off, MLX5_HW_MAX_ACTS);
	return NULL;
}

static void
flow_hw_set_vlan_vid(struct rte_eth_dev *dev,
		     struct rte_flow_action *ra,
		     struct rte_flow_action *rm,
		     struct rte_flow_action_modify_field *spec,
		     struct rte_flow_action_modify_field *mask,
		     int set_vlan_vid_ix)
{
	struct rte_flow_error error;
	const bool masked = rm[set_vlan_vid_ix].conf &&
		(((const struct rte_flow_action_of_set_vlan_vid *)
			rm[set_vlan_vid_ix].conf)->vlan_vid != 0);
	const struct rte_flow_action_of_set_vlan_vid *conf =
		ra[set_vlan_vid_ix].conf;
	rte_be16_t vid = masked ? conf->vlan_vid : 0;
	int width = mlx5_flow_item_field_width(dev, RTE_FLOW_FIELD_VLAN_ID, 0,
					       NULL, &error);
	*spec = (typeof(*spec)) {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = RTE_FLOW_FIELD_VLAN_ID,
			.level = 0, .offset = 0,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
			.level = vid,
			.offset = 0,
		},
		.width = width,
	};
	*mask = (typeof(*mask)) {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = RTE_FLOW_FIELD_VLAN_ID,
			.level = 0xff, .offset = 0xffffffff,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
			.level = masked ? (1U << width) - 1 : 0,
			.offset = 0,
		},
		.width = 0xffffffff,
	};
	ra[set_vlan_vid_ix].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
	ra[set_vlan_vid_ix].conf = spec;
	rm[set_vlan_vid_ix].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
	rm[set_vlan_vid_ix].conf = mask;
}

static __rte_always_inline int
flow_hw_set_vlan_vid_construct(struct rte_eth_dev *dev,
			       struct mlx5_hw_q_job *job,
			       struct mlx5_action_construct_data *act_data,
			       const struct mlx5_hw_actions *hw_acts,
			       const struct rte_flow_action *action)
{
	struct rte_flow_error error;
	rte_be16_t vid = ((const struct rte_flow_action_of_set_vlan_vid *)
			   action->conf)->vlan_vid;
	int width = mlx5_flow_item_field_width(dev, RTE_FLOW_FIELD_VLAN_ID, 0,
					       NULL, &error);
	struct rte_flow_action_modify_field conf = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = RTE_FLOW_FIELD_VLAN_ID,
			.level = 0, .offset = 0,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
			.level = vid,
			.offset = 0,
		},
		.width = width,
	};
	struct rte_flow_action modify_action = {
		.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
		.conf = &conf
	};

	return flow_hw_modify_field_construct(job, act_data, hw_acts,
					      &modify_action);
}

static int
flow_hw_flex_item_acquire(struct rte_eth_dev *dev,
			  struct rte_flow_item_flex_handle *handle,
			  uint8_t *flex_item)
{
	int index = mlx5_flex_acquire_index(dev, handle, false);

	MLX5_ASSERT(index >= 0 && index < (int)(sizeof(uint32_t) * CHAR_BIT));
	if (index < 0)
		return -1;
	if (!(*flex_item & RTE_BIT32(index))) {
		/* Don't count same flex item again. */
		if (mlx5_flex_acquire_index(dev, handle, true) != index)
			MLX5_ASSERT(false);
		*flex_item |= (uint8_t)RTE_BIT32(index);
	}
	return 0;
}

static void
flow_hw_flex_item_release(struct rte_eth_dev *dev, uint8_t *flex_item)
{
	while (*flex_item) {
		int index = rte_bsf32(*flex_item);

		mlx5_flex_release_index(dev, index);
		*flex_item &= ~(uint8_t)RTE_BIT32(index);
	}
}
static __rte_always_inline void
flow_hw_actions_template_replace_container(const
					   struct rte_flow_action *actions,
					   const
					   struct rte_flow_action *masks,
					   struct rte_flow_action *new_actions,
					   struct rte_flow_action *new_masks,
					   struct rte_flow_action **ra,
					   struct rte_flow_action **rm,
					   uint32_t act_num)
{
	memcpy(new_actions, actions, sizeof(actions[0]) * act_num);
	memcpy(new_masks, masks, sizeof(masks[0]) * act_num);
	*ra = (void *)(uintptr_t)new_actions;
	*rm = (void *)(uintptr_t)new_masks;
}

/* Action template copies these actions in rte_flow_conv() */

static const struct rte_flow_action rx_meta_copy_action =  {
	.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
	.conf = &(struct rte_flow_action_modify_field){
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)
				MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_B,
		},
		.src = {
			.field = (enum rte_flow_field_id)
				MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_C_1,
		},
		.width = 32,
	}
};

static const struct rte_flow_action rx_meta_copy_mask = {
	.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
	.conf = &(struct rte_flow_action_modify_field){
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)
				MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = (enum rte_flow_field_id)
				MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.width = UINT32_MAX,
	}
};

static const struct rte_flow_action quota_color_inc_action = {
	.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
	.conf = &(struct rte_flow_action_modify_field) {
		.operation = RTE_FLOW_MODIFY_ADD,
		.dst = {
			.field = RTE_FLOW_FIELD_METER_COLOR,
			.level = 0, .offset = 0
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
			.level = 1,
			.offset = 0,
		},
		.width = 2
	}
};

static const struct rte_flow_action quota_color_inc_mask = {
	.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
	.conf = &(struct rte_flow_action_modify_field) {
		.operation = RTE_FLOW_MODIFY_ADD,
		.dst = {
			.field = RTE_FLOW_FIELD_METER_COLOR,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
			.level = 3,
			.offset = 0
		},
		.width = UINT32_MAX
	}
};

/**
 * Create flow action template.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] attr
 *   Pointer to the action template attributes.
 * @param[in] actions
 *   Associated actions (list terminated by the END action).
 * @param[in] masks
 *   List of actions that marks which of the action's member is constant.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   Action template pointer on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow_actions_template *
flow_hw_actions_template_create(struct rte_eth_dev *dev,
			const struct rte_flow_actions_template_attr *attr,
			const struct rte_flow_action actions[],
			const struct rte_flow_action masks[],
			struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int len, act_len, mask_len;
	unsigned int act_num;
	unsigned int i;
	struct rte_flow_actions_template *at = NULL;
	uint16_t pos = UINT16_MAX;
	uint64_t action_flags = 0;
	struct rte_flow_action tmp_action[MLX5_HW_MAX_ACTS];
	struct rte_flow_action tmp_mask[MLX5_HW_MAX_ACTS];
	struct rte_flow_action *ra = (void *)(uintptr_t)actions;
	struct rte_flow_action *rm = (void *)(uintptr_t)masks;
	int set_vlan_vid_ix = -1;
	struct rte_flow_action_modify_field set_vlan_vid_spec = {0, };
	struct rte_flow_action_modify_field set_vlan_vid_mask = {0, };
	struct rte_flow_action mf_actions[MLX5_HW_MAX_ACTS];
	struct rte_flow_action mf_masks[MLX5_HW_MAX_ACTS];
	uint32_t expand_mf_num = 0;
	uint16_t src_off[MLX5_HW_MAX_ACTS] = {0, };

	if (mlx5_flow_hw_actions_validate(dev, attr, actions, masks,
					  &action_flags, error))
		return NULL;
	for (i = 0; ra[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
		switch (ra[i].type) {
		/* OF_PUSH_VLAN *MUST* come before OF_SET_VLAN_VID */
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			i += is_of_vlan_pcp_present(ra + i) ?
				MLX5_HW_VLAN_PUSH_PCP_IDX :
				MLX5_HW_VLAN_PUSH_VID_IDX;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
			set_vlan_vid_ix = i;
			break;
		default:
			break;
		}
	}
	/*
	 * Count flow actions to allocate required space for storing DR offsets and to check
	 * if temporary buffer would not be overrun.
	 */
	act_num = i + 1;
	if (act_num >= MLX5_HW_MAX_ACTS) {
		rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Too many actions");
		return NULL;
	}
	if (set_vlan_vid_ix != -1) {
		/* If temporary action buffer was not used, copy template actions to it */
		if (ra == actions)
			flow_hw_actions_template_replace_container(actions,
								   masks,
								   tmp_action,
								   tmp_mask,
								   &ra, &rm,
								   act_num);
		flow_hw_set_vlan_vid(dev, ra, rm,
				     &set_vlan_vid_spec, &set_vlan_vid_mask,
				     set_vlan_vid_ix);
		action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
	}
	if (action_flags & MLX5_FLOW_ACTION_QUOTA) {
		mf_actions[expand_mf_num] = quota_color_inc_action;
		mf_masks[expand_mf_num] = quota_color_inc_mask;
		expand_mf_num++;
	}
	if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS &&
	    priv->sh->config.dv_esw_en &&
	    (action_flags & (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS))) {
		/* Insert META copy */
		mf_actions[expand_mf_num] = rx_meta_copy_action;
		mf_masks[expand_mf_num] = rx_meta_copy_mask;
		expand_mf_num++;
	}
	if (expand_mf_num) {
		if (act_num + expand_mf_num > MLX5_HW_MAX_ACTS) {
			rte_flow_error_set(error, E2BIG,
					   RTE_FLOW_ERROR_TYPE_ACTION,
					   NULL, "cannot expand: too many actions");
			return NULL;
		}
		if (ra == actions)
			flow_hw_actions_template_replace_container(actions,
								   masks,
								   tmp_action,
								   tmp_mask,
								   &ra, &rm,
								   act_num);
		/* Application should make sure only one Q/RSS exist in one rule. */
		pos = flow_hw_template_expand_modify_field(ra, rm,
							   mf_actions,
							   mf_masks,
							   action_flags,
							   act_num,
							   expand_mf_num);
		act_num += expand_mf_num;
		for (i = pos + expand_mf_num; i < act_num; i++)
			src_off[i] += expand_mf_num;
		action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
	}
	act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, ra, error);
	if (act_len <= 0)
		return NULL;
	len = RTE_ALIGN(act_len, 16);
	mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, rm, error);
	if (mask_len <= 0)
		return NULL;
	len += RTE_ALIGN(mask_len, 16);
	len += RTE_ALIGN(act_num * sizeof(*at->dr_off), 16);
	len += RTE_ALIGN(act_num * sizeof(*at->src_off), 16);
	at = mlx5_malloc(MLX5_MEM_ZERO, len + sizeof(*at),
			 RTE_CACHE_LINE_SIZE, rte_socket_id());
	if (!at) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate action template");
		return NULL;
	}
	/* Actions part is in the first part. */
	at->attr = *attr;
	at->actions = (struct rte_flow_action *)(at + 1);
	act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->actions,
				len, ra, error);
	if (act_len <= 0)
		goto error;
	/* Masks part is in the second part. */
	at->masks = (struct rte_flow_action *)(((uint8_t *)at->actions) + act_len);
	mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->masks,
				 len - act_len, rm, error);
	if (mask_len <= 0)
		goto error;
	/* DR actions offsets in the third part. */
	at->dr_off = (uint16_t *)((uint8_t *)at->masks + mask_len);
	at->src_off = RTE_PTR_ADD(at->dr_off,
				  RTE_ALIGN(act_num * sizeof(*at->dr_off), 16));
	memcpy(at->src_off, src_off, act_num * sizeof(at->src_off[0]));
	at->actions_num = act_num;
	for (i = 0; i < at->actions_num; ++i)
		at->dr_off[i] = UINT16_MAX;
	at->reformat_off = UINT16_MAX;
	at->mhdr_off = UINT16_MAX;
	at->recom_off = UINT16_MAX;
	for (i = 0; actions->type != RTE_FLOW_ACTION_TYPE_END;
	     actions++, masks++, i++) {
		const struct rte_flow_action_modify_field *info;

		switch (actions->type) {
		/*
		 * mlx5 PMD hacks indirect action index directly to the action conf.
		 * The rte_flow_conv() function copies the content from conf pointer.
		 * Need to restore the indirect action index from action conf here.
		 */
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			at->actions[i].conf = ra[i].conf;
			at->masks[i].conf = rm[i].conf;
			break;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			info = actions->conf;
			if ((info->dst.field == RTE_FLOW_FIELD_FLEX_ITEM &&
			     flow_hw_flex_item_acquire(dev, info->dst.flex_handle,
						       &at->flex_item)) ||
			    (info->src.field == RTE_FLOW_FIELD_FLEX_ITEM &&
			     flow_hw_flex_item_acquire(dev, info->src.flex_handle,
						       &at->flex_item)))
				goto error;
			break;
		default:
			break;
		}
	}
	at->tmpl = flow_hw_dr_actions_template_create(dev, at);
	if (!at->tmpl)
		goto error;
	at->action_flags = action_flags;
	__atomic_fetch_add(&at->refcnt, 1, __ATOMIC_RELAXED);
	LIST_INSERT_HEAD(&priv->flow_hw_at, at, next);
	return at;
error:
	if (at) {
		if (at->tmpl)
			mlx5dr_action_template_destroy(at->tmpl);
		mlx5_free(at);
	}
	rte_flow_error_set(error, rte_errno,
			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
			   "Failed to create action template");
	return NULL;
}

/**
 * Destroy flow action template.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] template
 *   Pointer to the action template to be destroyed.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_actions_template_destroy(struct rte_eth_dev *dev,
				 struct rte_flow_actions_template *template,
				 struct rte_flow_error *error __rte_unused)
{
	uint64_t flag = MLX5_FLOW_ACTION_IPV6_ROUTING_REMOVE |
			MLX5_FLOW_ACTION_IPV6_ROUTING_PUSH;

	if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
		DRV_LOG(WARNING, "Action template %p is still in use.",
			(void *)template);
		return rte_flow_error_set(error, EBUSY,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "action template in using");
	}
	if (template->action_flags & flag)
		mlx5_free_srh_flex_parser(dev);
	LIST_REMOVE(template, next);
	flow_hw_flex_item_release(dev, &template->flex_item);
	if (template->tmpl)
		mlx5dr_action_template_destroy(template->tmpl);
	mlx5_free(template);
	return 0;
}

static uint32_t
flow_hw_count_items(const struct rte_flow_item *items)
{
	const struct rte_flow_item *curr_item;
	uint32_t nb_items;

	nb_items = 0;
	for (curr_item = items; curr_item->type != RTE_FLOW_ITEM_TYPE_END; ++curr_item)
		++nb_items;
	return ++nb_items;
}

static struct rte_flow_item *
flow_hw_prepend_item(const struct rte_flow_item *items,
		     const uint32_t nb_items,
		     const struct rte_flow_item *new_item,
		     struct rte_flow_error *error)
{
	struct rte_flow_item *copied_items;
	size_t size;

	/* Allocate new array of items. */
	size = sizeof(*copied_items) * (nb_items + 1);
	copied_items = mlx5_malloc(MLX5_MEM_ZERO, size, 0, rte_socket_id());
	if (!copied_items) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate item template");
		return NULL;
	}
	/* Put new item at the beginning and copy the rest. */
	copied_items[0] = *new_item;
	rte_memcpy(&copied_items[1], items, sizeof(*items) * nb_items);
	return copied_items;
}

static inline bool
flow_hw_item_compare_field_supported(enum rte_flow_field_id field)
{
	switch (field) {
	case RTE_FLOW_FIELD_TAG:
	case RTE_FLOW_FIELD_META:
	case RTE_FLOW_FIELD_VALUE:
		return true;
	default:
		break;
	}
	return false;
}

static int
flow_hw_validate_item_compare(const struct rte_flow_item *item,
			      struct rte_flow_error *error)
{
	const struct rte_flow_item_compare *comp_m = item->mask;
	const struct rte_flow_item_compare *comp_v = item->spec;

	if (unlikely(!comp_m))
		return rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "compare item mask is missing");
	if (comp_m->width != UINT32_MAX)
		return rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "compare item only support full mask");
	if (!flow_hw_item_compare_field_supported(comp_m->a.field) ||
	    !flow_hw_item_compare_field_supported(comp_m->b.field))
		return rte_flow_error_set(error, ENOTSUP,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "compare item field not support");
	if (comp_m->a.field == RTE_FLOW_FIELD_VALUE &&
	    comp_m->b.field == RTE_FLOW_FIELD_VALUE)
		return rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "compare between value is not valid");
	if (comp_v) {
		if (comp_v->operation != comp_m->operation ||
		    comp_v->a.field != comp_m->a.field ||
		    comp_v->b.field != comp_m->b.field)
			return rte_flow_error_set(error, EINVAL,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL,
					   "compare item spec/mask not matching");
		if ((comp_v->width & comp_m->width) != 32)
			return rte_flow_error_set(error, EINVAL,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL,
					   "compare item only support full mask");
	}
	return 0;
}

static int
flow_hw_pattern_validate(struct rte_eth_dev *dev,
			 const struct rte_flow_pattern_template_attr *attr,
			 const struct rte_flow_item items[],
			 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int i, tag_idx;
	bool items_end = false;
	uint32_t tag_bitmap = 0;
	int ret;

	if (!attr->ingress && !attr->egress && !attr->transfer)
		return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR, NULL,
					  "at least one of the direction attributes"
					  " must be specified");
	if (priv->sh->config.dv_esw_en) {
		MLX5_ASSERT(priv->master || priv->representor);
		if (priv->master) {
			if ((attr->ingress && attr->egress) ||
			    (attr->ingress && attr->transfer) ||
			    (attr->egress && attr->transfer))
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
							  "only one direction attribute at once"
							  " can be used on transfer proxy port");
		} else {
			if (attr->transfer)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, NULL,
							  "transfer attribute cannot be used with"
							  " port representors");
			if (attr->ingress && attr->egress)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
							  "ingress and egress direction attributes"
							  " cannot be used at the same time on"
							  " port representors");
		}
	} else {
		if (attr->transfer)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, NULL,
						  "transfer attribute cannot be used when"
						  " E-Switch is disabled");
	}
	for (i = 0; !items_end; i++) {
		int type = items[i].type;

		switch (type) {
		case RTE_FLOW_ITEM_TYPE_TAG:
		{
			const struct rte_flow_item_tag *tag =
				(const struct rte_flow_item_tag *)items[i].spec;

			if (tag == NULL)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  NULL,
							  "Tag spec is NULL");
			if (tag->index >= MLX5_FLOW_HW_TAGS_MAX &&
			    tag->index != RTE_PMD_MLX5_LINEAR_HASH_TAG_INDEX)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  NULL,
							  "Invalid tag index");
			tag_idx = flow_hw_get_reg_id(dev, RTE_FLOW_ITEM_TYPE_TAG, tag->index);
			if (tag_idx == REG_NON)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  NULL,
							  "Unsupported tag index");
			if (tag_bitmap & (1 << tag_idx))
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ITEM,
							  NULL,
							  "Duplicated tag index");
			tag_bitmap |= 1 << tag_idx;
			break;
		}
		case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
		{
			const struct rte_flow_item_tag *tag =
				(const struct rte_flow_item_tag *)items[i].spec;
			uint16_t regcs = (uint8_t)priv->sh->cdev->config.hca_attr.set_reg_c;

			if (!((1 << (tag->index - REG_C_0)) & regcs))
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  NULL,
							  "Unsupported internal tag index");
			if (tag_bitmap & (1 << tag->index))
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ITEM,
							  NULL,
							  "Duplicated tag index");
			tag_bitmap |= 1 << tag->index;
			break;
		}
		case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT:
			if (attr->ingress && priv->sh->config.repr_matching)
				return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
						  "represented port item cannot be used"
						  " when ingress attribute is set");
			if (attr->egress)
				return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
						  "represented port item cannot be used"
						  " when egress attribute is set");
			break;
		case RTE_FLOW_ITEM_TYPE_META:
			if (!priv->sh->config.dv_esw_en ||
			    priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) {
				if (attr->ingress)
					return rte_flow_error_set(error, EINVAL,
								  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
								  "META item is not supported"
								  " on current FW with ingress"
								  " attribute");
			}
			break;
		case RTE_FLOW_ITEM_TYPE_METER_COLOR:
		{
			int reg = flow_hw_get_reg_id(dev,
						     RTE_FLOW_ITEM_TYPE_METER_COLOR,
						     0);
			if (reg == REG_NON)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  NULL,
							  "Unsupported meter color register");
			break;
		}
		case RTE_FLOW_ITEM_TYPE_AGGR_AFFINITY:
		{
			if (!priv->sh->lag_rx_port_affinity_en)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
							  "Unsupported aggregated affinity with Older FW");
			if ((attr->transfer && priv->fdb_def_rule) || attr->egress)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
							  "Aggregated affinity item not supported"
							  " with egress or transfer"
							  " attribute");
			break;
		}
		case RTE_FLOW_ITEM_TYPE_COMPARE:
		{
			ret = flow_hw_validate_item_compare(&items[i], error);
			if (ret)
				return ret;
			break;
		}
		case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
		{
			int ret;

			ret = mlx5_flow_geneve_tlv_option_validate(priv,
								   &items[i],
								   error);
			if (ret < 0)
				return ret;
			break;
		}
		case RTE_FLOW_ITEM_TYPE_VOID:
		case RTE_FLOW_ITEM_TYPE_ETH:
		case RTE_FLOW_ITEM_TYPE_VLAN:
		case RTE_FLOW_ITEM_TYPE_IPV4:
		case RTE_FLOW_ITEM_TYPE_IPV6:
		case RTE_FLOW_ITEM_TYPE_UDP:
		case RTE_FLOW_ITEM_TYPE_TCP:
		case RTE_FLOW_ITEM_TYPE_GTP:
		case RTE_FLOW_ITEM_TYPE_GTP_PSC:
		case RTE_FLOW_ITEM_TYPE_VXLAN:
		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
		case RTE_FLOW_ITEM_TYPE_MPLS:
		case RTE_FLOW_ITEM_TYPE_GENEVE:
		case MLX5_RTE_FLOW_ITEM_TYPE_SQ:
		case RTE_FLOW_ITEM_TYPE_GRE:
		case RTE_FLOW_ITEM_TYPE_GRE_KEY:
		case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
		case RTE_FLOW_ITEM_TYPE_ICMP:
		case RTE_FLOW_ITEM_TYPE_ICMP6:
		case RTE_FLOW_ITEM_TYPE_ICMP6_ECHO_REQUEST:
		case RTE_FLOW_ITEM_TYPE_QUOTA:
		case RTE_FLOW_ITEM_TYPE_ICMP6_ECHO_REPLY:
		case RTE_FLOW_ITEM_TYPE_CONNTRACK:
		case RTE_FLOW_ITEM_TYPE_IPV6_ROUTING_EXT:
		case RTE_FLOW_ITEM_TYPE_ESP:
		case RTE_FLOW_ITEM_TYPE_FLEX:
		case RTE_FLOW_ITEM_TYPE_IB_BTH:
		case RTE_FLOW_ITEM_TYPE_PTYPE:
		case RTE_FLOW_ITEM_TYPE_RANDOM:
			break;
		case RTE_FLOW_ITEM_TYPE_INTEGRITY:
			/*
			 * Integrity flow item validation require access to
			 * both item mask and spec.
			 * Current HWS model allows item mask in pattern
			 * template and item spec in flow rule.
			 */
			break;
		case RTE_FLOW_ITEM_TYPE_END:
			items_end = true;
			break;
		default:
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL,
						  "Unsupported item type");
		}
	}
	return 0;
}

static bool
flow_hw_pattern_has_sq_match(const struct rte_flow_item *items)
{
	unsigned int i;

	for (i = 0; items[i].type != RTE_FLOW_ITEM_TYPE_END; ++i)
		if (items[i].type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ)
			return true;
	return false;
}

/**
 * Create flow item template.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] attr
 *   Pointer to the item template attributes.
 * @param[in] items
 *   The template item pattern.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *  Item template pointer on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow_pattern_template *
flow_hw_pattern_template_create(struct rte_eth_dev *dev,
			     const struct rte_flow_pattern_template_attr *attr,
			     const struct rte_flow_item items[],
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_pattern_template *it;
	struct rte_flow_item *copied_items = NULL;
	const struct rte_flow_item *tmpl_items;
	uint32_t orig_item_nb;
	struct rte_flow_item port = {
		.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
		.mask = &rte_flow_item_ethdev_mask,
	};
	struct rte_flow_item_tag tag_v = {
		.data = 0,
		.index = REG_C_0,
	};
	struct rte_flow_item_tag tag_m = {
		.data = flow_hw_tx_tag_regc_mask(dev),
		.index = 0xff,
	};
	struct rte_flow_item tag = {
		.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
		.spec = &tag_v,
		.mask = &tag_m,
		.last = NULL
	};
	unsigned int i = 0;

	if (flow_hw_pattern_validate(dev, attr, items, error))
		return NULL;
	orig_item_nb = flow_hw_count_items(items);
	if (priv->sh->config.dv_esw_en &&
	    priv->sh->config.repr_matching &&
	    attr->ingress && !attr->egress && !attr->transfer) {
		copied_items = flow_hw_prepend_item(items, orig_item_nb, &port, error);
		if (!copied_items)
			return NULL;
		tmpl_items = copied_items;
	} else if (priv->sh->config.dv_esw_en &&
		   priv->sh->config.repr_matching &&
		   !attr->ingress && attr->egress && !attr->transfer) {
		if (flow_hw_pattern_has_sq_match(items)) {
			DRV_LOG(DEBUG, "Port %u omitting implicit REG_C_0 match for egress "
				       "pattern template", dev->data->port_id);
			tmpl_items = items;
			goto setup_pattern_template;
		}
		copied_items = flow_hw_prepend_item(items, orig_item_nb, &tag, error);
		if (!copied_items)
			return NULL;
		tmpl_items = copied_items;
	} else {
		tmpl_items = items;
	}
setup_pattern_template:
	it = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*it), 0, rte_socket_id());
	if (!it) {
		if (copied_items)
			mlx5_free(copied_items);
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate item template");
		return NULL;
	}
	it->attr = *attr;
	it->orig_item_nb = orig_item_nb;
	it->mt = mlx5dr_match_template_create(tmpl_items, attr->relaxed_matching);
	if (!it->mt) {
		if (copied_items)
			mlx5_free(copied_items);
		mlx5_free(it);
		rte_flow_error_set(error, rte_errno,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot create match template");
		return NULL;
	}
	it->item_flags = flow_hw_matching_item_flags_get(tmpl_items);
	if (copied_items) {
		if (attr->ingress)
			it->implicit_port = true;
		else if (attr->egress)
			it->implicit_tag = true;
		mlx5_free(copied_items);
	}
	/* Either inner or outer, can't both. */
	if (it->item_flags & (MLX5_FLOW_ITEM_OUTER_IPV6_ROUTING_EXT |
			      MLX5_FLOW_ITEM_INNER_IPV6_ROUTING_EXT)) {
		if (((it->item_flags & MLX5_FLOW_ITEM_OUTER_IPV6_ROUTING_EXT) &&
		     (it->item_flags & MLX5_FLOW_ITEM_INNER_IPV6_ROUTING_EXT)) ||
		    (mlx5_alloc_srh_flex_parser(dev))) {
			claim_zero(mlx5dr_match_template_destroy(it->mt));
			mlx5_free(it);
			rte_flow_error_set(error, rte_errno,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					   "cannot create IPv6 routing extension support");
			return NULL;
		}
	}
	for (i = 0; items[i].type != RTE_FLOW_ITEM_TYPE_END; ++i) {
		switch (items[i].type) {
		case RTE_FLOW_ITEM_TYPE_FLEX: {
			const struct rte_flow_item_flex *spec =
				(const struct rte_flow_item_flex *)items[i].spec;
			struct rte_flow_item_flex_handle *handle = spec->handle;

			if (flow_hw_flex_item_acquire(dev, handle, &it->flex_item)) {
				rte_flow_error_set(error, rte_errno,
						   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
						   "Failed to acquire flex item");
				goto error;
			}
			break;
		}
		case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: {
			const struct rte_flow_item_geneve_opt *spec = items[i].spec;

			if (mlx5_geneve_tlv_option_register(priv, spec,
							    &it->geneve_opt_mng)) {
				rte_flow_error_set(error, rte_errno,
						   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
						   "Failed to register GENEVE TLV option");
				goto error;
			}
			break;
		}
		default:
			break;
		}
	}
	__atomic_fetch_add(&it->refcnt, 1, __ATOMIC_RELAXED);
	LIST_INSERT_HEAD(&priv->flow_hw_itt, it, next);
	return it;
error:
	flow_hw_flex_item_release(dev, &it->flex_item);
	mlx5_geneve_tlv_options_unregister(priv, &it->geneve_opt_mng);
	claim_zero(mlx5dr_match_template_destroy(it->mt));
	mlx5_free(it);
	return NULL;
}

/**
 * Destroy flow item template.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] template
 *   Pointer to the item template to be destroyed.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_pattern_template_destroy(struct rte_eth_dev *dev,
			      struct rte_flow_pattern_template *template,
			      struct rte_flow_error *error __rte_unused)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
		DRV_LOG(WARNING, "Item template %p is still in use.",
			(void *)template);
		return rte_flow_error_set(error, EBUSY,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "item template in using");
	}
	if (template->item_flags & (MLX5_FLOW_ITEM_OUTER_IPV6_ROUTING_EXT |
				    MLX5_FLOW_ITEM_INNER_IPV6_ROUTING_EXT))
		mlx5_free_srh_flex_parser(dev);
	LIST_REMOVE(template, next);
	flow_hw_flex_item_release(dev, &template->flex_item);
	mlx5_geneve_tlv_options_unregister(priv, &template->geneve_opt_mng);
	claim_zero(mlx5dr_match_template_destroy(template->mt));
	mlx5_free(template);
	return 0;
}

/*
 * Get information about HWS pre-configurable resources.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[out] port_info
 *   Pointer to port information.
 * @param[out] queue_info
 *   Pointer to queue information.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_info_get(struct rte_eth_dev *dev,
		 struct rte_flow_port_info *port_info,
		 struct rte_flow_queue_info *queue_info,
		 struct rte_flow_error *error __rte_unused)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint16_t port_id = dev->data->port_id;
	struct rte_mtr_capabilities mtr_cap;
	int ret;

	memset(port_info, 0, sizeof(*port_info));
	/* Queue size is unlimited from low-level. */
	port_info->max_nb_queues = UINT32_MAX;
	queue_info->max_size = UINT32_MAX;

	memset(&mtr_cap, 0, sizeof(struct rte_mtr_capabilities));
	ret = rte_mtr_capabilities_get(port_id, &mtr_cap, NULL);
	if (!ret)
		port_info->max_nb_meters = mtr_cap.n_max;
	port_info->max_nb_counters = priv->sh->hws_max_nb_counters;
	port_info->max_nb_aging_objects = port_info->max_nb_counters;
	return 0;
}

/**
 * Create group callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] cb_ctx
 *   Pointer to the group creation context.
 *
 * @return
 *   Group entry on success, NULL otherwise and rte_errno is set.
 */
struct mlx5_list_entry *
flow_hw_grp_create_cb(void *tool_ctx, void *cb_ctx)
{
	struct mlx5_dev_ctx_shared *sh = tool_ctx;
	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
	struct rte_eth_dev *dev = ctx->dev;
	struct rte_flow_attr *attr = (struct rte_flow_attr *)ctx->data;
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_table_attr dr_tbl_attr = {0};
	struct rte_flow_error *error = ctx->error;
	struct mlx5_flow_group *grp_data;
	struct mlx5dr_table *tbl = NULL;
	struct mlx5dr_action *jump;
	uint32_t idx = 0;

	grp_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
	if (!grp_data) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate flow table data entry");
		return NULL;
	}
	dr_tbl_attr.level = attr->group;
	if (attr->transfer)
		dr_tbl_attr.type = MLX5DR_TABLE_TYPE_FDB;
	else if (attr->egress)
		dr_tbl_attr.type = MLX5DR_TABLE_TYPE_NIC_TX;
	else
		dr_tbl_attr.type = MLX5DR_TABLE_TYPE_NIC_RX;
	tbl = mlx5dr_table_create(priv->dr_ctx, &dr_tbl_attr);
	if (!tbl)
		goto error;
	grp_data->tbl = tbl;
	if (attr->group) {
		/* Jump action be used by non-root table. */
		jump = mlx5dr_action_create_dest_table
			(priv->dr_ctx, tbl,
			 mlx5_hw_act_flag[!!attr->group][dr_tbl_attr.type]);
		if (!jump)
			goto error;
		grp_data->jump.hws_action = jump;
		/* Jump action be used by root table.  */
		jump = mlx5dr_action_create_dest_table
			(priv->dr_ctx, tbl,
			 mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_ROOT]
					 [dr_tbl_attr.type]);
		if (!jump)
			goto error;
		grp_data->jump.root_action = jump;
	}
	grp_data->dev = dev;
	grp_data->idx = idx;
	grp_data->group_id = attr->group;
	grp_data->type = dr_tbl_attr.type;
	return &grp_data->entry;
error:
	if (grp_data->jump.root_action)
		mlx5dr_action_destroy(grp_data->jump.root_action);
	if (grp_data->jump.hws_action)
		mlx5dr_action_destroy(grp_data->jump.hws_action);
	if (tbl)
		mlx5dr_table_destroy(tbl);
	if (idx)
		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], idx);
	rte_flow_error_set(error, ENOMEM,
			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
			   NULL,
			   "cannot allocate flow dr table");
	return NULL;
}

/**
 * Remove group callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] entry
 *   Pointer to the entry to be removed.
 */
void
flow_hw_grp_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
	struct mlx5_dev_ctx_shared *sh = tool_ctx;
	struct mlx5_flow_group *grp_data =
		    container_of(entry, struct mlx5_flow_group, entry);

	MLX5_ASSERT(entry && sh);
	/* To use the wrapper glue functions instead. */
	if (grp_data->jump.hws_action)
		mlx5dr_action_destroy(grp_data->jump.hws_action);
	if (grp_data->jump.root_action)
		mlx5dr_action_destroy(grp_data->jump.root_action);
	mlx5dr_table_destroy(grp_data->tbl);
	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
}

/**
 * Match group callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] entry
 *   Pointer to the group to be matched.
 * @param[in] cb_ctx
 *   Pointer to the group matching context.
 *
 * @return
 *   0 on matched, 1 on miss matched.
 */
int
flow_hw_grp_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
		     void *cb_ctx)
{
	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
	struct mlx5_flow_group *grp_data =
		container_of(entry, struct mlx5_flow_group, entry);
	struct rte_flow_attr *attr =
			(struct rte_flow_attr *)ctx->data;

	return (grp_data->dev != ctx->dev) ||
		(grp_data->group_id != attr->group) ||
		((grp_data->type != MLX5DR_TABLE_TYPE_FDB) &&
		attr->transfer) ||
		((grp_data->type != MLX5DR_TABLE_TYPE_NIC_TX) &&
		attr->egress) ||
		((grp_data->type != MLX5DR_TABLE_TYPE_NIC_RX) &&
		attr->ingress);
}

/**
 * Clone group entry callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] entry
 *   Pointer to the group to be matched.
 * @param[in] cb_ctx
 *   Pointer to the group matching context.
 *
 * @return
 *   0 on matched, 1 on miss matched.
 */
struct mlx5_list_entry *
flow_hw_grp_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
		     void *cb_ctx)
{
	struct mlx5_dev_ctx_shared *sh = tool_ctx;
	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
	struct mlx5_flow_group *grp_data;
	struct rte_flow_error *error = ctx->error;
	uint32_t idx = 0;

	grp_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
	if (!grp_data) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate flow table data entry");
		return NULL;
	}
	memcpy(grp_data, oentry, sizeof(*grp_data));
	grp_data->idx = idx;
	return &grp_data->entry;
}

/**
 * Free cloned group entry callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] entry
 *   Pointer to the group to be freed.
 */
void
flow_hw_grp_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
	struct mlx5_dev_ctx_shared *sh = tool_ctx;
	struct mlx5_flow_group *grp_data =
		    container_of(entry, struct mlx5_flow_group, entry);

	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
}

/**
 * Create and cache a vport action for given @p dev port. vport actions
 * cache is used in HWS with FDB flows.
 *
 * This function does not create any function if proxy port for @p dev port
 * was not configured for HW Steering.
 *
 * This function assumes that E-Switch is enabled and PMD is running with
 * HW Steering configured.
 *
 * @param dev
 *   Pointer to Ethernet device which will be the action destination.
 *
 * @return
 *   0 on success, positive value otherwise.
 */
int
flow_hw_create_vport_action(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	uint16_t port_id = dev->data->port_id;
	uint16_t proxy_port_id = port_id;
	int ret;

	ret = mlx5_flow_pick_transfer_proxy(dev, &proxy_port_id, NULL);
	if (ret)
		return ret;
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->hw_vport)
		return 0;
	if (proxy_priv->hw_vport[port_id]) {
		DRV_LOG(ERR, "port %u HWS vport action already created",
			port_id);
		return -EINVAL;
	}
	proxy_priv->hw_vport[port_id] = mlx5dr_action_create_dest_vport
			(proxy_priv->dr_ctx, priv->dev_port,
			 MLX5DR_ACTION_FLAG_HWS_FDB);
	if (!proxy_priv->hw_vport[port_id]) {
		DRV_LOG(ERR, "port %u unable to create HWS vport action",
			port_id);
		return -EINVAL;
	}
	return 0;
}

/**
 * Destroys the vport action associated with @p dev device
 * from actions' cache.
 *
 * This function does not destroy any action if there is no action cached
 * for @p dev or proxy port was not configured for HW Steering.
 *
 * This function assumes that E-Switch is enabled and PMD is running with
 * HW Steering configured.
 *
 * @param dev
 *   Pointer to Ethernet device which will be the action destination.
 */
void
flow_hw_destroy_vport_action(struct rte_eth_dev *dev)
{
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	uint16_t port_id = dev->data->port_id;
	uint16_t proxy_port_id = port_id;

	if (mlx5_flow_pick_transfer_proxy(dev, &proxy_port_id, NULL))
		return;
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->hw_vport || !proxy_priv->hw_vport[port_id])
		return;
	mlx5dr_action_destroy(proxy_priv->hw_vport[port_id]);
	proxy_priv->hw_vport[port_id] = NULL;
}

static int
flow_hw_create_vport_actions(struct mlx5_priv *priv)
{
	uint16_t port_id;

	MLX5_ASSERT(!priv->hw_vport);
	priv->hw_vport = mlx5_malloc(MLX5_MEM_ZERO,
				     sizeof(*priv->hw_vport) * RTE_MAX_ETHPORTS,
				     0, SOCKET_ID_ANY);
	if (!priv->hw_vport)
		return -ENOMEM;
	DRV_LOG(DEBUG, "port %u :: creating vport actions", priv->dev_data->port_id);
	DRV_LOG(DEBUG, "port %u ::    domain_id=%u", priv->dev_data->port_id, priv->domain_id);
	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
		struct mlx5_priv *port_priv = rte_eth_devices[port_id].data->dev_private;

		if (!port_priv ||
		    port_priv->domain_id != priv->domain_id)
			continue;
		DRV_LOG(DEBUG, "port %u :: for port_id=%u, calling mlx5dr_action_create_dest_vport() with ibport=%u",
			priv->dev_data->port_id, port_id, port_priv->dev_port);
		priv->hw_vport[port_id] = mlx5dr_action_create_dest_vport
				(priv->dr_ctx, port_priv->dev_port,
				 MLX5DR_ACTION_FLAG_HWS_FDB);
		DRV_LOG(DEBUG, "port %u :: priv->hw_vport[%u]=%p",
			priv->dev_data->port_id, port_id, (void *)priv->hw_vport[port_id]);
		if (!priv->hw_vport[port_id])
			return -EINVAL;
	}
	return 0;
}

static void
flow_hw_free_vport_actions(struct mlx5_priv *priv)
{
	uint16_t port_id;

	if (!priv->hw_vport)
		return;
	for (port_id = 0; port_id < RTE_MAX_ETHPORTS; ++port_id)
		if (priv->hw_vport[port_id])
			mlx5dr_action_destroy(priv->hw_vport[port_id]);
	mlx5_free(priv->hw_vport);
	priv->hw_vport = NULL;
}

static void
flow_hw_create_send_to_kernel_actions(struct mlx5_priv *priv __rte_unused)
{
#ifdef HAVE_MLX5DV_DR_ACTION_CREATE_DEST_ROOT_TABLE
	int action_flag;
	int i;
	bool is_vf_sf_dev = priv->sh->dev_cap.vf || priv->sh->dev_cap.sf;

	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX; i++) {
		if ((!priv->sh->config.dv_esw_en || is_vf_sf_dev) &&
		     i == MLX5DR_TABLE_TYPE_FDB)
			continue;
		action_flag = mlx5_hw_act_flag[1][i];
		priv->hw_send_to_kernel[i] =
				mlx5dr_action_create_dest_root(priv->dr_ctx,
							MLX5_HW_LOWEST_PRIO_ROOT,
							action_flag);
		if (!priv->hw_send_to_kernel[i]) {
			DRV_LOG(WARNING, "Unable to create HWS send to kernel action");
			return;
		}
	}
#endif
}

static void
flow_hw_destroy_send_to_kernel_action(struct mlx5_priv *priv)
{
	int i;
	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX; i++) {
		if (priv->hw_send_to_kernel[i]) {
			mlx5dr_action_destroy(priv->hw_send_to_kernel[i]);
			priv->hw_send_to_kernel[i] = NULL;
		}
	}
}

/**
 * Create an egress pattern template matching on source SQ.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to pattern template on success. NULL otherwise, and rte_errno is set.
 */
static struct rte_flow_pattern_template *
flow_hw_create_tx_repr_sq_pattern_tmpl(struct rte_eth_dev *dev, struct rte_flow_error *error)
{
	struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.egress = 1,
	};
	struct mlx5_rte_flow_item_sq sq_mask = {
		.queue = UINT32_MAX,
	};
	struct rte_flow_item items[] = {
		{
			.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
			.mask = &sq_mask,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &attr, items, error);
}

static __rte_always_inline uint32_t
flow_hw_tx_tag_regc_mask(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t mask = priv->sh->dv_regc0_mask;

	/* Mask is verified during device initialization. Sanity checking here. */
	MLX5_ASSERT(mask != 0);
	/*
	 * Availability of sufficient number of bits in REG_C_0 is verified on initialization.
	 * Sanity checking here.
	 */
	MLX5_ASSERT(rte_popcount32(mask) >= rte_popcount32(priv->vport_meta_mask));
	return mask;
}

static __rte_always_inline uint32_t
flow_hw_tx_tag_regc_value(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t tag;

	/* Mask is verified during device initialization. Sanity checking here. */
	MLX5_ASSERT(priv->vport_meta_mask != 0);
	tag = priv->vport_meta_tag >> (rte_bsf32(priv->vport_meta_mask));
	/*
	 * Availability of sufficient number of bits in REG_C_0 is verified on initialization.
	 * Sanity checking here.
	 */
	MLX5_ASSERT((tag & priv->sh->dv_regc0_mask) == tag);
	return tag;
}

static void
flow_hw_update_action_mask(struct rte_flow_action *action,
			   struct rte_flow_action *mask,
			   enum rte_flow_action_type type,
			   void *conf_v,
			   void *conf_m)
{
	action->type = type;
	action->conf = conf_v;
	mask->type = type;
	mask->conf = conf_m;
}

/**
 * Create an egress actions template with MODIFY_FIELD action for setting unused REG_C_0 bits
 * to vport tag and JUMP action to group 1.
 *
 * If extended metadata mode is enabled, then MODIFY_FIELD action for copying software metadata
 * to REG_C_1 is added as well.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to actions template on success. NULL otherwise, and rte_errno is set.
 */
static struct rte_flow_actions_template *
flow_hw_create_tx_repr_tag_jump_acts_tmpl(struct rte_eth_dev *dev,
					  struct rte_flow_error *error)
{
	uint32_t tag_mask = flow_hw_tx_tag_regc_mask(dev);
	uint32_t tag_value = flow_hw_tx_tag_regc_value(dev);
	struct rte_flow_actions_template_attr attr = {
		.egress = 1,
	};
	struct rte_flow_action_modify_field set_tag_v = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_C_0,
			.offset = rte_bsf32(tag_mask),
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
		},
		.width = rte_popcount32(tag_mask),
	};
	struct rte_flow_action_modify_field set_tag_m = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
		},
		.width = UINT32_MAX,
	};
	struct rte_flow_action_modify_field copy_metadata_v = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_C_1,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_A,
		},
		.width = 32,
	};
	struct rte_flow_action_modify_field copy_metadata_m = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.width = UINT32_MAX,
	};
	struct rte_flow_action_jump jump_v = {
		.group = MLX5_HW_LOWEST_USABLE_GROUP,
	};
	struct rte_flow_action_jump jump_m = {
		.group = UINT32_MAX,
	};
	struct rte_flow_action actions_v[4] = { { 0 } };
	struct rte_flow_action actions_m[4] = { { 0 } };
	unsigned int idx = 0;

	rte_memcpy(set_tag_v.src.value, &tag_value, sizeof(tag_value));
	rte_memcpy(set_tag_m.src.value, &tag_mask, sizeof(tag_mask));
	flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx],
				   RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
				   &set_tag_v, &set_tag_m);
	idx++;
	if (MLX5_SH(dev)->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) {
		flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx],
					   RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
					   &copy_metadata_v, &copy_metadata_m);
		idx++;
	}
	flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx], RTE_FLOW_ACTION_TYPE_JUMP,
				   &jump_v, &jump_m);
	idx++;
	flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx], RTE_FLOW_ACTION_TYPE_END,
				   NULL, NULL);
	idx++;
	MLX5_ASSERT(idx <= RTE_DIM(actions_v));
	return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, error);
}

static void
flow_hw_cleanup_tx_repr_tagging(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (priv->hw_tx_repr_tagging_tbl) {
		flow_hw_table_destroy(dev, priv->hw_tx_repr_tagging_tbl, NULL);
		priv->hw_tx_repr_tagging_tbl = NULL;
	}
	if (priv->hw_tx_repr_tagging_at) {
		flow_hw_actions_template_destroy(dev, priv->hw_tx_repr_tagging_at, NULL);
		priv->hw_tx_repr_tagging_at = NULL;
	}
	if (priv->hw_tx_repr_tagging_pt) {
		flow_hw_pattern_template_destroy(dev, priv->hw_tx_repr_tagging_pt, NULL);
		priv->hw_tx_repr_tagging_pt = NULL;
	}
}

/**
 * Setup templates and table used to create default Tx flow rules. These default rules
 * allow for matching Tx representor traffic using a vport tag placed in unused bits of
 * REG_C_0 register.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, negative errno value otherwise.
 */
static int
flow_hw_setup_tx_repr_tagging(struct rte_eth_dev *dev, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 0,
			.priority = MLX5_HW_LOWEST_PRIO_ROOT,
			.egress = 1,
		},
		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	MLX5_ASSERT(priv->sh->config.dv_esw_en);
	MLX5_ASSERT(priv->sh->config.repr_matching);
	priv->hw_tx_repr_tagging_pt =
		flow_hw_create_tx_repr_sq_pattern_tmpl(dev, error);
	if (!priv->hw_tx_repr_tagging_pt)
		goto err;
	priv->hw_tx_repr_tagging_at =
		flow_hw_create_tx_repr_tag_jump_acts_tmpl(dev, error);
	if (!priv->hw_tx_repr_tagging_at)
		goto err;
	priv->hw_tx_repr_tagging_tbl = flow_hw_table_create(dev, &cfg,
							    &priv->hw_tx_repr_tagging_pt, 1,
							    &priv->hw_tx_repr_tagging_at, 1,
							    error);
	if (!priv->hw_tx_repr_tagging_tbl)
		goto err;
	return 0;
err:
	flow_hw_cleanup_tx_repr_tagging(dev);
	return -rte_errno;
}

static uint32_t
flow_hw_esw_mgr_regc_marker_mask(struct rte_eth_dev *dev)
{
	uint32_t mask = MLX5_SH(dev)->dv_regc0_mask;

	/* Mask is verified during device initialization. */
	MLX5_ASSERT(mask != 0);
	return mask;
}

static uint32_t
flow_hw_esw_mgr_regc_marker(struct rte_eth_dev *dev)
{
	uint32_t mask = MLX5_SH(dev)->dv_regc0_mask;

	/* Mask is verified during device initialization. */
	MLX5_ASSERT(mask != 0);
	return RTE_BIT32(rte_bsf32(mask));
}

/**
 * Creates a flow pattern template used to match on E-Switch Manager.
 * This template is used to set up a table for SQ miss default flow.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev,
					     struct rte_flow_error *error)
{
	struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.transfer = 1,
	};
	struct rte_flow_item_ethdev port_spec = {
		.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
	};
	struct rte_flow_item_ethdev port_mask = {
		.port_id = UINT16_MAX,
	};
	struct mlx5_rte_flow_item_sq sq_mask = {
		.queue = UINT32_MAX,
	};
	struct rte_flow_item items[] = {
		{
			.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
			.spec = &port_spec,
			.mask = &port_mask,
		},
		{
			.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
			.mask = &sq_mask,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &attr, items, error);
}

/**
 * Creates a flow pattern template used to match REG_C_0 and a SQ.
 * Matching on REG_C_0 is set up to match on all bits usable by user-space.
 * If traffic was sent from E-Switch Manager, then all usable bits will be set to 0,
 * except the least significant bit, which will be set to 1.
 *
 * This template is used to set up a table for SQ miss default flow.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev,
					     struct rte_flow_error *error)
{
	struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.transfer = 1,
	};
	struct rte_flow_item_tag reg_c0_spec = {
		.index = (uint8_t)REG_C_0,
	};
	struct rte_flow_item_tag reg_c0_mask = {
		.index = 0xff,
		.data = flow_hw_esw_mgr_regc_marker_mask(dev),
	};
	struct mlx5_rte_flow_item_sq queue_mask = {
		.queue = UINT32_MAX,
	};
	struct rte_flow_item items[] = {
		{
			.type = (enum rte_flow_item_type)
				MLX5_RTE_FLOW_ITEM_TYPE_TAG,
			.spec = &reg_c0_spec,
			.mask = &reg_c0_mask,
		},
		{
			.type = (enum rte_flow_item_type)
				MLX5_RTE_FLOW_ITEM_TYPE_SQ,
			.mask = &queue_mask,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &attr, items, error);
}

/**
 * Creates a flow pattern template with unmasked represented port matching.
 * This template is used to set up a table for default transfer flows
 * directing packets to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_port_pattern_template(struct rte_eth_dev *dev,
					  struct rte_flow_error *error)
{
	struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.transfer = 1,
	};
	struct rte_flow_item_ethdev port_mask = {
		.port_id = UINT16_MAX,
	};
	struct rte_flow_item items[] = {
		{
			.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
			.mask = &port_mask,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &attr, items, error);
}

/*
 * Creating a flow pattern template with all ETH packets matching.
 * This template is used to set up a table for default Tx copy (Tx metadata
 * to REG_C_1) flow rule usage.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev,
						     struct rte_flow_error *error)
{
	struct rte_flow_pattern_template_attr tx_pa_attr = {
		.relaxed_matching = 0,
		.egress = 1,
	};
	struct rte_flow_item_eth promisc = {
		.hdr.dst_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.hdr.ether_type = 0,
	};
	struct rte_flow_item eth_all[] = {
		[0] = {
			.type = RTE_FLOW_ITEM_TYPE_ETH,
			.spec = &promisc,
			.mask = &promisc,
		},
		[1] = {
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &tx_pa_attr, eth_all, error);
}

/*
 * Creating a flow pattern template with all LACP packets matching, only for NIC
 * ingress domain.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_lacp_rx_pattern_template(struct rte_eth_dev *dev, struct rte_flow_error *error)
{
	struct rte_flow_pattern_template_attr pa_attr = {
		.relaxed_matching = 0,
		.ingress = 1,
	};
	struct rte_flow_item_eth lacp_mask = {
		.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.type = 0xFFFF,
	};
	struct rte_flow_item eth_all[] = {
		[0] = {
			.type = RTE_FLOW_ITEM_TYPE_ETH,
			.mask = &lacp_mask,
		},
		[1] = {
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	return flow_hw_pattern_template_create(dev, &pa_attr, eth_all, error);
}

/**
 * Creates a flow actions template with modify field action and masked jump action.
 * Modify field action sets the least significant bit of REG_C_0 (usable by user-space)
 * to 1, meaning that packet was originated from E-Switch Manager. Jump action
 * transfers steering to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow actions template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev,
					       struct rte_flow_error *error)
{
	uint32_t marker_mask = flow_hw_esw_mgr_regc_marker_mask(dev);
	uint32_t marker_bits = flow_hw_esw_mgr_regc_marker(dev);
	struct rte_flow_actions_template_attr attr = {
		.transfer = 1,
	};
	struct rte_flow_action_modify_field set_reg_v = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_C_0,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
		},
		.width = rte_popcount32(marker_mask),
	};
	struct rte_flow_action_modify_field set_reg_m = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
		},
		.width = UINT32_MAX,
	};
	struct rte_flow_action_jump jump_v = {
		.group = MLX5_HW_LOWEST_USABLE_GROUP,
	};
	struct rte_flow_action_jump jump_m = {
		.group = UINT32_MAX,
	};
	struct rte_flow_action actions_v[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &set_reg_v,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_v,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_action actions_m[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &set_reg_m,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_m,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};

	set_reg_v.dst.offset = rte_bsf32(marker_mask);
	rte_memcpy(set_reg_v.src.value, &marker_bits, sizeof(marker_bits));
	rte_memcpy(set_reg_m.src.value, &marker_mask, sizeof(marker_mask));
	return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, error);
}

/**
 * Creates a flow actions template with an unmasked JUMP action. Flows
 * based on this template will perform a jump to some group. This template
 * is used to set up tables for control flows.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param group
 *   Destination group for this action template.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow actions template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_ctrl_jump_actions_template(struct rte_eth_dev *dev,
					  uint32_t group,
					  struct rte_flow_error *error)
{
	struct rte_flow_actions_template_attr attr = {
		.transfer = 1,
	};
	struct rte_flow_action_jump jump_v = {
		.group = group,
	};
	struct rte_flow_action_jump jump_m = {
		.group = UINT32_MAX,
	};
	struct rte_flow_action actions_v[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_v,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_action actions_m[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_m,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};

	return flow_hw_actions_template_create(dev, &attr, actions_v,
					       actions_m, error);
}

/**
 * Creates a flow action template with a unmasked REPRESENTED_PORT action.
 * It is used to create control flow tables.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow action template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_ctrl_port_actions_template(struct rte_eth_dev *dev,
					  struct rte_flow_error *error)
{
	struct rte_flow_actions_template_attr attr = {
		.transfer = 1,
	};
	struct rte_flow_action_ethdev port_v = {
		.port_id = 0,
	};
	struct rte_flow_action actions_v[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
			.conf = &port_v,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_action_ethdev port_m = {
		.port_id = 0,
	};
	struct rte_flow_action actions_m[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
			.conf = &port_m,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};

	return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, error);
}

/*
 * Creating an actions template to use header modify action for register
 * copying. This template is used to set up a table for copy flow.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow actions template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev,
						     struct rte_flow_error *error)
{
	struct rte_flow_actions_template_attr tx_act_attr = {
		.egress = 1,
	};
	const struct rte_flow_action_modify_field mreg_action = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_C_1,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_A,
		},
		.width = 32,
	};
	const struct rte_flow_action_modify_field mreg_mask = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT8_MAX,
			.tag_index = UINT8_MAX,
			.offset = UINT32_MAX,
		},
		.width = UINT32_MAX,
	};
	const struct rte_flow_action_jump jump_action = {
		.group = 1,
	};
	const struct rte_flow_action_jump jump_mask = {
		.group = UINT32_MAX,
	};
	const struct rte_flow_action actions[] = {
		[0] = {
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &mreg_action,
		},
		[1] = {
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_action,
		},
		[2] = {
			.type = RTE_FLOW_ACTION_TYPE_END,
		},
	};
	const struct rte_flow_action masks[] = {
		[0] = {
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &mreg_mask,
		},
		[1] = {
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_mask,
		},
		[2] = {
			.type = RTE_FLOW_ACTION_TYPE_END,
		},
	};

	return flow_hw_actions_template_create(dev, &tx_act_attr, actions,
					       masks, error);
}

/*
 * Creating an actions template to use default miss to re-route packets to the
 * kernel driver stack.
 * On root table, only DEFAULT_MISS action can be used.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow actions template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_lacp_rx_actions_template(struct rte_eth_dev *dev, struct rte_flow_error *error)
{
	struct rte_flow_actions_template_attr act_attr = {
		.ingress = 1,
	};
	const struct rte_flow_action actions[] = {
		[0] = {
			.type = (enum rte_flow_action_type)
				MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
		},
		[1] = {
			.type = RTE_FLOW_ACTION_TYPE_END,
		},
	};

	return flow_hw_actions_template_create(dev, &act_attr, actions, actions, error);
}

/**
 * Creates a control flow table used to transfer traffic from E-Switch Manager
 * and TX queues from group 0 to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param it
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table*
flow_hw_create_ctrl_sq_miss_root_table(struct rte_eth_dev *dev,
				       struct rte_flow_pattern_template *it,
				       struct rte_flow_actions_template *at,
				       struct rte_flow_error *error)
{
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 0,
			.priority = MLX5_HW_LOWEST_PRIO_ROOT,
			.ingress = 0,
			.egress = 0,
			.transfer = 1,
		},
		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error);
}


/**
 * Creates a control flow table used to transfer traffic from E-Switch Manager
 * and TX queues from group 0 to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param it
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table*
flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev *dev,
				  struct rte_flow_pattern_template *it,
				  struct rte_flow_actions_template *at,
				  struct rte_flow_error *error)
{
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 1,
			.priority = MLX5_HW_LOWEST_PRIO_NON_ROOT,
			.ingress = 0,
			.egress = 0,
			.transfer = 1,
		},
		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error);
}

/*
 * Creating the default Tx metadata copy table on NIC Tx group 0.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param pt
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table*
flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev,
					  struct rte_flow_pattern_template *pt,
					  struct rte_flow_actions_template *at,
					  struct rte_flow_error *error)
{
	struct rte_flow_template_table_attr tx_tbl_attr = {
		.flow_attr = {
			.group = 0, /* Root */
			.priority = MLX5_HW_LOWEST_PRIO_ROOT,
			.egress = 1,
		},
		.nb_flows = 1, /* One default flow rule for all. */
	};
	struct mlx5_flow_template_table_cfg tx_tbl_cfg = {
		.attr = tx_tbl_attr,
		.external = false,
	};

	return flow_hw_table_create(dev, &tx_tbl_cfg, &pt, 1, &at, 1, error);
}

/**
 * Creates a control flow table used to transfer traffic
 * from group 0 to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param it
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table *
flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev,
			       struct rte_flow_pattern_template *it,
			       struct rte_flow_actions_template *at,
			       struct rte_flow_error *error)
{
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 0,
			.priority = 0,
			.ingress = 0,
			.egress = 0,
			.transfer = 1,
		},
		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error);
}

/*
 * Create a table on the root group to for the LACP traffic redirecting.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param it
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table *
flow_hw_create_lacp_rx_table(struct rte_eth_dev *dev,
			     struct rte_flow_pattern_template *it,
			     struct rte_flow_actions_template *at,
			     struct rte_flow_error *error)
{
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 0,
			.priority = 0,
			.ingress = 1,
			.egress = 0,
			.transfer = 0,
		},
		.nb_flows = 1,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, error);
}

/**
 * Creates a set of flow tables used to create control flows used
 * when E-Switch is engaged.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, negative values otherwise
 */
static __rte_unused int
flow_hw_create_ctrl_tables(struct rte_eth_dev *dev, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_pattern_template *esw_mgr_items_tmpl = NULL;
	struct rte_flow_pattern_template *regc_sq_items_tmpl = NULL;
	struct rte_flow_pattern_template *port_items_tmpl = NULL;
	struct rte_flow_pattern_template *tx_meta_items_tmpl = NULL;
	struct rte_flow_pattern_template *lacp_rx_items_tmpl = NULL;
	struct rte_flow_actions_template *regc_jump_actions_tmpl = NULL;
	struct rte_flow_actions_template *port_actions_tmpl = NULL;
	struct rte_flow_actions_template *jump_one_actions_tmpl = NULL;
	struct rte_flow_actions_template *tx_meta_actions_tmpl = NULL;
	struct rte_flow_actions_template *lacp_rx_actions_tmpl = NULL;
	uint32_t xmeta = priv->sh->config.dv_xmeta_en;
	uint32_t repr_matching = priv->sh->config.repr_matching;
	int ret;

	/* Create templates and table for default SQ miss flow rules - root table. */
	esw_mgr_items_tmpl = flow_hw_create_ctrl_esw_mgr_pattern_template(dev, error);
	if (!esw_mgr_items_tmpl) {
		DRV_LOG(ERR, "port %u failed to create E-Switch Manager item"
			" template for control flows", dev->data->port_id);
		goto err;
	}
	regc_jump_actions_tmpl = flow_hw_create_ctrl_regc_jump_actions_template(dev, error);
	if (!regc_jump_actions_tmpl) {
		DRV_LOG(ERR, "port %u failed to create REG_C set and jump action template"
			" for control flows", dev->data->port_id);
		goto err;
	}
	MLX5_ASSERT(priv->hw_esw_sq_miss_root_tbl == NULL);
	priv->hw_esw_sq_miss_root_tbl = flow_hw_create_ctrl_sq_miss_root_table
			(dev, esw_mgr_items_tmpl, regc_jump_actions_tmpl, error);
	if (!priv->hw_esw_sq_miss_root_tbl) {
		DRV_LOG(ERR, "port %u failed to create table for default sq miss (root table)"
			" for control flows", dev->data->port_id);
		goto err;
	}
	/* Create templates and table for default SQ miss flow rules - non-root table. */
	regc_sq_items_tmpl = flow_hw_create_ctrl_regc_sq_pattern_template(dev, error);
	if (!regc_sq_items_tmpl) {
		DRV_LOG(ERR, "port %u failed to create SQ item template for"
			" control flows", dev->data->port_id);
		goto err;
	}
	port_actions_tmpl = flow_hw_create_ctrl_port_actions_template(dev, error);
	if (!port_actions_tmpl) {
		DRV_LOG(ERR, "port %u failed to create port action template"
			" for control flows", dev->data->port_id);
		goto err;
	}
	MLX5_ASSERT(priv->hw_esw_sq_miss_tbl == NULL);
	priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev, regc_sq_items_tmpl,
								     port_actions_tmpl, error);
	if (!priv->hw_esw_sq_miss_tbl) {
		DRV_LOG(ERR, "port %u failed to create table for default sq miss (non-root table)"
			" for control flows", dev->data->port_id);
		goto err;
	}
	/* Create templates and table for default FDB jump flow rules. */
	port_items_tmpl = flow_hw_create_ctrl_port_pattern_template(dev, error);
	if (!port_items_tmpl) {
		DRV_LOG(ERR, "port %u failed to create SQ item template for"
			" control flows", dev->data->port_id);
		goto err;
	}
	jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template
			(dev, MLX5_HW_LOWEST_USABLE_GROUP, error);
	if (!jump_one_actions_tmpl) {
		DRV_LOG(ERR, "port %u failed to create jump action template"
			" for control flows", dev->data->port_id);
		goto err;
	}
	MLX5_ASSERT(priv->hw_esw_zero_tbl == NULL);
	priv->hw_esw_zero_tbl = flow_hw_create_ctrl_jump_table(dev, port_items_tmpl,
							       jump_one_actions_tmpl,
							       error);
	if (!priv->hw_esw_zero_tbl) {
		DRV_LOG(ERR, "port %u failed to create table for default jump to group 1"
			" for control flows", dev->data->port_id);
		goto err;
	}
	/* Create templates and table for default Tx metadata copy flow rule. */
	if (!repr_matching && xmeta == MLX5_XMETA_MODE_META32_HWS) {
		tx_meta_items_tmpl =
			flow_hw_create_tx_default_mreg_copy_pattern_template(dev, error);
		if (!tx_meta_items_tmpl) {
			DRV_LOG(ERR, "port %u failed to Tx metadata copy pattern"
				" template for control flows", dev->data->port_id);
			goto err;
		}
		tx_meta_actions_tmpl =
			flow_hw_create_tx_default_mreg_copy_actions_template(dev, error);
		if (!tx_meta_actions_tmpl) {
			DRV_LOG(ERR, "port %u failed to Tx metadata copy actions"
				" template for control flows", dev->data->port_id);
			goto err;
		}
		MLX5_ASSERT(priv->hw_tx_meta_cpy_tbl == NULL);
		priv->hw_tx_meta_cpy_tbl =
			flow_hw_create_tx_default_mreg_copy_table(dev, tx_meta_items_tmpl,
								  tx_meta_actions_tmpl, error);
		if (!priv->hw_tx_meta_cpy_tbl) {
			DRV_LOG(ERR, "port %u failed to create table for default"
				" Tx metadata copy flow rule", dev->data->port_id);
			goto err;
		}
	}
	/* Create LACP default miss table. */
	if (!priv->sh->config.lacp_by_user && priv->pf_bond >= 0) {
		lacp_rx_items_tmpl = flow_hw_create_lacp_rx_pattern_template(dev, error);
		if (!lacp_rx_items_tmpl) {
			DRV_LOG(ERR, "port %u failed to create pattern template"
				" for LACP Rx traffic", dev->data->port_id);
			goto err;
		}
		lacp_rx_actions_tmpl = flow_hw_create_lacp_rx_actions_template(dev, error);
		if (!lacp_rx_actions_tmpl) {
			DRV_LOG(ERR, "port %u failed to create actions template"
				" for LACP Rx traffic", dev->data->port_id);
			goto err;
		}
		priv->hw_lacp_rx_tbl = flow_hw_create_lacp_rx_table(dev, lacp_rx_items_tmpl,
								    lacp_rx_actions_tmpl, error);
		if (!priv->hw_lacp_rx_tbl) {
			DRV_LOG(ERR, "port %u failed to create template table for"
				" for LACP Rx traffic", dev->data->port_id);
			goto err;
		}
	}
	return 0;
err:
	/* Do not overwrite the rte_errno. */
	ret = -rte_errno;
	if (ret == 0)
		ret = rte_flow_error_set(error, EINVAL,
					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					 "Failed to create control tables.");
	if (priv->hw_tx_meta_cpy_tbl) {
		flow_hw_table_destroy(dev, priv->hw_tx_meta_cpy_tbl, NULL);
		priv->hw_tx_meta_cpy_tbl = NULL;
	}
	if (priv->hw_esw_zero_tbl) {
		flow_hw_table_destroy(dev, priv->hw_esw_zero_tbl, NULL);
		priv->hw_esw_zero_tbl = NULL;
	}
	if (priv->hw_esw_sq_miss_tbl) {
		flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_tbl, NULL);
		priv->hw_esw_sq_miss_tbl = NULL;
	}
	if (priv->hw_esw_sq_miss_root_tbl) {
		flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_root_tbl, NULL);
		priv->hw_esw_sq_miss_root_tbl = NULL;
	}
	if (lacp_rx_actions_tmpl)
		flow_hw_actions_template_destroy(dev, lacp_rx_actions_tmpl, NULL);
	if (tx_meta_actions_tmpl)
		flow_hw_actions_template_destroy(dev, tx_meta_actions_tmpl, NULL);
	if (jump_one_actions_tmpl)
		flow_hw_actions_template_destroy(dev, jump_one_actions_tmpl, NULL);
	if (port_actions_tmpl)
		flow_hw_actions_template_destroy(dev, port_actions_tmpl, NULL);
	if (regc_jump_actions_tmpl)
		flow_hw_actions_template_destroy(dev, regc_jump_actions_tmpl, NULL);
	if (lacp_rx_items_tmpl)
		flow_hw_pattern_template_destroy(dev, lacp_rx_items_tmpl, NULL);
	if (tx_meta_items_tmpl)
		flow_hw_pattern_template_destroy(dev, tx_meta_items_tmpl, NULL);
	if (port_items_tmpl)
		flow_hw_pattern_template_destroy(dev, port_items_tmpl, NULL);
	if (regc_sq_items_tmpl)
		flow_hw_pattern_template_destroy(dev, regc_sq_items_tmpl, NULL);
	if (esw_mgr_items_tmpl)
		flow_hw_pattern_template_destroy(dev, esw_mgr_items_tmpl, NULL);
	return ret;
}

static void
flow_hw_ct_mng_destroy(struct rte_eth_dev *dev,
		       struct mlx5_aso_ct_pools_mng *ct_mng)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	mlx5_aso_ct_queue_uninit(priv->sh, ct_mng);
	mlx5_free(ct_mng);
}

static void
flow_hw_ct_pool_destroy(struct rte_eth_dev *dev __rte_unused,
			struct mlx5_aso_ct_pool *pool)
{
	if (pool->dr_action)
		mlx5dr_action_destroy(pool->dr_action);
	if (pool->devx_obj)
		claim_zero(mlx5_devx_cmd_destroy(pool->devx_obj));
	if (pool->cts)
		mlx5_ipool_destroy(pool->cts);
	mlx5_free(pool);
}

static struct mlx5_aso_ct_pool *
flow_hw_ct_pool_create(struct rte_eth_dev *dev,
		       const struct rte_flow_port_attr *port_attr)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_pool *pool;
	struct mlx5_devx_obj *obj;
	uint32_t nb_cts = rte_align32pow2(port_attr->nb_conn_tracks);
	uint32_t log_obj_size = rte_log2_u32(nb_cts);
	struct mlx5_indexed_pool_config cfg = {
		.size = sizeof(struct mlx5_aso_ct_action),
		.trunk_size = 1 << 12,
		.per_core_cache = 1 << 13,
		.need_lock = 1,
		.release_mem_en = !!priv->sh->config.reclaim_mode,
		.malloc = mlx5_malloc,
		.free = mlx5_free,
		.type = "mlx5_hw_ct_action",
	};
	int reg_id;
	uint32_t flags;

	if (port_attr->flags & RTE_FLOW_PORT_FLAG_SHARE_INDIRECT) {
		DRV_LOG(ERR, "Connection tracking is not supported "
			     "in cross vHCA sharing mode");
		rte_errno = ENOTSUP;
		return NULL;
	}
	pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
	if (!pool) {
		rte_errno = ENOMEM;
		return NULL;
	}
	obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->cdev->ctx,
							  priv->sh->cdev->pdn,
							  log_obj_size);
	if (!obj) {
		rte_errno = ENODATA;
		DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
		goto err;
	}
	pool->devx_obj = obj;
	reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, NULL);
	flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX;
	if (priv->sh->config.dv_esw_en && priv->master)
		flags |= MLX5DR_ACTION_FLAG_HWS_FDB;
	pool->dr_action = mlx5dr_action_create_aso_ct(priv->dr_ctx,
						      (struct mlx5dr_devx_obj *)obj,
						      reg_id - REG_C_0, flags);
	if (!pool->dr_action)
		goto err;
	/*
	 * No need for local cache if CT number is a small number. Since
	 * flow insertion rate will be very limited in that case. Here let's
	 * set the number to less than default trunk size 4K.
	 */
	if (nb_cts <= cfg.trunk_size) {
		cfg.per_core_cache = 0;
		cfg.trunk_size = nb_cts;
	} else if (nb_cts <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
		cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
	}
	pool->cts = mlx5_ipool_create(&cfg);
	if (!pool->cts)
		goto err;
	pool->sq = priv->ct_mng->aso_sqs;
	/* Assign the last extra ASO SQ as public SQ. */
	pool->shared_sq = &priv->ct_mng->aso_sqs[priv->nb_queue - 1];
	return pool;
err:
	flow_hw_ct_pool_destroy(dev, pool);
	return NULL;
}

static void
flow_hw_destroy_vlan(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	enum mlx5dr_table_type i;

	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX; i++) {
		if (priv->hw_pop_vlan[i]) {
			mlx5dr_action_destroy(priv->hw_pop_vlan[i]);
			priv->hw_pop_vlan[i] = NULL;
		}
		if (priv->hw_push_vlan[i]) {
			mlx5dr_action_destroy(priv->hw_push_vlan[i]);
			priv->hw_push_vlan[i] = NULL;
		}
	}
}

static int
flow_hw_create_vlan(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	enum mlx5dr_table_type i;
	const enum mlx5dr_action_flags flags[MLX5DR_TABLE_TYPE_MAX] = {
		MLX5DR_ACTION_FLAG_HWS_RX,
		MLX5DR_ACTION_FLAG_HWS_TX,
		MLX5DR_ACTION_FLAG_HWS_FDB
	};

	/* rte_errno is set in the mlx5dr_action* functions. */
	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i <= MLX5DR_TABLE_TYPE_NIC_TX; i++) {
		priv->hw_pop_vlan[i] =
			mlx5dr_action_create_pop_vlan(priv->dr_ctx, flags[i]);
		if (!priv->hw_pop_vlan[i])
			return -rte_errno;
		priv->hw_push_vlan[i] =
			mlx5dr_action_create_push_vlan(priv->dr_ctx, flags[i]);
		if (!priv->hw_pop_vlan[i])
			return -rte_errno;
	}
	if (priv->sh->config.dv_esw_en && priv->master) {
		priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB] =
			mlx5dr_action_create_pop_vlan
				(priv->dr_ctx, MLX5DR_ACTION_FLAG_HWS_FDB);
		if (!priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB])
			return -rte_errno;
		priv->hw_push_vlan[MLX5DR_TABLE_TYPE_FDB] =
			mlx5dr_action_create_push_vlan
				(priv->dr_ctx, MLX5DR_ACTION_FLAG_HWS_FDB);
		if (!priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB])
			return -rte_errno;
	}
	return 0;
}

static void
flow_hw_cleanup_ctrl_rx_tables(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	unsigned int i;
	unsigned int j;

	if (!priv->hw_ctrl_rx)
		return;
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
		for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
			struct rte_flow_template_table *tbl = priv->hw_ctrl_rx->tables[i][j].tbl;
			struct rte_flow_pattern_template *pt = priv->hw_ctrl_rx->tables[i][j].pt;

			if (tbl)
				claim_zero(flow_hw_table_destroy(dev, tbl, NULL));
			if (pt)
				claim_zero(flow_hw_pattern_template_destroy(dev, pt, NULL));
		}
	}
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++i) {
		struct rte_flow_actions_template *at = priv->hw_ctrl_rx->rss[i];

		if (at)
			claim_zero(flow_hw_actions_template_destroy(dev, at, NULL));
	}
	mlx5_free(priv->hw_ctrl_rx);
	priv->hw_ctrl_rx = NULL;
}

static uint64_t
flow_hw_ctrl_rx_rss_type_hash_types(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	switch (rss_type) {
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_NON_IP:
		return 0;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4:
		return RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | RTE_ETH_RSS_NONFRAG_IPV4_OTHER;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
		return RTE_ETH_RSS_NONFRAG_IPV4_UDP;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
		return RTE_ETH_RSS_NONFRAG_IPV4_TCP;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6:
		return RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_OTHER;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
		return RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_IPV6_UDP_EX;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
		return RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_IPV6_TCP_EX;
	default:
		/* Should not reach here. */
		MLX5_ASSERT(false);
		return 0;
	}
}

static struct rte_flow_actions_template *
flow_hw_create_ctrl_rx_rss_template(struct rte_eth_dev *dev,
				    const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_actions_template_attr attr = {
		.ingress = 1,
	};
	uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
	struct rte_flow_action_rss rss_conf = {
		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
		.level = 0,
		.types = 0,
		.key_len = priv->rss_conf.rss_key_len,
		.key = priv->rss_conf.rss_key,
		.queue_num = priv->reta_idx_n,
		.queue = queue,
	};
	struct rte_flow_action actions[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_RSS,
			.conf = &rss_conf,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_action masks[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_RSS,
			.conf = &rss_conf,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_actions_template *at;
	struct rte_flow_error error;
	unsigned int i;

	MLX5_ASSERT(priv->reta_idx_n > 0 && priv->reta_idx);
	/* Select proper RSS hash types and based on that configure the actions template. */
	rss_conf.types = flow_hw_ctrl_rx_rss_type_hash_types(rss_type);
	if (rss_conf.types) {
		for (i = 0; i < priv->reta_idx_n; ++i)
			queue[i] = (*priv->reta_idx)[i];
	} else {
		rss_conf.queue_num = 1;
		queue[0] = (*priv->reta_idx)[0];
	}
	at = flow_hw_actions_template_create(dev, &attr, actions, masks, &error);
	if (!at)
		DRV_LOG(ERR,
			"Failed to create ctrl flow actions template: rte_errno(%d), type(%d): %s",
			rte_errno, error.type,
			error.message ? error.message : "(no stated reason)");
	return at;
}

static uint32_t ctrl_rx_rss_priority_map[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX] = {
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_NON_IP] = MLX5_HW_CTRL_RX_PRIO_L2,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4] = MLX5_HW_CTRL_RX_PRIO_L3,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP] = MLX5_HW_CTRL_RX_PRIO_L4,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP] = MLX5_HW_CTRL_RX_PRIO_L4,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6] = MLX5_HW_CTRL_RX_PRIO_L3,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP] = MLX5_HW_CTRL_RX_PRIO_L4,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP] = MLX5_HW_CTRL_RX_PRIO_L4,
};

static uint32_t ctrl_rx_nb_flows_map[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX] = {
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN] = MLX5_MAX_VLAN_IDS,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN] = MLX5_MAX_VLAN_IDS,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN] = MLX5_MAX_VLAN_IDS,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC] = MLX5_MAX_UC_MAC_ADDRESSES,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN] =
			MLX5_MAX_UC_MAC_ADDRESSES * MLX5_MAX_VLAN_IDS,
};

static struct rte_flow_template_table_attr
flow_hw_get_ctrl_rx_table_attr(enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
			       const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	return (struct rte_flow_template_table_attr){
		.flow_attr = {
			.group = 0,
			.priority = ctrl_rx_rss_priority_map[rss_type],
			.ingress = 1,
		},
		.nb_flows = ctrl_rx_nb_flows_map[eth_pattern_type],
	};
}

static struct rte_flow_item
flow_hw_get_ctrl_rx_eth_item(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
	struct rte_flow_item item = {
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.mask = NULL,
	};

	switch (eth_pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
		item.mask = &ctrl_rx_eth_promisc_mask;
		break;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
		item.mask = &ctrl_rx_eth_mcast_mask;
		break;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		item.mask = &ctrl_rx_eth_dmac_mask;
		break;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
		item.mask = &ctrl_rx_eth_ipv4_mcast_mask;
		break;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
		item.mask = &ctrl_rx_eth_ipv6_mcast_mask;
		break;
	default:
		/* Should not reach here - ETH mask must be present. */
		item.type = RTE_FLOW_ITEM_TYPE_END;
		MLX5_ASSERT(false);
		break;
	}
	return item;
}

static struct rte_flow_item
flow_hw_get_ctrl_rx_vlan_item(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
	struct rte_flow_item item = {
		.type = RTE_FLOW_ITEM_TYPE_VOID,
		.mask = NULL,
	};

	switch (eth_pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		item.type = RTE_FLOW_ITEM_TYPE_VLAN;
		item.mask = &rte_flow_item_vlan_mask;
		break;
	default:
		/* Nothing to update. */
		break;
	}
	return item;
}

static struct rte_flow_item
flow_hw_get_ctrl_rx_l3_item(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct rte_flow_item item = {
		.type = RTE_FLOW_ITEM_TYPE_VOID,
		.mask = NULL,
	};

	switch (rss_type) {
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
		item.type = RTE_FLOW_ITEM_TYPE_IPV4;
		break;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
		item.type = RTE_FLOW_ITEM_TYPE_IPV6;
		break;
	default:
		/* Nothing to update. */
		break;
	}
	return item;
}

static struct rte_flow_item
flow_hw_get_ctrl_rx_l4_item(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct rte_flow_item item = {
		.type = RTE_FLOW_ITEM_TYPE_VOID,
		.mask = NULL,
	};

	switch (rss_type) {
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
		item.type = RTE_FLOW_ITEM_TYPE_UDP;
		break;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
		item.type = RTE_FLOW_ITEM_TYPE_TCP;
		break;
	default:
		/* Nothing to update. */
		break;
	}
	return item;
}

static struct rte_flow_pattern_template *
flow_hw_create_ctrl_rx_pattern_template
		(struct rte_eth_dev *dev,
		 const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
		 const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	const struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.ingress = 1,
	};
	struct rte_flow_item items[] = {
		/* Matching patterns */
		flow_hw_get_ctrl_rx_eth_item(eth_pattern_type),
		flow_hw_get_ctrl_rx_vlan_item(eth_pattern_type),
		flow_hw_get_ctrl_rx_l3_item(rss_type),
		flow_hw_get_ctrl_rx_l4_item(rss_type),
		/* Terminate pattern */
		{ .type = RTE_FLOW_ITEM_TYPE_END }
	};

	return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}

static int
flow_hw_create_ctrl_rx_tables(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	unsigned int i;
	unsigned int j;
	int ret;

	MLX5_ASSERT(!priv->hw_ctrl_rx);
	priv->hw_ctrl_rx = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*priv->hw_ctrl_rx),
				       RTE_CACHE_LINE_SIZE, rte_socket_id());
	if (!priv->hw_ctrl_rx) {
		DRV_LOG(ERR, "Failed to allocate memory for Rx control flow tables");
		rte_errno = ENOMEM;
		return -rte_errno;
	}
	/* Create all pattern template variants. */
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
		enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type = i;

		for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
			const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type = j;
			struct rte_flow_template_table_attr attr;
			struct rte_flow_pattern_template *pt;

			attr = flow_hw_get_ctrl_rx_table_attr(eth_pattern_type, rss_type);
			pt = flow_hw_create_ctrl_rx_pattern_template(dev, eth_pattern_type,
								     rss_type);
			if (!pt)
				goto err;
			priv->hw_ctrl_rx->tables[i][j].attr = attr;
			priv->hw_ctrl_rx->tables[i][j].pt = pt;
		}
	}
	return 0;
err:
	ret = rte_errno;
	flow_hw_cleanup_ctrl_rx_tables(dev);
	rte_errno = ret;
	return -ret;
}

void
mlx5_flow_hw_cleanup_ctrl_rx_templates(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_hw_ctrl_rx *hw_ctrl_rx;
	unsigned int i;
	unsigned int j;

	if (!priv->dr_ctx)
		return;
	if (!priv->hw_ctrl_rx)
		return;
	hw_ctrl_rx = priv->hw_ctrl_rx;
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
		for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
			struct mlx5_flow_hw_ctrl_rx_table *tmpls = &hw_ctrl_rx->tables[i][j];

			if (tmpls->tbl) {
				claim_zero(flow_hw_table_destroy(dev, tmpls->tbl, NULL));
				tmpls->tbl = NULL;
			}
		}
	}
	for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
		if (hw_ctrl_rx->rss[j]) {
			claim_zero(flow_hw_actions_template_destroy(dev, hw_ctrl_rx->rss[j], NULL));
			hw_ctrl_rx->rss[j] = NULL;
		}
	}
}

/**
 * Copy the provided HWS configuration to a newly allocated buffer.
 *
 * @param[in] port_attr
 *   Port configuration attributes.
 * @param[in] nb_queue
 *   Number of queue.
 * @param[in] queue_attr
 *   Array that holds attributes for each flow queue.
 *
 * @return
 *   Pointer to copied HWS configuration is returned on success.
 *   Otherwise, NULL is returned and rte_errno is set.
 */
static struct mlx5_flow_hw_attr *
flow_hw_alloc_copy_config(const struct rte_flow_port_attr *port_attr,
			  const uint16_t nb_queue,
			  const struct rte_flow_queue_attr *queue_attr[],
			  struct rte_flow_error *error)
{
	struct mlx5_flow_hw_attr *hw_attr;
	size_t hw_attr_size;
	unsigned int i;

	hw_attr_size = sizeof(*hw_attr) + nb_queue * sizeof(*hw_attr->queue_attr);
	hw_attr = mlx5_malloc(MLX5_MEM_ZERO, hw_attr_size, 0, SOCKET_ID_ANY);
	if (!hw_attr) {
		rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				   "Not enough memory to store configuration");
		return NULL;
	}
	memcpy(&hw_attr->port_attr, port_attr, sizeof(*port_attr));
	hw_attr->nb_queue = nb_queue;
	/* Queue attributes are placed after the mlx5_flow_hw_attr. */
	hw_attr->queue_attr = (struct rte_flow_queue_attr *)(hw_attr + 1);
	for (i = 0; i < nb_queue; ++i)
		memcpy(&hw_attr->queue_attr[i], queue_attr[i], sizeof(hw_attr->queue_attr[i]));
	return hw_attr;
}

/**
 * Compares the preserved HWS configuration with the provided one.
 *
 * @param[in] hw_attr
 *   Pointer to preserved HWS configuration.
 * @param[in] new_pa
 *   Port configuration attributes to compare.
 * @param[in] new_nbq
 *   Number of queues to compare.
 * @param[in] new_qa
 *   Array that holds attributes for each flow queue.
 *
 * @return
 *   True if configurations are the same, false otherwise.
 */
static bool
flow_hw_compare_config(const struct mlx5_flow_hw_attr *hw_attr,
		       const struct rte_flow_port_attr *new_pa,
		       const uint16_t new_nbq,
		       const struct rte_flow_queue_attr *new_qa[])
{
	const struct rte_flow_port_attr *old_pa = &hw_attr->port_attr;
	const uint16_t old_nbq = hw_attr->nb_queue;
	const struct rte_flow_queue_attr *old_qa = hw_attr->queue_attr;
	unsigned int i;

	if (old_pa->nb_counters != new_pa->nb_counters ||
	    old_pa->nb_aging_objects != new_pa->nb_aging_objects ||
	    old_pa->nb_meters != new_pa->nb_meters ||
	    old_pa->nb_conn_tracks != new_pa->nb_conn_tracks ||
	    old_pa->flags != new_pa->flags)
		return false;
	if (old_nbq != new_nbq)
		return false;
	for (i = 0; i < old_nbq; ++i)
		if (old_qa[i].size != new_qa[i]->size)
			return false;
	return true;
}

/**
 * Configure port HWS resources.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] port_attr
 *   Port configuration attributes.
 * @param[in] nb_queue
 *   Number of queue.
 * @param[in] queue_attr
 *   Array that holds attributes for each flow queue.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_configure(struct rte_eth_dev *dev,
		  const struct rte_flow_port_attr *port_attr,
		  uint16_t nb_queue,
		  const struct rte_flow_queue_attr *queue_attr[],
		  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_priv *host_priv = NULL;
	struct mlx5dr_context *dr_ctx = NULL;
	struct mlx5dr_context_attr dr_ctx_attr = {0};
	struct mlx5_hw_q *hw_q;
	struct mlx5_hw_q_job *job = NULL;
	uint32_t mem_size, i, j;
	struct mlx5_indexed_pool_config cfg = {
		.size = sizeof(struct mlx5_action_construct_data),
		.trunk_size = 4096,
		.need_lock = 1,
		.release_mem_en = !!priv->sh->config.reclaim_mode,
		.malloc = mlx5_malloc,
		.free = mlx5_free,
		.type = "mlx5_hw_action_construct_data",
	};
	/*
	 * Adds one queue to be used by PMD.
	 * The last queue will be used by the PMD.
	 */
	uint16_t nb_q_updated = 0;
	struct rte_flow_queue_attr **_queue_attr = NULL;
	struct rte_flow_queue_attr ctrl_queue_attr = {0};
	bool is_proxy = !!(priv->sh->config.dv_esw_en && priv->master);
	int ret = 0;
	uint32_t action_flags;

	if (!port_attr || !nb_queue || !queue_attr) {
		rte_errno = EINVAL;
		goto err;
	}
	/*
	 * Calling rte_flow_configure() again is allowed if and only if
	 * provided configuration matches the initially provided one.
	 */
	if (priv->dr_ctx) {
		MLX5_ASSERT(priv->hw_attr != NULL);
		for (i = 0; i < priv->nb_queue; i++) {
			hw_q = &priv->hw_q[i];
			/* Make sure all queues are empty. */
			if (hw_q->size != hw_q->job_idx) {
				rte_errno = EBUSY;
				goto err;
			}
		}
		if (flow_hw_compare_config(priv->hw_attr, port_attr, nb_queue, queue_attr))
			return 0;
		else
			return rte_flow_error_set(error, ENOTSUP,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
						  "Changing HWS configuration attributes "
						  "is not supported");
	}
	priv->hw_attr = flow_hw_alloc_copy_config(port_attr, nb_queue, queue_attr, error);
	if (!priv->hw_attr) {
		ret = -rte_errno;
		goto err;
	}
	ctrl_queue_attr.size = queue_attr[0]->size;
	nb_q_updated = nb_queue + 1;
	_queue_attr = mlx5_malloc(MLX5_MEM_ZERO,
				  nb_q_updated *
				  sizeof(struct rte_flow_queue_attr *),
				  64, SOCKET_ID_ANY);
	if (!_queue_attr) {
		rte_errno = ENOMEM;
		goto err;
	}

	memcpy(_queue_attr, queue_attr, sizeof(void *) * nb_queue);
	_queue_attr[nb_queue] = &ctrl_queue_attr;
	priv->acts_ipool = mlx5_ipool_create(&cfg);
	if (!priv->acts_ipool)
		goto err;
	/* Allocate the queue job descriptor LIFO. */
	mem_size = sizeof(priv->hw_q[0]) * nb_q_updated;
	for (i = 0; i < nb_q_updated; i++) {
		/*
		 * Check if the queues' size are all the same as the
		 * limitation from HWS layer.
		 */
		if (_queue_attr[i]->size != _queue_attr[0]->size) {
			rte_errno = EINVAL;
			goto err;
		}
		mem_size += (sizeof(struct mlx5_hw_q_job *) +
			    sizeof(struct mlx5_hw_q_job) +
			    sizeof(uint8_t) * MLX5_ENCAP_MAX_LEN +
			    sizeof(uint8_t) * MLX5_PUSH_MAX_LEN +
			    sizeof(struct mlx5_modification_cmd) *
			    MLX5_MHDR_MAX_CMD +
			    sizeof(struct rte_flow_item) *
			    MLX5_HW_MAX_ITEMS +
				sizeof(struct rte_flow_hw)) *
			    _queue_attr[i]->size;
	}
	priv->hw_q = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
				 64, SOCKET_ID_ANY);
	if (!priv->hw_q) {
		rte_errno = ENOMEM;
		goto err;
	}
	for (i = 0; i < nb_q_updated; i++) {
		char mz_name[RTE_MEMZONE_NAMESIZE];
		uint8_t *encap = NULL, *push = NULL;
		struct mlx5_modification_cmd *mhdr_cmd = NULL;
		struct rte_flow_item *items = NULL;
		struct rte_flow_hw *upd_flow = NULL;

		priv->hw_q[i].job_idx = _queue_attr[i]->size;
		priv->hw_q[i].size = _queue_attr[i]->size;
		if (i == 0)
			priv->hw_q[i].job = (struct mlx5_hw_q_job **)
					    &priv->hw_q[nb_q_updated];
		else
			priv->hw_q[i].job = (struct mlx5_hw_q_job **)
				&job[_queue_attr[i - 1]->size - 1].upd_flow[1];
		job = (struct mlx5_hw_q_job *)
		      &priv->hw_q[i].job[_queue_attr[i]->size];
		mhdr_cmd = (struct mlx5_modification_cmd *)
			   &job[_queue_attr[i]->size];
		encap = (uint8_t *)
			 &mhdr_cmd[_queue_attr[i]->size * MLX5_MHDR_MAX_CMD];
		push = (uint8_t *)
			 &encap[_queue_attr[i]->size * MLX5_ENCAP_MAX_LEN];
		items = (struct rte_flow_item *)
			 &push[_queue_attr[i]->size * MLX5_PUSH_MAX_LEN];
		upd_flow = (struct rte_flow_hw *)
			&items[_queue_attr[i]->size * MLX5_HW_MAX_ITEMS];
		for (j = 0; j < _queue_attr[i]->size; j++) {
			job[j].mhdr_cmd = &mhdr_cmd[j * MLX5_MHDR_MAX_CMD];
			job[j].encap_data = &encap[j * MLX5_ENCAP_MAX_LEN];
			job[j].push_data = &push[j * MLX5_PUSH_MAX_LEN];
			job[j].items = &items[j * MLX5_HW_MAX_ITEMS];
			job[j].upd_flow = &upd_flow[j];
			priv->hw_q[i].job[j] = &job[j];
		}
		snprintf(mz_name, sizeof(mz_name), "port_%u_indir_act_cq_%u",
			 dev->data->port_id, i);
		priv->hw_q[i].indir_cq = rte_ring_create(mz_name,
				_queue_attr[i]->size, SOCKET_ID_ANY,
				RING_F_SP_ENQ | RING_F_SC_DEQ |
				RING_F_EXACT_SZ);
		if (!priv->hw_q[i].indir_cq)
			goto err;
		snprintf(mz_name, sizeof(mz_name), "port_%u_indir_act_iq_%u",
			 dev->data->port_id, i);
		priv->hw_q[i].indir_iq = rte_ring_create(mz_name,
				_queue_attr[i]->size, SOCKET_ID_ANY,
				RING_F_SP_ENQ | RING_F_SC_DEQ |
				RING_F_EXACT_SZ);
		if (!priv->hw_q[i].indir_iq)
			goto err;
	}
	dr_ctx_attr.pd = priv->sh->cdev->pd;
	dr_ctx_attr.queues = nb_q_updated;
	/* Queue size should all be the same. Take the first one. */
	dr_ctx_attr.queue_size = _queue_attr[0]->size;
	if (port_attr->flags & RTE_FLOW_PORT_FLAG_SHARE_INDIRECT) {
		struct rte_eth_dev *host_dev = NULL;
		uint16_t port_id;

		MLX5_ASSERT(rte_eth_dev_is_valid_port(port_attr->host_port_id));
		if (is_proxy) {
			DRV_LOG(ERR, "cross vHCA shared mode not supported "
				"for E-Switch confgiurations");
			rte_errno = ENOTSUP;
			goto err;
		}
		MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
			if (port_id == port_attr->host_port_id) {
				host_dev = &rte_eth_devices[port_id];
				break;
			}
		}
		if (!host_dev || host_dev == dev ||
		    !host_dev->data || !host_dev->data->dev_private) {
			DRV_LOG(ERR, "Invalid cross vHCA host port %u",
				port_attr->host_port_id);
			rte_errno = EINVAL;
			goto err;
		}
		host_priv = host_dev->data->dev_private;
		if (host_priv->sh->cdev->ctx == priv->sh->cdev->ctx) {
			DRV_LOG(ERR, "Sibling ports %u and %u do not "
				     "require cross vHCA sharing mode",
				dev->data->port_id, port_attr->host_port_id);
			rte_errno = EINVAL;
			goto err;
		}
		if (host_priv->shared_host) {
			DRV_LOG(ERR, "Host port %u is not the sharing base",
				port_attr->host_port_id);
			rte_errno = EINVAL;
			goto err;
		}
		if (port_attr->nb_counters ||
		    port_attr->nb_aging_objects ||
		    port_attr->nb_meters ||
		    port_attr->nb_conn_tracks) {
			DRV_LOG(ERR,
				"Object numbers on guest port must be zeros");
			rte_errno = EINVAL;
			goto err;
		}
		dr_ctx_attr.shared_ibv_ctx = host_priv->sh->cdev->ctx;
		priv->shared_host = host_dev;
		__atomic_fetch_add(&host_priv->shared_refcnt, 1, __ATOMIC_RELAXED);
	}
	dr_ctx = mlx5dr_context_open(priv->sh->cdev->ctx, &dr_ctx_attr);
	/* rte_errno has been updated by HWS layer. */
	if (!dr_ctx)
		goto err;
	priv->dr_ctx = dr_ctx;
	priv->nb_queue = nb_q_updated;
	rte_spinlock_init(&priv->hw_ctrl_lock);
	LIST_INIT(&priv->hw_ctrl_flows);
	LIST_INIT(&priv->hw_ext_ctrl_flows);
	ret = flow_hw_create_ctrl_rx_tables(dev);
	if (ret) {
		rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				   "Failed to set up Rx control flow templates");
		goto err;
	}
	/* Initialize quotas */
	if (port_attr->nb_quotas || (host_priv && host_priv->quota_ctx.devx_obj)) {
		ret = mlx5_flow_quota_init(dev, port_attr->nb_quotas);
		if (ret) {
			rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					   "Failed to initialize quota.");
			goto err;
		}
	}
	/* Initialize meter library*/
	if (port_attr->nb_meters || (host_priv && host_priv->hws_mpool))
		if (mlx5_flow_meter_init(dev, port_attr->nb_meters, 0, 0, nb_q_updated))
			goto err;
	/* Add global actions. */
	for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
		uint32_t act_flags = 0;

		act_flags = mlx5_hw_act_flag[i][0] | mlx5_hw_act_flag[i][1];
		if (is_proxy)
			act_flags |= mlx5_hw_act_flag[i][2];
		priv->hw_drop[i] = mlx5dr_action_create_dest_drop(priv->dr_ctx, act_flags);
		if (!priv->hw_drop[i])
			goto err;
		priv->hw_tag[i] = mlx5dr_action_create_tag
			(priv->dr_ctx, mlx5_hw_act_flag[i][0]);
		if (!priv->hw_tag[i])
			goto err;
	}
	if (priv->sh->config.dv_esw_en && priv->sh->config.repr_matching) {
		ret = flow_hw_setup_tx_repr_tagging(dev, error);
		if (ret)
			goto err;
	}
	/*
	 * DEFAULT_MISS action have different behaviors in different domains.
	 * In FDB, it will steering the packets to the E-switch manager.
	 * In NIC Rx root, it will steering the packet to the kernel driver stack.
	 * An action with all bits set in the flag can be created and the HWS
	 * layer will translate it properly when being used in different rules.
	 */
	action_flags = MLX5DR_ACTION_FLAG_ROOT_RX | MLX5DR_ACTION_FLAG_HWS_RX |
		       MLX5DR_ACTION_FLAG_ROOT_TX | MLX5DR_ACTION_FLAG_HWS_TX;
	if (is_proxy)
		action_flags |= (MLX5DR_ACTION_FLAG_ROOT_FDB | MLX5DR_ACTION_FLAG_HWS_FDB);
	priv->hw_def_miss = mlx5dr_action_create_default_miss(priv->dr_ctx, action_flags);
	if (!priv->hw_def_miss)
		goto err;
	if (is_proxy) {
		ret = flow_hw_create_vport_actions(priv);
		if (ret) {
			rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL, "Failed to create vport actions.");
			goto err;
		}
		ret = flow_hw_create_ctrl_tables(dev, error);
		if (ret)
			goto err;
	}
	if (!priv->shared_host)
		flow_hw_create_send_to_kernel_actions(priv);
	if (port_attr->nb_conn_tracks || (host_priv && host_priv->hws_ctpool)) {
		mem_size = sizeof(struct mlx5_aso_sq) * nb_q_updated +
			   sizeof(*priv->ct_mng);
		priv->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
					   RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
		if (!priv->ct_mng)
			goto err;
		if (mlx5_aso_ct_queue_init(priv->sh, priv->ct_mng, nb_q_updated))
			goto err;
		priv->hws_ctpool = flow_hw_ct_pool_create(dev, port_attr);
		if (!priv->hws_ctpool)
			goto err;
		priv->sh->ct_aso_en = 1;
	}
	if (port_attr->nb_counters || (host_priv && host_priv->hws_cpool)) {
		priv->hws_cpool = mlx5_hws_cnt_pool_create(dev, port_attr,
							   nb_queue);
		if (priv->hws_cpool == NULL)
			goto err;
	}
	if (port_attr->nb_aging_objects) {
		if (port_attr->nb_counters == 0) {
			/*
			 * Aging management uses counter. Number counters
			 * requesting should take into account a counter for
			 * each flow rules containing AGE without counter.
			 */
			DRV_LOG(ERR, "Port %u AGE objects are requested (%u) "
				"without counters requesting.",
				dev->data->port_id,
				port_attr->nb_aging_objects);
			rte_errno = EINVAL;
			goto err;
		}
		ret = mlx5_hws_age_pool_init(dev, port_attr, nb_queue);
		if (ret < 0) {
			rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL, "Failed to init age pool.");
			goto err;
		}
	}
	ret = flow_hw_create_vlan(dev);
	if (ret) {
		rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL, "Failed to VLAN actions.");
		goto err;
	}
	if (_queue_attr)
		mlx5_free(_queue_attr);
	if (port_attr->flags & RTE_FLOW_PORT_FLAG_STRICT_QUEUE)
		priv->hws_strict_queue = 1;
	dev->flow_fp_ops = &mlx5_flow_hw_fp_ops;
	return 0;
err:
	if (priv->hws_ctpool) {
		flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
		priv->hws_ctpool = NULL;
	}
	if (priv->ct_mng) {
		flow_hw_ct_mng_destroy(dev, priv->ct_mng);
		priv->ct_mng = NULL;
	}
	if (priv->hws_age_req)
		mlx5_hws_age_pool_destroy(priv);
	if (priv->hws_cpool) {
		mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
		priv->hws_cpool = NULL;
	}
	mlx5_flow_quota_destroy(dev);
	flow_hw_destroy_send_to_kernel_action(priv);
	flow_hw_free_vport_actions(priv);
	for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
		if (priv->hw_drop[i])
			mlx5dr_action_destroy(priv->hw_drop[i]);
		if (priv->hw_tag[i])
			mlx5dr_action_destroy(priv->hw_tag[i]);
	}
	if (priv->hw_def_miss)
		mlx5dr_action_destroy(priv->hw_def_miss);
	flow_hw_destroy_vlan(dev);
	if (dr_ctx)
		claim_zero(mlx5dr_context_close(dr_ctx));
	for (i = 0; i < nb_q_updated; i++) {
		rte_ring_free(priv->hw_q[i].indir_iq);
		rte_ring_free(priv->hw_q[i].indir_cq);
	}
	mlx5_free(priv->hw_q);
	priv->hw_q = NULL;
	if (priv->acts_ipool) {
		mlx5_ipool_destroy(priv->acts_ipool);
		priv->acts_ipool = NULL;
	}
	if (_queue_attr)
		mlx5_free(_queue_attr);
	if (priv->shared_host) {
		__atomic_fetch_sub(&host_priv->shared_refcnt, 1, __ATOMIC_RELAXED);
		priv->shared_host = NULL;
	}
	mlx5_free(priv->hw_attr);
	priv->hw_attr = NULL;
	/* Do not overwrite the internal errno information. */
	if (ret)
		return ret;
	return rte_flow_error_set(error, rte_errno,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				  "fail to configure port");
}

/**
 * Release HWS resources.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 */
void
flow_hw_resource_release(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_template_table *tbl;
	struct rte_flow_pattern_template *it;
	struct rte_flow_actions_template *at;
	struct mlx5_flow_group *grp;
	uint32_t i;

	if (!priv->dr_ctx)
		return;
	dev->flow_fp_ops = &rte_flow_fp_default_ops;
	flow_hw_rxq_flag_set(dev, false);
	flow_hw_flush_all_ctrl_flows(dev);
	flow_hw_cleanup_tx_repr_tagging(dev);
	flow_hw_cleanup_ctrl_rx_tables(dev);
	while (!LIST_EMPTY(&priv->flow_hw_grp)) {
		grp = LIST_FIRST(&priv->flow_hw_grp);
		flow_hw_group_unset_miss_group(dev, grp, NULL);
	}
	while (!LIST_EMPTY(&priv->flow_hw_tbl_ongo)) {
		tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo);
		flow_hw_table_destroy(dev, tbl, NULL);
	}
	while (!LIST_EMPTY(&priv->flow_hw_tbl)) {
		tbl = LIST_FIRST(&priv->flow_hw_tbl);
		flow_hw_table_destroy(dev, tbl, NULL);
	}
	while (!LIST_EMPTY(&priv->flow_hw_itt)) {
		it = LIST_FIRST(&priv->flow_hw_itt);
		flow_hw_pattern_template_destroy(dev, it, NULL);
	}
	while (!LIST_EMPTY(&priv->flow_hw_at)) {
		at = LIST_FIRST(&priv->flow_hw_at);
		flow_hw_actions_template_destroy(dev, at, NULL);
	}
	for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
		if (priv->hw_drop[i])
			mlx5dr_action_destroy(priv->hw_drop[i]);
		if (priv->hw_tag[i])
			mlx5dr_action_destroy(priv->hw_tag[i]);
	}
	if (priv->hw_def_miss)
		mlx5dr_action_destroy(priv->hw_def_miss);
	flow_hw_destroy_vlan(dev);
	flow_hw_destroy_send_to_kernel_action(priv);
	flow_hw_free_vport_actions(priv);
	if (priv->acts_ipool) {
		mlx5_ipool_destroy(priv->acts_ipool);
		priv->acts_ipool = NULL;
	}
	if (priv->hws_age_req)
		mlx5_hws_age_pool_destroy(priv);
	if (priv->hws_cpool) {
		mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
		priv->hws_cpool = NULL;
	}
	if (priv->hws_ctpool) {
		flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
		priv->hws_ctpool = NULL;
	}
	if (priv->ct_mng) {
		flow_hw_ct_mng_destroy(dev, priv->ct_mng);
		priv->ct_mng = NULL;
	}
	mlx5_flow_quota_destroy(dev);
	for (i = 0; i < priv->nb_queue; i++) {
		rte_ring_free(priv->hw_q[i].indir_iq);
		rte_ring_free(priv->hw_q[i].indir_cq);
	}
	mlx5_free(priv->hw_q);
	priv->hw_q = NULL;
	claim_zero(mlx5dr_context_close(priv->dr_ctx));
	if (priv->shared_host) {
		struct mlx5_priv *host_priv = priv->shared_host->data->dev_private;
		__atomic_fetch_sub(&host_priv->shared_refcnt, 1, __ATOMIC_RELAXED);
		priv->shared_host = NULL;
	}
	priv->dr_ctx = NULL;
	mlx5_free(priv->hw_attr);
	priv->hw_attr = NULL;
	priv->nb_queue = 0;
}

/* Sets vport tag and mask, for given port, used in HWS rules. */
void
flow_hw_set_port_info(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint16_t port_id = dev->data->port_id;
	struct flow_hw_port_info *info;

	MLX5_ASSERT(port_id < RTE_MAX_ETHPORTS);
	info = &mlx5_flow_hw_port_infos[port_id];
	info->regc_mask = priv->vport_meta_mask;
	info->regc_value = priv->vport_meta_tag;
	info->is_wire = mlx5_is_port_on_mpesw_device(priv) ? priv->mpesw_uplink : priv->master;
}

/* Clears vport tag and mask used for HWS rules. */
void
flow_hw_clear_port_info(struct rte_eth_dev *dev)
{
	uint16_t port_id = dev->data->port_id;
	struct flow_hw_port_info *info;

	MLX5_ASSERT(port_id < RTE_MAX_ETHPORTS);
	info = &mlx5_flow_hw_port_infos[port_id];
	info->regc_mask = 0;
	info->regc_value = 0;
	info->is_wire = 0;
}

static int
flow_hw_conntrack_destroy(struct rte_eth_dev *dev __rte_unused,
			  uint32_t idx,
			  struct rte_flow_error *error)
{
	uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
	uint32_t ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
	struct rte_eth_dev *owndev = &rte_eth_devices[owner];
	struct mlx5_priv *priv = owndev->data->dev_private;
	struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
	struct mlx5_aso_ct_action *ct;

	ct = mlx5_ipool_get(pool->cts, ct_idx);
	if (!ct) {
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Invalid CT destruction index");
	}
	__atomic_store_n(&ct->state, ASO_CONNTRACK_FREE,
				 __ATOMIC_RELAXED);
	mlx5_ipool_free(pool->cts, ct_idx);
	return 0;
}

static int
flow_hw_conntrack_query(struct rte_eth_dev *dev, uint32_t queue, uint32_t idx,
			struct rte_flow_action_conntrack *profile,
			void *user_data, bool push,
			struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
	struct mlx5_aso_ct_action *ct;
	uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
	uint32_t ct_idx;

	if (owner != PORT_ID(priv))
		return rte_flow_error_set(error, EACCES,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Can't query CT object owned by another port");
	ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
	ct = mlx5_ipool_get(pool->cts, ct_idx);
	if (!ct) {
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Invalid CT query index");
	}
	profile->peer_port = ct->peer;
	profile->is_original_dir = ct->is_original;
	if (mlx5_aso_ct_query_by_wqe(priv->sh, queue, ct, profile, user_data, push))
		return rte_flow_error_set(error, EIO,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Failed to query CT context");
	return 0;
}


static int
flow_hw_conntrack_update(struct rte_eth_dev *dev, uint32_t queue,
			 const struct rte_flow_modify_conntrack *action_conf,
			 uint32_t idx, void *user_data, bool push,
			 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
	struct mlx5_aso_ct_action *ct;
	const struct rte_flow_action_conntrack *new_prf;
	uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
	uint32_t ct_idx;
	int ret = 0;

	if (PORT_ID(priv) != owner)
		return rte_flow_error_set(error, EACCES,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL,
					  "Can't update CT object owned by another port");
	ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
	ct = mlx5_ipool_get(pool->cts, ct_idx);
	if (!ct) {
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Invalid CT update index");
	}
	new_prf = &action_conf->new_ct;
	if (action_conf->direction)
		ct->is_original = !!new_prf->is_original_dir;
	if (action_conf->state) {
		/* Only validate the profile when it needs to be updated. */
		ret = mlx5_validate_action_ct(dev, new_prf, error);
		if (ret)
			return ret;
		ret = mlx5_aso_ct_update_by_wqe(priv->sh, queue, ct, new_prf,
						user_data, push);
		if (ret)
			return rte_flow_error_set(error, EIO,
					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					NULL,
					"Failed to send CT context update WQE");
		if (queue != MLX5_HW_INV_QUEUE)
			return 0;
		/* Block until ready or a failure in synchronous mode. */
		ret = mlx5_aso_ct_available(priv->sh, queue, ct);
		if (ret)
			rte_flow_error_set(error, rte_errno,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL,
					   "Timeout to get the CT update");
	}
	return ret;
}

static struct rte_flow_action_handle *
flow_hw_conntrack_create(struct rte_eth_dev *dev, uint32_t queue,
			 const struct rte_flow_action_conntrack *pro,
			 void *user_data, bool push,
			 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
	struct mlx5_aso_ct_action *ct;
	uint32_t ct_idx = 0;
	int ret;
	bool async = !!(queue != MLX5_HW_INV_QUEUE);

	if (!pool) {
		rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
				   "CT is not enabled");
		return 0;
	}
	ct = mlx5_ipool_zmalloc(pool->cts, &ct_idx);
	if (!ct) {
		rte_flow_error_set(error, rte_errno,
				   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
				   "Failed to allocate CT object");
		return 0;
	}
	ct->offset = ct_idx - 1;
	ct->is_original = !!pro->is_original_dir;
	ct->peer = pro->peer_port;
	ct->pool = pool;
	if (mlx5_aso_ct_update_by_wqe(priv->sh, queue, ct, pro, user_data, push)) {
		mlx5_ipool_free(pool->cts, ct_idx);
		rte_flow_error_set(error, EBUSY,
				   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
				   "Failed to update CT");
		return 0;
	}
	if (!async) {
		ret = mlx5_aso_ct_available(priv->sh, queue, ct);
		if (ret) {
			mlx5_ipool_free(pool->cts, ct_idx);
			rte_flow_error_set(error, rte_errno,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL,
					   "Timeout to get the CT update");
			return 0;
		}
	}
	return (struct rte_flow_action_handle *)(uintptr_t)
		MLX5_ACTION_CTX_CT_GEN_IDX(PORT_ID(priv), ct_idx);
}

/**
 * Validate shared action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   Which queue to be used.
 * @param[in] attr
 *   Operation attribute.
 * @param[in] conf
 *   Indirect action configuration.
 * @param[in] action
 *   rte_flow action detail.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
static int
flow_hw_action_handle_validate(struct rte_eth_dev *dev, uint32_t queue,
			       const struct rte_flow_op_attr *attr,
			       const struct rte_flow_indir_action_conf *conf,
			       const struct rte_flow_action *action,
			       void *user_data,
			       struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	RTE_SET_USED(attr);
	RTE_SET_USED(queue);
	RTE_SET_USED(user_data);
	switch (action->type) {
	case RTE_FLOW_ACTION_TYPE_AGE:
		if (!priv->hws_age_req)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  NULL,
						  "aging pool not initialized");
		break;
	case RTE_FLOW_ACTION_TYPE_COUNT:
		if (!priv->hws_cpool)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  NULL,
						  "counters pool not initialized");
		break;
	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
		if (priv->hws_ctpool == NULL)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  NULL,
						  "CT pool not initialized");
		return mlx5_validate_action_ct(dev, action->conf, error);
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		return flow_hw_validate_action_meter_mark(dev, action, error);
	case RTE_FLOW_ACTION_TYPE_RSS:
		return flow_dv_action_validate(dev, conf, action, error);
	case RTE_FLOW_ACTION_TYPE_QUOTA:
		return 0;
	default:
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "action type not supported");
	}
	return 0;
}

static __rte_always_inline bool
flow_hw_action_push(const struct rte_flow_op_attr *attr)
{
	return attr ? !attr->postpone : true;
}

static __rte_always_inline struct mlx5_hw_q_job *
flow_hw_action_job_init(struct mlx5_priv *priv, uint32_t queue,
			const struct rte_flow_action_handle *handle,
			void *user_data, void *query_data,
			enum mlx5_hw_job_type type,
			struct rte_flow_error *error)
{
	struct mlx5_hw_q_job *job;

	MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
	job = flow_hw_job_get(priv, queue);
	if (!job) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
				   "Action destroy failed due to queue full.");
		return NULL;
	}
	job->type = type;
	job->action = handle;
	job->user_data = user_data;
	job->query.user = query_data;
	return job;
}

static __rte_always_inline void
flow_hw_action_finalize(struct rte_eth_dev *dev, uint32_t queue,
			struct mlx5_hw_q_job *job,
			bool push, bool aso, bool status)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (queue == MLX5_HW_INV_QUEUE)
		queue = CTRL_QUEUE_ID(priv);
	if (likely(status)) {
		/* 1. add new job to a queue */
		if (!aso)
			rte_ring_enqueue(push ?
					 priv->hw_q[queue].indir_cq :
					 priv->hw_q[queue].indir_iq,
					 job);
		/* 2. send pending jobs */
		if (push)
			__flow_hw_push_action(dev, queue);
	} else {
		flow_hw_job_put(priv, job, queue);
	}
}

/**
 * Create shared action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   Which queue to be used.
 * @param[in] attr
 *   Operation attribute.
 * @param[in] conf
 *   Indirect action configuration.
 * @param[in] action
 *   rte_flow action detail.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   Action handle on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow_action_handle *
flow_hw_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
			     const struct rte_flow_op_attr *attr,
			     const struct rte_flow_indir_action_conf *conf,
			     const struct rte_flow_action *action,
			     void *user_data,
			     struct rte_flow_error *error)
{
	struct rte_flow_action_handle *handle = NULL;
	struct mlx5_hw_q_job *job = NULL;
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_age *age;
	struct mlx5_aso_mtr *aso_mtr;
	cnt_id_t cnt_id;
	uint32_t mtr_id;
	uint32_t age_idx;
	bool push = flow_hw_action_push(attr);
	bool aso = false;

	if (attr) {
		job = flow_hw_action_job_init(priv, queue, NULL, user_data,
					      NULL, MLX5_HW_Q_JOB_TYPE_CREATE,
					      error);
		if (!job)
			return NULL;
	}
	switch (action->type) {
	case RTE_FLOW_ACTION_TYPE_AGE:
		if (priv->hws_strict_queue) {
			struct mlx5_age_info *info = GET_PORT_AGE_INFO(priv);

			if (queue >= info->hw_q_age->nb_rings) {
				rte_flow_error_set(error, EINVAL,
						   RTE_FLOW_ERROR_TYPE_ACTION,
						   NULL,
						   "Invalid queue ID for indirect AGE.");
				rte_errno = EINVAL;
				return NULL;
			}
		}
		age = action->conf;
		age_idx = mlx5_hws_age_action_create(priv, queue, true, age,
						     0, error);
		if (age_idx == 0) {
			rte_flow_error_set(error, ENODEV,
					   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					   "AGE are not configured!");
		} else {
			age_idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
				   MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
			handle =
			    (struct rte_flow_action_handle *)(uintptr_t)age_idx;
		}
		break;
	case RTE_FLOW_ACTION_TYPE_COUNT:
		if (mlx5_hws_cnt_shared_get(priv->hws_cpool, &cnt_id, 0))
			rte_flow_error_set(error, ENODEV,
					RTE_FLOW_ERROR_TYPE_ACTION,
					NULL,
					"counter are not configured!");
		else
			handle = (struct rte_flow_action_handle *)
				 (uintptr_t)cnt_id;
		break;
	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
		aso = true;
		handle = flow_hw_conntrack_create(dev, queue, action->conf, job,
						  push, error);
		break;
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		aso = true;
		aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, job, push);
		if (!aso_mtr)
			break;
		mtr_id = (MLX5_INDIRECT_ACTION_TYPE_METER_MARK <<
			MLX5_INDIRECT_ACTION_TYPE_OFFSET) | (aso_mtr->fm.meter_id);
		handle = (struct rte_flow_action_handle *)(uintptr_t)mtr_id;
		break;
	case RTE_FLOW_ACTION_TYPE_RSS:
		handle = flow_dv_action_create(dev, conf, action, error);
		break;
	case RTE_FLOW_ACTION_TYPE_QUOTA:
		aso = true;
		handle = mlx5_quota_alloc(dev, queue, action->conf,
					  job, push, error);
		break;
	default:
		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
				   NULL, "action type not supported");
		break;
	}
	if (job) {
		job->action = handle;
		job->indirect_type = MLX5_HW_INDIRECT_TYPE_LEGACY;
		flow_hw_action_finalize(dev, queue, job, push, aso,
					handle != NULL);
	}
	return handle;
}

static int
mlx5_flow_update_meter_mark(struct rte_eth_dev *dev, uint32_t queue,
			    const struct rte_flow_update_meter_mark *upd_meter_mark,
			    uint32_t idx, bool push,
			    struct mlx5_hw_q_job *job, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	const struct rte_flow_action_meter_mark *meter_mark = &upd_meter_mark->meter_mark;
	struct mlx5_aso_mtr *aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
	struct mlx5_flow_meter_info *fm;

	if (!aso_mtr)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "Invalid meter_mark update index");
	fm = &aso_mtr->fm;
	if (upd_meter_mark->profile_valid)
		fm->profile = (struct mlx5_flow_meter_profile *)
			(meter_mark->profile);
	if (upd_meter_mark->color_mode_valid)
		fm->color_aware = meter_mark->color_mode;
	if (upd_meter_mark->state_valid)
		fm->is_enable = meter_mark->state;
	/* Update ASO flow meter by wqe. */
	if (mlx5_aso_meter_update_by_wqe(priv->sh, queue,
					 aso_mtr, &priv->mtr_bulk, job, push))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "Unable to update ASO meter WQE");
	/* Wait for ASO object completion. */
	if (queue == MLX5_HW_INV_QUEUE &&
	    mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "Unable to wait for ASO meter CQE");
	return 0;
}

/**
 * Update shared action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   Which queue to be used.
 * @param[in] attr
 *   Operation attribute.
 * @param[in] handle
 *   Action handle to be updated.
 * @param[in] update
 *   Update value.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
			     const struct rte_flow_op_attr *attr,
			     struct rte_flow_action_handle *handle,
			     const void *update,
			     void *user_data,
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_modify_conntrack *ct_conf =
		(const struct rte_flow_modify_conntrack *)update;
	struct mlx5_hw_q_job *job = NULL;
	uint32_t act_idx = (uint32_t)(uintptr_t)handle;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
	int ret = 0;
	bool push = flow_hw_action_push(attr);
	bool aso = false;

	if (attr) {
		job = flow_hw_action_job_init(priv, queue, handle, user_data,
					      NULL, MLX5_HW_Q_JOB_TYPE_UPDATE,
					      error);
		if (!job)
			return -rte_errno;
	}
	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		ret = mlx5_hws_age_action_update(priv, idx, update, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		if (ct_conf->state)
			aso = true;
		ret = flow_hw_conntrack_update(dev, queue, update, act_idx,
					       job, push, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		aso = true;
		ret = mlx5_flow_update_meter_mark(dev, queue, update, idx, push,
						  job, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_RSS:
		ret = flow_dv_action_update(dev, handle, update, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_QUOTA:
		aso = true;
		ret = mlx5_quota_query_update(dev, queue, handle, update, NULL,
					      job, push, error);
		break;
	default:
		ret = -ENOTSUP;
		rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "action type not supported");
		break;
	}
	if (job)
		flow_hw_action_finalize(dev, queue, job, push, aso, ret == 0);
	return ret;
}

/**
 * Destroy shared action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   Which queue to be used.
 * @param[in] attr
 *   Operation attribute.
 * @param[in] handle
 *   Action handle to be destroyed.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
			      const struct rte_flow_op_attr *attr,
			      struct rte_flow_action_handle *handle,
			      void *user_data,
			      struct rte_flow_error *error)
{
	uint32_t act_idx = (uint32_t)(uintptr_t)handle;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t age_idx = act_idx & MLX5_HWS_AGE_IDX_MASK;
	uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct mlx5_hw_q_job *job = NULL;
	struct mlx5_aso_mtr *aso_mtr;
	struct mlx5_flow_meter_info *fm;
	bool push = flow_hw_action_push(attr);
	bool aso = false;
	int ret = 0;

	if (attr) {
		job = flow_hw_action_job_init(priv, queue, handle, user_data,
					      NULL, MLX5_HW_Q_JOB_TYPE_DESTROY,
					      error);
		if (!job)
			return -rte_errno;
	}
	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		ret = mlx5_hws_age_action_destroy(priv, age_idx, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_COUNT:
		age_idx = mlx5_hws_cnt_age_get(priv->hws_cpool, act_idx);
		if (age_idx != 0)
			/*
			 * If this counter belongs to indirect AGE, here is the
			 * time to update the AGE.
			 */
			mlx5_hws_age_nb_cnt_decrease(priv, age_idx);
		mlx5_hws_cnt_shared_put(priv->hws_cpool, &act_idx);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		ret = flow_hw_conntrack_destroy(dev, act_idx, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
		if (!aso_mtr) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Invalid meter_mark destroy index");
			break;
		}
		fm = &aso_mtr->fm;
		fm->is_enable = 0;
		/* Update ASO flow meter by wqe. */
		if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, aso_mtr,
						 &priv->mtr_bulk, job, push)) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Unable to update ASO meter WQE");
			break;
		}
		/* Wait for ASO object completion. */
		if (queue == MLX5_HW_INV_QUEUE &&
		    mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Unable to wait for ASO meter CQE");
			break;
		}
		if (!job)
			mlx5_ipool_free(pool->idx_pool, idx);
		else
			aso = true;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_RSS:
		ret = flow_dv_action_destroy(dev, handle, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_QUOTA:
		break;
	default:
		ret = -ENOTSUP;
		rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "action type not supported");
		break;
	}
	if (job)
		flow_hw_action_finalize(dev, queue, job, push, aso, ret == 0);
	return ret;
}

static int
flow_hw_query_counter(const struct rte_eth_dev *dev, uint32_t counter,
		      void *data, struct rte_flow_error *error)
{
	struct mlx5_hws_cnt_pool *hpool;
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hws_cnt *cnt;
	struct rte_flow_query_count *qc = data;
	uint32_t iidx;
	uint64_t pkts, bytes;

	if (!mlx5_hws_cnt_id_valid(counter))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				"counter are not available");
	hpool = mlx5_hws_cnt_host_pool(priv->hws_cpool);
	iidx = mlx5_hws_cnt_iidx(hpool, counter);
	cnt = &hpool->pool[iidx];
	__hws_cnt_query_raw(priv->hws_cpool, counter, &pkts, &bytes);
	qc->hits_set = 1;
	qc->bytes_set = 1;
	qc->hits = pkts - cnt->reset.hits;
	qc->bytes = bytes - cnt->reset.bytes;
	if (qc->reset) {
		cnt->reset.bytes = bytes;
		cnt->reset.hits = pkts;
	}
	return 0;
}

/**
 * Query a flow rule AGE action for aging information.
 *
 * @param[in] dev
 *   Pointer to Ethernet device.
 * @param[in] age_idx
 *   Index of AGE action parameter.
 * @param[out] data
 *   Data retrieved by the query.
 * @param[out] error
 *   Perform verbose error reporting if not NULL.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_query_age(const struct rte_eth_dev *dev, uint32_t age_idx, void *data,
		  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);
	struct mlx5_indexed_pool *ipool = age_info->ages_ipool;
	struct mlx5_hws_age_param *param = mlx5_ipool_get(ipool, age_idx);
	struct rte_flow_query_age *resp = data;

	if (!param || !param->timeout)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "age data not available");
	switch (__atomic_load_n(&param->state, __ATOMIC_RELAXED)) {
	case HWS_AGE_AGED_OUT_REPORTED:
	case HWS_AGE_AGED_OUT_NOT_REPORTED:
		resp->aged = 1;
		break;
	case HWS_AGE_CANDIDATE:
	case HWS_AGE_CANDIDATE_INSIDE_RING:
		resp->aged = 0;
		break;
	case HWS_AGE_FREE:
		/*
		 * When state is FREE the flow itself should be invalid.
		 * Fall-through.
		 */
	default:
		MLX5_ASSERT(0);
		break;
	}
	resp->sec_since_last_hit_valid = !resp->aged;
	if (resp->sec_since_last_hit_valid)
		resp->sec_since_last_hit = __atomic_load_n
				 (&param->sec_since_last_hit, __ATOMIC_RELAXED);
	return 0;
}

static int
flow_hw_query(struct rte_eth_dev *dev, struct rte_flow *flow,
	      const struct rte_flow_action *actions, void *data,
	      struct rte_flow_error *error)
{
	int ret = -EINVAL;
	struct rte_flow_hw *hw_flow = (struct rte_flow_hw *)flow;

	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
		switch (actions->type) {
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_COUNT:
			ret = flow_hw_query_counter(dev, hw_flow->cnt_id, data,
						    error);
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
			ret = flow_hw_query_age(dev, hw_flow->age_idx, data,
						error);
			break;
		default:
			return rte_flow_error_set(error, ENOTSUP,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  actions,
						  "action not supported");
		}
	}
	return ret;
}

/**
 * Validate indirect action.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] conf
 *   Shared action configuration.
 * @param[in] action
 *   Action specification used to create indirect action.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
static int
flow_hw_action_validate(struct rte_eth_dev *dev,
			const struct rte_flow_indir_action_conf *conf,
			const struct rte_flow_action *action,
			struct rte_flow_error *err)
{
	return flow_hw_action_handle_validate(dev, MLX5_HW_INV_QUEUE, NULL,
					      conf, action, NULL, err);
}

/**
 * Create indirect action.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] conf
 *   Shared action configuration.
 * @param[in] action
 *   Action specification used to create indirect action.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   A valid shared action handle in case of success, NULL otherwise and
 *   rte_errno is set.
 */
static struct rte_flow_action_handle *
flow_hw_action_create(struct rte_eth_dev *dev,
		       const struct rte_flow_indir_action_conf *conf,
		       const struct rte_flow_action *action,
		       struct rte_flow_error *err)
{
	return flow_hw_action_handle_create(dev, MLX5_HW_INV_QUEUE,
					    NULL, conf, action, NULL, err);
}

/**
 * Destroy the indirect action.
 * Release action related resources on the NIC and the memory.
 * Lock free, (mutex should be acquired by caller).
 * Dispatcher for action type specific call.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] handle
 *   The indirect action object handle to be removed.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
static int
flow_hw_action_destroy(struct rte_eth_dev *dev,
		       struct rte_flow_action_handle *handle,
		       struct rte_flow_error *error)
{
	return flow_hw_action_handle_destroy(dev, MLX5_HW_INV_QUEUE,
			NULL, handle, NULL, error);
}

/**
 * Updates in place shared action configuration.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] handle
 *   The indirect action object handle to be updated.
 * @param[in] update
 *   Action specification used to modify the action pointed by *handle*.
 *   *update* could be of same type with the action pointed by the *handle*
 *   handle argument, or some other structures like a wrapper, depending on
 *   the indirect action type.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
static int
flow_hw_action_update(struct rte_eth_dev *dev,
		      struct rte_flow_action_handle *handle,
		      const void *update,
		      struct rte_flow_error *err)
{
	return flow_hw_action_handle_update(dev, MLX5_HW_INV_QUEUE,
			NULL, handle, update, NULL, err);
}

static int
flow_hw_action_handle_query(struct rte_eth_dev *dev, uint32_t queue,
			    const struct rte_flow_op_attr *attr,
			    const struct rte_flow_action_handle *handle,
			    void *data, void *user_data,
			    struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_q_job *job = NULL;
	uint32_t act_idx = (uint32_t)(uintptr_t)handle;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t age_idx = act_idx & MLX5_HWS_AGE_IDX_MASK;
	int ret;
	bool push = flow_hw_action_push(attr);
	bool aso = false;

	if (attr) {
		job = flow_hw_action_job_init(priv, queue, handle, user_data,
					      data, MLX5_HW_Q_JOB_TYPE_QUERY,
					      error);
		if (!job)
			return -rte_errno;
	}
	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		ret = flow_hw_query_age(dev, age_idx, data, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_COUNT:
		ret = flow_hw_query_counter(dev, act_idx, data, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		aso = true;
		if (job)
			job->query.user = data;
		ret = flow_hw_conntrack_query(dev, queue, act_idx, data,
					      job, push, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_QUOTA:
		aso = true;
		ret = mlx5_quota_query(dev, queue, handle, data,
				       job, push, error);
		break;
	default:
		ret = -ENOTSUP;
		rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "action type not supported");
		break;
	}
	if (job)
		flow_hw_action_finalize(dev, queue, job, push, aso, ret == 0);
	return ret;
}

static int
flow_hw_async_action_handle_query_update
			(struct rte_eth_dev *dev, uint32_t queue,
			 const struct rte_flow_op_attr *attr,
			 struct rte_flow_action_handle *handle,
			 const void *update, void *query,
			 enum rte_flow_query_update_mode qu_mode,
			 void *user_data, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	bool push = flow_hw_action_push(attr);
	bool aso = false;
	struct mlx5_hw_q_job *job = NULL;
	int ret = 0;

	if (attr) {
		job = flow_hw_action_job_init(priv, queue, handle, user_data,
					      query,
					      MLX5_HW_Q_JOB_TYPE_UPDATE_QUERY,
					      error);
		if (!job)
			return -rte_errno;
	}
	switch (MLX5_INDIRECT_ACTION_TYPE_GET(handle)) {
	case MLX5_INDIRECT_ACTION_TYPE_QUOTA:
		if (qu_mode != RTE_FLOW_QU_QUERY_FIRST) {
			ret = rte_flow_error_set
				(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
				 NULL, "quota action must query before update");
			break;
		}
		aso = true;
		ret = mlx5_quota_query_update(dev, queue, handle,
					      update, query, job, push, error);
		break;
	default:
		ret = rte_flow_error_set(error, ENOTSUP,
					 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, "update and query not supportred");
	}
	if (job)
		flow_hw_action_finalize(dev, queue, job, push, aso, ret == 0);
	return ret;
}

static int
flow_hw_action_query(struct rte_eth_dev *dev,
		     const struct rte_flow_action_handle *handle, void *data,
		     struct rte_flow_error *error)
{
	return flow_hw_action_handle_query(dev, MLX5_HW_INV_QUEUE, NULL,
			handle, data, NULL, error);
}

static int
flow_hw_action_query_update(struct rte_eth_dev *dev,
			    struct rte_flow_action_handle *handle,
			    const void *update, void *query,
			    enum rte_flow_query_update_mode qu_mode,
			    struct rte_flow_error *error)
{
	return flow_hw_async_action_handle_query_update(dev, MLX5_HW_INV_QUEUE,
							NULL, handle, update,
							query, qu_mode, NULL,
							error);
}

/**
 * Get aged-out flows of a given port on the given HWS flow queue.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] queue_id
 *   Flow queue to query. Ignored when RTE_FLOW_PORT_FLAG_STRICT_QUEUE not set.
 * @param[in, out] contexts
 *   The address of an array of pointers to the aged-out flows contexts.
 * @param[in] nb_contexts
 *   The length of context array pointers.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   if nb_contexts is 0, return the amount of all aged contexts.
 *   if nb_contexts is not 0 , return the amount of aged flows reported
 *   in the context array, otherwise negative errno value.
 */
static int
flow_hw_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id,
			 void **contexts, uint32_t nb_contexts,
			 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);
	struct rte_ring *r;
	int nb_flows = 0;

	if (nb_contexts && !contexts)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "empty context");
	if (priv->hws_strict_queue) {
		if (queue_id >= age_info->hw_q_age->nb_rings)
			return rte_flow_error_set(error, EINVAL,
						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						NULL, "invalid queue id");
		r = age_info->hw_q_age->aged_lists[queue_id];
	} else {
		r = age_info->hw_age.aged_list;
		MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
	}
	if (nb_contexts == 0)
		return rte_ring_count(r);
	while ((uint32_t)nb_flows < nb_contexts) {
		uint32_t age_idx;

		if (rte_ring_dequeue_elem(r, &age_idx, sizeof(uint32_t)) < 0)
			break;
		/* get the AGE context if the aged-out index is still valid. */
		contexts[nb_flows] = mlx5_hws_age_context_get(priv, age_idx);
		if (!contexts[nb_flows])
			continue;
		nb_flows++;
	}
	return nb_flows;
}

/**
 * Get aged-out flows.
 *
 * This function is relevant only if RTE_FLOW_PORT_FLAG_STRICT_QUEUE isn't set.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] contexts
 *   The address of an array of pointers to the aged-out flows contexts.
 * @param[in] nb_contexts
 *   The length of context array pointers.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   how many contexts get in success, otherwise negative errno value.
 *   if nb_contexts is 0, return the amount of all aged contexts.
 *   if nb_contexts is not 0 , return the amount of aged flows reported
 *   in the context array.
 */
static int
flow_hw_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
		       uint32_t nb_contexts, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (priv->hws_strict_queue)
		DRV_LOG(WARNING,
			"port %u get aged flows called in strict queue mode.",
			dev->data->port_id);
	return flow_hw_get_q_aged_flows(dev, 0, contexts, nb_contexts, error);
}

static void
mlx5_mirror_destroy_clone(struct rte_eth_dev *dev,
			  struct mlx5_mirror_clone *clone)
{
	switch (clone->type) {
	case RTE_FLOW_ACTION_TYPE_RSS:
	case RTE_FLOW_ACTION_TYPE_QUEUE:
		mlx5_hrxq_release(dev,
				  ((struct mlx5_hrxq *)(clone->action_ctx))->idx);
		break;
	case RTE_FLOW_ACTION_TYPE_JUMP:
		flow_hw_jump_release(dev, clone->action_ctx);
		break;
	case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
	case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
	case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
	case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
	case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
	case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
	default:
		break;
	}
}

void
mlx5_hw_mirror_destroy(struct rte_eth_dev *dev, struct mlx5_mirror *mirror)
{
	uint32_t i;

	mlx5_indirect_list_remove_entry(&mirror->indirect);
	for (i = 0; i < mirror->clones_num; i++)
		mlx5_mirror_destroy_clone(dev, &mirror->clone[i]);
	if (mirror->mirror_action)
		mlx5dr_action_destroy(mirror->mirror_action);
	mlx5_free(mirror);
}

static __rte_always_inline bool
mlx5_mirror_terminal_action(const struct rte_flow_action *action)
{
	switch (action->type) {
	case RTE_FLOW_ACTION_TYPE_JUMP:
	case RTE_FLOW_ACTION_TYPE_RSS:
	case RTE_FLOW_ACTION_TYPE_QUEUE:
	case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
	case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
		return true;
	default:
		break;
	}
	return false;
}

static bool
mlx5_mirror_validate_sample_action(struct rte_eth_dev *dev,
				   const struct rte_flow_attr *flow_attr,
				   const struct rte_flow_action *action)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_ethdev *port = NULL;
	bool is_proxy = MLX5_HW_PORT_IS_PROXY(priv);

	if (!action)
		return false;
	switch (action->type) {
	case RTE_FLOW_ACTION_TYPE_QUEUE:
	case RTE_FLOW_ACTION_TYPE_RSS:
		if (flow_attr->transfer)
			return false;
		break;
	case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
		if (!is_proxy || !flow_attr->transfer)
			return false;
		port = action->conf;
		if (!port || port->port_id != MLX5_REPRESENTED_PORT_ESW_MGR)
			return false;
		break;
	case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
	case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
	case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
	case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
	case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
		if (!is_proxy || !flow_attr->transfer)
			return false;
		if (action[0].type == RTE_FLOW_ACTION_TYPE_RAW_DECAP &&
		    action[1].type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
			return false;
		break;
	default:
		return false;
	}
	return true;
}

/**
 * Valid mirror actions list includes one or two SAMPLE actions
 * followed by JUMP.
 *
 * @return
 * Number of mirrors *action* list was valid.
 * -EINVAL otherwise.
 */
static int
mlx5_hw_mirror_actions_list_validate(struct rte_eth_dev *dev,
				     const struct rte_flow_attr *flow_attr,
				     const struct rte_flow_action *actions)
{
	if (actions[0].type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
		int i = 1;
		bool valid;
		const struct rte_flow_action_sample *sample = actions[0].conf;
		valid = mlx5_mirror_validate_sample_action(dev, flow_attr,
							   sample->actions);
		if (!valid)
			return -EINVAL;
		if (actions[1].type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
			i = 2;
			sample = actions[1].conf;
			valid = mlx5_mirror_validate_sample_action(dev, flow_attr,
								   sample->actions);
			if (!valid)
				return -EINVAL;
		}
		return mlx5_mirror_terminal_action(actions + i) ? i + 1 : -EINVAL;
	}
	return -EINVAL;
}

static int
mirror_format_tir(struct rte_eth_dev *dev,
		  struct mlx5_mirror_clone *clone,
		  const struct mlx5_flow_template_table_cfg *table_cfg,
		  const struct rte_flow_action *action,
		  struct mlx5dr_action_dest_attr *dest_attr,
		  struct rte_flow_error *error)
{
	uint32_t hws_flags;
	enum mlx5dr_table_type table_type;
	struct mlx5_hrxq *tir_ctx;

	table_type = get_mlx5dr_table_type(&table_cfg->attr.flow_attr);
	hws_flags = mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_NONE_ROOT][table_type];
	tir_ctx = flow_hw_tir_action_register(dev, hws_flags, action);
	if (!tir_ctx)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION,
					  action, "failed to create QUEUE action for mirror clone");
	dest_attr->dest = tir_ctx->action;
	clone->action_ctx = tir_ctx;
	return 0;
}

static int
mirror_format_jump(struct rte_eth_dev *dev,
		   struct mlx5_mirror_clone *clone,
		   const struct mlx5_flow_template_table_cfg *table_cfg,
		   const struct rte_flow_action *action,
		   struct mlx5dr_action_dest_attr *dest_attr,
		   struct rte_flow_error *error)
{
	const struct rte_flow_action_jump *jump_conf = action->conf;
	struct mlx5_hw_jump_action *jump = flow_hw_jump_action_register
						(dev, table_cfg,
						 jump_conf->group, error);

	if (!jump)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION,
					  action, "failed to create JUMP action for mirror clone");
	dest_attr->dest = jump->hws_action;
	clone->action_ctx = jump;
	return 0;
}

static int
mirror_format_port(struct rte_eth_dev *dev,
		   const struct rte_flow_action *action,
		   struct mlx5dr_action_dest_attr *dest_attr,
		   struct rte_flow_error __rte_unused *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_ethdev *port_action = action->conf;

	dest_attr->dest = priv->hw_vport[port_action->port_id];
	return 0;
}

static int
hw_mirror_clone_reformat(const struct rte_flow_action *actions,
			 struct mlx5dr_action_dest_attr *dest_attr,
			 enum mlx5dr_action_type *action_type,
			 uint8_t *reformat_buf, bool decap)
{
	int ret;
	const struct rte_flow_item *encap_item = NULL;
	const struct rte_flow_action_raw_encap *encap_conf = NULL;
	typeof(dest_attr->reformat) *reformat = &dest_attr->reformat;

	switch (actions[0].type) {
	case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
		encap_conf = actions[0].conf;
		break;
	case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
		encap_item = MLX5_CONST_ENCAP_ITEM(rte_flow_action_vxlan_encap,
						   actions);
		break;
	case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
		encap_item = MLX5_CONST_ENCAP_ITEM(rte_flow_action_nvgre_encap,
						   actions);
		break;
	default:
		return -EINVAL;
	}
	*action_type = decap ?
		       MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L3 :
		       MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2;
	if (encap_item) {
		ret = flow_dv_convert_encap_data(encap_item, reformat_buf,
						 &reformat->reformat_data_sz, NULL);
		if (ret)
			return -EINVAL;
		reformat->reformat_data = reformat_buf;
	} else {
		reformat->reformat_data = (void *)(uintptr_t)encap_conf->data;
		reformat->reformat_data_sz = encap_conf->size;
	}
	return 0;
}

static int
hw_mirror_format_clone(struct rte_eth_dev *dev,
			struct mlx5_mirror_clone *clone,
			const struct mlx5_flow_template_table_cfg *table_cfg,
			const struct rte_flow_action *actions,
			struct mlx5dr_action_dest_attr *dest_attr,
			uint8_t *reformat_buf, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int ret;
	uint32_t i;
	bool decap_seen = false;

	for (i = 0; actions[i].type != RTE_FLOW_ACTION_TYPE_END; i++) {
		dest_attr->action_type[i] = mlx5_hw_dr_action_types[actions[i].type];
		switch (actions[i].type) {
		case RTE_FLOW_ACTION_TYPE_QUEUE:
		case RTE_FLOW_ACTION_TYPE_RSS:
			ret = mirror_format_tir(dev, clone, table_cfg,
						&actions[i], dest_attr, error);
			if (ret)
				return ret;
			break;
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
			ret = mirror_format_port(dev, &actions[i],
						 dest_attr, error);
			if (ret)
				return ret;
			break;
		case RTE_FLOW_ACTION_TYPE_JUMP:
			ret = mirror_format_jump(dev, clone, table_cfg,
						 &actions[i], dest_attr, error);
			if (ret)
				return ret;
			break;
		case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR:
			dest_attr->dest = priv->hw_def_miss;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
			decap_seen = true;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
			ret = hw_mirror_clone_reformat(&actions[i], dest_attr,
						       &dest_attr->action_type[i],
						       reformat_buf, decap_seen);
			if (ret < 0)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ACTION,
							  &actions[i],
							  "failed to create reformat action");
			break;
		default:
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  &actions[i], "unsupported sample action");
		}
		clone->type = actions->type;
	}
	dest_attr->action_type[i] = MLX5DR_ACTION_TYP_LAST;
	return 0;
}

static struct rte_flow_action_list_handle *
mlx5_hw_mirror_handle_create(struct rte_eth_dev *dev,
			     const struct mlx5_flow_template_table_cfg *table_cfg,
			     const struct rte_flow_action *actions,
			     struct rte_flow_error *error)
{
	uint32_t hws_flags;
	int ret = 0, i, clones_num;
	struct mlx5_mirror *mirror;
	enum mlx5dr_table_type table_type;
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_attr *flow_attr = &table_cfg->attr.flow_attr;
	uint8_t reformat_buf[MLX5_MIRROR_MAX_CLONES_NUM][MLX5_ENCAP_MAX_LEN];
	struct mlx5dr_action_dest_attr mirror_attr[MLX5_MIRROR_MAX_CLONES_NUM + 1];
	enum mlx5dr_action_type array_action_types[MLX5_MIRROR_MAX_CLONES_NUM + 1]
						  [MLX5_MIRROR_MAX_SAMPLE_ACTIONS_LEN + 1];

	memset(mirror_attr, 0, sizeof(mirror_attr));
	memset(array_action_types, 0, sizeof(array_action_types));
	table_type = get_mlx5dr_table_type(flow_attr);
	hws_flags = mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_NONE_ROOT][table_type];
	clones_num = mlx5_hw_mirror_actions_list_validate(dev, flow_attr,
							  actions);
	if (clones_num < 0) {
		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
				   actions, "Invalid mirror list format");
		return NULL;
	}
	mirror = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mirror),
			     0, SOCKET_ID_ANY);
	if (!mirror) {
		rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ACTION,
				   actions, "Failed to allocate mirror context");
		return NULL;
	}

	mirror->indirect.type = MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR;
	mirror->clones_num = clones_num;
	for (i = 0; i < clones_num; i++) {
		const struct rte_flow_action *clone_actions;

		mirror_attr[i].action_type = array_action_types[i];
		if (actions[i].type == RTE_FLOW_ACTION_TYPE_SAMPLE) {
			const struct rte_flow_action_sample *sample = actions[i].conf;

			clone_actions = sample->actions;
		} else {
			clone_actions = &actions[i];
		}
		ret = hw_mirror_format_clone(dev, &mirror->clone[i], table_cfg,
					     clone_actions, &mirror_attr[i],
					     reformat_buf[i], error);

		if (ret)
			goto error;
	}
	hws_flags |= MLX5DR_ACTION_FLAG_SHARED;
	mirror->mirror_action = mlx5dr_action_create_dest_array(priv->dr_ctx,
								clones_num,
								mirror_attr,
								hws_flags);
	if (!mirror->mirror_action) {
		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
				   actions, "Failed to create HWS mirror action");
		goto error;
	}

	mlx5_indirect_list_add_entry(&priv->indirect_list_head, &mirror->indirect);
	return (struct rte_flow_action_list_handle *)mirror;

error:
	mlx5_hw_mirror_destroy(dev, mirror);
	return NULL;
}

void
mlx5_destroy_legacy_indirect(__rte_unused struct rte_eth_dev *dev,
			     struct mlx5_indirect_list *ptr)
{
	struct mlx5_indlst_legacy *obj = (typeof(obj))ptr;

	switch (obj->legacy_type) {
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		break; /* ASO meters were released in mlx5_flow_meter_flush() */
	default:
		break;
	}
	mlx5_free(obj);
}

static struct rte_flow_action_list_handle *
mlx5_create_legacy_indlst(struct rte_eth_dev *dev, uint32_t queue,
			  const struct rte_flow_op_attr *attr,
			  const struct rte_flow_indir_action_conf *conf,
			  const struct rte_flow_action *actions,
			  void *user_data, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_indlst_legacy *indlst_obj = mlx5_malloc(MLX5_MEM_ZERO,
							    sizeof(*indlst_obj),
							    0, SOCKET_ID_ANY);

	if (!indlst_obj)
		return NULL;
	indlst_obj->handle = flow_hw_action_handle_create(dev, queue, attr, conf,
							  actions, user_data,
							  error);
	if (!indlst_obj->handle) {
		mlx5_free(indlst_obj);
		return NULL;
	}
	indlst_obj->legacy_type = actions[0].type;
	indlst_obj->indirect.type = MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY;
	mlx5_indirect_list_add_entry(&priv->indirect_list_head, &indlst_obj->indirect);
	return (struct rte_flow_action_list_handle *)indlst_obj;
}

static __rte_always_inline enum mlx5_indirect_list_type
flow_hw_inlist_type_get(const struct rte_flow_action *actions)
{
	switch (actions[0].type) {
	case RTE_FLOW_ACTION_TYPE_SAMPLE:
		return MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR;
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		return actions[1].type == RTE_FLOW_ACTION_TYPE_END ?
		       MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY :
		       MLX5_INDIRECT_ACTION_LIST_TYPE_ERR;
	case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
	case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
		return MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT;
	default:
		break;
	}
	return MLX5_INDIRECT_ACTION_LIST_TYPE_ERR;
}

static struct rte_flow_action_list_handle*
mlx5_hw_decap_encap_handle_create(struct rte_eth_dev *dev,
				  const struct mlx5_flow_template_table_cfg *table_cfg,
				  const struct rte_flow_action *actions,
				  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_attr *flow_attr = &table_cfg->attr.flow_attr;
	const struct rte_flow_action *encap = NULL;
	const struct rte_flow_action *decap = NULL;
	struct rte_flow_indir_action_conf indirect_conf = {
		.ingress = flow_attr->ingress,
		.egress = flow_attr->egress,
		.transfer = flow_attr->transfer,
	};
	struct mlx5_hw_encap_decap_action *handle;
	uint64_t action_flags = 0;

	/*
	 * Allow
	 * 1. raw_decap / raw_encap / end
	 * 2. raw_encap / end
	 * 3. raw_decap / end
	 */
	while (actions->type != RTE_FLOW_ACTION_TYPE_END) {
		if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_DECAP) {
			if (action_flags) {
				rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
						   actions, "Invalid indirect action list sequence");
				return NULL;
			}
			action_flags |= MLX5_FLOW_ACTION_DECAP;
			decap = actions;
		} else if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
			if (action_flags & MLX5_FLOW_ACTION_ENCAP) {
				rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
						   actions, "Invalid indirect action list sequence");
				return NULL;
			}
			action_flags |= MLX5_FLOW_ACTION_ENCAP;
			encap = actions;
		} else {
			rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
					   actions, "Invalid indirect action type in list");
			return NULL;
		}
		actions++;
	}
	if (!decap && !encap) {
		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
				   actions, "Invalid indirect action combinations");
		return NULL;
	}
	handle = mlx5_reformat_action_create(dev, &indirect_conf, encap, decap, error);
	if (!handle) {
		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
				   actions, "Failed to create HWS decap_encap action");
		return NULL;
	}
	handle->indirect.type = MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT;
	LIST_INSERT_HEAD(&priv->indirect_list_head, &handle->indirect, entry);
	return (struct rte_flow_action_list_handle *)handle;
}

static struct rte_flow_action_list_handle *
flow_hw_async_action_list_handle_create(struct rte_eth_dev *dev, uint32_t queue,
					const struct rte_flow_op_attr *attr,
					const struct rte_flow_indir_action_conf *conf,
					const struct rte_flow_action *actions,
					void *user_data,
					struct rte_flow_error *error)
{
	struct mlx5_hw_q_job *job = NULL;
	bool push = flow_hw_action_push(attr);
	enum mlx5_indirect_list_type list_type;
	struct rte_flow_action_list_handle *handle;
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct mlx5_flow_template_table_cfg table_cfg = {
		.external = true,
		.attr = {
			.flow_attr = {
				.ingress = conf->ingress,
				.egress = conf->egress,
				.transfer = conf->transfer
			}
		}
	};

	if (!actions) {
		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
				   NULL, "No action list");
		return NULL;
	}
	list_type = flow_hw_inlist_type_get(actions);
	if (list_type == MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY) {
		/*
		 * Legacy indirect actions already have
		 * async resources management. No need to do it twice.
		 */
		handle = mlx5_create_legacy_indlst(dev, queue, attr, conf,
						   actions, user_data, error);
		goto end;
	}
	if (attr) {
		job = flow_hw_action_job_init(priv, queue, NULL, user_data,
					      NULL, MLX5_HW_Q_JOB_TYPE_CREATE,
					      error);
		if (!job)
			return NULL;
	}
	switch (list_type) {
	case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR:
		handle = mlx5_hw_mirror_handle_create(dev, &table_cfg,
						      actions, error);
		break;
	case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT:
		handle = mlx5_hw_decap_encap_handle_create(dev, &table_cfg,
							   actions, error);
		break;
	default:
		handle = NULL;
		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
				   actions, "Invalid list");
	}
	if (job) {
		job->action = handle;
		job->indirect_type = MLX5_HW_INDIRECT_TYPE_LIST;
		flow_hw_action_finalize(dev, queue, job, push, false,
					handle != NULL);
	}
end:
	return handle;
}

static struct rte_flow_action_list_handle *
flow_hw_action_list_handle_create(struct rte_eth_dev *dev,
				  const struct rte_flow_indir_action_conf *conf,
				  const struct rte_flow_action *actions,
				  struct rte_flow_error *error)
{
	return flow_hw_async_action_list_handle_create(dev, MLX5_HW_INV_QUEUE,
						       NULL, conf, actions,
						       NULL, error);
}

static int
flow_hw_async_action_list_handle_destroy
			(struct rte_eth_dev *dev, uint32_t queue,
			 const struct rte_flow_op_attr *attr,
			 struct rte_flow_action_list_handle *handle,
			 void *user_data, struct rte_flow_error *error)
{
	int ret = 0;
	struct mlx5_hw_q_job *job = NULL;
	bool push = flow_hw_action_push(attr);
	struct mlx5_priv *priv = dev->data->dev_private;
	enum mlx5_indirect_list_type type =
		mlx5_get_indirect_list_type((void *)handle);

	if (type == MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY) {
		struct mlx5_indlst_legacy *legacy = (typeof(legacy))handle;

		ret = flow_hw_action_handle_destroy(dev, queue, attr,
						    legacy->handle,
						    user_data, error);
		mlx5_indirect_list_remove_entry(&legacy->indirect);
		goto end;
	}
	if (attr) {
		job = flow_hw_action_job_init(priv, queue, NULL, user_data,
					      NULL, MLX5_HW_Q_JOB_TYPE_DESTROY,
					      error);
		if (!job)
			return rte_errno;
	}
	switch (type) {
	case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR:
		mlx5_hw_mirror_destroy(dev, (struct mlx5_mirror *)handle);
		break;
	case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT:
		LIST_REMOVE(&((struct mlx5_hw_encap_decap_action *)handle)->indirect,
			    entry);
		mlx5_reformat_action_destroy(dev, handle, error);
		break;
	default:
		ret = rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION,
					  NULL, "Invalid indirect list handle");
	}
	if (job) {
		flow_hw_action_finalize(dev, queue, job, push, false, true);
	}
end:
	return ret;
}

static int
flow_hw_action_list_handle_destroy(struct rte_eth_dev *dev,
				   struct rte_flow_action_list_handle *handle,
				   struct rte_flow_error *error)
{
	return flow_hw_async_action_list_handle_destroy(dev, MLX5_HW_INV_QUEUE,
							NULL, handle, NULL,
							error);
}

static int
flow_hw_async_action_list_handle_query_update
		(struct rte_eth_dev *dev, uint32_t queue_id,
		 const struct rte_flow_op_attr *attr,
		 const struct rte_flow_action_list_handle *handle,
		 const void **update, void **query,
		 enum rte_flow_query_update_mode mode,
		 void *user_data, struct rte_flow_error *error)
{
	enum mlx5_indirect_list_type type =
		mlx5_get_indirect_list_type((const void *)handle);

	if (type == MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY) {
		struct mlx5_indlst_legacy *legacy = (void *)(uintptr_t)handle;

		if (update && query)
			return flow_hw_async_action_handle_query_update
				(dev, queue_id, attr, legacy->handle,
				 update, query, mode, user_data, error);
		else if (update && update[0])
			return flow_hw_action_handle_update(dev, queue_id, attr,
							    legacy->handle, update[0],
							    user_data, error);
		else if (query && query[0])
			return flow_hw_action_handle_query(dev, queue_id, attr,
							   legacy->handle, query[0],
							   user_data, error);
		else
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL, "invalid legacy handle query_update parameters");
	}
	return -ENOTSUP;
}

static int
flow_hw_action_list_handle_query_update(struct rte_eth_dev *dev,
					const struct rte_flow_action_list_handle *handle,
					const void **update, void **query,
					enum rte_flow_query_update_mode mode,
					struct rte_flow_error *error)
{
	return flow_hw_async_action_list_handle_query_update
					(dev, MLX5_HW_INV_QUEUE, NULL, handle,
					 update, query, mode, NULL, error);
}

static int
flow_hw_calc_table_hash(struct rte_eth_dev *dev,
			 const struct rte_flow_template_table *table,
			 const struct rte_flow_item pattern[],
			 uint8_t pattern_template_index,
			 uint32_t *hash, struct rte_flow_error *error)
{
	const struct rte_flow_item *items;
	/* Temp job to allow adding missing items */
	static struct rte_flow_item tmp_items[MLX5_HW_MAX_ITEMS];
	static struct mlx5_hw_q_job job = {.items = tmp_items};
	int res;

	items = flow_hw_get_rule_items(dev, table, pattern,
				       pattern_template_index,
				       &job);
	res = mlx5dr_rule_hash_calculate(table->matcher, items,
					 pattern_template_index,
					 MLX5DR_RULE_HASH_CALC_MODE_RAW,
					 hash);
	if (res)
		return rte_flow_error_set(error, res,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL,
					  "hash could not be calculated");
	return 0;
}

static int
flow_hw_calc_encap_hash(struct rte_eth_dev *dev,
			const struct rte_flow_item pattern[],
			enum rte_flow_encap_hash_field dest_field,
			uint8_t *hash,
			struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_crc_encap_entropy_hash_fields data;
	enum mlx5dr_crc_encap_entropy_hash_size res_size =
			dest_field == RTE_FLOW_ENCAP_HASH_FIELD_SRC_PORT ?
				MLX5DR_CRC_ENCAP_ENTROPY_HASH_SIZE_16 :
				MLX5DR_CRC_ENCAP_ENTROPY_HASH_SIZE_8;
	int res;

	memset(&data, 0, sizeof(struct mlx5dr_crc_encap_entropy_hash_fields));

	for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
		switch (pattern->type) {
		case RTE_FLOW_ITEM_TYPE_IPV4:
			data.dst.ipv4_addr =
				((const struct rte_flow_item_ipv4 *)(pattern->spec))->hdr.dst_addr;
			data.src.ipv4_addr =
				((const struct rte_flow_item_ipv4 *)(pattern->spec))->hdr.src_addr;
			break;
		case RTE_FLOW_ITEM_TYPE_IPV6:
			memcpy(data.dst.ipv6_addr,
			       ((const struct rte_flow_item_ipv6 *)(pattern->spec))->hdr.dst_addr,
			       sizeof(data.dst.ipv6_addr));
			memcpy(data.src.ipv6_addr,
			       ((const struct rte_flow_item_ipv6 *)(pattern->spec))->hdr.src_addr,
			       sizeof(data.src.ipv6_addr));
			break;
		case RTE_FLOW_ITEM_TYPE_UDP:
			data.next_protocol = IPPROTO_UDP;
			data.dst_port =
				((const struct rte_flow_item_udp *)(pattern->spec))->hdr.dst_port;
			data.src_port =
				((const struct rte_flow_item_udp *)(pattern->spec))->hdr.src_port;
			break;
		case RTE_FLOW_ITEM_TYPE_TCP:
			data.next_protocol = IPPROTO_TCP;
			data.dst_port =
				((const struct rte_flow_item_tcp *)(pattern->spec))->hdr.dst_port;
			data.src_port =
				((const struct rte_flow_item_tcp *)(pattern->spec))->hdr.src_port;
			break;
		case RTE_FLOW_ITEM_TYPE_ICMP:
			data.next_protocol = IPPROTO_ICMP;
			break;
		case RTE_FLOW_ITEM_TYPE_ICMP6:
			data.next_protocol = IPPROTO_ICMPV6;
			break;
		default:
			break;
		}
	}
	res = mlx5dr_crc_encap_entropy_hash_calc(priv->dr_ctx, &data, hash, res_size);
	if (res)
		return rte_flow_error_set(error, res,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "error while calculating encap hash");
	return 0;
}

const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
	.info_get = flow_hw_info_get,
	.configure = flow_hw_configure,
	.pattern_validate = flow_hw_pattern_validate,
	.pattern_template_create = flow_hw_pattern_template_create,
	.pattern_template_destroy = flow_hw_pattern_template_destroy,
	.actions_validate = flow_hw_actions_validate,
	.actions_template_create = flow_hw_actions_template_create,
	.actions_template_destroy = flow_hw_actions_template_destroy,
	.template_table_create = flow_hw_template_table_create,
	.template_table_destroy = flow_hw_table_destroy,
	.group_set_miss_actions = flow_hw_group_set_miss_actions,
	.async_flow_create = flow_hw_async_flow_create,
	.async_flow_create_by_index = flow_hw_async_flow_create_by_index,
	.async_flow_update = flow_hw_async_flow_update,
	.async_flow_destroy = flow_hw_async_flow_destroy,
	.pull = flow_hw_pull,
	.push = flow_hw_push,
	.async_action_create = flow_hw_action_handle_create,
	.async_action_destroy = flow_hw_action_handle_destroy,
	.async_action_update = flow_hw_action_handle_update,
	.async_action_query_update = flow_hw_async_action_handle_query_update,
	.async_action_query = flow_hw_action_handle_query,
	.action_validate = flow_hw_action_validate,
	.action_create = flow_hw_action_create,
	.action_destroy = flow_hw_action_destroy,
	.action_update = flow_hw_action_update,
	.action_query = flow_hw_action_query,
	.action_query_update = flow_hw_action_query_update,
	.action_list_handle_create = flow_hw_action_list_handle_create,
	.action_list_handle_destroy = flow_hw_action_list_handle_destroy,
	.action_list_handle_query_update =
		flow_hw_action_list_handle_query_update,
	.async_action_list_handle_create =
		flow_hw_async_action_list_handle_create,
	.async_action_list_handle_destroy =
		flow_hw_async_action_list_handle_destroy,
	.async_action_list_handle_query_update =
		flow_hw_async_action_list_handle_query_update,
	.query = flow_hw_query,
	.get_aged_flows = flow_hw_get_aged_flows,
	.get_q_aged_flows = flow_hw_get_q_aged_flows,
	.item_create = flow_dv_item_create,
	.item_release = flow_dv_item_release,
	.flow_calc_table_hash = flow_hw_calc_table_hash,
	.flow_calc_encap_hash = flow_hw_calc_encap_hash,
};

/**
 * Creates a control flow using flow template API on @p proxy_dev device,
 * on behalf of @p owner_dev device.
 *
 * This function uses locks internally to synchronize access to the
 * flow queue.
 *
 * Created flow is stored in private list associated with @p proxy_dev device.
 *
 * @param owner_dev
 *   Pointer to Ethernet device on behalf of which flow is created.
 * @param proxy_dev
 *   Pointer to Ethernet device on which flow is created.
 * @param table
 *   Pointer to flow table.
 * @param items
 *   Pointer to flow rule items.
 * @param item_template_idx
 *   Index of an item template associated with @p table.
 * @param actions
 *   Pointer to flow rule actions.
 * @param action_template_idx
 *   Index of an action template associated with @p table.
 * @param info
 *   Additional info about control flow rule.
 * @param external
 *   External ctrl flow.
 *
 * @return
 *   0 on success, negative errno value otherwise and rte_errno set.
 */
static __rte_unused int
flow_hw_create_ctrl_flow(struct rte_eth_dev *owner_dev,
			 struct rte_eth_dev *proxy_dev,
			 struct rte_flow_template_table *table,
			 struct rte_flow_item items[],
			 uint8_t item_template_idx,
			 struct rte_flow_action actions[],
			 uint8_t action_template_idx,
			 struct mlx5_hw_ctrl_flow_info *info,
			 bool external)
{
	struct mlx5_priv *priv = proxy_dev->data->dev_private;
	uint32_t queue = CTRL_QUEUE_ID(priv);
	struct rte_flow_op_attr op_attr = {
		.postpone = 0,
	};
	struct rte_flow *flow = NULL;
	struct mlx5_hw_ctrl_flow *entry = NULL;
	int ret;

	rte_spinlock_lock(&priv->hw_ctrl_lock);
	entry = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_SYS, sizeof(*entry),
			    0, SOCKET_ID_ANY);
	if (!entry) {
		DRV_LOG(ERR, "port %u not enough memory to create control flows",
			proxy_dev->data->port_id);
		rte_errno = ENOMEM;
		ret = -rte_errno;
		goto error;
	}
	flow = flow_hw_async_flow_create(proxy_dev, queue, &op_attr, table,
					 items, item_template_idx,
					 actions, action_template_idx,
					 NULL, NULL);
	if (!flow) {
		DRV_LOG(ERR, "port %u failed to enqueue create control"
			" flow operation", proxy_dev->data->port_id);
		ret = -rte_errno;
		goto error;
	}
	ret = __flow_hw_pull_comp(proxy_dev, queue, NULL);
	if (ret) {
		DRV_LOG(ERR, "port %u failed to insert control flow",
			proxy_dev->data->port_id);
		rte_errno = EINVAL;
		ret = -rte_errno;
		goto error;
	}
	entry->owner_dev = owner_dev;
	entry->flow = flow;
	if (info)
		entry->info = *info;
	else
		entry->info.type = MLX5_HW_CTRL_FLOW_TYPE_GENERAL;
	if (external)
		LIST_INSERT_HEAD(&priv->hw_ext_ctrl_flows, entry, next);
	else
		LIST_INSERT_HEAD(&priv->hw_ctrl_flows, entry, next);
	rte_spinlock_unlock(&priv->hw_ctrl_lock);
	return 0;
error:
	if (entry)
		mlx5_free(entry);
	rte_spinlock_unlock(&priv->hw_ctrl_lock);
	return ret;
}

/**
 * Destroys a control flow @p flow using flow template API on @p dev device.
 *
 * This function uses locks internally to synchronize access to the
 * flow queue.
 *
 * If the @p flow is stored on any private list/pool, then caller must free up
 * the relevant resources.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param flow
 *   Pointer to flow rule.
 *
 * @return
 *   0 on success, non-zero value otherwise.
 */
static int
flow_hw_destroy_ctrl_flow(struct rte_eth_dev *dev, struct rte_flow *flow)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t queue = CTRL_QUEUE_ID(priv);
	struct rte_flow_op_attr op_attr = {
		.postpone = 0,
	};
	int ret;

	rte_spinlock_lock(&priv->hw_ctrl_lock);
	ret = flow_hw_async_flow_destroy(dev, queue, &op_attr, flow, NULL, NULL);
	if (ret) {
		DRV_LOG(ERR, "port %u failed to enqueue destroy control"
			" flow operation", dev->data->port_id);
		goto exit;
	}
	ret = __flow_hw_pull_comp(dev, queue, NULL);
	if (ret) {
		DRV_LOG(ERR, "port %u failed to destroy control flow",
			dev->data->port_id);
		rte_errno = EINVAL;
		ret = -rte_errno;
		goto exit;
	}
exit:
	rte_spinlock_unlock(&priv->hw_ctrl_lock);
	return ret;
}

/**
 * Destroys control flows created on behalf of @p owner device on @p dev device.
 *
 * @param dev
 *   Pointer to Ethernet device on which control flows were created.
 * @param owner
 *   Pointer to Ethernet device owning control flows.
 *
 * @return
 *   0 on success, otherwise negative error code is returned and
 *   rte_errno is set.
 */
static int
flow_hw_flush_ctrl_flows_owned_by(struct rte_eth_dev *dev, struct rte_eth_dev *owner)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_ctrl_flow *cf;
	struct mlx5_hw_ctrl_flow *cf_next;
	int ret;

	cf = LIST_FIRST(&priv->hw_ctrl_flows);
	while (cf != NULL) {
		cf_next = LIST_NEXT(cf, next);
		if (cf->owner_dev == owner) {
			ret = flow_hw_destroy_ctrl_flow(dev, cf->flow);
			if (ret) {
				rte_errno = ret;
				return -ret;
			}
			LIST_REMOVE(cf, next);
			mlx5_free(cf);
		}
		cf = cf_next;
	}
	return 0;
}

/**
 * Destroys control flows created for @p owner_dev device.
 *
 * @param owner_dev
 *   Pointer to Ethernet device owning control flows.
 *
 * @return
 *   0 on success, otherwise negative error code is returned and
 *   rte_errno is set.
 */
int
mlx5_flow_hw_flush_ctrl_flows(struct rte_eth_dev *owner_dev)
{
	struct mlx5_priv *owner_priv = owner_dev->data->dev_private;
	struct rte_eth_dev *proxy_dev;
	uint16_t owner_port_id = owner_dev->data->port_id;
	uint16_t proxy_port_id = owner_dev->data->port_id;
	int ret;

	/* Flush all flows created by this port for itself. */
	ret = flow_hw_flush_ctrl_flows_owned_by(owner_dev, owner_dev);
	if (ret)
		return ret;
	/* Flush all flows created for this port on proxy port. */
	if (owner_priv->sh->config.dv_esw_en) {
		ret = rte_flow_pick_transfer_proxy(owner_port_id, &proxy_port_id, NULL);
		if (ret == -ENODEV) {
			DRV_LOG(DEBUG, "Unable to find transfer proxy port for port %u. It was "
				       "probably closed. Control flows were cleared.",
				       owner_port_id);
			rte_errno = 0;
			return 0;
		} else if (ret) {
			DRV_LOG(ERR, "Unable to find proxy port for port %u (ret = %d)",
				owner_port_id, ret);
			return ret;
		}
		proxy_dev = &rte_eth_devices[proxy_port_id];
	} else {
		proxy_dev = owner_dev;
	}
	return flow_hw_flush_ctrl_flows_owned_by(proxy_dev, owner_dev);
}

/**
 * Destroys all control flows created on @p dev device.
 *
 * @param owner_dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   0 on success, otherwise negative error code is returned and
 *   rte_errno is set.
 */
static int
flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_ctrl_flow *cf;
	struct mlx5_hw_ctrl_flow *cf_next;
	int ret;

	cf = LIST_FIRST(&priv->hw_ctrl_flows);
	while (cf != NULL) {
		cf_next = LIST_NEXT(cf, next);
		ret = flow_hw_destroy_ctrl_flow(dev, cf->flow);
		if (ret) {
			rte_errno = ret;
			return -ret;
		}
		LIST_REMOVE(cf, next);
		mlx5_free(cf);
		cf = cf_next;
	}
	cf = LIST_FIRST(&priv->hw_ext_ctrl_flows);
	while (cf != NULL) {
		cf_next = LIST_NEXT(cf, next);
		ret = flow_hw_destroy_ctrl_flow(dev, cf->flow);
		if (ret) {
			rte_errno = ret;
			return -ret;
		}
		LIST_REMOVE(cf, next);
		mlx5_free(cf);
		cf = cf_next;
	}
	return 0;
}

int
mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn, bool external)
{
	uint16_t port_id = dev->data->port_id;
	struct rte_flow_item_ethdev esw_mgr_spec = {
		.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
	};
	struct rte_flow_item_ethdev esw_mgr_mask = {
		.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
	};
	struct rte_flow_item_tag reg_c0_spec = {
		.index = (uint8_t)REG_C_0,
		.data = flow_hw_esw_mgr_regc_marker(dev),
	};
	struct rte_flow_item_tag reg_c0_mask = {
		.index = 0xff,
		.data = flow_hw_esw_mgr_regc_marker_mask(dev),
	};
	struct mlx5_rte_flow_item_sq sq_spec = {
		.queue = sqn,
	};
	struct rte_flow_action_ethdev port = {
		.port_id = port_id,
	};
	struct rte_flow_item items[3] = { { 0 } };
	struct rte_flow_action actions[3] = { { 0 } };
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS_ROOT,
		.esw_mgr_sq = sqn,
	};
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	uint16_t proxy_port_id = dev->data->port_id;
	int ret;

	ret = rte_flow_pick_transfer_proxy(port_id, &proxy_port_id, NULL);
	if (ret) {
		DRV_LOG(ERR, "Unable to pick transfer proxy port for port %u. Transfer proxy "
			     "port must be present to create default SQ miss flows.",
			     port_id);
		return ret;
	}
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->dr_ctx) {
		DRV_LOG(DEBUG, "Transfer proxy port (port %u) of port %u must be configured "
			       "for HWS to create default SQ miss flows. Default flows will "
			       "not be created.",
			       proxy_port_id, port_id);
		return 0;
	}
	if (!proxy_priv->hw_esw_sq_miss_root_tbl ||
	    !proxy_priv->hw_esw_sq_miss_tbl) {
		DRV_LOG(ERR, "Transfer proxy port (port %u) of port %u was configured, but "
			     "default flow tables were not created.",
			     proxy_port_id, port_id);
		rte_errno = ENOMEM;
		return -rte_errno;
	}
	/*
	 * Create a root SQ miss flow rule - match E-Switch Manager and SQ,
	 * and jump to group 1.
	 */
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
		.spec = &esw_mgr_spec,
		.mask = &esw_mgr_mask,
	};
	items[1] = (struct rte_flow_item){
		.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
		.spec = &sq_spec,
	};
	items[2] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_END,
	};
	actions[0] = (struct rte_flow_action){
		.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
	};
	actions[1] = (struct rte_flow_action){
		.type = RTE_FLOW_ACTION_TYPE_JUMP,
	};
	actions[2] = (struct rte_flow_action) {
		.type = RTE_FLOW_ACTION_TYPE_END,
	};
	ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_root_tbl,
				       items, 0, actions, 0, &flow_info, external);
	if (ret) {
		DRV_LOG(ERR, "Port %u failed to create root SQ miss flow rule for SQ %u, ret %d",
			port_id, sqn, ret);
		return ret;
	}
	/*
	 * Create a non-root SQ miss flow rule - match REG_C_0 marker and SQ,
	 * and forward to port.
	 */
	items[0] = (struct rte_flow_item){
		.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
		.spec = &reg_c0_spec,
		.mask = &reg_c0_mask,
	};
	items[1] = (struct rte_flow_item){
		.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
		.spec = &sq_spec,
	};
	items[2] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_END,
	};
	actions[0] = (struct rte_flow_action){
		.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
		.conf = &port,
	};
	actions[1] = (struct rte_flow_action){
		.type = RTE_FLOW_ACTION_TYPE_END,
	};
	flow_info.type = MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS;
	ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_tbl,
				       items, 0, actions, 0, &flow_info, external);
	if (ret) {
		DRV_LOG(ERR, "Port %u failed to create HWS SQ miss flow rule for SQ %u, ret %d",
			port_id, sqn, ret);
		return ret;
	}
	return 0;
}

static bool
flow_hw_is_matching_sq_miss_flow(struct mlx5_hw_ctrl_flow *cf,
				 struct rte_eth_dev *dev,
				 uint32_t sqn)
{
	if (cf->owner_dev != dev)
		return false;
	if (cf->info.type == MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS_ROOT && cf->info.esw_mgr_sq == sqn)
		return true;
	if (cf->info.type == MLX5_HW_CTRL_FLOW_TYPE_SQ_MISS && cf->info.esw_mgr_sq == sqn)
		return true;
	return false;
}

int
mlx5_flow_hw_esw_destroy_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn)
{
	uint16_t port_id = dev->data->port_id;
	uint16_t proxy_port_id = dev->data->port_id;
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	struct mlx5_hw_ctrl_flow *cf;
	struct mlx5_hw_ctrl_flow *cf_next;
	int ret;

	ret = rte_flow_pick_transfer_proxy(port_id, &proxy_port_id, NULL);
	if (ret) {
		DRV_LOG(ERR, "Unable to pick transfer proxy port for port %u. Transfer proxy "
			     "port must be present for default SQ miss flow rules to exist.",
			     port_id);
		return ret;
	}
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->dr_ctx)
		return 0;
	if (!proxy_priv->hw_esw_sq_miss_root_tbl ||
	    !proxy_priv->hw_esw_sq_miss_tbl)
		return 0;
	cf = LIST_FIRST(&proxy_priv->hw_ctrl_flows);
	while (cf != NULL) {
		cf_next = LIST_NEXT(cf, next);
		if (flow_hw_is_matching_sq_miss_flow(cf, dev, sqn)) {
			claim_zero(flow_hw_destroy_ctrl_flow(proxy_dev, cf->flow));
			LIST_REMOVE(cf, next);
			mlx5_free(cf);
		}
		cf = cf_next;
	}
	return 0;
}

int
mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev)
{
	uint16_t port_id = dev->data->port_id;
	struct rte_flow_item_ethdev port_spec = {
		.port_id = port_id,
	};
	struct rte_flow_item items[] = {
		{
			.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
			.spec = &port_spec,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	struct rte_flow_action_jump jump = {
		.group = 1,
	};
	struct rte_flow_action actions[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_JUMP,
	};
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	uint16_t proxy_port_id = dev->data->port_id;
	int ret;

	ret = rte_flow_pick_transfer_proxy(port_id, &proxy_port_id, NULL);
	if (ret) {
		DRV_LOG(ERR, "Unable to pick transfer proxy port for port %u. Transfer proxy "
			     "port must be present to create default FDB jump rule.",
			     port_id);
		return ret;
	}
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->dr_ctx) {
		DRV_LOG(DEBUG, "Transfer proxy port (port %u) of port %u must be configured "
			       "for HWS to create default FDB jump rule. Default rule will "
			       "not be created.",
			       proxy_port_id, port_id);
		return 0;
	}
	if (!proxy_priv->hw_esw_zero_tbl) {
		DRV_LOG(ERR, "Transfer proxy port (port %u) of port %u was configured, but "
			     "default flow tables were not created.",
			     proxy_port_id, port_id);
		rte_errno = EINVAL;
		return -rte_errno;
	}
	return flow_hw_create_ctrl_flow(dev, proxy_dev,
					proxy_priv->hw_esw_zero_tbl,
					items, 0, actions, 0, &flow_info, false);
}

int
mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_item_eth promisc = {
		.hdr.dst_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.hdr.src_addr.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.hdr.ether_type = 0,
	};
	struct rte_flow_item eth_all[] = {
		[0] = {
			.type = RTE_FLOW_ITEM_TYPE_ETH,
			.spec = &promisc,
			.mask = &promisc,
		},
		[1] = {
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	struct rte_flow_action_modify_field mreg_action = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_C_1,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.tag_index = REG_A,
		},
		.width = 32,
	};
	struct rte_flow_action copy_reg_action[] = {
		[0] = {
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &mreg_action,
		},
		[1] = {
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
		},
		[2] = {
			.type = RTE_FLOW_ACTION_TYPE_END,
		},
	};
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_TX_META_COPY,
	};

	MLX5_ASSERT(priv->master);
	if (!priv->dr_ctx || !priv->hw_tx_meta_cpy_tbl)
		return 0;
	return flow_hw_create_ctrl_flow(dev, dev,
					priv->hw_tx_meta_cpy_tbl,
					eth_all, 0, copy_reg_action, 0, &flow_info, false);
}

int
mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn, bool external)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_rte_flow_item_sq sq_spec = {
		.queue = sqn,
	};
	struct rte_flow_item items[] = {
		{
			.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
			.spec = &sq_spec,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	/*
	 * Allocate actions array suitable for all cases - extended metadata enabled or not.
	 * With extended metadata there will be an additional MODIFY_FIELD action before JUMP.
	 */
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD },
		{ .type = RTE_FLOW_ACTION_TYPE_JUMP },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_TX_REPR_MATCH,
		.tx_repr_sq = sqn,
	};

	/* It is assumed that caller checked for representor matching. */
	MLX5_ASSERT(priv->sh->config.repr_matching);
	if (!priv->dr_ctx) {
		DRV_LOG(DEBUG, "Port %u must be configured for HWS, before creating "
			       "default egress flow rules. Omitting creation.",
			       dev->data->port_id);
		return 0;
	}
	if (!priv->hw_tx_repr_tagging_tbl) {
		DRV_LOG(ERR, "Port %u is configured for HWS, but table for default "
			     "egress flow rules does not exist.",
			     dev->data->port_id);
		rte_errno = EINVAL;
		return -rte_errno;
	}
	/*
	 * If extended metadata mode is enabled, then an additional MODIFY_FIELD action must be
	 * placed before terminating JUMP action.
	 */
	if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) {
		actions[1].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
		actions[2].type = RTE_FLOW_ACTION_TYPE_JUMP;
	}
	return flow_hw_create_ctrl_flow(dev, dev, priv->hw_tx_repr_tagging_tbl,
					items, 0, actions, 0, &flow_info, external);
}

int
mlx5_flow_hw_lacp_rx_flow(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_item_eth lacp_item = {
		.type = RTE_BE16(RTE_ETHER_TYPE_SLOW),
	};
	struct rte_flow_item eth_lacp[] = {
		[0] = {
			.type = RTE_FLOW_ITEM_TYPE_ETH,
			.spec = &lacp_item,
			.mask = &lacp_item,
		},
		[1] = {
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	struct rte_flow_action miss_action[] = {
		[0] = {
			.type = (enum rte_flow_action_type)
				MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
		},
		[1] = {
			.type = RTE_FLOW_ACTION_TYPE_END,
		},
	};
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_LACP_RX,
	};

	MLX5_ASSERT(priv->master);
	if (!priv->dr_ctx || !priv->hw_lacp_rx_tbl)
		return 0;
	return flow_hw_create_ctrl_flow(dev, dev, priv->hw_lacp_rx_tbl, eth_lacp, 0,
					miss_action, 0, &flow_info, false);
}

static uint32_t
__calc_pattern_flags(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
	switch (eth_pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
		return MLX5_CTRL_PROMISCUOUS;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
		return MLX5_CTRL_ALL_MULTICAST;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
		return MLX5_CTRL_BROADCAST;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
		return MLX5_CTRL_IPV4_MULTICAST;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
		return MLX5_CTRL_IPV6_MULTICAST;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		return MLX5_CTRL_DMAC;
	default:
		/* Should not reach here. */
		MLX5_ASSERT(false);
		return 0;
	}
}

static uint32_t
__calc_vlan_flags(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
	switch (eth_pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		return MLX5_CTRL_VLAN_FILTER;
	default:
		return 0;
	}
}

static bool
eth_pattern_type_is_requested(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
			      uint32_t flags)
{
	uint32_t pattern_flags = __calc_pattern_flags(eth_pattern_type);
	uint32_t vlan_flags = __calc_vlan_flags(eth_pattern_type);
	bool pattern_requested = !!(pattern_flags & flags);
	bool consider_vlan = vlan_flags || (MLX5_CTRL_VLAN_FILTER & flags);
	bool vlan_requested = !!(vlan_flags & flags);

	if (consider_vlan)
		return pattern_requested && vlan_requested;
	else
		return pattern_requested;
}

static bool
rss_type_is_requested(struct mlx5_priv *priv,
		      const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct rte_flow_actions_template *at = priv->hw_ctrl_rx->rss[rss_type];
	unsigned int i;

	for (i = 0; at->actions[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
		if (at->actions[i].type == RTE_FLOW_ACTION_TYPE_RSS) {
			const struct rte_flow_action_rss *rss = at->actions[i].conf;
			uint64_t rss_types = rss->types;

			if ((rss_types & priv->rss_conf.rss_hf) != rss_types)
				return false;
		}
	}
	return true;
}

static const struct rte_flow_item_eth *
__get_eth_spec(const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern)
{
	switch (pattern) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
		return &ctrl_rx_eth_promisc_spec;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
		return &ctrl_rx_eth_mcast_spec;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
		return &ctrl_rx_eth_bcast_spec;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
		return &ctrl_rx_eth_ipv4_mcast_spec;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
		return &ctrl_rx_eth_ipv6_mcast_spec;
	default:
		/* This case should not be reached. */
		MLX5_ASSERT(false);
		return NULL;
	}
}

static int
__flow_hw_ctrl_flows_single(struct rte_eth_dev *dev,
			    struct rte_flow_template_table *tbl,
			    const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
			    const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	const struct rte_flow_item_eth *eth_spec = __get_eth_spec(pattern_type);
	struct rte_flow_item items[5];
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_RSS },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS,
	};

	if (!eth_spec)
		return -EINVAL;
	memset(items, 0, sizeof(items));
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.spec = eth_spec,
	};
	items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VOID };
	items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
	items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
	items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
	/* Without VLAN filtering, only a single flow rule must be created. */
	return flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0, &flow_info, false);
}

static int
__flow_hw_ctrl_flows_single_vlan(struct rte_eth_dev *dev,
				 struct rte_flow_template_table *tbl,
				 const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
				 const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_item_eth *eth_spec = __get_eth_spec(pattern_type);
	struct rte_flow_item items[5];
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_RSS },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS,
	};
	unsigned int i;

	if (!eth_spec)
		return -EINVAL;
	memset(items, 0, sizeof(items));
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.spec = eth_spec,
	};
	/* Optional VLAN for now will be VOID - will be filled later. */
	items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VLAN };
	items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
	items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
	items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
	/* Since VLAN filtering is done, create a single flow rule for each registered vid. */
	for (i = 0; i < priv->vlan_filter_n; ++i) {
		uint16_t vlan = priv->vlan_filter[i];
		struct rte_flow_item_vlan vlan_spec = {
			.hdr.vlan_tci = rte_cpu_to_be_16(vlan),
		};

		items[1].spec = &vlan_spec;
		if (flow_hw_create_ctrl_flow(dev, dev,
					     tbl, items, 0, actions, 0, &flow_info, false))
			return -rte_errno;
	}
	return 0;
}

static int
__flow_hw_ctrl_flows_unicast(struct rte_eth_dev *dev,
			     struct rte_flow_template_table *tbl,
			     const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
			     const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct rte_flow_item_eth eth_spec;
	struct rte_flow_item items[5];
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_RSS },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS,
	};
	const struct rte_ether_addr cmp = {
		.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	};
	unsigned int i;

	RTE_SET_USED(pattern_type);

	memset(&eth_spec, 0, sizeof(eth_spec));
	memset(items, 0, sizeof(items));
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.spec = &eth_spec,
	};
	items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VOID };
	items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
	items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
	items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
	for (i = 0; i < MLX5_MAX_MAC_ADDRESSES; ++i) {
		struct rte_ether_addr *mac = &dev->data->mac_addrs[i];

		if (!memcmp(mac, &cmp, sizeof(*mac)))
			continue;
		memcpy(&eth_spec.hdr.dst_addr.addr_bytes, mac->addr_bytes, RTE_ETHER_ADDR_LEN);
		if (flow_hw_create_ctrl_flow(dev, dev,
					     tbl, items, 0, actions, 0, &flow_info, false))
			return -rte_errno;
	}
	return 0;
}

static int
__flow_hw_ctrl_flows_unicast_vlan(struct rte_eth_dev *dev,
				  struct rte_flow_template_table *tbl,
				  const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
				  const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_item_eth eth_spec;
	struct rte_flow_item items[5];
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_RSS },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};
	struct mlx5_hw_ctrl_flow_info flow_info = {
		.type = MLX5_HW_CTRL_FLOW_TYPE_DEFAULT_RX_RSS,
	};
	const struct rte_ether_addr cmp = {
		.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	};
	unsigned int i;
	unsigned int j;

	RTE_SET_USED(pattern_type);

	memset(&eth_spec, 0, sizeof(eth_spec));
	memset(items, 0, sizeof(items));
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.spec = &eth_spec,
	};
	items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VLAN };
	items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
	items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
	items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
	for (i = 0; i < MLX5_MAX_MAC_ADDRESSES; ++i) {
		struct rte_ether_addr *mac = &dev->data->mac_addrs[i];

		if (!memcmp(mac, &cmp, sizeof(*mac)))
			continue;
		memcpy(&eth_spec.hdr.dst_addr.addr_bytes, mac->addr_bytes, RTE_ETHER_ADDR_LEN);
		for (j = 0; j < priv->vlan_filter_n; ++j) {
			uint16_t vlan = priv->vlan_filter[j];
			struct rte_flow_item_vlan vlan_spec = {
				.hdr.vlan_tci = rte_cpu_to_be_16(vlan),
			};

			items[1].spec = &vlan_spec;
			if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0,
						     &flow_info, false))
				return -rte_errno;
		}
	}
	return 0;
}

static int
__flow_hw_ctrl_flows(struct rte_eth_dev *dev,
		     struct rte_flow_template_table *tbl,
		     const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
		     const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	switch (pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
		return __flow_hw_ctrl_flows_single(dev, tbl, pattern_type, rss_type);
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
		return __flow_hw_ctrl_flows_single_vlan(dev, tbl, pattern_type, rss_type);
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
		return __flow_hw_ctrl_flows_unicast(dev, tbl, pattern_type, rss_type);
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		return __flow_hw_ctrl_flows_unicast_vlan(dev, tbl, pattern_type, rss_type);
	default:
		/* Should not reach here. */
		MLX5_ASSERT(false);
		rte_errno = EINVAL;
		return -EINVAL;
	}
}


int
mlx5_flow_hw_ctrl_flows(struct rte_eth_dev *dev, uint32_t flags)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_hw_ctrl_rx *hw_ctrl_rx;
	unsigned int i;
	unsigned int j;
	int ret = 0;

	RTE_SET_USED(priv);
	RTE_SET_USED(flags);
	if (!priv->dr_ctx) {
		DRV_LOG(DEBUG, "port %u Control flow rules will not be created. "
			       "HWS needs to be configured beforehand.",
			       dev->data->port_id);
		return 0;
	}
	if (!priv->hw_ctrl_rx) {
		DRV_LOG(ERR, "port %u Control flow rules templates were not created.",
			dev->data->port_id);
		rte_errno = EINVAL;
		return -rte_errno;
	}
	hw_ctrl_rx = priv->hw_ctrl_rx;
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
		const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type = i;

		if (!eth_pattern_type_is_requested(eth_pattern_type, flags))
			continue;
		for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
			const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type = j;
			struct rte_flow_actions_template *at;
			struct mlx5_flow_hw_ctrl_rx_table *tmpls = &hw_ctrl_rx->tables[i][j];
			const struct mlx5_flow_template_table_cfg cfg = {
				.attr = tmpls->attr,
				.external = 0,
			};

			if (!hw_ctrl_rx->rss[rss_type]) {
				at = flow_hw_create_ctrl_rx_rss_template(dev, rss_type);
				if (!at)
					return -rte_errno;
				hw_ctrl_rx->rss[rss_type] = at;
			} else {
				at = hw_ctrl_rx->rss[rss_type];
			}
			if (!rss_type_is_requested(priv, rss_type))
				continue;
			if (!tmpls->tbl) {
				tmpls->tbl = flow_hw_table_create(dev, &cfg,
								  &tmpls->pt, 1, &at, 1, NULL);
				if (!tmpls->tbl) {
					DRV_LOG(ERR, "port %u Failed to create template table "
						     "for control flow rules. Unable to create "
						     "control flow rules.",
						     dev->data->port_id);
					return -rte_errno;
				}
			}

			ret = __flow_hw_ctrl_flows(dev, tmpls->tbl, eth_pattern_type, rss_type);
			if (ret) {
				DRV_LOG(ERR, "port %u Failed to create control flow rule.",
					dev->data->port_id);
				return ret;
			}
		}
	}
	return 0;
}

void
mlx5_flow_meter_uninit(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (priv->mtr_policy_arr) {
		mlx5_free(priv->mtr_policy_arr);
		priv->mtr_policy_arr = NULL;
	}
	if (priv->mtr_profile_arr) {
		mlx5_free(priv->mtr_profile_arr);
		priv->mtr_profile_arr = NULL;
	}
	if (priv->hws_mpool) {
		mlx5_aso_mtr_queue_uninit(priv->sh, priv->hws_mpool, NULL);
		mlx5_ipool_destroy(priv->hws_mpool->idx_pool);
		mlx5_free(priv->hws_mpool);
		priv->hws_mpool = NULL;
	}
	if (priv->mtr_bulk.aso) {
		mlx5_free(priv->mtr_bulk.aso);
		priv->mtr_bulk.aso = NULL;
		priv->mtr_bulk.size = 0;
		mlx5_aso_queue_uninit(priv->sh, ASO_OPC_MOD_POLICER);
	}
	if (priv->mtr_bulk.action) {
		mlx5dr_action_destroy(priv->mtr_bulk.action);
		priv->mtr_bulk.action = NULL;
	}
	if (priv->mtr_bulk.devx_obj) {
		claim_zero(mlx5_devx_cmd_destroy(priv->mtr_bulk.devx_obj));
		priv->mtr_bulk.devx_obj = NULL;
	}
}

int
mlx5_flow_meter_init(struct rte_eth_dev *dev,
		     uint32_t nb_meters,
		     uint32_t nb_meter_profiles,
		     uint32_t nb_meter_policies,
		     uint32_t nb_queues)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_devx_obj *dcs = NULL;
	uint32_t log_obj_size;
	int ret = 0;
	int reg_id;
	struct mlx5_aso_mtr *aso;
	uint32_t i;
	struct rte_flow_error error;
	uint32_t flags;
	uint32_t nb_mtrs = rte_align32pow2(nb_meters);
	struct mlx5_indexed_pool_config cfg = {
		.size = sizeof(struct mlx5_aso_mtr),
		.trunk_size = 1 << 12,
		.per_core_cache = 1 << 13,
		.need_lock = 1,
		.release_mem_en = !!priv->sh->config.reclaim_mode,
		.malloc = mlx5_malloc,
		.max_idx = nb_meters,
		.free = mlx5_free,
		.type = "mlx5_hw_mtr_mark_action",
	};

	if (!nb_meters) {
		ret = ENOTSUP;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter configuration is invalid.");
		goto err;
	}
	if (!priv->mtr_en || !priv->sh->meter_aso_en) {
		ret = ENOTSUP;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter ASO is not supported.");
		goto err;
	}
	priv->mtr_config.nb_meters = nb_meters;
	log_obj_size = rte_log2_u32(nb_meters >> 1);
	dcs = mlx5_devx_cmd_create_flow_meter_aso_obj
		(priv->sh->cdev->ctx, priv->sh->cdev->pdn,
			log_obj_size);
	if (!dcs) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter ASO object allocation failed.");
		goto err;
	}
	priv->mtr_bulk.devx_obj = dcs;
	reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL);
	if (reg_id < 0) {
		ret = ENOTSUP;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter register is not available.");
		goto err;
	}
	flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX;
	if (priv->sh->config.dv_esw_en && priv->master)
		flags |= MLX5DR_ACTION_FLAG_HWS_FDB;
	priv->mtr_bulk.action = mlx5dr_action_create_aso_meter
			(priv->dr_ctx, (struct mlx5dr_devx_obj *)dcs,
				reg_id - REG_C_0, flags);
	if (!priv->mtr_bulk.action) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter action creation failed.");
		goto err;
	}
	priv->mtr_bulk.aso = mlx5_malloc(MLX5_MEM_ZERO,
					 sizeof(struct mlx5_aso_mtr) *
					 nb_meters,
					 RTE_CACHE_LINE_SIZE,
					 SOCKET_ID_ANY);
	if (!priv->mtr_bulk.aso) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter bulk ASO allocation failed.");
		goto err;
	}
	priv->mtr_bulk.size = nb_meters;
	aso = priv->mtr_bulk.aso;
	for (i = 0; i < priv->mtr_bulk.size; i++) {
		aso->type = ASO_METER_DIRECT;
		aso->state = ASO_METER_WAIT;
		aso->offset = i;
		aso++;
	}
	priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO,
				sizeof(struct mlx5_aso_mtr_pool),
				RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
	if (!priv->hws_mpool) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter ipool allocation failed.");
		goto err;
	}
	priv->hws_mpool->devx_obj = priv->mtr_bulk.devx_obj;
	priv->hws_mpool->action = priv->mtr_bulk.action;
	priv->hws_mpool->nb_sq = nb_queues;
	if (mlx5_aso_mtr_queue_init(priv->sh, priv->hws_mpool,
				    &priv->sh->mtrmng->pools_mng, nb_queues)) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter ASO queue allocation failed.");
		goto err;
	}
	/*
	 * No need for local cache if Meter number is a small number.
	 * Since flow insertion rate will be very limited in that case.
	 * Here let's set the number to less than default trunk size 4K.
	 */
	if (nb_mtrs <= cfg.trunk_size) {
		cfg.per_core_cache = 0;
		cfg.trunk_size = nb_mtrs;
	} else if (nb_mtrs <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
		cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
	}
	priv->hws_mpool->idx_pool = mlx5_ipool_create(&cfg);
	if (nb_meter_profiles) {
		priv->mtr_config.nb_meter_profiles = nb_meter_profiles;
		priv->mtr_profile_arr =
			mlx5_malloc(MLX5_MEM_ZERO,
				    sizeof(struct mlx5_flow_meter_profile) *
				    nb_meter_profiles,
				    RTE_CACHE_LINE_SIZE,
				    SOCKET_ID_ANY);
		if (!priv->mtr_profile_arr) {
			ret = ENOMEM;
			rte_flow_error_set(&error, ENOMEM,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL, "Meter profile allocation failed.");
			goto err;
		}
	}
	if (nb_meter_policies) {
		priv->mtr_config.nb_meter_policies = nb_meter_policies;
		priv->mtr_policy_arr =
			mlx5_malloc(MLX5_MEM_ZERO,
				    sizeof(struct mlx5_flow_meter_policy) *
				    nb_meter_policies,
				    RTE_CACHE_LINE_SIZE,
				    SOCKET_ID_ANY);
		if (!priv->mtr_policy_arr) {
			ret = ENOMEM;
			rte_flow_error_set(&error, ENOMEM,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL, "Meter policy allocation failed.");
			goto err;
		}
	}
	return 0;
err:
	mlx5_flow_meter_uninit(dev);
	return ret;
}

static __rte_always_inline uint32_t
mlx5_reformat_domain_to_tbl_type(const struct rte_flow_indir_action_conf *domain)
{
	uint32_t tbl_type;

	if (domain->transfer)
		tbl_type = MLX5DR_ACTION_FLAG_HWS_FDB;
	else if (domain->egress)
		tbl_type = MLX5DR_ACTION_FLAG_HWS_TX;
	else if (domain->ingress)
		tbl_type = MLX5DR_ACTION_FLAG_HWS_RX;
	else
		tbl_type = UINT32_MAX;
	return tbl_type;
}

static struct mlx5_hw_encap_decap_action *
__mlx5_reformat_create(struct rte_eth_dev *dev,
		       const struct rte_flow_action_raw_encap *encap_conf,
		       const struct rte_flow_indir_action_conf *domain,
		       enum mlx5dr_action_type type)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_encap_decap_action *handle;
	struct mlx5dr_action_reformat_header hdr;
	uint32_t flags;

	flags = mlx5_reformat_domain_to_tbl_type(domain);
	flags |= (uint32_t)MLX5DR_ACTION_FLAG_SHARED;
	if (flags == UINT32_MAX) {
		DRV_LOG(ERR, "Reformat: invalid indirect action configuration");
		return NULL;
	}
	/* Allocate new list entry. */
	handle = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*handle), 0, SOCKET_ID_ANY);
	if (!handle) {
		DRV_LOG(ERR, "Reformat: failed to allocate reformat entry");
		return NULL;
	}
	handle->action_type = type;
	hdr.sz = encap_conf ? encap_conf->size : 0;
	hdr.data = encap_conf ? encap_conf->data : NULL;
	handle->action = mlx5dr_action_create_reformat(priv->dr_ctx,
					type, 1, &hdr, 0, flags);
	if (!handle->action) {
		DRV_LOG(ERR, "Reformat: failed to create reformat action");
		mlx5_free(handle);
		return NULL;
	}
	return handle;
}

/**
 * Create mlx5 reformat action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] conf
 *   Pointer to the indirect action parameters.
 * @param[in] encap_action
 *   Pointer to the raw_encap action configuration.
 * @param[in] decap_action
 *   Pointer to the raw_decap action configuration.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   A valid shared action handle in case of success, NULL otherwise and
 *   rte_errno is set.
 */
struct mlx5_hw_encap_decap_action*
mlx5_reformat_action_create(struct rte_eth_dev *dev,
			    const struct rte_flow_indir_action_conf *conf,
			    const struct rte_flow_action *encap_action,
			    const struct rte_flow_action *decap_action,
			    struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_encap_decap_action *handle;
	const struct rte_flow_action_raw_encap *encap = NULL;
	const struct rte_flow_action_raw_decap *decap = NULL;
	enum mlx5dr_action_type type = MLX5DR_ACTION_TYP_LAST;

	MLX5_ASSERT(!encap_action || encap_action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP);
	MLX5_ASSERT(!decap_action || decap_action->type == RTE_FLOW_ACTION_TYPE_RAW_DECAP);
	if (priv->sh->config.dv_flow_en != 2) {
		rte_flow_error_set(error, ENOTSUP,
				   RTE_FLOW_ERROR_TYPE_ACTION, encap_action,
				   "Reformat: hardware does not support");
		return NULL;
	}
	if (!conf || (conf->transfer + conf->egress + conf->ingress != 1)) {
		rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_ACTION, encap_action,
				   "Reformat: domain should be specified");
		return NULL;
	}
	if ((encap_action && !encap_action->conf) || (decap_action && !decap_action->conf)) {
		rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_ACTION, encap_action,
				   "Reformat: missed action configuration");
		return NULL;
	}
	if (encap_action && !decap_action) {
		encap = (const struct rte_flow_action_raw_encap *)encap_action->conf;
		if (!encap->size || encap->size > MLX5_ENCAP_MAX_LEN ||
		    encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
			rte_flow_error_set(error, EINVAL,
					   RTE_FLOW_ERROR_TYPE_ACTION, encap_action,
					   "Reformat: Invalid encap length");
			return NULL;
		}
		type = MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2;
	} else if (decap_action && !encap_action) {
		decap = (const struct rte_flow_action_raw_decap *)decap_action->conf;
		if (!decap->size || decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
			rte_flow_error_set(error, EINVAL,
					   RTE_FLOW_ERROR_TYPE_ACTION, encap_action,
					   "Reformat: Invalid decap length");
			return NULL;
		}
		type = MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2;
	} else if (encap_action && decap_action) {
		decap = (const struct rte_flow_action_raw_decap *)decap_action->conf;
		encap = (const struct rte_flow_action_raw_encap *)encap_action->conf;
		if (decap->size < MLX5_ENCAPSULATION_DECISION_SIZE &&
		    encap->size >= MLX5_ENCAPSULATION_DECISION_SIZE &&
		    encap->size <= MLX5_ENCAP_MAX_LEN) {
			type = MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L3;
		} else if (decap->size >= MLX5_ENCAPSULATION_DECISION_SIZE &&
			   encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
			type = MLX5DR_ACTION_TYP_REFORMAT_TNL_L3_TO_L2;
		} else {
			rte_flow_error_set(error, EINVAL,
					   RTE_FLOW_ERROR_TYPE_ACTION, encap_action,
					   "Reformat: Invalid decap & encap length");
			return NULL;
		}
	} else if (!encap_action && !decap_action) {
		rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_ACTION, encap_action,
				   "Reformat: Invalid decap & encap configurations");
		return NULL;
	}
	if (!priv->dr_ctx) {
		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
				   encap_action, "Reformat: HWS not supported");
		return NULL;
	}
	handle = __mlx5_reformat_create(dev, encap, conf, type);
	if (!handle) {
		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, encap_action,
				   "Reformat: failed to create indirect action");
		return NULL;
	}
	return handle;
}

/**
 * Destroy the indirect reformat action.
 * Release action related resources on the NIC and the memory.
 * Lock free, (mutex should be acquired by caller).
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] handle
 *   The indirect action list handle to be removed.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
int
mlx5_reformat_action_destroy(struct rte_eth_dev *dev,
			     struct rte_flow_action_list_handle *handle,
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_encap_decap_action *action;

	action = (struct mlx5_hw_encap_decap_action *)handle;
	if (!priv->dr_ctx || !action)
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, handle,
					  "Reformat: invalid action handle");
	mlx5dr_action_destroy(action->action);
	mlx5_free(handle);
	return 0;
}

static struct rte_flow_fp_ops mlx5_flow_hw_fp_ops = {
	.async_create = flow_hw_async_flow_create,
	.async_create_by_index = flow_hw_async_flow_create_by_index,
	.async_actions_update = flow_hw_async_flow_update,
	.async_destroy = flow_hw_async_flow_destroy,
	.push = flow_hw_push,
	.pull = flow_hw_pull,
	.async_action_handle_create = flow_hw_action_handle_create,
	.async_action_handle_destroy = flow_hw_action_handle_destroy,
	.async_action_handle_update = flow_hw_action_handle_update,
	.async_action_handle_query = flow_hw_action_handle_query,
	.async_action_handle_query_update = flow_hw_async_action_handle_query_update,
	.async_action_list_handle_create = flow_hw_async_action_list_handle_create,
	.async_action_list_handle_destroy = flow_hw_async_action_list_handle_destroy,
	.async_action_list_handle_query_update =
		flow_hw_async_action_list_handle_query_update,
};

#endif