xref: /dpdk/drivers/net/mlx5/mlx5_flow.c (revision d6dc072aeb126c7ac2f282f7f78d7b84bde9eac8)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2016 6WIND S.A.
3  * Copyright 2016 Mellanox Technologies, Ltd
4  */
5 
6 #include <stdalign.h>
7 #include <stdint.h>
8 #include <string.h>
9 #include <stdbool.h>
10 #include <sys/queue.h>
11 
12 #include <rte_common.h>
13 #include <rte_ether.h>
14 #include <ethdev_driver.h>
15 #include <rte_eal_paging.h>
16 #include <rte_flow.h>
17 #include <rte_cycles.h>
18 #include <rte_flow_driver.h>
19 #include <rte_malloc.h>
20 #include <rte_ip.h>
21 
22 #include <mlx5_glue.h>
23 #include <mlx5_devx_cmds.h>
24 #include <mlx5_prm.h>
25 #include <mlx5_malloc.h>
26 
27 #include "mlx5_defs.h"
28 #include "mlx5.h"
29 #include "mlx5_flow.h"
30 #include "mlx5_flow_os.h"
31 #include "mlx5_rx.h"
32 #include "mlx5_tx.h"
33 #include "mlx5_common_os.h"
34 #include "rte_pmd_mlx5.h"
35 
36 /*
37  * Shared array for quick translation between port_id and vport mask/values
38  * used for HWS rules.
39  */
40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
41 
42 struct tunnel_default_miss_ctx {
43 	uint16_t *queue;
44 	__extension__
45 	union {
46 		struct rte_flow_action_rss action_rss;
47 		struct rte_flow_action_queue miss_queue;
48 		struct rte_flow_action_jump miss_jump;
49 		uint8_t raw[0];
50 	};
51 };
52 
53 void
54 mlx5_indirect_list_handles_release(struct rte_eth_dev *dev)
55 {
56 	struct mlx5_priv *priv = dev->data->dev_private;
57 #ifdef HAVE_MLX5_HWS_SUPPORT
58 	struct rte_flow_error error;
59 #endif
60 
61 	while (!LIST_EMPTY(&priv->indirect_list_head)) {
62 		struct mlx5_indirect_list *e =
63 			LIST_FIRST(&priv->indirect_list_head);
64 
65 		LIST_REMOVE(e, entry);
66 		switch (e->type) {
67 #ifdef HAVE_MLX5_HWS_SUPPORT
68 		case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR:
69 			mlx5_hw_mirror_destroy(dev, (struct mlx5_mirror *)e);
70 		break;
71 		case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY:
72 			mlx5_destroy_legacy_indirect(dev, e);
73 			break;
74 		case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT:
75 			mlx5_reformat_action_destroy(dev,
76 				(struct rte_flow_action_list_handle *)e, &error);
77 			break;
78 #endif
79 		default:
80 			DRV_LOG(ERR, "invalid indirect list type");
81 			MLX5_ASSERT(false);
82 			break;
83 		}
84 	}
85 }
86 
87 static int
88 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
89 			     struct rte_flow *flow,
90 			     const struct rte_flow_attr *attr,
91 			     const struct rte_flow_action *app_actions,
92 			     uint32_t flow_idx,
93 			     const struct mlx5_flow_tunnel *tunnel,
94 			     struct tunnel_default_miss_ctx *ctx,
95 			     struct rte_flow_error *error);
96 static struct mlx5_flow_tunnel *
97 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
98 static void
99 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
100 static uint32_t
101 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
102 				const struct mlx5_flow_tunnel *tunnel,
103 				uint32_t group, uint32_t *table,
104 				struct rte_flow_error *error);
105 
106 /** Device flow drivers. */
107 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
108 
109 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
110 
111 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
112 	[MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
113 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
114 	[MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
115 #endif
116 #ifdef HAVE_MLX5_HWS_SUPPORT
117 	[MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops,
118 #endif
119 	[MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
120 	[MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
121 };
122 
123 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
124 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
125 	(const int []){ \
126 		__VA_ARGS__, 0, \
127 	}
128 
129 /** Node object of input graph for mlx5_flow_expand_rss(). */
130 struct mlx5_flow_expand_node {
131 	const int *const next;
132 	/**<
133 	 * List of next node indexes. Index 0 is interpreted as a terminator.
134 	 */
135 	const enum rte_flow_item_type type;
136 	/**< Pattern item type of current node. */
137 	uint64_t rss_types;
138 	/**<
139 	 * RSS types bit-field associated with this node
140 	 * (see RTE_ETH_RSS_* definitions).
141 	 */
142 	uint64_t node_flags;
143 	/**<
144 	 *  Bit-fields that define how the node is used in the expansion.
145 	 * (see MLX5_EXPANSION_NODE_* definitions).
146 	 */
147 };
148 
149 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */
150 struct mlx5_flow_expand_sqn {
151 	uint32_t entries; /** Number of entries */
152 	struct {
153 		struct rte_flow_item *pattern; /**< Expanded pattern array. */
154 		uint32_t priority; /**< Priority offset for each expansion. */
155 	} entry[];
156 };
157 
158 /* Optional expand field. The expansion alg will not go deeper. */
159 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
160 
161 /* The node is not added implicitly as expansion to the flow pattern.
162  * If the node type does not match the flow pattern item type, the
163  * expansion alg will go deeper to its next items.
164  * In the current implementation, the list of next nodes indexes can
165  * have up to one node with this flag set and it has to be the last
166  * node index (before the list terminator).
167  */
168 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
169 
170 /** Object returned by mlx5_flow_expand_rss(). */
171 struct mlx5_flow_expand_rss {
172 	uint32_t entries;
173 	/**< Number of entries @p patterns and @p priorities. */
174 	struct {
175 		struct rte_flow_item *pattern; /**< Expanded pattern array. */
176 		uint32_t priority; /**< Priority offset for each expansion. */
177 	} entry[];
178 };
179 
180 static void
181 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
182 
183 static const struct mlx5_flow_expand_node *
184 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
185 		unsigned int item_idx,
186 		const struct mlx5_flow_expand_node graph[],
187 		const struct mlx5_flow_expand_node *node);
188 
189 static __rte_always_inline int
190 mlx5_need_cache_flow(const struct mlx5_priv *priv,
191 		     const struct rte_flow_attr *attr)
192 {
193 	return priv->isolated && priv->sh->config.dv_flow_en == 1 &&
194 		(attr ? !attr->group : true) &&
195 		priv->mode_info.mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY &&
196 		(!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule);
197 }
198 
199 static bool
200 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
201 {
202 	switch (item->type) {
203 	case RTE_FLOW_ITEM_TYPE_ETH:
204 	case RTE_FLOW_ITEM_TYPE_VLAN:
205 	case RTE_FLOW_ITEM_TYPE_IPV4:
206 	case RTE_FLOW_ITEM_TYPE_IPV6:
207 	case RTE_FLOW_ITEM_TYPE_UDP:
208 	case RTE_FLOW_ITEM_TYPE_TCP:
209 	case RTE_FLOW_ITEM_TYPE_ESP:
210 	case RTE_FLOW_ITEM_TYPE_ICMP:
211 	case RTE_FLOW_ITEM_TYPE_ICMP6:
212 	case RTE_FLOW_ITEM_TYPE_VXLAN:
213 	case RTE_FLOW_ITEM_TYPE_NVGRE:
214 	case RTE_FLOW_ITEM_TYPE_GRE:
215 	case RTE_FLOW_ITEM_TYPE_GENEVE:
216 	case RTE_FLOW_ITEM_TYPE_MPLS:
217 	case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
218 	case RTE_FLOW_ITEM_TYPE_GRE_KEY:
219 	case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
220 	case RTE_FLOW_ITEM_TYPE_GTP:
221 		return true;
222 	default:
223 		break;
224 	}
225 	return false;
226 }
227 
228 /**
229  * Network Service Header (NSH) and its next protocol values
230  * are described in RFC-8393.
231  */
232 static enum rte_flow_item_type
233 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask)
234 {
235 	enum rte_flow_item_type type;
236 
237 	switch (proto_mask & proto_spec) {
238 	case 0:
239 		type = RTE_FLOW_ITEM_TYPE_VOID;
240 		break;
241 	case RTE_VXLAN_GPE_TYPE_IPV4:
242 		type = RTE_FLOW_ITEM_TYPE_IPV4;
243 		break;
244 	case RTE_VXLAN_GPE_TYPE_IPV6:
245 		type = RTE_VXLAN_GPE_TYPE_IPV6;
246 		break;
247 	case RTE_VXLAN_GPE_TYPE_ETH:
248 		type = RTE_FLOW_ITEM_TYPE_ETH;
249 		break;
250 	default:
251 		type = RTE_FLOW_ITEM_TYPE_END;
252 	}
253 	return type;
254 }
255 
256 static enum rte_flow_item_type
257 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask)
258 {
259 	enum rte_flow_item_type type;
260 
261 	switch (proto_mask & proto_spec) {
262 	case 0:
263 		type = RTE_FLOW_ITEM_TYPE_VOID;
264 		break;
265 	case IPPROTO_UDP:
266 		type = RTE_FLOW_ITEM_TYPE_UDP;
267 		break;
268 	case IPPROTO_TCP:
269 		type = RTE_FLOW_ITEM_TYPE_TCP;
270 		break;
271 	case IPPROTO_IPIP:
272 		type = RTE_FLOW_ITEM_TYPE_IPV4;
273 		break;
274 	case IPPROTO_IPV6:
275 		type = RTE_FLOW_ITEM_TYPE_IPV6;
276 		break;
277 	case IPPROTO_ESP:
278 		type = RTE_FLOW_ITEM_TYPE_ESP;
279 		break;
280 	default:
281 		type = RTE_FLOW_ITEM_TYPE_END;
282 	}
283 	return type;
284 }
285 
286 static enum rte_flow_item_type
287 mlx5_ethertype_to_item_type(rte_be16_t type_spec,
288 			    rte_be16_t type_mask, bool is_tunnel)
289 {
290 	enum rte_flow_item_type type;
291 
292 	switch (rte_be_to_cpu_16(type_spec & type_mask)) {
293 	case 0:
294 		type = RTE_FLOW_ITEM_TYPE_VOID;
295 		break;
296 	case RTE_ETHER_TYPE_TEB:
297 		type = is_tunnel ?
298 		       RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END;
299 		break;
300 	case RTE_ETHER_TYPE_VLAN:
301 		type = !is_tunnel ?
302 		       RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END;
303 		break;
304 	case RTE_ETHER_TYPE_IPV4:
305 		type = RTE_FLOW_ITEM_TYPE_IPV4;
306 		break;
307 	case RTE_ETHER_TYPE_IPV6:
308 		type = RTE_FLOW_ITEM_TYPE_IPV6;
309 		break;
310 	default:
311 		type = RTE_FLOW_ITEM_TYPE_END;
312 	}
313 	return type;
314 }
315 
316 static enum rte_flow_item_type
317 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
318 {
319 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld)                              \
320 	do {                                                             \
321 		const void *m = item->mask;                              \
322 		const void *s = item->spec;                              \
323 		mask = m ?                                               \
324 			((const struct rte_flow_item_##type *)m)->fld :  \
325 			rte_flow_item_##type##_mask.fld;                 \
326 		spec = ((const struct rte_flow_item_##type *)s)->fld;    \
327 	} while (0)
328 
329 	enum rte_flow_item_type ret;
330 	uint16_t spec, mask;
331 
332 	if (item == NULL || item->spec == NULL)
333 		return RTE_FLOW_ITEM_TYPE_VOID;
334 	switch (item->type) {
335 	case RTE_FLOW_ITEM_TYPE_ETH:
336 		MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type);
337 		if (!mask)
338 			return RTE_FLOW_ITEM_TYPE_VOID;
339 		ret = mlx5_ethertype_to_item_type(spec, mask, false);
340 		break;
341 	case RTE_FLOW_ITEM_TYPE_VLAN:
342 		MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto);
343 		if (!mask)
344 			return RTE_FLOW_ITEM_TYPE_VOID;
345 		ret = mlx5_ethertype_to_item_type(spec, mask, false);
346 		break;
347 	case RTE_FLOW_ITEM_TYPE_IPV4:
348 		MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id);
349 		if (!mask)
350 			return RTE_FLOW_ITEM_TYPE_VOID;
351 		ret = mlx5_inet_proto_to_item_type(spec, mask);
352 		break;
353 	case RTE_FLOW_ITEM_TYPE_IPV6:
354 		MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto);
355 		if (!mask)
356 			return RTE_FLOW_ITEM_TYPE_VOID;
357 		ret = mlx5_inet_proto_to_item_type(spec, mask);
358 		break;
359 	case RTE_FLOW_ITEM_TYPE_GENEVE:
360 		MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol);
361 		ret = mlx5_ethertype_to_item_type(spec, mask, true);
362 		break;
363 	case RTE_FLOW_ITEM_TYPE_GRE:
364 		MLX5_XSET_ITEM_MASK_SPEC(gre, protocol);
365 		ret = mlx5_ethertype_to_item_type(spec, mask, true);
366 		break;
367 	case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
368 		MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto);
369 		ret = mlx5_nsh_proto_to_item_type(spec, mask);
370 		break;
371 	default:
372 		ret = RTE_FLOW_ITEM_TYPE_VOID;
373 		break;
374 	}
375 	return ret;
376 #undef MLX5_XSET_ITEM_MASK_SPEC
377 }
378 
379 static const int *
380 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[],
381 		const int *next_node)
382 {
383 	const struct mlx5_flow_expand_node *node = NULL;
384 	const int *next = next_node;
385 
386 	while (next && *next) {
387 		/*
388 		 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
389 		 * flag set, because they were not found in the flow pattern.
390 		 */
391 		node = &graph[*next];
392 		if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
393 			break;
394 		next = node->next;
395 	}
396 	return next;
397 }
398 
399 #define MLX5_RSS_EXP_ELT_N 32
400 
401 /**
402  * Expand RSS flows into several possible flows according to the RSS hash
403  * fields requested and the driver capabilities.
404  *
405  * @param[out] buf
406  *   Buffer to store the result expansion.
407  * @param[in] size
408  *   Buffer size in bytes. If 0, @p buf can be NULL.
409  * @param[in] pattern
410  *   User flow pattern.
411  * @param[in] types
412  *   RSS types to expand (see RTE_ETH_RSS_* definitions).
413  * @param[in] graph
414  *   Input graph to expand @p pattern according to @p types.
415  * @param[in] graph_root_index
416  *   Index of root node in @p graph, typically 0.
417  *
418  * @return
419  *   A positive value representing the size of @p buf in bytes regardless of
420  *   @p size on success, a negative errno value otherwise and rte_errno is
421  *   set, the following errors are defined:
422  *
423  *   -E2BIG: graph-depth @p graph is too deep.
424  *   -EINVAL: @p size has not enough space for expanded pattern.
425  */
426 static int
427 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
428 		     const struct rte_flow_item *pattern, uint64_t types,
429 		     const struct mlx5_flow_expand_node graph[],
430 		     int graph_root_index)
431 {
432 	const struct rte_flow_item *item;
433 	const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
434 	const int *next_node;
435 	const int *stack[MLX5_RSS_EXP_ELT_N];
436 	int stack_pos = 0;
437 	struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
438 	unsigned int i, item_idx, last_expand_item_idx = 0;
439 	size_t lsize;
440 	size_t user_pattern_size = 0;
441 	void *addr = NULL;
442 	const struct mlx5_flow_expand_node *next = NULL;
443 	struct rte_flow_item missed_item;
444 	int missed = 0;
445 	int elt = 0;
446 	const struct rte_flow_item *last_expand_item = NULL;
447 
448 	memset(&missed_item, 0, sizeof(missed_item));
449 	lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
450 		MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
451 	if (lsize > size)
452 		return -EINVAL;
453 	buf->entry[0].priority = 0;
454 	buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
455 	buf->entries = 0;
456 	addr = buf->entry[0].pattern;
457 	for (item = pattern, item_idx = 0;
458 			item->type != RTE_FLOW_ITEM_TYPE_END;
459 			item++, item_idx++) {
460 		if (!mlx5_flow_is_rss_expandable_item(item)) {
461 			user_pattern_size += sizeof(*item);
462 			continue;
463 		}
464 		last_expand_item = item;
465 		last_expand_item_idx = item_idx;
466 		i = 0;
467 		while (node->next && node->next[i]) {
468 			next = &graph[node->next[i]];
469 			if (next->type == item->type)
470 				break;
471 			if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
472 				node = next;
473 				i = 0;
474 			} else {
475 				++i;
476 			}
477 		}
478 		if (next)
479 			node = next;
480 		user_pattern_size += sizeof(*item);
481 	}
482 	user_pattern_size += sizeof(*item); /* Handle END item. */
483 	lsize += user_pattern_size;
484 	if (lsize > size)
485 		return -EINVAL;
486 	/* Copy the user pattern in the first entry of the buffer. */
487 	rte_memcpy(addr, pattern, user_pattern_size);
488 	addr = (void *)(((uintptr_t)addr) + user_pattern_size);
489 	buf->entries = 1;
490 	/* Start expanding. */
491 	memset(flow_items, 0, sizeof(flow_items));
492 	user_pattern_size -= sizeof(*item);
493 	/*
494 	 * Check if the last valid item has spec set, need complete pattern,
495 	 * and the pattern can be used for expansion.
496 	 */
497 	missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item);
498 	if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
499 		/* Item type END indicates expansion is not required. */
500 		return lsize;
501 	}
502 	if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
503 		next = NULL;
504 		missed = 1;
505 		i = 0;
506 		while (node->next && node->next[i]) {
507 			next = &graph[node->next[i]];
508 			if (next->type == missed_item.type) {
509 				flow_items[0].type = missed_item.type;
510 				flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
511 				break;
512 			}
513 			if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
514 				node = next;
515 				i = 0;
516 			} else {
517 				++i;
518 			}
519 			next = NULL;
520 		}
521 	}
522 	if (next && missed) {
523 		elt = 2; /* missed item + item end. */
524 		node = next;
525 		lsize += elt * sizeof(*item) + user_pattern_size;
526 		if (lsize > size)
527 			return -EINVAL;
528 		if (node->rss_types & types) {
529 			buf->entry[buf->entries].priority = 1;
530 			buf->entry[buf->entries].pattern = addr;
531 			buf->entries++;
532 			rte_memcpy(addr, buf->entry[0].pattern,
533 				   user_pattern_size);
534 			addr = (void *)(((uintptr_t)addr) + user_pattern_size);
535 			rte_memcpy(addr, flow_items, elt * sizeof(*item));
536 			addr = (void *)(((uintptr_t)addr) +
537 					elt * sizeof(*item));
538 		}
539 	} else if (last_expand_item != NULL) {
540 		node = mlx5_flow_expand_rss_adjust_node(pattern,
541 				last_expand_item_idx, graph, node);
542 	}
543 	memset(flow_items, 0, sizeof(flow_items));
544 	next_node = mlx5_flow_expand_rss_skip_explicit(graph,
545 			node->next);
546 	stack[stack_pos] = next_node;
547 	node = next_node ? &graph[*next_node] : NULL;
548 	while (node) {
549 		flow_items[stack_pos].type = node->type;
550 		if (node->rss_types & types) {
551 			size_t n;
552 			/*
553 			 * compute the number of items to copy from the
554 			 * expansion and copy it.
555 			 * When the stack_pos is 0, there are 1 element in it,
556 			 * plus the addition END item.
557 			 */
558 			elt = stack_pos + 2;
559 			flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
560 			lsize += elt * sizeof(*item) + user_pattern_size;
561 			if (lsize > size)
562 				return -EINVAL;
563 			n = elt * sizeof(*item);
564 			MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N);
565 			buf->entry[buf->entries].priority =
566 				stack_pos + 1 + missed;
567 			buf->entry[buf->entries].pattern = addr;
568 			buf->entries++;
569 			rte_memcpy(addr, buf->entry[0].pattern,
570 				   user_pattern_size);
571 			addr = (void *)(((uintptr_t)addr) +
572 					user_pattern_size);
573 			rte_memcpy(addr, &missed_item,
574 				   missed * sizeof(*item));
575 			addr = (void *)(((uintptr_t)addr) +
576 				missed * sizeof(*item));
577 			rte_memcpy(addr, flow_items, n);
578 			addr = (void *)(((uintptr_t)addr) + n);
579 		}
580 		/* Go deeper. */
581 		if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
582 				node->next) {
583 			next_node = mlx5_flow_expand_rss_skip_explicit(graph,
584 					node->next);
585 			if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
586 				rte_errno = E2BIG;
587 				return -rte_errno;
588 			}
589 			stack[stack_pos] = next_node;
590 		} else if (*(next_node + 1)) {
591 			/* Follow up with the next possibility. */
592 			next_node = mlx5_flow_expand_rss_skip_explicit(graph,
593 					++next_node);
594 		} else if (!stack_pos) {
595 			/*
596 			 * Completing the traverse over the different paths.
597 			 * The next_node is advanced to the terminator.
598 			 */
599 			++next_node;
600 		} else {
601 			/* Move to the next path. */
602 			while (stack_pos) {
603 				next_node = stack[--stack_pos];
604 				next_node++;
605 				if (*next_node)
606 					break;
607 			}
608 			next_node = mlx5_flow_expand_rss_skip_explicit(graph,
609 					next_node);
610 			stack[stack_pos] = next_node;
611 		}
612 		node = next_node && *next_node ? &graph[*next_node] : NULL;
613 	};
614 	return lsize;
615 }
616 
617 /**
618  * Expand SQN flows into several possible flows according to the Tx queue
619  * number
620  *
621  * @param[in] buf
622  *   Buffer to store the result expansion.
623  * @param[in] size
624  *   Buffer size in bytes. If 0, @p buf can be NULL.
625  * @param[in] pattern
626  *   User flow pattern.
627  * @param[in] sq_specs
628  *   Buffer to store sq spec.
629  *
630  * @return
631  *   0 for success and negative value for failure
632  *
633  */
634 static int
635 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size,
636 		     const struct rte_flow_item *pattern,
637 		     struct mlx5_rte_flow_item_sq *sq_specs)
638 {
639 	const struct rte_flow_item *item;
640 	bool port_representor = false;
641 	size_t user_pattern_size = 0;
642 	struct rte_eth_dev *dev;
643 	struct mlx5_priv *priv;
644 	void *addr = NULL;
645 	uint16_t port_id;
646 	size_t lsize;
647 	int elt = 2;
648 	uint16_t i;
649 
650 	buf->entries = 0;
651 	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
652 		if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) {
653 			const struct rte_flow_item_ethdev *pid_v = item->spec;
654 
655 			if (!pid_v)
656 				return 0;
657 			port_id = pid_v->port_id;
658 			port_representor = true;
659 		}
660 		user_pattern_size += sizeof(*item);
661 	}
662 	if (!port_representor)
663 		return 0;
664 	dev = &rte_eth_devices[port_id];
665 	priv = dev->data->dev_private;
666 	buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n];
667 	lsize = offsetof(struct mlx5_flow_expand_sqn, entry) +
668 		sizeof(buf->entry[0]) * priv->txqs_n;
669 	if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size)
670 		return -EINVAL;
671 	addr = buf->entry[0].pattern;
672 	for (i = 0; i != priv->txqs_n; ++i) {
673 		struct rte_flow_item pattern_add[] = {
674 			{
675 				.type = (enum rte_flow_item_type)
676 					MLX5_RTE_FLOW_ITEM_TYPE_SQ,
677 				.spec = &sq_specs[i],
678 			},
679 			{
680 				.type = RTE_FLOW_ITEM_TYPE_END,
681 			},
682 		};
683 		struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i);
684 
685 		if (txq == NULL)
686 			return -EINVAL;
687 		buf->entry[i].pattern = addr;
688 		sq_specs[i].queue = mlx5_txq_get_sqn(txq);
689 		mlx5_txq_release(dev, i);
690 		rte_memcpy(addr, pattern, user_pattern_size);
691 		addr = (void *)(((uintptr_t)addr) + user_pattern_size);
692 		rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt);
693 		addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt);
694 		buf->entries++;
695 	}
696 	return 0;
697 }
698 
699 enum mlx5_expansion {
700 	MLX5_EXPANSION_ROOT,
701 	MLX5_EXPANSION_ROOT_OUTER,
702 	MLX5_EXPANSION_OUTER_ETH,
703 	MLX5_EXPANSION_OUTER_VLAN,
704 	MLX5_EXPANSION_OUTER_IPV4,
705 	MLX5_EXPANSION_OUTER_IPV4_UDP,
706 	MLX5_EXPANSION_OUTER_IPV4_TCP,
707 	MLX5_EXPANSION_OUTER_IPV4_ESP,
708 	MLX5_EXPANSION_OUTER_IPV4_ICMP,
709 	MLX5_EXPANSION_OUTER_IPV6,
710 	MLX5_EXPANSION_OUTER_IPV6_UDP,
711 	MLX5_EXPANSION_OUTER_IPV6_TCP,
712 	MLX5_EXPANSION_OUTER_IPV6_ESP,
713 	MLX5_EXPANSION_OUTER_IPV6_ICMP6,
714 	MLX5_EXPANSION_VXLAN,
715 	MLX5_EXPANSION_STD_VXLAN,
716 	MLX5_EXPANSION_L3_VXLAN,
717 	MLX5_EXPANSION_VXLAN_GPE,
718 	MLX5_EXPANSION_GRE,
719 	MLX5_EXPANSION_NVGRE,
720 	MLX5_EXPANSION_GRE_KEY,
721 	MLX5_EXPANSION_MPLS,
722 	MLX5_EXPANSION_ETH,
723 	MLX5_EXPANSION_VLAN,
724 	MLX5_EXPANSION_IPV4,
725 	MLX5_EXPANSION_IPV4_UDP,
726 	MLX5_EXPANSION_IPV4_TCP,
727 	MLX5_EXPANSION_IPV4_ESP,
728 	MLX5_EXPANSION_IPV4_ICMP,
729 	MLX5_EXPANSION_IPV6,
730 	MLX5_EXPANSION_IPV6_UDP,
731 	MLX5_EXPANSION_IPV6_TCP,
732 	MLX5_EXPANSION_IPV6_ESP,
733 	MLX5_EXPANSION_IPV6_ICMP6,
734 	MLX5_EXPANSION_IPV6_FRAG_EXT,
735 	MLX5_EXPANSION_GTP,
736 	MLX5_EXPANSION_GENEVE,
737 };
738 
739 /** Supported expansion of items. */
740 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
741 	[MLX5_EXPANSION_ROOT] = {
742 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
743 						  MLX5_EXPANSION_IPV4,
744 						  MLX5_EXPANSION_IPV6),
745 		.type = RTE_FLOW_ITEM_TYPE_END,
746 	},
747 	[MLX5_EXPANSION_ROOT_OUTER] = {
748 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
749 						  MLX5_EXPANSION_OUTER_IPV4,
750 						  MLX5_EXPANSION_OUTER_IPV6),
751 		.type = RTE_FLOW_ITEM_TYPE_END,
752 	},
753 	[MLX5_EXPANSION_OUTER_ETH] = {
754 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
755 		.type = RTE_FLOW_ITEM_TYPE_ETH,
756 		.rss_types = 0,
757 	},
758 	[MLX5_EXPANSION_OUTER_VLAN] = {
759 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
760 						  MLX5_EXPANSION_OUTER_IPV6),
761 		.type = RTE_FLOW_ITEM_TYPE_VLAN,
762 		.node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
763 	},
764 	[MLX5_EXPANSION_OUTER_IPV4] = {
765 		.next = MLX5_FLOW_EXPAND_RSS_NEXT
766 			(MLX5_EXPANSION_OUTER_IPV4_UDP,
767 			 MLX5_EXPANSION_OUTER_IPV4_TCP,
768 			 MLX5_EXPANSION_OUTER_IPV4_ESP,
769 			 MLX5_EXPANSION_OUTER_IPV4_ICMP,
770 			 MLX5_EXPANSION_GRE,
771 			 MLX5_EXPANSION_NVGRE,
772 			 MLX5_EXPANSION_IPV4,
773 			 MLX5_EXPANSION_IPV6),
774 		.type = RTE_FLOW_ITEM_TYPE_IPV4,
775 		.rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
776 			RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
777 	},
778 	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
779 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
780 						  MLX5_EXPANSION_VXLAN_GPE,
781 						  MLX5_EXPANSION_MPLS,
782 						  MLX5_EXPANSION_GENEVE,
783 						  MLX5_EXPANSION_GTP),
784 		.type = RTE_FLOW_ITEM_TYPE_UDP,
785 		.rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
786 	},
787 	[MLX5_EXPANSION_OUTER_IPV4_TCP] = {
788 		.type = RTE_FLOW_ITEM_TYPE_TCP,
789 		.rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
790 	},
791 	[MLX5_EXPANSION_OUTER_IPV4_ESP] = {
792 		.type = RTE_FLOW_ITEM_TYPE_ESP,
793 		.rss_types = RTE_ETH_RSS_ESP,
794 	},
795 	[MLX5_EXPANSION_OUTER_IPV4_ICMP] = {
796 		.type = RTE_FLOW_ITEM_TYPE_ICMP,
797 	},
798 	[MLX5_EXPANSION_OUTER_IPV6] = {
799 		.next = MLX5_FLOW_EXPAND_RSS_NEXT
800 			(MLX5_EXPANSION_OUTER_IPV6_UDP,
801 			 MLX5_EXPANSION_OUTER_IPV6_TCP,
802 			 MLX5_EXPANSION_OUTER_IPV6_ESP,
803 			 MLX5_EXPANSION_OUTER_IPV6_ICMP6,
804 			 MLX5_EXPANSION_IPV4,
805 			 MLX5_EXPANSION_IPV6,
806 			 MLX5_EXPANSION_GRE,
807 			 MLX5_EXPANSION_NVGRE),
808 		.type = RTE_FLOW_ITEM_TYPE_IPV6,
809 		.rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
810 			RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
811 	},
812 	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
813 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
814 						  MLX5_EXPANSION_VXLAN_GPE,
815 						  MLX5_EXPANSION_MPLS,
816 						  MLX5_EXPANSION_GENEVE,
817 						  MLX5_EXPANSION_GTP),
818 		.type = RTE_FLOW_ITEM_TYPE_UDP,
819 		.rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
820 	},
821 	[MLX5_EXPANSION_OUTER_IPV6_TCP] = {
822 		.type = RTE_FLOW_ITEM_TYPE_TCP,
823 		.rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
824 	},
825 	[MLX5_EXPANSION_OUTER_IPV6_ESP] = {
826 		.type = RTE_FLOW_ITEM_TYPE_ESP,
827 		.rss_types = RTE_ETH_RSS_ESP,
828 	},
829 	[MLX5_EXPANSION_OUTER_IPV6_ICMP6] = {
830 		.type = RTE_FLOW_ITEM_TYPE_ICMP6,
831 	},
832 	[MLX5_EXPANSION_VXLAN] = {
833 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
834 						  MLX5_EXPANSION_IPV4,
835 						  MLX5_EXPANSION_IPV6),
836 		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
837 	},
838 	[MLX5_EXPANSION_STD_VXLAN] = {
839 			.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
840 					.type = RTE_FLOW_ITEM_TYPE_VXLAN,
841 	},
842 	[MLX5_EXPANSION_L3_VXLAN] = {
843 			.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
844 					MLX5_EXPANSION_IPV6),
845 					.type = RTE_FLOW_ITEM_TYPE_VXLAN,
846 	},
847 	[MLX5_EXPANSION_VXLAN_GPE] = {
848 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
849 						  MLX5_EXPANSION_IPV4,
850 						  MLX5_EXPANSION_IPV6),
851 		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
852 	},
853 	[MLX5_EXPANSION_GRE] = {
854 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
855 						  MLX5_EXPANSION_IPV4,
856 						  MLX5_EXPANSION_IPV6,
857 						  MLX5_EXPANSION_GRE_KEY,
858 						  MLX5_EXPANSION_MPLS),
859 		.type = RTE_FLOW_ITEM_TYPE_GRE,
860 	},
861 	[MLX5_EXPANSION_GRE_KEY] = {
862 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
863 						  MLX5_EXPANSION_IPV6,
864 						  MLX5_EXPANSION_MPLS),
865 		.type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
866 		.node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
867 	},
868 	[MLX5_EXPANSION_NVGRE] = {
869 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
870 		.type = RTE_FLOW_ITEM_TYPE_NVGRE,
871 	},
872 	[MLX5_EXPANSION_MPLS] = {
873 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
874 						  MLX5_EXPANSION_IPV6,
875 						  MLX5_EXPANSION_ETH),
876 		.type = RTE_FLOW_ITEM_TYPE_MPLS,
877 		.node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
878 	},
879 	[MLX5_EXPANSION_ETH] = {
880 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
881 		.type = RTE_FLOW_ITEM_TYPE_ETH,
882 	},
883 	[MLX5_EXPANSION_VLAN] = {
884 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
885 						  MLX5_EXPANSION_IPV6),
886 		.type = RTE_FLOW_ITEM_TYPE_VLAN,
887 		.node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
888 	},
889 	[MLX5_EXPANSION_IPV4] = {
890 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
891 						  MLX5_EXPANSION_IPV4_TCP,
892 						  MLX5_EXPANSION_IPV4_ESP,
893 						  MLX5_EXPANSION_IPV4_ICMP),
894 		.type = RTE_FLOW_ITEM_TYPE_IPV4,
895 		.rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
896 			RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
897 	},
898 	[MLX5_EXPANSION_IPV4_UDP] = {
899 		.type = RTE_FLOW_ITEM_TYPE_UDP,
900 		.rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
901 	},
902 	[MLX5_EXPANSION_IPV4_TCP] = {
903 		.type = RTE_FLOW_ITEM_TYPE_TCP,
904 		.rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
905 	},
906 	[MLX5_EXPANSION_IPV4_ESP] = {
907 		.type = RTE_FLOW_ITEM_TYPE_ESP,
908 		.rss_types = RTE_ETH_RSS_ESP,
909 	},
910 	[MLX5_EXPANSION_IPV4_ICMP] = {
911 		.type = RTE_FLOW_ITEM_TYPE_ICMP,
912 	},
913 	[MLX5_EXPANSION_IPV6] = {
914 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
915 						  MLX5_EXPANSION_IPV6_TCP,
916 						  MLX5_EXPANSION_IPV6_ESP,
917 						  MLX5_EXPANSION_IPV6_ICMP6,
918 						  MLX5_EXPANSION_IPV6_FRAG_EXT),
919 		.type = RTE_FLOW_ITEM_TYPE_IPV6,
920 		.rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
921 			RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
922 	},
923 	[MLX5_EXPANSION_IPV6_UDP] = {
924 		.type = RTE_FLOW_ITEM_TYPE_UDP,
925 		.rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
926 	},
927 	[MLX5_EXPANSION_IPV6_TCP] = {
928 		.type = RTE_FLOW_ITEM_TYPE_TCP,
929 		.rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
930 	},
931 	[MLX5_EXPANSION_IPV6_ESP] = {
932 		.type = RTE_FLOW_ITEM_TYPE_ESP,
933 		.rss_types = RTE_ETH_RSS_ESP,
934 	},
935 	[MLX5_EXPANSION_IPV6_FRAG_EXT] = {
936 		.type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
937 	},
938 	[MLX5_EXPANSION_IPV6_ICMP6] = {
939 		.type = RTE_FLOW_ITEM_TYPE_ICMP6,
940 	},
941 	[MLX5_EXPANSION_GTP] = {
942 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
943 						  MLX5_EXPANSION_IPV6),
944 		.type = RTE_FLOW_ITEM_TYPE_GTP,
945 	},
946 	[MLX5_EXPANSION_GENEVE] = {
947 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
948 						  MLX5_EXPANSION_IPV4,
949 						  MLX5_EXPANSION_IPV6),
950 		.type = RTE_FLOW_ITEM_TYPE_GENEVE,
951 	},
952 };
953 
954 static struct rte_flow_action_handle *
955 mlx5_action_handle_create(struct rte_eth_dev *dev,
956 			  const struct rte_flow_indir_action_conf *conf,
957 			  const struct rte_flow_action *action,
958 			  struct rte_flow_error *error);
959 static int mlx5_action_handle_destroy
960 				(struct rte_eth_dev *dev,
961 				 struct rte_flow_action_handle *handle,
962 				 struct rte_flow_error *error);
963 static int mlx5_action_handle_update
964 				(struct rte_eth_dev *dev,
965 				 struct rte_flow_action_handle *handle,
966 				 const void *update,
967 				 struct rte_flow_error *error);
968 static int mlx5_action_handle_query
969 				(struct rte_eth_dev *dev,
970 				 const struct rte_flow_action_handle *handle,
971 				 void *data,
972 				 struct rte_flow_error *error);
973 static int
974 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
975 		    struct rte_flow_tunnel *app_tunnel,
976 		    struct rte_flow_action **actions,
977 		    uint32_t *num_of_actions,
978 		    struct rte_flow_error *error);
979 static int
980 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
981 		       struct rte_flow_tunnel *app_tunnel,
982 		       struct rte_flow_item **items,
983 		       uint32_t *num_of_items,
984 		       struct rte_flow_error *error);
985 static int
986 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
987 			      struct rte_flow_item *pmd_items,
988 			      uint32_t num_items, struct rte_flow_error *err);
989 static int
990 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
991 				struct rte_flow_action *pmd_actions,
992 				uint32_t num_actions,
993 				struct rte_flow_error *err);
994 static int
995 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
996 				  struct rte_mbuf *m,
997 				  struct rte_flow_restore_info *info,
998 				  struct rte_flow_error *err);
999 static struct rte_flow_item_flex_handle *
1000 mlx5_flow_flex_item_create(struct rte_eth_dev *dev,
1001 			   const struct rte_flow_item_flex_conf *conf,
1002 			   struct rte_flow_error *error);
1003 static int
1004 mlx5_flow_flex_item_release(struct rte_eth_dev *dev,
1005 			    const struct rte_flow_item_flex_handle *handle,
1006 			    struct rte_flow_error *error);
1007 static int
1008 mlx5_flow_info_get(struct rte_eth_dev *dev,
1009 		   struct rte_flow_port_info *port_info,
1010 		   struct rte_flow_queue_info *queue_info,
1011 		   struct rte_flow_error *error);
1012 static int
1013 mlx5_flow_port_configure(struct rte_eth_dev *dev,
1014 			 const struct rte_flow_port_attr *port_attr,
1015 			 uint16_t nb_queue,
1016 			 const struct rte_flow_queue_attr *queue_attr[],
1017 			 struct rte_flow_error *err);
1018 
1019 static struct rte_flow_pattern_template *
1020 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev,
1021 		const struct rte_flow_pattern_template_attr *attr,
1022 		const struct rte_flow_item items[],
1023 		struct rte_flow_error *error);
1024 
1025 static int
1026 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev,
1027 				   struct rte_flow_pattern_template *template,
1028 				   struct rte_flow_error *error);
1029 static struct rte_flow_actions_template *
1030 mlx5_flow_actions_template_create(struct rte_eth_dev *dev,
1031 			const struct rte_flow_actions_template_attr *attr,
1032 			const struct rte_flow_action actions[],
1033 			const struct rte_flow_action masks[],
1034 			struct rte_flow_error *error);
1035 static int
1036 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev,
1037 				   struct rte_flow_actions_template *template,
1038 				   struct rte_flow_error *error);
1039 
1040 static struct rte_flow_template_table *
1041 mlx5_flow_table_create(struct rte_eth_dev *dev,
1042 		       const struct rte_flow_template_table_attr *attr,
1043 		       struct rte_flow_pattern_template *item_templates[],
1044 		       uint8_t nb_item_templates,
1045 		       struct rte_flow_actions_template *action_templates[],
1046 		       uint8_t nb_action_templates,
1047 		       struct rte_flow_error *error);
1048 static int
1049 mlx5_flow_table_destroy(struct rte_eth_dev *dev,
1050 			struct rte_flow_template_table *table,
1051 			struct rte_flow_error *error);
1052 static int
1053 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev,
1054 				 uint32_t group_id,
1055 				 const struct rte_flow_group_attr *attr,
1056 				 const struct rte_flow_action actions[],
1057 				 struct rte_flow_error *error);
1058 
1059 static int
1060 mlx5_action_handle_query_update(struct rte_eth_dev *dev,
1061 				struct rte_flow_action_handle *handle,
1062 				const void *update, void *query,
1063 				enum rte_flow_query_update_mode qu_mode,
1064 				struct rte_flow_error *error);
1065 
1066 static struct rte_flow_action_list_handle *
1067 mlx5_action_list_handle_create(struct rte_eth_dev *dev,
1068 			       const struct rte_flow_indir_action_conf *conf,
1069 			       const struct rte_flow_action *actions,
1070 			       struct rte_flow_error *error);
1071 
1072 static int
1073 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev,
1074 				struct rte_flow_action_list_handle *handle,
1075 				struct rte_flow_error *error);
1076 
1077 static int
1078 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev,
1079 					  const
1080 					  struct rte_flow_action_list_handle *handle,
1081 					  const void **update, void **query,
1082 					  enum rte_flow_query_update_mode mode,
1083 					  struct rte_flow_error *error);
1084 
1085 static int
1086 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev,
1087 			  const struct rte_flow_template_table *table,
1088 			  const struct rte_flow_item pattern[],
1089 			  uint8_t pattern_template_index,
1090 			  uint32_t *hash, struct rte_flow_error *error);
1091 static int
1092 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev,
1093 			  const struct rte_flow_item pattern[],
1094 			  enum rte_flow_encap_hash_field dest_field,
1095 			  uint8_t *hash,
1096 			  struct rte_flow_error *error);
1097 
1098 static int
1099 mlx5_template_table_resize(struct rte_eth_dev *dev,
1100 			   struct rte_flow_template_table *table,
1101 			   uint32_t nb_rules, struct rte_flow_error *error);
1102 static int
1103 mlx5_flow_async_update_resized(struct rte_eth_dev *dev, uint32_t queue,
1104 			       const struct rte_flow_op_attr *attr,
1105 			       struct rte_flow *rule, void *user_data,
1106 			       struct rte_flow_error *error);
1107 static int
1108 mlx5_table_resize_complete(struct rte_eth_dev *dev,
1109 			   struct rte_flow_template_table *table,
1110 			   struct rte_flow_error *error);
1111 
1112 static const struct rte_flow_ops mlx5_flow_ops = {
1113 	.validate = mlx5_flow_validate,
1114 	.create = mlx5_flow_create,
1115 	.destroy = mlx5_flow_destroy,
1116 	.flush = mlx5_flow_flush,
1117 	.isolate = mlx5_flow_isolate,
1118 	.query = mlx5_flow_query,
1119 	.dev_dump = mlx5_flow_dev_dump,
1120 	.get_q_aged_flows = mlx5_flow_get_q_aged_flows,
1121 	.get_aged_flows = mlx5_flow_get_aged_flows,
1122 	.action_handle_create = mlx5_action_handle_create,
1123 	.action_handle_destroy = mlx5_action_handle_destroy,
1124 	.action_handle_update = mlx5_action_handle_update,
1125 	.action_handle_query = mlx5_action_handle_query,
1126 	.action_handle_query_update = mlx5_action_handle_query_update,
1127 	.action_list_handle_create = mlx5_action_list_handle_create,
1128 	.action_list_handle_destroy = mlx5_action_list_handle_destroy,
1129 	.tunnel_decap_set = mlx5_flow_tunnel_decap_set,
1130 	.tunnel_match = mlx5_flow_tunnel_match,
1131 	.tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
1132 	.tunnel_item_release = mlx5_flow_tunnel_item_release,
1133 	.get_restore_info = mlx5_flow_tunnel_get_restore_info,
1134 	.flex_item_create = mlx5_flow_flex_item_create,
1135 	.flex_item_release = mlx5_flow_flex_item_release,
1136 	.info_get = mlx5_flow_info_get,
1137 	.pick_transfer_proxy = mlx5_flow_pick_transfer_proxy,
1138 	.configure = mlx5_flow_port_configure,
1139 	.pattern_template_create = mlx5_flow_pattern_template_create,
1140 	.pattern_template_destroy = mlx5_flow_pattern_template_destroy,
1141 	.actions_template_create = mlx5_flow_actions_template_create,
1142 	.actions_template_destroy = mlx5_flow_actions_template_destroy,
1143 	.template_table_create = mlx5_flow_table_create,
1144 	.template_table_destroy = mlx5_flow_table_destroy,
1145 	.group_set_miss_actions = mlx5_flow_group_set_miss_actions,
1146 	.action_list_handle_query_update =
1147 		mlx5_flow_action_list_handle_query_update,
1148 	.flow_calc_table_hash = mlx5_flow_calc_table_hash,
1149 	.flow_calc_encap_hash = mlx5_flow_calc_encap_hash,
1150 	.flow_template_table_resize = mlx5_template_table_resize,
1151 	.flow_update_resized = mlx5_flow_async_update_resized,
1152 	.flow_template_table_resize_complete = mlx5_table_resize_complete,
1153 };
1154 
1155 /* Tunnel information. */
1156 struct mlx5_flow_tunnel_info {
1157 	uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
1158 	uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
1159 };
1160 
1161 static struct mlx5_flow_tunnel_info tunnels_info[] = {
1162 	{
1163 		.tunnel = MLX5_FLOW_LAYER_VXLAN,
1164 		.ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
1165 	},
1166 	{
1167 		.tunnel = MLX5_FLOW_LAYER_GENEVE,
1168 		.ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
1169 	},
1170 	{
1171 		.tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
1172 		.ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
1173 	},
1174 	{
1175 		.tunnel = MLX5_FLOW_LAYER_GRE,
1176 		.ptype = RTE_PTYPE_TUNNEL_GRE,
1177 	},
1178 	{
1179 		.tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
1180 		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
1181 	},
1182 	{
1183 		.tunnel = MLX5_FLOW_LAYER_MPLS,
1184 		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
1185 	},
1186 	{
1187 		.tunnel = MLX5_FLOW_LAYER_NVGRE,
1188 		.ptype = RTE_PTYPE_TUNNEL_NVGRE,
1189 	},
1190 	{
1191 		.tunnel = MLX5_FLOW_LAYER_IPIP,
1192 		.ptype = RTE_PTYPE_TUNNEL_IP,
1193 	},
1194 	{
1195 		.tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
1196 		.ptype = RTE_PTYPE_TUNNEL_IP,
1197 	},
1198 	{
1199 		.tunnel = MLX5_FLOW_LAYER_GTP,
1200 		.ptype = RTE_PTYPE_TUNNEL_GTPU,
1201 	},
1202 };
1203 
1204 
1205 
1206 /**
1207  * Translate tag ID to register.
1208  *
1209  * @param[in] dev
1210  *   Pointer to the Ethernet device structure.
1211  * @param[in] feature
1212  *   The feature that request the register.
1213  * @param[in] id
1214  *   The request register ID.
1215  * @param[out] error
1216  *   Error description in case of any.
1217  *
1218  * @return
1219  *   The request register on success, a negative errno
1220  *   value otherwise and rte_errno is set.
1221  */
1222 int
1223 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
1224 		     enum mlx5_feature_name feature,
1225 		     uint32_t id,
1226 		     struct rte_flow_error *error)
1227 {
1228 	struct mlx5_priv *priv = dev->data->dev_private;
1229 	struct mlx5_sh_config *config = &priv->sh->config;
1230 	struct mlx5_dev_registers *reg = &priv->sh->registers;
1231 	enum modify_reg start_reg;
1232 	bool skip_mtr_reg = false;
1233 
1234 	switch (feature) {
1235 	case MLX5_HAIRPIN_RX:
1236 		return REG_B;
1237 	case MLX5_HAIRPIN_TX:
1238 		return REG_A;
1239 	case MLX5_METADATA_RX:
1240 		switch (config->dv_xmeta_en) {
1241 		case MLX5_XMETA_MODE_LEGACY:
1242 			return REG_B;
1243 		case MLX5_XMETA_MODE_META16:
1244 			return REG_C_0;
1245 		case MLX5_XMETA_MODE_META32:
1246 			return REG_C_1;
1247 		case MLX5_XMETA_MODE_META32_HWS:
1248 			return REG_C_1;
1249 		}
1250 		break;
1251 	case MLX5_METADATA_TX:
1252 		if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) {
1253 			return REG_C_1;
1254 		} else {
1255 			return REG_A;
1256 		}
1257 	case MLX5_METADATA_FDB:
1258 		switch (config->dv_xmeta_en) {
1259 		case MLX5_XMETA_MODE_LEGACY:
1260 			return REG_NON;
1261 		case MLX5_XMETA_MODE_META16:
1262 			return REG_C_0;
1263 		case MLX5_XMETA_MODE_META32:
1264 			return REG_C_1;
1265 		case MLX5_XMETA_MODE_META32_HWS:
1266 			return REG_C_1;
1267 		}
1268 		break;
1269 	case MLX5_FLOW_MARK:
1270 		switch (config->dv_xmeta_en) {
1271 		case MLX5_XMETA_MODE_LEGACY:
1272 		case MLX5_XMETA_MODE_META32_HWS:
1273 			return REG_NON;
1274 		case MLX5_XMETA_MODE_META16:
1275 			return REG_C_1;
1276 		case MLX5_XMETA_MODE_META32:
1277 			return REG_C_0;
1278 		}
1279 		break;
1280 	case MLX5_MTR_ID:
1281 		/*
1282 		 * If meter color and meter id share one register, flow match
1283 		 * should use the meter color register for match.
1284 		 */
1285 		if (priv->mtr_reg_share)
1286 			return reg->aso_reg;
1287 		else
1288 			return reg->aso_reg != REG_C_2 ? REG_C_2 :
1289 			       REG_C_3;
1290 	case MLX5_MTR_COLOR:
1291 	case MLX5_ASO_FLOW_HIT:
1292 	case MLX5_ASO_CONNTRACK:
1293 	case MLX5_SAMPLE_ID:
1294 		/* All features use the same REG_C. */
1295 		MLX5_ASSERT(reg->aso_reg != REG_NON);
1296 		return reg->aso_reg;
1297 	case MLX5_COPY_MARK:
1298 		/*
1299 		 * Metadata COPY_MARK register using is in meter suffix sub
1300 		 * flow while with meter. It's safe to share the same register.
1301 		 */
1302 		return reg->aso_reg != REG_C_2 ? REG_C_2 : REG_C_3;
1303 	case MLX5_APP_TAG:
1304 		/*
1305 		 * If meter is enable, it will engage the register for color
1306 		 * match and flow match. If meter color match is not using the
1307 		 * REG_C_2, need to skip the REG_C_x be used by meter color
1308 		 * match.
1309 		 * If meter is disable, free to use all available registers.
1310 		 */
1311 		start_reg = reg->aso_reg != REG_C_2 ? REG_C_2 :
1312 			    (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
1313 		skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
1314 		if (id > (uint32_t)(REG_C_7 - start_reg))
1315 			return rte_flow_error_set(error, EINVAL,
1316 						  RTE_FLOW_ERROR_TYPE_ITEM,
1317 						  NULL, "invalid tag id");
1318 		if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
1319 			return rte_flow_error_set(error, ENOTSUP,
1320 						  RTE_FLOW_ERROR_TYPE_ITEM,
1321 						  NULL, "unsupported tag id");
1322 		/*
1323 		 * This case means meter is using the REG_C_x great than 2.
1324 		 * Take care not to conflict with meter color REG_C_x.
1325 		 * If the available index REG_C_y >= REG_C_x, skip the
1326 		 * color register.
1327 		 */
1328 		if (skip_mtr_reg && priv->sh->flow_mreg_c
1329 		    [id + start_reg - REG_C_0] >= reg->aso_reg) {
1330 			if (id >= (uint32_t)(REG_C_7 - start_reg))
1331 				return rte_flow_error_set(error, EINVAL,
1332 						       RTE_FLOW_ERROR_TYPE_ITEM,
1333 							NULL, "invalid tag id");
1334 			if (priv->sh->flow_mreg_c
1335 			    [id + 1 + start_reg - REG_C_0] != REG_NON)
1336 				return priv->sh->flow_mreg_c
1337 					       [id + 1 + start_reg - REG_C_0];
1338 			return rte_flow_error_set(error, ENOTSUP,
1339 						  RTE_FLOW_ERROR_TYPE_ITEM,
1340 						  NULL, "unsupported tag id");
1341 		}
1342 		return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
1343 	}
1344 	MLX5_ASSERT(false);
1345 	return rte_flow_error_set(error, EINVAL,
1346 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1347 				  NULL, "invalid feature name");
1348 }
1349 
1350 /**
1351  * Check extensive flow metadata register support.
1352  *
1353  * @param dev
1354  *   Pointer to rte_eth_dev structure.
1355  *
1356  * @return
1357  *   True if device supports extensive flow metadata register, otherwise false.
1358  */
1359 bool
1360 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
1361 {
1362 	struct mlx5_priv *priv = dev->data->dev_private;
1363 
1364 	/*
1365 	 * Having available reg_c can be regarded inclusively as supporting
1366 	 * extensive flow metadata register, which could mean,
1367 	 * - metadata register copy action by modify header.
1368 	 * - 16 modify header actions is supported.
1369 	 * - reg_c's are preserved across different domain (FDB and NIC) on
1370 	 *   packet loopback by flow lookup miss.
1371 	 */
1372 	return priv->sh->flow_mreg_c[2] != REG_NON;
1373 }
1374 
1375 /**
1376  * Get the lowest priority.
1377  *
1378  * @param[in] dev
1379  *   Pointer to the Ethernet device structure.
1380  * @param[in] attributes
1381  *   Pointer to device flow rule attributes.
1382  *
1383  * @return
1384  *   The value of lowest priority of flow.
1385  */
1386 uint32_t
1387 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
1388 			  const struct rte_flow_attr *attr)
1389 {
1390 	struct mlx5_priv *priv = dev->data->dev_private;
1391 
1392 	if (!attr->group && !(attr->transfer && priv->fdb_def_rule))
1393 		return priv->sh->flow_max_priority - 2;
1394 	return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
1395 }
1396 
1397 /**
1398  * Calculate matcher priority of the flow.
1399  *
1400  * @param[in] dev
1401  *   Pointer to the Ethernet device structure.
1402  * @param[in] attr
1403  *   Pointer to device flow rule attributes.
1404  * @param[in] subpriority
1405  *   The priority based on the items.
1406  * @param[in] external
1407  *   Flow is user flow.
1408  * @return
1409  *   The matcher priority of the flow.
1410  */
1411 uint16_t
1412 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
1413 			  const struct rte_flow_attr *attr,
1414 			  uint32_t subpriority, bool external)
1415 {
1416 	uint16_t priority = (uint16_t)attr->priority;
1417 	struct mlx5_priv *priv = dev->data->dev_private;
1418 
1419 	/* NIC root rules */
1420 	if (!attr->group && !attr->transfer) {
1421 		if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1422 			priority = priv->sh->flow_max_priority - 1;
1423 		return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
1424 	/* FDB root rules */
1425 	} else if (attr->transfer && (!external || !priv->fdb_def_rule) &&
1426 		   attr->group == 0 &&
1427 		   attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) {
1428 		return (priv->sh->flow_max_priority - 1) * 3;
1429 	}
1430 	if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1431 		priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
1432 	return priority * 3 + subpriority;
1433 }
1434 
1435 /**
1436  * Verify the @p item specifications (spec, last, mask) are compatible with the
1437  * NIC capabilities.
1438  *
1439  * @param[in] item
1440  *   Item specification.
1441  * @param[in] mask
1442  *   @p item->mask or flow default bit-masks.
1443  * @param[in] nic_mask
1444  *   Bit-masks covering supported fields by the NIC to compare with user mask.
1445  * @param[in] size
1446  *   Bit-masks size in bytes.
1447  * @param[in] range_accepted
1448  *   True if range of values is accepted for specific fields, false otherwise.
1449  * @param[out] error
1450  *   Pointer to error structure.
1451  *
1452  * @return
1453  *   0 on success, a negative errno value otherwise and rte_errno is set.
1454  */
1455 int
1456 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1457 			  const uint8_t *mask,
1458 			  const uint8_t *nic_mask,
1459 			  unsigned int size,
1460 			  bool range_accepted,
1461 			  struct rte_flow_error *error)
1462 {
1463 	unsigned int i;
1464 
1465 	MLX5_ASSERT(nic_mask);
1466 	for (i = 0; i < size; ++i)
1467 		if ((nic_mask[i] | mask[i]) != nic_mask[i])
1468 			return rte_flow_error_set(error, ENOTSUP,
1469 						  RTE_FLOW_ERROR_TYPE_ITEM,
1470 						  item,
1471 						  "mask enables non supported"
1472 						  " bits");
1473 	if (!item->spec && (item->mask || item->last))
1474 		return rte_flow_error_set(error, EINVAL,
1475 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
1476 					  "mask/last without a spec is not"
1477 					  " supported");
1478 	if (item->spec && item->last && !range_accepted) {
1479 		uint8_t spec[size];
1480 		uint8_t last[size];
1481 		unsigned int i;
1482 		int ret;
1483 
1484 		for (i = 0; i < size; ++i) {
1485 			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1486 			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1487 		}
1488 		ret = memcmp(spec, last, size);
1489 		if (ret != 0)
1490 			return rte_flow_error_set(error, EINVAL,
1491 						  RTE_FLOW_ERROR_TYPE_ITEM,
1492 						  item,
1493 						  "range is not valid");
1494 	}
1495 	return 0;
1496 }
1497 
1498 /**
1499  * Adjust the hash fields according to the @p flow information.
1500  *
1501  * @param[in] dev_flow.
1502  *   Pointer to the mlx5_flow.
1503  * @param[in] tunnel
1504  *   1 when the hash field is for a tunnel item.
1505  * @param[in] layer_types
1506  *   RTE_ETH_RSS_* types.
1507  * @param[in] hash_fields
1508  *   Item hash fields.
1509  *
1510  * @return
1511  *   The hash fields that should be used.
1512  */
1513 uint64_t
1514 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1515 			    int tunnel __rte_unused, uint64_t layer_types,
1516 			    uint64_t hash_fields)
1517 {
1518 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1519 	int rss_request_inner = rss_desc->level >= 2;
1520 
1521 	/* Check RSS hash level for tunnel. */
1522 	if (tunnel && rss_request_inner)
1523 		hash_fields |= IBV_RX_HASH_INNER;
1524 	else if (tunnel || rss_request_inner)
1525 		return 0;
1526 #endif
1527 	/* Check if requested layer matches RSS hash fields. */
1528 	if (!(rss_desc->types & layer_types))
1529 		return 0;
1530 	return hash_fields;
1531 }
1532 
1533 /**
1534  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1535  * if several tunnel rules are used on this queue, the tunnel ptype will be
1536  * cleared.
1537  *
1538  * @param rxq_ctrl
1539  *   Rx queue to update.
1540  */
1541 static void
1542 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1543 {
1544 	unsigned int i;
1545 	uint32_t tunnel_ptype = 0;
1546 
1547 	/* Look up for the ptype to use. */
1548 	for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1549 		if (!rxq_ctrl->flow_tunnels_n[i])
1550 			continue;
1551 		if (!tunnel_ptype) {
1552 			tunnel_ptype = tunnels_info[i].ptype;
1553 		} else {
1554 			tunnel_ptype = 0;
1555 			break;
1556 		}
1557 	}
1558 	rxq_ctrl->rxq.tunnel = tunnel_ptype;
1559 }
1560 
1561 /**
1562  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device
1563  * flow.
1564  *
1565  * @param[in] dev
1566  *   Pointer to the Ethernet device structure.
1567  * @param[in] dev_handle
1568  *   Pointer to device flow handle structure.
1569  */
1570 void
1571 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1572 		       struct mlx5_flow_handle *dev_handle)
1573 {
1574 	struct mlx5_priv *priv = dev->data->dev_private;
1575 	const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1576 	struct mlx5_ind_table_obj *ind_tbl = NULL;
1577 	unsigned int i;
1578 
1579 	if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1580 		struct mlx5_hrxq *hrxq;
1581 
1582 		hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1583 			      dev_handle->rix_hrxq);
1584 		if (hrxq)
1585 			ind_tbl = hrxq->ind_table;
1586 	} else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1587 		struct mlx5_shared_action_rss *shared_rss;
1588 
1589 		shared_rss = mlx5_ipool_get
1590 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1591 			 dev_handle->rix_srss);
1592 		if (shared_rss)
1593 			ind_tbl = shared_rss->ind_tbl;
1594 	}
1595 	if (!ind_tbl)
1596 		return;
1597 	for (i = 0; i != ind_tbl->queues_n; ++i) {
1598 		int idx = ind_tbl->queues[i];
1599 		struct mlx5_rxq_ctrl *rxq_ctrl;
1600 
1601 		if (mlx5_is_external_rxq(dev, idx))
1602 			continue;
1603 		rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx);
1604 		MLX5_ASSERT(rxq_ctrl != NULL);
1605 		if (rxq_ctrl == NULL)
1606 			continue;
1607 		/*
1608 		 * To support metadata register copy on Tx loopback,
1609 		 * this must be always enabled (metadata may arive
1610 		 * from other port - not from local flows only.
1611 		 */
1612 		if (tunnel) {
1613 			unsigned int j;
1614 
1615 			/* Increase the counter matching the flow. */
1616 			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1617 				if ((tunnels_info[j].tunnel &
1618 				     dev_handle->layers) ==
1619 				    tunnels_info[j].tunnel) {
1620 					rxq_ctrl->flow_tunnels_n[j]++;
1621 					break;
1622 				}
1623 			}
1624 			flow_rxq_tunnel_ptype_update(rxq_ctrl);
1625 		}
1626 	}
1627 }
1628 
1629 static void
1630 flow_rxq_mark_flag_set(struct rte_eth_dev *dev)
1631 {
1632 	struct mlx5_priv *priv = dev->data->dev_private;
1633 	struct mlx5_rxq_ctrl *rxq_ctrl;
1634 	uint16_t port_id;
1635 
1636 	if (priv->sh->shared_mark_enabled)
1637 		return;
1638 	if (priv->master || priv->representor) {
1639 		MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
1640 			struct mlx5_priv *opriv =
1641 				rte_eth_devices[port_id].data->dev_private;
1642 
1643 			if (!opriv ||
1644 			    opriv->sh != priv->sh ||
1645 			    opriv->domain_id != priv->domain_id ||
1646 			    opriv->mark_enabled)
1647 				continue;
1648 			LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) {
1649 				rxq_ctrl->rxq.mark = 1;
1650 			}
1651 			opriv->mark_enabled = 1;
1652 		}
1653 	} else {
1654 		LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) {
1655 			rxq_ctrl->rxq.mark = 1;
1656 		}
1657 		priv->mark_enabled = 1;
1658 	}
1659 	priv->sh->shared_mark_enabled = 1;
1660 }
1661 
1662 /**
1663  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1664  *
1665  * @param[in] dev
1666  *   Pointer to the Ethernet device structure.
1667  * @param[in] flow
1668  *   Pointer to flow structure.
1669  */
1670 static void
1671 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1672 {
1673 	struct mlx5_priv *priv = dev->data->dev_private;
1674 	uint32_t handle_idx;
1675 	struct mlx5_flow_handle *dev_handle;
1676 	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1677 
1678 	MLX5_ASSERT(wks);
1679 	if (wks->mark)
1680 		flow_rxq_mark_flag_set(dev);
1681 	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1682 		       handle_idx, dev_handle, next)
1683 		flow_drv_rxq_flags_set(dev, dev_handle);
1684 }
1685 
1686 /**
1687  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1688  * device flow if no other flow uses it with the same kind of request.
1689  *
1690  * @param dev
1691  *   Pointer to Ethernet device.
1692  * @param[in] dev_handle
1693  *   Pointer to the device flow handle structure.
1694  */
1695 static void
1696 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1697 			struct mlx5_flow_handle *dev_handle)
1698 {
1699 	struct mlx5_priv *priv = dev->data->dev_private;
1700 	const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1701 	struct mlx5_ind_table_obj *ind_tbl = NULL;
1702 	unsigned int i;
1703 
1704 	if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1705 		struct mlx5_hrxq *hrxq;
1706 
1707 		hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1708 			      dev_handle->rix_hrxq);
1709 		if (hrxq)
1710 			ind_tbl = hrxq->ind_table;
1711 	} else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1712 		struct mlx5_shared_action_rss *shared_rss;
1713 
1714 		shared_rss = mlx5_ipool_get
1715 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1716 			 dev_handle->rix_srss);
1717 		if (shared_rss)
1718 			ind_tbl = shared_rss->ind_tbl;
1719 	}
1720 	if (!ind_tbl)
1721 		return;
1722 	MLX5_ASSERT(dev->data->dev_started);
1723 	for (i = 0; i != ind_tbl->queues_n; ++i) {
1724 		int idx = ind_tbl->queues[i];
1725 		struct mlx5_rxq_ctrl *rxq_ctrl;
1726 
1727 		if (mlx5_is_external_rxq(dev, idx))
1728 			continue;
1729 		rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx);
1730 		MLX5_ASSERT(rxq_ctrl != NULL);
1731 		if (rxq_ctrl == NULL)
1732 			continue;
1733 		if (tunnel) {
1734 			unsigned int j;
1735 
1736 			/* Decrease the counter matching the flow. */
1737 			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1738 				if ((tunnels_info[j].tunnel &
1739 				     dev_handle->layers) ==
1740 				    tunnels_info[j].tunnel) {
1741 					rxq_ctrl->flow_tunnels_n[j]--;
1742 					break;
1743 				}
1744 			}
1745 			flow_rxq_tunnel_ptype_update(rxq_ctrl);
1746 		}
1747 	}
1748 }
1749 
1750 /**
1751  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1752  * @p flow if no other flow uses it with the same kind of request.
1753  *
1754  * @param dev
1755  *   Pointer to Ethernet device.
1756  * @param[in] flow
1757  *   Pointer to the flow.
1758  */
1759 static void
1760 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1761 {
1762 	struct mlx5_priv *priv = dev->data->dev_private;
1763 	uint32_t handle_idx;
1764 	struct mlx5_flow_handle *dev_handle;
1765 
1766 	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1767 		       handle_idx, dev_handle, next)
1768 		flow_drv_rxq_flags_trim(dev, dev_handle);
1769 }
1770 
1771 /**
1772  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1773  *
1774  * @param dev
1775  *   Pointer to Ethernet device.
1776  */
1777 static void
1778 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1779 {
1780 	struct mlx5_priv *priv = dev->data->dev_private;
1781 	unsigned int i;
1782 
1783 	for (i = 0; i != priv->rxqs_n; ++i) {
1784 		struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i);
1785 		unsigned int j;
1786 
1787 		if (rxq == NULL || rxq->ctrl == NULL)
1788 			continue;
1789 		rxq->ctrl->rxq.mark = 0;
1790 		for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1791 			rxq->ctrl->flow_tunnels_n[j] = 0;
1792 		rxq->ctrl->rxq.tunnel = 0;
1793 	}
1794 	priv->mark_enabled = 0;
1795 	priv->sh->shared_mark_enabled = 0;
1796 }
1797 
1798 static uint64_t mlx5_restore_info_dynflag;
1799 
1800 int
1801 mlx5_flow_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features)
1802 {
1803 	struct mlx5_priv *priv = dev->data->dev_private;
1804 	uint64_t supported = 0;
1805 
1806 	if (!is_tunnel_offload_active(dev)) {
1807 		supported |= RTE_ETH_RX_METADATA_USER_FLAG;
1808 		supported |= RTE_ETH_RX_METADATA_USER_MARK;
1809 		if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) {
1810 			DRV_LOG(DEBUG,
1811 				"tunnel offload was not activated, consider setting dv_xmeta_en=%d",
1812 				MLX5_XMETA_MODE_MISS_INFO);
1813 		}
1814 	} else {
1815 		supported |= RTE_ETH_RX_METADATA_TUNNEL_ID;
1816 		if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0 &&
1817 				mlx5_restore_info_dynflag == 0)
1818 			mlx5_restore_info_dynflag = rte_flow_restore_info_dynflag();
1819 	}
1820 
1821 	if (((*features & supported) & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0)
1822 		priv->tunnel_enabled = 1;
1823 	else
1824 		priv->tunnel_enabled = 0;
1825 
1826 	*features &= supported;
1827 	return 0;
1828 }
1829 
1830 /**
1831  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1832  *
1833  * @param[in] dev
1834  *   Pointer to the Ethernet device structure.
1835  */
1836 void
1837 mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev)
1838 {
1839 	struct mlx5_priv *priv = dev->data->dev_private;
1840 	uint64_t mark_flag = RTE_MBUF_F_RX_FDIR_ID;
1841 	unsigned int i;
1842 
1843 	if (priv->tunnel_enabled)
1844 		mark_flag |= mlx5_restore_info_dynflag;
1845 
1846 	for (i = 0; i != priv->rxqs_n; ++i) {
1847 		struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i);
1848 		struct mlx5_rxq_data *data;
1849 
1850 		if (rxq == NULL || rxq->ctrl == NULL)
1851 			continue;
1852 		data = &rxq->ctrl->rxq;
1853 		if (!rte_flow_dynf_metadata_avail()) {
1854 			data->dynf_meta = 0;
1855 			data->flow_meta_mask = 0;
1856 			data->flow_meta_offset = -1;
1857 			data->flow_meta_port_mask = 0;
1858 		} else {
1859 			data->dynf_meta = 1;
1860 			data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1861 			data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1862 			data->flow_meta_port_mask = priv->sh->dv_meta_mask;
1863 		}
1864 		data->mark_flag = mark_flag;
1865 	}
1866 }
1867 
1868 /*
1869  * return a pointer to the desired action in the list of actions.
1870  *
1871  * @param[in] actions
1872  *   The list of actions to search the action in.
1873  * @param[in] action
1874  *   The action to find.
1875  *
1876  * @return
1877  *   Pointer to the action in the list, if found. NULL otherwise.
1878  */
1879 const struct rte_flow_action *
1880 mlx5_flow_find_action(const struct rte_flow_action *actions,
1881 		      enum rte_flow_action_type action)
1882 {
1883 	if (actions == NULL)
1884 		return NULL;
1885 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1886 		if (actions->type == action)
1887 			return actions;
1888 	return NULL;
1889 }
1890 
1891 /*
1892  * Validate the flag action.
1893  *
1894  * @param[in] action_flags
1895  *   Bit-fields that holds the actions detected until now.
1896  * @param[in] attr
1897  *   Attributes of flow that includes this action.
1898  * @param[out] error
1899  *   Pointer to error structure.
1900  *
1901  * @return
1902  *   0 on success, a negative errno value otherwise and rte_errno is set.
1903  */
1904 int
1905 mlx5_flow_validate_action_flag(uint64_t action_flags,
1906 			       const struct rte_flow_attr *attr,
1907 			       struct rte_flow_error *error)
1908 {
1909 	if (action_flags & MLX5_FLOW_ACTION_MARK)
1910 		return rte_flow_error_set(error, EINVAL,
1911 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1912 					  "can't mark and flag in same flow");
1913 	if (action_flags & MLX5_FLOW_ACTION_FLAG)
1914 		return rte_flow_error_set(error, EINVAL,
1915 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1916 					  "can't have 2 flag"
1917 					  " actions in same flow");
1918 	if (attr->egress)
1919 		return rte_flow_error_set(error, ENOTSUP,
1920 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1921 					  "flag action not supported for "
1922 					  "egress");
1923 	return 0;
1924 }
1925 
1926 /*
1927  * Validate the mark action.
1928  *
1929  * @param[in] action
1930  *   Pointer to the queue action.
1931  * @param[in] action_flags
1932  *   Bit-fields that holds the actions detected until now.
1933  * @param[in] attr
1934  *   Attributes of flow that includes this action.
1935  * @param[out] error
1936  *   Pointer to error structure.
1937  *
1938  * @return
1939  *   0 on success, a negative errno value otherwise and rte_errno is set.
1940  */
1941 int
1942 mlx5_flow_validate_action_mark(struct rte_eth_dev *dev,
1943 			       const struct rte_flow_action *action,
1944 			       uint64_t action_flags,
1945 			       const struct rte_flow_attr *attr,
1946 			       struct rte_flow_error *error)
1947 {
1948 	const struct rte_flow_action_mark *mark = action->conf;
1949 
1950 	if (!mark)
1951 		return rte_flow_error_set(error, EINVAL,
1952 					  RTE_FLOW_ERROR_TYPE_ACTION,
1953 					  action,
1954 					  "configuration cannot be null");
1955 	if (mark->id >= MLX5_FLOW_MARK_MAX)
1956 		return rte_flow_error_set(error, EINVAL,
1957 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1958 					  &mark->id,
1959 					  "mark id must in 0 <= id < "
1960 					  RTE_STR(MLX5_FLOW_MARK_MAX));
1961 	if (action_flags & MLX5_FLOW_ACTION_FLAG)
1962 		return rte_flow_error_set(error, EINVAL,
1963 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1964 					  "can't flag and mark in same flow");
1965 	if (action_flags & MLX5_FLOW_ACTION_MARK)
1966 		return rte_flow_error_set(error, EINVAL,
1967 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1968 					  "can't have 2 mark actions in same"
1969 					  " flow");
1970 	if (attr->egress)
1971 		return rte_flow_error_set(error, ENOTSUP,
1972 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1973 					  "mark action not supported for "
1974 					  "egress");
1975 	if (attr->transfer && mlx5_hws_active(dev))
1976 		return rte_flow_error_set(error, ENOTSUP,
1977 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1978 					  "non-template mark action not supported for transfer");
1979 	return 0;
1980 }
1981 
1982 /*
1983  * Validate the drop action.
1984  *
1985  * @param[in] dev
1986  *   Pointer to the Ethernet device structure.
1987  * @param[in] is_root
1988  *   True if flow is validated for root table. False otherwise.
1989  * @param[in] attr
1990  *   Attributes of flow that includes this action.
1991  * @param[out] error
1992  *   Pointer to error structure.
1993  *
1994  * @return
1995  *   0 on success, a negative errno value otherwise and rte_errno is set.
1996  */
1997 int
1998 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev,
1999 			       bool is_root,
2000 			       const struct rte_flow_attr *attr,
2001 			       struct rte_flow_error *error)
2002 {
2003 	struct mlx5_priv *priv = dev->data->dev_private;
2004 
2005 	if (priv->sh->config.dv_flow_en == 0 && attr->egress)
2006 		return rte_flow_error_set(error, ENOTSUP,
2007 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2008 					  "drop action not supported for "
2009 					  "egress");
2010 	if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) &&
2011 	    !priv->sh->dr_root_drop_action_en) {
2012 		return rte_flow_error_set(error, ENOTSUP,
2013 					  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
2014 					  "drop action not supported for "
2015 					  "egress and transfer on group 0");
2016 	}
2017 	return 0;
2018 }
2019 
2020 /*
2021  * Validate the queue action.
2022  *
2023  * @param[in] action
2024  *   Pointer to the queue action.
2025  * @param[in] action_flags
2026  *   Bit-fields that holds the actions detected until now.
2027  * @param[in] dev
2028  *   Pointer to the Ethernet device structure.
2029  * @param[in] attr
2030  *   Attributes of flow that includes this action.
2031  * @param[out] error
2032  *   Pointer to error structure.
2033  *
2034  * @return
2035  *   0 on success, a negative errno value otherwise and rte_errno is set.
2036  */
2037 int
2038 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
2039 				uint64_t action_flags,
2040 				struct rte_eth_dev *dev,
2041 				const struct rte_flow_attr *attr,
2042 				struct rte_flow_error *error)
2043 {
2044 	struct mlx5_priv *priv = dev->data->dev_private;
2045 	const struct rte_flow_action_queue *queue = action->conf;
2046 
2047 	if (!queue)
2048 		return rte_flow_error_set(error, EINVAL,
2049 					  RTE_FLOW_ERROR_TYPE_ACTION, action,
2050 					  "no QUEUE action configuration");
2051 	if (action_flags & MLX5_FLOW_FATE_ACTIONS)
2052 		return rte_flow_error_set(error, EINVAL,
2053 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2054 					  "can't have 2 fate actions in"
2055 					  " same flow");
2056 	if (attr->egress)
2057 		return rte_flow_error_set(error, ENOTSUP,
2058 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2059 					  "queue action not supported for egress.");
2060 	if (mlx5_is_external_rxq(dev, queue->index))
2061 		return 0;
2062 	if (!priv->rxqs_n)
2063 		return rte_flow_error_set(error, EINVAL,
2064 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2065 					  NULL, "No Rx queues configured");
2066 	if (queue->index >= priv->rxqs_n)
2067 		return rte_flow_error_set(error, EINVAL,
2068 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2069 					  &queue->index,
2070 					  "queue index out of range");
2071 	if (mlx5_rxq_get(dev, queue->index) == NULL)
2072 		return rte_flow_error_set(error, EINVAL,
2073 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2074 					  &queue->index,
2075 					  "queue is not configured");
2076 	return 0;
2077 }
2078 
2079 /**
2080  * Validate queue numbers for device RSS.
2081  *
2082  * @param[in] dev
2083  *   Configured device.
2084  * @param[in] queues
2085  *   Array of queue numbers.
2086  * @param[in] queues_n
2087  *   Size of the @p queues array.
2088  * @param[out] error
2089  *   On error, filled with a textual error description.
2090  * @param[out] queue_idx
2091  *   On error, filled with an offending queue index in @p queues array.
2092  *
2093  * @return
2094  *   0 on success, a negative errno code on error.
2095  */
2096 static int
2097 mlx5_validate_rss_queues(struct rte_eth_dev *dev,
2098 			 const uint16_t *queues, uint32_t queues_n,
2099 			 const char **error, uint32_t *queue_idx)
2100 {
2101 	const struct mlx5_priv *priv = dev->data->dev_private;
2102 	bool is_hairpin = false;
2103 	bool is_ext_rss = false;
2104 	uint32_t i;
2105 
2106 	for (i = 0; i != queues_n; ++i) {
2107 		struct mlx5_rxq_ctrl *rxq_ctrl;
2108 
2109 		if (mlx5_is_external_rxq(dev, queues[0])) {
2110 			is_ext_rss = true;
2111 			continue;
2112 		}
2113 		if (is_ext_rss) {
2114 			*error = "Combining external and regular RSS queues is not supported";
2115 			*queue_idx = i;
2116 			return -ENOTSUP;
2117 		}
2118 		if (queues[i] >= priv->rxqs_n) {
2119 			*error = "queue index out of range";
2120 			*queue_idx = i;
2121 			return -EINVAL;
2122 		}
2123 		rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]);
2124 		if (rxq_ctrl == NULL) {
2125 			*error =  "queue is not configured";
2126 			*queue_idx = i;
2127 			return -EINVAL;
2128 		}
2129 		if (i == 0 && rxq_ctrl->is_hairpin)
2130 			is_hairpin = true;
2131 		if (is_hairpin != rxq_ctrl->is_hairpin) {
2132 			*error = "combining hairpin and regular RSS queues is not supported";
2133 			*queue_idx = i;
2134 			return -ENOTSUP;
2135 		}
2136 	}
2137 	return 0;
2138 }
2139 
2140 /*
2141  * Validate the rss action.
2142  *
2143  * @param[in] dev
2144  *   Pointer to the Ethernet device structure.
2145  * @param[in] action
2146  *   Pointer to the queue action.
2147  * @param[out] error
2148  *   Pointer to error structure.
2149  *
2150  * @return
2151  *   0 on success, a negative errno value otherwise and rte_errno is set.
2152  */
2153 int
2154 mlx5_validate_action_rss(struct rte_eth_dev *dev,
2155 			 const struct rte_flow_action *action,
2156 			 struct rte_flow_error *error)
2157 {
2158 	struct mlx5_priv *priv = dev->data->dev_private;
2159 	const struct rte_flow_action_rss *rss = action->conf;
2160 	int ret;
2161 	const char *message;
2162 	uint32_t queue_idx;
2163 
2164 	if (!rss)
2165 		return rte_flow_error_set
2166 			(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
2167 			 action, "no RSS action configuration");
2168 	if (rss->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
2169 		DRV_LOG(WARNING, "port %u symmetric RSS supported with SORT",
2170 			dev->data->port_id);
2171 	} else if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
2172 		   rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
2173 		return rte_flow_error_set(error, ENOTSUP,
2174 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2175 					  &rss->func,
2176 					  "RSS hash function not supported");
2177 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
2178 	if (rss->level > 2)
2179 #else
2180 	if (rss->level > 1)
2181 #endif
2182 		return rte_flow_error_set(error, ENOTSUP,
2183 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2184 					  &rss->level,
2185 					  "tunnel RSS is not supported");
2186 	/* allow RSS key_len 0 in case of NULL (default) RSS key. */
2187 	if (rss->key_len == 0 && rss->key != NULL)
2188 		return rte_flow_error_set(error, ENOTSUP,
2189 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2190 					  &rss->key_len,
2191 					  "RSS hash key length 0");
2192 	if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
2193 		return rte_flow_error_set(error, ENOTSUP,
2194 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2195 					  &rss->key_len,
2196 					  "RSS hash key too small");
2197 	if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
2198 		return rte_flow_error_set(error, ENOTSUP,
2199 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2200 					  &rss->key_len,
2201 					  "RSS hash key too large");
2202 	if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size)
2203 		return rte_flow_error_set(error, ENOTSUP,
2204 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2205 					  &rss->queue_num,
2206 					  "number of queues too large");
2207 	if (rss->types & MLX5_RSS_HF_MASK)
2208 		return rte_flow_error_set(error, ENOTSUP,
2209 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2210 					  &rss->types,
2211 					  "some RSS protocols are not"
2212 					  " supported");
2213 	if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) &&
2214 	    !(rss->types & RTE_ETH_RSS_IP))
2215 		return rte_flow_error_set(error, EINVAL,
2216 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
2217 					  "L3 partial RSS requested but L3 RSS"
2218 					  " type not specified");
2219 	if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) &&
2220 	    !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP)))
2221 		return rte_flow_error_set(error, EINVAL,
2222 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
2223 					  "L4 partial RSS requested but L4 RSS"
2224 					  " type not specified");
2225 	if (!priv->rxqs_n && priv->ext_rxqs == NULL)
2226 		return rte_flow_error_set(error, EINVAL,
2227 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2228 					  NULL, "No Rx queues configured");
2229 	if (!rss->queue_num)
2230 		return rte_flow_error_set(error, EINVAL,
2231 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2232 					  NULL, "No queues configured");
2233 	ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num,
2234 				       &message, &queue_idx);
2235 	if (ret != 0) {
2236 		return rte_flow_error_set(error, -ret,
2237 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2238 					  &rss->queue[queue_idx], message);
2239 	}
2240 	return 0;
2241 }
2242 
2243 /*
2244  * Validate the rss action.
2245  *
2246  * @param[in] action
2247  *   Pointer to the queue action.
2248  * @param[in] action_flags
2249  *   Bit-fields that holds the actions detected until now.
2250  * @param[in] dev
2251  *   Pointer to the Ethernet device structure.
2252  * @param[in] attr
2253  *   Attributes of flow that includes this action.
2254  * @param[in] item_flags
2255  *   Items that were detected.
2256  * @param[out] error
2257  *   Pointer to error structure.
2258  *
2259  * @return
2260  *   0 on success, a negative errno value otherwise and rte_errno is set.
2261  */
2262 int
2263 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
2264 			      uint64_t action_flags,
2265 			      struct rte_eth_dev *dev,
2266 			      const struct rte_flow_attr *attr,
2267 			      uint64_t item_flags,
2268 			      struct rte_flow_error *error)
2269 {
2270 	const struct rte_flow_action_rss *rss = action->conf;
2271 	int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2272 	int ret;
2273 
2274 	if (action_flags & MLX5_FLOW_FATE_ACTIONS)
2275 		return rte_flow_error_set(error, EINVAL,
2276 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2277 					  "can't have 2 fate actions"
2278 					  " in same flow");
2279 	ret = mlx5_validate_action_rss(dev, action, error);
2280 	if (ret)
2281 		return ret;
2282 	if (attr->egress)
2283 		return rte_flow_error_set(error, ENOTSUP,
2284 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2285 					  "rss action not supported for "
2286 					  "egress");
2287 	if (rss->level > 1 && !tunnel)
2288 		return rte_flow_error_set(error, EINVAL,
2289 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
2290 					  "inner RSS is not supported for "
2291 					  "non-tunnel flows");
2292 	if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
2293 	    !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
2294 		return rte_flow_error_set(error, EINVAL,
2295 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
2296 					  "RSS on eCPRI is not supported now");
2297 	}
2298 	if ((item_flags & MLX5_FLOW_LAYER_MPLS) &&
2299 	    !(item_flags &
2300 	      (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) &&
2301 	    rss->level > 1)
2302 		return rte_flow_error_set(error, EINVAL,
2303 					  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2304 					  "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern");
2305 	return 0;
2306 }
2307 
2308 /*
2309  * Validate the default miss action.
2310  *
2311  * @param[in] action_flags
2312  *   Bit-fields that holds the actions detected until now.
2313  * @param[out] error
2314  *   Pointer to error structure.
2315  *
2316  * @return
2317  *   0 on success, a negative errno value otherwise and rte_errno is set.
2318  */
2319 int
2320 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
2321 				const struct rte_flow_attr *attr,
2322 				struct rte_flow_error *error)
2323 {
2324 	if (action_flags & MLX5_FLOW_FATE_ACTIONS)
2325 		return rte_flow_error_set(error, EINVAL,
2326 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2327 					  "can't have 2 fate actions in"
2328 					  " same flow");
2329 	if (attr->egress)
2330 		return rte_flow_error_set(error, ENOTSUP,
2331 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2332 					  "default miss action not supported "
2333 					  "for egress");
2334 	if (attr->group)
2335 		return rte_flow_error_set(error, ENOTSUP,
2336 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
2337 					  "only group 0 is supported");
2338 	if (attr->transfer)
2339 		return rte_flow_error_set(error, ENOTSUP,
2340 					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2341 					  NULL, "transfer is not supported");
2342 	return 0;
2343 }
2344 
2345 /*
2346  * Validate the count action.
2347  *
2348  * @param[in] dev
2349  *   Pointer to the Ethernet device structure.
2350  * @param[in] attr
2351  *   Attributes of flow that includes this action.
2352  * @param[out] error
2353  *   Pointer to error structure.
2354  *
2355  * @return
2356  *   0 on success, a negative errno value otherwise and rte_errno is set.
2357  */
2358 int
2359 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
2360 				const struct rte_flow_attr *attr,
2361 				struct rte_flow_error *error)
2362 {
2363 	if (attr->egress)
2364 		return rte_flow_error_set(error, ENOTSUP,
2365 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2366 					  "count action not supported for "
2367 					  "egress");
2368 	return 0;
2369 }
2370 
2371 /*
2372  * Validate the ASO CT action.
2373  *
2374  * @param[in] dev
2375  *   Pointer to the Ethernet device structure.
2376  * @param[in] conntrack
2377  *   Pointer to the CT action profile.
2378  * @param[out] error
2379  *   Pointer to error structure.
2380  *
2381  * @return
2382  *   0 on success, a negative errno value otherwise and rte_errno is set.
2383  */
2384 int
2385 mlx5_validate_action_ct(struct rte_eth_dev *dev,
2386 			const struct rte_flow_action_conntrack *conntrack,
2387 			struct rte_flow_error *error)
2388 {
2389 	RTE_SET_USED(dev);
2390 
2391 	if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
2392 		return rte_flow_error_set(error, EINVAL,
2393 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2394 					  "Invalid CT state");
2395 	if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
2396 		return rte_flow_error_set(error, EINVAL,
2397 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2398 					  "Invalid last TCP packet flag");
2399 	return 0;
2400 }
2401 
2402 /**
2403  * Validate the level value for modify field action.
2404  *
2405  * @param[in] data
2406  *   Pointer to the rte_flow_field_data structure either src or dst.
2407  * @param[out] error
2408  *   Pointer to error structure.
2409  *
2410  * @return
2411  *   0 on success, a negative errno value otherwise and rte_errno is set.
2412  */
2413 int
2414 flow_validate_modify_field_level(const struct rte_flow_field_data *data,
2415 				 struct rte_flow_error *error)
2416 {
2417 	if (data->level == 0 || data->field == RTE_FLOW_FIELD_FLEX_ITEM)
2418 		return 0;
2419 	if (data->field != RTE_FLOW_FIELD_TAG &&
2420 	    data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
2421 		if (data->level > 1)
2422 			return rte_flow_error_set(error, ENOTSUP,
2423 						  RTE_FLOW_ERROR_TYPE_ACTION,
2424 						  NULL,
2425 						  "inner header fields modification is not supported");
2426 		return 0;
2427 	}
2428 	if (data->tag_index != 0)
2429 		return rte_flow_error_set(error, EINVAL,
2430 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2431 					  "tag array can be provided using 'level' or 'tag_index' fields, not both");
2432 	/*
2433 	 * The tag array for RTE_FLOW_FIELD_TAG type is provided using
2434 	 * 'tag_index' field. In old API, it was provided using 'level' field
2435 	 * and it is still supported for backwards compatibility.
2436 	 */
2437 	DRV_LOG(DEBUG, "tag array provided in 'level' field instead of 'tag_index' field.");
2438 	return 0;
2439 }
2440 
2441 /**
2442  * Validate ICMP6 item.
2443  *
2444  * @param[in] item
2445  *   Item specification.
2446  * @param[in] item_flags
2447  *   Bit-fields that holds the items detected until now.
2448  * @param[in] ext_vlan_sup
2449  *   Whether extended VLAN features are supported or not.
2450  * @param[out] error
2451  *   Pointer to error structure.
2452  *
2453  * @return
2454  *   0 on success, a negative errno value otherwise and rte_errno is set.
2455  */
2456 int
2457 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
2458 			       uint64_t item_flags,
2459 			       uint8_t target_protocol,
2460 			       struct rte_flow_error *error)
2461 {
2462 	const struct rte_flow_item_icmp6 *mask = item->mask;
2463 	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2464 	const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2465 				      MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2466 	const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2467 				      MLX5_FLOW_LAYER_OUTER_L4;
2468 	int ret;
2469 
2470 	if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
2471 		return rte_flow_error_set(error, EINVAL,
2472 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2473 					  "protocol filtering not compatible"
2474 					  " with ICMP6 layer");
2475 	if (!(item_flags & l3m))
2476 		return rte_flow_error_set(error, EINVAL,
2477 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2478 					  "IPv6 is mandatory to filter on"
2479 					  " ICMP6");
2480 	if (item_flags & l4m)
2481 		return rte_flow_error_set(error, EINVAL,
2482 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2483 					  "multiple L4 layers not supported");
2484 	if (!mask)
2485 		mask = &rte_flow_item_icmp6_mask;
2486 	ret = mlx5_flow_item_acceptable
2487 		(item, (const uint8_t *)mask,
2488 		 (const uint8_t *)&rte_flow_item_icmp6_mask,
2489 		 sizeof(struct rte_flow_item_icmp6),
2490 		 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2491 	if (ret < 0)
2492 		return ret;
2493 	return 0;
2494 }
2495 
2496 /**
2497  * Validate ICMP6 echo request/reply item.
2498  *
2499  * @param[in] item
2500  *   Item specification.
2501  * @param[in] item_flags
2502  *   Bit-fields that holds the items detected until now.
2503  * @param[in] ext_vlan_sup
2504  *   Whether extended VLAN features are supported or not.
2505  * @param[out] error
2506  *   Pointer to error structure.
2507  *
2508  * @return
2509  *   0 on success, a negative errno value otherwise and rte_errno is set.
2510  */
2511 int
2512 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item,
2513 				   uint64_t item_flags,
2514 				   uint8_t target_protocol,
2515 				   struct rte_flow_error *error)
2516 {
2517 	const struct rte_flow_item_icmp6_echo *mask = item->mask;
2518 	const struct rte_flow_item_icmp6_echo nic_mask = {
2519 		.hdr.base.type = 0xff,
2520 		.hdr.base.code = 0xff,
2521 		.hdr.identifier = RTE_BE16(0xffff),
2522 		.hdr.sequence = RTE_BE16(0xffff),
2523 	};
2524 	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2525 	const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2526 				      MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2527 	const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2528 				      MLX5_FLOW_LAYER_OUTER_L4;
2529 	int ret;
2530 
2531 	if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
2532 		return rte_flow_error_set(error, EINVAL,
2533 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2534 					  "protocol filtering not compatible"
2535 					  " with ICMP6 layer");
2536 	if (!(item_flags & l3m))
2537 		return rte_flow_error_set(error, EINVAL,
2538 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2539 					  "IPv6 is mandatory to filter on"
2540 					  " ICMP6");
2541 	if (item_flags & l4m)
2542 		return rte_flow_error_set(error, EINVAL,
2543 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2544 					  "multiple L4 layers not supported");
2545 	if (!mask)
2546 		mask = &nic_mask;
2547 	ret = mlx5_flow_item_acceptable
2548 		(item, (const uint8_t *)mask,
2549 		 (const uint8_t *)&nic_mask,
2550 		 sizeof(struct rte_flow_item_icmp6_echo),
2551 		 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2552 	if (ret < 0)
2553 		return ret;
2554 	return 0;
2555 }
2556 
2557 /**
2558  * Validate ICMP item.
2559  *
2560  * @param[in] item
2561  *   Item specification.
2562  * @param[in] item_flags
2563  *   Bit-fields that holds the items detected until now.
2564  * @param[out] error
2565  *   Pointer to error structure.
2566  *
2567  * @return
2568  *   0 on success, a negative errno value otherwise and rte_errno is set.
2569  */
2570 int
2571 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
2572 			     uint64_t item_flags,
2573 			     uint8_t target_protocol,
2574 			     struct rte_flow_error *error)
2575 {
2576 	const struct rte_flow_item_icmp *mask = item->mask;
2577 	const struct rte_flow_item_icmp nic_mask = {
2578 		.hdr.icmp_type = 0xff,
2579 		.hdr.icmp_code = 0xff,
2580 		.hdr.icmp_ident = RTE_BE16(0xffff),
2581 		.hdr.icmp_seq_nb = RTE_BE16(0xffff),
2582 	};
2583 	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2584 	const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2585 				      MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2586 	const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2587 				      MLX5_FLOW_LAYER_OUTER_L4;
2588 	int ret;
2589 
2590 	if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
2591 		return rte_flow_error_set(error, EINVAL,
2592 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2593 					  "protocol filtering not compatible"
2594 					  " with ICMP layer");
2595 	if (!(item_flags & l3m))
2596 		return rte_flow_error_set(error, EINVAL,
2597 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2598 					  "IPv4 is mandatory to filter"
2599 					  " on ICMP");
2600 	if (item_flags & l4m)
2601 		return rte_flow_error_set(error, EINVAL,
2602 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2603 					  "multiple L4 layers not supported");
2604 	if (!mask)
2605 		mask = &nic_mask;
2606 	ret = mlx5_flow_item_acceptable
2607 		(item, (const uint8_t *)mask,
2608 		 (const uint8_t *)&nic_mask,
2609 		 sizeof(struct rte_flow_item_icmp),
2610 		 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2611 	if (ret < 0)
2612 		return ret;
2613 	return 0;
2614 }
2615 
2616 /**
2617  * Validate Ethernet item.
2618  *
2619  * @param[in] item
2620  *   Item specification.
2621  * @param[in] item_flags
2622  *   Bit-fields that holds the items detected until now.
2623  * @param[out] error
2624  *   Pointer to error structure.
2625  *
2626  * @return
2627  *   0 on success, a negative errno value otherwise and rte_errno is set.
2628  */
2629 int
2630 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
2631 			    uint64_t item_flags, bool ext_vlan_sup,
2632 			    struct rte_flow_error *error)
2633 {
2634 	const struct rte_flow_item_eth *mask = item->mask;
2635 	const struct rte_flow_item_eth nic_mask = {
2636 		.hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2637 		.hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2638 		.hdr.ether_type = RTE_BE16(0xffff),
2639 		.has_vlan = ext_vlan_sup ? 1 : 0,
2640 	};
2641 	int ret;
2642 	int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2643 	const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2	:
2644 				       MLX5_FLOW_LAYER_OUTER_L2;
2645 
2646 	if (item_flags & ethm)
2647 		return rte_flow_error_set(error, ENOTSUP,
2648 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2649 					  "multiple L2 layers not supported");
2650 	if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
2651 	    (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
2652 		return rte_flow_error_set(error, EINVAL,
2653 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2654 					  "L2 layer should not follow "
2655 					  "L3 layers");
2656 	if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
2657 	    (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
2658 		return rte_flow_error_set(error, EINVAL,
2659 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2660 					  "L2 layer should not follow VLAN");
2661 	if (item_flags & MLX5_FLOW_LAYER_GTP)
2662 		return rte_flow_error_set(error, EINVAL,
2663 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2664 					  "L2 layer should not follow GTP");
2665 	if (!mask)
2666 		mask = &rte_flow_item_eth_mask;
2667 	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2668 					(const uint8_t *)&nic_mask,
2669 					sizeof(struct rte_flow_item_eth),
2670 					MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2671 	return ret;
2672 }
2673 
2674 /**
2675  * Validate VLAN item.
2676  *
2677  * @param[in] item
2678  *   Item specification.
2679  * @param[in] item_flags
2680  *   Bit-fields that holds the items detected until now.
2681  * @param[in] dev
2682  *   Ethernet device flow is being created on.
2683  * @param[out] error
2684  *   Pointer to error structure.
2685  *
2686  * @return
2687  *   0 on success, a negative errno value otherwise and rte_errno is set.
2688  */
2689 int
2690 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
2691 			     uint64_t item_flags,
2692 			     struct rte_eth_dev *dev,
2693 			     struct rte_flow_error *error)
2694 {
2695 	const struct rte_flow_item_vlan *spec = item->spec;
2696 	const struct rte_flow_item_vlan *mask = item->mask;
2697 	const struct rte_flow_item_vlan nic_mask = {
2698 		.hdr.vlan_tci = RTE_BE16(UINT16_MAX),
2699 		.hdr.eth_proto = RTE_BE16(UINT16_MAX),
2700 	};
2701 	uint16_t vlan_tag = 0;
2702 	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2703 	int ret;
2704 	const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2705 					MLX5_FLOW_LAYER_INNER_L4) :
2706 				       (MLX5_FLOW_LAYER_OUTER_L3 |
2707 					MLX5_FLOW_LAYER_OUTER_L4);
2708 	const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2709 					MLX5_FLOW_LAYER_OUTER_VLAN;
2710 
2711 	if (item_flags & vlanm)
2712 		return rte_flow_error_set(error, EINVAL,
2713 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2714 					  "multiple VLAN layers not supported");
2715 	else if ((item_flags & l34m) != 0)
2716 		return rte_flow_error_set(error, EINVAL,
2717 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2718 					  "VLAN cannot follow L3/L4 layer");
2719 	if (!mask)
2720 		mask = &rte_flow_item_vlan_mask;
2721 	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2722 					(const uint8_t *)&nic_mask,
2723 					sizeof(struct rte_flow_item_vlan),
2724 					MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2725 	if (ret)
2726 		return ret;
2727 	if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) {
2728 		struct mlx5_priv *priv = dev->data->dev_private;
2729 
2730 		if (priv->vmwa_context) {
2731 			/*
2732 			 * Non-NULL context means we have a virtual machine
2733 			 * and SR-IOV enabled, we have to create VLAN interface
2734 			 * to make hypervisor to setup E-Switch vport
2735 			 * context correctly. We avoid creating the multiple
2736 			 * VLAN interfaces, so we cannot support VLAN tag mask.
2737 			 */
2738 			return rte_flow_error_set(error, EINVAL,
2739 						  RTE_FLOW_ERROR_TYPE_ITEM,
2740 						  item,
2741 						  "VLAN tag mask is not"
2742 						  " supported in virtual"
2743 						  " environment");
2744 		}
2745 	}
2746 	if (spec) {
2747 		vlan_tag = spec->hdr.vlan_tci;
2748 		vlan_tag &= mask->hdr.vlan_tci;
2749 	}
2750 	/*
2751 	 * From verbs perspective an empty VLAN is equivalent
2752 	 * to a packet without VLAN layer.
2753 	 */
2754 	if (!vlan_tag)
2755 		return rte_flow_error_set(error, EINVAL,
2756 					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2757 					  item->spec,
2758 					  "VLAN cannot be empty");
2759 	return 0;
2760 }
2761 
2762 /**
2763  * Validate IPV4 item.
2764  *
2765  * @param[in] item
2766  *   Item specification.
2767  * @param[in] item_flags
2768  *   Bit-fields that holds the items detected until now.
2769  * @param[in] last_item
2770  *   Previous validated item in the pattern items.
2771  * @param[in] ether_type
2772  *   Type in the ethernet layer header (including dot1q).
2773  * @param[in] acc_mask
2774  *   Acceptable mask, if NULL default internal default mask
2775  *   will be used to check whether item fields are supported.
2776  * @param[in] range_accepted
2777  *   True if range of values is accepted for specific fields, false otherwise.
2778  * @param[out] error
2779  *   Pointer to error structure.
2780  *
2781  * @return
2782  *   0 on success, a negative errno value otherwise and rte_errno is set.
2783  */
2784 int
2785 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2786 			     uint64_t item_flags,
2787 			     uint64_t last_item,
2788 			     uint16_t ether_type,
2789 			     const struct rte_flow_item_ipv4 *acc_mask,
2790 			     bool range_accepted,
2791 			     struct rte_flow_error *error)
2792 {
2793 	const struct rte_flow_item_ipv4 *mask = item->mask;
2794 	const struct rte_flow_item_ipv4 *spec = item->spec;
2795 	const struct rte_flow_item_ipv4 nic_mask = {
2796 		.hdr = {
2797 			.src_addr = RTE_BE32(0xffffffff),
2798 			.dst_addr = RTE_BE32(0xffffffff),
2799 			.type_of_service = 0xff,
2800 			.next_proto_id = 0xff,
2801 		},
2802 	};
2803 	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2804 	const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2805 				      MLX5_FLOW_LAYER_OUTER_L3;
2806 	const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2807 				      MLX5_FLOW_LAYER_OUTER_L4;
2808 	int ret;
2809 	uint8_t next_proto = 0xFF;
2810 	const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2811 				  MLX5_FLOW_LAYER_OUTER_VLAN |
2812 				  MLX5_FLOW_LAYER_INNER_VLAN);
2813 
2814 	if ((last_item & l2_vlan) && ether_type &&
2815 	    ether_type != RTE_ETHER_TYPE_IPV4)
2816 		return rte_flow_error_set(error, EINVAL,
2817 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2818 					  "IPv4 cannot follow L2/VLAN layer "
2819 					  "which ether type is not IPv4");
2820 	if (item_flags & MLX5_FLOW_LAYER_IPIP) {
2821 		if (mask && spec)
2822 			next_proto = mask->hdr.next_proto_id &
2823 				     spec->hdr.next_proto_id;
2824 		if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2825 			return rte_flow_error_set(error, EINVAL,
2826 						  RTE_FLOW_ERROR_TYPE_ITEM,
2827 						  item,
2828 						  "multiple tunnel "
2829 						  "not supported");
2830 	}
2831 	if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2832 		return rte_flow_error_set(error, EINVAL,
2833 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2834 					  "wrong tunnel type - IPv6 specified "
2835 					  "but IPv4 item provided");
2836 	if (item_flags & l3m)
2837 		return rte_flow_error_set(error, ENOTSUP,
2838 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2839 					  "multiple L3 layers not supported");
2840 	else if (item_flags & l4m)
2841 		return rte_flow_error_set(error, EINVAL,
2842 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2843 					  "L3 cannot follow an L4 layer.");
2844 	else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2845 		  !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2846 		return rte_flow_error_set(error, EINVAL,
2847 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2848 					  "L3 cannot follow an NVGRE layer.");
2849 	if (!mask)
2850 		mask = &rte_flow_item_ipv4_mask;
2851 	else if (mask->hdr.next_proto_id != 0 &&
2852 		 mask->hdr.next_proto_id != 0xff)
2853 		return rte_flow_error_set(error, EINVAL,
2854 					  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2855 					  "partial mask is not supported"
2856 					  " for protocol");
2857 	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2858 					acc_mask ? (const uint8_t *)acc_mask
2859 						 : (const uint8_t *)&nic_mask,
2860 					sizeof(struct rte_flow_item_ipv4),
2861 					range_accepted, error);
2862 	if (ret < 0)
2863 		return ret;
2864 	return 0;
2865 }
2866 
2867 /**
2868  * Validate IPV6 item.
2869  *
2870  * @param[in] item
2871  *   Item specification.
2872  * @param[in] item_flags
2873  *   Bit-fields that holds the items detected until now.
2874  * @param[in] last_item
2875  *   Previous validated item in the pattern items.
2876  * @param[in] ether_type
2877  *   Type in the ethernet layer header (including dot1q).
2878  * @param[in] acc_mask
2879  *   Acceptable mask, if NULL default internal default mask
2880  *   will be used to check whether item fields are supported.
2881  * @param[out] error
2882  *   Pointer to error structure.
2883  *
2884  * @return
2885  *   0 on success, a negative errno value otherwise and rte_errno is set.
2886  */
2887 int
2888 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2889 			     uint64_t item_flags,
2890 			     uint64_t last_item,
2891 			     uint16_t ether_type,
2892 			     const struct rte_flow_item_ipv6 *acc_mask,
2893 			     struct rte_flow_error *error)
2894 {
2895 	const struct rte_flow_item_ipv6 *mask = item->mask;
2896 	const struct rte_flow_item_ipv6 *spec = item->spec;
2897 	const struct rte_flow_item_ipv6 nic_mask = {
2898 		.hdr = {
2899 			.src_addr =
2900 				"\xff\xff\xff\xff\xff\xff\xff\xff"
2901 				"\xff\xff\xff\xff\xff\xff\xff\xff",
2902 			.dst_addr =
2903 				"\xff\xff\xff\xff\xff\xff\xff\xff"
2904 				"\xff\xff\xff\xff\xff\xff\xff\xff",
2905 			.vtc_flow = RTE_BE32(0xffffffff),
2906 			.proto = 0xff,
2907 		},
2908 	};
2909 	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2910 	const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2911 				      MLX5_FLOW_LAYER_OUTER_L3;
2912 	const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2913 				      MLX5_FLOW_LAYER_OUTER_L4;
2914 	int ret;
2915 	uint8_t next_proto = 0xFF;
2916 	const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2917 				  MLX5_FLOW_LAYER_OUTER_VLAN |
2918 				  MLX5_FLOW_LAYER_INNER_VLAN);
2919 
2920 	if ((last_item & l2_vlan) && ether_type &&
2921 	    ether_type != RTE_ETHER_TYPE_IPV6)
2922 		return rte_flow_error_set(error, EINVAL,
2923 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2924 					  "IPv6 cannot follow L2/VLAN layer "
2925 					  "which ether type is not IPv6");
2926 	if (mask && mask->hdr.proto == UINT8_MAX && spec)
2927 		next_proto = spec->hdr.proto;
2928 	if (item_flags & MLX5_FLOW_LAYER_IPIP) {
2929 		if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2930 			return rte_flow_error_set(error, EINVAL,
2931 						  RTE_FLOW_ERROR_TYPE_ITEM,
2932 						  item,
2933 						  "multiple tunnel "
2934 						  "not supported");
2935 	}
2936 	if (next_proto == IPPROTO_HOPOPTS  ||
2937 	    next_proto == IPPROTO_ROUTING  ||
2938 	    next_proto == IPPROTO_FRAGMENT ||
2939 	    next_proto == IPPROTO_ESP	   ||
2940 	    next_proto == IPPROTO_AH	   ||
2941 	    next_proto == IPPROTO_DSTOPTS)
2942 		return rte_flow_error_set(error, EINVAL,
2943 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2944 					  "IPv6 proto (next header) should "
2945 					  "not be set as extension header");
2946 	if (item_flags & MLX5_FLOW_LAYER_IPIP)
2947 		return rte_flow_error_set(error, EINVAL,
2948 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2949 					  "wrong tunnel type - IPv4 specified "
2950 					  "but IPv6 item provided");
2951 	if (item_flags & l3m)
2952 		return rte_flow_error_set(error, ENOTSUP,
2953 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2954 					  "multiple L3 layers not supported");
2955 	else if (item_flags & l4m)
2956 		return rte_flow_error_set(error, EINVAL,
2957 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2958 					  "L3 cannot follow an L4 layer.");
2959 	else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2960 		  !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2961 		return rte_flow_error_set(error, EINVAL,
2962 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
2963 					  "L3 cannot follow an NVGRE layer.");
2964 	if (!mask)
2965 		mask = &rte_flow_item_ipv6_mask;
2966 	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2967 					acc_mask ? (const uint8_t *)acc_mask
2968 						 : (const uint8_t *)&nic_mask,
2969 					sizeof(struct rte_flow_item_ipv6),
2970 					MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2971 	if (ret < 0)
2972 		return ret;
2973 	return 0;
2974 }
2975 
2976 /**
2977  * Validate UDP item.
2978  *
2979  * @param[in] item
2980  *   Item specification.
2981  * @param[in] item_flags
2982  *   Bit-fields that holds the items detected until now.
2983  * @param[in] target_protocol
2984  *   The next protocol in the previous item.
2985  * @param[in] flow_mask
2986  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2987  * @param[out] error
2988  *   Pointer to error structure.
2989  *
2990  * @return
2991  *   0 on success, a negative errno value otherwise and rte_errno is set.
2992  */
2993 int
2994 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2995 			    uint64_t item_flags,
2996 			    uint8_t target_protocol,
2997 			    struct rte_flow_error *error)
2998 {
2999 	const struct rte_flow_item_udp *mask = item->mask;
3000 	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
3001 	const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
3002 				      MLX5_FLOW_LAYER_OUTER_L3;
3003 	const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
3004 				      MLX5_FLOW_LAYER_OUTER_L4;
3005 	int ret;
3006 
3007 	if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
3008 		return rte_flow_error_set(error, EINVAL,
3009 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3010 					  "protocol filtering not compatible"
3011 					  " with UDP layer");
3012 	if (!(item_flags & l3m))
3013 		return rte_flow_error_set(error, EINVAL,
3014 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3015 					  "L3 is mandatory to filter on L4");
3016 	if (item_flags & l4m)
3017 		return rte_flow_error_set(error, EINVAL,
3018 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3019 					  "multiple L4 layers not supported");
3020 	if (!mask)
3021 		mask = &rte_flow_item_udp_mask;
3022 	ret = mlx5_flow_item_acceptable
3023 		(item, (const uint8_t *)mask,
3024 		 (const uint8_t *)&rte_flow_item_udp_mask,
3025 		 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
3026 		 error);
3027 	if (ret < 0)
3028 		return ret;
3029 	return 0;
3030 }
3031 
3032 /**
3033  * Validate TCP item.
3034  *
3035  * @param[in] item
3036  *   Item specification.
3037  * @param[in] item_flags
3038  *   Bit-fields that holds the items detected until now.
3039  * @param[in] target_protocol
3040  *   The next protocol in the previous item.
3041  * @param[out] error
3042  *   Pointer to error structure.
3043  *
3044  * @return
3045  *   0 on success, a negative errno value otherwise and rte_errno is set.
3046  */
3047 int
3048 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
3049 			    uint64_t item_flags,
3050 			    uint8_t target_protocol,
3051 			    const struct rte_flow_item_tcp *flow_mask,
3052 			    struct rte_flow_error *error)
3053 {
3054 	const struct rte_flow_item_tcp *mask = item->mask;
3055 	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
3056 	const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
3057 				      MLX5_FLOW_LAYER_OUTER_L3;
3058 	const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
3059 				      MLX5_FLOW_LAYER_OUTER_L4;
3060 	int ret;
3061 
3062 	MLX5_ASSERT(flow_mask);
3063 	if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
3064 		return rte_flow_error_set(error, EINVAL,
3065 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3066 					  "protocol filtering not compatible"
3067 					  " with TCP layer");
3068 	if (!(item_flags & l3m))
3069 		return rte_flow_error_set(error, EINVAL,
3070 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3071 					  "L3 is mandatory to filter on L4");
3072 	if (item_flags & l4m)
3073 		return rte_flow_error_set(error, EINVAL,
3074 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3075 					  "multiple L4 layers not supported");
3076 	if (!mask)
3077 		mask = &rte_flow_item_tcp_mask;
3078 	ret = mlx5_flow_item_acceptable
3079 		(item, (const uint8_t *)mask,
3080 		 (const uint8_t *)flow_mask,
3081 		 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
3082 		 error);
3083 	if (ret < 0)
3084 		return ret;
3085 	return 0;
3086 }
3087 
3088 /**
3089  * Validate VXLAN item.
3090  *
3091  * @param[in] dev
3092  *   Pointer to the Ethernet device structure.
3093  * @param[in] udp_dport
3094  *   UDP destination port
3095  * @param[in] item
3096  *   Item specification.
3097  * @param[in] item_flags
3098  *   Bit-fields that holds the items detected until now.
3099  * @param root
3100  *   Whether action is on root table.
3101  * @param[out] error
3102  *   Pointer to error structure.
3103  *
3104  * @return
3105  *   0 on success, a negative errno value otherwise and rte_errno is set.
3106  */
3107 int
3108 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev,
3109 			      uint16_t udp_dport,
3110 			      const struct rte_flow_item *item,
3111 			      uint64_t item_flags,
3112 			      bool root,
3113 			      struct rte_flow_error *error)
3114 {
3115 	const struct rte_flow_item_vxlan *spec = item->spec;
3116 	const struct rte_flow_item_vxlan *mask = item->mask;
3117 	int ret;
3118 	struct mlx5_priv *priv = dev->data->dev_private;
3119 	union vni {
3120 		uint32_t vlan_id;
3121 		uint8_t vni[4];
3122 	} id = { .vlan_id = 0, };
3123 	const struct rte_flow_item_vxlan nic_mask = {
3124 		.hdr.vni = "\xff\xff\xff",
3125 		.hdr.rsvd1 = 0xff,
3126 	};
3127 	const struct rte_flow_item_vxlan *valid_mask;
3128 
3129 	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3130 		return rte_flow_error_set(error, ENOTSUP,
3131 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3132 					  "multiple tunnel layers not"
3133 					  " supported");
3134 	valid_mask = &rte_flow_item_vxlan_mask;
3135 	/*
3136 	 * Verify only UDPv4 is present as defined in
3137 	 * https://tools.ietf.org/html/rfc7348
3138 	 */
3139 	if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
3140 		return rte_flow_error_set(error, EINVAL,
3141 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3142 					  "no outer UDP layer found");
3143 	if (!mask)
3144 		mask = &rte_flow_item_vxlan_mask;
3145 
3146 	if (priv->sh->steering_format_version !=
3147 	    MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
3148 	    !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) {
3149 		/* non-root table */
3150 		if (!root && priv->sh->misc5_cap)
3151 			valid_mask = &nic_mask;
3152 		/* Group zero in NIC domain */
3153 		if (!root && priv->sh->tunnel_header_0_1)
3154 			valid_mask = &nic_mask;
3155 	}
3156 	ret = mlx5_flow_item_acceptable
3157 		(item, (const uint8_t *)mask,
3158 		 (const uint8_t *)valid_mask,
3159 		 sizeof(struct rte_flow_item_vxlan),
3160 		 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3161 	if (ret < 0)
3162 		return ret;
3163 	if (spec) {
3164 		memcpy(&id.vni[1], spec->hdr.vni, 3);
3165 		memcpy(&id.vni[1], mask->hdr.vni, 3);
3166 	}
3167 	if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
3168 		return rte_flow_error_set(error, ENOTSUP,
3169 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3170 					  "VXLAN tunnel must be fully defined");
3171 	return 0;
3172 }
3173 
3174 /**
3175  * Validate VXLAN_GPE item.
3176  *
3177  * @param[in] item
3178  *   Item specification.
3179  * @param[in] item_flags
3180  *   Bit-fields that holds the items detected until now.
3181  * @param[in] priv
3182  *   Pointer to the private data structure.
3183  * @param[in] target_protocol
3184  *   The next protocol in the previous item.
3185  * @param[out] error
3186  *   Pointer to error structure.
3187  *
3188  * @return
3189  *   0 on success, a negative errno value otherwise and rte_errno is set.
3190  */
3191 int
3192 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
3193 				  uint64_t item_flags,
3194 				  struct rte_eth_dev *dev,
3195 				  struct rte_flow_error *error)
3196 {
3197 	struct mlx5_priv *priv = dev->data->dev_private;
3198 	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
3199 	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
3200 	int ret;
3201 	union vni {
3202 		uint32_t vlan_id;
3203 		uint8_t vni[4];
3204 	} id = { .vlan_id = 0, };
3205 
3206 	struct rte_flow_item_vxlan_gpe nic_mask = {
3207 		.vni = "\xff\xff\xff",
3208 		.protocol = 0xff,
3209 		.flags = 0xff,
3210 	};
3211 
3212 	if (!priv->sh->config.l3_vxlan_en)
3213 		return rte_flow_error_set(error, ENOTSUP,
3214 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3215 					  "L3 VXLAN is not enabled by device"
3216 					  " parameter and/or not configured in"
3217 					  " firmware");
3218 	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3219 		return rte_flow_error_set(error, ENOTSUP,
3220 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3221 					  "multiple tunnel layers not"
3222 					  " supported");
3223 	/*
3224 	 * Verify only UDPv4 is present as defined in
3225 	 * https://tools.ietf.org/html/rfc7348
3226 	 */
3227 	if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
3228 		return rte_flow_error_set(error, EINVAL,
3229 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3230 					  "no outer UDP layer found");
3231 	if (!mask)
3232 		mask = &rte_flow_item_vxlan_gpe_mask;
3233 	if (priv->sh->misc5_cap && priv->sh->tunnel_header_0_1) {
3234 		nic_mask.rsvd0[0] = 0xff;
3235 		nic_mask.rsvd0[1] = 0xff;
3236 		nic_mask.rsvd1 = 0xff;
3237 	}
3238 	ret = mlx5_flow_item_acceptable
3239 		(item, (const uint8_t *)mask,
3240 		 (const uint8_t *)&nic_mask,
3241 		 sizeof(struct rte_flow_item_vxlan_gpe),
3242 		 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3243 	if (ret < 0)
3244 		return ret;
3245 	if (spec) {
3246 		memcpy(&id.vni[1], spec->hdr.vni, 3);
3247 		memcpy(&id.vni[1], mask->hdr.vni, 3);
3248 	}
3249 	if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
3250 		return rte_flow_error_set(error, ENOTSUP,
3251 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3252 					  "VXLAN-GPE tunnel must be fully"
3253 					  " defined");
3254 	return 0;
3255 }
3256 /**
3257  * Validate GRE Key item.
3258  *
3259  * @param[in] item
3260  *   Item specification.
3261  * @param[in] item_flags
3262  *   Bit flags to mark detected items.
3263  * @param[in] gre_item
3264  *   Pointer to gre_item
3265  * @param[out] error
3266  *   Pointer to error structure.
3267  *
3268  * @return
3269  *   0 on success, a negative errno value otherwise and rte_errno is set.
3270  */
3271 int
3272 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
3273 				uint64_t item_flags,
3274 				const struct rte_flow_item *gre_item,
3275 				struct rte_flow_error *error)
3276 {
3277 	const rte_be32_t *mask = item->mask;
3278 	int ret = 0;
3279 	rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
3280 	const struct rte_flow_item_gre *gre_spec;
3281 	const struct rte_flow_item_gre *gre_mask;
3282 
3283 	if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
3284 		return rte_flow_error_set(error, ENOTSUP,
3285 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3286 					  "Multiple GRE key not support");
3287 	if (!(item_flags & MLX5_FLOW_LAYER_GRE))
3288 		return rte_flow_error_set(error, ENOTSUP,
3289 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3290 					  "No preceding GRE header");
3291 	if (item_flags & MLX5_FLOW_LAYER_INNER)
3292 		return rte_flow_error_set(error, ENOTSUP,
3293 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3294 					  "GRE key following a wrong item");
3295 	gre_mask = gre_item->mask;
3296 	if (!gre_mask)
3297 		gre_mask = &rte_flow_item_gre_mask;
3298 	gre_spec = gre_item->spec;
3299 	if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
3300 			 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
3301 		return rte_flow_error_set(error, EINVAL,
3302 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3303 					  "Key bit must be on");
3304 
3305 	if (!mask)
3306 		mask = &gre_key_default_mask;
3307 	ret = mlx5_flow_item_acceptable
3308 		(item, (const uint8_t *)mask,
3309 		 (const uint8_t *)&gre_key_default_mask,
3310 		 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3311 	return ret;
3312 }
3313 
3314 /**
3315  * Validate GRE optional item.
3316  *
3317  * @param[in] dev
3318  *   Pointer to the Ethernet device structure.
3319  * @param[in] item
3320  *   Item specification.
3321  * @param[in] item_flags
3322  *   Bit flags to mark detected items.
3323  * @param[in] attr
3324  *   Flow rule attributes.
3325  * @param[in] gre_item
3326  *   Pointer to gre_item
3327  * @param[out] error
3328  *   Pointer to error structure.
3329  *
3330  * @return
3331  *   0 on success, a negative errno value otherwise and rte_errno is set.
3332  */
3333 int
3334 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev,
3335 				   const struct rte_flow_item *item,
3336 				   uint64_t item_flags,
3337 				   const struct rte_flow_attr *attr,
3338 				   const struct rte_flow_item *gre_item,
3339 				   struct rte_flow_error *error)
3340 {
3341 	const struct rte_flow_item_gre *gre_spec = gre_item->spec;
3342 	const struct rte_flow_item_gre *gre_mask = gre_item->mask;
3343 	const struct rte_flow_item_gre_opt *spec = item->spec;
3344 	const struct rte_flow_item_gre_opt *mask = item->mask;
3345 	struct mlx5_priv *priv = dev->data->dev_private;
3346 	int ret = 0;
3347 	struct rte_flow_item_gre_opt nic_mask = {
3348 		.checksum_rsvd = {
3349 			.checksum = RTE_BE16(UINT16_MAX),
3350 			.reserved1 = 0x0,
3351 		},
3352 		.key = {
3353 			.key = RTE_BE32(UINT32_MAX),
3354 		},
3355 		.sequence = {
3356 			.sequence = RTE_BE32(UINT32_MAX),
3357 		},
3358 	};
3359 
3360 	if (!(item_flags & MLX5_FLOW_LAYER_GRE))
3361 		return rte_flow_error_set(error, ENOTSUP,
3362 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3363 					  "No preceding GRE header");
3364 	if (item_flags & MLX5_FLOW_LAYER_INNER)
3365 		return rte_flow_error_set(error, ENOTSUP,
3366 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3367 					  "GRE option following a wrong item");
3368 	if (!spec || !mask)
3369 		return rte_flow_error_set(error, EINVAL,
3370 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3371 					  "At least one field gre_option(checksum/key/sequence) must be specified");
3372 	if (!gre_mask)
3373 		gre_mask = &rte_flow_item_gre_mask;
3374 	if (mask->checksum_rsvd.checksum)
3375 		if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) &&
3376 				 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000)))
3377 			return rte_flow_error_set(error, EINVAL,
3378 						  RTE_FLOW_ERROR_TYPE_ITEM,
3379 						  item,
3380 						  "Checksum bit must be on");
3381 	if (mask->key.key)
3382 		if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
3383 				 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
3384 			return rte_flow_error_set(error, EINVAL,
3385 						  RTE_FLOW_ERROR_TYPE_ITEM,
3386 						  item, "Key bit must be on");
3387 	if (mask->sequence.sequence)
3388 		if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) &&
3389 				 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000)))
3390 			return rte_flow_error_set(error, EINVAL,
3391 						  RTE_FLOW_ERROR_TYPE_ITEM,
3392 						  item,
3393 						  "Sequence bit must be on");
3394 	if (mask->checksum_rsvd.checksum || mask->sequence.sequence) {
3395 		if (priv->sh->steering_format_version ==
3396 		    MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
3397 		    ((attr->group || (attr->transfer && priv->fdb_def_rule)) &&
3398 		     !priv->sh->misc5_cap) ||
3399 		    (!(priv->sh->tunnel_header_0_1 &&
3400 		       priv->sh->tunnel_header_2_3) &&
3401 		    !attr->group && (!attr->transfer || !priv->fdb_def_rule)))
3402 			return rte_flow_error_set(error, EINVAL,
3403 						  RTE_FLOW_ERROR_TYPE_ITEM,
3404 						  item,
3405 						  "Checksum/Sequence not supported");
3406 	}
3407 	ret = mlx5_flow_item_acceptable
3408 		(item, (const uint8_t *)mask,
3409 		 (const uint8_t *)&nic_mask,
3410 		 sizeof(struct rte_flow_item_gre_opt),
3411 		 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3412 	return ret;
3413 }
3414 
3415 /**
3416  * Validate GRE item.
3417  *
3418  * @param[in] item
3419  *   Item specification.
3420  * @param[in] item_flags
3421  *   Bit flags to mark detected items.
3422  * @param[in] target_protocol
3423  *   The next protocol in the previous item.
3424  * @param[out] error
3425  *   Pointer to error structure.
3426  *
3427  * @return
3428  *   0 on success, a negative errno value otherwise and rte_errno is set.
3429  */
3430 int
3431 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
3432 			    uint64_t item_flags,
3433 			    uint8_t target_protocol,
3434 			    struct rte_flow_error *error)
3435 {
3436 	const struct rte_flow_item_gre *spec __rte_unused = item->spec;
3437 	const struct rte_flow_item_gre *mask = item->mask;
3438 	int ret;
3439 	const struct rte_flow_item_gre nic_mask = {
3440 		.c_rsvd0_ver = RTE_BE16(0xB000),
3441 		.protocol = RTE_BE16(UINT16_MAX),
3442 	};
3443 
3444 	if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3445 		return rte_flow_error_set(error, EINVAL,
3446 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3447 					  "protocol filtering not compatible"
3448 					  " with this GRE layer");
3449 	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3450 		return rte_flow_error_set(error, ENOTSUP,
3451 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3452 					  "multiple tunnel layers not"
3453 					  " supported");
3454 	if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3455 		return rte_flow_error_set(error, ENOTSUP,
3456 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3457 					  "L3 Layer is missing");
3458 	if (!mask)
3459 		mask = &rte_flow_item_gre_mask;
3460 	ret = mlx5_flow_item_acceptable
3461 		(item, (const uint8_t *)mask,
3462 		 (const uint8_t *)&nic_mask,
3463 		 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
3464 		 error);
3465 	if (ret < 0)
3466 		return ret;
3467 #ifndef HAVE_MLX5DV_DR
3468 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
3469 	if (spec && (spec->protocol & mask->protocol))
3470 		return rte_flow_error_set(error, ENOTSUP,
3471 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3472 					  "without MPLS support the"
3473 					  " specification cannot be used for"
3474 					  " filtering");
3475 #endif
3476 #endif
3477 	return 0;
3478 }
3479 
3480 /**
3481  * Validate Geneve item.
3482  *
3483  * @param[in] item
3484  *   Item specification.
3485  * @param[in] itemFlags
3486  *   Bit-fields that holds the items detected until now.
3487  * @param[in] enPriv
3488  *   Pointer to the private data structure.
3489  * @param[out] error
3490  *   Pointer to error structure.
3491  *
3492  * @return
3493  *   0 on success, a negative errno value otherwise and rte_errno is set.
3494  */
3495 
3496 int
3497 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
3498 			       uint64_t item_flags,
3499 			       struct rte_eth_dev *dev,
3500 			       struct rte_flow_error *error)
3501 {
3502 	struct mlx5_priv *priv = dev->data->dev_private;
3503 	const struct rte_flow_item_geneve *spec = item->spec;
3504 	const struct rte_flow_item_geneve *mask = item->mask;
3505 	int ret;
3506 	uint16_t gbhdr;
3507 	uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ?
3508 			  MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
3509 	const struct rte_flow_item_geneve nic_mask = {
3510 		.ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
3511 		.vni = "\xff\xff\xff",
3512 		.protocol = RTE_BE16(UINT16_MAX),
3513 	};
3514 
3515 	if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx)
3516 		return rte_flow_error_set(error, ENOTSUP,
3517 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3518 					  "L3 Geneve is not enabled by device"
3519 					  " parameter and/or not configured in"
3520 					  " firmware");
3521 	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3522 		return rte_flow_error_set(error, ENOTSUP,
3523 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3524 					  "multiple tunnel layers not"
3525 					  " supported");
3526 	/*
3527 	 * Verify only UDPv4 is present as defined in
3528 	 * https://tools.ietf.org/html/rfc7348
3529 	 */
3530 	if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
3531 		return rte_flow_error_set(error, EINVAL,
3532 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3533 					  "no outer UDP layer found");
3534 	if (!mask)
3535 		mask = &rte_flow_item_geneve_mask;
3536 	ret = mlx5_flow_item_acceptable
3537 				  (item, (const uint8_t *)mask,
3538 				   (const uint8_t *)&nic_mask,
3539 				   sizeof(struct rte_flow_item_geneve),
3540 				   MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3541 	if (ret)
3542 		return ret;
3543 	if (spec) {
3544 		gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
3545 		if (MLX5_GENEVE_VER_VAL(gbhdr) ||
3546 		     MLX5_GENEVE_CRITO_VAL(gbhdr) ||
3547 		     MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
3548 			return rte_flow_error_set(error, ENOTSUP,
3549 						  RTE_FLOW_ERROR_TYPE_ITEM,
3550 						  item,
3551 						  "Geneve protocol unsupported"
3552 						  " fields are being used");
3553 		if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
3554 			return rte_flow_error_set
3555 					(error, ENOTSUP,
3556 					 RTE_FLOW_ERROR_TYPE_ITEM,
3557 					 item,
3558 					 "Unsupported Geneve options length");
3559 	}
3560 	if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
3561 		return rte_flow_error_set
3562 				    (error, ENOTSUP,
3563 				     RTE_FLOW_ERROR_TYPE_ITEM, item,
3564 				     "Geneve tunnel must be fully defined");
3565 	return 0;
3566 }
3567 
3568 /**
3569  * Validate Geneve TLV option item.
3570  *
3571  * @param[in] item
3572  *   Item specification.
3573  * @param[in] last_item
3574  *   Previous validated item in the pattern items.
3575  * @param[in] geneve_item
3576  *   Previous GENEVE item specification.
3577  * @param[in] dev
3578  *   Pointer to the rte_eth_dev structure.
3579  * @param[out] error
3580  *   Pointer to error structure.
3581  *
3582  * @return
3583  *   0 on success, a negative errno value otherwise and rte_errno is set.
3584  */
3585 int
3586 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
3587 				   uint64_t last_item,
3588 				   const struct rte_flow_item *geneve_item,
3589 				   struct rte_eth_dev *dev,
3590 				   struct rte_flow_error *error)
3591 {
3592 	struct mlx5_priv *priv = dev->data->dev_private;
3593 	struct mlx5_dev_ctx_shared *sh = priv->sh;
3594 	struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
3595 	struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr;
3596 	uint8_t data_max_supported =
3597 			hca_attr->max_geneve_tlv_option_data_len * 4;
3598 	const struct rte_flow_item_geneve *geneve_spec;
3599 	const struct rte_flow_item_geneve *geneve_mask;
3600 	const struct rte_flow_item_geneve_opt *spec = item->spec;
3601 	const struct rte_flow_item_geneve_opt *mask = item->mask;
3602 	unsigned int i;
3603 	unsigned int data_len;
3604 	uint8_t tlv_option_len;
3605 	uint16_t optlen_m, optlen_v;
3606 	const struct rte_flow_item_geneve_opt full_mask = {
3607 		.option_class = RTE_BE16(0xffff),
3608 		.option_type = 0xff,
3609 		.option_len = 0x1f,
3610 	};
3611 
3612 	if (!mask)
3613 		mask = &rte_flow_item_geneve_opt_mask;
3614 	if (!spec)
3615 		return rte_flow_error_set
3616 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3617 			"Geneve TLV opt class/type/length must be specified");
3618 	if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
3619 		return rte_flow_error_set
3620 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3621 			"Geneve TLV opt length exceeds the limit (31)");
3622 	/* Check if class type and length masks are full. */
3623 	if (full_mask.option_class != mask->option_class ||
3624 	    full_mask.option_type != mask->option_type ||
3625 	    full_mask.option_len != (mask->option_len & full_mask.option_len))
3626 		return rte_flow_error_set
3627 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3628 			"Geneve TLV opt class/type/length masks must be full");
3629 	/* Check if length is supported */
3630 	if ((uint32_t)spec->option_len >
3631 			hca_attr->max_geneve_tlv_option_data_len)
3632 		return rte_flow_error_set
3633 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3634 			"Geneve TLV opt length not supported");
3635 	if (hca_attr->max_geneve_tlv_options > 1)
3636 		DRV_LOG(DEBUG,
3637 			"max_geneve_tlv_options supports more than 1 option");
3638 	/* Check GENEVE item preceding. */
3639 	if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
3640 		return rte_flow_error_set
3641 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3642 			"Geneve opt item must be preceded with Geneve item");
3643 	geneve_spec = geneve_item->spec;
3644 	geneve_mask = geneve_item->mask ? geneve_item->mask :
3645 					  &rte_flow_item_geneve_mask;
3646 	/* Check if GENEVE TLV option size doesn't exceed option length */
3647 	if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
3648 			    geneve_spec->ver_opt_len_o_c_rsvd0)) {
3649 		tlv_option_len = spec->option_len & mask->option_len;
3650 		optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
3651 		optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
3652 		optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
3653 		optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
3654 		if ((optlen_v & optlen_m) <= tlv_option_len)
3655 			return rte_flow_error_set
3656 				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3657 				 "GENEVE TLV option length exceeds optlen");
3658 	}
3659 	/* Check if length is 0 or data is 0. */
3660 	if (spec->data == NULL || spec->option_len == 0)
3661 		return rte_flow_error_set
3662 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3663 			"Geneve TLV opt with zero data/length not supported");
3664 	/* Check not all data & mask are 0. */
3665 	data_len = spec->option_len * 4;
3666 	if (mask->data == NULL) {
3667 		for (i = 0; i < data_len; i++)
3668 			if (spec->data[i])
3669 				break;
3670 		if (i == data_len)
3671 			return rte_flow_error_set(error, ENOTSUP,
3672 				RTE_FLOW_ERROR_TYPE_ITEM, item,
3673 				"Can't match on Geneve option data 0");
3674 	} else {
3675 		for (i = 0; i < data_len; i++)
3676 			if (spec->data[i] & mask->data[i])
3677 				break;
3678 		if (i == data_len)
3679 			return rte_flow_error_set(error, ENOTSUP,
3680 				RTE_FLOW_ERROR_TYPE_ITEM, item,
3681 				"Can't match on Geneve option data and mask 0");
3682 		/* Check data mask supported. */
3683 		for (i = data_max_supported; i < data_len ; i++)
3684 			if (mask->data[i])
3685 				return rte_flow_error_set(error, ENOTSUP,
3686 					RTE_FLOW_ERROR_TYPE_ITEM, item,
3687 					"Data mask is of unsupported size");
3688 	}
3689 	/* Check GENEVE option is supported in NIC. */
3690 	if (!hca_attr->geneve_tlv_opt)
3691 		return rte_flow_error_set
3692 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3693 			"Geneve TLV opt not supported");
3694 	/* Check if we already have geneve option with different type/class. */
3695 	rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
3696 	geneve_opt_resource = sh->geneve_tlv_option_resource;
3697 	if (geneve_opt_resource != NULL)
3698 		if (geneve_opt_resource->option_class != spec->option_class ||
3699 		    geneve_opt_resource->option_type != spec->option_type ||
3700 		    geneve_opt_resource->length != spec->option_len) {
3701 			rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3702 			return rte_flow_error_set(error, ENOTSUP,
3703 				RTE_FLOW_ERROR_TYPE_ITEM, item,
3704 				"Only one Geneve TLV option supported");
3705 		}
3706 	rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3707 	return 0;
3708 }
3709 
3710 /**
3711  * Validate MPLS item.
3712  *
3713  * @param[in] dev
3714  *   Pointer to the rte_eth_dev structure.
3715  * @param[in] item
3716  *   Item specification.
3717  * @param[in] item_flags
3718  *   Bit-fields that holds the items detected until now.
3719  * @param[in] prev_layer
3720  *   The protocol layer indicated in previous item.
3721  * @param[out] error
3722  *   Pointer to error structure.
3723  *
3724  * @return
3725  *   0 on success, a negative errno value otherwise and rte_errno is set.
3726  */
3727 int
3728 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
3729 			     const struct rte_flow_item *item __rte_unused,
3730 			     uint64_t item_flags __rte_unused,
3731 			     uint64_t prev_layer __rte_unused,
3732 			     struct rte_flow_error *error)
3733 {
3734 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
3735 	const struct rte_flow_item_mpls *mask = item->mask;
3736 	struct mlx5_priv *priv = dev->data->dev_private;
3737 	int ret;
3738 
3739 	if (!priv->sh->dev_cap.mpls_en)
3740 		return rte_flow_error_set(error, ENOTSUP,
3741 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3742 					  "MPLS not supported or"
3743 					  " disabled in firmware"
3744 					  " configuration.");
3745 	/* MPLS over UDP, GRE is allowed */
3746 	if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP |
3747 			    MLX5_FLOW_LAYER_GRE |
3748 			    MLX5_FLOW_LAYER_GRE_KEY)))
3749 		return rte_flow_error_set(error, EINVAL,
3750 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3751 					  "protocol filtering not compatible"
3752 					  " with MPLS layer");
3753 	/* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
3754 	if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3755 	    !(item_flags & MLX5_FLOW_LAYER_GRE))
3756 		return rte_flow_error_set(error, ENOTSUP,
3757 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3758 					  "multiple tunnel layers not"
3759 					  " supported");
3760 	if (!mask)
3761 		mask = &rte_flow_item_mpls_mask;
3762 	ret = mlx5_flow_item_acceptable
3763 		(item, (const uint8_t *)mask,
3764 		 (const uint8_t *)&rte_flow_item_mpls_mask,
3765 		 sizeof(struct rte_flow_item_mpls),
3766 		 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3767 	if (ret < 0)
3768 		return ret;
3769 	return 0;
3770 #else
3771 	return rte_flow_error_set(error, ENOTSUP,
3772 				  RTE_FLOW_ERROR_TYPE_ITEM, item,
3773 				  "MPLS is not supported by Verbs, please"
3774 				  " update.");
3775 #endif
3776 }
3777 
3778 /**
3779  * Validate NVGRE item.
3780  *
3781  * @param[in] item
3782  *   Item specification.
3783  * @param[in] item_flags
3784  *   Bit flags to mark detected items.
3785  * @param[in] target_protocol
3786  *   The next protocol in the previous item.
3787  * @param[out] error
3788  *   Pointer to error structure.
3789  *
3790  * @return
3791  *   0 on success, a negative errno value otherwise and rte_errno is set.
3792  */
3793 int
3794 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
3795 			      uint64_t item_flags,
3796 			      uint8_t target_protocol,
3797 			      struct rte_flow_error *error)
3798 {
3799 	const struct rte_flow_item_nvgre *mask = item->mask;
3800 	int ret;
3801 
3802 	if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3803 		return rte_flow_error_set(error, EINVAL,
3804 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3805 					  "protocol filtering not compatible"
3806 					  " with this GRE layer");
3807 	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3808 		return rte_flow_error_set(error, ENOTSUP,
3809 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3810 					  "multiple tunnel layers not"
3811 					  " supported");
3812 	if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3813 		return rte_flow_error_set(error, ENOTSUP,
3814 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3815 					  "L3 Layer is missing");
3816 	if (!mask)
3817 		mask = &rte_flow_item_nvgre_mask;
3818 	ret = mlx5_flow_item_acceptable
3819 		(item, (const uint8_t *)mask,
3820 		 (const uint8_t *)&rte_flow_item_nvgre_mask,
3821 		 sizeof(struct rte_flow_item_nvgre),
3822 		 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3823 	if (ret < 0)
3824 		return ret;
3825 	return 0;
3826 }
3827 
3828 /**
3829  * Validate eCPRI item.
3830  *
3831  * @param[in] item
3832  *   Item specification.
3833  * @param[in] item_flags
3834  *   Bit-fields that holds the items detected until now.
3835  * @param[in] last_item
3836  *   Previous validated item in the pattern items.
3837  * @param[in] ether_type
3838  *   Type in the ethernet layer header (including dot1q).
3839  * @param[in] acc_mask
3840  *   Acceptable mask, if NULL default internal default mask
3841  *   will be used to check whether item fields are supported.
3842  * @param[out] error
3843  *   Pointer to error structure.
3844  *
3845  * @return
3846  *   0 on success, a negative errno value otherwise and rte_errno is set.
3847  */
3848 int
3849 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3850 			      uint64_t item_flags,
3851 			      uint64_t last_item,
3852 			      uint16_t ether_type,
3853 			      const struct rte_flow_item_ecpri *acc_mask,
3854 			      struct rte_flow_error *error)
3855 {
3856 	const struct rte_flow_item_ecpri *mask = item->mask;
3857 	const struct rte_flow_item_ecpri nic_mask = {
3858 		.hdr = {
3859 			.common = {
3860 				.u32 =
3861 				RTE_BE32(((const struct rte_ecpri_common_hdr) {
3862 					.type = 0xFF,
3863 					}).u32),
3864 			},
3865 			.dummy[0] = 0xFFFFFFFF,
3866 		},
3867 	};
3868 	const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3869 					MLX5_FLOW_LAYER_OUTER_VLAN);
3870 	struct rte_flow_item_ecpri mask_lo;
3871 
3872 	if (!(last_item & outer_l2_vlan) &&
3873 	    last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3874 		return rte_flow_error_set(error, EINVAL,
3875 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3876 					  "eCPRI can only follow L2/VLAN layer or UDP layer");
3877 	if ((last_item & outer_l2_vlan) && ether_type &&
3878 	    ether_type != RTE_ETHER_TYPE_ECPRI)
3879 		return rte_flow_error_set(error, EINVAL,
3880 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3881 					  "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3882 	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3883 		return rte_flow_error_set(error, EINVAL,
3884 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3885 					  "eCPRI with tunnel is not supported right now");
3886 	if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3887 		return rte_flow_error_set(error, ENOTSUP,
3888 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3889 					  "multiple L3 layers not supported");
3890 	else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3891 		return rte_flow_error_set(error, EINVAL,
3892 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3893 					  "eCPRI cannot coexist with a TCP layer");
3894 	/* In specification, eCPRI could be over UDP layer. */
3895 	else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3896 		return rte_flow_error_set(error, EINVAL,
3897 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3898 					  "eCPRI over UDP layer is not yet supported right now");
3899 	/* Mask for type field in common header could be zero. */
3900 	if (!mask)
3901 		mask = &rte_flow_item_ecpri_mask;
3902 	mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3903 	/* Input mask is in big-endian format. */
3904 	if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3905 		return rte_flow_error_set(error, EINVAL,
3906 					  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3907 					  "partial mask is not supported for protocol");
3908 	else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3909 		return rte_flow_error_set(error, EINVAL,
3910 					  RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3911 					  "message header mask must be after a type mask");
3912 	return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3913 					 acc_mask ? (const uint8_t *)acc_mask
3914 						  : (const uint8_t *)&nic_mask,
3915 					 sizeof(struct rte_flow_item_ecpri),
3916 					 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3917 }
3918 
3919 /**
3920  * Validate the NSH item.
3921  *
3922  * @param[in] dev
3923  *   Pointer to Ethernet device on which flow rule is being created on.
3924  * @param[out] error
3925  *   Pointer to error structure.
3926  *
3927  * @return
3928  *   0 on success, a negative errno value otherwise and rte_errno is set.
3929  */
3930 int
3931 mlx5_flow_validate_item_nsh(struct rte_eth_dev *dev,
3932 			    const struct rte_flow_item *item,
3933 			    struct rte_flow_error *error)
3934 {
3935 	struct mlx5_priv *priv = dev->data->dev_private;
3936 
3937 	if (item->mask) {
3938 		return rte_flow_error_set(error, ENOTSUP,
3939 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3940 					  "NSH fields matching is not supported");
3941 	}
3942 
3943 	if (!priv->sh->config.dv_flow_en) {
3944 		return rte_flow_error_set(error, ENOTSUP,
3945 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3946 					  NULL, "NSH support requires DV flow interface");
3947 	}
3948 
3949 	if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_vxlan_gpe_nsh) {
3950 		return rte_flow_error_set(error, ENOTSUP,
3951 					  RTE_FLOW_ERROR_TYPE_ITEM, item,
3952 					  "Current FW does not support matching on NSH");
3953 	}
3954 
3955 	return 0;
3956 }
3957 
3958 static int
3959 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3960 		   const struct rte_flow_attr *attr __rte_unused,
3961 		   const struct rte_flow_item items[] __rte_unused,
3962 		   const struct rte_flow_action actions[] __rte_unused,
3963 		   bool external __rte_unused,
3964 		   int hairpin __rte_unused,
3965 		   struct rte_flow_error *error)
3966 {
3967 	return rte_flow_error_set(error, ENOTSUP,
3968 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3969 }
3970 
3971 static struct mlx5_flow *
3972 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3973 		  const struct rte_flow_attr *attr __rte_unused,
3974 		  const struct rte_flow_item items[] __rte_unused,
3975 		  const struct rte_flow_action actions[] __rte_unused,
3976 		  struct rte_flow_error *error)
3977 {
3978 	rte_flow_error_set(error, ENOTSUP,
3979 			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3980 	return NULL;
3981 }
3982 
3983 static int
3984 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3985 		    struct mlx5_flow *dev_flow __rte_unused,
3986 		    const struct rte_flow_attr *attr __rte_unused,
3987 		    const struct rte_flow_item items[] __rte_unused,
3988 		    const struct rte_flow_action actions[] __rte_unused,
3989 		    struct rte_flow_error *error)
3990 {
3991 	return rte_flow_error_set(error, ENOTSUP,
3992 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3993 }
3994 
3995 static int
3996 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3997 		struct rte_flow *flow __rte_unused,
3998 		struct rte_flow_error *error)
3999 {
4000 	return rte_flow_error_set(error, ENOTSUP,
4001 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
4002 }
4003 
4004 static void
4005 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
4006 		 struct rte_flow *flow __rte_unused)
4007 {
4008 }
4009 
4010 static void
4011 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
4012 		  struct rte_flow *flow __rte_unused)
4013 {
4014 }
4015 
4016 static int
4017 flow_null_query(struct rte_eth_dev *dev __rte_unused,
4018 		struct rte_flow *flow __rte_unused,
4019 		const struct rte_flow_action *actions __rte_unused,
4020 		void *data __rte_unused,
4021 		struct rte_flow_error *error)
4022 {
4023 	return rte_flow_error_set(error, ENOTSUP,
4024 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
4025 }
4026 
4027 static int
4028 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
4029 		      uint32_t domains __rte_unused,
4030 		      uint32_t flags __rte_unused)
4031 {
4032 	return 0;
4033 }
4034 
4035 int
4036 flow_null_get_aged_flows(struct rte_eth_dev *dev,
4037 		    void **context __rte_unused,
4038 		    uint32_t nb_contexts __rte_unused,
4039 		    struct rte_flow_error *error __rte_unused)
4040 {
4041 	DRV_LOG(ERR, "port %u get aged flows is not supported.",
4042 		dev->data->port_id);
4043 	return -ENOTSUP;
4044 }
4045 
4046 uint32_t
4047 flow_null_counter_allocate(struct rte_eth_dev *dev)
4048 {
4049 	DRV_LOG(ERR, "port %u counter allocate is not supported.",
4050 		dev->data->port_id);
4051 	return 0;
4052 }
4053 
4054 void
4055 flow_null_counter_free(struct rte_eth_dev *dev,
4056 			uint32_t counter __rte_unused)
4057 {
4058 	DRV_LOG(ERR, "port %u counter free is not supported.",
4059 		 dev->data->port_id);
4060 }
4061 
4062 int
4063 flow_null_counter_query(struct rte_eth_dev *dev,
4064 			uint32_t counter __rte_unused,
4065 			bool clear __rte_unused,
4066 			uint64_t *pkts __rte_unused,
4067 			uint64_t *bytes __rte_unused,
4068 			void **action __rte_unused)
4069 {
4070 	DRV_LOG(ERR, "port %u counter query is not supported.",
4071 		 dev->data->port_id);
4072 	return -ENOTSUP;
4073 }
4074 
4075 /* Void driver to protect from null pointer reference. */
4076 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
4077 	.validate = flow_null_validate,
4078 	.prepare = flow_null_prepare,
4079 	.translate = flow_null_translate,
4080 	.apply = flow_null_apply,
4081 	.remove = flow_null_remove,
4082 	.destroy = flow_null_destroy,
4083 	.query = flow_null_query,
4084 	.sync_domain = flow_null_sync_domain,
4085 	.get_aged_flows = flow_null_get_aged_flows,
4086 	.counter_alloc = flow_null_counter_allocate,
4087 	.counter_free = flow_null_counter_free,
4088 	.counter_query = flow_null_counter_query
4089 };
4090 
4091 /**
4092  * Select flow driver type according to flow attributes and device
4093  * configuration.
4094  *
4095  * @param[in] dev
4096  *   Pointer to the dev structure.
4097  * @param[in] attr
4098  *   Pointer to the flow attributes.
4099  *
4100  * @return
4101  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
4102  */
4103 static enum mlx5_flow_drv_type
4104 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
4105 {
4106 	struct mlx5_priv *priv = dev->data->dev_private;
4107 	/* The OS can determine first a specific flow type (DV, VERBS) */
4108 	enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
4109 
4110 	if (type != MLX5_FLOW_TYPE_MAX)
4111 		return type;
4112 	/*
4113 	 * Currently when dv_flow_en == 2, only HW steering engine is
4114 	 * supported. New engines can also be chosen here if ready.
4115 	 */
4116 	if (priv->sh->config.dv_flow_en == 2)
4117 		return MLX5_FLOW_TYPE_HW;
4118 	if (!attr)
4119 		return MLX5_FLOW_TYPE_MIN;
4120 	/* If no OS specific type - continue with DV/VERBS selection */
4121 	if (attr->transfer && priv->sh->config.dv_esw_en)
4122 		type = MLX5_FLOW_TYPE_DV;
4123 	if (!attr->transfer)
4124 		type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
4125 						     MLX5_FLOW_TYPE_VERBS;
4126 	return type;
4127 }
4128 
4129 #define flow_get_drv_ops(type) flow_drv_ops[type]
4130 
4131 /**
4132  * Flow driver validation API. This abstracts calling driver specific functions.
4133  * The type of flow driver is determined according to flow attributes.
4134  *
4135  * @param[in] dev
4136  *   Pointer to the dev structure.
4137  * @param[in] attr
4138  *   Pointer to the flow attributes.
4139  * @param[in] items
4140  *   Pointer to the list of items.
4141  * @param[in] actions
4142  *   Pointer to the list of actions.
4143  * @param[in] external
4144  *   This flow rule is created by request external to PMD.
4145  * @param[in] hairpin
4146  *   Number of hairpin TX actions, 0 means classic flow.
4147  * @param[out] error
4148  *   Pointer to the error structure.
4149  *
4150  * @return
4151  *   0 on success, a negative errno value otherwise and rte_errno is set.
4152  */
4153 static inline int
4154 flow_drv_validate(struct rte_eth_dev *dev,
4155 		  const struct rte_flow_attr *attr,
4156 		  const struct rte_flow_item items[],
4157 		  const struct rte_flow_action actions[],
4158 		  bool external, int hairpin, struct rte_flow_error *error)
4159 {
4160 	const struct mlx5_flow_driver_ops *fops;
4161 	enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
4162 
4163 	fops = flow_get_drv_ops(type);
4164 	return fops->validate(dev, attr, items, actions, external,
4165 			      hairpin, error);
4166 }
4167 
4168 /**
4169  * Flow driver preparation API. This abstracts calling driver specific
4170  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
4171  * calculates the size of memory required for device flow, allocates the memory,
4172  * initializes the device flow and returns the pointer.
4173  *
4174  * @note
4175  *   This function initializes device flow structure such as dv or verbs in
4176  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
4177  *   rest. For example, adding returning device flow to flow->dev_flow list and
4178  *   setting backward reference to the flow should be done out of this function.
4179  *   layers field is not filled either.
4180  *
4181  * @param[in] dev
4182  *   Pointer to the dev structure.
4183  * @param[in] attr
4184  *   Pointer to the flow attributes.
4185  * @param[in] items
4186  *   Pointer to the list of items.
4187  * @param[in] actions
4188  *   Pointer to the list of actions.
4189  * @param[in] flow_idx
4190  *   This memory pool index to the flow.
4191  * @param[out] error
4192  *   Pointer to the error structure.
4193  *
4194  * @return
4195  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
4196  */
4197 static inline struct mlx5_flow *
4198 flow_drv_prepare(struct rte_eth_dev *dev,
4199 		 const struct rte_flow *flow,
4200 		 const struct rte_flow_attr *attr,
4201 		 const struct rte_flow_item items[],
4202 		 const struct rte_flow_action actions[],
4203 		 uint32_t flow_idx,
4204 		 struct rte_flow_error *error)
4205 {
4206 	const struct mlx5_flow_driver_ops *fops;
4207 	enum mlx5_flow_drv_type type = flow->drv_type;
4208 	struct mlx5_flow *mlx5_flow = NULL;
4209 
4210 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
4211 	fops = flow_get_drv_ops(type);
4212 	mlx5_flow = fops->prepare(dev, attr, items, actions, error);
4213 	if (mlx5_flow)
4214 		mlx5_flow->flow_idx = flow_idx;
4215 	return mlx5_flow;
4216 }
4217 
4218 /**
4219  * Flow driver translation API. This abstracts calling driver specific
4220  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
4221  * translates a generic flow into a driver flow. flow_drv_prepare() must
4222  * precede.
4223  *
4224  * @note
4225  *   dev_flow->layers could be filled as a result of parsing during translation
4226  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
4227  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
4228  *   flow->actions could be overwritten even though all the expanded dev_flows
4229  *   have the same actions.
4230  *
4231  * @param[in] dev
4232  *   Pointer to the rte dev structure.
4233  * @param[in, out] dev_flow
4234  *   Pointer to the mlx5 flow.
4235  * @param[in] attr
4236  *   Pointer to the flow attributes.
4237  * @param[in] items
4238  *   Pointer to the list of items.
4239  * @param[in] actions
4240  *   Pointer to the list of actions.
4241  * @param[out] error
4242  *   Pointer to the error structure.
4243  *
4244  * @return
4245  *   0 on success, a negative errno value otherwise and rte_errno is set.
4246  */
4247 static inline int
4248 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
4249 		   const struct rte_flow_attr *attr,
4250 		   const struct rte_flow_item items[],
4251 		   const struct rte_flow_action actions[],
4252 		   struct rte_flow_error *error)
4253 {
4254 	const struct mlx5_flow_driver_ops *fops;
4255 	enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
4256 
4257 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
4258 	fops = flow_get_drv_ops(type);
4259 	return fops->translate(dev, dev_flow, attr, items, actions, error);
4260 }
4261 
4262 /**
4263  * Flow driver apply API. This abstracts calling driver specific functions.
4264  * Parent flow (rte_flow) should have driver type (drv_type). It applies
4265  * translated driver flows on to device. flow_drv_translate() must precede.
4266  *
4267  * @param[in] dev
4268  *   Pointer to Ethernet device structure.
4269  * @param[in, out] flow
4270  *   Pointer to flow structure.
4271  * @param[out] error
4272  *   Pointer to error structure.
4273  *
4274  * @return
4275  *   0 on success, a negative errno value otherwise and rte_errno is set.
4276  */
4277 static inline int
4278 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
4279 	       struct rte_flow_error *error)
4280 {
4281 	const struct mlx5_flow_driver_ops *fops;
4282 	enum mlx5_flow_drv_type type = flow->drv_type;
4283 
4284 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
4285 	fops = flow_get_drv_ops(type);
4286 	return fops->apply(dev, flow, error);
4287 }
4288 
4289 /**
4290  * Flow driver destroy API. This abstracts calling driver specific functions.
4291  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
4292  * on device and releases resources of the flow.
4293  *
4294  * @param[in] dev
4295  *   Pointer to Ethernet device.
4296  * @param[in, out] flow
4297  *   Pointer to flow structure.
4298  */
4299 static inline void
4300 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
4301 {
4302 	const struct mlx5_flow_driver_ops *fops;
4303 	enum mlx5_flow_drv_type type = flow->drv_type;
4304 
4305 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
4306 	fops = flow_get_drv_ops(type);
4307 	fops->destroy(dev, flow);
4308 }
4309 
4310 /**
4311  * Flow driver find RSS policy tbl API. This abstracts calling driver
4312  * specific functions. Parent flow (rte_flow) should have driver
4313  * type (drv_type). It will find the RSS policy table that has the rss_desc.
4314  *
4315  * @param[in] dev
4316  *   Pointer to Ethernet device.
4317  * @param[in, out] flow
4318  *   Pointer to flow structure.
4319  * @param[in] policy
4320  *   Pointer to meter policy table.
4321  * @param[in] rss_desc
4322  *   Pointer to rss_desc
4323  */
4324 static struct mlx5_flow_meter_sub_policy *
4325 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
4326 		struct rte_flow *flow,
4327 		struct mlx5_flow_meter_policy *policy,
4328 		struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
4329 {
4330 	const struct mlx5_flow_driver_ops *fops;
4331 	enum mlx5_flow_drv_type type = flow->drv_type;
4332 
4333 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
4334 	fops = flow_get_drv_ops(type);
4335 	return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
4336 }
4337 
4338 /**
4339  * Flow driver color tag rule API. This abstracts calling driver
4340  * specific functions. Parent flow (rte_flow) should have driver
4341  * type (drv_type). It will create the color tag rules in hierarchy meter.
4342  *
4343  * @param[in] dev
4344  *   Pointer to Ethernet device.
4345  * @param[in, out] flow
4346  *   Pointer to flow structure.
4347  * @param[in] fm
4348  *   Pointer to flow meter structure.
4349  * @param[in] src_port
4350  *   The src port this extra rule should use.
4351  * @param[in] item
4352  *   The src port id match item.
4353  * @param[out] error
4354  *   Pointer to error structure.
4355  */
4356 static int
4357 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev,
4358 		struct rte_flow *flow,
4359 		struct mlx5_flow_meter_info *fm,
4360 		int32_t src_port,
4361 		const struct rte_flow_item *item,
4362 		struct rte_flow_error *error)
4363 {
4364 	const struct mlx5_flow_driver_ops *fops;
4365 	enum mlx5_flow_drv_type type = flow->drv_type;
4366 
4367 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
4368 	fops = flow_get_drv_ops(type);
4369 	return fops->meter_hierarchy_rule_create(dev, fm,
4370 						src_port, item, error);
4371 }
4372 
4373 /**
4374  * Get RSS action from the action list.
4375  *
4376  * @param[in] dev
4377  *   Pointer to Ethernet device.
4378  * @param[in] actions
4379  *   Pointer to the list of actions.
4380  * @param[in] flow
4381  *   Parent flow structure pointer.
4382  *
4383  * @return
4384  *   Pointer to the RSS action if exist, else return NULL.
4385  */
4386 static const struct rte_flow_action_rss*
4387 flow_get_rss_action(struct rte_eth_dev *dev,
4388 		    const struct rte_flow_action actions[])
4389 {
4390 	struct mlx5_priv *priv = dev->data->dev_private;
4391 	const struct rte_flow_action_rss *rss = NULL;
4392 	struct mlx5_meter_policy_action_container *acg;
4393 	struct mlx5_meter_policy_action_container *acy;
4394 
4395 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4396 		switch (actions->type) {
4397 		case RTE_FLOW_ACTION_TYPE_RSS:
4398 			rss = actions->conf;
4399 			break;
4400 		case RTE_FLOW_ACTION_TYPE_SAMPLE:
4401 		{
4402 			const struct rte_flow_action_sample *sample =
4403 								actions->conf;
4404 			const struct rte_flow_action *act = sample->actions;
4405 			for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
4406 				if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
4407 					rss = act->conf;
4408 			break;
4409 		}
4410 		case RTE_FLOW_ACTION_TYPE_METER:
4411 		{
4412 			uint32_t mtr_idx;
4413 			struct mlx5_flow_meter_info *fm;
4414 			struct mlx5_flow_meter_policy *policy;
4415 			const struct rte_flow_action_meter *mtr = actions->conf;
4416 
4417 			fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
4418 			if (fm && !fm->def_policy) {
4419 				policy = mlx5_flow_meter_policy_find(dev,
4420 						fm->policy_id, NULL);
4421 				MLX5_ASSERT(policy);
4422 				if (policy->is_hierarchy) {
4423 					policy =
4424 				mlx5_flow_meter_hierarchy_get_final_policy(dev,
4425 									policy);
4426 					if (!policy)
4427 						return NULL;
4428 				}
4429 				if (policy->is_rss) {
4430 					acg =
4431 					&policy->act_cnt[RTE_COLOR_GREEN];
4432 					acy =
4433 					&policy->act_cnt[RTE_COLOR_YELLOW];
4434 					if (acg->fate_action ==
4435 					    MLX5_FLOW_FATE_SHARED_RSS)
4436 						rss = acg->rss->conf;
4437 					else if (acy->fate_action ==
4438 						 MLX5_FLOW_FATE_SHARED_RSS)
4439 						rss = acy->rss->conf;
4440 				}
4441 			}
4442 			break;
4443 		}
4444 		default:
4445 			break;
4446 		}
4447 	}
4448 	return rss;
4449 }
4450 
4451 /**
4452  * Get ASO age action by index.
4453  *
4454  * @param[in] dev
4455  *   Pointer to the Ethernet device structure.
4456  * @param[in] age_idx
4457  *   Index to the ASO age action.
4458  *
4459  * @return
4460  *   The specified ASO age action.
4461  */
4462 struct mlx5_aso_age_action*
4463 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
4464 {
4465 	uint16_t pool_idx = age_idx & UINT16_MAX;
4466 	uint16_t offset = (age_idx >> 16) & UINT16_MAX;
4467 	struct mlx5_priv *priv = dev->data->dev_private;
4468 	struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
4469 	struct mlx5_aso_age_pool *pool;
4470 
4471 	rte_rwlock_read_lock(&mng->resize_rwl);
4472 	pool = mng->pools[pool_idx];
4473 	rte_rwlock_read_unlock(&mng->resize_rwl);
4474 	return &pool->actions[offset - 1];
4475 }
4476 
4477 /* maps indirect action to translated direct in some actions array */
4478 struct mlx5_translated_action_handle {
4479 	struct rte_flow_action_handle *action; /**< Indirect action handle. */
4480 	int index; /**< Index in related array of rte_flow_action. */
4481 };
4482 
4483 /**
4484  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
4485  * direct action if translation possible.
4486  * This functionality used to run same execution path for both direct and
4487  * indirect actions on flow create. All necessary preparations for indirect
4488  * action handling should be performed on *handle* actions list returned
4489  * from this call.
4490  *
4491  * @param[in] dev
4492  *   Pointer to Ethernet device.
4493  * @param[in] actions
4494  *   List of actions to translate.
4495  * @param[out] handle
4496  *   List to store translated indirect action object handles.
4497  * @param[in, out] indir_n
4498  *   Size of *handle* array. On return should be updated with number of
4499  *   indirect actions retrieved from the *actions* list.
4500  * @param[out] translated_actions
4501  *   List of actions where all indirect actions were translated to direct
4502  *   if possible. NULL if no translation took place.
4503  * @param[out] error
4504  *   Pointer to the error structure.
4505  *
4506  * @return
4507  *   0 on success, a negative errno value otherwise and rte_errno is set.
4508  */
4509 static int
4510 flow_action_handles_translate(struct rte_eth_dev *dev,
4511 			      const struct rte_flow_action actions[],
4512 			      struct mlx5_translated_action_handle *handle,
4513 			      int *indir_n,
4514 			      struct rte_flow_action **translated_actions,
4515 			      struct rte_flow_error *error)
4516 {
4517 	struct mlx5_priv *priv = dev->data->dev_private;
4518 	struct rte_flow_action *translated = NULL;
4519 	size_t actions_size;
4520 	int n;
4521 	int copied_n = 0;
4522 	struct mlx5_translated_action_handle *handle_end = NULL;
4523 
4524 	for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
4525 		if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
4526 			continue;
4527 		if (copied_n == *indir_n) {
4528 			return rte_flow_error_set
4529 				(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
4530 				 NULL, "too many shared actions");
4531 		}
4532 		rte_memcpy(&handle[copied_n].action, &actions[n].conf,
4533 			   sizeof(actions[n].conf));
4534 		handle[copied_n].index = n;
4535 		copied_n++;
4536 	}
4537 	n++;
4538 	*indir_n = copied_n;
4539 	if (!copied_n)
4540 		return 0;
4541 	actions_size = sizeof(struct rte_flow_action) * n;
4542 	translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
4543 	if (!translated) {
4544 		rte_errno = ENOMEM;
4545 		return -ENOMEM;
4546 	}
4547 	memcpy(translated, actions, actions_size);
4548 	for (handle_end = handle + copied_n; handle < handle_end; handle++) {
4549 		struct mlx5_shared_action_rss *shared_rss;
4550 		uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
4551 		uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
4552 		uint32_t idx = act_idx &
4553 			       ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
4554 
4555 		switch (type) {
4556 		case MLX5_INDIRECT_ACTION_TYPE_RSS:
4557 			shared_rss = mlx5_ipool_get
4558 			  (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
4559 			translated[handle->index].type =
4560 				RTE_FLOW_ACTION_TYPE_RSS;
4561 			translated[handle->index].conf =
4562 				&shared_rss->origin;
4563 			break;
4564 		case MLX5_INDIRECT_ACTION_TYPE_COUNT:
4565 			translated[handle->index].type =
4566 						(enum rte_flow_action_type)
4567 						MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
4568 			translated[handle->index].conf = (void *)(uintptr_t)idx;
4569 			break;
4570 		case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
4571 			translated[handle->index].type =
4572 						(enum rte_flow_action_type)
4573 						MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK;
4574 			translated[handle->index].conf = (void *)(uintptr_t)idx;
4575 			break;
4576 		case MLX5_INDIRECT_ACTION_TYPE_AGE:
4577 			if (priv->sh->flow_hit_aso_en) {
4578 				translated[handle->index].type =
4579 					(enum rte_flow_action_type)
4580 					MLX5_RTE_FLOW_ACTION_TYPE_AGE;
4581 				translated[handle->index].conf =
4582 							 (void *)(uintptr_t)idx;
4583 				break;
4584 			}
4585 			/* Fall-through */
4586 		case MLX5_INDIRECT_ACTION_TYPE_CT:
4587 			if (priv->sh->ct_aso_en) {
4588 				translated[handle->index].type =
4589 					RTE_FLOW_ACTION_TYPE_CONNTRACK;
4590 				translated[handle->index].conf =
4591 							 (void *)(uintptr_t)idx;
4592 				break;
4593 			}
4594 			/* Fall-through */
4595 		default:
4596 			mlx5_free(translated);
4597 			return rte_flow_error_set
4598 				(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
4599 				 NULL, "invalid indirect action type");
4600 		}
4601 	}
4602 	*translated_actions = translated;
4603 	return 0;
4604 }
4605 
4606 /**
4607  * Get Shared RSS action from the action list.
4608  *
4609  * @param[in] dev
4610  *   Pointer to Ethernet device.
4611  * @param[in] shared
4612  *   Pointer to the list of actions.
4613  * @param[in] shared_n
4614  *   Actions list length.
4615  *
4616  * @return
4617  *   The MLX5 RSS action ID if exists, otherwise return 0.
4618  */
4619 static uint32_t
4620 flow_get_shared_rss_action(struct rte_eth_dev *dev,
4621 			   struct mlx5_translated_action_handle *handle,
4622 			   int shared_n)
4623 {
4624 	struct mlx5_translated_action_handle *handle_end;
4625 	struct mlx5_priv *priv = dev->data->dev_private;
4626 	struct mlx5_shared_action_rss *shared_rss;
4627 
4628 
4629 	for (handle_end = handle + shared_n; handle < handle_end; handle++) {
4630 		uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
4631 		uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
4632 		uint32_t idx = act_idx &
4633 			       ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
4634 		switch (type) {
4635 		case MLX5_INDIRECT_ACTION_TYPE_RSS:
4636 			shared_rss = mlx5_ipool_get
4637 				(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
4638 									   idx);
4639 			rte_atomic_fetch_add_explicit(&shared_rss->refcnt, 1,
4640 					   rte_memory_order_relaxed);
4641 			return idx;
4642 		default:
4643 			break;
4644 		}
4645 	}
4646 	return 0;
4647 }
4648 
4649 static unsigned int
4650 find_graph_root(uint32_t rss_level)
4651 {
4652 	return rss_level < 2 ? MLX5_EXPANSION_ROOT :
4653 			       MLX5_EXPANSION_ROOT_OUTER;
4654 }
4655 
4656 /**
4657  *  Get layer flags from the prefix flow.
4658  *
4659  *  Some flows may be split to several subflows, the prefix subflow gets the
4660  *  match items and the suffix sub flow gets the actions.
4661  *  Some actions need the user defined match item flags to get the detail for
4662  *  the action.
4663  *  This function helps the suffix flow to get the item layer flags from prefix
4664  *  subflow.
4665  *
4666  * @param[in] dev_flow
4667  *   Pointer the created prefix subflow.
4668  *
4669  * @return
4670  *   The layers get from prefix subflow.
4671  */
4672 static inline uint64_t
4673 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
4674 {
4675 	uint64_t layers = 0;
4676 
4677 	/*
4678 	 * Layers bits could be localization, but usually the compiler will
4679 	 * help to do the optimization work for source code.
4680 	 * If no decap actions, use the layers directly.
4681 	 */
4682 	if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
4683 		return dev_flow->handle->layers;
4684 	/* Convert L3 layers with decap action. */
4685 	if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
4686 		layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4687 	else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
4688 		layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4689 	/* Convert L4 layers with decap action.  */
4690 	if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
4691 		layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
4692 	else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
4693 		layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
4694 	return layers;
4695 }
4696 
4697 /**
4698  * Get metadata split action information.
4699  *
4700  * @param[in] actions
4701  *   Pointer to the list of actions.
4702  * @param[out] qrss
4703  *   Pointer to the return pointer.
4704  * @param[out] qrss_type
4705  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
4706  *   if no QUEUE/RSS is found.
4707  * @param[out] encap_idx
4708  *   Pointer to the index of the encap action if exists, otherwise the last
4709  *   action index.
4710  *
4711  * @return
4712  *   Total number of actions.
4713  */
4714 static int
4715 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
4716 				       const struct rte_flow_action **qrss,
4717 				       int *encap_idx)
4718 {
4719 	const struct rte_flow_action_raw_encap *raw_encap;
4720 	int actions_n = 0;
4721 	int raw_decap_idx = -1;
4722 
4723 	*encap_idx = -1;
4724 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4725 		switch (actions->type) {
4726 		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4727 		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4728 			*encap_idx = actions_n;
4729 			break;
4730 		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4731 			raw_decap_idx = actions_n;
4732 			break;
4733 		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4734 			raw_encap = actions->conf;
4735 			if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4736 				*encap_idx = raw_decap_idx != -1 ?
4737 						      raw_decap_idx : actions_n;
4738 			break;
4739 		case RTE_FLOW_ACTION_TYPE_QUEUE:
4740 		case RTE_FLOW_ACTION_TYPE_RSS:
4741 			*qrss = actions;
4742 			break;
4743 		default:
4744 			break;
4745 		}
4746 		actions_n++;
4747 	}
4748 	if (*encap_idx == -1)
4749 		*encap_idx = actions_n;
4750 	/* Count RTE_FLOW_ACTION_TYPE_END. */
4751 	return actions_n + 1;
4752 }
4753 
4754 /**
4755  * Check if the action will change packet.
4756  *
4757  * @param dev
4758  *   Pointer to Ethernet device.
4759  * @param[in] type
4760  *   action type.
4761  *
4762  * @return
4763  *   true if action will change packet, false otherwise.
4764  */
4765 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
4766 					  enum rte_flow_action_type type)
4767 {
4768 	struct mlx5_priv *priv = dev->data->dev_private;
4769 
4770 	switch (type) {
4771 	case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4772 	case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4773 	case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4774 	case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4775 	case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4776 	case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4777 	case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
4778 	case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
4779 	case RTE_FLOW_ACTION_TYPE_DEC_TTL:
4780 	case RTE_FLOW_ACTION_TYPE_SET_TTL:
4781 	case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
4782 	case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
4783 	case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4784 	case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4785 	case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
4786 	case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
4787 	case RTE_FLOW_ACTION_TYPE_SET_META:
4788 	case RTE_FLOW_ACTION_TYPE_SET_TAG:
4789 	case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
4790 	case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4791 	case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4792 	case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4793 	case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4794 	case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4795 	case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4796 	case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4797 	case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4798 	case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4799 	case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
4800 		return true;
4801 	case RTE_FLOW_ACTION_TYPE_FLAG:
4802 	case RTE_FLOW_ACTION_TYPE_MARK:
4803 		if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
4804 		    priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS)
4805 			return true;
4806 		else
4807 			return false;
4808 	default:
4809 		return false;
4810 	}
4811 }
4812 
4813 /**
4814  * Check meter action from the action list.
4815  *
4816  * @param dev
4817  *   Pointer to Ethernet device.
4818  * @param[in] actions
4819  *   Pointer to the list of actions.
4820  * @param[out] has_mtr
4821  *   Pointer to the meter exist flag.
4822  * @param[out] has_modify
4823  *   Pointer to the flag showing there's packet change action.
4824  * @param[out] meter_id
4825  *   Pointer to the meter id.
4826  *
4827  * @return
4828  *   Total number of actions.
4829  */
4830 static int
4831 flow_check_meter_action(struct rte_eth_dev *dev,
4832 			const struct rte_flow_action actions[],
4833 			bool *has_mtr, bool *has_modify, uint32_t *meter_id)
4834 {
4835 	const struct rte_flow_action_meter *mtr = NULL;
4836 	int actions_n = 0;
4837 
4838 	MLX5_ASSERT(has_mtr);
4839 	*has_mtr = false;
4840 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4841 		switch (actions->type) {
4842 		case RTE_FLOW_ACTION_TYPE_METER:
4843 			mtr = actions->conf;
4844 			*meter_id = mtr->mtr_id;
4845 			*has_mtr = true;
4846 			break;
4847 		default:
4848 			break;
4849 		}
4850 		if (!*has_mtr)
4851 			*has_modify |= flow_check_modify_action_type(dev,
4852 								actions->type);
4853 		actions_n++;
4854 	}
4855 	/* Count RTE_FLOW_ACTION_TYPE_END. */
4856 	return actions_n + 1;
4857 }
4858 
4859 /**
4860  * Check if the flow should be split due to hairpin.
4861  * The reason for the split is that in current HW we can't
4862  * support encap and push-vlan on Rx, so if a flow contains
4863  * these actions we move it to Tx.
4864  *
4865  * @param dev
4866  *   Pointer to Ethernet device.
4867  * @param[in] attr
4868  *   Flow rule attributes.
4869  * @param[in] actions
4870  *   Associated actions (list terminated by the END action).
4871  *
4872  * @return
4873  *   > 0 the number of actions and the flow should be split,
4874  *   0 when no split required.
4875  */
4876 static int
4877 flow_check_hairpin_split(struct rte_eth_dev *dev,
4878 			 const struct rte_flow_attr *attr,
4879 			 const struct rte_flow_action actions[])
4880 {
4881 	int queue_action = 0;
4882 	int action_n = 0;
4883 	int split = 0;
4884 	int push_vlan = 0;
4885 	const struct rte_flow_action_queue *queue;
4886 	const struct rte_flow_action_rss *rss;
4887 	const struct rte_flow_action_raw_encap *raw_encap;
4888 	const struct rte_eth_hairpin_conf *conf;
4889 
4890 	if (!attr->ingress)
4891 		return 0;
4892 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4893 		if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN)
4894 			push_vlan = 1;
4895 		switch (actions->type) {
4896 		case RTE_FLOW_ACTION_TYPE_QUEUE:
4897 			queue = actions->conf;
4898 			if (queue == NULL)
4899 				return 0;
4900 			conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
4901 			if (conf == NULL || conf->tx_explicit != 0)
4902 				return 0;
4903 			queue_action = 1;
4904 			action_n++;
4905 			break;
4906 		case RTE_FLOW_ACTION_TYPE_RSS:
4907 			rss = actions->conf;
4908 			if (rss == NULL || rss->queue_num == 0)
4909 				return 0;
4910 			conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
4911 			if (conf == NULL || conf->tx_explicit != 0)
4912 				return 0;
4913 			queue_action = 1;
4914 			action_n++;
4915 			break;
4916 		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4917 		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4918 		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4919 		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4920 			split++;
4921 			action_n++;
4922 			break;
4923 		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4924 			if (push_vlan)
4925 				split++;
4926 			action_n++;
4927 			break;
4928 		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4929 			raw_encap = actions->conf;
4930 			if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4931 				split++;
4932 			action_n++;
4933 			break;
4934 		default:
4935 			action_n++;
4936 			break;
4937 		}
4938 	}
4939 	if (split && queue_action)
4940 		return action_n;
4941 	return 0;
4942 }
4943 
4944 /* Declare flow create/destroy prototype in advance. */
4945 static uint32_t
4946 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4947 		 const struct rte_flow_attr *attr,
4948 		 const struct rte_flow_item items[],
4949 		 const struct rte_flow_action actions[],
4950 		 bool external, struct rte_flow_error *error);
4951 
4952 static void
4953 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4954 		  uint32_t flow_idx);
4955 
4956 int
4957 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
4958 		      struct mlx5_list_entry *entry, void *cb_ctx)
4959 {
4960 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4961 	struct mlx5_flow_mreg_copy_resource *mcp_res =
4962 			       container_of(entry, typeof(*mcp_res), hlist_ent);
4963 
4964 	return mcp_res->mark_id != *(uint32_t *)(ctx->data);
4965 }
4966 
4967 struct mlx5_list_entry *
4968 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
4969 {
4970 	struct rte_eth_dev *dev = tool_ctx;
4971 	struct mlx5_priv *priv = dev->data->dev_private;
4972 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4973 	struct mlx5_flow_mreg_copy_resource *mcp_res;
4974 	struct rte_flow_error *error = ctx->error;
4975 	uint32_t idx = 0;
4976 	int ret;
4977 	uint32_t mark_id = *(uint32_t *)(ctx->data);
4978 	struct rte_flow_attr attr = {
4979 		.group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4980 		.ingress = 1,
4981 	};
4982 	struct mlx5_rte_flow_item_tag tag_spec = {
4983 		.data = mark_id,
4984 	};
4985 	struct rte_flow_item items[] = {
4986 		[1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4987 	};
4988 	struct rte_flow_action_mark ftag = {
4989 		.id = mark_id,
4990 	};
4991 	struct mlx5_flow_action_copy_mreg cp_mreg = {
4992 		.dst = REG_B,
4993 		.src = REG_NON,
4994 	};
4995 	struct rte_flow_action_jump jump = {
4996 		.group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4997 	};
4998 	struct rte_flow_action actions[] = {
4999 		[3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
5000 	};
5001 
5002 	/* Fill the register fields in the flow. */
5003 	ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
5004 	if (ret < 0)
5005 		return NULL;
5006 	tag_spec.id = ret;
5007 	ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5008 	if (ret < 0)
5009 		return NULL;
5010 	cp_mreg.src = ret;
5011 	/* Provide the full width of FLAG specific value. */
5012 	if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
5013 		tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
5014 	/* Build a new flow. */
5015 	if (mark_id != MLX5_DEFAULT_COPY_ID) {
5016 		items[0] = (struct rte_flow_item){
5017 			.type = (enum rte_flow_item_type)
5018 				MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5019 			.spec = &tag_spec,
5020 		};
5021 		items[1] = (struct rte_flow_item){
5022 			.type = RTE_FLOW_ITEM_TYPE_END,
5023 		};
5024 		actions[0] = (struct rte_flow_action){
5025 			.type = (enum rte_flow_action_type)
5026 				MLX5_RTE_FLOW_ACTION_TYPE_MARK,
5027 			.conf = &ftag,
5028 		};
5029 		actions[1] = (struct rte_flow_action){
5030 			.type = (enum rte_flow_action_type)
5031 				MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5032 			.conf = &cp_mreg,
5033 		};
5034 		actions[2] = (struct rte_flow_action){
5035 			.type = RTE_FLOW_ACTION_TYPE_JUMP,
5036 			.conf = &jump,
5037 		};
5038 		actions[3] = (struct rte_flow_action){
5039 			.type = RTE_FLOW_ACTION_TYPE_END,
5040 		};
5041 	} else {
5042 		/* Default rule, wildcard match. */
5043 		attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
5044 		items[0] = (struct rte_flow_item){
5045 			.type = RTE_FLOW_ITEM_TYPE_END,
5046 		};
5047 		actions[0] = (struct rte_flow_action){
5048 			.type = (enum rte_flow_action_type)
5049 				MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5050 			.conf = &cp_mreg,
5051 		};
5052 		actions[1] = (struct rte_flow_action){
5053 			.type = RTE_FLOW_ACTION_TYPE_JUMP,
5054 			.conf = &jump,
5055 		};
5056 		actions[2] = (struct rte_flow_action){
5057 			.type = RTE_FLOW_ACTION_TYPE_END,
5058 		};
5059 	}
5060 	/* Build a new entry. */
5061 	mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
5062 	if (!mcp_res) {
5063 		rte_errno = ENOMEM;
5064 		return NULL;
5065 	}
5066 	mcp_res->idx = idx;
5067 	mcp_res->mark_id = mark_id;
5068 	/*
5069 	 * The copy Flows are not included in any list. There
5070 	 * ones are referenced from other Flows and can not
5071 	 * be applied, removed, deleted in arbitrary order
5072 	 * by list traversing.
5073 	 */
5074 	mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
5075 					&attr, items, actions, false, error);
5076 	if (!mcp_res->rix_flow) {
5077 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
5078 		return NULL;
5079 	}
5080 	return &mcp_res->hlist_ent;
5081 }
5082 
5083 struct mlx5_list_entry *
5084 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
5085 		      void *cb_ctx __rte_unused)
5086 {
5087 	struct rte_eth_dev *dev = tool_ctx;
5088 	struct mlx5_priv *priv = dev->data->dev_private;
5089 	struct mlx5_flow_mreg_copy_resource *mcp_res;
5090 	uint32_t idx = 0;
5091 
5092 	mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
5093 	if (!mcp_res) {
5094 		rte_errno = ENOMEM;
5095 		return NULL;
5096 	}
5097 	memcpy(mcp_res, oentry, sizeof(*mcp_res));
5098 	mcp_res->idx = idx;
5099 	return &mcp_res->hlist_ent;
5100 }
5101 
5102 void
5103 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
5104 {
5105 	struct mlx5_flow_mreg_copy_resource *mcp_res =
5106 			       container_of(entry, typeof(*mcp_res), hlist_ent);
5107 	struct rte_eth_dev *dev = tool_ctx;
5108 	struct mlx5_priv *priv = dev->data->dev_private;
5109 
5110 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
5111 }
5112 
5113 /**
5114  * Add a flow of copying flow metadata registers in RX_CP_TBL.
5115  *
5116  * As mark_id is unique, if there's already a registered flow for the mark_id,
5117  * return by increasing the reference counter of the resource. Otherwise, create
5118  * the resource (mcp_res) and flow.
5119  *
5120  * Flow looks like,
5121  *   - If ingress port is ANY and reg_c[1] is mark_id,
5122  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
5123  *
5124  * For default flow (zero mark_id), flow is like,
5125  *   - If ingress port is ANY,
5126  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
5127  *
5128  * @param dev
5129  *   Pointer to Ethernet device.
5130  * @param mark_id
5131  *   ID of MARK action, zero means default flow for META.
5132  * @param[out] error
5133  *   Perform verbose error reporting if not NULL.
5134  *
5135  * @return
5136  *   Associated resource on success, NULL otherwise and rte_errno is set.
5137  */
5138 static struct mlx5_flow_mreg_copy_resource *
5139 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
5140 			  struct rte_flow_error *error)
5141 {
5142 	struct mlx5_priv *priv = dev->data->dev_private;
5143 	struct mlx5_list_entry *entry;
5144 	struct mlx5_flow_cb_ctx ctx = {
5145 		.dev = dev,
5146 		.error = error,
5147 		.data = &mark_id,
5148 	};
5149 
5150 	/* Check if already registered. */
5151 	MLX5_ASSERT(priv->mreg_cp_tbl);
5152 	entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
5153 	if (!entry)
5154 		return NULL;
5155 	return container_of(entry, struct mlx5_flow_mreg_copy_resource,
5156 			    hlist_ent);
5157 }
5158 
5159 void
5160 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
5161 {
5162 	struct mlx5_flow_mreg_copy_resource *mcp_res =
5163 			       container_of(entry, typeof(*mcp_res), hlist_ent);
5164 	struct rte_eth_dev *dev = tool_ctx;
5165 	struct mlx5_priv *priv = dev->data->dev_private;
5166 
5167 	MLX5_ASSERT(mcp_res->rix_flow);
5168 	flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
5169 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
5170 }
5171 
5172 /**
5173  * Release flow in RX_CP_TBL.
5174  *
5175  * @param dev
5176  *   Pointer to Ethernet device.
5177  * @flow
5178  *   Parent flow for wich copying is provided.
5179  */
5180 static void
5181 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
5182 			  struct rte_flow *flow)
5183 {
5184 	struct mlx5_flow_mreg_copy_resource *mcp_res;
5185 	struct mlx5_priv *priv = dev->data->dev_private;
5186 
5187 	if (!flow->rix_mreg_copy)
5188 		return;
5189 	mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
5190 				 flow->rix_mreg_copy);
5191 	if (!mcp_res || !priv->mreg_cp_tbl)
5192 		return;
5193 	MLX5_ASSERT(mcp_res->rix_flow);
5194 	mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
5195 	flow->rix_mreg_copy = 0;
5196 }
5197 
5198 /**
5199  * Remove the default copy action from RX_CP_TBL.
5200  *
5201  * This functions is called in the mlx5_dev_start(). No thread safe
5202  * is guaranteed.
5203  *
5204  * @param dev
5205  *   Pointer to Ethernet device.
5206  */
5207 static void
5208 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
5209 {
5210 	struct mlx5_list_entry *entry;
5211 	struct mlx5_priv *priv = dev->data->dev_private;
5212 	struct mlx5_flow_cb_ctx ctx;
5213 	uint32_t mark_id;
5214 
5215 	/* Check if default flow is registered. */
5216 	if (!priv->mreg_cp_tbl)
5217 		return;
5218 	mark_id = MLX5_DEFAULT_COPY_ID;
5219 	ctx.data = &mark_id;
5220 	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
5221 	if (!entry)
5222 		return;
5223 	mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
5224 }
5225 
5226 /**
5227  * Add the default copy action in RX_CP_TBL.
5228  *
5229  * This functions is called in the mlx5_dev_start(). No thread safe
5230  * is guaranteed.
5231  *
5232  * @param dev
5233  *   Pointer to Ethernet device.
5234  * @param[out] error
5235  *   Perform verbose error reporting if not NULL.
5236  *
5237  * @return
5238  *   0 for success, negative value otherwise and rte_errno is set.
5239  */
5240 static int
5241 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
5242 				  struct rte_flow_error *error)
5243 {
5244 	struct mlx5_priv *priv = dev->data->dev_private;
5245 	struct mlx5_flow_mreg_copy_resource *mcp_res;
5246 	struct mlx5_flow_cb_ctx ctx;
5247 	uint32_t mark_id;
5248 
5249 	/* Check whether extensive metadata feature is engaged. */
5250 	if (!priv->sh->config.dv_flow_en ||
5251 	    priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5252 	    !mlx5_flow_ext_mreg_supported(dev) ||
5253 	    !priv->sh->dv_regc0_mask)
5254 		return 0;
5255 	/*
5256 	 * Add default mreg copy flow may be called multiple time, but
5257 	 * only be called once in stop. Avoid register it twice.
5258 	 */
5259 	mark_id = MLX5_DEFAULT_COPY_ID;
5260 	ctx.data = &mark_id;
5261 	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
5262 		return 0;
5263 	mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
5264 	if (!mcp_res)
5265 		return -rte_errno;
5266 	return 0;
5267 }
5268 
5269 /**
5270  * Add a flow of copying flow metadata registers in RX_CP_TBL.
5271  *
5272  * All the flow having Q/RSS action should be split by
5273  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
5274  * performs the following,
5275  *   - CQE->flow_tag := reg_c[1] (MARK)
5276  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
5277  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
5278  * but there should be a flow per each MARK ID set by MARK action.
5279  *
5280  * For the aforementioned reason, if there's a MARK action in flow's action
5281  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
5282  * the MARK ID to CQE's flow_tag like,
5283  *   - If reg_c[1] is mark_id,
5284  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
5285  *
5286  * For SET_META action which stores value in reg_c[0], as the destination is
5287  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
5288  * MARK ID means the default flow. The default flow looks like,
5289  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
5290  *
5291  * @param dev
5292  *   Pointer to Ethernet device.
5293  * @param flow
5294  *   Pointer to flow structure.
5295  * @param[in] actions
5296  *   Pointer to the list of actions.
5297  * @param[out] error
5298  *   Perform verbose error reporting if not NULL.
5299  *
5300  * @return
5301  *   0 on success, negative value otherwise and rte_errno is set.
5302  */
5303 static int
5304 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
5305 			    struct rte_flow *flow,
5306 			    const struct rte_flow_action *actions,
5307 			    struct rte_flow_error *error)
5308 {
5309 	struct mlx5_priv *priv = dev->data->dev_private;
5310 	struct mlx5_sh_config *config = &priv->sh->config;
5311 	struct mlx5_flow_mreg_copy_resource *mcp_res;
5312 	const struct rte_flow_action_mark *mark;
5313 
5314 	/* Check whether extensive metadata feature is engaged. */
5315 	if (!config->dv_flow_en ||
5316 	    config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5317 	    !mlx5_flow_ext_mreg_supported(dev) ||
5318 	    !priv->sh->dv_regc0_mask)
5319 		return 0;
5320 	/* Find MARK action. */
5321 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5322 		switch (actions->type) {
5323 		case RTE_FLOW_ACTION_TYPE_FLAG:
5324 			mcp_res = flow_mreg_add_copy_action
5325 				(dev, MLX5_FLOW_MARK_DEFAULT, error);
5326 			if (!mcp_res)
5327 				return -rte_errno;
5328 			flow->rix_mreg_copy = mcp_res->idx;
5329 			return 0;
5330 		case RTE_FLOW_ACTION_TYPE_MARK:
5331 			mark = (const struct rte_flow_action_mark *)
5332 				actions->conf;
5333 			mcp_res =
5334 				flow_mreg_add_copy_action(dev, mark->id, error);
5335 			if (!mcp_res)
5336 				return -rte_errno;
5337 			flow->rix_mreg_copy = mcp_res->idx;
5338 			return 0;
5339 		default:
5340 			break;
5341 		}
5342 	}
5343 	return 0;
5344 }
5345 
5346 #define MLX5_MAX_SPLIT_ACTIONS 24
5347 #define MLX5_MAX_SPLIT_ITEMS 24
5348 
5349 /**
5350  * Split the hairpin flow.
5351  * Since HW can't support encap and push-vlan on Rx, we move these
5352  * actions to Tx.
5353  * If the count action is after the encap then we also
5354  * move the count action. in this case the count will also measure
5355  * the outer bytes.
5356  *
5357  * @param dev
5358  *   Pointer to Ethernet device.
5359  * @param[in] actions
5360  *   Associated actions (list terminated by the END action).
5361  * @param[out] actions_rx
5362  *   Rx flow actions.
5363  * @param[out] actions_tx
5364  *   Tx flow actions..
5365  * @param[out] pattern_tx
5366  *   The pattern items for the Tx flow.
5367  * @param[out] flow_id
5368  *   The flow ID connected to this flow.
5369  *
5370  * @return
5371  *   0 on success.
5372  */
5373 static int
5374 flow_hairpin_split(struct rte_eth_dev *dev,
5375 		   const struct rte_flow_action actions[],
5376 		   struct rte_flow_action actions_rx[],
5377 		   struct rte_flow_action actions_tx[],
5378 		   struct rte_flow_item pattern_tx[],
5379 		   uint32_t flow_id)
5380 {
5381 	const struct rte_flow_action_raw_encap *raw_encap;
5382 	const struct rte_flow_action_raw_decap *raw_decap;
5383 	struct mlx5_rte_flow_action_set_tag *set_tag;
5384 	struct rte_flow_action *tag_action;
5385 	struct mlx5_rte_flow_item_tag *tag_item;
5386 	struct rte_flow_item *item;
5387 	char *addr;
5388 	int push_vlan = 0;
5389 	int encap = 0;
5390 
5391 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5392 		if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN)
5393 			push_vlan = 1;
5394 		switch (actions->type) {
5395 		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5396 		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5397 		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5398 		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5399 			rte_memcpy(actions_tx, actions,
5400 			       sizeof(struct rte_flow_action));
5401 			actions_tx++;
5402 			break;
5403 		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5404 			if (push_vlan) {
5405 				rte_memcpy(actions_tx, actions,
5406 					   sizeof(struct rte_flow_action));
5407 				actions_tx++;
5408 			} else {
5409 				rte_memcpy(actions_rx, actions,
5410 					   sizeof(struct rte_flow_action));
5411 				actions_rx++;
5412 			}
5413 			break;
5414 		case RTE_FLOW_ACTION_TYPE_COUNT:
5415 		case RTE_FLOW_ACTION_TYPE_AGE:
5416 			if (encap) {
5417 				rte_memcpy(actions_tx, actions,
5418 					   sizeof(struct rte_flow_action));
5419 				actions_tx++;
5420 			} else {
5421 				rte_memcpy(actions_rx, actions,
5422 					   sizeof(struct rte_flow_action));
5423 				actions_rx++;
5424 			}
5425 			break;
5426 		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5427 			raw_encap = actions->conf;
5428 			if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
5429 				memcpy(actions_tx, actions,
5430 				       sizeof(struct rte_flow_action));
5431 				actions_tx++;
5432 				encap = 1;
5433 			} else {
5434 				rte_memcpy(actions_rx, actions,
5435 					   sizeof(struct rte_flow_action));
5436 				actions_rx++;
5437 			}
5438 			break;
5439 		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5440 			raw_decap = actions->conf;
5441 			if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
5442 				memcpy(actions_tx, actions,
5443 				       sizeof(struct rte_flow_action));
5444 				actions_tx++;
5445 			} else {
5446 				rte_memcpy(actions_rx, actions,
5447 					   sizeof(struct rte_flow_action));
5448 				actions_rx++;
5449 			}
5450 			break;
5451 		default:
5452 			rte_memcpy(actions_rx, actions,
5453 				   sizeof(struct rte_flow_action));
5454 			actions_rx++;
5455 			break;
5456 		}
5457 	}
5458 	/* Add set meta action and end action for the Rx flow. */
5459 	tag_action = actions_rx;
5460 	tag_action->type = (enum rte_flow_action_type)
5461 			   MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5462 	actions_rx++;
5463 	rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
5464 	actions_rx++;
5465 	set_tag = (void *)actions_rx;
5466 	*set_tag = (struct mlx5_rte_flow_action_set_tag) {
5467 		.id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
5468 		.data = flow_id,
5469 	};
5470 	MLX5_ASSERT(set_tag->id > REG_NON);
5471 	tag_action->conf = set_tag;
5472 	/* Create Tx item list. */
5473 	rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
5474 	addr = (void *)&pattern_tx[2];
5475 	item = pattern_tx;
5476 	item->type = (enum rte_flow_item_type)
5477 		     MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5478 	tag_item = (void *)addr;
5479 	tag_item->data = flow_id;
5480 	tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
5481 	MLX5_ASSERT(set_tag->id > REG_NON);
5482 	item->spec = tag_item;
5483 	addr += sizeof(struct mlx5_rte_flow_item_tag);
5484 	tag_item = (void *)addr;
5485 	tag_item->data = UINT32_MAX;
5486 	tag_item->id = UINT16_MAX;
5487 	item->mask = tag_item;
5488 	item->last = NULL;
5489 	item++;
5490 	item->type = RTE_FLOW_ITEM_TYPE_END;
5491 	return 0;
5492 }
5493 
5494 /**
5495  * The last stage of splitting chain, just creates the subflow
5496  * without any modification.
5497  *
5498  * @param[in] dev
5499  *   Pointer to Ethernet device.
5500  * @param[in] flow
5501  *   Parent flow structure pointer.
5502  * @param[in, out] sub_flow
5503  *   Pointer to return the created subflow, may be NULL.
5504  * @param[in] attr
5505  *   Flow rule attributes.
5506  * @param[in] items
5507  *   Pattern specification (list terminated by the END pattern item).
5508  * @param[in] actions
5509  *   Associated actions (list terminated by the END action).
5510  * @param[in] flow_split_info
5511  *   Pointer to flow split info structure.
5512  * @param[out] error
5513  *   Perform verbose error reporting if not NULL.
5514  * @return
5515  *   0 on success, negative value otherwise
5516  */
5517 static int
5518 flow_create_split_inner(struct rte_eth_dev *dev,
5519 			struct rte_flow *flow,
5520 			struct mlx5_flow **sub_flow,
5521 			const struct rte_flow_attr *attr,
5522 			const struct rte_flow_item items[],
5523 			const struct rte_flow_action actions[],
5524 			struct mlx5_flow_split_info *flow_split_info,
5525 			struct rte_flow_error *error)
5526 {
5527 	struct mlx5_flow *dev_flow;
5528 	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5529 
5530 	dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
5531 				    flow_split_info->flow_idx, error);
5532 	if (!dev_flow)
5533 		return -rte_errno;
5534 	dev_flow->flow = flow;
5535 	dev_flow->external = flow_split_info->external;
5536 	dev_flow->skip_scale = flow_split_info->skip_scale;
5537 	/* Subflow object was created, we must include one in the list. */
5538 	SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5539 		      dev_flow->handle, next);
5540 	/*
5541 	 * If dev_flow is as one of the suffix flow, some actions in suffix
5542 	 * flow may need some user defined item layer flags, and pass the
5543 	 * Metadata rxq mark flag to suffix flow as well.
5544 	 */
5545 	if (flow_split_info->prefix_layers)
5546 		dev_flow->handle->layers = flow_split_info->prefix_layers;
5547 	if (flow_split_info->prefix_mark) {
5548 		MLX5_ASSERT(wks);
5549 		wks->mark = 1;
5550 	}
5551 	if (sub_flow)
5552 		*sub_flow = dev_flow;
5553 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5554 	dev_flow->dv.table_id = flow_split_info->table_id;
5555 #endif
5556 	return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
5557 }
5558 
5559 /**
5560  * Get the sub policy of a meter.
5561  *
5562  * @param[in] dev
5563  *   Pointer to Ethernet device.
5564  * @param[in] flow
5565  *   Parent flow structure pointer.
5566  * @param wks
5567  *   Pointer to thread flow work space.
5568  * @param[in] attr
5569  *   Flow rule attributes.
5570  * @param[in] items
5571  *   Pattern specification (list terminated by the END pattern item).
5572  * @param[out] error
5573  *   Perform verbose error reporting if not NULL.
5574  *
5575  * @return
5576  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
5577  */
5578 static struct mlx5_flow_meter_sub_policy *
5579 get_meter_sub_policy(struct rte_eth_dev *dev,
5580 		     struct rte_flow *flow,
5581 		     struct mlx5_flow_workspace *wks,
5582 		     const struct rte_flow_attr *attr,
5583 		     const struct rte_flow_item items[],
5584 		     struct rte_flow_error *error)
5585 {
5586 	struct mlx5_flow_meter_policy *policy;
5587 	struct mlx5_flow_meter_policy *final_policy;
5588 	struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
5589 
5590 	policy = wks->policy;
5591 	final_policy = policy->is_hierarchy ? wks->final_policy : policy;
5592 	if (final_policy->is_rss || final_policy->is_queue) {
5593 		struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
5594 		struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
5595 		uint32_t i;
5596 
5597 		/*
5598 		 * This is a tmp dev_flow,
5599 		 * no need to register any matcher for it in translate.
5600 		 */
5601 		wks->skip_matcher_reg = 1;
5602 		for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
5603 			struct mlx5_flow dev_flow = {0};
5604 			struct mlx5_flow_handle dev_handle = { {0} };
5605 			uint8_t fate = final_policy->act_cnt[i].fate_action;
5606 
5607 			if (fate == MLX5_FLOW_FATE_SHARED_RSS) {
5608 				const struct rte_flow_action_rss *rss_act =
5609 					final_policy->act_cnt[i].rss->conf;
5610 				struct rte_flow_action rss_actions[2] = {
5611 					[0] = {
5612 					.type = RTE_FLOW_ACTION_TYPE_RSS,
5613 					.conf = rss_act,
5614 					},
5615 					[1] = {
5616 					.type = RTE_FLOW_ACTION_TYPE_END,
5617 					.conf = NULL,
5618 					}
5619 				};
5620 
5621 				dev_flow.handle = &dev_handle;
5622 				dev_flow.ingress = attr->ingress;
5623 				dev_flow.flow = flow;
5624 				dev_flow.external = 0;
5625 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5626 				dev_flow.dv.transfer = attr->transfer;
5627 #endif
5628 				/**
5629 				 * Translate RSS action to get rss hash fields.
5630 				 */
5631 				if (flow_drv_translate(dev, &dev_flow, attr,
5632 						items, rss_actions, error))
5633 					goto exit;
5634 				rss_desc_v[i] = wks->rss_desc;
5635 				rss_desc_v[i].symmetric_hash_function =
5636 						dev_flow.symmetric_hash_function;
5637 				rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
5638 				rss_desc_v[i].hash_fields =
5639 						dev_flow.hash_fields;
5640 				rss_desc_v[i].queue_num =
5641 						rss_desc_v[i].hash_fields ?
5642 						rss_desc_v[i].queue_num : 1;
5643 				rss_desc_v[i].tunnel =
5644 						!!(dev_flow.handle->layers &
5645 						   MLX5_FLOW_LAYER_TUNNEL);
5646 				/* Use the RSS queues in the containers. */
5647 				rss_desc_v[i].queue =
5648 					(uint16_t *)(uintptr_t)rss_act->queue;
5649 				rss_desc[i] = &rss_desc_v[i];
5650 			} else if (fate == MLX5_FLOW_FATE_QUEUE) {
5651 				/* This is queue action. */
5652 				rss_desc_v[i] = wks->rss_desc;
5653 				rss_desc_v[i].key_len = 0;
5654 				rss_desc_v[i].hash_fields = 0;
5655 				rss_desc_v[i].queue =
5656 					&final_policy->act_cnt[i].queue;
5657 				rss_desc_v[i].queue_num = 1;
5658 				rss_desc[i] = &rss_desc_v[i];
5659 			} else {
5660 				rss_desc[i] = NULL;
5661 			}
5662 		}
5663 		sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
5664 						flow, policy, rss_desc);
5665 	} else {
5666 		enum mlx5_meter_domain mtr_domain =
5667 			attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5668 				(attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5669 						MLX5_MTR_DOMAIN_INGRESS);
5670 		sub_policy = policy->sub_policys[mtr_domain][0];
5671 	}
5672 	if (!sub_policy)
5673 		rte_flow_error_set(error, EINVAL,
5674 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5675 				   "Failed to get meter sub-policy.");
5676 exit:
5677 	return sub_policy;
5678 }
5679 
5680 /**
5681  * Split the meter flow.
5682  *
5683  * As meter flow will split to three sub flow, other than meter
5684  * action, the other actions make sense to only meter accepts
5685  * the packet. If it need to be dropped, no other additional
5686  * actions should be take.
5687  *
5688  * One kind of special action which decapsulates the L3 tunnel
5689  * header will be in the prefix sub flow, as not to take the
5690  * L3 tunnel header into account.
5691  *
5692  * @param[in] dev
5693  *   Pointer to Ethernet device.
5694  * @param[in] flow
5695  *   Parent flow structure pointer.
5696  * @param wks
5697  *   Pointer to thread flow work space.
5698  * @param[in] attr
5699  *   Flow rule attributes.
5700  * @param[in] items
5701  *   Pattern specification (list terminated by the END pattern item).
5702  * @param[out] sfx_items
5703  *   Suffix flow match items (list terminated by the END pattern item).
5704  * @param[in] actions
5705  *   Associated actions (list terminated by the END action).
5706  * @param[out] actions_sfx
5707  *   Suffix flow actions.
5708  * @param[out] actions_pre
5709  *   Prefix flow actions.
5710  * @param[out] mtr_flow_id
5711  *   Pointer to meter flow id.
5712  * @param[out] error
5713  *   Perform verbose error reporting if not NULL.
5714  *
5715  * @return
5716  *   0 on success, a negative errno value otherwise and rte_errno is set.
5717  */
5718 static int
5719 flow_meter_split_prep(struct rte_eth_dev *dev,
5720 		      struct rte_flow *flow,
5721 		      struct mlx5_flow_workspace *wks,
5722 		      const struct rte_flow_attr *attr,
5723 		      const struct rte_flow_item items[],
5724 		      struct rte_flow_item sfx_items[],
5725 		      const struct rte_flow_action actions[],
5726 		      struct rte_flow_action actions_sfx[],
5727 		      struct rte_flow_action actions_pre[],
5728 		      uint32_t *mtr_flow_id,
5729 		      struct rte_flow_error *error)
5730 {
5731 	struct mlx5_priv *priv = dev->data->dev_private;
5732 	struct mlx5_flow_meter_info *fm = wks->fm;
5733 	struct rte_flow_action *tag_action = NULL;
5734 	struct rte_flow_item *tag_item;
5735 	struct mlx5_rte_flow_action_set_tag *set_tag;
5736 	const struct rte_flow_action_raw_encap *raw_encap;
5737 	const struct rte_flow_action_raw_decap *raw_decap;
5738 	struct mlx5_rte_flow_item_tag *tag_item_spec;
5739 	struct mlx5_rte_flow_item_tag *tag_item_mask;
5740 	uint32_t tag_id = 0;
5741 	bool vlan_actions;
5742 	struct rte_flow_item *orig_sfx_items = sfx_items;
5743 	const struct rte_flow_item *orig_items = items;
5744 	struct rte_flow_action *hw_mtr_action;
5745 	struct rte_flow_action *action_pre_head = NULL;
5746 	uint16_t flow_src_port = priv->representor_id;
5747 	bool mtr_first;
5748 	uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
5749 	uint8_t mtr_reg_bits = priv->mtr_reg_share ?
5750 				MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
5751 	uint32_t flow_id = 0;
5752 	uint32_t flow_id_reversed = 0;
5753 	uint8_t flow_id_bits = 0;
5754 	bool after_meter = false;
5755 	int shift;
5756 
5757 	/* Prepare the suffix subflow items. */
5758 	tag_item = sfx_items++;
5759 	tag_item->type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5760 	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5761 		int item_type = items->type;
5762 
5763 		switch (item_type) {
5764 		case RTE_FLOW_ITEM_TYPE_PORT_ID:
5765 		case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT:
5766 		case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR:
5767 			if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error))
5768 				return -rte_errno;
5769 			if (!fm->def_policy && wks->policy->hierarchy_match_port &&
5770 			    flow_src_port != priv->representor_id) {
5771 				if (flow_drv_mtr_hierarchy_rule_create(dev,
5772 								flow, fm,
5773 								flow_src_port,
5774 								items,
5775 								error))
5776 					return -rte_errno;
5777 			}
5778 			memcpy(sfx_items, items, sizeof(*sfx_items));
5779 			sfx_items++;
5780 			break;
5781 		case RTE_FLOW_ITEM_TYPE_VLAN:
5782 			/*
5783 			 * Copy VLAN items in case VLAN actions are performed.
5784 			 * If there are no VLAN actions, these items will be VOID.
5785 			 */
5786 			memcpy(sfx_items, items, sizeof(*sfx_items));
5787 			sfx_items->type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
5788 			sfx_items++;
5789 			break;
5790 		default:
5791 			break;
5792 		}
5793 	}
5794 	sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
5795 	sfx_items++;
5796 	mtr_first = priv->sh->meter_aso_en &&
5797 		(attr->egress || (attr->transfer && flow_src_port != UINT16_MAX));
5798 	/* For ASO meter, meter must be before tag in TX direction. */
5799 	if (mtr_first) {
5800 		action_pre_head = actions_pre++;
5801 		/* Leave space for tag action. */
5802 		tag_action = actions_pre++;
5803 	}
5804 	/* Prepare the actions for prefix and suffix flow. */
5805 	vlan_actions = false;
5806 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5807 		struct rte_flow_action *action_cur = NULL;
5808 
5809 		switch (actions->type) {
5810 		case RTE_FLOW_ACTION_TYPE_METER:
5811 			if (mtr_first) {
5812 				action_cur = action_pre_head;
5813 			} else {
5814 				/* Leave space for tag action. */
5815 				tag_action = actions_pre++;
5816 				action_cur = actions_pre++;
5817 			}
5818 			after_meter = true;
5819 			break;
5820 		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5821 		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5822 			action_cur = actions_pre++;
5823 			break;
5824 		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5825 			raw_encap = actions->conf;
5826 			if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
5827 				action_cur = actions_pre++;
5828 			break;
5829 		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5830 			raw_decap = actions->conf;
5831 			if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
5832 				action_cur = actions_pre++;
5833 			break;
5834 		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5835 		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5836 			vlan_actions = true;
5837 			break;
5838 		case RTE_FLOW_ACTION_TYPE_COUNT:
5839 			if (fm->def_policy)
5840 				action_cur = after_meter ?
5841 						actions_sfx++ : actions_pre++;
5842 			break;
5843 		default:
5844 			break;
5845 		}
5846 		if (!action_cur)
5847 			action_cur = (fm->def_policy) ?
5848 					actions_sfx++ : actions_pre++;
5849 		memcpy(action_cur, actions, sizeof(struct rte_flow_action));
5850 	}
5851 	/* If there are no VLAN actions, convert VLAN items to VOID in suffix flow items. */
5852 	if (!vlan_actions) {
5853 		struct rte_flow_item *it = orig_sfx_items;
5854 
5855 		for (; it->type != RTE_FLOW_ITEM_TYPE_END; it++)
5856 			if (it->type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
5857 				it->type = RTE_FLOW_ITEM_TYPE_VOID;
5858 	}
5859 	/* Add end action to the actions. */
5860 	actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
5861 	if (priv->sh->meter_aso_en) {
5862 		/**
5863 		 * For ASO meter, need to add an extra jump action explicitly,
5864 		 * to jump from meter to policer table.
5865 		 */
5866 		struct mlx5_flow_meter_sub_policy *sub_policy;
5867 		struct mlx5_flow_tbl_data_entry *tbl_data;
5868 
5869 		if (!fm->def_policy) {
5870 			sub_policy = get_meter_sub_policy(dev, flow, wks,
5871 							  attr, orig_items,
5872 							  error);
5873 			if (!sub_policy)
5874 				return -rte_errno;
5875 		} else {
5876 			enum mlx5_meter_domain mtr_domain =
5877 			attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5878 				(attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5879 						MLX5_MTR_DOMAIN_INGRESS);
5880 
5881 			sub_policy =
5882 			&priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
5883 		}
5884 		tbl_data = container_of(sub_policy->tbl_rsc,
5885 					struct mlx5_flow_tbl_data_entry, tbl);
5886 		hw_mtr_action = actions_pre++;
5887 		hw_mtr_action->type = (enum rte_flow_action_type)
5888 				      MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
5889 		hw_mtr_action->conf = tbl_data->jump.action;
5890 	}
5891 	actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
5892 	actions_pre++;
5893 	if (!tag_action)
5894 		return rte_flow_error_set(error, ENOMEM,
5895 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5896 					  NULL, "No tag action space.");
5897 	if (!mtr_flow_id) {
5898 		tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
5899 		goto exit;
5900 	}
5901 	/* Only default-policy Meter creates mtr flow id. */
5902 	if (fm->def_policy) {
5903 		mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
5904 		if (!tag_id)
5905 			return rte_flow_error_set(error, ENOMEM,
5906 					RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5907 					"Failed to allocate meter flow id.");
5908 		flow_id = tag_id - 1;
5909 		flow_id_bits = (!flow_id) ? 1 :
5910 				(MLX5_REG_BITS - rte_clz32(flow_id));
5911 		if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
5912 		    mtr_reg_bits) {
5913 			mlx5_ipool_free(fm->flow_ipool, tag_id);
5914 			return rte_flow_error_set(error, EINVAL,
5915 					RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5916 					"Meter flow id exceeds max limit.");
5917 		}
5918 		if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
5919 			priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
5920 	}
5921 	/* Build tag actions and items for meter_id/meter flow_id. */
5922 	set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
5923 	tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
5924 	tag_item_mask = tag_item_spec + 1;
5925 	/* Both flow_id and meter_id share the same register. */
5926 	*set_tag = (struct mlx5_rte_flow_action_set_tag) {
5927 		.id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
5928 							    0, error),
5929 		.offset = mtr_id_offset,
5930 		.length = mtr_reg_bits,
5931 		.data = flow->meter,
5932 	};
5933 	/*
5934 	 * The color Reg bits used by flow_id are growing from
5935 	 * msb to lsb, so must do bit reverse for flow_id val in RegC.
5936 	 */
5937 	for (shift = 0; shift < flow_id_bits; shift++)
5938 		flow_id_reversed = (flow_id_reversed << 1) |
5939 				((flow_id >> shift) & 0x1);
5940 	set_tag->data |=
5941 		flow_id_reversed << (mtr_reg_bits - flow_id_bits);
5942 	tag_item_spec->id = set_tag->id;
5943 	tag_item_spec->data = set_tag->data << mtr_id_offset;
5944 	tag_item_mask->data = UINT32_MAX << mtr_id_offset;
5945 	tag_action->type = (enum rte_flow_action_type)
5946 				MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5947 	tag_action->conf = set_tag;
5948 	tag_item->spec = tag_item_spec;
5949 	tag_item->last = NULL;
5950 	tag_item->mask = tag_item_mask;
5951 exit:
5952 	if (mtr_flow_id)
5953 		*mtr_flow_id = tag_id;
5954 	return 0;
5955 }
5956 
5957 /**
5958  * Split action list having QUEUE/RSS for metadata register copy.
5959  *
5960  * Once Q/RSS action is detected in user's action list, the flow action
5961  * should be split in order to copy metadata registers, which will happen in
5962  * RX_CP_TBL like,
5963  *   - CQE->flow_tag := reg_c[1] (MARK)
5964  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
5965  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
5966  * This is because the last action of each flow must be a terminal action
5967  * (QUEUE, RSS or DROP).
5968  *
5969  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
5970  * stored and kept in the mlx5_flow structure per each sub_flow.
5971  *
5972  * The Q/RSS action is replaced with,
5973  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
5974  * And the following JUMP action is added at the end,
5975  *   - JUMP, to RX_CP_TBL.
5976  *
5977  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
5978  * flow_create_split_metadata() routine. The flow will look like,
5979  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
5980  *
5981  * @param dev
5982  *   Pointer to Ethernet device.
5983  * @param[out] split_actions
5984  *   Pointer to store split actions to jump to CP_TBL.
5985  * @param[in] actions
5986  *   Pointer to the list of original flow actions.
5987  * @param[in] qrss
5988  *   Pointer to the Q/RSS action.
5989  * @param[in] actions_n
5990  *   Number of original actions.
5991  * @param[in] mtr_sfx
5992  *   Check if it is in meter suffix table.
5993  * @param[out] error
5994  *   Perform verbose error reporting if not NULL.
5995  *
5996  * @return
5997  *   non-zero unique flow_id on success, otherwise 0 and
5998  *   error/rte_error are set.
5999  */
6000 static uint32_t
6001 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
6002 			  struct rte_flow_action *split_actions,
6003 			  const struct rte_flow_action *actions,
6004 			  const struct rte_flow_action *qrss,
6005 			  int actions_n, int mtr_sfx,
6006 			  struct rte_flow_error *error)
6007 {
6008 	struct mlx5_priv *priv = dev->data->dev_private;
6009 	struct mlx5_rte_flow_action_set_tag *set_tag;
6010 	struct rte_flow_action_jump *jump;
6011 	const int qrss_idx = qrss - actions;
6012 	uint32_t flow_id = 0;
6013 	int ret = 0;
6014 
6015 	/*
6016 	 * Given actions will be split
6017 	 * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
6018 	 * - Add jump to mreg CP_TBL.
6019 	 * As a result, there will be one more action.
6020 	 */
6021 	memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
6022 	/* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */
6023 	++actions_n;
6024 	set_tag = (void *)(split_actions + actions_n);
6025 	/*
6026 	 * If we are not the meter suffix flow, add the tag action.
6027 	 * Since meter suffix flow already has the tag added.
6028 	 */
6029 	if (!mtr_sfx) {
6030 		/*
6031 		 * Allocate the new subflow ID. This one is unique within
6032 		 * device and not shared with representors. Otherwise,
6033 		 * we would have to resolve multi-thread access synch
6034 		 * issue. Each flow on the shared device is appended
6035 		 * with source vport identifier, so the resulting
6036 		 * flows will be unique in the shared (by master and
6037 		 * representors) domain even if they have coinciding
6038 		 * IDs.
6039 		 */
6040 		mlx5_ipool_malloc(priv->sh->ipool
6041 				  [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
6042 		if (!flow_id)
6043 			return rte_flow_error_set(error, ENOMEM,
6044 						  RTE_FLOW_ERROR_TYPE_ACTION,
6045 						  NULL, "can't allocate id "
6046 						  "for split Q/RSS subflow");
6047 		/* Internal SET_TAG action to set flow ID. */
6048 		*set_tag = (struct mlx5_rte_flow_action_set_tag){
6049 			.data = flow_id,
6050 		};
6051 		ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
6052 		if (ret < 0)
6053 			return ret;
6054 		set_tag->id = ret;
6055 		/* Construct new actions array. */
6056 		/* Replace QUEUE/RSS action. */
6057 		split_actions[qrss_idx] = (struct rte_flow_action){
6058 			.type = (enum rte_flow_action_type)
6059 				MLX5_RTE_FLOW_ACTION_TYPE_TAG,
6060 			.conf = set_tag,
6061 		};
6062 	} else {
6063 		/*
6064 		 * If we are the suffix flow of meter, tag already exist.
6065 		 * Set the QUEUE/RSS action to void.
6066 		 */
6067 		split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID;
6068 	}
6069 	/* JUMP action to jump to mreg copy table (CP_TBL). */
6070 	jump = (void *)(set_tag + 1);
6071 	*jump = (struct rte_flow_action_jump){
6072 		.group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
6073 	};
6074 	split_actions[actions_n - 2] = (struct rte_flow_action){
6075 		.type = RTE_FLOW_ACTION_TYPE_JUMP,
6076 		.conf = jump,
6077 	};
6078 	split_actions[actions_n - 1] = (struct rte_flow_action){
6079 		.type = RTE_FLOW_ACTION_TYPE_END,
6080 	};
6081 	return flow_id;
6082 }
6083 
6084 /**
6085  * Extend the given action list for Tx metadata copy.
6086  *
6087  * Copy the given action list to the ext_actions and add flow metadata register
6088  * copy action in order to copy reg_a set by WQE to reg_c[0].
6089  *
6090  * @param[out] ext_actions
6091  *   Pointer to the extended action list.
6092  * @param[in] actions
6093  *   Pointer to the list of actions.
6094  * @param[in] actions_n
6095  *   Number of actions in the list.
6096  * @param[out] error
6097  *   Perform verbose error reporting if not NULL.
6098  * @param[in] encap_idx
6099  *   The encap action index.
6100  *
6101  * @return
6102  *   0 on success, negative value otherwise
6103  */
6104 static int
6105 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
6106 		       struct rte_flow_action *ext_actions,
6107 		       const struct rte_flow_action *actions,
6108 		       int actions_n, struct rte_flow_error *error,
6109 		       int encap_idx)
6110 {
6111 	struct mlx5_flow_action_copy_mreg *cp_mreg =
6112 		(struct mlx5_flow_action_copy_mreg *)
6113 			(ext_actions + actions_n + 1);
6114 	int ret;
6115 
6116 	ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
6117 	if (ret < 0)
6118 		return ret;
6119 	cp_mreg->dst = ret;
6120 	ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
6121 	if (ret < 0)
6122 		return ret;
6123 	cp_mreg->src = ret;
6124 	if (encap_idx != 0)
6125 		memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
6126 	if (encap_idx == actions_n - 1) {
6127 		ext_actions[actions_n - 1] = (struct rte_flow_action){
6128 			.type = (enum rte_flow_action_type)
6129 				MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6130 			.conf = cp_mreg,
6131 		};
6132 		ext_actions[actions_n] = (struct rte_flow_action){
6133 			.type = RTE_FLOW_ACTION_TYPE_END,
6134 		};
6135 	} else {
6136 		ext_actions[encap_idx] = (struct rte_flow_action){
6137 			.type = (enum rte_flow_action_type)
6138 				MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
6139 			.conf = cp_mreg,
6140 		};
6141 		memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
6142 				sizeof(*ext_actions) * (actions_n - encap_idx));
6143 	}
6144 	return 0;
6145 }
6146 
6147 /**
6148  * Check the match action from the action list.
6149  *
6150  * @param[in] actions
6151  *   Pointer to the list of actions.
6152  * @param[in] attr
6153  *   Flow rule attributes.
6154  * @param[in] action
6155  *   The action to be check if exist.
6156  * @param[out] match_action_pos
6157  *   Pointer to the position of the matched action if exists, otherwise is -1.
6158  * @param[out] qrss_action_pos
6159  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
6160  * @param[out] modify_after_mirror
6161  *   Pointer to the flag of modify action after FDB mirroring.
6162  *
6163  * @return
6164  *   > 0 the total number of actions.
6165  *   0 if not found match action in action list.
6166  */
6167 static int
6168 flow_check_match_action(const struct rte_flow_action actions[],
6169 			const struct rte_flow_attr *attr,
6170 			enum rte_flow_action_type action,
6171 			int *match_action_pos, int *qrss_action_pos,
6172 			int *modify_after_mirror)
6173 {
6174 	const struct rte_flow_action_sample *sample;
6175 	const struct rte_flow_action_raw_decap *decap;
6176 	const struct rte_flow_action *action_cur = NULL;
6177 	int actions_n = 0;
6178 	uint32_t ratio = 0;
6179 	int sub_type = 0;
6180 	int flag = 0;
6181 	int fdb_mirror = 0;
6182 
6183 	*match_action_pos = -1;
6184 	*qrss_action_pos = -1;
6185 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
6186 		if (actions->type == action) {
6187 			flag = 1;
6188 			*match_action_pos = actions_n;
6189 		}
6190 		switch (actions->type) {
6191 		case RTE_FLOW_ACTION_TYPE_QUEUE:
6192 		case RTE_FLOW_ACTION_TYPE_RSS:
6193 			*qrss_action_pos = actions_n;
6194 			break;
6195 		case RTE_FLOW_ACTION_TYPE_SAMPLE:
6196 			sample = actions->conf;
6197 			ratio = sample->ratio;
6198 			sub_type = ((const struct rte_flow_action *)
6199 					(sample->actions))->type;
6200 			if (ratio == 1 && attr->transfer &&
6201 			    sub_type != RTE_FLOW_ACTION_TYPE_END)
6202 				fdb_mirror = 1;
6203 			break;
6204 		case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
6205 		case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
6206 		case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
6207 		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
6208 		case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
6209 		case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
6210 		case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
6211 		case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
6212 		case RTE_FLOW_ACTION_TYPE_DEC_TTL:
6213 		case RTE_FLOW_ACTION_TYPE_SET_TTL:
6214 		case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
6215 		case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
6216 		case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
6217 		case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
6218 		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
6219 		case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
6220 		case RTE_FLOW_ACTION_TYPE_FLAG:
6221 		case RTE_FLOW_ACTION_TYPE_MARK:
6222 		case RTE_FLOW_ACTION_TYPE_SET_META:
6223 		case RTE_FLOW_ACTION_TYPE_SET_TAG:
6224 		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
6225 		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
6226 		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
6227 		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
6228 		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
6229 		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
6230 		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
6231 		case RTE_FLOW_ACTION_TYPE_METER:
6232 			if (fdb_mirror)
6233 				*modify_after_mirror = 1;
6234 			break;
6235 		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
6236 			decap = actions->conf;
6237 			action_cur = actions;
6238 			while ((++action_cur)->type == RTE_FLOW_ACTION_TYPE_VOID)
6239 				;
6240 			if (action_cur->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
6241 				const struct rte_flow_action_raw_encap *encap =
6242 								action_cur->conf;
6243 				if (decap->size <=
6244 					MLX5_ENCAPSULATION_DECISION_SIZE &&
6245 				    encap->size >
6246 					MLX5_ENCAPSULATION_DECISION_SIZE)
6247 					/* L3 encap. */
6248 					break;
6249 			}
6250 			if (fdb_mirror)
6251 				*modify_after_mirror = 1;
6252 			break;
6253 		default:
6254 			break;
6255 		}
6256 		actions_n++;
6257 	}
6258 	if (flag && fdb_mirror && !*modify_after_mirror) {
6259 		/* FDB mirroring uses the destination array to implement
6260 		 * instead of FLOW_SAMPLER object.
6261 		 */
6262 		if (sub_type != RTE_FLOW_ACTION_TYPE_END)
6263 			flag = 0;
6264 	}
6265 	/* Count RTE_FLOW_ACTION_TYPE_END. */
6266 	return flag ? actions_n + 1 : 0;
6267 }
6268 
6269 #define SAMPLE_SUFFIX_ITEM 3
6270 
6271 /**
6272  * Split the sample flow.
6273  *
6274  * As sample flow will split to two sub flow, sample flow with
6275  * sample action, the other actions will move to new suffix flow.
6276  *
6277  * Also add unique tag id with tag action in the sample flow,
6278  * the same tag id will be as match in the suffix flow.
6279  *
6280  * @param dev
6281  *   Pointer to Ethernet device.
6282  * @param[in] add_tag
6283  *   Add extra tag action flag.
6284  * @param[out] sfx_items
6285  *   Suffix flow match items (list terminated by the END pattern item).
6286  * @param[in] actions
6287  *   Associated actions (list terminated by the END action).
6288  * @param[out] actions_sfx
6289  *   Suffix flow actions.
6290  * @param[out] actions_pre
6291  *   Prefix flow actions.
6292  * @param[in] actions_n
6293  *  The total number of actions.
6294  * @param[in] sample_action_pos
6295  *   The sample action position.
6296  * @param[in] qrss_action_pos
6297  *   The Queue/RSS action position.
6298  * @param[in] jump_table
6299  *   Add extra jump action flag.
6300  * @param[out] error
6301  *   Perform verbose error reporting if not NULL.
6302  *
6303  * @return
6304  *   0 on success, or unique flow_id, a negative errno value
6305  *   otherwise and rte_errno is set.
6306  */
6307 static int
6308 flow_sample_split_prep(struct rte_eth_dev *dev,
6309 		       int add_tag,
6310 		       const struct rte_flow_item items[],
6311 		       struct rte_flow_item sfx_items[],
6312 		       const struct rte_flow_action actions[],
6313 		       struct rte_flow_action actions_sfx[],
6314 		       struct rte_flow_action actions_pre[],
6315 		       int actions_n,
6316 		       int sample_action_pos,
6317 		       int qrss_action_pos,
6318 		       int jump_table,
6319 		       struct rte_flow_error *error)
6320 {
6321 	struct mlx5_priv *priv = dev->data->dev_private;
6322 	struct mlx5_rte_flow_action_set_tag *set_tag;
6323 	struct mlx5_rte_flow_item_tag *tag_spec;
6324 	struct mlx5_rte_flow_item_tag *tag_mask;
6325 	struct rte_flow_action_jump *jump_action;
6326 	uint32_t tag_id = 0;
6327 	int append_index = 0;
6328 	int set_tag_idx = -1;
6329 	int index;
6330 	int ret;
6331 
6332 	if (sample_action_pos < 0)
6333 		return rte_flow_error_set(error, EINVAL,
6334 					  RTE_FLOW_ERROR_TYPE_ACTION,
6335 					  NULL, "invalid position of sample "
6336 					  "action in list");
6337 	/* Prepare the actions for prefix and suffix flow. */
6338 	if (add_tag) {
6339 		/* Update the new added tag action index preceding
6340 		 * the PUSH_VLAN or ENCAP action.
6341 		 */
6342 		const struct rte_flow_action_raw_encap *raw_encap;
6343 		const struct rte_flow_action *action = actions;
6344 		int encap_idx;
6345 		int action_idx = 0;
6346 		int raw_decap_idx = -1;
6347 		int push_vlan_idx = -1;
6348 		for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) {
6349 			switch (action->type) {
6350 			case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
6351 				raw_decap_idx = action_idx;
6352 				break;
6353 			case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
6354 				raw_encap = action->conf;
6355 				if (raw_encap->size >
6356 					MLX5_ENCAPSULATION_DECISION_SIZE) {
6357 					encap_idx = raw_decap_idx != -1 ?
6358 						    raw_decap_idx : action_idx;
6359 					if (encap_idx < sample_action_pos &&
6360 					    push_vlan_idx == -1)
6361 						set_tag_idx = encap_idx;
6362 				}
6363 				break;
6364 			case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
6365 			case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
6366 				encap_idx = action_idx;
6367 				if (encap_idx < sample_action_pos &&
6368 				    push_vlan_idx == -1)
6369 					set_tag_idx = encap_idx;
6370 				break;
6371 			case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
6372 			case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
6373 				if (action_idx < sample_action_pos &&
6374 				    push_vlan_idx == -1) {
6375 					set_tag_idx = action_idx;
6376 					push_vlan_idx = action_idx;
6377 				}
6378 				break;
6379 			default:
6380 				break;
6381 			}
6382 			action_idx++;
6383 		}
6384 	}
6385 	/* Prepare the actions for prefix and suffix flow. */
6386 	if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
6387 		index = qrss_action_pos;
6388 		/* Put the preceding the Queue/RSS action into prefix flow. */
6389 		if (index != 0)
6390 			memcpy(actions_pre, actions,
6391 			       sizeof(struct rte_flow_action) * index);
6392 		/* Put others preceding the sample action into prefix flow. */
6393 		if (sample_action_pos > index + 1)
6394 			memcpy(actions_pre + index, actions + index + 1,
6395 			       sizeof(struct rte_flow_action) *
6396 			       (sample_action_pos - index - 1));
6397 		index = sample_action_pos - 1;
6398 		/* Put Queue/RSS action into Suffix flow. */
6399 		memcpy(actions_sfx, actions + qrss_action_pos,
6400 		       sizeof(struct rte_flow_action));
6401 		actions_sfx++;
6402 	} else if (add_tag && set_tag_idx >= 0) {
6403 		if (set_tag_idx > 0)
6404 			memcpy(actions_pre, actions,
6405 			       sizeof(struct rte_flow_action) * set_tag_idx);
6406 		memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx,
6407 		       sizeof(struct rte_flow_action) *
6408 		       (sample_action_pos - set_tag_idx));
6409 		index = sample_action_pos;
6410 	} else {
6411 		index = sample_action_pos;
6412 		if (index != 0)
6413 			memcpy(actions_pre, actions,
6414 			       sizeof(struct rte_flow_action) * index);
6415 	}
6416 	/* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
6417 	 * For CX6DX and above, metadata registers Cx preserve their value,
6418 	 * add an extra tag action for NIC-RX and E-Switch Domain.
6419 	 */
6420 	if (add_tag) {
6421 		/* Prepare the prefix tag action. */
6422 		append_index++;
6423 		set_tag = (void *)(actions_pre + actions_n + append_index);
6424 		/* Trust VF/SF on CX5 not supported meter so that the reserved
6425 		 * metadata regC is REG_NON, back to use application tag
6426 		 * index 0.
6427 		 */
6428 		if (unlikely(priv->sh->registers.aso_reg == REG_NON))
6429 			ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
6430 		else
6431 			ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error);
6432 		if (ret < 0)
6433 			return ret;
6434 		mlx5_ipool_malloc(priv->sh->ipool
6435 				  [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
6436 		*set_tag = (struct mlx5_rte_flow_action_set_tag) {
6437 			.id = ret,
6438 			.data = tag_id,
6439 		};
6440 		/* Prepare the suffix subflow items. */
6441 		tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
6442 		tag_spec->data = tag_id;
6443 		tag_spec->id = set_tag->id;
6444 		tag_mask = tag_spec + 1;
6445 		tag_mask->data = UINT32_MAX;
6446 		for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6447 			if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID ||
6448 			    items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR ||
6449 			    items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) {
6450 				memcpy(sfx_items, items, sizeof(*sfx_items));
6451 				sfx_items++;
6452 				break;
6453 			}
6454 		}
6455 		sfx_items[0] = (struct rte_flow_item){
6456 			.type = (enum rte_flow_item_type)
6457 				MLX5_RTE_FLOW_ITEM_TYPE_TAG,
6458 			.spec = tag_spec,
6459 			.last = NULL,
6460 			.mask = tag_mask,
6461 		};
6462 		sfx_items[1] = (struct rte_flow_item){
6463 			.type = (enum rte_flow_item_type)
6464 				RTE_FLOW_ITEM_TYPE_END,
6465 		};
6466 		/* Prepare the tag action in prefix subflow. */
6467 		set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx;
6468 		actions_pre[set_tag_idx] =
6469 			(struct rte_flow_action){
6470 			.type = (enum rte_flow_action_type)
6471 				MLX5_RTE_FLOW_ACTION_TYPE_TAG,
6472 			.conf = set_tag,
6473 		};
6474 		/* Update next sample position due to add one tag action */
6475 		index += 1;
6476 	}
6477 	/* Copy the sample action into prefix flow. */
6478 	memcpy(actions_pre + index, actions + sample_action_pos,
6479 	       sizeof(struct rte_flow_action));
6480 	index += 1;
6481 	/* For the modify action after the sample action in E-Switch mirroring,
6482 	 * Add the extra jump action in prefix subflow and jump into the next
6483 	 * table, then do the modify action in the new table.
6484 	 */
6485 	if (jump_table) {
6486 		/* Prepare the prefix jump action. */
6487 		append_index++;
6488 		jump_action = (void *)(actions_pre + actions_n + append_index);
6489 		jump_action->group = jump_table;
6490 		actions_pre[index++] =
6491 			(struct rte_flow_action){
6492 			.type = (enum rte_flow_action_type)
6493 				RTE_FLOW_ACTION_TYPE_JUMP,
6494 			.conf = jump_action,
6495 		};
6496 	}
6497 	actions_pre[index] = (struct rte_flow_action){
6498 		.type = (enum rte_flow_action_type)
6499 			RTE_FLOW_ACTION_TYPE_END,
6500 	};
6501 	/* Put the actions after sample into Suffix flow. */
6502 	memcpy(actions_sfx, actions + sample_action_pos + 1,
6503 	       sizeof(struct rte_flow_action) *
6504 	       (actions_n - sample_action_pos - 1));
6505 	return tag_id;
6506 }
6507 
6508 /**
6509  * The splitting for metadata feature.
6510  *
6511  * - Q/RSS action on NIC Rx should be split in order to pass by
6512  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
6513  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
6514  *
6515  * - All the actions on NIC Tx should have a mreg copy action to
6516  *   copy reg_a from WQE to reg_c[0].
6517  *
6518  * @param dev
6519  *   Pointer to Ethernet device.
6520  * @param[in] flow
6521  *   Parent flow structure pointer.
6522  * @param[in] attr
6523  *   Flow rule attributes.
6524  * @param[in] items
6525  *   Pattern specification (list terminated by the END pattern item).
6526  * @param[in] actions
6527  *   Associated actions (list terminated by the END action).
6528  * @param[in] flow_split_info
6529  *   Pointer to flow split info structure.
6530  * @param[out] error
6531  *   Perform verbose error reporting if not NULL.
6532  * @return
6533  *   0 on success, negative value otherwise
6534  */
6535 static int
6536 flow_create_split_metadata(struct rte_eth_dev *dev,
6537 			   struct rte_flow *flow,
6538 			   const struct rte_flow_attr *attr,
6539 			   const struct rte_flow_item items[],
6540 			   const struct rte_flow_action actions[],
6541 			   struct mlx5_flow_split_info *flow_split_info,
6542 			   struct rte_flow_error *error)
6543 {
6544 	struct mlx5_priv *priv = dev->data->dev_private;
6545 	struct mlx5_sh_config *config = &priv->sh->config;
6546 	const struct rte_flow_action *qrss = NULL;
6547 	struct rte_flow_action *ext_actions = NULL;
6548 	struct mlx5_flow *dev_flow = NULL;
6549 	uint32_t qrss_id = 0;
6550 	int mtr_sfx = 0;
6551 	size_t act_size;
6552 	int actions_n;
6553 	int encap_idx;
6554 	int ret;
6555 
6556 	/* Check whether extensive metadata feature is engaged. */
6557 	if (!config->dv_flow_en ||
6558 	    config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
6559 	    !mlx5_flow_ext_mreg_supported(dev))
6560 		return flow_create_split_inner(dev, flow, NULL, attr, items,
6561 					       actions, flow_split_info, error);
6562 	actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
6563 							   &encap_idx);
6564 	if (qrss) {
6565 		/* Exclude hairpin flows from splitting. */
6566 		if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
6567 			const struct rte_flow_action_queue *queue;
6568 
6569 			queue = qrss->conf;
6570 			if (mlx5_rxq_is_hairpin(dev, queue->index))
6571 				qrss = NULL;
6572 		} else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
6573 			const struct rte_flow_action_rss *rss;
6574 
6575 			rss = qrss->conf;
6576 			if (mlx5_rxq_is_hairpin(dev, rss->queue[0]))
6577 				qrss = NULL;
6578 		}
6579 	}
6580 	if (qrss) {
6581 		/* Check if it is in meter suffix table. */
6582 		mtr_sfx = attr->group ==
6583 			  ((attr->transfer && priv->fdb_def_rule) ?
6584 			  (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6585 			  MLX5_FLOW_TABLE_LEVEL_METER);
6586 		/*
6587 		 * Q/RSS action on NIC Rx should be split in order to pass by
6588 		 * the mreg copy table (RX_CP_TBL) and then it jumps to the
6589 		 * action table (RX_ACT_TBL) which has the split Q/RSS action.
6590 		 */
6591 		act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
6592 			   sizeof(struct rte_flow_action_set_tag) +
6593 			   sizeof(struct rte_flow_action_jump);
6594 		ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
6595 					  SOCKET_ID_ANY);
6596 		if (!ext_actions)
6597 			return rte_flow_error_set(error, ENOMEM,
6598 						  RTE_FLOW_ERROR_TYPE_ACTION,
6599 						  NULL, "no memory to split "
6600 						  "metadata flow");
6601 		/*
6602 		 * Create the new actions list with removed Q/RSS action
6603 		 * and appended set tag and jump to register copy table
6604 		 * (RX_CP_TBL). We should preallocate unique tag ID here
6605 		 * in advance, because it is needed for set tag action.
6606 		 */
6607 		qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
6608 						    qrss, actions_n,
6609 						    mtr_sfx, error);
6610 		if (!mtr_sfx && !qrss_id) {
6611 			ret = -rte_errno;
6612 			goto exit;
6613 		}
6614 	} else if (attr->egress) {
6615 		/*
6616 		 * All the actions on NIC Tx should have a metadata register
6617 		 * copy action to copy reg_a from WQE to reg_c[meta]
6618 		 */
6619 		act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
6620 			   sizeof(struct mlx5_flow_action_copy_mreg);
6621 		ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
6622 					  SOCKET_ID_ANY);
6623 		if (!ext_actions)
6624 			return rte_flow_error_set(error, ENOMEM,
6625 						  RTE_FLOW_ERROR_TYPE_ACTION,
6626 						  NULL, "no memory to split "
6627 						  "metadata flow");
6628 		/* Create the action list appended with copy register. */
6629 		ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
6630 					     actions_n, error, encap_idx);
6631 		if (ret < 0)
6632 			goto exit;
6633 	}
6634 	/* Add the unmodified original or prefix subflow. */
6635 	ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6636 				      items, ext_actions ? ext_actions :
6637 				      actions, flow_split_info, error);
6638 	if (ret < 0)
6639 		goto exit;
6640 	MLX5_ASSERT(dev_flow);
6641 	if (qrss) {
6642 		const struct rte_flow_attr q_attr = {
6643 			.group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6644 			.ingress = 1,
6645 		};
6646 		/* Internal PMD action to set register. */
6647 		struct mlx5_rte_flow_item_tag q_tag_spec = {
6648 			.data = qrss_id,
6649 			.id = REG_NON,
6650 		};
6651 		struct rte_flow_item q_items[] = {
6652 			{
6653 				.type = (enum rte_flow_item_type)
6654 					MLX5_RTE_FLOW_ITEM_TYPE_TAG,
6655 				.spec = &q_tag_spec,
6656 				.last = NULL,
6657 				.mask = NULL,
6658 			},
6659 			{
6660 				.type = RTE_FLOW_ITEM_TYPE_END,
6661 			},
6662 		};
6663 		struct rte_flow_action q_actions[] = {
6664 			{
6665 				.type = qrss->type,
6666 				.conf = qrss->conf,
6667 			},
6668 			{
6669 				.type = RTE_FLOW_ACTION_TYPE_END,
6670 			},
6671 		};
6672 		uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
6673 
6674 		/*
6675 		 * Configure the tag item only if there is no meter subflow.
6676 		 * Since tag is already marked in the meter suffix subflow
6677 		 * we can just use the meter suffix items as is.
6678 		 */
6679 		if (qrss_id) {
6680 			/* Not meter subflow. */
6681 			MLX5_ASSERT(!mtr_sfx);
6682 			/*
6683 			 * Put unique id in prefix flow due to it is destroyed
6684 			 * after suffix flow and id will be freed after there
6685 			 * is no actual flows with this id and identifier
6686 			 * reallocation becomes possible (for example, for
6687 			 * other flows in other threads).
6688 			 */
6689 			dev_flow->handle->split_flow_id = qrss_id;
6690 			ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
6691 						   error);
6692 			if (ret < 0)
6693 				goto exit;
6694 			q_tag_spec.id = ret;
6695 		}
6696 		dev_flow = NULL;
6697 		/* Add suffix subflow to execute Q/RSS. */
6698 		flow_split_info->prefix_layers = layers;
6699 		flow_split_info->prefix_mark = 0;
6700 		flow_split_info->table_id = 0;
6701 		ret = flow_create_split_inner(dev, flow, &dev_flow,
6702 					      &q_attr, mtr_sfx ? items :
6703 					      q_items, q_actions,
6704 					      flow_split_info, error);
6705 		if (ret < 0)
6706 			goto exit;
6707 		/* qrss ID should be freed if failed. */
6708 		qrss_id = 0;
6709 		MLX5_ASSERT(dev_flow);
6710 	}
6711 
6712 exit:
6713 	/*
6714 	 * We do not destroy the partially created sub_flows in case of error.
6715 	 * These ones are included into parent flow list and will be destroyed
6716 	 * by flow_drv_destroy.
6717 	 */
6718 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
6719 			qrss_id);
6720 	mlx5_free(ext_actions);
6721 	return ret;
6722 }
6723 
6724 /**
6725  * Create meter internal drop flow with the original pattern.
6726  *
6727  * @param dev
6728  *   Pointer to Ethernet device.
6729  * @param[in] flow
6730  *   Parent flow structure pointer.
6731  * @param[in] attr
6732  *   Flow rule attributes.
6733  * @param[in] items
6734  *   Pattern specification (list terminated by the END pattern item).
6735  * @param[in] flow_split_info
6736  *   Pointer to flow split info structure.
6737  * @param[in] fm
6738  *   Pointer to flow meter structure.
6739  * @param[out] error
6740  *   Perform verbose error reporting if not NULL.
6741  * @return
6742  *   0 on success, negative value otherwise
6743  */
6744 static uint32_t
6745 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
6746 			struct rte_flow *flow,
6747 			const struct rte_flow_attr *attr,
6748 			const struct rte_flow_item items[],
6749 			struct mlx5_flow_split_info *flow_split_info,
6750 			struct mlx5_flow_meter_info *fm,
6751 			struct rte_flow_error *error)
6752 {
6753 	struct mlx5_flow *dev_flow = NULL;
6754 	struct rte_flow_attr drop_attr = *attr;
6755 	struct rte_flow_action drop_actions[3];
6756 	struct mlx5_flow_split_info drop_split_info = *flow_split_info;
6757 
6758 	MLX5_ASSERT(fm->drop_cnt);
6759 	drop_actions[0].type =
6760 		(enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
6761 	drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
6762 	drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
6763 	drop_actions[1].conf = NULL;
6764 	drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
6765 	drop_actions[2].conf = NULL;
6766 	drop_split_info.external = false;
6767 	drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6768 	drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
6769 	drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
6770 	return flow_create_split_inner(dev, flow, &dev_flow,
6771 				&drop_attr, items, drop_actions,
6772 				&drop_split_info, error);
6773 }
6774 
6775 static int
6776 flow_count_vlan_items(const struct rte_flow_item items[])
6777 {
6778 	int items_n = 0;
6779 
6780 	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6781 		if (items->type == RTE_FLOW_ITEM_TYPE_VLAN ||
6782 		    items->type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
6783 			items_n++;
6784 	}
6785 	return items_n;
6786 }
6787 
6788 /**
6789  * The splitting for meter feature.
6790  *
6791  * - The meter flow will be split to two flows as prefix and
6792  *   suffix flow. The packets make sense only it pass the prefix
6793  *   meter action.
6794  *
6795  * - Reg_C_5 is used for the packet to match betweend prefix and
6796  *   suffix flow.
6797  *
6798  * @param dev
6799  *   Pointer to Ethernet device.
6800  * @param[in] flow
6801  *   Parent flow structure pointer.
6802  * @param[in] attr
6803  *   Flow rule attributes.
6804  * @param[in] items
6805  *   Pattern specification (list terminated by the END pattern item).
6806  * @param[in] actions
6807  *   Associated actions (list terminated by the END action).
6808  * @param[in] flow_split_info
6809  *   Pointer to flow split info structure.
6810  * @param[out] error
6811  *   Perform verbose error reporting if not NULL.
6812  * @return
6813  *   0 on success, negative value otherwise
6814  */
6815 static int
6816 flow_create_split_meter(struct rte_eth_dev *dev,
6817 			struct rte_flow *flow,
6818 			const struct rte_flow_attr *attr,
6819 			const struct rte_flow_item items[],
6820 			const struct rte_flow_action actions[],
6821 			struct mlx5_flow_split_info *flow_split_info,
6822 			struct rte_flow_error *error)
6823 {
6824 	struct mlx5_priv *priv = dev->data->dev_private;
6825 	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
6826 	struct rte_flow_action *sfx_actions = NULL;
6827 	struct rte_flow_action *pre_actions = NULL;
6828 	struct rte_flow_item *sfx_items = NULL;
6829 	struct mlx5_flow *dev_flow = NULL;
6830 	struct rte_flow_attr sfx_attr = *attr;
6831 	struct mlx5_flow_meter_info *fm = NULL;
6832 	uint8_t skip_scale_restore;
6833 	bool has_mtr = false;
6834 	bool has_modify = false;
6835 	bool set_mtr_reg = true;
6836 	bool is_mtr_hierarchy = false;
6837 	uint32_t meter_id = 0;
6838 	uint32_t mtr_idx = 0;
6839 	uint32_t mtr_flow_id = 0;
6840 	size_t act_size;
6841 	size_t item_size;
6842 	int actions_n = 0;
6843 	int vlan_items_n = 0;
6844 	int ret = 0;
6845 
6846 	if (priv->mtr_en)
6847 		actions_n = flow_check_meter_action(dev, actions, &has_mtr,
6848 						    &has_modify, &meter_id);
6849 	if (has_mtr) {
6850 		if (flow->meter) {
6851 			fm = flow_dv_meter_find_by_idx(priv, flow->meter);
6852 			if (!fm)
6853 				return rte_flow_error_set(error, EINVAL,
6854 						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6855 						NULL, "Meter not found.");
6856 		} else {
6857 			fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
6858 			if (!fm)
6859 				return rte_flow_error_set(error, EINVAL,
6860 						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6861 						NULL, "Meter not found.");
6862 			ret = mlx5_flow_meter_attach(priv, fm,
6863 						     &sfx_attr, error);
6864 			if (ret)
6865 				return -rte_errno;
6866 			flow->meter = mtr_idx;
6867 		}
6868 		MLX5_ASSERT(wks);
6869 		wks->fm = fm;
6870 		if (!fm->def_policy) {
6871 			wks->policy = mlx5_flow_meter_policy_find(dev,
6872 								  fm->policy_id,
6873 								  NULL);
6874 			MLX5_ASSERT(wks->policy);
6875 			if (wks->policy->mark)
6876 				wks->mark = 1;
6877 			if (wks->policy->is_hierarchy) {
6878 				wks->final_policy =
6879 				mlx5_flow_meter_hierarchy_get_final_policy(dev,
6880 								wks->policy);
6881 				if (!wks->final_policy)
6882 					return rte_flow_error_set(error,
6883 					EINVAL,
6884 					RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6885 				"Failed to find terminal policy of hierarchy.");
6886 				is_mtr_hierarchy = true;
6887 			}
6888 		}
6889 		/*
6890 		 * If it isn't default-policy Meter, and
6891 		 * 1. Not meter hierarchy and there's no action in flow to change
6892 		 *    packet (modify/encap/decap etc.), OR
6893 		 * 2. No drop count needed for this meter.
6894 		 * Then no need to use regC to save meter id anymore.
6895 		 */
6896 		if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt))
6897 			set_mtr_reg = false;
6898 		/* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */
6899 #define METER_PREFIX_ACTION 7
6900 		act_size = (sizeof(struct rte_flow_action) *
6901 			    (actions_n + METER_PREFIX_ACTION)) +
6902 			   sizeof(struct mlx5_rte_flow_action_set_tag);
6903 		/* Flow can have multiple VLAN items. Account for them in suffix items. */
6904 		vlan_items_n = flow_count_vlan_items(items);
6905 		/* Suffix items: tag, [vlans], port id, end. */
6906 #define METER_SUFFIX_ITEM 3
6907 		item_size = sizeof(struct rte_flow_item) * (METER_SUFFIX_ITEM + vlan_items_n) +
6908 			    sizeof(struct mlx5_rte_flow_item_tag) * 2;
6909 		sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
6910 					  0, SOCKET_ID_ANY);
6911 		if (!sfx_actions)
6912 			return rte_flow_error_set(error, ENOMEM,
6913 						  RTE_FLOW_ERROR_TYPE_ACTION,
6914 						  NULL, "no memory to split "
6915 						  "meter flow");
6916 		sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
6917 			     act_size);
6918 		/* There's no suffix flow for meter of non-default policy. */
6919 		if (!fm->def_policy)
6920 			pre_actions = sfx_actions + 1;
6921 		else
6922 			pre_actions = sfx_actions + actions_n;
6923 		ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr,
6924 					    items, sfx_items, actions,
6925 					    sfx_actions, pre_actions,
6926 					    (set_mtr_reg ? &mtr_flow_id : NULL),
6927 					    error);
6928 		if (ret) {
6929 			ret = -rte_errno;
6930 			goto exit;
6931 		}
6932 		/* Add the prefix subflow. */
6933 		skip_scale_restore = flow_split_info->skip_scale;
6934 		flow_split_info->skip_scale |=
6935 			1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6936 		ret = flow_create_split_inner(dev, flow, &dev_flow,
6937 					      attr, items, pre_actions,
6938 					      flow_split_info, error);
6939 		flow_split_info->skip_scale = skip_scale_restore;
6940 		if (ret) {
6941 			if (mtr_flow_id)
6942 				mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
6943 			ret = -rte_errno;
6944 			goto exit;
6945 		}
6946 		if (mtr_flow_id) {
6947 			dev_flow->handle->split_flow_id = mtr_flow_id;
6948 			dev_flow->handle->is_meter_flow_id = 1;
6949 		}
6950 		if (!fm->def_policy) {
6951 			if (!set_mtr_reg && fm->drop_cnt)
6952 				ret =
6953 			flow_meter_create_drop_flow_with_org_pattern(dev, flow,
6954 							&sfx_attr, items,
6955 							flow_split_info,
6956 							fm, error);
6957 			goto exit;
6958 		}
6959 		/* Setting the sfx group atrr. */
6960 		sfx_attr.group = sfx_attr.transfer ?
6961 				(MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6962 				 MLX5_FLOW_TABLE_LEVEL_METER;
6963 		flow_split_info->prefix_layers =
6964 				flow_get_prefix_layer_flags(dev_flow);
6965 		flow_split_info->prefix_mark |= wks->mark;
6966 		flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
6967 	}
6968 	/* Add the prefix subflow. */
6969 	ret = flow_create_split_metadata(dev, flow,
6970 					 &sfx_attr, sfx_items ?
6971 					 sfx_items : items,
6972 					 sfx_actions ? sfx_actions : actions,
6973 					 flow_split_info, error);
6974 exit:
6975 	if (sfx_actions)
6976 		mlx5_free(sfx_actions);
6977 	return ret;
6978 }
6979 
6980 /**
6981  * The splitting for sample feature.
6982  *
6983  * Once Sample action is detected in the action list, the flow actions should
6984  * be split into prefix sub flow and suffix sub flow.
6985  *
6986  * The original items remain in the prefix sub flow, all actions preceding the
6987  * sample action and the sample action itself will be copied to the prefix
6988  * sub flow, the actions following the sample action will be copied to the
6989  * suffix sub flow, Queue action always be located in the suffix sub flow.
6990  *
6991  * In order to make the packet from prefix sub flow matches with suffix sub
6992  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
6993  * flow uses tag item with the unique flow id.
6994  *
6995  * @param dev
6996  *   Pointer to Ethernet device.
6997  * @param[in] flow
6998  *   Parent flow structure pointer.
6999  * @param[in] attr
7000  *   Flow rule attributes.
7001  * @param[in] items
7002  *   Pattern specification (list terminated by the END pattern item).
7003  * @param[in] actions
7004  *   Associated actions (list terminated by the END action).
7005  * @param[in] flow_split_info
7006  *   Pointer to flow split info structure.
7007  * @param[out] error
7008  *   Perform verbose error reporting if not NULL.
7009  * @return
7010  *   0 on success, negative value otherwise
7011  */
7012 static int
7013 flow_create_split_sample(struct rte_eth_dev *dev,
7014 			 struct rte_flow *flow,
7015 			 const struct rte_flow_attr *attr,
7016 			 const struct rte_flow_item items[],
7017 			 const struct rte_flow_action actions[],
7018 			 struct mlx5_flow_split_info *flow_split_info,
7019 			 struct rte_flow_error *error)
7020 {
7021 	struct mlx5_priv *priv = dev->data->dev_private;
7022 	struct rte_flow_action *sfx_actions = NULL;
7023 	struct rte_flow_action *pre_actions = NULL;
7024 	struct rte_flow_item *sfx_items = NULL;
7025 	struct mlx5_flow *dev_flow = NULL;
7026 	struct rte_flow_attr sfx_attr = *attr;
7027 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7028 	struct mlx5_flow_dv_sample_resource *sample_res;
7029 	struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
7030 	struct mlx5_flow_tbl_resource *sfx_tbl;
7031 	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
7032 #endif
7033 	size_t act_size;
7034 	size_t item_size;
7035 	uint32_t fdb_tx = 0;
7036 	int32_t tag_id = 0;
7037 	int actions_n = 0;
7038 	int sample_action_pos;
7039 	int qrss_action_pos;
7040 	int add_tag = 0;
7041 	int modify_after_mirror = 0;
7042 	uint16_t jump_table = 0;
7043 	const uint32_t next_ft_step = 1;
7044 	int ret = 0;
7045 	struct mlx5_priv *item_port_priv = NULL;
7046 	const struct rte_flow_item *item;
7047 
7048 	if (priv->sampler_en)
7049 		actions_n = flow_check_match_action(actions, attr,
7050 					RTE_FLOW_ACTION_TYPE_SAMPLE,
7051 					&sample_action_pos, &qrss_action_pos,
7052 					&modify_after_mirror);
7053 	if (actions_n) {
7054 		/* The prefix actions must includes sample, tag, end. */
7055 		act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
7056 			   + sizeof(struct mlx5_rte_flow_action_set_tag);
7057 		item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
7058 			    sizeof(struct mlx5_rte_flow_item_tag) * 2;
7059 		sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
7060 					  item_size), 0, SOCKET_ID_ANY);
7061 		if (!sfx_actions)
7062 			return rte_flow_error_set(error, ENOMEM,
7063 						  RTE_FLOW_ERROR_TYPE_ACTION,
7064 						  NULL, "no memory to split "
7065 						  "sample flow");
7066 		for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
7067 			if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) {
7068 				const struct rte_flow_item_port_id *spec;
7069 
7070 				spec = (const struct rte_flow_item_port_id *)item->spec;
7071 				if (spec)
7072 					item_port_priv =
7073 						mlx5_port_to_eswitch_info(spec->id, true);
7074 				break;
7075 			} else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) {
7076 				const struct rte_flow_item_ethdev *spec;
7077 
7078 				spec = (const struct rte_flow_item_ethdev *)item->spec;
7079 				if (spec)
7080 					item_port_priv =
7081 						mlx5_port_to_eswitch_info(spec->port_id, true);
7082 				break;
7083 			} else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) {
7084 				const struct rte_flow_item_ethdev *spec;
7085 
7086 				spec = (const struct rte_flow_item_ethdev *)item->spec;
7087 				if (spec)
7088 					item_port_priv =
7089 						mlx5_port_to_eswitch_info(spec->port_id, true);
7090 				break;
7091 			}
7092 		}
7093 		/* The representor_id is UINT16_MAX for uplink. */
7094 		fdb_tx = (attr->transfer &&
7095 			  flow_source_vport_representor(priv, item_port_priv));
7096 		/*
7097 		 * When reg_c_preserve is set, metadata registers Cx preserve
7098 		 * their value even through packet duplication.
7099 		 */
7100 		add_tag = (!fdb_tx ||
7101 			   priv->sh->cdev->config.hca_attr.reg_c_preserve);
7102 		if (add_tag)
7103 			sfx_items = (struct rte_flow_item *)((char *)sfx_actions
7104 					+ act_size);
7105 		if (modify_after_mirror)
7106 			jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
7107 				     next_ft_step;
7108 		pre_actions = sfx_actions + actions_n;
7109 		tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items,
7110 						actions, sfx_actions,
7111 						pre_actions, actions_n,
7112 						sample_action_pos,
7113 						qrss_action_pos, jump_table,
7114 						error);
7115 		if (tag_id < 0 || (add_tag && !tag_id)) {
7116 			ret = -rte_errno;
7117 			goto exit;
7118 		}
7119 		if (modify_after_mirror)
7120 			flow_split_info->skip_scale =
7121 					1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
7122 		/* Add the prefix subflow. */
7123 		ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
7124 					      items, pre_actions,
7125 					      flow_split_info, error);
7126 		if (ret) {
7127 			ret = -rte_errno;
7128 			goto exit;
7129 		}
7130 		dev_flow->handle->split_flow_id = tag_id;
7131 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7132 		if (!modify_after_mirror) {
7133 			/* Set the sfx group attr. */
7134 			sample_res = (struct mlx5_flow_dv_sample_resource *)
7135 						dev_flow->dv.sample_res;
7136 			sfx_tbl = (struct mlx5_flow_tbl_resource *)
7137 						sample_res->normal_path_tbl;
7138 			sfx_tbl_data = container_of(sfx_tbl,
7139 						struct mlx5_flow_tbl_data_entry,
7140 						tbl);
7141 			sfx_attr.group = sfx_attr.transfer ?
7142 			(sfx_tbl_data->level - 1) : sfx_tbl_data->level;
7143 		} else {
7144 			MLX5_ASSERT(attr->transfer);
7145 			sfx_attr.group = jump_table;
7146 		}
7147 		flow_split_info->prefix_layers =
7148 				flow_get_prefix_layer_flags(dev_flow);
7149 		MLX5_ASSERT(wks);
7150 		flow_split_info->prefix_mark |= wks->mark;
7151 		/* Suffix group level already be scaled with factor, set
7152 		 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
7153 		 * again in translation.
7154 		 */
7155 		flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
7156 #endif
7157 	}
7158 	/* Add the suffix subflow. */
7159 	ret = flow_create_split_meter(dev, flow, &sfx_attr,
7160 				      sfx_items ? sfx_items : items,
7161 				      sfx_actions ? sfx_actions : actions,
7162 				      flow_split_info, error);
7163 exit:
7164 	if (sfx_actions)
7165 		mlx5_free(sfx_actions);
7166 	return ret;
7167 }
7168 
7169 /**
7170  * Split the flow to subflow set. The splitters might be linked
7171  * in the chain, like this:
7172  * flow_create_split_outer() calls:
7173  *   flow_create_split_meter() calls:
7174  *     flow_create_split_metadata(meter_subflow_0) calls:
7175  *       flow_create_split_inner(metadata_subflow_0)
7176  *       flow_create_split_inner(metadata_subflow_1)
7177  *       flow_create_split_inner(metadata_subflow_2)
7178  *     flow_create_split_metadata(meter_subflow_1) calls:
7179  *       flow_create_split_inner(metadata_subflow_0)
7180  *       flow_create_split_inner(metadata_subflow_1)
7181  *       flow_create_split_inner(metadata_subflow_2)
7182  *
7183  * This provide flexible way to add new levels of flow splitting.
7184  * The all of successfully created subflows are included to the
7185  * parent flow dev_flow list.
7186  *
7187  * @param dev
7188  *   Pointer to Ethernet device.
7189  * @param[in] flow
7190  *   Parent flow structure pointer.
7191  * @param[in] attr
7192  *   Flow rule attributes.
7193  * @param[in] items
7194  *   Pattern specification (list terminated by the END pattern item).
7195  * @param[in] actions
7196  *   Associated actions (list terminated by the END action).
7197  * @param[in] flow_split_info
7198  *   Pointer to flow split info structure.
7199  * @param[out] error
7200  *   Perform verbose error reporting if not NULL.
7201  * @return
7202  *   0 on success, negative value otherwise
7203  */
7204 static int
7205 flow_create_split_outer(struct rte_eth_dev *dev,
7206 			struct rte_flow *flow,
7207 			const struct rte_flow_attr *attr,
7208 			const struct rte_flow_item items[],
7209 			const struct rte_flow_action actions[],
7210 			struct mlx5_flow_split_info *flow_split_info,
7211 			struct rte_flow_error *error)
7212 {
7213 	int ret;
7214 
7215 	ret = flow_create_split_sample(dev, flow, attr, items,
7216 				       actions, flow_split_info, error);
7217 	MLX5_ASSERT(ret <= 0);
7218 	return ret;
7219 }
7220 
7221 static inline struct mlx5_flow_tunnel *
7222 flow_tunnel_from_rule(const struct mlx5_flow *flow)
7223 {
7224 	struct mlx5_flow_tunnel *tunnel;
7225 
7226 #pragma GCC diagnostic push
7227 #pragma GCC diagnostic ignored "-Wcast-qual"
7228 	tunnel = (typeof(tunnel))flow->tunnel;
7229 #pragma GCC diagnostic pop
7230 
7231 	return tunnel;
7232 }
7233 
7234 /**
7235  * Create a flow and add it to @p list.
7236  *
7237  * @param dev
7238  *   Pointer to Ethernet device.
7239  * @param list
7240  *   Pointer to a TAILQ flow list. If this parameter NULL,
7241  *   no list insertion occurred, flow is just created,
7242  *   this is caller's responsibility to track the
7243  *   created flow.
7244  * @param[in] attr
7245  *   Flow rule attributes.
7246  * @param[in] items
7247  *   Pattern specification (list terminated by the END pattern item).
7248  * @param[in] actions
7249  *   Associated actions (list terminated by the END action).
7250  * @param[in] external
7251  *   This flow rule is created by request external to PMD.
7252  * @param[out] error
7253  *   Perform verbose error reporting if not NULL.
7254  *
7255  * @return
7256  *   A flow index on success, 0 otherwise and rte_errno is set.
7257  */
7258 static uint32_t
7259 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
7260 		 const struct rte_flow_attr *attr,
7261 		 const struct rte_flow_item items[],
7262 		 const struct rte_flow_action original_actions[],
7263 		 bool external, struct rte_flow_error *error)
7264 {
7265 	struct mlx5_priv *priv = dev->data->dev_private;
7266 	struct rte_flow *flow = NULL;
7267 	struct mlx5_flow *dev_flow;
7268 	const struct rte_flow_action_rss *rss = NULL;
7269 	struct mlx5_translated_action_handle
7270 		indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
7271 	int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
7272 	union {
7273 		struct mlx5_flow_expand_rss buf;
7274 		uint8_t buffer[8192];
7275 	} expand_buffer;
7276 	union {
7277 		struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
7278 		uint8_t buffer[2048];
7279 	} actions_rx;
7280 	union {
7281 		struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
7282 		uint8_t buffer[2048];
7283 	} actions_hairpin_tx;
7284 	union {
7285 		struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
7286 		uint8_t buffer[2048];
7287 	} items_tx;
7288 	struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT];
7289 	struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
7290 	struct mlx5_flow_rss_desc *rss_desc;
7291 	const struct rte_flow_action *p_actions_rx;
7292 	uint32_t i;
7293 	uint32_t idx = 0;
7294 	int hairpin_flow;
7295 	struct rte_flow_attr attr_tx = { .priority = 0 };
7296 	const struct rte_flow_action *actions;
7297 	struct rte_flow_action *translated_actions = NULL;
7298 	struct mlx5_flow_tunnel *tunnel;
7299 	struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
7300 	struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
7301 	struct mlx5_flow_split_info flow_split_info = {
7302 		.external = !!external,
7303 		.skip_scale = 0,
7304 		.flow_idx = 0,
7305 		.prefix_mark = 0,
7306 		.prefix_layers = 0,
7307 		.table_id = 0
7308 	};
7309 	int ret;
7310 
7311 	MLX5_ASSERT(wks);
7312 	rss_desc = &wks->rss_desc;
7313 	ret = flow_action_handles_translate(dev, original_actions,
7314 					    indir_actions,
7315 					    &indir_actions_n,
7316 					    &translated_actions, error);
7317 	if (ret < 0) {
7318 		MLX5_ASSERT(translated_actions == NULL);
7319 		return 0;
7320 	}
7321 	actions = translated_actions ? translated_actions : original_actions;
7322 	p_actions_rx = actions;
7323 	hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
7324 	ret = flow_drv_validate(dev, attr, items, p_actions_rx,
7325 				external, hairpin_flow, error);
7326 	if (ret < 0)
7327 		goto error_before_hairpin_split;
7328 	flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
7329 	if (!flow) {
7330 		rte_errno = ENOMEM;
7331 		goto error_before_hairpin_split;
7332 	}
7333 	if (hairpin_flow > 0) {
7334 		if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
7335 			rte_errno = EINVAL;
7336 			goto error_before_hairpin_split;
7337 		}
7338 		flow_hairpin_split(dev, actions, actions_rx.actions,
7339 				   actions_hairpin_tx.actions, items_tx.items,
7340 				   idx);
7341 		p_actions_rx = actions_rx.actions;
7342 	}
7343 	flow_split_info.flow_idx = idx;
7344 	flow->drv_type = flow_get_drv_type(dev, attr);
7345 	MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
7346 		    flow->drv_type < MLX5_FLOW_TYPE_MAX);
7347 	memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
7348 	/* RSS Action only works on NIC RX domain */
7349 	if (attr->ingress)
7350 		rss = flow_get_rss_action(dev, p_actions_rx);
7351 	if (rss) {
7352 		MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512);
7353 		rss_desc->symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func);
7354 		/*
7355 		 * The following information is required by
7356 		 * mlx5_flow_hashfields_adjust() in advance.
7357 		 */
7358 		rss_desc->level = rss->level;
7359 		/* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */
7360 		rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
7361 	}
7362 	flow->dev_handles = 0;
7363 	if (rss && rss->types) {
7364 		unsigned int graph_root;
7365 
7366 		graph_root = find_graph_root(rss->level);
7367 		ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
7368 					   items, rss->types,
7369 					   mlx5_support_expansion, graph_root);
7370 		MLX5_ASSERT(ret > 0 &&
7371 		       (unsigned int)ret < sizeof(expand_buffer.buffer));
7372 		if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
7373 			for (i = 0; i < buf->entries; ++i)
7374 				mlx5_dbg__print_pattern(buf->entry[i].pattern);
7375 		}
7376 	} else {
7377 		ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf,
7378 					   sizeof(expand_buffer.buffer),
7379 					   items, sq_specs);
7380 		if (ret) {
7381 			rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
7382 					   NULL, "not enough memory for rte_flow");
7383 			goto error;
7384 		}
7385 		if (buf->entries == 0) {
7386 			buf->entries = 1;
7387 			buf->entry[0].pattern = (void *)(uintptr_t)items;
7388 		}
7389 	}
7390 	rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
7391 						      indir_actions_n);
7392 	for (i = 0; i < buf->entries; ++i) {
7393 		/* Initialize flow split data. */
7394 		flow_split_info.prefix_layers = 0;
7395 		flow_split_info.prefix_mark = 0;
7396 		flow_split_info.skip_scale = 0;
7397 		/*
7398 		 * The splitter may create multiple dev_flows,
7399 		 * depending on configuration. In the simplest
7400 		 * case it just creates unmodified original flow.
7401 		 */
7402 		ret = flow_create_split_outer(dev, flow, attr,
7403 					      buf->entry[i].pattern,
7404 					      p_actions_rx, &flow_split_info,
7405 					      error);
7406 		if (ret < 0)
7407 			goto error;
7408 		if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
7409 			ret = flow_tunnel_add_default_miss(dev, flow, attr,
7410 							   p_actions_rx,
7411 							   idx,
7412 							   wks->flows[0].tunnel,
7413 							   &default_miss_ctx,
7414 							   error);
7415 			if (ret < 0) {
7416 				mlx5_free(default_miss_ctx.queue);
7417 				goto error;
7418 			}
7419 		}
7420 	}
7421 	/* Create the tx flow. */
7422 	if (hairpin_flow) {
7423 		attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
7424 		attr_tx.ingress = 0;
7425 		attr_tx.egress = 1;
7426 		dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
7427 					 actions_hairpin_tx.actions,
7428 					 idx, error);
7429 		if (!dev_flow)
7430 			goto error;
7431 		dev_flow->flow = flow;
7432 		dev_flow->external = 0;
7433 		SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
7434 			      dev_flow->handle, next);
7435 		ret = flow_drv_translate(dev, dev_flow, &attr_tx,
7436 					 items_tx.items,
7437 					 actions_hairpin_tx.actions, error);
7438 		if (ret < 0)
7439 			goto error;
7440 	}
7441 	/*
7442 	 * Update the metadata register copy table. If extensive
7443 	 * metadata feature is enabled and registers are supported
7444 	 * we might create the extra rte_flow for each unique
7445 	 * MARK/FLAG action ID.
7446 	 *
7447 	 * The table is updated for ingress and transfer flows only, because
7448 	 * the egress Flows belong to the different device and
7449 	 * copy table should be updated in peer NIC Rx domain.
7450 	 */
7451 	if ((attr->ingress || attr->transfer) &&
7452 	    (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
7453 		ret = flow_mreg_update_copy_table(dev, flow, actions, error);
7454 		if (ret)
7455 			goto error;
7456 	}
7457 	/*
7458 	 * If the flow is external (from application) OR device is started,
7459 	 * OR mreg discover, then apply immediately.
7460 	 */
7461 	if (external || dev->data->dev_started ||
7462 	    (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
7463 	     attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
7464 		ret = flow_drv_apply(dev, flow, error);
7465 		if (ret < 0)
7466 			goto error;
7467 	}
7468 	flow->type = type;
7469 	flow_rxq_flags_set(dev, flow);
7470 	rte_free(translated_actions);
7471 	tunnel = flow_tunnel_from_rule(wks->flows);
7472 	if (tunnel) {
7473 		flow->tunnel = 1;
7474 		flow->tunnel_id = tunnel->tunnel_id;
7475 		rte_atomic_fetch_add_explicit(&tunnel->refctn, 1, rte_memory_order_relaxed);
7476 		mlx5_free(default_miss_ctx.queue);
7477 	}
7478 	mlx5_flow_pop_thread_workspace();
7479 	return idx;
7480 error:
7481 	MLX5_ASSERT(flow);
7482 	ret = rte_errno; /* Save rte_errno before cleanup. */
7483 	flow_mreg_del_copy_action(dev, flow);
7484 	flow_drv_destroy(dev, flow);
7485 	if (rss_desc->shared_rss)
7486 		rte_atomic_fetch_sub_explicit(&((struct mlx5_shared_action_rss *)
7487 			mlx5_ipool_get
7488 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
7489 			rss_desc->shared_rss))->refcnt, 1, rte_memory_order_relaxed);
7490 	mlx5_ipool_free(priv->flows[type], idx);
7491 	rte_errno = ret; /* Restore rte_errno. */
7492 	ret = rte_errno;
7493 	rte_errno = ret;
7494 error_before_hairpin_split:
7495 	mlx5_flow_pop_thread_workspace();
7496 	rte_free(translated_actions);
7497 	return 0;
7498 }
7499 
7500 /**
7501  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
7502  * incoming packets to table 1.
7503  *
7504  * Other flow rules, requested for group n, will be created in
7505  * e-switch table n+1.
7506  * Jump action to e-switch group n will be created to group n+1.
7507  *
7508  * Used when working in switchdev mode, to utilise advantages of table 1
7509  * and above.
7510  *
7511  * @param dev
7512  *   Pointer to Ethernet device.
7513  *
7514  * @return
7515  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
7516  */
7517 struct rte_flow *
7518 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
7519 {
7520 	const struct rte_flow_attr attr = {
7521 		.group = 0,
7522 		.priority = 0,
7523 		.ingress = 0,
7524 		.egress = 0,
7525 		.transfer = 1,
7526 	};
7527 	const struct rte_flow_item pattern = {
7528 		.type = RTE_FLOW_ITEM_TYPE_END,
7529 	};
7530 	struct rte_flow_action_jump jump = {
7531 		.group = 1,
7532 	};
7533 	const struct rte_flow_action actions[] = {
7534 		{
7535 			.type = RTE_FLOW_ACTION_TYPE_JUMP,
7536 			.conf = &jump,
7537 		},
7538 		{
7539 			.type = RTE_FLOW_ACTION_TYPE_END,
7540 		},
7541 	};
7542 	struct rte_flow_error error;
7543 
7544 	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7545 						   &attr, &pattern,
7546 						   actions, false, &error);
7547 }
7548 
7549 /**
7550  * Create a dedicated flow rule on e-switch table 1, matches ESW manager
7551  * and sq number, directs all packets to peer vport.
7552  *
7553  * @param dev
7554  *   Pointer to Ethernet device.
7555  * @param sq_num
7556  *   SQ number.
7557  *
7558  * @return
7559  *   Flow ID on success, 0 otherwise and rte_errno is set.
7560  */
7561 uint32_t
7562 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num)
7563 {
7564 	struct rte_flow_attr attr = {
7565 		.group = 0,
7566 		.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7567 		.ingress = 0,
7568 		.egress = 0,
7569 		.transfer = 1,
7570 	};
7571 	struct rte_flow_item_port_id port_spec = {
7572 		.id = MLX5_PORT_ESW_MGR,
7573 	};
7574 	struct mlx5_rte_flow_item_sq sq_spec = {
7575 		.queue = sq_num,
7576 	};
7577 	struct rte_flow_item pattern[] = {
7578 		{
7579 			.type = RTE_FLOW_ITEM_TYPE_PORT_ID,
7580 			.spec = &port_spec,
7581 		},
7582 		{
7583 			.type = (enum rte_flow_item_type)
7584 				MLX5_RTE_FLOW_ITEM_TYPE_SQ,
7585 			.spec = &sq_spec,
7586 		},
7587 		{
7588 			.type = RTE_FLOW_ITEM_TYPE_END,
7589 		},
7590 	};
7591 	struct rte_flow_action_jump jump = {
7592 		.group = 1,
7593 	};
7594 	struct rte_flow_action_port_id port = {
7595 		.id = dev->data->port_id,
7596 	};
7597 	struct rte_flow_action actions[] = {
7598 		{
7599 			.type = RTE_FLOW_ACTION_TYPE_JUMP,
7600 			.conf = &jump,
7601 		},
7602 		{
7603 			.type = RTE_FLOW_ACTION_TYPE_END,
7604 		},
7605 	};
7606 	struct rte_flow_error error;
7607 
7608 	/*
7609 	 * Creates group 0, highest priority jump flow.
7610 	 * Matches txq to bypass kernel packets.
7611 	 */
7612 	if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
7613 			     false, &error) == 0)
7614 		return 0;
7615 	/* Create group 1, lowest priority redirect flow for txq. */
7616 	attr.group = 1;
7617 	actions[0].conf = &port;
7618 	actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID;
7619 	return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
7620 				actions, false, &error);
7621 }
7622 
7623 /**
7624  * Validate a flow supported by the NIC.
7625  *
7626  * @see rte_flow_validate()
7627  * @see rte_flow_ops
7628  */
7629 int
7630 mlx5_flow_validate(struct rte_eth_dev *dev,
7631 		   const struct rte_flow_attr *attr,
7632 		   const struct rte_flow_item items[],
7633 		   const struct rte_flow_action original_actions[],
7634 		   struct rte_flow_error *error)
7635 {
7636 	int hairpin_flow;
7637 	struct mlx5_translated_action_handle
7638 		indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
7639 	int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
7640 	const struct rte_flow_action *actions;
7641 	struct rte_flow_action *translated_actions = NULL;
7642 	int ret = flow_action_handles_translate(dev, original_actions,
7643 						indir_actions,
7644 						&indir_actions_n,
7645 						&translated_actions, error);
7646 
7647 	if (ret)
7648 		return ret;
7649 	actions = translated_actions ? translated_actions : original_actions;
7650 	hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
7651 	ret = flow_drv_validate(dev, attr, items, actions,
7652 				true, hairpin_flow, error);
7653 	rte_free(translated_actions);
7654 	return ret;
7655 }
7656 
7657 static int
7658 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev,
7659 			  const struct rte_flow_attr *attr,
7660 			  const uint32_t orig_prio,
7661 			  const struct rte_flow_item *items,
7662 			  const struct rte_flow_action *actions,
7663 			  uint32_t flow_idx)
7664 {
7665 	struct mlx5_priv *priv = dev->data->dev_private;
7666 	struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info;
7667 	struct mlx5_dv_flow_info *flow_info, *tmp_info;
7668 	struct rte_flow_error error;
7669 	int len, ret;
7670 
7671 	flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY);
7672 	if (!flow_info) {
7673 		DRV_LOG(ERR, "No enough memory for flow_info caching.");
7674 		return -1;
7675 	}
7676 	flow_info->orig_prio = orig_prio;
7677 	flow_info->attr = *attr;
7678 	/* Standby mode rule awlays saves it in low priority entry. */
7679 	flow_info->flow_idx_low_prio = flow_idx;
7680 
7681 	/* Store matching items. */
7682 	ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error);
7683 	if (ret <= 0) {
7684 		DRV_LOG(ERR, "Can't get items length.");
7685 		goto end;
7686 	}
7687 	len = RTE_ALIGN(ret, 16);
7688 	flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY);
7689 	if (!flow_info->items) {
7690 		DRV_LOG(ERR, "No enough memory for items caching.");
7691 		goto end;
7692 	}
7693 	ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error);
7694 	if (ret <= 0) {
7695 		DRV_LOG(ERR, "Can't duplicate items.");
7696 		goto end;
7697 	}
7698 
7699 	/* Store flow actions. */
7700 	ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error);
7701 	if (ret <= 0) {
7702 		DRV_LOG(ERR, "Can't get actions length.");
7703 		goto end;
7704 	}
7705 	len = RTE_ALIGN(ret, 16);
7706 	flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY);
7707 	if (!flow_info->actions) {
7708 		DRV_LOG(ERR, "No enough memory for actions caching.");
7709 		goto end;
7710 	}
7711 	ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error);
7712 	if (ret <= 0) {
7713 		DRV_LOG(ERR, "Can't duplicate actions.");
7714 		goto end;
7715 	}
7716 
7717 	/* Insert to the list end. */
7718 	if (LIST_EMPTY(&mode_info->hot_upgrade)) {
7719 		LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info,  next);
7720 	} else {
7721 		tmp_info = LIST_FIRST(&mode_info->hot_upgrade);
7722 		while (LIST_NEXT(tmp_info, next))
7723 			tmp_info = LIST_NEXT(tmp_info, next);
7724 		LIST_INSERT_AFTER(tmp_info, flow_info, next);
7725 	}
7726 	return 0;
7727 end:
7728 	if (flow_info->items)
7729 		mlx5_free(flow_info->items);
7730 	if (flow_info->actions)
7731 		mlx5_free(flow_info->actions);
7732 	mlx5_free(flow_info);
7733 	return -1;
7734 }
7735 
7736 static int
7737 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio)
7738 {
7739 	struct mlx5_priv *priv = dev->data->dev_private;
7740 	struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info;
7741 	struct mlx5_dv_flow_info *flow_info;
7742 	struct rte_flow_attr attr;
7743 	struct rte_flow_error error;
7744 	struct rte_flow *high, *low;
7745 
7746 	flow_info = LIST_FIRST(&mode_info->hot_upgrade);
7747 	while (flow_info) {
7748 		/* DUP flow may have the same priority. */
7749 		if (flow_info->orig_prio != flow_info->attr.priority) {
7750 			attr = flow_info->attr;
7751 			if (orig_prio)
7752 				attr.priority = flow_info->orig_prio;
7753 			flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
7754 					&attr, flow_info->items, flow_info->actions,
7755 					true, &error);
7756 			if (!flow_info->flow_idx_high_prio) {
7757 				DRV_LOG(ERR, "Priority toggle failed internally.");
7758 				goto err;
7759 			}
7760 		}
7761 		flow_info = LIST_NEXT(flow_info, next);
7762 	}
7763 	/* Delete the low priority rules and swap the flow handle. */
7764 	flow_info = LIST_FIRST(&mode_info->hot_upgrade);
7765 	while (flow_info) {
7766 		MLX5_ASSERT(flow_info->flow_idx_low_prio);
7767 		if (flow_info->orig_prio != flow_info->attr.priority) {
7768 			high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
7769 					flow_info->flow_idx_high_prio);
7770 			low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
7771 					flow_info->flow_idx_low_prio);
7772 			if (high && low) {
7773 				RTE_SWAP(*low, *high);
7774 				flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
7775 						  flow_info->flow_idx_low_prio);
7776 				flow_info->flow_idx_high_prio = 0;
7777 			}
7778 		}
7779 		flow_info = LIST_NEXT(flow_info, next);
7780 	}
7781 	return 0;
7782 err:
7783 	/* Destroy preceding successful high priority rules. */
7784 	flow_info = LIST_FIRST(&mode_info->hot_upgrade);
7785 	while (flow_info) {
7786 		if (flow_info->orig_prio != flow_info->attr.priority) {
7787 			if (flow_info->flow_idx_high_prio)
7788 				flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
7789 						  flow_info->flow_idx_high_prio);
7790 			else
7791 				break;
7792 			flow_info->flow_idx_high_prio = 0;
7793 		}
7794 		flow_info = LIST_NEXT(flow_info, next);
7795 	}
7796 	return -1;
7797 }
7798 
7799 /**
7800  * Set the mode of the flow engine of a process to active or standby during live migration.
7801  *
7802  * @param[in] mode
7803  *   MLX5 flow engine mode, @see `enum rte_pmd_mlx5_flow_engine_mode`.
7804  * @param[in] flags
7805  *   Flow engine mode specific flags.
7806  *
7807  * @return
7808  *   Negative value on error, positive on success.
7809  */
7810 int
7811 rte_pmd_mlx5_flow_engine_set_mode(enum rte_pmd_mlx5_flow_engine_mode mode, uint32_t flags)
7812 {
7813 	struct mlx5_priv *priv;
7814 	struct rte_pmd_mlx5_flow_engine_mode_info *mode_info;
7815 	struct mlx5_dv_flow_info *flow_info, *tmp_info;
7816 	uint16_t port, port_id;
7817 	uint16_t toggle_num = 0;
7818 	struct rte_eth_dev *dev;
7819 	enum rte_pmd_mlx5_flow_engine_mode orig_mode;
7820 	uint32_t orig_flags;
7821 	bool need_toggle = false;
7822 
7823 	/* Check if flags combinations are supported. */
7824 	if (flags && flags != RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) {
7825 		DRV_LOG(ERR, "Doesn't support such flags %u", flags);
7826 		return -1;
7827 	}
7828 	MLX5_ETH_FOREACH_DEV(port, NULL) {
7829 		dev = &rte_eth_devices[port];
7830 		priv = dev->data->dev_private;
7831 		mode_info = &priv->mode_info;
7832 		/* No mode change. Assume all devices hold the same mode. */
7833 		if (mode_info->mode == mode) {
7834 			DRV_LOG(INFO, "Process flow engine has been in mode %u", mode);
7835 			if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) {
7836 				DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n",
7837 						port, mode_info->mode_flag);
7838 				orig_mode = mode_info->mode;
7839 				orig_flags = mode_info->mode_flag;
7840 				goto err;
7841 			}
7842 			mode_info->mode_flag = flags;
7843 			toggle_num++;
7844 			continue;
7845 		}
7846 		/* Active -> standby. */
7847 		if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY) {
7848 			if (!LIST_EMPTY(&mode_info->hot_upgrade)) {
7849 				DRV_LOG(ERR, "Cached rule existed");
7850 				orig_mode = mode_info->mode;
7851 				orig_flags = mode_info->mode_flag;
7852 				goto err;
7853 			}
7854 			mode_info->mode_flag = flags;
7855 			mode_info->mode = mode;
7856 			toggle_num++;
7857 		/* Standby -> active. */
7858 		} else if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) {
7859 			if (LIST_EMPTY(&mode_info->hot_upgrade)) {
7860 				DRV_LOG(INFO, "No cached rule existed");
7861 			} else {
7862 				if (mlx5_flow_cache_flow_toggle(dev, true)) {
7863 					orig_mode = mode_info->mode;
7864 					orig_flags = mode_info->mode_flag;
7865 					need_toggle = true;
7866 					goto err;
7867 				}
7868 			}
7869 			toggle_num++;
7870 		}
7871 	}
7872 	if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) {
7873 		/* Clear cache flow rules. */
7874 		MLX5_ETH_FOREACH_DEV(port, NULL) {
7875 			priv = rte_eth_devices[port].data->dev_private;
7876 			mode_info = &priv->mode_info;
7877 			flow_info = LIST_FIRST(&mode_info->hot_upgrade);
7878 			while (flow_info) {
7879 				tmp_info = LIST_NEXT(flow_info, next);
7880 				LIST_REMOVE(flow_info, next);
7881 				mlx5_free(flow_info->actions);
7882 				mlx5_free(flow_info->items);
7883 				mlx5_free(flow_info);
7884 				flow_info = tmp_info;
7885 			}
7886 			MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade));
7887 		}
7888 	}
7889 	return toggle_num;
7890 err:
7891 	/* Rollback all preceding successful ports. */
7892 	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
7893 		if (port_id == port)
7894 			break;
7895 		priv = rte_eth_devices[port_id].data->dev_private;
7896 		mode_info = &priv->mode_info;
7897 		if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) &&
7898 		    mlx5_flow_cache_flow_toggle(dev, false))
7899 			return -EPERM;
7900 		mode_info->mode = orig_mode;
7901 		mode_info->mode_flag = orig_flags;
7902 	}
7903 	return -EINVAL;
7904 }
7905 /**
7906  * Create a flow.
7907  *
7908  * @see rte_flow_create()
7909  * @see rte_flow_ops
7910  */
7911 struct rte_flow *
7912 mlx5_flow_create(struct rte_eth_dev *dev,
7913 		 const struct rte_flow_attr *attr,
7914 		 const struct rte_flow_item items[],
7915 		 const struct rte_flow_action actions[],
7916 		 struct rte_flow_error *error)
7917 {
7918 	struct mlx5_priv *priv = dev->data->dev_private;
7919 	struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr;
7920 	uint32_t prio = attr->priority;
7921 	uint32_t flow_idx;
7922 
7923 	if (priv->sh->config.dv_flow_en == 2) {
7924 		rte_flow_error_set(error, ENOTSUP,
7925 			  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7926 			  NULL,
7927 			  "Flow non-Q creation not supported");
7928 		return NULL;
7929 	}
7930 	/*
7931 	 * If the device is not started yet, it is not allowed to created a
7932 	 * flow from application. PMD default flows and traffic control flows
7933 	 * are not affected.
7934 	 */
7935 	if (unlikely(!dev->data->dev_started)) {
7936 		DRV_LOG(DEBUG, "port %u is not started when "
7937 			"inserting a flow", dev->data->port_id);
7938 		rte_flow_error_set(error, ENODEV,
7939 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7940 				   NULL,
7941 				   "port not started");
7942 		return NULL;
7943 	}
7944 	if (unlikely(mlx5_need_cache_flow(priv, attr))) {
7945 		if (attr->transfer ||
7946 				(attr->ingress && !(priv->mode_info.mode_flag &
7947 				RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS)))
7948 			new_attr->priority += 1;
7949 	}
7950 	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error);
7951 	if (!flow_idx)
7952 		return NULL;
7953 	if (unlikely(mlx5_need_cache_flow(priv, attr))) {
7954 		if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) {
7955 			flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
7956 			flow_idx = 0;
7957 		}
7958 	}
7959 	return (void *)(uintptr_t)flow_idx;
7960 }
7961 
7962 /**
7963  * Destroy a flow in a list.
7964  *
7965  * @param dev
7966  *   Pointer to Ethernet device.
7967  * @param[in] flow_idx
7968  *   Index of flow to destroy.
7969  */
7970 static void
7971 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
7972 		  uint32_t flow_idx)
7973 {
7974 	struct mlx5_priv *priv = dev->data->dev_private;
7975 	struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
7976 
7977 	if (!flow)
7978 		return;
7979 	MLX5_ASSERT(flow->type == type);
7980 	/*
7981 	 * Update RX queue flags only if port is started, otherwise it is
7982 	 * already clean.
7983 	 */
7984 	if (dev->data->dev_started)
7985 		flow_rxq_flags_trim(dev, flow);
7986 	flow_drv_destroy(dev, flow);
7987 	if (flow->tunnel) {
7988 		struct mlx5_flow_tunnel *tunnel;
7989 
7990 		tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
7991 		RTE_VERIFY(tunnel);
7992 		if (!(rte_atomic_fetch_sub_explicit(&tunnel->refctn, 1,
7993 				rte_memory_order_relaxed) - 1))
7994 			mlx5_flow_tunnel_free(dev, tunnel);
7995 	}
7996 	flow_mreg_del_copy_action(dev, flow);
7997 	mlx5_ipool_free(priv->flows[type], flow_idx);
7998 }
7999 
8000 /**
8001  * Destroy all flows.
8002  *
8003  * @param dev
8004  *   Pointer to Ethernet device.
8005  * @param type
8006  *   Flow type to be flushed.
8007  * @param active
8008  *   If flushing is called actively.
8009  */
8010 void
8011 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
8012 		     bool active)
8013 {
8014 	struct mlx5_priv *priv = dev->data->dev_private;
8015 	uint32_t num_flushed = 0, fidx = 1;
8016 	struct rte_flow *flow;
8017 	struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info;
8018 	struct mlx5_dv_flow_info *flow_info;
8019 
8020 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8021 	if (priv->sh->config.dv_flow_en == 2 &&
8022 	    type == MLX5_FLOW_TYPE_GEN) {
8023 		flow_hw_q_flow_flush(dev, NULL);
8024 		return;
8025 	}
8026 #endif
8027 
8028 	MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
8029 		flow_list_destroy(dev, type, fidx);
8030 		if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) {
8031 			flow_info = LIST_FIRST(&mode_info->hot_upgrade);
8032 			while (flow_info) {
8033 				/* Romove the cache flow info. */
8034 				if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) {
8035 					MLX5_ASSERT(!flow_info->flow_idx_high_prio);
8036 					LIST_REMOVE(flow_info, next);
8037 					mlx5_free(flow_info->items);
8038 					mlx5_free(flow_info->actions);
8039 					mlx5_free(flow_info);
8040 					break;
8041 				}
8042 				flow_info = LIST_NEXT(flow_info, next);
8043 			}
8044 		}
8045 		num_flushed++;
8046 	}
8047 	if (active) {
8048 		DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
8049 			dev->data->port_id, num_flushed);
8050 	}
8051 }
8052 
8053 /**
8054  * Stop all default actions for flows.
8055  *
8056  * @param dev
8057  *   Pointer to Ethernet device.
8058  */
8059 void
8060 mlx5_flow_stop_default(struct rte_eth_dev *dev)
8061 {
8062 	flow_mreg_del_default_copy_action(dev);
8063 	flow_rxq_flags_clear(dev);
8064 }
8065 
8066 /**
8067  * Set rxq flag.
8068  *
8069  * @param[in] dev
8070  *   Pointer to the rte_eth_dev structure.
8071  * @param[in] enable
8072  *   Flag to enable or not.
8073  */
8074 void
8075 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable)
8076 {
8077 	struct mlx5_priv *priv = dev->data->dev_private;
8078 	unsigned int i;
8079 
8080 	if ((!priv->mark_enabled && !enable) ||
8081 	    (priv->mark_enabled && enable))
8082 		return;
8083 	for (i = 0; i < priv->rxqs_n; ++i) {
8084 		struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i);
8085 
8086 		/* With RXQ start/stop feature, RXQ might be stopped. */
8087 		if (!rxq_ctrl)
8088 			continue;
8089 		rxq_ctrl->rxq.mark = enable;
8090 	}
8091 	priv->mark_enabled = enable;
8092 }
8093 
8094 /**
8095  * Start all default actions for flows.
8096  *
8097  * @param dev
8098  *   Pointer to Ethernet device.
8099  * @return
8100  *   0 on success, a negative errno value otherwise and rte_errno is set.
8101  */
8102 int
8103 mlx5_flow_start_default(struct rte_eth_dev *dev)
8104 {
8105 	struct rte_flow_error error;
8106 
8107 	/* Make sure default copy action (reg_c[0] -> reg_b) is created. */
8108 	return flow_mreg_add_default_copy_action(dev, &error);
8109 }
8110 
8111 /**
8112  * Release key of thread specific flow workspace data.
8113  */
8114 void
8115 flow_release_workspace(void *data)
8116 {
8117 	struct mlx5_flow_workspace *wks = data;
8118 	struct mlx5_flow_workspace *next;
8119 
8120 	while (wks) {
8121 		next = wks->next;
8122 		free(wks);
8123 		wks = next;
8124 	}
8125 }
8126 
8127 /**
8128  * Get thread specific current flow workspace.
8129  *
8130  * @return pointer to thread specific flow workspace data, NULL on error.
8131  */
8132 struct mlx5_flow_workspace*
8133 mlx5_flow_get_thread_workspace(void)
8134 {
8135 	struct mlx5_flow_workspace *data;
8136 
8137 	data = mlx5_flow_os_get_specific_workspace();
8138 	MLX5_ASSERT(data && data->inuse);
8139 	if (!data || !data->inuse)
8140 		DRV_LOG(ERR, "flow workspace not initialized.");
8141 	return data;
8142 }
8143 
8144 /**
8145  * Allocate and init new flow workspace.
8146  *
8147  * @return pointer to flow workspace data, NULL on error.
8148  */
8149 static struct mlx5_flow_workspace*
8150 flow_alloc_thread_workspace(void)
8151 {
8152 	size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long));
8153 	size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512;
8154 	struct mlx5_flow_workspace *data = calloc(1, data_size +
8155 						     rss_queue_array_size);
8156 
8157 	if (!data) {
8158 		DRV_LOG(ERR, "Failed to allocate flow workspace memory.");
8159 		return NULL;
8160 	}
8161 	data->rss_desc.queue = RTE_PTR_ADD(data, data_size);
8162 	return data;
8163 }
8164 
8165 /**
8166  * Get new thread specific flow workspace.
8167  *
8168  * If current workspace inuse, create new one and set as current.
8169  *
8170  * @return pointer to thread specific flow workspace data, NULL on error.
8171  */
8172 struct mlx5_flow_workspace*
8173 mlx5_flow_push_thread_workspace(void)
8174 {
8175 	struct mlx5_flow_workspace *curr;
8176 	struct mlx5_flow_workspace *data;
8177 
8178 	curr = mlx5_flow_os_get_specific_workspace();
8179 	if (!curr) {
8180 		data = flow_alloc_thread_workspace();
8181 		if (!data)
8182 			return NULL;
8183 		mlx5_flow_os_workspace_gc_add(data);
8184 	} else if (!curr->inuse) {
8185 		data = curr;
8186 	} else if (curr->next) {
8187 		data = curr->next;
8188 	} else {
8189 		data = flow_alloc_thread_workspace();
8190 		if (!data)
8191 			return NULL;
8192 		curr->next = data;
8193 		data->prev = curr;
8194 	}
8195 	data->inuse = 1;
8196 	data->flow_idx = 0;
8197 	/* Set as current workspace */
8198 	if (mlx5_flow_os_set_specific_workspace(data))
8199 		DRV_LOG(ERR, "Failed to set flow workspace to thread.");
8200 	return data;
8201 }
8202 
8203 /**
8204  * Close current thread specific flow workspace.
8205  *
8206  * If previous workspace available, set it as current.
8207  *
8208  * @return pointer to thread specific flow workspace data, NULL on error.
8209  */
8210 void
8211 mlx5_flow_pop_thread_workspace(void)
8212 {
8213 	struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
8214 
8215 	if (!data)
8216 		return;
8217 	if (!data->inuse) {
8218 		DRV_LOG(ERR, "Failed to close unused flow workspace.");
8219 		return;
8220 	}
8221 	data->inuse = 0;
8222 	if (!data->prev)
8223 		return;
8224 	if (mlx5_flow_os_set_specific_workspace(data->prev))
8225 		DRV_LOG(ERR, "Failed to set flow workspace to thread.");
8226 }
8227 
8228 /**
8229  * Verify the flow list is empty
8230  *
8231  * @param dev
8232  *  Pointer to Ethernet device.
8233  *
8234  * @return the number of flows not released.
8235  */
8236 int
8237 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
8238 {
8239 	struct mlx5_priv *priv = dev->data->dev_private;
8240 	struct rte_flow *flow;
8241 	uint32_t idx = 0;
8242 	int ret = 0, i;
8243 
8244 	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
8245 		MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
8246 			DRV_LOG(DEBUG, "port %u flow %p still referenced",
8247 				dev->data->port_id, (void *)flow);
8248 			ret++;
8249 		}
8250 	}
8251 	return ret;
8252 }
8253 
8254 /**
8255  * Enable default hairpin egress flow.
8256  *
8257  * @param dev
8258  *   Pointer to Ethernet device.
8259  * @param sq_num
8260  *   The SQ hw number.
8261  *
8262  * @return
8263  *   0 on success, a negative errno value otherwise and rte_errno is set.
8264  */
8265 int
8266 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
8267 			    uint32_t sq_num)
8268 {
8269 	const struct rte_flow_attr attr = {
8270 		.egress = 1,
8271 		.priority = 0,
8272 	};
8273 	struct mlx5_rte_flow_item_sq queue_spec = {
8274 		.queue = sq_num,
8275 	};
8276 	struct mlx5_rte_flow_item_sq queue_mask = {
8277 		.queue = UINT32_MAX,
8278 	};
8279 	struct rte_flow_item items[] = {
8280 		{
8281 			.type = (enum rte_flow_item_type)
8282 				MLX5_RTE_FLOW_ITEM_TYPE_SQ,
8283 			.spec = &queue_spec,
8284 			.last = NULL,
8285 			.mask = &queue_mask,
8286 		},
8287 		{
8288 			.type = RTE_FLOW_ITEM_TYPE_END,
8289 		},
8290 	};
8291 	struct rte_flow_action_jump jump = {
8292 		.group = MLX5_HAIRPIN_TX_TABLE,
8293 	};
8294 	struct rte_flow_action actions[2];
8295 	uint32_t flow_idx;
8296 	struct rte_flow_error error;
8297 
8298 	actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
8299 	actions[0].conf = &jump;
8300 	actions[1].type = RTE_FLOW_ACTION_TYPE_END;
8301 	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
8302 				    &attr, items, actions, false, &error);
8303 	if (!flow_idx) {
8304 		DRV_LOG(DEBUG,
8305 			"Failed to create ctrl flow: rte_errno(%d),"
8306 			" type(%d), message(%s)",
8307 			rte_errno, error.type,
8308 			error.message ? error.message : " (no stated reason)");
8309 		return -rte_errno;
8310 	}
8311 	return 0;
8312 }
8313 
8314 /**
8315  * Enable a control flow configured from the control plane.
8316  *
8317  * @param dev
8318  *   Pointer to Ethernet device.
8319  * @param eth_spec
8320  *   An Ethernet flow spec to apply.
8321  * @param eth_mask
8322  *   An Ethernet flow mask to apply.
8323  * @param vlan_spec
8324  *   A VLAN flow spec to apply.
8325  * @param vlan_mask
8326  *   A VLAN flow mask to apply.
8327  *
8328  * @return
8329  *   0 on success, a negative errno value otherwise and rte_errno is set.
8330  */
8331 int
8332 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
8333 		    struct rte_flow_item_eth *eth_spec,
8334 		    struct rte_flow_item_eth *eth_mask,
8335 		    struct rte_flow_item_vlan *vlan_spec,
8336 		    struct rte_flow_item_vlan *vlan_mask)
8337 {
8338 	struct mlx5_priv *priv = dev->data->dev_private;
8339 	const struct rte_flow_attr attr = {
8340 		.ingress = 1,
8341 		.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
8342 	};
8343 	struct rte_flow_item items[] = {
8344 		{
8345 			.type = RTE_FLOW_ITEM_TYPE_ETH,
8346 			.spec = eth_spec,
8347 			.last = NULL,
8348 			.mask = eth_mask,
8349 		},
8350 		{
8351 			.type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
8352 					      RTE_FLOW_ITEM_TYPE_END,
8353 			.spec = vlan_spec,
8354 			.last = NULL,
8355 			.mask = vlan_mask,
8356 		},
8357 		{
8358 			.type = RTE_FLOW_ITEM_TYPE_END,
8359 		},
8360 	};
8361 	uint16_t queue[priv->reta_idx_n];
8362 	struct rte_flow_action_rss action_rss = {
8363 		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
8364 		.level = 0,
8365 		.types = priv->rss_conf.rss_hf,
8366 		.key_len = priv->rss_conf.rss_key_len,
8367 		.queue_num = priv->reta_idx_n,
8368 		.key = priv->rss_conf.rss_key,
8369 		.queue = queue,
8370 	};
8371 	struct rte_flow_action actions[] = {
8372 		{
8373 			.type = RTE_FLOW_ACTION_TYPE_RSS,
8374 			.conf = &action_rss,
8375 		},
8376 		{
8377 			.type = RTE_FLOW_ACTION_TYPE_END,
8378 		},
8379 	};
8380 	uint32_t flow_idx;
8381 	struct rte_flow_error error;
8382 	unsigned int i;
8383 
8384 	if (!priv->reta_idx_n || !priv->rxqs_n) {
8385 		return 0;
8386 	}
8387 	if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
8388 		action_rss.types = 0;
8389 	for (i = 0; i != priv->reta_idx_n; ++i)
8390 		queue[i] = (*priv->reta_idx)[i];
8391 	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
8392 				    &attr, items, actions, false, &error);
8393 	if (!flow_idx)
8394 		return -rte_errno;
8395 	return 0;
8396 }
8397 
8398 /**
8399  * Enable a flow control configured from the control plane.
8400  *
8401  * @param dev
8402  *   Pointer to Ethernet device.
8403  * @param eth_spec
8404  *   An Ethernet flow spec to apply.
8405  * @param eth_mask
8406  *   An Ethernet flow mask to apply.
8407  *
8408  * @return
8409  *   0 on success, a negative errno value otherwise and rte_errno is set.
8410  */
8411 int
8412 mlx5_ctrl_flow(struct rte_eth_dev *dev,
8413 	       struct rte_flow_item_eth *eth_spec,
8414 	       struct rte_flow_item_eth *eth_mask)
8415 {
8416 	return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
8417 }
8418 
8419 /**
8420  * Create default miss flow rule matching lacp traffic
8421  *
8422  * @param dev
8423  *   Pointer to Ethernet device.
8424  * @param eth_spec
8425  *   An Ethernet flow spec to apply.
8426  *
8427  * @return
8428  *   0 on success, a negative errno value otherwise and rte_errno is set.
8429  */
8430 int
8431 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
8432 {
8433 	/*
8434 	 * The LACP matching is done by only using ether type since using
8435 	 * a multicast dst mac causes kernel to give low priority to this flow.
8436 	 */
8437 	static const struct rte_flow_item_eth lacp_spec = {
8438 		.hdr.ether_type = RTE_BE16(0x8809),
8439 	};
8440 	static const struct rte_flow_item_eth lacp_mask = {
8441 		.hdr.ether_type = 0xffff,
8442 	};
8443 	const struct rte_flow_attr attr = {
8444 		.ingress = 1,
8445 	};
8446 	struct rte_flow_item items[] = {
8447 		{
8448 			.type = RTE_FLOW_ITEM_TYPE_ETH,
8449 			.spec = &lacp_spec,
8450 			.mask = &lacp_mask,
8451 		},
8452 		{
8453 			.type = RTE_FLOW_ITEM_TYPE_END,
8454 		},
8455 	};
8456 	struct rte_flow_action actions[] = {
8457 		{
8458 			.type = (enum rte_flow_action_type)
8459 				MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
8460 		},
8461 		{
8462 			.type = RTE_FLOW_ACTION_TYPE_END,
8463 		},
8464 	};
8465 	struct rte_flow_error error;
8466 	uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
8467 					&attr, items, actions,
8468 					false, &error);
8469 
8470 	if (!flow_idx)
8471 		return -rte_errno;
8472 	return 0;
8473 }
8474 
8475 /**
8476  * Destroy a flow.
8477  *
8478  * @see rte_flow_destroy()
8479  * @see rte_flow_ops
8480  */
8481 int
8482 mlx5_flow_destroy(struct rte_eth_dev *dev,
8483 		  struct rte_flow *flow,
8484 		  struct rte_flow_error *error __rte_unused)
8485 {
8486 	struct mlx5_priv *priv = dev->data->dev_private;
8487 	struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info;
8488 	struct mlx5_dv_flow_info *flow_info;
8489 
8490 	if (priv->sh->config.dv_flow_en == 2)
8491 		return rte_flow_error_set(error, ENOTSUP,
8492 			  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8493 			  NULL,
8494 			  "Flow non-Q destruction not supported");
8495 	flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
8496 				(uintptr_t)(void *)flow);
8497 	if (unlikely(mlx5_need_cache_flow(priv, NULL))) {
8498 		flow_info = LIST_FIRST(&mode_info->hot_upgrade);
8499 		while (flow_info) {
8500 			/* Romove the cache flow info. */
8501 			if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) {
8502 				MLX5_ASSERT(!flow_info->flow_idx_high_prio);
8503 				LIST_REMOVE(flow_info, next);
8504 				mlx5_free(flow_info->items);
8505 				mlx5_free(flow_info->actions);
8506 				mlx5_free(flow_info);
8507 				break;
8508 			}
8509 			flow_info = LIST_NEXT(flow_info, next);
8510 		}
8511 	}
8512 	return 0;
8513 }
8514 
8515 /**
8516  * Destroy all flows.
8517  *
8518  * @see rte_flow_flush()
8519  * @see rte_flow_ops
8520  */
8521 int
8522 mlx5_flow_flush(struct rte_eth_dev *dev,
8523 		struct rte_flow_error *error __rte_unused)
8524 {
8525 	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
8526 	return 0;
8527 }
8528 
8529 /**
8530  * Isolated mode.
8531  *
8532  * @see rte_flow_isolate()
8533  * @see rte_flow_ops
8534  */
8535 int
8536 mlx5_flow_isolate(struct rte_eth_dev *dev,
8537 		  int enable,
8538 		  struct rte_flow_error *error)
8539 {
8540 	struct mlx5_priv *priv = dev->data->dev_private;
8541 
8542 	if (dev->data->dev_started) {
8543 		rte_flow_error_set(error, EBUSY,
8544 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8545 				   NULL,
8546 				   "port must be stopped first");
8547 		return -rte_errno;
8548 	}
8549 	if (!enable && !priv->sh->config.repr_matching)
8550 		return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8551 					  "isolated mode cannot be disabled when "
8552 					  "representor matching is disabled");
8553 	priv->isolated = !!enable;
8554 	if (enable)
8555 		dev->dev_ops = &mlx5_dev_ops_isolate;
8556 	else
8557 		dev->dev_ops = &mlx5_dev_ops;
8558 
8559 	dev->rx_descriptor_status = mlx5_rx_descriptor_status;
8560 	dev->tx_descriptor_status = mlx5_tx_descriptor_status;
8561 
8562 	return 0;
8563 }
8564 
8565 /**
8566  * Query a flow.
8567  *
8568  * @see rte_flow_query()
8569  * @see rte_flow_ops
8570  */
8571 static int
8572 flow_drv_query(struct rte_eth_dev *dev,
8573 	       struct rte_flow *eflow,
8574 	       const struct rte_flow_action *actions,
8575 	       void *data,
8576 	       struct rte_flow_error *error)
8577 {
8578 	struct mlx5_priv *priv = dev->data->dev_private;
8579 	const struct mlx5_flow_driver_ops *fops;
8580 	struct rte_flow *flow = NULL;
8581 	enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN;
8582 
8583 	if (priv->sh->config.dv_flow_en == 2) {
8584 #ifdef HAVE_MLX5_HWS_SUPPORT
8585 		flow = eflow;
8586 		ftype = MLX5_FLOW_TYPE_HW;
8587 #endif
8588 	} else {
8589 		flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
8590 				(uintptr_t)(void *)eflow);
8591 	}
8592 	if (!flow) {
8593 		return rte_flow_error_set(error, ENOENT,
8594 			  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8595 			  NULL,
8596 			  "invalid flow handle");
8597 	}
8598 	if (ftype == MLX5_FLOW_TYPE_MIN)
8599 		ftype = flow->drv_type;
8600 	MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
8601 	fops = flow_get_drv_ops(ftype);
8602 
8603 	return fops->query(dev, flow, actions, data, error);
8604 }
8605 
8606 /**
8607  * Query a flow.
8608  *
8609  * @see rte_flow_query()
8610  * @see rte_flow_ops
8611  */
8612 int
8613 mlx5_flow_query(struct rte_eth_dev *dev,
8614 		struct rte_flow *flow,
8615 		const struct rte_flow_action *actions,
8616 		void *data,
8617 		struct rte_flow_error *error)
8618 {
8619 	int ret;
8620 
8621 	ret = flow_drv_query(dev, flow, actions, data,
8622 			     error);
8623 	if (ret < 0)
8624 		return ret;
8625 	return 0;
8626 }
8627 
8628 /**
8629  * Get rte_flow callbacks.
8630  *
8631  * @param dev
8632  *   Pointer to Ethernet device structure.
8633  * @param ops
8634  *   Pointer to operation-specific structure.
8635  *
8636  * @return 0
8637  */
8638 int
8639 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
8640 		  const struct rte_flow_ops **ops)
8641 {
8642 	*ops = &mlx5_flow_ops;
8643 	return 0;
8644 }
8645 
8646 /**
8647  * Validate meter policy actions.
8648  * Dispatcher for action type specific validation.
8649  *
8650  * @param[in] dev
8651  *   Pointer to the Ethernet device structure.
8652  * @param[in] action
8653  *   The meter policy action object to validate.
8654  * @param[in] attr
8655  *   Attributes of flow to determine steering domain.
8656  * @param[out] is_rss
8657  *   Is RSS or not.
8658  * @param[out] domain_bitmap
8659  *   Domain bitmap.
8660  * @param[out] is_def_policy
8661  *   Is default policy or not.
8662  * @param[out] error
8663  *   Perform verbose error reporting if not NULL. Initialized in case of
8664  *   error only.
8665  *
8666  * @return
8667  *   0 on success, otherwise negative errno value.
8668  */
8669 int
8670 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
8671 			const struct rte_flow_action *actions[RTE_COLORS],
8672 			struct rte_flow_attr *attr,
8673 			bool *is_rss,
8674 			uint8_t *domain_bitmap,
8675 			uint8_t *policy_mode,
8676 			struct rte_mtr_error *error)
8677 {
8678 	const struct mlx5_flow_driver_ops *fops;
8679 
8680 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8681 	return fops->validate_mtr_acts(dev, actions, attr, is_rss,
8682 				       domain_bitmap, policy_mode, error);
8683 }
8684 
8685 /**
8686  * Destroy the meter table set.
8687  *
8688  * @param[in] dev
8689  *   Pointer to Ethernet device.
8690  * @param[in] mtr_policy
8691  *   Meter policy struct.
8692  */
8693 void
8694 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
8695 		      struct mlx5_flow_meter_policy *mtr_policy)
8696 {
8697 	const struct mlx5_flow_driver_ops *fops;
8698 
8699 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8700 	fops->destroy_mtr_acts(dev, mtr_policy);
8701 }
8702 
8703 /**
8704  * Create policy action, lock free,
8705  * (mutex should be acquired by caller).
8706  * Dispatcher for action type specific call.
8707  *
8708  * @param[in] dev
8709  *   Pointer to the Ethernet device structure.
8710  * @param[in] mtr_policy
8711  *   Meter policy struct.
8712  * @param[in] action
8713  *   Action specification used to create meter actions.
8714  * @param[in] attr
8715  *   Flow rule attributes.
8716  * @param[out] error
8717  *   Perform verbose error reporting if not NULL. Initialized in case of
8718  *   error only.
8719  *
8720  * @return
8721  *   0 on success, otherwise negative errno value.
8722  */
8723 int
8724 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
8725 		      struct mlx5_flow_meter_policy *mtr_policy,
8726 		      const struct rte_flow_action *actions[RTE_COLORS],
8727 		      struct rte_flow_attr *attr,
8728 		      struct rte_mtr_error *error)
8729 {
8730 	const struct mlx5_flow_driver_ops *fops;
8731 
8732 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8733 	return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error);
8734 }
8735 
8736 /**
8737  * Create policy rules, lock free,
8738  * (mutex should be acquired by caller).
8739  * Dispatcher for action type specific call.
8740  *
8741  * @param[in] dev
8742  *   Pointer to the Ethernet device structure.
8743  * @param[in] mtr_policy
8744  *   Meter policy struct.
8745  *
8746  * @return
8747  *   0 on success, -1 otherwise.
8748  */
8749 int
8750 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
8751 			     struct mlx5_flow_meter_policy *mtr_policy)
8752 {
8753 	const struct mlx5_flow_driver_ops *fops;
8754 
8755 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8756 	return fops->create_policy_rules(dev, mtr_policy);
8757 }
8758 
8759 /**
8760  * Destroy policy rules, lock free,
8761  * (mutex should be acquired by caller).
8762  * Dispatcher for action type specific call.
8763  *
8764  * @param[in] dev
8765  *   Pointer to the Ethernet device structure.
8766  * @param[in] mtr_policy
8767  *   Meter policy struct.
8768  */
8769 void
8770 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
8771 			     struct mlx5_flow_meter_policy *mtr_policy)
8772 {
8773 	const struct mlx5_flow_driver_ops *fops;
8774 
8775 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8776 	fops->destroy_policy_rules(dev, mtr_policy);
8777 }
8778 
8779 /**
8780  * Destroy the default policy table set.
8781  *
8782  * @param[in] dev
8783  *   Pointer to Ethernet device.
8784  */
8785 void
8786 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
8787 {
8788 	const struct mlx5_flow_driver_ops *fops;
8789 
8790 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8791 	fops->destroy_def_policy(dev);
8792 }
8793 
8794 /**
8795  * Destroy the default policy table set.
8796  *
8797  * @param[in] dev
8798  *   Pointer to Ethernet device.
8799  *
8800  * @return
8801  *   0 on success, -1 otherwise.
8802  */
8803 int
8804 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
8805 {
8806 	const struct mlx5_flow_driver_ops *fops;
8807 
8808 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8809 	return fops->create_def_policy(dev);
8810 }
8811 
8812 /**
8813  * Create the needed meter and suffix tables.
8814  *
8815  * @param[in] dev
8816  *   Pointer to Ethernet device.
8817  *
8818  * @return
8819  *   0 on success, -1 otherwise.
8820  */
8821 int
8822 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
8823 			struct mlx5_flow_meter_info *fm,
8824 			uint32_t mtr_idx,
8825 			uint8_t domain_bitmap)
8826 {
8827 	const struct mlx5_flow_driver_ops *fops;
8828 
8829 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8830 	return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
8831 }
8832 
8833 /**
8834  * Destroy the meter table set.
8835  *
8836  * @param[in] dev
8837  *   Pointer to Ethernet device.
8838  * @param[in] tbl
8839  *   Pointer to the meter table set.
8840  */
8841 void
8842 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
8843 			   struct mlx5_flow_meter_info *fm)
8844 {
8845 	const struct mlx5_flow_driver_ops *fops;
8846 
8847 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8848 	fops->destroy_mtr_tbls(dev, fm);
8849 }
8850 
8851 /**
8852  * Destroy the global meter drop table.
8853  *
8854  * @param[in] dev
8855  *   Pointer to Ethernet device.
8856  */
8857 void
8858 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
8859 {
8860 	const struct mlx5_flow_driver_ops *fops;
8861 
8862 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8863 	fops->destroy_mtr_drop_tbls(dev);
8864 }
8865 
8866 /**
8867  * Destroy the sub policy table with RX queue.
8868  *
8869  * @param[in] dev
8870  *   Pointer to Ethernet device.
8871  * @param[in] mtr_policy
8872  *   Pointer to meter policy table.
8873  */
8874 void
8875 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
8876 		struct mlx5_flow_meter_policy *mtr_policy)
8877 {
8878 	const struct mlx5_flow_driver_ops *fops;
8879 
8880 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8881 	fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
8882 }
8883 
8884 /**
8885  * Allocate the needed aso flow meter id.
8886  *
8887  * @param[in] dev
8888  *   Pointer to Ethernet device.
8889  *
8890  * @return
8891  *   Index to aso flow meter on success, NULL otherwise.
8892  */
8893 uint32_t
8894 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
8895 {
8896 	const struct mlx5_flow_driver_ops *fops;
8897 
8898 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8899 	return fops->create_meter(dev);
8900 }
8901 
8902 /**
8903  * Free the aso flow meter id.
8904  *
8905  * @param[in] dev
8906  *   Pointer to Ethernet device.
8907  * @param[in] mtr_idx
8908  *  Index to aso flow meter to be free.
8909  *
8910  * @return
8911  *   0 on success.
8912  */
8913 void
8914 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
8915 {
8916 	const struct mlx5_flow_driver_ops *fops;
8917 
8918 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8919 	fops->free_meter(dev, mtr_idx);
8920 }
8921 
8922 /**
8923  * Allocate a counter.
8924  *
8925  * @param[in] dev
8926  *   Pointer to Ethernet device structure.
8927  *
8928  * @return
8929  *   Index to allocated counter  on success, 0 otherwise.
8930  */
8931 uint32_t
8932 mlx5_counter_alloc(struct rte_eth_dev *dev)
8933 {
8934 	struct rte_flow_attr attr = { .transfer = 0 };
8935 
8936 	return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc
8937 		(dev);
8938 }
8939 
8940 /**
8941  * Free a counter.
8942  *
8943  * @param[in] dev
8944  *   Pointer to Ethernet device structure.
8945  * @param[in] cnt
8946  *   Index to counter to be free.
8947  */
8948 void
8949 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
8950 {
8951 	struct rte_flow_attr attr = { .transfer = 0 };
8952 
8953 	flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt);
8954 }
8955 
8956 /**
8957  * Query counter statistics.
8958  *
8959  * @param[in] dev
8960  *   Pointer to Ethernet device structure.
8961  * @param[in] cnt
8962  *   Index to counter to query.
8963  * @param[in] clear
8964  *   Set to clear counter statistics.
8965  * @param[out] pkts
8966  *   The counter hits packets number to save.
8967  * @param[out] bytes
8968  *   The counter hits bytes number to save.
8969  *
8970  * @return
8971  *   0 on success, a negative errno value otherwise.
8972  */
8973 int
8974 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
8975 		   bool clear, uint64_t *pkts, uint64_t *bytes, void **action)
8976 {
8977 	struct rte_flow_attr attr = { .transfer = 0 };
8978 
8979 	return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query
8980 		(dev, cnt, clear, pkts, bytes, action);
8981 }
8982 
8983 /**
8984  * Get information about HWS pre-configurable resources.
8985  *
8986  * @param[in] dev
8987  *   Pointer to the rte_eth_dev structure.
8988  * @param[out] port_info
8989  *   Pointer to port information.
8990  * @param[out] queue_info
8991  *   Pointer to queue information.
8992  * @param[out] error
8993  *   Pointer to error structure.
8994  *
8995  * @return
8996  *   0 on success, a negative errno value otherwise and rte_errno is set.
8997  */
8998 static int
8999 mlx5_flow_info_get(struct rte_eth_dev *dev,
9000 		   struct rte_flow_port_info *port_info,
9001 		   struct rte_flow_queue_info *queue_info,
9002 		   struct rte_flow_error *error)
9003 {
9004 	const struct mlx5_flow_driver_ops *fops;
9005 	struct rte_flow_attr attr = {0};
9006 
9007 	if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW)
9008 		return rte_flow_error_set(error, ENOTSUP,
9009 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9010 				NULL,
9011 				"info get with incorrect steering mode");
9012 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9013 	return fops->info_get(dev, port_info, queue_info, error);
9014 }
9015 
9016 /**
9017  * Configure port HWS resources.
9018  *
9019  * @param[in] dev
9020  *   Pointer to the rte_eth_dev structure.
9021  * @param[in] port_attr
9022  *   Port configuration attributes.
9023  * @param[in] nb_queue
9024  *   Number of queue.
9025  * @param[in] queue_attr
9026  *   Array that holds attributes for each flow queue.
9027  * @param[out] error
9028  *   Pointer to error structure.
9029  *
9030  * @return
9031  *   0 on success, a negative errno value otherwise and rte_errno is set.
9032  */
9033 static int
9034 mlx5_flow_port_configure(struct rte_eth_dev *dev,
9035 			 const struct rte_flow_port_attr *port_attr,
9036 			 uint16_t nb_queue,
9037 			 const struct rte_flow_queue_attr *queue_attr[],
9038 			 struct rte_flow_error *error)
9039 {
9040 	const struct mlx5_flow_driver_ops *fops;
9041 	struct rte_flow_attr attr = {0};
9042 
9043 	if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW)
9044 		return rte_flow_error_set(error, ENOTSUP,
9045 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9046 				NULL,
9047 				"port configure with incorrect steering mode");
9048 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9049 	return fops->configure(dev, port_attr, nb_queue, queue_attr, error);
9050 }
9051 
9052 /**
9053  * Validate item template.
9054  *
9055  * @param[in] dev
9056  *   Pointer to the rte_eth_dev structure.
9057  * @param[in] attr
9058  *   Pointer to the item template attributes.
9059  * @param[in] items
9060  *   The template item pattern.
9061  * @param[out] error
9062  *   Pointer to error structure.
9063  *
9064  * @return
9065  *   0 on success, a negative errno value otherwise and rte_errno is set.
9066  */
9067 int
9068 mlx5_flow_pattern_validate(struct rte_eth_dev *dev,
9069 		const struct rte_flow_pattern_template_attr *attr,
9070 		const struct rte_flow_item items[],
9071 		struct rte_flow_error *error)
9072 {
9073 	const struct mlx5_flow_driver_ops *fops;
9074 	struct rte_flow_attr fattr = {0};
9075 
9076 	if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) {
9077 		rte_flow_error_set(error, ENOTSUP,
9078 			RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9079 			"pattern validate with incorrect steering mode");
9080 		return -ENOTSUP;
9081 	}
9082 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9083 	return fops->pattern_validate(dev, attr, items, error);
9084 }
9085 
9086 /**
9087  * Create flow item template.
9088  *
9089  * @param[in] dev
9090  *   Pointer to the rte_eth_dev structure.
9091  * @param[in] attr
9092  *   Pointer to the item template attributes.
9093  * @param[in] items
9094  *   The template item pattern.
9095  * @param[out] error
9096  *   Pointer to error structure.
9097  *
9098  * @return
9099  *   0 on success, a negative errno value otherwise and rte_errno is set.
9100  */
9101 static struct rte_flow_pattern_template *
9102 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev,
9103 		const struct rte_flow_pattern_template_attr *attr,
9104 		const struct rte_flow_item items[],
9105 		struct rte_flow_error *error)
9106 {
9107 	const struct mlx5_flow_driver_ops *fops;
9108 	struct rte_flow_attr fattr = {0};
9109 
9110 	if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) {
9111 		rte_flow_error_set(error, ENOTSUP,
9112 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9113 				NULL,
9114 				"pattern create with incorrect steering mode");
9115 		return NULL;
9116 	}
9117 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9118 	return fops->pattern_template_create(dev, attr, items, error);
9119 }
9120 
9121 /**
9122  * Destroy flow item template.
9123  *
9124  * @param[in] dev
9125  *   Pointer to the rte_eth_dev structure.
9126  * @param[in] template
9127  *   Pointer to the item template to be destroyed.
9128  * @param[out] error
9129  *   Pointer to error structure.
9130  *
9131  * @return
9132  *   0 on success, a negative errno value otherwise and rte_errno is set.
9133  */
9134 static int
9135 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev,
9136 				   struct rte_flow_pattern_template *template,
9137 				   struct rte_flow_error *error)
9138 {
9139 	const struct mlx5_flow_driver_ops *fops;
9140 	struct rte_flow_attr attr = {0};
9141 
9142 	if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW)
9143 		return rte_flow_error_set(error, ENOTSUP,
9144 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9145 				NULL,
9146 				"pattern destroy with incorrect steering mode");
9147 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9148 	return fops->pattern_template_destroy(dev, template, error);
9149 }
9150 
9151 /**
9152  * Validate flow actions template.
9153  *
9154  * @param[in] dev
9155  *   Pointer to the rte_eth_dev structure.
9156  * @param[in] attr
9157  *   Pointer to the action template attributes.
9158  * @param[in] actions
9159  *   Associated actions (list terminated by the END action).
9160  * @param[in] masks
9161  *   List of actions that marks which of the action's member is constant.
9162  * @param[out] error
9163  *   Pointer to error structure.
9164  *
9165  * @return
9166  *   0 on success, a negative errno value otherwise and rte_errno is set.
9167  */
9168 int
9169 mlx5_flow_actions_validate(struct rte_eth_dev *dev,
9170 			const struct rte_flow_actions_template_attr *attr,
9171 			const struct rte_flow_action actions[],
9172 			const struct rte_flow_action masks[],
9173 			struct rte_flow_error *error)
9174 {
9175 	const struct mlx5_flow_driver_ops *fops;
9176 	struct rte_flow_attr fattr = {0};
9177 
9178 	if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) {
9179 		rte_flow_error_set(error, ENOTSUP,
9180 			RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9181 			"actions validate with incorrect steering mode");
9182 		return -ENOTSUP;
9183 	}
9184 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9185 	return fops->actions_validate(dev, attr, actions, masks, error);
9186 }
9187 
9188 /**
9189  * Create flow item template.
9190  *
9191  * @param[in] dev
9192  *   Pointer to the rte_eth_dev structure.
9193  * @param[in] attr
9194  *   Pointer to the action template attributes.
9195  * @param[in] actions
9196  *   Associated actions (list terminated by the END action).
9197  * @param[in] masks
9198  *   List of actions that marks which of the action's member is constant.
9199  * @param[out] error
9200  *   Pointer to error structure.
9201  *
9202  * @return
9203  *   0 on success, a negative errno value otherwise and rte_errno is set.
9204  */
9205 static struct rte_flow_actions_template *
9206 mlx5_flow_actions_template_create(struct rte_eth_dev *dev,
9207 			const struct rte_flow_actions_template_attr *attr,
9208 			const struct rte_flow_action actions[],
9209 			const struct rte_flow_action masks[],
9210 			struct rte_flow_error *error)
9211 {
9212 	const struct mlx5_flow_driver_ops *fops;
9213 	struct rte_flow_attr fattr = {0};
9214 
9215 	if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) {
9216 		rte_flow_error_set(error, ENOTSUP,
9217 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9218 				NULL,
9219 				"action create with incorrect steering mode");
9220 		return NULL;
9221 	}
9222 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9223 	return fops->actions_template_create(dev, attr, actions, masks, error);
9224 }
9225 
9226 /**
9227  * Destroy flow action template.
9228  *
9229  * @param[in] dev
9230  *   Pointer to the rte_eth_dev structure.
9231  * @param[in] template
9232  *   Pointer to the action template to be destroyed.
9233  * @param[out] error
9234  *   Pointer to error structure.
9235  *
9236  * @return
9237  *   0 on success, a negative errno value otherwise and rte_errno is set.
9238  */
9239 static int
9240 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev,
9241 				   struct rte_flow_actions_template *template,
9242 				   struct rte_flow_error *error)
9243 {
9244 	const struct mlx5_flow_driver_ops *fops;
9245 	struct rte_flow_attr attr = {0};
9246 
9247 	if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW)
9248 		return rte_flow_error_set(error, ENOTSUP,
9249 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9250 				NULL,
9251 				"action destroy with incorrect steering mode");
9252 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9253 	return fops->actions_template_destroy(dev, template, error);
9254 }
9255 
9256 /**
9257  * Create flow table.
9258  *
9259  * @param[in] dev
9260  *   Pointer to the rte_eth_dev structure.
9261  * @param[in] attr
9262  *   Pointer to the table attributes.
9263  * @param[in] item_templates
9264  *   Item template array to be binded to the table.
9265  * @param[in] nb_item_templates
9266  *   Number of item template.
9267  * @param[in] action_templates
9268  *   Action template array to be binded to the table.
9269  * @param[in] nb_action_templates
9270  *   Number of action template.
9271  * @param[out] error
9272  *   Pointer to error structure.
9273  *
9274  * @return
9275  *    Table on success, NULL otherwise and rte_errno is set.
9276  */
9277 static struct rte_flow_template_table *
9278 mlx5_flow_table_create(struct rte_eth_dev *dev,
9279 		       const struct rte_flow_template_table_attr *attr,
9280 		       struct rte_flow_pattern_template *item_templates[],
9281 		       uint8_t nb_item_templates,
9282 		       struct rte_flow_actions_template *action_templates[],
9283 		       uint8_t nb_action_templates,
9284 		       struct rte_flow_error *error)
9285 {
9286 	const struct mlx5_flow_driver_ops *fops;
9287 	struct rte_flow_attr fattr = {0};
9288 
9289 	if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) {
9290 		rte_flow_error_set(error, ENOTSUP,
9291 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9292 				NULL,
9293 				"table create with incorrect steering mode");
9294 		return NULL;
9295 	}
9296 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9297 	return fops->template_table_create(dev,
9298 					   attr,
9299 					   item_templates,
9300 					   nb_item_templates,
9301 					   action_templates,
9302 					   nb_action_templates,
9303 					   error);
9304 }
9305 
9306 /**
9307  * PMD destroy flow table.
9308  *
9309  * @param[in] dev
9310  *   Pointer to the rte_eth_dev structure.
9311  * @param[in] table
9312  *   Pointer to the table to be destroyed.
9313  * @param[out] error
9314  *   Pointer to error structure.
9315  *
9316  * @return
9317  *   0 on success, a negative errno value otherwise and rte_errno is set.
9318  */
9319 static int
9320 mlx5_flow_table_destroy(struct rte_eth_dev *dev,
9321 			struct rte_flow_template_table *table,
9322 			struct rte_flow_error *error)
9323 {
9324 	const struct mlx5_flow_driver_ops *fops;
9325 	struct rte_flow_attr attr = {0};
9326 
9327 	if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW)
9328 		return rte_flow_error_set(error, ENOTSUP,
9329 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9330 				NULL,
9331 				"table destroy with incorrect steering mode");
9332 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9333 	return fops->template_table_destroy(dev, table, error);
9334 }
9335 
9336 /**
9337  * PMD group set miss actions.
9338  *
9339  * @param[in] dev
9340  *   Pointer to the rte_eth_dev structure.
9341  * @param[in] attr
9342  *   Pointer to group attributes
9343  * @param[in] actions
9344  *   Array of actions
9345  * @param[out] error
9346  *   Pointer to error structure.
9347  *
9348  * @return
9349  *   0 on success, a negative errno value otherwise and rte_errno is set.
9350  */
9351 static int
9352 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev,
9353 				 uint32_t group_id,
9354 				 const struct rte_flow_group_attr *attr,
9355 				 const struct rte_flow_action actions[],
9356 				 struct rte_flow_error *error)
9357 {
9358 	const struct mlx5_flow_driver_ops *fops;
9359 	struct rte_flow_attr fattr = {0};
9360 
9361 	if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW)
9362 		return rte_flow_error_set(error, ENOTSUP,
9363 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9364 				NULL,
9365 				"group set miss actions with incorrect steering mode");
9366 	fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
9367 	return fops->group_set_miss_actions(dev, group_id, attr, actions, error);
9368 }
9369 
9370 /**
9371  * Allocate a new memory for the counter values wrapped by all the needed
9372  * management.
9373  *
9374  * @param[in] sh
9375  *   Pointer to mlx5_dev_ctx_shared object.
9376  *
9377  * @return
9378  *   0 on success, a negative errno value otherwise.
9379  */
9380 static int
9381 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
9382 {
9383 	struct mlx5_counter_stats_mem_mng *mem_mng;
9384 	volatile struct flow_counter_stats *raw_data;
9385 	int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES;
9386 	int size = (sizeof(struct flow_counter_stats) *
9387 			MLX5_COUNTERS_PER_POOL +
9388 			sizeof(struct mlx5_counter_stats_raw)) * raws_n +
9389 			sizeof(struct mlx5_counter_stats_mem_mng);
9390 	size_t pgsize = rte_mem_page_size();
9391 	uint8_t *mem;
9392 	int ret;
9393 	int i;
9394 
9395 	if (pgsize == (size_t)-1) {
9396 		DRV_LOG(ERR, "Failed to get mem page size");
9397 		rte_errno = ENOMEM;
9398 		return -ENOMEM;
9399 	}
9400 	mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
9401 	if (!mem) {
9402 		rte_errno = ENOMEM;
9403 		return -ENOMEM;
9404 	}
9405 	mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
9406 	size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
9407 	ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd,
9408 					  sh->cdev->pdn, mem, size,
9409 					  &mem_mng->wm);
9410 	if (ret) {
9411 		rte_errno = errno;
9412 		mlx5_free(mem);
9413 		return -rte_errno;
9414 	}
9415 	mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
9416 	raw_data = (volatile struct flow_counter_stats *)mem;
9417 	for (i = 0; i < raws_n; ++i) {
9418 		mem_mng->raws[i].mem_mng = mem_mng;
9419 		mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
9420 	}
9421 	for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
9422 		LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws,
9423 				 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i,
9424 				 next);
9425 	LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next);
9426 	sh->sws_cmng.mem_mng = mem_mng;
9427 	return 0;
9428 }
9429 
9430 /**
9431  * Set the statistic memory to the new counter pool.
9432  *
9433  * @param[in] sh
9434  *   Pointer to mlx5_dev_ctx_shared object.
9435  * @param[in] pool
9436  *   Pointer to the pool to set the statistic memory.
9437  *
9438  * @return
9439  *   0 on success, a negative errno value otherwise.
9440  */
9441 static int
9442 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
9443 			       struct mlx5_flow_counter_pool *pool)
9444 {
9445 	struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng;
9446 	/* Resize statistic memory once used out. */
9447 	if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) &&
9448 	    mlx5_flow_create_counter_stat_mem_mng(sh)) {
9449 		DRV_LOG(ERR, "Cannot resize counter stat mem.");
9450 		return -1;
9451 	}
9452 	rte_spinlock_lock(&pool->sl);
9453 	pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK;
9454 	rte_spinlock_unlock(&pool->sl);
9455 	pool->raw_hw = NULL;
9456 	return 0;
9457 }
9458 
9459 #define MLX5_POOL_QUERY_FREQ_US 1000000
9460 
9461 /**
9462  * Set the periodic procedure for triggering asynchronous batch queries for all
9463  * the counter pools.
9464  *
9465  * @param[in] sh
9466  *   Pointer to mlx5_dev_ctx_shared object.
9467  */
9468 void
9469 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
9470 {
9471 	uint32_t pools_n, us;
9472 
9473 	pools_n = rte_atomic_load_explicit(&sh->sws_cmng.n_valid, rte_memory_order_relaxed);
9474 	us = MLX5_POOL_QUERY_FREQ_US / pools_n;
9475 	DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
9476 	if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
9477 		sh->sws_cmng.query_thread_on = 0;
9478 		DRV_LOG(ERR, "Cannot reinitialize query alarm");
9479 	} else {
9480 		sh->sws_cmng.query_thread_on = 1;
9481 	}
9482 }
9483 
9484 /**
9485  * The periodic procedure for triggering asynchronous batch queries for all the
9486  * counter pools. This function is probably called by the host thread.
9487  *
9488  * @param[in] arg
9489  *   The parameter for the alarm process.
9490  */
9491 void
9492 mlx5_flow_query_alarm(void *arg)
9493 {
9494 	struct mlx5_dev_ctx_shared *sh = arg;
9495 	struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng;
9496 	uint16_t pool_index = cmng->pool_index;
9497 	struct mlx5_flow_counter_pool *pool;
9498 	uint16_t n_valid;
9499 	int ret;
9500 
9501 	if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES)
9502 		goto set_alarm;
9503 	rte_spinlock_lock(&cmng->pool_update_sl);
9504 	pool = cmng->pools[pool_index];
9505 	n_valid = cmng->n_valid;
9506 	rte_spinlock_unlock(&cmng->pool_update_sl);
9507 	/* Set the statistic memory to the new created pool. */
9508 	if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
9509 		goto set_alarm;
9510 	if (pool->raw_hw)
9511 		/* There is a pool query in progress. */
9512 		goto set_alarm;
9513 	pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws);
9514 	if (!pool->raw_hw)
9515 		/* No free counter statistics raw memory. */
9516 		goto set_alarm;
9517 	/*
9518 	 * Identify the counters released between query trigger and query
9519 	 * handle more efficiently. The counter released in this gap period
9520 	 * should wait for a new round of query as the new arrived packets
9521 	 * will not be taken into account.
9522 	 */
9523 	pool->query_gen++;
9524 	ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
9525 					       MLX5_COUNTERS_PER_POOL,
9526 					       NULL, NULL,
9527 					       pool->raw_hw->mem_mng->wm.lkey,
9528 					       (void *)(uintptr_t)
9529 					       pool->raw_hw->data,
9530 					       sh->devx_comp,
9531 					       (uint64_t)(uintptr_t)pool);
9532 	if (ret) {
9533 		DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
9534 			" %d", pool->min_dcs->id);
9535 		pool->raw_hw = NULL;
9536 		goto set_alarm;
9537 	}
9538 	LIST_REMOVE(pool->raw_hw, next);
9539 	cmng->pending_queries++;
9540 	pool_index++;
9541 	if (pool_index >= n_valid)
9542 		pool_index = 0;
9543 set_alarm:
9544 	cmng->pool_index = pool_index;
9545 	mlx5_set_query_alarm(sh);
9546 }
9547 
9548 /**
9549  * Check and callback event for new aged flow in the counter pool
9550  *
9551  * @param[in] sh
9552  *   Pointer to mlx5_dev_ctx_shared object.
9553  * @param[in] pool
9554  *   Pointer to Current counter pool.
9555  */
9556 static void
9557 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
9558 		   struct mlx5_flow_counter_pool *pool)
9559 {
9560 	struct mlx5_priv *priv;
9561 	struct mlx5_flow_counter *cnt;
9562 	struct mlx5_age_info *age_info;
9563 	struct mlx5_age_param *age_param;
9564 	struct mlx5_counter_stats_raw *cur = pool->raw_hw;
9565 	struct mlx5_counter_stats_raw *prev = pool->raw;
9566 	const uint64_t curr_time = MLX5_CURR_TIME_SEC;
9567 	const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
9568 	uint16_t expected = AGE_CANDIDATE;
9569 	uint32_t i;
9570 
9571 	pool->time_of_last_age_check = curr_time;
9572 	for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
9573 		cnt = MLX5_POOL_GET_CNT(pool, i);
9574 		age_param = MLX5_CNT_TO_AGE(cnt);
9575 		if (rte_atomic_load_explicit(&age_param->state,
9576 				    rte_memory_order_relaxed) != AGE_CANDIDATE)
9577 			continue;
9578 		if (cur->data[i].hits != prev->data[i].hits) {
9579 			rte_atomic_store_explicit(&age_param->sec_since_last_hit, 0,
9580 					 rte_memory_order_relaxed);
9581 			continue;
9582 		}
9583 		if (rte_atomic_fetch_add_explicit(&age_param->sec_since_last_hit,
9584 				       time_delta,
9585 				       rte_memory_order_relaxed) + time_delta <= age_param->timeout)
9586 			continue;
9587 		/**
9588 		 * Hold the lock first, or if between the
9589 		 * state AGE_TMOUT and tailq operation the
9590 		 * release happened, the release procedure
9591 		 * may delete a non-existent tailq node.
9592 		 */
9593 		priv = rte_eth_devices[age_param->port_id].data->dev_private;
9594 		age_info = GET_PORT_AGE_INFO(priv);
9595 		rte_spinlock_lock(&age_info->aged_sl);
9596 		if (rte_atomic_compare_exchange_strong_explicit(&age_param->state, &expected,
9597 						AGE_TMOUT,
9598 						rte_memory_order_relaxed,
9599 						rte_memory_order_relaxed)) {
9600 			TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
9601 			MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
9602 		}
9603 		rte_spinlock_unlock(&age_info->aged_sl);
9604 	}
9605 	mlx5_age_event_prepare(sh);
9606 }
9607 
9608 /**
9609  * Handler for the HW respond about ready values from an asynchronous batch
9610  * query. This function is probably called by the host thread.
9611  *
9612  * @param[in] sh
9613  *   The pointer to the shared device context.
9614  * @param[in] async_id
9615  *   The Devx async ID.
9616  * @param[in] status
9617  *   The status of the completion.
9618  */
9619 void
9620 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
9621 				  uint64_t async_id, int status)
9622 {
9623 	struct mlx5_flow_counter_pool *pool =
9624 		(struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
9625 	struct mlx5_counter_stats_raw *raw_to_free;
9626 	uint8_t query_gen = pool->query_gen ^ 1;
9627 	struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng;
9628 	enum mlx5_counter_type cnt_type =
9629 		pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
9630 				MLX5_COUNTER_TYPE_ORIGIN;
9631 
9632 	if (unlikely(status)) {
9633 		raw_to_free = pool->raw_hw;
9634 	} else {
9635 		raw_to_free = pool->raw;
9636 		if (pool->is_aged)
9637 			mlx5_flow_aging_check(sh, pool);
9638 		rte_spinlock_lock(&pool->sl);
9639 		pool->raw = pool->raw_hw;
9640 		rte_spinlock_unlock(&pool->sl);
9641 		/* Be sure the new raw counters data is updated in memory. */
9642 		rte_io_wmb();
9643 		if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
9644 			rte_spinlock_lock(&cmng->csl[cnt_type]);
9645 			TAILQ_CONCAT(&cmng->counters[cnt_type],
9646 				     &pool->counters[query_gen], next);
9647 			rte_spinlock_unlock(&cmng->csl[cnt_type]);
9648 		}
9649 	}
9650 	LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next);
9651 	pool->raw_hw = NULL;
9652 	sh->sws_cmng.pending_queries--;
9653 }
9654 
9655 static int
9656 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
9657 		    const struct flow_grp_info *grp_info,
9658 		    struct rte_flow_error *error)
9659 {
9660 	if (grp_info->transfer && grp_info->external &&
9661 	    grp_info->fdb_def_rule) {
9662 		if (group == UINT32_MAX)
9663 			return rte_flow_error_set
9664 						(error, EINVAL,
9665 						 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
9666 						 NULL,
9667 						 "group index not supported");
9668 		*table = group + 1;
9669 	} else {
9670 		*table = group;
9671 	}
9672 	DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
9673 	return 0;
9674 }
9675 
9676 /**
9677  * Translate the rte_flow group index to HW table value.
9678  *
9679  * If tunnel offload is disabled, all group ids converted to flow table
9680  * id using the standard method.
9681  * If tunnel offload is enabled, group id can be converted using the
9682  * standard or tunnel conversion method. Group conversion method
9683  * selection depends on flags in `grp_info` parameter:
9684  * - Internal (grp_info.external == 0) groups conversion uses the
9685  *   standard method.
9686  * - Group ids in JUMP action converted with the tunnel conversion.
9687  * - Group id in rule attribute conversion depends on a rule type and
9688  *   group id value:
9689  *   ** non zero group attributes converted with the tunnel method
9690  *   ** zero group attribute in non-tunnel rule is converted using the
9691  *      standard method - there's only one root table
9692  *   ** zero group attribute in steer tunnel rule is converted with the
9693  *      standard method - single root table
9694  *   ** zero group attribute in match tunnel rule is a special OvS
9695  *      case: that value is used for portability reasons. That group
9696  *      id is converted with the tunnel conversion method.
9697  *
9698  * @param[in] dev
9699  *   Port device
9700  * @param[in] tunnel
9701  *   PMD tunnel offload object
9702  * @param[in] group
9703  *   rte_flow group index value.
9704  * @param[out] table
9705  *   HW table value.
9706  * @param[in] grp_info
9707  *   flags used for conversion
9708  * @param[out] error
9709  *   Pointer to error structure.
9710  *
9711  * @return
9712  *   0 on success, a negative errno value otherwise and rte_errno is set.
9713  */
9714 int
9715 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
9716 			 const struct mlx5_flow_tunnel *tunnel,
9717 			 uint32_t group, uint32_t *table,
9718 			 const struct flow_grp_info *grp_info,
9719 			 struct rte_flow_error *error)
9720 {
9721 	int ret;
9722 	bool standard_translation;
9723 
9724 	if (!grp_info->skip_scale && grp_info->external &&
9725 	    group < MLX5_MAX_TABLES_EXTERNAL)
9726 		group *= MLX5_FLOW_TABLE_FACTOR;
9727 	if (is_tunnel_offload_active(dev)) {
9728 		standard_translation = !grp_info->external ||
9729 					grp_info->std_tbl_fix;
9730 	} else {
9731 		standard_translation = true;
9732 	}
9733 	DRV_LOG(DEBUG,
9734 		"port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
9735 		dev->data->port_id, group, grp_info->transfer,
9736 		grp_info->external, grp_info->fdb_def_rule,
9737 		standard_translation ? "STANDARD" : "TUNNEL");
9738 	if (standard_translation)
9739 		ret = flow_group_to_table(dev->data->port_id, group, table,
9740 					  grp_info, error);
9741 	else
9742 		ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
9743 						      table, error);
9744 
9745 	return ret;
9746 }
9747 
9748 /**
9749  * Discover availability of metadata reg_c's.
9750  *
9751  * Iteratively use test flows to check availability.
9752  *
9753  * @param[in] dev
9754  *   Pointer to the Ethernet device structure.
9755  *
9756  * @return
9757  *   0 on success, a negative errno value otherwise and rte_errno is set.
9758  */
9759 int
9760 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
9761 {
9762 	struct mlx5_priv *priv = dev->data->dev_private;
9763 	enum modify_reg idx;
9764 	int n = 0;
9765 
9766 	/* reg_c[0] and reg_c[1] are reserved. */
9767 	priv->sh->flow_mreg_c[n++] = REG_C_0;
9768 	priv->sh->flow_mreg_c[n++] = REG_C_1;
9769 	/* Discover availability of other reg_c's. */
9770 	for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
9771 		struct rte_flow_attr attr = {
9772 			.group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
9773 			.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
9774 			.ingress = 1,
9775 		};
9776 		struct rte_flow_item items[] = {
9777 			[0] = {
9778 				.type = RTE_FLOW_ITEM_TYPE_END,
9779 			},
9780 		};
9781 		struct rte_flow_action actions[] = {
9782 			[0] = {
9783 				.type = (enum rte_flow_action_type)
9784 					MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
9785 				.conf = &(struct mlx5_flow_action_copy_mreg){
9786 					.src = REG_C_1,
9787 					.dst = idx,
9788 				},
9789 			},
9790 			[1] = {
9791 				.type = RTE_FLOW_ACTION_TYPE_JUMP,
9792 				.conf = &(struct rte_flow_action_jump){
9793 					.group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
9794 				},
9795 			},
9796 			[2] = {
9797 				.type = RTE_FLOW_ACTION_TYPE_END,
9798 			},
9799 		};
9800 		uint32_t flow_idx;
9801 		struct rte_flow *flow;
9802 		struct rte_flow_error error;
9803 
9804 		if (!priv->sh->config.dv_flow_en)
9805 			break;
9806 		/* Create internal flow, validation skips copy action. */
9807 		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
9808 					items, actions, false, &error);
9809 		flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
9810 				      flow_idx);
9811 		if (!flow)
9812 			continue;
9813 		priv->sh->flow_mreg_c[n++] = idx;
9814 		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
9815 	}
9816 	for (; n < MLX5_MREG_C_NUM; ++n)
9817 		priv->sh->flow_mreg_c[n] = REG_NON;
9818 	priv->sh->metadata_regc_check_flag = 1;
9819 	return 0;
9820 }
9821 
9822 int
9823 save_dump_file(const uint8_t *data, uint32_t size,
9824 	uint32_t type, uint64_t id, void *arg, FILE *file)
9825 {
9826 	char line[BUF_SIZE];
9827 	uint32_t out = 0;
9828 	uint32_t k;
9829 	uint32_t actions_num;
9830 	struct rte_flow_query_count *count;
9831 
9832 	memset(line, 0, BUF_SIZE);
9833 	switch (type) {
9834 	case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR:
9835 		actions_num = *(uint32_t *)(arg);
9836 		out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,",
9837 				type, id, actions_num);
9838 		break;
9839 	case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT:
9840 		out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",",
9841 				type, id);
9842 		break;
9843 	case DR_DUMP_REC_TYPE_PMD_COUNTER:
9844 		count = (struct rte_flow_query_count *)arg;
9845 		fprintf(file,
9846 			"%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n",
9847 			type, id, count->hits, count->bytes);
9848 		return 0;
9849 	default:
9850 		return -1;
9851 	}
9852 
9853 	for (k = 0; k < size; k++) {
9854 		/* Make sure we do not overrun the line buffer length. */
9855 		if (out >= BUF_SIZE - 4) {
9856 			line[out] = '\0';
9857 			break;
9858 		}
9859 		out += snprintf(line + out, BUF_SIZE - out, "%02x",
9860 				(data[k]) & 0xff);
9861 	}
9862 	fprintf(file, "%s\n", line);
9863 	return 0;
9864 }
9865 
9866 int
9867 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow,
9868 	struct rte_flow_query_count *count, struct rte_flow_error *error)
9869 {
9870 	struct rte_flow_action action[2];
9871 	enum mlx5_flow_drv_type ftype;
9872 	const struct mlx5_flow_driver_ops *fops;
9873 
9874 	if (!flow) {
9875 		return rte_flow_error_set(error, ENOENT,
9876 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9877 				NULL,
9878 				"invalid flow handle");
9879 	}
9880 	action[0].type = RTE_FLOW_ACTION_TYPE_COUNT;
9881 	action[1].type = RTE_FLOW_ACTION_TYPE_END;
9882 	if (flow->counter) {
9883 		memset(count, 0, sizeof(struct rte_flow_query_count));
9884 		ftype = (enum mlx5_flow_drv_type)(flow->drv_type);
9885 		MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN &&
9886 						ftype < MLX5_FLOW_TYPE_MAX);
9887 		fops = flow_get_drv_ops(ftype);
9888 		return fops->query(dev, flow, action, count, error);
9889 	}
9890 	return -1;
9891 }
9892 
9893 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9894 /**
9895  * Dump flow ipool data to file
9896  *
9897  * @param[in] dev
9898  *   The pointer to Ethernet device.
9899  * @param[in] file
9900  *   A pointer to a file for output.
9901  * @param[out] error
9902  *   Perform verbose error reporting if not NULL. PMDs initialize this
9903  *   structure in case of error only.
9904  * @return
9905  *   0 on success, a negative value otherwise.
9906  */
9907 int
9908 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev,
9909 	struct rte_flow *flow, FILE *file,
9910 	struct rte_flow_error *error)
9911 {
9912 	struct mlx5_priv *priv = dev->data->dev_private;
9913 	struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
9914 	struct mlx5_flow_dv_encap_decap_resource *encap_decap;
9915 	uint32_t handle_idx;
9916 	struct mlx5_flow_handle *dh;
9917 	struct rte_flow_query_count count;
9918 	uint32_t actions_num;
9919 	const uint8_t *data;
9920 	size_t size;
9921 	uint64_t id;
9922 	uint32_t type;
9923 	void *action = NULL;
9924 
9925 	if (!flow) {
9926 		return rte_flow_error_set(error, ENOENT,
9927 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9928 				NULL,
9929 				"invalid flow handle");
9930 	}
9931 	handle_idx = flow->dev_handles;
9932 	/* query counter */
9933 	if (flow->counter &&
9934 	(!mlx5_counter_query(dev, flow->counter, false,
9935 	&count.hits, &count.bytes, &action)) && action) {
9936 		id = (uint64_t)(uintptr_t)action;
9937 		type = DR_DUMP_REC_TYPE_PMD_COUNTER;
9938 		save_dump_file(NULL, 0, type,
9939 			id, (void *)&count, file);
9940 	}
9941 
9942 	while (handle_idx) {
9943 		dh = mlx5_ipool_get(priv->sh->ipool
9944 				[MLX5_IPOOL_MLX5_FLOW], handle_idx);
9945 		if (!dh)
9946 			continue;
9947 		handle_idx = dh->next.next;
9948 
9949 		/* Get modify_hdr and encap_decap buf from ipools. */
9950 		encap_decap = NULL;
9951 		modify_hdr = dh->dvh.modify_hdr;
9952 
9953 		if (dh->dvh.rix_encap_decap) {
9954 			encap_decap = mlx5_ipool_get(priv->sh->ipool
9955 						[MLX5_IPOOL_DECAP_ENCAP],
9956 						dh->dvh.rix_encap_decap);
9957 		}
9958 		if (modify_hdr) {
9959 			data = (const uint8_t *)modify_hdr->actions;
9960 			size = (size_t)(modify_hdr->actions_num) * 8;
9961 			id = (uint64_t)(uintptr_t)modify_hdr->action;
9962 			actions_num = modify_hdr->actions_num;
9963 			type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
9964 			save_dump_file(data, size, type, id,
9965 						(void *)(&actions_num), file);
9966 		}
9967 		if (encap_decap) {
9968 			data = encap_decap->buf;
9969 			size = encap_decap->size;
9970 			id = (uint64_t)(uintptr_t)encap_decap->action;
9971 			type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
9972 			save_dump_file(data, size, type,
9973 						id, NULL, file);
9974 		}
9975 	}
9976 	return 0;
9977 }
9978 
9979 /**
9980  * Dump all flow's encap_decap/modify_hdr/counter data to file
9981  *
9982  * @param[in] dev
9983  *   The pointer to Ethernet device.
9984  * @param[in] file
9985  *   A pointer to a file for output.
9986  * @param[out] error
9987  *   Perform verbose error reporting if not NULL. PMDs initialize this
9988  *   structure in case of error only.
9989  * @return
9990  *   0 on success, a negative value otherwise.
9991  */
9992 static int
9993 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev,
9994 	FILE *file, struct rte_flow_error *error __rte_unused)
9995 {
9996 	struct mlx5_priv *priv = dev->data->dev_private;
9997 	struct mlx5_dev_ctx_shared *sh = priv->sh;
9998 	struct mlx5_hlist *h;
9999 	struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
10000 	struct mlx5_flow_dv_encap_decap_resource *encap_decap;
10001 	struct rte_flow_query_count count;
10002 	uint32_t actions_num;
10003 	const uint8_t *data;
10004 	size_t size;
10005 	uint64_t id;
10006 	uint32_t type;
10007 	uint32_t i;
10008 	uint32_t j;
10009 	struct mlx5_list_inconst *l_inconst;
10010 	struct mlx5_list_entry *e;
10011 	int lcore_index;
10012 	struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng;
10013 	uint32_t max;
10014 	void *action;
10015 
10016 	/* encap_decap hlist is lcore_share, get global core cache. */
10017 	i = MLX5_LIST_GLOBAL;
10018 	h = sh->encaps_decaps;
10019 	if (h) {
10020 		for (j = 0; j <= h->mask; j++) {
10021 			l_inconst = &h->buckets[j].l;
10022 			if (!l_inconst || !l_inconst->cache[i])
10023 				continue;
10024 
10025 			e = LIST_FIRST(&l_inconst->cache[i]->h);
10026 			while (e) {
10027 				encap_decap =
10028 				(struct mlx5_flow_dv_encap_decap_resource *)e;
10029 				data = encap_decap->buf;
10030 				size = encap_decap->size;
10031 				id = (uint64_t)(uintptr_t)encap_decap->action;
10032 				type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
10033 				save_dump_file(data, size, type,
10034 					id, NULL, file);
10035 				e = LIST_NEXT(e, next);
10036 			}
10037 		}
10038 	}
10039 
10040 	/* get modify_hdr */
10041 	h = sh->modify_cmds;
10042 	if (h) {
10043 		lcore_index = rte_lcore_index(rte_lcore_id());
10044 		if (unlikely(lcore_index == -1)) {
10045 			lcore_index = MLX5_LIST_NLCORE;
10046 			rte_spinlock_lock(&h->l_const.lcore_lock);
10047 		}
10048 		i = lcore_index;
10049 
10050 		if (lcore_index == MLX5_LIST_NLCORE) {
10051 			for (i = 0; i <= (uint32_t)lcore_index; i++) {
10052 				for (j = 0; j <= h->mask; j++) {
10053 					l_inconst = &h->buckets[j].l;
10054 					if (!l_inconst || !l_inconst->cache[i])
10055 						continue;
10056 
10057 					e = LIST_FIRST(&l_inconst->cache[i]->h);
10058 					while (e) {
10059 						modify_hdr =
10060 						(struct mlx5_flow_dv_modify_hdr_resource *)e;
10061 						data = (const uint8_t *)modify_hdr->actions;
10062 						size = (size_t)(modify_hdr->actions_num) * 8;
10063 						actions_num = modify_hdr->actions_num;
10064 						id = (uint64_t)(uintptr_t)modify_hdr->action;
10065 						type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
10066 						save_dump_file(data, size, type, id,
10067 								(void *)(&actions_num), file);
10068 						e = LIST_NEXT(e, next);
10069 					}
10070 				}
10071 			}
10072 		} else {
10073 			for (j = 0; j <= h->mask; j++) {
10074 				l_inconst = &h->buckets[j].l;
10075 				if (!l_inconst || !l_inconst->cache[i])
10076 					continue;
10077 
10078 				e = LIST_FIRST(&l_inconst->cache[i]->h);
10079 				while (e) {
10080 					modify_hdr =
10081 					(struct mlx5_flow_dv_modify_hdr_resource *)e;
10082 					data = (const uint8_t *)modify_hdr->actions;
10083 					size = (size_t)(modify_hdr->actions_num) * 8;
10084 					actions_num = modify_hdr->actions_num;
10085 					id = (uint64_t)(uintptr_t)modify_hdr->action;
10086 					type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
10087 					save_dump_file(data, size, type, id,
10088 							(void *)(&actions_num), file);
10089 					e = LIST_NEXT(e, next);
10090 				}
10091 			}
10092 		}
10093 
10094 		if (unlikely(lcore_index == MLX5_LIST_NLCORE))
10095 			rte_spinlock_unlock(&h->l_const.lcore_lock);
10096 	}
10097 
10098 	/* get counter */
10099 	MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM);
10100 	max = MLX5_COUNTERS_PER_POOL * cmng->n_valid;
10101 	for (j = 1; j <= max; j++) {
10102 		action = NULL;
10103 		if ((!mlx5_counter_query(dev, j, false, &count.hits,
10104 		&count.bytes, &action)) && action) {
10105 			id = (uint64_t)(uintptr_t)action;
10106 			type = DR_DUMP_REC_TYPE_PMD_COUNTER;
10107 			save_dump_file(NULL, 0, type,
10108 					id, (void *)&count, file);
10109 		}
10110 	}
10111 	return 0;
10112 }
10113 #endif
10114 
10115 /**
10116  * Dump flow raw hw data to file
10117  *
10118  * @param[in] dev
10119  *    The pointer to Ethernet device.
10120  * @param[in] file
10121  *   A pointer to a file for output.
10122  * @param[out] error
10123  *   Perform verbose error reporting if not NULL. PMDs initialize this
10124  *   structure in case of error only.
10125  * @return
10126  *   0 on success, a negative value otherwise.
10127  */
10128 int
10129 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
10130 		   FILE *file,
10131 		   struct rte_flow_error *error __rte_unused)
10132 {
10133 	struct mlx5_priv *priv = dev->data->dev_private;
10134 	struct mlx5_dev_ctx_shared *sh = priv->sh;
10135 	uint32_t handle_idx;
10136 	int ret;
10137 	struct mlx5_flow_handle *dh;
10138 	struct rte_flow *flow;
10139 
10140 	if (!sh->config.dv_flow_en) {
10141 		if (fputs("device dv flow disabled\n", file) <= 0)
10142 			return -errno;
10143 		return -ENOTSUP;
10144 	}
10145 
10146 	/* dump all */
10147 	if (!flow_idx) {
10148 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
10149 		if (mlx5_flow_dev_dump_sh_all(dev, file, error))
10150 			return -EINVAL;
10151 
10152 		if (sh->config.dv_flow_en == 2)
10153 			return mlx5dr_debug_dump(priv->dr_ctx, file);
10154 #endif
10155 		return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
10156 					       sh->rx_domain,
10157 					       sh->tx_domain, file);
10158 	}
10159 	/* dump one */
10160 	flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
10161 			(uintptr_t)(void *)flow_idx);
10162 	if (!flow)
10163 		return -EINVAL;
10164 
10165 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
10166 	mlx5_flow_dev_dump_ipool(dev, flow, file, error);
10167 #endif
10168 	handle_idx = flow->dev_handles;
10169 	while (handle_idx) {
10170 		dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10171 				handle_idx);
10172 		if (!dh)
10173 			return -ENOENT;
10174 		if (dh->drv_flow) {
10175 			if (sh->config.dv_flow_en == 2)
10176 				return -ENOTSUP;
10177 
10178 			ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
10179 							     file);
10180 			if (ret)
10181 				return -ENOENT;
10182 		}
10183 		handle_idx = dh->next.next;
10184 	}
10185 	return 0;
10186 }
10187 
10188 /**
10189  * Get aged-out flows.
10190  *
10191  * @param[in] dev
10192  *   Pointer to the Ethernet device structure.
10193  * @param[in] context
10194  *   The address of an array of pointers to the aged-out flows contexts.
10195  * @param[in] nb_countexts
10196  *   The length of context array pointers.
10197  * @param[out] error
10198  *   Perform verbose error reporting if not NULL. Initialized in case of
10199  *   error only.
10200  *
10201  * @return
10202  *   how many contexts get in success, otherwise negative errno value.
10203  *   if nb_contexts is 0, return the amount of all aged contexts.
10204  *   if nb_contexts is not 0 , return the amount of aged flows reported
10205  *   in the context array.
10206  */
10207 int
10208 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
10209 			uint32_t nb_contexts, struct rte_flow_error *error)
10210 {
10211 	struct rte_flow_attr attr = { .transfer = 0 };
10212 
10213 	return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows
10214 		(dev, contexts, nb_contexts, error);
10215 }
10216 
10217 /**
10218  * Get aged-out flows per HWS queue.
10219  *
10220  * @param[in] dev
10221  *   Pointer to the Ethernet device structure.
10222  * @param[in] queue_id
10223  *   Flow queue to query.
10224  * @param[in] context
10225  *   The address of an array of pointers to the aged-out flows contexts.
10226  * @param[in] nb_countexts
10227  *   The length of context array pointers.
10228  * @param[out] error
10229  *   Perform verbose error reporting if not NULL. Initialized in case of
10230  *   error only.
10231  *
10232  * @return
10233  *   how many contexts get in success, otherwise negative errno value.
10234  *   if nb_contexts is 0, return the amount of all aged contexts.
10235  *   if nb_contexts is not 0 , return the amount of aged flows reported
10236  *   in the context array.
10237  */
10238 int
10239 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id,
10240 			   void **contexts, uint32_t nb_contexts,
10241 			   struct rte_flow_error *error)
10242 {
10243 	const struct mlx5_flow_driver_ops *fops;
10244 	struct rte_flow_attr attr = { 0 };
10245 
10246 	if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) {
10247 		fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
10248 		return fops->get_q_aged_flows(dev, queue_id, contexts,
10249 					      nb_contexts, error);
10250 	}
10251 	DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.",
10252 		dev->data->port_id, queue_id);
10253 	return rte_flow_error_set(error, ENOTSUP,
10254 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10255 				  "get Q aged flows with incorrect steering mode");
10256 }
10257 
10258 /* Wrapper for driver action_validate op callback */
10259 static int
10260 flow_drv_action_validate(struct rte_eth_dev *dev,
10261 			 const struct rte_flow_indir_action_conf *conf,
10262 			 const struct rte_flow_action *action,
10263 			 const struct mlx5_flow_driver_ops *fops,
10264 			 struct rte_flow_error *error)
10265 {
10266 	static const char err_msg[] = "indirect action validation unsupported";
10267 
10268 	if (!fops->action_validate) {
10269 		DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10270 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10271 				   NULL, err_msg);
10272 		return -rte_errno;
10273 	}
10274 	return fops->action_validate(dev, conf, action, error);
10275 }
10276 
10277 /**
10278  * Destroys the shared action by handle.
10279  *
10280  * @param dev
10281  *   Pointer to Ethernet device structure.
10282  * @param[in] handle
10283  *   Handle for the indirect action object to be destroyed.
10284  * @param[out] error
10285  *   Perform verbose error reporting if not NULL. PMDs initialize this
10286  *   structure in case of error only.
10287  *
10288  * @return
10289  *   0 on success, a negative errno value otherwise and rte_errno is set.
10290  *
10291  * @note: wrapper for driver action_create op callback.
10292  */
10293 static int
10294 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
10295 			   struct rte_flow_action_handle *handle,
10296 			   struct rte_flow_error *error)
10297 {
10298 	static const char err_msg[] = "indirect action destruction unsupported";
10299 	struct rte_flow_attr attr = { .transfer = 0 };
10300 	const struct mlx5_flow_driver_ops *fops =
10301 			flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10302 
10303 	if (!fops->action_destroy) {
10304 		DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10305 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10306 				   NULL, err_msg);
10307 		return -rte_errno;
10308 	}
10309 	return fops->action_destroy(dev, handle, error);
10310 }
10311 
10312 /* Wrapper for driver action_destroy op callback */
10313 static int
10314 flow_drv_action_update(struct rte_eth_dev *dev,
10315 		       struct rte_flow_action_handle *handle,
10316 		       const void *update,
10317 		       const struct mlx5_flow_driver_ops *fops,
10318 		       struct rte_flow_error *error)
10319 {
10320 	static const char err_msg[] = "indirect action update unsupported";
10321 
10322 	if (!fops->action_update) {
10323 		DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10324 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10325 				   NULL, err_msg);
10326 		return -rte_errno;
10327 	}
10328 	return fops->action_update(dev, handle, update, error);
10329 }
10330 
10331 /* Wrapper for driver action_destroy op callback */
10332 static int
10333 flow_drv_action_query(struct rte_eth_dev *dev,
10334 		      const struct rte_flow_action_handle *handle,
10335 		      void *data,
10336 		      const struct mlx5_flow_driver_ops *fops,
10337 		      struct rte_flow_error *error)
10338 {
10339 	static const char err_msg[] = "indirect action query unsupported";
10340 
10341 	if (!fops->action_query) {
10342 		DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10343 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10344 				   NULL, err_msg);
10345 		return -rte_errno;
10346 	}
10347 	return fops->action_query(dev, handle, data, error);
10348 }
10349 
10350 /**
10351  * Create indirect action for reuse in multiple flow rules.
10352  *
10353  * @param dev
10354  *   Pointer to Ethernet device structure.
10355  * @param conf
10356  *   Pointer to indirect action object configuration.
10357  * @param[in] action
10358  *   Action configuration for indirect action object creation.
10359  * @param[out] error
10360  *   Perform verbose error reporting if not NULL. PMDs initialize this
10361  *   structure in case of error only.
10362  * @return
10363  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
10364  */
10365 static struct rte_flow_action_handle *
10366 mlx5_action_handle_create(struct rte_eth_dev *dev,
10367 			  const struct rte_flow_indir_action_conf *conf,
10368 			  const struct rte_flow_action *action,
10369 			  struct rte_flow_error *error)
10370 {
10371 	static const char err_msg[] = "indirect action creation unsupported";
10372 	struct rte_flow_attr attr = { .transfer = 0 };
10373 	const struct mlx5_flow_driver_ops *fops =
10374 			flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10375 
10376 	if (flow_drv_action_validate(dev, conf, action, fops, error))
10377 		return NULL;
10378 	if (!fops->action_create) {
10379 		DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10380 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10381 				   NULL, err_msg);
10382 		return NULL;
10383 	}
10384 	return fops->action_create(dev, conf, action, error);
10385 }
10386 
10387 /**
10388  * Updates inplace the indirect action configuration pointed by *handle*
10389  * with the configuration provided as *update* argument.
10390  * The update of the indirect action configuration effects all flow rules
10391  * reusing the action via handle.
10392  *
10393  * @param dev
10394  *   Pointer to Ethernet device structure.
10395  * @param[in] handle
10396  *   Handle for the indirect action to be updated.
10397  * @param[in] update
10398  *   Action specification used to modify the action pointed by handle.
10399  *   *update* could be of same type with the action pointed by the *handle*
10400  *   handle argument, or some other structures like a wrapper, depending on
10401  *   the indirect action type.
10402  * @param[out] error
10403  *   Perform verbose error reporting if not NULL. PMDs initialize this
10404  *   structure in case of error only.
10405  *
10406  * @return
10407  *   0 on success, a negative errno value otherwise and rte_errno is set.
10408  */
10409 static int
10410 mlx5_action_handle_update(struct rte_eth_dev *dev,
10411 		struct rte_flow_action_handle *handle,
10412 		const void *update,
10413 		struct rte_flow_error *error)
10414 {
10415 	struct rte_flow_attr attr = { .transfer = 0 };
10416 	const struct mlx5_flow_driver_ops *fops =
10417 			flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10418 	int ret;
10419 	uint32_t act_idx = (uint32_t)(uintptr_t)handle;
10420 	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
10421 
10422 	switch (type) {
10423 	case MLX5_INDIRECT_ACTION_TYPE_CT:
10424 	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
10425 		ret = 0;
10426 		break;
10427 	default:
10428 		ret = flow_drv_action_validate(dev, NULL,
10429 				(const struct rte_flow_action *)update,
10430 				fops, error);
10431 	}
10432 	if (ret)
10433 		return ret;
10434 	return flow_drv_action_update(dev, handle, update, fops,
10435 				      error);
10436 }
10437 
10438 /**
10439  * Query the indirect action by handle.
10440  *
10441  * This function allows retrieving action-specific data such as counters.
10442  * Data is gathered by special action which may be present/referenced in
10443  * more than one flow rule definition.
10444  *
10445  * see @RTE_FLOW_ACTION_TYPE_COUNT
10446  *
10447  * @param dev
10448  *   Pointer to Ethernet device structure.
10449  * @param[in] handle
10450  *   Handle for the indirect action to query.
10451  * @param[in, out] data
10452  *   Pointer to storage for the associated query data type.
10453  * @param[out] error
10454  *   Perform verbose error reporting if not NULL. PMDs initialize this
10455  *   structure in case of error only.
10456  *
10457  * @return
10458  *   0 on success, a negative errno value otherwise and rte_errno is set.
10459  */
10460 static int
10461 mlx5_action_handle_query(struct rte_eth_dev *dev,
10462 			 const struct rte_flow_action_handle *handle,
10463 			 void *data,
10464 			 struct rte_flow_error *error)
10465 {
10466 	struct rte_flow_attr attr = { .transfer = 0 };
10467 	const struct mlx5_flow_driver_ops *fops =
10468 			flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10469 
10470 	return flow_drv_action_query(dev, handle, data, fops, error);
10471 }
10472 
10473 static int
10474 mlx5_action_handle_query_update(struct rte_eth_dev *dev,
10475 				struct rte_flow_action_handle *handle,
10476 				const void *update, void *query,
10477 				enum rte_flow_query_update_mode qu_mode,
10478 				struct rte_flow_error *error)
10479 {
10480 	struct rte_flow_attr attr = { .transfer = 0 };
10481 	enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr);
10482 	const struct mlx5_flow_driver_ops *fops;
10483 
10484 	if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX)
10485 		return rte_flow_error_set(error, ENOTSUP,
10486 					  RTE_FLOW_ERROR_TYPE_ACTION,
10487 					  NULL, "invalid driver type");
10488 	fops = flow_get_drv_ops(drv_type);
10489 	if (!fops || !fops->action_query_update)
10490 		return rte_flow_error_set(error, ENOTSUP,
10491 					  RTE_FLOW_ERROR_TYPE_ACTION,
10492 					  NULL, "no query_update handler");
10493 	return fops->action_query_update(dev, handle, update,
10494 					 query, qu_mode, error);
10495 }
10496 
10497 
10498 #define MLX5_DRV_FOPS_OR_ERR(dev, fops, drv_cb, ret)                           \
10499 {                                                                              \
10500 	struct rte_flow_attr attr = { .transfer = 0 };                         \
10501 	enum mlx5_flow_drv_type drv_type = flow_get_drv_type((dev), &attr);    \
10502 	if (drv_type == MLX5_FLOW_TYPE_MIN ||                                  \
10503 	    drv_type == MLX5_FLOW_TYPE_MAX) {                                  \
10504 		rte_flow_error_set(error, ENOTSUP,                             \
10505 				   RTE_FLOW_ERROR_TYPE_ACTION,                 \
10506 				   NULL, "invalid driver type");               \
10507 		return ret;                                                    \
10508 	}                                                                      \
10509 	(fops) = flow_get_drv_ops(drv_type);                                   \
10510 	if (!(fops) || !(fops)->drv_cb) {                                      \
10511 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, \
10512 				   NULL, "no action_list handler");            \
10513 		return ret;                                                    \
10514 	}                                                                      \
10515 }
10516 
10517 static struct rte_flow_action_list_handle *
10518 mlx5_action_list_handle_create(struct rte_eth_dev *dev,
10519 			       const struct rte_flow_indir_action_conf *conf,
10520 			       const struct rte_flow_action *actions,
10521 			       struct rte_flow_error *error)
10522 {
10523 	const struct mlx5_flow_driver_ops *fops;
10524 
10525 	MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_create, NULL);
10526 	return fops->action_list_handle_create(dev, conf, actions, error);
10527 }
10528 
10529 static int
10530 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev,
10531 				struct rte_flow_action_list_handle *handle,
10532 				struct rte_flow_error *error)
10533 {
10534 	const struct mlx5_flow_driver_ops *fops;
10535 
10536 	MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_destroy, ENOTSUP);
10537 	return fops->action_list_handle_destroy(dev, handle, error);
10538 }
10539 
10540 static int
10541 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev,
10542 					  const
10543 					  struct rte_flow_action_list_handle *handle,
10544 					  const void **update, void **query,
10545 					  enum rte_flow_query_update_mode mode,
10546 					  struct rte_flow_error *error)
10547 {
10548 	const struct mlx5_flow_driver_ops *fops;
10549 
10550 	MLX5_DRV_FOPS_OR_ERR(dev, fops,
10551 			     action_list_handle_query_update, ENOTSUP);
10552 	return fops->action_list_handle_query_update(dev, handle, update, query,
10553 						     mode, error);
10554 }
10555 static int
10556 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev,
10557 			  const struct rte_flow_template_table *table,
10558 			  const struct rte_flow_item pattern[],
10559 			  uint8_t pattern_template_index,
10560 			  uint32_t *hash, struct rte_flow_error *error)
10561 {
10562 	struct rte_flow_attr attr = { .transfer = 0 };
10563 	enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr);
10564 	const struct mlx5_flow_driver_ops *fops;
10565 
10566 	if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX)
10567 		return rte_flow_error_set(error, ENOTSUP,
10568 					  RTE_FLOW_ERROR_TYPE_ACTION,
10569 					  NULL, "invalid driver type");
10570 	fops = flow_get_drv_ops(drv_type);
10571 	if (!fops || !fops->action_query_update)
10572 		return rte_flow_error_set(error, ENOTSUP,
10573 					  RTE_FLOW_ERROR_TYPE_ACTION,
10574 					  NULL, "no query_update handler");
10575 	return fops->flow_calc_table_hash(dev, table, pattern, pattern_template_index,
10576 					  hash, error);
10577 }
10578 
10579 static int
10580 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev,
10581 			  const struct rte_flow_item pattern[],
10582 			  enum rte_flow_encap_hash_field dest_field,
10583 			  uint8_t *hash,
10584 			  struct rte_flow_error *error)
10585 {
10586 	enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, NULL);
10587 	const struct mlx5_flow_driver_ops *fops;
10588 
10589 	if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX)
10590 		return rte_flow_error_set(error, ENOTSUP,
10591 					  RTE_FLOW_ERROR_TYPE_ACTION,
10592 					  NULL, "invalid driver type");
10593 	fops = flow_get_drv_ops(drv_type);
10594 	if (!fops || !fops->flow_calc_encap_hash)
10595 		return rte_flow_error_set(error, ENOTSUP,
10596 					  RTE_FLOW_ERROR_TYPE_ACTION,
10597 					  NULL, "no calc encap hash handler");
10598 	return fops->flow_calc_encap_hash(dev, pattern, dest_field, hash, error);
10599 }
10600 
10601 static int
10602 mlx5_template_table_resize(struct rte_eth_dev *dev,
10603 			   struct rte_flow_template_table *table,
10604 			   uint32_t nb_rules, struct rte_flow_error *error)
10605 {
10606 	const struct mlx5_flow_driver_ops *fops;
10607 
10608 	MLX5_DRV_FOPS_OR_ERR(dev, fops, table_resize, ENOTSUP);
10609 	return fops->table_resize(dev, table, nb_rules, error);
10610 }
10611 
10612 static int
10613 mlx5_table_resize_complete(struct rte_eth_dev *dev,
10614 			   struct rte_flow_template_table *table,
10615 			   struct rte_flow_error *error)
10616 {
10617 	const struct mlx5_flow_driver_ops *fops;
10618 
10619 	MLX5_DRV_FOPS_OR_ERR(dev, fops, table_resize_complete, ENOTSUP);
10620 	return fops->table_resize_complete(dev, table, error);
10621 }
10622 
10623 static int
10624 mlx5_flow_async_update_resized(struct rte_eth_dev *dev, uint32_t queue,
10625 			       const struct rte_flow_op_attr *op_attr,
10626 			       struct rte_flow *rule, void *user_data,
10627 			       struct rte_flow_error *error)
10628 {
10629 	const struct mlx5_flow_driver_ops *fops;
10630 
10631 	MLX5_DRV_FOPS_OR_ERR(dev, fops, flow_update_resized, ENOTSUP);
10632 	return fops->flow_update_resized(dev, queue, op_attr, rule, user_data, error);
10633 }
10634 
10635 /**
10636  * Destroy all indirect actions (shared RSS).
10637  *
10638  * @param dev
10639  *   Pointer to Ethernet device.
10640  *
10641  * @return
10642  *   0 on success, a negative errno value otherwise and rte_errno is set.
10643  */
10644 int
10645 mlx5_action_handle_flush(struct rte_eth_dev *dev)
10646 {
10647 	struct rte_flow_error error;
10648 	struct mlx5_priv *priv = dev->data->dev_private;
10649 	struct mlx5_shared_action_rss *shared_rss;
10650 	int ret = 0;
10651 	uint32_t idx;
10652 
10653 	ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
10654 		      priv->rss_shared_actions, idx, shared_rss, next) {
10655 		ret |= mlx5_action_handle_destroy(dev,
10656 		       (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
10657 	}
10658 	return ret;
10659 }
10660 
10661 /**
10662  * Validate existing indirect actions against current device configuration
10663  * and attach them to device resources.
10664  *
10665  * @param dev
10666  *   Pointer to Ethernet device.
10667  *
10668  * @return
10669  *   0 on success, a negative errno value otherwise and rte_errno is set.
10670  */
10671 int
10672 mlx5_action_handle_attach(struct rte_eth_dev *dev)
10673 {
10674 	struct mlx5_priv *priv = dev->data->dev_private;
10675 	int ret = 0;
10676 	struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last;
10677 
10678 	LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
10679 		const char *message;
10680 		uint32_t queue_idx;
10681 
10682 		ret = mlx5_validate_rss_queues(dev, ind_tbl->queues,
10683 					       ind_tbl->queues_n,
10684 					       &message, &queue_idx);
10685 		if (ret != 0) {
10686 			DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s",
10687 				dev->data->port_id, ind_tbl->queues[queue_idx],
10688 				message);
10689 			break;
10690 		}
10691 	}
10692 	if (ret != 0)
10693 		return ret;
10694 	LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
10695 		ret = mlx5_ind_table_obj_attach(dev, ind_tbl);
10696 		if (ret != 0) {
10697 			DRV_LOG(ERR, "Port %u could not attach "
10698 				"indirection table obj %p",
10699 				dev->data->port_id, (void *)ind_tbl);
10700 			goto error;
10701 		}
10702 	}
10703 
10704 	return 0;
10705 error:
10706 	ind_tbl_last = ind_tbl;
10707 	LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
10708 		if (ind_tbl == ind_tbl_last)
10709 			break;
10710 		if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0)
10711 			DRV_LOG(CRIT, "Port %u could not detach "
10712 				"indirection table obj %p on rollback",
10713 				dev->data->port_id, (void *)ind_tbl);
10714 	}
10715 	return ret;
10716 }
10717 
10718 /**
10719  * Detach indirect actions of the device from its resources.
10720  *
10721  * @param dev
10722  *   Pointer to Ethernet device.
10723  *
10724  * @return
10725  *   0 on success, a negative errno value otherwise and rte_errno is set.
10726  */
10727 int
10728 mlx5_action_handle_detach(struct rte_eth_dev *dev)
10729 {
10730 	struct mlx5_priv *priv = dev->data->dev_private;
10731 	int ret = 0;
10732 	struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last;
10733 
10734 	LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
10735 		ret = mlx5_ind_table_obj_detach(dev, ind_tbl);
10736 		if (ret != 0) {
10737 			DRV_LOG(ERR, "Port %u could not detach "
10738 				"indirection table obj %p",
10739 				dev->data->port_id, (void *)ind_tbl);
10740 			goto error;
10741 		}
10742 	}
10743 	return 0;
10744 error:
10745 	ind_tbl_last = ind_tbl;
10746 	LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
10747 		if (ind_tbl == ind_tbl_last)
10748 			break;
10749 		if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0)
10750 			DRV_LOG(CRIT, "Port %u could not attach "
10751 				"indirection table obj %p on rollback",
10752 				dev->data->port_id, (void *)ind_tbl);
10753 	}
10754 	return ret;
10755 }
10756 
10757 #ifndef HAVE_MLX5DV_DR
10758 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
10759 #else
10760 #define MLX5_DOMAIN_SYNC_FLOW \
10761 	(MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
10762 #endif
10763 
10764 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
10765 {
10766 	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
10767 	const struct mlx5_flow_driver_ops *fops;
10768 	int ret;
10769 	struct rte_flow_attr attr = { .transfer = 0 };
10770 
10771 	fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10772 	ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
10773 	if (ret > 0)
10774 		ret = -ret;
10775 	return ret;
10776 }
10777 
10778 const struct mlx5_flow_tunnel *
10779 mlx5_get_tof(const struct rte_flow_item *item,
10780 	     const struct rte_flow_action *action,
10781 	     enum mlx5_tof_rule_type *rule_type)
10782 {
10783 	for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
10784 		if (item->type == (typeof(item->type))
10785 				  MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
10786 			*rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
10787 			return flow_items_to_tunnel(item);
10788 		}
10789 	}
10790 	for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
10791 		if (action->type == (typeof(action->type))
10792 				    MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
10793 			*rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
10794 			return flow_actions_to_tunnel(action);
10795 		}
10796 	}
10797 	return NULL;
10798 }
10799 
10800 /**
10801  * tunnel offload functionality is defined for DV environment only
10802  */
10803 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
10804 __extension__
10805 union tunnel_offload_mark {
10806 	uint32_t val;
10807 	struct {
10808 		uint32_t app_reserve:8;
10809 		uint32_t table_id:15;
10810 		uint32_t transfer:1;
10811 		uint32_t _unused_:8;
10812 	};
10813 };
10814 
10815 static bool
10816 mlx5_access_tunnel_offload_db
10817 	(struct rte_eth_dev *dev,
10818 	 bool (*match)(struct rte_eth_dev *,
10819 		       struct mlx5_flow_tunnel *, const void *),
10820 	 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
10821 	 void (*miss)(struct rte_eth_dev *, void *),
10822 	 void *ctx, bool lock_op);
10823 
10824 static int
10825 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
10826 			     struct rte_flow *flow,
10827 			     const struct rte_flow_attr *attr,
10828 			     const struct rte_flow_action *app_actions,
10829 			     uint32_t flow_idx,
10830 			     const struct mlx5_flow_tunnel *tunnel,
10831 			     struct tunnel_default_miss_ctx *ctx,
10832 			     struct rte_flow_error *error)
10833 {
10834 	struct mlx5_priv *priv = dev->data->dev_private;
10835 	struct mlx5_flow *dev_flow;
10836 	struct rte_flow_attr miss_attr = *attr;
10837 	const struct rte_flow_item miss_items[2] = {
10838 		{
10839 			.type = RTE_FLOW_ITEM_TYPE_ETH,
10840 			.spec = NULL,
10841 			.last = NULL,
10842 			.mask = NULL
10843 		},
10844 		{
10845 			.type = RTE_FLOW_ITEM_TYPE_END,
10846 			.spec = NULL,
10847 			.last = NULL,
10848 			.mask = NULL
10849 		}
10850 	};
10851 	union tunnel_offload_mark mark_id;
10852 	struct rte_flow_action_mark miss_mark;
10853 	struct rte_flow_action miss_actions[3] = {
10854 		[0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
10855 		[2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
10856 	};
10857 	const struct rte_flow_action_jump *jump_data;
10858 	uint32_t i, flow_table = 0; /* prevent compilation warning */
10859 	struct flow_grp_info grp_info = {
10860 		.external = 1,
10861 		.transfer = attr->transfer,
10862 		.fdb_def_rule = !!priv->fdb_def_rule,
10863 		.std_tbl_fix = 0,
10864 	};
10865 	int ret;
10866 
10867 	if (!attr->transfer) {
10868 		uint32_t q_size;
10869 
10870 		miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
10871 		q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
10872 		ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
10873 					 0, SOCKET_ID_ANY);
10874 		if (!ctx->queue)
10875 			return rte_flow_error_set
10876 				(error, ENOMEM,
10877 				RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10878 				NULL, "invalid default miss RSS");
10879 		ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
10880 		ctx->action_rss.level = 0,
10881 		ctx->action_rss.types = priv->rss_conf.rss_hf,
10882 		ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
10883 		ctx->action_rss.queue_num = priv->reta_idx_n,
10884 		ctx->action_rss.key = priv->rss_conf.rss_key,
10885 		ctx->action_rss.queue = ctx->queue;
10886 		if (!priv->reta_idx_n || !priv->rxqs_n)
10887 			return rte_flow_error_set
10888 				(error, EINVAL,
10889 				RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10890 				NULL, "invalid port configuration");
10891 		if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
10892 			ctx->action_rss.types = 0;
10893 		for (i = 0; i != priv->reta_idx_n; ++i)
10894 			ctx->queue[i] = (*priv->reta_idx)[i];
10895 	} else {
10896 		miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
10897 		ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
10898 	}
10899 	miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
10900 	for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
10901 	jump_data = app_actions->conf;
10902 	miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
10903 	miss_attr.group = jump_data->group;
10904 	ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
10905 				       &flow_table, &grp_info, error);
10906 	if (ret)
10907 		return rte_flow_error_set(error, EINVAL,
10908 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10909 					  NULL, "invalid tunnel id");
10910 	mark_id.app_reserve = 0;
10911 	mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
10912 	mark_id.transfer = !!attr->transfer;
10913 	mark_id._unused_ = 0;
10914 	miss_mark.id = mark_id.val;
10915 	dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
10916 				    miss_items, miss_actions, flow_idx, error);
10917 	if (!dev_flow)
10918 		return -rte_errno;
10919 	dev_flow->flow = flow;
10920 	dev_flow->external = true;
10921 	dev_flow->tunnel = tunnel;
10922 	dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
10923 	/* Subflow object was created, we must include one in the list. */
10924 	SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
10925 		      dev_flow->handle, next);
10926 	DRV_LOG(DEBUG,
10927 		"port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
10928 		dev->data->port_id, tunnel->app_tunnel.type,
10929 		tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
10930 	ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
10931 				  miss_actions, error);
10932 	if (!ret)
10933 		ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
10934 						  error);
10935 
10936 	return ret;
10937 }
10938 
10939 static const struct mlx5_flow_tbl_data_entry  *
10940 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
10941 {
10942 	struct mlx5_priv *priv = dev->data->dev_private;
10943 	struct mlx5_dev_ctx_shared *sh = priv->sh;
10944 	struct mlx5_list_entry *he;
10945 	union tunnel_offload_mark mbits = { .val = mark };
10946 	union mlx5_flow_tbl_key table_key = {
10947 		{
10948 			.level = tunnel_id_to_flow_tbl(mbits.table_id),
10949 			.id = 0,
10950 			.reserved = 0,
10951 			.dummy = 0,
10952 			.is_fdb = !!mbits.transfer,
10953 			.is_egress = 0,
10954 		}
10955 	};
10956 	struct mlx5_flow_cb_ctx ctx = {
10957 		.data = &table_key.v64,
10958 	};
10959 
10960 	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
10961 	return he ?
10962 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
10963 }
10964 
10965 static void
10966 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
10967 				   struct mlx5_list_entry *entry)
10968 {
10969 	struct mlx5_dev_ctx_shared *sh = tool_ctx;
10970 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10971 
10972 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10973 			tunnel_flow_tbl_to_id(tte->flow_table));
10974 	mlx5_free(tte);
10975 }
10976 
10977 static int
10978 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
10979 				  struct mlx5_list_entry *entry, void *cb_ctx)
10980 {
10981 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10982 	union tunnel_tbl_key tbl = {
10983 		.val = *(uint64_t *)(ctx->data),
10984 	};
10985 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10986 
10987 	return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
10988 }
10989 
10990 static struct mlx5_list_entry *
10991 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
10992 {
10993 	struct mlx5_dev_ctx_shared *sh = tool_ctx;
10994 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10995 	struct tunnel_tbl_entry *tte;
10996 	union tunnel_tbl_key tbl = {
10997 		.val = *(uint64_t *)(ctx->data),
10998 	};
10999 
11000 	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
11001 			  sizeof(*tte), 0,
11002 			  SOCKET_ID_ANY);
11003 	if (!tte)
11004 		goto err;
11005 	mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
11006 			  &tte->flow_table);
11007 	if (tte->flow_table >= MLX5_MAX_TABLES) {
11008 		DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
11009 			tte->flow_table);
11010 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
11011 				tte->flow_table);
11012 		goto err;
11013 	} else if (!tte->flow_table) {
11014 		goto err;
11015 	}
11016 	tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
11017 	tte->tunnel_id = tbl.tunnel_id;
11018 	tte->group = tbl.group;
11019 	return &tte->hash;
11020 err:
11021 	if (tte)
11022 		mlx5_free(tte);
11023 	return NULL;
11024 }
11025 
11026 static struct mlx5_list_entry *
11027 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
11028 				  struct mlx5_list_entry *oentry,
11029 				  void *cb_ctx __rte_unused)
11030 {
11031 	struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
11032 						   0, SOCKET_ID_ANY);
11033 
11034 	if (!tte)
11035 		return NULL;
11036 	memcpy(tte, oentry, sizeof(*tte));
11037 	return &tte->hash;
11038 }
11039 
11040 static void
11041 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
11042 				       struct mlx5_list_entry *entry)
11043 {
11044 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
11045 
11046 	mlx5_free(tte);
11047 }
11048 
11049 static uint32_t
11050 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
11051 				const struct mlx5_flow_tunnel *tunnel,
11052 				uint32_t group, uint32_t *table,
11053 				struct rte_flow_error *error)
11054 {
11055 	struct mlx5_list_entry *he;
11056 	struct tunnel_tbl_entry *tte;
11057 	union tunnel_tbl_key key = {
11058 		.tunnel_id = tunnel ? tunnel->tunnel_id : 0,
11059 		.group = group
11060 	};
11061 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
11062 	struct mlx5_hlist *group_hash;
11063 	struct mlx5_flow_cb_ctx ctx = {
11064 		.data = &key.val,
11065 	};
11066 
11067 	group_hash = tunnel ? tunnel->groups : thub->groups;
11068 	he = mlx5_hlist_register(group_hash, key.val, &ctx);
11069 	if (!he)
11070 		return rte_flow_error_set(error, EINVAL,
11071 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
11072 					  NULL,
11073 					  "tunnel group index not supported");
11074 	tte = container_of(he, typeof(*tte), hash);
11075 	*table = tte->flow_table;
11076 	DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
11077 		dev->data->port_id, key.tunnel_id, group, *table);
11078 	return 0;
11079 }
11080 
11081 static void
11082 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
11083 		      struct mlx5_flow_tunnel *tunnel)
11084 {
11085 	struct mlx5_priv *priv = dev->data->dev_private;
11086 	struct mlx5_indexed_pool *ipool;
11087 
11088 	DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
11089 		dev->data->port_id, tunnel->tunnel_id);
11090 	LIST_REMOVE(tunnel, chain);
11091 	mlx5_hlist_destroy(tunnel->groups);
11092 	ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
11093 	mlx5_ipool_free(ipool, tunnel->tunnel_id);
11094 }
11095 
11096 static bool
11097 mlx5_access_tunnel_offload_db
11098 	(struct rte_eth_dev *dev,
11099 	 bool (*match)(struct rte_eth_dev *,
11100 		       struct mlx5_flow_tunnel *, const void *),
11101 	 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
11102 	 void (*miss)(struct rte_eth_dev *, void *),
11103 	 void *ctx, bool lock_op)
11104 {
11105 	bool verdict = false;
11106 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
11107 	struct mlx5_flow_tunnel *tunnel;
11108 
11109 	rte_spinlock_lock(&thub->sl);
11110 	LIST_FOREACH(tunnel, &thub->tunnels, chain) {
11111 		verdict = match(dev, tunnel, (const void *)ctx);
11112 		if (verdict)
11113 			break;
11114 	}
11115 	if (!lock_op)
11116 		rte_spinlock_unlock(&thub->sl);
11117 	if (verdict && hit)
11118 		hit(dev, tunnel, ctx);
11119 	if (!verdict && miss)
11120 		miss(dev, ctx);
11121 	if (lock_op)
11122 		rte_spinlock_unlock(&thub->sl);
11123 
11124 	return verdict;
11125 }
11126 
11127 struct tunnel_db_find_tunnel_id_ctx {
11128 	uint32_t tunnel_id;
11129 	struct mlx5_flow_tunnel *tunnel;
11130 };
11131 
11132 static bool
11133 find_tunnel_id_match(struct rte_eth_dev *dev,
11134 		     struct mlx5_flow_tunnel *tunnel, const void *x)
11135 {
11136 	const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
11137 
11138 	RTE_SET_USED(dev);
11139 	return tunnel->tunnel_id == ctx->tunnel_id;
11140 }
11141 
11142 static void
11143 find_tunnel_id_hit(struct rte_eth_dev *dev,
11144 		   struct mlx5_flow_tunnel *tunnel, void *x)
11145 {
11146 	struct tunnel_db_find_tunnel_id_ctx *ctx = x;
11147 	RTE_SET_USED(dev);
11148 	ctx->tunnel = tunnel;
11149 }
11150 
11151 static struct mlx5_flow_tunnel *
11152 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
11153 {
11154 	struct tunnel_db_find_tunnel_id_ctx ctx = {
11155 		.tunnel_id = id,
11156 	};
11157 
11158 	mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
11159 				      find_tunnel_id_hit, NULL, &ctx, true);
11160 
11161 	return ctx.tunnel;
11162 }
11163 
11164 static struct mlx5_flow_tunnel *
11165 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
11166 			  const struct rte_flow_tunnel *app_tunnel)
11167 {
11168 	struct mlx5_priv *priv = dev->data->dev_private;
11169 	struct mlx5_indexed_pool *ipool;
11170 	struct mlx5_flow_tunnel *tunnel;
11171 	uint32_t id;
11172 
11173 	ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
11174 	tunnel = mlx5_ipool_zmalloc(ipool, &id);
11175 	if (!tunnel)
11176 		return NULL;
11177 	if (id >= MLX5_MAX_TUNNELS) {
11178 		mlx5_ipool_free(ipool, id);
11179 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
11180 		return NULL;
11181 	}
11182 	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
11183 					   priv->sh,
11184 					   mlx5_flow_tunnel_grp2tbl_create_cb,
11185 					   mlx5_flow_tunnel_grp2tbl_match_cb,
11186 					   mlx5_flow_tunnel_grp2tbl_remove_cb,
11187 					   mlx5_flow_tunnel_grp2tbl_clone_cb,
11188 					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
11189 	if (!tunnel->groups) {
11190 		mlx5_ipool_free(ipool, id);
11191 		return NULL;
11192 	}
11193 	/* initiate new PMD tunnel */
11194 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
11195 	tunnel->tunnel_id = id;
11196 	tunnel->action.type = (typeof(tunnel->action.type))
11197 			      MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
11198 	tunnel->action.conf = tunnel;
11199 	tunnel->item.type = (typeof(tunnel->item.type))
11200 			    MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
11201 	tunnel->item.spec = tunnel;
11202 	tunnel->item.last = NULL;
11203 	tunnel->item.mask = NULL;
11204 
11205 	DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
11206 		dev->data->port_id, tunnel->tunnel_id);
11207 
11208 	return tunnel;
11209 }
11210 
11211 struct tunnel_db_get_tunnel_ctx {
11212 	const struct rte_flow_tunnel *app_tunnel;
11213 	struct mlx5_flow_tunnel *tunnel;
11214 };
11215 
11216 static bool get_tunnel_match(struct rte_eth_dev *dev,
11217 			     struct mlx5_flow_tunnel *tunnel, const void *x)
11218 {
11219 	const struct tunnel_db_get_tunnel_ctx *ctx = x;
11220 
11221 	RTE_SET_USED(dev);
11222 	return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
11223 		       sizeof(*ctx->app_tunnel));
11224 }
11225 
11226 static void get_tunnel_hit(struct rte_eth_dev *dev,
11227 			   struct mlx5_flow_tunnel *tunnel, void *x)
11228 {
11229 	/* called under tunnel spinlock protection */
11230 	struct tunnel_db_get_tunnel_ctx *ctx = x;
11231 
11232 	RTE_SET_USED(dev);
11233 	tunnel->refctn++;
11234 	ctx->tunnel = tunnel;
11235 }
11236 
11237 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
11238 {
11239 	/* called under tunnel spinlock protection */
11240 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
11241 	struct tunnel_db_get_tunnel_ctx *ctx = x;
11242 
11243 	rte_spinlock_unlock(&thub->sl);
11244 	ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
11245 	rte_spinlock_lock(&thub->sl);
11246 	if (ctx->tunnel) {
11247 		ctx->tunnel->refctn = 1;
11248 		LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
11249 	}
11250 }
11251 
11252 
11253 static int
11254 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
11255 		     const struct rte_flow_tunnel *app_tunnel,
11256 		     struct mlx5_flow_tunnel **tunnel)
11257 {
11258 	struct tunnel_db_get_tunnel_ctx ctx = {
11259 		.app_tunnel = app_tunnel,
11260 	};
11261 
11262 	mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
11263 				      get_tunnel_miss, &ctx, true);
11264 	*tunnel = ctx.tunnel;
11265 	return ctx.tunnel ? 0 : -ENOMEM;
11266 }
11267 
11268 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
11269 {
11270 	struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
11271 
11272 	if (!thub)
11273 		return;
11274 	if (!LIST_EMPTY(&thub->tunnels))
11275 		DRV_LOG(WARNING, "port %u tunnels present", port_id);
11276 	mlx5_hlist_destroy(thub->groups);
11277 	mlx5_free(thub);
11278 }
11279 
11280 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
11281 {
11282 	int err;
11283 	struct mlx5_flow_tunnel_hub *thub;
11284 
11285 	thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
11286 			   0, SOCKET_ID_ANY);
11287 	if (!thub)
11288 		return -ENOMEM;
11289 	LIST_INIT(&thub->tunnels);
11290 	rte_spinlock_init(&thub->sl);
11291 	thub->groups = mlx5_hlist_create("flow groups", 64,
11292 					 false, true, sh,
11293 					 mlx5_flow_tunnel_grp2tbl_create_cb,
11294 					 mlx5_flow_tunnel_grp2tbl_match_cb,
11295 					 mlx5_flow_tunnel_grp2tbl_remove_cb,
11296 					 mlx5_flow_tunnel_grp2tbl_clone_cb,
11297 					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
11298 	if (!thub->groups) {
11299 		err = -rte_errno;
11300 		goto err;
11301 	}
11302 	sh->tunnel_hub = thub;
11303 
11304 	return 0;
11305 
11306 err:
11307 	if (thub->groups)
11308 		mlx5_hlist_destroy(thub->groups);
11309 	if (thub)
11310 		mlx5_free(thub);
11311 	return err;
11312 }
11313 
11314 static inline int
11315 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
11316 			  struct rte_flow_tunnel *tunnel,
11317 			  struct rte_flow_error *error)
11318 {
11319 	struct mlx5_priv *priv = dev->data->dev_private;
11320 
11321 	if (!priv->sh->config.dv_flow_en)
11322 		return rte_flow_error_set(error, ENOTSUP,
11323 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
11324 					  "flow DV interface is off");
11325 	if (!is_tunnel_offload_active(dev))
11326 		return rte_flow_error_set(error, ENOTSUP,
11327 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
11328 					  "tunnel offload was not activated, consider setting dv_xmeta_en=3");
11329 	if (!tunnel)
11330 		return rte_flow_error_set(error, EINVAL,
11331 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
11332 					  "no application tunnel");
11333 	switch (tunnel->type) {
11334 	default:
11335 		return rte_flow_error_set(error, EINVAL,
11336 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
11337 					  "unsupported tunnel type");
11338 	case RTE_FLOW_ITEM_TYPE_VXLAN:
11339 	case RTE_FLOW_ITEM_TYPE_GRE:
11340 	case RTE_FLOW_ITEM_TYPE_NVGRE:
11341 	case RTE_FLOW_ITEM_TYPE_GENEVE:
11342 		break;
11343 	}
11344 	return 0;
11345 }
11346 
11347 static int
11348 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
11349 		    struct rte_flow_tunnel *app_tunnel,
11350 		    struct rte_flow_action **actions,
11351 		    uint32_t *num_of_actions,
11352 		    struct rte_flow_error *error)
11353 {
11354 	struct mlx5_flow_tunnel *tunnel;
11355 	int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error);
11356 
11357 	if (ret)
11358 		return ret;
11359 	ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
11360 	if (ret < 0) {
11361 		return rte_flow_error_set(error, ret,
11362 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
11363 					  "failed to initialize pmd tunnel");
11364 	}
11365 	*actions = &tunnel->action;
11366 	*num_of_actions = 1;
11367 	return 0;
11368 }
11369 
11370 static int
11371 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
11372 		       struct rte_flow_tunnel *app_tunnel,
11373 		       struct rte_flow_item **items,
11374 		       uint32_t *num_of_items,
11375 		       struct rte_flow_error *error)
11376 {
11377 	struct mlx5_flow_tunnel *tunnel;
11378 	int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error);
11379 
11380 	if (ret)
11381 		return ret;
11382 	ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
11383 	if (ret < 0) {
11384 		return rte_flow_error_set(error, ret,
11385 					  RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
11386 					  "failed to initialize pmd tunnel");
11387 	}
11388 	*items = &tunnel->item;
11389 	*num_of_items = 1;
11390 	return 0;
11391 }
11392 
11393 struct tunnel_db_element_release_ctx {
11394 	struct rte_flow_item *items;
11395 	struct rte_flow_action *actions;
11396 	uint32_t num_elements;
11397 	struct rte_flow_error *error;
11398 	int ret;
11399 };
11400 
11401 static bool
11402 tunnel_element_release_match(struct rte_eth_dev *dev,
11403 			     struct mlx5_flow_tunnel *tunnel, const void *x)
11404 {
11405 	const struct tunnel_db_element_release_ctx *ctx = x;
11406 
11407 	RTE_SET_USED(dev);
11408 	if (ctx->num_elements != 1)
11409 		return false;
11410 	else if (ctx->items)
11411 		return ctx->items == &tunnel->item;
11412 	else if (ctx->actions)
11413 		return ctx->actions == &tunnel->action;
11414 
11415 	return false;
11416 }
11417 
11418 static void
11419 tunnel_element_release_hit(struct rte_eth_dev *dev,
11420 			   struct mlx5_flow_tunnel *tunnel, void *x)
11421 {
11422 	struct tunnel_db_element_release_ctx *ctx = x;
11423 	ctx->ret = 0;
11424 	if (!(rte_atomic_fetch_sub_explicit(&tunnel->refctn, 1, rte_memory_order_relaxed) - 1))
11425 		mlx5_flow_tunnel_free(dev, tunnel);
11426 }
11427 
11428 static void
11429 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
11430 {
11431 	struct tunnel_db_element_release_ctx *ctx = x;
11432 	RTE_SET_USED(dev);
11433 	ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
11434 				      RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
11435 				      "invalid argument");
11436 }
11437 
11438 static int
11439 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
11440 		       struct rte_flow_item *pmd_items,
11441 		       uint32_t num_items, struct rte_flow_error *err)
11442 {
11443 	struct tunnel_db_element_release_ctx ctx = {
11444 		.items = pmd_items,
11445 		.actions = NULL,
11446 		.num_elements = num_items,
11447 		.error = err,
11448 	};
11449 
11450 	mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
11451 				      tunnel_element_release_hit,
11452 				      tunnel_element_release_miss, &ctx, false);
11453 
11454 	return ctx.ret;
11455 }
11456 
11457 static int
11458 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
11459 			 struct rte_flow_action *pmd_actions,
11460 			 uint32_t num_actions, struct rte_flow_error *err)
11461 {
11462 	struct tunnel_db_element_release_ctx ctx = {
11463 		.items = NULL,
11464 		.actions = pmd_actions,
11465 		.num_elements = num_actions,
11466 		.error = err,
11467 	};
11468 
11469 	mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
11470 				      tunnel_element_release_hit,
11471 				      tunnel_element_release_miss, &ctx, false);
11472 
11473 	return ctx.ret;
11474 }
11475 
11476 static int
11477 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
11478 				  struct rte_mbuf *m,
11479 				  struct rte_flow_restore_info *info,
11480 				  struct rte_flow_error *err)
11481 {
11482 	uint64_t ol_flags = m->ol_flags;
11483 	const struct mlx5_flow_tbl_data_entry *tble;
11484 	const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
11485 	struct mlx5_priv *priv = dev->data->dev_private;
11486 
11487 	if (priv->tunnel_enabled == 0)
11488 		goto err;
11489 	if ((ol_flags & mask) != mask)
11490 		goto err;
11491 	tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
11492 	if (!tble) {
11493 		DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
11494 			dev->data->port_id, m->hash.fdir.hi);
11495 		goto err;
11496 	}
11497 	MLX5_ASSERT(tble->tunnel);
11498 	memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
11499 	info->group_id = tble->group_id;
11500 	info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
11501 		      RTE_FLOW_RESTORE_INFO_GROUP_ID |
11502 		      RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
11503 
11504 	return 0;
11505 
11506 err:
11507 	return rte_flow_error_set(err, EINVAL,
11508 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
11509 				  "failed to get restore info");
11510 }
11511 
11512 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
11513 static int
11514 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
11515 			   __rte_unused struct rte_flow_tunnel *app_tunnel,
11516 			   __rte_unused struct rte_flow_action **actions,
11517 			   __rte_unused uint32_t *num_of_actions,
11518 			   __rte_unused struct rte_flow_error *error)
11519 {
11520 	return -ENOTSUP;
11521 }
11522 
11523 static int
11524 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
11525 		       __rte_unused struct rte_flow_tunnel *app_tunnel,
11526 		       __rte_unused struct rte_flow_item **items,
11527 		       __rte_unused uint32_t *num_of_items,
11528 		       __rte_unused struct rte_flow_error *error)
11529 {
11530 	return -ENOTSUP;
11531 }
11532 
11533 static int
11534 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
11535 			      __rte_unused struct rte_flow_item *pmd_items,
11536 			      __rte_unused uint32_t num_items,
11537 			      __rte_unused struct rte_flow_error *err)
11538 {
11539 	return -ENOTSUP;
11540 }
11541 
11542 static int
11543 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
11544 				__rte_unused struct rte_flow_action *pmd_action,
11545 				__rte_unused uint32_t num_actions,
11546 				__rte_unused struct rte_flow_error *err)
11547 {
11548 	return -ENOTSUP;
11549 }
11550 
11551 static int
11552 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
11553 				  __rte_unused struct rte_mbuf *m,
11554 				  __rte_unused struct rte_flow_restore_info *i,
11555 				  __rte_unused struct rte_flow_error *err)
11556 {
11557 	return -ENOTSUP;
11558 }
11559 
11560 static int
11561 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
11562 			     __rte_unused struct rte_flow *flow,
11563 			     __rte_unused const struct rte_flow_attr *attr,
11564 			     __rte_unused const struct rte_flow_action *actions,
11565 			     __rte_unused uint32_t flow_idx,
11566 			     __rte_unused const struct mlx5_flow_tunnel *tunnel,
11567 			     __rte_unused struct tunnel_default_miss_ctx *ctx,
11568 			     __rte_unused struct rte_flow_error *error)
11569 {
11570 	return -ENOTSUP;
11571 }
11572 
11573 static struct mlx5_flow_tunnel *
11574 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
11575 		    __rte_unused uint32_t id)
11576 {
11577 	return NULL;
11578 }
11579 
11580 static void
11581 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
11582 		      __rte_unused struct mlx5_flow_tunnel *tunnel)
11583 {
11584 }
11585 
11586 static uint32_t
11587 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
11588 				__rte_unused const struct mlx5_flow_tunnel *t,
11589 				__rte_unused uint32_t group,
11590 				__rte_unused uint32_t *table,
11591 				struct rte_flow_error *error)
11592 {
11593 	return rte_flow_error_set(error, ENOTSUP,
11594 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
11595 				  "tunnel offload requires DV support");
11596 }
11597 
11598 void
11599 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
11600 			__rte_unused  uint16_t port_id)
11601 {
11602 }
11603 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
11604 
11605 /* Flex flow item API */
11606 static struct rte_flow_item_flex_handle *
11607 mlx5_flow_flex_item_create(struct rte_eth_dev *dev,
11608 			   const struct rte_flow_item_flex_conf *conf,
11609 			   struct rte_flow_error *error)
11610 {
11611 	static const char err_msg[] = "flex item creation unsupported";
11612 	struct mlx5_priv *priv = dev->data->dev_private;
11613 	struct rte_flow_attr attr = { .transfer = 0 };
11614 	const struct mlx5_flow_driver_ops *fops =
11615 			flow_get_drv_ops(flow_get_drv_type(dev, &attr));
11616 
11617 	if (!priv->pci_dev) {
11618 		rte_flow_error_set(error, ENOTSUP,
11619 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
11620 				   "create flex item on PF only");
11621 		return NULL;
11622 	}
11623 	switch (priv->pci_dev->id.device_id) {
11624 	case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2:
11625 	case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3:
11626 		break;
11627 	default:
11628 		rte_flow_error_set(error, ENOTSUP,
11629 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
11630 				   "flex item available on BlueField ports only");
11631 		return NULL;
11632 	}
11633 	if (!fops->item_create) {
11634 		DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
11635 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
11636 				   NULL, err_msg);
11637 		return NULL;
11638 	}
11639 	return fops->item_create(dev, conf, error);
11640 }
11641 
11642 static int
11643 mlx5_flow_flex_item_release(struct rte_eth_dev *dev,
11644 			    const struct rte_flow_item_flex_handle *handle,
11645 			    struct rte_flow_error *error)
11646 {
11647 	static const char err_msg[] = "flex item release unsupported";
11648 	struct rte_flow_attr attr = { .transfer = 0 };
11649 	const struct mlx5_flow_driver_ops *fops =
11650 			flow_get_drv_ops(flow_get_drv_type(dev, &attr));
11651 
11652 	if (!fops->item_release) {
11653 		DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
11654 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
11655 				   NULL, err_msg);
11656 		return -rte_errno;
11657 	}
11658 	return fops->item_release(dev, handle, error);
11659 }
11660 
11661 static void
11662 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
11663 {
11664 	int ret;
11665 	struct rte_flow_error error;
11666 
11667 	for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
11668 		char *item_name;
11669 		ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
11670 				    sizeof(item_name),
11671 				    (void *)(uintptr_t)item->type, &error);
11672 		if (ret > 0)
11673 			printf("%s ", item_name);
11674 		else
11675 			printf("%d\n", (int)item->type);
11676 	}
11677 	printf("END\n");
11678 }
11679 
11680 static int
11681 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item)
11682 {
11683 	const struct rte_flow_item_udp *spec = udp_item->spec;
11684 	const struct rte_flow_item_udp *mask = udp_item->mask;
11685 	uint16_t udp_dport = 0;
11686 
11687 	if (spec != NULL) {
11688 		if (!mask)
11689 			mask = &rte_flow_item_udp_mask;
11690 		udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port &
11691 				mask->hdr.dst_port);
11692 	}
11693 	return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN);
11694 }
11695 
11696 static const struct mlx5_flow_expand_node *
11697 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
11698 		unsigned int item_idx,
11699 		const struct mlx5_flow_expand_node graph[],
11700 		const struct mlx5_flow_expand_node *node)
11701 {
11702 	const struct rte_flow_item *item = pattern + item_idx, *prev_item;
11703 
11704 	if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN &&
11705 			node != NULL &&
11706 			node->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
11707 		/*
11708 		 * The expansion node is VXLAN and it is also the last
11709 		 * expandable item in the pattern, so need to continue
11710 		 * expansion of the inner tunnel.
11711 		 */
11712 		MLX5_ASSERT(item_idx > 0);
11713 		prev_item = pattern + item_idx - 1;
11714 		MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP);
11715 		if (mlx5_flow_is_std_vxlan_port(prev_item))
11716 			return &graph[MLX5_EXPANSION_STD_VXLAN];
11717 		return &graph[MLX5_EXPANSION_L3_VXLAN];
11718 	}
11719 	return node;
11720 }
11721 
11722 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
11723 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
11724 	{ 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
11725 };
11726 
11727 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
11728 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
11729 	{ 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
11730 	{ 9, 10, 11 }, { 12, 13, 14 },
11731 };
11732 
11733 /**
11734  * Discover the number of available flow priorities.
11735  *
11736  * @param dev
11737  *   Ethernet device.
11738  *
11739  * @return
11740  *   On success, number of available flow priorities.
11741  *   On failure, a negative errno-style code and rte_errno is set.
11742  */
11743 int
11744 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
11745 {
11746 	static const uint16_t vprio[] = {8, 16};
11747 	const struct mlx5_priv *priv = dev->data->dev_private;
11748 	const struct mlx5_flow_driver_ops *fops;
11749 	enum mlx5_flow_drv_type type;
11750 	int ret;
11751 
11752 	type = mlx5_flow_os_get_type();
11753 	if (type == MLX5_FLOW_TYPE_MAX) {
11754 		type = MLX5_FLOW_TYPE_VERBS;
11755 		if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en)
11756 			type = MLX5_FLOW_TYPE_DV;
11757 	}
11758 	fops = flow_get_drv_ops(type);
11759 	if (fops->discover_priorities == NULL) {
11760 		DRV_LOG(ERR, "Priority discovery not supported");
11761 		rte_errno = ENOTSUP;
11762 		return -rte_errno;
11763 	}
11764 	ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio));
11765 	if (ret < 0)
11766 		return ret;
11767 	switch (ret) {
11768 	case 8:
11769 		ret = RTE_DIM(priority_map_3);
11770 		break;
11771 	case 16:
11772 		ret = RTE_DIM(priority_map_5);
11773 		break;
11774 	default:
11775 		rte_errno = ENOTSUP;
11776 		DRV_LOG(ERR,
11777 			"port %u maximum priority: %d expected 8/16",
11778 			dev->data->port_id, ret);
11779 		return -rte_errno;
11780 	}
11781 	DRV_LOG(INFO, "port %u supported flow priorities:"
11782 		" 0-%d for ingress or egress root table,"
11783 		" 0-%d for non-root table or transfer root table.",
11784 		dev->data->port_id, ret - 2,
11785 		MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
11786 	return ret;
11787 }
11788 
11789 /**
11790  * Adjust flow priority based on the highest layer and the request priority.
11791  *
11792  * @param[in] dev
11793  *   Pointer to the Ethernet device structure.
11794  * @param[in] priority
11795  *   The rule base priority.
11796  * @param[in] subpriority
11797  *   The priority based on the items.
11798  *
11799  * @return
11800  *   The new priority.
11801  */
11802 uint32_t
11803 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
11804 			  uint32_t subpriority)
11805 {
11806 	uint32_t res = 0;
11807 	struct mlx5_priv *priv = dev->data->dev_private;
11808 
11809 	switch (priv->sh->flow_max_priority) {
11810 	case RTE_DIM(priority_map_3):
11811 		res = priority_map_3[priority][subpriority];
11812 		break;
11813 	case RTE_DIM(priority_map_5):
11814 		res = priority_map_5[priority][subpriority];
11815 		break;
11816 	}
11817 	return  res;
11818 }
11819 
11820 /**
11821  * Get the priority for sending traffic to kernel table.
11822  *
11823  * @param[in] dev
11824  *   Pointer to the Ethernet device structure.
11825  *
11826  * @return
11827  *   On success: the value of priority for sending traffic to kernel table
11828  *   On failure: -1
11829  */
11830 uint32_t
11831 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev)
11832 {
11833 	struct mlx5_priv *priv = dev->data->dev_private;
11834 	uint32_t res;
11835 
11836 	switch (priv->sh->flow_max_priority) {
11837 	case RTE_DIM(priority_map_5):
11838 		res = 15;
11839 		break;
11840 	case RTE_DIM(priority_map_3):
11841 		res = 7;
11842 		break;
11843 	default:
11844 		DRV_LOG(ERR,
11845 			"port %u maximum priority: %d expected 8/16",
11846 			dev->data->port_id, priv->sh->flow_max_priority);
11847 		res = (uint32_t)-1;
11848 	}
11849 	return res;
11850 }
11851 
11852 /**
11853  * Get the E-Switch Manager vport id.
11854  *
11855  * @param[in] dev
11856  *   Pointer to the Ethernet device structure.
11857  *
11858  * @return
11859  *   The vport id.
11860  */
11861 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev)
11862 {
11863 	struct mlx5_priv *priv = dev->data->dev_private;
11864 	struct mlx5_common_device *cdev = priv->sh->cdev;
11865 
11866 	/* New FW exposes E-Switch Manager vport ID, can use it directly. */
11867 	if (cdev->config.hca_attr.esw_mgr_vport_id_valid)
11868 		return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id;
11869 
11870 	if (priv->pci_dev == NULL)
11871 		return 0;
11872 	switch (priv->pci_dev->id.device_id) {
11873 	case PCI_DEVICE_ID_MELLANOX_BLUEFIELD:
11874 	case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2:
11875 	case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3:
11876 	/*
11877 	 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability,
11878 	 * only the BF embedded CPUs control the E-Switch Manager port. Hence,
11879 	 * ECPF vport ID is selected and not the host port (0) in any BF case.
11880 	 */
11881 		return (int16_t)MLX5_ECPF_VPORT_ID;
11882 	default:
11883 		return MLX5_PF_VPORT_ID;
11884 	}
11885 }
11886 
11887 /**
11888  * Parse item to get the vport id.
11889  *
11890  * @param[in] dev
11891  *   Pointer to the Ethernet device structure.
11892  * @param[in] item
11893  *   The src port id match item.
11894  * @param[out] vport_id
11895  *   Pointer to put the vport id.
11896  * @param[out] all_ports
11897  *   Indicate if the item matches all ports.
11898  * @param[out] error
11899  *   Pointer to error structure.
11900  *
11901  * @return
11902  *   0 on success, a negative errno value otherwise and rte_errno is set.
11903  */
11904 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev,
11905 				const struct rte_flow_item *item,
11906 				uint16_t *vport_id,
11907 				bool *all_ports,
11908 				struct rte_flow_error *error)
11909 {
11910 	struct mlx5_priv *port_priv;
11911 	const struct rte_flow_item_port_id *pid_v = NULL;
11912 	const struct rte_flow_item_ethdev *dev_v = NULL;
11913 	uint32_t esw_mgr_port;
11914 	uint32_t src_port;
11915 
11916 	if (all_ports)
11917 		*all_ports = false;
11918 	switch (item->type) {
11919 	case RTE_FLOW_ITEM_TYPE_PORT_ID:
11920 		pid_v = item->spec;
11921 		if (!pid_v)
11922 			return 0;
11923 		src_port = pid_v->id;
11924 		esw_mgr_port = MLX5_PORT_ESW_MGR;
11925 		break;
11926 	case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT:
11927 		dev_v = item->spec;
11928 		if (!dev_v) {
11929 			if (all_ports)
11930 				*all_ports = true;
11931 			return 0;
11932 		}
11933 		src_port = dev_v->port_id;
11934 		esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR;
11935 		break;
11936 	case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR:
11937 		src_port = MLX5_REPRESENTED_PORT_ESW_MGR;
11938 		esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR;
11939 		break;
11940 	default:
11941 		return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
11942 					  NULL, "Incorrect item type.");
11943 	}
11944 	if (src_port == esw_mgr_port) {
11945 		*vport_id = mlx5_flow_get_esw_manager_vport_id(dev);
11946 	} else {
11947 		port_priv = mlx5_port_to_eswitch_info(src_port, false);
11948 		if (!port_priv)
11949 			return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
11950 						  NULL, "Failed to get port info.");
11951 		*vport_id = port_priv->representor_id;
11952 	}
11953 
11954 	return 0;
11955 }
11956 
11957 int
11958 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev,
11959 			      uint16_t *proxy_port_id,
11960 			      struct rte_flow_error *error)
11961 {
11962 	const struct mlx5_priv *priv = dev->data->dev_private;
11963 	uint16_t port_id;
11964 
11965 	if (!priv->sh->config.dv_esw_en)
11966 		return rte_flow_error_set(error, EINVAL,
11967 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11968 					  NULL,
11969 					  "unable to provide a proxy port"
11970 					  " without E-Switch configured");
11971 	if (!priv->master && !priv->representor)
11972 		return rte_flow_error_set(error, EINVAL,
11973 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11974 					  NULL,
11975 					  "unable to provide a proxy port"
11976 					  " for port which is not a master"
11977 					  " or a representor port");
11978 	if (priv->master) {
11979 		*proxy_port_id = dev->data->port_id;
11980 		return 0;
11981 	}
11982 	MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
11983 		const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id];
11984 		const struct mlx5_priv *port_priv = port_dev->data->dev_private;
11985 
11986 		if (port_priv->master &&
11987 		    port_priv->domain_id == priv->domain_id) {
11988 			*proxy_port_id = port_id;
11989 			return 0;
11990 		}
11991 	}
11992 	return rte_flow_error_set(error, ENODEV,
11993 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11994 				  NULL, "unable to find a proxy port");
11995 }
11996 
11997 /**
11998  * Discover IPv6 traffic class ID support in rdma-core and firmware.
11999  *
12000  * @param dev
12001  *   Ethernet device.
12002  *
12003  * @return
12004  *   0, rdma-core is good to work with firmware.
12005  *   -EOPNOTSUPP, rdma-core could not work with new IPv6 TC ID.
12006  */
12007 int
12008 mlx5_flow_discover_ipv6_tc_support(struct rte_eth_dev *dev)
12009 {
12010 	struct rte_flow_action_set_dscp set_dscp;
12011 	struct rte_flow_attr attr;
12012 	struct rte_flow_action actions[2];
12013 	struct rte_flow_item items[3];
12014 	struct rte_flow_error error;
12015 	uint32_t flow_idx;
12016 
12017 	memset(&attr, 0, sizeof(attr));
12018 	memset(actions, 0, sizeof(actions));
12019 	memset(items, 0, sizeof(items));
12020 	attr.group = 1;
12021 	attr.egress = 1;
12022 	items[0].type = RTE_FLOW_ITEM_TYPE_ETH;
12023 	items[1].type = RTE_FLOW_ITEM_TYPE_IPV6;
12024 	items[2].type = RTE_FLOW_ITEM_TYPE_END;
12025 	/* Random value */
12026 	set_dscp.dscp = 9;
12027 	actions[0].type = RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP;
12028 	actions[0].conf = &set_dscp;
12029 	actions[1].type = RTE_FLOW_ACTION_TYPE_END;
12030 
12031 	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, items, actions, true, &error);
12032 	if (!flow_idx)
12033 		return -EOPNOTSUPP;
12034 
12035 	flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
12036 	return 0;
12037 }
12038 
12039 void *
12040 rte_pmd_mlx5_create_geneve_tlv_parser(uint16_t port_id,
12041 				      const struct rte_pmd_mlx5_geneve_tlv tlv_list[],
12042 				      uint8_t nb_options)
12043 {
12044 #ifdef HAVE_MLX5_HWS_SUPPORT
12045 	return mlx5_geneve_tlv_parser_create(port_id, tlv_list, nb_options);
12046 #else
12047 	(void)port_id;
12048 	(void)tlv_list;
12049 	(void)nb_options;
12050 	DRV_LOG(ERR, "%s is not supported.", __func__);
12051 	rte_errno = ENOTSUP;
12052 	return NULL;
12053 #endif
12054 }
12055 
12056 int
12057 rte_pmd_mlx5_destroy_geneve_tlv_parser(void *handle)
12058 {
12059 #ifdef HAVE_MLX5_HWS_SUPPORT
12060 	return mlx5_geneve_tlv_parser_destroy(handle);
12061 #else
12062 	(void)handle;
12063 	DRV_LOG(ERR, "%s is not supported.", __func__);
12064 	rte_errno = ENOTSUP;
12065 	return -rte_errno;
12066 #endif
12067 }
12068