xref: /dpdk/drivers/net/sfc/sfc_flow.c (revision d77d07391d4def5b8bac2ad92d46488d7ca3737c)
1 /*-
2  *   BSD LICENSE
3  *
4  * Copyright (c) 2017 Solarflare Communications Inc.
5  * All rights reserved.
6  *
7  * This software was jointly developed between OKTET Labs (under contract
8  * for Solarflare) and Solarflare Communications, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * 1. Redistributions of source code must retain the above copyright notice,
14  *    this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright notice,
16  *    this list of conditions and the following disclaimer in the documentation
17  *    and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 #include <rte_tailq.h>
33 #include <rte_common.h>
34 #include <rte_ethdev.h>
35 #include <rte_eth_ctrl.h>
36 #include <rte_ether.h>
37 #include <rte_flow.h>
38 #include <rte_flow_driver.h>
39 
40 #include "efx.h"
41 
42 #include "sfc.h"
43 #include "sfc_rx.h"
44 #include "sfc_filter.h"
45 #include "sfc_flow.h"
46 #include "sfc_log.h"
47 
48 /*
49  * At now flow API is implemented in such a manner that each
50  * flow rule is converted to a hardware filter.
51  * All elements of flow rule (attributes, pattern items, actions)
52  * correspond to one or more fields in the efx_filter_spec_s structure
53  * that is responsible for the hardware filter.
54  */
55 
56 enum sfc_flow_item_layers {
57 	SFC_FLOW_ITEM_ANY_LAYER,
58 	SFC_FLOW_ITEM_START_LAYER,
59 	SFC_FLOW_ITEM_L2,
60 	SFC_FLOW_ITEM_L3,
61 	SFC_FLOW_ITEM_L4,
62 };
63 
64 typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item,
65 				  efx_filter_spec_t *spec,
66 				  struct rte_flow_error *error);
67 
68 struct sfc_flow_item {
69 	enum rte_flow_item_type type;		/* Type of item */
70 	enum sfc_flow_item_layers layer;	/* Layer of item */
71 	enum sfc_flow_item_layers prev_layer;	/* Previous layer of item */
72 	sfc_flow_item_parse *parse;		/* Parsing function */
73 };
74 
75 static sfc_flow_item_parse sfc_flow_parse_void;
76 static sfc_flow_item_parse sfc_flow_parse_eth;
77 static sfc_flow_item_parse sfc_flow_parse_vlan;
78 static sfc_flow_item_parse sfc_flow_parse_ipv4;
79 static sfc_flow_item_parse sfc_flow_parse_ipv6;
80 static sfc_flow_item_parse sfc_flow_parse_tcp;
81 static sfc_flow_item_parse sfc_flow_parse_udp;
82 
83 static boolean_t
84 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
85 {
86 	uint8_t sum = 0;
87 	unsigned int i;
88 
89 	for (i = 0; i < size; i++)
90 		sum |= buf[i];
91 
92 	return (sum == 0) ? B_TRUE : B_FALSE;
93 }
94 
95 /*
96  * Validate item and prepare structures spec and mask for parsing
97  */
98 static int
99 sfc_flow_parse_init(const struct rte_flow_item *item,
100 		    const void **spec_ptr,
101 		    const void **mask_ptr,
102 		    const void *supp_mask,
103 		    const void *def_mask,
104 		    unsigned int size,
105 		    struct rte_flow_error *error)
106 {
107 	const uint8_t *spec;
108 	const uint8_t *mask;
109 	const uint8_t *last;
110 	uint8_t match;
111 	uint8_t supp;
112 	unsigned int i;
113 
114 	if (item == NULL) {
115 		rte_flow_error_set(error, EINVAL,
116 				   RTE_FLOW_ERROR_TYPE_ITEM, NULL,
117 				   "NULL item");
118 		return -rte_errno;
119 	}
120 
121 	if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) {
122 		rte_flow_error_set(error, EINVAL,
123 				   RTE_FLOW_ERROR_TYPE_ITEM, item,
124 				   "Mask or last is set without spec");
125 		return -rte_errno;
126 	}
127 
128 	/*
129 	 * If "mask" is not set, default mask is used,
130 	 * but if default mask is NULL, "mask" should be set
131 	 */
132 	if (item->mask == NULL) {
133 		if (def_mask == NULL) {
134 			rte_flow_error_set(error, EINVAL,
135 				RTE_FLOW_ERROR_TYPE_ITEM, NULL,
136 				"Mask should be specified");
137 			return -rte_errno;
138 		}
139 
140 		mask = (const uint8_t *)def_mask;
141 	} else {
142 		mask = (const uint8_t *)item->mask;
143 	}
144 
145 	spec = (const uint8_t *)item->spec;
146 	last = (const uint8_t *)item->last;
147 
148 	if (spec == NULL)
149 		goto exit;
150 
151 	/*
152 	 * If field values in "last" are either 0 or equal to the corresponding
153 	 * values in "spec" then they are ignored
154 	 */
155 	if (last != NULL &&
156 	    !sfc_flow_is_zero(last, size) &&
157 	    memcmp(last, spec, size) != 0) {
158 		rte_flow_error_set(error, ENOTSUP,
159 				   RTE_FLOW_ERROR_TYPE_ITEM, item,
160 				   "Ranging is not supported");
161 		return -rte_errno;
162 	}
163 
164 	if (supp_mask == NULL) {
165 		rte_flow_error_set(error, EINVAL,
166 			RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
167 			"Supported mask for item should be specified");
168 		return -rte_errno;
169 	}
170 
171 	/* Check that mask and spec not asks for more match than supp_mask */
172 	for (i = 0; i < size; i++) {
173 		match = spec[i] | mask[i];
174 		supp = ((const uint8_t *)supp_mask)[i];
175 
176 		if ((match | supp) != supp) {
177 			rte_flow_error_set(error, ENOTSUP,
178 					   RTE_FLOW_ERROR_TYPE_ITEM, item,
179 					   "Item's field is not supported");
180 			return -rte_errno;
181 		}
182 	}
183 
184 exit:
185 	*spec_ptr = spec;
186 	*mask_ptr = mask;
187 	return 0;
188 }
189 
190 /*
191  * Protocol parsers.
192  * Masking is not supported, so masks in items should be either
193  * full or empty (zeroed) and set only for supported fields which
194  * are specified in the supp_mask.
195  */
196 
197 static int
198 sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item,
199 		    __rte_unused efx_filter_spec_t *efx_spec,
200 		    __rte_unused struct rte_flow_error *error)
201 {
202 	return 0;
203 }
204 
205 /**
206  * Convert Ethernet item to EFX filter specification.
207  *
208  * @param item[in]
209  *   Item specification. Only source and destination addresses and
210  *   Ethernet type fields are supported. In addition to full and
211  *   empty masks of destination address, individual/group mask is
212  *   also supported. If the mask is NULL, default mask will be used.
213  *   Ranging is not supported.
214  * @param efx_spec[in, out]
215  *   EFX filter specification to update.
216  * @param[out] error
217  *   Perform verbose error reporting if not NULL.
218  */
219 static int
220 sfc_flow_parse_eth(const struct rte_flow_item *item,
221 		   efx_filter_spec_t *efx_spec,
222 		   struct rte_flow_error *error)
223 {
224 	int rc;
225 	const struct rte_flow_item_eth *spec = NULL;
226 	const struct rte_flow_item_eth *mask = NULL;
227 	const struct rte_flow_item_eth supp_mask = {
228 		.dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
229 		.src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
230 		.type = 0xffff,
231 	};
232 	const uint8_t ig_mask[EFX_MAC_ADDR_LEN] = {
233 		0x01, 0x00, 0x00, 0x00, 0x00, 0x00
234 	};
235 
236 	rc = sfc_flow_parse_init(item,
237 				 (const void **)&spec,
238 				 (const void **)&mask,
239 				 &supp_mask,
240 				 &rte_flow_item_eth_mask,
241 				 sizeof(struct rte_flow_item_eth),
242 				 error);
243 	if (rc != 0)
244 		return rc;
245 
246 	/* If "spec" is not set, could be any Ethernet */
247 	if (spec == NULL)
248 		return 0;
249 
250 	if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) {
251 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
252 		rte_memcpy(efx_spec->efs_loc_mac, spec->dst.addr_bytes,
253 			   EFX_MAC_ADDR_LEN);
254 	} else if (memcmp(mask->dst.addr_bytes, ig_mask,
255 			  EFX_MAC_ADDR_LEN) == 0) {
256 		if (is_unicast_ether_addr(&spec->dst))
257 			efx_spec->efs_match_flags |=
258 				EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
259 		else
260 			efx_spec->efs_match_flags |=
261 				EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
262 	} else if (!is_zero_ether_addr(&mask->dst)) {
263 		goto fail_bad_mask;
264 	}
265 
266 	if (is_same_ether_addr(&mask->src, &supp_mask.src)) {
267 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC;
268 		rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes,
269 			   EFX_MAC_ADDR_LEN);
270 	} else if (!is_zero_ether_addr(&mask->src)) {
271 		goto fail_bad_mask;
272 	}
273 
274 	/*
275 	 * Ether type is in big-endian byte order in item and
276 	 * in little-endian in efx_spec, so byte swap is used
277 	 */
278 	if (mask->type == supp_mask.type) {
279 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
280 		efx_spec->efs_ether_type = rte_bswap16(spec->type);
281 	} else if (mask->type != 0) {
282 		goto fail_bad_mask;
283 	}
284 
285 	return 0;
286 
287 fail_bad_mask:
288 	rte_flow_error_set(error, EINVAL,
289 			   RTE_FLOW_ERROR_TYPE_ITEM, item,
290 			   "Bad mask in the ETH pattern item");
291 	return -rte_errno;
292 }
293 
294 /**
295  * Convert VLAN item to EFX filter specification.
296  *
297  * @param item[in]
298  *   Item specification. Only VID field is supported.
299  *   The mask can not be NULL. Ranging is not supported.
300  * @param efx_spec[in, out]
301  *   EFX filter specification to update.
302  * @param[out] error
303  *   Perform verbose error reporting if not NULL.
304  */
305 static int
306 sfc_flow_parse_vlan(const struct rte_flow_item *item,
307 		    efx_filter_spec_t *efx_spec,
308 		    struct rte_flow_error *error)
309 {
310 	int rc;
311 	uint16_t vid;
312 	const struct rte_flow_item_vlan *spec = NULL;
313 	const struct rte_flow_item_vlan *mask = NULL;
314 	const struct rte_flow_item_vlan supp_mask = {
315 		.tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX),
316 	};
317 
318 	rc = sfc_flow_parse_init(item,
319 				 (const void **)&spec,
320 				 (const void **)&mask,
321 				 &supp_mask,
322 				 NULL,
323 				 sizeof(struct rte_flow_item_vlan),
324 				 error);
325 	if (rc != 0)
326 		return rc;
327 
328 	/*
329 	 * VID is in big-endian byte order in item and
330 	 * in little-endian in efx_spec, so byte swap is used.
331 	 * If two VLAN items are included, the first matches
332 	 * the outer tag and the next matches the inner tag.
333 	 */
334 	if (mask->tci == supp_mask.tci) {
335 		vid = rte_bswap16(spec->tci);
336 
337 		if (!(efx_spec->efs_match_flags &
338 		      EFX_FILTER_MATCH_OUTER_VID)) {
339 			efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
340 			efx_spec->efs_outer_vid = vid;
341 		} else if (!(efx_spec->efs_match_flags &
342 			     EFX_FILTER_MATCH_INNER_VID)) {
343 			efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID;
344 			efx_spec->efs_inner_vid = vid;
345 		} else {
346 			rte_flow_error_set(error, EINVAL,
347 					   RTE_FLOW_ERROR_TYPE_ITEM, item,
348 					   "More than two VLAN items");
349 			return -rte_errno;
350 		}
351 	} else {
352 		rte_flow_error_set(error, EINVAL,
353 				   RTE_FLOW_ERROR_TYPE_ITEM, item,
354 				   "VLAN ID in TCI match is required");
355 		return -rte_errno;
356 	}
357 
358 	return 0;
359 }
360 
361 /**
362  * Convert IPv4 item to EFX filter specification.
363  *
364  * @param item[in]
365  *   Item specification. Only source and destination addresses and
366  *   protocol fields are supported. If the mask is NULL, default
367  *   mask will be used. Ranging is not supported.
368  * @param efx_spec[in, out]
369  *   EFX filter specification to update.
370  * @param[out] error
371  *   Perform verbose error reporting if not NULL.
372  */
373 static int
374 sfc_flow_parse_ipv4(const struct rte_flow_item *item,
375 		    efx_filter_spec_t *efx_spec,
376 		    struct rte_flow_error *error)
377 {
378 	int rc;
379 	const struct rte_flow_item_ipv4 *spec = NULL;
380 	const struct rte_flow_item_ipv4 *mask = NULL;
381 	const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4);
382 	const struct rte_flow_item_ipv4 supp_mask = {
383 		.hdr = {
384 			.src_addr = 0xffffffff,
385 			.dst_addr = 0xffffffff,
386 			.next_proto_id = 0xff,
387 		}
388 	};
389 
390 	rc = sfc_flow_parse_init(item,
391 				 (const void **)&spec,
392 				 (const void **)&mask,
393 				 &supp_mask,
394 				 &rte_flow_item_ipv4_mask,
395 				 sizeof(struct rte_flow_item_ipv4),
396 				 error);
397 	if (rc != 0)
398 		return rc;
399 
400 	/*
401 	 * Filtering by IPv4 source and destination addresses requires
402 	 * the appropriate ETHER_TYPE in hardware filters
403 	 */
404 	if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
405 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
406 		efx_spec->efs_ether_type = ether_type_ipv4;
407 	} else if (efx_spec->efs_ether_type != ether_type_ipv4) {
408 		rte_flow_error_set(error, EINVAL,
409 			RTE_FLOW_ERROR_TYPE_ITEM, item,
410 			"Ethertype in pattern with IPV4 item should be appropriate");
411 		return -rte_errno;
412 	}
413 
414 	if (spec == NULL)
415 		return 0;
416 
417 	/*
418 	 * IPv4 addresses are in big-endian byte order in item and in
419 	 * efx_spec
420 	 */
421 	if (mask->hdr.src_addr == supp_mask.hdr.src_addr) {
422 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
423 		efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr;
424 	} else if (mask->hdr.src_addr != 0) {
425 		goto fail_bad_mask;
426 	}
427 
428 	if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) {
429 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
430 		efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr;
431 	} else if (mask->hdr.dst_addr != 0) {
432 		goto fail_bad_mask;
433 	}
434 
435 	if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) {
436 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
437 		efx_spec->efs_ip_proto = spec->hdr.next_proto_id;
438 	} else if (mask->hdr.next_proto_id != 0) {
439 		goto fail_bad_mask;
440 	}
441 
442 	return 0;
443 
444 fail_bad_mask:
445 	rte_flow_error_set(error, EINVAL,
446 			   RTE_FLOW_ERROR_TYPE_ITEM, item,
447 			   "Bad mask in the IPV4 pattern item");
448 	return -rte_errno;
449 }
450 
451 /**
452  * Convert IPv6 item to EFX filter specification.
453  *
454  * @param item[in]
455  *   Item specification. Only source and destination addresses and
456  *   next header fields are supported. If the mask is NULL, default
457  *   mask will be used. Ranging is not supported.
458  * @param efx_spec[in, out]
459  *   EFX filter specification to update.
460  * @param[out] error
461  *   Perform verbose error reporting if not NULL.
462  */
463 static int
464 sfc_flow_parse_ipv6(const struct rte_flow_item *item,
465 		    efx_filter_spec_t *efx_spec,
466 		    struct rte_flow_error *error)
467 {
468 	int rc;
469 	const struct rte_flow_item_ipv6 *spec = NULL;
470 	const struct rte_flow_item_ipv6 *mask = NULL;
471 	const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6);
472 	const struct rte_flow_item_ipv6 supp_mask = {
473 		.hdr = {
474 			.src_addr = { 0xff, 0xff, 0xff, 0xff,
475 				      0xff, 0xff, 0xff, 0xff,
476 				      0xff, 0xff, 0xff, 0xff,
477 				      0xff, 0xff, 0xff, 0xff },
478 			.dst_addr = { 0xff, 0xff, 0xff, 0xff,
479 				      0xff, 0xff, 0xff, 0xff,
480 				      0xff, 0xff, 0xff, 0xff,
481 				      0xff, 0xff, 0xff, 0xff },
482 			.proto = 0xff,
483 		}
484 	};
485 
486 	rc = sfc_flow_parse_init(item,
487 				 (const void **)&spec,
488 				 (const void **)&mask,
489 				 &supp_mask,
490 				 &rte_flow_item_ipv6_mask,
491 				 sizeof(struct rte_flow_item_ipv6),
492 				 error);
493 	if (rc != 0)
494 		return rc;
495 
496 	/*
497 	 * Filtering by IPv6 source and destination addresses requires
498 	 * the appropriate ETHER_TYPE in hardware filters
499 	 */
500 	if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
501 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
502 		efx_spec->efs_ether_type = ether_type_ipv6;
503 	} else if (efx_spec->efs_ether_type != ether_type_ipv6) {
504 		rte_flow_error_set(error, EINVAL,
505 			RTE_FLOW_ERROR_TYPE_ITEM, item,
506 			"Ethertype in pattern with IPV6 item should be appropriate");
507 		return -rte_errno;
508 	}
509 
510 	if (spec == NULL)
511 		return 0;
512 
513 	/*
514 	 * IPv6 addresses are in big-endian byte order in item and in
515 	 * efx_spec
516 	 */
517 	if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr,
518 		   sizeof(mask->hdr.src_addr)) == 0) {
519 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
520 
521 		RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) !=
522 				 sizeof(spec->hdr.src_addr));
523 		rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr,
524 			   sizeof(efx_spec->efs_rem_host));
525 	} else if (!sfc_flow_is_zero(mask->hdr.src_addr,
526 				     sizeof(mask->hdr.src_addr))) {
527 		goto fail_bad_mask;
528 	}
529 
530 	if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr,
531 		   sizeof(mask->hdr.dst_addr)) == 0) {
532 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
533 
534 		RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) !=
535 				 sizeof(spec->hdr.dst_addr));
536 		rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr,
537 			   sizeof(efx_spec->efs_loc_host));
538 	} else if (!sfc_flow_is_zero(mask->hdr.dst_addr,
539 				     sizeof(mask->hdr.dst_addr))) {
540 		goto fail_bad_mask;
541 	}
542 
543 	if (mask->hdr.proto == supp_mask.hdr.proto) {
544 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
545 		efx_spec->efs_ip_proto = spec->hdr.proto;
546 	} else if (mask->hdr.proto != 0) {
547 		goto fail_bad_mask;
548 	}
549 
550 	return 0;
551 
552 fail_bad_mask:
553 	rte_flow_error_set(error, EINVAL,
554 			   RTE_FLOW_ERROR_TYPE_ITEM, item,
555 			   "Bad mask in the IPV6 pattern item");
556 	return -rte_errno;
557 }
558 
559 /**
560  * Convert TCP item to EFX filter specification.
561  *
562  * @param item[in]
563  *   Item specification. Only source and destination ports fields
564  *   are supported. If the mask is NULL, default mask will be used.
565  *   Ranging is not supported.
566  * @param efx_spec[in, out]
567  *   EFX filter specification to update.
568  * @param[out] error
569  *   Perform verbose error reporting if not NULL.
570  */
571 static int
572 sfc_flow_parse_tcp(const struct rte_flow_item *item,
573 		   efx_filter_spec_t *efx_spec,
574 		   struct rte_flow_error *error)
575 {
576 	int rc;
577 	const struct rte_flow_item_tcp *spec = NULL;
578 	const struct rte_flow_item_tcp *mask = NULL;
579 	const struct rte_flow_item_tcp supp_mask = {
580 		.hdr = {
581 			.src_port = 0xffff,
582 			.dst_port = 0xffff,
583 		}
584 	};
585 
586 	rc = sfc_flow_parse_init(item,
587 				 (const void **)&spec,
588 				 (const void **)&mask,
589 				 &supp_mask,
590 				 &rte_flow_item_tcp_mask,
591 				 sizeof(struct rte_flow_item_tcp),
592 				 error);
593 	if (rc != 0)
594 		return rc;
595 
596 	/*
597 	 * Filtering by TCP source and destination ports requires
598 	 * the appropriate IP_PROTO in hardware filters
599 	 */
600 	if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
601 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
602 		efx_spec->efs_ip_proto = EFX_IPPROTO_TCP;
603 	} else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) {
604 		rte_flow_error_set(error, EINVAL,
605 			RTE_FLOW_ERROR_TYPE_ITEM, item,
606 			"IP proto in pattern with TCP item should be appropriate");
607 		return -rte_errno;
608 	}
609 
610 	if (spec == NULL)
611 		return 0;
612 
613 	/*
614 	 * Source and destination ports are in big-endian byte order in item and
615 	 * in little-endian in efx_spec, so byte swap is used
616 	 */
617 	if (mask->hdr.src_port == supp_mask.hdr.src_port) {
618 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
619 		efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
620 	} else if (mask->hdr.src_port != 0) {
621 		goto fail_bad_mask;
622 	}
623 
624 	if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
625 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
626 		efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
627 	} else if (mask->hdr.dst_port != 0) {
628 		goto fail_bad_mask;
629 	}
630 
631 	return 0;
632 
633 fail_bad_mask:
634 	rte_flow_error_set(error, EINVAL,
635 			   RTE_FLOW_ERROR_TYPE_ITEM, item,
636 			   "Bad mask in the TCP pattern item");
637 	return -rte_errno;
638 }
639 
640 /**
641  * Convert UDP item to EFX filter specification.
642  *
643  * @param item[in]
644  *   Item specification. Only source and destination ports fields
645  *   are supported. If the mask is NULL, default mask will be used.
646  *   Ranging is not supported.
647  * @param efx_spec[in, out]
648  *   EFX filter specification to update.
649  * @param[out] error
650  *   Perform verbose error reporting if not NULL.
651  */
652 static int
653 sfc_flow_parse_udp(const struct rte_flow_item *item,
654 		   efx_filter_spec_t *efx_spec,
655 		   struct rte_flow_error *error)
656 {
657 	int rc;
658 	const struct rte_flow_item_udp *spec = NULL;
659 	const struct rte_flow_item_udp *mask = NULL;
660 	const struct rte_flow_item_udp supp_mask = {
661 		.hdr = {
662 			.src_port = 0xffff,
663 			.dst_port = 0xffff,
664 		}
665 	};
666 
667 	rc = sfc_flow_parse_init(item,
668 				 (const void **)&spec,
669 				 (const void **)&mask,
670 				 &supp_mask,
671 				 &rte_flow_item_udp_mask,
672 				 sizeof(struct rte_flow_item_udp),
673 				 error);
674 	if (rc != 0)
675 		return rc;
676 
677 	/*
678 	 * Filtering by UDP source and destination ports requires
679 	 * the appropriate IP_PROTO in hardware filters
680 	 */
681 	if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
682 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
683 		efx_spec->efs_ip_proto = EFX_IPPROTO_UDP;
684 	} else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) {
685 		rte_flow_error_set(error, EINVAL,
686 			RTE_FLOW_ERROR_TYPE_ITEM, item,
687 			"IP proto in pattern with UDP item should be appropriate");
688 		return -rte_errno;
689 	}
690 
691 	if (spec == NULL)
692 		return 0;
693 
694 	/*
695 	 * Source and destination ports are in big-endian byte order in item and
696 	 * in little-endian in efx_spec, so byte swap is used
697 	 */
698 	if (mask->hdr.src_port == supp_mask.hdr.src_port) {
699 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
700 		efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
701 	} else if (mask->hdr.src_port != 0) {
702 		goto fail_bad_mask;
703 	}
704 
705 	if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
706 		efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
707 		efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
708 	} else if (mask->hdr.dst_port != 0) {
709 		goto fail_bad_mask;
710 	}
711 
712 	return 0;
713 
714 fail_bad_mask:
715 	rte_flow_error_set(error, EINVAL,
716 			   RTE_FLOW_ERROR_TYPE_ITEM, item,
717 			   "Bad mask in the UDP pattern item");
718 	return -rte_errno;
719 }
720 
721 static const struct sfc_flow_item sfc_flow_items[] = {
722 	{
723 		.type = RTE_FLOW_ITEM_TYPE_VOID,
724 		.prev_layer = SFC_FLOW_ITEM_ANY_LAYER,
725 		.layer = SFC_FLOW_ITEM_ANY_LAYER,
726 		.parse = sfc_flow_parse_void,
727 	},
728 	{
729 		.type = RTE_FLOW_ITEM_TYPE_ETH,
730 		.prev_layer = SFC_FLOW_ITEM_START_LAYER,
731 		.layer = SFC_FLOW_ITEM_L2,
732 		.parse = sfc_flow_parse_eth,
733 	},
734 	{
735 		.type = RTE_FLOW_ITEM_TYPE_VLAN,
736 		.prev_layer = SFC_FLOW_ITEM_L2,
737 		.layer = SFC_FLOW_ITEM_L2,
738 		.parse = sfc_flow_parse_vlan,
739 	},
740 	{
741 		.type = RTE_FLOW_ITEM_TYPE_IPV4,
742 		.prev_layer = SFC_FLOW_ITEM_L2,
743 		.layer = SFC_FLOW_ITEM_L3,
744 		.parse = sfc_flow_parse_ipv4,
745 	},
746 	{
747 		.type = RTE_FLOW_ITEM_TYPE_IPV6,
748 		.prev_layer = SFC_FLOW_ITEM_L2,
749 		.layer = SFC_FLOW_ITEM_L3,
750 		.parse = sfc_flow_parse_ipv6,
751 	},
752 	{
753 		.type = RTE_FLOW_ITEM_TYPE_TCP,
754 		.prev_layer = SFC_FLOW_ITEM_L3,
755 		.layer = SFC_FLOW_ITEM_L4,
756 		.parse = sfc_flow_parse_tcp,
757 	},
758 	{
759 		.type = RTE_FLOW_ITEM_TYPE_UDP,
760 		.prev_layer = SFC_FLOW_ITEM_L3,
761 		.layer = SFC_FLOW_ITEM_L4,
762 		.parse = sfc_flow_parse_udp,
763 	},
764 };
765 
766 /*
767  * Protocol-independent flow API support
768  */
769 static int
770 sfc_flow_parse_attr(const struct rte_flow_attr *attr,
771 		    struct rte_flow *flow,
772 		    struct rte_flow_error *error)
773 {
774 	if (attr == NULL) {
775 		rte_flow_error_set(error, EINVAL,
776 				   RTE_FLOW_ERROR_TYPE_ATTR, NULL,
777 				   "NULL attribute");
778 		return -rte_errno;
779 	}
780 	if (attr->group != 0) {
781 		rte_flow_error_set(error, ENOTSUP,
782 				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
783 				   "Groups are not supported");
784 		return -rte_errno;
785 	}
786 	if (attr->priority != 0) {
787 		rte_flow_error_set(error, ENOTSUP,
788 				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
789 				   "Priorities are not supported");
790 		return -rte_errno;
791 	}
792 	if (attr->egress != 0) {
793 		rte_flow_error_set(error, ENOTSUP,
794 				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
795 				   "Egress is not supported");
796 		return -rte_errno;
797 	}
798 	if (attr->ingress == 0) {
799 		rte_flow_error_set(error, ENOTSUP,
800 				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
801 				   "Only ingress is supported");
802 		return -rte_errno;
803 	}
804 
805 	flow->spec.efs_flags |= EFX_FILTER_FLAG_RX;
806 	flow->spec.efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
807 
808 	return 0;
809 }
810 
811 /* Get item from array sfc_flow_items */
812 static const struct sfc_flow_item *
813 sfc_flow_get_item(enum rte_flow_item_type type)
814 {
815 	unsigned int i;
816 
817 	for (i = 0; i < RTE_DIM(sfc_flow_items); i++)
818 		if (sfc_flow_items[i].type == type)
819 			return &sfc_flow_items[i];
820 
821 	return NULL;
822 }
823 
824 static int
825 sfc_flow_parse_pattern(const struct rte_flow_item pattern[],
826 		       struct rte_flow *flow,
827 		       struct rte_flow_error *error)
828 {
829 	int rc;
830 	unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER;
831 	const struct sfc_flow_item *item;
832 
833 	if (pattern == NULL) {
834 		rte_flow_error_set(error, EINVAL,
835 				   RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
836 				   "NULL pattern");
837 		return -rte_errno;
838 	}
839 
840 	for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
841 		item = sfc_flow_get_item(pattern->type);
842 		if (item == NULL) {
843 			rte_flow_error_set(error, ENOTSUP,
844 					   RTE_FLOW_ERROR_TYPE_ITEM, pattern,
845 					   "Unsupported pattern item");
846 			return -rte_errno;
847 		}
848 
849 		/*
850 		 * Omitting one or several protocol layers at the beginning
851 		 * of pattern is supported
852 		 */
853 		if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
854 		    prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
855 		    item->prev_layer != prev_layer) {
856 			rte_flow_error_set(error, ENOTSUP,
857 					   RTE_FLOW_ERROR_TYPE_ITEM, pattern,
858 					   "Unexpected sequence of pattern items");
859 			return -rte_errno;
860 		}
861 
862 		rc = item->parse(pattern, &flow->spec, error);
863 		if (rc != 0)
864 			return rc;
865 
866 		if (item->layer != SFC_FLOW_ITEM_ANY_LAYER)
867 			prev_layer = item->layer;
868 	}
869 
870 	return 0;
871 }
872 
873 static int
874 sfc_flow_parse_queue(struct sfc_adapter *sa,
875 		     const struct rte_flow_action_queue *queue,
876 		     struct rte_flow *flow)
877 {
878 	struct sfc_rxq *rxq;
879 
880 	if (queue->index >= sa->rxq_count)
881 		return -EINVAL;
882 
883 	rxq = sa->rxq_info[queue->index].rxq;
884 	flow->spec.efs_dmaq_id = (uint16_t)rxq->hw_index;
885 
886 	return 0;
887 }
888 
889 #if EFSYS_OPT_RX_SCALE
890 static int
891 sfc_flow_parse_rss(struct sfc_adapter *sa,
892 		   const struct rte_flow_action_rss *rss,
893 		   struct rte_flow *flow)
894 {
895 	unsigned int rxq_sw_index;
896 	struct sfc_rxq *rxq;
897 	unsigned int rxq_hw_index_min;
898 	unsigned int rxq_hw_index_max;
899 	const struct rte_eth_rss_conf *rss_conf = rss->rss_conf;
900 	uint64_t rss_hf;
901 	uint8_t *rss_key = NULL;
902 	struct sfc_flow_rss *sfc_rss_conf = &flow->rss_conf;
903 	unsigned int i;
904 
905 	if (rss->num == 0)
906 		return -EINVAL;
907 
908 	rxq_sw_index = sa->rxq_count - 1;
909 	rxq = sa->rxq_info[rxq_sw_index].rxq;
910 	rxq_hw_index_min = rxq->hw_index;
911 	rxq_hw_index_max = 0;
912 
913 	for (i = 0; i < rss->num; ++i) {
914 		rxq_sw_index = rss->queue[i];
915 
916 		if (rxq_sw_index >= sa->rxq_count)
917 			return -EINVAL;
918 
919 		rxq = sa->rxq_info[rxq_sw_index].rxq;
920 
921 		if (rxq->hw_index < rxq_hw_index_min)
922 			rxq_hw_index_min = rxq->hw_index;
923 
924 		if (rxq->hw_index > rxq_hw_index_max)
925 			rxq_hw_index_max = rxq->hw_index;
926 	}
927 
928 	rss_hf = (rss_conf != NULL) ? rss_conf->rss_hf : SFC_RSS_OFFLOADS;
929 	if ((rss_hf & ~SFC_RSS_OFFLOADS) != 0)
930 		return -EINVAL;
931 
932 	if (rss_conf != NULL) {
933 		if (rss_conf->rss_key_len != sizeof(sa->rss_key))
934 			return -EINVAL;
935 
936 		rss_key = rss_conf->rss_key;
937 	} else {
938 		rss_key = sa->rss_key;
939 	}
940 
941 	flow->rss = B_TRUE;
942 
943 	sfc_rss_conf->rxq_hw_index_min = rxq_hw_index_min;
944 	sfc_rss_conf->rxq_hw_index_max = rxq_hw_index_max;
945 	sfc_rss_conf->rss_hash_types = sfc_rte_to_efx_hash_type(rss_hf);
946 	rte_memcpy(sfc_rss_conf->rss_key, rss_key, sizeof(sa->rss_key));
947 
948 	for (i = 0; i < RTE_DIM(sfc_rss_conf->rss_tbl); ++i) {
949 		unsigned int rxq_sw_index = rss->queue[i % rss->num];
950 		struct sfc_rxq *rxq = sa->rxq_info[rxq_sw_index].rxq;
951 
952 		sfc_rss_conf->rss_tbl[i] = rxq->hw_index - rxq_hw_index_min;
953 	}
954 
955 	return 0;
956 }
957 #endif /* EFSYS_OPT_RX_SCALE */
958 
959 static int
960 sfc_flow_filter_insert(struct sfc_adapter *sa,
961 		       struct rte_flow *flow)
962 {
963 	efx_filter_spec_t *spec = &flow->spec;
964 
965 #if EFSYS_OPT_RX_SCALE
966 	struct sfc_flow_rss *rss = &flow->rss_conf;
967 	int rc = 0;
968 
969 	if (flow->rss) {
970 		unsigned int rss_spread = MIN(rss->rxq_hw_index_max -
971 					      rss->rxq_hw_index_min + 1,
972 					      EFX_MAXRSS);
973 
974 		rc = efx_rx_scale_context_alloc(sa->nic,
975 						EFX_RX_SCALE_EXCLUSIVE,
976 						rss_spread,
977 						&spec->efs_rss_context);
978 		if (rc != 0)
979 			goto fail_scale_context_alloc;
980 
981 		rc = efx_rx_scale_mode_set(sa->nic, spec->efs_rss_context,
982 					   EFX_RX_HASHALG_TOEPLITZ,
983 					   rss->rss_hash_types, B_TRUE);
984 		if (rc != 0)
985 			goto fail_scale_mode_set;
986 
987 		rc = efx_rx_scale_key_set(sa->nic, spec->efs_rss_context,
988 					  rss->rss_key,
989 					  sizeof(sa->rss_key));
990 		if (rc != 0)
991 			goto fail_scale_key_set;
992 
993 		spec->efs_dmaq_id = rss->rxq_hw_index_min;
994 		spec->efs_flags |= EFX_FILTER_FLAG_RX_RSS;
995 	}
996 
997 	rc = efx_filter_insert(sa->nic, spec);
998 	if (rc != 0)
999 		goto fail_filter_insert;
1000 
1001 	if (flow->rss) {
1002 		/*
1003 		 * Scale table is set after filter insertion because
1004 		 * the table entries are relative to the base RxQ ID
1005 		 * and the latter is submitted to the HW by means of
1006 		 * inserting a filter, so by the time of the request
1007 		 * the HW knows all the information needed to verify
1008 		 * the table entries, and the operation will succeed
1009 		 */
1010 		rc = efx_rx_scale_tbl_set(sa->nic, spec->efs_rss_context,
1011 					  rss->rss_tbl, RTE_DIM(rss->rss_tbl));
1012 		if (rc != 0)
1013 			goto fail_scale_tbl_set;
1014 	}
1015 
1016 	return 0;
1017 
1018 fail_scale_tbl_set:
1019 	efx_filter_remove(sa->nic, spec);
1020 
1021 fail_filter_insert:
1022 fail_scale_key_set:
1023 fail_scale_mode_set:
1024 	if (rss != NULL)
1025 		efx_rx_scale_context_free(sa->nic, spec->efs_rss_context);
1026 
1027 fail_scale_context_alloc:
1028 	return rc;
1029 #else /* !EFSYS_OPT_RX_SCALE */
1030 	return efx_filter_insert(sa->nic, spec);
1031 #endif /* EFSYS_OPT_RX_SCALE */
1032 }
1033 
1034 static int
1035 sfc_flow_filter_remove(struct sfc_adapter *sa,
1036 		       struct rte_flow *flow)
1037 {
1038 	efx_filter_spec_t *spec = &flow->spec;
1039 	int rc = 0;
1040 
1041 	rc = efx_filter_remove(sa->nic, spec);
1042 	if (rc != 0)
1043 		return rc;
1044 
1045 #if EFSYS_OPT_RX_SCALE
1046 	if (flow->rss)
1047 		rc = efx_rx_scale_context_free(sa->nic, spec->efs_rss_context);
1048 #endif /* EFSYS_OPT_RX_SCALE */
1049 
1050 	return rc;
1051 }
1052 
1053 static int
1054 sfc_flow_parse_actions(struct sfc_adapter *sa,
1055 		       const struct rte_flow_action actions[],
1056 		       struct rte_flow *flow,
1057 		       struct rte_flow_error *error)
1058 {
1059 	int rc;
1060 	boolean_t is_specified = B_FALSE;
1061 
1062 	if (actions == NULL) {
1063 		rte_flow_error_set(error, EINVAL,
1064 				   RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
1065 				   "NULL actions");
1066 		return -rte_errno;
1067 	}
1068 
1069 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1070 		switch (actions->type) {
1071 		case RTE_FLOW_ACTION_TYPE_VOID:
1072 			break;
1073 
1074 		case RTE_FLOW_ACTION_TYPE_QUEUE:
1075 			rc = sfc_flow_parse_queue(sa, actions->conf, flow);
1076 			if (rc != 0) {
1077 				rte_flow_error_set(error, EINVAL,
1078 					RTE_FLOW_ERROR_TYPE_ACTION, actions,
1079 					"Bad QUEUE action");
1080 				return -rte_errno;
1081 			}
1082 
1083 			is_specified = B_TRUE;
1084 			break;
1085 
1086 #if EFSYS_OPT_RX_SCALE
1087 		case RTE_FLOW_ACTION_TYPE_RSS:
1088 			rc = sfc_flow_parse_rss(sa, actions->conf, flow);
1089 			if (rc != 0) {
1090 				rte_flow_error_set(error, rc,
1091 					RTE_FLOW_ERROR_TYPE_ACTION, actions,
1092 					"Bad RSS action");
1093 				return -rte_errno;
1094 			}
1095 
1096 			is_specified = B_TRUE;
1097 			break;
1098 #endif /* EFSYS_OPT_RX_SCALE */
1099 
1100 		default:
1101 			rte_flow_error_set(error, ENOTSUP,
1102 					   RTE_FLOW_ERROR_TYPE_ACTION, actions,
1103 					   "Action is not supported");
1104 			return -rte_errno;
1105 		}
1106 	}
1107 
1108 	if (!is_specified) {
1109 		rte_flow_error_set(error, EINVAL,
1110 				   RTE_FLOW_ERROR_TYPE_ACTION_NUM, actions,
1111 				   "Action is unspecified");
1112 		return -rte_errno;
1113 	}
1114 
1115 	return 0;
1116 }
1117 
1118 static int
1119 sfc_flow_parse(struct rte_eth_dev *dev,
1120 	       const struct rte_flow_attr *attr,
1121 	       const struct rte_flow_item pattern[],
1122 	       const struct rte_flow_action actions[],
1123 	       struct rte_flow *flow,
1124 	       struct rte_flow_error *error)
1125 {
1126 	struct sfc_adapter *sa = dev->data->dev_private;
1127 	int rc;
1128 
1129 	memset(&flow->spec, 0, sizeof(flow->spec));
1130 
1131 	rc = sfc_flow_parse_attr(attr, flow, error);
1132 	if (rc != 0)
1133 		goto fail_bad_value;
1134 
1135 	rc = sfc_flow_parse_pattern(pattern, flow, error);
1136 	if (rc != 0)
1137 		goto fail_bad_value;
1138 
1139 	rc = sfc_flow_parse_actions(sa, actions, flow, error);
1140 	if (rc != 0)
1141 		goto fail_bad_value;
1142 
1143 	if (!sfc_filter_is_match_supported(sa, flow->spec.efs_match_flags)) {
1144 		rte_flow_error_set(error, ENOTSUP,
1145 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1146 				   "Flow rule pattern is not supported");
1147 		return -rte_errno;
1148 	}
1149 
1150 fail_bad_value:
1151 	return rc;
1152 }
1153 
1154 static int
1155 sfc_flow_validate(struct rte_eth_dev *dev,
1156 		  const struct rte_flow_attr *attr,
1157 		  const struct rte_flow_item pattern[],
1158 		  const struct rte_flow_action actions[],
1159 		  struct rte_flow_error *error)
1160 {
1161 	struct rte_flow flow;
1162 
1163 	return sfc_flow_parse(dev, attr, pattern, actions, &flow, error);
1164 }
1165 
1166 static struct rte_flow *
1167 sfc_flow_create(struct rte_eth_dev *dev,
1168 		const struct rte_flow_attr *attr,
1169 		const struct rte_flow_item pattern[],
1170 		const struct rte_flow_action actions[],
1171 		struct rte_flow_error *error)
1172 {
1173 	struct sfc_adapter *sa = dev->data->dev_private;
1174 	struct rte_flow *flow = NULL;
1175 	int rc;
1176 
1177 	flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0);
1178 	if (flow == NULL) {
1179 		rte_flow_error_set(error, ENOMEM,
1180 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1181 				   "Failed to allocate memory");
1182 		goto fail_no_mem;
1183 	}
1184 
1185 	rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error);
1186 	if (rc != 0)
1187 		goto fail_bad_value;
1188 
1189 	TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries);
1190 
1191 	sfc_adapter_lock(sa);
1192 
1193 	if (sa->state == SFC_ADAPTER_STARTED) {
1194 		rc = sfc_flow_filter_insert(sa, flow);
1195 		if (rc != 0) {
1196 			rte_flow_error_set(error, rc,
1197 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1198 				"Failed to insert filter");
1199 			goto fail_filter_insert;
1200 		}
1201 	}
1202 
1203 	sfc_adapter_unlock(sa);
1204 
1205 	return flow;
1206 
1207 fail_filter_insert:
1208 	TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1209 
1210 fail_bad_value:
1211 	rte_free(flow);
1212 	sfc_adapter_unlock(sa);
1213 
1214 fail_no_mem:
1215 	return NULL;
1216 }
1217 
1218 static int
1219 sfc_flow_remove(struct sfc_adapter *sa,
1220 		struct rte_flow *flow,
1221 		struct rte_flow_error *error)
1222 {
1223 	int rc = 0;
1224 
1225 	SFC_ASSERT(sfc_adapter_is_locked(sa));
1226 
1227 	if (sa->state == SFC_ADAPTER_STARTED) {
1228 		rc = sfc_flow_filter_remove(sa, flow);
1229 		if (rc != 0)
1230 			rte_flow_error_set(error, rc,
1231 				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1232 				"Failed to destroy flow rule");
1233 	}
1234 
1235 	TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1236 	rte_free(flow);
1237 
1238 	return rc;
1239 }
1240 
1241 static int
1242 sfc_flow_destroy(struct rte_eth_dev *dev,
1243 		 struct rte_flow *flow,
1244 		 struct rte_flow_error *error)
1245 {
1246 	struct sfc_adapter *sa = dev->data->dev_private;
1247 	struct rte_flow *flow_ptr;
1248 	int rc = EINVAL;
1249 
1250 	sfc_adapter_lock(sa);
1251 
1252 	TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) {
1253 		if (flow_ptr == flow)
1254 			rc = 0;
1255 	}
1256 	if (rc != 0) {
1257 		rte_flow_error_set(error, rc,
1258 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1259 				   "Failed to find flow rule to destroy");
1260 		goto fail_bad_value;
1261 	}
1262 
1263 	rc = sfc_flow_remove(sa, flow, error);
1264 
1265 fail_bad_value:
1266 	sfc_adapter_unlock(sa);
1267 
1268 	return -rc;
1269 }
1270 
1271 static int
1272 sfc_flow_flush(struct rte_eth_dev *dev,
1273 	       struct rte_flow_error *error)
1274 {
1275 	struct sfc_adapter *sa = dev->data->dev_private;
1276 	struct rte_flow *flow;
1277 	int rc = 0;
1278 	int ret = 0;
1279 
1280 	sfc_adapter_lock(sa);
1281 
1282 	while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1283 		rc = sfc_flow_remove(sa, flow, error);
1284 		if (rc != 0)
1285 			ret = rc;
1286 	}
1287 
1288 	sfc_adapter_unlock(sa);
1289 
1290 	return -ret;
1291 }
1292 
1293 static int
1294 sfc_flow_isolate(struct rte_eth_dev *dev, int enable,
1295 		 struct rte_flow_error *error)
1296 {
1297 	struct sfc_adapter *sa = dev->data->dev_private;
1298 	struct sfc_port *port = &sa->port;
1299 	int ret = 0;
1300 
1301 	sfc_adapter_lock(sa);
1302 	if (sa->state != SFC_ADAPTER_INITIALIZED) {
1303 		rte_flow_error_set(error, EBUSY,
1304 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1305 				   NULL, "please close the port first");
1306 		ret = -rte_errno;
1307 	} else {
1308 		port->isolated = (enable) ? B_TRUE : B_FALSE;
1309 	}
1310 	sfc_adapter_unlock(sa);
1311 
1312 	return ret;
1313 }
1314 
1315 const struct rte_flow_ops sfc_flow_ops = {
1316 	.validate = sfc_flow_validate,
1317 	.create = sfc_flow_create,
1318 	.destroy = sfc_flow_destroy,
1319 	.flush = sfc_flow_flush,
1320 	.query = NULL,
1321 	.isolate = sfc_flow_isolate,
1322 };
1323 
1324 void
1325 sfc_flow_init(struct sfc_adapter *sa)
1326 {
1327 	SFC_ASSERT(sfc_adapter_is_locked(sa));
1328 
1329 	TAILQ_INIT(&sa->filter.flow_list);
1330 }
1331 
1332 void
1333 sfc_flow_fini(struct sfc_adapter *sa)
1334 {
1335 	struct rte_flow *flow;
1336 
1337 	SFC_ASSERT(sfc_adapter_is_locked(sa));
1338 
1339 	while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1340 		TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1341 		rte_free(flow);
1342 	}
1343 }
1344 
1345 void
1346 sfc_flow_stop(struct sfc_adapter *sa)
1347 {
1348 	struct rte_flow *flow;
1349 
1350 	SFC_ASSERT(sfc_adapter_is_locked(sa));
1351 
1352 	TAILQ_FOREACH(flow, &sa->filter.flow_list, entries)
1353 		sfc_flow_filter_remove(sa, flow);
1354 }
1355 
1356 int
1357 sfc_flow_start(struct sfc_adapter *sa)
1358 {
1359 	struct rte_flow *flow;
1360 	int rc = 0;
1361 
1362 	sfc_log_init(sa, "entry");
1363 
1364 	SFC_ASSERT(sfc_adapter_is_locked(sa));
1365 
1366 	TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) {
1367 		rc = sfc_flow_filter_insert(sa, flow);
1368 		if (rc != 0)
1369 			goto fail_bad_flow;
1370 	}
1371 
1372 	sfc_log_init(sa, "done");
1373 
1374 fail_bad_flow:
1375 	return rc;
1376 }
1377