xref: /dpdk/drivers/net/sfc/sfc_rx.c (revision a3a2e2c8f7de433e10b1548df65b20bf10086d9c)
1 /*-
2  *   BSD LICENSE
3  *
4  * Copyright (c) 2016-2017 Solarflare Communications Inc.
5  * All rights reserved.
6  *
7  * This software was jointly developed between OKTET Labs (under contract
8  * for Solarflare) and Solarflare Communications, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * 1. Redistributions of source code must retain the above copyright notice,
14  *    this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright notice,
16  *    this list of conditions and the following disclaimer in the documentation
17  *    and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 #include <rte_mempool.h>
33 
34 #include "efx.h"
35 
36 #include "sfc.h"
37 #include "sfc_debug.h"
38 #include "sfc_log.h"
39 #include "sfc_ev.h"
40 #include "sfc_rx.h"
41 #include "sfc_kvargs.h"
42 #include "sfc_tweak.h"
43 
44 /*
45  * Maximum number of Rx queue flush attempt in the case of failure or
46  * flush timeout
47  */
48 #define SFC_RX_QFLUSH_ATTEMPTS		(3)
49 
50 /*
51  * Time to wait between event queue polling attempts when waiting for Rx
52  * queue flush done or failed events.
53  */
54 #define SFC_RX_QFLUSH_POLL_WAIT_MS	(1)
55 
56 /*
57  * Maximum number of event queue polling attempts when waiting for Rx queue
58  * flush done or failed events. It defines Rx queue flush attempt timeout
59  * together with SFC_RX_QFLUSH_POLL_WAIT_MS.
60  */
61 #define SFC_RX_QFLUSH_POLL_ATTEMPTS	(2000)
62 
63 void
64 sfc_rx_qflush_done(struct sfc_rxq *rxq)
65 {
66 	rxq->state |= SFC_RXQ_FLUSHED;
67 	rxq->state &= ~SFC_RXQ_FLUSHING;
68 }
69 
70 void
71 sfc_rx_qflush_failed(struct sfc_rxq *rxq)
72 {
73 	rxq->state |= SFC_RXQ_FLUSH_FAILED;
74 	rxq->state &= ~SFC_RXQ_FLUSHING;
75 }
76 
77 static void
78 sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq)
79 {
80 	unsigned int free_space;
81 	unsigned int bulks;
82 	void *objs[SFC_RX_REFILL_BULK];
83 	efsys_dma_addr_t addr[RTE_DIM(objs)];
84 	unsigned int added = rxq->added;
85 	unsigned int id;
86 	unsigned int i;
87 	struct sfc_efx_rx_sw_desc *rxd;
88 	struct rte_mbuf *m;
89 	uint16_t port_id = rxq->dp.dpq.port_id;
90 
91 	free_space = EFX_RXQ_LIMIT(rxq->ptr_mask + 1) -
92 		(added - rxq->completed);
93 
94 	if (free_space < rxq->refill_threshold)
95 		return;
96 
97 	bulks = free_space / RTE_DIM(objs);
98 	/* refill_threshold guarantees that bulks is positive */
99 	SFC_ASSERT(bulks > 0);
100 
101 	id = added & rxq->ptr_mask;
102 	do {
103 		if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs,
104 						  RTE_DIM(objs)) < 0)) {
105 			/*
106 			 * It is hardly a safe way to increment counter
107 			 * from different contexts, but all PMDs do it.
108 			 */
109 			rxq->evq->sa->eth_dev->data->rx_mbuf_alloc_failed +=
110 				RTE_DIM(objs);
111 			/* Return if we have posted nothing yet */
112 			if (added == rxq->added)
113 				return;
114 			/* Push posted */
115 			break;
116 		}
117 
118 		for (i = 0; i < RTE_DIM(objs);
119 		     ++i, id = (id + 1) & rxq->ptr_mask) {
120 			m = objs[i];
121 
122 			rxd = &rxq->sw_desc[id];
123 			rxd->mbuf = m;
124 
125 			SFC_ASSERT(rte_mbuf_refcnt_read(m) == 1);
126 			m->data_off = RTE_PKTMBUF_HEADROOM;
127 			SFC_ASSERT(m->next == NULL);
128 			SFC_ASSERT(m->nb_segs == 1);
129 			m->port = port_id;
130 
131 			addr[i] = rte_pktmbuf_mtophys(m);
132 		}
133 
134 		efx_rx_qpost(rxq->common, addr, rxq->buf_size,
135 			     RTE_DIM(objs), rxq->completed, added);
136 		added += RTE_DIM(objs);
137 	} while (--bulks > 0);
138 
139 	SFC_ASSERT(added != rxq->added);
140 	rxq->added = added;
141 	efx_rx_qpush(rxq->common, added, &rxq->pushed);
142 }
143 
144 static uint64_t
145 sfc_efx_rx_desc_flags_to_offload_flags(const unsigned int desc_flags)
146 {
147 	uint64_t mbuf_flags = 0;
148 
149 	switch (desc_flags & (EFX_PKT_IPV4 | EFX_CKSUM_IPV4)) {
150 	case (EFX_PKT_IPV4 | EFX_CKSUM_IPV4):
151 		mbuf_flags |= PKT_RX_IP_CKSUM_GOOD;
152 		break;
153 	case EFX_PKT_IPV4:
154 		mbuf_flags |= PKT_RX_IP_CKSUM_BAD;
155 		break;
156 	default:
157 		RTE_BUILD_BUG_ON(PKT_RX_IP_CKSUM_UNKNOWN != 0);
158 		SFC_ASSERT((mbuf_flags & PKT_RX_IP_CKSUM_MASK) ==
159 			   PKT_RX_IP_CKSUM_UNKNOWN);
160 		break;
161 	}
162 
163 	switch ((desc_flags &
164 		 (EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP))) {
165 	case (EFX_PKT_TCP | EFX_CKSUM_TCPUDP):
166 	case (EFX_PKT_UDP | EFX_CKSUM_TCPUDP):
167 		mbuf_flags |= PKT_RX_L4_CKSUM_GOOD;
168 		break;
169 	case EFX_PKT_TCP:
170 	case EFX_PKT_UDP:
171 		mbuf_flags |= PKT_RX_L4_CKSUM_BAD;
172 		break;
173 	default:
174 		RTE_BUILD_BUG_ON(PKT_RX_L4_CKSUM_UNKNOWN != 0);
175 		SFC_ASSERT((mbuf_flags & PKT_RX_L4_CKSUM_MASK) ==
176 			   PKT_RX_L4_CKSUM_UNKNOWN);
177 		break;
178 	}
179 
180 	return mbuf_flags;
181 }
182 
183 static uint32_t
184 sfc_efx_rx_desc_flags_to_packet_type(const unsigned int desc_flags)
185 {
186 	return RTE_PTYPE_L2_ETHER |
187 		((desc_flags & EFX_PKT_IPV4) ?
188 			RTE_PTYPE_L3_IPV4_EXT_UNKNOWN : 0) |
189 		((desc_flags & EFX_PKT_IPV6) ?
190 			RTE_PTYPE_L3_IPV6_EXT_UNKNOWN : 0) |
191 		((desc_flags & EFX_PKT_TCP) ? RTE_PTYPE_L4_TCP : 0) |
192 		((desc_flags & EFX_PKT_UDP) ? RTE_PTYPE_L4_UDP : 0);
193 }
194 
195 static const uint32_t *
196 sfc_efx_supported_ptypes_get(void)
197 {
198 	static const uint32_t ptypes[] = {
199 		RTE_PTYPE_L2_ETHER,
200 		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
201 		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
202 		RTE_PTYPE_L4_TCP,
203 		RTE_PTYPE_L4_UDP,
204 		RTE_PTYPE_UNKNOWN
205 	};
206 
207 	return ptypes;
208 }
209 
210 static void
211 sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags,
212 			struct rte_mbuf *m)
213 {
214 #if EFSYS_OPT_RX_SCALE
215 	uint8_t *mbuf_data;
216 
217 
218 	if ((rxq->flags & SFC_EFX_RXQ_FLAG_RSS_HASH) == 0)
219 		return;
220 
221 	mbuf_data = rte_pktmbuf_mtod(m, uint8_t *);
222 
223 	if (flags & (EFX_PKT_IPV4 | EFX_PKT_IPV6)) {
224 		m->hash.rss = efx_pseudo_hdr_hash_get(rxq->common,
225 						      EFX_RX_HASHALG_TOEPLITZ,
226 						      mbuf_data);
227 
228 		m->ol_flags |= PKT_RX_RSS_HASH;
229 	}
230 #endif
231 }
232 
233 static uint16_t
234 sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
235 {
236 	struct sfc_dp_rxq *dp_rxq = rx_queue;
237 	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
238 	unsigned int completed;
239 	unsigned int prefix_size = rxq->prefix_size;
240 	unsigned int done_pkts = 0;
241 	boolean_t discard_next = B_FALSE;
242 	struct rte_mbuf *scatter_pkt = NULL;
243 
244 	if (unlikely((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0))
245 		return 0;
246 
247 	sfc_ev_qpoll(rxq->evq);
248 
249 	completed = rxq->completed;
250 	while (completed != rxq->pending && done_pkts < nb_pkts) {
251 		unsigned int id;
252 		struct sfc_efx_rx_sw_desc *rxd;
253 		struct rte_mbuf *m;
254 		unsigned int seg_len;
255 		unsigned int desc_flags;
256 
257 		id = completed++ & rxq->ptr_mask;
258 		rxd = &rxq->sw_desc[id];
259 		m = rxd->mbuf;
260 		desc_flags = rxd->flags;
261 
262 		if (discard_next)
263 			goto discard;
264 
265 		if (desc_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
266 			goto discard;
267 
268 		if (desc_flags & EFX_PKT_PREFIX_LEN) {
269 			uint16_t tmp_size;
270 			int rc __rte_unused;
271 
272 			rc = efx_pseudo_hdr_pkt_length_get(rxq->common,
273 				rte_pktmbuf_mtod(m, uint8_t *), &tmp_size);
274 			SFC_ASSERT(rc == 0);
275 			seg_len = tmp_size;
276 		} else {
277 			seg_len = rxd->size - prefix_size;
278 		}
279 
280 		rte_pktmbuf_data_len(m) = seg_len;
281 		rte_pktmbuf_pkt_len(m) = seg_len;
282 
283 		if (scatter_pkt != NULL) {
284 			if (rte_pktmbuf_chain(scatter_pkt, m) != 0) {
285 				rte_pktmbuf_free(scatter_pkt);
286 				goto discard;
287 			}
288 			/* The packet to deliver */
289 			m = scatter_pkt;
290 		}
291 
292 		if (desc_flags & EFX_PKT_CONT) {
293 			/* The packet is scattered, more fragments to come */
294 			scatter_pkt = m;
295 			/* Further fragments have no prefix */
296 			prefix_size = 0;
297 			continue;
298 		}
299 
300 		/* Scattered packet is done */
301 		scatter_pkt = NULL;
302 		/* The first fragment of the packet has prefix */
303 		prefix_size = rxq->prefix_size;
304 
305 		m->ol_flags =
306 			sfc_efx_rx_desc_flags_to_offload_flags(desc_flags);
307 		m->packet_type =
308 			sfc_efx_rx_desc_flags_to_packet_type(desc_flags);
309 
310 		/*
311 		 * Extract RSS hash from the packet prefix and
312 		 * set the corresponding field (if needed and possible)
313 		 */
314 		sfc_efx_rx_set_rss_hash(rxq, desc_flags, m);
315 
316 		m->data_off += prefix_size;
317 
318 		*rx_pkts++ = m;
319 		done_pkts++;
320 		continue;
321 
322 discard:
323 		discard_next = ((desc_flags & EFX_PKT_CONT) != 0);
324 		rte_mempool_put(rxq->refill_mb_pool, m);
325 		rxd->mbuf = NULL;
326 	}
327 
328 	/* pending is only moved when entire packet is received */
329 	SFC_ASSERT(scatter_pkt == NULL);
330 
331 	rxq->completed = completed;
332 
333 	sfc_efx_rx_qrefill(rxq);
334 
335 	return done_pkts;
336 }
337 
338 static sfc_dp_rx_qdesc_npending_t sfc_efx_rx_qdesc_npending;
339 static unsigned int
340 sfc_efx_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
341 {
342 	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
343 
344 	if ((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0)
345 		return 0;
346 
347 	sfc_ev_qpoll(rxq->evq);
348 
349 	return rxq->pending - rxq->completed;
350 }
351 
352 struct sfc_rxq *
353 sfc_rxq_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq)
354 {
355 	const struct sfc_dp_queue *dpq = &dp_rxq->dpq;
356 	struct rte_eth_dev *eth_dev;
357 	struct sfc_adapter *sa;
358 	struct sfc_rxq *rxq;
359 
360 	SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id));
361 	eth_dev = &rte_eth_devices[dpq->port_id];
362 
363 	sa = eth_dev->data->dev_private;
364 
365 	SFC_ASSERT(dpq->queue_id < sa->rxq_count);
366 	rxq = sa->rxq_info[dpq->queue_id].rxq;
367 
368 	SFC_ASSERT(rxq != NULL);
369 	return rxq;
370 }
371 
372 static sfc_dp_rx_qcreate_t sfc_efx_rx_qcreate;
373 static int
374 sfc_efx_rx_qcreate(uint16_t port_id, uint16_t queue_id,
375 		   const struct rte_pci_addr *pci_addr, int socket_id,
376 		   const struct sfc_dp_rx_qcreate_info *info,
377 		   struct sfc_dp_rxq **dp_rxqp)
378 {
379 	struct sfc_efx_rxq *rxq;
380 	int rc;
381 
382 	rc = ENOMEM;
383 	rxq = rte_zmalloc_socket("sfc-efx-rxq", sizeof(*rxq),
384 				 RTE_CACHE_LINE_SIZE, socket_id);
385 	if (rxq == NULL)
386 		goto fail_rxq_alloc;
387 
388 	sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr);
389 
390 	rc = ENOMEM;
391 	rxq->sw_desc = rte_calloc_socket("sfc-efx-rxq-sw_desc",
392 					 info->rxq_entries,
393 					 sizeof(*rxq->sw_desc),
394 					 RTE_CACHE_LINE_SIZE, socket_id);
395 	if (rxq->sw_desc == NULL)
396 		goto fail_desc_alloc;
397 
398 	/* efx datapath is bound to efx control path */
399 	rxq->evq = sfc_rxq_by_dp_rxq(&rxq->dp)->evq;
400 	if (info->flags & SFC_RXQ_FLAG_RSS_HASH)
401 		rxq->flags |= SFC_EFX_RXQ_FLAG_RSS_HASH;
402 	rxq->ptr_mask = info->rxq_entries - 1;
403 	rxq->batch_max = info->batch_max;
404 	rxq->prefix_size = info->prefix_size;
405 	rxq->refill_threshold = info->refill_threshold;
406 	rxq->buf_size = info->buf_size;
407 	rxq->refill_mb_pool = info->refill_mb_pool;
408 
409 	*dp_rxqp = &rxq->dp;
410 	return 0;
411 
412 fail_desc_alloc:
413 	rte_free(rxq);
414 
415 fail_rxq_alloc:
416 	return rc;
417 }
418 
419 static sfc_dp_rx_qdestroy_t sfc_efx_rx_qdestroy;
420 static void
421 sfc_efx_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
422 {
423 	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
424 
425 	rte_free(rxq->sw_desc);
426 	rte_free(rxq);
427 }
428 
429 static sfc_dp_rx_qstart_t sfc_efx_rx_qstart;
430 static int
431 sfc_efx_rx_qstart(struct sfc_dp_rxq *dp_rxq,
432 		  __rte_unused unsigned int evq_read_ptr)
433 {
434 	/* libefx-based datapath is specific to libefx-based PMD */
435 	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
436 	struct sfc_rxq *crxq = sfc_rxq_by_dp_rxq(dp_rxq);
437 
438 	rxq->common = crxq->common;
439 
440 	rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0;
441 
442 	sfc_efx_rx_qrefill(rxq);
443 
444 	rxq->flags |= (SFC_EFX_RXQ_FLAG_STARTED | SFC_EFX_RXQ_FLAG_RUNNING);
445 
446 	return 0;
447 }
448 
449 static sfc_dp_rx_qstop_t sfc_efx_rx_qstop;
450 static void
451 sfc_efx_rx_qstop(struct sfc_dp_rxq *dp_rxq,
452 		 __rte_unused unsigned int *evq_read_ptr)
453 {
454 	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
455 
456 	rxq->flags &= ~SFC_EFX_RXQ_FLAG_RUNNING;
457 
458 	/* libefx-based datapath is bound to libefx-based PMD and uses
459 	 * event queue structure directly. So, there is no necessity to
460 	 * return EvQ read pointer.
461 	 */
462 }
463 
464 static sfc_dp_rx_qpurge_t sfc_efx_rx_qpurge;
465 static void
466 sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
467 {
468 	struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
469 	unsigned int i;
470 	struct sfc_efx_rx_sw_desc *rxd;
471 
472 	for (i = rxq->completed; i != rxq->added; ++i) {
473 		rxd = &rxq->sw_desc[i & rxq->ptr_mask];
474 		rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
475 		rxd->mbuf = NULL;
476 		/* Packed stream relies on 0 in inactive SW desc.
477 		 * Rx queue stop is not performance critical, so
478 		 * there is no harm to do it always.
479 		 */
480 		rxd->flags = 0;
481 		rxd->size = 0;
482 	}
483 
484 	rxq->flags &= ~SFC_EFX_RXQ_FLAG_STARTED;
485 }
486 
487 struct sfc_dp_rx sfc_efx_rx = {
488 	.dp = {
489 		.name		= SFC_KVARG_DATAPATH_EFX,
490 		.type		= SFC_DP_RX,
491 		.hw_fw_caps	= 0,
492 	},
493 	.features		= SFC_DP_RX_FEAT_SCATTER,
494 	.qcreate		= sfc_efx_rx_qcreate,
495 	.qdestroy		= sfc_efx_rx_qdestroy,
496 	.qstart			= sfc_efx_rx_qstart,
497 	.qstop			= sfc_efx_rx_qstop,
498 	.qpurge			= sfc_efx_rx_qpurge,
499 	.supported_ptypes_get	= sfc_efx_supported_ptypes_get,
500 	.qdesc_npending		= sfc_efx_rx_qdesc_npending,
501 	.pkt_burst		= sfc_efx_recv_pkts,
502 };
503 
504 unsigned int
505 sfc_rx_qdesc_npending(struct sfc_adapter *sa, unsigned int sw_index)
506 {
507 	struct sfc_rxq *rxq;
508 
509 	SFC_ASSERT(sw_index < sa->rxq_count);
510 	rxq = sa->rxq_info[sw_index].rxq;
511 
512 	if (rxq == NULL || (rxq->state & SFC_RXQ_STARTED) == 0)
513 		return 0;
514 
515 	return sa->dp_rx->qdesc_npending(rxq->dp);
516 }
517 
518 int
519 sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset)
520 {
521 	struct sfc_rxq *rxq = sfc_rxq_by_dp_rxq(dp_rxq);
522 
523 	return offset < rxq->evq->sa->dp_rx->qdesc_npending(dp_rxq);
524 }
525 
526 static void
527 sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index)
528 {
529 	struct sfc_rxq *rxq;
530 	unsigned int retry_count;
531 	unsigned int wait_count;
532 	int rc;
533 
534 	rxq = sa->rxq_info[sw_index].rxq;
535 	SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
536 
537 	/*
538 	 * Retry Rx queue flushing in the case of flush failed or
539 	 * timeout. In the worst case it can delay for 6 seconds.
540 	 */
541 	for (retry_count = 0;
542 	     ((rxq->state & SFC_RXQ_FLUSHED) == 0) &&
543 	     (retry_count < SFC_RX_QFLUSH_ATTEMPTS);
544 	     ++retry_count) {
545 		rc = efx_rx_qflush(rxq->common);
546 		if (rc != 0) {
547 			rxq->state |= (rc == EALREADY) ?
548 				SFC_RXQ_FLUSHED : SFC_RXQ_FLUSH_FAILED;
549 			break;
550 		}
551 		rxq->state &= ~SFC_RXQ_FLUSH_FAILED;
552 		rxq->state |= SFC_RXQ_FLUSHING;
553 
554 		/*
555 		 * Wait for Rx queue flush done or failed event at least
556 		 * SFC_RX_QFLUSH_POLL_WAIT_MS milliseconds and not more
557 		 * than 2 seconds (SFC_RX_QFLUSH_POLL_WAIT_MS multiplied
558 		 * by SFC_RX_QFLUSH_POLL_ATTEMPTS).
559 		 */
560 		wait_count = 0;
561 		do {
562 			rte_delay_ms(SFC_RX_QFLUSH_POLL_WAIT_MS);
563 			sfc_ev_qpoll(rxq->evq);
564 		} while ((rxq->state & SFC_RXQ_FLUSHING) &&
565 			 (wait_count++ < SFC_RX_QFLUSH_POLL_ATTEMPTS));
566 
567 		if (rxq->state & SFC_RXQ_FLUSHING)
568 			sfc_err(sa, "RxQ %u flush timed out", sw_index);
569 
570 		if (rxq->state & SFC_RXQ_FLUSH_FAILED)
571 			sfc_err(sa, "RxQ %u flush failed", sw_index);
572 
573 		if (rxq->state & SFC_RXQ_FLUSHED)
574 			sfc_info(sa, "RxQ %u flushed", sw_index);
575 	}
576 
577 	sa->dp_rx->qpurge(rxq->dp);
578 }
579 
580 static int
581 sfc_rx_default_rxq_set_filter(struct sfc_adapter *sa, struct sfc_rxq *rxq)
582 {
583 	boolean_t rss = (sa->rss_channels > 0) ? B_TRUE : B_FALSE;
584 	struct sfc_port *port = &sa->port;
585 	int rc;
586 
587 	/*
588 	 * If promiscuous or all-multicast mode has been requested, setting
589 	 * filter for the default Rx queue might fail, in particular, while
590 	 * running over PCI function which is not a member of corresponding
591 	 * privilege groups; if this occurs, few iterations will be made to
592 	 * repeat this step without promiscuous and all-multicast flags set
593 	 */
594 retry:
595 	rc = efx_mac_filter_default_rxq_set(sa->nic, rxq->common, rss);
596 	if (rc == 0)
597 		return 0;
598 	else if (rc != EOPNOTSUPP)
599 		return rc;
600 
601 	if (port->promisc) {
602 		sfc_warn(sa, "promiscuous mode has been requested, "
603 			     "but the HW rejects it");
604 		sfc_warn(sa, "promiscuous mode will be disabled");
605 
606 		port->promisc = B_FALSE;
607 		rc = sfc_set_rx_mode(sa);
608 		if (rc != 0)
609 			return rc;
610 
611 		goto retry;
612 	}
613 
614 	if (port->allmulti) {
615 		sfc_warn(sa, "all-multicast mode has been requested, "
616 			     "but the HW rejects it");
617 		sfc_warn(sa, "all-multicast mode will be disabled");
618 
619 		port->allmulti = B_FALSE;
620 		rc = sfc_set_rx_mode(sa);
621 		if (rc != 0)
622 			return rc;
623 
624 		goto retry;
625 	}
626 
627 	return rc;
628 }
629 
630 int
631 sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index)
632 {
633 	struct sfc_rxq_info *rxq_info;
634 	struct sfc_rxq *rxq;
635 	struct sfc_evq *evq;
636 	int rc;
637 
638 	sfc_log_init(sa, "sw_index=%u", sw_index);
639 
640 	SFC_ASSERT(sw_index < sa->rxq_count);
641 
642 	rxq_info = &sa->rxq_info[sw_index];
643 	rxq = rxq_info->rxq;
644 	SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED);
645 
646 	evq = rxq->evq;
647 
648 	rc = sfc_ev_qstart(evq, sfc_evq_index_by_rxq_sw_index(sa, sw_index));
649 	if (rc != 0)
650 		goto fail_ev_qstart;
651 
652 	rc = efx_rx_qcreate(sa->nic, rxq->hw_index, 0, rxq_info->type,
653 			    &rxq->mem, rxq_info->entries,
654 			    0 /* not used on EF10 */, evq->common,
655 			    &rxq->common);
656 	if (rc != 0)
657 		goto fail_rx_qcreate;
658 
659 	efx_rx_qenable(rxq->common);
660 
661 	rc = sa->dp_rx->qstart(rxq->dp, evq->read_ptr);
662 	if (rc != 0)
663 		goto fail_dp_qstart;
664 
665 	rxq->state |= SFC_RXQ_STARTED;
666 
667 	if (sw_index == 0) {
668 		rc = sfc_rx_default_rxq_set_filter(sa, rxq);
669 		if (rc != 0)
670 			goto fail_mac_filter_default_rxq_set;
671 	}
672 
673 	/* It seems to be used by DPDK for debug purposes only ('rte_ether') */
674 	sa->eth_dev->data->rx_queue_state[sw_index] =
675 		RTE_ETH_QUEUE_STATE_STARTED;
676 
677 	return 0;
678 
679 fail_mac_filter_default_rxq_set:
680 	sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr);
681 
682 fail_dp_qstart:
683 	sfc_rx_qflush(sa, sw_index);
684 
685 fail_rx_qcreate:
686 	sfc_ev_qstop(evq);
687 
688 fail_ev_qstart:
689 	return rc;
690 }
691 
692 void
693 sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index)
694 {
695 	struct sfc_rxq_info *rxq_info;
696 	struct sfc_rxq *rxq;
697 
698 	sfc_log_init(sa, "sw_index=%u", sw_index);
699 
700 	SFC_ASSERT(sw_index < sa->rxq_count);
701 
702 	rxq_info = &sa->rxq_info[sw_index];
703 	rxq = rxq_info->rxq;
704 
705 	if (rxq->state == SFC_RXQ_INITIALIZED)
706 		return;
707 	SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
708 
709 	/* It seems to be used by DPDK for debug purposes only ('rte_ether') */
710 	sa->eth_dev->data->rx_queue_state[sw_index] =
711 		RTE_ETH_QUEUE_STATE_STOPPED;
712 
713 	sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr);
714 
715 	if (sw_index == 0)
716 		efx_mac_filter_default_rxq_clear(sa->nic);
717 
718 	sfc_rx_qflush(sa, sw_index);
719 
720 	rxq->state = SFC_RXQ_INITIALIZED;
721 
722 	efx_rx_qdestroy(rxq->common);
723 
724 	sfc_ev_qstop(rxq->evq);
725 }
726 
727 static int
728 sfc_rx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_rx_desc,
729 		   const struct rte_eth_rxconf *rx_conf)
730 {
731 	const uint16_t rx_free_thresh_max = EFX_RXQ_LIMIT(nb_rx_desc);
732 	int rc = 0;
733 
734 	if (rx_conf->rx_thresh.pthresh != 0 ||
735 	    rx_conf->rx_thresh.hthresh != 0 ||
736 	    rx_conf->rx_thresh.wthresh != 0) {
737 		sfc_err(sa,
738 			"RxQ prefetch/host/writeback thresholds are not supported");
739 		rc = EINVAL;
740 	}
741 
742 	if (rx_conf->rx_free_thresh > rx_free_thresh_max) {
743 		sfc_err(sa,
744 			"RxQ free threshold too large: %u vs maximum %u",
745 			rx_conf->rx_free_thresh, rx_free_thresh_max);
746 		rc = EINVAL;
747 	}
748 
749 	if (rx_conf->rx_drop_en == 0) {
750 		sfc_err(sa, "RxQ drop disable is not supported");
751 		rc = EINVAL;
752 	}
753 
754 	return rc;
755 }
756 
757 static unsigned int
758 sfc_rx_mbuf_data_alignment(struct rte_mempool *mb_pool)
759 {
760 	uint32_t data_off;
761 	uint32_t order;
762 
763 	/* The mbuf object itself is always cache line aligned */
764 	order = rte_bsf32(RTE_CACHE_LINE_SIZE);
765 
766 	/* Data offset from mbuf object start */
767 	data_off = sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mb_pool) +
768 		RTE_PKTMBUF_HEADROOM;
769 
770 	order = MIN(order, rte_bsf32(data_off));
771 
772 	return 1u << (order - 1);
773 }
774 
775 static uint16_t
776 sfc_rx_mb_pool_buf_size(struct sfc_adapter *sa, struct rte_mempool *mb_pool)
777 {
778 	const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
779 	const uint32_t nic_align_start = MAX(1, encp->enc_rx_buf_align_start);
780 	const uint32_t nic_align_end = MAX(1, encp->enc_rx_buf_align_end);
781 	uint16_t buf_size;
782 	unsigned int buf_aligned;
783 	unsigned int start_alignment;
784 	unsigned int end_padding_alignment;
785 
786 	/* Below it is assumed that both alignments are power of 2 */
787 	SFC_ASSERT(rte_is_power_of_2(nic_align_start));
788 	SFC_ASSERT(rte_is_power_of_2(nic_align_end));
789 
790 	/*
791 	 * mbuf is always cache line aligned, double-check
792 	 * that it meets rx buffer start alignment requirements.
793 	 */
794 
795 	/* Start from mbuf pool data room size */
796 	buf_size = rte_pktmbuf_data_room_size(mb_pool);
797 
798 	/* Remove headroom */
799 	if (buf_size <= RTE_PKTMBUF_HEADROOM) {
800 		sfc_err(sa,
801 			"RxQ mbuf pool %s object data room size %u is smaller than headroom %u",
802 			mb_pool->name, buf_size, RTE_PKTMBUF_HEADROOM);
803 		return 0;
804 	}
805 	buf_size -= RTE_PKTMBUF_HEADROOM;
806 
807 	/* Calculate guaranteed data start alignment */
808 	buf_aligned = sfc_rx_mbuf_data_alignment(mb_pool);
809 
810 	/* Reserve space for start alignment */
811 	if (buf_aligned < nic_align_start) {
812 		start_alignment = nic_align_start - buf_aligned;
813 		if (buf_size <= start_alignment) {
814 			sfc_err(sa,
815 				"RxQ mbuf pool %s object data room size %u is insufficient for headroom %u and buffer start alignment %u required by NIC",
816 				mb_pool->name,
817 				rte_pktmbuf_data_room_size(mb_pool),
818 				RTE_PKTMBUF_HEADROOM, start_alignment);
819 			return 0;
820 		}
821 		buf_aligned = nic_align_start;
822 		buf_size -= start_alignment;
823 	} else {
824 		start_alignment = 0;
825 	}
826 
827 	/* Make sure that end padding does not write beyond the buffer */
828 	if (buf_aligned < nic_align_end) {
829 		/*
830 		 * Estimate space which can be lost. If guarnteed buffer
831 		 * size is odd, lost space is (nic_align_end - 1). More
832 		 * accurate formula is below.
833 		 */
834 		end_padding_alignment = nic_align_end -
835 			MIN(buf_aligned, 1u << (rte_bsf32(buf_size) - 1));
836 		if (buf_size <= end_padding_alignment) {
837 			sfc_err(sa,
838 				"RxQ mbuf pool %s object data room size %u is insufficient for headroom %u, buffer start alignment %u and end padding alignment %u required by NIC",
839 				mb_pool->name,
840 				rte_pktmbuf_data_room_size(mb_pool),
841 				RTE_PKTMBUF_HEADROOM, start_alignment,
842 				end_padding_alignment);
843 			return 0;
844 		}
845 		buf_size -= end_padding_alignment;
846 	} else {
847 		/*
848 		 * Start is aligned the same or better than end,
849 		 * just align length.
850 		 */
851 		buf_size = P2ALIGN(buf_size, nic_align_end);
852 	}
853 
854 	return buf_size;
855 }
856 
857 int
858 sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
859 	     uint16_t nb_rx_desc, unsigned int socket_id,
860 	     const struct rte_eth_rxconf *rx_conf,
861 	     struct rte_mempool *mb_pool)
862 {
863 	const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
864 	int rc;
865 	uint16_t buf_size;
866 	struct sfc_rxq_info *rxq_info;
867 	struct sfc_evq *evq;
868 	struct sfc_rxq *rxq;
869 	struct sfc_dp_rx_qcreate_info info;
870 
871 	rc = sfc_rx_qcheck_conf(sa, nb_rx_desc, rx_conf);
872 	if (rc != 0)
873 		goto fail_bad_conf;
874 
875 	buf_size = sfc_rx_mb_pool_buf_size(sa, mb_pool);
876 	if (buf_size == 0) {
877 		sfc_err(sa, "RxQ %u mbuf pool object size is too small",
878 			sw_index);
879 		rc = EINVAL;
880 		goto fail_bad_conf;
881 	}
882 
883 	if ((buf_size < sa->port.pdu + encp->enc_rx_prefix_size) &&
884 	    !sa->eth_dev->data->dev_conf.rxmode.enable_scatter) {
885 		sfc_err(sa, "Rx scatter is disabled and RxQ %u mbuf pool "
886 			"object size is too small", sw_index);
887 		sfc_err(sa, "RxQ %u calculated Rx buffer size is %u vs "
888 			"PDU size %u plus Rx prefix %u bytes",
889 			sw_index, buf_size, (unsigned int)sa->port.pdu,
890 			encp->enc_rx_prefix_size);
891 		rc = EINVAL;
892 		goto fail_bad_conf;
893 	}
894 
895 	SFC_ASSERT(sw_index < sa->rxq_count);
896 	rxq_info = &sa->rxq_info[sw_index];
897 
898 	SFC_ASSERT(nb_rx_desc <= rxq_info->max_entries);
899 	rxq_info->entries = nb_rx_desc;
900 	rxq_info->type =
901 		sa->eth_dev->data->dev_conf.rxmode.enable_scatter ?
902 		EFX_RXQ_TYPE_SCATTER : EFX_RXQ_TYPE_DEFAULT;
903 
904 	rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_RX, sw_index,
905 			  rxq_info->entries, socket_id, &evq);
906 	if (rc != 0)
907 		goto fail_ev_qinit;
908 
909 	rc = ENOMEM;
910 	rxq = rte_zmalloc_socket("sfc-rxq", sizeof(*rxq), RTE_CACHE_LINE_SIZE,
911 				 socket_id);
912 	if (rxq == NULL)
913 		goto fail_rxq_alloc;
914 
915 	rxq_info->rxq = rxq;
916 
917 	rxq->evq = evq;
918 	rxq->hw_index = sw_index;
919 	rxq->refill_threshold =
920 		RTE_MAX(rx_conf->rx_free_thresh, SFC_RX_REFILL_BULK);
921 	rxq->refill_mb_pool = mb_pool;
922 
923 	rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries),
924 			   socket_id, &rxq->mem);
925 	if (rc != 0)
926 		goto fail_dma_alloc;
927 
928 	memset(&info, 0, sizeof(info));
929 	info.refill_mb_pool = rxq->refill_mb_pool;
930 	info.refill_threshold = rxq->refill_threshold;
931 	info.buf_size = buf_size;
932 	info.batch_max = encp->enc_rx_batch_max;
933 	info.prefix_size = encp->enc_rx_prefix_size;
934 
935 #if EFSYS_OPT_RX_SCALE
936 	if (sa->hash_support == EFX_RX_HASH_AVAILABLE && sa->rss_channels > 0)
937 		info.flags |= SFC_RXQ_FLAG_RSS_HASH;
938 #endif
939 
940 	info.rxq_entries = rxq_info->entries;
941 	info.rxq_hw_ring = rxq->mem.esm_base;
942 	info.evq_entries = rxq_info->entries;
943 	info.evq_hw_ring = evq->mem.esm_base;
944 	info.hw_index = rxq->hw_index;
945 	info.mem_bar = sa->mem_bar.esb_base;
946 
947 	rc = sa->dp_rx->qcreate(sa->eth_dev->data->port_id, sw_index,
948 				&RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr,
949 				socket_id, &info, &rxq->dp);
950 	if (rc != 0)
951 		goto fail_dp_rx_qcreate;
952 
953 	evq->dp_rxq = rxq->dp;
954 
955 	rxq->state = SFC_RXQ_INITIALIZED;
956 
957 	rxq_info->deferred_start = (rx_conf->rx_deferred_start != 0);
958 
959 	return 0;
960 
961 fail_dp_rx_qcreate:
962 	sfc_dma_free(sa, &rxq->mem);
963 
964 fail_dma_alloc:
965 	rxq_info->rxq = NULL;
966 	rte_free(rxq);
967 
968 fail_rxq_alloc:
969 	sfc_ev_qfini(evq);
970 
971 fail_ev_qinit:
972 	rxq_info->entries = 0;
973 
974 fail_bad_conf:
975 	sfc_log_init(sa, "failed %d", rc);
976 	return rc;
977 }
978 
979 void
980 sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index)
981 {
982 	struct sfc_rxq_info *rxq_info;
983 	struct sfc_rxq *rxq;
984 
985 	SFC_ASSERT(sw_index < sa->rxq_count);
986 
987 	rxq_info = &sa->rxq_info[sw_index];
988 
989 	rxq = rxq_info->rxq;
990 	SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED);
991 
992 	sa->dp_rx->qdestroy(rxq->dp);
993 	rxq->dp = NULL;
994 
995 	rxq_info->rxq = NULL;
996 	rxq_info->entries = 0;
997 
998 	sfc_dma_free(sa, &rxq->mem);
999 
1000 	sfc_ev_qfini(rxq->evq);
1001 	rxq->evq = NULL;
1002 
1003 	rte_free(rxq);
1004 }
1005 
1006 #if EFSYS_OPT_RX_SCALE
1007 efx_rx_hash_type_t
1008 sfc_rte_to_efx_hash_type(uint64_t rss_hf)
1009 {
1010 	efx_rx_hash_type_t efx_hash_types = 0;
1011 
1012 	if ((rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1013 		       ETH_RSS_NONFRAG_IPV4_OTHER)) != 0)
1014 		efx_hash_types |= EFX_RX_HASH_IPV4;
1015 
1016 	if ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != 0)
1017 		efx_hash_types |= EFX_RX_HASH_TCPIPV4;
1018 
1019 	if ((rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
1020 			ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX)) != 0)
1021 		efx_hash_types |= EFX_RX_HASH_IPV6;
1022 
1023 	if ((rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX)) != 0)
1024 		efx_hash_types |= EFX_RX_HASH_TCPIPV6;
1025 
1026 	return efx_hash_types;
1027 }
1028 
1029 uint64_t
1030 sfc_efx_to_rte_hash_type(efx_rx_hash_type_t efx_hash_types)
1031 {
1032 	uint64_t rss_hf = 0;
1033 
1034 	if ((efx_hash_types & EFX_RX_HASH_IPV4) != 0)
1035 		rss_hf |= (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1036 			   ETH_RSS_NONFRAG_IPV4_OTHER);
1037 
1038 	if ((efx_hash_types & EFX_RX_HASH_TCPIPV4) != 0)
1039 		rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
1040 
1041 	if ((efx_hash_types & EFX_RX_HASH_IPV6) != 0)
1042 		rss_hf |= (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
1043 			   ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX);
1044 
1045 	if ((efx_hash_types & EFX_RX_HASH_TCPIPV6) != 0)
1046 		rss_hf |= (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX);
1047 
1048 	return rss_hf;
1049 }
1050 #endif
1051 
1052 static int
1053 sfc_rx_rss_config(struct sfc_adapter *sa)
1054 {
1055 	int rc = 0;
1056 
1057 #if EFSYS_OPT_RX_SCALE
1058 	if (sa->rss_channels > 0) {
1059 		rc = efx_rx_scale_mode_set(sa->nic, EFX_RX_HASHALG_TOEPLITZ,
1060 					   sa->rss_hash_types, B_TRUE);
1061 		if (rc != 0)
1062 			goto finish;
1063 
1064 		rc = efx_rx_scale_key_set(sa->nic, sa->rss_key,
1065 					  sizeof(sa->rss_key));
1066 		if (rc != 0)
1067 			goto finish;
1068 
1069 		rc = efx_rx_scale_tbl_set(sa->nic, sa->rss_tbl,
1070 					  sizeof(sa->rss_tbl));
1071 	}
1072 
1073 finish:
1074 #endif
1075 	return rc;
1076 }
1077 
1078 int
1079 sfc_rx_start(struct sfc_adapter *sa)
1080 {
1081 	unsigned int sw_index;
1082 	int rc;
1083 
1084 	sfc_log_init(sa, "rxq_count=%u", sa->rxq_count);
1085 
1086 	rc = efx_rx_init(sa->nic);
1087 	if (rc != 0)
1088 		goto fail_rx_init;
1089 
1090 	rc = sfc_rx_rss_config(sa);
1091 	if (rc != 0)
1092 		goto fail_rss_config;
1093 
1094 	for (sw_index = 0; sw_index < sa->rxq_count; ++sw_index) {
1095 		if ((!sa->rxq_info[sw_index].deferred_start ||
1096 		     sa->rxq_info[sw_index].deferred_started)) {
1097 			rc = sfc_rx_qstart(sa, sw_index);
1098 			if (rc != 0)
1099 				goto fail_rx_qstart;
1100 		}
1101 	}
1102 
1103 	return 0;
1104 
1105 fail_rx_qstart:
1106 	while (sw_index-- > 0)
1107 		sfc_rx_qstop(sa, sw_index);
1108 
1109 fail_rss_config:
1110 	efx_rx_fini(sa->nic);
1111 
1112 fail_rx_init:
1113 	sfc_log_init(sa, "failed %d", rc);
1114 	return rc;
1115 }
1116 
1117 void
1118 sfc_rx_stop(struct sfc_adapter *sa)
1119 {
1120 	unsigned int sw_index;
1121 
1122 	sfc_log_init(sa, "rxq_count=%u", sa->rxq_count);
1123 
1124 	sw_index = sa->rxq_count;
1125 	while (sw_index-- > 0) {
1126 		if (sa->rxq_info[sw_index].rxq != NULL)
1127 			sfc_rx_qstop(sa, sw_index);
1128 	}
1129 
1130 	efx_rx_fini(sa->nic);
1131 }
1132 
1133 static int
1134 sfc_rx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index)
1135 {
1136 	struct sfc_rxq_info *rxq_info = &sa->rxq_info[sw_index];
1137 	unsigned int max_entries;
1138 
1139 	max_entries = EFX_RXQ_MAXNDESCS;
1140 	SFC_ASSERT(rte_is_power_of_2(max_entries));
1141 
1142 	rxq_info->max_entries = max_entries;
1143 
1144 	return 0;
1145 }
1146 
1147 static int
1148 sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode)
1149 {
1150 	int rc = 0;
1151 
1152 	switch (rxmode->mq_mode) {
1153 	case ETH_MQ_RX_NONE:
1154 		/* No special checks are required */
1155 		break;
1156 #if EFSYS_OPT_RX_SCALE
1157 	case ETH_MQ_RX_RSS:
1158 		if (sa->rss_support == EFX_RX_SCALE_UNAVAILABLE) {
1159 			sfc_err(sa, "RSS is not available");
1160 			rc = EINVAL;
1161 		}
1162 		break;
1163 #endif
1164 	default:
1165 		sfc_err(sa, "Rx multi-queue mode %u not supported",
1166 			rxmode->mq_mode);
1167 		rc = EINVAL;
1168 	}
1169 
1170 	if (rxmode->header_split) {
1171 		sfc_err(sa, "Header split on Rx not supported");
1172 		rc = EINVAL;
1173 	}
1174 
1175 	if (rxmode->hw_vlan_filter) {
1176 		sfc_err(sa, "HW VLAN filtering not supported");
1177 		rc = EINVAL;
1178 	}
1179 
1180 	if (rxmode->hw_vlan_strip) {
1181 		sfc_err(sa, "HW VLAN stripping not supported");
1182 		rc = EINVAL;
1183 	}
1184 
1185 	if (rxmode->hw_vlan_extend) {
1186 		sfc_err(sa,
1187 			"Q-in-Q HW VLAN stripping not supported");
1188 		rc = EINVAL;
1189 	}
1190 
1191 	if (!rxmode->hw_strip_crc) {
1192 		sfc_warn(sa,
1193 			 "FCS stripping control not supported - always stripped");
1194 		rxmode->hw_strip_crc = 1;
1195 	}
1196 
1197 	if (rxmode->enable_scatter &&
1198 	    (~sa->dp_rx->features & SFC_DP_RX_FEAT_SCATTER)) {
1199 		sfc_err(sa, "Rx scatter not supported by %s datapath",
1200 			sa->dp_rx->dp.name);
1201 		rc = EINVAL;
1202 	}
1203 
1204 	if (rxmode->enable_lro) {
1205 		sfc_err(sa, "LRO not supported");
1206 		rc = EINVAL;
1207 	}
1208 
1209 	return rc;
1210 }
1211 
1212 /**
1213  * Destroy excess queues that are no longer needed after reconfiguration
1214  * or complete close.
1215  */
1216 static void
1217 sfc_rx_fini_queues(struct sfc_adapter *sa, unsigned int nb_rx_queues)
1218 {
1219 	int sw_index;
1220 
1221 	SFC_ASSERT(nb_rx_queues <= sa->rxq_count);
1222 
1223 	sw_index = sa->rxq_count;
1224 	while (--sw_index >= (int)nb_rx_queues) {
1225 		if (sa->rxq_info[sw_index].rxq != NULL)
1226 			sfc_rx_qfini(sa, sw_index);
1227 	}
1228 
1229 	sa->rxq_count = nb_rx_queues;
1230 }
1231 
1232 /**
1233  * Initialize Rx subsystem.
1234  *
1235  * Called at device (re)configuration stage when number of receive queues is
1236  * specified together with other device level receive configuration.
1237  *
1238  * It should be used to allocate NUMA-unaware resources.
1239  */
1240 int
1241 sfc_rx_configure(struct sfc_adapter *sa)
1242 {
1243 	struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf;
1244 	const unsigned int nb_rx_queues = sa->eth_dev->data->nb_rx_queues;
1245 	unsigned int sw_index;
1246 	int rc;
1247 
1248 	sfc_log_init(sa, "nb_rx_queues=%u (old %u)",
1249 		     nb_rx_queues, sa->rxq_count);
1250 
1251 	rc = sfc_rx_check_mode(sa, &dev_conf->rxmode);
1252 	if (rc != 0)
1253 		goto fail_check_mode;
1254 
1255 	if (nb_rx_queues == sa->rxq_count)
1256 		goto done;
1257 
1258 	if (sa->rxq_info == NULL) {
1259 		rc = ENOMEM;
1260 		sa->rxq_info = rte_calloc_socket("sfc-rxqs", nb_rx_queues,
1261 						 sizeof(sa->rxq_info[0]), 0,
1262 						 sa->socket_id);
1263 		if (sa->rxq_info == NULL)
1264 			goto fail_rxqs_alloc;
1265 	} else {
1266 		struct sfc_rxq_info *new_rxq_info;
1267 
1268 		if (nb_rx_queues < sa->rxq_count)
1269 			sfc_rx_fini_queues(sa, nb_rx_queues);
1270 
1271 		rc = ENOMEM;
1272 		new_rxq_info =
1273 			rte_realloc(sa->rxq_info,
1274 				    nb_rx_queues * sizeof(sa->rxq_info[0]), 0);
1275 		if (new_rxq_info == NULL && nb_rx_queues > 0)
1276 			goto fail_rxqs_realloc;
1277 
1278 		sa->rxq_info = new_rxq_info;
1279 		if (nb_rx_queues > sa->rxq_count)
1280 			memset(&sa->rxq_info[sa->rxq_count], 0,
1281 			       (nb_rx_queues - sa->rxq_count) *
1282 			       sizeof(sa->rxq_info[0]));
1283 	}
1284 
1285 	while (sa->rxq_count < nb_rx_queues) {
1286 		rc = sfc_rx_qinit_info(sa, sa->rxq_count);
1287 		if (rc != 0)
1288 			goto fail_rx_qinit_info;
1289 
1290 		sa->rxq_count++;
1291 	}
1292 
1293 #if EFSYS_OPT_RX_SCALE
1294 	sa->rss_channels = (dev_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) ?
1295 			   MIN(sa->rxq_count, EFX_MAXRSS) : 0;
1296 
1297 	if (sa->rss_channels > 0) {
1298 		for (sw_index = 0; sw_index < EFX_RSS_TBL_SIZE; ++sw_index)
1299 			sa->rss_tbl[sw_index] = sw_index % sa->rss_channels;
1300 	}
1301 #endif
1302 
1303 done:
1304 	return 0;
1305 
1306 fail_rx_qinit_info:
1307 fail_rxqs_realloc:
1308 fail_rxqs_alloc:
1309 	sfc_rx_close(sa);
1310 
1311 fail_check_mode:
1312 	sfc_log_init(sa, "failed %d", rc);
1313 	return rc;
1314 }
1315 
1316 /**
1317  * Shutdown Rx subsystem.
1318  *
1319  * Called at device close stage, for example, before device shutdown.
1320  */
1321 void
1322 sfc_rx_close(struct sfc_adapter *sa)
1323 {
1324 	sfc_rx_fini_queues(sa, 0);
1325 
1326 	sa->rss_channels = 0;
1327 
1328 	rte_free(sa->rxq_info);
1329 	sa->rxq_info = NULL;
1330 }
1331