xref: /dpdk/drivers/net/sfc/sfc_ethdev.c (revision dda791c2b6b28bece51928aa688f12e9f7b0e394)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  *
3  * Copyright (c) 2016-2018 Solarflare Communications Inc.
4  * All rights reserved.
5  *
6  * This software was jointly developed between OKTET Labs (under contract
7  * for Solarflare) and Solarflare Communications, Inc.
8  */
9 
10 #include <rte_dev.h>
11 #include <rte_ethdev_driver.h>
12 #include <rte_ethdev_pci.h>
13 #include <rte_pci.h>
14 #include <rte_bus_pci.h>
15 #include <rte_errno.h>
16 #include <rte_string_fns.h>
17 
18 #include "efx.h"
19 
20 #include "sfc.h"
21 #include "sfc_debug.h"
22 #include "sfc_log.h"
23 #include "sfc_kvargs.h"
24 #include "sfc_ev.h"
25 #include "sfc_rx.h"
26 #include "sfc_tx.h"
27 #include "sfc_flow.h"
28 #include "sfc_dp.h"
29 #include "sfc_dp_rx.h"
30 
31 uint32_t sfc_logtype_driver;
32 
33 static struct sfc_dp_list sfc_dp_head =
34 	TAILQ_HEAD_INITIALIZER(sfc_dp_head);
35 
36 static int
37 sfc_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
38 {
39 	struct sfc_adapter *sa = dev->data->dev_private;
40 	efx_nic_fw_info_t enfi;
41 	int ret;
42 	int rc;
43 
44 	/*
45 	 * Return value of the callback is likely supposed to be
46 	 * equal to or greater than 0, nevertheless, if an error
47 	 * occurs, it will be desirable to pass it to the caller
48 	 */
49 	if ((fw_version == NULL) || (fw_size == 0))
50 		return -EINVAL;
51 
52 	rc = efx_nic_get_fw_version(sa->nic, &enfi);
53 	if (rc != 0)
54 		return -rc;
55 
56 	ret = snprintf(fw_version, fw_size,
57 		       "%" PRIu16 ".%" PRIu16 ".%" PRIu16 ".%" PRIu16,
58 		       enfi.enfi_mc_fw_version[0], enfi.enfi_mc_fw_version[1],
59 		       enfi.enfi_mc_fw_version[2], enfi.enfi_mc_fw_version[3]);
60 	if (ret < 0)
61 		return ret;
62 
63 	if (enfi.enfi_dpcpu_fw_ids_valid) {
64 		size_t dpcpu_fw_ids_offset = MIN(fw_size - 1, (size_t)ret);
65 		int ret_extra;
66 
67 		ret_extra = snprintf(fw_version + dpcpu_fw_ids_offset,
68 				     fw_size - dpcpu_fw_ids_offset,
69 				     " rx%" PRIx16 " tx%" PRIx16,
70 				     enfi.enfi_rx_dpcpu_fw_id,
71 				     enfi.enfi_tx_dpcpu_fw_id);
72 		if (ret_extra < 0)
73 			return ret_extra;
74 
75 		ret += ret_extra;
76 	}
77 
78 	if (fw_size < (size_t)(++ret))
79 		return ret;
80 	else
81 		return 0;
82 }
83 
84 static void
85 sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
86 {
87 	const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
88 	struct sfc_adapter *sa = dev->data->dev_private;
89 	struct sfc_rss *rss = &sa->rss;
90 	uint64_t txq_offloads_def = 0;
91 
92 	sfc_log_init(sa, "entry");
93 
94 	dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX;
95 
96 	/* Autonegotiation may be disabled */
97 	dev_info->speed_capa = ETH_LINK_SPEED_FIXED;
98 	if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_1000FDX)
99 		dev_info->speed_capa |= ETH_LINK_SPEED_1G;
100 	if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_10000FDX)
101 		dev_info->speed_capa |= ETH_LINK_SPEED_10G;
102 	if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_25000FDX)
103 		dev_info->speed_capa |= ETH_LINK_SPEED_25G;
104 	if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_40000FDX)
105 		dev_info->speed_capa |= ETH_LINK_SPEED_40G;
106 	if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_50000FDX)
107 		dev_info->speed_capa |= ETH_LINK_SPEED_50G;
108 	if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_100000FDX)
109 		dev_info->speed_capa |= ETH_LINK_SPEED_100G;
110 
111 	dev_info->max_rx_queues = sa->rxq_max;
112 	dev_info->max_tx_queues = sa->txq_max;
113 
114 	/* By default packets are dropped if no descriptors are available */
115 	dev_info->default_rxconf.rx_drop_en = 1;
116 
117 	dev_info->rx_queue_offload_capa = sfc_rx_get_queue_offload_caps(sa);
118 
119 	/*
120 	 * rx_offload_capa includes both device and queue offloads since
121 	 * the latter may be requested on a per device basis which makes
122 	 * sense when some offloads are needed to be set on all queues.
123 	 */
124 	dev_info->rx_offload_capa = sfc_rx_get_dev_offload_caps(sa) |
125 				    dev_info->rx_queue_offload_capa;
126 
127 	dev_info->tx_queue_offload_capa = sfc_tx_get_queue_offload_caps(sa);
128 
129 	/*
130 	 * tx_offload_capa includes both device and queue offloads since
131 	 * the latter may be requested on a per device basis which makes
132 	 * sense when some offloads are needed to be set on all queues.
133 	 */
134 	dev_info->tx_offload_capa = sfc_tx_get_dev_offload_caps(sa) |
135 				    dev_info->tx_queue_offload_capa;
136 
137 	if (dev_info->tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
138 		txq_offloads_def |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
139 
140 	dev_info->default_txconf.offloads |= txq_offloads_def;
141 
142 	if (rss->context_type != EFX_RX_SCALE_UNAVAILABLE) {
143 		uint64_t rte_hf = 0;
144 		unsigned int i;
145 
146 		for (i = 0; i < rss->hf_map_nb_entries; ++i)
147 			rte_hf |= rss->hf_map[i].rte;
148 
149 		dev_info->reta_size = EFX_RSS_TBL_SIZE;
150 		dev_info->hash_key_size = EFX_RSS_KEY_SIZE;
151 		dev_info->flow_type_rss_offloads = rte_hf;
152 	}
153 
154 	/* Initialize to hardware limits */
155 	dev_info->rx_desc_lim.nb_max = EFX_RXQ_MAXNDESCS;
156 	dev_info->rx_desc_lim.nb_min = EFX_RXQ_MINNDESCS;
157 	/* The RXQ hardware requires that the descriptor count is a power
158 	 * of 2, but rx_desc_lim cannot properly describe that constraint.
159 	 */
160 	dev_info->rx_desc_lim.nb_align = EFX_RXQ_MINNDESCS;
161 
162 	/* Initialize to hardware limits */
163 	dev_info->tx_desc_lim.nb_max = sa->txq_max_entries;
164 	dev_info->tx_desc_lim.nb_min = EFX_TXQ_MINNDESCS;
165 	/*
166 	 * The TXQ hardware requires that the descriptor count is a power
167 	 * of 2, but tx_desc_lim cannot properly describe that constraint
168 	 */
169 	dev_info->tx_desc_lim.nb_align = EFX_TXQ_MINNDESCS;
170 
171 	if (sap->dp_rx->get_dev_info != NULL)
172 		sap->dp_rx->get_dev_info(dev_info);
173 	if (sap->dp_tx->get_dev_info != NULL)
174 		sap->dp_tx->get_dev_info(dev_info);
175 
176 	dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP |
177 			     RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP;
178 }
179 
180 static const uint32_t *
181 sfc_dev_supported_ptypes_get(struct rte_eth_dev *dev)
182 {
183 	const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
184 	struct sfc_adapter *sa = dev->data->dev_private;
185 	const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
186 	uint32_t tunnel_encaps = encp->enc_tunnel_encapsulations_supported;
187 
188 	return sap->dp_rx->supported_ptypes_get(tunnel_encaps);
189 }
190 
191 static int
192 sfc_dev_configure(struct rte_eth_dev *dev)
193 {
194 	struct rte_eth_dev_data *dev_data = dev->data;
195 	struct sfc_adapter *sa = dev_data->dev_private;
196 	int rc;
197 
198 	sfc_log_init(sa, "entry n_rxq=%u n_txq=%u",
199 		     dev_data->nb_rx_queues, dev_data->nb_tx_queues);
200 
201 	sfc_adapter_lock(sa);
202 	switch (sa->state) {
203 	case SFC_ADAPTER_CONFIGURED:
204 		/* FALLTHROUGH */
205 	case SFC_ADAPTER_INITIALIZED:
206 		rc = sfc_configure(sa);
207 		break;
208 	default:
209 		sfc_err(sa, "unexpected adapter state %u to configure",
210 			sa->state);
211 		rc = EINVAL;
212 		break;
213 	}
214 	sfc_adapter_unlock(sa);
215 
216 	sfc_log_init(sa, "done %d", rc);
217 	SFC_ASSERT(rc >= 0);
218 	return -rc;
219 }
220 
221 static int
222 sfc_dev_start(struct rte_eth_dev *dev)
223 {
224 	struct sfc_adapter *sa = dev->data->dev_private;
225 	int rc;
226 
227 	sfc_log_init(sa, "entry");
228 
229 	sfc_adapter_lock(sa);
230 	rc = sfc_start(sa);
231 	sfc_adapter_unlock(sa);
232 
233 	sfc_log_init(sa, "done %d", rc);
234 	SFC_ASSERT(rc >= 0);
235 	return -rc;
236 }
237 
238 static int
239 sfc_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
240 {
241 	struct sfc_adapter *sa = dev->data->dev_private;
242 	struct rte_eth_link current_link;
243 	int ret;
244 
245 	sfc_log_init(sa, "entry");
246 
247 	if (sa->state != SFC_ADAPTER_STARTED) {
248 		sfc_port_link_mode_to_info(EFX_LINK_UNKNOWN, &current_link);
249 	} else if (wait_to_complete) {
250 		efx_link_mode_t link_mode;
251 
252 		if (efx_port_poll(sa->nic, &link_mode) != 0)
253 			link_mode = EFX_LINK_UNKNOWN;
254 		sfc_port_link_mode_to_info(link_mode, &current_link);
255 
256 	} else {
257 		sfc_ev_mgmt_qpoll(sa);
258 		rte_eth_linkstatus_get(dev, &current_link);
259 	}
260 
261 	ret = rte_eth_linkstatus_set(dev, &current_link);
262 	if (ret == 0)
263 		sfc_notice(sa, "Link status is %s",
264 			   current_link.link_status ? "UP" : "DOWN");
265 
266 	return ret;
267 }
268 
269 static void
270 sfc_dev_stop(struct rte_eth_dev *dev)
271 {
272 	struct sfc_adapter *sa = dev->data->dev_private;
273 
274 	sfc_log_init(sa, "entry");
275 
276 	sfc_adapter_lock(sa);
277 	sfc_stop(sa);
278 	sfc_adapter_unlock(sa);
279 
280 	sfc_log_init(sa, "done");
281 }
282 
283 static int
284 sfc_dev_set_link_up(struct rte_eth_dev *dev)
285 {
286 	struct sfc_adapter *sa = dev->data->dev_private;
287 	int rc;
288 
289 	sfc_log_init(sa, "entry");
290 
291 	sfc_adapter_lock(sa);
292 	rc = sfc_start(sa);
293 	sfc_adapter_unlock(sa);
294 
295 	SFC_ASSERT(rc >= 0);
296 	return -rc;
297 }
298 
299 static int
300 sfc_dev_set_link_down(struct rte_eth_dev *dev)
301 {
302 	struct sfc_adapter *sa = dev->data->dev_private;
303 
304 	sfc_log_init(sa, "entry");
305 
306 	sfc_adapter_lock(sa);
307 	sfc_stop(sa);
308 	sfc_adapter_unlock(sa);
309 
310 	return 0;
311 }
312 
313 static void
314 sfc_dev_close(struct rte_eth_dev *dev)
315 {
316 	struct sfc_adapter *sa = dev->data->dev_private;
317 
318 	sfc_log_init(sa, "entry");
319 
320 	sfc_adapter_lock(sa);
321 	switch (sa->state) {
322 	case SFC_ADAPTER_STARTED:
323 		sfc_stop(sa);
324 		SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED);
325 		/* FALLTHROUGH */
326 	case SFC_ADAPTER_CONFIGURED:
327 		sfc_close(sa);
328 		SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED);
329 		/* FALLTHROUGH */
330 	case SFC_ADAPTER_INITIALIZED:
331 		break;
332 	default:
333 		sfc_err(sa, "unexpected adapter state %u on close", sa->state);
334 		break;
335 	}
336 	sfc_adapter_unlock(sa);
337 
338 	sfc_log_init(sa, "done");
339 }
340 
341 static void
342 sfc_dev_filter_set(struct rte_eth_dev *dev, enum sfc_dev_filter_mode mode,
343 		   boolean_t enabled)
344 {
345 	struct sfc_port *port;
346 	boolean_t *toggle;
347 	struct sfc_adapter *sa = dev->data->dev_private;
348 	boolean_t allmulti = (mode == SFC_DEV_FILTER_MODE_ALLMULTI);
349 	const char *desc = (allmulti) ? "all-multi" : "promiscuous";
350 
351 	sfc_adapter_lock(sa);
352 
353 	port = &sa->port;
354 	toggle = (allmulti) ? (&port->allmulti) : (&port->promisc);
355 
356 	if (*toggle != enabled) {
357 		*toggle = enabled;
358 
359 		if (port->isolated) {
360 			sfc_warn(sa, "isolated mode is active on the port");
361 			sfc_warn(sa, "the change is to be applied on the next "
362 				     "start provided that isolated mode is "
363 				     "disabled prior the next start");
364 		} else if ((sa->state == SFC_ADAPTER_STARTED) &&
365 			   (sfc_set_rx_mode(sa) != 0)) {
366 			*toggle = !(enabled);
367 			sfc_warn(sa, "Failed to %s %s mode",
368 				 ((enabled) ? "enable" : "disable"), desc);
369 		}
370 	}
371 
372 	sfc_adapter_unlock(sa);
373 }
374 
375 static void
376 sfc_dev_promisc_enable(struct rte_eth_dev *dev)
377 {
378 	sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_TRUE);
379 }
380 
381 static void
382 sfc_dev_promisc_disable(struct rte_eth_dev *dev)
383 {
384 	sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_FALSE);
385 }
386 
387 static void
388 sfc_dev_allmulti_enable(struct rte_eth_dev *dev)
389 {
390 	sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_TRUE);
391 }
392 
393 static void
394 sfc_dev_allmulti_disable(struct rte_eth_dev *dev)
395 {
396 	sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_FALSE);
397 }
398 
399 static int
400 sfc_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
401 		   uint16_t nb_rx_desc, unsigned int socket_id,
402 		   const struct rte_eth_rxconf *rx_conf,
403 		   struct rte_mempool *mb_pool)
404 {
405 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
406 	struct sfc_adapter *sa = dev->data->dev_private;
407 	int rc;
408 
409 	sfc_log_init(sa, "RxQ=%u nb_rx_desc=%u socket_id=%u",
410 		     rx_queue_id, nb_rx_desc, socket_id);
411 
412 	sfc_adapter_lock(sa);
413 
414 	rc = sfc_rx_qinit(sa, rx_queue_id, nb_rx_desc, socket_id,
415 			  rx_conf, mb_pool);
416 	if (rc != 0)
417 		goto fail_rx_qinit;
418 
419 	dev->data->rx_queues[rx_queue_id] = sas->rxq_info[rx_queue_id].dp;
420 
421 	sfc_adapter_unlock(sa);
422 
423 	return 0;
424 
425 fail_rx_qinit:
426 	sfc_adapter_unlock(sa);
427 	SFC_ASSERT(rc > 0);
428 	return -rc;
429 }
430 
431 static void
432 sfc_rx_queue_release(void *queue)
433 {
434 	struct sfc_dp_rxq *dp_rxq = queue;
435 	struct sfc_rxq *rxq;
436 	struct sfc_adapter *sa;
437 	unsigned int sw_index;
438 
439 	if (dp_rxq == NULL)
440 		return;
441 
442 	rxq = sfc_rxq_by_dp_rxq(dp_rxq);
443 	sa = rxq->evq->sa;
444 	sfc_adapter_lock(sa);
445 
446 	sw_index = dp_rxq->dpq.queue_id;
447 
448 	sfc_log_init(sa, "RxQ=%u", sw_index);
449 
450 	sfc_rx_qfini(sa, sw_index);
451 
452 	sfc_adapter_unlock(sa);
453 }
454 
455 static int
456 sfc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
457 		   uint16_t nb_tx_desc, unsigned int socket_id,
458 		   const struct rte_eth_txconf *tx_conf)
459 {
460 	struct sfc_adapter *sa = dev->data->dev_private;
461 	int rc;
462 
463 	sfc_log_init(sa, "TxQ = %u, nb_tx_desc = %u, socket_id = %u",
464 		     tx_queue_id, nb_tx_desc, socket_id);
465 
466 	sfc_adapter_lock(sa);
467 
468 	rc = sfc_tx_qinit(sa, tx_queue_id, nb_tx_desc, socket_id, tx_conf);
469 	if (rc != 0)
470 		goto fail_tx_qinit;
471 
472 	dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].dp;
473 
474 	sfc_adapter_unlock(sa);
475 	return 0;
476 
477 fail_tx_qinit:
478 	sfc_adapter_unlock(sa);
479 	SFC_ASSERT(rc > 0);
480 	return -rc;
481 }
482 
483 static void
484 sfc_tx_queue_release(void *queue)
485 {
486 	struct sfc_dp_txq *dp_txq = queue;
487 	struct sfc_txq *txq;
488 	unsigned int sw_index;
489 	struct sfc_adapter *sa;
490 
491 	if (dp_txq == NULL)
492 		return;
493 
494 	txq = sfc_txq_by_dp_txq(dp_txq);
495 	sw_index = dp_txq->dpq.queue_id;
496 
497 	SFC_ASSERT(txq->evq != NULL);
498 	sa = txq->evq->sa;
499 
500 	sfc_log_init(sa, "TxQ = %u", sw_index);
501 
502 	sfc_adapter_lock(sa);
503 
504 	sfc_tx_qfini(sa, sw_index);
505 
506 	sfc_adapter_unlock(sa);
507 }
508 
509 /*
510  * Some statistics are computed as A - B where A and B each increase
511  * monotonically with some hardware counter(s) and the counters are read
512  * asynchronously.
513  *
514  * If packet X is counted in A, but not counted in B yet, computed value is
515  * greater than real.
516  *
517  * If packet X is not counted in A at the moment of reading the counter,
518  * but counted in B at the moment of reading the counter, computed value
519  * is less than real.
520  *
521  * However, counter which grows backward is worse evil than slightly wrong
522  * value. So, let's try to guarantee that it never happens except may be
523  * the case when the MAC stats are zeroed as a result of a NIC reset.
524  */
525 static void
526 sfc_update_diff_stat(uint64_t *stat, uint64_t newval)
527 {
528 	if ((int64_t)(newval - *stat) > 0 || newval == 0)
529 		*stat = newval;
530 }
531 
532 static int
533 sfc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
534 {
535 	struct sfc_adapter *sa = dev->data->dev_private;
536 	struct sfc_port *port = &sa->port;
537 	uint64_t *mac_stats;
538 	int ret;
539 
540 	rte_spinlock_lock(&port->mac_stats_lock);
541 
542 	ret = sfc_port_update_mac_stats(sa);
543 	if (ret != 0)
544 		goto unlock;
545 
546 	mac_stats = port->mac_stats_buf;
547 
548 	if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask,
549 				   EFX_MAC_VADAPTER_RX_UNICAST_PACKETS)) {
550 		stats->ipackets =
551 			mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS] +
552 			mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS] +
553 			mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS];
554 		stats->opackets =
555 			mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS] +
556 			mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS] +
557 			mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS];
558 		stats->ibytes =
559 			mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_BYTES] +
560 			mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES] +
561 			mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES];
562 		stats->obytes =
563 			mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_BYTES] +
564 			mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES] +
565 			mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES];
566 		stats->imissed = mac_stats[EFX_MAC_VADAPTER_RX_BAD_PACKETS];
567 		stats->oerrors = mac_stats[EFX_MAC_VADAPTER_TX_BAD_PACKETS];
568 	} else {
569 		stats->opackets = mac_stats[EFX_MAC_TX_PKTS];
570 		stats->ibytes = mac_stats[EFX_MAC_RX_OCTETS];
571 		stats->obytes = mac_stats[EFX_MAC_TX_OCTETS];
572 		/*
573 		 * Take into account stats which are whenever supported
574 		 * on EF10. If some stat is not supported by current
575 		 * firmware variant or HW revision, it is guaranteed
576 		 * to be zero in mac_stats.
577 		 */
578 		stats->imissed =
579 			mac_stats[EFX_MAC_RX_NODESC_DROP_CNT] +
580 			mac_stats[EFX_MAC_PM_TRUNC_BB_OVERFLOW] +
581 			mac_stats[EFX_MAC_PM_DISCARD_BB_OVERFLOW] +
582 			mac_stats[EFX_MAC_PM_TRUNC_VFIFO_FULL] +
583 			mac_stats[EFX_MAC_PM_DISCARD_VFIFO_FULL] +
584 			mac_stats[EFX_MAC_PM_TRUNC_QBB] +
585 			mac_stats[EFX_MAC_PM_DISCARD_QBB] +
586 			mac_stats[EFX_MAC_PM_DISCARD_MAPPING] +
587 			mac_stats[EFX_MAC_RXDP_Q_DISABLED_PKTS] +
588 			mac_stats[EFX_MAC_RXDP_DI_DROPPED_PKTS];
589 		stats->ierrors =
590 			mac_stats[EFX_MAC_RX_FCS_ERRORS] +
591 			mac_stats[EFX_MAC_RX_ALIGN_ERRORS] +
592 			mac_stats[EFX_MAC_RX_JABBER_PKTS];
593 		/* no oerrors counters supported on EF10 */
594 
595 		/* Exclude missed, errors and pauses from Rx packets */
596 		sfc_update_diff_stat(&port->ipackets,
597 			mac_stats[EFX_MAC_RX_PKTS] -
598 			mac_stats[EFX_MAC_RX_PAUSE_PKTS] -
599 			stats->imissed - stats->ierrors);
600 		stats->ipackets = port->ipackets;
601 	}
602 
603 unlock:
604 	rte_spinlock_unlock(&port->mac_stats_lock);
605 	SFC_ASSERT(ret >= 0);
606 	return -ret;
607 }
608 
609 static void
610 sfc_stats_reset(struct rte_eth_dev *dev)
611 {
612 	struct sfc_adapter *sa = dev->data->dev_private;
613 	struct sfc_port *port = &sa->port;
614 	int rc;
615 
616 	if (sa->state != SFC_ADAPTER_STARTED) {
617 		/*
618 		 * The operation cannot be done if port is not started; it
619 		 * will be scheduled to be done during the next port start
620 		 */
621 		port->mac_stats_reset_pending = B_TRUE;
622 		return;
623 	}
624 
625 	rc = sfc_port_reset_mac_stats(sa);
626 	if (rc != 0)
627 		sfc_err(sa, "failed to reset statistics (rc = %d)", rc);
628 }
629 
630 static int
631 sfc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
632 	       unsigned int xstats_count)
633 {
634 	struct sfc_adapter *sa = dev->data->dev_private;
635 	struct sfc_port *port = &sa->port;
636 	uint64_t *mac_stats;
637 	int rc;
638 	unsigned int i;
639 	int nstats = 0;
640 
641 	rte_spinlock_lock(&port->mac_stats_lock);
642 
643 	rc = sfc_port_update_mac_stats(sa);
644 	if (rc != 0) {
645 		SFC_ASSERT(rc > 0);
646 		nstats = -rc;
647 		goto unlock;
648 	}
649 
650 	mac_stats = port->mac_stats_buf;
651 
652 	for (i = 0; i < EFX_MAC_NSTATS; ++i) {
653 		if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) {
654 			if (xstats != NULL && nstats < (int)xstats_count) {
655 				xstats[nstats].id = nstats;
656 				xstats[nstats].value = mac_stats[i];
657 			}
658 			nstats++;
659 		}
660 	}
661 
662 unlock:
663 	rte_spinlock_unlock(&port->mac_stats_lock);
664 
665 	return nstats;
666 }
667 
668 static int
669 sfc_xstats_get_names(struct rte_eth_dev *dev,
670 		     struct rte_eth_xstat_name *xstats_names,
671 		     unsigned int xstats_count)
672 {
673 	struct sfc_adapter *sa = dev->data->dev_private;
674 	struct sfc_port *port = &sa->port;
675 	unsigned int i;
676 	unsigned int nstats = 0;
677 
678 	for (i = 0; i < EFX_MAC_NSTATS; ++i) {
679 		if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) {
680 			if (xstats_names != NULL && nstats < xstats_count)
681 				strlcpy(xstats_names[nstats].name,
682 					efx_mac_stat_name(sa->nic, i),
683 					sizeof(xstats_names[0].name));
684 			nstats++;
685 		}
686 	}
687 
688 	return nstats;
689 }
690 
691 static int
692 sfc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
693 		     uint64_t *values, unsigned int n)
694 {
695 	struct sfc_adapter *sa = dev->data->dev_private;
696 	struct sfc_port *port = &sa->port;
697 	uint64_t *mac_stats;
698 	unsigned int nb_supported = 0;
699 	unsigned int nb_written = 0;
700 	unsigned int i;
701 	int ret;
702 	int rc;
703 
704 	if (unlikely(values == NULL) ||
705 	    unlikely((ids == NULL) && (n < port->mac_stats_nb_supported)))
706 		return port->mac_stats_nb_supported;
707 
708 	rte_spinlock_lock(&port->mac_stats_lock);
709 
710 	rc = sfc_port_update_mac_stats(sa);
711 	if (rc != 0) {
712 		SFC_ASSERT(rc > 0);
713 		ret = -rc;
714 		goto unlock;
715 	}
716 
717 	mac_stats = port->mac_stats_buf;
718 
719 	for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < n); ++i) {
720 		if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i))
721 			continue;
722 
723 		if ((ids == NULL) || (ids[nb_written] == nb_supported))
724 			values[nb_written++] = mac_stats[i];
725 
726 		++nb_supported;
727 	}
728 
729 	ret = nb_written;
730 
731 unlock:
732 	rte_spinlock_unlock(&port->mac_stats_lock);
733 
734 	return ret;
735 }
736 
737 static int
738 sfc_xstats_get_names_by_id(struct rte_eth_dev *dev,
739 			   struct rte_eth_xstat_name *xstats_names,
740 			   const uint64_t *ids, unsigned int size)
741 {
742 	struct sfc_adapter *sa = dev->data->dev_private;
743 	struct sfc_port *port = &sa->port;
744 	unsigned int nb_supported = 0;
745 	unsigned int nb_written = 0;
746 	unsigned int i;
747 
748 	if (unlikely(xstats_names == NULL) ||
749 	    unlikely((ids == NULL) && (size < port->mac_stats_nb_supported)))
750 		return port->mac_stats_nb_supported;
751 
752 	for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < size); ++i) {
753 		if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i))
754 			continue;
755 
756 		if ((ids == NULL) || (ids[nb_written] == nb_supported)) {
757 			char *name = xstats_names[nb_written++].name;
758 
759 			strlcpy(name, efx_mac_stat_name(sa->nic, i),
760 				sizeof(xstats_names[0].name));
761 		}
762 
763 		++nb_supported;
764 	}
765 
766 	return nb_written;
767 }
768 
769 static int
770 sfc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
771 {
772 	struct sfc_adapter *sa = dev->data->dev_private;
773 	unsigned int wanted_fc, link_fc;
774 
775 	memset(fc_conf, 0, sizeof(*fc_conf));
776 
777 	sfc_adapter_lock(sa);
778 
779 	if (sa->state == SFC_ADAPTER_STARTED)
780 		efx_mac_fcntl_get(sa->nic, &wanted_fc, &link_fc);
781 	else
782 		link_fc = sa->port.flow_ctrl;
783 
784 	switch (link_fc) {
785 	case 0:
786 		fc_conf->mode = RTE_FC_NONE;
787 		break;
788 	case EFX_FCNTL_RESPOND:
789 		fc_conf->mode = RTE_FC_RX_PAUSE;
790 		break;
791 	case EFX_FCNTL_GENERATE:
792 		fc_conf->mode = RTE_FC_TX_PAUSE;
793 		break;
794 	case (EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE):
795 		fc_conf->mode = RTE_FC_FULL;
796 		break;
797 	default:
798 		sfc_err(sa, "%s: unexpected flow control value %#x",
799 			__func__, link_fc);
800 	}
801 
802 	fc_conf->autoneg = sa->port.flow_ctrl_autoneg;
803 
804 	sfc_adapter_unlock(sa);
805 
806 	return 0;
807 }
808 
809 static int
810 sfc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
811 {
812 	struct sfc_adapter *sa = dev->data->dev_private;
813 	struct sfc_port *port = &sa->port;
814 	unsigned int fcntl;
815 	int rc;
816 
817 	if (fc_conf->high_water != 0 || fc_conf->low_water != 0 ||
818 	    fc_conf->pause_time != 0 || fc_conf->send_xon != 0 ||
819 	    fc_conf->mac_ctrl_frame_fwd != 0) {
820 		sfc_err(sa, "unsupported flow control settings specified");
821 		rc = EINVAL;
822 		goto fail_inval;
823 	}
824 
825 	switch (fc_conf->mode) {
826 	case RTE_FC_NONE:
827 		fcntl = 0;
828 		break;
829 	case RTE_FC_RX_PAUSE:
830 		fcntl = EFX_FCNTL_RESPOND;
831 		break;
832 	case RTE_FC_TX_PAUSE:
833 		fcntl = EFX_FCNTL_GENERATE;
834 		break;
835 	case RTE_FC_FULL:
836 		fcntl = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE;
837 		break;
838 	default:
839 		rc = EINVAL;
840 		goto fail_inval;
841 	}
842 
843 	sfc_adapter_lock(sa);
844 
845 	if (sa->state == SFC_ADAPTER_STARTED) {
846 		rc = efx_mac_fcntl_set(sa->nic, fcntl, fc_conf->autoneg);
847 		if (rc != 0)
848 			goto fail_mac_fcntl_set;
849 	}
850 
851 	port->flow_ctrl = fcntl;
852 	port->flow_ctrl_autoneg = fc_conf->autoneg;
853 
854 	sfc_adapter_unlock(sa);
855 
856 	return 0;
857 
858 fail_mac_fcntl_set:
859 	sfc_adapter_unlock(sa);
860 fail_inval:
861 	SFC_ASSERT(rc > 0);
862 	return -rc;
863 }
864 
865 static int
866 sfc_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
867 {
868 	struct sfc_adapter *sa = dev->data->dev_private;
869 	size_t pdu = EFX_MAC_PDU(mtu);
870 	size_t old_pdu;
871 	int rc;
872 
873 	sfc_log_init(sa, "mtu=%u", mtu);
874 
875 	rc = EINVAL;
876 	if (pdu < EFX_MAC_PDU_MIN) {
877 		sfc_err(sa, "too small MTU %u (PDU size %u less than min %u)",
878 			(unsigned int)mtu, (unsigned int)pdu,
879 			EFX_MAC_PDU_MIN);
880 		goto fail_inval;
881 	}
882 	if (pdu > EFX_MAC_PDU_MAX) {
883 		sfc_err(sa, "too big MTU %u (PDU size %u greater than max %u)",
884 			(unsigned int)mtu, (unsigned int)pdu,
885 			EFX_MAC_PDU_MAX);
886 		goto fail_inval;
887 	}
888 
889 	sfc_adapter_lock(sa);
890 
891 	if (pdu != sa->port.pdu) {
892 		if (sa->state == SFC_ADAPTER_STARTED) {
893 			sfc_stop(sa);
894 
895 			old_pdu = sa->port.pdu;
896 			sa->port.pdu = pdu;
897 			rc = sfc_start(sa);
898 			if (rc != 0)
899 				goto fail_start;
900 		} else {
901 			sa->port.pdu = pdu;
902 		}
903 	}
904 
905 	/*
906 	 * The driver does not use it, but other PMDs update jumbo frame
907 	 * flag and max_rx_pkt_len when MTU is set.
908 	 */
909 	if (mtu > ETHER_MAX_LEN) {
910 		struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
911 		rxmode->offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
912 	}
913 
914 	dev->data->dev_conf.rxmode.max_rx_pkt_len = sa->port.pdu;
915 
916 	sfc_adapter_unlock(sa);
917 
918 	sfc_log_init(sa, "done");
919 	return 0;
920 
921 fail_start:
922 	sa->port.pdu = old_pdu;
923 	if (sfc_start(sa) != 0)
924 		sfc_err(sa, "cannot start with neither new (%u) nor old (%u) "
925 			"PDU max size - port is stopped",
926 			(unsigned int)pdu, (unsigned int)old_pdu);
927 	sfc_adapter_unlock(sa);
928 
929 fail_inval:
930 	sfc_log_init(sa, "failed %d", rc);
931 	SFC_ASSERT(rc > 0);
932 	return -rc;
933 }
934 static int
935 sfc_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
936 {
937 	struct sfc_adapter *sa = dev->data->dev_private;
938 	const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
939 	struct sfc_port *port = &sa->port;
940 	struct ether_addr *old_addr = &dev->data->mac_addrs[0];
941 	int rc = 0;
942 
943 	sfc_adapter_lock(sa);
944 
945 	/*
946 	 * Copy the address to the device private data so that
947 	 * it could be recalled in the case of adapter restart.
948 	 */
949 	ether_addr_copy(mac_addr, &port->default_mac_addr);
950 
951 	/*
952 	 * Neither of the two following checks can return
953 	 * an error. The new MAC address is preserved in
954 	 * the device private data and can be activated
955 	 * on the next port start if the user prevents
956 	 * isolated mode from being enabled.
957 	 */
958 	if (port->isolated) {
959 		sfc_warn(sa, "isolated mode is active on the port");
960 		sfc_warn(sa, "will not set MAC address");
961 		goto unlock;
962 	}
963 
964 	if (sa->state != SFC_ADAPTER_STARTED) {
965 		sfc_notice(sa, "the port is not started");
966 		sfc_notice(sa, "the new MAC address will be set on port start");
967 
968 		goto unlock;
969 	}
970 
971 	if (encp->enc_allow_set_mac_with_installed_filters) {
972 		rc = efx_mac_addr_set(sa->nic, mac_addr->addr_bytes);
973 		if (rc != 0) {
974 			sfc_err(sa, "cannot set MAC address (rc = %u)", rc);
975 			goto unlock;
976 		}
977 
978 		/*
979 		 * Changing the MAC address by means of MCDI request
980 		 * has no effect on received traffic, therefore
981 		 * we also need to update unicast filters
982 		 */
983 		rc = sfc_set_rx_mode(sa);
984 		if (rc != 0) {
985 			sfc_err(sa, "cannot set filter (rc = %u)", rc);
986 			/* Rollback the old address */
987 			(void)efx_mac_addr_set(sa->nic, old_addr->addr_bytes);
988 			(void)sfc_set_rx_mode(sa);
989 		}
990 	} else {
991 		sfc_warn(sa, "cannot set MAC address with filters installed");
992 		sfc_warn(sa, "adapter will be restarted to pick the new MAC");
993 		sfc_warn(sa, "(some traffic may be dropped)");
994 
995 		/*
996 		 * Since setting MAC address with filters installed is not
997 		 * allowed on the adapter, the new MAC address will be set
998 		 * by means of adapter restart. sfc_start() shall retrieve
999 		 * the new address from the device private data and set it.
1000 		 */
1001 		sfc_stop(sa);
1002 		rc = sfc_start(sa);
1003 		if (rc != 0)
1004 			sfc_err(sa, "cannot restart adapter (rc = %u)", rc);
1005 	}
1006 
1007 unlock:
1008 	if (rc != 0)
1009 		ether_addr_copy(old_addr, &port->default_mac_addr);
1010 
1011 	sfc_adapter_unlock(sa);
1012 
1013 	SFC_ASSERT(rc >= 0);
1014 	return -rc;
1015 }
1016 
1017 
1018 static int
1019 sfc_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set,
1020 		     uint32_t nb_mc_addr)
1021 {
1022 	struct sfc_adapter *sa = dev->data->dev_private;
1023 	struct sfc_port *port = &sa->port;
1024 	uint8_t *mc_addrs = port->mcast_addrs;
1025 	int rc;
1026 	unsigned int i;
1027 
1028 	if (port->isolated) {
1029 		sfc_err(sa, "isolated mode is active on the port");
1030 		sfc_err(sa, "will not set multicast address list");
1031 		return -ENOTSUP;
1032 	}
1033 
1034 	if (mc_addrs == NULL)
1035 		return -ENOBUFS;
1036 
1037 	if (nb_mc_addr > port->max_mcast_addrs) {
1038 		sfc_err(sa, "too many multicast addresses: %u > %u",
1039 			 nb_mc_addr, port->max_mcast_addrs);
1040 		return -EINVAL;
1041 	}
1042 
1043 	for (i = 0; i < nb_mc_addr; ++i) {
1044 		rte_memcpy(mc_addrs, mc_addr_set[i].addr_bytes,
1045 				 EFX_MAC_ADDR_LEN);
1046 		mc_addrs += EFX_MAC_ADDR_LEN;
1047 	}
1048 
1049 	port->nb_mcast_addrs = nb_mc_addr;
1050 
1051 	if (sa->state != SFC_ADAPTER_STARTED)
1052 		return 0;
1053 
1054 	rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs,
1055 					port->nb_mcast_addrs);
1056 	if (rc != 0)
1057 		sfc_err(sa, "cannot set multicast address list (rc = %u)", rc);
1058 
1059 	SFC_ASSERT(rc >= 0);
1060 	return -rc;
1061 }
1062 
1063 /*
1064  * The function is used by the secondary process as well. It must not
1065  * use any process-local pointers from the adapter data.
1066  */
1067 static void
1068 sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1069 		      struct rte_eth_rxq_info *qinfo)
1070 {
1071 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1072 	struct sfc_adapter *sa = dev->data->dev_private;
1073 	struct sfc_rxq_info *rxq_info;
1074 
1075 	sfc_adapter_lock(sa);
1076 
1077 	SFC_ASSERT(rx_queue_id < sas->rxq_count);
1078 
1079 	rxq_info = &sas->rxq_info[rx_queue_id];
1080 
1081 	qinfo->mp = rxq_info->refill_mb_pool;
1082 	qinfo->conf.rx_free_thresh = rxq_info->refill_threshold;
1083 	qinfo->conf.rx_drop_en = 1;
1084 	qinfo->conf.rx_deferred_start = rxq_info->deferred_start;
1085 	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
1086 	if (rxq_info->type_flags & EFX_RXQ_FLAG_SCATTER) {
1087 		qinfo->conf.offloads |= DEV_RX_OFFLOAD_SCATTER;
1088 		qinfo->scattered_rx = 1;
1089 	}
1090 	qinfo->nb_desc = rxq_info->entries;
1091 
1092 	sfc_adapter_unlock(sa);
1093 }
1094 
1095 /*
1096  * The function is used by the secondary process as well. It must not
1097  * use any process-local pointers from the adapter data.
1098  */
1099 static void
1100 sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1101 		      struct rte_eth_txq_info *qinfo)
1102 {
1103 	struct sfc_adapter *sa = dev->data->dev_private;
1104 	struct sfc_txq_info *txq_info;
1105 
1106 	sfc_adapter_lock(sa);
1107 
1108 	SFC_ASSERT(tx_queue_id < sa->txq_count);
1109 
1110 	txq_info = &sa->txq_info[tx_queue_id];
1111 
1112 	memset(qinfo, 0, sizeof(*qinfo));
1113 
1114 	qinfo->conf.offloads = txq_info->offloads;
1115 	qinfo->conf.tx_free_thresh = txq_info->free_thresh;
1116 	qinfo->conf.tx_deferred_start = txq_info->deferred_start;
1117 	qinfo->nb_desc = txq_info->entries;
1118 
1119 	sfc_adapter_unlock(sa);
1120 }
1121 
1122 /*
1123  * The function is used by the secondary process as well. It must not
1124  * use any process-local pointers from the adapter data.
1125  */
1126 static uint32_t
1127 sfc_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1128 {
1129 	const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
1130 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1131 	struct sfc_rxq_info *rxq_info;
1132 
1133 	SFC_ASSERT(rx_queue_id < sas->rxq_count);
1134 	rxq_info = &sas->rxq_info[rx_queue_id];
1135 
1136 	if ((rxq_info->state & SFC_RXQ_STARTED) == 0)
1137 		return 0;
1138 
1139 	return sap->dp_rx->qdesc_npending(rxq_info->dp);
1140 }
1141 
1142 /*
1143  * The function is used by the secondary process as well. It must not
1144  * use any process-local pointers from the adapter data.
1145  */
1146 static int
1147 sfc_rx_descriptor_done(void *queue, uint16_t offset)
1148 {
1149 	struct sfc_dp_rxq *dp_rxq = queue;
1150 	const struct sfc_dp_rx *dp_rx;
1151 
1152 	dp_rx = sfc_dp_rx_by_dp_rxq(dp_rxq);
1153 
1154 	return offset < dp_rx->qdesc_npending(dp_rxq);
1155 }
1156 
1157 /*
1158  * The function is used by the secondary process as well. It must not
1159  * use any process-local pointers from the adapter data.
1160  */
1161 static int
1162 sfc_rx_descriptor_status(void *queue, uint16_t offset)
1163 {
1164 	struct sfc_dp_rxq *dp_rxq = queue;
1165 	const struct sfc_dp_rx *dp_rx;
1166 
1167 	dp_rx = sfc_dp_rx_by_dp_rxq(dp_rxq);
1168 
1169 	return dp_rx->qdesc_status(dp_rxq, offset);
1170 }
1171 
1172 /*
1173  * The function is used by the secondary process as well. It must not
1174  * use any process-local pointers from the adapter data.
1175  */
1176 static int
1177 sfc_tx_descriptor_status(void *queue, uint16_t offset)
1178 {
1179 	struct sfc_dp_txq *dp_txq = queue;
1180 	const struct sfc_dp_tx *dp_tx;
1181 
1182 	dp_tx = sfc_dp_tx_by_dp_txq(dp_txq);
1183 
1184 	return dp_tx->qdesc_status(dp_txq, offset);
1185 }
1186 
1187 static int
1188 sfc_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1189 {
1190 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1191 	struct sfc_adapter *sa = dev->data->dev_private;
1192 	int rc;
1193 
1194 	sfc_log_init(sa, "RxQ=%u", rx_queue_id);
1195 
1196 	sfc_adapter_lock(sa);
1197 
1198 	rc = EINVAL;
1199 	if (sa->state != SFC_ADAPTER_STARTED)
1200 		goto fail_not_started;
1201 
1202 	if (sas->rxq_info[rx_queue_id].state != SFC_RXQ_INITIALIZED)
1203 		goto fail_not_setup;
1204 
1205 	rc = sfc_rx_qstart(sa, rx_queue_id);
1206 	if (rc != 0)
1207 		goto fail_rx_qstart;
1208 
1209 	sas->rxq_info[rx_queue_id].deferred_started = B_TRUE;
1210 
1211 	sfc_adapter_unlock(sa);
1212 
1213 	return 0;
1214 
1215 fail_rx_qstart:
1216 fail_not_setup:
1217 fail_not_started:
1218 	sfc_adapter_unlock(sa);
1219 	SFC_ASSERT(rc > 0);
1220 	return -rc;
1221 }
1222 
1223 static int
1224 sfc_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1225 {
1226 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1227 	struct sfc_adapter *sa = dev->data->dev_private;
1228 
1229 	sfc_log_init(sa, "RxQ=%u", rx_queue_id);
1230 
1231 	sfc_adapter_lock(sa);
1232 	sfc_rx_qstop(sa, rx_queue_id);
1233 
1234 	sas->rxq_info[rx_queue_id].deferred_started = B_FALSE;
1235 
1236 	sfc_adapter_unlock(sa);
1237 
1238 	return 0;
1239 }
1240 
1241 static int
1242 sfc_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1243 {
1244 	struct sfc_adapter *sa = dev->data->dev_private;
1245 	int rc;
1246 
1247 	sfc_log_init(sa, "TxQ = %u", tx_queue_id);
1248 
1249 	sfc_adapter_lock(sa);
1250 
1251 	rc = EINVAL;
1252 	if (sa->state != SFC_ADAPTER_STARTED)
1253 		goto fail_not_started;
1254 
1255 	if (sa->txq_info[tx_queue_id].state != SFC_TXQ_INITIALIZED)
1256 		goto fail_not_setup;
1257 
1258 	rc = sfc_tx_qstart(sa, tx_queue_id);
1259 	if (rc != 0)
1260 		goto fail_tx_qstart;
1261 
1262 	sa->txq_info[tx_queue_id].deferred_started = B_TRUE;
1263 
1264 	sfc_adapter_unlock(sa);
1265 	return 0;
1266 
1267 fail_tx_qstart:
1268 
1269 fail_not_setup:
1270 fail_not_started:
1271 	sfc_adapter_unlock(sa);
1272 	SFC_ASSERT(rc > 0);
1273 	return -rc;
1274 }
1275 
1276 static int
1277 sfc_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1278 {
1279 	struct sfc_adapter *sa = dev->data->dev_private;
1280 
1281 	sfc_log_init(sa, "TxQ = %u", tx_queue_id);
1282 
1283 	sfc_adapter_lock(sa);
1284 
1285 	sfc_tx_qstop(sa, tx_queue_id);
1286 
1287 	sa->txq_info[tx_queue_id].deferred_started = B_FALSE;
1288 
1289 	sfc_adapter_unlock(sa);
1290 	return 0;
1291 }
1292 
1293 static efx_tunnel_protocol_t
1294 sfc_tunnel_rte_type_to_efx_udp_proto(enum rte_eth_tunnel_type rte_type)
1295 {
1296 	switch (rte_type) {
1297 	case RTE_TUNNEL_TYPE_VXLAN:
1298 		return EFX_TUNNEL_PROTOCOL_VXLAN;
1299 	case RTE_TUNNEL_TYPE_GENEVE:
1300 		return EFX_TUNNEL_PROTOCOL_GENEVE;
1301 	default:
1302 		return EFX_TUNNEL_NPROTOS;
1303 	}
1304 }
1305 
1306 enum sfc_udp_tunnel_op_e {
1307 	SFC_UDP_TUNNEL_ADD_PORT,
1308 	SFC_UDP_TUNNEL_DEL_PORT,
1309 };
1310 
1311 static int
1312 sfc_dev_udp_tunnel_op(struct rte_eth_dev *dev,
1313 		      struct rte_eth_udp_tunnel *tunnel_udp,
1314 		      enum sfc_udp_tunnel_op_e op)
1315 {
1316 	struct sfc_adapter *sa = dev->data->dev_private;
1317 	efx_tunnel_protocol_t tunnel_proto;
1318 	int rc;
1319 
1320 	sfc_log_init(sa, "%s udp_port=%u prot_type=%u",
1321 		     (op == SFC_UDP_TUNNEL_ADD_PORT) ? "add" :
1322 		     (op == SFC_UDP_TUNNEL_DEL_PORT) ? "delete" : "unknown",
1323 		     tunnel_udp->udp_port, tunnel_udp->prot_type);
1324 
1325 	tunnel_proto =
1326 		sfc_tunnel_rte_type_to_efx_udp_proto(tunnel_udp->prot_type);
1327 	if (tunnel_proto >= EFX_TUNNEL_NPROTOS) {
1328 		rc = ENOTSUP;
1329 		goto fail_bad_proto;
1330 	}
1331 
1332 	sfc_adapter_lock(sa);
1333 
1334 	switch (op) {
1335 	case SFC_UDP_TUNNEL_ADD_PORT:
1336 		rc = efx_tunnel_config_udp_add(sa->nic,
1337 					       tunnel_udp->udp_port,
1338 					       tunnel_proto);
1339 		break;
1340 	case SFC_UDP_TUNNEL_DEL_PORT:
1341 		rc = efx_tunnel_config_udp_remove(sa->nic,
1342 						  tunnel_udp->udp_port,
1343 						  tunnel_proto);
1344 		break;
1345 	default:
1346 		rc = EINVAL;
1347 		goto fail_bad_op;
1348 	}
1349 
1350 	if (rc != 0)
1351 		goto fail_op;
1352 
1353 	if (sa->state == SFC_ADAPTER_STARTED) {
1354 		rc = efx_tunnel_reconfigure(sa->nic);
1355 		if (rc == EAGAIN) {
1356 			/*
1357 			 * Configuration is accepted by FW and MC reboot
1358 			 * is initiated to apply the changes. MC reboot
1359 			 * will be handled in a usual way (MC reboot
1360 			 * event on management event queue and adapter
1361 			 * restart).
1362 			 */
1363 			rc = 0;
1364 		} else if (rc != 0) {
1365 			goto fail_reconfigure;
1366 		}
1367 	}
1368 
1369 	sfc_adapter_unlock(sa);
1370 	return 0;
1371 
1372 fail_reconfigure:
1373 	/* Remove/restore entry since the change makes the trouble */
1374 	switch (op) {
1375 	case SFC_UDP_TUNNEL_ADD_PORT:
1376 		(void)efx_tunnel_config_udp_remove(sa->nic,
1377 						   tunnel_udp->udp_port,
1378 						   tunnel_proto);
1379 		break;
1380 	case SFC_UDP_TUNNEL_DEL_PORT:
1381 		(void)efx_tunnel_config_udp_add(sa->nic,
1382 						tunnel_udp->udp_port,
1383 						tunnel_proto);
1384 		break;
1385 	}
1386 
1387 fail_op:
1388 fail_bad_op:
1389 	sfc_adapter_unlock(sa);
1390 
1391 fail_bad_proto:
1392 	SFC_ASSERT(rc > 0);
1393 	return -rc;
1394 }
1395 
1396 static int
1397 sfc_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
1398 			    struct rte_eth_udp_tunnel *tunnel_udp)
1399 {
1400 	return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_ADD_PORT);
1401 }
1402 
1403 static int
1404 sfc_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
1405 			    struct rte_eth_udp_tunnel *tunnel_udp)
1406 {
1407 	return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_DEL_PORT);
1408 }
1409 
1410 /*
1411  * The function is used by the secondary process as well. It must not
1412  * use any process-local pointers from the adapter data.
1413  */
1414 static int
1415 sfc_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
1416 			  struct rte_eth_rss_conf *rss_conf)
1417 {
1418 	struct sfc_adapter *sa = dev->data->dev_private;
1419 	struct sfc_rss *rss = &sa->rss;
1420 
1421 	if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE)
1422 		return -ENOTSUP;
1423 
1424 	sfc_adapter_lock(sa);
1425 
1426 	/*
1427 	 * Mapping of hash configuration between RTE and EFX is not one-to-one,
1428 	 * hence, conversion is done here to derive a correct set of ETH_RSS
1429 	 * flags which corresponds to the active EFX configuration stored
1430 	 * locally in 'sfc_adapter' and kept up-to-date
1431 	 */
1432 	rss_conf->rss_hf = sfc_rx_hf_efx_to_rte(sa, rss->hash_types);
1433 	rss_conf->rss_key_len = EFX_RSS_KEY_SIZE;
1434 	if (rss_conf->rss_key != NULL)
1435 		rte_memcpy(rss_conf->rss_key, rss->key, EFX_RSS_KEY_SIZE);
1436 
1437 	sfc_adapter_unlock(sa);
1438 
1439 	return 0;
1440 }
1441 
1442 static int
1443 sfc_dev_rss_hash_update(struct rte_eth_dev *dev,
1444 			struct rte_eth_rss_conf *rss_conf)
1445 {
1446 	struct sfc_adapter *sa = dev->data->dev_private;
1447 	struct sfc_rss *rss = &sa->rss;
1448 	struct sfc_port *port = &sa->port;
1449 	unsigned int efx_hash_types;
1450 	int rc = 0;
1451 
1452 	if (port->isolated)
1453 		return -ENOTSUP;
1454 
1455 	if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) {
1456 		sfc_err(sa, "RSS is not available");
1457 		return -ENOTSUP;
1458 	}
1459 
1460 	if (rss->channels == 0) {
1461 		sfc_err(sa, "RSS is not configured");
1462 		return -EINVAL;
1463 	}
1464 
1465 	if ((rss_conf->rss_key != NULL) &&
1466 	    (rss_conf->rss_key_len != sizeof(rss->key))) {
1467 		sfc_err(sa, "RSS key size is wrong (should be %lu)",
1468 			sizeof(rss->key));
1469 		return -EINVAL;
1470 	}
1471 
1472 	sfc_adapter_lock(sa);
1473 
1474 	rc = sfc_rx_hf_rte_to_efx(sa, rss_conf->rss_hf, &efx_hash_types);
1475 	if (rc != 0)
1476 		goto fail_rx_hf_rte_to_efx;
1477 
1478 	rc = efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1479 				   rss->hash_alg, efx_hash_types, B_TRUE);
1480 	if (rc != 0)
1481 		goto fail_scale_mode_set;
1482 
1483 	if (rss_conf->rss_key != NULL) {
1484 		if (sa->state == SFC_ADAPTER_STARTED) {
1485 			rc = efx_rx_scale_key_set(sa->nic,
1486 						  EFX_RSS_CONTEXT_DEFAULT,
1487 						  rss_conf->rss_key,
1488 						  sizeof(rss->key));
1489 			if (rc != 0)
1490 				goto fail_scale_key_set;
1491 		}
1492 
1493 		rte_memcpy(rss->key, rss_conf->rss_key, sizeof(rss->key));
1494 	}
1495 
1496 	rss->hash_types = efx_hash_types;
1497 
1498 	sfc_adapter_unlock(sa);
1499 
1500 	return 0;
1501 
1502 fail_scale_key_set:
1503 	if (efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1504 				  EFX_RX_HASHALG_TOEPLITZ,
1505 				  rss->hash_types, B_TRUE) != 0)
1506 		sfc_err(sa, "failed to restore RSS mode");
1507 
1508 fail_scale_mode_set:
1509 fail_rx_hf_rte_to_efx:
1510 	sfc_adapter_unlock(sa);
1511 	return -rc;
1512 }
1513 
1514 /*
1515  * The function is used by the secondary process as well. It must not
1516  * use any process-local pointers from the adapter data.
1517  */
1518 static int
1519 sfc_dev_rss_reta_query(struct rte_eth_dev *dev,
1520 		       struct rte_eth_rss_reta_entry64 *reta_conf,
1521 		       uint16_t reta_size)
1522 {
1523 	struct sfc_adapter *sa = dev->data->dev_private;
1524 	struct sfc_rss *rss = &sa->rss;
1525 	struct sfc_port *port = &sa->port;
1526 	int entry;
1527 
1528 	if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE || port->isolated)
1529 		return -ENOTSUP;
1530 
1531 	if (rss->channels == 0)
1532 		return -EINVAL;
1533 
1534 	if (reta_size != EFX_RSS_TBL_SIZE)
1535 		return -EINVAL;
1536 
1537 	sfc_adapter_lock(sa);
1538 
1539 	for (entry = 0; entry < reta_size; entry++) {
1540 		int grp = entry / RTE_RETA_GROUP_SIZE;
1541 		int grp_idx = entry % RTE_RETA_GROUP_SIZE;
1542 
1543 		if ((reta_conf[grp].mask >> grp_idx) & 1)
1544 			reta_conf[grp].reta[grp_idx] = rss->tbl[entry];
1545 	}
1546 
1547 	sfc_adapter_unlock(sa);
1548 
1549 	return 0;
1550 }
1551 
1552 static int
1553 sfc_dev_rss_reta_update(struct rte_eth_dev *dev,
1554 			struct rte_eth_rss_reta_entry64 *reta_conf,
1555 			uint16_t reta_size)
1556 {
1557 	struct sfc_adapter *sa = dev->data->dev_private;
1558 	struct sfc_rss *rss = &sa->rss;
1559 	struct sfc_port *port = &sa->port;
1560 	unsigned int *rss_tbl_new;
1561 	uint16_t entry;
1562 	int rc = 0;
1563 
1564 
1565 	if (port->isolated)
1566 		return -ENOTSUP;
1567 
1568 	if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) {
1569 		sfc_err(sa, "RSS is not available");
1570 		return -ENOTSUP;
1571 	}
1572 
1573 	if (rss->channels == 0) {
1574 		sfc_err(sa, "RSS is not configured");
1575 		return -EINVAL;
1576 	}
1577 
1578 	if (reta_size != EFX_RSS_TBL_SIZE) {
1579 		sfc_err(sa, "RETA size is wrong (should be %u)",
1580 			EFX_RSS_TBL_SIZE);
1581 		return -EINVAL;
1582 	}
1583 
1584 	rss_tbl_new = rte_zmalloc("rss_tbl_new", sizeof(rss->tbl), 0);
1585 	if (rss_tbl_new == NULL)
1586 		return -ENOMEM;
1587 
1588 	sfc_adapter_lock(sa);
1589 
1590 	rte_memcpy(rss_tbl_new, rss->tbl, sizeof(rss->tbl));
1591 
1592 	for (entry = 0; entry < reta_size; entry++) {
1593 		int grp_idx = entry % RTE_RETA_GROUP_SIZE;
1594 		struct rte_eth_rss_reta_entry64 *grp;
1595 
1596 		grp = &reta_conf[entry / RTE_RETA_GROUP_SIZE];
1597 
1598 		if (grp->mask & (1ull << grp_idx)) {
1599 			if (grp->reta[grp_idx] >= rss->channels) {
1600 				rc = EINVAL;
1601 				goto bad_reta_entry;
1602 			}
1603 			rss_tbl_new[entry] = grp->reta[grp_idx];
1604 		}
1605 	}
1606 
1607 	if (sa->state == SFC_ADAPTER_STARTED) {
1608 		rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1609 					  rss_tbl_new, EFX_RSS_TBL_SIZE);
1610 		if (rc != 0)
1611 			goto fail_scale_tbl_set;
1612 	}
1613 
1614 	rte_memcpy(rss->tbl, rss_tbl_new, sizeof(rss->tbl));
1615 
1616 fail_scale_tbl_set:
1617 bad_reta_entry:
1618 	sfc_adapter_unlock(sa);
1619 
1620 	rte_free(rss_tbl_new);
1621 
1622 	SFC_ASSERT(rc >= 0);
1623 	return -rc;
1624 }
1625 
1626 static int
1627 sfc_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type filter_type,
1628 		    enum rte_filter_op filter_op,
1629 		    void *arg)
1630 {
1631 	struct sfc_adapter *sa = dev->data->dev_private;
1632 	int rc = ENOTSUP;
1633 
1634 	sfc_log_init(sa, "entry");
1635 
1636 	switch (filter_type) {
1637 	case RTE_ETH_FILTER_NONE:
1638 		sfc_err(sa, "Global filters configuration not supported");
1639 		break;
1640 	case RTE_ETH_FILTER_MACVLAN:
1641 		sfc_err(sa, "MACVLAN filters not supported");
1642 		break;
1643 	case RTE_ETH_FILTER_ETHERTYPE:
1644 		sfc_err(sa, "EtherType filters not supported");
1645 		break;
1646 	case RTE_ETH_FILTER_FLEXIBLE:
1647 		sfc_err(sa, "Flexible filters not supported");
1648 		break;
1649 	case RTE_ETH_FILTER_SYN:
1650 		sfc_err(sa, "SYN filters not supported");
1651 		break;
1652 	case RTE_ETH_FILTER_NTUPLE:
1653 		sfc_err(sa, "NTUPLE filters not supported");
1654 		break;
1655 	case RTE_ETH_FILTER_TUNNEL:
1656 		sfc_err(sa, "Tunnel filters not supported");
1657 		break;
1658 	case RTE_ETH_FILTER_FDIR:
1659 		sfc_err(sa, "Flow Director filters not supported");
1660 		break;
1661 	case RTE_ETH_FILTER_HASH:
1662 		sfc_err(sa, "Hash filters not supported");
1663 		break;
1664 	case RTE_ETH_FILTER_GENERIC:
1665 		if (filter_op != RTE_ETH_FILTER_GET) {
1666 			rc = EINVAL;
1667 		} else {
1668 			*(const void **)arg = &sfc_flow_ops;
1669 			rc = 0;
1670 		}
1671 		break;
1672 	default:
1673 		sfc_err(sa, "Unknown filter type %u", filter_type);
1674 		break;
1675 	}
1676 
1677 	sfc_log_init(sa, "exit: %d", -rc);
1678 	SFC_ASSERT(rc >= 0);
1679 	return -rc;
1680 }
1681 
1682 static int
1683 sfc_pool_ops_supported(struct rte_eth_dev *dev, const char *pool)
1684 {
1685 	const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
1686 
1687 	/*
1688 	 * If Rx datapath does not provide callback to check mempool,
1689 	 * all pools are supported.
1690 	 */
1691 	if (sap->dp_rx->pool_ops_supported == NULL)
1692 		return 1;
1693 
1694 	return sap->dp_rx->pool_ops_supported(pool);
1695 }
1696 
1697 static const struct eth_dev_ops sfc_eth_dev_ops = {
1698 	.dev_configure			= sfc_dev_configure,
1699 	.dev_start			= sfc_dev_start,
1700 	.dev_stop			= sfc_dev_stop,
1701 	.dev_set_link_up		= sfc_dev_set_link_up,
1702 	.dev_set_link_down		= sfc_dev_set_link_down,
1703 	.dev_close			= sfc_dev_close,
1704 	.promiscuous_enable		= sfc_dev_promisc_enable,
1705 	.promiscuous_disable		= sfc_dev_promisc_disable,
1706 	.allmulticast_enable		= sfc_dev_allmulti_enable,
1707 	.allmulticast_disable		= sfc_dev_allmulti_disable,
1708 	.link_update			= sfc_dev_link_update,
1709 	.stats_get			= sfc_stats_get,
1710 	.stats_reset			= sfc_stats_reset,
1711 	.xstats_get			= sfc_xstats_get,
1712 	.xstats_reset			= sfc_stats_reset,
1713 	.xstats_get_names		= sfc_xstats_get_names,
1714 	.dev_infos_get			= sfc_dev_infos_get,
1715 	.dev_supported_ptypes_get	= sfc_dev_supported_ptypes_get,
1716 	.mtu_set			= sfc_dev_set_mtu,
1717 	.rx_queue_start			= sfc_rx_queue_start,
1718 	.rx_queue_stop			= sfc_rx_queue_stop,
1719 	.tx_queue_start			= sfc_tx_queue_start,
1720 	.tx_queue_stop			= sfc_tx_queue_stop,
1721 	.rx_queue_setup			= sfc_rx_queue_setup,
1722 	.rx_queue_release		= sfc_rx_queue_release,
1723 	.rx_queue_count			= sfc_rx_queue_count,
1724 	.rx_descriptor_done		= sfc_rx_descriptor_done,
1725 	.rx_descriptor_status		= sfc_rx_descriptor_status,
1726 	.tx_descriptor_status		= sfc_tx_descriptor_status,
1727 	.tx_queue_setup			= sfc_tx_queue_setup,
1728 	.tx_queue_release		= sfc_tx_queue_release,
1729 	.flow_ctrl_get			= sfc_flow_ctrl_get,
1730 	.flow_ctrl_set			= sfc_flow_ctrl_set,
1731 	.mac_addr_set			= sfc_mac_addr_set,
1732 	.udp_tunnel_port_add		= sfc_dev_udp_tunnel_port_add,
1733 	.udp_tunnel_port_del		= sfc_dev_udp_tunnel_port_del,
1734 	.reta_update			= sfc_dev_rss_reta_update,
1735 	.reta_query			= sfc_dev_rss_reta_query,
1736 	.rss_hash_update		= sfc_dev_rss_hash_update,
1737 	.rss_hash_conf_get		= sfc_dev_rss_hash_conf_get,
1738 	.filter_ctrl			= sfc_dev_filter_ctrl,
1739 	.set_mc_addr_list		= sfc_set_mc_addr_list,
1740 	.rxq_info_get			= sfc_rx_queue_info_get,
1741 	.txq_info_get			= sfc_tx_queue_info_get,
1742 	.fw_version_get			= sfc_fw_version_get,
1743 	.xstats_get_by_id		= sfc_xstats_get_by_id,
1744 	.xstats_get_names_by_id		= sfc_xstats_get_names_by_id,
1745 	.pool_ops_supported		= sfc_pool_ops_supported,
1746 };
1747 
1748 /**
1749  * Duplicate a string in potentially shared memory required for
1750  * multi-process support.
1751  *
1752  * strdup() allocates from process-local heap/memory.
1753  */
1754 static char *
1755 sfc_strdup(const char *str)
1756 {
1757 	size_t size;
1758 	char *copy;
1759 
1760 	if (str == NULL)
1761 		return NULL;
1762 
1763 	size = strlen(str) + 1;
1764 	copy = rte_malloc(__func__, size, 0);
1765 	if (copy != NULL)
1766 		rte_memcpy(copy, str, size);
1767 
1768 	return copy;
1769 }
1770 
1771 static int
1772 sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
1773 {
1774 	struct sfc_adapter *sa = dev->data->dev_private;
1775 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1776 	const struct sfc_dp_rx *dp_rx;
1777 	const struct sfc_dp_tx *dp_tx;
1778 	const efx_nic_cfg_t *encp;
1779 	unsigned int avail_caps = 0;
1780 	const char *rx_name = NULL;
1781 	const char *tx_name = NULL;
1782 	int rc;
1783 
1784 	switch (sa->family) {
1785 	case EFX_FAMILY_HUNTINGTON:
1786 	case EFX_FAMILY_MEDFORD:
1787 	case EFX_FAMILY_MEDFORD2:
1788 		avail_caps |= SFC_DP_HW_FW_CAP_EF10;
1789 		break;
1790 	default:
1791 		break;
1792 	}
1793 
1794 	encp = efx_nic_cfg_get(sa->nic);
1795 	if (encp->enc_rx_es_super_buffer_supported)
1796 		avail_caps |= SFC_DP_HW_FW_CAP_RX_ES_SUPER_BUFFER;
1797 
1798 	rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH,
1799 				sfc_kvarg_string_handler, &rx_name);
1800 	if (rc != 0)
1801 		goto fail_kvarg_rx_datapath;
1802 
1803 	if (rx_name != NULL) {
1804 		dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, rx_name);
1805 		if (dp_rx == NULL) {
1806 			sfc_err(sa, "Rx datapath %s not found", rx_name);
1807 			rc = ENOENT;
1808 			goto fail_dp_rx;
1809 		}
1810 		if (!sfc_dp_match_hw_fw_caps(&dp_rx->dp, avail_caps)) {
1811 			sfc_err(sa,
1812 				"Insufficient Hw/FW capabilities to use Rx datapath %s",
1813 				rx_name);
1814 			rc = EINVAL;
1815 			goto fail_dp_rx_caps;
1816 		}
1817 	} else {
1818 		dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps);
1819 		if (dp_rx == NULL) {
1820 			sfc_err(sa, "Rx datapath by caps %#x not found",
1821 				avail_caps);
1822 			rc = ENOENT;
1823 			goto fail_dp_rx;
1824 		}
1825 	}
1826 
1827 	sas->dp_rx_name = sfc_strdup(dp_rx->dp.name);
1828 	if (sas->dp_rx_name == NULL) {
1829 		rc = ENOMEM;
1830 		goto fail_dp_rx_name;
1831 	}
1832 
1833 	sfc_notice(sa, "use %s Rx datapath", sas->dp_rx_name);
1834 
1835 	rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH,
1836 				sfc_kvarg_string_handler, &tx_name);
1837 	if (rc != 0)
1838 		goto fail_kvarg_tx_datapath;
1839 
1840 	if (tx_name != NULL) {
1841 		dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name);
1842 		if (dp_tx == NULL) {
1843 			sfc_err(sa, "Tx datapath %s not found", tx_name);
1844 			rc = ENOENT;
1845 			goto fail_dp_tx;
1846 		}
1847 		if (!sfc_dp_match_hw_fw_caps(&dp_tx->dp, avail_caps)) {
1848 			sfc_err(sa,
1849 				"Insufficient Hw/FW capabilities to use Tx datapath %s",
1850 				tx_name);
1851 			rc = EINVAL;
1852 			goto fail_dp_tx_caps;
1853 		}
1854 	} else {
1855 		dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps);
1856 		if (dp_tx == NULL) {
1857 			sfc_err(sa, "Tx datapath by caps %#x not found",
1858 				avail_caps);
1859 			rc = ENOENT;
1860 			goto fail_dp_tx;
1861 		}
1862 	}
1863 
1864 	sas->dp_tx_name = sfc_strdup(dp_tx->dp.name);
1865 	if (sas->dp_tx_name == NULL) {
1866 		rc = ENOMEM;
1867 		goto fail_dp_tx_name;
1868 	}
1869 
1870 	sfc_notice(sa, "use %s Tx datapath", sas->dp_tx_name);
1871 
1872 	sa->priv.dp_rx = dp_rx;
1873 	sa->priv.dp_tx = dp_tx;
1874 
1875 	dev->rx_pkt_burst = dp_rx->pkt_burst;
1876 	dev->tx_pkt_burst = dp_tx->pkt_burst;
1877 
1878 	dev->dev_ops = &sfc_eth_dev_ops;
1879 
1880 	return 0;
1881 
1882 fail_dp_tx_name:
1883 fail_dp_tx_caps:
1884 fail_dp_tx:
1885 fail_kvarg_tx_datapath:
1886 	rte_free(sas->dp_rx_name);
1887 	sas->dp_rx_name = NULL;
1888 
1889 fail_dp_rx_name:
1890 fail_dp_rx_caps:
1891 fail_dp_rx:
1892 fail_kvarg_rx_datapath:
1893 	return rc;
1894 }
1895 
1896 static void
1897 sfc_eth_dev_clear_ops(struct rte_eth_dev *dev)
1898 {
1899 	struct sfc_adapter *sa = dev->data->dev_private;
1900 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1901 
1902 	dev->dev_ops = NULL;
1903 	dev->rx_pkt_burst = NULL;
1904 	dev->tx_pkt_burst = NULL;
1905 
1906 	rte_free(sas->dp_tx_name);
1907 	sas->dp_tx_name = NULL;
1908 	sa->priv.dp_tx = NULL;
1909 
1910 	rte_free(sas->dp_rx_name);
1911 	sas->dp_rx_name = NULL;
1912 	sa->priv.dp_rx = NULL;
1913 }
1914 
1915 static const struct eth_dev_ops sfc_eth_dev_secondary_ops = {
1916 	.rx_queue_count			= sfc_rx_queue_count,
1917 	.rx_descriptor_done		= sfc_rx_descriptor_done,
1918 	.rx_descriptor_status		= sfc_rx_descriptor_status,
1919 	.tx_descriptor_status		= sfc_tx_descriptor_status,
1920 	.reta_query			= sfc_dev_rss_reta_query,
1921 	.rss_hash_conf_get		= sfc_dev_rss_hash_conf_get,
1922 	.rxq_info_get			= sfc_rx_queue_info_get,
1923 	.txq_info_get			= sfc_tx_queue_info_get,
1924 };
1925 
1926 static int
1927 sfc_eth_dev_secondary_init(struct rte_eth_dev *dev, uint32_t logtype_main)
1928 {
1929 	struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1930 	struct sfc_adapter_priv *sap;
1931 	const struct sfc_dp_rx *dp_rx;
1932 	const struct sfc_dp_tx *dp_tx;
1933 	int rc;
1934 
1935 	/*
1936 	 * Allocate process private data from heap, since it should not
1937 	 * be located in shared memory allocated using rte_malloc() API.
1938 	 */
1939 	sap = calloc(1, sizeof(*sap));
1940 	if (sap == NULL) {
1941 		rc = ENOMEM;
1942 		goto fail_alloc_priv;
1943 	}
1944 
1945 	sap->logtype_main = logtype_main;
1946 
1947 	dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, sas->dp_rx_name);
1948 	if (dp_rx == NULL) {
1949 		SFC_LOG(sas, RTE_LOG_ERR, logtype_main,
1950 			"cannot find %s Rx datapath", sas->dp_rx_name);
1951 		rc = ENOENT;
1952 		goto fail_dp_rx;
1953 	}
1954 	if (~dp_rx->features & SFC_DP_RX_FEAT_MULTI_PROCESS) {
1955 		SFC_LOG(sas, RTE_LOG_ERR, logtype_main,
1956 			"%s Rx datapath does not support multi-process",
1957 			sas->dp_rx_name);
1958 		rc = EINVAL;
1959 		goto fail_dp_rx_multi_process;
1960 	}
1961 
1962 	dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, sas->dp_tx_name);
1963 	if (dp_tx == NULL) {
1964 		SFC_LOG(sas, RTE_LOG_ERR, logtype_main,
1965 			"cannot find %s Tx datapath", sas->dp_tx_name);
1966 		rc = ENOENT;
1967 		goto fail_dp_tx;
1968 	}
1969 	if (~dp_tx->features & SFC_DP_TX_FEAT_MULTI_PROCESS) {
1970 		SFC_LOG(sas, RTE_LOG_ERR, logtype_main,
1971 			"%s Tx datapath does not support multi-process",
1972 			sas->dp_tx_name);
1973 		rc = EINVAL;
1974 		goto fail_dp_tx_multi_process;
1975 	}
1976 
1977 	sap->dp_rx = dp_rx;
1978 	sap->dp_tx = dp_tx;
1979 
1980 	dev->process_private = sap;
1981 	dev->rx_pkt_burst = dp_rx->pkt_burst;
1982 	dev->tx_pkt_burst = dp_tx->pkt_burst;
1983 	dev->dev_ops = &sfc_eth_dev_secondary_ops;
1984 
1985 	return 0;
1986 
1987 fail_dp_tx_multi_process:
1988 fail_dp_tx:
1989 fail_dp_rx_multi_process:
1990 fail_dp_rx:
1991 	free(sap);
1992 
1993 fail_alloc_priv:
1994 	return rc;
1995 }
1996 
1997 static void
1998 sfc_eth_dev_secondary_clear_ops(struct rte_eth_dev *dev)
1999 {
2000 	free(dev->process_private);
2001 	dev->process_private = NULL;
2002 	dev->dev_ops = NULL;
2003 	dev->tx_pkt_burst = NULL;
2004 	dev->rx_pkt_burst = NULL;
2005 }
2006 
2007 static void
2008 sfc_register_dp(void)
2009 {
2010 	/* Register once */
2011 	if (TAILQ_EMPTY(&sfc_dp_head)) {
2012 		/* Prefer EF10 datapath */
2013 		sfc_dp_register(&sfc_dp_head, &sfc_ef10_essb_rx.dp);
2014 		sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp);
2015 		sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
2016 
2017 		sfc_dp_register(&sfc_dp_head, &sfc_ef10_tx.dp);
2018 		sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp);
2019 		sfc_dp_register(&sfc_dp_head, &sfc_ef10_simple_tx.dp);
2020 	}
2021 }
2022 
2023 static int
2024 sfc_eth_dev_init(struct rte_eth_dev *dev)
2025 {
2026 	struct sfc_adapter *sa = dev->data->dev_private;
2027 	struct sfc_adapter_shared *sas;
2028 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2029 	uint32_t logtype_main;
2030 	int rc;
2031 	const efx_nic_cfg_t *encp;
2032 	const struct ether_addr *from;
2033 
2034 	sfc_register_dp();
2035 
2036 	logtype_main = sfc_register_logtype(&pci_dev->addr,
2037 					    SFC_LOGTYPE_MAIN_STR,
2038 					    RTE_LOG_NOTICE);
2039 
2040 	sa->priv.shared = &sa->_shared;
2041 	sas = sa->priv.shared;
2042 
2043 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2044 		return -sfc_eth_dev_secondary_init(dev, logtype_main);
2045 
2046 	/*
2047 	 * sfc_adapter is a mixture of shared and process private data.
2048 	 * During transition period use it in both kinds. When the
2049 	 * driver becomes ready to separate it, sfc_adapter will become
2050 	 * primary process private only.
2051 	 */
2052 	dev->process_private = sa;
2053 
2054 	/* Required for logging */
2055 	sas->pci_addr = pci_dev->addr;
2056 	sas->port_id = dev->data->port_id;
2057 	sa->priv.logtype_main = logtype_main;
2058 
2059 	sa->eth_dev = dev;
2060 
2061 	/* Copy PCI device info to the dev->data */
2062 	rte_eth_copy_pci_info(dev, pci_dev);
2063 
2064 	rc = sfc_kvargs_parse(sa);
2065 	if (rc != 0)
2066 		goto fail_kvargs_parse;
2067 
2068 	sfc_log_init(sa, "entry");
2069 
2070 	dev->data->mac_addrs = rte_zmalloc("sfc", ETHER_ADDR_LEN, 0);
2071 	if (dev->data->mac_addrs == NULL) {
2072 		rc = ENOMEM;
2073 		goto fail_mac_addrs;
2074 	}
2075 
2076 	sfc_adapter_lock_init(sa);
2077 	sfc_adapter_lock(sa);
2078 
2079 	sfc_log_init(sa, "probing");
2080 	rc = sfc_probe(sa);
2081 	if (rc != 0)
2082 		goto fail_probe;
2083 
2084 	sfc_log_init(sa, "set device ops");
2085 	rc = sfc_eth_dev_set_ops(dev);
2086 	if (rc != 0)
2087 		goto fail_set_ops;
2088 
2089 	sfc_log_init(sa, "attaching");
2090 	rc = sfc_attach(sa);
2091 	if (rc != 0)
2092 		goto fail_attach;
2093 
2094 	encp = efx_nic_cfg_get(sa->nic);
2095 
2096 	/*
2097 	 * The arguments are really reverse order in comparison to
2098 	 * Linux kernel. Copy from NIC config to Ethernet device data.
2099 	 */
2100 	from = (const struct ether_addr *)(encp->enc_mac_addr);
2101 	ether_addr_copy(from, &dev->data->mac_addrs[0]);
2102 
2103 	sfc_adapter_unlock(sa);
2104 
2105 	sfc_log_init(sa, "done");
2106 	return 0;
2107 
2108 fail_attach:
2109 	sfc_eth_dev_clear_ops(dev);
2110 
2111 fail_set_ops:
2112 	sfc_unprobe(sa);
2113 
2114 fail_probe:
2115 	sfc_adapter_unlock(sa);
2116 	sfc_adapter_lock_fini(sa);
2117 	rte_free(dev->data->mac_addrs);
2118 	dev->data->mac_addrs = NULL;
2119 
2120 fail_mac_addrs:
2121 	sfc_kvargs_cleanup(sa);
2122 
2123 fail_kvargs_parse:
2124 	sfc_log_init(sa, "failed %d", rc);
2125 	dev->process_private = NULL;
2126 	SFC_ASSERT(rc > 0);
2127 	return -rc;
2128 }
2129 
2130 static int
2131 sfc_eth_dev_uninit(struct rte_eth_dev *dev)
2132 {
2133 	struct sfc_adapter *sa;
2134 
2135 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2136 		sfc_eth_dev_secondary_clear_ops(dev);
2137 		return 0;
2138 	}
2139 
2140 	sa = dev->data->dev_private;
2141 	sfc_log_init(sa, "entry");
2142 
2143 	sfc_adapter_lock(sa);
2144 
2145 	sfc_eth_dev_clear_ops(dev);
2146 
2147 	sfc_detach(sa);
2148 	sfc_unprobe(sa);
2149 
2150 	sfc_kvargs_cleanup(sa);
2151 
2152 	sfc_adapter_unlock(sa);
2153 	sfc_adapter_lock_fini(sa);
2154 
2155 	sfc_log_init(sa, "done");
2156 
2157 	/* Required for logging, so cleanup last */
2158 	sa->eth_dev = NULL;
2159 	return 0;
2160 }
2161 
2162 static const struct rte_pci_id pci_id_sfc_efx_map[] = {
2163 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE) },
2164 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE_VF) },
2165 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT) },
2166 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT_VF) },
2167 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD) },
2168 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD_VF) },
2169 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2) },
2170 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2_VF) },
2171 	{ .vendor_id = 0 /* sentinel */ }
2172 };
2173 
2174 static int sfc_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2175 	struct rte_pci_device *pci_dev)
2176 {
2177 	return rte_eth_dev_pci_generic_probe(pci_dev,
2178 		sizeof(struct sfc_adapter), sfc_eth_dev_init);
2179 }
2180 
2181 static int sfc_eth_dev_pci_remove(struct rte_pci_device *pci_dev)
2182 {
2183 	return rte_eth_dev_pci_generic_remove(pci_dev, sfc_eth_dev_uninit);
2184 }
2185 
2186 static struct rte_pci_driver sfc_efx_pmd = {
2187 	.id_table = pci_id_sfc_efx_map,
2188 	.drv_flags =
2189 		RTE_PCI_DRV_INTR_LSC |
2190 		RTE_PCI_DRV_NEED_MAPPING,
2191 	.probe = sfc_eth_dev_pci_probe,
2192 	.remove = sfc_eth_dev_pci_remove,
2193 };
2194 
2195 RTE_PMD_REGISTER_PCI(net_sfc_efx, sfc_efx_pmd);
2196 RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map);
2197 RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio-pci");
2198 RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
2199 	SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " "
2200 	SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " "
2201 	SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " "
2202 	SFC_KVARG_FW_VARIANT "=" SFC_KVARG_VALUES_FW_VARIANT " "
2203 	SFC_KVARG_RXD_WAIT_TIMEOUT_NS "=<long> "
2204 	SFC_KVARG_STATS_UPDATE_PERIOD_MS "=<long>");
2205 
2206 RTE_INIT(sfc_driver_register_logtype)
2207 {
2208 	int ret;
2209 
2210 	ret = rte_log_register_type_and_pick_level(SFC_LOGTYPE_PREFIX "driver",
2211 						   RTE_LOG_NOTICE);
2212 	sfc_logtype_driver = (ret < 0) ? RTE_LOGTYPE_PMD : ret;
2213 }
2214