xref: /dpdk/drivers/net/thunderx/nicvf_ethdev.c (revision 8809f78c7dd9f33a44a4f89c58fc91ded34296ed)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2016 Cavium, Inc
3  */
4 
5 #include <assert.h>
6 #include <stdio.h>
7 #include <stdbool.h>
8 #include <errno.h>
9 #include <stdint.h>
10 #include <string.h>
11 #include <unistd.h>
12 #include <stdarg.h>
13 #include <inttypes.h>
14 #include <netinet/in.h>
15 #include <sys/queue.h>
16 
17 #include <rte_alarm.h>
18 #include <rte_branch_prediction.h>
19 #include <rte_byteorder.h>
20 #include <rte_common.h>
21 #include <rte_cycles.h>
22 #include <rte_debug.h>
23 #include <rte_dev.h>
24 #include <rte_eal.h>
25 #include <rte_ether.h>
26 #include <rte_ethdev_driver.h>
27 #include <rte_ethdev_pci.h>
28 #include <rte_interrupts.h>
29 #include <rte_log.h>
30 #include <rte_memory.h>
31 #include <rte_memzone.h>
32 #include <rte_malloc.h>
33 #include <rte_random.h>
34 #include <rte_pci.h>
35 #include <rte_bus_pci.h>
36 #include <rte_tailq.h>
37 #include <rte_devargs.h>
38 #include <rte_kvargs.h>
39 
40 #include "base/nicvf_plat.h"
41 
42 #include "nicvf_ethdev.h"
43 #include "nicvf_rxtx.h"
44 #include "nicvf_svf.h"
45 #include "nicvf_logs.h"
46 
47 static int nicvf_dev_stop(struct rte_eth_dev *dev);
48 static void nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup);
49 static void nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic,
50 			  bool cleanup);
51 static int nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask);
52 static int nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
53 
54 RTE_LOG_REGISTER(nicvf_logtype_mbox, pmd.net.thunderx.mbox, NOTICE);
55 RTE_LOG_REGISTER(nicvf_logtype_init, pmd.net.thunderx.init, NOTICE);
56 RTE_LOG_REGISTER(nicvf_logtype_driver, pmd.net.thunderx.driver, NOTICE);
57 
58 static void
59 nicvf_link_status_update(struct nicvf *nic,
60 			 struct rte_eth_link *link)
61 {
62 	memset(link, 0, sizeof(*link));
63 
64 	link->link_status = nic->link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
65 
66 	if (nic->duplex == NICVF_HALF_DUPLEX)
67 		link->link_duplex = ETH_LINK_HALF_DUPLEX;
68 	else if (nic->duplex == NICVF_FULL_DUPLEX)
69 		link->link_duplex = ETH_LINK_FULL_DUPLEX;
70 	link->link_speed = nic->speed;
71 	link->link_autoneg = ETH_LINK_AUTONEG;
72 }
73 
74 static void
75 nicvf_interrupt(void *arg)
76 {
77 	struct rte_eth_dev *dev = arg;
78 	struct nicvf *nic = nicvf_pmd_priv(dev);
79 	struct rte_eth_link link;
80 
81 	if (nicvf_reg_poll_interrupts(nic) == NIC_MBOX_MSG_BGX_LINK_CHANGE) {
82 		if (dev->data->dev_conf.intr_conf.lsc) {
83 			nicvf_link_status_update(nic, &link);
84 			rte_eth_linkstatus_set(dev, &link);
85 
86 			rte_eth_dev_callback_process(dev,
87 						     RTE_ETH_EVENT_INTR_LSC,
88 						     NULL);
89 		}
90 	}
91 
92 	rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
93 				nicvf_interrupt, dev);
94 }
95 
96 static void
97 nicvf_vf_interrupt(void *arg)
98 {
99 	struct nicvf *nic = arg;
100 
101 	nicvf_reg_poll_interrupts(nic);
102 
103 	rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
104 				nicvf_vf_interrupt, nic);
105 }
106 
107 static int
108 nicvf_periodic_alarm_start(void (fn)(void *), void *arg)
109 {
110 	return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, fn, arg);
111 }
112 
113 static int
114 nicvf_periodic_alarm_stop(void (fn)(void *), void *arg)
115 {
116 	return rte_eal_alarm_cancel(fn, arg);
117 }
118 
119 /*
120  * Return 0 means link status changed, -1 means not changed
121  */
122 static int
123 nicvf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
124 {
125 #define CHECK_INTERVAL 100  /* 100ms */
126 #define MAX_CHECK_TIME 90   /* 9s (90 * 100ms) in total */
127 	struct rte_eth_link link;
128 	struct nicvf *nic = nicvf_pmd_priv(dev);
129 	int i;
130 
131 	PMD_INIT_FUNC_TRACE();
132 
133 	if (wait_to_complete) {
134 		/* rte_eth_link_get() might need to wait up to 9 seconds */
135 		for (i = 0; i < MAX_CHECK_TIME; i++) {
136 			nicvf_link_status_update(nic, &link);
137 			if (link.link_status == ETH_LINK_UP)
138 				break;
139 			rte_delay_ms(CHECK_INTERVAL);
140 		}
141 	} else {
142 		nicvf_link_status_update(nic, &link);
143 	}
144 
145 	return rte_eth_linkstatus_set(dev, &link);
146 }
147 
148 static int
149 nicvf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
150 {
151 	struct nicvf *nic = nicvf_pmd_priv(dev);
152 	uint32_t buffsz, frame_size = mtu + NIC_HW_L2_OVERHEAD;
153 	size_t i;
154 	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
155 
156 	PMD_INIT_FUNC_TRACE();
157 
158 	if (frame_size > NIC_HW_MAX_FRS)
159 		return -EINVAL;
160 
161 	if (frame_size < NIC_HW_MIN_FRS)
162 		return -EINVAL;
163 
164 	buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
165 
166 	/*
167 	 * Refuse mtu that requires the support of scattered packets
168 	 * when this feature has not been enabled before.
169 	 */
170 	if (dev->data->dev_started && !dev->data->scattered_rx &&
171 		(frame_size + 2 * VLAN_TAG_SIZE > buffsz))
172 		return -EINVAL;
173 
174 	/* check <seg size> * <max_seg>  >= max_frame */
175 	if (dev->data->scattered_rx &&
176 		(frame_size + 2 * VLAN_TAG_SIZE > buffsz * NIC_HW_MAX_SEGS))
177 		return -EINVAL;
178 
179 	if (frame_size > RTE_ETHER_MAX_LEN)
180 		rxmode->offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
181 	else
182 		rxmode->offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
183 
184 	if (nicvf_mbox_update_hw_max_frs(nic, mtu))
185 		return -EINVAL;
186 
187 	/* Update max_rx_pkt_len */
188 	rxmode->max_rx_pkt_len = mtu + RTE_ETHER_HDR_LEN;
189 	nic->mtu = mtu;
190 
191 	for (i = 0; i < nic->sqs_count; i++)
192 		nic->snicvf[i]->mtu = mtu;
193 
194 	return 0;
195 }
196 
197 static int
198 nicvf_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs)
199 {
200 	uint64_t *data = regs->data;
201 	struct nicvf *nic = nicvf_pmd_priv(dev);
202 
203 	if (data == NULL) {
204 		regs->length = nicvf_reg_get_count();
205 		regs->width = THUNDERX_REG_BYTES;
206 		return 0;
207 	}
208 
209 	/* Support only full register dump */
210 	if ((regs->length == 0) ||
211 		(regs->length == (uint32_t)nicvf_reg_get_count())) {
212 		regs->version = nic->vendor_id << 16 | nic->device_id;
213 		nicvf_reg_dump(nic, data);
214 		return 0;
215 	}
216 	return -ENOTSUP;
217 }
218 
219 static int
220 nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
221 {
222 	uint16_t qidx;
223 	struct nicvf_hw_rx_qstats rx_qstats;
224 	struct nicvf_hw_tx_qstats tx_qstats;
225 	struct nicvf_hw_stats port_stats;
226 	struct nicvf *nic = nicvf_pmd_priv(dev);
227 	uint16_t rx_start, rx_end;
228 	uint16_t tx_start, tx_end;
229 	size_t i;
230 
231 	/* RX queue indices for the first VF */
232 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
233 
234 	/* Reading per RX ring stats */
235 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
236 		if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
237 			break;
238 
239 		nicvf_hw_get_rx_qstats(nic, &rx_qstats, qidx);
240 		stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes;
241 		stats->q_ipackets[qidx] = rx_qstats.q_rx_packets;
242 	}
243 
244 	/* TX queue indices for the first VF */
245 	nicvf_tx_range(dev, nic, &tx_start, &tx_end);
246 
247 	/* Reading per TX ring stats */
248 	for (qidx = tx_start; qidx <= tx_end; qidx++) {
249 		if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
250 			break;
251 
252 		nicvf_hw_get_tx_qstats(nic, &tx_qstats, qidx);
253 		stats->q_obytes[qidx] = tx_qstats.q_tx_bytes;
254 		stats->q_opackets[qidx] = tx_qstats.q_tx_packets;
255 	}
256 
257 	for (i = 0; i < nic->sqs_count; i++) {
258 		struct nicvf *snic = nic->snicvf[i];
259 
260 		if (snic == NULL)
261 			break;
262 
263 		/* RX queue indices for a secondary VF */
264 		nicvf_rx_range(dev, snic, &rx_start, &rx_end);
265 
266 		/* Reading per RX ring stats */
267 		for (qidx = rx_start; qidx <= rx_end; qidx++) {
268 			if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
269 				break;
270 
271 			nicvf_hw_get_rx_qstats(snic, &rx_qstats,
272 					       qidx % MAX_RCV_QUEUES_PER_QS);
273 			stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes;
274 			stats->q_ipackets[qidx] = rx_qstats.q_rx_packets;
275 		}
276 
277 		/* TX queue indices for a secondary VF */
278 		nicvf_tx_range(dev, snic, &tx_start, &tx_end);
279 		/* Reading per TX ring stats */
280 		for (qidx = tx_start; qidx <= tx_end; qidx++) {
281 			if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
282 				break;
283 
284 			nicvf_hw_get_tx_qstats(snic, &tx_qstats,
285 					       qidx % MAX_SND_QUEUES_PER_QS);
286 			stats->q_obytes[qidx] = tx_qstats.q_tx_bytes;
287 			stats->q_opackets[qidx] = tx_qstats.q_tx_packets;
288 		}
289 	}
290 
291 	nicvf_hw_get_stats(nic, &port_stats);
292 	stats->ibytes = port_stats.rx_bytes;
293 	stats->ipackets = port_stats.rx_ucast_frames;
294 	stats->ipackets += port_stats.rx_bcast_frames;
295 	stats->ipackets += port_stats.rx_mcast_frames;
296 	stats->ierrors = port_stats.rx_l2_errors;
297 	stats->imissed = port_stats.rx_drop_red;
298 	stats->imissed += port_stats.rx_drop_overrun;
299 	stats->imissed += port_stats.rx_drop_bcast;
300 	stats->imissed += port_stats.rx_drop_mcast;
301 	stats->imissed += port_stats.rx_drop_l3_bcast;
302 	stats->imissed += port_stats.rx_drop_l3_mcast;
303 
304 	stats->obytes = port_stats.tx_bytes_ok;
305 	stats->opackets = port_stats.tx_ucast_frames_ok;
306 	stats->opackets += port_stats.tx_bcast_frames_ok;
307 	stats->opackets += port_stats.tx_mcast_frames_ok;
308 	stats->oerrors = port_stats.tx_drops;
309 
310 	return 0;
311 }
312 
313 static const uint32_t *
314 nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev)
315 {
316 	size_t copied;
317 	static uint32_t ptypes[32];
318 	struct nicvf *nic = nicvf_pmd_priv(dev);
319 	static const uint32_t ptypes_common[] = {
320 		RTE_PTYPE_L3_IPV4,
321 		RTE_PTYPE_L3_IPV4_EXT,
322 		RTE_PTYPE_L3_IPV6,
323 		RTE_PTYPE_L3_IPV6_EXT,
324 		RTE_PTYPE_L4_TCP,
325 		RTE_PTYPE_L4_UDP,
326 		RTE_PTYPE_L4_FRAG,
327 	};
328 	static const uint32_t ptypes_tunnel[] = {
329 		RTE_PTYPE_TUNNEL_GRE,
330 		RTE_PTYPE_TUNNEL_GENEVE,
331 		RTE_PTYPE_TUNNEL_VXLAN,
332 		RTE_PTYPE_TUNNEL_NVGRE,
333 	};
334 	static const uint32_t ptypes_end = RTE_PTYPE_UNKNOWN;
335 
336 	copied = sizeof(ptypes_common);
337 	memcpy(ptypes, ptypes_common, copied);
338 	if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
339 		memcpy((char *)ptypes + copied, ptypes_tunnel,
340 			sizeof(ptypes_tunnel));
341 		copied += sizeof(ptypes_tunnel);
342 	}
343 
344 	memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end));
345 
346 	/* All Ptypes are supported in all Rx functions. */
347 	return ptypes;
348 }
349 
350 static int
351 nicvf_dev_stats_reset(struct rte_eth_dev *dev)
352 {
353 	int i;
354 	uint16_t rxqs = 0, txqs = 0;
355 	struct nicvf *nic = nicvf_pmd_priv(dev);
356 	uint16_t rx_start, rx_end;
357 	uint16_t tx_start, tx_end;
358 	int ret;
359 
360 	/* Reset all primary nic counters */
361 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
362 	for (i = rx_start; i <= rx_end; i++)
363 		rxqs |= (0x3 << (i * 2));
364 
365 	nicvf_tx_range(dev, nic, &tx_start, &tx_end);
366 	for (i = tx_start; i <= tx_end; i++)
367 		txqs |= (0x3 << (i * 2));
368 
369 	ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, rxqs, txqs);
370 	if (ret != 0)
371 		return ret;
372 
373 	/* Reset secondary nic queue counters */
374 	for (i = 0; i < nic->sqs_count; i++) {
375 		struct nicvf *snic = nic->snicvf[i];
376 		if (snic == NULL)
377 			break;
378 
379 		nicvf_rx_range(dev, snic, &rx_start, &rx_end);
380 		for (i = rx_start; i <= rx_end; i++)
381 			rxqs |= (0x3 << ((i % MAX_CMP_QUEUES_PER_QS) * 2));
382 
383 		nicvf_tx_range(dev, snic, &tx_start, &tx_end);
384 		for (i = tx_start; i <= tx_end; i++)
385 			txqs |= (0x3 << ((i % MAX_SND_QUEUES_PER_QS) * 2));
386 
387 		ret = nicvf_mbox_reset_stat_counters(snic, 0, 0, rxqs, txqs);
388 		if (ret != 0)
389 			return ret;
390 	}
391 
392 	return 0;
393 }
394 
395 /* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */
396 static int
397 nicvf_dev_promisc_enable(struct rte_eth_dev *dev __rte_unused)
398 {
399 	return 0;
400 }
401 
402 static inline uint64_t
403 nicvf_rss_ethdev_to_nic(struct nicvf *nic, uint64_t ethdev_rss)
404 {
405 	uint64_t nic_rss = 0;
406 
407 	if (ethdev_rss & ETH_RSS_IPV4)
408 		nic_rss |= RSS_IP_ENA;
409 
410 	if (ethdev_rss & ETH_RSS_IPV6)
411 		nic_rss |= RSS_IP_ENA;
412 
413 	if (ethdev_rss & ETH_RSS_NONFRAG_IPV4_UDP)
414 		nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
415 
416 	if (ethdev_rss & ETH_RSS_NONFRAG_IPV4_TCP)
417 		nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
418 
419 	if (ethdev_rss & ETH_RSS_NONFRAG_IPV6_UDP)
420 		nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
421 
422 	if (ethdev_rss & ETH_RSS_NONFRAG_IPV6_TCP)
423 		nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
424 
425 	if (ethdev_rss & ETH_RSS_PORT)
426 		nic_rss |= RSS_L2_EXTENDED_HASH_ENA;
427 
428 	if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
429 		if (ethdev_rss & ETH_RSS_VXLAN)
430 			nic_rss |= RSS_TUN_VXLAN_ENA;
431 
432 		if (ethdev_rss & ETH_RSS_GENEVE)
433 			nic_rss |= RSS_TUN_GENEVE_ENA;
434 
435 		if (ethdev_rss & ETH_RSS_NVGRE)
436 			nic_rss |= RSS_TUN_NVGRE_ENA;
437 	}
438 
439 	return nic_rss;
440 }
441 
442 static inline uint64_t
443 nicvf_rss_nic_to_ethdev(struct nicvf *nic,  uint64_t nic_rss)
444 {
445 	uint64_t ethdev_rss = 0;
446 
447 	if (nic_rss & RSS_IP_ENA)
448 		ethdev_rss |= (ETH_RSS_IPV4 | ETH_RSS_IPV6);
449 
450 	if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_TCP_ENA))
451 		ethdev_rss |= (ETH_RSS_NONFRAG_IPV4_TCP |
452 				ETH_RSS_NONFRAG_IPV6_TCP);
453 
454 	if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_UDP_ENA))
455 		ethdev_rss |= (ETH_RSS_NONFRAG_IPV4_UDP |
456 				ETH_RSS_NONFRAG_IPV6_UDP);
457 
458 	if (nic_rss & RSS_L2_EXTENDED_HASH_ENA)
459 		ethdev_rss |= ETH_RSS_PORT;
460 
461 	if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
462 		if (nic_rss & RSS_TUN_VXLAN_ENA)
463 			ethdev_rss |= ETH_RSS_VXLAN;
464 
465 		if (nic_rss & RSS_TUN_GENEVE_ENA)
466 			ethdev_rss |= ETH_RSS_GENEVE;
467 
468 		if (nic_rss & RSS_TUN_NVGRE_ENA)
469 			ethdev_rss |= ETH_RSS_NVGRE;
470 	}
471 	return ethdev_rss;
472 }
473 
474 static int
475 nicvf_dev_reta_query(struct rte_eth_dev *dev,
476 		     struct rte_eth_rss_reta_entry64 *reta_conf,
477 		     uint16_t reta_size)
478 {
479 	struct nicvf *nic = nicvf_pmd_priv(dev);
480 	uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
481 	int ret, i, j;
482 
483 	if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
484 		PMD_DRV_LOG(ERR,
485 			    "The size of hash lookup table configured "
486 			    "(%u) doesn't match the number hardware can supported "
487 			    "(%u)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
488 		return -EINVAL;
489 	}
490 
491 	ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
492 	if (ret)
493 		return ret;
494 
495 	/* Copy RETA table */
496 	for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_RETA_GROUP_SIZE); i++) {
497 		for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
498 			if ((reta_conf[i].mask >> j) & 0x01)
499 				reta_conf[i].reta[j] = tbl[j];
500 	}
501 
502 	return 0;
503 }
504 
505 static int
506 nicvf_dev_reta_update(struct rte_eth_dev *dev,
507 		      struct rte_eth_rss_reta_entry64 *reta_conf,
508 		      uint16_t reta_size)
509 {
510 	struct nicvf *nic = nicvf_pmd_priv(dev);
511 	uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
512 	int ret, i, j;
513 
514 	if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
515 		PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
516 			"(%u) doesn't match the number hardware can supported "
517 			"(%u)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
518 		return -EINVAL;
519 	}
520 
521 	ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
522 	if (ret)
523 		return ret;
524 
525 	/* Copy RETA table */
526 	for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_RETA_GROUP_SIZE); i++) {
527 		for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
528 			if ((reta_conf[i].mask >> j) & 0x01)
529 				tbl[j] = reta_conf[i].reta[j];
530 	}
531 
532 	return nicvf_rss_reta_update(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
533 }
534 
535 static int
536 nicvf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
537 			    struct rte_eth_rss_conf *rss_conf)
538 {
539 	struct nicvf *nic = nicvf_pmd_priv(dev);
540 
541 	if (rss_conf->rss_key)
542 		nicvf_rss_get_key(nic, rss_conf->rss_key);
543 
544 	rss_conf->rss_key_len =  RSS_HASH_KEY_BYTE_SIZE;
545 	rss_conf->rss_hf = nicvf_rss_nic_to_ethdev(nic, nicvf_rss_get_cfg(nic));
546 	return 0;
547 }
548 
549 static int
550 nicvf_dev_rss_hash_update(struct rte_eth_dev *dev,
551 			  struct rte_eth_rss_conf *rss_conf)
552 {
553 	struct nicvf *nic = nicvf_pmd_priv(dev);
554 	uint64_t nic_rss;
555 
556 	if (rss_conf->rss_key &&
557 		rss_conf->rss_key_len != RSS_HASH_KEY_BYTE_SIZE) {
558 		PMD_DRV_LOG(ERR, "Hash key size mismatch %u",
559 			    rss_conf->rss_key_len);
560 		return -EINVAL;
561 	}
562 
563 	if (rss_conf->rss_key)
564 		nicvf_rss_set_key(nic, rss_conf->rss_key);
565 
566 	nic_rss = nicvf_rss_ethdev_to_nic(nic, rss_conf->rss_hf);
567 	nicvf_rss_set_cfg(nic, nic_rss);
568 	return 0;
569 }
570 
571 static int
572 nicvf_qset_cq_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
573 		    struct nicvf_rxq *rxq, uint16_t qidx, uint32_t desc_cnt)
574 {
575 	const struct rte_memzone *rz;
576 	uint32_t ring_size = CMP_QUEUE_SZ_MAX * sizeof(union cq_entry_t);
577 
578 	rz = rte_eth_dma_zone_reserve(dev, "cq_ring",
579 				      nicvf_netdev_qidx(nic, qidx), ring_size,
580 				      NICVF_CQ_BASE_ALIGN_BYTES, nic->node);
581 	if (rz == NULL) {
582 		PMD_INIT_LOG(ERR, "Failed to allocate mem for cq hw ring");
583 		return -ENOMEM;
584 	}
585 
586 	memset(rz->addr, 0, ring_size);
587 
588 	rxq->phys = rz->iova;
589 	rxq->desc = rz->addr;
590 	rxq->qlen_mask = desc_cnt - 1;
591 
592 	return 0;
593 }
594 
595 static int
596 nicvf_qset_sq_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
597 		    struct nicvf_txq *sq, uint16_t qidx, uint32_t desc_cnt)
598 {
599 	const struct rte_memzone *rz;
600 	uint32_t ring_size = SND_QUEUE_SZ_MAX * sizeof(union sq_entry_t);
601 
602 	rz = rte_eth_dma_zone_reserve(dev, "sq",
603 				      nicvf_netdev_qidx(nic, qidx), ring_size,
604 				      NICVF_SQ_BASE_ALIGN_BYTES, nic->node);
605 	if (rz == NULL) {
606 		PMD_INIT_LOG(ERR, "Failed allocate mem for sq hw ring");
607 		return -ENOMEM;
608 	}
609 
610 	memset(rz->addr, 0, ring_size);
611 
612 	sq->phys = rz->iova;
613 	sq->desc = rz->addr;
614 	sq->qlen_mask = desc_cnt - 1;
615 
616 	return 0;
617 }
618 
619 static int
620 nicvf_qset_rbdr_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
621 		      uint32_t desc_cnt, uint32_t buffsz)
622 {
623 	struct nicvf_rbdr *rbdr;
624 	const struct rte_memzone *rz;
625 	uint32_t ring_size;
626 
627 	assert(nic->rbdr == NULL);
628 	rbdr = rte_zmalloc_socket("rbdr", sizeof(struct nicvf_rbdr),
629 				  RTE_CACHE_LINE_SIZE, nic->node);
630 	if (rbdr == NULL) {
631 		PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr");
632 		return -ENOMEM;
633 	}
634 
635 	ring_size = sizeof(struct rbdr_entry_t) * RBDR_QUEUE_SZ_MAX;
636 	rz = rte_eth_dma_zone_reserve(dev, "rbdr",
637 				      nicvf_netdev_qidx(nic, 0), ring_size,
638 				      NICVF_RBDR_BASE_ALIGN_BYTES, nic->node);
639 	if (rz == NULL) {
640 		PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr desc ring");
641 		return -ENOMEM;
642 	}
643 
644 	memset(rz->addr, 0, ring_size);
645 
646 	rbdr->phys = rz->iova;
647 	rbdr->tail = 0;
648 	rbdr->next_tail = 0;
649 	rbdr->desc = rz->addr;
650 	rbdr->buffsz = buffsz;
651 	rbdr->qlen_mask = desc_cnt - 1;
652 	rbdr->rbdr_status =
653 		nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_STATUS0;
654 	rbdr->rbdr_door =
655 		nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_DOOR;
656 
657 	nic->rbdr = rbdr;
658 	return 0;
659 }
660 
661 static void
662 nicvf_rbdr_release_mbuf(struct rte_eth_dev *dev, struct nicvf *nic,
663 			nicvf_iova_addr_t phy)
664 {
665 	uint16_t qidx;
666 	void *obj;
667 	struct nicvf_rxq *rxq;
668 	uint16_t rx_start, rx_end;
669 
670 	/* Get queue ranges for this VF */
671 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
672 
673 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
674 		rxq = dev->data->rx_queues[qidx];
675 		if (rxq->precharge_cnt) {
676 			obj = (void *)nicvf_mbuff_phy2virt(phy,
677 							   rxq->mbuf_phys_off);
678 			rte_mempool_put(rxq->pool, obj);
679 			rxq->precharge_cnt--;
680 			break;
681 		}
682 	}
683 }
684 
685 static inline void
686 nicvf_rbdr_release_mbufs(struct rte_eth_dev *dev, struct nicvf *nic)
687 {
688 	uint32_t qlen_mask, head;
689 	struct rbdr_entry_t *entry;
690 	struct nicvf_rbdr *rbdr = nic->rbdr;
691 
692 	qlen_mask = rbdr->qlen_mask;
693 	head = rbdr->head;
694 	while (head != rbdr->tail) {
695 		entry = rbdr->desc + head;
696 		nicvf_rbdr_release_mbuf(dev, nic, entry->full_addr);
697 		head++;
698 		head = head & qlen_mask;
699 	}
700 }
701 
702 static inline void
703 nicvf_tx_queue_release_mbufs(struct nicvf_txq *txq)
704 {
705 	uint32_t head;
706 
707 	head = txq->head;
708 	while (head != txq->tail) {
709 		if (txq->txbuffs[head]) {
710 			rte_pktmbuf_free_seg(txq->txbuffs[head]);
711 			txq->txbuffs[head] = NULL;
712 		}
713 		head++;
714 		head = head & txq->qlen_mask;
715 	}
716 }
717 
718 static void
719 nicvf_tx_queue_reset(struct nicvf_txq *txq)
720 {
721 	uint32_t txq_desc_cnt = txq->qlen_mask + 1;
722 
723 	memset(txq->desc, 0, sizeof(union sq_entry_t) * txq_desc_cnt);
724 	memset(txq->txbuffs, 0, sizeof(struct rte_mbuf *) * txq_desc_cnt);
725 	txq->tail = 0;
726 	txq->head = 0;
727 	txq->xmit_bufs = 0;
728 }
729 
730 static inline int
731 nicvf_vf_start_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
732 			uint16_t qidx)
733 {
734 	struct nicvf_txq *txq;
735 	int ret;
736 
737 	assert(qidx < MAX_SND_QUEUES_PER_QS);
738 
739 	if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
740 		RTE_ETH_QUEUE_STATE_STARTED)
741 		return 0;
742 
743 	txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)];
744 	txq->pool = NULL;
745 	ret = nicvf_qset_sq_config(nic, qidx, txq);
746 	if (ret) {
747 		PMD_INIT_LOG(ERR, "Failed to configure sq VF%d %d %d",
748 			     nic->vf_id, qidx, ret);
749 		goto config_sq_error;
750 	}
751 
752 	dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
753 		RTE_ETH_QUEUE_STATE_STARTED;
754 	return ret;
755 
756 config_sq_error:
757 	nicvf_qset_sq_reclaim(nic, qidx);
758 	return ret;
759 }
760 
761 static inline int
762 nicvf_vf_stop_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
763 		       uint16_t qidx)
764 {
765 	struct nicvf_txq *txq;
766 	int ret;
767 
768 	assert(qidx < MAX_SND_QUEUES_PER_QS);
769 
770 	if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
771 		RTE_ETH_QUEUE_STATE_STOPPED)
772 		return 0;
773 
774 	ret = nicvf_qset_sq_reclaim(nic, qidx);
775 	if (ret)
776 		PMD_INIT_LOG(ERR, "Failed to reclaim sq VF%d %d %d",
777 			     nic->vf_id, qidx, ret);
778 
779 	txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)];
780 	nicvf_tx_queue_release_mbufs(txq);
781 	nicvf_tx_queue_reset(txq);
782 
783 	dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
784 		RTE_ETH_QUEUE_STATE_STOPPED;
785 	return ret;
786 }
787 
788 static inline int
789 nicvf_configure_cpi(struct rte_eth_dev *dev)
790 {
791 	struct nicvf *nic = nicvf_pmd_priv(dev);
792 	uint16_t qidx, qcnt;
793 	int ret;
794 
795 	/* Count started rx queues */
796 	for (qidx = qcnt = 0; qidx < dev->data->nb_rx_queues; qidx++)
797 		if (dev->data->rx_queue_state[qidx] ==
798 		    RTE_ETH_QUEUE_STATE_STARTED)
799 			qcnt++;
800 
801 	nic->cpi_alg = CPI_ALG_NONE;
802 	ret = nicvf_mbox_config_cpi(nic, qcnt);
803 	if (ret)
804 		PMD_INIT_LOG(ERR, "Failed to configure CPI %d", ret);
805 
806 	return ret;
807 }
808 
809 static inline int
810 nicvf_configure_rss(struct rte_eth_dev *dev)
811 {
812 	struct nicvf *nic = nicvf_pmd_priv(dev);
813 	uint64_t rsshf;
814 	int ret = -EINVAL;
815 
816 	rsshf = nicvf_rss_ethdev_to_nic(nic,
817 			dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf);
818 	PMD_DRV_LOG(INFO, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64,
819 		    dev->data->dev_conf.rxmode.mq_mode,
820 		    dev->data->nb_rx_queues,
821 		    dev->data->dev_conf.lpbk_mode, rsshf);
822 
823 	if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_NONE)
824 		ret = nicvf_rss_term(nic);
825 	else if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS)
826 		ret = nicvf_rss_config(nic, dev->data->nb_rx_queues, rsshf);
827 	if (ret)
828 		PMD_INIT_LOG(ERR, "Failed to configure RSS %d", ret);
829 
830 	return ret;
831 }
832 
833 static int
834 nicvf_configure_rss_reta(struct rte_eth_dev *dev)
835 {
836 	struct nicvf *nic = nicvf_pmd_priv(dev);
837 	unsigned int idx, qmap_size;
838 	uint8_t qmap[RTE_MAX_QUEUES_PER_PORT];
839 	uint8_t default_reta[NIC_MAX_RSS_IDR_TBL_SIZE];
840 
841 	if (nic->cpi_alg != CPI_ALG_NONE)
842 		return -EINVAL;
843 
844 	/* Prepare queue map */
845 	for (idx = 0, qmap_size = 0; idx < dev->data->nb_rx_queues; idx++) {
846 		if (dev->data->rx_queue_state[idx] ==
847 				RTE_ETH_QUEUE_STATE_STARTED)
848 			qmap[qmap_size++] = idx;
849 	}
850 
851 	/* Update default RSS RETA */
852 	for (idx = 0; idx < NIC_MAX_RSS_IDR_TBL_SIZE; idx++)
853 		default_reta[idx] = qmap[idx % qmap_size];
854 
855 	return nicvf_rss_reta_update(nic, default_reta,
856 				     NIC_MAX_RSS_IDR_TBL_SIZE);
857 }
858 
859 static void
860 nicvf_dev_tx_queue_release(void *sq)
861 {
862 	struct nicvf_txq *txq;
863 
864 	PMD_INIT_FUNC_TRACE();
865 
866 	txq = (struct nicvf_txq *)sq;
867 	if (txq) {
868 		if (txq->txbuffs != NULL) {
869 			nicvf_tx_queue_release_mbufs(txq);
870 			rte_free(txq->txbuffs);
871 			txq->txbuffs = NULL;
872 		}
873 		rte_free(txq);
874 	}
875 }
876 
877 static void
878 nicvf_set_tx_function(struct rte_eth_dev *dev)
879 {
880 	struct nicvf_txq *txq = NULL;
881 	size_t i;
882 	bool multiseg = false;
883 
884 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
885 		txq = dev->data->tx_queues[i];
886 		if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
887 			multiseg = true;
888 			break;
889 		}
890 	}
891 
892 	/* Use a simple Tx queue (no offloads, no multi segs) if possible */
893 	if (multiseg) {
894 		PMD_DRV_LOG(DEBUG, "Using multi-segment tx callback");
895 		dev->tx_pkt_burst = nicvf_xmit_pkts_multiseg;
896 	} else {
897 		PMD_DRV_LOG(DEBUG, "Using single-segment tx callback");
898 		dev->tx_pkt_burst = nicvf_xmit_pkts;
899 	}
900 
901 	if (!txq)
902 		return;
903 
904 	if (txq->pool_free == nicvf_single_pool_free_xmited_buffers)
905 		PMD_DRV_LOG(DEBUG, "Using single-mempool tx free method");
906 	else
907 		PMD_DRV_LOG(DEBUG, "Using multi-mempool tx free method");
908 }
909 
910 static void
911 nicvf_set_rx_function(struct rte_eth_dev *dev)
912 {
913 	struct nicvf *nic = nicvf_pmd_priv(dev);
914 
915 	const eth_rx_burst_t rx_burst_func[2][2][2] = {
916 	/* [NORMAL/SCATTER] [CKSUM/NO_CKSUM] [VLAN_STRIP/NO_VLAN_STRIP] */
917 		[0][0][0] = nicvf_recv_pkts_no_offload,
918 		[0][0][1] = nicvf_recv_pkts_vlan_strip,
919 		[0][1][0] = nicvf_recv_pkts_cksum,
920 		[0][1][1] = nicvf_recv_pkts_cksum_vlan_strip,
921 		[1][0][0] = nicvf_recv_pkts_multiseg_no_offload,
922 		[1][0][1] = nicvf_recv_pkts_multiseg_vlan_strip,
923 		[1][1][0] = nicvf_recv_pkts_multiseg_cksum,
924 		[1][1][1] = nicvf_recv_pkts_multiseg_cksum_vlan_strip,
925 	};
926 
927 	dev->rx_pkt_burst =
928 		rx_burst_func[dev->data->scattered_rx]
929 			[nic->offload_cksum][nic->vlan_strip];
930 }
931 
932 static int
933 nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
934 			 uint16_t nb_desc, unsigned int socket_id,
935 			 const struct rte_eth_txconf *tx_conf)
936 {
937 	uint16_t tx_free_thresh;
938 	bool is_single_pool;
939 	struct nicvf_txq *txq;
940 	struct nicvf *nic = nicvf_pmd_priv(dev);
941 	uint64_t offloads;
942 
943 	PMD_INIT_FUNC_TRACE();
944 
945 	if (qidx >= MAX_SND_QUEUES_PER_QS)
946 		nic = nic->snicvf[qidx / MAX_SND_QUEUES_PER_QS - 1];
947 
948 	qidx = qidx % MAX_SND_QUEUES_PER_QS;
949 
950 	/* Socket id check */
951 	if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
952 		PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
953 		socket_id, nic->node);
954 
955 	/* Tx deferred start is not supported */
956 	if (tx_conf->tx_deferred_start) {
957 		PMD_INIT_LOG(ERR, "Tx deferred start not supported");
958 		return -EINVAL;
959 	}
960 
961 	/* Roundup nb_desc to available qsize and validate max number of desc */
962 	nb_desc = nicvf_qsize_sq_roundup(nb_desc);
963 	if (nb_desc == 0) {
964 		PMD_INIT_LOG(ERR, "Value of nb_desc beyond available sq qsize");
965 		return -EINVAL;
966 	}
967 
968 	/* Validate tx_free_thresh */
969 	tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
970 				tx_conf->tx_free_thresh :
971 				NICVF_DEFAULT_TX_FREE_THRESH);
972 
973 	if (tx_free_thresh > (nb_desc) ||
974 		tx_free_thresh > NICVF_MAX_TX_FREE_THRESH) {
975 		PMD_INIT_LOG(ERR,
976 			"tx_free_thresh must be less than the number of TX "
977 			"descriptors. (tx_free_thresh=%u port=%d "
978 			"queue=%d)", (unsigned int)tx_free_thresh,
979 			(int)dev->data->port_id, (int)qidx);
980 		return -EINVAL;
981 	}
982 
983 	/* Free memory prior to re-allocation if needed. */
984 	if (dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) {
985 		PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
986 				nicvf_netdev_qidx(nic, qidx));
987 		nicvf_dev_tx_queue_release(
988 			dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)]);
989 		dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL;
990 	}
991 
992 	/* Allocating tx queue data structure */
993 	txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq),
994 					RTE_CACHE_LINE_SIZE, nic->node);
995 	if (txq == NULL) {
996 		PMD_INIT_LOG(ERR, "Failed to allocate txq=%d",
997 			     nicvf_netdev_qidx(nic, qidx));
998 		return -ENOMEM;
999 	}
1000 
1001 	txq->nic = nic;
1002 	txq->queue_id = qidx;
1003 	txq->tx_free_thresh = tx_free_thresh;
1004 	txq->sq_head = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_HEAD;
1005 	txq->sq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_DOOR;
1006 	offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
1007 	txq->offloads = offloads;
1008 
1009 	is_single_pool = !!(offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE);
1010 
1011 	/* Choose optimum free threshold value for multipool case */
1012 	if (!is_single_pool) {
1013 		txq->tx_free_thresh = (uint16_t)
1014 		(tx_conf->tx_free_thresh == NICVF_DEFAULT_TX_FREE_THRESH ?
1015 				NICVF_TX_FREE_MPOOL_THRESH :
1016 				tx_conf->tx_free_thresh);
1017 		txq->pool_free = nicvf_multi_pool_free_xmited_buffers;
1018 	} else {
1019 		txq->pool_free = nicvf_single_pool_free_xmited_buffers;
1020 	}
1021 
1022 	/* Allocate software ring */
1023 	txq->txbuffs = rte_zmalloc_socket("txq->txbuffs",
1024 				nb_desc * sizeof(struct rte_mbuf *),
1025 				RTE_CACHE_LINE_SIZE, nic->node);
1026 
1027 	if (txq->txbuffs == NULL) {
1028 		nicvf_dev_tx_queue_release(txq);
1029 		return -ENOMEM;
1030 	}
1031 
1032 	if (nicvf_qset_sq_alloc(dev, nic, txq, qidx, nb_desc)) {
1033 		PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx);
1034 		nicvf_dev_tx_queue_release(txq);
1035 		return -ENOMEM;
1036 	}
1037 
1038 	nicvf_tx_queue_reset(txq);
1039 
1040 	PMD_INIT_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p"
1041 			" phys=0x%" PRIx64 " offloads=0x%" PRIx64,
1042 			nicvf_netdev_qidx(nic, qidx), txq, nb_desc, txq->desc,
1043 			txq->phys, txq->offloads);
1044 
1045 	dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = txq;
1046 	dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1047 		RTE_ETH_QUEUE_STATE_STOPPED;
1048 	return 0;
1049 }
1050 
1051 static inline void
1052 nicvf_rx_queue_release_mbufs(struct rte_eth_dev *dev, struct nicvf_rxq *rxq)
1053 {
1054 	uint32_t rxq_cnt;
1055 	uint32_t nb_pkts, released_pkts = 0;
1056 	uint32_t refill_cnt = 0;
1057 	struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH];
1058 
1059 	if (dev->rx_pkt_burst == NULL)
1060 		return;
1061 
1062 	while ((rxq_cnt = nicvf_dev_rx_queue_count(dev,
1063 				nicvf_netdev_qidx(rxq->nic, rxq->queue_id)))) {
1064 		nb_pkts = dev->rx_pkt_burst(rxq, rx_pkts,
1065 					NICVF_MAX_RX_FREE_THRESH);
1066 		PMD_DRV_LOG(INFO, "nb_pkts=%d  rxq_cnt=%d", nb_pkts, rxq_cnt);
1067 		while (nb_pkts) {
1068 			rte_pktmbuf_free_seg(rx_pkts[--nb_pkts]);
1069 			released_pkts++;
1070 		}
1071 	}
1072 
1073 
1074 	refill_cnt += nicvf_dev_rbdr_refill(dev,
1075 			nicvf_netdev_qidx(rxq->nic, rxq->queue_id));
1076 
1077 	PMD_DRV_LOG(INFO, "free_cnt=%d  refill_cnt=%d",
1078 		    released_pkts, refill_cnt);
1079 }
1080 
1081 static void
1082 nicvf_rx_queue_reset(struct nicvf_rxq *rxq)
1083 {
1084 	rxq->head = 0;
1085 	rxq->available_space = 0;
1086 	rxq->recv_buffers = 0;
1087 }
1088 
1089 static inline int
1090 nicvf_vf_start_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
1091 			uint16_t qidx)
1092 {
1093 	struct nicvf_rxq *rxq;
1094 	int ret;
1095 
1096 	assert(qidx < MAX_RCV_QUEUES_PER_QS);
1097 
1098 	if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
1099 		RTE_ETH_QUEUE_STATE_STARTED)
1100 		return 0;
1101 
1102 	/* Update rbdr pointer to all rxq */
1103 	rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)];
1104 	rxq->shared_rbdr = nic->rbdr;
1105 
1106 	ret = nicvf_qset_rq_config(nic, qidx, rxq);
1107 	if (ret) {
1108 		PMD_INIT_LOG(ERR, "Failed to configure rq VF%d %d %d",
1109 			     nic->vf_id, qidx, ret);
1110 		goto config_rq_error;
1111 	}
1112 	ret = nicvf_qset_cq_config(nic, qidx, rxq);
1113 	if (ret) {
1114 		PMD_INIT_LOG(ERR, "Failed to configure cq VF%d %d %d",
1115 			     nic->vf_id, qidx, ret);
1116 		goto config_cq_error;
1117 	}
1118 
1119 	dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1120 		RTE_ETH_QUEUE_STATE_STARTED;
1121 	return 0;
1122 
1123 config_cq_error:
1124 	nicvf_qset_cq_reclaim(nic, qidx);
1125 config_rq_error:
1126 	nicvf_qset_rq_reclaim(nic, qidx);
1127 	return ret;
1128 }
1129 
1130 static inline int
1131 nicvf_vf_stop_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
1132 		       uint16_t qidx)
1133 {
1134 	struct nicvf_rxq *rxq;
1135 	int ret, other_error;
1136 
1137 	if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
1138 		RTE_ETH_QUEUE_STATE_STOPPED)
1139 		return 0;
1140 
1141 	ret = nicvf_qset_rq_reclaim(nic, qidx);
1142 	if (ret)
1143 		PMD_INIT_LOG(ERR, "Failed to reclaim rq VF%d %d %d",
1144 			     nic->vf_id, qidx, ret);
1145 
1146 	other_error = ret;
1147 	rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)];
1148 	nicvf_rx_queue_release_mbufs(dev, rxq);
1149 	nicvf_rx_queue_reset(rxq);
1150 
1151 	ret = nicvf_qset_cq_reclaim(nic, qidx);
1152 	if (ret)
1153 		PMD_INIT_LOG(ERR, "Failed to reclaim cq VF%d %d %d",
1154 			     nic->vf_id, qidx, ret);
1155 
1156 	other_error |= ret;
1157 	dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1158 		RTE_ETH_QUEUE_STATE_STOPPED;
1159 	return other_error;
1160 }
1161 
1162 static void
1163 nicvf_dev_rx_queue_release(void *rx_queue)
1164 {
1165 	PMD_INIT_FUNC_TRACE();
1166 
1167 	rte_free(rx_queue);
1168 }
1169 
1170 static int
1171 nicvf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1172 {
1173 	struct nicvf *nic = nicvf_pmd_priv(dev);
1174 	int ret;
1175 
1176 	if (qidx >= MAX_RCV_QUEUES_PER_QS)
1177 		nic = nic->snicvf[(qidx / MAX_RCV_QUEUES_PER_QS - 1)];
1178 
1179 	qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1180 
1181 	ret = nicvf_vf_start_rx_queue(dev, nic, qidx);
1182 	if (ret)
1183 		return ret;
1184 
1185 	ret = nicvf_configure_cpi(dev);
1186 	if (ret)
1187 		return ret;
1188 
1189 	return nicvf_configure_rss_reta(dev);
1190 }
1191 
1192 static int
1193 nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1194 {
1195 	int ret;
1196 	struct nicvf *nic = nicvf_pmd_priv(dev);
1197 
1198 	if (qidx >= MAX_SND_QUEUES_PER_QS)
1199 		nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1200 
1201 	qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1202 
1203 	ret = nicvf_vf_stop_rx_queue(dev, nic, qidx);
1204 	ret |= nicvf_configure_cpi(dev);
1205 	ret |= nicvf_configure_rss_reta(dev);
1206 	return ret;
1207 }
1208 
1209 static int
1210 nicvf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1211 {
1212 	struct nicvf *nic = nicvf_pmd_priv(dev);
1213 
1214 	if (qidx >= MAX_SND_QUEUES_PER_QS)
1215 		nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1216 
1217 	qidx = qidx % MAX_SND_QUEUES_PER_QS;
1218 
1219 	return nicvf_vf_start_tx_queue(dev, nic, qidx);
1220 }
1221 
1222 static int
1223 nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1224 {
1225 	struct nicvf *nic = nicvf_pmd_priv(dev);
1226 
1227 	if (qidx >= MAX_SND_QUEUES_PER_QS)
1228 		nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1229 
1230 	qidx = qidx % MAX_SND_QUEUES_PER_QS;
1231 
1232 	return nicvf_vf_stop_tx_queue(dev, nic, qidx);
1233 }
1234 
1235 static inline void
1236 nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq)
1237 {
1238 	uintptr_t p;
1239 	struct rte_mbuf mb_def;
1240 	struct nicvf *nic = rxq->nic;
1241 
1242 	RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8);
1243 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0);
1244 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) -
1245 				offsetof(struct rte_mbuf, data_off) != 2);
1246 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) -
1247 				offsetof(struct rte_mbuf, data_off) != 4);
1248 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) -
1249 				offsetof(struct rte_mbuf, data_off) != 6);
1250 	RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq, rxq_fastpath_data_end) -
1251 				offsetof(struct nicvf_rxq,
1252 					rxq_fastpath_data_start) > 128);
1253 	mb_def.nb_segs = 1;
1254 	mb_def.data_off = RTE_PKTMBUF_HEADROOM + (nic->skip_bytes);
1255 	mb_def.port = rxq->port_id;
1256 	rte_mbuf_refcnt_set(&mb_def, 1);
1257 
1258 	/* Prevent compiler reordering: rearm_data covers previous fields */
1259 	rte_compiler_barrier();
1260 	p = (uintptr_t)&mb_def.rearm_data;
1261 	rxq->mbuf_initializer.value = *(uint64_t *)p;
1262 }
1263 
1264 static int
1265 nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
1266 			 uint16_t nb_desc, unsigned int socket_id,
1267 			 const struct rte_eth_rxconf *rx_conf,
1268 			 struct rte_mempool *mp)
1269 {
1270 	uint16_t rx_free_thresh;
1271 	struct nicvf_rxq *rxq;
1272 	struct nicvf *nic = nicvf_pmd_priv(dev);
1273 	uint64_t offloads;
1274 	uint32_t buffsz;
1275 	struct rte_pktmbuf_pool_private *mbp_priv;
1276 
1277 	PMD_INIT_FUNC_TRACE();
1278 
1279 	/* First skip check */
1280 	mbp_priv = rte_mempool_get_priv(mp);
1281 	buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
1282 	if (buffsz < (uint32_t)(nic->skip_bytes)) {
1283 		PMD_INIT_LOG(ERR, "First skip is more than configured buffer size");
1284 		return -EINVAL;
1285 	}
1286 
1287 	if (qidx >= MAX_RCV_QUEUES_PER_QS)
1288 		nic = nic->snicvf[qidx / MAX_RCV_QUEUES_PER_QS - 1];
1289 
1290 	qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1291 
1292 	/* Socket id check */
1293 	if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
1294 		PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
1295 		socket_id, nic->node);
1296 
1297 	/* Mempool memory must be contiguous, so must be one memory segment*/
1298 	if (mp->nb_mem_chunks != 1) {
1299 		PMD_INIT_LOG(ERR, "Non-contiguous mempool, add more huge pages");
1300 		return -EINVAL;
1301 	}
1302 
1303 	/* Mempool memory must be physically contiguous */
1304 	if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG) {
1305 		PMD_INIT_LOG(ERR, "Mempool memory must be physically contiguous");
1306 		return -EINVAL;
1307 	}
1308 
1309 	/* Rx deferred start is not supported */
1310 	if (rx_conf->rx_deferred_start) {
1311 		PMD_INIT_LOG(ERR, "Rx deferred start not supported");
1312 		return -EINVAL;
1313 	}
1314 
1315 	/* Roundup nb_desc to available qsize and validate max number of desc */
1316 	nb_desc = nicvf_qsize_cq_roundup(nb_desc);
1317 	if (nb_desc == 0) {
1318 		PMD_INIT_LOG(ERR, "Value nb_desc beyond available hw cq qsize");
1319 		return -EINVAL;
1320 	}
1321 
1322 
1323 	/* Check rx_free_thresh upper bound */
1324 	rx_free_thresh = (uint16_t)((rx_conf->rx_free_thresh) ?
1325 				rx_conf->rx_free_thresh :
1326 				NICVF_DEFAULT_RX_FREE_THRESH);
1327 	if (rx_free_thresh > NICVF_MAX_RX_FREE_THRESH ||
1328 		rx_free_thresh >= nb_desc * .75) {
1329 		PMD_INIT_LOG(ERR, "rx_free_thresh greater than expected %d",
1330 				rx_free_thresh);
1331 		return -EINVAL;
1332 	}
1333 
1334 	/* Free memory prior to re-allocation if needed */
1335 	if (dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) {
1336 		PMD_RX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
1337 				nicvf_netdev_qidx(nic, qidx));
1338 		nicvf_dev_rx_queue_release(
1339 			dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)]);
1340 		dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL;
1341 	}
1342 
1343 	/* Allocate rxq memory */
1344 	rxq = rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq),
1345 					RTE_CACHE_LINE_SIZE, nic->node);
1346 	if (rxq == NULL) {
1347 		PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d",
1348 			     nicvf_netdev_qidx(nic, qidx));
1349 		return -ENOMEM;
1350 	}
1351 
1352 	rxq->nic = nic;
1353 	rxq->pool = mp;
1354 	rxq->queue_id = qidx;
1355 	rxq->port_id = dev->data->port_id;
1356 	rxq->rx_free_thresh = rx_free_thresh;
1357 	rxq->rx_drop_en = rx_conf->rx_drop_en;
1358 	rxq->cq_status = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_STATUS;
1359 	rxq->cq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_DOOR;
1360 	rxq->precharge_cnt = 0;
1361 
1362 	if (nicvf_hw_cap(nic) & NICVF_CAP_CQE_RX2)
1363 		rxq->rbptr_offset = NICVF_CQE_RX2_RBPTR_WORD;
1364 	else
1365 		rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD;
1366 
1367 	nicvf_rxq_mbuf_setup(rxq);
1368 
1369 	/* Alloc completion queue */
1370 	if (nicvf_qset_cq_alloc(dev, nic, rxq, rxq->queue_id, nb_desc)) {
1371 		PMD_INIT_LOG(ERR, "failed to allocate cq %u", rxq->queue_id);
1372 		nicvf_dev_rx_queue_release(rxq);
1373 		return -ENOMEM;
1374 	}
1375 
1376 	nicvf_rx_queue_reset(rxq);
1377 
1378 	offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
1379 	PMD_INIT_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d)"
1380 			" phy=0x%" PRIx64 " offloads=0x%" PRIx64,
1381 			nicvf_netdev_qidx(nic, qidx), rxq, mp->name, nb_desc,
1382 			rte_mempool_avail_count(mp), rxq->phys, offloads);
1383 
1384 	dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = rxq;
1385 	dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1386 		RTE_ETH_QUEUE_STATE_STOPPED;
1387 	return 0;
1388 }
1389 
1390 static int
1391 nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1392 {
1393 	struct nicvf *nic = nicvf_pmd_priv(dev);
1394 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1395 
1396 	PMD_INIT_FUNC_TRACE();
1397 
1398 	/* Autonegotiation may be disabled */
1399 	dev_info->speed_capa = ETH_LINK_SPEED_FIXED;
1400 	dev_info->speed_capa |= ETH_LINK_SPEED_10M | ETH_LINK_SPEED_100M |
1401 				 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
1402 	if (nicvf_hw_version(nic) != PCI_SUB_DEVICE_ID_CN81XX_NICVF)
1403 		dev_info->speed_capa |= ETH_LINK_SPEED_40G;
1404 
1405 	dev_info->min_rx_bufsize = RTE_ETHER_MIN_MTU;
1406 	dev_info->max_rx_pktlen = NIC_HW_MAX_MTU + RTE_ETHER_HDR_LEN;
1407 	dev_info->max_rx_queues =
1408 			(uint16_t)MAX_RCV_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
1409 	dev_info->max_tx_queues =
1410 			(uint16_t)MAX_SND_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
1411 	dev_info->max_mac_addrs = 1;
1412 	dev_info->max_vfs = pci_dev->max_vfs;
1413 
1414 	dev_info->rx_offload_capa = NICVF_RX_OFFLOAD_CAPA;
1415 	dev_info->tx_offload_capa = NICVF_TX_OFFLOAD_CAPA;
1416 	dev_info->rx_queue_offload_capa = NICVF_RX_OFFLOAD_CAPA;
1417 	dev_info->tx_queue_offload_capa = NICVF_TX_OFFLOAD_CAPA;
1418 
1419 	dev_info->reta_size = nic->rss_info.rss_size;
1420 	dev_info->hash_key_size = RSS_HASH_KEY_BYTE_SIZE;
1421 	dev_info->flow_type_rss_offloads = NICVF_RSS_OFFLOAD_PASS1;
1422 	if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING)
1423 		dev_info->flow_type_rss_offloads |= NICVF_RSS_OFFLOAD_TUNNEL;
1424 
1425 	dev_info->default_rxconf = (struct rte_eth_rxconf) {
1426 		.rx_free_thresh = NICVF_DEFAULT_RX_FREE_THRESH,
1427 		.rx_drop_en = 0,
1428 	};
1429 
1430 	dev_info->default_txconf = (struct rte_eth_txconf) {
1431 		.tx_free_thresh = NICVF_DEFAULT_TX_FREE_THRESH,
1432 		.offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE |
1433 			DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM   |
1434 			DEV_TX_OFFLOAD_UDP_CKSUM          |
1435 			DEV_TX_OFFLOAD_TCP_CKSUM,
1436 	};
1437 
1438 	return 0;
1439 }
1440 
1441 static nicvf_iova_addr_t
1442 rbdr_rte_mempool_get(void *dev, void *opaque)
1443 {
1444 	uint16_t qidx;
1445 	uintptr_t mbuf;
1446 	struct nicvf_rxq *rxq;
1447 	struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)dev;
1448 	struct nicvf *nic = (struct nicvf *)opaque;
1449 	uint16_t rx_start, rx_end;
1450 
1451 	/* Get queue ranges for this VF */
1452 	nicvf_rx_range(eth_dev, nic, &rx_start, &rx_end);
1453 
1454 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
1455 		rxq = eth_dev->data->rx_queues[qidx];
1456 		/* Maintain equal buffer count across all pools */
1457 		if (rxq->precharge_cnt >= rxq->qlen_mask)
1458 			continue;
1459 		rxq->precharge_cnt++;
1460 		mbuf = (uintptr_t)rte_pktmbuf_alloc(rxq->pool);
1461 		if (mbuf)
1462 			return nicvf_mbuff_virt2phy(mbuf, rxq->mbuf_phys_off);
1463 	}
1464 	return 0;
1465 }
1466 
1467 static int
1468 nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
1469 {
1470 	int ret;
1471 	uint16_t qidx, data_off;
1472 	uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs;
1473 	uint64_t mbuf_phys_off = 0;
1474 	struct nicvf_rxq *rxq;
1475 	struct rte_mbuf *mbuf;
1476 	uint16_t rx_start, rx_end;
1477 	uint16_t tx_start, tx_end;
1478 	int mask;
1479 
1480 	PMD_INIT_FUNC_TRACE();
1481 
1482 	/* Userspace process exited without proper shutdown in last run */
1483 	if (nicvf_qset_rbdr_active(nic, 0))
1484 		nicvf_vf_stop(dev, nic, false);
1485 
1486 	/* Get queue ranges for this VF */
1487 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
1488 
1489 	/*
1490 	 * Thunderx nicvf PMD can support more than one pool per port only when
1491 	 * 1) Data payload size is same across all the pools in given port
1492 	 * AND
1493 	 * 2) All mbuffs in the pools are from the same hugepage
1494 	 * AND
1495 	 * 3) Mbuff metadata size is same across all the pools in given port
1496 	 *
1497 	 * This is to support existing application that uses multiple pool/port.
1498 	 * But, the purpose of using multipool for QoS will not be addressed.
1499 	 *
1500 	 */
1501 
1502 	/* Validate mempool attributes */
1503 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
1504 		rxq = dev->data->rx_queues[qidx];
1505 		rxq->mbuf_phys_off = nicvf_mempool_phy_offset(rxq->pool);
1506 		mbuf = rte_pktmbuf_alloc(rxq->pool);
1507 		if (mbuf == NULL) {
1508 			PMD_INIT_LOG(ERR, "Failed allocate mbuf VF%d qid=%d "
1509 				     "pool=%s",
1510 				     nic->vf_id, qidx, rxq->pool->name);
1511 			return -ENOMEM;
1512 		}
1513 		data_off = nicvf_mbuff_meta_length(mbuf);
1514 		data_off += RTE_PKTMBUF_HEADROOM;
1515 		rte_pktmbuf_free(mbuf);
1516 
1517 		if (data_off % RTE_CACHE_LINE_SIZE) {
1518 			PMD_INIT_LOG(ERR, "%s: unaligned data_off=%d delta=%d",
1519 				rxq->pool->name, data_off,
1520 				data_off % RTE_CACHE_LINE_SIZE);
1521 			return -EINVAL;
1522 		}
1523 		rxq->mbuf_phys_off -= data_off;
1524 		rxq->mbuf_phys_off -= nic->skip_bytes;
1525 
1526 		if (mbuf_phys_off == 0)
1527 			mbuf_phys_off = rxq->mbuf_phys_off;
1528 		if (mbuf_phys_off != rxq->mbuf_phys_off) {
1529 			PMD_INIT_LOG(ERR, "pool params not same,%s VF%d %"
1530 				     PRIx64, rxq->pool->name, nic->vf_id,
1531 				     mbuf_phys_off);
1532 			return -EINVAL;
1533 		}
1534 	}
1535 
1536 	/* Check the level of buffers in the pool */
1537 	total_rxq_desc = 0;
1538 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
1539 		rxq = dev->data->rx_queues[qidx];
1540 		/* Count total numbers of rxq descs */
1541 		total_rxq_desc += rxq->qlen_mask + 1;
1542 		exp_buffs = RTE_MEMPOOL_CACHE_MAX_SIZE + rxq->rx_free_thresh;
1543 		exp_buffs *= dev->data->nb_rx_queues;
1544 		if (rte_mempool_avail_count(rxq->pool) < exp_buffs) {
1545 			PMD_INIT_LOG(ERR, "Buff shortage in pool=%s (%d/%d)",
1546 				     rxq->pool->name,
1547 				     rte_mempool_avail_count(rxq->pool),
1548 				     exp_buffs);
1549 			return -ENOENT;
1550 		}
1551 	}
1552 
1553 	/* Check RBDR desc overflow */
1554 	ret = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1555 	if (ret == 0) {
1556 		PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc "
1557 			     "VF%d", nic->vf_id);
1558 		return -ENOMEM;
1559 	}
1560 
1561 	/* Enable qset */
1562 	ret = nicvf_qset_config(nic);
1563 	if (ret) {
1564 		PMD_INIT_LOG(ERR, "Failed to enable qset %d VF%d", ret,
1565 			     nic->vf_id);
1566 		return ret;
1567 	}
1568 
1569 	/* Allocate RBDR and RBDR ring desc */
1570 	nb_rbdr_desc = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1571 	ret = nicvf_qset_rbdr_alloc(dev, nic, nb_rbdr_desc, rbdrsz);
1572 	if (ret) {
1573 		PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc "
1574 			     "VF%d", nic->vf_id);
1575 		goto qset_reclaim;
1576 	}
1577 
1578 	/* Enable and configure RBDR registers */
1579 	ret = nicvf_qset_rbdr_config(nic, 0);
1580 	if (ret) {
1581 		PMD_INIT_LOG(ERR, "Failed to configure rbdr %d VF%d", ret,
1582 			     nic->vf_id);
1583 		goto qset_rbdr_free;
1584 	}
1585 
1586 	/* Fill rte_mempool buffers in RBDR pool and precharge it */
1587 	ret = nicvf_qset_rbdr_precharge(dev, nic, 0, rbdr_rte_mempool_get,
1588 					total_rxq_desc);
1589 	if (ret) {
1590 		PMD_INIT_LOG(ERR, "Failed to fill rbdr %d VF%d", ret,
1591 			     nic->vf_id);
1592 		goto qset_rbdr_reclaim;
1593 	}
1594 
1595 	PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR VF%d",
1596 		     nic->rbdr->tail, nb_rbdr_desc, nic->vf_id);
1597 
1598 	/* Configure VLAN Strip */
1599 	mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
1600 		ETH_VLAN_EXTEND_MASK;
1601 	ret = nicvf_vlan_offload_config(dev, mask);
1602 
1603 	/* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data
1604 	 * to the 64bit memory address.
1605 	 * The alignment creates a hole in mbuf(between the end of headroom and
1606 	 * packet data start). The new revision of the HW provides an option to
1607 	 * disable the L3 alignment feature and make mbuf layout looks
1608 	 * more like other NICs. For better application compatibility, disabling
1609 	 * l3 alignment feature on the hardware revisions it supports
1610 	 */
1611 	nicvf_apad_config(nic, false);
1612 
1613 	/* Get queue ranges for this VF */
1614 	nicvf_tx_range(dev, nic, &tx_start, &tx_end);
1615 
1616 	/* Configure TX queues */
1617 	for (qidx = tx_start; qidx <= tx_end; qidx++) {
1618 		ret = nicvf_vf_start_tx_queue(dev, nic,
1619 			qidx % MAX_SND_QUEUES_PER_QS);
1620 		if (ret)
1621 			goto start_txq_error;
1622 	}
1623 
1624 	/* Configure RX queues */
1625 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
1626 		ret = nicvf_vf_start_rx_queue(dev, nic,
1627 			qidx % MAX_RCV_QUEUES_PER_QS);
1628 		if (ret)
1629 			goto start_rxq_error;
1630 	}
1631 
1632 	if (!nic->sqs_mode) {
1633 		/* Configure CPI algorithm */
1634 		ret = nicvf_configure_cpi(dev);
1635 		if (ret)
1636 			goto start_txq_error;
1637 
1638 		ret = nicvf_mbox_get_rss_size(nic);
1639 		if (ret) {
1640 			PMD_INIT_LOG(ERR, "Failed to get rss table size");
1641 			goto qset_rss_error;
1642 		}
1643 
1644 		/* Configure RSS */
1645 		ret = nicvf_configure_rss(dev);
1646 		if (ret)
1647 			goto qset_rss_error;
1648 	}
1649 
1650 	/* Done; Let PF make the BGX's RX and TX switches to ON position */
1651 	nicvf_mbox_cfg_done(nic);
1652 	return 0;
1653 
1654 qset_rss_error:
1655 	nicvf_rss_term(nic);
1656 start_rxq_error:
1657 	for (qidx = rx_start; qidx <= rx_end; qidx++)
1658 		nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS);
1659 start_txq_error:
1660 	for (qidx = tx_start; qidx <= tx_end; qidx++)
1661 		nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS);
1662 qset_rbdr_reclaim:
1663 	nicvf_qset_rbdr_reclaim(nic, 0);
1664 	nicvf_rbdr_release_mbufs(dev, nic);
1665 qset_rbdr_free:
1666 	if (nic->rbdr) {
1667 		rte_free(nic->rbdr);
1668 		nic->rbdr = NULL;
1669 	}
1670 qset_reclaim:
1671 	nicvf_qset_reclaim(nic);
1672 	return ret;
1673 }
1674 
1675 static int
1676 nicvf_dev_start(struct rte_eth_dev *dev)
1677 {
1678 	uint16_t qidx;
1679 	int ret;
1680 	size_t i;
1681 	struct nicvf *nic = nicvf_pmd_priv(dev);
1682 	struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
1683 	uint16_t mtu;
1684 	uint32_t buffsz = 0, rbdrsz = 0;
1685 	struct rte_pktmbuf_pool_private *mbp_priv;
1686 	struct nicvf_rxq *rxq;
1687 
1688 	PMD_INIT_FUNC_TRACE();
1689 
1690 	/* This function must be called for a primary device */
1691 	assert_primary(nic);
1692 
1693 	/* Validate RBDR buff size */
1694 	for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) {
1695 		rxq = dev->data->rx_queues[qidx];
1696 		mbp_priv = rte_mempool_get_priv(rxq->pool);
1697 		buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
1698 		if (buffsz % 128) {
1699 			PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128");
1700 			return -EINVAL;
1701 		}
1702 		if (rbdrsz == 0)
1703 			rbdrsz = buffsz;
1704 		if (rbdrsz != buffsz) {
1705 			PMD_INIT_LOG(ERR, "buffsz not same, qidx=%d (%d/%d)",
1706 				     qidx, rbdrsz, buffsz);
1707 			return -EINVAL;
1708 		}
1709 	}
1710 
1711 	/* Configure loopback */
1712 	ret = nicvf_loopback_config(nic, dev->data->dev_conf.lpbk_mode);
1713 	if (ret) {
1714 		PMD_INIT_LOG(ERR, "Failed to configure loopback %d", ret);
1715 		return ret;
1716 	}
1717 
1718 	/* Reset all statistics counters attached to this port */
1719 	ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF);
1720 	if (ret) {
1721 		PMD_INIT_LOG(ERR, "Failed to reset stat counters %d", ret);
1722 		return ret;
1723 	}
1724 
1725 	/* Setup scatter mode if needed by jumbo */
1726 	if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
1727 					    2 * VLAN_TAG_SIZE > buffsz)
1728 		dev->data->scattered_rx = 1;
1729 	if ((rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER) != 0)
1730 		dev->data->scattered_rx = 1;
1731 
1732 	/* Setup MTU based on max_rx_pkt_len or default */
1733 	mtu = dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME ?
1734 		dev->data->dev_conf.rxmode.max_rx_pkt_len
1735 			-  RTE_ETHER_HDR_LEN : RTE_ETHER_MTU;
1736 
1737 	if (nicvf_dev_set_mtu(dev, mtu)) {
1738 		PMD_INIT_LOG(ERR, "Failed to set default mtu size");
1739 		return -EBUSY;
1740 	}
1741 
1742 	ret = nicvf_vf_start(dev, nic, rbdrsz);
1743 	if (ret != 0)
1744 		return ret;
1745 
1746 	for (i = 0; i < nic->sqs_count; i++) {
1747 		assert(nic->snicvf[i]);
1748 
1749 		ret = nicvf_vf_start(dev, nic->snicvf[i], rbdrsz);
1750 		if (ret != 0)
1751 			return ret;
1752 	}
1753 
1754 	/* Configure callbacks based on offloads */
1755 	nicvf_set_tx_function(dev);
1756 	nicvf_set_rx_function(dev);
1757 
1758 	return 0;
1759 }
1760 
1761 static void
1762 nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup)
1763 {
1764 	size_t i;
1765 	int ret;
1766 	struct nicvf *nic = nicvf_pmd_priv(dev);
1767 
1768 	PMD_INIT_FUNC_TRACE();
1769 	dev->data->dev_started = 0;
1770 
1771 	/* Teardown secondary vf first */
1772 	for (i = 0; i < nic->sqs_count; i++) {
1773 		if (!nic->snicvf[i])
1774 			continue;
1775 
1776 		nicvf_vf_stop(dev, nic->snicvf[i], cleanup);
1777 	}
1778 
1779 	/* Stop the primary VF now */
1780 	nicvf_vf_stop(dev, nic, cleanup);
1781 
1782 	/* Disable loopback */
1783 	ret = nicvf_loopback_config(nic, 0);
1784 	if (ret)
1785 		PMD_INIT_LOG(ERR, "Failed to disable loopback %d", ret);
1786 
1787 	/* Reclaim CPI configuration */
1788 	ret = nicvf_mbox_config_cpi(nic, 0);
1789 	if (ret)
1790 		PMD_INIT_LOG(ERR, "Failed to reclaim CPI config %d", ret);
1791 }
1792 
1793 static int
1794 nicvf_dev_stop(struct rte_eth_dev *dev)
1795 {
1796 	PMD_INIT_FUNC_TRACE();
1797 
1798 	nicvf_dev_stop_cleanup(dev, false);
1799 
1800 	return 0;
1801 }
1802 
1803 static void
1804 nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic, bool cleanup)
1805 {
1806 	int ret;
1807 	uint16_t qidx;
1808 	uint16_t tx_start, tx_end;
1809 	uint16_t rx_start, rx_end;
1810 
1811 	PMD_INIT_FUNC_TRACE();
1812 
1813 	if (cleanup) {
1814 		/* Let PF make the BGX's RX and TX switches to OFF position */
1815 		nicvf_mbox_shutdown(nic);
1816 	}
1817 
1818 	/* Disable VLAN Strip */
1819 	nicvf_vlan_hw_strip(nic, 0);
1820 
1821 	/* Get queue ranges for this VF */
1822 	nicvf_tx_range(dev, nic, &tx_start, &tx_end);
1823 
1824 	for (qidx = tx_start; qidx <= tx_end; qidx++)
1825 		nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS);
1826 
1827 	/* Get queue ranges for this VF */
1828 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
1829 
1830 	/* Reclaim rq */
1831 	for (qidx = rx_start; qidx <= rx_end; qidx++)
1832 		nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS);
1833 
1834 	/* Reclaim RBDR */
1835 	ret = nicvf_qset_rbdr_reclaim(nic, 0);
1836 	if (ret)
1837 		PMD_INIT_LOG(ERR, "Failed to reclaim RBDR %d", ret);
1838 
1839 	/* Move all charged buffers in RBDR back to pool */
1840 	if (nic->rbdr != NULL)
1841 		nicvf_rbdr_release_mbufs(dev, nic);
1842 
1843 	/* Disable qset */
1844 	ret = nicvf_qset_reclaim(nic);
1845 	if (ret)
1846 		PMD_INIT_LOG(ERR, "Failed to disable qset %d", ret);
1847 
1848 	/* Disable all interrupts */
1849 	nicvf_disable_all_interrupts(nic);
1850 
1851 	/* Free RBDR SW structure */
1852 	if (nic->rbdr) {
1853 		rte_free(nic->rbdr);
1854 		nic->rbdr = NULL;
1855 	}
1856 }
1857 
1858 static int
1859 nicvf_dev_close(struct rte_eth_dev *dev)
1860 {
1861 	size_t i;
1862 	struct nicvf *nic = nicvf_pmd_priv(dev);
1863 
1864 	PMD_INIT_FUNC_TRACE();
1865 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1866 		return 0;
1867 
1868 	nicvf_dev_stop_cleanup(dev, true);
1869 	nicvf_periodic_alarm_stop(nicvf_interrupt, dev);
1870 
1871 	for (i = 0; i < nic->sqs_count; i++) {
1872 		if (!nic->snicvf[i])
1873 			continue;
1874 
1875 		nicvf_periodic_alarm_stop(nicvf_vf_interrupt, nic->snicvf[i]);
1876 	}
1877 
1878 	return 0;
1879 }
1880 
1881 static int
1882 nicvf_request_sqs(struct nicvf *nic)
1883 {
1884 	size_t i;
1885 
1886 	assert_primary(nic);
1887 	assert(nic->sqs_count > 0);
1888 	assert(nic->sqs_count <= MAX_SQS_PER_VF);
1889 
1890 	/* Set no of Rx/Tx queues in each of the SQsets */
1891 	for (i = 0; i < nic->sqs_count; i++) {
1892 		if (nicvf_svf_empty())
1893 			rte_panic("Cannot assign sufficient number of "
1894 				  "secondary queues to primary VF%" PRIu8 "\n",
1895 				  nic->vf_id);
1896 
1897 		nic->snicvf[i] = nicvf_svf_pop();
1898 		nic->snicvf[i]->sqs_id = i;
1899 	}
1900 
1901 	return nicvf_mbox_request_sqs(nic);
1902 }
1903 
1904 static int
1905 nicvf_dev_configure(struct rte_eth_dev *dev)
1906 {
1907 	struct rte_eth_dev_data *data = dev->data;
1908 	struct rte_eth_conf *conf = &data->dev_conf;
1909 	struct rte_eth_rxmode *rxmode = &conf->rxmode;
1910 	struct rte_eth_txmode *txmode = &conf->txmode;
1911 	struct nicvf *nic = nicvf_pmd_priv(dev);
1912 	uint8_t cqcount;
1913 
1914 	PMD_INIT_FUNC_TRACE();
1915 
1916 	if (rxmode->mq_mode & ETH_MQ_RX_RSS_FLAG)
1917 		rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH;
1918 
1919 	if (!rte_eal_has_hugepages()) {
1920 		PMD_INIT_LOG(INFO, "Huge page is not configured");
1921 		return -EINVAL;
1922 	}
1923 
1924 	if (txmode->mq_mode) {
1925 		PMD_INIT_LOG(INFO, "Tx mq_mode DCB or VMDq not supported");
1926 		return -EINVAL;
1927 	}
1928 
1929 	if (rxmode->mq_mode != ETH_MQ_RX_NONE &&
1930 		rxmode->mq_mode != ETH_MQ_RX_RSS) {
1931 		PMD_INIT_LOG(INFO, "Unsupported rx qmode %d", rxmode->mq_mode);
1932 		return -EINVAL;
1933 	}
1934 
1935 	if (rxmode->split_hdr_size) {
1936 		PMD_INIT_LOG(INFO, "Rxmode does not support split header");
1937 		return -EINVAL;
1938 	}
1939 
1940 	if (conf->link_speeds & ETH_LINK_SPEED_FIXED) {
1941 		PMD_INIT_LOG(INFO, "Setting link speed/duplex not supported");
1942 		return -EINVAL;
1943 	}
1944 
1945 	if (conf->dcb_capability_en) {
1946 		PMD_INIT_LOG(INFO, "DCB enable not supported");
1947 		return -EINVAL;
1948 	}
1949 
1950 	if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1951 		PMD_INIT_LOG(INFO, "Flow director not supported");
1952 		return -EINVAL;
1953 	}
1954 
1955 	assert_primary(nic);
1956 	NICVF_STATIC_ASSERT(MAX_RCV_QUEUES_PER_QS == MAX_SND_QUEUES_PER_QS);
1957 	cqcount = RTE_MAX(data->nb_tx_queues, data->nb_rx_queues);
1958 	if (cqcount > MAX_RCV_QUEUES_PER_QS) {
1959 		nic->sqs_count = RTE_ALIGN_CEIL(cqcount, MAX_RCV_QUEUES_PER_QS);
1960 		nic->sqs_count = (nic->sqs_count / MAX_RCV_QUEUES_PER_QS) - 1;
1961 	} else {
1962 		nic->sqs_count = 0;
1963 	}
1964 
1965 	assert(nic->sqs_count <= MAX_SQS_PER_VF);
1966 
1967 	if (nic->sqs_count > 0) {
1968 		if (nicvf_request_sqs(nic)) {
1969 			rte_panic("Cannot assign sufficient number of "
1970 				  "secondary queues to PORT%d VF%" PRIu8 "\n",
1971 				  dev->data->port_id, nic->vf_id);
1972 		}
1973 	}
1974 
1975 	if (rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM)
1976 		nic->offload_cksum = 1;
1977 
1978 	PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64,
1979 		dev->data->port_id, nicvf_hw_cap(nic));
1980 
1981 	return 0;
1982 }
1983 
1984 static int
1985 nicvf_dev_set_link_up(struct rte_eth_dev *dev)
1986 {
1987 	struct nicvf *nic = nicvf_pmd_priv(dev);
1988 	int rc, i;
1989 
1990 	rc = nicvf_mbox_set_link_up_down(nic, true);
1991 	if (rc)
1992 		goto done;
1993 
1994 	/* Start tx queues  */
1995 	for (i = 0; i < dev->data->nb_tx_queues; i++)
1996 		nicvf_dev_tx_queue_start(dev, i);
1997 
1998 done:
1999 	return rc;
2000 }
2001 
2002 static int
2003 nicvf_dev_set_link_down(struct rte_eth_dev *dev)
2004 {
2005 	struct nicvf *nic = nicvf_pmd_priv(dev);
2006 	int i;
2007 
2008 	/* Stop tx queues  */
2009 	for (i = 0; i < dev->data->nb_tx_queues; i++)
2010 		nicvf_dev_tx_queue_stop(dev, i);
2011 
2012 	return nicvf_mbox_set_link_up_down(nic, false);
2013 }
2014 
2015 /* Initialize and register driver with DPDK Application */
2016 static const struct eth_dev_ops nicvf_eth_dev_ops = {
2017 	.dev_configure            = nicvf_dev_configure,
2018 	.dev_start                = nicvf_dev_start,
2019 	.dev_stop                 = nicvf_dev_stop,
2020 	.link_update              = nicvf_dev_link_update,
2021 	.dev_close                = nicvf_dev_close,
2022 	.stats_get                = nicvf_dev_stats_get,
2023 	.stats_reset              = nicvf_dev_stats_reset,
2024 	.promiscuous_enable       = nicvf_dev_promisc_enable,
2025 	.dev_infos_get            = nicvf_dev_info_get,
2026 	.dev_supported_ptypes_get = nicvf_dev_supported_ptypes_get,
2027 	.mtu_set                  = nicvf_dev_set_mtu,
2028 	.vlan_offload_set         = nicvf_vlan_offload_set,
2029 	.reta_update              = nicvf_dev_reta_update,
2030 	.reta_query               = nicvf_dev_reta_query,
2031 	.rss_hash_update          = nicvf_dev_rss_hash_update,
2032 	.rss_hash_conf_get        = nicvf_dev_rss_hash_conf_get,
2033 	.rx_queue_start           = nicvf_dev_rx_queue_start,
2034 	.rx_queue_stop            = nicvf_dev_rx_queue_stop,
2035 	.tx_queue_start           = nicvf_dev_tx_queue_start,
2036 	.tx_queue_stop            = nicvf_dev_tx_queue_stop,
2037 	.rx_queue_setup           = nicvf_dev_rx_queue_setup,
2038 	.rx_queue_release         = nicvf_dev_rx_queue_release,
2039 	.tx_queue_setup           = nicvf_dev_tx_queue_setup,
2040 	.tx_queue_release         = nicvf_dev_tx_queue_release,
2041 	.dev_set_link_up          = nicvf_dev_set_link_up,
2042 	.dev_set_link_down        = nicvf_dev_set_link_down,
2043 	.get_reg                  = nicvf_dev_get_regs,
2044 };
2045 
2046 static int
2047 nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask)
2048 {
2049 	struct rte_eth_rxmode *rxmode;
2050 	struct nicvf *nic = nicvf_pmd_priv(dev);
2051 	rxmode = &dev->data->dev_conf.rxmode;
2052 	if (mask & ETH_VLAN_STRIP_MASK) {
2053 		if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
2054 			nicvf_vlan_hw_strip(nic, true);
2055 		else
2056 			nicvf_vlan_hw_strip(nic, false);
2057 	}
2058 
2059 	return 0;
2060 }
2061 
2062 static int
2063 nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2064 {
2065 	nicvf_vlan_offload_config(dev, mask);
2066 
2067 	return 0;
2068 }
2069 
2070 static inline int
2071 nicvf_set_first_skip(struct rte_eth_dev *dev)
2072 {
2073 	int bytes_to_skip = 0;
2074 	int ret = 0;
2075 	unsigned int i;
2076 	struct rte_kvargs *kvlist;
2077 	static const char *const skip[] = {
2078 		SKIP_DATA_BYTES,
2079 		NULL};
2080 	struct nicvf *nic = nicvf_pmd_priv(dev);
2081 
2082 	if (!dev->device->devargs) {
2083 		nicvf_first_skip_config(nic, 0);
2084 		return ret;
2085 	}
2086 
2087 	kvlist = rte_kvargs_parse(dev->device->devargs->args, skip);
2088 	if (!kvlist)
2089 		return -EINVAL;
2090 
2091 	if (kvlist->count == 0)
2092 		goto exit;
2093 
2094 	for (i = 0; i != kvlist->count; ++i) {
2095 		const struct rte_kvargs_pair *pair = &kvlist->pairs[i];
2096 
2097 		if (!strcmp(pair->key, SKIP_DATA_BYTES))
2098 			bytes_to_skip = atoi(pair->value);
2099 	}
2100 
2101 	/*128 bytes amounts to one cache line*/
2102 	if (bytes_to_skip >= 0 && bytes_to_skip < 128) {
2103 		if (!(bytes_to_skip % 8)) {
2104 			nicvf_first_skip_config(nic, (bytes_to_skip / 8));
2105 			nic->skip_bytes = bytes_to_skip;
2106 			goto kvlist_free;
2107 		} else {
2108 			PMD_INIT_LOG(ERR, "skip_data_bytes should be multiple of 8");
2109 			ret = -EINVAL;
2110 			goto exit;
2111 		}
2112 	} else {
2113 		PMD_INIT_LOG(ERR, "skip_data_bytes should be less than 128");
2114 		ret = -EINVAL;
2115 		goto exit;
2116 	}
2117 exit:
2118 	nicvf_first_skip_config(nic, 0);
2119 kvlist_free:
2120 	rte_kvargs_free(kvlist);
2121 	return ret;
2122 }
2123 static int
2124 nicvf_eth_dev_uninit(struct rte_eth_dev *dev)
2125 {
2126 	PMD_INIT_FUNC_TRACE();
2127 	nicvf_dev_close(dev);
2128 	return 0;
2129 }
2130 static int
2131 nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
2132 {
2133 	int ret;
2134 	struct rte_pci_device *pci_dev;
2135 	struct nicvf *nic = nicvf_pmd_priv(eth_dev);
2136 
2137 	PMD_INIT_FUNC_TRACE();
2138 
2139 	eth_dev->dev_ops = &nicvf_eth_dev_ops;
2140 	eth_dev->rx_queue_count = nicvf_dev_rx_queue_count;
2141 
2142 	/* For secondary processes, the primary has done all the work */
2143 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2144 		if (nic) {
2145 			/* Setup callbacks for secondary process */
2146 			nicvf_set_tx_function(eth_dev);
2147 			nicvf_set_rx_function(eth_dev);
2148 			return 0;
2149 		} else {
2150 			/* If nic == NULL than it is secondary function
2151 			 * so ethdev need to be released by caller */
2152 			return ENOTSUP;
2153 		}
2154 	}
2155 
2156 	pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2157 	rte_eth_copy_pci_info(eth_dev, pci_dev);
2158 	eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
2159 
2160 	nic->device_id = pci_dev->id.device_id;
2161 	nic->vendor_id = pci_dev->id.vendor_id;
2162 	nic->subsystem_device_id = pci_dev->id.subsystem_device_id;
2163 	nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
2164 
2165 	PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) %u:%u:%u:%u",
2166 			pci_dev->id.vendor_id, pci_dev->id.device_id,
2167 			pci_dev->addr.domain, pci_dev->addr.bus,
2168 			pci_dev->addr.devid, pci_dev->addr.function);
2169 
2170 	nic->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
2171 	if (!nic->reg_base) {
2172 		PMD_INIT_LOG(ERR, "Failed to map BAR0");
2173 		ret = -ENODEV;
2174 		goto fail;
2175 	}
2176 
2177 	nicvf_disable_all_interrupts(nic);
2178 
2179 	ret = nicvf_periodic_alarm_start(nicvf_interrupt, eth_dev);
2180 	if (ret) {
2181 		PMD_INIT_LOG(ERR, "Failed to start period alarm");
2182 		goto fail;
2183 	}
2184 
2185 	ret = nicvf_mbox_check_pf_ready(nic);
2186 	if (ret) {
2187 		PMD_INIT_LOG(ERR, "Failed to get ready message from PF");
2188 		goto alarm_fail;
2189 	} else {
2190 		PMD_INIT_LOG(INFO,
2191 			"node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
2192 			nic->node, nic->vf_id,
2193 			nic->tns_mode == NIC_TNS_MODE ? "tns" : "tns-bypass",
2194 			nic->sqs_mode ? "true" : "false",
2195 			nic->loopback_supported ? "true" : "false"
2196 			);
2197 	}
2198 
2199 	ret = nicvf_base_init(nic);
2200 	if (ret) {
2201 		PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init");
2202 		goto malloc_fail;
2203 	}
2204 
2205 	if (nic->sqs_mode) {
2206 		/* Push nic to stack of secondary vfs */
2207 		nicvf_svf_push(nic);
2208 
2209 		/* Steal nic pointer from the device for further reuse */
2210 		eth_dev->data->dev_private = NULL;
2211 
2212 		nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev);
2213 		ret = nicvf_periodic_alarm_start(nicvf_vf_interrupt, nic);
2214 		if (ret) {
2215 			PMD_INIT_LOG(ERR, "Failed to start period alarm");
2216 			goto fail;
2217 		}
2218 
2219 		/* Detach port by returning positive error number */
2220 		return ENOTSUP;
2221 	}
2222 
2223 	eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
2224 					RTE_ETHER_ADDR_LEN, 0);
2225 	if (eth_dev->data->mac_addrs == NULL) {
2226 		PMD_INIT_LOG(ERR, "Failed to allocate memory for mac addr");
2227 		ret = -ENOMEM;
2228 		goto alarm_fail;
2229 	}
2230 	if (rte_is_zero_ether_addr((struct rte_ether_addr *)nic->mac_addr))
2231 		rte_eth_random_addr(&nic->mac_addr[0]);
2232 
2233 	rte_ether_addr_copy((struct rte_ether_addr *)nic->mac_addr,
2234 			&eth_dev->data->mac_addrs[0]);
2235 
2236 	ret = nicvf_mbox_set_mac_addr(nic, nic->mac_addr);
2237 	if (ret) {
2238 		PMD_INIT_LOG(ERR, "Failed to set mac addr");
2239 		goto malloc_fail;
2240 	}
2241 
2242 	ret = nicvf_set_first_skip(eth_dev);
2243 	if (ret) {
2244 		PMD_INIT_LOG(ERR, "Failed to configure first skip");
2245 		goto malloc_fail;
2246 	}
2247 	PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
2248 		eth_dev->data->port_id, nic->vendor_id, nic->device_id,
2249 		nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],
2250 		nic->mac_addr[3], nic->mac_addr[4], nic->mac_addr[5]);
2251 
2252 	return 0;
2253 
2254 malloc_fail:
2255 	rte_free(eth_dev->data->mac_addrs);
2256 	eth_dev->data->mac_addrs = NULL;
2257 alarm_fail:
2258 	nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev);
2259 fail:
2260 	return ret;
2261 }
2262 
2263 static const struct rte_pci_id pci_id_nicvf_map[] = {
2264 	{
2265 		.class_id = RTE_CLASS_ANY_ID,
2266 		.vendor_id = PCI_VENDOR_ID_CAVIUM,
2267 		.device_id = PCI_DEVICE_ID_THUNDERX_CN88XX_PASS1_NICVF,
2268 		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2269 		.subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS1_NICVF,
2270 	},
2271 	{
2272 		.class_id = RTE_CLASS_ANY_ID,
2273 		.vendor_id = PCI_VENDOR_ID_CAVIUM,
2274 		.device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2275 		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2276 		.subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF,
2277 	},
2278 	{
2279 		.class_id = RTE_CLASS_ANY_ID,
2280 		.vendor_id = PCI_VENDOR_ID_CAVIUM,
2281 		.device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2282 		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2283 		.subsystem_device_id = PCI_SUB_DEVICE_ID_CN81XX_NICVF,
2284 	},
2285 	{
2286 		.class_id = RTE_CLASS_ANY_ID,
2287 		.vendor_id = PCI_VENDOR_ID_CAVIUM,
2288 		.device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2289 		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2290 		.subsystem_device_id = PCI_SUB_DEVICE_ID_CN83XX_NICVF,
2291 	},
2292 	{
2293 		.vendor_id = 0,
2294 	},
2295 };
2296 
2297 static int nicvf_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2298 	struct rte_pci_device *pci_dev)
2299 {
2300 	return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct nicvf),
2301 		nicvf_eth_dev_init);
2302 }
2303 
2304 static int nicvf_eth_pci_remove(struct rte_pci_device *pci_dev)
2305 {
2306 	return rte_eth_dev_pci_generic_remove(pci_dev, nicvf_eth_dev_uninit);
2307 }
2308 
2309 static struct rte_pci_driver rte_nicvf_pmd = {
2310 	.id_table = pci_id_nicvf_map,
2311 	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_KEEP_MAPPED_RES |
2312 			RTE_PCI_DRV_INTR_LSC,
2313 	.probe = nicvf_eth_pci_probe,
2314 	.remove = nicvf_eth_pci_remove,
2315 };
2316 
2317 RTE_PMD_REGISTER_PCI(net_thunderx, rte_nicvf_pmd);
2318 RTE_PMD_REGISTER_PCI_TABLE(net_thunderx, pci_id_nicvf_map);
2319 RTE_PMD_REGISTER_KMOD_DEP(net_thunderx, "* igb_uio | uio_pci_generic | vfio-pci");
2320 RTE_PMD_REGISTER_PARAM_STRING(net_thunderx, SKIP_DATA_BYTES "=<int>");
2321