xref: /dpdk/drivers/net/thunderx/nicvf_ethdev.c (revision f665790a5dbad7b645ff46f31d65e977324e7bfc)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2016 Cavium, Inc
3  */
4 
5 #include <assert.h>
6 #include <stdio.h>
7 #include <stdbool.h>
8 #include <errno.h>
9 #include <stdint.h>
10 #include <string.h>
11 #include <unistd.h>
12 #include <stdarg.h>
13 #include <inttypes.h>
14 #include <netinet/in.h>
15 #include <sys/queue.h>
16 
17 #include <rte_alarm.h>
18 #include <rte_branch_prediction.h>
19 #include <rte_byteorder.h>
20 #include <rte_common.h>
21 #include <rte_cycles.h>
22 #include <rte_debug.h>
23 #include <dev_driver.h>
24 #include <rte_eal.h>
25 #include <rte_ether.h>
26 #include <ethdev_driver.h>
27 #include <ethdev_pci.h>
28 #include <rte_interrupts.h>
29 #include <rte_log.h>
30 #include <rte_memory.h>
31 #include <rte_memzone.h>
32 #include <rte_malloc.h>
33 #include <rte_random.h>
34 #include <rte_pci.h>
35 #include <bus_pci_driver.h>
36 #include <rte_tailq.h>
37 #include <rte_devargs.h>
38 #include <rte_kvargs.h>
39 
40 #include "base/nicvf_plat.h"
41 
42 #include "nicvf_ethdev.h"
43 #include "nicvf_rxtx.h"
44 #include "nicvf_svf.h"
45 #include "nicvf_logs.h"
46 
47 static int nicvf_dev_stop(struct rte_eth_dev *dev);
48 static void nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup);
49 static void nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic,
50 			  bool cleanup);
51 static int nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask);
52 static int nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
53 
54 RTE_LOG_REGISTER_SUFFIX(nicvf_logtype_mbox, mbox, NOTICE);
55 RTE_LOG_REGISTER_SUFFIX(nicvf_logtype_init, init, NOTICE);
56 RTE_LOG_REGISTER_SUFFIX(nicvf_logtype_driver, driver, NOTICE);
57 
58 #define NICVF_QLM_MODE_SGMII  7
59 #define NICVF_QLM_MODE_XFI   12
60 
61 #define BCAST_ACCEPT      0x01
62 #define CAM_ACCEPT        (1 << 3)
63 #define BGX_MCAST_MODE(x) ((x) << 1)
64 
65 enum nicvf_link_speed {
66 	NICVF_LINK_SPEED_SGMII,
67 	NICVF_LINK_SPEED_XAUI,
68 	NICVF_LINK_SPEED_RXAUI,
69 	NICVF_LINK_SPEED_10G_R,
70 	NICVF_LINK_SPEED_40G_R,
71 	NICVF_LINK_SPEED_RESERVE1,
72 	NICVF_LINK_SPEED_QSGMII,
73 	NICVF_LINK_SPEED_RESERVE2,
74 	NICVF_LINK_SPEED_UNKNOWN = 255
75 };
76 
77 static inline uint32_t
78 nicvf_parse_link_speeds(uint32_t link_speeds)
79 {
80 	uint32_t link_speed = NICVF_LINK_SPEED_UNKNOWN;
81 
82 	if (link_speeds & RTE_ETH_LINK_SPEED_40G)
83 		link_speed = NICVF_LINK_SPEED_40G_R;
84 
85 	if (link_speeds & RTE_ETH_LINK_SPEED_10G) {
86 		link_speed  = NICVF_LINK_SPEED_XAUI;
87 		link_speed |= NICVF_LINK_SPEED_RXAUI;
88 		link_speed |= NICVF_LINK_SPEED_10G_R;
89 	}
90 
91 	if (link_speeds & RTE_ETH_LINK_SPEED_5G)
92 		link_speed = NICVF_LINK_SPEED_QSGMII;
93 
94 	if (link_speeds & RTE_ETH_LINK_SPEED_1G)
95 		link_speed = NICVF_LINK_SPEED_SGMII;
96 
97 	return link_speed;
98 }
99 
100 static inline uint8_t
101 nicvf_parse_eth_link_duplex(uint32_t link_speeds)
102 {
103 	if ((link_speeds & RTE_ETH_LINK_SPEED_10M_HD) ||
104 			(link_speeds & RTE_ETH_LINK_SPEED_100M_HD))
105 		return RTE_ETH_LINK_HALF_DUPLEX;
106 	else
107 		return RTE_ETH_LINK_FULL_DUPLEX;
108 }
109 
110 static int
111 nicvf_apply_link_speed(struct rte_eth_dev *dev)
112 {
113 	struct nicvf *nic = nicvf_pmd_priv(dev);
114 	struct rte_eth_conf *conf = &dev->data->dev_conf;
115 	struct change_link_mode cfg;
116 	if (conf->link_speeds == RTE_ETH_LINK_SPEED_AUTONEG)
117 		/* TODO: Handle this case */
118 		return 0;
119 
120 	cfg.speed = nicvf_parse_link_speeds(conf->link_speeds);
121 	cfg.autoneg = (conf->link_speeds & RTE_ETH_LINK_SPEED_FIXED) ? 1 : 0;
122 	cfg.duplex = nicvf_parse_eth_link_duplex(conf->link_speeds);
123 	cfg.qlm_mode = ((conf->link_speeds & RTE_ETH_LINK_SPEED_1G) ?
124 			NICVF_QLM_MODE_SGMII :
125 			(conf->link_speeds & RTE_ETH_LINK_SPEED_10G) ?
126 			NICVF_QLM_MODE_XFI : 0);
127 
128 	if (cfg.speed != NICVF_LINK_SPEED_UNKNOWN &&
129 	    (cfg.speed != nic->speed || cfg.duplex != nic->duplex)) {
130 		nic->speed = cfg.speed;
131 		nic->duplex = cfg.duplex;
132 		return nicvf_mbox_change_mode(nic, &cfg);
133 	} else {
134 		return 0;
135 	}
136 }
137 
138 static void
139 nicvf_link_status_update(struct nicvf *nic,
140 			 struct rte_eth_link *link)
141 {
142 	memset(link, 0, sizeof(*link));
143 
144 	link->link_status = nic->link_up ? RTE_ETH_LINK_UP : RTE_ETH_LINK_DOWN;
145 
146 	if (nic->duplex == NICVF_HALF_DUPLEX)
147 		link->link_duplex = RTE_ETH_LINK_HALF_DUPLEX;
148 	else if (nic->duplex == NICVF_FULL_DUPLEX)
149 		link->link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
150 	link->link_speed = nic->speed;
151 	link->link_autoneg = RTE_ETH_LINK_AUTONEG;
152 }
153 
154 /*Poll for link status change by sending NIC_MBOX_MSG_BGX_LINK_CHANGE msg
155  * periodically to PF.
156  */
157 static void
158 nicvf_interrupt(void *arg)
159 {
160 	struct rte_eth_dev *dev = arg;
161 	struct nicvf *nic = nicvf_pmd_priv(dev);
162 	struct rte_eth_link link;
163 
164 	rte_eth_linkstatus_get(dev, &link);
165 
166 	nicvf_mbox_link_change(nic);
167 	if (nic->link_up != link.link_status) {
168 		if (dev->data->dev_conf.intr_conf.lsc) {
169 			nicvf_link_status_update(nic, &link);
170 			rte_eth_linkstatus_set(dev, &link);
171 
172 			rte_eth_dev_callback_process(dev,
173 						     RTE_ETH_EVENT_INTR_LSC,
174 						     NULL);
175 		}
176 	}
177 
178 	rte_eal_alarm_set(NICVF_INTR_LINK_POLL_INTERVAL_MS * 1000,
179 				nicvf_interrupt, dev);
180 }
181 
182 static void
183 nicvf_vf_interrupt(void *arg)
184 {
185 	struct nicvf *nic = arg;
186 
187 	nicvf_reg_poll_interrupts(nic);
188 
189 	rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
190 				nicvf_vf_interrupt, nic);
191 }
192 
193 static int
194 nicvf_periodic_alarm_start(void (fn)(void *), void *arg)
195 {
196 	return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, fn, arg);
197 }
198 
199 static int
200 nicvf_periodic_alarm_stop(void (fn)(void *), void *arg)
201 {
202 	return rte_eal_alarm_cancel(fn, arg);
203 }
204 
205 /*
206  * Return 0 means link status changed, -1 means not changed
207  */
208 static int
209 nicvf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
210 {
211 #define CHECK_INTERVAL 100  /* 100ms */
212 #define MAX_CHECK_TIME 90   /* 9s (90 * 100ms) in total */
213 	struct rte_eth_link link;
214 	struct nicvf *nic = nicvf_pmd_priv(dev);
215 	int i;
216 
217 	PMD_INIT_FUNC_TRACE();
218 
219 	if (wait_to_complete) {
220 		/* rte_eth_link_get() might need to wait up to 9 seconds */
221 		for (i = 0; i < MAX_CHECK_TIME; i++) {
222 			nicvf_link_status_update(nic, &link);
223 			if (link.link_status == RTE_ETH_LINK_UP)
224 				break;
225 			rte_delay_ms(CHECK_INTERVAL);
226 		}
227 	} else {
228 		nicvf_link_status_update(nic, &link);
229 	}
230 
231 	return rte_eth_linkstatus_set(dev, &link);
232 }
233 
234 static int
235 nicvf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
236 {
237 	struct nicvf *nic = nicvf_pmd_priv(dev);
238 	uint32_t buffsz, frame_size = mtu + NIC_HW_L2_OVERHEAD;
239 	size_t i;
240 
241 	PMD_INIT_FUNC_TRACE();
242 
243 	buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
244 
245 	/*
246 	 * Refuse mtu that requires the support of scattered packets
247 	 * when this feature has not been enabled before.
248 	 */
249 	if (dev->data->dev_started && !dev->data->scattered_rx &&
250 		(frame_size + 2 * VLAN_TAG_SIZE > buffsz))
251 		return -EINVAL;
252 
253 	/* check <seg size> * <max_seg>  >= max_frame */
254 	if (dev->data->scattered_rx &&
255 		(frame_size + 2 * VLAN_TAG_SIZE > buffsz * NIC_HW_MAX_SEGS))
256 		return -EINVAL;
257 
258 	if (nicvf_mbox_update_hw_max_frs(nic, mtu))
259 		return -EINVAL;
260 
261 	nic->mtu = mtu;
262 
263 	for (i = 0; i < nic->sqs_count; i++)
264 		nic->snicvf[i]->mtu = mtu;
265 
266 	return 0;
267 }
268 
269 static int
270 nicvf_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs)
271 {
272 	uint64_t *data = regs->data;
273 	struct nicvf *nic = nicvf_pmd_priv(dev);
274 
275 	if (data == NULL) {
276 		regs->length = nicvf_reg_get_count();
277 		regs->width = THUNDERX_REG_BYTES;
278 		return 0;
279 	}
280 
281 	/* Support only full register dump */
282 	if ((regs->length == 0) ||
283 		(regs->length == (uint32_t)nicvf_reg_get_count())) {
284 		regs->version = nic->vendor_id << 16 | nic->device_id;
285 		nicvf_reg_dump(nic, data);
286 		return 0;
287 	}
288 	return -ENOTSUP;
289 }
290 
291 static int
292 nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
293 {
294 	uint16_t qidx;
295 	struct nicvf_hw_rx_qstats rx_qstats;
296 	struct nicvf_hw_tx_qstats tx_qstats;
297 	struct nicvf_hw_stats port_stats;
298 	struct nicvf *nic = nicvf_pmd_priv(dev);
299 	uint16_t rx_start, rx_end;
300 	uint16_t tx_start, tx_end;
301 	size_t i;
302 
303 	/* RX queue indices for the first VF */
304 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
305 
306 	/* Reading per RX ring stats */
307 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
308 		if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
309 			break;
310 
311 		nicvf_hw_get_rx_qstats(nic, &rx_qstats, qidx);
312 		stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes;
313 		stats->q_ipackets[qidx] = rx_qstats.q_rx_packets;
314 	}
315 
316 	/* TX queue indices for the first VF */
317 	nicvf_tx_range(dev, nic, &tx_start, &tx_end);
318 
319 	/* Reading per TX ring stats */
320 	for (qidx = tx_start; qidx <= tx_end; qidx++) {
321 		if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
322 			break;
323 
324 		nicvf_hw_get_tx_qstats(nic, &tx_qstats, qidx);
325 		stats->q_obytes[qidx] = tx_qstats.q_tx_bytes;
326 		stats->q_opackets[qidx] = tx_qstats.q_tx_packets;
327 	}
328 
329 	for (i = 0; i < nic->sqs_count; i++) {
330 		struct nicvf *snic = nic->snicvf[i];
331 
332 		if (snic == NULL)
333 			break;
334 
335 		/* RX queue indices for a secondary VF */
336 		nicvf_rx_range(dev, snic, &rx_start, &rx_end);
337 
338 		/* Reading per RX ring stats */
339 		for (qidx = rx_start; qidx <= rx_end; qidx++) {
340 			if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
341 				break;
342 
343 			nicvf_hw_get_rx_qstats(snic, &rx_qstats,
344 					       qidx % MAX_RCV_QUEUES_PER_QS);
345 			stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes;
346 			stats->q_ipackets[qidx] = rx_qstats.q_rx_packets;
347 		}
348 
349 		/* TX queue indices for a secondary VF */
350 		nicvf_tx_range(dev, snic, &tx_start, &tx_end);
351 		/* Reading per TX ring stats */
352 		for (qidx = tx_start; qidx <= tx_end; qidx++) {
353 			if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
354 				break;
355 
356 			nicvf_hw_get_tx_qstats(snic, &tx_qstats,
357 					       qidx % MAX_SND_QUEUES_PER_QS);
358 			stats->q_obytes[qidx] = tx_qstats.q_tx_bytes;
359 			stats->q_opackets[qidx] = tx_qstats.q_tx_packets;
360 		}
361 	}
362 
363 	nicvf_hw_get_stats(nic, &port_stats);
364 	stats->ibytes = port_stats.rx_bytes;
365 	stats->ipackets = port_stats.rx_ucast_frames;
366 	stats->ipackets += port_stats.rx_bcast_frames;
367 	stats->ipackets += port_stats.rx_mcast_frames;
368 	stats->ierrors = port_stats.rx_l2_errors;
369 	stats->imissed = port_stats.rx_drop_red;
370 	stats->imissed += port_stats.rx_drop_overrun;
371 	stats->imissed += port_stats.rx_drop_bcast;
372 	stats->imissed += port_stats.rx_drop_mcast;
373 	stats->imissed += port_stats.rx_drop_l3_bcast;
374 	stats->imissed += port_stats.rx_drop_l3_mcast;
375 
376 	stats->obytes = port_stats.tx_bytes_ok;
377 	stats->opackets = port_stats.tx_ucast_frames_ok;
378 	stats->opackets += port_stats.tx_bcast_frames_ok;
379 	stats->opackets += port_stats.tx_mcast_frames_ok;
380 	stats->oerrors = port_stats.tx_drops;
381 
382 	return 0;
383 }
384 
385 static const uint32_t *
386 nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev, size_t *no_of_elements)
387 {
388 	size_t copied;
389 	static uint32_t ptypes[32];
390 	struct nicvf *nic = nicvf_pmd_priv(dev);
391 	static const uint32_t ptypes_common[] = {
392 		RTE_PTYPE_L3_IPV4,
393 		RTE_PTYPE_L3_IPV4_EXT,
394 		RTE_PTYPE_L3_IPV6,
395 		RTE_PTYPE_L3_IPV6_EXT,
396 		RTE_PTYPE_L4_TCP,
397 		RTE_PTYPE_L4_UDP,
398 		RTE_PTYPE_L4_FRAG,
399 	};
400 	static const uint32_t ptypes_tunnel[] = {
401 		RTE_PTYPE_TUNNEL_GRE,
402 		RTE_PTYPE_TUNNEL_GENEVE,
403 		RTE_PTYPE_TUNNEL_VXLAN,
404 		RTE_PTYPE_TUNNEL_NVGRE,
405 	};
406 
407 	copied = sizeof(ptypes_common);
408 	memcpy(ptypes, ptypes_common, copied);
409 	if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
410 		memcpy((char *)ptypes + copied, ptypes_tunnel,
411 			sizeof(ptypes_tunnel));
412 		copied += sizeof(ptypes_tunnel);
413 	}
414 
415 
416 	/* All Ptypes are supported in all Rx functions. */
417 	*no_of_elements = copied / sizeof(ptypes[0]);
418 	return ptypes;
419 }
420 
421 static int
422 nicvf_dev_stats_reset(struct rte_eth_dev *dev)
423 {
424 	int i;
425 	uint16_t rxqs = 0, txqs = 0;
426 	struct nicvf *nic = nicvf_pmd_priv(dev);
427 	uint16_t rx_start, rx_end;
428 	uint16_t tx_start, tx_end;
429 	int ret;
430 
431 	/* Reset all primary nic counters */
432 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
433 	for (i = rx_start; i <= rx_end; i++)
434 		rxqs |= (0x3 << (i * 2));
435 
436 	nicvf_tx_range(dev, nic, &tx_start, &tx_end);
437 	for (i = tx_start; i <= tx_end; i++)
438 		txqs |= (0x3 << (i * 2));
439 
440 	ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, rxqs, txqs);
441 	if (ret != 0)
442 		return ret;
443 
444 	/* Reset secondary nic queue counters */
445 	for (i = 0; i < nic->sqs_count; i++) {
446 		struct nicvf *snic = nic->snicvf[i];
447 		if (snic == NULL)
448 			break;
449 
450 		nicvf_rx_range(dev, snic, &rx_start, &rx_end);
451 		for (i = rx_start; i <= rx_end; i++)
452 			rxqs |= (0x3 << ((i % MAX_CMP_QUEUES_PER_QS) * 2));
453 
454 		nicvf_tx_range(dev, snic, &tx_start, &tx_end);
455 		for (i = tx_start; i <= tx_end; i++)
456 			txqs |= (0x3 << ((i % MAX_SND_QUEUES_PER_QS) * 2));
457 
458 		ret = nicvf_mbox_reset_stat_counters(snic, 0, 0, rxqs, txqs);
459 		if (ret != 0)
460 			return ret;
461 	}
462 
463 	return 0;
464 }
465 
466 /* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */
467 static int
468 nicvf_dev_promisc_enable(struct rte_eth_dev *dev __rte_unused)
469 {
470 	return 0;
471 }
472 
473 static inline uint64_t
474 nicvf_rss_ethdev_to_nic(struct nicvf *nic, uint64_t ethdev_rss)
475 {
476 	uint64_t nic_rss = 0;
477 
478 	if (ethdev_rss & RTE_ETH_RSS_IPV4)
479 		nic_rss |= RSS_IP_ENA;
480 
481 	if (ethdev_rss & RTE_ETH_RSS_IPV6)
482 		nic_rss |= RSS_IP_ENA;
483 
484 	if (ethdev_rss & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
485 		nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
486 
487 	if (ethdev_rss & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
488 		nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
489 
490 	if (ethdev_rss & RTE_ETH_RSS_NONFRAG_IPV6_UDP)
491 		nic_rss |= (RSS_IP_ENA | RSS_UDP_ENA);
492 
493 	if (ethdev_rss & RTE_ETH_RSS_NONFRAG_IPV6_TCP)
494 		nic_rss |= (RSS_IP_ENA | RSS_TCP_ENA);
495 
496 	if (ethdev_rss & RTE_ETH_RSS_PORT)
497 		nic_rss |= RSS_L2_EXTENDED_HASH_ENA;
498 
499 	if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
500 		if (ethdev_rss & RTE_ETH_RSS_VXLAN)
501 			nic_rss |= RSS_TUN_VXLAN_ENA;
502 
503 		if (ethdev_rss & RTE_ETH_RSS_GENEVE)
504 			nic_rss |= RSS_TUN_GENEVE_ENA;
505 
506 		if (ethdev_rss & RTE_ETH_RSS_NVGRE)
507 			nic_rss |= RSS_TUN_NVGRE_ENA;
508 	}
509 
510 	return nic_rss;
511 }
512 
513 static inline uint64_t
514 nicvf_rss_nic_to_ethdev(struct nicvf *nic,  uint64_t nic_rss)
515 {
516 	uint64_t ethdev_rss = 0;
517 
518 	if (nic_rss & RSS_IP_ENA)
519 		ethdev_rss |= (RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_IPV6);
520 
521 	if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_TCP_ENA))
522 		ethdev_rss |= (RTE_ETH_RSS_NONFRAG_IPV4_TCP |
523 				RTE_ETH_RSS_NONFRAG_IPV6_TCP);
524 
525 	if ((nic_rss & RSS_IP_ENA) && (nic_rss & RSS_UDP_ENA))
526 		ethdev_rss |= (RTE_ETH_RSS_NONFRAG_IPV4_UDP |
527 				RTE_ETH_RSS_NONFRAG_IPV6_UDP);
528 
529 	if (nic_rss & RSS_L2_EXTENDED_HASH_ENA)
530 		ethdev_rss |= RTE_ETH_RSS_PORT;
531 
532 	if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) {
533 		if (nic_rss & RSS_TUN_VXLAN_ENA)
534 			ethdev_rss |= RTE_ETH_RSS_VXLAN;
535 
536 		if (nic_rss & RSS_TUN_GENEVE_ENA)
537 			ethdev_rss |= RTE_ETH_RSS_GENEVE;
538 
539 		if (nic_rss & RSS_TUN_NVGRE_ENA)
540 			ethdev_rss |= RTE_ETH_RSS_NVGRE;
541 	}
542 	return ethdev_rss;
543 }
544 
545 static int
546 nicvf_dev_reta_query(struct rte_eth_dev *dev,
547 		     struct rte_eth_rss_reta_entry64 *reta_conf,
548 		     uint16_t reta_size)
549 {
550 	struct nicvf *nic = nicvf_pmd_priv(dev);
551 	uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
552 	int ret, i, j;
553 
554 	if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
555 		PMD_DRV_LOG(ERR,
556 			    "The size of hash lookup table configured "
557 			    "(%u) doesn't match the number hardware can supported "
558 			    "(%u)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
559 		return -EINVAL;
560 	}
561 
562 	ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
563 	if (ret)
564 		return ret;
565 
566 	/* Copy RETA table */
567 	for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_ETH_RETA_GROUP_SIZE); i++) {
568 		for (j = 0; j < RTE_ETH_RETA_GROUP_SIZE; j++)
569 			if ((reta_conf[i].mask >> j) & 0x01)
570 				reta_conf[i].reta[j] = tbl[j];
571 	}
572 
573 	return 0;
574 }
575 
576 static int
577 nicvf_dev_reta_update(struct rte_eth_dev *dev,
578 		      struct rte_eth_rss_reta_entry64 *reta_conf,
579 		      uint16_t reta_size)
580 {
581 	struct nicvf *nic = nicvf_pmd_priv(dev);
582 	uint8_t tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
583 	int ret, i, j;
584 
585 	if (reta_size != NIC_MAX_RSS_IDR_TBL_SIZE) {
586 		PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
587 			"(%u) doesn't match the number hardware can supported "
588 			"(%u)", reta_size, NIC_MAX_RSS_IDR_TBL_SIZE);
589 		return -EINVAL;
590 	}
591 
592 	ret = nicvf_rss_reta_query(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
593 	if (ret)
594 		return ret;
595 
596 	/* Copy RETA table */
597 	for (i = 0; i < (NIC_MAX_RSS_IDR_TBL_SIZE / RTE_ETH_RETA_GROUP_SIZE); i++) {
598 		for (j = 0; j < RTE_ETH_RETA_GROUP_SIZE; j++)
599 			if ((reta_conf[i].mask >> j) & 0x01)
600 				tbl[j] = reta_conf[i].reta[j];
601 	}
602 
603 	return nicvf_rss_reta_update(nic, tbl, NIC_MAX_RSS_IDR_TBL_SIZE);
604 }
605 
606 static int
607 nicvf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
608 			    struct rte_eth_rss_conf *rss_conf)
609 {
610 	struct nicvf *nic = nicvf_pmd_priv(dev);
611 
612 	if (rss_conf->rss_key)
613 		nicvf_rss_get_key(nic, rss_conf->rss_key);
614 
615 	rss_conf->rss_key_len =  RSS_HASH_KEY_BYTE_SIZE;
616 	rss_conf->rss_hf = nicvf_rss_nic_to_ethdev(nic, nicvf_rss_get_cfg(nic));
617 	return 0;
618 }
619 
620 static int
621 nicvf_dev_rss_hash_update(struct rte_eth_dev *dev,
622 			  struct rte_eth_rss_conf *rss_conf)
623 {
624 	struct nicvf *nic = nicvf_pmd_priv(dev);
625 	uint64_t nic_rss;
626 
627 	if (rss_conf->rss_key &&
628 		rss_conf->rss_key_len != RSS_HASH_KEY_BYTE_SIZE) {
629 		PMD_DRV_LOG(ERR, "Hash key size mismatch %u",
630 			    rss_conf->rss_key_len);
631 		return -EINVAL;
632 	}
633 
634 	if (rss_conf->rss_key)
635 		nicvf_rss_set_key(nic, rss_conf->rss_key);
636 
637 	nic_rss = nicvf_rss_ethdev_to_nic(nic, rss_conf->rss_hf);
638 	nicvf_rss_set_cfg(nic, nic_rss);
639 	return 0;
640 }
641 
642 static int
643 nicvf_qset_cq_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
644 		    struct nicvf_rxq *rxq, uint16_t qidx, uint32_t desc_cnt)
645 {
646 	const struct rte_memzone *rz;
647 	uint32_t ring_size = CMP_QUEUE_SZ_MAX * sizeof(union cq_entry_t);
648 
649 	rz = rte_eth_dma_zone_reserve(dev, "cq_ring",
650 				      nicvf_netdev_qidx(nic, qidx), ring_size,
651 				      NICVF_CQ_BASE_ALIGN_BYTES, nic->node);
652 	if (rz == NULL) {
653 		PMD_INIT_LOG(ERR, "Failed to allocate mem for cq hw ring");
654 		return -ENOMEM;
655 	}
656 
657 	memset(rz->addr, 0, ring_size);
658 
659 	rxq->phys = rz->iova;
660 	rxq->desc = rz->addr;
661 	rxq->qlen_mask = desc_cnt - 1;
662 
663 	return 0;
664 }
665 
666 static int
667 nicvf_qset_sq_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
668 		    struct nicvf_txq *sq, uint16_t qidx, uint32_t desc_cnt)
669 {
670 	const struct rte_memzone *rz;
671 	uint32_t ring_size = SND_QUEUE_SZ_MAX * sizeof(union sq_entry_t);
672 
673 	rz = rte_eth_dma_zone_reserve(dev, "sq",
674 				      nicvf_netdev_qidx(nic, qidx), ring_size,
675 				      NICVF_SQ_BASE_ALIGN_BYTES, nic->node);
676 	if (rz == NULL) {
677 		PMD_INIT_LOG(ERR, "Failed allocate mem for sq hw ring");
678 		return -ENOMEM;
679 	}
680 
681 	memset(rz->addr, 0, ring_size);
682 
683 	sq->phys = rz->iova;
684 	sq->desc = rz->addr;
685 	sq->qlen_mask = desc_cnt - 1;
686 
687 	return 0;
688 }
689 
690 static int
691 nicvf_qset_rbdr_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
692 		      uint32_t desc_cnt, uint32_t buffsz)
693 {
694 	struct nicvf_rbdr *rbdr;
695 	const struct rte_memzone *rz;
696 	uint32_t ring_size;
697 
698 	assert(nic->rbdr == NULL);
699 	rbdr = rte_zmalloc_socket("rbdr", sizeof(struct nicvf_rbdr),
700 				  RTE_CACHE_LINE_SIZE, nic->node);
701 	if (rbdr == NULL) {
702 		PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr");
703 		return -ENOMEM;
704 	}
705 
706 	ring_size = sizeof(struct rbdr_entry_t) * RBDR_QUEUE_SZ_MAX;
707 	rz = rte_eth_dma_zone_reserve(dev, "rbdr",
708 				      nicvf_netdev_qidx(nic, 0), ring_size,
709 				      NICVF_RBDR_BASE_ALIGN_BYTES, nic->node);
710 	if (rz == NULL) {
711 		PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr desc ring");
712 		rte_free(rbdr);
713 		return -ENOMEM;
714 	}
715 
716 	memset(rz->addr, 0, ring_size);
717 
718 	rbdr->phys = rz->iova;
719 	rbdr->tail = 0;
720 	rbdr->next_tail = 0;
721 	rbdr->desc = rz->addr;
722 	rbdr->buffsz = buffsz;
723 	rbdr->qlen_mask = desc_cnt - 1;
724 	rbdr->rbdr_status =
725 		nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_STATUS0;
726 	rbdr->rbdr_door =
727 		nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_DOOR;
728 
729 	nic->rbdr = rbdr;
730 	return 0;
731 }
732 
733 static void
734 nicvf_rbdr_release_mbuf(struct rte_eth_dev *dev, struct nicvf *nic,
735 			nicvf_iova_addr_t phy)
736 {
737 	uint16_t qidx;
738 	void *obj;
739 	struct nicvf_rxq *rxq;
740 	uint16_t rx_start, rx_end;
741 
742 	/* Get queue ranges for this VF */
743 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
744 
745 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
746 		rxq = dev->data->rx_queues[qidx];
747 		if (rxq->precharge_cnt) {
748 			obj = (void *)nicvf_mbuff_phy2virt(phy,
749 							   rxq->mbuf_phys_off);
750 			rte_mempool_put(rxq->pool, obj);
751 			rxq->precharge_cnt--;
752 			break;
753 		}
754 	}
755 }
756 
757 static inline void
758 nicvf_rbdr_release_mbufs(struct rte_eth_dev *dev, struct nicvf *nic)
759 {
760 	uint32_t qlen_mask, head;
761 	struct rbdr_entry_t *entry;
762 	struct nicvf_rbdr *rbdr = nic->rbdr;
763 
764 	qlen_mask = rbdr->qlen_mask;
765 	head = rbdr->head;
766 	while (head != rbdr->tail) {
767 		entry = rbdr->desc + head;
768 		nicvf_rbdr_release_mbuf(dev, nic, entry->full_addr);
769 		head++;
770 		head = head & qlen_mask;
771 	}
772 }
773 
774 static inline void
775 nicvf_tx_queue_release_mbufs(struct nicvf_txq *txq)
776 {
777 	uint32_t head;
778 
779 	head = txq->head;
780 	while (head != txq->tail) {
781 		if (txq->txbuffs[head]) {
782 			rte_pktmbuf_free_seg(txq->txbuffs[head]);
783 			txq->txbuffs[head] = NULL;
784 		}
785 		head++;
786 		head = head & txq->qlen_mask;
787 	}
788 }
789 
790 static void
791 nicvf_tx_queue_reset(struct nicvf_txq *txq)
792 {
793 	uint32_t txq_desc_cnt = txq->qlen_mask + 1;
794 
795 	memset(txq->desc, 0, sizeof(union sq_entry_t) * txq_desc_cnt);
796 	memset(txq->txbuffs, 0, sizeof(struct rte_mbuf *) * txq_desc_cnt);
797 	txq->tail = 0;
798 	txq->head = 0;
799 	txq->xmit_bufs = 0;
800 }
801 
802 static inline int
803 nicvf_vf_start_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
804 			uint16_t qidx)
805 {
806 	struct nicvf_txq *txq;
807 	int ret;
808 
809 	assert(qidx < MAX_SND_QUEUES_PER_QS);
810 
811 	if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
812 		RTE_ETH_QUEUE_STATE_STARTED)
813 		return 0;
814 
815 	txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)];
816 	txq->pool = NULL;
817 	ret = nicvf_qset_sq_config(nic, qidx, txq);
818 	if (ret) {
819 		PMD_INIT_LOG(ERR, "Failed to configure sq VF%d %d %d",
820 			     nic->vf_id, qidx, ret);
821 		goto config_sq_error;
822 	}
823 
824 	dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
825 		RTE_ETH_QUEUE_STATE_STARTED;
826 	return ret;
827 
828 config_sq_error:
829 	nicvf_qset_sq_reclaim(nic, qidx);
830 	return ret;
831 }
832 
833 static inline int
834 nicvf_vf_stop_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
835 		       uint16_t qidx)
836 {
837 	struct nicvf_txq *txq;
838 	int ret;
839 
840 	assert(qidx < MAX_SND_QUEUES_PER_QS);
841 
842 	if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
843 		RTE_ETH_QUEUE_STATE_STOPPED)
844 		return 0;
845 
846 	ret = nicvf_qset_sq_reclaim(nic, qidx);
847 	if (ret)
848 		PMD_INIT_LOG(ERR, "Failed to reclaim sq VF%d %d %d",
849 			     nic->vf_id, qidx, ret);
850 
851 	txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)];
852 	nicvf_tx_queue_release_mbufs(txq);
853 	nicvf_tx_queue_reset(txq);
854 
855 	dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
856 		RTE_ETH_QUEUE_STATE_STOPPED;
857 	return ret;
858 }
859 
860 static inline int
861 nicvf_configure_cpi(struct rte_eth_dev *dev)
862 {
863 	struct nicvf *nic = nicvf_pmd_priv(dev);
864 	uint16_t qidx, qcnt;
865 	int ret;
866 
867 	/* Count started rx queues */
868 	for (qidx = qcnt = 0; qidx < dev->data->nb_rx_queues; qidx++)
869 		if (dev->data->rx_queue_state[qidx] ==
870 		    RTE_ETH_QUEUE_STATE_STARTED)
871 			qcnt++;
872 
873 	nic->cpi_alg = CPI_ALG_NONE;
874 	ret = nicvf_mbox_config_cpi(nic, qcnt);
875 	if (ret)
876 		PMD_INIT_LOG(ERR, "Failed to configure CPI %d", ret);
877 
878 	return ret;
879 }
880 
881 static inline int
882 nicvf_configure_rss(struct rte_eth_dev *dev)
883 {
884 	struct nicvf *nic = nicvf_pmd_priv(dev);
885 	uint64_t rsshf;
886 	int ret = -EINVAL;
887 
888 	rsshf = nicvf_rss_ethdev_to_nic(nic,
889 			dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf);
890 	PMD_DRV_LOG(INFO, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64,
891 		    dev->data->dev_conf.rxmode.mq_mode,
892 		    dev->data->nb_rx_queues,
893 		    dev->data->dev_conf.lpbk_mode, rsshf);
894 
895 	if (dev->data->dev_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_NONE)
896 		ret = nicvf_rss_term(nic);
897 	else if (dev->data->dev_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS)
898 		ret = nicvf_rss_config(nic, dev->data->nb_rx_queues, rsshf);
899 	if (ret)
900 		PMD_INIT_LOG(ERR, "Failed to configure RSS %d", ret);
901 
902 	return ret;
903 }
904 
905 static int
906 nicvf_configure_rss_reta(struct rte_eth_dev *dev)
907 {
908 	struct nicvf *nic = nicvf_pmd_priv(dev);
909 	unsigned int idx, qmap_size;
910 	uint8_t qmap[RTE_MAX_QUEUES_PER_PORT];
911 	uint8_t default_reta[NIC_MAX_RSS_IDR_TBL_SIZE];
912 
913 	if (nic->cpi_alg != CPI_ALG_NONE)
914 		return -EINVAL;
915 
916 	/* Prepare queue map */
917 	for (idx = 0, qmap_size = 0; idx < dev->data->nb_rx_queues; idx++) {
918 		if (dev->data->rx_queue_state[idx] ==
919 				RTE_ETH_QUEUE_STATE_STARTED)
920 			qmap[qmap_size++] = idx;
921 	}
922 
923 	/* Update default RSS RETA */
924 	for (idx = 0; idx < NIC_MAX_RSS_IDR_TBL_SIZE; idx++)
925 		default_reta[idx] = qmap[idx % qmap_size];
926 
927 	return nicvf_rss_reta_update(nic, default_reta,
928 				     NIC_MAX_RSS_IDR_TBL_SIZE);
929 }
930 
931 static void
932 nicvf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
933 {
934 	struct nicvf_txq *txq = dev->data->tx_queues[qid];
935 
936 	PMD_INIT_FUNC_TRACE();
937 
938 	if (txq) {
939 		if (txq->txbuffs != NULL) {
940 			nicvf_tx_queue_release_mbufs(txq);
941 			rte_free(txq->txbuffs);
942 			txq->txbuffs = NULL;
943 		}
944 		rte_free(txq);
945 		dev->data->tx_queues[qid] = NULL;
946 	}
947 }
948 
949 static void
950 nicvf_set_tx_function(struct rte_eth_dev *dev)
951 {
952 	struct nicvf_txq *txq = NULL;
953 	size_t i;
954 	bool multiseg = false;
955 
956 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
957 		txq = dev->data->tx_queues[i];
958 		if (txq->offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS) {
959 			multiseg = true;
960 			break;
961 		}
962 	}
963 
964 	/* Use a simple Tx queue (no offloads, no multi segs) if possible */
965 	if (multiseg) {
966 		PMD_DRV_LOG(DEBUG, "Using multi-segment tx callback");
967 		dev->tx_pkt_burst = nicvf_xmit_pkts_multiseg;
968 	} else {
969 		PMD_DRV_LOG(DEBUG, "Using single-segment tx callback");
970 		dev->tx_pkt_burst = nicvf_xmit_pkts;
971 	}
972 
973 	if (!txq)
974 		return;
975 
976 	if (txq->pool_free == nicvf_single_pool_free_xmited_buffers)
977 		PMD_DRV_LOG(DEBUG, "Using single-mempool tx free method");
978 	else
979 		PMD_DRV_LOG(DEBUG, "Using multi-mempool tx free method");
980 }
981 
982 static void
983 nicvf_set_rx_function(struct rte_eth_dev *dev)
984 {
985 	struct nicvf *nic = nicvf_pmd_priv(dev);
986 
987 	const eth_rx_burst_t rx_burst_func[2][2][2] = {
988 	/* [NORMAL/SCATTER] [CKSUM/NO_CKSUM] [VLAN_STRIP/NO_VLAN_STRIP] */
989 		[0][0][0] = nicvf_recv_pkts_no_offload,
990 		[0][0][1] = nicvf_recv_pkts_vlan_strip,
991 		[0][1][0] = nicvf_recv_pkts_cksum,
992 		[0][1][1] = nicvf_recv_pkts_cksum_vlan_strip,
993 		[1][0][0] = nicvf_recv_pkts_multiseg_no_offload,
994 		[1][0][1] = nicvf_recv_pkts_multiseg_vlan_strip,
995 		[1][1][0] = nicvf_recv_pkts_multiseg_cksum,
996 		[1][1][1] = nicvf_recv_pkts_multiseg_cksum_vlan_strip,
997 	};
998 
999 	dev->rx_pkt_burst =
1000 		rx_burst_func[dev->data->scattered_rx]
1001 			[nic->offload_cksum][nic->vlan_strip];
1002 }
1003 
1004 static int
1005 nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
1006 			 uint16_t nb_desc, unsigned int socket_id,
1007 			 const struct rte_eth_txconf *tx_conf)
1008 {
1009 	uint16_t tx_free_thresh;
1010 	bool is_single_pool;
1011 	struct nicvf_txq *txq;
1012 	struct nicvf *nic = nicvf_pmd_priv(dev);
1013 	uint64_t offloads;
1014 
1015 	PMD_INIT_FUNC_TRACE();
1016 
1017 	if (qidx >= MAX_SND_QUEUES_PER_QS)
1018 		nic = nic->snicvf[qidx / MAX_SND_QUEUES_PER_QS - 1];
1019 
1020 	qidx = qidx % MAX_SND_QUEUES_PER_QS;
1021 
1022 	/* Socket id check */
1023 	if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
1024 		PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
1025 		socket_id, nic->node);
1026 
1027 	/* Tx deferred start is not supported */
1028 	if (tx_conf->tx_deferred_start) {
1029 		PMD_INIT_LOG(ERR, "Tx deferred start not supported");
1030 		return -EINVAL;
1031 	}
1032 
1033 	/* Roundup nb_desc to available qsize and validate max number of desc */
1034 	nb_desc = nicvf_qsize_sq_roundup(nb_desc);
1035 	if (nb_desc == 0) {
1036 		PMD_INIT_LOG(ERR, "Value of nb_desc beyond available sq qsize");
1037 		return -EINVAL;
1038 	}
1039 
1040 	/* Validate tx_free_thresh */
1041 	tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
1042 				tx_conf->tx_free_thresh :
1043 				NICVF_DEFAULT_TX_FREE_THRESH);
1044 
1045 	if (tx_free_thresh > (nb_desc) ||
1046 		tx_free_thresh > NICVF_MAX_TX_FREE_THRESH) {
1047 		PMD_INIT_LOG(ERR,
1048 			"tx_free_thresh must be less than the number of TX "
1049 			"descriptors. (tx_free_thresh=%u port=%d "
1050 			"queue=%d)", (unsigned int)tx_free_thresh,
1051 			(int)dev->data->port_id, (int)qidx);
1052 		return -EINVAL;
1053 	}
1054 
1055 	/* Free memory prior to re-allocation if needed. */
1056 	if (dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) {
1057 		PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
1058 				nicvf_netdev_qidx(nic, qidx));
1059 		nicvf_dev_tx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1060 		dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL;
1061 	}
1062 
1063 	/* Allocating tx queue data structure */
1064 	txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq),
1065 					RTE_CACHE_LINE_SIZE, nic->node);
1066 	if (txq == NULL) {
1067 		PMD_INIT_LOG(ERR, "Failed to allocate txq=%d",
1068 			     nicvf_netdev_qidx(nic, qidx));
1069 		return -ENOMEM;
1070 	}
1071 
1072 	txq->nic = nic;
1073 	txq->queue_id = qidx;
1074 	txq->tx_free_thresh = tx_free_thresh;
1075 	txq->sq_head = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_HEAD;
1076 	txq->sq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_DOOR;
1077 	offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
1078 	txq->offloads = offloads;
1079 
1080 	is_single_pool = !!(offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE);
1081 
1082 	/* Choose optimum free threshold value for multipool case */
1083 	if (!is_single_pool) {
1084 		txq->tx_free_thresh = (uint16_t)
1085 		(tx_conf->tx_free_thresh == NICVF_DEFAULT_TX_FREE_THRESH ?
1086 				NICVF_TX_FREE_MPOOL_THRESH :
1087 				tx_conf->tx_free_thresh);
1088 		txq->pool_free = nicvf_multi_pool_free_xmited_buffers;
1089 	} else {
1090 		txq->pool_free = nicvf_single_pool_free_xmited_buffers;
1091 	}
1092 
1093 	dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = txq;
1094 
1095 	/* Allocate software ring */
1096 	txq->txbuffs = rte_zmalloc_socket("txq->txbuffs",
1097 				nb_desc * sizeof(struct rte_mbuf *),
1098 				RTE_CACHE_LINE_SIZE, nic->node);
1099 
1100 	if (txq->txbuffs == NULL) {
1101 		nicvf_dev_tx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1102 		return -ENOMEM;
1103 	}
1104 
1105 	if (nicvf_qset_sq_alloc(dev, nic, txq, qidx, nb_desc)) {
1106 		PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx);
1107 		nicvf_dev_tx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1108 		return -ENOMEM;
1109 	}
1110 
1111 	nicvf_tx_queue_reset(txq);
1112 
1113 	PMD_INIT_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p"
1114 			" phys=0x%" PRIx64 " offloads=0x%" PRIx64,
1115 			nicvf_netdev_qidx(nic, qidx), txq, nb_desc, txq->desc,
1116 			txq->phys, txq->offloads);
1117 
1118 	dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1119 		RTE_ETH_QUEUE_STATE_STOPPED;
1120 	return 0;
1121 }
1122 
1123 static inline void
1124 nicvf_rx_queue_release_mbufs(struct rte_eth_dev *dev, struct nicvf_rxq *rxq)
1125 {
1126 	uint32_t rxq_cnt;
1127 	uint32_t nb_pkts, released_pkts = 0;
1128 	uint32_t refill_cnt = 0;
1129 	struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH];
1130 
1131 	if (dev->rx_pkt_burst == NULL)
1132 		return;
1133 
1134 	while ((rxq_cnt = nicvf_dev_rx_queue_count(rxq))) {
1135 		nb_pkts = dev->rx_pkt_burst(rxq, rx_pkts,
1136 					NICVF_MAX_RX_FREE_THRESH);
1137 		PMD_DRV_LOG(INFO, "nb_pkts=%d  rxq_cnt=%d", nb_pkts, rxq_cnt);
1138 		while (nb_pkts) {
1139 			rte_pktmbuf_free_seg(rx_pkts[--nb_pkts]);
1140 			released_pkts++;
1141 		}
1142 	}
1143 
1144 
1145 	refill_cnt += nicvf_dev_rbdr_refill(dev,
1146 			nicvf_netdev_qidx(rxq->nic, rxq->queue_id));
1147 
1148 	PMD_DRV_LOG(INFO, "free_cnt=%d  refill_cnt=%d",
1149 		    released_pkts, refill_cnt);
1150 }
1151 
1152 static void
1153 nicvf_rx_queue_reset(struct nicvf_rxq *rxq)
1154 {
1155 	rxq->head = 0;
1156 	rxq->available_space = 0;
1157 	rxq->recv_buffers = 0;
1158 }
1159 
1160 static inline int
1161 nicvf_vf_start_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
1162 			uint16_t qidx)
1163 {
1164 	struct nicvf_rxq *rxq;
1165 	int ret;
1166 
1167 	assert(qidx < MAX_RCV_QUEUES_PER_QS);
1168 
1169 	if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
1170 		RTE_ETH_QUEUE_STATE_STARTED)
1171 		return 0;
1172 
1173 	/* Update rbdr pointer to all rxq */
1174 	rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)];
1175 	rxq->shared_rbdr = nic->rbdr;
1176 
1177 	ret = nicvf_qset_rq_config(nic, qidx, rxq);
1178 	if (ret) {
1179 		PMD_INIT_LOG(ERR, "Failed to configure rq VF%d %d %d",
1180 			     nic->vf_id, qidx, ret);
1181 		goto config_rq_error;
1182 	}
1183 	ret = nicvf_qset_cq_config(nic, qidx, rxq);
1184 	if (ret) {
1185 		PMD_INIT_LOG(ERR, "Failed to configure cq VF%d %d %d",
1186 			     nic->vf_id, qidx, ret);
1187 		goto config_cq_error;
1188 	}
1189 
1190 	dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1191 		RTE_ETH_QUEUE_STATE_STARTED;
1192 	return 0;
1193 
1194 config_cq_error:
1195 	nicvf_qset_cq_reclaim(nic, qidx);
1196 config_rq_error:
1197 	nicvf_qset_rq_reclaim(nic, qidx);
1198 	return ret;
1199 }
1200 
1201 static inline int
1202 nicvf_vf_stop_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
1203 		       uint16_t qidx)
1204 {
1205 	struct nicvf_rxq *rxq;
1206 	int ret, other_error;
1207 
1208 	if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
1209 		RTE_ETH_QUEUE_STATE_STOPPED)
1210 		return 0;
1211 
1212 	ret = nicvf_qset_rq_reclaim(nic, qidx);
1213 	if (ret)
1214 		PMD_INIT_LOG(ERR, "Failed to reclaim rq VF%d %d %d",
1215 			     nic->vf_id, qidx, ret);
1216 
1217 	other_error = ret;
1218 	rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)];
1219 	nicvf_rx_queue_release_mbufs(dev, rxq);
1220 	nicvf_rx_queue_reset(rxq);
1221 
1222 	ret = nicvf_qset_cq_reclaim(nic, qidx);
1223 	if (ret)
1224 		PMD_INIT_LOG(ERR, "Failed to reclaim cq VF%d %d %d",
1225 			     nic->vf_id, qidx, ret);
1226 
1227 	other_error |= ret;
1228 	dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1229 		RTE_ETH_QUEUE_STATE_STOPPED;
1230 	return other_error;
1231 }
1232 
1233 static void
1234 nicvf_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1235 {
1236 	PMD_INIT_FUNC_TRACE();
1237 
1238 	rte_free(dev->data->rx_queues[qid]);
1239 }
1240 
1241 static int
1242 nicvf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1243 {
1244 	struct nicvf *nic = nicvf_pmd_priv(dev);
1245 	int ret;
1246 
1247 	if (qidx >= MAX_RCV_QUEUES_PER_QS)
1248 		nic = nic->snicvf[(qidx / MAX_RCV_QUEUES_PER_QS - 1)];
1249 
1250 	qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1251 
1252 	ret = nicvf_vf_start_rx_queue(dev, nic, qidx);
1253 	if (ret)
1254 		return ret;
1255 
1256 	ret = nicvf_configure_cpi(dev);
1257 	if (ret)
1258 		return ret;
1259 
1260 	return nicvf_configure_rss_reta(dev);
1261 }
1262 
1263 static int
1264 nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1265 {
1266 	int ret;
1267 	struct nicvf *nic = nicvf_pmd_priv(dev);
1268 
1269 	if (qidx >= MAX_SND_QUEUES_PER_QS)
1270 		nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1271 
1272 	qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1273 
1274 	ret = nicvf_vf_stop_rx_queue(dev, nic, qidx);
1275 	ret |= nicvf_configure_cpi(dev);
1276 	ret |= nicvf_configure_rss_reta(dev);
1277 	return ret;
1278 }
1279 
1280 static int
1281 nicvf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
1282 {
1283 	struct nicvf *nic = nicvf_pmd_priv(dev);
1284 
1285 	if (qidx >= MAX_SND_QUEUES_PER_QS)
1286 		nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1287 
1288 	qidx = qidx % MAX_SND_QUEUES_PER_QS;
1289 
1290 	return nicvf_vf_start_tx_queue(dev, nic, qidx);
1291 }
1292 
1293 static int
1294 nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
1295 {
1296 	struct nicvf *nic = nicvf_pmd_priv(dev);
1297 
1298 	if (qidx >= MAX_SND_QUEUES_PER_QS)
1299 		nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
1300 
1301 	qidx = qidx % MAX_SND_QUEUES_PER_QS;
1302 
1303 	return nicvf_vf_stop_tx_queue(dev, nic, qidx);
1304 }
1305 
1306 static inline void
1307 nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq)
1308 {
1309 	uintptr_t p;
1310 	struct rte_mbuf mb_def;
1311 	struct nicvf *nic = rxq->nic;
1312 
1313 	RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8);
1314 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0);
1315 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) -
1316 				offsetof(struct rte_mbuf, data_off) != 2);
1317 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) -
1318 				offsetof(struct rte_mbuf, data_off) != 4);
1319 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) -
1320 				offsetof(struct rte_mbuf, data_off) != 6);
1321 	RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq, rxq_fastpath_data_end) -
1322 				offsetof(struct nicvf_rxq,
1323 					rxq_fastpath_data_start) > 128);
1324 	mb_def.nb_segs = 1;
1325 	mb_def.data_off = RTE_PKTMBUF_HEADROOM + (nic->skip_bytes);
1326 	mb_def.port = rxq->port_id;
1327 	rte_mbuf_refcnt_set(&mb_def, 1);
1328 
1329 	/* Prevent compiler reordering: rearm_data covers previous fields */
1330 	rte_compiler_barrier();
1331 	p = (uintptr_t)&mb_def.rearm_data;
1332 	rxq->mbuf_initializer.value = *(uint64_t *)p;
1333 }
1334 
1335 static int
1336 nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
1337 			 uint16_t nb_desc, unsigned int socket_id,
1338 			 const struct rte_eth_rxconf *rx_conf,
1339 			 struct rte_mempool *mp)
1340 {
1341 	uint16_t rx_free_thresh;
1342 	struct nicvf_rxq *rxq;
1343 	struct nicvf *nic = nicvf_pmd_priv(dev);
1344 	uint64_t offloads;
1345 	uint32_t buffsz;
1346 	struct rte_pktmbuf_pool_private *mbp_priv;
1347 
1348 	PMD_INIT_FUNC_TRACE();
1349 
1350 	/* First skip check */
1351 	mbp_priv = rte_mempool_get_priv(mp);
1352 	buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
1353 	if (buffsz < (uint32_t)(nic->skip_bytes)) {
1354 		PMD_INIT_LOG(ERR, "First skip is more than configured buffer size");
1355 		return -EINVAL;
1356 	}
1357 
1358 	if (qidx >= MAX_RCV_QUEUES_PER_QS)
1359 		nic = nic->snicvf[qidx / MAX_RCV_QUEUES_PER_QS - 1];
1360 
1361 	qidx = qidx % MAX_RCV_QUEUES_PER_QS;
1362 
1363 	/* Socket id check */
1364 	if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
1365 		PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
1366 		socket_id, nic->node);
1367 
1368 	/* Mempool memory must be contiguous, so must be one memory segment*/
1369 	if (mp->nb_mem_chunks != 1) {
1370 		PMD_INIT_LOG(ERR, "Non-contiguous mempool, add more huge pages");
1371 		return -EINVAL;
1372 	}
1373 
1374 	/* Mempool memory must be physically contiguous */
1375 	if (mp->flags & RTE_MEMPOOL_F_NO_IOVA_CONTIG) {
1376 		PMD_INIT_LOG(ERR, "Mempool memory must be physically contiguous");
1377 		return -EINVAL;
1378 	}
1379 
1380 	/* Rx deferred start is not supported */
1381 	if (rx_conf->rx_deferred_start) {
1382 		PMD_INIT_LOG(ERR, "Rx deferred start not supported");
1383 		return -EINVAL;
1384 	}
1385 
1386 	/* Roundup nb_desc to available qsize and validate max number of desc */
1387 	nb_desc = nicvf_qsize_cq_roundup(nb_desc);
1388 	if (nb_desc == 0) {
1389 		PMD_INIT_LOG(ERR, "Value nb_desc beyond available hw cq qsize");
1390 		return -EINVAL;
1391 	}
1392 
1393 
1394 	/* Check rx_free_thresh upper bound */
1395 	rx_free_thresh = (uint16_t)((rx_conf->rx_free_thresh) ?
1396 				rx_conf->rx_free_thresh :
1397 				NICVF_DEFAULT_RX_FREE_THRESH);
1398 	if (rx_free_thresh > NICVF_MAX_RX_FREE_THRESH ||
1399 		rx_free_thresh >= nb_desc * .75) {
1400 		PMD_INIT_LOG(ERR, "rx_free_thresh greater than expected %d",
1401 				rx_free_thresh);
1402 		return -EINVAL;
1403 	}
1404 
1405 	/* Free memory prior to re-allocation if needed */
1406 	if (dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) {
1407 		PMD_RX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
1408 				nicvf_netdev_qidx(nic, qidx));
1409 		nicvf_dev_rx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1410 		dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL;
1411 	}
1412 
1413 	/* Allocate rxq memory */
1414 	rxq = rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq),
1415 					RTE_CACHE_LINE_SIZE, nic->node);
1416 	if (rxq == NULL) {
1417 		PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d",
1418 			     nicvf_netdev_qidx(nic, qidx));
1419 		return -ENOMEM;
1420 	}
1421 
1422 	rxq->nic = nic;
1423 	rxq->pool = mp;
1424 	rxq->queue_id = qidx;
1425 	rxq->port_id = dev->data->port_id;
1426 	rxq->rx_free_thresh = rx_free_thresh;
1427 	rxq->rx_drop_en = rx_conf->rx_drop_en;
1428 	rxq->cq_status = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_STATUS;
1429 	rxq->cq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_DOOR;
1430 	rxq->precharge_cnt = 0;
1431 
1432 	if (nicvf_hw_cap(nic) & NICVF_CAP_CQE_RX2)
1433 		rxq->rbptr_offset = NICVF_CQE_RX2_RBPTR_WORD;
1434 	else
1435 		rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD;
1436 
1437 	dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = rxq;
1438 
1439 	nicvf_rxq_mbuf_setup(rxq);
1440 
1441 	/* Alloc completion queue */
1442 	if (nicvf_qset_cq_alloc(dev, nic, rxq, rxq->queue_id, nb_desc)) {
1443 		PMD_INIT_LOG(ERR, "failed to allocate cq %u", rxq->queue_id);
1444 		nicvf_dev_rx_queue_release(dev, nicvf_netdev_qidx(nic, qidx));
1445 		return -ENOMEM;
1446 	}
1447 
1448 	nicvf_rx_queue_reset(rxq);
1449 
1450 	offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
1451 	PMD_INIT_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d)"
1452 			" phy=0x%" PRIx64 " offloads=0x%" PRIx64,
1453 			nicvf_netdev_qidx(nic, qidx), rxq, mp->name, nb_desc,
1454 			rte_mempool_avail_count(mp), rxq->phys, offloads);
1455 
1456 	dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
1457 		RTE_ETH_QUEUE_STATE_STOPPED;
1458 	return 0;
1459 }
1460 
1461 static int
1462 nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1463 {
1464 	struct nicvf *nic = nicvf_pmd_priv(dev);
1465 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1466 
1467 	PMD_INIT_FUNC_TRACE();
1468 
1469 	/* Autonegotiation may be disabled */
1470 	dev_info->speed_capa = RTE_ETH_LINK_SPEED_FIXED;
1471 	dev_info->speed_capa |= RTE_ETH_LINK_SPEED_10M | RTE_ETH_LINK_SPEED_100M |
1472 				 RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_10G;
1473 	if (nicvf_hw_version(nic) != PCI_SUB_DEVICE_ID_CN81XX_NICVF)
1474 		dev_info->speed_capa |= RTE_ETH_LINK_SPEED_40G;
1475 
1476 	dev_info->min_rx_bufsize = RTE_ETHER_MIN_MTU;
1477 	dev_info->max_rx_pktlen = NIC_HW_MAX_MTU + RTE_ETHER_HDR_LEN;
1478 	dev_info->max_rx_queues =
1479 			(uint16_t)MAX_RCV_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
1480 	dev_info->max_tx_queues =
1481 			(uint16_t)MAX_SND_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
1482 	dev_info->max_mac_addrs = 1;
1483 	dev_info->max_vfs = pci_dev->max_vfs;
1484 
1485 	dev_info->max_mtu = dev_info->max_rx_pktlen -
1486 				(RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN);
1487 	dev_info->min_mtu = dev_info->min_rx_bufsize - NIC_HW_L2_OVERHEAD;
1488 
1489 	dev_info->rx_offload_capa = NICVF_RX_OFFLOAD_CAPA;
1490 	dev_info->tx_offload_capa = NICVF_TX_OFFLOAD_CAPA;
1491 	dev_info->rx_queue_offload_capa = NICVF_RX_OFFLOAD_CAPA;
1492 	dev_info->tx_queue_offload_capa = NICVF_TX_OFFLOAD_CAPA;
1493 
1494 	dev_info->reta_size = nic->rss_info.rss_size;
1495 	dev_info->hash_key_size = RSS_HASH_KEY_BYTE_SIZE;
1496 	dev_info->flow_type_rss_offloads = NICVF_RSS_OFFLOAD_PASS1;
1497 	if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING)
1498 		dev_info->flow_type_rss_offloads |= NICVF_RSS_OFFLOAD_TUNNEL;
1499 
1500 	dev_info->default_rxconf = (struct rte_eth_rxconf) {
1501 		.rx_free_thresh = NICVF_DEFAULT_RX_FREE_THRESH,
1502 		.rx_drop_en = 0,
1503 	};
1504 
1505 	dev_info->default_txconf = (struct rte_eth_txconf) {
1506 		.tx_free_thresh = NICVF_DEFAULT_TX_FREE_THRESH,
1507 		.offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE |
1508 			RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM   |
1509 			RTE_ETH_TX_OFFLOAD_UDP_CKSUM          |
1510 			RTE_ETH_TX_OFFLOAD_TCP_CKSUM,
1511 	};
1512 
1513 	return 0;
1514 }
1515 
1516 static nicvf_iova_addr_t
1517 rbdr_rte_mempool_get(void *dev, void *opaque)
1518 {
1519 	uint16_t qidx;
1520 	uintptr_t mbuf;
1521 	struct nicvf_rxq *rxq;
1522 	struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)dev;
1523 	struct nicvf *nic = (struct nicvf *)opaque;
1524 	uint16_t rx_start, rx_end;
1525 
1526 	/* Get queue ranges for this VF */
1527 	nicvf_rx_range(eth_dev, nic, &rx_start, &rx_end);
1528 
1529 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
1530 		rxq = eth_dev->data->rx_queues[qidx];
1531 		/* Maintain equal buffer count across all pools */
1532 		if (rxq->precharge_cnt >= rxq->qlen_mask)
1533 			continue;
1534 		rxq->precharge_cnt++;
1535 		mbuf = (uintptr_t)rte_pktmbuf_alloc(rxq->pool);
1536 		if (mbuf)
1537 			return nicvf_mbuff_virt2phy(mbuf, rxq->mbuf_phys_off);
1538 	}
1539 	return 0;
1540 }
1541 
1542 static int
1543 nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
1544 {
1545 	int ret;
1546 	uint16_t qidx, data_off;
1547 	uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs;
1548 	uint64_t mbuf_phys_off = 0;
1549 	struct nicvf_rxq *rxq;
1550 	struct rte_mbuf *mbuf;
1551 	uint16_t rx_start, rx_end;
1552 	uint16_t tx_start, tx_end;
1553 	int mask;
1554 
1555 	PMD_INIT_FUNC_TRACE();
1556 
1557 	/* Userspace process exited without proper shutdown in last run */
1558 	if (nicvf_qset_rbdr_active(nic, 0))
1559 		nicvf_vf_stop(dev, nic, false);
1560 
1561 	/* Get queue ranges for this VF */
1562 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
1563 
1564 	/*
1565 	 * Thunderx nicvf PMD can support more than one pool per port only when
1566 	 * 1) Data payload size is same across all the pools in given port
1567 	 * AND
1568 	 * 2) All mbuffs in the pools are from the same hugepage
1569 	 * AND
1570 	 * 3) Mbuff metadata size is same across all the pools in given port
1571 	 *
1572 	 * This is to support existing application that uses multiple pool/port.
1573 	 * But, the purpose of using multipool for QoS will not be addressed.
1574 	 *
1575 	 */
1576 
1577 	/* Validate mempool attributes */
1578 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
1579 		rxq = dev->data->rx_queues[qidx];
1580 		rxq->mbuf_phys_off = nicvf_mempool_phy_offset(rxq->pool);
1581 		mbuf = rte_pktmbuf_alloc(rxq->pool);
1582 		if (mbuf == NULL) {
1583 			PMD_INIT_LOG(ERR, "Failed allocate mbuf VF%d qid=%d "
1584 				     "pool=%s",
1585 				     nic->vf_id, qidx, rxq->pool->name);
1586 			return -ENOMEM;
1587 		}
1588 		data_off = nicvf_mbuff_meta_length(mbuf);
1589 		data_off += RTE_PKTMBUF_HEADROOM;
1590 		rte_pktmbuf_free(mbuf);
1591 
1592 		if (data_off % RTE_CACHE_LINE_SIZE) {
1593 			PMD_INIT_LOG(ERR, "%s: unaligned data_off=%d delta=%d",
1594 				rxq->pool->name, data_off,
1595 				data_off % RTE_CACHE_LINE_SIZE);
1596 			return -EINVAL;
1597 		}
1598 		rxq->mbuf_phys_off -= data_off;
1599 		rxq->mbuf_phys_off -= nic->skip_bytes;
1600 
1601 		if (mbuf_phys_off == 0)
1602 			mbuf_phys_off = rxq->mbuf_phys_off;
1603 		if (mbuf_phys_off != rxq->mbuf_phys_off) {
1604 			PMD_INIT_LOG(ERR, "pool params not same,%s VF%d %"
1605 				     PRIx64, rxq->pool->name, nic->vf_id,
1606 				     mbuf_phys_off);
1607 			return -EINVAL;
1608 		}
1609 	}
1610 
1611 	/* Check the level of buffers in the pool */
1612 	total_rxq_desc = 0;
1613 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
1614 		rxq = dev->data->rx_queues[qidx];
1615 		/* Count total numbers of rxq descs */
1616 		total_rxq_desc += rxq->qlen_mask + 1;
1617 		exp_buffs = RTE_MEMPOOL_CACHE_MAX_SIZE + rxq->rx_free_thresh;
1618 		exp_buffs *= dev->data->nb_rx_queues;
1619 		if (rte_mempool_avail_count(rxq->pool) < exp_buffs) {
1620 			PMD_INIT_LOG(ERR, "Buff shortage in pool=%s (%d/%d)",
1621 				     rxq->pool->name,
1622 				     rte_mempool_avail_count(rxq->pool),
1623 				     exp_buffs);
1624 			return -ENOENT;
1625 		}
1626 	}
1627 
1628 	/* Check RBDR desc overflow */
1629 	ret = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1630 	if (ret == 0) {
1631 		PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc "
1632 			     "VF%d", nic->vf_id);
1633 		return -ENOMEM;
1634 	}
1635 
1636 	/* Enable qset */
1637 	ret = nicvf_qset_config(nic);
1638 	if (ret) {
1639 		PMD_INIT_LOG(ERR, "Failed to enable qset %d VF%d", ret,
1640 			     nic->vf_id);
1641 		return ret;
1642 	}
1643 
1644 	/* Allocate RBDR and RBDR ring desc */
1645 	nb_rbdr_desc = nicvf_qsize_rbdr_roundup(total_rxq_desc);
1646 	ret = nicvf_qset_rbdr_alloc(dev, nic, nb_rbdr_desc, rbdrsz);
1647 	if (ret) {
1648 		PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc "
1649 			     "VF%d", nic->vf_id);
1650 		goto qset_reclaim;
1651 	}
1652 
1653 	/* Enable and configure RBDR registers */
1654 	ret = nicvf_qset_rbdr_config(nic, 0);
1655 	if (ret) {
1656 		PMD_INIT_LOG(ERR, "Failed to configure rbdr %d VF%d", ret,
1657 			     nic->vf_id);
1658 		goto qset_rbdr_free;
1659 	}
1660 
1661 	/* Fill rte_mempool buffers in RBDR pool and precharge it */
1662 	ret = nicvf_qset_rbdr_precharge(dev, nic, 0, rbdr_rte_mempool_get,
1663 					total_rxq_desc);
1664 	if (ret) {
1665 		PMD_INIT_LOG(ERR, "Failed to fill rbdr %d VF%d", ret,
1666 			     nic->vf_id);
1667 		goto qset_rbdr_reclaim;
1668 	}
1669 
1670 	PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR VF%d",
1671 		     nic->rbdr->tail, nb_rbdr_desc, nic->vf_id);
1672 
1673 	/* Configure VLAN Strip */
1674 	mask = RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK |
1675 		RTE_ETH_VLAN_EXTEND_MASK;
1676 	ret = nicvf_vlan_offload_config(dev, mask);
1677 
1678 	/* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data
1679 	 * to the 64bit memory address.
1680 	 * The alignment creates a hole in mbuf(between the end of headroom and
1681 	 * packet data start). The new revision of the HW provides an option to
1682 	 * disable the L3 alignment feature and make mbuf layout looks
1683 	 * more like other NICs. For better application compatibility, disabling
1684 	 * l3 alignment feature on the hardware revisions it supports
1685 	 */
1686 	nicvf_apad_config(nic, false);
1687 
1688 	/* Get queue ranges for this VF */
1689 	nicvf_tx_range(dev, nic, &tx_start, &tx_end);
1690 
1691 	/* Configure TX queues */
1692 	for (qidx = tx_start; qidx <= tx_end; qidx++) {
1693 		ret = nicvf_vf_start_tx_queue(dev, nic,
1694 			qidx % MAX_SND_QUEUES_PER_QS);
1695 		if (ret)
1696 			goto start_txq_error;
1697 	}
1698 
1699 	/* Configure RX queues */
1700 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
1701 		ret = nicvf_vf_start_rx_queue(dev, nic,
1702 			qidx % MAX_RCV_QUEUES_PER_QS);
1703 		if (ret)
1704 			goto start_rxq_error;
1705 	}
1706 
1707 	if (!nic->sqs_mode) {
1708 		/* Configure CPI algorithm */
1709 		ret = nicvf_configure_cpi(dev);
1710 		if (ret)
1711 			goto start_txq_error;
1712 
1713 		ret = nicvf_mbox_get_rss_size(nic);
1714 		if (ret) {
1715 			PMD_INIT_LOG(ERR, "Failed to get rss table size");
1716 			goto qset_rss_error;
1717 		}
1718 
1719 		/* Configure RSS */
1720 		ret = nicvf_configure_rss(dev);
1721 		if (ret)
1722 			goto qset_rss_error;
1723 	}
1724 
1725 	/* Done; Let PF make the BGX's RX and TX switches to ON position */
1726 	nicvf_mbox_cfg_done(nic);
1727 	return 0;
1728 
1729 qset_rss_error:
1730 	nicvf_rss_term(nic);
1731 start_rxq_error:
1732 	for (qidx = rx_start; qidx <= rx_end; qidx++)
1733 		nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS);
1734 start_txq_error:
1735 	for (qidx = tx_start; qidx <= tx_end; qidx++)
1736 		nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS);
1737 qset_rbdr_reclaim:
1738 	nicvf_qset_rbdr_reclaim(nic, 0);
1739 	nicvf_rbdr_release_mbufs(dev, nic);
1740 qset_rbdr_free:
1741 	if (nic->rbdr) {
1742 		rte_free(nic->rbdr);
1743 		nic->rbdr = NULL;
1744 	}
1745 qset_reclaim:
1746 	nicvf_qset_reclaim(nic);
1747 	return ret;
1748 }
1749 
1750 static int
1751 nicvf_dev_start(struct rte_eth_dev *dev)
1752 {
1753 	uint16_t qidx;
1754 	int ret;
1755 	size_t i;
1756 	struct nicvf *nic = nicvf_pmd_priv(dev);
1757 	struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
1758 	uint16_t mtu;
1759 	uint32_t buffsz = 0, rbdrsz = 0;
1760 	struct rte_pktmbuf_pool_private *mbp_priv;
1761 	struct nicvf_rxq *rxq;
1762 
1763 	PMD_INIT_FUNC_TRACE();
1764 
1765 	/* This function must be called for a primary device */
1766 	assert_primary(nic);
1767 
1768 	/* Validate RBDR buff size */
1769 	for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) {
1770 		rxq = dev->data->rx_queues[qidx];
1771 		mbp_priv = rte_mempool_get_priv(rxq->pool);
1772 		buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
1773 		if (buffsz % 128) {
1774 			PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128");
1775 			return -EINVAL;
1776 		}
1777 		if (rbdrsz == 0)
1778 			rbdrsz = buffsz;
1779 		if (rbdrsz != buffsz) {
1780 			PMD_INIT_LOG(ERR, "buffsz not same, qidx=%d (%d/%d)",
1781 				     qidx, rbdrsz, buffsz);
1782 			return -EINVAL;
1783 		}
1784 	}
1785 
1786 	/* Configure loopback */
1787 	ret = nicvf_loopback_config(nic, dev->data->dev_conf.lpbk_mode);
1788 	if (ret) {
1789 		PMD_INIT_LOG(ERR, "Failed to configure loopback %d", ret);
1790 		return ret;
1791 	}
1792 
1793 	/* Reset all statistics counters attached to this port */
1794 	ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF);
1795 	if (ret) {
1796 		PMD_INIT_LOG(ERR, "Failed to reset stat counters %d", ret);
1797 		return ret;
1798 	}
1799 
1800 	/* Setup scatter mode if needed by jumbo */
1801 	if (dev->data->mtu + (uint32_t)NIC_HW_L2_OVERHEAD + 2 * VLAN_TAG_SIZE > buffsz)
1802 		dev->data->scattered_rx = 1;
1803 	if ((rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) != 0)
1804 		dev->data->scattered_rx = 1;
1805 
1806 	/* Setup MTU */
1807 	mtu = dev->data->mtu;
1808 
1809 	if (nicvf_dev_set_mtu(dev, mtu)) {
1810 		PMD_INIT_LOG(ERR, "Failed to set default mtu size");
1811 		return -EBUSY;
1812 	}
1813 
1814 	/* Apply new link configurations if changed */
1815 	ret = nicvf_apply_link_speed(dev);
1816 	if (ret) {
1817 		PMD_INIT_LOG(ERR, "Failed to set link configuration");
1818 		return ret;
1819 	}
1820 
1821 	ret = nicvf_vf_start(dev, nic, rbdrsz);
1822 	if (ret != 0)
1823 		return ret;
1824 
1825 	for (i = 0; i < nic->sqs_count; i++) {
1826 		assert(nic->snicvf[i]);
1827 
1828 		ret = nicvf_vf_start(dev, nic->snicvf[i], rbdrsz);
1829 		if (ret != 0)
1830 			return ret;
1831 	}
1832 
1833 	/* Configure callbacks based on offloads */
1834 	nicvf_set_tx_function(dev);
1835 	nicvf_set_rx_function(dev);
1836 
1837 	return 0;
1838 }
1839 
1840 static void
1841 nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup)
1842 {
1843 	size_t i;
1844 	int ret;
1845 	struct nicvf *nic = nicvf_pmd_priv(dev);
1846 
1847 	PMD_INIT_FUNC_TRACE();
1848 	dev->data->dev_started = 0;
1849 
1850 	/* Teardown secondary vf first */
1851 	for (i = 0; i < nic->sqs_count; i++) {
1852 		if (!nic->snicvf[i])
1853 			continue;
1854 
1855 		nicvf_vf_stop(dev, nic->snicvf[i], cleanup);
1856 	}
1857 
1858 	/* Stop the primary VF now */
1859 	nicvf_vf_stop(dev, nic, cleanup);
1860 
1861 	/* Disable loopback */
1862 	ret = nicvf_loopback_config(nic, 0);
1863 	if (ret)
1864 		PMD_INIT_LOG(ERR, "Failed to disable loopback %d", ret);
1865 
1866 	/* Reclaim CPI configuration */
1867 	ret = nicvf_mbox_config_cpi(nic, 0);
1868 	if (ret)
1869 		PMD_INIT_LOG(ERR, "Failed to reclaim CPI config %d", ret);
1870 }
1871 
1872 static int
1873 nicvf_dev_stop(struct rte_eth_dev *dev)
1874 {
1875 	PMD_INIT_FUNC_TRACE();
1876 
1877 	nicvf_dev_stop_cleanup(dev, false);
1878 
1879 	return 0;
1880 }
1881 
1882 static void
1883 nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic, bool cleanup)
1884 {
1885 	int ret;
1886 	uint16_t qidx;
1887 	uint16_t tx_start, tx_end;
1888 	uint16_t rx_start, rx_end;
1889 
1890 	PMD_INIT_FUNC_TRACE();
1891 
1892 	if (cleanup) {
1893 		/* Let PF make the BGX's RX and TX switches to OFF position */
1894 		nicvf_mbox_shutdown(nic);
1895 	}
1896 
1897 	/* Disable VLAN Strip */
1898 	nicvf_vlan_hw_strip(nic, 0);
1899 
1900 	/* Get queue ranges for this VF */
1901 	nicvf_tx_range(dev, nic, &tx_start, &tx_end);
1902 
1903 	for (qidx = tx_start; qidx <= tx_end; qidx++)
1904 		nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS);
1905 
1906 	/* Get queue ranges for this VF */
1907 	nicvf_rx_range(dev, nic, &rx_start, &rx_end);
1908 
1909 	/* Reclaim rq */
1910 	for (qidx = rx_start; qidx <= rx_end; qidx++)
1911 		nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS);
1912 
1913 	/* Reclaim RBDR */
1914 	ret = nicvf_qset_rbdr_reclaim(nic, 0);
1915 	if (ret)
1916 		PMD_INIT_LOG(ERR, "Failed to reclaim RBDR %d", ret);
1917 
1918 	/* Move all charged buffers in RBDR back to pool */
1919 	if (nic->rbdr != NULL)
1920 		nicvf_rbdr_release_mbufs(dev, nic);
1921 
1922 	/* Disable qset */
1923 	ret = nicvf_qset_reclaim(nic);
1924 	if (ret)
1925 		PMD_INIT_LOG(ERR, "Failed to disable qset %d", ret);
1926 
1927 	/* Disable all interrupts */
1928 	nicvf_disable_all_interrupts(nic);
1929 
1930 	/* Free RBDR SW structure */
1931 	if (nic->rbdr) {
1932 		rte_free(nic->rbdr);
1933 		nic->rbdr = NULL;
1934 	}
1935 }
1936 
1937 static int
1938 nicvf_dev_close(struct rte_eth_dev *dev)
1939 {
1940 	struct nicvf *nic = nicvf_pmd_priv(dev);
1941 
1942 	PMD_INIT_FUNC_TRACE();
1943 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1944 		return 0;
1945 
1946 	nicvf_dev_stop_cleanup(dev, true);
1947 	nicvf_periodic_alarm_stop(nicvf_interrupt, dev);
1948 	nicvf_periodic_alarm_stop(nicvf_vf_interrupt, nic);
1949 
1950 	rte_intr_instance_free(nic->intr_handle);
1951 
1952 	return 0;
1953 }
1954 
1955 static int
1956 nicvf_request_sqs(struct nicvf *nic)
1957 {
1958 	size_t i;
1959 
1960 	assert_primary(nic);
1961 	assert(nic->sqs_count > 0);
1962 	assert(nic->sqs_count <= MAX_SQS_PER_VF);
1963 
1964 	/* Set no of Rx/Tx queues in each of the SQsets */
1965 	for (i = 0; i < nic->sqs_count; i++) {
1966 		if (nicvf_svf_empty())
1967 			rte_panic("Cannot assign sufficient number of "
1968 				  "secondary queues to primary VF%" PRIu8 "\n",
1969 				  nic->vf_id);
1970 
1971 		nic->snicvf[i] = nicvf_svf_pop();
1972 		nic->snicvf[i]->sqs_id = i;
1973 	}
1974 
1975 	return nicvf_mbox_request_sqs(nic);
1976 }
1977 
1978 static int
1979 nicvf_dev_configure(struct rte_eth_dev *dev)
1980 {
1981 	struct rte_eth_dev_data *data = dev->data;
1982 	struct rte_eth_conf *conf = &data->dev_conf;
1983 	struct rte_eth_rxmode *rxmode = &conf->rxmode;
1984 	struct rte_eth_txmode *txmode = &conf->txmode;
1985 	struct nicvf *nic = nicvf_pmd_priv(dev);
1986 	uint8_t cqcount;
1987 
1988 	PMD_INIT_FUNC_TRACE();
1989 
1990 	if (rxmode->mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
1991 		rxmode->offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
1992 
1993 	if (!rte_eal_has_hugepages()) {
1994 		PMD_INIT_LOG(INFO, "Huge page is not configured");
1995 		return -EINVAL;
1996 	}
1997 
1998 	if (txmode->mq_mode) {
1999 		PMD_INIT_LOG(INFO, "Tx mq_mode DCB or VMDq not supported");
2000 		return -EINVAL;
2001 	}
2002 
2003 	if (rxmode->mq_mode != RTE_ETH_MQ_RX_NONE &&
2004 		rxmode->mq_mode != RTE_ETH_MQ_RX_RSS) {
2005 		PMD_INIT_LOG(INFO, "Unsupported rx qmode %d", rxmode->mq_mode);
2006 		return -EINVAL;
2007 	}
2008 
2009 	if (conf->dcb_capability_en) {
2010 		PMD_INIT_LOG(INFO, "DCB enable not supported");
2011 		return -EINVAL;
2012 	}
2013 
2014 	assert_primary(nic);
2015 	NICVF_STATIC_ASSERT(MAX_RCV_QUEUES_PER_QS == MAX_SND_QUEUES_PER_QS);
2016 	cqcount = RTE_MAX(data->nb_tx_queues, data->nb_rx_queues);
2017 	if (cqcount > MAX_RCV_QUEUES_PER_QS) {
2018 		nic->sqs_count = RTE_ALIGN_CEIL(cqcount, MAX_RCV_QUEUES_PER_QS);
2019 		nic->sqs_count = (nic->sqs_count / MAX_RCV_QUEUES_PER_QS) - 1;
2020 	} else {
2021 		nic->sqs_count = 0;
2022 	}
2023 
2024 	assert(nic->sqs_count <= MAX_SQS_PER_VF);
2025 
2026 	if (nic->sqs_count > 0) {
2027 		if (nicvf_request_sqs(nic)) {
2028 			rte_panic("Cannot assign sufficient number of "
2029 				  "secondary queues to PORT%d VF%" PRIu8 "\n",
2030 				  dev->data->port_id, nic->vf_id);
2031 		}
2032 	}
2033 
2034 	if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM)
2035 		nic->offload_cksum = 1;
2036 
2037 	PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64,
2038 		dev->data->port_id, nicvf_hw_cap(nic));
2039 
2040 	return 0;
2041 }
2042 
2043 static int
2044 nicvf_dev_set_link_up(struct rte_eth_dev *dev)
2045 {
2046 	struct nicvf *nic = nicvf_pmd_priv(dev);
2047 	int rc, i;
2048 
2049 	rc = nicvf_mbox_set_link_up_down(nic, true);
2050 	if (rc)
2051 		goto done;
2052 
2053 	/* Start tx queues  */
2054 	for (i = 0; i < dev->data->nb_tx_queues; i++)
2055 		nicvf_dev_tx_queue_start(dev, i);
2056 
2057 done:
2058 	return rc;
2059 }
2060 
2061 static int
2062 nicvf_dev_set_link_down(struct rte_eth_dev *dev)
2063 {
2064 	struct nicvf *nic = nicvf_pmd_priv(dev);
2065 	int i;
2066 
2067 	/* Stop tx queues  */
2068 	for (i = 0; i < dev->data->nb_tx_queues; i++)
2069 		nicvf_dev_tx_queue_stop(dev, i);
2070 
2071 	return nicvf_mbox_set_link_up_down(nic, false);
2072 }
2073 
2074 /* Initialize and register driver with DPDK Application */
2075 static const struct eth_dev_ops nicvf_eth_dev_ops = {
2076 	.dev_configure            = nicvf_dev_configure,
2077 	.dev_start                = nicvf_dev_start,
2078 	.dev_stop                 = nicvf_dev_stop,
2079 	.link_update              = nicvf_dev_link_update,
2080 	.dev_close                = nicvf_dev_close,
2081 	.stats_get                = nicvf_dev_stats_get,
2082 	.stats_reset              = nicvf_dev_stats_reset,
2083 	.promiscuous_enable       = nicvf_dev_promisc_enable,
2084 	.dev_infos_get            = nicvf_dev_info_get,
2085 	.dev_supported_ptypes_get = nicvf_dev_supported_ptypes_get,
2086 	.mtu_set                  = nicvf_dev_set_mtu,
2087 	.vlan_offload_set         = nicvf_vlan_offload_set,
2088 	.reta_update              = nicvf_dev_reta_update,
2089 	.reta_query               = nicvf_dev_reta_query,
2090 	.rss_hash_update          = nicvf_dev_rss_hash_update,
2091 	.rss_hash_conf_get        = nicvf_dev_rss_hash_conf_get,
2092 	.rx_queue_start           = nicvf_dev_rx_queue_start,
2093 	.rx_queue_stop            = nicvf_dev_rx_queue_stop,
2094 	.tx_queue_start           = nicvf_dev_tx_queue_start,
2095 	.tx_queue_stop            = nicvf_dev_tx_queue_stop,
2096 	.rx_queue_setup           = nicvf_dev_rx_queue_setup,
2097 	.rx_queue_release         = nicvf_dev_rx_queue_release,
2098 	.tx_queue_setup           = nicvf_dev_tx_queue_setup,
2099 	.tx_queue_release         = nicvf_dev_tx_queue_release,
2100 	.dev_set_link_up          = nicvf_dev_set_link_up,
2101 	.dev_set_link_down        = nicvf_dev_set_link_down,
2102 	.get_reg                  = nicvf_dev_get_regs,
2103 };
2104 
2105 static int
2106 nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask)
2107 {
2108 	struct rte_eth_rxmode *rxmode;
2109 	struct nicvf *nic = nicvf_pmd_priv(dev);
2110 	rxmode = &dev->data->dev_conf.rxmode;
2111 	if (mask & RTE_ETH_VLAN_STRIP_MASK) {
2112 		if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
2113 			nicvf_vlan_hw_strip(nic, true);
2114 		else
2115 			nicvf_vlan_hw_strip(nic, false);
2116 	}
2117 
2118 	return 0;
2119 }
2120 
2121 static int
2122 nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2123 {
2124 	nicvf_vlan_offload_config(dev, mask);
2125 
2126 	return 0;
2127 }
2128 
2129 static inline int
2130 nicvf_set_first_skip(struct rte_eth_dev *dev)
2131 {
2132 	int bytes_to_skip = 0;
2133 	int ret = 0;
2134 	unsigned int i;
2135 	struct rte_kvargs *kvlist;
2136 	static const char *const skip[] = {
2137 		SKIP_DATA_BYTES,
2138 		NULL};
2139 	struct nicvf *nic = nicvf_pmd_priv(dev);
2140 
2141 	if (!dev->device->devargs) {
2142 		nicvf_first_skip_config(nic, 0);
2143 		return ret;
2144 	}
2145 
2146 	kvlist = rte_kvargs_parse(dev->device->devargs->args, skip);
2147 	if (!kvlist)
2148 		return -EINVAL;
2149 
2150 	if (kvlist->count == 0)
2151 		goto exit;
2152 
2153 	for (i = 0; i != kvlist->count; ++i) {
2154 		const struct rte_kvargs_pair *pair = &kvlist->pairs[i];
2155 
2156 		if (!strcmp(pair->key, SKIP_DATA_BYTES))
2157 			bytes_to_skip = atoi(pair->value);
2158 	}
2159 
2160 	/*128 bytes amounts to one cache line*/
2161 	if (bytes_to_skip >= 0 && bytes_to_skip < 128) {
2162 		if (!(bytes_to_skip % 8)) {
2163 			nicvf_first_skip_config(nic, (bytes_to_skip / 8));
2164 			nic->skip_bytes = bytes_to_skip;
2165 			goto kvlist_free;
2166 		} else {
2167 			PMD_INIT_LOG(ERR, "skip_data_bytes should be multiple of 8");
2168 			ret = -EINVAL;
2169 			goto exit;
2170 		}
2171 	} else {
2172 		PMD_INIT_LOG(ERR, "skip_data_bytes should be less than 128");
2173 		ret = -EINVAL;
2174 		goto exit;
2175 	}
2176 exit:
2177 	nicvf_first_skip_config(nic, 0);
2178 kvlist_free:
2179 	rte_kvargs_free(kvlist);
2180 	return ret;
2181 }
2182 static int
2183 nicvf_eth_dev_uninit(struct rte_eth_dev *dev)
2184 {
2185 	PMD_INIT_FUNC_TRACE();
2186 	nicvf_dev_close(dev);
2187 	return 0;
2188 }
2189 
2190 static inline uint64_t ether_addr_to_u64(uint8_t *addr)
2191 {
2192 	uint64_t u = 0;
2193 	int i;
2194 
2195 	for (i = 0; i < RTE_ETHER_ADDR_LEN; i++)
2196 		u = u << 8 | addr[i];
2197 
2198 	return u;
2199 }
2200 
2201 static int
2202 nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
2203 {
2204 	uint8_t dmac_ctrl_reg = 0;
2205 	int ret;
2206 	struct rte_pci_device *pci_dev;
2207 	struct nicvf *nic = nicvf_pmd_priv(eth_dev);
2208 
2209 	PMD_INIT_FUNC_TRACE();
2210 
2211 	eth_dev->dev_ops = &nicvf_eth_dev_ops;
2212 	eth_dev->rx_queue_count = nicvf_dev_rx_queue_count;
2213 
2214 	/* For secondary processes, the primary has done all the work */
2215 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2216 		if (nic) {
2217 			/* Setup callbacks for secondary process */
2218 			nicvf_set_tx_function(eth_dev);
2219 			nicvf_set_rx_function(eth_dev);
2220 			return 0;
2221 		} else {
2222 			/* If nic == NULL than it is secondary function
2223 			 * so ethdev need to be released by caller */
2224 			return ENOTSUP;
2225 		}
2226 	}
2227 
2228 	pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2229 	rte_eth_copy_pci_info(eth_dev, pci_dev);
2230 	eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
2231 
2232 	nic->device_id = pci_dev->id.device_id;
2233 	nic->vendor_id = pci_dev->id.vendor_id;
2234 	nic->subsystem_device_id = pci_dev->id.subsystem_device_id;
2235 	nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
2236 
2237 	PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) " PCI_PRI_FMT,
2238 			pci_dev->id.vendor_id, pci_dev->id.device_id,
2239 			pci_dev->addr.domain, pci_dev->addr.bus,
2240 			pci_dev->addr.devid, pci_dev->addr.function);
2241 
2242 	nic->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
2243 	if (!nic->reg_base) {
2244 		PMD_INIT_LOG(ERR, "Failed to map BAR0");
2245 		ret = -ENODEV;
2246 		goto fail;
2247 	}
2248 
2249 	/* Allocate interrupt instance */
2250 	nic->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
2251 	if (nic->intr_handle == NULL) {
2252 		PMD_INIT_LOG(ERR, "Failed to allocate intr handle");
2253 		ret = -ENODEV;
2254 		goto fail;
2255 	}
2256 
2257 	nicvf_disable_all_interrupts(nic);
2258 
2259 	/* To read mbox messages */
2260 	ret = nicvf_periodic_alarm_start(nicvf_vf_interrupt, nic);
2261 	if (ret) {
2262 		PMD_INIT_LOG(ERR, "Failed to start period alarm");
2263 		goto fail;
2264 	}
2265 
2266 	/* To poll link status change*/
2267 	ret = nicvf_periodic_alarm_start(nicvf_interrupt, eth_dev);
2268 	if (ret) {
2269 		PMD_INIT_LOG(ERR, "Failed to start period alarm");
2270 		goto fail;
2271 	}
2272 
2273 	ret = nicvf_mbox_check_pf_ready(nic);
2274 	if (ret) {
2275 		PMD_INIT_LOG(ERR, "Failed to get ready message from PF");
2276 		goto alarm_fail;
2277 	} else {
2278 		PMD_INIT_LOG(INFO,
2279 			"node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
2280 			nic->node, nic->vf_id,
2281 			nic->tns_mode == NIC_TNS_MODE ? "tns" : "tns-bypass",
2282 			nic->sqs_mode ? "true" : "false",
2283 			nic->loopback_supported ? "true" : "false"
2284 			);
2285 	}
2286 
2287 	/* To make sure RX DMAC register is set to default value (0x3) */
2288 	nicvf_mbox_reset_xcast(nic);
2289 
2290 	ret = nicvf_base_init(nic);
2291 	if (ret) {
2292 		PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init");
2293 		goto malloc_fail;
2294 	}
2295 
2296 	if (nic->sqs_mode) {
2297 		/* Push nic to stack of secondary vfs */
2298 		nicvf_svf_push(nic);
2299 
2300 		/* Steal nic pointer from the device for further reuse */
2301 		eth_dev->data->dev_private = NULL;
2302 
2303 		nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev);
2304 
2305 		/* Detach port by returning positive error number */
2306 		return ENOTSUP;
2307 	}
2308 
2309 	eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
2310 					RTE_ETHER_ADDR_LEN, 0);
2311 	if (eth_dev->data->mac_addrs == NULL) {
2312 		PMD_INIT_LOG(ERR, "Failed to allocate memory for mac addr");
2313 		ret = -ENOMEM;
2314 		goto alarm_fail;
2315 	}
2316 	if (rte_is_zero_ether_addr((struct rte_ether_addr *)nic->mac_addr))
2317 		rte_eth_random_addr(&nic->mac_addr[0]);
2318 
2319 	rte_ether_addr_copy((struct rte_ether_addr *)nic->mac_addr,
2320 			&eth_dev->data->mac_addrs[0]);
2321 
2322 	ret = nicvf_mbox_set_mac_addr(nic, nic->mac_addr);
2323 	if (ret) {
2324 		PMD_INIT_LOG(ERR, "Failed to set mac addr");
2325 		goto malloc_fail;
2326 	}
2327 
2328 	/* set DMAC CTRL reg to allow MAC */
2329 	dmac_ctrl_reg = BCAST_ACCEPT | BGX_MCAST_MODE(2) | CAM_ACCEPT;
2330 	ret = nicvf_mbox_set_xcast(nic, dmac_ctrl_reg,
2331 			ether_addr_to_u64(nic->mac_addr));
2332 	if (ret) {
2333 		PMD_INIT_LOG(ERR, "Failed to set mac addr");
2334 		goto malloc_fail;
2335 	}
2336 
2337 	ret = nicvf_set_first_skip(eth_dev);
2338 	if (ret) {
2339 		PMD_INIT_LOG(ERR, "Failed to configure first skip");
2340 		goto malloc_fail;
2341 	}
2342 	PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=" RTE_ETHER_ADDR_PRT_FMT,
2343 		eth_dev->data->port_id, nic->vendor_id, nic->device_id,
2344 		nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],
2345 		nic->mac_addr[3], nic->mac_addr[4], nic->mac_addr[5]);
2346 
2347 	return 0;
2348 
2349 malloc_fail:
2350 	rte_free(eth_dev->data->mac_addrs);
2351 	eth_dev->data->mac_addrs = NULL;
2352 alarm_fail:
2353 	nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev);
2354 fail:
2355 	return ret;
2356 }
2357 
2358 static const struct rte_pci_id pci_id_nicvf_map[] = {
2359 	{
2360 		.class_id = RTE_CLASS_ANY_ID,
2361 		.vendor_id = PCI_VENDOR_ID_CAVIUM,
2362 		.device_id = PCI_DEVICE_ID_THUNDERX_CN88XX_PASS1_NICVF,
2363 		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2364 		.subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS1_NICVF,
2365 	},
2366 	{
2367 		.class_id = RTE_CLASS_ANY_ID,
2368 		.vendor_id = PCI_VENDOR_ID_CAVIUM,
2369 		.device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2370 		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2371 		.subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF,
2372 	},
2373 	{
2374 		.class_id = RTE_CLASS_ANY_ID,
2375 		.vendor_id = PCI_VENDOR_ID_CAVIUM,
2376 		.device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2377 		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2378 		.subsystem_device_id = PCI_SUB_DEVICE_ID_CN81XX_NICVF,
2379 	},
2380 	{
2381 		.class_id = RTE_CLASS_ANY_ID,
2382 		.vendor_id = PCI_VENDOR_ID_CAVIUM,
2383 		.device_id = PCI_DEVICE_ID_THUNDERX_NICVF,
2384 		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
2385 		.subsystem_device_id = PCI_SUB_DEVICE_ID_CN83XX_NICVF,
2386 	},
2387 	{
2388 		.vendor_id = 0,
2389 	},
2390 };
2391 
2392 static int nicvf_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2393 	struct rte_pci_device *pci_dev)
2394 {
2395 	return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct nicvf),
2396 		nicvf_eth_dev_init);
2397 }
2398 
2399 static int nicvf_eth_pci_remove(struct rte_pci_device *pci_dev)
2400 {
2401 	return rte_eth_dev_pci_generic_remove(pci_dev, nicvf_eth_dev_uninit);
2402 }
2403 
2404 static struct rte_pci_driver rte_nicvf_pmd = {
2405 	.id_table = pci_id_nicvf_map,
2406 	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_KEEP_MAPPED_RES |
2407 			RTE_PCI_DRV_INTR_LSC,
2408 	.probe = nicvf_eth_pci_probe,
2409 	.remove = nicvf_eth_pci_remove,
2410 };
2411 
2412 RTE_PMD_REGISTER_PCI(net_thunderx, rte_nicvf_pmd);
2413 RTE_PMD_REGISTER_PCI_TABLE(net_thunderx, pci_id_nicvf_map);
2414 RTE_PMD_REGISTER_KMOD_DEP(net_thunderx, "* igb_uio | uio_pci_generic | vfio-pci");
2415 RTE_PMD_REGISTER_PARAM_STRING(net_thunderx, SKIP_DATA_BYTES "=<int>");
2416