xref: /dpdk/app/test-pmd/testpmd.c (revision 487eec3401b7a1664982f39da139980a4f5b3adc)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2017 Intel Corporation
3  */
4 
5 #include <stdarg.h>
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <signal.h>
9 #include <string.h>
10 #include <time.h>
11 #include <fcntl.h>
12 #include <sys/mman.h>
13 #include <sys/types.h>
14 #include <errno.h>
15 #include <stdbool.h>
16 
17 #include <sys/queue.h>
18 #include <sys/stat.h>
19 
20 #include <stdint.h>
21 #include <unistd.h>
22 #include <inttypes.h>
23 
24 #include <rte_common.h>
25 #include <rte_errno.h>
26 #include <rte_byteorder.h>
27 #include <rte_log.h>
28 #include <rte_debug.h>
29 #include <rte_cycles.h>
30 #include <rte_memory.h>
31 #include <rte_memcpy.h>
32 #include <rte_launch.h>
33 #include <rte_eal.h>
34 #include <rte_alarm.h>
35 #include <rte_per_lcore.h>
36 #include <rte_lcore.h>
37 #include <rte_atomic.h>
38 #include <rte_branch_prediction.h>
39 #include <rte_mempool.h>
40 #include <rte_malloc.h>
41 #include <rte_mbuf.h>
42 #include <rte_mbuf_pool_ops.h>
43 #include <rte_interrupts.h>
44 #include <rte_pci.h>
45 #include <rte_ether.h>
46 #include <rte_ethdev.h>
47 #include <rte_dev.h>
48 #include <rte_string_fns.h>
49 #ifdef RTE_LIBRTE_IXGBE_PMD
50 #include <rte_pmd_ixgbe.h>
51 #endif
52 #ifdef RTE_LIBRTE_PDUMP
53 #include <rte_pdump.h>
54 #endif
55 #include <rte_flow.h>
56 #include <rte_metrics.h>
57 #ifdef RTE_LIBRTE_BITRATE
58 #include <rte_bitrate.h>
59 #endif
60 #ifdef RTE_LIBRTE_LATENCY_STATS
61 #include <rte_latencystats.h>
62 #endif
63 
64 #include "testpmd.h"
65 
66 #ifndef MAP_HUGETLB
67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
68 #define HUGE_FLAG (0x40000)
69 #else
70 #define HUGE_FLAG MAP_HUGETLB
71 #endif
72 
73 #ifndef MAP_HUGE_SHIFT
74 /* older kernels (or FreeBSD) will not have this define */
75 #define HUGE_SHIFT (26)
76 #else
77 #define HUGE_SHIFT MAP_HUGE_SHIFT
78 #endif
79 
80 #define EXTMEM_HEAP_NAME "extmem"
81 
82 uint16_t verbose_level = 0; /**< Silent by default. */
83 int testpmd_logtype; /**< Log type for testpmd logs */
84 
85 /* use master core for command line ? */
86 uint8_t interactive = 0;
87 uint8_t auto_start = 0;
88 uint8_t tx_first;
89 char cmdline_filename[PATH_MAX] = {0};
90 
91 /*
92  * NUMA support configuration.
93  * When set, the NUMA support attempts to dispatch the allocation of the
94  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
95  * probed ports among the CPU sockets 0 and 1.
96  * Otherwise, all memory is allocated from CPU socket 0.
97  */
98 uint8_t numa_support = 1; /**< numa enabled by default */
99 
100 /*
101  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
102  * not configured.
103  */
104 uint8_t socket_num = UMA_NO_CONFIG;
105 
106 /*
107  * Select mempool allocation type:
108  * - native: use regular DPDK memory
109  * - anon: use regular DPDK memory to create mempool, but populate using
110  *         anonymous memory (may not be IOVA-contiguous)
111  * - xmem: use externally allocated hugepage memory
112  */
113 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
114 
115 /*
116  * Store specified sockets on which memory pool to be used by ports
117  * is allocated.
118  */
119 uint8_t port_numa[RTE_MAX_ETHPORTS];
120 
121 /*
122  * Store specified sockets on which RX ring to be used by ports
123  * is allocated.
124  */
125 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
126 
127 /*
128  * Store specified sockets on which TX ring to be used by ports
129  * is allocated.
130  */
131 uint8_t txring_numa[RTE_MAX_ETHPORTS];
132 
133 /*
134  * Record the Ethernet address of peer target ports to which packets are
135  * forwarded.
136  * Must be instantiated with the ethernet addresses of peer traffic generator
137  * ports.
138  */
139 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
140 portid_t nb_peer_eth_addrs = 0;
141 
142 /*
143  * Probed Target Environment.
144  */
145 struct rte_port *ports;	       /**< For all probed ethernet ports. */
146 portid_t nb_ports;             /**< Number of probed ethernet ports. */
147 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
148 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
149 
150 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
151 
152 /*
153  * Test Forwarding Configuration.
154  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
155  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
156  */
157 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
158 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
159 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
160 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
161 
162 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
163 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
164 
165 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
166 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
167 
168 /*
169  * Forwarding engines.
170  */
171 struct fwd_engine * fwd_engines[] = {
172 	&io_fwd_engine,
173 	&mac_fwd_engine,
174 	&mac_swap_engine,
175 	&flow_gen_engine,
176 	&rx_only_engine,
177 	&tx_only_engine,
178 	&csum_fwd_engine,
179 	&icmp_echo_engine,
180 	&noisy_vnf_engine,
181 #if defined RTE_LIBRTE_PMD_SOFTNIC
182 	&softnic_fwd_engine,
183 #endif
184 #ifdef RTE_LIBRTE_IEEE1588
185 	&ieee1588_fwd_engine,
186 #endif
187 	NULL,
188 };
189 
190 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
191 uint16_t mempool_flags;
192 
193 struct fwd_config cur_fwd_config;
194 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
195 uint32_t retry_enabled;
196 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
197 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
198 
199 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
200 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
201                                       * specified on command-line. */
202 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
203 
204 /*
205  * In container, it cannot terminate the process which running with 'stats-period'
206  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
207  */
208 uint8_t f_quit;
209 
210 /*
211  * Configuration of packet segments used by the "txonly" processing engine.
212  */
213 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
214 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
215 	TXONLY_DEF_PACKET_LEN,
216 };
217 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
218 
219 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
220 /**< Split policy for packets to TX. */
221 
222 uint8_t txonly_multi_flow;
223 /**< Whether multiple flows are generated in TXONLY mode. */
224 
225 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
226 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
227 
228 /* current configuration is in DCB or not,0 means it is not in DCB mode */
229 uint8_t dcb_config = 0;
230 
231 /* Whether the dcb is in testing status */
232 uint8_t dcb_test = 0;
233 
234 /*
235  * Configurable number of RX/TX queues.
236  */
237 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
238 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
239 
240 /*
241  * Configurable number of RX/TX ring descriptors.
242  * Defaults are supplied by drivers via ethdev.
243  */
244 #define RTE_TEST_RX_DESC_DEFAULT 0
245 #define RTE_TEST_TX_DESC_DEFAULT 0
246 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
247 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
248 
249 #define RTE_PMD_PARAM_UNSET -1
250 /*
251  * Configurable values of RX and TX ring threshold registers.
252  */
253 
254 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
255 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
256 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
257 
258 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
259 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
260 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
261 
262 /*
263  * Configurable value of RX free threshold.
264  */
265 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
266 
267 /*
268  * Configurable value of RX drop enable.
269  */
270 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
271 
272 /*
273  * Configurable value of TX free threshold.
274  */
275 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
276 
277 /*
278  * Configurable value of TX RS bit threshold.
279  */
280 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
281 
282 /*
283  * Configurable value of buffered packets before sending.
284  */
285 uint16_t noisy_tx_sw_bufsz;
286 
287 /*
288  * Configurable value of packet buffer timeout.
289  */
290 uint16_t noisy_tx_sw_buf_flush_time;
291 
292 /*
293  * Configurable value for size of VNF internal memory area
294  * used for simulating noisy neighbour behaviour
295  */
296 uint64_t noisy_lkup_mem_sz;
297 
298 /*
299  * Configurable value of number of random writes done in
300  * VNF simulation memory area.
301  */
302 uint64_t noisy_lkup_num_writes;
303 
304 /*
305  * Configurable value of number of random reads done in
306  * VNF simulation memory area.
307  */
308 uint64_t noisy_lkup_num_reads;
309 
310 /*
311  * Configurable value of number of random reads/writes done in
312  * VNF simulation memory area.
313  */
314 uint64_t noisy_lkup_num_reads_writes;
315 
316 /*
317  * Receive Side Scaling (RSS) configuration.
318  */
319 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
320 
321 /*
322  * Port topology configuration
323  */
324 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
325 
326 /*
327  * Avoids to flush all the RX streams before starts forwarding.
328  */
329 uint8_t no_flush_rx = 0; /* flush by default */
330 
331 /*
332  * Flow API isolated mode.
333  */
334 uint8_t flow_isolate_all;
335 
336 /*
337  * Avoids to check link status when starting/stopping a port.
338  */
339 uint8_t no_link_check = 0; /* check by default */
340 
341 /*
342  * Don't automatically start all ports in interactive mode.
343  */
344 uint8_t no_device_start = 0;
345 
346 /*
347  * Enable link status change notification
348  */
349 uint8_t lsc_interrupt = 1; /* enabled by default */
350 
351 /*
352  * Enable device removal notification.
353  */
354 uint8_t rmv_interrupt = 1; /* enabled by default */
355 
356 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
357 
358 /* After attach, port setup is called on event or by iterator */
359 bool setup_on_probe_event = true;
360 
361 /* Pretty printing of ethdev events */
362 static const char * const eth_event_desc[] = {
363 	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
364 	[RTE_ETH_EVENT_INTR_LSC] = "link state change",
365 	[RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
366 	[RTE_ETH_EVENT_INTR_RESET] = "reset",
367 	[RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
368 	[RTE_ETH_EVENT_IPSEC] = "IPsec",
369 	[RTE_ETH_EVENT_MACSEC] = "MACsec",
370 	[RTE_ETH_EVENT_INTR_RMV] = "device removal",
371 	[RTE_ETH_EVENT_NEW] = "device probed",
372 	[RTE_ETH_EVENT_DESTROY] = "device released",
373 	[RTE_ETH_EVENT_MAX] = NULL,
374 };
375 
376 /*
377  * Display or mask ether events
378  * Default to all events except VF_MBOX
379  */
380 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
381 			    (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
382 			    (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
383 			    (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
384 			    (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
385 			    (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
386 			    (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
387 /*
388  * Decide if all memory are locked for performance.
389  */
390 int do_mlockall = 0;
391 
392 /*
393  * NIC bypass mode configuration options.
394  */
395 
396 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
397 /* The NIC bypass watchdog timeout. */
398 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
399 #endif
400 
401 
402 #ifdef RTE_LIBRTE_LATENCY_STATS
403 
404 /*
405  * Set when latency stats is enabled in the commandline
406  */
407 uint8_t latencystats_enabled;
408 
409 /*
410  * Lcore ID to serive latency statistics.
411  */
412 lcoreid_t latencystats_lcore_id = -1;
413 
414 #endif
415 
416 /*
417  * Ethernet device configuration.
418  */
419 struct rte_eth_rxmode rx_mode = {
420 	.max_rx_pkt_len = RTE_ETHER_MAX_LEN,
421 		/**< Default maximum frame length. */
422 };
423 
424 struct rte_eth_txmode tx_mode = {
425 	.offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
426 };
427 
428 struct rte_fdir_conf fdir_conf = {
429 	.mode = RTE_FDIR_MODE_NONE,
430 	.pballoc = RTE_FDIR_PBALLOC_64K,
431 	.status = RTE_FDIR_REPORT_STATUS,
432 	.mask = {
433 		.vlan_tci_mask = 0xFFEF,
434 		.ipv4_mask     = {
435 			.src_ip = 0xFFFFFFFF,
436 			.dst_ip = 0xFFFFFFFF,
437 		},
438 		.ipv6_mask     = {
439 			.src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
440 			.dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
441 		},
442 		.src_port_mask = 0xFFFF,
443 		.dst_port_mask = 0xFFFF,
444 		.mac_addr_byte_mask = 0xFF,
445 		.tunnel_type_mask = 1,
446 		.tunnel_id_mask = 0xFFFFFFFF,
447 	},
448 	.drop_queue = 127,
449 };
450 
451 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
452 
453 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
454 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
455 
456 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
457 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
458 
459 uint16_t nb_tx_queue_stats_mappings = 0;
460 uint16_t nb_rx_queue_stats_mappings = 0;
461 
462 /*
463  * Display zero values by default for xstats
464  */
465 uint8_t xstats_hide_zero;
466 
467 unsigned int num_sockets = 0;
468 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
469 
470 #ifdef RTE_LIBRTE_BITRATE
471 /* Bitrate statistics */
472 struct rte_stats_bitrates *bitrate_data;
473 lcoreid_t bitrate_lcore_id;
474 uint8_t bitrate_enabled;
475 #endif
476 
477 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
478 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
479 
480 /* Forward function declarations */
481 static void setup_attached_port(portid_t pi);
482 static void map_port_queue_stats_mapping_registers(portid_t pi,
483 						   struct rte_port *port);
484 static void check_all_ports_link_status(uint32_t port_mask);
485 static int eth_event_callback(portid_t port_id,
486 			      enum rte_eth_event_type type,
487 			      void *param, void *ret_param);
488 static void dev_event_callback(const char *device_name,
489 				enum rte_dev_event_type type,
490 				void *param);
491 
492 /*
493  * Check if all the ports are started.
494  * If yes, return positive value. If not, return zero.
495  */
496 static int all_ports_started(void);
497 
498 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
499 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
500 
501 /*
502  * Helper function to check if socket is already discovered.
503  * If yes, return positive value. If not, return zero.
504  */
505 int
506 new_socket_id(unsigned int socket_id)
507 {
508 	unsigned int i;
509 
510 	for (i = 0; i < num_sockets; i++) {
511 		if (socket_ids[i] == socket_id)
512 			return 0;
513 	}
514 	return 1;
515 }
516 
517 /*
518  * Setup default configuration.
519  */
520 static void
521 set_default_fwd_lcores_config(void)
522 {
523 	unsigned int i;
524 	unsigned int nb_lc;
525 	unsigned int sock_num;
526 
527 	nb_lc = 0;
528 	for (i = 0; i < RTE_MAX_LCORE; i++) {
529 		if (!rte_lcore_is_enabled(i))
530 			continue;
531 		sock_num = rte_lcore_to_socket_id(i);
532 		if (new_socket_id(sock_num)) {
533 			if (num_sockets >= RTE_MAX_NUMA_NODES) {
534 				rte_exit(EXIT_FAILURE,
535 					 "Total sockets greater than %u\n",
536 					 RTE_MAX_NUMA_NODES);
537 			}
538 			socket_ids[num_sockets++] = sock_num;
539 		}
540 		if (i == rte_get_master_lcore())
541 			continue;
542 		fwd_lcores_cpuids[nb_lc++] = i;
543 	}
544 	nb_lcores = (lcoreid_t) nb_lc;
545 	nb_cfg_lcores = nb_lcores;
546 	nb_fwd_lcores = 1;
547 }
548 
549 static void
550 set_def_peer_eth_addrs(void)
551 {
552 	portid_t i;
553 
554 	for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
555 		peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
556 		peer_eth_addrs[i].addr_bytes[5] = i;
557 	}
558 }
559 
560 static void
561 set_default_fwd_ports_config(void)
562 {
563 	portid_t pt_id;
564 	int i = 0;
565 
566 	RTE_ETH_FOREACH_DEV(pt_id) {
567 		fwd_ports_ids[i++] = pt_id;
568 
569 		/* Update sockets info according to the attached device */
570 		int socket_id = rte_eth_dev_socket_id(pt_id);
571 		if (socket_id >= 0 && new_socket_id(socket_id)) {
572 			if (num_sockets >= RTE_MAX_NUMA_NODES) {
573 				rte_exit(EXIT_FAILURE,
574 					 "Total sockets greater than %u\n",
575 					 RTE_MAX_NUMA_NODES);
576 			}
577 			socket_ids[num_sockets++] = socket_id;
578 		}
579 	}
580 
581 	nb_cfg_ports = nb_ports;
582 	nb_fwd_ports = nb_ports;
583 }
584 
585 void
586 set_def_fwd_config(void)
587 {
588 	set_default_fwd_lcores_config();
589 	set_def_peer_eth_addrs();
590 	set_default_fwd_ports_config();
591 }
592 
593 /* extremely pessimistic estimation of memory required to create a mempool */
594 static int
595 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
596 {
597 	unsigned int n_pages, mbuf_per_pg, leftover;
598 	uint64_t total_mem, mbuf_mem, obj_sz;
599 
600 	/* there is no good way to predict how much space the mempool will
601 	 * occupy because it will allocate chunks on the fly, and some of those
602 	 * will come from default DPDK memory while some will come from our
603 	 * external memory, so just assume 128MB will be enough for everyone.
604 	 */
605 	uint64_t hdr_mem = 128 << 20;
606 
607 	/* account for possible non-contiguousness */
608 	obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
609 	if (obj_sz > pgsz) {
610 		TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
611 		return -1;
612 	}
613 
614 	mbuf_per_pg = pgsz / obj_sz;
615 	leftover = (nb_mbufs % mbuf_per_pg) > 0;
616 	n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
617 
618 	mbuf_mem = n_pages * pgsz;
619 
620 	total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
621 
622 	if (total_mem > SIZE_MAX) {
623 		TESTPMD_LOG(ERR, "Memory size too big\n");
624 		return -1;
625 	}
626 	*out = (size_t)total_mem;
627 
628 	return 0;
629 }
630 
631 static int
632 pagesz_flags(uint64_t page_sz)
633 {
634 	/* as per mmap() manpage, all page sizes are log2 of page size
635 	 * shifted by MAP_HUGE_SHIFT
636 	 */
637 	int log2 = rte_log2_u64(page_sz);
638 
639 	return (log2 << HUGE_SHIFT);
640 }
641 
642 static void *
643 alloc_mem(size_t memsz, size_t pgsz, bool huge)
644 {
645 	void *addr;
646 	int flags;
647 
648 	/* allocate anonymous hugepages */
649 	flags = MAP_ANONYMOUS | MAP_PRIVATE;
650 	if (huge)
651 		flags |= HUGE_FLAG | pagesz_flags(pgsz);
652 
653 	addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
654 	if (addr == MAP_FAILED)
655 		return NULL;
656 
657 	return addr;
658 }
659 
660 struct extmem_param {
661 	void *addr;
662 	size_t len;
663 	size_t pgsz;
664 	rte_iova_t *iova_table;
665 	unsigned int iova_table_len;
666 };
667 
668 static int
669 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
670 		bool huge)
671 {
672 	uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
673 			RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
674 	unsigned int cur_page, n_pages, pgsz_idx;
675 	size_t mem_sz, cur_pgsz;
676 	rte_iova_t *iovas = NULL;
677 	void *addr;
678 	int ret;
679 
680 	for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
681 		/* skip anything that is too big */
682 		if (pgsizes[pgsz_idx] > SIZE_MAX)
683 			continue;
684 
685 		cur_pgsz = pgsizes[pgsz_idx];
686 
687 		/* if we were told not to allocate hugepages, override */
688 		if (!huge)
689 			cur_pgsz = sysconf(_SC_PAGESIZE);
690 
691 		ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
692 		if (ret < 0) {
693 			TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
694 			return -1;
695 		}
696 
697 		/* allocate our memory */
698 		addr = alloc_mem(mem_sz, cur_pgsz, huge);
699 
700 		/* if we couldn't allocate memory with a specified page size,
701 		 * that doesn't mean we can't do it with other page sizes, so
702 		 * try another one.
703 		 */
704 		if (addr == NULL)
705 			continue;
706 
707 		/* store IOVA addresses for every page in this memory area */
708 		n_pages = mem_sz / cur_pgsz;
709 
710 		iovas = malloc(sizeof(*iovas) * n_pages);
711 
712 		if (iovas == NULL) {
713 			TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
714 			goto fail;
715 		}
716 		/* lock memory if it's not huge pages */
717 		if (!huge)
718 			mlock(addr, mem_sz);
719 
720 		/* populate IOVA addresses */
721 		for (cur_page = 0; cur_page < n_pages; cur_page++) {
722 			rte_iova_t iova;
723 			size_t offset;
724 			void *cur;
725 
726 			offset = cur_pgsz * cur_page;
727 			cur = RTE_PTR_ADD(addr, offset);
728 
729 			/* touch the page before getting its IOVA */
730 			*(volatile char *)cur = 0;
731 
732 			iova = rte_mem_virt2iova(cur);
733 
734 			iovas[cur_page] = iova;
735 		}
736 
737 		break;
738 	}
739 	/* if we couldn't allocate anything */
740 	if (iovas == NULL)
741 		return -1;
742 
743 	param->addr = addr;
744 	param->len = mem_sz;
745 	param->pgsz = cur_pgsz;
746 	param->iova_table = iovas;
747 	param->iova_table_len = n_pages;
748 
749 	return 0;
750 fail:
751 	if (iovas)
752 		free(iovas);
753 	if (addr)
754 		munmap(addr, mem_sz);
755 
756 	return -1;
757 }
758 
759 static int
760 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
761 {
762 	struct extmem_param param;
763 	int socket_id, ret;
764 
765 	memset(&param, 0, sizeof(param));
766 
767 	/* check if our heap exists */
768 	socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
769 	if (socket_id < 0) {
770 		/* create our heap */
771 		ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
772 		if (ret < 0) {
773 			TESTPMD_LOG(ERR, "Cannot create heap\n");
774 			return -1;
775 		}
776 	}
777 
778 	ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
779 	if (ret < 0) {
780 		TESTPMD_LOG(ERR, "Cannot create memory area\n");
781 		return -1;
782 	}
783 
784 	/* we now have a valid memory area, so add it to heap */
785 	ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
786 			param.addr, param.len, param.iova_table,
787 			param.iova_table_len, param.pgsz);
788 
789 	/* when using VFIO, memory is automatically mapped for DMA by EAL */
790 
791 	/* not needed any more */
792 	free(param.iova_table);
793 
794 	if (ret < 0) {
795 		TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
796 		munmap(param.addr, param.len);
797 		return -1;
798 	}
799 
800 	/* success */
801 
802 	TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
803 			param.len >> 20);
804 
805 	return 0;
806 }
807 static void
808 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
809 	     struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
810 {
811 	uint16_t pid = 0;
812 	int ret;
813 
814 	RTE_ETH_FOREACH_DEV(pid) {
815 		struct rte_eth_dev *dev =
816 			&rte_eth_devices[pid];
817 
818 		ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
819 					memhdr->len);
820 		if (ret) {
821 			TESTPMD_LOG(DEBUG,
822 				    "unable to DMA unmap addr 0x%p "
823 				    "for device %s\n",
824 				    memhdr->addr, dev->data->name);
825 		}
826 	}
827 	ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
828 	if (ret) {
829 		TESTPMD_LOG(DEBUG,
830 			    "unable to un-register addr 0x%p\n", memhdr->addr);
831 	}
832 }
833 
834 static void
835 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
836 	   struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
837 {
838 	uint16_t pid = 0;
839 	size_t page_size = sysconf(_SC_PAGESIZE);
840 	int ret;
841 
842 	ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
843 				  page_size);
844 	if (ret) {
845 		TESTPMD_LOG(DEBUG,
846 			    "unable to register addr 0x%p\n", memhdr->addr);
847 		return;
848 	}
849 	RTE_ETH_FOREACH_DEV(pid) {
850 		struct rte_eth_dev *dev =
851 			&rte_eth_devices[pid];
852 
853 		ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
854 				      memhdr->len);
855 		if (ret) {
856 			TESTPMD_LOG(DEBUG,
857 				    "unable to DMA map addr 0x%p "
858 				    "for device %s\n",
859 				    memhdr->addr, dev->data->name);
860 		}
861 	}
862 }
863 
864 /*
865  * Configuration initialisation done once at init time.
866  */
867 static struct rte_mempool *
868 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
869 		 unsigned int socket_id)
870 {
871 	char pool_name[RTE_MEMPOOL_NAMESIZE];
872 	struct rte_mempool *rte_mp = NULL;
873 	uint32_t mb_size;
874 
875 	mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
876 	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
877 
878 	TESTPMD_LOG(INFO,
879 		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
880 		pool_name, nb_mbuf, mbuf_seg_size, socket_id);
881 
882 	switch (mp_alloc_type) {
883 	case MP_ALLOC_NATIVE:
884 		{
885 			/* wrapper to rte_mempool_create() */
886 			TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
887 					rte_mbuf_best_mempool_ops());
888 			rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
889 				mb_mempool_cache, 0, mbuf_seg_size, socket_id);
890 			break;
891 		}
892 	case MP_ALLOC_ANON:
893 		{
894 			rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
895 				mb_size, (unsigned int) mb_mempool_cache,
896 				sizeof(struct rte_pktmbuf_pool_private),
897 				socket_id, mempool_flags);
898 			if (rte_mp == NULL)
899 				goto err;
900 
901 			if (rte_mempool_populate_anon(rte_mp) == 0) {
902 				rte_mempool_free(rte_mp);
903 				rte_mp = NULL;
904 				goto err;
905 			}
906 			rte_pktmbuf_pool_init(rte_mp, NULL);
907 			rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
908 			rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
909 			break;
910 		}
911 	case MP_ALLOC_XMEM:
912 	case MP_ALLOC_XMEM_HUGE:
913 		{
914 			int heap_socket;
915 			bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
916 
917 			if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
918 				rte_exit(EXIT_FAILURE, "Could not create external memory\n");
919 
920 			heap_socket =
921 				rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
922 			if (heap_socket < 0)
923 				rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
924 
925 			TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
926 					rte_mbuf_best_mempool_ops());
927 			rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
928 					mb_mempool_cache, 0, mbuf_seg_size,
929 					heap_socket);
930 			break;
931 		}
932 	default:
933 		{
934 			rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
935 		}
936 	}
937 
938 err:
939 	if (rte_mp == NULL) {
940 		rte_exit(EXIT_FAILURE,
941 			"Creation of mbuf pool for socket %u failed: %s\n",
942 			socket_id, rte_strerror(rte_errno));
943 	} else if (verbose_level > 0) {
944 		rte_mempool_dump(stdout, rte_mp);
945 	}
946 	return rte_mp;
947 }
948 
949 /*
950  * Check given socket id is valid or not with NUMA mode,
951  * if valid, return 0, else return -1
952  */
953 static int
954 check_socket_id(const unsigned int socket_id)
955 {
956 	static int warning_once = 0;
957 
958 	if (new_socket_id(socket_id)) {
959 		if (!warning_once && numa_support)
960 			printf("Warning: NUMA should be configured manually by"
961 			       " using --port-numa-config and"
962 			       " --ring-numa-config parameters along with"
963 			       " --numa.\n");
964 		warning_once = 1;
965 		return -1;
966 	}
967 	return 0;
968 }
969 
970 /*
971  * Get the allowed maximum number of RX queues.
972  * *pid return the port id which has minimal value of
973  * max_rx_queues in all ports.
974  */
975 queueid_t
976 get_allowed_max_nb_rxq(portid_t *pid)
977 {
978 	queueid_t allowed_max_rxq = MAX_QUEUE_ID;
979 	bool max_rxq_valid = false;
980 	portid_t pi;
981 	struct rte_eth_dev_info dev_info;
982 
983 	RTE_ETH_FOREACH_DEV(pi) {
984 		if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
985 			continue;
986 
987 		max_rxq_valid = true;
988 		if (dev_info.max_rx_queues < allowed_max_rxq) {
989 			allowed_max_rxq = dev_info.max_rx_queues;
990 			*pid = pi;
991 		}
992 	}
993 	return max_rxq_valid ? allowed_max_rxq : 0;
994 }
995 
996 /*
997  * Check input rxq is valid or not.
998  * If input rxq is not greater than any of maximum number
999  * of RX queues of all ports, it is valid.
1000  * if valid, return 0, else return -1
1001  */
1002 int
1003 check_nb_rxq(queueid_t rxq)
1004 {
1005 	queueid_t allowed_max_rxq;
1006 	portid_t pid = 0;
1007 
1008 	allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1009 	if (rxq > allowed_max_rxq) {
1010 		printf("Fail: input rxq (%u) can't be greater "
1011 		       "than max_rx_queues (%u) of port %u\n",
1012 		       rxq,
1013 		       allowed_max_rxq,
1014 		       pid);
1015 		return -1;
1016 	}
1017 	return 0;
1018 }
1019 
1020 /*
1021  * Get the allowed maximum number of TX queues.
1022  * *pid return the port id which has minimal value of
1023  * max_tx_queues in all ports.
1024  */
1025 queueid_t
1026 get_allowed_max_nb_txq(portid_t *pid)
1027 {
1028 	queueid_t allowed_max_txq = MAX_QUEUE_ID;
1029 	bool max_txq_valid = false;
1030 	portid_t pi;
1031 	struct rte_eth_dev_info dev_info;
1032 
1033 	RTE_ETH_FOREACH_DEV(pi) {
1034 		if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1035 			continue;
1036 
1037 		max_txq_valid = true;
1038 		if (dev_info.max_tx_queues < allowed_max_txq) {
1039 			allowed_max_txq = dev_info.max_tx_queues;
1040 			*pid = pi;
1041 		}
1042 	}
1043 	return max_txq_valid ? allowed_max_txq : 0;
1044 }
1045 
1046 /*
1047  * Check input txq is valid or not.
1048  * If input txq is not greater than any of maximum number
1049  * of TX queues of all ports, it is valid.
1050  * if valid, return 0, else return -1
1051  */
1052 int
1053 check_nb_txq(queueid_t txq)
1054 {
1055 	queueid_t allowed_max_txq;
1056 	portid_t pid = 0;
1057 
1058 	allowed_max_txq = get_allowed_max_nb_txq(&pid);
1059 	if (txq > allowed_max_txq) {
1060 		printf("Fail: input txq (%u) can't be greater "
1061 		       "than max_tx_queues (%u) of port %u\n",
1062 		       txq,
1063 		       allowed_max_txq,
1064 		       pid);
1065 		return -1;
1066 	}
1067 	return 0;
1068 }
1069 
1070 static void
1071 init_config(void)
1072 {
1073 	portid_t pid;
1074 	struct rte_port *port;
1075 	struct rte_mempool *mbp;
1076 	unsigned int nb_mbuf_per_pool;
1077 	lcoreid_t  lc_id;
1078 	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1079 	struct rte_gro_param gro_param;
1080 	uint32_t gso_types;
1081 	uint16_t data_size;
1082 	bool warning = 0;
1083 	int k;
1084 	int ret;
1085 
1086 	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1087 
1088 	/* Configuration of logical cores. */
1089 	fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1090 				sizeof(struct fwd_lcore *) * nb_lcores,
1091 				RTE_CACHE_LINE_SIZE);
1092 	if (fwd_lcores == NULL) {
1093 		rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1094 							"failed\n", nb_lcores);
1095 	}
1096 	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1097 		fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1098 					       sizeof(struct fwd_lcore),
1099 					       RTE_CACHE_LINE_SIZE);
1100 		if (fwd_lcores[lc_id] == NULL) {
1101 			rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1102 								"failed\n");
1103 		}
1104 		fwd_lcores[lc_id]->cpuid_idx = lc_id;
1105 	}
1106 
1107 	RTE_ETH_FOREACH_DEV(pid) {
1108 		port = &ports[pid];
1109 		/* Apply default TxRx configuration for all ports */
1110 		port->dev_conf.txmode = tx_mode;
1111 		port->dev_conf.rxmode = rx_mode;
1112 
1113 		ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1114 		if (ret != 0)
1115 			rte_exit(EXIT_FAILURE,
1116 				 "rte_eth_dev_info_get() failed\n");
1117 
1118 		if (!(port->dev_info.tx_offload_capa &
1119 		      DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1120 			port->dev_conf.txmode.offloads &=
1121 				~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1122 		if (!(port->dev_info.tx_offload_capa &
1123 			DEV_TX_OFFLOAD_MATCH_METADATA))
1124 			port->dev_conf.txmode.offloads &=
1125 				~DEV_TX_OFFLOAD_MATCH_METADATA;
1126 		if (numa_support) {
1127 			if (port_numa[pid] != NUMA_NO_CONFIG)
1128 				port_per_socket[port_numa[pid]]++;
1129 			else {
1130 				uint32_t socket_id = rte_eth_dev_socket_id(pid);
1131 
1132 				/*
1133 				 * if socket_id is invalid,
1134 				 * set to the first available socket.
1135 				 */
1136 				if (check_socket_id(socket_id) < 0)
1137 					socket_id = socket_ids[0];
1138 				port_per_socket[socket_id]++;
1139 			}
1140 		}
1141 
1142 		/* Apply Rx offloads configuration */
1143 		for (k = 0; k < port->dev_info.max_rx_queues; k++)
1144 			port->rx_conf[k].offloads =
1145 				port->dev_conf.rxmode.offloads;
1146 		/* Apply Tx offloads configuration */
1147 		for (k = 0; k < port->dev_info.max_tx_queues; k++)
1148 			port->tx_conf[k].offloads =
1149 				port->dev_conf.txmode.offloads;
1150 
1151 		/* set flag to initialize port/queue */
1152 		port->need_reconfig = 1;
1153 		port->need_reconfig_queues = 1;
1154 		port->tx_metadata = 0;
1155 
1156 		/* Check for maximum number of segments per MTU. Accordingly
1157 		 * update the mbuf data size.
1158 		 */
1159 		if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1160 				port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1161 			data_size = rx_mode.max_rx_pkt_len /
1162 				port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1163 
1164 			if ((data_size + RTE_PKTMBUF_HEADROOM) >
1165 							mbuf_data_size) {
1166 				mbuf_data_size = data_size +
1167 						 RTE_PKTMBUF_HEADROOM;
1168 				warning = 1;
1169 			}
1170 		}
1171 	}
1172 
1173 	if (warning)
1174 		TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1175 			    mbuf_data_size);
1176 
1177 	/*
1178 	 * Create pools of mbuf.
1179 	 * If NUMA support is disabled, create a single pool of mbuf in
1180 	 * socket 0 memory by default.
1181 	 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1182 	 *
1183 	 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1184 	 * nb_txd can be configured at run time.
1185 	 */
1186 	if (param_total_num_mbufs)
1187 		nb_mbuf_per_pool = param_total_num_mbufs;
1188 	else {
1189 		nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1190 			(nb_lcores * mb_mempool_cache) +
1191 			RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1192 		nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1193 	}
1194 
1195 	if (numa_support) {
1196 		uint8_t i;
1197 
1198 		for (i = 0; i < num_sockets; i++)
1199 			mempools[i] = mbuf_pool_create(mbuf_data_size,
1200 						       nb_mbuf_per_pool,
1201 						       socket_ids[i]);
1202 	} else {
1203 		if (socket_num == UMA_NO_CONFIG)
1204 			mempools[0] = mbuf_pool_create(mbuf_data_size,
1205 						       nb_mbuf_per_pool, 0);
1206 		else
1207 			mempools[socket_num] = mbuf_pool_create
1208 							(mbuf_data_size,
1209 							 nb_mbuf_per_pool,
1210 							 socket_num);
1211 	}
1212 
1213 	init_port_config();
1214 
1215 	gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1216 		DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1217 	/*
1218 	 * Records which Mbuf pool to use by each logical core, if needed.
1219 	 */
1220 	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1221 		mbp = mbuf_pool_find(
1222 			rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1223 
1224 		if (mbp == NULL)
1225 			mbp = mbuf_pool_find(0);
1226 		fwd_lcores[lc_id]->mbp = mbp;
1227 		/* initialize GSO context */
1228 		fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1229 		fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1230 		fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1231 		fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1232 			RTE_ETHER_CRC_LEN;
1233 		fwd_lcores[lc_id]->gso_ctx.flag = 0;
1234 	}
1235 
1236 	/* Configuration of packet forwarding streams. */
1237 	if (init_fwd_streams() < 0)
1238 		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1239 
1240 	fwd_config_setup();
1241 
1242 	/* create a gro context for each lcore */
1243 	gro_param.gro_types = RTE_GRO_TCP_IPV4;
1244 	gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1245 	gro_param.max_item_per_flow = MAX_PKT_BURST;
1246 	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1247 		gro_param.socket_id = rte_lcore_to_socket_id(
1248 				fwd_lcores_cpuids[lc_id]);
1249 		fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1250 		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1251 			rte_exit(EXIT_FAILURE,
1252 					"rte_gro_ctx_create() failed\n");
1253 		}
1254 	}
1255 
1256 #if defined RTE_LIBRTE_PMD_SOFTNIC
1257 	if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1258 		RTE_ETH_FOREACH_DEV(pid) {
1259 			port = &ports[pid];
1260 			const char *driver = port->dev_info.driver_name;
1261 
1262 			if (strcmp(driver, "net_softnic") == 0)
1263 				port->softport.fwd_lcore_arg = fwd_lcores;
1264 		}
1265 	}
1266 #endif
1267 
1268 }
1269 
1270 
1271 void
1272 reconfig(portid_t new_port_id, unsigned socket_id)
1273 {
1274 	struct rte_port *port;
1275 	int ret;
1276 
1277 	/* Reconfiguration of Ethernet ports. */
1278 	port = &ports[new_port_id];
1279 
1280 	ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1281 	if (ret != 0)
1282 		return;
1283 
1284 	/* set flag to initialize port/queue */
1285 	port->need_reconfig = 1;
1286 	port->need_reconfig_queues = 1;
1287 	port->socket_id = socket_id;
1288 
1289 	init_port_config();
1290 }
1291 
1292 
1293 int
1294 init_fwd_streams(void)
1295 {
1296 	portid_t pid;
1297 	struct rte_port *port;
1298 	streamid_t sm_id, nb_fwd_streams_new;
1299 	queueid_t q;
1300 
1301 	/* set socket id according to numa or not */
1302 	RTE_ETH_FOREACH_DEV(pid) {
1303 		port = &ports[pid];
1304 		if (nb_rxq > port->dev_info.max_rx_queues) {
1305 			printf("Fail: nb_rxq(%d) is greater than "
1306 				"max_rx_queues(%d)\n", nb_rxq,
1307 				port->dev_info.max_rx_queues);
1308 			return -1;
1309 		}
1310 		if (nb_txq > port->dev_info.max_tx_queues) {
1311 			printf("Fail: nb_txq(%d) is greater than "
1312 				"max_tx_queues(%d)\n", nb_txq,
1313 				port->dev_info.max_tx_queues);
1314 			return -1;
1315 		}
1316 		if (numa_support) {
1317 			if (port_numa[pid] != NUMA_NO_CONFIG)
1318 				port->socket_id = port_numa[pid];
1319 			else {
1320 				port->socket_id = rte_eth_dev_socket_id(pid);
1321 
1322 				/*
1323 				 * if socket_id is invalid,
1324 				 * set to the first available socket.
1325 				 */
1326 				if (check_socket_id(port->socket_id) < 0)
1327 					port->socket_id = socket_ids[0];
1328 			}
1329 		}
1330 		else {
1331 			if (socket_num == UMA_NO_CONFIG)
1332 				port->socket_id = 0;
1333 			else
1334 				port->socket_id = socket_num;
1335 		}
1336 	}
1337 
1338 	q = RTE_MAX(nb_rxq, nb_txq);
1339 	if (q == 0) {
1340 		printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1341 		return -1;
1342 	}
1343 	nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1344 	if (nb_fwd_streams_new == nb_fwd_streams)
1345 		return 0;
1346 	/* clear the old */
1347 	if (fwd_streams != NULL) {
1348 		for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1349 			if (fwd_streams[sm_id] == NULL)
1350 				continue;
1351 			rte_free(fwd_streams[sm_id]);
1352 			fwd_streams[sm_id] = NULL;
1353 		}
1354 		rte_free(fwd_streams);
1355 		fwd_streams = NULL;
1356 	}
1357 
1358 	/* init new */
1359 	nb_fwd_streams = nb_fwd_streams_new;
1360 	if (nb_fwd_streams) {
1361 		fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1362 			sizeof(struct fwd_stream *) * nb_fwd_streams,
1363 			RTE_CACHE_LINE_SIZE);
1364 		if (fwd_streams == NULL)
1365 			rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1366 				 " (struct fwd_stream *)) failed\n",
1367 				 nb_fwd_streams);
1368 
1369 		for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1370 			fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1371 				" struct fwd_stream", sizeof(struct fwd_stream),
1372 				RTE_CACHE_LINE_SIZE);
1373 			if (fwd_streams[sm_id] == NULL)
1374 				rte_exit(EXIT_FAILURE, "rte_zmalloc"
1375 					 "(struct fwd_stream) failed\n");
1376 		}
1377 	}
1378 
1379 	return 0;
1380 }
1381 
1382 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1383 static void
1384 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1385 {
1386 	unsigned int total_burst;
1387 	unsigned int nb_burst;
1388 	unsigned int burst_stats[3];
1389 	uint16_t pktnb_stats[3];
1390 	uint16_t nb_pkt;
1391 	int burst_percent[3];
1392 
1393 	/*
1394 	 * First compute the total number of packet bursts and the
1395 	 * two highest numbers of bursts of the same number of packets.
1396 	 */
1397 	total_burst = 0;
1398 	burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1399 	pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1400 	for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1401 		nb_burst = pbs->pkt_burst_spread[nb_pkt];
1402 		if (nb_burst == 0)
1403 			continue;
1404 		total_burst += nb_burst;
1405 		if (nb_burst > burst_stats[0]) {
1406 			burst_stats[1] = burst_stats[0];
1407 			pktnb_stats[1] = pktnb_stats[0];
1408 			burst_stats[0] = nb_burst;
1409 			pktnb_stats[0] = nb_pkt;
1410 		} else if (nb_burst > burst_stats[1]) {
1411 			burst_stats[1] = nb_burst;
1412 			pktnb_stats[1] = nb_pkt;
1413 		}
1414 	}
1415 	if (total_burst == 0)
1416 		return;
1417 	burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1418 	printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1419 	       burst_percent[0], (int) pktnb_stats[0]);
1420 	if (burst_stats[0] == total_burst) {
1421 		printf("]\n");
1422 		return;
1423 	}
1424 	if (burst_stats[0] + burst_stats[1] == total_burst) {
1425 		printf(" + %d%% of %d pkts]\n",
1426 		       100 - burst_percent[0], pktnb_stats[1]);
1427 		return;
1428 	}
1429 	burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1430 	burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1431 	if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1432 		printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1433 		return;
1434 	}
1435 	printf(" + %d%% of %d pkts + %d%% of others]\n",
1436 	       burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1437 }
1438 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1439 
1440 static void
1441 fwd_stream_stats_display(streamid_t stream_id)
1442 {
1443 	struct fwd_stream *fs;
1444 	static const char *fwd_top_stats_border = "-------";
1445 
1446 	fs = fwd_streams[stream_id];
1447 	if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1448 	    (fs->fwd_dropped == 0))
1449 		return;
1450 	printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1451 	       "TX Port=%2d/Queue=%2d %s\n",
1452 	       fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1453 	       fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1454 	printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1455 	       " TX-dropped: %-14"PRIu64,
1456 	       fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1457 
1458 	/* if checksum mode */
1459 	if (cur_fwd_eng == &csum_fwd_engine) {
1460 		printf("  RX- bad IP checksum: %-14"PRIu64
1461 		       "  Rx- bad L4 checksum: %-14"PRIu64
1462 		       " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1463 			fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1464 			fs->rx_bad_outer_l4_csum);
1465 	} else {
1466 		printf("\n");
1467 	}
1468 
1469 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1470 	pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1471 	pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1472 #endif
1473 }
1474 
1475 void
1476 fwd_stats_display(void)
1477 {
1478 	static const char *fwd_stats_border = "----------------------";
1479 	static const char *acc_stats_border = "+++++++++++++++";
1480 	struct {
1481 		struct fwd_stream *rx_stream;
1482 		struct fwd_stream *tx_stream;
1483 		uint64_t tx_dropped;
1484 		uint64_t rx_bad_ip_csum;
1485 		uint64_t rx_bad_l4_csum;
1486 		uint64_t rx_bad_outer_l4_csum;
1487 	} ports_stats[RTE_MAX_ETHPORTS];
1488 	uint64_t total_rx_dropped = 0;
1489 	uint64_t total_tx_dropped = 0;
1490 	uint64_t total_rx_nombuf = 0;
1491 	struct rte_eth_stats stats;
1492 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1493 	uint64_t fwd_cycles = 0;
1494 #endif
1495 	uint64_t total_recv = 0;
1496 	uint64_t total_xmit = 0;
1497 	struct rte_port *port;
1498 	streamid_t sm_id;
1499 	portid_t pt_id;
1500 	int i;
1501 
1502 	memset(ports_stats, 0, sizeof(ports_stats));
1503 
1504 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1505 		struct fwd_stream *fs = fwd_streams[sm_id];
1506 
1507 		if (cur_fwd_config.nb_fwd_streams >
1508 		    cur_fwd_config.nb_fwd_ports) {
1509 			fwd_stream_stats_display(sm_id);
1510 		} else {
1511 			ports_stats[fs->tx_port].tx_stream = fs;
1512 			ports_stats[fs->rx_port].rx_stream = fs;
1513 		}
1514 
1515 		ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1516 
1517 		ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1518 		ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1519 		ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1520 				fs->rx_bad_outer_l4_csum;
1521 
1522 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1523 		fwd_cycles += fs->core_cycles;
1524 #endif
1525 	}
1526 	for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1527 		uint8_t j;
1528 
1529 		pt_id = fwd_ports_ids[i];
1530 		port = &ports[pt_id];
1531 
1532 		rte_eth_stats_get(pt_id, &stats);
1533 		stats.ipackets -= port->stats.ipackets;
1534 		stats.opackets -= port->stats.opackets;
1535 		stats.ibytes -= port->stats.ibytes;
1536 		stats.obytes -= port->stats.obytes;
1537 		stats.imissed -= port->stats.imissed;
1538 		stats.oerrors -= port->stats.oerrors;
1539 		stats.rx_nombuf -= port->stats.rx_nombuf;
1540 
1541 		total_recv += stats.ipackets;
1542 		total_xmit += stats.opackets;
1543 		total_rx_dropped += stats.imissed;
1544 		total_tx_dropped += ports_stats[pt_id].tx_dropped;
1545 		total_tx_dropped += stats.oerrors;
1546 		total_rx_nombuf  += stats.rx_nombuf;
1547 
1548 		printf("\n  %s Forward statistics for port %-2d %s\n",
1549 		       fwd_stats_border, pt_id, fwd_stats_border);
1550 
1551 		if (!port->rx_queue_stats_mapping_enabled &&
1552 		    !port->tx_queue_stats_mapping_enabled) {
1553 			printf("  RX-packets: %-14"PRIu64
1554 			       " RX-dropped: %-14"PRIu64
1555 			       "RX-total: %-"PRIu64"\n",
1556 			       stats.ipackets, stats.imissed,
1557 			       stats.ipackets + stats.imissed);
1558 
1559 			if (cur_fwd_eng == &csum_fwd_engine)
1560 				printf("  Bad-ipcsum: %-14"PRIu64
1561 				       " Bad-l4csum: %-14"PRIu64
1562 				       "Bad-outer-l4csum: %-14"PRIu64"\n",
1563 				       ports_stats[pt_id].rx_bad_ip_csum,
1564 				       ports_stats[pt_id].rx_bad_l4_csum,
1565 				       ports_stats[pt_id].rx_bad_outer_l4_csum);
1566 			if (stats.ierrors + stats.rx_nombuf > 0) {
1567 				printf("  RX-error: %-"PRIu64"\n",
1568 				       stats.ierrors);
1569 				printf("  RX-nombufs: %-14"PRIu64"\n",
1570 				       stats.rx_nombuf);
1571 			}
1572 
1573 			printf("  TX-packets: %-14"PRIu64
1574 			       " TX-dropped: %-14"PRIu64
1575 			       "TX-total: %-"PRIu64"\n",
1576 			       stats.opackets, ports_stats[pt_id].tx_dropped,
1577 			       stats.opackets + ports_stats[pt_id].tx_dropped);
1578 		} else {
1579 			printf("  RX-packets:             %14"PRIu64
1580 			       "    RX-dropped:%14"PRIu64
1581 			       "    RX-total:%14"PRIu64"\n",
1582 			       stats.ipackets, stats.imissed,
1583 			       stats.ipackets + stats.imissed);
1584 
1585 			if (cur_fwd_eng == &csum_fwd_engine)
1586 				printf("  Bad-ipcsum:%14"PRIu64
1587 				       "    Bad-l4csum:%14"PRIu64
1588 				       "    Bad-outer-l4csum: %-14"PRIu64"\n",
1589 				       ports_stats[pt_id].rx_bad_ip_csum,
1590 				       ports_stats[pt_id].rx_bad_l4_csum,
1591 				       ports_stats[pt_id].rx_bad_outer_l4_csum);
1592 			if ((stats.ierrors + stats.rx_nombuf) > 0) {
1593 				printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1594 				printf("  RX-nombufs:             %14"PRIu64"\n",
1595 				       stats.rx_nombuf);
1596 			}
1597 
1598 			printf("  TX-packets:             %14"PRIu64
1599 			       "    TX-dropped:%14"PRIu64
1600 			       "    TX-total:%14"PRIu64"\n",
1601 			       stats.opackets, ports_stats[pt_id].tx_dropped,
1602 			       stats.opackets + ports_stats[pt_id].tx_dropped);
1603 		}
1604 
1605 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1606 		if (ports_stats[pt_id].rx_stream)
1607 			pkt_burst_stats_display("RX",
1608 				&ports_stats[pt_id].rx_stream->rx_burst_stats);
1609 		if (ports_stats[pt_id].tx_stream)
1610 			pkt_burst_stats_display("TX",
1611 				&ports_stats[pt_id].tx_stream->tx_burst_stats);
1612 #endif
1613 
1614 		if (port->rx_queue_stats_mapping_enabled) {
1615 			printf("\n");
1616 			for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1617 				printf("  Stats reg %2d RX-packets:%14"PRIu64
1618 				       "     RX-errors:%14"PRIu64
1619 				       "    RX-bytes:%14"PRIu64"\n",
1620 				       j, stats.q_ipackets[j],
1621 				       stats.q_errors[j], stats.q_ibytes[j]);
1622 			}
1623 			printf("\n");
1624 		}
1625 		if (port->tx_queue_stats_mapping_enabled) {
1626 			for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1627 				printf("  Stats reg %2d TX-packets:%14"PRIu64
1628 				       "                                 TX-bytes:%14"
1629 				       PRIu64"\n",
1630 				       j, stats.q_opackets[j],
1631 				       stats.q_obytes[j]);
1632 			}
1633 		}
1634 
1635 		printf("  %s--------------------------------%s\n",
1636 		       fwd_stats_border, fwd_stats_border);
1637 	}
1638 
1639 	printf("\n  %s Accumulated forward statistics for all ports"
1640 	       "%s\n",
1641 	       acc_stats_border, acc_stats_border);
1642 	printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1643 	       "%-"PRIu64"\n"
1644 	       "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1645 	       "%-"PRIu64"\n",
1646 	       total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1647 	       total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1648 	if (total_rx_nombuf > 0)
1649 		printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1650 	printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1651 	       "%s\n",
1652 	       acc_stats_border, acc_stats_border);
1653 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1654 	if (total_recv > 0)
1655 		printf("\n  CPU cycles/packet=%u (total cycles="
1656 		       "%"PRIu64" / total RX packets=%"PRIu64")\n",
1657 		       (unsigned int)(fwd_cycles / total_recv),
1658 		       fwd_cycles, total_recv);
1659 #endif
1660 }
1661 
1662 void
1663 fwd_stats_reset(void)
1664 {
1665 	streamid_t sm_id;
1666 	portid_t pt_id;
1667 	int i;
1668 
1669 	for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1670 		pt_id = fwd_ports_ids[i];
1671 		rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1672 	}
1673 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1674 		struct fwd_stream *fs = fwd_streams[sm_id];
1675 
1676 		fs->rx_packets = 0;
1677 		fs->tx_packets = 0;
1678 		fs->fwd_dropped = 0;
1679 		fs->rx_bad_ip_csum = 0;
1680 		fs->rx_bad_l4_csum = 0;
1681 		fs->rx_bad_outer_l4_csum = 0;
1682 
1683 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1684 		memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1685 		memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1686 #endif
1687 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1688 		fs->core_cycles = 0;
1689 #endif
1690 	}
1691 }
1692 
1693 static void
1694 flush_fwd_rx_queues(void)
1695 {
1696 	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1697 	portid_t  rxp;
1698 	portid_t port_id;
1699 	queueid_t rxq;
1700 	uint16_t  nb_rx;
1701 	uint16_t  i;
1702 	uint8_t   j;
1703 	uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1704 	uint64_t timer_period;
1705 
1706 	/* convert to number of cycles */
1707 	timer_period = rte_get_timer_hz(); /* 1 second timeout */
1708 
1709 	for (j = 0; j < 2; j++) {
1710 		for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1711 			for (rxq = 0; rxq < nb_rxq; rxq++) {
1712 				port_id = fwd_ports_ids[rxp];
1713 				/**
1714 				* testpmd can stuck in the below do while loop
1715 				* if rte_eth_rx_burst() always returns nonzero
1716 				* packets. So timer is added to exit this loop
1717 				* after 1sec timer expiry.
1718 				*/
1719 				prev_tsc = rte_rdtsc();
1720 				do {
1721 					nb_rx = rte_eth_rx_burst(port_id, rxq,
1722 						pkts_burst, MAX_PKT_BURST);
1723 					for (i = 0; i < nb_rx; i++)
1724 						rte_pktmbuf_free(pkts_burst[i]);
1725 
1726 					cur_tsc = rte_rdtsc();
1727 					diff_tsc = cur_tsc - prev_tsc;
1728 					timer_tsc += diff_tsc;
1729 				} while ((nb_rx > 0) &&
1730 					(timer_tsc < timer_period));
1731 				timer_tsc = 0;
1732 			}
1733 		}
1734 		rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1735 	}
1736 }
1737 
1738 static void
1739 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1740 {
1741 	struct fwd_stream **fsm;
1742 	streamid_t nb_fs;
1743 	streamid_t sm_id;
1744 #ifdef RTE_LIBRTE_BITRATE
1745 	uint64_t tics_per_1sec;
1746 	uint64_t tics_datum;
1747 	uint64_t tics_current;
1748 	uint16_t i, cnt_ports;
1749 
1750 	cnt_ports = nb_ports;
1751 	tics_datum = rte_rdtsc();
1752 	tics_per_1sec = rte_get_timer_hz();
1753 #endif
1754 	fsm = &fwd_streams[fc->stream_idx];
1755 	nb_fs = fc->stream_nb;
1756 	do {
1757 		for (sm_id = 0; sm_id < nb_fs; sm_id++)
1758 			(*pkt_fwd)(fsm[sm_id]);
1759 #ifdef RTE_LIBRTE_BITRATE
1760 		if (bitrate_enabled != 0 &&
1761 				bitrate_lcore_id == rte_lcore_id()) {
1762 			tics_current = rte_rdtsc();
1763 			if (tics_current - tics_datum >= tics_per_1sec) {
1764 				/* Periodic bitrate calculation */
1765 				for (i = 0; i < cnt_ports; i++)
1766 					rte_stats_bitrate_calc(bitrate_data,
1767 						ports_ids[i]);
1768 				tics_datum = tics_current;
1769 			}
1770 		}
1771 #endif
1772 #ifdef RTE_LIBRTE_LATENCY_STATS
1773 		if (latencystats_enabled != 0 &&
1774 				latencystats_lcore_id == rte_lcore_id())
1775 			rte_latencystats_update();
1776 #endif
1777 
1778 	} while (! fc->stopped);
1779 }
1780 
1781 static int
1782 start_pkt_forward_on_core(void *fwd_arg)
1783 {
1784 	run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1785 			     cur_fwd_config.fwd_eng->packet_fwd);
1786 	return 0;
1787 }
1788 
1789 /*
1790  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1791  * Used to start communication flows in network loopback test configurations.
1792  */
1793 static int
1794 run_one_txonly_burst_on_core(void *fwd_arg)
1795 {
1796 	struct fwd_lcore *fwd_lc;
1797 	struct fwd_lcore tmp_lcore;
1798 
1799 	fwd_lc = (struct fwd_lcore *) fwd_arg;
1800 	tmp_lcore = *fwd_lc;
1801 	tmp_lcore.stopped = 1;
1802 	run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1803 	return 0;
1804 }
1805 
1806 /*
1807  * Launch packet forwarding:
1808  *     - Setup per-port forwarding context.
1809  *     - launch logical cores with their forwarding configuration.
1810  */
1811 static void
1812 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1813 {
1814 	port_fwd_begin_t port_fwd_begin;
1815 	unsigned int i;
1816 	unsigned int lc_id;
1817 	int diag;
1818 
1819 	port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1820 	if (port_fwd_begin != NULL) {
1821 		for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1822 			(*port_fwd_begin)(fwd_ports_ids[i]);
1823 	}
1824 	for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1825 		lc_id = fwd_lcores_cpuids[i];
1826 		if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1827 			fwd_lcores[i]->stopped = 0;
1828 			diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1829 						     fwd_lcores[i], lc_id);
1830 			if (diag != 0)
1831 				printf("launch lcore %u failed - diag=%d\n",
1832 				       lc_id, diag);
1833 		}
1834 	}
1835 }
1836 
1837 /*
1838  * Launch packet forwarding configuration.
1839  */
1840 void
1841 start_packet_forwarding(int with_tx_first)
1842 {
1843 	port_fwd_begin_t port_fwd_begin;
1844 	port_fwd_end_t  port_fwd_end;
1845 	struct rte_port *port;
1846 	unsigned int i;
1847 	portid_t   pt_id;
1848 
1849 	if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1850 		rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1851 
1852 	if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1853 		rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1854 
1855 	if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1856 		strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1857 		(!nb_rxq || !nb_txq))
1858 		rte_exit(EXIT_FAILURE,
1859 			"Either rxq or txq are 0, cannot use %s fwd mode\n",
1860 			cur_fwd_eng->fwd_mode_name);
1861 
1862 	if (all_ports_started() == 0) {
1863 		printf("Not all ports were started\n");
1864 		return;
1865 	}
1866 	if (test_done == 0) {
1867 		printf("Packet forwarding already started\n");
1868 		return;
1869 	}
1870 
1871 
1872 	if(dcb_test) {
1873 		for (i = 0; i < nb_fwd_ports; i++) {
1874 			pt_id = fwd_ports_ids[i];
1875 			port = &ports[pt_id];
1876 			if (!port->dcb_flag) {
1877 				printf("In DCB mode, all forwarding ports must "
1878                                        "be configured in this mode.\n");
1879 				return;
1880 			}
1881 		}
1882 		if (nb_fwd_lcores == 1) {
1883 			printf("In DCB mode,the nb forwarding cores "
1884                                "should be larger than 1.\n");
1885 			return;
1886 		}
1887 	}
1888 	test_done = 0;
1889 
1890 	fwd_config_setup();
1891 
1892 	if(!no_flush_rx)
1893 		flush_fwd_rx_queues();
1894 
1895 	pkt_fwd_config_display(&cur_fwd_config);
1896 	rxtx_config_display();
1897 
1898 	fwd_stats_reset();
1899 	for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1900 		pt_id = fwd_ports_ids[i];
1901 		port = &ports[pt_id];
1902 		map_port_queue_stats_mapping_registers(pt_id, port);
1903 	}
1904 	if (with_tx_first) {
1905 		port_fwd_begin = tx_only_engine.port_fwd_begin;
1906 		if (port_fwd_begin != NULL) {
1907 			for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1908 				(*port_fwd_begin)(fwd_ports_ids[i]);
1909 		}
1910 		while (with_tx_first--) {
1911 			launch_packet_forwarding(
1912 					run_one_txonly_burst_on_core);
1913 			rte_eal_mp_wait_lcore();
1914 		}
1915 		port_fwd_end = tx_only_engine.port_fwd_end;
1916 		if (port_fwd_end != NULL) {
1917 			for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1918 				(*port_fwd_end)(fwd_ports_ids[i]);
1919 		}
1920 	}
1921 	launch_packet_forwarding(start_pkt_forward_on_core);
1922 }
1923 
1924 void
1925 stop_packet_forwarding(void)
1926 {
1927 	port_fwd_end_t port_fwd_end;
1928 	lcoreid_t lc_id;
1929 	portid_t pt_id;
1930 	int i;
1931 
1932 	if (test_done) {
1933 		printf("Packet forwarding not started\n");
1934 		return;
1935 	}
1936 	printf("Telling cores to stop...");
1937 	for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1938 		fwd_lcores[lc_id]->stopped = 1;
1939 	printf("\nWaiting for lcores to finish...\n");
1940 	rte_eal_mp_wait_lcore();
1941 	port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1942 	if (port_fwd_end != NULL) {
1943 		for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1944 			pt_id = fwd_ports_ids[i];
1945 			(*port_fwd_end)(pt_id);
1946 		}
1947 	}
1948 
1949 	fwd_stats_display();
1950 
1951 	printf("\nDone.\n");
1952 	test_done = 1;
1953 }
1954 
1955 void
1956 dev_set_link_up(portid_t pid)
1957 {
1958 	if (rte_eth_dev_set_link_up(pid) < 0)
1959 		printf("\nSet link up fail.\n");
1960 }
1961 
1962 void
1963 dev_set_link_down(portid_t pid)
1964 {
1965 	if (rte_eth_dev_set_link_down(pid) < 0)
1966 		printf("\nSet link down fail.\n");
1967 }
1968 
1969 static int
1970 all_ports_started(void)
1971 {
1972 	portid_t pi;
1973 	struct rte_port *port;
1974 
1975 	RTE_ETH_FOREACH_DEV(pi) {
1976 		port = &ports[pi];
1977 		/* Check if there is a port which is not started */
1978 		if ((port->port_status != RTE_PORT_STARTED) &&
1979 			(port->slave_flag == 0))
1980 			return 0;
1981 	}
1982 
1983 	/* No port is not started */
1984 	return 1;
1985 }
1986 
1987 int
1988 port_is_stopped(portid_t port_id)
1989 {
1990 	struct rte_port *port = &ports[port_id];
1991 
1992 	if ((port->port_status != RTE_PORT_STOPPED) &&
1993 	    (port->slave_flag == 0))
1994 		return 0;
1995 	return 1;
1996 }
1997 
1998 int
1999 all_ports_stopped(void)
2000 {
2001 	portid_t pi;
2002 
2003 	RTE_ETH_FOREACH_DEV(pi) {
2004 		if (!port_is_stopped(pi))
2005 			return 0;
2006 	}
2007 
2008 	return 1;
2009 }
2010 
2011 int
2012 port_is_started(portid_t port_id)
2013 {
2014 	if (port_id_is_invalid(port_id, ENABLED_WARN))
2015 		return 0;
2016 
2017 	if (ports[port_id].port_status != RTE_PORT_STARTED)
2018 		return 0;
2019 
2020 	return 1;
2021 }
2022 
2023 int
2024 start_port(portid_t pid)
2025 {
2026 	int diag, need_check_link_status = -1;
2027 	portid_t pi;
2028 	queueid_t qi;
2029 	struct rte_port *port;
2030 	struct rte_ether_addr mac_addr;
2031 
2032 	if (port_id_is_invalid(pid, ENABLED_WARN))
2033 		return 0;
2034 
2035 	if(dcb_config)
2036 		dcb_test = 1;
2037 	RTE_ETH_FOREACH_DEV(pi) {
2038 		if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2039 			continue;
2040 
2041 		need_check_link_status = 0;
2042 		port = &ports[pi];
2043 		if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2044 						 RTE_PORT_HANDLING) == 0) {
2045 			printf("Port %d is now not stopped\n", pi);
2046 			continue;
2047 		}
2048 
2049 		if (port->need_reconfig > 0) {
2050 			port->need_reconfig = 0;
2051 
2052 			if (flow_isolate_all) {
2053 				int ret = port_flow_isolate(pi, 1);
2054 				if (ret) {
2055 					printf("Failed to apply isolated"
2056 					       " mode on port %d\n", pi);
2057 					return -1;
2058 				}
2059 			}
2060 			configure_rxtx_dump_callbacks(0);
2061 			printf("Configuring Port %d (socket %u)\n", pi,
2062 					port->socket_id);
2063 			/* configure port */
2064 			diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2065 						&(port->dev_conf));
2066 			if (diag != 0) {
2067 				if (rte_atomic16_cmpset(&(port->port_status),
2068 				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2069 					printf("Port %d can not be set back "
2070 							"to stopped\n", pi);
2071 				printf("Fail to configure port %d\n", pi);
2072 				/* try to reconfigure port next time */
2073 				port->need_reconfig = 1;
2074 				return -1;
2075 			}
2076 		}
2077 		if (port->need_reconfig_queues > 0) {
2078 			port->need_reconfig_queues = 0;
2079 			/* setup tx queues */
2080 			for (qi = 0; qi < nb_txq; qi++) {
2081 				if ((numa_support) &&
2082 					(txring_numa[pi] != NUMA_NO_CONFIG))
2083 					diag = rte_eth_tx_queue_setup(pi, qi,
2084 						port->nb_tx_desc[qi],
2085 						txring_numa[pi],
2086 						&(port->tx_conf[qi]));
2087 				else
2088 					diag = rte_eth_tx_queue_setup(pi, qi,
2089 						port->nb_tx_desc[qi],
2090 						port->socket_id,
2091 						&(port->tx_conf[qi]));
2092 
2093 				if (diag == 0)
2094 					continue;
2095 
2096 				/* Fail to setup tx queue, return */
2097 				if (rte_atomic16_cmpset(&(port->port_status),
2098 							RTE_PORT_HANDLING,
2099 							RTE_PORT_STOPPED) == 0)
2100 					printf("Port %d can not be set back "
2101 							"to stopped\n", pi);
2102 				printf("Fail to configure port %d tx queues\n",
2103 				       pi);
2104 				/* try to reconfigure queues next time */
2105 				port->need_reconfig_queues = 1;
2106 				return -1;
2107 			}
2108 			for (qi = 0; qi < nb_rxq; qi++) {
2109 				/* setup rx queues */
2110 				if ((numa_support) &&
2111 					(rxring_numa[pi] != NUMA_NO_CONFIG)) {
2112 					struct rte_mempool * mp =
2113 						mbuf_pool_find(rxring_numa[pi]);
2114 					if (mp == NULL) {
2115 						printf("Failed to setup RX queue:"
2116 							"No mempool allocation"
2117 							" on the socket %d\n",
2118 							rxring_numa[pi]);
2119 						return -1;
2120 					}
2121 
2122 					diag = rte_eth_rx_queue_setup(pi, qi,
2123 					     port->nb_rx_desc[qi],
2124 					     rxring_numa[pi],
2125 					     &(port->rx_conf[qi]),
2126 					     mp);
2127 				} else {
2128 					struct rte_mempool *mp =
2129 						mbuf_pool_find(port->socket_id);
2130 					if (mp == NULL) {
2131 						printf("Failed to setup RX queue:"
2132 							"No mempool allocation"
2133 							" on the socket %d\n",
2134 							port->socket_id);
2135 						return -1;
2136 					}
2137 					diag = rte_eth_rx_queue_setup(pi, qi,
2138 					     port->nb_rx_desc[qi],
2139 					     port->socket_id,
2140 					     &(port->rx_conf[qi]),
2141 					     mp);
2142 				}
2143 				if (diag == 0)
2144 					continue;
2145 
2146 				/* Fail to setup rx queue, return */
2147 				if (rte_atomic16_cmpset(&(port->port_status),
2148 							RTE_PORT_HANDLING,
2149 							RTE_PORT_STOPPED) == 0)
2150 					printf("Port %d can not be set back "
2151 							"to stopped\n", pi);
2152 				printf("Fail to configure port %d rx queues\n",
2153 				       pi);
2154 				/* try to reconfigure queues next time */
2155 				port->need_reconfig_queues = 1;
2156 				return -1;
2157 			}
2158 		}
2159 		configure_rxtx_dump_callbacks(verbose_level);
2160 		/* start port */
2161 		if (rte_eth_dev_start(pi) < 0) {
2162 			printf("Fail to start port %d\n", pi);
2163 
2164 			/* Fail to setup rx queue, return */
2165 			if (rte_atomic16_cmpset(&(port->port_status),
2166 				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2167 				printf("Port %d can not be set back to "
2168 							"stopped\n", pi);
2169 			continue;
2170 		}
2171 
2172 		if (rte_atomic16_cmpset(&(port->port_status),
2173 			RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2174 			printf("Port %d can not be set into started\n", pi);
2175 
2176 		if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2177 			printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2178 				mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2179 				mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2180 				mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2181 
2182 		/* at least one port started, need checking link status */
2183 		need_check_link_status = 1;
2184 	}
2185 
2186 	if (need_check_link_status == 1 && !no_link_check)
2187 		check_all_ports_link_status(RTE_PORT_ALL);
2188 	else if (need_check_link_status == 0)
2189 		printf("Please stop the ports first\n");
2190 
2191 	printf("Done\n");
2192 	return 0;
2193 }
2194 
2195 void
2196 stop_port(portid_t pid)
2197 {
2198 	portid_t pi;
2199 	struct rte_port *port;
2200 	int need_check_link_status = 0;
2201 
2202 	if (dcb_test) {
2203 		dcb_test = 0;
2204 		dcb_config = 0;
2205 	}
2206 
2207 	if (port_id_is_invalid(pid, ENABLED_WARN))
2208 		return;
2209 
2210 	printf("Stopping ports...\n");
2211 
2212 	RTE_ETH_FOREACH_DEV(pi) {
2213 		if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2214 			continue;
2215 
2216 		if (port_is_forwarding(pi) != 0 && test_done == 0) {
2217 			printf("Please remove port %d from forwarding configuration.\n", pi);
2218 			continue;
2219 		}
2220 
2221 		if (port_is_bonding_slave(pi)) {
2222 			printf("Please remove port %d from bonded device.\n", pi);
2223 			continue;
2224 		}
2225 
2226 		port = &ports[pi];
2227 		if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2228 						RTE_PORT_HANDLING) == 0)
2229 			continue;
2230 
2231 		rte_eth_dev_stop(pi);
2232 
2233 		if (rte_atomic16_cmpset(&(port->port_status),
2234 			RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2235 			printf("Port %d can not be set into stopped\n", pi);
2236 		need_check_link_status = 1;
2237 	}
2238 	if (need_check_link_status && !no_link_check)
2239 		check_all_ports_link_status(RTE_PORT_ALL);
2240 
2241 	printf("Done\n");
2242 }
2243 
2244 static void
2245 remove_invalid_ports_in(portid_t *array, portid_t *total)
2246 {
2247 	portid_t i;
2248 	portid_t new_total = 0;
2249 
2250 	for (i = 0; i < *total; i++)
2251 		if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2252 			array[new_total] = array[i];
2253 			new_total++;
2254 		}
2255 	*total = new_total;
2256 }
2257 
2258 static void
2259 remove_invalid_ports(void)
2260 {
2261 	remove_invalid_ports_in(ports_ids, &nb_ports);
2262 	remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2263 	nb_cfg_ports = nb_fwd_ports;
2264 }
2265 
2266 void
2267 close_port(portid_t pid)
2268 {
2269 	portid_t pi;
2270 	struct rte_port *port;
2271 
2272 	if (port_id_is_invalid(pid, ENABLED_WARN))
2273 		return;
2274 
2275 	printf("Closing ports...\n");
2276 
2277 	RTE_ETH_FOREACH_DEV(pi) {
2278 		if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2279 			continue;
2280 
2281 		if (port_is_forwarding(pi) != 0 && test_done == 0) {
2282 			printf("Please remove port %d from forwarding configuration.\n", pi);
2283 			continue;
2284 		}
2285 
2286 		if (port_is_bonding_slave(pi)) {
2287 			printf("Please remove port %d from bonded device.\n", pi);
2288 			continue;
2289 		}
2290 
2291 		port = &ports[pi];
2292 		if (rte_atomic16_cmpset(&(port->port_status),
2293 			RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2294 			printf("Port %d is already closed\n", pi);
2295 			continue;
2296 		}
2297 
2298 		if (rte_atomic16_cmpset(&(port->port_status),
2299 			RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2300 			printf("Port %d is now not stopped\n", pi);
2301 			continue;
2302 		}
2303 
2304 		if (port->flow_list)
2305 			port_flow_flush(pi);
2306 		rte_eth_dev_close(pi);
2307 
2308 		remove_invalid_ports();
2309 
2310 		if (rte_atomic16_cmpset(&(port->port_status),
2311 			RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2312 			printf("Port %d cannot be set to closed\n", pi);
2313 	}
2314 
2315 	printf("Done\n");
2316 }
2317 
2318 void
2319 reset_port(portid_t pid)
2320 {
2321 	int diag;
2322 	portid_t pi;
2323 	struct rte_port *port;
2324 
2325 	if (port_id_is_invalid(pid, ENABLED_WARN))
2326 		return;
2327 
2328 	if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2329 		(pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2330 		printf("Can not reset port(s), please stop port(s) first.\n");
2331 		return;
2332 	}
2333 
2334 	printf("Resetting ports...\n");
2335 
2336 	RTE_ETH_FOREACH_DEV(pi) {
2337 		if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2338 			continue;
2339 
2340 		if (port_is_forwarding(pi) != 0 && test_done == 0) {
2341 			printf("Please remove port %d from forwarding "
2342 			       "configuration.\n", pi);
2343 			continue;
2344 		}
2345 
2346 		if (port_is_bonding_slave(pi)) {
2347 			printf("Please remove port %d from bonded device.\n",
2348 			       pi);
2349 			continue;
2350 		}
2351 
2352 		diag = rte_eth_dev_reset(pi);
2353 		if (diag == 0) {
2354 			port = &ports[pi];
2355 			port->need_reconfig = 1;
2356 			port->need_reconfig_queues = 1;
2357 		} else {
2358 			printf("Failed to reset port %d. diag=%d\n", pi, diag);
2359 		}
2360 	}
2361 
2362 	printf("Done\n");
2363 }
2364 
2365 void
2366 attach_port(char *identifier)
2367 {
2368 	portid_t pi;
2369 	struct rte_dev_iterator iterator;
2370 
2371 	printf("Attaching a new port...\n");
2372 
2373 	if (identifier == NULL) {
2374 		printf("Invalid parameters are specified\n");
2375 		return;
2376 	}
2377 
2378 	if (rte_dev_probe(identifier) < 0) {
2379 		TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2380 		return;
2381 	}
2382 
2383 	/* first attach mode: event */
2384 	if (setup_on_probe_event) {
2385 		/* new ports are detected on RTE_ETH_EVENT_NEW event */
2386 		for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2387 			if (ports[pi].port_status == RTE_PORT_HANDLING &&
2388 					ports[pi].need_setup != 0)
2389 				setup_attached_port(pi);
2390 		return;
2391 	}
2392 
2393 	/* second attach mode: iterator */
2394 	RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2395 		/* setup ports matching the devargs used for probing */
2396 		if (port_is_forwarding(pi))
2397 			continue; /* port was already attached before */
2398 		setup_attached_port(pi);
2399 	}
2400 }
2401 
2402 static void
2403 setup_attached_port(portid_t pi)
2404 {
2405 	unsigned int socket_id;
2406 	int ret;
2407 
2408 	socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2409 	/* if socket_id is invalid, set to the first available socket. */
2410 	if (check_socket_id(socket_id) < 0)
2411 		socket_id = socket_ids[0];
2412 	reconfig(pi, socket_id);
2413 	ret = rte_eth_promiscuous_enable(pi);
2414 	if (ret != 0)
2415 		printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2416 			pi, rte_strerror(-ret));
2417 
2418 	ports_ids[nb_ports++] = pi;
2419 	fwd_ports_ids[nb_fwd_ports++] = pi;
2420 	nb_cfg_ports = nb_fwd_ports;
2421 	ports[pi].need_setup = 0;
2422 	ports[pi].port_status = RTE_PORT_STOPPED;
2423 
2424 	printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2425 	printf("Done\n");
2426 }
2427 
2428 void
2429 detach_port_device(portid_t port_id)
2430 {
2431 	struct rte_device *dev;
2432 	portid_t sibling;
2433 
2434 	printf("Removing a device...\n");
2435 
2436 	dev = rte_eth_devices[port_id].device;
2437 	if (dev == NULL) {
2438 		printf("Device already removed\n");
2439 		return;
2440 	}
2441 
2442 	if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2443 		if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2444 			printf("Port not stopped\n");
2445 			return;
2446 		}
2447 		printf("Port was not closed\n");
2448 		if (ports[port_id].flow_list)
2449 			port_flow_flush(port_id);
2450 	}
2451 
2452 	if (rte_dev_remove(dev) < 0) {
2453 		TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2454 		return;
2455 	}
2456 	RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2457 		/* reset mapping between old ports and removed device */
2458 		rte_eth_devices[sibling].device = NULL;
2459 		if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2460 			/* sibling ports are forced to be closed */
2461 			ports[sibling].port_status = RTE_PORT_CLOSED;
2462 			printf("Port %u is closed\n", sibling);
2463 		}
2464 	}
2465 
2466 	remove_invalid_ports();
2467 
2468 	printf("Device of port %u is detached\n", port_id);
2469 	printf("Now total ports is %d\n", nb_ports);
2470 	printf("Done\n");
2471 	return;
2472 }
2473 
2474 void
2475 detach_device(char *identifier)
2476 {
2477 	struct rte_dev_iterator iterator;
2478 	struct rte_devargs da;
2479 	portid_t port_id;
2480 
2481 	printf("Removing a device...\n");
2482 
2483 	memset(&da, 0, sizeof(da));
2484 	if (rte_devargs_parsef(&da, "%s", identifier)) {
2485 		printf("cannot parse identifier\n");
2486 		if (da.args)
2487 			free(da.args);
2488 		return;
2489 	}
2490 
2491 	RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2492 		if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2493 			if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2494 				printf("Port %u not stopped\n", port_id);
2495 				return;
2496 			}
2497 
2498 			/* sibling ports are forced to be closed */
2499 			if (ports[port_id].flow_list)
2500 				port_flow_flush(port_id);
2501 			ports[port_id].port_status = RTE_PORT_CLOSED;
2502 			printf("Port %u is now closed\n", port_id);
2503 		}
2504 	}
2505 
2506 	if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2507 		TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2508 			    da.name, da.bus->name);
2509 		return;
2510 	}
2511 
2512 	remove_invalid_ports();
2513 
2514 	printf("Device %s is detached\n", identifier);
2515 	printf("Now total ports is %d\n", nb_ports);
2516 	printf("Done\n");
2517 }
2518 
2519 void
2520 pmd_test_exit(void)
2521 {
2522 	portid_t pt_id;
2523 	int ret;
2524 	int i;
2525 
2526 	if (test_done == 0)
2527 		stop_packet_forwarding();
2528 
2529 	for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2530 		if (mempools[i]) {
2531 			if (mp_alloc_type == MP_ALLOC_ANON)
2532 				rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2533 						     NULL);
2534 		}
2535 	}
2536 	if (ports != NULL) {
2537 		no_link_check = 1;
2538 		RTE_ETH_FOREACH_DEV(pt_id) {
2539 			printf("\nStopping port %d...\n", pt_id);
2540 			fflush(stdout);
2541 			stop_port(pt_id);
2542 		}
2543 		RTE_ETH_FOREACH_DEV(pt_id) {
2544 			printf("\nShutting down port %d...\n", pt_id);
2545 			fflush(stdout);
2546 			close_port(pt_id);
2547 		}
2548 	}
2549 
2550 	if (hot_plug) {
2551 		ret = rte_dev_event_monitor_stop();
2552 		if (ret) {
2553 			RTE_LOG(ERR, EAL,
2554 				"fail to stop device event monitor.");
2555 			return;
2556 		}
2557 
2558 		ret = rte_dev_event_callback_unregister(NULL,
2559 			dev_event_callback, NULL);
2560 		if (ret < 0) {
2561 			RTE_LOG(ERR, EAL,
2562 				"fail to unregister device event callback.\n");
2563 			return;
2564 		}
2565 
2566 		ret = rte_dev_hotplug_handle_disable();
2567 		if (ret) {
2568 			RTE_LOG(ERR, EAL,
2569 				"fail to disable hotplug handling.\n");
2570 			return;
2571 		}
2572 	}
2573 	for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2574 		if (mempools[i])
2575 			rte_mempool_free(mempools[i]);
2576 	}
2577 
2578 	printf("\nBye...\n");
2579 }
2580 
2581 typedef void (*cmd_func_t)(void);
2582 struct pmd_test_command {
2583 	const char *cmd_name;
2584 	cmd_func_t cmd_func;
2585 };
2586 
2587 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2588 
2589 /* Check the link status of all ports in up to 9s, and print them finally */
2590 static void
2591 check_all_ports_link_status(uint32_t port_mask)
2592 {
2593 #define CHECK_INTERVAL 100 /* 100ms */
2594 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2595 	portid_t portid;
2596 	uint8_t count, all_ports_up, print_flag = 0;
2597 	struct rte_eth_link link;
2598 	int ret;
2599 
2600 	printf("Checking link statuses...\n");
2601 	fflush(stdout);
2602 	for (count = 0; count <= MAX_CHECK_TIME; count++) {
2603 		all_ports_up = 1;
2604 		RTE_ETH_FOREACH_DEV(portid) {
2605 			if ((port_mask & (1 << portid)) == 0)
2606 				continue;
2607 			memset(&link, 0, sizeof(link));
2608 			ret = rte_eth_link_get_nowait(portid, &link);
2609 			if (ret < 0) {
2610 				all_ports_up = 0;
2611 				if (print_flag == 1)
2612 					printf("Port %u link get failed: %s\n",
2613 						portid, rte_strerror(-ret));
2614 				continue;
2615 			}
2616 			/* print link status if flag set */
2617 			if (print_flag == 1) {
2618 				if (link.link_status)
2619 					printf(
2620 					"Port%d Link Up. speed %u Mbps- %s\n",
2621 					portid, link.link_speed,
2622 				(link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2623 					("full-duplex") : ("half-duplex\n"));
2624 				else
2625 					printf("Port %d Link Down\n", portid);
2626 				continue;
2627 			}
2628 			/* clear all_ports_up flag if any link down */
2629 			if (link.link_status == ETH_LINK_DOWN) {
2630 				all_ports_up = 0;
2631 				break;
2632 			}
2633 		}
2634 		/* after finally printing all link status, get out */
2635 		if (print_flag == 1)
2636 			break;
2637 
2638 		if (all_ports_up == 0) {
2639 			fflush(stdout);
2640 			rte_delay_ms(CHECK_INTERVAL);
2641 		}
2642 
2643 		/* set the print_flag if all ports up or timeout */
2644 		if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2645 			print_flag = 1;
2646 		}
2647 
2648 		if (lsc_interrupt)
2649 			break;
2650 	}
2651 }
2652 
2653 /*
2654  * This callback is for remove a port for a device. It has limitation because
2655  * it is not for multiple port removal for a device.
2656  * TODO: the device detach invoke will plan to be removed from user side to
2657  * eal. And convert all PMDs to free port resources on ether device closing.
2658  */
2659 static void
2660 rmv_port_callback(void *arg)
2661 {
2662 	int need_to_start = 0;
2663 	int org_no_link_check = no_link_check;
2664 	portid_t port_id = (intptr_t)arg;
2665 
2666 	RTE_ETH_VALID_PORTID_OR_RET(port_id);
2667 
2668 	if (!test_done && port_is_forwarding(port_id)) {
2669 		need_to_start = 1;
2670 		stop_packet_forwarding();
2671 	}
2672 	no_link_check = 1;
2673 	stop_port(port_id);
2674 	no_link_check = org_no_link_check;
2675 	close_port(port_id);
2676 	detach_port_device(port_id);
2677 	if (need_to_start)
2678 		start_packet_forwarding(0);
2679 }
2680 
2681 /* This function is used by the interrupt thread */
2682 static int
2683 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2684 		  void *ret_param)
2685 {
2686 	RTE_SET_USED(param);
2687 	RTE_SET_USED(ret_param);
2688 
2689 	if (type >= RTE_ETH_EVENT_MAX) {
2690 		fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2691 			port_id, __func__, type);
2692 		fflush(stderr);
2693 	} else if (event_print_mask & (UINT32_C(1) << type)) {
2694 		printf("\nPort %" PRIu16 ": %s event\n", port_id,
2695 			eth_event_desc[type]);
2696 		fflush(stdout);
2697 	}
2698 
2699 	switch (type) {
2700 	case RTE_ETH_EVENT_NEW:
2701 		ports[port_id].need_setup = 1;
2702 		ports[port_id].port_status = RTE_PORT_HANDLING;
2703 		break;
2704 	case RTE_ETH_EVENT_INTR_RMV:
2705 		if (port_id_is_invalid(port_id, DISABLED_WARN))
2706 			break;
2707 		if (rte_eal_alarm_set(100000,
2708 				rmv_port_callback, (void *)(intptr_t)port_id))
2709 			fprintf(stderr, "Could not set up deferred device removal\n");
2710 		break;
2711 	default:
2712 		break;
2713 	}
2714 	return 0;
2715 }
2716 
2717 static int
2718 register_eth_event_callback(void)
2719 {
2720 	int ret;
2721 	enum rte_eth_event_type event;
2722 
2723 	for (event = RTE_ETH_EVENT_UNKNOWN;
2724 			event < RTE_ETH_EVENT_MAX; event++) {
2725 		ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2726 				event,
2727 				eth_event_callback,
2728 				NULL);
2729 		if (ret != 0) {
2730 			TESTPMD_LOG(ERR, "Failed to register callback for "
2731 					"%s event\n", eth_event_desc[event]);
2732 			return -1;
2733 		}
2734 	}
2735 
2736 	return 0;
2737 }
2738 
2739 /* This function is used by the interrupt thread */
2740 static void
2741 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2742 			     __rte_unused void *arg)
2743 {
2744 	uint16_t port_id;
2745 	int ret;
2746 
2747 	if (type >= RTE_DEV_EVENT_MAX) {
2748 		fprintf(stderr, "%s called upon invalid event %d\n",
2749 			__func__, type);
2750 		fflush(stderr);
2751 	}
2752 
2753 	switch (type) {
2754 	case RTE_DEV_EVENT_REMOVE:
2755 		RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2756 			device_name);
2757 		ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2758 		if (ret) {
2759 			RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2760 				device_name);
2761 			return;
2762 		}
2763 		/*
2764 		 * Because the user's callback is invoked in eal interrupt
2765 		 * callback, the interrupt callback need to be finished before
2766 		 * it can be unregistered when detaching device. So finish
2767 		 * callback soon and use a deferred removal to detach device
2768 		 * is need. It is a workaround, once the device detaching be
2769 		 * moved into the eal in the future, the deferred removal could
2770 		 * be deleted.
2771 		 */
2772 		if (rte_eal_alarm_set(100000,
2773 				rmv_port_callback, (void *)(intptr_t)port_id))
2774 			RTE_LOG(ERR, EAL,
2775 				"Could not set up deferred device removal\n");
2776 		break;
2777 	case RTE_DEV_EVENT_ADD:
2778 		RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2779 			device_name);
2780 		/* TODO: After finish kernel driver binding,
2781 		 * begin to attach port.
2782 		 */
2783 		break;
2784 	default:
2785 		break;
2786 	}
2787 }
2788 
2789 static int
2790 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2791 {
2792 	uint16_t i;
2793 	int diag;
2794 	uint8_t mapping_found = 0;
2795 
2796 	for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2797 		if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2798 				(tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2799 			diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2800 					tx_queue_stats_mappings[i].queue_id,
2801 					tx_queue_stats_mappings[i].stats_counter_id);
2802 			if (diag != 0)
2803 				return diag;
2804 			mapping_found = 1;
2805 		}
2806 	}
2807 	if (mapping_found)
2808 		port->tx_queue_stats_mapping_enabled = 1;
2809 	return 0;
2810 }
2811 
2812 static int
2813 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2814 {
2815 	uint16_t i;
2816 	int diag;
2817 	uint8_t mapping_found = 0;
2818 
2819 	for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2820 		if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2821 				(rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2822 			diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2823 					rx_queue_stats_mappings[i].queue_id,
2824 					rx_queue_stats_mappings[i].stats_counter_id);
2825 			if (diag != 0)
2826 				return diag;
2827 			mapping_found = 1;
2828 		}
2829 	}
2830 	if (mapping_found)
2831 		port->rx_queue_stats_mapping_enabled = 1;
2832 	return 0;
2833 }
2834 
2835 static void
2836 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2837 {
2838 	int diag = 0;
2839 
2840 	diag = set_tx_queue_stats_mapping_registers(pi, port);
2841 	if (diag != 0) {
2842 		if (diag == -ENOTSUP) {
2843 			port->tx_queue_stats_mapping_enabled = 0;
2844 			printf("TX queue stats mapping not supported port id=%d\n", pi);
2845 		}
2846 		else
2847 			rte_exit(EXIT_FAILURE,
2848 					"set_tx_queue_stats_mapping_registers "
2849 					"failed for port id=%d diag=%d\n",
2850 					pi, diag);
2851 	}
2852 
2853 	diag = set_rx_queue_stats_mapping_registers(pi, port);
2854 	if (diag != 0) {
2855 		if (diag == -ENOTSUP) {
2856 			port->rx_queue_stats_mapping_enabled = 0;
2857 			printf("RX queue stats mapping not supported port id=%d\n", pi);
2858 		}
2859 		else
2860 			rte_exit(EXIT_FAILURE,
2861 					"set_rx_queue_stats_mapping_registers "
2862 					"failed for port id=%d diag=%d\n",
2863 					pi, diag);
2864 	}
2865 }
2866 
2867 static void
2868 rxtx_port_config(struct rte_port *port)
2869 {
2870 	uint16_t qid;
2871 	uint64_t offloads;
2872 
2873 	for (qid = 0; qid < nb_rxq; qid++) {
2874 		offloads = port->rx_conf[qid].offloads;
2875 		port->rx_conf[qid] = port->dev_info.default_rxconf;
2876 		if (offloads != 0)
2877 			port->rx_conf[qid].offloads = offloads;
2878 
2879 		/* Check if any Rx parameters have been passed */
2880 		if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2881 			port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2882 
2883 		if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2884 			port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2885 
2886 		if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2887 			port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2888 
2889 		if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2890 			port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2891 
2892 		if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2893 			port->rx_conf[qid].rx_drop_en = rx_drop_en;
2894 
2895 		port->nb_rx_desc[qid] = nb_rxd;
2896 	}
2897 
2898 	for (qid = 0; qid < nb_txq; qid++) {
2899 		offloads = port->tx_conf[qid].offloads;
2900 		port->tx_conf[qid] = port->dev_info.default_txconf;
2901 		if (offloads != 0)
2902 			port->tx_conf[qid].offloads = offloads;
2903 
2904 		/* Check if any Tx parameters have been passed */
2905 		if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2906 			port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2907 
2908 		if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2909 			port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2910 
2911 		if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2912 			port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2913 
2914 		if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2915 			port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2916 
2917 		if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2918 			port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2919 
2920 		port->nb_tx_desc[qid] = nb_txd;
2921 	}
2922 }
2923 
2924 void
2925 init_port_config(void)
2926 {
2927 	portid_t pid;
2928 	struct rte_port *port;
2929 	int ret;
2930 
2931 	RTE_ETH_FOREACH_DEV(pid) {
2932 		port = &ports[pid];
2933 		port->dev_conf.fdir_conf = fdir_conf;
2934 
2935 		ret = eth_dev_info_get_print_err(pid, &port->dev_info);
2936 		if (ret != 0)
2937 			return;
2938 
2939 		if (nb_rxq > 1) {
2940 			port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2941 			port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2942 				rss_hf & port->dev_info.flow_type_rss_offloads;
2943 		} else {
2944 			port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2945 			port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2946 		}
2947 
2948 		if (port->dcb_flag == 0) {
2949 			if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2950 				port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2951 			else
2952 				port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2953 		}
2954 
2955 		rxtx_port_config(port);
2956 
2957 		ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
2958 		if (ret != 0)
2959 			return;
2960 
2961 		map_port_queue_stats_mapping_registers(pid, port);
2962 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2963 		rte_pmd_ixgbe_bypass_init(pid);
2964 #endif
2965 
2966 		if (lsc_interrupt &&
2967 		    (rte_eth_devices[pid].data->dev_flags &
2968 		     RTE_ETH_DEV_INTR_LSC))
2969 			port->dev_conf.intr_conf.lsc = 1;
2970 		if (rmv_interrupt &&
2971 		    (rte_eth_devices[pid].data->dev_flags &
2972 		     RTE_ETH_DEV_INTR_RMV))
2973 			port->dev_conf.intr_conf.rmv = 1;
2974 	}
2975 }
2976 
2977 void set_port_slave_flag(portid_t slave_pid)
2978 {
2979 	struct rte_port *port;
2980 
2981 	port = &ports[slave_pid];
2982 	port->slave_flag = 1;
2983 }
2984 
2985 void clear_port_slave_flag(portid_t slave_pid)
2986 {
2987 	struct rte_port *port;
2988 
2989 	port = &ports[slave_pid];
2990 	port->slave_flag = 0;
2991 }
2992 
2993 uint8_t port_is_bonding_slave(portid_t slave_pid)
2994 {
2995 	struct rte_port *port;
2996 
2997 	port = &ports[slave_pid];
2998 	if ((rte_eth_devices[slave_pid].data->dev_flags &
2999 	    RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3000 		return 1;
3001 	return 0;
3002 }
3003 
3004 const uint16_t vlan_tags[] = {
3005 		0,  1,  2,  3,  4,  5,  6,  7,
3006 		8,  9, 10, 11,  12, 13, 14, 15,
3007 		16, 17, 18, 19, 20, 21, 22, 23,
3008 		24, 25, 26, 27, 28, 29, 30, 31
3009 };
3010 
3011 static  int
3012 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3013 		 enum dcb_mode_enable dcb_mode,
3014 		 enum rte_eth_nb_tcs num_tcs,
3015 		 uint8_t pfc_en)
3016 {
3017 	uint8_t i;
3018 	int32_t rc;
3019 	struct rte_eth_rss_conf rss_conf;
3020 
3021 	/*
3022 	 * Builds up the correct configuration for dcb+vt based on the vlan tags array
3023 	 * given above, and the number of traffic classes available for use.
3024 	 */
3025 	if (dcb_mode == DCB_VT_ENABLED) {
3026 		struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3027 				&eth_conf->rx_adv_conf.vmdq_dcb_conf;
3028 		struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3029 				&eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3030 
3031 		/* VMDQ+DCB RX and TX configurations */
3032 		vmdq_rx_conf->enable_default_pool = 0;
3033 		vmdq_rx_conf->default_pool = 0;
3034 		vmdq_rx_conf->nb_queue_pools =
3035 			(num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3036 		vmdq_tx_conf->nb_queue_pools =
3037 			(num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3038 
3039 		vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3040 		for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3041 			vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3042 			vmdq_rx_conf->pool_map[i].pools =
3043 				1 << (i % vmdq_rx_conf->nb_queue_pools);
3044 		}
3045 		for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3046 			vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3047 			vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3048 		}
3049 
3050 		/* set DCB mode of RX and TX of multiple queues */
3051 		eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3052 		eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3053 	} else {
3054 		struct rte_eth_dcb_rx_conf *rx_conf =
3055 				&eth_conf->rx_adv_conf.dcb_rx_conf;
3056 		struct rte_eth_dcb_tx_conf *tx_conf =
3057 				&eth_conf->tx_adv_conf.dcb_tx_conf;
3058 
3059 		rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3060 		if (rc != 0)
3061 			return rc;
3062 
3063 		rx_conf->nb_tcs = num_tcs;
3064 		tx_conf->nb_tcs = num_tcs;
3065 
3066 		for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3067 			rx_conf->dcb_tc[i] = i % num_tcs;
3068 			tx_conf->dcb_tc[i] = i % num_tcs;
3069 		}
3070 
3071 		eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3072 		eth_conf->rx_adv_conf.rss_conf = rss_conf;
3073 		eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3074 	}
3075 
3076 	if (pfc_en)
3077 		eth_conf->dcb_capability_en =
3078 				ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3079 	else
3080 		eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3081 
3082 	return 0;
3083 }
3084 
3085 int
3086 init_port_dcb_config(portid_t pid,
3087 		     enum dcb_mode_enable dcb_mode,
3088 		     enum rte_eth_nb_tcs num_tcs,
3089 		     uint8_t pfc_en)
3090 {
3091 	struct rte_eth_conf port_conf;
3092 	struct rte_port *rte_port;
3093 	int retval;
3094 	uint16_t i;
3095 
3096 	rte_port = &ports[pid];
3097 
3098 	memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3099 	/* Enter DCB configuration status */
3100 	dcb_config = 1;
3101 
3102 	port_conf.rxmode = rte_port->dev_conf.rxmode;
3103 	port_conf.txmode = rte_port->dev_conf.txmode;
3104 
3105 	/*set configuration of DCB in vt mode and DCB in non-vt mode*/
3106 	retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3107 	if (retval < 0)
3108 		return retval;
3109 	port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3110 
3111 	/* re-configure the device . */
3112 	retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3113 	if (retval < 0)
3114 		return retval;
3115 
3116 	retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3117 	if (retval != 0)
3118 		return retval;
3119 
3120 	/* If dev_info.vmdq_pool_base is greater than 0,
3121 	 * the queue id of vmdq pools is started after pf queues.
3122 	 */
3123 	if (dcb_mode == DCB_VT_ENABLED &&
3124 	    rte_port->dev_info.vmdq_pool_base > 0) {
3125 		printf("VMDQ_DCB multi-queue mode is nonsensical"
3126 			" for port %d.", pid);
3127 		return -1;
3128 	}
3129 
3130 	/* Assume the ports in testpmd have the same dcb capability
3131 	 * and has the same number of rxq and txq in dcb mode
3132 	 */
3133 	if (dcb_mode == DCB_VT_ENABLED) {
3134 		if (rte_port->dev_info.max_vfs > 0) {
3135 			nb_rxq = rte_port->dev_info.nb_rx_queues;
3136 			nb_txq = rte_port->dev_info.nb_tx_queues;
3137 		} else {
3138 			nb_rxq = rte_port->dev_info.max_rx_queues;
3139 			nb_txq = rte_port->dev_info.max_tx_queues;
3140 		}
3141 	} else {
3142 		/*if vt is disabled, use all pf queues */
3143 		if (rte_port->dev_info.vmdq_pool_base == 0) {
3144 			nb_rxq = rte_port->dev_info.max_rx_queues;
3145 			nb_txq = rte_port->dev_info.max_tx_queues;
3146 		} else {
3147 			nb_rxq = (queueid_t)num_tcs;
3148 			nb_txq = (queueid_t)num_tcs;
3149 
3150 		}
3151 	}
3152 	rx_free_thresh = 64;
3153 
3154 	memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3155 
3156 	rxtx_port_config(rte_port);
3157 	/* VLAN filter */
3158 	rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3159 	for (i = 0; i < RTE_DIM(vlan_tags); i++)
3160 		rx_vft_set(pid, vlan_tags[i], 1);
3161 
3162 	retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3163 	if (retval != 0)
3164 		return retval;
3165 
3166 	map_port_queue_stats_mapping_registers(pid, rte_port);
3167 
3168 	rte_port->dcb_flag = 1;
3169 
3170 	return 0;
3171 }
3172 
3173 static void
3174 init_port(void)
3175 {
3176 	/* Configuration of Ethernet ports. */
3177 	ports = rte_zmalloc("testpmd: ports",
3178 			    sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3179 			    RTE_CACHE_LINE_SIZE);
3180 	if (ports == NULL) {
3181 		rte_exit(EXIT_FAILURE,
3182 				"rte_zmalloc(%d struct rte_port) failed\n",
3183 				RTE_MAX_ETHPORTS);
3184 	}
3185 
3186 	/* Initialize ports NUMA structures */
3187 	memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3188 	memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3189 	memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3190 }
3191 
3192 static void
3193 force_quit(void)
3194 {
3195 	pmd_test_exit();
3196 	prompt_exit();
3197 }
3198 
3199 static void
3200 print_stats(void)
3201 {
3202 	uint8_t i;
3203 	const char clr[] = { 27, '[', '2', 'J', '\0' };
3204 	const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3205 
3206 	/* Clear screen and move to top left */
3207 	printf("%s%s", clr, top_left);
3208 
3209 	printf("\nPort statistics ====================================");
3210 	for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3211 		nic_stats_display(fwd_ports_ids[i]);
3212 
3213 	fflush(stdout);
3214 }
3215 
3216 static void
3217 signal_handler(int signum)
3218 {
3219 	if (signum == SIGINT || signum == SIGTERM) {
3220 		printf("\nSignal %d received, preparing to exit...\n",
3221 				signum);
3222 #ifdef RTE_LIBRTE_PDUMP
3223 		/* uninitialize packet capture framework */
3224 		rte_pdump_uninit();
3225 #endif
3226 #ifdef RTE_LIBRTE_LATENCY_STATS
3227 		if (latencystats_enabled != 0)
3228 			rte_latencystats_uninit();
3229 #endif
3230 		force_quit();
3231 		/* Set flag to indicate the force termination. */
3232 		f_quit = 1;
3233 		/* exit with the expected status */
3234 		signal(signum, SIG_DFL);
3235 		kill(getpid(), signum);
3236 	}
3237 }
3238 
3239 int
3240 main(int argc, char** argv)
3241 {
3242 	int diag;
3243 	portid_t port_id;
3244 	uint16_t count;
3245 	int ret;
3246 
3247 	signal(SIGINT, signal_handler);
3248 	signal(SIGTERM, signal_handler);
3249 
3250 	testpmd_logtype = rte_log_register("testpmd");
3251 	if (testpmd_logtype < 0)
3252 		rte_exit(EXIT_FAILURE, "Cannot register log type");
3253 	rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3254 
3255 	diag = rte_eal_init(argc, argv);
3256 	if (diag < 0)
3257 		rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3258 			 rte_strerror(rte_errno));
3259 
3260 	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3261 		rte_exit(EXIT_FAILURE,
3262 			 "Secondary process type not supported.\n");
3263 
3264 	ret = register_eth_event_callback();
3265 	if (ret != 0)
3266 		rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3267 
3268 #ifdef RTE_LIBRTE_PDUMP
3269 	/* initialize packet capture framework */
3270 	rte_pdump_init();
3271 #endif
3272 
3273 	count = 0;
3274 	RTE_ETH_FOREACH_DEV(port_id) {
3275 		ports_ids[count] = port_id;
3276 		count++;
3277 	}
3278 	nb_ports = (portid_t) count;
3279 	if (nb_ports == 0)
3280 		TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3281 
3282 	/* allocate port structures, and init them */
3283 	init_port();
3284 
3285 	set_def_fwd_config();
3286 	if (nb_lcores == 0)
3287 		rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3288 			 "Check the core mask argument\n");
3289 
3290 	/* Bitrate/latency stats disabled by default */
3291 #ifdef RTE_LIBRTE_BITRATE
3292 	bitrate_enabled = 0;
3293 #endif
3294 #ifdef RTE_LIBRTE_LATENCY_STATS
3295 	latencystats_enabled = 0;
3296 #endif
3297 
3298 	/* on FreeBSD, mlockall() is disabled by default */
3299 #ifdef RTE_EXEC_ENV_FREEBSD
3300 	do_mlockall = 0;
3301 #else
3302 	do_mlockall = 1;
3303 #endif
3304 
3305 	argc -= diag;
3306 	argv += diag;
3307 	if (argc > 1)
3308 		launch_args_parse(argc, argv);
3309 
3310 	if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3311 		TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3312 			strerror(errno));
3313 	}
3314 
3315 	if (tx_first && interactive)
3316 		rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3317 				"interactive mode.\n");
3318 
3319 	if (tx_first && lsc_interrupt) {
3320 		printf("Warning: lsc_interrupt needs to be off when "
3321 				" using tx_first. Disabling.\n");
3322 		lsc_interrupt = 0;
3323 	}
3324 
3325 	if (!nb_rxq && !nb_txq)
3326 		printf("Warning: Either rx or tx queues should be non-zero\n");
3327 
3328 	if (nb_rxq > 1 && nb_rxq > nb_txq)
3329 		printf("Warning: nb_rxq=%d enables RSS configuration, "
3330 		       "but nb_txq=%d will prevent to fully test it.\n",
3331 		       nb_rxq, nb_txq);
3332 
3333 	init_config();
3334 
3335 	if (hot_plug) {
3336 		ret = rte_dev_hotplug_handle_enable();
3337 		if (ret) {
3338 			RTE_LOG(ERR, EAL,
3339 				"fail to enable hotplug handling.");
3340 			return -1;
3341 		}
3342 
3343 		ret = rte_dev_event_monitor_start();
3344 		if (ret) {
3345 			RTE_LOG(ERR, EAL,
3346 				"fail to start device event monitoring.");
3347 			return -1;
3348 		}
3349 
3350 		ret = rte_dev_event_callback_register(NULL,
3351 			dev_event_callback, NULL);
3352 		if (ret) {
3353 			RTE_LOG(ERR, EAL,
3354 				"fail  to register device event callback\n");
3355 			return -1;
3356 		}
3357 	}
3358 
3359 	if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3360 		rte_exit(EXIT_FAILURE, "Start ports failed\n");
3361 
3362 	/* set all ports to promiscuous mode by default */
3363 	RTE_ETH_FOREACH_DEV(port_id) {
3364 		ret = rte_eth_promiscuous_enable(port_id);
3365 		if (ret != 0)
3366 			printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3367 				port_id, rte_strerror(-ret));
3368 	}
3369 
3370 	/* Init metrics library */
3371 	rte_metrics_init(rte_socket_id());
3372 
3373 #ifdef RTE_LIBRTE_LATENCY_STATS
3374 	if (latencystats_enabled != 0) {
3375 		int ret = rte_latencystats_init(1, NULL);
3376 		if (ret)
3377 			printf("Warning: latencystats init()"
3378 				" returned error %d\n",	ret);
3379 		printf("Latencystats running on lcore %d\n",
3380 			latencystats_lcore_id);
3381 	}
3382 #endif
3383 
3384 	/* Setup bitrate stats */
3385 #ifdef RTE_LIBRTE_BITRATE
3386 	if (bitrate_enabled != 0) {
3387 		bitrate_data = rte_stats_bitrate_create();
3388 		if (bitrate_data == NULL)
3389 			rte_exit(EXIT_FAILURE,
3390 				"Could not allocate bitrate data.\n");
3391 		rte_stats_bitrate_reg(bitrate_data);
3392 	}
3393 #endif
3394 
3395 #ifdef RTE_LIBRTE_CMDLINE
3396 	if (strlen(cmdline_filename) != 0)
3397 		cmdline_read_from_file(cmdline_filename);
3398 
3399 	if (interactive == 1) {
3400 		if (auto_start) {
3401 			printf("Start automatic packet forwarding\n");
3402 			start_packet_forwarding(0);
3403 		}
3404 		prompt();
3405 		pmd_test_exit();
3406 	} else
3407 #endif
3408 	{
3409 		char c;
3410 		int rc;
3411 
3412 		f_quit = 0;
3413 
3414 		printf("No commandline core given, start packet forwarding\n");
3415 		start_packet_forwarding(tx_first);
3416 		if (stats_period != 0) {
3417 			uint64_t prev_time = 0, cur_time, diff_time = 0;
3418 			uint64_t timer_period;
3419 
3420 			/* Convert to number of cycles */
3421 			timer_period = stats_period * rte_get_timer_hz();
3422 
3423 			while (f_quit == 0) {
3424 				cur_time = rte_get_timer_cycles();
3425 				diff_time += cur_time - prev_time;
3426 
3427 				if (diff_time >= timer_period) {
3428 					print_stats();
3429 					/* Reset the timer */
3430 					diff_time = 0;
3431 				}
3432 				/* Sleep to avoid unnecessary checks */
3433 				prev_time = cur_time;
3434 				sleep(1);
3435 			}
3436 		}
3437 
3438 		printf("Press enter to exit\n");
3439 		rc = read(0, &c, 1);
3440 		pmd_test_exit();
3441 		if (rc < 0)
3442 			return 1;
3443 	}
3444 
3445 	return 0;
3446 }
3447