1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIB_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIB_LATENCYSTATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use main core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 220 221 /* 222 * Configuration of packet segments used by the "txonly" processing engine. 223 */ 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 226 TXONLY_DEF_PACKET_LEN, 227 }; 228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 229 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 231 /**< Split policy for packets to TX. */ 232 233 uint8_t txonly_multi_flow; 234 /**< Whether multiple flows are generated in TXONLY mode. */ 235 236 uint32_t tx_pkt_times_inter; 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 238 239 uint32_t tx_pkt_times_intra; 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 241 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 243 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 244 245 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 246 uint8_t dcb_config = 0; 247 248 /* Whether the dcb is in testing status */ 249 uint8_t dcb_test = 0; 250 251 /* 252 * Configurable number of RX/TX queues. 253 */ 254 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 255 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 256 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 257 258 /* 259 * Configurable number of RX/TX ring descriptors. 260 * Defaults are supplied by drivers via ethdev. 261 */ 262 #define RTE_TEST_RX_DESC_DEFAULT 0 263 #define RTE_TEST_TX_DESC_DEFAULT 0 264 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 265 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 266 267 #define RTE_PMD_PARAM_UNSET -1 268 /* 269 * Configurable values of RX and TX ring threshold registers. 270 */ 271 272 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 273 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 274 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 275 276 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 277 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 278 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 279 280 /* 281 * Configurable value of RX free threshold. 282 */ 283 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 284 285 /* 286 * Configurable value of RX drop enable. 287 */ 288 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 289 290 /* 291 * Configurable value of TX free threshold. 292 */ 293 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 294 295 /* 296 * Configurable value of TX RS bit threshold. 297 */ 298 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 299 300 /* 301 * Configurable value of buffered packets before sending. 302 */ 303 uint16_t noisy_tx_sw_bufsz; 304 305 /* 306 * Configurable value of packet buffer timeout. 307 */ 308 uint16_t noisy_tx_sw_buf_flush_time; 309 310 /* 311 * Configurable value for size of VNF internal memory area 312 * used for simulating noisy neighbour behaviour 313 */ 314 uint64_t noisy_lkup_mem_sz; 315 316 /* 317 * Configurable value of number of random writes done in 318 * VNF simulation memory area. 319 */ 320 uint64_t noisy_lkup_num_writes; 321 322 /* 323 * Configurable value of number of random reads done in 324 * VNF simulation memory area. 325 */ 326 uint64_t noisy_lkup_num_reads; 327 328 /* 329 * Configurable value of number of random reads/writes done in 330 * VNF simulation memory area. 331 */ 332 uint64_t noisy_lkup_num_reads_writes; 333 334 /* 335 * Receive Side Scaling (RSS) configuration. 336 */ 337 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 338 339 /* 340 * Port topology configuration 341 */ 342 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 343 344 /* 345 * Avoids to flush all the RX streams before starts forwarding. 346 */ 347 uint8_t no_flush_rx = 0; /* flush by default */ 348 349 /* 350 * Flow API isolated mode. 351 */ 352 uint8_t flow_isolate_all; 353 354 /* 355 * Avoids to check link status when starting/stopping a port. 356 */ 357 uint8_t no_link_check = 0; /* check by default */ 358 359 /* 360 * Don't automatically start all ports in interactive mode. 361 */ 362 uint8_t no_device_start = 0; 363 364 /* 365 * Enable link status change notification 366 */ 367 uint8_t lsc_interrupt = 1; /* enabled by default */ 368 369 /* 370 * Enable device removal notification. 371 */ 372 uint8_t rmv_interrupt = 1; /* enabled by default */ 373 374 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 375 376 /* After attach, port setup is called on event or by iterator */ 377 bool setup_on_probe_event = true; 378 379 /* Clear ptypes on port initialization. */ 380 uint8_t clear_ptypes = true; 381 382 /* Hairpin ports configuration mode. */ 383 uint16_t hairpin_mode; 384 385 /* Pretty printing of ethdev events */ 386 static const char * const eth_event_desc[] = { 387 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 388 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 389 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 390 [RTE_ETH_EVENT_INTR_RESET] = "reset", 391 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 392 [RTE_ETH_EVENT_IPSEC] = "IPsec", 393 [RTE_ETH_EVENT_MACSEC] = "MACsec", 394 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 395 [RTE_ETH_EVENT_NEW] = "device probed", 396 [RTE_ETH_EVENT_DESTROY] = "device released", 397 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 398 [RTE_ETH_EVENT_MAX] = NULL, 399 }; 400 401 /* 402 * Display or mask ether events 403 * Default to all events except VF_MBOX 404 */ 405 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 406 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 407 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 408 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 409 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 410 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 411 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 412 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 413 /* 414 * Decide if all memory are locked for performance. 415 */ 416 int do_mlockall = 0; 417 418 /* 419 * NIC bypass mode configuration options. 420 */ 421 422 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 423 /* The NIC bypass watchdog timeout. */ 424 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 425 #endif 426 427 428 #ifdef RTE_LIB_LATENCYSTATS 429 430 /* 431 * Set when latency stats is enabled in the commandline 432 */ 433 uint8_t latencystats_enabled; 434 435 /* 436 * Lcore ID to serive latency statistics. 437 */ 438 lcoreid_t latencystats_lcore_id = -1; 439 440 #endif 441 442 /* 443 * Ethernet device configuration. 444 */ 445 struct rte_eth_rxmode rx_mode = { 446 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 447 /**< Default maximum frame length. */ 448 }; 449 450 struct rte_eth_txmode tx_mode = { 451 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 452 }; 453 454 struct rte_fdir_conf fdir_conf = { 455 .mode = RTE_FDIR_MODE_NONE, 456 .pballoc = RTE_FDIR_PBALLOC_64K, 457 .status = RTE_FDIR_REPORT_STATUS, 458 .mask = { 459 .vlan_tci_mask = 0xFFEF, 460 .ipv4_mask = { 461 .src_ip = 0xFFFFFFFF, 462 .dst_ip = 0xFFFFFFFF, 463 }, 464 .ipv6_mask = { 465 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 466 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 467 }, 468 .src_port_mask = 0xFFFF, 469 .dst_port_mask = 0xFFFF, 470 .mac_addr_byte_mask = 0xFF, 471 .tunnel_type_mask = 1, 472 .tunnel_id_mask = 0xFFFFFFFF, 473 }, 474 .drop_queue = 127, 475 }; 476 477 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 478 479 /* 480 * Display zero values by default for xstats 481 */ 482 uint8_t xstats_hide_zero; 483 484 /* 485 * Measure of CPU cycles disabled by default 486 */ 487 uint8_t record_core_cycles; 488 489 /* 490 * Display of RX and TX bursts disabled by default 491 */ 492 uint8_t record_burst_stats; 493 494 unsigned int num_sockets = 0; 495 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 496 497 #ifdef RTE_LIB_BITRATESTATS 498 /* Bitrate statistics */ 499 struct rte_stats_bitrates *bitrate_data; 500 lcoreid_t bitrate_lcore_id; 501 uint8_t bitrate_enabled; 502 #endif 503 504 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 505 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 506 507 /* 508 * hexadecimal bitmask of RX mq mode can be enabled. 509 */ 510 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 511 512 /* Forward function declarations */ 513 static void setup_attached_port(portid_t pi); 514 static void check_all_ports_link_status(uint32_t port_mask); 515 static int eth_event_callback(portid_t port_id, 516 enum rte_eth_event_type type, 517 void *param, void *ret_param); 518 static void dev_event_callback(const char *device_name, 519 enum rte_dev_event_type type, 520 void *param); 521 522 /* 523 * Check if all the ports are started. 524 * If yes, return positive value. If not, return zero. 525 */ 526 static int all_ports_started(void); 527 528 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 529 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 530 531 /* Holds the registered mbuf dynamic flags names. */ 532 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 533 534 /* 535 * Helper function to check if socket is already discovered. 536 * If yes, return positive value. If not, return zero. 537 */ 538 int 539 new_socket_id(unsigned int socket_id) 540 { 541 unsigned int i; 542 543 for (i = 0; i < num_sockets; i++) { 544 if (socket_ids[i] == socket_id) 545 return 0; 546 } 547 return 1; 548 } 549 550 /* 551 * Setup default configuration. 552 */ 553 static void 554 set_default_fwd_lcores_config(void) 555 { 556 unsigned int i; 557 unsigned int nb_lc; 558 unsigned int sock_num; 559 560 nb_lc = 0; 561 for (i = 0; i < RTE_MAX_LCORE; i++) { 562 if (!rte_lcore_is_enabled(i)) 563 continue; 564 sock_num = rte_lcore_to_socket_id(i); 565 if (new_socket_id(sock_num)) { 566 if (num_sockets >= RTE_MAX_NUMA_NODES) { 567 rte_exit(EXIT_FAILURE, 568 "Total sockets greater than %u\n", 569 RTE_MAX_NUMA_NODES); 570 } 571 socket_ids[num_sockets++] = sock_num; 572 } 573 if (i == rte_get_main_lcore()) 574 continue; 575 fwd_lcores_cpuids[nb_lc++] = i; 576 } 577 nb_lcores = (lcoreid_t) nb_lc; 578 nb_cfg_lcores = nb_lcores; 579 nb_fwd_lcores = 1; 580 } 581 582 static void 583 set_def_peer_eth_addrs(void) 584 { 585 portid_t i; 586 587 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 588 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 589 peer_eth_addrs[i].addr_bytes[5] = i; 590 } 591 } 592 593 static void 594 set_default_fwd_ports_config(void) 595 { 596 portid_t pt_id; 597 int i = 0; 598 599 RTE_ETH_FOREACH_DEV(pt_id) { 600 fwd_ports_ids[i++] = pt_id; 601 602 /* Update sockets info according to the attached device */ 603 int socket_id = rte_eth_dev_socket_id(pt_id); 604 if (socket_id >= 0 && new_socket_id(socket_id)) { 605 if (num_sockets >= RTE_MAX_NUMA_NODES) { 606 rte_exit(EXIT_FAILURE, 607 "Total sockets greater than %u\n", 608 RTE_MAX_NUMA_NODES); 609 } 610 socket_ids[num_sockets++] = socket_id; 611 } 612 } 613 614 nb_cfg_ports = nb_ports; 615 nb_fwd_ports = nb_ports; 616 } 617 618 void 619 set_def_fwd_config(void) 620 { 621 set_default_fwd_lcores_config(); 622 set_def_peer_eth_addrs(); 623 set_default_fwd_ports_config(); 624 } 625 626 /* extremely pessimistic estimation of memory required to create a mempool */ 627 static int 628 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 629 { 630 unsigned int n_pages, mbuf_per_pg, leftover; 631 uint64_t total_mem, mbuf_mem, obj_sz; 632 633 /* there is no good way to predict how much space the mempool will 634 * occupy because it will allocate chunks on the fly, and some of those 635 * will come from default DPDK memory while some will come from our 636 * external memory, so just assume 128MB will be enough for everyone. 637 */ 638 uint64_t hdr_mem = 128 << 20; 639 640 /* account for possible non-contiguousness */ 641 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 642 if (obj_sz > pgsz) { 643 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 644 return -1; 645 } 646 647 mbuf_per_pg = pgsz / obj_sz; 648 leftover = (nb_mbufs % mbuf_per_pg) > 0; 649 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 650 651 mbuf_mem = n_pages * pgsz; 652 653 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 654 655 if (total_mem > SIZE_MAX) { 656 TESTPMD_LOG(ERR, "Memory size too big\n"); 657 return -1; 658 } 659 *out = (size_t)total_mem; 660 661 return 0; 662 } 663 664 static int 665 pagesz_flags(uint64_t page_sz) 666 { 667 /* as per mmap() manpage, all page sizes are log2 of page size 668 * shifted by MAP_HUGE_SHIFT 669 */ 670 int log2 = rte_log2_u64(page_sz); 671 672 return (log2 << HUGE_SHIFT); 673 } 674 675 static void * 676 alloc_mem(size_t memsz, size_t pgsz, bool huge) 677 { 678 void *addr; 679 int flags; 680 681 /* allocate anonymous hugepages */ 682 flags = MAP_ANONYMOUS | MAP_PRIVATE; 683 if (huge) 684 flags |= HUGE_FLAG | pagesz_flags(pgsz); 685 686 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 687 if (addr == MAP_FAILED) 688 return NULL; 689 690 return addr; 691 } 692 693 struct extmem_param { 694 void *addr; 695 size_t len; 696 size_t pgsz; 697 rte_iova_t *iova_table; 698 unsigned int iova_table_len; 699 }; 700 701 static int 702 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 703 bool huge) 704 { 705 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 706 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 707 unsigned int cur_page, n_pages, pgsz_idx; 708 size_t mem_sz, cur_pgsz; 709 rte_iova_t *iovas = NULL; 710 void *addr; 711 int ret; 712 713 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 714 /* skip anything that is too big */ 715 if (pgsizes[pgsz_idx] > SIZE_MAX) 716 continue; 717 718 cur_pgsz = pgsizes[pgsz_idx]; 719 720 /* if we were told not to allocate hugepages, override */ 721 if (!huge) 722 cur_pgsz = sysconf(_SC_PAGESIZE); 723 724 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 725 if (ret < 0) { 726 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 727 return -1; 728 } 729 730 /* allocate our memory */ 731 addr = alloc_mem(mem_sz, cur_pgsz, huge); 732 733 /* if we couldn't allocate memory with a specified page size, 734 * that doesn't mean we can't do it with other page sizes, so 735 * try another one. 736 */ 737 if (addr == NULL) 738 continue; 739 740 /* store IOVA addresses for every page in this memory area */ 741 n_pages = mem_sz / cur_pgsz; 742 743 iovas = malloc(sizeof(*iovas) * n_pages); 744 745 if (iovas == NULL) { 746 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 747 goto fail; 748 } 749 /* lock memory if it's not huge pages */ 750 if (!huge) 751 mlock(addr, mem_sz); 752 753 /* populate IOVA addresses */ 754 for (cur_page = 0; cur_page < n_pages; cur_page++) { 755 rte_iova_t iova; 756 size_t offset; 757 void *cur; 758 759 offset = cur_pgsz * cur_page; 760 cur = RTE_PTR_ADD(addr, offset); 761 762 /* touch the page before getting its IOVA */ 763 *(volatile char *)cur = 0; 764 765 iova = rte_mem_virt2iova(cur); 766 767 iovas[cur_page] = iova; 768 } 769 770 break; 771 } 772 /* if we couldn't allocate anything */ 773 if (iovas == NULL) 774 return -1; 775 776 param->addr = addr; 777 param->len = mem_sz; 778 param->pgsz = cur_pgsz; 779 param->iova_table = iovas; 780 param->iova_table_len = n_pages; 781 782 return 0; 783 fail: 784 if (iovas) 785 free(iovas); 786 if (addr) 787 munmap(addr, mem_sz); 788 789 return -1; 790 } 791 792 static int 793 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 794 { 795 struct extmem_param param; 796 int socket_id, ret; 797 798 memset(¶m, 0, sizeof(param)); 799 800 /* check if our heap exists */ 801 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 802 if (socket_id < 0) { 803 /* create our heap */ 804 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 805 if (ret < 0) { 806 TESTPMD_LOG(ERR, "Cannot create heap\n"); 807 return -1; 808 } 809 } 810 811 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 812 if (ret < 0) { 813 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 814 return -1; 815 } 816 817 /* we now have a valid memory area, so add it to heap */ 818 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 819 param.addr, param.len, param.iova_table, 820 param.iova_table_len, param.pgsz); 821 822 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 823 824 /* not needed any more */ 825 free(param.iova_table); 826 827 if (ret < 0) { 828 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 829 munmap(param.addr, param.len); 830 return -1; 831 } 832 833 /* success */ 834 835 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 836 param.len >> 20); 837 838 return 0; 839 } 840 static void 841 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 842 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 843 { 844 uint16_t pid = 0; 845 int ret; 846 847 RTE_ETH_FOREACH_DEV(pid) { 848 struct rte_eth_dev *dev = 849 &rte_eth_devices[pid]; 850 851 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 852 memhdr->len); 853 if (ret) { 854 TESTPMD_LOG(DEBUG, 855 "unable to DMA unmap addr 0x%p " 856 "for device %s\n", 857 memhdr->addr, dev->data->name); 858 } 859 } 860 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 861 if (ret) { 862 TESTPMD_LOG(DEBUG, 863 "unable to un-register addr 0x%p\n", memhdr->addr); 864 } 865 } 866 867 static void 868 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 869 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 870 { 871 uint16_t pid = 0; 872 size_t page_size = sysconf(_SC_PAGESIZE); 873 int ret; 874 875 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 876 page_size); 877 if (ret) { 878 TESTPMD_LOG(DEBUG, 879 "unable to register addr 0x%p\n", memhdr->addr); 880 return; 881 } 882 RTE_ETH_FOREACH_DEV(pid) { 883 struct rte_eth_dev *dev = 884 &rte_eth_devices[pid]; 885 886 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 887 memhdr->len); 888 if (ret) { 889 TESTPMD_LOG(DEBUG, 890 "unable to DMA map addr 0x%p " 891 "for device %s\n", 892 memhdr->addr, dev->data->name); 893 } 894 } 895 } 896 897 static unsigned int 898 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 899 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 900 { 901 struct rte_pktmbuf_extmem *xmem; 902 unsigned int ext_num, zone_num, elt_num; 903 uint16_t elt_size; 904 905 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 906 elt_num = EXTBUF_ZONE_SIZE / elt_size; 907 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 908 909 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 910 if (xmem == NULL) { 911 TESTPMD_LOG(ERR, "Cannot allocate memory for " 912 "external buffer descriptors\n"); 913 *ext_mem = NULL; 914 return 0; 915 } 916 for (ext_num = 0; ext_num < zone_num; ext_num++) { 917 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 918 const struct rte_memzone *mz; 919 char mz_name[RTE_MEMZONE_NAMESIZE]; 920 int ret; 921 922 ret = snprintf(mz_name, sizeof(mz_name), 923 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 924 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 925 errno = ENAMETOOLONG; 926 ext_num = 0; 927 break; 928 } 929 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 930 socket_id, 931 RTE_MEMZONE_IOVA_CONTIG | 932 RTE_MEMZONE_1GB | 933 RTE_MEMZONE_SIZE_HINT_ONLY, 934 EXTBUF_ZONE_SIZE); 935 if (mz == NULL) { 936 /* 937 * The caller exits on external buffer creation 938 * error, so there is no need to free memzones. 939 */ 940 errno = ENOMEM; 941 ext_num = 0; 942 break; 943 } 944 xseg->buf_ptr = mz->addr; 945 xseg->buf_iova = mz->iova; 946 xseg->buf_len = EXTBUF_ZONE_SIZE; 947 xseg->elt_size = elt_size; 948 } 949 if (ext_num == 0 && xmem != NULL) { 950 free(xmem); 951 xmem = NULL; 952 } 953 *ext_mem = xmem; 954 return ext_num; 955 } 956 957 /* 958 * Configuration initialisation done once at init time. 959 */ 960 static struct rte_mempool * 961 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 962 unsigned int socket_id, uint16_t size_idx) 963 { 964 char pool_name[RTE_MEMPOOL_NAMESIZE]; 965 struct rte_mempool *rte_mp = NULL; 966 uint32_t mb_size; 967 968 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 969 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 970 971 TESTPMD_LOG(INFO, 972 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 973 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 974 975 switch (mp_alloc_type) { 976 case MP_ALLOC_NATIVE: 977 { 978 /* wrapper to rte_mempool_create() */ 979 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 980 rte_mbuf_best_mempool_ops()); 981 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 982 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 983 break; 984 } 985 case MP_ALLOC_ANON: 986 { 987 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 988 mb_size, (unsigned int) mb_mempool_cache, 989 sizeof(struct rte_pktmbuf_pool_private), 990 socket_id, mempool_flags); 991 if (rte_mp == NULL) 992 goto err; 993 994 if (rte_mempool_populate_anon(rte_mp) == 0) { 995 rte_mempool_free(rte_mp); 996 rte_mp = NULL; 997 goto err; 998 } 999 rte_pktmbuf_pool_init(rte_mp, NULL); 1000 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1001 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1002 break; 1003 } 1004 case MP_ALLOC_XMEM: 1005 case MP_ALLOC_XMEM_HUGE: 1006 { 1007 int heap_socket; 1008 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1009 1010 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1011 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1012 1013 heap_socket = 1014 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1015 if (heap_socket < 0) 1016 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1017 1018 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1019 rte_mbuf_best_mempool_ops()); 1020 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1021 mb_mempool_cache, 0, mbuf_seg_size, 1022 heap_socket); 1023 break; 1024 } 1025 case MP_ALLOC_XBUF: 1026 { 1027 struct rte_pktmbuf_extmem *ext_mem; 1028 unsigned int ext_num; 1029 1030 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1031 socket_id, pool_name, &ext_mem); 1032 if (ext_num == 0) 1033 rte_exit(EXIT_FAILURE, 1034 "Can't create pinned data buffers\n"); 1035 1036 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1037 rte_mbuf_best_mempool_ops()); 1038 rte_mp = rte_pktmbuf_pool_create_extbuf 1039 (pool_name, nb_mbuf, mb_mempool_cache, 1040 0, mbuf_seg_size, socket_id, 1041 ext_mem, ext_num); 1042 free(ext_mem); 1043 break; 1044 } 1045 default: 1046 { 1047 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1048 } 1049 } 1050 1051 err: 1052 if (rte_mp == NULL) { 1053 rte_exit(EXIT_FAILURE, 1054 "Creation of mbuf pool for socket %u failed: %s\n", 1055 socket_id, rte_strerror(rte_errno)); 1056 } else if (verbose_level > 0) { 1057 rte_mempool_dump(stdout, rte_mp); 1058 } 1059 return rte_mp; 1060 } 1061 1062 /* 1063 * Check given socket id is valid or not with NUMA mode, 1064 * if valid, return 0, else return -1 1065 */ 1066 static int 1067 check_socket_id(const unsigned int socket_id) 1068 { 1069 static int warning_once = 0; 1070 1071 if (new_socket_id(socket_id)) { 1072 if (!warning_once && numa_support) 1073 printf("Warning: NUMA should be configured manually by" 1074 " using --port-numa-config and" 1075 " --ring-numa-config parameters along with" 1076 " --numa.\n"); 1077 warning_once = 1; 1078 return -1; 1079 } 1080 return 0; 1081 } 1082 1083 /* 1084 * Get the allowed maximum number of RX queues. 1085 * *pid return the port id which has minimal value of 1086 * max_rx_queues in all ports. 1087 */ 1088 queueid_t 1089 get_allowed_max_nb_rxq(portid_t *pid) 1090 { 1091 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1092 bool max_rxq_valid = false; 1093 portid_t pi; 1094 struct rte_eth_dev_info dev_info; 1095 1096 RTE_ETH_FOREACH_DEV(pi) { 1097 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1098 continue; 1099 1100 max_rxq_valid = true; 1101 if (dev_info.max_rx_queues < allowed_max_rxq) { 1102 allowed_max_rxq = dev_info.max_rx_queues; 1103 *pid = pi; 1104 } 1105 } 1106 return max_rxq_valid ? allowed_max_rxq : 0; 1107 } 1108 1109 /* 1110 * Check input rxq is valid or not. 1111 * If input rxq is not greater than any of maximum number 1112 * of RX queues of all ports, it is valid. 1113 * if valid, return 0, else return -1 1114 */ 1115 int 1116 check_nb_rxq(queueid_t rxq) 1117 { 1118 queueid_t allowed_max_rxq; 1119 portid_t pid = 0; 1120 1121 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1122 if (rxq > allowed_max_rxq) { 1123 printf("Fail: input rxq (%u) can't be greater " 1124 "than max_rx_queues (%u) of port %u\n", 1125 rxq, 1126 allowed_max_rxq, 1127 pid); 1128 return -1; 1129 } 1130 return 0; 1131 } 1132 1133 /* 1134 * Get the allowed maximum number of TX queues. 1135 * *pid return the port id which has minimal value of 1136 * max_tx_queues in all ports. 1137 */ 1138 queueid_t 1139 get_allowed_max_nb_txq(portid_t *pid) 1140 { 1141 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1142 bool max_txq_valid = false; 1143 portid_t pi; 1144 struct rte_eth_dev_info dev_info; 1145 1146 RTE_ETH_FOREACH_DEV(pi) { 1147 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1148 continue; 1149 1150 max_txq_valid = true; 1151 if (dev_info.max_tx_queues < allowed_max_txq) { 1152 allowed_max_txq = dev_info.max_tx_queues; 1153 *pid = pi; 1154 } 1155 } 1156 return max_txq_valid ? allowed_max_txq : 0; 1157 } 1158 1159 /* 1160 * Check input txq is valid or not. 1161 * If input txq is not greater than any of maximum number 1162 * of TX queues of all ports, it is valid. 1163 * if valid, return 0, else return -1 1164 */ 1165 int 1166 check_nb_txq(queueid_t txq) 1167 { 1168 queueid_t allowed_max_txq; 1169 portid_t pid = 0; 1170 1171 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1172 if (txq > allowed_max_txq) { 1173 printf("Fail: input txq (%u) can't be greater " 1174 "than max_tx_queues (%u) of port %u\n", 1175 txq, 1176 allowed_max_txq, 1177 pid); 1178 return -1; 1179 } 1180 return 0; 1181 } 1182 1183 /* 1184 * Get the allowed maximum number of RXDs of every rx queue. 1185 * *pid return the port id which has minimal value of 1186 * max_rxd in all queues of all ports. 1187 */ 1188 static uint16_t 1189 get_allowed_max_nb_rxd(portid_t *pid) 1190 { 1191 uint16_t allowed_max_rxd = UINT16_MAX; 1192 portid_t pi; 1193 struct rte_eth_dev_info dev_info; 1194 1195 RTE_ETH_FOREACH_DEV(pi) { 1196 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1197 continue; 1198 1199 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1200 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1201 *pid = pi; 1202 } 1203 } 1204 return allowed_max_rxd; 1205 } 1206 1207 /* 1208 * Get the allowed minimal number of RXDs of every rx queue. 1209 * *pid return the port id which has minimal value of 1210 * min_rxd in all queues of all ports. 1211 */ 1212 static uint16_t 1213 get_allowed_min_nb_rxd(portid_t *pid) 1214 { 1215 uint16_t allowed_min_rxd = 0; 1216 portid_t pi; 1217 struct rte_eth_dev_info dev_info; 1218 1219 RTE_ETH_FOREACH_DEV(pi) { 1220 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1221 continue; 1222 1223 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1224 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1225 *pid = pi; 1226 } 1227 } 1228 1229 return allowed_min_rxd; 1230 } 1231 1232 /* 1233 * Check input rxd is valid or not. 1234 * If input rxd is not greater than any of maximum number 1235 * of RXDs of every Rx queues and is not less than any of 1236 * minimal number of RXDs of every Rx queues, it is valid. 1237 * if valid, return 0, else return -1 1238 */ 1239 int 1240 check_nb_rxd(queueid_t rxd) 1241 { 1242 uint16_t allowed_max_rxd; 1243 uint16_t allowed_min_rxd; 1244 portid_t pid = 0; 1245 1246 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1247 if (rxd > allowed_max_rxd) { 1248 printf("Fail: input rxd (%u) can't be greater " 1249 "than max_rxds (%u) of port %u\n", 1250 rxd, 1251 allowed_max_rxd, 1252 pid); 1253 return -1; 1254 } 1255 1256 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1257 if (rxd < allowed_min_rxd) { 1258 printf("Fail: input rxd (%u) can't be less " 1259 "than min_rxds (%u) of port %u\n", 1260 rxd, 1261 allowed_min_rxd, 1262 pid); 1263 return -1; 1264 } 1265 1266 return 0; 1267 } 1268 1269 /* 1270 * Get the allowed maximum number of TXDs of every rx queues. 1271 * *pid return the port id which has minimal value of 1272 * max_txd in every tx queue. 1273 */ 1274 static uint16_t 1275 get_allowed_max_nb_txd(portid_t *pid) 1276 { 1277 uint16_t allowed_max_txd = UINT16_MAX; 1278 portid_t pi; 1279 struct rte_eth_dev_info dev_info; 1280 1281 RTE_ETH_FOREACH_DEV(pi) { 1282 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1283 continue; 1284 1285 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1286 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1287 *pid = pi; 1288 } 1289 } 1290 return allowed_max_txd; 1291 } 1292 1293 /* 1294 * Get the allowed maximum number of TXDs of every tx queues. 1295 * *pid return the port id which has minimal value of 1296 * min_txd in every tx queue. 1297 */ 1298 static uint16_t 1299 get_allowed_min_nb_txd(portid_t *pid) 1300 { 1301 uint16_t allowed_min_txd = 0; 1302 portid_t pi; 1303 struct rte_eth_dev_info dev_info; 1304 1305 RTE_ETH_FOREACH_DEV(pi) { 1306 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1307 continue; 1308 1309 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1310 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1311 *pid = pi; 1312 } 1313 } 1314 1315 return allowed_min_txd; 1316 } 1317 1318 /* 1319 * Check input txd is valid or not. 1320 * If input txd is not greater than any of maximum number 1321 * of TXDs of every Rx queues, it is valid. 1322 * if valid, return 0, else return -1 1323 */ 1324 int 1325 check_nb_txd(queueid_t txd) 1326 { 1327 uint16_t allowed_max_txd; 1328 uint16_t allowed_min_txd; 1329 portid_t pid = 0; 1330 1331 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1332 if (txd > allowed_max_txd) { 1333 printf("Fail: input txd (%u) can't be greater " 1334 "than max_txds (%u) of port %u\n", 1335 txd, 1336 allowed_max_txd, 1337 pid); 1338 return -1; 1339 } 1340 1341 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1342 if (txd < allowed_min_txd) { 1343 printf("Fail: input txd (%u) can't be less " 1344 "than min_txds (%u) of port %u\n", 1345 txd, 1346 allowed_min_txd, 1347 pid); 1348 return -1; 1349 } 1350 return 0; 1351 } 1352 1353 1354 /* 1355 * Get the allowed maximum number of hairpin queues. 1356 * *pid return the port id which has minimal value of 1357 * max_hairpin_queues in all ports. 1358 */ 1359 queueid_t 1360 get_allowed_max_nb_hairpinq(portid_t *pid) 1361 { 1362 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1363 portid_t pi; 1364 struct rte_eth_hairpin_cap cap; 1365 1366 RTE_ETH_FOREACH_DEV(pi) { 1367 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1368 *pid = pi; 1369 return 0; 1370 } 1371 if (cap.max_nb_queues < allowed_max_hairpinq) { 1372 allowed_max_hairpinq = cap.max_nb_queues; 1373 *pid = pi; 1374 } 1375 } 1376 return allowed_max_hairpinq; 1377 } 1378 1379 /* 1380 * Check input hairpin is valid or not. 1381 * If input hairpin is not greater than any of maximum number 1382 * of hairpin queues of all ports, it is valid. 1383 * if valid, return 0, else return -1 1384 */ 1385 int 1386 check_nb_hairpinq(queueid_t hairpinq) 1387 { 1388 queueid_t allowed_max_hairpinq; 1389 portid_t pid = 0; 1390 1391 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1392 if (hairpinq > allowed_max_hairpinq) { 1393 printf("Fail: input hairpin (%u) can't be greater " 1394 "than max_hairpin_queues (%u) of port %u\n", 1395 hairpinq, allowed_max_hairpinq, pid); 1396 return -1; 1397 } 1398 return 0; 1399 } 1400 1401 static void 1402 init_config(void) 1403 { 1404 portid_t pid; 1405 struct rte_port *port; 1406 struct rte_mempool *mbp; 1407 unsigned int nb_mbuf_per_pool; 1408 lcoreid_t lc_id; 1409 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1410 struct rte_gro_param gro_param; 1411 uint32_t gso_types; 1412 uint16_t data_size; 1413 uint16_t eth_overhead; 1414 bool warning = 0; 1415 int k; 1416 int ret; 1417 1418 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1419 1420 /* Configuration of logical cores. */ 1421 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1422 sizeof(struct fwd_lcore *) * nb_lcores, 1423 RTE_CACHE_LINE_SIZE); 1424 if (fwd_lcores == NULL) { 1425 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1426 "failed\n", nb_lcores); 1427 } 1428 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1429 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1430 sizeof(struct fwd_lcore), 1431 RTE_CACHE_LINE_SIZE); 1432 if (fwd_lcores[lc_id] == NULL) { 1433 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1434 "failed\n"); 1435 } 1436 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1437 } 1438 1439 RTE_ETH_FOREACH_DEV(pid) { 1440 port = &ports[pid]; 1441 /* Apply default TxRx configuration for all ports */ 1442 port->dev_conf.txmode = tx_mode; 1443 port->dev_conf.rxmode = rx_mode; 1444 1445 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1446 if (ret != 0) 1447 rte_exit(EXIT_FAILURE, 1448 "rte_eth_dev_info_get() failed\n"); 1449 1450 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */ 1451 if (port->dev_info.max_mtu != UINT16_MAX && 1452 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu) 1453 eth_overhead = port->dev_info.max_rx_pktlen - 1454 port->dev_info.max_mtu; 1455 else 1456 eth_overhead = 1457 RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1458 1459 if (port->dev_conf.rxmode.max_rx_pkt_len <= 1460 (uint32_t)(RTE_ETHER_MTU + eth_overhead)) 1461 port->dev_conf.rxmode.max_rx_pkt_len = 1462 RTE_ETHER_MTU + eth_overhead; 1463 else 1464 port->dev_conf.rxmode.offloads |= 1465 DEV_RX_OFFLOAD_JUMBO_FRAME; 1466 1467 if (!(port->dev_info.tx_offload_capa & 1468 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1469 port->dev_conf.txmode.offloads &= 1470 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1471 if (numa_support) { 1472 if (port_numa[pid] != NUMA_NO_CONFIG) 1473 port_per_socket[port_numa[pid]]++; 1474 else { 1475 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1476 1477 /* 1478 * if socket_id is invalid, 1479 * set to the first available socket. 1480 */ 1481 if (check_socket_id(socket_id) < 0) 1482 socket_id = socket_ids[0]; 1483 port_per_socket[socket_id]++; 1484 } 1485 } 1486 1487 /* Apply Rx offloads configuration */ 1488 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1489 port->rx_conf[k].offloads = 1490 port->dev_conf.rxmode.offloads; 1491 /* Apply Tx offloads configuration */ 1492 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1493 port->tx_conf[k].offloads = 1494 port->dev_conf.txmode.offloads; 1495 1496 /* set flag to initialize port/queue */ 1497 port->need_reconfig = 1; 1498 port->need_reconfig_queues = 1; 1499 port->tx_metadata = 0; 1500 1501 /* Check for maximum number of segments per MTU. Accordingly 1502 * update the mbuf data size. 1503 */ 1504 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1505 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1506 data_size = rx_mode.max_rx_pkt_len / 1507 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1508 1509 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1510 mbuf_data_size[0]) { 1511 mbuf_data_size[0] = data_size + 1512 RTE_PKTMBUF_HEADROOM; 1513 warning = 1; 1514 } 1515 } 1516 } 1517 1518 if (warning) 1519 TESTPMD_LOG(WARNING, 1520 "Configured mbuf size of the first segment %hu\n", 1521 mbuf_data_size[0]); 1522 /* 1523 * Create pools of mbuf. 1524 * If NUMA support is disabled, create a single pool of mbuf in 1525 * socket 0 memory by default. 1526 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1527 * 1528 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1529 * nb_txd can be configured at run time. 1530 */ 1531 if (param_total_num_mbufs) 1532 nb_mbuf_per_pool = param_total_num_mbufs; 1533 else { 1534 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1535 (nb_lcores * mb_mempool_cache) + 1536 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1537 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1538 } 1539 1540 if (numa_support) { 1541 uint8_t i, j; 1542 1543 for (i = 0; i < num_sockets; i++) 1544 for (j = 0; j < mbuf_data_size_n; j++) 1545 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1546 mbuf_pool_create(mbuf_data_size[j], 1547 nb_mbuf_per_pool, 1548 socket_ids[i], j); 1549 } else { 1550 uint8_t i; 1551 1552 for (i = 0; i < mbuf_data_size_n; i++) 1553 mempools[i] = mbuf_pool_create 1554 (mbuf_data_size[i], 1555 nb_mbuf_per_pool, 1556 socket_num == UMA_NO_CONFIG ? 1557 0 : socket_num, i); 1558 } 1559 1560 init_port_config(); 1561 1562 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1563 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1564 /* 1565 * Records which Mbuf pool to use by each logical core, if needed. 1566 */ 1567 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1568 mbp = mbuf_pool_find( 1569 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1570 1571 if (mbp == NULL) 1572 mbp = mbuf_pool_find(0, 0); 1573 fwd_lcores[lc_id]->mbp = mbp; 1574 /* initialize GSO context */ 1575 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1576 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1577 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1578 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1579 RTE_ETHER_CRC_LEN; 1580 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1581 } 1582 1583 /* Configuration of packet forwarding streams. */ 1584 if (init_fwd_streams() < 0) 1585 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1586 1587 fwd_config_setup(); 1588 1589 /* create a gro context for each lcore */ 1590 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1591 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1592 gro_param.max_item_per_flow = MAX_PKT_BURST; 1593 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1594 gro_param.socket_id = rte_lcore_to_socket_id( 1595 fwd_lcores_cpuids[lc_id]); 1596 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1597 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1598 rte_exit(EXIT_FAILURE, 1599 "rte_gro_ctx_create() failed\n"); 1600 } 1601 } 1602 } 1603 1604 1605 void 1606 reconfig(portid_t new_port_id, unsigned socket_id) 1607 { 1608 struct rte_port *port; 1609 int ret; 1610 1611 /* Reconfiguration of Ethernet ports. */ 1612 port = &ports[new_port_id]; 1613 1614 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1615 if (ret != 0) 1616 return; 1617 1618 /* set flag to initialize port/queue */ 1619 port->need_reconfig = 1; 1620 port->need_reconfig_queues = 1; 1621 port->socket_id = socket_id; 1622 1623 init_port_config(); 1624 } 1625 1626 1627 int 1628 init_fwd_streams(void) 1629 { 1630 portid_t pid; 1631 struct rte_port *port; 1632 streamid_t sm_id, nb_fwd_streams_new; 1633 queueid_t q; 1634 1635 /* set socket id according to numa or not */ 1636 RTE_ETH_FOREACH_DEV(pid) { 1637 port = &ports[pid]; 1638 if (nb_rxq > port->dev_info.max_rx_queues) { 1639 printf("Fail: nb_rxq(%d) is greater than " 1640 "max_rx_queues(%d)\n", nb_rxq, 1641 port->dev_info.max_rx_queues); 1642 return -1; 1643 } 1644 if (nb_txq > port->dev_info.max_tx_queues) { 1645 printf("Fail: nb_txq(%d) is greater than " 1646 "max_tx_queues(%d)\n", nb_txq, 1647 port->dev_info.max_tx_queues); 1648 return -1; 1649 } 1650 if (numa_support) { 1651 if (port_numa[pid] != NUMA_NO_CONFIG) 1652 port->socket_id = port_numa[pid]; 1653 else { 1654 port->socket_id = rte_eth_dev_socket_id(pid); 1655 1656 /* 1657 * if socket_id is invalid, 1658 * set to the first available socket. 1659 */ 1660 if (check_socket_id(port->socket_id) < 0) 1661 port->socket_id = socket_ids[0]; 1662 } 1663 } 1664 else { 1665 if (socket_num == UMA_NO_CONFIG) 1666 port->socket_id = 0; 1667 else 1668 port->socket_id = socket_num; 1669 } 1670 } 1671 1672 q = RTE_MAX(nb_rxq, nb_txq); 1673 if (q == 0) { 1674 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1675 return -1; 1676 } 1677 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1678 if (nb_fwd_streams_new == nb_fwd_streams) 1679 return 0; 1680 /* clear the old */ 1681 if (fwd_streams != NULL) { 1682 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1683 if (fwd_streams[sm_id] == NULL) 1684 continue; 1685 rte_free(fwd_streams[sm_id]); 1686 fwd_streams[sm_id] = NULL; 1687 } 1688 rte_free(fwd_streams); 1689 fwd_streams = NULL; 1690 } 1691 1692 /* init new */ 1693 nb_fwd_streams = nb_fwd_streams_new; 1694 if (nb_fwd_streams) { 1695 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1696 sizeof(struct fwd_stream *) * nb_fwd_streams, 1697 RTE_CACHE_LINE_SIZE); 1698 if (fwd_streams == NULL) 1699 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1700 " (struct fwd_stream *)) failed\n", 1701 nb_fwd_streams); 1702 1703 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1704 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1705 " struct fwd_stream", sizeof(struct fwd_stream), 1706 RTE_CACHE_LINE_SIZE); 1707 if (fwd_streams[sm_id] == NULL) 1708 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1709 "(struct fwd_stream) failed\n"); 1710 } 1711 } 1712 1713 return 0; 1714 } 1715 1716 static void 1717 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1718 { 1719 uint64_t total_burst, sburst; 1720 uint64_t nb_burst; 1721 uint64_t burst_stats[4]; 1722 uint16_t pktnb_stats[4]; 1723 uint16_t nb_pkt; 1724 int burst_percent[4], sburstp; 1725 int i; 1726 1727 /* 1728 * First compute the total number of packet bursts and the 1729 * two highest numbers of bursts of the same number of packets. 1730 */ 1731 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1732 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1733 1734 /* Show stats for 0 burst size always */ 1735 total_burst = pbs->pkt_burst_spread[0]; 1736 burst_stats[0] = pbs->pkt_burst_spread[0]; 1737 pktnb_stats[0] = 0; 1738 1739 /* Find the next 2 burst sizes with highest occurrences. */ 1740 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1741 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1742 1743 if (nb_burst == 0) 1744 continue; 1745 1746 total_burst += nb_burst; 1747 1748 if (nb_burst > burst_stats[1]) { 1749 burst_stats[2] = burst_stats[1]; 1750 pktnb_stats[2] = pktnb_stats[1]; 1751 burst_stats[1] = nb_burst; 1752 pktnb_stats[1] = nb_pkt; 1753 } else if (nb_burst > burst_stats[2]) { 1754 burst_stats[2] = nb_burst; 1755 pktnb_stats[2] = nb_pkt; 1756 } 1757 } 1758 if (total_burst == 0) 1759 return; 1760 1761 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1762 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1763 if (i == 3) { 1764 printf("%d%% of other]\n", 100 - sburstp); 1765 return; 1766 } 1767 1768 sburst += burst_stats[i]; 1769 if (sburst == total_burst) { 1770 printf("%d%% of %d pkts]\n", 1771 100 - sburstp, (int) pktnb_stats[i]); 1772 return; 1773 } 1774 1775 burst_percent[i] = 1776 (double)burst_stats[i] / total_burst * 100; 1777 printf("%d%% of %d pkts + ", 1778 burst_percent[i], (int) pktnb_stats[i]); 1779 sburstp += burst_percent[i]; 1780 } 1781 } 1782 1783 static void 1784 fwd_stream_stats_display(streamid_t stream_id) 1785 { 1786 struct fwd_stream *fs; 1787 static const char *fwd_top_stats_border = "-------"; 1788 1789 fs = fwd_streams[stream_id]; 1790 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1791 (fs->fwd_dropped == 0)) 1792 return; 1793 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1794 "TX Port=%2d/Queue=%2d %s\n", 1795 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1796 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1797 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1798 " TX-dropped: %-14"PRIu64, 1799 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1800 1801 /* if checksum mode */ 1802 if (cur_fwd_eng == &csum_fwd_engine) { 1803 printf(" RX- bad IP checksum: %-14"PRIu64 1804 " Rx- bad L4 checksum: %-14"PRIu64 1805 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1806 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1807 fs->rx_bad_outer_l4_csum); 1808 } else { 1809 printf("\n"); 1810 } 1811 1812 if (record_burst_stats) { 1813 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1814 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1815 } 1816 } 1817 1818 void 1819 fwd_stats_display(void) 1820 { 1821 static const char *fwd_stats_border = "----------------------"; 1822 static const char *acc_stats_border = "+++++++++++++++"; 1823 struct { 1824 struct fwd_stream *rx_stream; 1825 struct fwd_stream *tx_stream; 1826 uint64_t tx_dropped; 1827 uint64_t rx_bad_ip_csum; 1828 uint64_t rx_bad_l4_csum; 1829 uint64_t rx_bad_outer_l4_csum; 1830 } ports_stats[RTE_MAX_ETHPORTS]; 1831 uint64_t total_rx_dropped = 0; 1832 uint64_t total_tx_dropped = 0; 1833 uint64_t total_rx_nombuf = 0; 1834 struct rte_eth_stats stats; 1835 uint64_t fwd_cycles = 0; 1836 uint64_t total_recv = 0; 1837 uint64_t total_xmit = 0; 1838 struct rte_port *port; 1839 streamid_t sm_id; 1840 portid_t pt_id; 1841 int i; 1842 1843 memset(ports_stats, 0, sizeof(ports_stats)); 1844 1845 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1846 struct fwd_stream *fs = fwd_streams[sm_id]; 1847 1848 if (cur_fwd_config.nb_fwd_streams > 1849 cur_fwd_config.nb_fwd_ports) { 1850 fwd_stream_stats_display(sm_id); 1851 } else { 1852 ports_stats[fs->tx_port].tx_stream = fs; 1853 ports_stats[fs->rx_port].rx_stream = fs; 1854 } 1855 1856 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1857 1858 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1859 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1860 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1861 fs->rx_bad_outer_l4_csum; 1862 1863 if (record_core_cycles) 1864 fwd_cycles += fs->core_cycles; 1865 } 1866 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1867 pt_id = fwd_ports_ids[i]; 1868 port = &ports[pt_id]; 1869 1870 rte_eth_stats_get(pt_id, &stats); 1871 stats.ipackets -= port->stats.ipackets; 1872 stats.opackets -= port->stats.opackets; 1873 stats.ibytes -= port->stats.ibytes; 1874 stats.obytes -= port->stats.obytes; 1875 stats.imissed -= port->stats.imissed; 1876 stats.oerrors -= port->stats.oerrors; 1877 stats.rx_nombuf -= port->stats.rx_nombuf; 1878 1879 total_recv += stats.ipackets; 1880 total_xmit += stats.opackets; 1881 total_rx_dropped += stats.imissed; 1882 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1883 total_tx_dropped += stats.oerrors; 1884 total_rx_nombuf += stats.rx_nombuf; 1885 1886 printf("\n %s Forward statistics for port %-2d %s\n", 1887 fwd_stats_border, pt_id, fwd_stats_border); 1888 1889 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 1890 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 1891 stats.ipackets + stats.imissed); 1892 1893 if (cur_fwd_eng == &csum_fwd_engine) 1894 printf(" Bad-ipcsum: %-14"PRIu64 1895 " Bad-l4csum: %-14"PRIu64 1896 "Bad-outer-l4csum: %-14"PRIu64"\n", 1897 ports_stats[pt_id].rx_bad_ip_csum, 1898 ports_stats[pt_id].rx_bad_l4_csum, 1899 ports_stats[pt_id].rx_bad_outer_l4_csum); 1900 if (stats.ierrors + stats.rx_nombuf > 0) { 1901 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 1902 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 1903 } 1904 1905 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 1906 "TX-total: %-"PRIu64"\n", 1907 stats.opackets, ports_stats[pt_id].tx_dropped, 1908 stats.opackets + ports_stats[pt_id].tx_dropped); 1909 1910 if (record_burst_stats) { 1911 if (ports_stats[pt_id].rx_stream) 1912 pkt_burst_stats_display("RX", 1913 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1914 if (ports_stats[pt_id].tx_stream) 1915 pkt_burst_stats_display("TX", 1916 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1917 } 1918 1919 printf(" %s--------------------------------%s\n", 1920 fwd_stats_border, fwd_stats_border); 1921 } 1922 1923 printf("\n %s Accumulated forward statistics for all ports" 1924 "%s\n", 1925 acc_stats_border, acc_stats_border); 1926 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1927 "%-"PRIu64"\n" 1928 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1929 "%-"PRIu64"\n", 1930 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1931 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1932 if (total_rx_nombuf > 0) 1933 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1934 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1935 "%s\n", 1936 acc_stats_border, acc_stats_border); 1937 if (record_core_cycles) { 1938 #define CYC_PER_MHZ 1E6 1939 if (total_recv > 0 || total_xmit > 0) { 1940 uint64_t total_pkts = 0; 1941 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1942 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1943 total_pkts = total_xmit; 1944 else 1945 total_pkts = total_recv; 1946 1947 printf("\n CPU cycles/packet=%.2F (total cycles=" 1948 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1949 " MHz Clock\n", 1950 (double) fwd_cycles / total_pkts, 1951 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1952 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1953 } 1954 } 1955 } 1956 1957 void 1958 fwd_stats_reset(void) 1959 { 1960 streamid_t sm_id; 1961 portid_t pt_id; 1962 int i; 1963 1964 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1965 pt_id = fwd_ports_ids[i]; 1966 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1967 } 1968 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1969 struct fwd_stream *fs = fwd_streams[sm_id]; 1970 1971 fs->rx_packets = 0; 1972 fs->tx_packets = 0; 1973 fs->fwd_dropped = 0; 1974 fs->rx_bad_ip_csum = 0; 1975 fs->rx_bad_l4_csum = 0; 1976 fs->rx_bad_outer_l4_csum = 0; 1977 1978 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1979 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1980 fs->core_cycles = 0; 1981 } 1982 } 1983 1984 static void 1985 flush_fwd_rx_queues(void) 1986 { 1987 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1988 portid_t rxp; 1989 portid_t port_id; 1990 queueid_t rxq; 1991 uint16_t nb_rx; 1992 uint16_t i; 1993 uint8_t j; 1994 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1995 uint64_t timer_period; 1996 1997 /* convert to number of cycles */ 1998 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1999 2000 for (j = 0; j < 2; j++) { 2001 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2002 for (rxq = 0; rxq < nb_rxq; rxq++) { 2003 port_id = fwd_ports_ids[rxp]; 2004 /** 2005 * testpmd can stuck in the below do while loop 2006 * if rte_eth_rx_burst() always returns nonzero 2007 * packets. So timer is added to exit this loop 2008 * after 1sec timer expiry. 2009 */ 2010 prev_tsc = rte_rdtsc(); 2011 do { 2012 nb_rx = rte_eth_rx_burst(port_id, rxq, 2013 pkts_burst, MAX_PKT_BURST); 2014 for (i = 0; i < nb_rx; i++) 2015 rte_pktmbuf_free(pkts_burst[i]); 2016 2017 cur_tsc = rte_rdtsc(); 2018 diff_tsc = cur_tsc - prev_tsc; 2019 timer_tsc += diff_tsc; 2020 } while ((nb_rx > 0) && 2021 (timer_tsc < timer_period)); 2022 timer_tsc = 0; 2023 } 2024 } 2025 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2026 } 2027 } 2028 2029 static void 2030 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2031 { 2032 struct fwd_stream **fsm; 2033 streamid_t nb_fs; 2034 streamid_t sm_id; 2035 #ifdef RTE_LIB_BITRATESTATS 2036 uint64_t tics_per_1sec; 2037 uint64_t tics_datum; 2038 uint64_t tics_current; 2039 uint16_t i, cnt_ports; 2040 2041 cnt_ports = nb_ports; 2042 tics_datum = rte_rdtsc(); 2043 tics_per_1sec = rte_get_timer_hz(); 2044 #endif 2045 fsm = &fwd_streams[fc->stream_idx]; 2046 nb_fs = fc->stream_nb; 2047 do { 2048 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2049 (*pkt_fwd)(fsm[sm_id]); 2050 #ifdef RTE_LIB_BITRATESTATS 2051 if (bitrate_enabled != 0 && 2052 bitrate_lcore_id == rte_lcore_id()) { 2053 tics_current = rte_rdtsc(); 2054 if (tics_current - tics_datum >= tics_per_1sec) { 2055 /* Periodic bitrate calculation */ 2056 for (i = 0; i < cnt_ports; i++) 2057 rte_stats_bitrate_calc(bitrate_data, 2058 ports_ids[i]); 2059 tics_datum = tics_current; 2060 } 2061 } 2062 #endif 2063 #ifdef RTE_LIB_LATENCYSTATS 2064 if (latencystats_enabled != 0 && 2065 latencystats_lcore_id == rte_lcore_id()) 2066 rte_latencystats_update(); 2067 #endif 2068 2069 } while (! fc->stopped); 2070 } 2071 2072 static int 2073 start_pkt_forward_on_core(void *fwd_arg) 2074 { 2075 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2076 cur_fwd_config.fwd_eng->packet_fwd); 2077 return 0; 2078 } 2079 2080 /* 2081 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2082 * Used to start communication flows in network loopback test configurations. 2083 */ 2084 static int 2085 run_one_txonly_burst_on_core(void *fwd_arg) 2086 { 2087 struct fwd_lcore *fwd_lc; 2088 struct fwd_lcore tmp_lcore; 2089 2090 fwd_lc = (struct fwd_lcore *) fwd_arg; 2091 tmp_lcore = *fwd_lc; 2092 tmp_lcore.stopped = 1; 2093 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2094 return 0; 2095 } 2096 2097 /* 2098 * Launch packet forwarding: 2099 * - Setup per-port forwarding context. 2100 * - launch logical cores with their forwarding configuration. 2101 */ 2102 static void 2103 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2104 { 2105 port_fwd_begin_t port_fwd_begin; 2106 unsigned int i; 2107 unsigned int lc_id; 2108 int diag; 2109 2110 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2111 if (port_fwd_begin != NULL) { 2112 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2113 (*port_fwd_begin)(fwd_ports_ids[i]); 2114 } 2115 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2116 lc_id = fwd_lcores_cpuids[i]; 2117 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2118 fwd_lcores[i]->stopped = 0; 2119 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2120 fwd_lcores[i], lc_id); 2121 if (diag != 0) 2122 printf("launch lcore %u failed - diag=%d\n", 2123 lc_id, diag); 2124 } 2125 } 2126 } 2127 2128 /* 2129 * Launch packet forwarding configuration. 2130 */ 2131 void 2132 start_packet_forwarding(int with_tx_first) 2133 { 2134 port_fwd_begin_t port_fwd_begin; 2135 port_fwd_end_t port_fwd_end; 2136 struct rte_port *port; 2137 unsigned int i; 2138 portid_t pt_id; 2139 2140 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2141 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2142 2143 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2144 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2145 2146 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2147 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2148 (!nb_rxq || !nb_txq)) 2149 rte_exit(EXIT_FAILURE, 2150 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2151 cur_fwd_eng->fwd_mode_name); 2152 2153 if (all_ports_started() == 0) { 2154 printf("Not all ports were started\n"); 2155 return; 2156 } 2157 if (test_done == 0) { 2158 printf("Packet forwarding already started\n"); 2159 return; 2160 } 2161 2162 2163 if(dcb_test) { 2164 for (i = 0; i < nb_fwd_ports; i++) { 2165 pt_id = fwd_ports_ids[i]; 2166 port = &ports[pt_id]; 2167 if (!port->dcb_flag) { 2168 printf("In DCB mode, all forwarding ports must " 2169 "be configured in this mode.\n"); 2170 return; 2171 } 2172 } 2173 if (nb_fwd_lcores == 1) { 2174 printf("In DCB mode,the nb forwarding cores " 2175 "should be larger than 1.\n"); 2176 return; 2177 } 2178 } 2179 test_done = 0; 2180 2181 fwd_config_setup(); 2182 2183 if(!no_flush_rx) 2184 flush_fwd_rx_queues(); 2185 2186 pkt_fwd_config_display(&cur_fwd_config); 2187 rxtx_config_display(); 2188 2189 fwd_stats_reset(); 2190 if (with_tx_first) { 2191 port_fwd_begin = tx_only_engine.port_fwd_begin; 2192 if (port_fwd_begin != NULL) { 2193 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2194 (*port_fwd_begin)(fwd_ports_ids[i]); 2195 } 2196 while (with_tx_first--) { 2197 launch_packet_forwarding( 2198 run_one_txonly_burst_on_core); 2199 rte_eal_mp_wait_lcore(); 2200 } 2201 port_fwd_end = tx_only_engine.port_fwd_end; 2202 if (port_fwd_end != NULL) { 2203 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2204 (*port_fwd_end)(fwd_ports_ids[i]); 2205 } 2206 } 2207 launch_packet_forwarding(start_pkt_forward_on_core); 2208 } 2209 2210 void 2211 stop_packet_forwarding(void) 2212 { 2213 port_fwd_end_t port_fwd_end; 2214 lcoreid_t lc_id; 2215 portid_t pt_id; 2216 int i; 2217 2218 if (test_done) { 2219 printf("Packet forwarding not started\n"); 2220 return; 2221 } 2222 printf("Telling cores to stop..."); 2223 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2224 fwd_lcores[lc_id]->stopped = 1; 2225 printf("\nWaiting for lcores to finish...\n"); 2226 rte_eal_mp_wait_lcore(); 2227 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2228 if (port_fwd_end != NULL) { 2229 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2230 pt_id = fwd_ports_ids[i]; 2231 (*port_fwd_end)(pt_id); 2232 } 2233 } 2234 2235 fwd_stats_display(); 2236 2237 printf("\nDone.\n"); 2238 test_done = 1; 2239 } 2240 2241 void 2242 dev_set_link_up(portid_t pid) 2243 { 2244 if (rte_eth_dev_set_link_up(pid) < 0) 2245 printf("\nSet link up fail.\n"); 2246 } 2247 2248 void 2249 dev_set_link_down(portid_t pid) 2250 { 2251 if (rte_eth_dev_set_link_down(pid) < 0) 2252 printf("\nSet link down fail.\n"); 2253 } 2254 2255 static int 2256 all_ports_started(void) 2257 { 2258 portid_t pi; 2259 struct rte_port *port; 2260 2261 RTE_ETH_FOREACH_DEV(pi) { 2262 port = &ports[pi]; 2263 /* Check if there is a port which is not started */ 2264 if ((port->port_status != RTE_PORT_STARTED) && 2265 (port->slave_flag == 0)) 2266 return 0; 2267 } 2268 2269 /* No port is not started */ 2270 return 1; 2271 } 2272 2273 int 2274 port_is_stopped(portid_t port_id) 2275 { 2276 struct rte_port *port = &ports[port_id]; 2277 2278 if ((port->port_status != RTE_PORT_STOPPED) && 2279 (port->slave_flag == 0)) 2280 return 0; 2281 return 1; 2282 } 2283 2284 int 2285 all_ports_stopped(void) 2286 { 2287 portid_t pi; 2288 2289 RTE_ETH_FOREACH_DEV(pi) { 2290 if (!port_is_stopped(pi)) 2291 return 0; 2292 } 2293 2294 return 1; 2295 } 2296 2297 int 2298 port_is_started(portid_t port_id) 2299 { 2300 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2301 return 0; 2302 2303 if (ports[port_id].port_status != RTE_PORT_STARTED) 2304 return 0; 2305 2306 return 1; 2307 } 2308 2309 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2310 static int 2311 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2312 { 2313 queueid_t qi; 2314 struct rte_eth_hairpin_conf hairpin_conf = { 2315 .peer_count = 1, 2316 }; 2317 int i; 2318 int diag; 2319 struct rte_port *port = &ports[pi]; 2320 uint16_t peer_rx_port = pi; 2321 uint16_t peer_tx_port = pi; 2322 uint32_t manual = 1; 2323 uint32_t tx_exp = hairpin_mode & 0x10; 2324 2325 if (!(hairpin_mode & 0xf)) { 2326 peer_rx_port = pi; 2327 peer_tx_port = pi; 2328 manual = 0; 2329 } else if (hairpin_mode & 0x1) { 2330 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2331 RTE_ETH_DEV_NO_OWNER); 2332 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2333 peer_tx_port = rte_eth_find_next_owned_by(0, 2334 RTE_ETH_DEV_NO_OWNER); 2335 if (p_pi != RTE_MAX_ETHPORTS) { 2336 peer_rx_port = p_pi; 2337 } else { 2338 uint16_t next_pi; 2339 2340 /* Last port will be the peer RX port of the first. */ 2341 RTE_ETH_FOREACH_DEV(next_pi) 2342 peer_rx_port = next_pi; 2343 } 2344 manual = 1; 2345 } else if (hairpin_mode & 0x2) { 2346 if (cnt_pi & 0x1) { 2347 peer_rx_port = p_pi; 2348 } else { 2349 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2350 RTE_ETH_DEV_NO_OWNER); 2351 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2352 peer_rx_port = pi; 2353 } 2354 peer_tx_port = peer_rx_port; 2355 manual = 1; 2356 } 2357 2358 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2359 hairpin_conf.peers[0].port = peer_rx_port; 2360 hairpin_conf.peers[0].queue = i + nb_rxq; 2361 hairpin_conf.manual_bind = !!manual; 2362 hairpin_conf.tx_explicit = !!tx_exp; 2363 diag = rte_eth_tx_hairpin_queue_setup 2364 (pi, qi, nb_txd, &hairpin_conf); 2365 i++; 2366 if (diag == 0) 2367 continue; 2368 2369 /* Fail to setup rx queue, return */ 2370 if (rte_atomic16_cmpset(&(port->port_status), 2371 RTE_PORT_HANDLING, 2372 RTE_PORT_STOPPED) == 0) 2373 printf("Port %d can not be set back " 2374 "to stopped\n", pi); 2375 printf("Fail to configure port %d hairpin " 2376 "queues\n", pi); 2377 /* try to reconfigure queues next time */ 2378 port->need_reconfig_queues = 1; 2379 return -1; 2380 } 2381 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2382 hairpin_conf.peers[0].port = peer_tx_port; 2383 hairpin_conf.peers[0].queue = i + nb_txq; 2384 hairpin_conf.manual_bind = !!manual; 2385 hairpin_conf.tx_explicit = !!tx_exp; 2386 diag = rte_eth_rx_hairpin_queue_setup 2387 (pi, qi, nb_rxd, &hairpin_conf); 2388 i++; 2389 if (diag == 0) 2390 continue; 2391 2392 /* Fail to setup rx queue, return */ 2393 if (rte_atomic16_cmpset(&(port->port_status), 2394 RTE_PORT_HANDLING, 2395 RTE_PORT_STOPPED) == 0) 2396 printf("Port %d can not be set back " 2397 "to stopped\n", pi); 2398 printf("Fail to configure port %d hairpin " 2399 "queues\n", pi); 2400 /* try to reconfigure queues next time */ 2401 port->need_reconfig_queues = 1; 2402 return -1; 2403 } 2404 return 0; 2405 } 2406 2407 /* Configure the Rx with optional split. */ 2408 int 2409 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2410 uint16_t nb_rx_desc, unsigned int socket_id, 2411 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2412 { 2413 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2414 unsigned int i, mp_n; 2415 int ret; 2416 2417 if (rx_pkt_nb_segs <= 1 || 2418 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2419 rx_conf->rx_seg = NULL; 2420 rx_conf->rx_nseg = 0; 2421 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2422 nb_rx_desc, socket_id, 2423 rx_conf, mp); 2424 return ret; 2425 } 2426 for (i = 0; i < rx_pkt_nb_segs; i++) { 2427 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2428 struct rte_mempool *mpx; 2429 /* 2430 * Use last valid pool for the segments with number 2431 * exceeding the pool index. 2432 */ 2433 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2434 mpx = mbuf_pool_find(socket_id, mp_n); 2435 /* Handle zero as mbuf data buffer size. */ 2436 rx_seg->length = rx_pkt_seg_lengths[i] ? 2437 rx_pkt_seg_lengths[i] : 2438 mbuf_data_size[mp_n]; 2439 rx_seg->offset = i < rx_pkt_nb_offs ? 2440 rx_pkt_seg_offsets[i] : 0; 2441 rx_seg->mp = mpx ? mpx : mp; 2442 } 2443 rx_conf->rx_nseg = rx_pkt_nb_segs; 2444 rx_conf->rx_seg = rx_useg; 2445 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2446 socket_id, rx_conf, NULL); 2447 rx_conf->rx_seg = NULL; 2448 rx_conf->rx_nseg = 0; 2449 return ret; 2450 } 2451 2452 int 2453 start_port(portid_t pid) 2454 { 2455 int diag, need_check_link_status = -1; 2456 portid_t pi; 2457 portid_t p_pi = RTE_MAX_ETHPORTS; 2458 portid_t pl[RTE_MAX_ETHPORTS]; 2459 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2460 uint16_t cnt_pi = 0; 2461 uint16_t cfg_pi = 0; 2462 int peer_pi; 2463 queueid_t qi; 2464 struct rte_port *port; 2465 struct rte_ether_addr mac_addr; 2466 struct rte_eth_hairpin_cap cap; 2467 2468 if (port_id_is_invalid(pid, ENABLED_WARN)) 2469 return 0; 2470 2471 if(dcb_config) 2472 dcb_test = 1; 2473 RTE_ETH_FOREACH_DEV(pi) { 2474 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2475 continue; 2476 2477 need_check_link_status = 0; 2478 port = &ports[pi]; 2479 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2480 RTE_PORT_HANDLING) == 0) { 2481 printf("Port %d is now not stopped\n", pi); 2482 continue; 2483 } 2484 2485 if (port->need_reconfig > 0) { 2486 port->need_reconfig = 0; 2487 2488 if (flow_isolate_all) { 2489 int ret = port_flow_isolate(pi, 1); 2490 if (ret) { 2491 printf("Failed to apply isolated" 2492 " mode on port %d\n", pi); 2493 return -1; 2494 } 2495 } 2496 configure_rxtx_dump_callbacks(0); 2497 printf("Configuring Port %d (socket %u)\n", pi, 2498 port->socket_id); 2499 if (nb_hairpinq > 0 && 2500 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2501 printf("Port %d doesn't support hairpin " 2502 "queues\n", pi); 2503 return -1; 2504 } 2505 /* configure port */ 2506 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2507 nb_txq + nb_hairpinq, 2508 &(port->dev_conf)); 2509 if (diag != 0) { 2510 if (rte_atomic16_cmpset(&(port->port_status), 2511 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2512 printf("Port %d can not be set back " 2513 "to stopped\n", pi); 2514 printf("Fail to configure port %d\n", pi); 2515 /* try to reconfigure port next time */ 2516 port->need_reconfig = 1; 2517 return -1; 2518 } 2519 } 2520 if (port->need_reconfig_queues > 0) { 2521 port->need_reconfig_queues = 0; 2522 /* setup tx queues */ 2523 for (qi = 0; qi < nb_txq; qi++) { 2524 if ((numa_support) && 2525 (txring_numa[pi] != NUMA_NO_CONFIG)) 2526 diag = rte_eth_tx_queue_setup(pi, qi, 2527 port->nb_tx_desc[qi], 2528 txring_numa[pi], 2529 &(port->tx_conf[qi])); 2530 else 2531 diag = rte_eth_tx_queue_setup(pi, qi, 2532 port->nb_tx_desc[qi], 2533 port->socket_id, 2534 &(port->tx_conf[qi])); 2535 2536 if (diag == 0) 2537 continue; 2538 2539 /* Fail to setup tx queue, return */ 2540 if (rte_atomic16_cmpset(&(port->port_status), 2541 RTE_PORT_HANDLING, 2542 RTE_PORT_STOPPED) == 0) 2543 printf("Port %d can not be set back " 2544 "to stopped\n", pi); 2545 printf("Fail to configure port %d tx queues\n", 2546 pi); 2547 /* try to reconfigure queues next time */ 2548 port->need_reconfig_queues = 1; 2549 return -1; 2550 } 2551 for (qi = 0; qi < nb_rxq; qi++) { 2552 /* setup rx queues */ 2553 if ((numa_support) && 2554 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2555 struct rte_mempool * mp = 2556 mbuf_pool_find 2557 (rxring_numa[pi], 0); 2558 if (mp == NULL) { 2559 printf("Failed to setup RX queue:" 2560 "No mempool allocation" 2561 " on the socket %d\n", 2562 rxring_numa[pi]); 2563 return -1; 2564 } 2565 2566 diag = rx_queue_setup(pi, qi, 2567 port->nb_rx_desc[qi], 2568 rxring_numa[pi], 2569 &(port->rx_conf[qi]), 2570 mp); 2571 } else { 2572 struct rte_mempool *mp = 2573 mbuf_pool_find 2574 (port->socket_id, 0); 2575 if (mp == NULL) { 2576 printf("Failed to setup RX queue:" 2577 "No mempool allocation" 2578 " on the socket %d\n", 2579 port->socket_id); 2580 return -1; 2581 } 2582 diag = rx_queue_setup(pi, qi, 2583 port->nb_rx_desc[qi], 2584 port->socket_id, 2585 &(port->rx_conf[qi]), 2586 mp); 2587 } 2588 if (diag == 0) 2589 continue; 2590 2591 /* Fail to setup rx queue, return */ 2592 if (rte_atomic16_cmpset(&(port->port_status), 2593 RTE_PORT_HANDLING, 2594 RTE_PORT_STOPPED) == 0) 2595 printf("Port %d can not be set back " 2596 "to stopped\n", pi); 2597 printf("Fail to configure port %d rx queues\n", 2598 pi); 2599 /* try to reconfigure queues next time */ 2600 port->need_reconfig_queues = 1; 2601 return -1; 2602 } 2603 /* setup hairpin queues */ 2604 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2605 return -1; 2606 } 2607 configure_rxtx_dump_callbacks(verbose_level); 2608 if (clear_ptypes) { 2609 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2610 NULL, 0); 2611 if (diag < 0) 2612 printf( 2613 "Port %d: Failed to disable Ptype parsing\n", 2614 pi); 2615 } 2616 2617 p_pi = pi; 2618 cnt_pi++; 2619 2620 /* start port */ 2621 if (rte_eth_dev_start(pi) < 0) { 2622 printf("Fail to start port %d\n", pi); 2623 2624 /* Fail to setup rx queue, return */ 2625 if (rte_atomic16_cmpset(&(port->port_status), 2626 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2627 printf("Port %d can not be set back to " 2628 "stopped\n", pi); 2629 continue; 2630 } 2631 2632 if (rte_atomic16_cmpset(&(port->port_status), 2633 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2634 printf("Port %d can not be set into started\n", pi); 2635 2636 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2637 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2638 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2639 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2640 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2641 2642 /* at least one port started, need checking link status */ 2643 need_check_link_status = 1; 2644 2645 pl[cfg_pi++] = pi; 2646 } 2647 2648 if (need_check_link_status == 1 && !no_link_check) 2649 check_all_ports_link_status(RTE_PORT_ALL); 2650 else if (need_check_link_status == 0) 2651 printf("Please stop the ports first\n"); 2652 2653 if (hairpin_mode & 0xf) { 2654 uint16_t i; 2655 int j; 2656 2657 /* bind all started hairpin ports */ 2658 for (i = 0; i < cfg_pi; i++) { 2659 pi = pl[i]; 2660 /* bind current Tx to all peer Rx */ 2661 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2662 RTE_MAX_ETHPORTS, 1); 2663 if (peer_pi < 0) 2664 return peer_pi; 2665 for (j = 0; j < peer_pi; j++) { 2666 if (!port_is_started(peer_pl[j])) 2667 continue; 2668 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2669 if (diag < 0) { 2670 printf("Error during binding hairpin" 2671 " Tx port %u to %u: %s\n", 2672 pi, peer_pl[j], 2673 rte_strerror(-diag)); 2674 return -1; 2675 } 2676 } 2677 /* bind all peer Tx to current Rx */ 2678 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2679 RTE_MAX_ETHPORTS, 0); 2680 if (peer_pi < 0) 2681 return peer_pi; 2682 for (j = 0; j < peer_pi; j++) { 2683 if (!port_is_started(peer_pl[j])) 2684 continue; 2685 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2686 if (diag < 0) { 2687 printf("Error during binding hairpin" 2688 " Tx port %u to %u: %s\n", 2689 peer_pl[j], pi, 2690 rte_strerror(-diag)); 2691 return -1; 2692 } 2693 } 2694 } 2695 } 2696 2697 printf("Done\n"); 2698 return 0; 2699 } 2700 2701 void 2702 stop_port(portid_t pid) 2703 { 2704 portid_t pi; 2705 struct rte_port *port; 2706 int need_check_link_status = 0; 2707 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2708 int peer_pi; 2709 2710 if (dcb_test) { 2711 dcb_test = 0; 2712 dcb_config = 0; 2713 } 2714 2715 if (port_id_is_invalid(pid, ENABLED_WARN)) 2716 return; 2717 2718 printf("Stopping ports...\n"); 2719 2720 RTE_ETH_FOREACH_DEV(pi) { 2721 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2722 continue; 2723 2724 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2725 printf("Please remove port %d from forwarding configuration.\n", pi); 2726 continue; 2727 } 2728 2729 if (port_is_bonding_slave(pi)) { 2730 printf("Please remove port %d from bonded device.\n", pi); 2731 continue; 2732 } 2733 2734 port = &ports[pi]; 2735 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2736 RTE_PORT_HANDLING) == 0) 2737 continue; 2738 2739 if (hairpin_mode & 0xf) { 2740 int j; 2741 2742 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2743 /* unbind all peer Tx from current Rx */ 2744 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2745 RTE_MAX_ETHPORTS, 0); 2746 if (peer_pi < 0) 2747 continue; 2748 for (j = 0; j < peer_pi; j++) { 2749 if (!port_is_started(peer_pl[j])) 2750 continue; 2751 rte_eth_hairpin_unbind(peer_pl[j], pi); 2752 } 2753 } 2754 2755 if (port->flow_list) 2756 port_flow_flush(pi); 2757 2758 if (rte_eth_dev_stop(pi) != 0) 2759 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2760 pi); 2761 2762 if (rte_atomic16_cmpset(&(port->port_status), 2763 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2764 printf("Port %d can not be set into stopped\n", pi); 2765 need_check_link_status = 1; 2766 } 2767 if (need_check_link_status && !no_link_check) 2768 check_all_ports_link_status(RTE_PORT_ALL); 2769 2770 printf("Done\n"); 2771 } 2772 2773 static void 2774 remove_invalid_ports_in(portid_t *array, portid_t *total) 2775 { 2776 portid_t i; 2777 portid_t new_total = 0; 2778 2779 for (i = 0; i < *total; i++) 2780 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2781 array[new_total] = array[i]; 2782 new_total++; 2783 } 2784 *total = new_total; 2785 } 2786 2787 static void 2788 remove_invalid_ports(void) 2789 { 2790 remove_invalid_ports_in(ports_ids, &nb_ports); 2791 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2792 nb_cfg_ports = nb_fwd_ports; 2793 } 2794 2795 void 2796 close_port(portid_t pid) 2797 { 2798 portid_t pi; 2799 struct rte_port *port; 2800 2801 if (port_id_is_invalid(pid, ENABLED_WARN)) 2802 return; 2803 2804 printf("Closing ports...\n"); 2805 2806 RTE_ETH_FOREACH_DEV(pi) { 2807 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2808 continue; 2809 2810 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2811 printf("Please remove port %d from forwarding configuration.\n", pi); 2812 continue; 2813 } 2814 2815 if (port_is_bonding_slave(pi)) { 2816 printf("Please remove port %d from bonded device.\n", pi); 2817 continue; 2818 } 2819 2820 port = &ports[pi]; 2821 if (rte_atomic16_cmpset(&(port->port_status), 2822 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2823 printf("Port %d is already closed\n", pi); 2824 continue; 2825 } 2826 2827 port_flow_flush(pi); 2828 rte_eth_dev_close(pi); 2829 } 2830 2831 remove_invalid_ports(); 2832 printf("Done\n"); 2833 } 2834 2835 void 2836 reset_port(portid_t pid) 2837 { 2838 int diag; 2839 portid_t pi; 2840 struct rte_port *port; 2841 2842 if (port_id_is_invalid(pid, ENABLED_WARN)) 2843 return; 2844 2845 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2846 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2847 printf("Can not reset port(s), please stop port(s) first.\n"); 2848 return; 2849 } 2850 2851 printf("Resetting ports...\n"); 2852 2853 RTE_ETH_FOREACH_DEV(pi) { 2854 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2855 continue; 2856 2857 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2858 printf("Please remove port %d from forwarding " 2859 "configuration.\n", pi); 2860 continue; 2861 } 2862 2863 if (port_is_bonding_slave(pi)) { 2864 printf("Please remove port %d from bonded device.\n", 2865 pi); 2866 continue; 2867 } 2868 2869 diag = rte_eth_dev_reset(pi); 2870 if (diag == 0) { 2871 port = &ports[pi]; 2872 port->need_reconfig = 1; 2873 port->need_reconfig_queues = 1; 2874 } else { 2875 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2876 } 2877 } 2878 2879 printf("Done\n"); 2880 } 2881 2882 void 2883 attach_port(char *identifier) 2884 { 2885 portid_t pi; 2886 struct rte_dev_iterator iterator; 2887 2888 printf("Attaching a new port...\n"); 2889 2890 if (identifier == NULL) { 2891 printf("Invalid parameters are specified\n"); 2892 return; 2893 } 2894 2895 if (rte_dev_probe(identifier) < 0) { 2896 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2897 return; 2898 } 2899 2900 /* first attach mode: event */ 2901 if (setup_on_probe_event) { 2902 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2903 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2904 if (ports[pi].port_status == RTE_PORT_HANDLING && 2905 ports[pi].need_setup != 0) 2906 setup_attached_port(pi); 2907 return; 2908 } 2909 2910 /* second attach mode: iterator */ 2911 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2912 /* setup ports matching the devargs used for probing */ 2913 if (port_is_forwarding(pi)) 2914 continue; /* port was already attached before */ 2915 setup_attached_port(pi); 2916 } 2917 } 2918 2919 static void 2920 setup_attached_port(portid_t pi) 2921 { 2922 unsigned int socket_id; 2923 int ret; 2924 2925 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2926 /* if socket_id is invalid, set to the first available socket. */ 2927 if (check_socket_id(socket_id) < 0) 2928 socket_id = socket_ids[0]; 2929 reconfig(pi, socket_id); 2930 ret = rte_eth_promiscuous_enable(pi); 2931 if (ret != 0) 2932 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2933 pi, rte_strerror(-ret)); 2934 2935 ports_ids[nb_ports++] = pi; 2936 fwd_ports_ids[nb_fwd_ports++] = pi; 2937 nb_cfg_ports = nb_fwd_ports; 2938 ports[pi].need_setup = 0; 2939 ports[pi].port_status = RTE_PORT_STOPPED; 2940 2941 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2942 printf("Done\n"); 2943 } 2944 2945 static void 2946 detach_device(struct rte_device *dev) 2947 { 2948 portid_t sibling; 2949 2950 if (dev == NULL) { 2951 printf("Device already removed\n"); 2952 return; 2953 } 2954 2955 printf("Removing a device...\n"); 2956 2957 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2958 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2959 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2960 printf("Port %u not stopped\n", sibling); 2961 return; 2962 } 2963 port_flow_flush(sibling); 2964 } 2965 } 2966 2967 if (rte_dev_remove(dev) < 0) { 2968 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2969 return; 2970 } 2971 remove_invalid_ports(); 2972 2973 printf("Device is detached\n"); 2974 printf("Now total ports is %d\n", nb_ports); 2975 printf("Done\n"); 2976 return; 2977 } 2978 2979 void 2980 detach_port_device(portid_t port_id) 2981 { 2982 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2983 return; 2984 2985 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2986 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2987 printf("Port not stopped\n"); 2988 return; 2989 } 2990 printf("Port was not closed\n"); 2991 } 2992 2993 detach_device(rte_eth_devices[port_id].device); 2994 } 2995 2996 void 2997 detach_devargs(char *identifier) 2998 { 2999 struct rte_dev_iterator iterator; 3000 struct rte_devargs da; 3001 portid_t port_id; 3002 3003 printf("Removing a device...\n"); 3004 3005 memset(&da, 0, sizeof(da)); 3006 if (rte_devargs_parsef(&da, "%s", identifier)) { 3007 printf("cannot parse identifier\n"); 3008 if (da.args) 3009 free(da.args); 3010 return; 3011 } 3012 3013 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3014 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3015 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3016 printf("Port %u not stopped\n", port_id); 3017 rte_eth_iterator_cleanup(&iterator); 3018 return; 3019 } 3020 port_flow_flush(port_id); 3021 } 3022 } 3023 3024 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3025 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3026 da.name, da.bus->name); 3027 return; 3028 } 3029 3030 remove_invalid_ports(); 3031 3032 printf("Device %s is detached\n", identifier); 3033 printf("Now total ports is %d\n", nb_ports); 3034 printf("Done\n"); 3035 } 3036 3037 void 3038 pmd_test_exit(void) 3039 { 3040 portid_t pt_id; 3041 unsigned int i; 3042 int ret; 3043 3044 if (test_done == 0) 3045 stop_packet_forwarding(); 3046 3047 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3048 if (mempools[i]) { 3049 if (mp_alloc_type == MP_ALLOC_ANON) 3050 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3051 NULL); 3052 } 3053 } 3054 if (ports != NULL) { 3055 no_link_check = 1; 3056 RTE_ETH_FOREACH_DEV(pt_id) { 3057 printf("\nStopping port %d...\n", pt_id); 3058 fflush(stdout); 3059 stop_port(pt_id); 3060 } 3061 RTE_ETH_FOREACH_DEV(pt_id) { 3062 printf("\nShutting down port %d...\n", pt_id); 3063 fflush(stdout); 3064 close_port(pt_id); 3065 } 3066 } 3067 3068 if (hot_plug) { 3069 ret = rte_dev_event_monitor_stop(); 3070 if (ret) { 3071 RTE_LOG(ERR, EAL, 3072 "fail to stop device event monitor."); 3073 return; 3074 } 3075 3076 ret = rte_dev_event_callback_unregister(NULL, 3077 dev_event_callback, NULL); 3078 if (ret < 0) { 3079 RTE_LOG(ERR, EAL, 3080 "fail to unregister device event callback.\n"); 3081 return; 3082 } 3083 3084 ret = rte_dev_hotplug_handle_disable(); 3085 if (ret) { 3086 RTE_LOG(ERR, EAL, 3087 "fail to disable hotplug handling.\n"); 3088 return; 3089 } 3090 } 3091 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3092 if (mempools[i]) 3093 rte_mempool_free(mempools[i]); 3094 } 3095 3096 printf("\nBye...\n"); 3097 } 3098 3099 typedef void (*cmd_func_t)(void); 3100 struct pmd_test_command { 3101 const char *cmd_name; 3102 cmd_func_t cmd_func; 3103 }; 3104 3105 /* Check the link status of all ports in up to 9s, and print them finally */ 3106 static void 3107 check_all_ports_link_status(uint32_t port_mask) 3108 { 3109 #define CHECK_INTERVAL 100 /* 100ms */ 3110 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3111 portid_t portid; 3112 uint8_t count, all_ports_up, print_flag = 0; 3113 struct rte_eth_link link; 3114 int ret; 3115 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3116 3117 printf("Checking link statuses...\n"); 3118 fflush(stdout); 3119 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3120 all_ports_up = 1; 3121 RTE_ETH_FOREACH_DEV(portid) { 3122 if ((port_mask & (1 << portid)) == 0) 3123 continue; 3124 memset(&link, 0, sizeof(link)); 3125 ret = rte_eth_link_get_nowait(portid, &link); 3126 if (ret < 0) { 3127 all_ports_up = 0; 3128 if (print_flag == 1) 3129 printf("Port %u link get failed: %s\n", 3130 portid, rte_strerror(-ret)); 3131 continue; 3132 } 3133 /* print link status if flag set */ 3134 if (print_flag == 1) { 3135 rte_eth_link_to_str(link_status, 3136 sizeof(link_status), &link); 3137 printf("Port %d %s\n", portid, link_status); 3138 continue; 3139 } 3140 /* clear all_ports_up flag if any link down */ 3141 if (link.link_status == ETH_LINK_DOWN) { 3142 all_ports_up = 0; 3143 break; 3144 } 3145 } 3146 /* after finally printing all link status, get out */ 3147 if (print_flag == 1) 3148 break; 3149 3150 if (all_ports_up == 0) { 3151 fflush(stdout); 3152 rte_delay_ms(CHECK_INTERVAL); 3153 } 3154 3155 /* set the print_flag if all ports up or timeout */ 3156 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3157 print_flag = 1; 3158 } 3159 3160 if (lsc_interrupt) 3161 break; 3162 } 3163 } 3164 3165 static void 3166 rmv_port_callback(void *arg) 3167 { 3168 int need_to_start = 0; 3169 int org_no_link_check = no_link_check; 3170 portid_t port_id = (intptr_t)arg; 3171 struct rte_device *dev; 3172 3173 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3174 3175 if (!test_done && port_is_forwarding(port_id)) { 3176 need_to_start = 1; 3177 stop_packet_forwarding(); 3178 } 3179 no_link_check = 1; 3180 stop_port(port_id); 3181 no_link_check = org_no_link_check; 3182 3183 /* Save rte_device pointer before closing ethdev port */ 3184 dev = rte_eth_devices[port_id].device; 3185 close_port(port_id); 3186 detach_device(dev); /* might be already removed or have more ports */ 3187 3188 if (need_to_start) 3189 start_packet_forwarding(0); 3190 } 3191 3192 /* This function is used by the interrupt thread */ 3193 static int 3194 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3195 void *ret_param) 3196 { 3197 RTE_SET_USED(param); 3198 RTE_SET_USED(ret_param); 3199 3200 if (type >= RTE_ETH_EVENT_MAX) { 3201 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3202 port_id, __func__, type); 3203 fflush(stderr); 3204 } else if (event_print_mask & (UINT32_C(1) << type)) { 3205 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3206 eth_event_desc[type]); 3207 fflush(stdout); 3208 } 3209 3210 switch (type) { 3211 case RTE_ETH_EVENT_NEW: 3212 ports[port_id].need_setup = 1; 3213 ports[port_id].port_status = RTE_PORT_HANDLING; 3214 break; 3215 case RTE_ETH_EVENT_INTR_RMV: 3216 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3217 break; 3218 if (rte_eal_alarm_set(100000, 3219 rmv_port_callback, (void *)(intptr_t)port_id)) 3220 fprintf(stderr, "Could not set up deferred device removal\n"); 3221 break; 3222 case RTE_ETH_EVENT_DESTROY: 3223 ports[port_id].port_status = RTE_PORT_CLOSED; 3224 printf("Port %u is closed\n", port_id); 3225 break; 3226 default: 3227 break; 3228 } 3229 return 0; 3230 } 3231 3232 static int 3233 register_eth_event_callback(void) 3234 { 3235 int ret; 3236 enum rte_eth_event_type event; 3237 3238 for (event = RTE_ETH_EVENT_UNKNOWN; 3239 event < RTE_ETH_EVENT_MAX; event++) { 3240 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3241 event, 3242 eth_event_callback, 3243 NULL); 3244 if (ret != 0) { 3245 TESTPMD_LOG(ERR, "Failed to register callback for " 3246 "%s event\n", eth_event_desc[event]); 3247 return -1; 3248 } 3249 } 3250 3251 return 0; 3252 } 3253 3254 /* This function is used by the interrupt thread */ 3255 static void 3256 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3257 __rte_unused void *arg) 3258 { 3259 uint16_t port_id; 3260 int ret; 3261 3262 if (type >= RTE_DEV_EVENT_MAX) { 3263 fprintf(stderr, "%s called upon invalid event %d\n", 3264 __func__, type); 3265 fflush(stderr); 3266 } 3267 3268 switch (type) { 3269 case RTE_DEV_EVENT_REMOVE: 3270 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3271 device_name); 3272 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3273 if (ret) { 3274 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3275 device_name); 3276 return; 3277 } 3278 /* 3279 * Because the user's callback is invoked in eal interrupt 3280 * callback, the interrupt callback need to be finished before 3281 * it can be unregistered when detaching device. So finish 3282 * callback soon and use a deferred removal to detach device 3283 * is need. It is a workaround, once the device detaching be 3284 * moved into the eal in the future, the deferred removal could 3285 * be deleted. 3286 */ 3287 if (rte_eal_alarm_set(100000, 3288 rmv_port_callback, (void *)(intptr_t)port_id)) 3289 RTE_LOG(ERR, EAL, 3290 "Could not set up deferred device removal\n"); 3291 break; 3292 case RTE_DEV_EVENT_ADD: 3293 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3294 device_name); 3295 /* TODO: After finish kernel driver binding, 3296 * begin to attach port. 3297 */ 3298 break; 3299 default: 3300 break; 3301 } 3302 } 3303 3304 static void 3305 rxtx_port_config(struct rte_port *port) 3306 { 3307 uint16_t qid; 3308 uint64_t offloads; 3309 3310 for (qid = 0; qid < nb_rxq; qid++) { 3311 offloads = port->rx_conf[qid].offloads; 3312 port->rx_conf[qid] = port->dev_info.default_rxconf; 3313 if (offloads != 0) 3314 port->rx_conf[qid].offloads = offloads; 3315 3316 /* Check if any Rx parameters have been passed */ 3317 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3318 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3319 3320 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3321 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3322 3323 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3324 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3325 3326 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3327 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3328 3329 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3330 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3331 3332 port->nb_rx_desc[qid] = nb_rxd; 3333 } 3334 3335 for (qid = 0; qid < nb_txq; qid++) { 3336 offloads = port->tx_conf[qid].offloads; 3337 port->tx_conf[qid] = port->dev_info.default_txconf; 3338 if (offloads != 0) 3339 port->tx_conf[qid].offloads = offloads; 3340 3341 /* Check if any Tx parameters have been passed */ 3342 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3343 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3344 3345 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3346 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3347 3348 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3349 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3350 3351 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3352 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3353 3354 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3355 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3356 3357 port->nb_tx_desc[qid] = nb_txd; 3358 } 3359 } 3360 3361 void 3362 init_port_config(void) 3363 { 3364 portid_t pid; 3365 struct rte_port *port; 3366 int ret; 3367 3368 RTE_ETH_FOREACH_DEV(pid) { 3369 port = &ports[pid]; 3370 port->dev_conf.fdir_conf = fdir_conf; 3371 3372 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3373 if (ret != 0) 3374 return; 3375 3376 if (nb_rxq > 1) { 3377 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3378 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3379 rss_hf & port->dev_info.flow_type_rss_offloads; 3380 } else { 3381 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3382 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3383 } 3384 3385 if (port->dcb_flag == 0) { 3386 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3387 port->dev_conf.rxmode.mq_mode = 3388 (enum rte_eth_rx_mq_mode) 3389 (rx_mq_mode & ETH_MQ_RX_RSS); 3390 else 3391 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3392 } 3393 3394 rxtx_port_config(port); 3395 3396 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3397 if (ret != 0) 3398 return; 3399 3400 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3401 rte_pmd_ixgbe_bypass_init(pid); 3402 #endif 3403 3404 if (lsc_interrupt && 3405 (rte_eth_devices[pid].data->dev_flags & 3406 RTE_ETH_DEV_INTR_LSC)) 3407 port->dev_conf.intr_conf.lsc = 1; 3408 if (rmv_interrupt && 3409 (rte_eth_devices[pid].data->dev_flags & 3410 RTE_ETH_DEV_INTR_RMV)) 3411 port->dev_conf.intr_conf.rmv = 1; 3412 } 3413 } 3414 3415 void set_port_slave_flag(portid_t slave_pid) 3416 { 3417 struct rte_port *port; 3418 3419 port = &ports[slave_pid]; 3420 port->slave_flag = 1; 3421 } 3422 3423 void clear_port_slave_flag(portid_t slave_pid) 3424 { 3425 struct rte_port *port; 3426 3427 port = &ports[slave_pid]; 3428 port->slave_flag = 0; 3429 } 3430 3431 uint8_t port_is_bonding_slave(portid_t slave_pid) 3432 { 3433 struct rte_port *port; 3434 3435 port = &ports[slave_pid]; 3436 if ((rte_eth_devices[slave_pid].data->dev_flags & 3437 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3438 return 1; 3439 return 0; 3440 } 3441 3442 const uint16_t vlan_tags[] = { 3443 0, 1, 2, 3, 4, 5, 6, 7, 3444 8, 9, 10, 11, 12, 13, 14, 15, 3445 16, 17, 18, 19, 20, 21, 22, 23, 3446 24, 25, 26, 27, 28, 29, 30, 31 3447 }; 3448 3449 static int 3450 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3451 enum dcb_mode_enable dcb_mode, 3452 enum rte_eth_nb_tcs num_tcs, 3453 uint8_t pfc_en) 3454 { 3455 uint8_t i; 3456 int32_t rc; 3457 struct rte_eth_rss_conf rss_conf; 3458 3459 /* 3460 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3461 * given above, and the number of traffic classes available for use. 3462 */ 3463 if (dcb_mode == DCB_VT_ENABLED) { 3464 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3465 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3466 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3467 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3468 3469 /* VMDQ+DCB RX and TX configurations */ 3470 vmdq_rx_conf->enable_default_pool = 0; 3471 vmdq_rx_conf->default_pool = 0; 3472 vmdq_rx_conf->nb_queue_pools = 3473 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3474 vmdq_tx_conf->nb_queue_pools = 3475 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3476 3477 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3478 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3479 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3480 vmdq_rx_conf->pool_map[i].pools = 3481 1 << (i % vmdq_rx_conf->nb_queue_pools); 3482 } 3483 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3484 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3485 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3486 } 3487 3488 /* set DCB mode of RX and TX of multiple queues */ 3489 eth_conf->rxmode.mq_mode = 3490 (enum rte_eth_rx_mq_mode) 3491 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3492 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3493 } else { 3494 struct rte_eth_dcb_rx_conf *rx_conf = 3495 ð_conf->rx_adv_conf.dcb_rx_conf; 3496 struct rte_eth_dcb_tx_conf *tx_conf = 3497 ð_conf->tx_adv_conf.dcb_tx_conf; 3498 3499 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3500 3501 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3502 if (rc != 0) 3503 return rc; 3504 3505 rx_conf->nb_tcs = num_tcs; 3506 tx_conf->nb_tcs = num_tcs; 3507 3508 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3509 rx_conf->dcb_tc[i] = i % num_tcs; 3510 tx_conf->dcb_tc[i] = i % num_tcs; 3511 } 3512 3513 eth_conf->rxmode.mq_mode = 3514 (enum rte_eth_rx_mq_mode) 3515 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3516 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3517 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3518 } 3519 3520 if (pfc_en) 3521 eth_conf->dcb_capability_en = 3522 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3523 else 3524 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3525 3526 return 0; 3527 } 3528 3529 int 3530 init_port_dcb_config(portid_t pid, 3531 enum dcb_mode_enable dcb_mode, 3532 enum rte_eth_nb_tcs num_tcs, 3533 uint8_t pfc_en) 3534 { 3535 struct rte_eth_conf port_conf; 3536 struct rte_port *rte_port; 3537 int retval; 3538 uint16_t i; 3539 3540 rte_port = &ports[pid]; 3541 3542 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3543 /* Enter DCB configuration status */ 3544 dcb_config = 1; 3545 3546 port_conf.rxmode = rte_port->dev_conf.rxmode; 3547 port_conf.txmode = rte_port->dev_conf.txmode; 3548 3549 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3550 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3551 if (retval < 0) 3552 return retval; 3553 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3554 3555 /* re-configure the device . */ 3556 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3557 if (retval < 0) 3558 return retval; 3559 3560 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3561 if (retval != 0) 3562 return retval; 3563 3564 /* If dev_info.vmdq_pool_base is greater than 0, 3565 * the queue id of vmdq pools is started after pf queues. 3566 */ 3567 if (dcb_mode == DCB_VT_ENABLED && 3568 rte_port->dev_info.vmdq_pool_base > 0) { 3569 printf("VMDQ_DCB multi-queue mode is nonsensical" 3570 " for port %d.", pid); 3571 return -1; 3572 } 3573 3574 /* Assume the ports in testpmd have the same dcb capability 3575 * and has the same number of rxq and txq in dcb mode 3576 */ 3577 if (dcb_mode == DCB_VT_ENABLED) { 3578 if (rte_port->dev_info.max_vfs > 0) { 3579 nb_rxq = rte_port->dev_info.nb_rx_queues; 3580 nb_txq = rte_port->dev_info.nb_tx_queues; 3581 } else { 3582 nb_rxq = rte_port->dev_info.max_rx_queues; 3583 nb_txq = rte_port->dev_info.max_tx_queues; 3584 } 3585 } else { 3586 /*if vt is disabled, use all pf queues */ 3587 if (rte_port->dev_info.vmdq_pool_base == 0) { 3588 nb_rxq = rte_port->dev_info.max_rx_queues; 3589 nb_txq = rte_port->dev_info.max_tx_queues; 3590 } else { 3591 nb_rxq = (queueid_t)num_tcs; 3592 nb_txq = (queueid_t)num_tcs; 3593 3594 } 3595 } 3596 rx_free_thresh = 64; 3597 3598 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3599 3600 rxtx_port_config(rte_port); 3601 /* VLAN filter */ 3602 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3603 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3604 rx_vft_set(pid, vlan_tags[i], 1); 3605 3606 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3607 if (retval != 0) 3608 return retval; 3609 3610 rte_port->dcb_flag = 1; 3611 3612 return 0; 3613 } 3614 3615 static void 3616 init_port(void) 3617 { 3618 int i; 3619 3620 /* Configuration of Ethernet ports. */ 3621 ports = rte_zmalloc("testpmd: ports", 3622 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3623 RTE_CACHE_LINE_SIZE); 3624 if (ports == NULL) { 3625 rte_exit(EXIT_FAILURE, 3626 "rte_zmalloc(%d struct rte_port) failed\n", 3627 RTE_MAX_ETHPORTS); 3628 } 3629 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3630 LIST_INIT(&ports[i].flow_tunnel_list); 3631 /* Initialize ports NUMA structures */ 3632 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3633 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3634 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3635 } 3636 3637 static void 3638 force_quit(void) 3639 { 3640 pmd_test_exit(); 3641 prompt_exit(); 3642 } 3643 3644 static void 3645 print_stats(void) 3646 { 3647 uint8_t i; 3648 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3649 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3650 3651 /* Clear screen and move to top left */ 3652 printf("%s%s", clr, top_left); 3653 3654 printf("\nPort statistics ===================================="); 3655 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3656 nic_stats_display(fwd_ports_ids[i]); 3657 3658 fflush(stdout); 3659 } 3660 3661 static void 3662 signal_handler(int signum) 3663 { 3664 if (signum == SIGINT || signum == SIGTERM) { 3665 printf("\nSignal %d received, preparing to exit...\n", 3666 signum); 3667 #ifdef RTE_LIB_PDUMP 3668 /* uninitialize packet capture framework */ 3669 rte_pdump_uninit(); 3670 #endif 3671 #ifdef RTE_LIB_LATENCYSTATS 3672 if (latencystats_enabled != 0) 3673 rte_latencystats_uninit(); 3674 #endif 3675 force_quit(); 3676 /* Set flag to indicate the force termination. */ 3677 f_quit = 1; 3678 /* exit with the expected status */ 3679 signal(signum, SIG_DFL); 3680 kill(getpid(), signum); 3681 } 3682 } 3683 3684 int 3685 main(int argc, char** argv) 3686 { 3687 int diag; 3688 portid_t port_id; 3689 uint16_t count; 3690 int ret; 3691 3692 signal(SIGINT, signal_handler); 3693 signal(SIGTERM, signal_handler); 3694 3695 testpmd_logtype = rte_log_register("testpmd"); 3696 if (testpmd_logtype < 0) 3697 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3698 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3699 3700 diag = rte_eal_init(argc, argv); 3701 if (diag < 0) 3702 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3703 rte_strerror(rte_errno)); 3704 3705 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3706 rte_exit(EXIT_FAILURE, 3707 "Secondary process type not supported.\n"); 3708 3709 ret = register_eth_event_callback(); 3710 if (ret != 0) 3711 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3712 3713 #ifdef RTE_LIB_PDUMP 3714 /* initialize packet capture framework */ 3715 rte_pdump_init(); 3716 #endif 3717 3718 count = 0; 3719 RTE_ETH_FOREACH_DEV(port_id) { 3720 ports_ids[count] = port_id; 3721 count++; 3722 } 3723 nb_ports = (portid_t) count; 3724 if (nb_ports == 0) 3725 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3726 3727 /* allocate port structures, and init them */ 3728 init_port(); 3729 3730 set_def_fwd_config(); 3731 if (nb_lcores == 0) 3732 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3733 "Check the core mask argument\n"); 3734 3735 /* Bitrate/latency stats disabled by default */ 3736 #ifdef RTE_LIB_BITRATESTATS 3737 bitrate_enabled = 0; 3738 #endif 3739 #ifdef RTE_LIB_LATENCYSTATS 3740 latencystats_enabled = 0; 3741 #endif 3742 3743 /* on FreeBSD, mlockall() is disabled by default */ 3744 #ifdef RTE_EXEC_ENV_FREEBSD 3745 do_mlockall = 0; 3746 #else 3747 do_mlockall = 1; 3748 #endif 3749 3750 argc -= diag; 3751 argv += diag; 3752 if (argc > 1) 3753 launch_args_parse(argc, argv); 3754 3755 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3756 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3757 strerror(errno)); 3758 } 3759 3760 if (tx_first && interactive) 3761 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3762 "interactive mode.\n"); 3763 3764 if (tx_first && lsc_interrupt) { 3765 printf("Warning: lsc_interrupt needs to be off when " 3766 " using tx_first. Disabling.\n"); 3767 lsc_interrupt = 0; 3768 } 3769 3770 if (!nb_rxq && !nb_txq) 3771 printf("Warning: Either rx or tx queues should be non-zero\n"); 3772 3773 if (nb_rxq > 1 && nb_rxq > nb_txq) 3774 printf("Warning: nb_rxq=%d enables RSS configuration, " 3775 "but nb_txq=%d will prevent to fully test it.\n", 3776 nb_rxq, nb_txq); 3777 3778 init_config(); 3779 3780 if (hot_plug) { 3781 ret = rte_dev_hotplug_handle_enable(); 3782 if (ret) { 3783 RTE_LOG(ERR, EAL, 3784 "fail to enable hotplug handling."); 3785 return -1; 3786 } 3787 3788 ret = rte_dev_event_monitor_start(); 3789 if (ret) { 3790 RTE_LOG(ERR, EAL, 3791 "fail to start device event monitoring."); 3792 return -1; 3793 } 3794 3795 ret = rte_dev_event_callback_register(NULL, 3796 dev_event_callback, NULL); 3797 if (ret) { 3798 RTE_LOG(ERR, EAL, 3799 "fail to register device event callback\n"); 3800 return -1; 3801 } 3802 } 3803 3804 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3805 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3806 3807 /* set all ports to promiscuous mode by default */ 3808 RTE_ETH_FOREACH_DEV(port_id) { 3809 ret = rte_eth_promiscuous_enable(port_id); 3810 if (ret != 0) 3811 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3812 port_id, rte_strerror(-ret)); 3813 } 3814 3815 /* Init metrics library */ 3816 rte_metrics_init(rte_socket_id()); 3817 3818 #ifdef RTE_LIB_LATENCYSTATS 3819 if (latencystats_enabled != 0) { 3820 int ret = rte_latencystats_init(1, NULL); 3821 if (ret) 3822 printf("Warning: latencystats init()" 3823 " returned error %d\n", ret); 3824 printf("Latencystats running on lcore %d\n", 3825 latencystats_lcore_id); 3826 } 3827 #endif 3828 3829 /* Setup bitrate stats */ 3830 #ifdef RTE_LIB_BITRATESTATS 3831 if (bitrate_enabled != 0) { 3832 bitrate_data = rte_stats_bitrate_create(); 3833 if (bitrate_data == NULL) 3834 rte_exit(EXIT_FAILURE, 3835 "Could not allocate bitrate data.\n"); 3836 rte_stats_bitrate_reg(bitrate_data); 3837 } 3838 #endif 3839 3840 #ifdef RTE_LIB_CMDLINE 3841 if (strlen(cmdline_filename) != 0) 3842 cmdline_read_from_file(cmdline_filename); 3843 3844 if (interactive == 1) { 3845 if (auto_start) { 3846 printf("Start automatic packet forwarding\n"); 3847 start_packet_forwarding(0); 3848 } 3849 prompt(); 3850 pmd_test_exit(); 3851 } else 3852 #endif 3853 { 3854 char c; 3855 int rc; 3856 3857 f_quit = 0; 3858 3859 printf("No commandline core given, start packet forwarding\n"); 3860 start_packet_forwarding(tx_first); 3861 if (stats_period != 0) { 3862 uint64_t prev_time = 0, cur_time, diff_time = 0; 3863 uint64_t timer_period; 3864 3865 /* Convert to number of cycles */ 3866 timer_period = stats_period * rte_get_timer_hz(); 3867 3868 while (f_quit == 0) { 3869 cur_time = rte_get_timer_cycles(); 3870 diff_time += cur_time - prev_time; 3871 3872 if (diff_time >= timer_period) { 3873 print_stats(); 3874 /* Reset the timer */ 3875 diff_time = 0; 3876 } 3877 /* Sleep to avoid unnecessary checks */ 3878 prev_time = cur_time; 3879 sleep(1); 3880 } 3881 } 3882 3883 printf("Press enter to exit\n"); 3884 rc = read(0, &c, 1); 3885 pmd_test_exit(); 3886 if (rc < 0) 3887 return 1; 3888 } 3889 3890 ret = rte_eal_cleanup(); 3891 if (ret != 0) 3892 rte_exit(EXIT_FAILURE, 3893 "EAL cleanup failed: %s\n", strerror(-ret)); 3894 3895 return EXIT_SUCCESS; 3896 } 3897