1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct fwd_config cur_fwd_config; 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 193 uint32_t retry_enabled; 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 196 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 198 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 199 * specified on command-line. */ 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 201 202 /* 203 * In container, it cannot terminate the process which running with 'stats-period' 204 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 205 */ 206 uint8_t f_quit; 207 208 /* 209 * Configuration of packet segments used by the "txonly" processing engine. 210 */ 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 213 TXONLY_DEF_PACKET_LEN, 214 }; 215 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 216 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 218 /**< Split policy for packets to TX. */ 219 220 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 221 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 222 223 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 224 uint8_t dcb_config = 0; 225 226 /* Whether the dcb is in testing status */ 227 uint8_t dcb_test = 0; 228 229 /* 230 * Configurable number of RX/TX queues. 231 */ 232 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 233 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 234 235 /* 236 * Configurable number of RX/TX ring descriptors. 237 * Defaults are supplied by drivers via ethdev. 238 */ 239 #define RTE_TEST_RX_DESC_DEFAULT 0 240 #define RTE_TEST_TX_DESC_DEFAULT 0 241 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 242 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 243 244 #define RTE_PMD_PARAM_UNSET -1 245 /* 246 * Configurable values of RX and TX ring threshold registers. 247 */ 248 249 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 250 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 251 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 252 253 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 254 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 255 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 256 257 /* 258 * Configurable value of RX free threshold. 259 */ 260 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 261 262 /* 263 * Configurable value of RX drop enable. 264 */ 265 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 266 267 /* 268 * Configurable value of TX free threshold. 269 */ 270 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 271 272 /* 273 * Configurable value of TX RS bit threshold. 274 */ 275 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 276 277 /* 278 * Configurable value of buffered packets before sending. 279 */ 280 uint16_t noisy_tx_sw_bufsz; 281 282 /* 283 * Configurable value of packet buffer timeout. 284 */ 285 uint16_t noisy_tx_sw_buf_flush_time; 286 287 /* 288 * Configurable value for size of VNF internal memory area 289 * used for simulating noisy neighbour behaviour 290 */ 291 uint64_t noisy_lkup_mem_sz; 292 293 /* 294 * Configurable value of number of random writes done in 295 * VNF simulation memory area. 296 */ 297 uint64_t noisy_lkup_num_writes; 298 299 /* 300 * Configurable value of number of random reads done in 301 * VNF simulation memory area. 302 */ 303 uint64_t noisy_lkup_num_reads; 304 305 /* 306 * Configurable value of number of random reads/writes done in 307 * VNF simulation memory area. 308 */ 309 uint64_t noisy_lkup_num_reads_writes; 310 311 /* 312 * Receive Side Scaling (RSS) configuration. 313 */ 314 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 315 316 /* 317 * Port topology configuration 318 */ 319 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 320 321 /* 322 * Avoids to flush all the RX streams before starts forwarding. 323 */ 324 uint8_t no_flush_rx = 0; /* flush by default */ 325 326 /* 327 * Flow API isolated mode. 328 */ 329 uint8_t flow_isolate_all; 330 331 /* 332 * Avoids to check link status when starting/stopping a port. 333 */ 334 uint8_t no_link_check = 0; /* check by default */ 335 336 /* 337 * Enable link status change notification 338 */ 339 uint8_t lsc_interrupt = 1; /* enabled by default */ 340 341 /* 342 * Enable device removal notification. 343 */ 344 uint8_t rmv_interrupt = 1; /* enabled by default */ 345 346 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 347 348 /* Pretty printing of ethdev events */ 349 static const char * const eth_event_desc[] = { 350 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 351 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 352 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 353 [RTE_ETH_EVENT_INTR_RESET] = "reset", 354 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 355 [RTE_ETH_EVENT_IPSEC] = "IPsec", 356 [RTE_ETH_EVENT_MACSEC] = "MACsec", 357 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 358 [RTE_ETH_EVENT_NEW] = "device probed", 359 [RTE_ETH_EVENT_DESTROY] = "device released", 360 [RTE_ETH_EVENT_MAX] = NULL, 361 }; 362 363 /* 364 * Display or mask ether events 365 * Default to all events except VF_MBOX 366 */ 367 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 368 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 369 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 370 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 371 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 372 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 373 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 374 /* 375 * Decide if all memory are locked for performance. 376 */ 377 int do_mlockall = 0; 378 379 /* 380 * NIC bypass mode configuration options. 381 */ 382 383 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 384 /* The NIC bypass watchdog timeout. */ 385 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 386 #endif 387 388 389 #ifdef RTE_LIBRTE_LATENCY_STATS 390 391 /* 392 * Set when latency stats is enabled in the commandline 393 */ 394 uint8_t latencystats_enabled; 395 396 /* 397 * Lcore ID to serive latency statistics. 398 */ 399 lcoreid_t latencystats_lcore_id = -1; 400 401 #endif 402 403 /* 404 * Ethernet device configuration. 405 */ 406 struct rte_eth_rxmode rx_mode = { 407 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 408 }; 409 410 struct rte_eth_txmode tx_mode = { 411 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 412 }; 413 414 struct rte_fdir_conf fdir_conf = { 415 .mode = RTE_FDIR_MODE_NONE, 416 .pballoc = RTE_FDIR_PBALLOC_64K, 417 .status = RTE_FDIR_REPORT_STATUS, 418 .mask = { 419 .vlan_tci_mask = 0xFFEF, 420 .ipv4_mask = { 421 .src_ip = 0xFFFFFFFF, 422 .dst_ip = 0xFFFFFFFF, 423 }, 424 .ipv6_mask = { 425 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 426 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 427 }, 428 .src_port_mask = 0xFFFF, 429 .dst_port_mask = 0xFFFF, 430 .mac_addr_byte_mask = 0xFF, 431 .tunnel_type_mask = 1, 432 .tunnel_id_mask = 0xFFFFFFFF, 433 }, 434 .drop_queue = 127, 435 }; 436 437 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 438 439 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 440 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 441 442 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 443 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 444 445 uint16_t nb_tx_queue_stats_mappings = 0; 446 uint16_t nb_rx_queue_stats_mappings = 0; 447 448 /* 449 * Display zero values by default for xstats 450 */ 451 uint8_t xstats_hide_zero; 452 453 unsigned int num_sockets = 0; 454 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 455 456 #ifdef RTE_LIBRTE_BITRATE 457 /* Bitrate statistics */ 458 struct rte_stats_bitrates *bitrate_data; 459 lcoreid_t bitrate_lcore_id; 460 uint8_t bitrate_enabled; 461 #endif 462 463 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 464 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 465 466 struct vxlan_encap_conf vxlan_encap_conf = { 467 .select_ipv4 = 1, 468 .select_vlan = 0, 469 .vni = "\x00\x00\x00", 470 .udp_src = 0, 471 .udp_dst = RTE_BE16(4789), 472 .ipv4_src = IPv4(127, 0, 0, 1), 473 .ipv4_dst = IPv4(255, 255, 255, 255), 474 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 475 "\x00\x00\x00\x00\x00\x00\x00\x01", 476 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 477 "\x00\x00\x00\x00\x00\x00\x11\x11", 478 .vlan_tci = 0, 479 .eth_src = "\x00\x00\x00\x00\x00\x00", 480 .eth_dst = "\xff\xff\xff\xff\xff\xff", 481 }; 482 483 struct nvgre_encap_conf nvgre_encap_conf = { 484 .select_ipv4 = 1, 485 .select_vlan = 0, 486 .tni = "\x00\x00\x00", 487 .ipv4_src = IPv4(127, 0, 0, 1), 488 .ipv4_dst = IPv4(255, 255, 255, 255), 489 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 490 "\x00\x00\x00\x00\x00\x00\x00\x01", 491 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 492 "\x00\x00\x00\x00\x00\x00\x11\x11", 493 .vlan_tci = 0, 494 .eth_src = "\x00\x00\x00\x00\x00\x00", 495 .eth_dst = "\xff\xff\xff\xff\xff\xff", 496 }; 497 498 /* Forward function declarations */ 499 static void setup_attached_port(portid_t pi); 500 static void map_port_queue_stats_mapping_registers(portid_t pi, 501 struct rte_port *port); 502 static void check_all_ports_link_status(uint32_t port_mask); 503 static int eth_event_callback(portid_t port_id, 504 enum rte_eth_event_type type, 505 void *param, void *ret_param); 506 static void eth_dev_event_callback(const char *device_name, 507 enum rte_dev_event_type type, 508 void *param); 509 510 /* 511 * Check if all the ports are started. 512 * If yes, return positive value. If not, return zero. 513 */ 514 static int all_ports_started(void); 515 516 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 517 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN; 518 519 /* 520 * Helper function to check if socket is already discovered. 521 * If yes, return positive value. If not, return zero. 522 */ 523 int 524 new_socket_id(unsigned int socket_id) 525 { 526 unsigned int i; 527 528 for (i = 0; i < num_sockets; i++) { 529 if (socket_ids[i] == socket_id) 530 return 0; 531 } 532 return 1; 533 } 534 535 /* 536 * Setup default configuration. 537 */ 538 static void 539 set_default_fwd_lcores_config(void) 540 { 541 unsigned int i; 542 unsigned int nb_lc; 543 unsigned int sock_num; 544 545 nb_lc = 0; 546 for (i = 0; i < RTE_MAX_LCORE; i++) { 547 if (!rte_lcore_is_enabled(i)) 548 continue; 549 sock_num = rte_lcore_to_socket_id(i); 550 if (new_socket_id(sock_num)) { 551 if (num_sockets >= RTE_MAX_NUMA_NODES) { 552 rte_exit(EXIT_FAILURE, 553 "Total sockets greater than %u\n", 554 RTE_MAX_NUMA_NODES); 555 } 556 socket_ids[num_sockets++] = sock_num; 557 } 558 if (i == rte_get_master_lcore()) 559 continue; 560 fwd_lcores_cpuids[nb_lc++] = i; 561 } 562 nb_lcores = (lcoreid_t) nb_lc; 563 nb_cfg_lcores = nb_lcores; 564 nb_fwd_lcores = 1; 565 } 566 567 static void 568 set_def_peer_eth_addrs(void) 569 { 570 portid_t i; 571 572 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 573 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 574 peer_eth_addrs[i].addr_bytes[5] = i; 575 } 576 } 577 578 static void 579 set_default_fwd_ports_config(void) 580 { 581 portid_t pt_id; 582 int i = 0; 583 584 RTE_ETH_FOREACH_DEV(pt_id) { 585 fwd_ports_ids[i++] = pt_id; 586 587 /* Update sockets info according to the attached device */ 588 int socket_id = rte_eth_dev_socket_id(pt_id); 589 if (socket_id >= 0 && new_socket_id(socket_id)) { 590 if (num_sockets >= RTE_MAX_NUMA_NODES) { 591 rte_exit(EXIT_FAILURE, 592 "Total sockets greater than %u\n", 593 RTE_MAX_NUMA_NODES); 594 } 595 socket_ids[num_sockets++] = socket_id; 596 } 597 } 598 599 nb_cfg_ports = nb_ports; 600 nb_fwd_ports = nb_ports; 601 } 602 603 void 604 set_def_fwd_config(void) 605 { 606 set_default_fwd_lcores_config(); 607 set_def_peer_eth_addrs(); 608 set_default_fwd_ports_config(); 609 } 610 611 /* extremely pessimistic estimation of memory required to create a mempool */ 612 static int 613 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 614 { 615 unsigned int n_pages, mbuf_per_pg, leftover; 616 uint64_t total_mem, mbuf_mem, obj_sz; 617 618 /* there is no good way to predict how much space the mempool will 619 * occupy because it will allocate chunks on the fly, and some of those 620 * will come from default DPDK memory while some will come from our 621 * external memory, so just assume 128MB will be enough for everyone. 622 */ 623 uint64_t hdr_mem = 128 << 20; 624 625 /* account for possible non-contiguousness */ 626 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 627 if (obj_sz > pgsz) { 628 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 629 return -1; 630 } 631 632 mbuf_per_pg = pgsz / obj_sz; 633 leftover = (nb_mbufs % mbuf_per_pg) > 0; 634 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 635 636 mbuf_mem = n_pages * pgsz; 637 638 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 639 640 if (total_mem > SIZE_MAX) { 641 TESTPMD_LOG(ERR, "Memory size too big\n"); 642 return -1; 643 } 644 *out = (size_t)total_mem; 645 646 return 0; 647 } 648 649 static inline uint32_t 650 bsf64(uint64_t v) 651 { 652 return (uint32_t)__builtin_ctzll(v); 653 } 654 655 static inline uint32_t 656 log2_u64(uint64_t v) 657 { 658 if (v == 0) 659 return 0; 660 v = rte_align64pow2(v); 661 return bsf64(v); 662 } 663 664 static int 665 pagesz_flags(uint64_t page_sz) 666 { 667 /* as per mmap() manpage, all page sizes are log2 of page size 668 * shifted by MAP_HUGE_SHIFT 669 */ 670 int log2 = log2_u64(page_sz); 671 672 return (log2 << HUGE_SHIFT); 673 } 674 675 static void * 676 alloc_mem(size_t memsz, size_t pgsz, bool huge) 677 { 678 void *addr; 679 int flags; 680 681 /* allocate anonymous hugepages */ 682 flags = MAP_ANONYMOUS | MAP_PRIVATE; 683 if (huge) 684 flags |= HUGE_FLAG | pagesz_flags(pgsz); 685 686 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 687 if (addr == MAP_FAILED) 688 return NULL; 689 690 return addr; 691 } 692 693 struct extmem_param { 694 void *addr; 695 size_t len; 696 size_t pgsz; 697 rte_iova_t *iova_table; 698 unsigned int iova_table_len; 699 }; 700 701 static int 702 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 703 bool huge) 704 { 705 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 706 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 707 unsigned int cur_page, n_pages, pgsz_idx; 708 size_t mem_sz, cur_pgsz; 709 rte_iova_t *iovas = NULL; 710 void *addr; 711 int ret; 712 713 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 714 /* skip anything that is too big */ 715 if (pgsizes[pgsz_idx] > SIZE_MAX) 716 continue; 717 718 cur_pgsz = pgsizes[pgsz_idx]; 719 720 /* if we were told not to allocate hugepages, override */ 721 if (!huge) 722 cur_pgsz = sysconf(_SC_PAGESIZE); 723 724 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 725 if (ret < 0) { 726 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 727 return -1; 728 } 729 730 /* allocate our memory */ 731 addr = alloc_mem(mem_sz, cur_pgsz, huge); 732 733 /* if we couldn't allocate memory with a specified page size, 734 * that doesn't mean we can't do it with other page sizes, so 735 * try another one. 736 */ 737 if (addr == NULL) 738 continue; 739 740 /* store IOVA addresses for every page in this memory area */ 741 n_pages = mem_sz / cur_pgsz; 742 743 iovas = malloc(sizeof(*iovas) * n_pages); 744 745 if (iovas == NULL) { 746 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 747 goto fail; 748 } 749 /* lock memory if it's not huge pages */ 750 if (!huge) 751 mlock(addr, mem_sz); 752 753 /* populate IOVA addresses */ 754 for (cur_page = 0; cur_page < n_pages; cur_page++) { 755 rte_iova_t iova; 756 size_t offset; 757 void *cur; 758 759 offset = cur_pgsz * cur_page; 760 cur = RTE_PTR_ADD(addr, offset); 761 762 /* touch the page before getting its IOVA */ 763 *(volatile char *)cur = 0; 764 765 iova = rte_mem_virt2iova(cur); 766 767 iovas[cur_page] = iova; 768 } 769 770 break; 771 } 772 /* if we couldn't allocate anything */ 773 if (iovas == NULL) 774 return -1; 775 776 param->addr = addr; 777 param->len = mem_sz; 778 param->pgsz = cur_pgsz; 779 param->iova_table = iovas; 780 param->iova_table_len = n_pages; 781 782 return 0; 783 fail: 784 if (iovas) 785 free(iovas); 786 if (addr) 787 munmap(addr, mem_sz); 788 789 return -1; 790 } 791 792 static int 793 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 794 { 795 struct extmem_param param; 796 int socket_id, ret; 797 798 memset(¶m, 0, sizeof(param)); 799 800 /* check if our heap exists */ 801 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 802 if (socket_id < 0) { 803 /* create our heap */ 804 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 805 if (ret < 0) { 806 TESTPMD_LOG(ERR, "Cannot create heap\n"); 807 return -1; 808 } 809 } 810 811 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 812 if (ret < 0) { 813 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 814 return -1; 815 } 816 817 /* we now have a valid memory area, so add it to heap */ 818 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 819 param.addr, param.len, param.iova_table, 820 param.iova_table_len, param.pgsz); 821 822 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 823 824 /* not needed any more */ 825 free(param.iova_table); 826 827 if (ret < 0) { 828 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 829 munmap(param.addr, param.len); 830 return -1; 831 } 832 833 /* success */ 834 835 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 836 param.len >> 20); 837 838 return 0; 839 } 840 841 /* 842 * Configuration initialisation done once at init time. 843 */ 844 static void 845 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 846 unsigned int socket_id) 847 { 848 char pool_name[RTE_MEMPOOL_NAMESIZE]; 849 struct rte_mempool *rte_mp = NULL; 850 uint32_t mb_size; 851 852 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 853 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 854 855 TESTPMD_LOG(INFO, 856 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 857 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 858 859 switch (mp_alloc_type) { 860 case MP_ALLOC_NATIVE: 861 { 862 /* wrapper to rte_mempool_create() */ 863 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 864 rte_mbuf_best_mempool_ops()); 865 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 866 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 867 break; 868 } 869 case MP_ALLOC_ANON: 870 { 871 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 872 mb_size, (unsigned int) mb_mempool_cache, 873 sizeof(struct rte_pktmbuf_pool_private), 874 socket_id, 0); 875 if (rte_mp == NULL) 876 goto err; 877 878 if (rte_mempool_populate_anon(rte_mp) == 0) { 879 rte_mempool_free(rte_mp); 880 rte_mp = NULL; 881 goto err; 882 } 883 rte_pktmbuf_pool_init(rte_mp, NULL); 884 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 885 break; 886 } 887 case MP_ALLOC_XMEM: 888 case MP_ALLOC_XMEM_HUGE: 889 { 890 int heap_socket; 891 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 892 893 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 894 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 895 896 heap_socket = 897 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 898 if (heap_socket < 0) 899 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 900 901 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 902 rte_mbuf_best_mempool_ops()); 903 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 904 mb_mempool_cache, 0, mbuf_seg_size, 905 heap_socket); 906 break; 907 } 908 default: 909 { 910 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 911 } 912 } 913 914 err: 915 if (rte_mp == NULL) { 916 rte_exit(EXIT_FAILURE, 917 "Creation of mbuf pool for socket %u failed: %s\n", 918 socket_id, rte_strerror(rte_errno)); 919 } else if (verbose_level > 0) { 920 rte_mempool_dump(stdout, rte_mp); 921 } 922 } 923 924 /* 925 * Check given socket id is valid or not with NUMA mode, 926 * if valid, return 0, else return -1 927 */ 928 static int 929 check_socket_id(const unsigned int socket_id) 930 { 931 static int warning_once = 0; 932 933 if (new_socket_id(socket_id)) { 934 if (!warning_once && numa_support) 935 printf("Warning: NUMA should be configured manually by" 936 " using --port-numa-config and" 937 " --ring-numa-config parameters along with" 938 " --numa.\n"); 939 warning_once = 1; 940 return -1; 941 } 942 return 0; 943 } 944 945 /* 946 * Get the allowed maximum number of RX queues. 947 * *pid return the port id which has minimal value of 948 * max_rx_queues in all ports. 949 */ 950 queueid_t 951 get_allowed_max_nb_rxq(portid_t *pid) 952 { 953 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 954 portid_t pi; 955 struct rte_eth_dev_info dev_info; 956 957 RTE_ETH_FOREACH_DEV(pi) { 958 rte_eth_dev_info_get(pi, &dev_info); 959 if (dev_info.max_rx_queues < allowed_max_rxq) { 960 allowed_max_rxq = dev_info.max_rx_queues; 961 *pid = pi; 962 } 963 } 964 return allowed_max_rxq; 965 } 966 967 /* 968 * Check input rxq is valid or not. 969 * If input rxq is not greater than any of maximum number 970 * of RX queues of all ports, it is valid. 971 * if valid, return 0, else return -1 972 */ 973 int 974 check_nb_rxq(queueid_t rxq) 975 { 976 queueid_t allowed_max_rxq; 977 portid_t pid = 0; 978 979 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 980 if (rxq > allowed_max_rxq) { 981 printf("Fail: input rxq (%u) can't be greater " 982 "than max_rx_queues (%u) of port %u\n", 983 rxq, 984 allowed_max_rxq, 985 pid); 986 return -1; 987 } 988 return 0; 989 } 990 991 /* 992 * Get the allowed maximum number of TX queues. 993 * *pid return the port id which has minimal value of 994 * max_tx_queues in all ports. 995 */ 996 queueid_t 997 get_allowed_max_nb_txq(portid_t *pid) 998 { 999 queueid_t allowed_max_txq = MAX_QUEUE_ID; 1000 portid_t pi; 1001 struct rte_eth_dev_info dev_info; 1002 1003 RTE_ETH_FOREACH_DEV(pi) { 1004 rte_eth_dev_info_get(pi, &dev_info); 1005 if (dev_info.max_tx_queues < allowed_max_txq) { 1006 allowed_max_txq = dev_info.max_tx_queues; 1007 *pid = pi; 1008 } 1009 } 1010 return allowed_max_txq; 1011 } 1012 1013 /* 1014 * Check input txq is valid or not. 1015 * If input txq is not greater than any of maximum number 1016 * of TX queues of all ports, it is valid. 1017 * if valid, return 0, else return -1 1018 */ 1019 int 1020 check_nb_txq(queueid_t txq) 1021 { 1022 queueid_t allowed_max_txq; 1023 portid_t pid = 0; 1024 1025 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1026 if (txq > allowed_max_txq) { 1027 printf("Fail: input txq (%u) can't be greater " 1028 "than max_tx_queues (%u) of port %u\n", 1029 txq, 1030 allowed_max_txq, 1031 pid); 1032 return -1; 1033 } 1034 return 0; 1035 } 1036 1037 static void 1038 init_config(void) 1039 { 1040 portid_t pid; 1041 struct rte_port *port; 1042 struct rte_mempool *mbp; 1043 unsigned int nb_mbuf_per_pool; 1044 lcoreid_t lc_id; 1045 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1046 struct rte_gro_param gro_param; 1047 uint32_t gso_types; 1048 int k; 1049 1050 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1051 1052 /* Configuration of logical cores. */ 1053 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1054 sizeof(struct fwd_lcore *) * nb_lcores, 1055 RTE_CACHE_LINE_SIZE); 1056 if (fwd_lcores == NULL) { 1057 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1058 "failed\n", nb_lcores); 1059 } 1060 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1061 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1062 sizeof(struct fwd_lcore), 1063 RTE_CACHE_LINE_SIZE); 1064 if (fwd_lcores[lc_id] == NULL) { 1065 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1066 "failed\n"); 1067 } 1068 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1069 } 1070 1071 RTE_ETH_FOREACH_DEV(pid) { 1072 port = &ports[pid]; 1073 /* Apply default TxRx configuration for all ports */ 1074 port->dev_conf.txmode = tx_mode; 1075 port->dev_conf.rxmode = rx_mode; 1076 rte_eth_dev_info_get(pid, &port->dev_info); 1077 1078 if (!(port->dev_info.tx_offload_capa & 1079 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1080 port->dev_conf.txmode.offloads &= 1081 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1082 if (!(port->dev_info.tx_offload_capa & 1083 DEV_TX_OFFLOAD_MATCH_METADATA)) 1084 port->dev_conf.txmode.offloads &= 1085 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1086 if (numa_support) { 1087 if (port_numa[pid] != NUMA_NO_CONFIG) 1088 port_per_socket[port_numa[pid]]++; 1089 else { 1090 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1091 1092 /* 1093 * if socket_id is invalid, 1094 * set to the first available socket. 1095 */ 1096 if (check_socket_id(socket_id) < 0) 1097 socket_id = socket_ids[0]; 1098 port_per_socket[socket_id]++; 1099 } 1100 } 1101 1102 /* Apply Rx offloads configuration */ 1103 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1104 port->rx_conf[k].offloads = 1105 port->dev_conf.rxmode.offloads; 1106 /* Apply Tx offloads configuration */ 1107 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1108 port->tx_conf[k].offloads = 1109 port->dev_conf.txmode.offloads; 1110 1111 /* set flag to initialize port/queue */ 1112 port->need_reconfig = 1; 1113 port->need_reconfig_queues = 1; 1114 port->tx_metadata = 0; 1115 } 1116 1117 /* 1118 * Create pools of mbuf. 1119 * If NUMA support is disabled, create a single pool of mbuf in 1120 * socket 0 memory by default. 1121 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1122 * 1123 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1124 * nb_txd can be configured at run time. 1125 */ 1126 if (param_total_num_mbufs) 1127 nb_mbuf_per_pool = param_total_num_mbufs; 1128 else { 1129 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1130 (nb_lcores * mb_mempool_cache) + 1131 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1132 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1133 } 1134 1135 if (numa_support) { 1136 uint8_t i; 1137 1138 for (i = 0; i < num_sockets; i++) 1139 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1140 socket_ids[i]); 1141 } else { 1142 if (socket_num == UMA_NO_CONFIG) 1143 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 1144 else 1145 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1146 socket_num); 1147 } 1148 1149 init_port_config(); 1150 1151 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1152 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1153 /* 1154 * Records which Mbuf pool to use by each logical core, if needed. 1155 */ 1156 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1157 mbp = mbuf_pool_find( 1158 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1159 1160 if (mbp == NULL) 1161 mbp = mbuf_pool_find(0); 1162 fwd_lcores[lc_id]->mbp = mbp; 1163 /* initialize GSO context */ 1164 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1165 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1166 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1167 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN - 1168 ETHER_CRC_LEN; 1169 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1170 } 1171 1172 /* Configuration of packet forwarding streams. */ 1173 if (init_fwd_streams() < 0) 1174 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1175 1176 fwd_config_setup(); 1177 1178 /* create a gro context for each lcore */ 1179 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1180 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1181 gro_param.max_item_per_flow = MAX_PKT_BURST; 1182 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1183 gro_param.socket_id = rte_lcore_to_socket_id( 1184 fwd_lcores_cpuids[lc_id]); 1185 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1186 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1187 rte_exit(EXIT_FAILURE, 1188 "rte_gro_ctx_create() failed\n"); 1189 } 1190 } 1191 1192 #if defined RTE_LIBRTE_PMD_SOFTNIC 1193 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1194 RTE_ETH_FOREACH_DEV(pid) { 1195 port = &ports[pid]; 1196 const char *driver = port->dev_info.driver_name; 1197 1198 if (strcmp(driver, "net_softnic") == 0) 1199 port->softport.fwd_lcore_arg = fwd_lcores; 1200 } 1201 } 1202 #endif 1203 1204 } 1205 1206 1207 void 1208 reconfig(portid_t new_port_id, unsigned socket_id) 1209 { 1210 struct rte_port *port; 1211 1212 /* Reconfiguration of Ethernet ports. */ 1213 port = &ports[new_port_id]; 1214 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1215 1216 /* set flag to initialize port/queue */ 1217 port->need_reconfig = 1; 1218 port->need_reconfig_queues = 1; 1219 port->socket_id = socket_id; 1220 1221 init_port_config(); 1222 } 1223 1224 1225 int 1226 init_fwd_streams(void) 1227 { 1228 portid_t pid; 1229 struct rte_port *port; 1230 streamid_t sm_id, nb_fwd_streams_new; 1231 queueid_t q; 1232 1233 /* set socket id according to numa or not */ 1234 RTE_ETH_FOREACH_DEV(pid) { 1235 port = &ports[pid]; 1236 if (nb_rxq > port->dev_info.max_rx_queues) { 1237 printf("Fail: nb_rxq(%d) is greater than " 1238 "max_rx_queues(%d)\n", nb_rxq, 1239 port->dev_info.max_rx_queues); 1240 return -1; 1241 } 1242 if (nb_txq > port->dev_info.max_tx_queues) { 1243 printf("Fail: nb_txq(%d) is greater than " 1244 "max_tx_queues(%d)\n", nb_txq, 1245 port->dev_info.max_tx_queues); 1246 return -1; 1247 } 1248 if (numa_support) { 1249 if (port_numa[pid] != NUMA_NO_CONFIG) 1250 port->socket_id = port_numa[pid]; 1251 else { 1252 port->socket_id = rte_eth_dev_socket_id(pid); 1253 1254 /* 1255 * if socket_id is invalid, 1256 * set to the first available socket. 1257 */ 1258 if (check_socket_id(port->socket_id) < 0) 1259 port->socket_id = socket_ids[0]; 1260 } 1261 } 1262 else { 1263 if (socket_num == UMA_NO_CONFIG) 1264 port->socket_id = 0; 1265 else 1266 port->socket_id = socket_num; 1267 } 1268 } 1269 1270 q = RTE_MAX(nb_rxq, nb_txq); 1271 if (q == 0) { 1272 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1273 return -1; 1274 } 1275 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1276 if (nb_fwd_streams_new == nb_fwd_streams) 1277 return 0; 1278 /* clear the old */ 1279 if (fwd_streams != NULL) { 1280 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1281 if (fwd_streams[sm_id] == NULL) 1282 continue; 1283 rte_free(fwd_streams[sm_id]); 1284 fwd_streams[sm_id] = NULL; 1285 } 1286 rte_free(fwd_streams); 1287 fwd_streams = NULL; 1288 } 1289 1290 /* init new */ 1291 nb_fwd_streams = nb_fwd_streams_new; 1292 if (nb_fwd_streams) { 1293 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1294 sizeof(struct fwd_stream *) * nb_fwd_streams, 1295 RTE_CACHE_LINE_SIZE); 1296 if (fwd_streams == NULL) 1297 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1298 " (struct fwd_stream *)) failed\n", 1299 nb_fwd_streams); 1300 1301 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1302 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1303 " struct fwd_stream", sizeof(struct fwd_stream), 1304 RTE_CACHE_LINE_SIZE); 1305 if (fwd_streams[sm_id] == NULL) 1306 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1307 "(struct fwd_stream) failed\n"); 1308 } 1309 } 1310 1311 return 0; 1312 } 1313 1314 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1315 static void 1316 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1317 { 1318 unsigned int total_burst; 1319 unsigned int nb_burst; 1320 unsigned int burst_stats[3]; 1321 uint16_t pktnb_stats[3]; 1322 uint16_t nb_pkt; 1323 int burst_percent[3]; 1324 1325 /* 1326 * First compute the total number of packet bursts and the 1327 * two highest numbers of bursts of the same number of packets. 1328 */ 1329 total_burst = 0; 1330 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1331 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1332 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1333 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1334 if (nb_burst == 0) 1335 continue; 1336 total_burst += nb_burst; 1337 if (nb_burst > burst_stats[0]) { 1338 burst_stats[1] = burst_stats[0]; 1339 pktnb_stats[1] = pktnb_stats[0]; 1340 burst_stats[0] = nb_burst; 1341 pktnb_stats[0] = nb_pkt; 1342 } else if (nb_burst > burst_stats[1]) { 1343 burst_stats[1] = nb_burst; 1344 pktnb_stats[1] = nb_pkt; 1345 } 1346 } 1347 if (total_burst == 0) 1348 return; 1349 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1350 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1351 burst_percent[0], (int) pktnb_stats[0]); 1352 if (burst_stats[0] == total_burst) { 1353 printf("]\n"); 1354 return; 1355 } 1356 if (burst_stats[0] + burst_stats[1] == total_burst) { 1357 printf(" + %d%% of %d pkts]\n", 1358 100 - burst_percent[0], pktnb_stats[1]); 1359 return; 1360 } 1361 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1362 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1363 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1364 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1365 return; 1366 } 1367 printf(" + %d%% of %d pkts + %d%% of others]\n", 1368 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1369 } 1370 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1371 1372 static void 1373 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats) 1374 { 1375 struct rte_port *port; 1376 uint8_t i; 1377 1378 static const char *fwd_stats_border = "----------------------"; 1379 1380 port = &ports[port_id]; 1381 printf("\n %s Forward statistics for port %-2d %s\n", 1382 fwd_stats_border, port_id, fwd_stats_border); 1383 1384 if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) { 1385 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1386 "%-"PRIu64"\n", 1387 stats->ipackets, stats->imissed, 1388 (uint64_t) (stats->ipackets + stats->imissed)); 1389 1390 if (cur_fwd_eng == &csum_fwd_engine) 1391 printf(" Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64"Bad-outer-l4csum: %-14"PRIu64"\n", 1392 port->rx_bad_ip_csum, port->rx_bad_l4_csum, 1393 port->rx_bad_outer_l4_csum); 1394 if ((stats->ierrors + stats->rx_nombuf) > 0) { 1395 printf(" RX-error: %-"PRIu64"\n", stats->ierrors); 1396 printf(" RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf); 1397 } 1398 1399 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1400 "%-"PRIu64"\n", 1401 stats->opackets, port->tx_dropped, 1402 (uint64_t) (stats->opackets + port->tx_dropped)); 1403 } 1404 else { 1405 printf(" RX-packets: %14"PRIu64" RX-dropped:%14"PRIu64" RX-total:" 1406 "%14"PRIu64"\n", 1407 stats->ipackets, stats->imissed, 1408 (uint64_t) (stats->ipackets + stats->imissed)); 1409 1410 if (cur_fwd_eng == &csum_fwd_engine) 1411 printf(" Bad-ipcsum:%14"PRIu64" Bad-l4csum:%14"PRIu64" Bad-outer-l4csum: %-14"PRIu64"\n", 1412 port->rx_bad_ip_csum, port->rx_bad_l4_csum, 1413 port->rx_bad_outer_l4_csum); 1414 if ((stats->ierrors + stats->rx_nombuf) > 0) { 1415 printf(" RX-error:%"PRIu64"\n", stats->ierrors); 1416 printf(" RX-nombufs: %14"PRIu64"\n", 1417 stats->rx_nombuf); 1418 } 1419 1420 printf(" TX-packets: %14"PRIu64" TX-dropped:%14"PRIu64" TX-total:" 1421 "%14"PRIu64"\n", 1422 stats->opackets, port->tx_dropped, 1423 (uint64_t) (stats->opackets + port->tx_dropped)); 1424 } 1425 1426 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1427 if (port->rx_stream) 1428 pkt_burst_stats_display("RX", 1429 &port->rx_stream->rx_burst_stats); 1430 if (port->tx_stream) 1431 pkt_burst_stats_display("TX", 1432 &port->tx_stream->tx_burst_stats); 1433 #endif 1434 1435 if (port->rx_queue_stats_mapping_enabled) { 1436 printf("\n"); 1437 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 1438 printf(" Stats reg %2d RX-packets:%14"PRIu64 1439 " RX-errors:%14"PRIu64 1440 " RX-bytes:%14"PRIu64"\n", 1441 i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]); 1442 } 1443 printf("\n"); 1444 } 1445 if (port->tx_queue_stats_mapping_enabled) { 1446 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 1447 printf(" Stats reg %2d TX-packets:%14"PRIu64 1448 " TX-bytes:%14"PRIu64"\n", 1449 i, stats->q_opackets[i], stats->q_obytes[i]); 1450 } 1451 } 1452 1453 printf(" %s--------------------------------%s\n", 1454 fwd_stats_border, fwd_stats_border); 1455 } 1456 1457 static void 1458 fwd_stream_stats_display(streamid_t stream_id) 1459 { 1460 struct fwd_stream *fs; 1461 static const char *fwd_top_stats_border = "-------"; 1462 1463 fs = fwd_streams[stream_id]; 1464 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1465 (fs->fwd_dropped == 0)) 1466 return; 1467 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1468 "TX Port=%2d/Queue=%2d %s\n", 1469 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1470 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1471 printf(" RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u", 1472 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1473 1474 /* if checksum mode */ 1475 if (cur_fwd_eng == &csum_fwd_engine) { 1476 printf(" RX- bad IP checksum: %-14u Rx- bad L4 checksum: " 1477 "%-14u Rx- bad outer L4 checksum: %-14u\n", 1478 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1479 fs->rx_bad_outer_l4_csum); 1480 } 1481 1482 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1483 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1484 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1485 #endif 1486 } 1487 1488 static void 1489 flush_fwd_rx_queues(void) 1490 { 1491 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1492 portid_t rxp; 1493 portid_t port_id; 1494 queueid_t rxq; 1495 uint16_t nb_rx; 1496 uint16_t i; 1497 uint8_t j; 1498 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1499 uint64_t timer_period; 1500 1501 /* convert to number of cycles */ 1502 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1503 1504 for (j = 0; j < 2; j++) { 1505 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1506 for (rxq = 0; rxq < nb_rxq; rxq++) { 1507 port_id = fwd_ports_ids[rxp]; 1508 /** 1509 * testpmd can stuck in the below do while loop 1510 * if rte_eth_rx_burst() always returns nonzero 1511 * packets. So timer is added to exit this loop 1512 * after 1sec timer expiry. 1513 */ 1514 prev_tsc = rte_rdtsc(); 1515 do { 1516 nb_rx = rte_eth_rx_burst(port_id, rxq, 1517 pkts_burst, MAX_PKT_BURST); 1518 for (i = 0; i < nb_rx; i++) 1519 rte_pktmbuf_free(pkts_burst[i]); 1520 1521 cur_tsc = rte_rdtsc(); 1522 diff_tsc = cur_tsc - prev_tsc; 1523 timer_tsc += diff_tsc; 1524 } while ((nb_rx > 0) && 1525 (timer_tsc < timer_period)); 1526 timer_tsc = 0; 1527 } 1528 } 1529 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1530 } 1531 } 1532 1533 static void 1534 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1535 { 1536 struct fwd_stream **fsm; 1537 streamid_t nb_fs; 1538 streamid_t sm_id; 1539 #ifdef RTE_LIBRTE_BITRATE 1540 uint64_t tics_per_1sec; 1541 uint64_t tics_datum; 1542 uint64_t tics_current; 1543 uint16_t i, cnt_ports; 1544 1545 cnt_ports = nb_ports; 1546 tics_datum = rte_rdtsc(); 1547 tics_per_1sec = rte_get_timer_hz(); 1548 #endif 1549 fsm = &fwd_streams[fc->stream_idx]; 1550 nb_fs = fc->stream_nb; 1551 do { 1552 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1553 (*pkt_fwd)(fsm[sm_id]); 1554 #ifdef RTE_LIBRTE_BITRATE 1555 if (bitrate_enabled != 0 && 1556 bitrate_lcore_id == rte_lcore_id()) { 1557 tics_current = rte_rdtsc(); 1558 if (tics_current - tics_datum >= tics_per_1sec) { 1559 /* Periodic bitrate calculation */ 1560 for (i = 0; i < cnt_ports; i++) 1561 rte_stats_bitrate_calc(bitrate_data, 1562 ports_ids[i]); 1563 tics_datum = tics_current; 1564 } 1565 } 1566 #endif 1567 #ifdef RTE_LIBRTE_LATENCY_STATS 1568 if (latencystats_enabled != 0 && 1569 latencystats_lcore_id == rte_lcore_id()) 1570 rte_latencystats_update(); 1571 #endif 1572 1573 } while (! fc->stopped); 1574 } 1575 1576 static int 1577 start_pkt_forward_on_core(void *fwd_arg) 1578 { 1579 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1580 cur_fwd_config.fwd_eng->packet_fwd); 1581 return 0; 1582 } 1583 1584 /* 1585 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1586 * Used to start communication flows in network loopback test configurations. 1587 */ 1588 static int 1589 run_one_txonly_burst_on_core(void *fwd_arg) 1590 { 1591 struct fwd_lcore *fwd_lc; 1592 struct fwd_lcore tmp_lcore; 1593 1594 fwd_lc = (struct fwd_lcore *) fwd_arg; 1595 tmp_lcore = *fwd_lc; 1596 tmp_lcore.stopped = 1; 1597 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1598 return 0; 1599 } 1600 1601 /* 1602 * Launch packet forwarding: 1603 * - Setup per-port forwarding context. 1604 * - launch logical cores with their forwarding configuration. 1605 */ 1606 static void 1607 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1608 { 1609 port_fwd_begin_t port_fwd_begin; 1610 unsigned int i; 1611 unsigned int lc_id; 1612 int diag; 1613 1614 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1615 if (port_fwd_begin != NULL) { 1616 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1617 (*port_fwd_begin)(fwd_ports_ids[i]); 1618 } 1619 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1620 lc_id = fwd_lcores_cpuids[i]; 1621 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1622 fwd_lcores[i]->stopped = 0; 1623 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1624 fwd_lcores[i], lc_id); 1625 if (diag != 0) 1626 printf("launch lcore %u failed - diag=%d\n", 1627 lc_id, diag); 1628 } 1629 } 1630 } 1631 1632 /* 1633 * Launch packet forwarding configuration. 1634 */ 1635 void 1636 start_packet_forwarding(int with_tx_first) 1637 { 1638 port_fwd_begin_t port_fwd_begin; 1639 port_fwd_end_t port_fwd_end; 1640 struct rte_port *port; 1641 unsigned int i; 1642 portid_t pt_id; 1643 streamid_t sm_id; 1644 1645 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1646 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1647 1648 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1649 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1650 1651 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1652 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1653 (!nb_rxq || !nb_txq)) 1654 rte_exit(EXIT_FAILURE, 1655 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1656 cur_fwd_eng->fwd_mode_name); 1657 1658 if (all_ports_started() == 0) { 1659 printf("Not all ports were started\n"); 1660 return; 1661 } 1662 if (test_done == 0) { 1663 printf("Packet forwarding already started\n"); 1664 return; 1665 } 1666 1667 1668 if(dcb_test) { 1669 for (i = 0; i < nb_fwd_ports; i++) { 1670 pt_id = fwd_ports_ids[i]; 1671 port = &ports[pt_id]; 1672 if (!port->dcb_flag) { 1673 printf("In DCB mode, all forwarding ports must " 1674 "be configured in this mode.\n"); 1675 return; 1676 } 1677 } 1678 if (nb_fwd_lcores == 1) { 1679 printf("In DCB mode,the nb forwarding cores " 1680 "should be larger than 1.\n"); 1681 return; 1682 } 1683 } 1684 test_done = 0; 1685 1686 fwd_config_setup(); 1687 1688 if(!no_flush_rx) 1689 flush_fwd_rx_queues(); 1690 1691 pkt_fwd_config_display(&cur_fwd_config); 1692 rxtx_config_display(); 1693 1694 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1695 pt_id = fwd_ports_ids[i]; 1696 port = &ports[pt_id]; 1697 rte_eth_stats_get(pt_id, &port->stats); 1698 port->tx_dropped = 0; 1699 1700 map_port_queue_stats_mapping_registers(pt_id, port); 1701 } 1702 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1703 fwd_streams[sm_id]->rx_packets = 0; 1704 fwd_streams[sm_id]->tx_packets = 0; 1705 fwd_streams[sm_id]->fwd_dropped = 0; 1706 fwd_streams[sm_id]->rx_bad_ip_csum = 0; 1707 fwd_streams[sm_id]->rx_bad_l4_csum = 0; 1708 fwd_streams[sm_id]->rx_bad_outer_l4_csum = 0; 1709 1710 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1711 memset(&fwd_streams[sm_id]->rx_burst_stats, 0, 1712 sizeof(fwd_streams[sm_id]->rx_burst_stats)); 1713 memset(&fwd_streams[sm_id]->tx_burst_stats, 0, 1714 sizeof(fwd_streams[sm_id]->tx_burst_stats)); 1715 #endif 1716 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1717 fwd_streams[sm_id]->core_cycles = 0; 1718 #endif 1719 } 1720 if (with_tx_first) { 1721 port_fwd_begin = tx_only_engine.port_fwd_begin; 1722 if (port_fwd_begin != NULL) { 1723 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1724 (*port_fwd_begin)(fwd_ports_ids[i]); 1725 } 1726 while (with_tx_first--) { 1727 launch_packet_forwarding( 1728 run_one_txonly_burst_on_core); 1729 rte_eal_mp_wait_lcore(); 1730 } 1731 port_fwd_end = tx_only_engine.port_fwd_end; 1732 if (port_fwd_end != NULL) { 1733 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1734 (*port_fwd_end)(fwd_ports_ids[i]); 1735 } 1736 } 1737 launch_packet_forwarding(start_pkt_forward_on_core); 1738 } 1739 1740 void 1741 stop_packet_forwarding(void) 1742 { 1743 struct rte_eth_stats stats; 1744 struct rte_port *port; 1745 port_fwd_end_t port_fwd_end; 1746 int i; 1747 portid_t pt_id; 1748 streamid_t sm_id; 1749 lcoreid_t lc_id; 1750 uint64_t total_recv; 1751 uint64_t total_xmit; 1752 uint64_t total_rx_dropped; 1753 uint64_t total_tx_dropped; 1754 uint64_t total_rx_nombuf; 1755 uint64_t tx_dropped; 1756 uint64_t rx_bad_ip_csum; 1757 uint64_t rx_bad_l4_csum; 1758 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1759 uint64_t fwd_cycles; 1760 #endif 1761 1762 static const char *acc_stats_border = "+++++++++++++++"; 1763 1764 if (test_done) { 1765 printf("Packet forwarding not started\n"); 1766 return; 1767 } 1768 printf("Telling cores to stop..."); 1769 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1770 fwd_lcores[lc_id]->stopped = 1; 1771 printf("\nWaiting for lcores to finish...\n"); 1772 rte_eal_mp_wait_lcore(); 1773 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1774 if (port_fwd_end != NULL) { 1775 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1776 pt_id = fwd_ports_ids[i]; 1777 (*port_fwd_end)(pt_id); 1778 } 1779 } 1780 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1781 fwd_cycles = 0; 1782 #endif 1783 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1784 if (cur_fwd_config.nb_fwd_streams > 1785 cur_fwd_config.nb_fwd_ports) { 1786 fwd_stream_stats_display(sm_id); 1787 ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL; 1788 ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL; 1789 } else { 1790 ports[fwd_streams[sm_id]->tx_port].tx_stream = 1791 fwd_streams[sm_id]; 1792 ports[fwd_streams[sm_id]->rx_port].rx_stream = 1793 fwd_streams[sm_id]; 1794 } 1795 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped; 1796 tx_dropped = (uint64_t) (tx_dropped + 1797 fwd_streams[sm_id]->fwd_dropped); 1798 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped; 1799 1800 rx_bad_ip_csum = 1801 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum; 1802 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum + 1803 fwd_streams[sm_id]->rx_bad_ip_csum); 1804 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum = 1805 rx_bad_ip_csum; 1806 1807 rx_bad_l4_csum = 1808 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum; 1809 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum + 1810 fwd_streams[sm_id]->rx_bad_l4_csum); 1811 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum = 1812 rx_bad_l4_csum; 1813 1814 ports[fwd_streams[sm_id]->rx_port].rx_bad_outer_l4_csum += 1815 fwd_streams[sm_id]->rx_bad_outer_l4_csum; 1816 1817 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1818 fwd_cycles = (uint64_t) (fwd_cycles + 1819 fwd_streams[sm_id]->core_cycles); 1820 #endif 1821 } 1822 total_recv = 0; 1823 total_xmit = 0; 1824 total_rx_dropped = 0; 1825 total_tx_dropped = 0; 1826 total_rx_nombuf = 0; 1827 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1828 pt_id = fwd_ports_ids[i]; 1829 1830 port = &ports[pt_id]; 1831 rte_eth_stats_get(pt_id, &stats); 1832 stats.ipackets -= port->stats.ipackets; 1833 port->stats.ipackets = 0; 1834 stats.opackets -= port->stats.opackets; 1835 port->stats.opackets = 0; 1836 stats.ibytes -= port->stats.ibytes; 1837 port->stats.ibytes = 0; 1838 stats.obytes -= port->stats.obytes; 1839 port->stats.obytes = 0; 1840 stats.imissed -= port->stats.imissed; 1841 port->stats.imissed = 0; 1842 stats.oerrors -= port->stats.oerrors; 1843 port->stats.oerrors = 0; 1844 stats.rx_nombuf -= port->stats.rx_nombuf; 1845 port->stats.rx_nombuf = 0; 1846 1847 total_recv += stats.ipackets; 1848 total_xmit += stats.opackets; 1849 total_rx_dropped += stats.imissed; 1850 total_tx_dropped += port->tx_dropped; 1851 total_rx_nombuf += stats.rx_nombuf; 1852 1853 fwd_port_stats_display(pt_id, &stats); 1854 } 1855 1856 printf("\n %s Accumulated forward statistics for all ports" 1857 "%s\n", 1858 acc_stats_border, acc_stats_border); 1859 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1860 "%-"PRIu64"\n" 1861 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1862 "%-"PRIu64"\n", 1863 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1864 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1865 if (total_rx_nombuf > 0) 1866 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1867 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1868 "%s\n", 1869 acc_stats_border, acc_stats_border); 1870 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1871 if (total_recv > 0) 1872 printf("\n CPU cycles/packet=%u (total cycles=" 1873 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1874 (unsigned int)(fwd_cycles / total_recv), 1875 fwd_cycles, total_recv); 1876 #endif 1877 printf("\nDone.\n"); 1878 test_done = 1; 1879 } 1880 1881 void 1882 dev_set_link_up(portid_t pid) 1883 { 1884 if (rte_eth_dev_set_link_up(pid) < 0) 1885 printf("\nSet link up fail.\n"); 1886 } 1887 1888 void 1889 dev_set_link_down(portid_t pid) 1890 { 1891 if (rte_eth_dev_set_link_down(pid) < 0) 1892 printf("\nSet link down fail.\n"); 1893 } 1894 1895 static int 1896 all_ports_started(void) 1897 { 1898 portid_t pi; 1899 struct rte_port *port; 1900 1901 RTE_ETH_FOREACH_DEV(pi) { 1902 port = &ports[pi]; 1903 /* Check if there is a port which is not started */ 1904 if ((port->port_status != RTE_PORT_STARTED) && 1905 (port->slave_flag == 0)) 1906 return 0; 1907 } 1908 1909 /* No port is not started */ 1910 return 1; 1911 } 1912 1913 int 1914 port_is_stopped(portid_t port_id) 1915 { 1916 struct rte_port *port = &ports[port_id]; 1917 1918 if ((port->port_status != RTE_PORT_STOPPED) && 1919 (port->slave_flag == 0)) 1920 return 0; 1921 return 1; 1922 } 1923 1924 int 1925 all_ports_stopped(void) 1926 { 1927 portid_t pi; 1928 1929 RTE_ETH_FOREACH_DEV(pi) { 1930 if (!port_is_stopped(pi)) 1931 return 0; 1932 } 1933 1934 return 1; 1935 } 1936 1937 int 1938 port_is_started(portid_t port_id) 1939 { 1940 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1941 return 0; 1942 1943 if (ports[port_id].port_status != RTE_PORT_STARTED) 1944 return 0; 1945 1946 return 1; 1947 } 1948 1949 int 1950 start_port(portid_t pid) 1951 { 1952 int diag, need_check_link_status = -1; 1953 portid_t pi; 1954 queueid_t qi; 1955 struct rte_port *port; 1956 struct ether_addr mac_addr; 1957 1958 if (port_id_is_invalid(pid, ENABLED_WARN)) 1959 return 0; 1960 1961 if(dcb_config) 1962 dcb_test = 1; 1963 RTE_ETH_FOREACH_DEV(pi) { 1964 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1965 continue; 1966 1967 need_check_link_status = 0; 1968 port = &ports[pi]; 1969 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1970 RTE_PORT_HANDLING) == 0) { 1971 printf("Port %d is now not stopped\n", pi); 1972 continue; 1973 } 1974 1975 if (port->need_reconfig > 0) { 1976 port->need_reconfig = 0; 1977 1978 if (flow_isolate_all) { 1979 int ret = port_flow_isolate(pi, 1); 1980 if (ret) { 1981 printf("Failed to apply isolated" 1982 " mode on port %d\n", pi); 1983 return -1; 1984 } 1985 } 1986 configure_rxtx_dump_callbacks(0); 1987 printf("Configuring Port %d (socket %u)\n", pi, 1988 port->socket_id); 1989 /* configure port */ 1990 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1991 &(port->dev_conf)); 1992 if (diag != 0) { 1993 if (rte_atomic16_cmpset(&(port->port_status), 1994 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1995 printf("Port %d can not be set back " 1996 "to stopped\n", pi); 1997 printf("Fail to configure port %d\n", pi); 1998 /* try to reconfigure port next time */ 1999 port->need_reconfig = 1; 2000 return -1; 2001 } 2002 } 2003 if (port->need_reconfig_queues > 0) { 2004 port->need_reconfig_queues = 0; 2005 /* setup tx queues */ 2006 for (qi = 0; qi < nb_txq; qi++) { 2007 if ((numa_support) && 2008 (txring_numa[pi] != NUMA_NO_CONFIG)) 2009 diag = rte_eth_tx_queue_setup(pi, qi, 2010 port->nb_tx_desc[qi], 2011 txring_numa[pi], 2012 &(port->tx_conf[qi])); 2013 else 2014 diag = rte_eth_tx_queue_setup(pi, qi, 2015 port->nb_tx_desc[qi], 2016 port->socket_id, 2017 &(port->tx_conf[qi])); 2018 2019 if (diag == 0) 2020 continue; 2021 2022 /* Fail to setup tx queue, return */ 2023 if (rte_atomic16_cmpset(&(port->port_status), 2024 RTE_PORT_HANDLING, 2025 RTE_PORT_STOPPED) == 0) 2026 printf("Port %d can not be set back " 2027 "to stopped\n", pi); 2028 printf("Fail to configure port %d tx queues\n", 2029 pi); 2030 /* try to reconfigure queues next time */ 2031 port->need_reconfig_queues = 1; 2032 return -1; 2033 } 2034 for (qi = 0; qi < nb_rxq; qi++) { 2035 /* setup rx queues */ 2036 if ((numa_support) && 2037 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2038 struct rte_mempool * mp = 2039 mbuf_pool_find(rxring_numa[pi]); 2040 if (mp == NULL) { 2041 printf("Failed to setup RX queue:" 2042 "No mempool allocation" 2043 " on the socket %d\n", 2044 rxring_numa[pi]); 2045 return -1; 2046 } 2047 2048 diag = rte_eth_rx_queue_setup(pi, qi, 2049 port->nb_rx_desc[qi], 2050 rxring_numa[pi], 2051 &(port->rx_conf[qi]), 2052 mp); 2053 } else { 2054 struct rte_mempool *mp = 2055 mbuf_pool_find(port->socket_id); 2056 if (mp == NULL) { 2057 printf("Failed to setup RX queue:" 2058 "No mempool allocation" 2059 " on the socket %d\n", 2060 port->socket_id); 2061 return -1; 2062 } 2063 diag = rte_eth_rx_queue_setup(pi, qi, 2064 port->nb_rx_desc[qi], 2065 port->socket_id, 2066 &(port->rx_conf[qi]), 2067 mp); 2068 } 2069 if (diag == 0) 2070 continue; 2071 2072 /* Fail to setup rx queue, return */ 2073 if (rte_atomic16_cmpset(&(port->port_status), 2074 RTE_PORT_HANDLING, 2075 RTE_PORT_STOPPED) == 0) 2076 printf("Port %d can not be set back " 2077 "to stopped\n", pi); 2078 printf("Fail to configure port %d rx queues\n", 2079 pi); 2080 /* try to reconfigure queues next time */ 2081 port->need_reconfig_queues = 1; 2082 return -1; 2083 } 2084 } 2085 configure_rxtx_dump_callbacks(verbose_level); 2086 /* start port */ 2087 if (rte_eth_dev_start(pi) < 0) { 2088 printf("Fail to start port %d\n", pi); 2089 2090 /* Fail to setup rx queue, return */ 2091 if (rte_atomic16_cmpset(&(port->port_status), 2092 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2093 printf("Port %d can not be set back to " 2094 "stopped\n", pi); 2095 continue; 2096 } 2097 2098 if (rte_atomic16_cmpset(&(port->port_status), 2099 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2100 printf("Port %d can not be set into started\n", pi); 2101 2102 rte_eth_macaddr_get(pi, &mac_addr); 2103 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2104 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2105 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2106 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2107 2108 /* at least one port started, need checking link status */ 2109 need_check_link_status = 1; 2110 } 2111 2112 if (need_check_link_status == 1 && !no_link_check) 2113 check_all_ports_link_status(RTE_PORT_ALL); 2114 else if (need_check_link_status == 0) 2115 printf("Please stop the ports first\n"); 2116 2117 printf("Done\n"); 2118 return 0; 2119 } 2120 2121 void 2122 stop_port(portid_t pid) 2123 { 2124 portid_t pi; 2125 struct rte_port *port; 2126 int need_check_link_status = 0; 2127 2128 if (dcb_test) { 2129 dcb_test = 0; 2130 dcb_config = 0; 2131 } 2132 2133 if (port_id_is_invalid(pid, ENABLED_WARN)) 2134 return; 2135 2136 printf("Stopping ports...\n"); 2137 2138 RTE_ETH_FOREACH_DEV(pi) { 2139 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2140 continue; 2141 2142 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2143 printf("Please remove port %d from forwarding configuration.\n", pi); 2144 continue; 2145 } 2146 2147 if (port_is_bonding_slave(pi)) { 2148 printf("Please remove port %d from bonded device.\n", pi); 2149 continue; 2150 } 2151 2152 port = &ports[pi]; 2153 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2154 RTE_PORT_HANDLING) == 0) 2155 continue; 2156 2157 rte_eth_dev_stop(pi); 2158 2159 if (rte_atomic16_cmpset(&(port->port_status), 2160 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2161 printf("Port %d can not be set into stopped\n", pi); 2162 need_check_link_status = 1; 2163 } 2164 if (need_check_link_status && !no_link_check) 2165 check_all_ports_link_status(RTE_PORT_ALL); 2166 2167 printf("Done\n"); 2168 } 2169 2170 static void 2171 remove_invalid_ports_in(portid_t *array, portid_t *total) 2172 { 2173 portid_t i; 2174 portid_t new_total = 0; 2175 2176 for (i = 0; i < *total; i++) 2177 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2178 array[new_total] = array[i]; 2179 new_total++; 2180 } 2181 *total = new_total; 2182 } 2183 2184 static void 2185 remove_invalid_ports(void) 2186 { 2187 remove_invalid_ports_in(ports_ids, &nb_ports); 2188 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2189 nb_cfg_ports = nb_fwd_ports; 2190 } 2191 2192 void 2193 close_port(portid_t pid) 2194 { 2195 portid_t pi; 2196 struct rte_port *port; 2197 2198 if (port_id_is_invalid(pid, ENABLED_WARN)) 2199 return; 2200 2201 printf("Closing ports...\n"); 2202 2203 RTE_ETH_FOREACH_DEV(pi) { 2204 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2205 continue; 2206 2207 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2208 printf("Please remove port %d from forwarding configuration.\n", pi); 2209 continue; 2210 } 2211 2212 if (port_is_bonding_slave(pi)) { 2213 printf("Please remove port %d from bonded device.\n", pi); 2214 continue; 2215 } 2216 2217 port = &ports[pi]; 2218 if (rte_atomic16_cmpset(&(port->port_status), 2219 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2220 printf("Port %d is already closed\n", pi); 2221 continue; 2222 } 2223 2224 if (rte_atomic16_cmpset(&(port->port_status), 2225 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2226 printf("Port %d is now not stopped\n", pi); 2227 continue; 2228 } 2229 2230 if (port->flow_list) 2231 port_flow_flush(pi); 2232 rte_eth_dev_close(pi); 2233 2234 remove_invalid_ports(); 2235 2236 if (rte_atomic16_cmpset(&(port->port_status), 2237 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2238 printf("Port %d cannot be set to closed\n", pi); 2239 } 2240 2241 printf("Done\n"); 2242 } 2243 2244 void 2245 reset_port(portid_t pid) 2246 { 2247 int diag; 2248 portid_t pi; 2249 struct rte_port *port; 2250 2251 if (port_id_is_invalid(pid, ENABLED_WARN)) 2252 return; 2253 2254 printf("Resetting ports...\n"); 2255 2256 RTE_ETH_FOREACH_DEV(pi) { 2257 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2258 continue; 2259 2260 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2261 printf("Please remove port %d from forwarding " 2262 "configuration.\n", pi); 2263 continue; 2264 } 2265 2266 if (port_is_bonding_slave(pi)) { 2267 printf("Please remove port %d from bonded device.\n", 2268 pi); 2269 continue; 2270 } 2271 2272 diag = rte_eth_dev_reset(pi); 2273 if (diag == 0) { 2274 port = &ports[pi]; 2275 port->need_reconfig = 1; 2276 port->need_reconfig_queues = 1; 2277 } else { 2278 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2279 } 2280 } 2281 2282 printf("Done\n"); 2283 } 2284 2285 void 2286 attach_port(char *identifier) 2287 { 2288 portid_t pi = 0; 2289 struct rte_dev_iterator iterator; 2290 2291 printf("Attaching a new port...\n"); 2292 2293 if (identifier == NULL) { 2294 printf("Invalid parameters are specified\n"); 2295 return; 2296 } 2297 2298 if (rte_dev_probe(identifier) != 0) { 2299 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2300 return; 2301 } 2302 2303 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2304 if (port_is_forwarding(pi)) 2305 continue; /* port was already attached before */ 2306 setup_attached_port(pi); 2307 } 2308 } 2309 2310 static void 2311 setup_attached_port(portid_t pi) 2312 { 2313 unsigned int socket_id; 2314 2315 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2316 /* if socket_id is invalid, set to the first available socket. */ 2317 if (check_socket_id(socket_id) < 0) 2318 socket_id = socket_ids[0]; 2319 reconfig(pi, socket_id); 2320 rte_eth_promiscuous_enable(pi); 2321 2322 ports_ids[nb_ports++] = pi; 2323 fwd_ports_ids[nb_fwd_ports++] = pi; 2324 nb_cfg_ports = nb_fwd_ports; 2325 ports[pi].port_status = RTE_PORT_STOPPED; 2326 2327 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2328 printf("Done\n"); 2329 } 2330 2331 void 2332 detach_port_device(portid_t port_id) 2333 { 2334 struct rte_device *dev; 2335 portid_t sibling; 2336 2337 printf("Removing a device...\n"); 2338 2339 dev = rte_eth_devices[port_id].device; 2340 if (dev == NULL) { 2341 printf("Device already removed\n"); 2342 return; 2343 } 2344 2345 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2346 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2347 printf("Port not stopped\n"); 2348 return; 2349 } 2350 printf("Port was not closed\n"); 2351 if (ports[port_id].flow_list) 2352 port_flow_flush(port_id); 2353 } 2354 2355 if (rte_dev_remove(dev) != 0) { 2356 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2357 return; 2358 } 2359 2360 for (sibling = 0; sibling < RTE_MAX_ETHPORTS; sibling++) { 2361 if (rte_eth_devices[sibling].device != dev) 2362 continue; 2363 /* reset mapping between old ports and removed device */ 2364 rte_eth_devices[sibling].device = NULL; 2365 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2366 /* sibling ports are forced to be closed */ 2367 ports[sibling].port_status = RTE_PORT_CLOSED; 2368 printf("Port %u is closed\n", sibling); 2369 } 2370 } 2371 2372 remove_invalid_ports(); 2373 2374 printf("Device of port %u is detached\n", port_id); 2375 printf("Now total ports is %d\n", nb_ports); 2376 printf("Done\n"); 2377 return; 2378 } 2379 2380 void 2381 pmd_test_exit(void) 2382 { 2383 struct rte_device *device; 2384 portid_t pt_id; 2385 int ret; 2386 2387 if (test_done == 0) 2388 stop_packet_forwarding(); 2389 2390 if (ports != NULL) { 2391 no_link_check = 1; 2392 RTE_ETH_FOREACH_DEV(pt_id) { 2393 printf("\nShutting down port %d...\n", pt_id); 2394 fflush(stdout); 2395 stop_port(pt_id); 2396 close_port(pt_id); 2397 2398 /* 2399 * This is a workaround to fix a virtio-user issue that 2400 * requires to call clean-up routine to remove existing 2401 * socket. 2402 * This workaround valid only for testpmd, needs a fix 2403 * valid for all applications. 2404 * TODO: Implement proper resource cleanup 2405 */ 2406 device = rte_eth_devices[pt_id].device; 2407 if (device && !strcmp(device->driver->name, "net_virtio_user")) 2408 detach_port_device(pt_id); 2409 } 2410 } 2411 2412 if (hot_plug) { 2413 ret = rte_dev_event_monitor_stop(); 2414 if (ret) { 2415 RTE_LOG(ERR, EAL, 2416 "fail to stop device event monitor."); 2417 return; 2418 } 2419 2420 ret = rte_dev_event_callback_unregister(NULL, 2421 eth_dev_event_callback, NULL); 2422 if (ret < 0) { 2423 RTE_LOG(ERR, EAL, 2424 "fail to unregister device event callback.\n"); 2425 return; 2426 } 2427 2428 ret = rte_dev_hotplug_handle_disable(); 2429 if (ret) { 2430 RTE_LOG(ERR, EAL, 2431 "fail to disable hotplug handling.\n"); 2432 return; 2433 } 2434 } 2435 2436 printf("\nBye...\n"); 2437 } 2438 2439 typedef void (*cmd_func_t)(void); 2440 struct pmd_test_command { 2441 const char *cmd_name; 2442 cmd_func_t cmd_func; 2443 }; 2444 2445 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2446 2447 /* Check the link status of all ports in up to 9s, and print them finally */ 2448 static void 2449 check_all_ports_link_status(uint32_t port_mask) 2450 { 2451 #define CHECK_INTERVAL 100 /* 100ms */ 2452 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2453 portid_t portid; 2454 uint8_t count, all_ports_up, print_flag = 0; 2455 struct rte_eth_link link; 2456 2457 printf("Checking link statuses...\n"); 2458 fflush(stdout); 2459 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2460 all_ports_up = 1; 2461 RTE_ETH_FOREACH_DEV(portid) { 2462 if ((port_mask & (1 << portid)) == 0) 2463 continue; 2464 memset(&link, 0, sizeof(link)); 2465 rte_eth_link_get_nowait(portid, &link); 2466 /* print link status if flag set */ 2467 if (print_flag == 1) { 2468 if (link.link_status) 2469 printf( 2470 "Port%d Link Up. speed %u Mbps- %s\n", 2471 portid, link.link_speed, 2472 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2473 ("full-duplex") : ("half-duplex\n")); 2474 else 2475 printf("Port %d Link Down\n", portid); 2476 continue; 2477 } 2478 /* clear all_ports_up flag if any link down */ 2479 if (link.link_status == ETH_LINK_DOWN) { 2480 all_ports_up = 0; 2481 break; 2482 } 2483 } 2484 /* after finally printing all link status, get out */ 2485 if (print_flag == 1) 2486 break; 2487 2488 if (all_ports_up == 0) { 2489 fflush(stdout); 2490 rte_delay_ms(CHECK_INTERVAL); 2491 } 2492 2493 /* set the print_flag if all ports up or timeout */ 2494 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2495 print_flag = 1; 2496 } 2497 2498 if (lsc_interrupt) 2499 break; 2500 } 2501 } 2502 2503 static void 2504 rmv_event_callback(void *arg) 2505 { 2506 int need_to_start = 0; 2507 int org_no_link_check = no_link_check; 2508 portid_t port_id = (intptr_t)arg; 2509 2510 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2511 2512 if (!test_done && port_is_forwarding(port_id)) { 2513 need_to_start = 1; 2514 stop_packet_forwarding(); 2515 } 2516 no_link_check = 1; 2517 stop_port(port_id); 2518 no_link_check = org_no_link_check; 2519 close_port(port_id); 2520 detach_port_device(port_id); 2521 if (need_to_start) 2522 start_packet_forwarding(0); 2523 } 2524 2525 /* This function is used by the interrupt thread */ 2526 static int 2527 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2528 void *ret_param) 2529 { 2530 RTE_SET_USED(param); 2531 RTE_SET_USED(ret_param); 2532 2533 if (type >= RTE_ETH_EVENT_MAX) { 2534 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2535 port_id, __func__, type); 2536 fflush(stderr); 2537 } else if (event_print_mask & (UINT32_C(1) << type)) { 2538 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2539 eth_event_desc[type]); 2540 fflush(stdout); 2541 } 2542 2543 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2544 return 0; 2545 2546 switch (type) { 2547 case RTE_ETH_EVENT_INTR_RMV: 2548 if (rte_eal_alarm_set(100000, 2549 rmv_event_callback, (void *)(intptr_t)port_id)) 2550 fprintf(stderr, "Could not set up deferred device removal\n"); 2551 break; 2552 default: 2553 break; 2554 } 2555 return 0; 2556 } 2557 2558 static int 2559 register_eth_event_callback(void) 2560 { 2561 int ret; 2562 enum rte_eth_event_type event; 2563 2564 for (event = RTE_ETH_EVENT_UNKNOWN; 2565 event < RTE_ETH_EVENT_MAX; event++) { 2566 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2567 event, 2568 eth_event_callback, 2569 NULL); 2570 if (ret != 0) { 2571 TESTPMD_LOG(ERR, "Failed to register callback for " 2572 "%s event\n", eth_event_desc[event]); 2573 return -1; 2574 } 2575 } 2576 2577 return 0; 2578 } 2579 2580 /* This function is used by the interrupt thread */ 2581 static void 2582 eth_dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2583 __rte_unused void *arg) 2584 { 2585 uint16_t port_id; 2586 int ret; 2587 2588 if (type >= RTE_DEV_EVENT_MAX) { 2589 fprintf(stderr, "%s called upon invalid event %d\n", 2590 __func__, type); 2591 fflush(stderr); 2592 } 2593 2594 switch (type) { 2595 case RTE_DEV_EVENT_REMOVE: 2596 RTE_LOG(ERR, EAL, "The device: %s has been removed!\n", 2597 device_name); 2598 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2599 if (ret) { 2600 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2601 device_name); 2602 return; 2603 } 2604 rmv_event_callback((void *)(intptr_t)port_id); 2605 break; 2606 case RTE_DEV_EVENT_ADD: 2607 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2608 device_name); 2609 /* TODO: After finish kernel driver binding, 2610 * begin to attach port. 2611 */ 2612 break; 2613 default: 2614 break; 2615 } 2616 } 2617 2618 static int 2619 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2620 { 2621 uint16_t i; 2622 int diag; 2623 uint8_t mapping_found = 0; 2624 2625 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2626 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2627 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2628 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2629 tx_queue_stats_mappings[i].queue_id, 2630 tx_queue_stats_mappings[i].stats_counter_id); 2631 if (diag != 0) 2632 return diag; 2633 mapping_found = 1; 2634 } 2635 } 2636 if (mapping_found) 2637 port->tx_queue_stats_mapping_enabled = 1; 2638 return 0; 2639 } 2640 2641 static int 2642 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2643 { 2644 uint16_t i; 2645 int diag; 2646 uint8_t mapping_found = 0; 2647 2648 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2649 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2650 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2651 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2652 rx_queue_stats_mappings[i].queue_id, 2653 rx_queue_stats_mappings[i].stats_counter_id); 2654 if (diag != 0) 2655 return diag; 2656 mapping_found = 1; 2657 } 2658 } 2659 if (mapping_found) 2660 port->rx_queue_stats_mapping_enabled = 1; 2661 return 0; 2662 } 2663 2664 static void 2665 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2666 { 2667 int diag = 0; 2668 2669 diag = set_tx_queue_stats_mapping_registers(pi, port); 2670 if (diag != 0) { 2671 if (diag == -ENOTSUP) { 2672 port->tx_queue_stats_mapping_enabled = 0; 2673 printf("TX queue stats mapping not supported port id=%d\n", pi); 2674 } 2675 else 2676 rte_exit(EXIT_FAILURE, 2677 "set_tx_queue_stats_mapping_registers " 2678 "failed for port id=%d diag=%d\n", 2679 pi, diag); 2680 } 2681 2682 diag = set_rx_queue_stats_mapping_registers(pi, port); 2683 if (diag != 0) { 2684 if (diag == -ENOTSUP) { 2685 port->rx_queue_stats_mapping_enabled = 0; 2686 printf("RX queue stats mapping not supported port id=%d\n", pi); 2687 } 2688 else 2689 rte_exit(EXIT_FAILURE, 2690 "set_rx_queue_stats_mapping_registers " 2691 "failed for port id=%d diag=%d\n", 2692 pi, diag); 2693 } 2694 } 2695 2696 static void 2697 rxtx_port_config(struct rte_port *port) 2698 { 2699 uint16_t qid; 2700 2701 for (qid = 0; qid < nb_rxq; qid++) { 2702 port->rx_conf[qid] = port->dev_info.default_rxconf; 2703 2704 /* Check if any Rx parameters have been passed */ 2705 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2706 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2707 2708 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2709 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2710 2711 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2712 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2713 2714 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2715 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2716 2717 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2718 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2719 2720 port->nb_rx_desc[qid] = nb_rxd; 2721 } 2722 2723 for (qid = 0; qid < nb_txq; qid++) { 2724 port->tx_conf[qid] = port->dev_info.default_txconf; 2725 2726 /* Check if any Tx parameters have been passed */ 2727 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2728 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2729 2730 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2731 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2732 2733 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2734 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2735 2736 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2737 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2738 2739 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2740 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2741 2742 port->nb_tx_desc[qid] = nb_txd; 2743 } 2744 } 2745 2746 void 2747 init_port_config(void) 2748 { 2749 portid_t pid; 2750 struct rte_port *port; 2751 2752 RTE_ETH_FOREACH_DEV(pid) { 2753 port = &ports[pid]; 2754 port->dev_conf.fdir_conf = fdir_conf; 2755 rte_eth_dev_info_get(pid, &port->dev_info); 2756 if (nb_rxq > 1) { 2757 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2758 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2759 rss_hf & port->dev_info.flow_type_rss_offloads; 2760 } else { 2761 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2762 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2763 } 2764 2765 if (port->dcb_flag == 0) { 2766 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2767 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2768 else 2769 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2770 } 2771 2772 rxtx_port_config(port); 2773 2774 rte_eth_macaddr_get(pid, &port->eth_addr); 2775 2776 map_port_queue_stats_mapping_registers(pid, port); 2777 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2778 rte_pmd_ixgbe_bypass_init(pid); 2779 #endif 2780 2781 if (lsc_interrupt && 2782 (rte_eth_devices[pid].data->dev_flags & 2783 RTE_ETH_DEV_INTR_LSC)) 2784 port->dev_conf.intr_conf.lsc = 1; 2785 if (rmv_interrupt && 2786 (rte_eth_devices[pid].data->dev_flags & 2787 RTE_ETH_DEV_INTR_RMV)) 2788 port->dev_conf.intr_conf.rmv = 1; 2789 } 2790 } 2791 2792 void set_port_slave_flag(portid_t slave_pid) 2793 { 2794 struct rte_port *port; 2795 2796 port = &ports[slave_pid]; 2797 port->slave_flag = 1; 2798 } 2799 2800 void clear_port_slave_flag(portid_t slave_pid) 2801 { 2802 struct rte_port *port; 2803 2804 port = &ports[slave_pid]; 2805 port->slave_flag = 0; 2806 } 2807 2808 uint8_t port_is_bonding_slave(portid_t slave_pid) 2809 { 2810 struct rte_port *port; 2811 2812 port = &ports[slave_pid]; 2813 if ((rte_eth_devices[slave_pid].data->dev_flags & 2814 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2815 return 1; 2816 return 0; 2817 } 2818 2819 const uint16_t vlan_tags[] = { 2820 0, 1, 2, 3, 4, 5, 6, 7, 2821 8, 9, 10, 11, 12, 13, 14, 15, 2822 16, 17, 18, 19, 20, 21, 22, 23, 2823 24, 25, 26, 27, 28, 29, 30, 31 2824 }; 2825 2826 static int 2827 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 2828 enum dcb_mode_enable dcb_mode, 2829 enum rte_eth_nb_tcs num_tcs, 2830 uint8_t pfc_en) 2831 { 2832 uint8_t i; 2833 int32_t rc; 2834 struct rte_eth_rss_conf rss_conf; 2835 2836 /* 2837 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2838 * given above, and the number of traffic classes available for use. 2839 */ 2840 if (dcb_mode == DCB_VT_ENABLED) { 2841 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2842 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2843 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2844 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2845 2846 /* VMDQ+DCB RX and TX configurations */ 2847 vmdq_rx_conf->enable_default_pool = 0; 2848 vmdq_rx_conf->default_pool = 0; 2849 vmdq_rx_conf->nb_queue_pools = 2850 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2851 vmdq_tx_conf->nb_queue_pools = 2852 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2853 2854 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2855 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2856 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2857 vmdq_rx_conf->pool_map[i].pools = 2858 1 << (i % vmdq_rx_conf->nb_queue_pools); 2859 } 2860 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2861 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 2862 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 2863 } 2864 2865 /* set DCB mode of RX and TX of multiple queues */ 2866 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2867 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2868 } else { 2869 struct rte_eth_dcb_rx_conf *rx_conf = 2870 ð_conf->rx_adv_conf.dcb_rx_conf; 2871 struct rte_eth_dcb_tx_conf *tx_conf = 2872 ð_conf->tx_adv_conf.dcb_tx_conf; 2873 2874 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 2875 if (rc != 0) 2876 return rc; 2877 2878 rx_conf->nb_tcs = num_tcs; 2879 tx_conf->nb_tcs = num_tcs; 2880 2881 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2882 rx_conf->dcb_tc[i] = i % num_tcs; 2883 tx_conf->dcb_tc[i] = i % num_tcs; 2884 } 2885 2886 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2887 eth_conf->rx_adv_conf.rss_conf = rss_conf; 2888 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2889 } 2890 2891 if (pfc_en) 2892 eth_conf->dcb_capability_en = 2893 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2894 else 2895 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2896 2897 return 0; 2898 } 2899 2900 int 2901 init_port_dcb_config(portid_t pid, 2902 enum dcb_mode_enable dcb_mode, 2903 enum rte_eth_nb_tcs num_tcs, 2904 uint8_t pfc_en) 2905 { 2906 struct rte_eth_conf port_conf; 2907 struct rte_port *rte_port; 2908 int retval; 2909 uint16_t i; 2910 2911 rte_port = &ports[pid]; 2912 2913 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2914 /* Enter DCB configuration status */ 2915 dcb_config = 1; 2916 2917 port_conf.rxmode = rte_port->dev_conf.rxmode; 2918 port_conf.txmode = rte_port->dev_conf.txmode; 2919 2920 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2921 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 2922 if (retval < 0) 2923 return retval; 2924 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2925 2926 /* re-configure the device . */ 2927 rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 2928 2929 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2930 2931 /* If dev_info.vmdq_pool_base is greater than 0, 2932 * the queue id of vmdq pools is started after pf queues. 2933 */ 2934 if (dcb_mode == DCB_VT_ENABLED && 2935 rte_port->dev_info.vmdq_pool_base > 0) { 2936 printf("VMDQ_DCB multi-queue mode is nonsensical" 2937 " for port %d.", pid); 2938 return -1; 2939 } 2940 2941 /* Assume the ports in testpmd have the same dcb capability 2942 * and has the same number of rxq and txq in dcb mode 2943 */ 2944 if (dcb_mode == DCB_VT_ENABLED) { 2945 if (rte_port->dev_info.max_vfs > 0) { 2946 nb_rxq = rte_port->dev_info.nb_rx_queues; 2947 nb_txq = rte_port->dev_info.nb_tx_queues; 2948 } else { 2949 nb_rxq = rte_port->dev_info.max_rx_queues; 2950 nb_txq = rte_port->dev_info.max_tx_queues; 2951 } 2952 } else { 2953 /*if vt is disabled, use all pf queues */ 2954 if (rte_port->dev_info.vmdq_pool_base == 0) { 2955 nb_rxq = rte_port->dev_info.max_rx_queues; 2956 nb_txq = rte_port->dev_info.max_tx_queues; 2957 } else { 2958 nb_rxq = (queueid_t)num_tcs; 2959 nb_txq = (queueid_t)num_tcs; 2960 2961 } 2962 } 2963 rx_free_thresh = 64; 2964 2965 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 2966 2967 rxtx_port_config(rte_port); 2968 /* VLAN filter */ 2969 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2970 for (i = 0; i < RTE_DIM(vlan_tags); i++) 2971 rx_vft_set(pid, vlan_tags[i], 1); 2972 2973 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 2974 map_port_queue_stats_mapping_registers(pid, rte_port); 2975 2976 rte_port->dcb_flag = 1; 2977 2978 return 0; 2979 } 2980 2981 static void 2982 init_port(void) 2983 { 2984 /* Configuration of Ethernet ports. */ 2985 ports = rte_zmalloc("testpmd: ports", 2986 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 2987 RTE_CACHE_LINE_SIZE); 2988 if (ports == NULL) { 2989 rte_exit(EXIT_FAILURE, 2990 "rte_zmalloc(%d struct rte_port) failed\n", 2991 RTE_MAX_ETHPORTS); 2992 } 2993 2994 /* Initialize ports NUMA structures */ 2995 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 2996 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 2997 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 2998 } 2999 3000 static void 3001 force_quit(void) 3002 { 3003 pmd_test_exit(); 3004 prompt_exit(); 3005 } 3006 3007 static void 3008 print_stats(void) 3009 { 3010 uint8_t i; 3011 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3012 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3013 3014 /* Clear screen and move to top left */ 3015 printf("%s%s", clr, top_left); 3016 3017 printf("\nPort statistics ===================================="); 3018 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3019 nic_stats_display(fwd_ports_ids[i]); 3020 } 3021 3022 static void 3023 signal_handler(int signum) 3024 { 3025 if (signum == SIGINT || signum == SIGTERM) { 3026 printf("\nSignal %d received, preparing to exit...\n", 3027 signum); 3028 #ifdef RTE_LIBRTE_PDUMP 3029 /* uninitialize packet capture framework */ 3030 rte_pdump_uninit(); 3031 #endif 3032 #ifdef RTE_LIBRTE_LATENCY_STATS 3033 rte_latencystats_uninit(); 3034 #endif 3035 force_quit(); 3036 /* Set flag to indicate the force termination. */ 3037 f_quit = 1; 3038 /* exit with the expected status */ 3039 signal(signum, SIG_DFL); 3040 kill(getpid(), signum); 3041 } 3042 } 3043 3044 int 3045 main(int argc, char** argv) 3046 { 3047 int diag; 3048 portid_t port_id; 3049 uint16_t count; 3050 int ret; 3051 3052 signal(SIGINT, signal_handler); 3053 signal(SIGTERM, signal_handler); 3054 3055 diag = rte_eal_init(argc, argv); 3056 if (diag < 0) 3057 rte_panic("Cannot init EAL\n"); 3058 3059 testpmd_logtype = rte_log_register("testpmd"); 3060 if (testpmd_logtype < 0) 3061 rte_panic("Cannot register log type"); 3062 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3063 3064 ret = register_eth_event_callback(); 3065 if (ret != 0) 3066 rte_panic("Cannot register for ethdev events"); 3067 3068 #ifdef RTE_LIBRTE_PDUMP 3069 /* initialize packet capture framework */ 3070 rte_pdump_init(NULL); 3071 #endif 3072 3073 count = 0; 3074 RTE_ETH_FOREACH_DEV(port_id) { 3075 ports_ids[count] = port_id; 3076 count++; 3077 } 3078 nb_ports = (portid_t) count; 3079 if (nb_ports == 0) 3080 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3081 3082 /* allocate port structures, and init them */ 3083 init_port(); 3084 3085 set_def_fwd_config(); 3086 if (nb_lcores == 0) 3087 rte_panic("Empty set of forwarding logical cores - check the " 3088 "core mask supplied in the command parameters\n"); 3089 3090 /* Bitrate/latency stats disabled by default */ 3091 #ifdef RTE_LIBRTE_BITRATE 3092 bitrate_enabled = 0; 3093 #endif 3094 #ifdef RTE_LIBRTE_LATENCY_STATS 3095 latencystats_enabled = 0; 3096 #endif 3097 3098 /* on FreeBSD, mlockall() is disabled by default */ 3099 #ifdef RTE_EXEC_ENV_BSDAPP 3100 do_mlockall = 0; 3101 #else 3102 do_mlockall = 1; 3103 #endif 3104 3105 argc -= diag; 3106 argv += diag; 3107 if (argc > 1) 3108 launch_args_parse(argc, argv); 3109 3110 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3111 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3112 strerror(errno)); 3113 } 3114 3115 if (tx_first && interactive) 3116 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3117 "interactive mode.\n"); 3118 3119 if (tx_first && lsc_interrupt) { 3120 printf("Warning: lsc_interrupt needs to be off when " 3121 " using tx_first. Disabling.\n"); 3122 lsc_interrupt = 0; 3123 } 3124 3125 if (!nb_rxq && !nb_txq) 3126 printf("Warning: Either rx or tx queues should be non-zero\n"); 3127 3128 if (nb_rxq > 1 && nb_rxq > nb_txq) 3129 printf("Warning: nb_rxq=%d enables RSS configuration, " 3130 "but nb_txq=%d will prevent to fully test it.\n", 3131 nb_rxq, nb_txq); 3132 3133 init_config(); 3134 3135 if (hot_plug) { 3136 ret = rte_dev_hotplug_handle_enable(); 3137 if (ret) { 3138 RTE_LOG(ERR, EAL, 3139 "fail to enable hotplug handling."); 3140 return -1; 3141 } 3142 3143 ret = rte_dev_event_monitor_start(); 3144 if (ret) { 3145 RTE_LOG(ERR, EAL, 3146 "fail to start device event monitoring."); 3147 return -1; 3148 } 3149 3150 ret = rte_dev_event_callback_register(NULL, 3151 eth_dev_event_callback, NULL); 3152 if (ret) { 3153 RTE_LOG(ERR, EAL, 3154 "fail to register device event callback\n"); 3155 return -1; 3156 } 3157 } 3158 3159 if (start_port(RTE_PORT_ALL) != 0) 3160 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3161 3162 /* set all ports to promiscuous mode by default */ 3163 RTE_ETH_FOREACH_DEV(port_id) 3164 rte_eth_promiscuous_enable(port_id); 3165 3166 /* Init metrics library */ 3167 rte_metrics_init(rte_socket_id()); 3168 3169 #ifdef RTE_LIBRTE_LATENCY_STATS 3170 if (latencystats_enabled != 0) { 3171 int ret = rte_latencystats_init(1, NULL); 3172 if (ret) 3173 printf("Warning: latencystats init()" 3174 " returned error %d\n", ret); 3175 printf("Latencystats running on lcore %d\n", 3176 latencystats_lcore_id); 3177 } 3178 #endif 3179 3180 /* Setup bitrate stats */ 3181 #ifdef RTE_LIBRTE_BITRATE 3182 if (bitrate_enabled != 0) { 3183 bitrate_data = rte_stats_bitrate_create(); 3184 if (bitrate_data == NULL) 3185 rte_exit(EXIT_FAILURE, 3186 "Could not allocate bitrate data.\n"); 3187 rte_stats_bitrate_reg(bitrate_data); 3188 } 3189 #endif 3190 3191 #ifdef RTE_LIBRTE_CMDLINE 3192 if (strlen(cmdline_filename) != 0) 3193 cmdline_read_from_file(cmdline_filename); 3194 3195 if (interactive == 1) { 3196 if (auto_start) { 3197 printf("Start automatic packet forwarding\n"); 3198 start_packet_forwarding(0); 3199 } 3200 prompt(); 3201 pmd_test_exit(); 3202 } else 3203 #endif 3204 { 3205 char c; 3206 int rc; 3207 3208 f_quit = 0; 3209 3210 printf("No commandline core given, start packet forwarding\n"); 3211 start_packet_forwarding(tx_first); 3212 if (stats_period != 0) { 3213 uint64_t prev_time = 0, cur_time, diff_time = 0; 3214 uint64_t timer_period; 3215 3216 /* Convert to number of cycles */ 3217 timer_period = stats_period * rte_get_timer_hz(); 3218 3219 while (f_quit == 0) { 3220 cur_time = rte_get_timer_cycles(); 3221 diff_time += cur_time - prev_time; 3222 3223 if (diff_time >= timer_period) { 3224 print_stats(); 3225 /* Reset the timer */ 3226 diff_time = 0; 3227 } 3228 /* Sleep to avoid unnecessary checks */ 3229 prev_time = cur_time; 3230 sleep(1); 3231 } 3232 } 3233 3234 printf("Press enter to exit\n"); 3235 rc = read(0, &c, 1); 3236 pmd_test_exit(); 3237 if (rc < 0) 3238 return 1; 3239 } 3240 3241 return 0; 3242 } 3243