1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct fwd_config cur_fwd_config; 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 193 uint32_t retry_enabled; 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 196 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 198 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 199 * specified on command-line. */ 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 201 202 /* 203 * In container, it cannot terminate the process which running with 'stats-period' 204 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 205 */ 206 uint8_t f_quit; 207 208 /* 209 * Configuration of packet segments used by the "txonly" processing engine. 210 */ 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 213 TXONLY_DEF_PACKET_LEN, 214 }; 215 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 216 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 218 /**< Split policy for packets to TX. */ 219 220 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 221 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 222 223 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 224 uint8_t dcb_config = 0; 225 226 /* Whether the dcb is in testing status */ 227 uint8_t dcb_test = 0; 228 229 /* 230 * Configurable number of RX/TX queues. 231 */ 232 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 233 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 234 235 /* 236 * Configurable number of RX/TX ring descriptors. 237 * Defaults are supplied by drivers via ethdev. 238 */ 239 #define RTE_TEST_RX_DESC_DEFAULT 0 240 #define RTE_TEST_TX_DESC_DEFAULT 0 241 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 242 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 243 244 #define RTE_PMD_PARAM_UNSET -1 245 /* 246 * Configurable values of RX and TX ring threshold registers. 247 */ 248 249 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 250 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 251 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 252 253 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 254 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 255 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 256 257 /* 258 * Configurable value of RX free threshold. 259 */ 260 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 261 262 /* 263 * Configurable value of RX drop enable. 264 */ 265 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 266 267 /* 268 * Configurable value of TX free threshold. 269 */ 270 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 271 272 /* 273 * Configurable value of TX RS bit threshold. 274 */ 275 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 276 277 /* 278 * Configurable value of buffered packets before sending. 279 */ 280 uint16_t noisy_tx_sw_bufsz; 281 282 /* 283 * Configurable value of packet buffer timeout. 284 */ 285 uint16_t noisy_tx_sw_buf_flush_time; 286 287 /* 288 * Configurable value for size of VNF internal memory area 289 * used for simulating noisy neighbour behaviour 290 */ 291 uint64_t noisy_lkup_mem_sz; 292 293 /* 294 * Configurable value of number of random writes done in 295 * VNF simulation memory area. 296 */ 297 uint64_t noisy_lkup_num_writes; 298 299 /* 300 * Configurable value of number of random reads done in 301 * VNF simulation memory area. 302 */ 303 uint64_t noisy_lkup_num_reads; 304 305 /* 306 * Configurable value of number of random reads/writes done in 307 * VNF simulation memory area. 308 */ 309 uint64_t noisy_lkup_num_reads_writes; 310 311 /* 312 * Receive Side Scaling (RSS) configuration. 313 */ 314 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 315 316 /* 317 * Port topology configuration 318 */ 319 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 320 321 /* 322 * Avoids to flush all the RX streams before starts forwarding. 323 */ 324 uint8_t no_flush_rx = 0; /* flush by default */ 325 326 /* 327 * Flow API isolated mode. 328 */ 329 uint8_t flow_isolate_all; 330 331 /* 332 * Avoids to check link status when starting/stopping a port. 333 */ 334 uint8_t no_link_check = 0; /* check by default */ 335 336 /* 337 * Enable link status change notification 338 */ 339 uint8_t lsc_interrupt = 1; /* enabled by default */ 340 341 /* 342 * Enable device removal notification. 343 */ 344 uint8_t rmv_interrupt = 1; /* enabled by default */ 345 346 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 347 348 /* After attach, port setup is called on event or by iterator */ 349 bool setup_on_probe_event = true; 350 351 /* Pretty printing of ethdev events */ 352 static const char * const eth_event_desc[] = { 353 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 354 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 355 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 356 [RTE_ETH_EVENT_INTR_RESET] = "reset", 357 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 358 [RTE_ETH_EVENT_IPSEC] = "IPsec", 359 [RTE_ETH_EVENT_MACSEC] = "MACsec", 360 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 361 [RTE_ETH_EVENT_NEW] = "device probed", 362 [RTE_ETH_EVENT_DESTROY] = "device released", 363 [RTE_ETH_EVENT_MAX] = NULL, 364 }; 365 366 /* 367 * Display or mask ether events 368 * Default to all events except VF_MBOX 369 */ 370 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 371 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 372 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 373 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 374 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 375 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 376 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 377 /* 378 * Decide if all memory are locked for performance. 379 */ 380 int do_mlockall = 0; 381 382 /* 383 * NIC bypass mode configuration options. 384 */ 385 386 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 387 /* The NIC bypass watchdog timeout. */ 388 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 389 #endif 390 391 392 #ifdef RTE_LIBRTE_LATENCY_STATS 393 394 /* 395 * Set when latency stats is enabled in the commandline 396 */ 397 uint8_t latencystats_enabled; 398 399 /* 400 * Lcore ID to serive latency statistics. 401 */ 402 lcoreid_t latencystats_lcore_id = -1; 403 404 #endif 405 406 /* 407 * Ethernet device configuration. 408 */ 409 struct rte_eth_rxmode rx_mode = { 410 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 411 }; 412 413 struct rte_eth_txmode tx_mode = { 414 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 415 }; 416 417 struct rte_fdir_conf fdir_conf = { 418 .mode = RTE_FDIR_MODE_NONE, 419 .pballoc = RTE_FDIR_PBALLOC_64K, 420 .status = RTE_FDIR_REPORT_STATUS, 421 .mask = { 422 .vlan_tci_mask = 0xFFEF, 423 .ipv4_mask = { 424 .src_ip = 0xFFFFFFFF, 425 .dst_ip = 0xFFFFFFFF, 426 }, 427 .ipv6_mask = { 428 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 429 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 430 }, 431 .src_port_mask = 0xFFFF, 432 .dst_port_mask = 0xFFFF, 433 .mac_addr_byte_mask = 0xFF, 434 .tunnel_type_mask = 1, 435 .tunnel_id_mask = 0xFFFFFFFF, 436 }, 437 .drop_queue = 127, 438 }; 439 440 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 441 442 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 443 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 444 445 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 446 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 447 448 uint16_t nb_tx_queue_stats_mappings = 0; 449 uint16_t nb_rx_queue_stats_mappings = 0; 450 451 /* 452 * Display zero values by default for xstats 453 */ 454 uint8_t xstats_hide_zero; 455 456 unsigned int num_sockets = 0; 457 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 458 459 #ifdef RTE_LIBRTE_BITRATE 460 /* Bitrate statistics */ 461 struct rte_stats_bitrates *bitrate_data; 462 lcoreid_t bitrate_lcore_id; 463 uint8_t bitrate_enabled; 464 #endif 465 466 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 467 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 468 469 struct vxlan_encap_conf vxlan_encap_conf = { 470 .select_ipv4 = 1, 471 .select_vlan = 0, 472 .select_tos_ttl = 0, 473 .vni = "\x00\x00\x00", 474 .udp_src = 0, 475 .udp_dst = RTE_BE16(4789), 476 .ipv4_src = IPv4(127, 0, 0, 1), 477 .ipv4_dst = IPv4(255, 255, 255, 255), 478 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 479 "\x00\x00\x00\x00\x00\x00\x00\x01", 480 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 481 "\x00\x00\x00\x00\x00\x00\x11\x11", 482 .vlan_tci = 0, 483 .ip_tos = 0, 484 .ip_ttl = 255, 485 .eth_src = "\x00\x00\x00\x00\x00\x00", 486 .eth_dst = "\xff\xff\xff\xff\xff\xff", 487 }; 488 489 struct nvgre_encap_conf nvgre_encap_conf = { 490 .select_ipv4 = 1, 491 .select_vlan = 0, 492 .tni = "\x00\x00\x00", 493 .ipv4_src = IPv4(127, 0, 0, 1), 494 .ipv4_dst = IPv4(255, 255, 255, 255), 495 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 496 "\x00\x00\x00\x00\x00\x00\x00\x01", 497 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 498 "\x00\x00\x00\x00\x00\x00\x11\x11", 499 .vlan_tci = 0, 500 .eth_src = "\x00\x00\x00\x00\x00\x00", 501 .eth_dst = "\xff\xff\xff\xff\xff\xff", 502 }; 503 504 /* Forward function declarations */ 505 static void setup_attached_port(portid_t pi); 506 static void map_port_queue_stats_mapping_registers(portid_t pi, 507 struct rte_port *port); 508 static void check_all_ports_link_status(uint32_t port_mask); 509 static int eth_event_callback(portid_t port_id, 510 enum rte_eth_event_type type, 511 void *param, void *ret_param); 512 static void dev_event_callback(const char *device_name, 513 enum rte_dev_event_type type, 514 void *param); 515 516 /* 517 * Check if all the ports are started. 518 * If yes, return positive value. If not, return zero. 519 */ 520 static int all_ports_started(void); 521 522 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 523 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN; 524 525 /* 526 * Helper function to check if socket is already discovered. 527 * If yes, return positive value. If not, return zero. 528 */ 529 int 530 new_socket_id(unsigned int socket_id) 531 { 532 unsigned int i; 533 534 for (i = 0; i < num_sockets; i++) { 535 if (socket_ids[i] == socket_id) 536 return 0; 537 } 538 return 1; 539 } 540 541 /* 542 * Setup default configuration. 543 */ 544 static void 545 set_default_fwd_lcores_config(void) 546 { 547 unsigned int i; 548 unsigned int nb_lc; 549 unsigned int sock_num; 550 551 nb_lc = 0; 552 for (i = 0; i < RTE_MAX_LCORE; i++) { 553 if (!rte_lcore_is_enabled(i)) 554 continue; 555 sock_num = rte_lcore_to_socket_id(i); 556 if (new_socket_id(sock_num)) { 557 if (num_sockets >= RTE_MAX_NUMA_NODES) { 558 rte_exit(EXIT_FAILURE, 559 "Total sockets greater than %u\n", 560 RTE_MAX_NUMA_NODES); 561 } 562 socket_ids[num_sockets++] = sock_num; 563 } 564 if (i == rte_get_master_lcore()) 565 continue; 566 fwd_lcores_cpuids[nb_lc++] = i; 567 } 568 nb_lcores = (lcoreid_t) nb_lc; 569 nb_cfg_lcores = nb_lcores; 570 nb_fwd_lcores = 1; 571 } 572 573 static void 574 set_def_peer_eth_addrs(void) 575 { 576 portid_t i; 577 578 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 579 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 580 peer_eth_addrs[i].addr_bytes[5] = i; 581 } 582 } 583 584 static void 585 set_default_fwd_ports_config(void) 586 { 587 portid_t pt_id; 588 int i = 0; 589 590 RTE_ETH_FOREACH_DEV(pt_id) { 591 fwd_ports_ids[i++] = pt_id; 592 593 /* Update sockets info according to the attached device */ 594 int socket_id = rte_eth_dev_socket_id(pt_id); 595 if (socket_id >= 0 && new_socket_id(socket_id)) { 596 if (num_sockets >= RTE_MAX_NUMA_NODES) { 597 rte_exit(EXIT_FAILURE, 598 "Total sockets greater than %u\n", 599 RTE_MAX_NUMA_NODES); 600 } 601 socket_ids[num_sockets++] = socket_id; 602 } 603 } 604 605 nb_cfg_ports = nb_ports; 606 nb_fwd_ports = nb_ports; 607 } 608 609 void 610 set_def_fwd_config(void) 611 { 612 set_default_fwd_lcores_config(); 613 set_def_peer_eth_addrs(); 614 set_default_fwd_ports_config(); 615 } 616 617 /* extremely pessimistic estimation of memory required to create a mempool */ 618 static int 619 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 620 { 621 unsigned int n_pages, mbuf_per_pg, leftover; 622 uint64_t total_mem, mbuf_mem, obj_sz; 623 624 /* there is no good way to predict how much space the mempool will 625 * occupy because it will allocate chunks on the fly, and some of those 626 * will come from default DPDK memory while some will come from our 627 * external memory, so just assume 128MB will be enough for everyone. 628 */ 629 uint64_t hdr_mem = 128 << 20; 630 631 /* account for possible non-contiguousness */ 632 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 633 if (obj_sz > pgsz) { 634 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 635 return -1; 636 } 637 638 mbuf_per_pg = pgsz / obj_sz; 639 leftover = (nb_mbufs % mbuf_per_pg) > 0; 640 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 641 642 mbuf_mem = n_pages * pgsz; 643 644 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 645 646 if (total_mem > SIZE_MAX) { 647 TESTPMD_LOG(ERR, "Memory size too big\n"); 648 return -1; 649 } 650 *out = (size_t)total_mem; 651 652 return 0; 653 } 654 655 static int 656 pagesz_flags(uint64_t page_sz) 657 { 658 /* as per mmap() manpage, all page sizes are log2 of page size 659 * shifted by MAP_HUGE_SHIFT 660 */ 661 int log2 = rte_log2_u64(page_sz); 662 663 return (log2 << HUGE_SHIFT); 664 } 665 666 static void * 667 alloc_mem(size_t memsz, size_t pgsz, bool huge) 668 { 669 void *addr; 670 int flags; 671 672 /* allocate anonymous hugepages */ 673 flags = MAP_ANONYMOUS | MAP_PRIVATE; 674 if (huge) 675 flags |= HUGE_FLAG | pagesz_flags(pgsz); 676 677 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 678 if (addr == MAP_FAILED) 679 return NULL; 680 681 return addr; 682 } 683 684 struct extmem_param { 685 void *addr; 686 size_t len; 687 size_t pgsz; 688 rte_iova_t *iova_table; 689 unsigned int iova_table_len; 690 }; 691 692 static int 693 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 694 bool huge) 695 { 696 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 697 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 698 unsigned int cur_page, n_pages, pgsz_idx; 699 size_t mem_sz, cur_pgsz; 700 rte_iova_t *iovas = NULL; 701 void *addr; 702 int ret; 703 704 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 705 /* skip anything that is too big */ 706 if (pgsizes[pgsz_idx] > SIZE_MAX) 707 continue; 708 709 cur_pgsz = pgsizes[pgsz_idx]; 710 711 /* if we were told not to allocate hugepages, override */ 712 if (!huge) 713 cur_pgsz = sysconf(_SC_PAGESIZE); 714 715 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 716 if (ret < 0) { 717 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 718 return -1; 719 } 720 721 /* allocate our memory */ 722 addr = alloc_mem(mem_sz, cur_pgsz, huge); 723 724 /* if we couldn't allocate memory with a specified page size, 725 * that doesn't mean we can't do it with other page sizes, so 726 * try another one. 727 */ 728 if (addr == NULL) 729 continue; 730 731 /* store IOVA addresses for every page in this memory area */ 732 n_pages = mem_sz / cur_pgsz; 733 734 iovas = malloc(sizeof(*iovas) * n_pages); 735 736 if (iovas == NULL) { 737 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 738 goto fail; 739 } 740 /* lock memory if it's not huge pages */ 741 if (!huge) 742 mlock(addr, mem_sz); 743 744 /* populate IOVA addresses */ 745 for (cur_page = 0; cur_page < n_pages; cur_page++) { 746 rte_iova_t iova; 747 size_t offset; 748 void *cur; 749 750 offset = cur_pgsz * cur_page; 751 cur = RTE_PTR_ADD(addr, offset); 752 753 /* touch the page before getting its IOVA */ 754 *(volatile char *)cur = 0; 755 756 iova = rte_mem_virt2iova(cur); 757 758 iovas[cur_page] = iova; 759 } 760 761 break; 762 } 763 /* if we couldn't allocate anything */ 764 if (iovas == NULL) 765 return -1; 766 767 param->addr = addr; 768 param->len = mem_sz; 769 param->pgsz = cur_pgsz; 770 param->iova_table = iovas; 771 param->iova_table_len = n_pages; 772 773 return 0; 774 fail: 775 if (iovas) 776 free(iovas); 777 if (addr) 778 munmap(addr, mem_sz); 779 780 return -1; 781 } 782 783 static int 784 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 785 { 786 struct extmem_param param; 787 int socket_id, ret; 788 789 memset(¶m, 0, sizeof(param)); 790 791 /* check if our heap exists */ 792 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 793 if (socket_id < 0) { 794 /* create our heap */ 795 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 796 if (ret < 0) { 797 TESTPMD_LOG(ERR, "Cannot create heap\n"); 798 return -1; 799 } 800 } 801 802 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 803 if (ret < 0) { 804 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 805 return -1; 806 } 807 808 /* we now have a valid memory area, so add it to heap */ 809 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 810 param.addr, param.len, param.iova_table, 811 param.iova_table_len, param.pgsz); 812 813 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 814 815 /* not needed any more */ 816 free(param.iova_table); 817 818 if (ret < 0) { 819 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 820 munmap(param.addr, param.len); 821 return -1; 822 } 823 824 /* success */ 825 826 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 827 param.len >> 20); 828 829 return 0; 830 } 831 832 /* 833 * Configuration initialisation done once at init time. 834 */ 835 static void 836 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 837 unsigned int socket_id) 838 { 839 char pool_name[RTE_MEMPOOL_NAMESIZE]; 840 struct rte_mempool *rte_mp = NULL; 841 uint32_t mb_size; 842 843 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 844 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 845 846 TESTPMD_LOG(INFO, 847 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 848 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 849 850 switch (mp_alloc_type) { 851 case MP_ALLOC_NATIVE: 852 { 853 /* wrapper to rte_mempool_create() */ 854 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 855 rte_mbuf_best_mempool_ops()); 856 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 857 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 858 break; 859 } 860 case MP_ALLOC_ANON: 861 { 862 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 863 mb_size, (unsigned int) mb_mempool_cache, 864 sizeof(struct rte_pktmbuf_pool_private), 865 socket_id, 0); 866 if (rte_mp == NULL) 867 goto err; 868 869 if (rte_mempool_populate_anon(rte_mp) == 0) { 870 rte_mempool_free(rte_mp); 871 rte_mp = NULL; 872 goto err; 873 } 874 rte_pktmbuf_pool_init(rte_mp, NULL); 875 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 876 break; 877 } 878 case MP_ALLOC_XMEM: 879 case MP_ALLOC_XMEM_HUGE: 880 { 881 int heap_socket; 882 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 883 884 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 885 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 886 887 heap_socket = 888 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 889 if (heap_socket < 0) 890 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 891 892 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 893 rte_mbuf_best_mempool_ops()); 894 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 895 mb_mempool_cache, 0, mbuf_seg_size, 896 heap_socket); 897 break; 898 } 899 default: 900 { 901 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 902 } 903 } 904 905 err: 906 if (rte_mp == NULL) { 907 rte_exit(EXIT_FAILURE, 908 "Creation of mbuf pool for socket %u failed: %s\n", 909 socket_id, rte_strerror(rte_errno)); 910 } else if (verbose_level > 0) { 911 rte_mempool_dump(stdout, rte_mp); 912 } 913 } 914 915 /* 916 * Check given socket id is valid or not with NUMA mode, 917 * if valid, return 0, else return -1 918 */ 919 static int 920 check_socket_id(const unsigned int socket_id) 921 { 922 static int warning_once = 0; 923 924 if (new_socket_id(socket_id)) { 925 if (!warning_once && numa_support) 926 printf("Warning: NUMA should be configured manually by" 927 " using --port-numa-config and" 928 " --ring-numa-config parameters along with" 929 " --numa.\n"); 930 warning_once = 1; 931 return -1; 932 } 933 return 0; 934 } 935 936 /* 937 * Get the allowed maximum number of RX queues. 938 * *pid return the port id which has minimal value of 939 * max_rx_queues in all ports. 940 */ 941 queueid_t 942 get_allowed_max_nb_rxq(portid_t *pid) 943 { 944 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 945 portid_t pi; 946 struct rte_eth_dev_info dev_info; 947 948 RTE_ETH_FOREACH_DEV(pi) { 949 rte_eth_dev_info_get(pi, &dev_info); 950 if (dev_info.max_rx_queues < allowed_max_rxq) { 951 allowed_max_rxq = dev_info.max_rx_queues; 952 *pid = pi; 953 } 954 } 955 return allowed_max_rxq; 956 } 957 958 /* 959 * Check input rxq is valid or not. 960 * If input rxq is not greater than any of maximum number 961 * of RX queues of all ports, it is valid. 962 * if valid, return 0, else return -1 963 */ 964 int 965 check_nb_rxq(queueid_t rxq) 966 { 967 queueid_t allowed_max_rxq; 968 portid_t pid = 0; 969 970 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 971 if (rxq > allowed_max_rxq) { 972 printf("Fail: input rxq (%u) can't be greater " 973 "than max_rx_queues (%u) of port %u\n", 974 rxq, 975 allowed_max_rxq, 976 pid); 977 return -1; 978 } 979 return 0; 980 } 981 982 /* 983 * Get the allowed maximum number of TX queues. 984 * *pid return the port id which has minimal value of 985 * max_tx_queues in all ports. 986 */ 987 queueid_t 988 get_allowed_max_nb_txq(portid_t *pid) 989 { 990 queueid_t allowed_max_txq = MAX_QUEUE_ID; 991 portid_t pi; 992 struct rte_eth_dev_info dev_info; 993 994 RTE_ETH_FOREACH_DEV(pi) { 995 rte_eth_dev_info_get(pi, &dev_info); 996 if (dev_info.max_tx_queues < allowed_max_txq) { 997 allowed_max_txq = dev_info.max_tx_queues; 998 *pid = pi; 999 } 1000 } 1001 return allowed_max_txq; 1002 } 1003 1004 /* 1005 * Check input txq is valid or not. 1006 * If input txq is not greater than any of maximum number 1007 * of TX queues of all ports, it is valid. 1008 * if valid, return 0, else return -1 1009 */ 1010 int 1011 check_nb_txq(queueid_t txq) 1012 { 1013 queueid_t allowed_max_txq; 1014 portid_t pid = 0; 1015 1016 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1017 if (txq > allowed_max_txq) { 1018 printf("Fail: input txq (%u) can't be greater " 1019 "than max_tx_queues (%u) of port %u\n", 1020 txq, 1021 allowed_max_txq, 1022 pid); 1023 return -1; 1024 } 1025 return 0; 1026 } 1027 1028 static void 1029 init_config(void) 1030 { 1031 portid_t pid; 1032 struct rte_port *port; 1033 struct rte_mempool *mbp; 1034 unsigned int nb_mbuf_per_pool; 1035 lcoreid_t lc_id; 1036 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1037 struct rte_gro_param gro_param; 1038 uint32_t gso_types; 1039 int k; 1040 1041 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1042 1043 /* Configuration of logical cores. */ 1044 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1045 sizeof(struct fwd_lcore *) * nb_lcores, 1046 RTE_CACHE_LINE_SIZE); 1047 if (fwd_lcores == NULL) { 1048 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1049 "failed\n", nb_lcores); 1050 } 1051 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1052 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1053 sizeof(struct fwd_lcore), 1054 RTE_CACHE_LINE_SIZE); 1055 if (fwd_lcores[lc_id] == NULL) { 1056 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1057 "failed\n"); 1058 } 1059 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1060 } 1061 1062 RTE_ETH_FOREACH_DEV(pid) { 1063 port = &ports[pid]; 1064 /* Apply default TxRx configuration for all ports */ 1065 port->dev_conf.txmode = tx_mode; 1066 port->dev_conf.rxmode = rx_mode; 1067 rte_eth_dev_info_get(pid, &port->dev_info); 1068 1069 if (!(port->dev_info.tx_offload_capa & 1070 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1071 port->dev_conf.txmode.offloads &= 1072 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1073 if (!(port->dev_info.tx_offload_capa & 1074 DEV_TX_OFFLOAD_MATCH_METADATA)) 1075 port->dev_conf.txmode.offloads &= 1076 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1077 if (numa_support) { 1078 if (port_numa[pid] != NUMA_NO_CONFIG) 1079 port_per_socket[port_numa[pid]]++; 1080 else { 1081 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1082 1083 /* 1084 * if socket_id is invalid, 1085 * set to the first available socket. 1086 */ 1087 if (check_socket_id(socket_id) < 0) 1088 socket_id = socket_ids[0]; 1089 port_per_socket[socket_id]++; 1090 } 1091 } 1092 1093 /* Apply Rx offloads configuration */ 1094 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1095 port->rx_conf[k].offloads = 1096 port->dev_conf.rxmode.offloads; 1097 /* Apply Tx offloads configuration */ 1098 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1099 port->tx_conf[k].offloads = 1100 port->dev_conf.txmode.offloads; 1101 1102 /* set flag to initialize port/queue */ 1103 port->need_reconfig = 1; 1104 port->need_reconfig_queues = 1; 1105 port->tx_metadata = 0; 1106 } 1107 1108 /* 1109 * Create pools of mbuf. 1110 * If NUMA support is disabled, create a single pool of mbuf in 1111 * socket 0 memory by default. 1112 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1113 * 1114 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1115 * nb_txd can be configured at run time. 1116 */ 1117 if (param_total_num_mbufs) 1118 nb_mbuf_per_pool = param_total_num_mbufs; 1119 else { 1120 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1121 (nb_lcores * mb_mempool_cache) + 1122 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1123 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1124 } 1125 1126 if (numa_support) { 1127 uint8_t i; 1128 1129 for (i = 0; i < num_sockets; i++) 1130 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1131 socket_ids[i]); 1132 } else { 1133 if (socket_num == UMA_NO_CONFIG) 1134 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 1135 else 1136 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1137 socket_num); 1138 } 1139 1140 init_port_config(); 1141 1142 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1143 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1144 /* 1145 * Records which Mbuf pool to use by each logical core, if needed. 1146 */ 1147 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1148 mbp = mbuf_pool_find( 1149 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1150 1151 if (mbp == NULL) 1152 mbp = mbuf_pool_find(0); 1153 fwd_lcores[lc_id]->mbp = mbp; 1154 /* initialize GSO context */ 1155 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1156 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1157 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1158 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN - 1159 ETHER_CRC_LEN; 1160 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1161 } 1162 1163 /* Configuration of packet forwarding streams. */ 1164 if (init_fwd_streams() < 0) 1165 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1166 1167 fwd_config_setup(); 1168 1169 /* create a gro context for each lcore */ 1170 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1171 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1172 gro_param.max_item_per_flow = MAX_PKT_BURST; 1173 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1174 gro_param.socket_id = rte_lcore_to_socket_id( 1175 fwd_lcores_cpuids[lc_id]); 1176 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1177 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1178 rte_exit(EXIT_FAILURE, 1179 "rte_gro_ctx_create() failed\n"); 1180 } 1181 } 1182 1183 #if defined RTE_LIBRTE_PMD_SOFTNIC 1184 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1185 RTE_ETH_FOREACH_DEV(pid) { 1186 port = &ports[pid]; 1187 const char *driver = port->dev_info.driver_name; 1188 1189 if (strcmp(driver, "net_softnic") == 0) 1190 port->softport.fwd_lcore_arg = fwd_lcores; 1191 } 1192 } 1193 #endif 1194 1195 } 1196 1197 1198 void 1199 reconfig(portid_t new_port_id, unsigned socket_id) 1200 { 1201 struct rte_port *port; 1202 1203 /* Reconfiguration of Ethernet ports. */ 1204 port = &ports[new_port_id]; 1205 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1206 1207 /* set flag to initialize port/queue */ 1208 port->need_reconfig = 1; 1209 port->need_reconfig_queues = 1; 1210 port->socket_id = socket_id; 1211 1212 init_port_config(); 1213 } 1214 1215 1216 int 1217 init_fwd_streams(void) 1218 { 1219 portid_t pid; 1220 struct rte_port *port; 1221 streamid_t sm_id, nb_fwd_streams_new; 1222 queueid_t q; 1223 1224 /* set socket id according to numa or not */ 1225 RTE_ETH_FOREACH_DEV(pid) { 1226 port = &ports[pid]; 1227 if (nb_rxq > port->dev_info.max_rx_queues) { 1228 printf("Fail: nb_rxq(%d) is greater than " 1229 "max_rx_queues(%d)\n", nb_rxq, 1230 port->dev_info.max_rx_queues); 1231 return -1; 1232 } 1233 if (nb_txq > port->dev_info.max_tx_queues) { 1234 printf("Fail: nb_txq(%d) is greater than " 1235 "max_tx_queues(%d)\n", nb_txq, 1236 port->dev_info.max_tx_queues); 1237 return -1; 1238 } 1239 if (numa_support) { 1240 if (port_numa[pid] != NUMA_NO_CONFIG) 1241 port->socket_id = port_numa[pid]; 1242 else { 1243 port->socket_id = rte_eth_dev_socket_id(pid); 1244 1245 /* 1246 * if socket_id is invalid, 1247 * set to the first available socket. 1248 */ 1249 if (check_socket_id(port->socket_id) < 0) 1250 port->socket_id = socket_ids[0]; 1251 } 1252 } 1253 else { 1254 if (socket_num == UMA_NO_CONFIG) 1255 port->socket_id = 0; 1256 else 1257 port->socket_id = socket_num; 1258 } 1259 } 1260 1261 q = RTE_MAX(nb_rxq, nb_txq); 1262 if (q == 0) { 1263 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1264 return -1; 1265 } 1266 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1267 if (nb_fwd_streams_new == nb_fwd_streams) 1268 return 0; 1269 /* clear the old */ 1270 if (fwd_streams != NULL) { 1271 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1272 if (fwd_streams[sm_id] == NULL) 1273 continue; 1274 rte_free(fwd_streams[sm_id]); 1275 fwd_streams[sm_id] = NULL; 1276 } 1277 rte_free(fwd_streams); 1278 fwd_streams = NULL; 1279 } 1280 1281 /* init new */ 1282 nb_fwd_streams = nb_fwd_streams_new; 1283 if (nb_fwd_streams) { 1284 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1285 sizeof(struct fwd_stream *) * nb_fwd_streams, 1286 RTE_CACHE_LINE_SIZE); 1287 if (fwd_streams == NULL) 1288 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1289 " (struct fwd_stream *)) failed\n", 1290 nb_fwd_streams); 1291 1292 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1293 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1294 " struct fwd_stream", sizeof(struct fwd_stream), 1295 RTE_CACHE_LINE_SIZE); 1296 if (fwd_streams[sm_id] == NULL) 1297 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1298 "(struct fwd_stream) failed\n"); 1299 } 1300 } 1301 1302 return 0; 1303 } 1304 1305 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1306 static void 1307 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1308 { 1309 unsigned int total_burst; 1310 unsigned int nb_burst; 1311 unsigned int burst_stats[3]; 1312 uint16_t pktnb_stats[3]; 1313 uint16_t nb_pkt; 1314 int burst_percent[3]; 1315 1316 /* 1317 * First compute the total number of packet bursts and the 1318 * two highest numbers of bursts of the same number of packets. 1319 */ 1320 total_burst = 0; 1321 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1322 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1323 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1324 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1325 if (nb_burst == 0) 1326 continue; 1327 total_burst += nb_burst; 1328 if (nb_burst > burst_stats[0]) { 1329 burst_stats[1] = burst_stats[0]; 1330 pktnb_stats[1] = pktnb_stats[0]; 1331 burst_stats[0] = nb_burst; 1332 pktnb_stats[0] = nb_pkt; 1333 } else if (nb_burst > burst_stats[1]) { 1334 burst_stats[1] = nb_burst; 1335 pktnb_stats[1] = nb_pkt; 1336 } 1337 } 1338 if (total_burst == 0) 1339 return; 1340 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1341 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1342 burst_percent[0], (int) pktnb_stats[0]); 1343 if (burst_stats[0] == total_burst) { 1344 printf("]\n"); 1345 return; 1346 } 1347 if (burst_stats[0] + burst_stats[1] == total_burst) { 1348 printf(" + %d%% of %d pkts]\n", 1349 100 - burst_percent[0], pktnb_stats[1]); 1350 return; 1351 } 1352 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1353 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1354 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1355 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1356 return; 1357 } 1358 printf(" + %d%% of %d pkts + %d%% of others]\n", 1359 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1360 } 1361 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1362 1363 static void 1364 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats) 1365 { 1366 struct rte_port *port; 1367 uint8_t i; 1368 1369 static const char *fwd_stats_border = "----------------------"; 1370 1371 port = &ports[port_id]; 1372 printf("\n %s Forward statistics for port %-2d %s\n", 1373 fwd_stats_border, port_id, fwd_stats_border); 1374 1375 if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) { 1376 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1377 "%-"PRIu64"\n", 1378 stats->ipackets, stats->imissed, 1379 (uint64_t) (stats->ipackets + stats->imissed)); 1380 1381 if (cur_fwd_eng == &csum_fwd_engine) 1382 printf(" Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64"Bad-outer-l4csum: %-14"PRIu64"\n", 1383 port->rx_bad_ip_csum, port->rx_bad_l4_csum, 1384 port->rx_bad_outer_l4_csum); 1385 if ((stats->ierrors + stats->rx_nombuf) > 0) { 1386 printf(" RX-error: %-"PRIu64"\n", stats->ierrors); 1387 printf(" RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf); 1388 } 1389 1390 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1391 "%-"PRIu64"\n", 1392 stats->opackets, port->tx_dropped, 1393 (uint64_t) (stats->opackets + port->tx_dropped)); 1394 } 1395 else { 1396 printf(" RX-packets: %14"PRIu64" RX-dropped:%14"PRIu64" RX-total:" 1397 "%14"PRIu64"\n", 1398 stats->ipackets, stats->imissed, 1399 (uint64_t) (stats->ipackets + stats->imissed)); 1400 1401 if (cur_fwd_eng == &csum_fwd_engine) 1402 printf(" Bad-ipcsum:%14"PRIu64" Bad-l4csum:%14"PRIu64" Bad-outer-l4csum: %-14"PRIu64"\n", 1403 port->rx_bad_ip_csum, port->rx_bad_l4_csum, 1404 port->rx_bad_outer_l4_csum); 1405 if ((stats->ierrors + stats->rx_nombuf) > 0) { 1406 printf(" RX-error:%"PRIu64"\n", stats->ierrors); 1407 printf(" RX-nombufs: %14"PRIu64"\n", 1408 stats->rx_nombuf); 1409 } 1410 1411 printf(" TX-packets: %14"PRIu64" TX-dropped:%14"PRIu64" TX-total:" 1412 "%14"PRIu64"\n", 1413 stats->opackets, port->tx_dropped, 1414 (uint64_t) (stats->opackets + port->tx_dropped)); 1415 } 1416 1417 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1418 if (port->rx_stream) 1419 pkt_burst_stats_display("RX", 1420 &port->rx_stream->rx_burst_stats); 1421 if (port->tx_stream) 1422 pkt_burst_stats_display("TX", 1423 &port->tx_stream->tx_burst_stats); 1424 #endif 1425 1426 if (port->rx_queue_stats_mapping_enabled) { 1427 printf("\n"); 1428 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 1429 printf(" Stats reg %2d RX-packets:%14"PRIu64 1430 " RX-errors:%14"PRIu64 1431 " RX-bytes:%14"PRIu64"\n", 1432 i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]); 1433 } 1434 printf("\n"); 1435 } 1436 if (port->tx_queue_stats_mapping_enabled) { 1437 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 1438 printf(" Stats reg %2d TX-packets:%14"PRIu64 1439 " TX-bytes:%14"PRIu64"\n", 1440 i, stats->q_opackets[i], stats->q_obytes[i]); 1441 } 1442 } 1443 1444 printf(" %s--------------------------------%s\n", 1445 fwd_stats_border, fwd_stats_border); 1446 } 1447 1448 static void 1449 fwd_stream_stats_display(streamid_t stream_id) 1450 { 1451 struct fwd_stream *fs; 1452 static const char *fwd_top_stats_border = "-------"; 1453 1454 fs = fwd_streams[stream_id]; 1455 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1456 (fs->fwd_dropped == 0)) 1457 return; 1458 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1459 "TX Port=%2d/Queue=%2d %s\n", 1460 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1461 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1462 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1463 " TX-dropped: %-14"PRIu64, 1464 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1465 1466 /* if checksum mode */ 1467 if (cur_fwd_eng == &csum_fwd_engine) { 1468 printf(" RX- bad IP checksum: %-14"PRIu64 1469 " Rx- bad L4 checksum: %-14"PRIu64 1470 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1471 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1472 fs->rx_bad_outer_l4_csum); 1473 } else { 1474 printf("\n"); 1475 } 1476 1477 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1478 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1479 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1480 #endif 1481 } 1482 1483 static void 1484 flush_fwd_rx_queues(void) 1485 { 1486 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1487 portid_t rxp; 1488 portid_t port_id; 1489 queueid_t rxq; 1490 uint16_t nb_rx; 1491 uint16_t i; 1492 uint8_t j; 1493 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1494 uint64_t timer_period; 1495 1496 /* convert to number of cycles */ 1497 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1498 1499 for (j = 0; j < 2; j++) { 1500 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1501 for (rxq = 0; rxq < nb_rxq; rxq++) { 1502 port_id = fwd_ports_ids[rxp]; 1503 /** 1504 * testpmd can stuck in the below do while loop 1505 * if rte_eth_rx_burst() always returns nonzero 1506 * packets. So timer is added to exit this loop 1507 * after 1sec timer expiry. 1508 */ 1509 prev_tsc = rte_rdtsc(); 1510 do { 1511 nb_rx = rte_eth_rx_burst(port_id, rxq, 1512 pkts_burst, MAX_PKT_BURST); 1513 for (i = 0; i < nb_rx; i++) 1514 rte_pktmbuf_free(pkts_burst[i]); 1515 1516 cur_tsc = rte_rdtsc(); 1517 diff_tsc = cur_tsc - prev_tsc; 1518 timer_tsc += diff_tsc; 1519 } while ((nb_rx > 0) && 1520 (timer_tsc < timer_period)); 1521 timer_tsc = 0; 1522 } 1523 } 1524 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1525 } 1526 } 1527 1528 static void 1529 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1530 { 1531 struct fwd_stream **fsm; 1532 streamid_t nb_fs; 1533 streamid_t sm_id; 1534 #ifdef RTE_LIBRTE_BITRATE 1535 uint64_t tics_per_1sec; 1536 uint64_t tics_datum; 1537 uint64_t tics_current; 1538 uint16_t i, cnt_ports; 1539 1540 cnt_ports = nb_ports; 1541 tics_datum = rte_rdtsc(); 1542 tics_per_1sec = rte_get_timer_hz(); 1543 #endif 1544 fsm = &fwd_streams[fc->stream_idx]; 1545 nb_fs = fc->stream_nb; 1546 do { 1547 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1548 (*pkt_fwd)(fsm[sm_id]); 1549 #ifdef RTE_LIBRTE_BITRATE 1550 if (bitrate_enabled != 0 && 1551 bitrate_lcore_id == rte_lcore_id()) { 1552 tics_current = rte_rdtsc(); 1553 if (tics_current - tics_datum >= tics_per_1sec) { 1554 /* Periodic bitrate calculation */ 1555 for (i = 0; i < cnt_ports; i++) 1556 rte_stats_bitrate_calc(bitrate_data, 1557 ports_ids[i]); 1558 tics_datum = tics_current; 1559 } 1560 } 1561 #endif 1562 #ifdef RTE_LIBRTE_LATENCY_STATS 1563 if (latencystats_enabled != 0 && 1564 latencystats_lcore_id == rte_lcore_id()) 1565 rte_latencystats_update(); 1566 #endif 1567 1568 } while (! fc->stopped); 1569 } 1570 1571 static int 1572 start_pkt_forward_on_core(void *fwd_arg) 1573 { 1574 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1575 cur_fwd_config.fwd_eng->packet_fwd); 1576 return 0; 1577 } 1578 1579 /* 1580 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1581 * Used to start communication flows in network loopback test configurations. 1582 */ 1583 static int 1584 run_one_txonly_burst_on_core(void *fwd_arg) 1585 { 1586 struct fwd_lcore *fwd_lc; 1587 struct fwd_lcore tmp_lcore; 1588 1589 fwd_lc = (struct fwd_lcore *) fwd_arg; 1590 tmp_lcore = *fwd_lc; 1591 tmp_lcore.stopped = 1; 1592 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1593 return 0; 1594 } 1595 1596 /* 1597 * Launch packet forwarding: 1598 * - Setup per-port forwarding context. 1599 * - launch logical cores with their forwarding configuration. 1600 */ 1601 static void 1602 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1603 { 1604 port_fwd_begin_t port_fwd_begin; 1605 unsigned int i; 1606 unsigned int lc_id; 1607 int diag; 1608 1609 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1610 if (port_fwd_begin != NULL) { 1611 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1612 (*port_fwd_begin)(fwd_ports_ids[i]); 1613 } 1614 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1615 lc_id = fwd_lcores_cpuids[i]; 1616 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1617 fwd_lcores[i]->stopped = 0; 1618 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1619 fwd_lcores[i], lc_id); 1620 if (diag != 0) 1621 printf("launch lcore %u failed - diag=%d\n", 1622 lc_id, diag); 1623 } 1624 } 1625 } 1626 1627 /* 1628 * Launch packet forwarding configuration. 1629 */ 1630 void 1631 start_packet_forwarding(int with_tx_first) 1632 { 1633 port_fwd_begin_t port_fwd_begin; 1634 port_fwd_end_t port_fwd_end; 1635 struct rte_port *port; 1636 unsigned int i; 1637 portid_t pt_id; 1638 streamid_t sm_id; 1639 1640 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1641 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1642 1643 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1644 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1645 1646 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1647 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1648 (!nb_rxq || !nb_txq)) 1649 rte_exit(EXIT_FAILURE, 1650 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1651 cur_fwd_eng->fwd_mode_name); 1652 1653 if (all_ports_started() == 0) { 1654 printf("Not all ports were started\n"); 1655 return; 1656 } 1657 if (test_done == 0) { 1658 printf("Packet forwarding already started\n"); 1659 return; 1660 } 1661 1662 1663 if(dcb_test) { 1664 for (i = 0; i < nb_fwd_ports; i++) { 1665 pt_id = fwd_ports_ids[i]; 1666 port = &ports[pt_id]; 1667 if (!port->dcb_flag) { 1668 printf("In DCB mode, all forwarding ports must " 1669 "be configured in this mode.\n"); 1670 return; 1671 } 1672 } 1673 if (nb_fwd_lcores == 1) { 1674 printf("In DCB mode,the nb forwarding cores " 1675 "should be larger than 1.\n"); 1676 return; 1677 } 1678 } 1679 test_done = 0; 1680 1681 fwd_config_setup(); 1682 1683 if(!no_flush_rx) 1684 flush_fwd_rx_queues(); 1685 1686 pkt_fwd_config_display(&cur_fwd_config); 1687 rxtx_config_display(); 1688 1689 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1690 pt_id = fwd_ports_ids[i]; 1691 port = &ports[pt_id]; 1692 rte_eth_stats_get(pt_id, &port->stats); 1693 port->tx_dropped = 0; 1694 1695 map_port_queue_stats_mapping_registers(pt_id, port); 1696 } 1697 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1698 fwd_streams[sm_id]->rx_packets = 0; 1699 fwd_streams[sm_id]->tx_packets = 0; 1700 fwd_streams[sm_id]->fwd_dropped = 0; 1701 fwd_streams[sm_id]->rx_bad_ip_csum = 0; 1702 fwd_streams[sm_id]->rx_bad_l4_csum = 0; 1703 fwd_streams[sm_id]->rx_bad_outer_l4_csum = 0; 1704 1705 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1706 memset(&fwd_streams[sm_id]->rx_burst_stats, 0, 1707 sizeof(fwd_streams[sm_id]->rx_burst_stats)); 1708 memset(&fwd_streams[sm_id]->tx_burst_stats, 0, 1709 sizeof(fwd_streams[sm_id]->tx_burst_stats)); 1710 #endif 1711 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1712 fwd_streams[sm_id]->core_cycles = 0; 1713 #endif 1714 } 1715 if (with_tx_first) { 1716 port_fwd_begin = tx_only_engine.port_fwd_begin; 1717 if (port_fwd_begin != NULL) { 1718 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1719 (*port_fwd_begin)(fwd_ports_ids[i]); 1720 } 1721 while (with_tx_first--) { 1722 launch_packet_forwarding( 1723 run_one_txonly_burst_on_core); 1724 rte_eal_mp_wait_lcore(); 1725 } 1726 port_fwd_end = tx_only_engine.port_fwd_end; 1727 if (port_fwd_end != NULL) { 1728 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1729 (*port_fwd_end)(fwd_ports_ids[i]); 1730 } 1731 } 1732 launch_packet_forwarding(start_pkt_forward_on_core); 1733 } 1734 1735 void 1736 stop_packet_forwarding(void) 1737 { 1738 struct rte_eth_stats stats; 1739 struct rte_port *port; 1740 port_fwd_end_t port_fwd_end; 1741 int i; 1742 portid_t pt_id; 1743 streamid_t sm_id; 1744 lcoreid_t lc_id; 1745 uint64_t total_recv; 1746 uint64_t total_xmit; 1747 uint64_t total_rx_dropped; 1748 uint64_t total_tx_dropped; 1749 uint64_t total_rx_nombuf; 1750 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1751 uint64_t fwd_cycles; 1752 #endif 1753 1754 static const char *acc_stats_border = "+++++++++++++++"; 1755 1756 if (test_done) { 1757 printf("Packet forwarding not started\n"); 1758 return; 1759 } 1760 printf("Telling cores to stop..."); 1761 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1762 fwd_lcores[lc_id]->stopped = 1; 1763 printf("\nWaiting for lcores to finish...\n"); 1764 rte_eal_mp_wait_lcore(); 1765 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1766 if (port_fwd_end != NULL) { 1767 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1768 pt_id = fwd_ports_ids[i]; 1769 (*port_fwd_end)(pt_id); 1770 } 1771 } 1772 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1773 fwd_cycles = 0; 1774 #endif 1775 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1776 struct fwd_stream *fs = fwd_streams[sm_id]; 1777 1778 if (cur_fwd_config.nb_fwd_streams > 1779 cur_fwd_config.nb_fwd_ports) { 1780 fwd_stream_stats_display(sm_id); 1781 ports[fs->tx_port].tx_stream = NULL; 1782 ports[fs->rx_port].rx_stream = NULL; 1783 } else { 1784 ports[fs->tx_port].tx_stream = fs; 1785 ports[fs->rx_port].rx_stream = fs; 1786 } 1787 ports[fs->tx_port].tx_dropped += fs->fwd_dropped; 1788 ports[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1789 ports[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1790 ports[fs->rx_port].rx_bad_outer_l4_csum += 1791 fs->rx_bad_outer_l4_csum; 1792 1793 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1794 fwd_cycles = (uint64_t) (fwd_cycles + 1795 fwd_streams[sm_id]->core_cycles); 1796 #endif 1797 } 1798 total_recv = 0; 1799 total_xmit = 0; 1800 total_rx_dropped = 0; 1801 total_tx_dropped = 0; 1802 total_rx_nombuf = 0; 1803 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1804 pt_id = fwd_ports_ids[i]; 1805 1806 port = &ports[pt_id]; 1807 rte_eth_stats_get(pt_id, &stats); 1808 stats.ipackets -= port->stats.ipackets; 1809 port->stats.ipackets = 0; 1810 stats.opackets -= port->stats.opackets; 1811 port->stats.opackets = 0; 1812 stats.ibytes -= port->stats.ibytes; 1813 port->stats.ibytes = 0; 1814 stats.obytes -= port->stats.obytes; 1815 port->stats.obytes = 0; 1816 stats.imissed -= port->stats.imissed; 1817 port->stats.imissed = 0; 1818 stats.oerrors -= port->stats.oerrors; 1819 port->stats.oerrors = 0; 1820 stats.rx_nombuf -= port->stats.rx_nombuf; 1821 port->stats.rx_nombuf = 0; 1822 1823 total_recv += stats.ipackets; 1824 total_xmit += stats.opackets; 1825 total_rx_dropped += stats.imissed; 1826 total_tx_dropped += port->tx_dropped; 1827 total_rx_nombuf += stats.rx_nombuf; 1828 1829 fwd_port_stats_display(pt_id, &stats); 1830 } 1831 1832 printf("\n %s Accumulated forward statistics for all ports" 1833 "%s\n", 1834 acc_stats_border, acc_stats_border); 1835 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1836 "%-"PRIu64"\n" 1837 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1838 "%-"PRIu64"\n", 1839 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1840 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1841 if (total_rx_nombuf > 0) 1842 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1843 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1844 "%s\n", 1845 acc_stats_border, acc_stats_border); 1846 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1847 if (total_recv > 0) 1848 printf("\n CPU cycles/packet=%u (total cycles=" 1849 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1850 (unsigned int)(fwd_cycles / total_recv), 1851 fwd_cycles, total_recv); 1852 #endif 1853 printf("\nDone.\n"); 1854 test_done = 1; 1855 } 1856 1857 void 1858 dev_set_link_up(portid_t pid) 1859 { 1860 if (rte_eth_dev_set_link_up(pid) < 0) 1861 printf("\nSet link up fail.\n"); 1862 } 1863 1864 void 1865 dev_set_link_down(portid_t pid) 1866 { 1867 if (rte_eth_dev_set_link_down(pid) < 0) 1868 printf("\nSet link down fail.\n"); 1869 } 1870 1871 static int 1872 all_ports_started(void) 1873 { 1874 portid_t pi; 1875 struct rte_port *port; 1876 1877 RTE_ETH_FOREACH_DEV(pi) { 1878 port = &ports[pi]; 1879 /* Check if there is a port which is not started */ 1880 if ((port->port_status != RTE_PORT_STARTED) && 1881 (port->slave_flag == 0)) 1882 return 0; 1883 } 1884 1885 /* No port is not started */ 1886 return 1; 1887 } 1888 1889 int 1890 port_is_stopped(portid_t port_id) 1891 { 1892 struct rte_port *port = &ports[port_id]; 1893 1894 if ((port->port_status != RTE_PORT_STOPPED) && 1895 (port->slave_flag == 0)) 1896 return 0; 1897 return 1; 1898 } 1899 1900 int 1901 all_ports_stopped(void) 1902 { 1903 portid_t pi; 1904 1905 RTE_ETH_FOREACH_DEV(pi) { 1906 if (!port_is_stopped(pi)) 1907 return 0; 1908 } 1909 1910 return 1; 1911 } 1912 1913 int 1914 port_is_started(portid_t port_id) 1915 { 1916 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1917 return 0; 1918 1919 if (ports[port_id].port_status != RTE_PORT_STARTED) 1920 return 0; 1921 1922 return 1; 1923 } 1924 1925 int 1926 start_port(portid_t pid) 1927 { 1928 int diag, need_check_link_status = -1; 1929 portid_t pi; 1930 queueid_t qi; 1931 struct rte_port *port; 1932 struct ether_addr mac_addr; 1933 1934 if (port_id_is_invalid(pid, ENABLED_WARN)) 1935 return 0; 1936 1937 if(dcb_config) 1938 dcb_test = 1; 1939 RTE_ETH_FOREACH_DEV(pi) { 1940 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1941 continue; 1942 1943 need_check_link_status = 0; 1944 port = &ports[pi]; 1945 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1946 RTE_PORT_HANDLING) == 0) { 1947 printf("Port %d is now not stopped\n", pi); 1948 continue; 1949 } 1950 1951 if (port->need_reconfig > 0) { 1952 port->need_reconfig = 0; 1953 1954 if (flow_isolate_all) { 1955 int ret = port_flow_isolate(pi, 1); 1956 if (ret) { 1957 printf("Failed to apply isolated" 1958 " mode on port %d\n", pi); 1959 return -1; 1960 } 1961 } 1962 configure_rxtx_dump_callbacks(0); 1963 printf("Configuring Port %d (socket %u)\n", pi, 1964 port->socket_id); 1965 /* configure port */ 1966 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1967 &(port->dev_conf)); 1968 if (diag != 0) { 1969 if (rte_atomic16_cmpset(&(port->port_status), 1970 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1971 printf("Port %d can not be set back " 1972 "to stopped\n", pi); 1973 printf("Fail to configure port %d\n", pi); 1974 /* try to reconfigure port next time */ 1975 port->need_reconfig = 1; 1976 return -1; 1977 } 1978 } 1979 if (port->need_reconfig_queues > 0) { 1980 port->need_reconfig_queues = 0; 1981 /* setup tx queues */ 1982 for (qi = 0; qi < nb_txq; qi++) { 1983 if ((numa_support) && 1984 (txring_numa[pi] != NUMA_NO_CONFIG)) 1985 diag = rte_eth_tx_queue_setup(pi, qi, 1986 port->nb_tx_desc[qi], 1987 txring_numa[pi], 1988 &(port->tx_conf[qi])); 1989 else 1990 diag = rte_eth_tx_queue_setup(pi, qi, 1991 port->nb_tx_desc[qi], 1992 port->socket_id, 1993 &(port->tx_conf[qi])); 1994 1995 if (diag == 0) 1996 continue; 1997 1998 /* Fail to setup tx queue, return */ 1999 if (rte_atomic16_cmpset(&(port->port_status), 2000 RTE_PORT_HANDLING, 2001 RTE_PORT_STOPPED) == 0) 2002 printf("Port %d can not be set back " 2003 "to stopped\n", pi); 2004 printf("Fail to configure port %d tx queues\n", 2005 pi); 2006 /* try to reconfigure queues next time */ 2007 port->need_reconfig_queues = 1; 2008 return -1; 2009 } 2010 for (qi = 0; qi < nb_rxq; qi++) { 2011 /* setup rx queues */ 2012 if ((numa_support) && 2013 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2014 struct rte_mempool * mp = 2015 mbuf_pool_find(rxring_numa[pi]); 2016 if (mp == NULL) { 2017 printf("Failed to setup RX queue:" 2018 "No mempool allocation" 2019 " on the socket %d\n", 2020 rxring_numa[pi]); 2021 return -1; 2022 } 2023 2024 diag = rte_eth_rx_queue_setup(pi, qi, 2025 port->nb_rx_desc[qi], 2026 rxring_numa[pi], 2027 &(port->rx_conf[qi]), 2028 mp); 2029 } else { 2030 struct rte_mempool *mp = 2031 mbuf_pool_find(port->socket_id); 2032 if (mp == NULL) { 2033 printf("Failed to setup RX queue:" 2034 "No mempool allocation" 2035 " on the socket %d\n", 2036 port->socket_id); 2037 return -1; 2038 } 2039 diag = rte_eth_rx_queue_setup(pi, qi, 2040 port->nb_rx_desc[qi], 2041 port->socket_id, 2042 &(port->rx_conf[qi]), 2043 mp); 2044 } 2045 if (diag == 0) 2046 continue; 2047 2048 /* Fail to setup rx queue, return */ 2049 if (rte_atomic16_cmpset(&(port->port_status), 2050 RTE_PORT_HANDLING, 2051 RTE_PORT_STOPPED) == 0) 2052 printf("Port %d can not be set back " 2053 "to stopped\n", pi); 2054 printf("Fail to configure port %d rx queues\n", 2055 pi); 2056 /* try to reconfigure queues next time */ 2057 port->need_reconfig_queues = 1; 2058 return -1; 2059 } 2060 } 2061 configure_rxtx_dump_callbacks(verbose_level); 2062 /* start port */ 2063 if (rte_eth_dev_start(pi) < 0) { 2064 printf("Fail to start port %d\n", pi); 2065 2066 /* Fail to setup rx queue, return */ 2067 if (rte_atomic16_cmpset(&(port->port_status), 2068 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2069 printf("Port %d can not be set back to " 2070 "stopped\n", pi); 2071 continue; 2072 } 2073 2074 if (rte_atomic16_cmpset(&(port->port_status), 2075 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2076 printf("Port %d can not be set into started\n", pi); 2077 2078 rte_eth_macaddr_get(pi, &mac_addr); 2079 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2080 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2081 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2082 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2083 2084 /* at least one port started, need checking link status */ 2085 need_check_link_status = 1; 2086 } 2087 2088 if (need_check_link_status == 1 && !no_link_check) 2089 check_all_ports_link_status(RTE_PORT_ALL); 2090 else if (need_check_link_status == 0) 2091 printf("Please stop the ports first\n"); 2092 2093 printf("Done\n"); 2094 return 0; 2095 } 2096 2097 void 2098 stop_port(portid_t pid) 2099 { 2100 portid_t pi; 2101 struct rte_port *port; 2102 int need_check_link_status = 0; 2103 2104 if (dcb_test) { 2105 dcb_test = 0; 2106 dcb_config = 0; 2107 } 2108 2109 if (port_id_is_invalid(pid, ENABLED_WARN)) 2110 return; 2111 2112 printf("Stopping ports...\n"); 2113 2114 RTE_ETH_FOREACH_DEV(pi) { 2115 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2116 continue; 2117 2118 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2119 printf("Please remove port %d from forwarding configuration.\n", pi); 2120 continue; 2121 } 2122 2123 if (port_is_bonding_slave(pi)) { 2124 printf("Please remove port %d from bonded device.\n", pi); 2125 continue; 2126 } 2127 2128 port = &ports[pi]; 2129 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2130 RTE_PORT_HANDLING) == 0) 2131 continue; 2132 2133 rte_eth_dev_stop(pi); 2134 2135 if (rte_atomic16_cmpset(&(port->port_status), 2136 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2137 printf("Port %d can not be set into stopped\n", pi); 2138 need_check_link_status = 1; 2139 } 2140 if (need_check_link_status && !no_link_check) 2141 check_all_ports_link_status(RTE_PORT_ALL); 2142 2143 printf("Done\n"); 2144 } 2145 2146 static void 2147 remove_invalid_ports_in(portid_t *array, portid_t *total) 2148 { 2149 portid_t i; 2150 portid_t new_total = 0; 2151 2152 for (i = 0; i < *total; i++) 2153 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2154 array[new_total] = array[i]; 2155 new_total++; 2156 } 2157 *total = new_total; 2158 } 2159 2160 static void 2161 remove_invalid_ports(void) 2162 { 2163 remove_invalid_ports_in(ports_ids, &nb_ports); 2164 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2165 nb_cfg_ports = nb_fwd_ports; 2166 } 2167 2168 void 2169 close_port(portid_t pid) 2170 { 2171 portid_t pi; 2172 struct rte_port *port; 2173 2174 if (port_id_is_invalid(pid, ENABLED_WARN)) 2175 return; 2176 2177 printf("Closing ports...\n"); 2178 2179 RTE_ETH_FOREACH_DEV(pi) { 2180 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2181 continue; 2182 2183 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2184 printf("Please remove port %d from forwarding configuration.\n", pi); 2185 continue; 2186 } 2187 2188 if (port_is_bonding_slave(pi)) { 2189 printf("Please remove port %d from bonded device.\n", pi); 2190 continue; 2191 } 2192 2193 port = &ports[pi]; 2194 if (rte_atomic16_cmpset(&(port->port_status), 2195 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2196 printf("Port %d is already closed\n", pi); 2197 continue; 2198 } 2199 2200 if (rte_atomic16_cmpset(&(port->port_status), 2201 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2202 printf("Port %d is now not stopped\n", pi); 2203 continue; 2204 } 2205 2206 if (port->flow_list) 2207 port_flow_flush(pi); 2208 rte_eth_dev_close(pi); 2209 2210 remove_invalid_ports(); 2211 2212 if (rte_atomic16_cmpset(&(port->port_status), 2213 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2214 printf("Port %d cannot be set to closed\n", pi); 2215 } 2216 2217 printf("Done\n"); 2218 } 2219 2220 void 2221 reset_port(portid_t pid) 2222 { 2223 int diag; 2224 portid_t pi; 2225 struct rte_port *port; 2226 2227 if (port_id_is_invalid(pid, ENABLED_WARN)) 2228 return; 2229 2230 printf("Resetting ports...\n"); 2231 2232 RTE_ETH_FOREACH_DEV(pi) { 2233 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2234 continue; 2235 2236 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2237 printf("Please remove port %d from forwarding " 2238 "configuration.\n", pi); 2239 continue; 2240 } 2241 2242 if (port_is_bonding_slave(pi)) { 2243 printf("Please remove port %d from bonded device.\n", 2244 pi); 2245 continue; 2246 } 2247 2248 diag = rte_eth_dev_reset(pi); 2249 if (diag == 0) { 2250 port = &ports[pi]; 2251 port->need_reconfig = 1; 2252 port->need_reconfig_queues = 1; 2253 } else { 2254 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2255 } 2256 } 2257 2258 printf("Done\n"); 2259 } 2260 2261 void 2262 attach_port(char *identifier) 2263 { 2264 portid_t pi; 2265 struct rte_dev_iterator iterator; 2266 2267 printf("Attaching a new port...\n"); 2268 2269 if (identifier == NULL) { 2270 printf("Invalid parameters are specified\n"); 2271 return; 2272 } 2273 2274 if (rte_dev_probe(identifier) != 0) { 2275 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2276 return; 2277 } 2278 2279 /* first attach mode: event */ 2280 if (setup_on_probe_event) { 2281 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2282 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2283 if (ports[pi].port_status == RTE_PORT_HANDLING && 2284 ports[pi].need_setup != 0) 2285 setup_attached_port(pi); 2286 return; 2287 } 2288 2289 /* second attach mode: iterator */ 2290 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2291 /* setup ports matching the devargs used for probing */ 2292 if (port_is_forwarding(pi)) 2293 continue; /* port was already attached before */ 2294 setup_attached_port(pi); 2295 } 2296 } 2297 2298 static void 2299 setup_attached_port(portid_t pi) 2300 { 2301 unsigned int socket_id; 2302 2303 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2304 /* if socket_id is invalid, set to the first available socket. */ 2305 if (check_socket_id(socket_id) < 0) 2306 socket_id = socket_ids[0]; 2307 reconfig(pi, socket_id); 2308 rte_eth_promiscuous_enable(pi); 2309 2310 ports_ids[nb_ports++] = pi; 2311 fwd_ports_ids[nb_fwd_ports++] = pi; 2312 nb_cfg_ports = nb_fwd_ports; 2313 ports[pi].need_setup = 0; 2314 ports[pi].port_status = RTE_PORT_STOPPED; 2315 2316 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2317 printf("Done\n"); 2318 } 2319 2320 void 2321 detach_port_device(portid_t port_id) 2322 { 2323 struct rte_device *dev; 2324 portid_t sibling; 2325 2326 printf("Removing a device...\n"); 2327 2328 dev = rte_eth_devices[port_id].device; 2329 if (dev == NULL) { 2330 printf("Device already removed\n"); 2331 return; 2332 } 2333 2334 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2335 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2336 printf("Port not stopped\n"); 2337 return; 2338 } 2339 printf("Port was not closed\n"); 2340 if (ports[port_id].flow_list) 2341 port_flow_flush(port_id); 2342 } 2343 2344 if (rte_dev_remove(dev) != 0) { 2345 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2346 return; 2347 } 2348 2349 for (sibling = 0; sibling < RTE_MAX_ETHPORTS; sibling++) { 2350 if (rte_eth_devices[sibling].device != dev) 2351 continue; 2352 /* reset mapping between old ports and removed device */ 2353 rte_eth_devices[sibling].device = NULL; 2354 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2355 /* sibling ports are forced to be closed */ 2356 ports[sibling].port_status = RTE_PORT_CLOSED; 2357 printf("Port %u is closed\n", sibling); 2358 } 2359 } 2360 2361 remove_invalid_ports(); 2362 2363 printf("Device of port %u is detached\n", port_id); 2364 printf("Now total ports is %d\n", nb_ports); 2365 printf("Done\n"); 2366 return; 2367 } 2368 2369 void 2370 pmd_test_exit(void) 2371 { 2372 struct rte_device *device; 2373 portid_t pt_id; 2374 int ret; 2375 2376 if (test_done == 0) 2377 stop_packet_forwarding(); 2378 2379 if (ports != NULL) { 2380 no_link_check = 1; 2381 RTE_ETH_FOREACH_DEV(pt_id) { 2382 printf("\nStopping port %d...\n", pt_id); 2383 fflush(stdout); 2384 stop_port(pt_id); 2385 } 2386 RTE_ETH_FOREACH_DEV(pt_id) { 2387 printf("\nShutting down port %d...\n", pt_id); 2388 fflush(stdout); 2389 close_port(pt_id); 2390 2391 /* 2392 * This is a workaround to fix a virtio-user issue that 2393 * requires to call clean-up routine to remove existing 2394 * socket. 2395 * This workaround valid only for testpmd, needs a fix 2396 * valid for all applications. 2397 * TODO: Implement proper resource cleanup 2398 */ 2399 device = rte_eth_devices[pt_id].device; 2400 if (device && !strcmp(device->driver->name, "net_virtio_user")) 2401 detach_port_device(pt_id); 2402 } 2403 } 2404 2405 if (hot_plug) { 2406 ret = rte_dev_event_monitor_stop(); 2407 if (ret) { 2408 RTE_LOG(ERR, EAL, 2409 "fail to stop device event monitor."); 2410 return; 2411 } 2412 2413 ret = rte_dev_event_callback_unregister(NULL, 2414 dev_event_callback, NULL); 2415 if (ret < 0) { 2416 RTE_LOG(ERR, EAL, 2417 "fail to unregister device event callback.\n"); 2418 return; 2419 } 2420 2421 ret = rte_dev_hotplug_handle_disable(); 2422 if (ret) { 2423 RTE_LOG(ERR, EAL, 2424 "fail to disable hotplug handling.\n"); 2425 return; 2426 } 2427 } 2428 2429 printf("\nBye...\n"); 2430 } 2431 2432 typedef void (*cmd_func_t)(void); 2433 struct pmd_test_command { 2434 const char *cmd_name; 2435 cmd_func_t cmd_func; 2436 }; 2437 2438 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2439 2440 /* Check the link status of all ports in up to 9s, and print them finally */ 2441 static void 2442 check_all_ports_link_status(uint32_t port_mask) 2443 { 2444 #define CHECK_INTERVAL 100 /* 100ms */ 2445 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2446 portid_t portid; 2447 uint8_t count, all_ports_up, print_flag = 0; 2448 struct rte_eth_link link; 2449 2450 printf("Checking link statuses...\n"); 2451 fflush(stdout); 2452 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2453 all_ports_up = 1; 2454 RTE_ETH_FOREACH_DEV(portid) { 2455 if ((port_mask & (1 << portid)) == 0) 2456 continue; 2457 memset(&link, 0, sizeof(link)); 2458 rte_eth_link_get_nowait(portid, &link); 2459 /* print link status if flag set */ 2460 if (print_flag == 1) { 2461 if (link.link_status) 2462 printf( 2463 "Port%d Link Up. speed %u Mbps- %s\n", 2464 portid, link.link_speed, 2465 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2466 ("full-duplex") : ("half-duplex\n")); 2467 else 2468 printf("Port %d Link Down\n", portid); 2469 continue; 2470 } 2471 /* clear all_ports_up flag if any link down */ 2472 if (link.link_status == ETH_LINK_DOWN) { 2473 all_ports_up = 0; 2474 break; 2475 } 2476 } 2477 /* after finally printing all link status, get out */ 2478 if (print_flag == 1) 2479 break; 2480 2481 if (all_ports_up == 0) { 2482 fflush(stdout); 2483 rte_delay_ms(CHECK_INTERVAL); 2484 } 2485 2486 /* set the print_flag if all ports up or timeout */ 2487 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2488 print_flag = 1; 2489 } 2490 2491 if (lsc_interrupt) 2492 break; 2493 } 2494 } 2495 2496 /* 2497 * This callback is for remove a port for a device. It has limitation because 2498 * it is not for multiple port removal for a device. 2499 * TODO: the device detach invoke will plan to be removed from user side to 2500 * eal. And convert all PMDs to free port resources on ether device closing. 2501 */ 2502 static void 2503 rmv_port_callback(void *arg) 2504 { 2505 int need_to_start = 0; 2506 int org_no_link_check = no_link_check; 2507 portid_t port_id = (intptr_t)arg; 2508 2509 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2510 2511 if (!test_done && port_is_forwarding(port_id)) { 2512 need_to_start = 1; 2513 stop_packet_forwarding(); 2514 } 2515 no_link_check = 1; 2516 stop_port(port_id); 2517 no_link_check = org_no_link_check; 2518 close_port(port_id); 2519 detach_port_device(port_id); 2520 if (need_to_start) 2521 start_packet_forwarding(0); 2522 } 2523 2524 /* This function is used by the interrupt thread */ 2525 static int 2526 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2527 void *ret_param) 2528 { 2529 RTE_SET_USED(param); 2530 RTE_SET_USED(ret_param); 2531 2532 if (type >= RTE_ETH_EVENT_MAX) { 2533 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2534 port_id, __func__, type); 2535 fflush(stderr); 2536 } else if (event_print_mask & (UINT32_C(1) << type)) { 2537 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2538 eth_event_desc[type]); 2539 fflush(stdout); 2540 } 2541 2542 switch (type) { 2543 case RTE_ETH_EVENT_NEW: 2544 ports[port_id].need_setup = 1; 2545 ports[port_id].port_status = RTE_PORT_HANDLING; 2546 break; 2547 case RTE_ETH_EVENT_INTR_RMV: 2548 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2549 break; 2550 if (rte_eal_alarm_set(100000, 2551 rmv_port_callback, (void *)(intptr_t)port_id)) 2552 fprintf(stderr, "Could not set up deferred device removal\n"); 2553 break; 2554 default: 2555 break; 2556 } 2557 return 0; 2558 } 2559 2560 static int 2561 register_eth_event_callback(void) 2562 { 2563 int ret; 2564 enum rte_eth_event_type event; 2565 2566 for (event = RTE_ETH_EVENT_UNKNOWN; 2567 event < RTE_ETH_EVENT_MAX; event++) { 2568 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2569 event, 2570 eth_event_callback, 2571 NULL); 2572 if (ret != 0) { 2573 TESTPMD_LOG(ERR, "Failed to register callback for " 2574 "%s event\n", eth_event_desc[event]); 2575 return -1; 2576 } 2577 } 2578 2579 return 0; 2580 } 2581 2582 /* This function is used by the interrupt thread */ 2583 static void 2584 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2585 __rte_unused void *arg) 2586 { 2587 uint16_t port_id; 2588 int ret; 2589 2590 if (type >= RTE_DEV_EVENT_MAX) { 2591 fprintf(stderr, "%s called upon invalid event %d\n", 2592 __func__, type); 2593 fflush(stderr); 2594 } 2595 2596 switch (type) { 2597 case RTE_DEV_EVENT_REMOVE: 2598 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2599 device_name); 2600 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2601 if (ret) { 2602 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2603 device_name); 2604 return; 2605 } 2606 /* 2607 * Because the user's callback is invoked in eal interrupt 2608 * callback, the interrupt callback need to be finished before 2609 * it can be unregistered when detaching device. So finish 2610 * callback soon and use a deferred removal to detach device 2611 * is need. It is a workaround, once the device detaching be 2612 * moved into the eal in the future, the deferred removal could 2613 * be deleted. 2614 */ 2615 if (rte_eal_alarm_set(100000, 2616 rmv_port_callback, (void *)(intptr_t)port_id)) 2617 RTE_LOG(ERR, EAL, 2618 "Could not set up deferred device removal\n"); 2619 break; 2620 case RTE_DEV_EVENT_ADD: 2621 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2622 device_name); 2623 /* TODO: After finish kernel driver binding, 2624 * begin to attach port. 2625 */ 2626 break; 2627 default: 2628 break; 2629 } 2630 } 2631 2632 static int 2633 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2634 { 2635 uint16_t i; 2636 int diag; 2637 uint8_t mapping_found = 0; 2638 2639 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2640 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2641 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2642 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2643 tx_queue_stats_mappings[i].queue_id, 2644 tx_queue_stats_mappings[i].stats_counter_id); 2645 if (diag != 0) 2646 return diag; 2647 mapping_found = 1; 2648 } 2649 } 2650 if (mapping_found) 2651 port->tx_queue_stats_mapping_enabled = 1; 2652 return 0; 2653 } 2654 2655 static int 2656 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2657 { 2658 uint16_t i; 2659 int diag; 2660 uint8_t mapping_found = 0; 2661 2662 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2663 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2664 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2665 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2666 rx_queue_stats_mappings[i].queue_id, 2667 rx_queue_stats_mappings[i].stats_counter_id); 2668 if (diag != 0) 2669 return diag; 2670 mapping_found = 1; 2671 } 2672 } 2673 if (mapping_found) 2674 port->rx_queue_stats_mapping_enabled = 1; 2675 return 0; 2676 } 2677 2678 static void 2679 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2680 { 2681 int diag = 0; 2682 2683 diag = set_tx_queue_stats_mapping_registers(pi, port); 2684 if (diag != 0) { 2685 if (diag == -ENOTSUP) { 2686 port->tx_queue_stats_mapping_enabled = 0; 2687 printf("TX queue stats mapping not supported port id=%d\n", pi); 2688 } 2689 else 2690 rte_exit(EXIT_FAILURE, 2691 "set_tx_queue_stats_mapping_registers " 2692 "failed for port id=%d diag=%d\n", 2693 pi, diag); 2694 } 2695 2696 diag = set_rx_queue_stats_mapping_registers(pi, port); 2697 if (diag != 0) { 2698 if (diag == -ENOTSUP) { 2699 port->rx_queue_stats_mapping_enabled = 0; 2700 printf("RX queue stats mapping not supported port id=%d\n", pi); 2701 } 2702 else 2703 rte_exit(EXIT_FAILURE, 2704 "set_rx_queue_stats_mapping_registers " 2705 "failed for port id=%d diag=%d\n", 2706 pi, diag); 2707 } 2708 } 2709 2710 static void 2711 rxtx_port_config(struct rte_port *port) 2712 { 2713 uint16_t qid; 2714 2715 for (qid = 0; qid < nb_rxq; qid++) { 2716 port->rx_conf[qid] = port->dev_info.default_rxconf; 2717 2718 /* Check if any Rx parameters have been passed */ 2719 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2720 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2721 2722 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2723 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2724 2725 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2726 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2727 2728 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2729 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2730 2731 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2732 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2733 2734 port->nb_rx_desc[qid] = nb_rxd; 2735 } 2736 2737 for (qid = 0; qid < nb_txq; qid++) { 2738 port->tx_conf[qid] = port->dev_info.default_txconf; 2739 2740 /* Check if any Tx parameters have been passed */ 2741 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2742 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2743 2744 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2745 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2746 2747 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2748 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2749 2750 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2751 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2752 2753 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2754 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2755 2756 port->nb_tx_desc[qid] = nb_txd; 2757 } 2758 } 2759 2760 void 2761 init_port_config(void) 2762 { 2763 portid_t pid; 2764 struct rte_port *port; 2765 2766 RTE_ETH_FOREACH_DEV(pid) { 2767 port = &ports[pid]; 2768 port->dev_conf.fdir_conf = fdir_conf; 2769 rte_eth_dev_info_get(pid, &port->dev_info); 2770 if (nb_rxq > 1) { 2771 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2772 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2773 rss_hf & port->dev_info.flow_type_rss_offloads; 2774 } else { 2775 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2776 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2777 } 2778 2779 if (port->dcb_flag == 0) { 2780 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2781 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2782 else 2783 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2784 } 2785 2786 rxtx_port_config(port); 2787 2788 rte_eth_macaddr_get(pid, &port->eth_addr); 2789 2790 map_port_queue_stats_mapping_registers(pid, port); 2791 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2792 rte_pmd_ixgbe_bypass_init(pid); 2793 #endif 2794 2795 if (lsc_interrupt && 2796 (rte_eth_devices[pid].data->dev_flags & 2797 RTE_ETH_DEV_INTR_LSC)) 2798 port->dev_conf.intr_conf.lsc = 1; 2799 if (rmv_interrupt && 2800 (rte_eth_devices[pid].data->dev_flags & 2801 RTE_ETH_DEV_INTR_RMV)) 2802 port->dev_conf.intr_conf.rmv = 1; 2803 } 2804 } 2805 2806 void set_port_slave_flag(portid_t slave_pid) 2807 { 2808 struct rte_port *port; 2809 2810 port = &ports[slave_pid]; 2811 port->slave_flag = 1; 2812 } 2813 2814 void clear_port_slave_flag(portid_t slave_pid) 2815 { 2816 struct rte_port *port; 2817 2818 port = &ports[slave_pid]; 2819 port->slave_flag = 0; 2820 } 2821 2822 uint8_t port_is_bonding_slave(portid_t slave_pid) 2823 { 2824 struct rte_port *port; 2825 2826 port = &ports[slave_pid]; 2827 if ((rte_eth_devices[slave_pid].data->dev_flags & 2828 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2829 return 1; 2830 return 0; 2831 } 2832 2833 const uint16_t vlan_tags[] = { 2834 0, 1, 2, 3, 4, 5, 6, 7, 2835 8, 9, 10, 11, 12, 13, 14, 15, 2836 16, 17, 18, 19, 20, 21, 22, 23, 2837 24, 25, 26, 27, 28, 29, 30, 31 2838 }; 2839 2840 static int 2841 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 2842 enum dcb_mode_enable dcb_mode, 2843 enum rte_eth_nb_tcs num_tcs, 2844 uint8_t pfc_en) 2845 { 2846 uint8_t i; 2847 int32_t rc; 2848 struct rte_eth_rss_conf rss_conf; 2849 2850 /* 2851 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2852 * given above, and the number of traffic classes available for use. 2853 */ 2854 if (dcb_mode == DCB_VT_ENABLED) { 2855 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2856 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2857 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2858 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2859 2860 /* VMDQ+DCB RX and TX configurations */ 2861 vmdq_rx_conf->enable_default_pool = 0; 2862 vmdq_rx_conf->default_pool = 0; 2863 vmdq_rx_conf->nb_queue_pools = 2864 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2865 vmdq_tx_conf->nb_queue_pools = 2866 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2867 2868 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2869 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2870 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2871 vmdq_rx_conf->pool_map[i].pools = 2872 1 << (i % vmdq_rx_conf->nb_queue_pools); 2873 } 2874 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2875 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 2876 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 2877 } 2878 2879 /* set DCB mode of RX and TX of multiple queues */ 2880 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2881 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2882 } else { 2883 struct rte_eth_dcb_rx_conf *rx_conf = 2884 ð_conf->rx_adv_conf.dcb_rx_conf; 2885 struct rte_eth_dcb_tx_conf *tx_conf = 2886 ð_conf->tx_adv_conf.dcb_tx_conf; 2887 2888 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 2889 if (rc != 0) 2890 return rc; 2891 2892 rx_conf->nb_tcs = num_tcs; 2893 tx_conf->nb_tcs = num_tcs; 2894 2895 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2896 rx_conf->dcb_tc[i] = i % num_tcs; 2897 tx_conf->dcb_tc[i] = i % num_tcs; 2898 } 2899 2900 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2901 eth_conf->rx_adv_conf.rss_conf = rss_conf; 2902 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2903 } 2904 2905 if (pfc_en) 2906 eth_conf->dcb_capability_en = 2907 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2908 else 2909 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2910 2911 return 0; 2912 } 2913 2914 int 2915 init_port_dcb_config(portid_t pid, 2916 enum dcb_mode_enable dcb_mode, 2917 enum rte_eth_nb_tcs num_tcs, 2918 uint8_t pfc_en) 2919 { 2920 struct rte_eth_conf port_conf; 2921 struct rte_port *rte_port; 2922 int retval; 2923 uint16_t i; 2924 2925 rte_port = &ports[pid]; 2926 2927 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2928 /* Enter DCB configuration status */ 2929 dcb_config = 1; 2930 2931 port_conf.rxmode = rte_port->dev_conf.rxmode; 2932 port_conf.txmode = rte_port->dev_conf.txmode; 2933 2934 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2935 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 2936 if (retval < 0) 2937 return retval; 2938 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2939 2940 /* re-configure the device . */ 2941 rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 2942 2943 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2944 2945 /* If dev_info.vmdq_pool_base is greater than 0, 2946 * the queue id of vmdq pools is started after pf queues. 2947 */ 2948 if (dcb_mode == DCB_VT_ENABLED && 2949 rte_port->dev_info.vmdq_pool_base > 0) { 2950 printf("VMDQ_DCB multi-queue mode is nonsensical" 2951 " for port %d.", pid); 2952 return -1; 2953 } 2954 2955 /* Assume the ports in testpmd have the same dcb capability 2956 * and has the same number of rxq and txq in dcb mode 2957 */ 2958 if (dcb_mode == DCB_VT_ENABLED) { 2959 if (rte_port->dev_info.max_vfs > 0) { 2960 nb_rxq = rte_port->dev_info.nb_rx_queues; 2961 nb_txq = rte_port->dev_info.nb_tx_queues; 2962 } else { 2963 nb_rxq = rte_port->dev_info.max_rx_queues; 2964 nb_txq = rte_port->dev_info.max_tx_queues; 2965 } 2966 } else { 2967 /*if vt is disabled, use all pf queues */ 2968 if (rte_port->dev_info.vmdq_pool_base == 0) { 2969 nb_rxq = rte_port->dev_info.max_rx_queues; 2970 nb_txq = rte_port->dev_info.max_tx_queues; 2971 } else { 2972 nb_rxq = (queueid_t)num_tcs; 2973 nb_txq = (queueid_t)num_tcs; 2974 2975 } 2976 } 2977 rx_free_thresh = 64; 2978 2979 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 2980 2981 rxtx_port_config(rte_port); 2982 /* VLAN filter */ 2983 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2984 for (i = 0; i < RTE_DIM(vlan_tags); i++) 2985 rx_vft_set(pid, vlan_tags[i], 1); 2986 2987 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 2988 map_port_queue_stats_mapping_registers(pid, rte_port); 2989 2990 rte_port->dcb_flag = 1; 2991 2992 return 0; 2993 } 2994 2995 static void 2996 init_port(void) 2997 { 2998 /* Configuration of Ethernet ports. */ 2999 ports = rte_zmalloc("testpmd: ports", 3000 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3001 RTE_CACHE_LINE_SIZE); 3002 if (ports == NULL) { 3003 rte_exit(EXIT_FAILURE, 3004 "rte_zmalloc(%d struct rte_port) failed\n", 3005 RTE_MAX_ETHPORTS); 3006 } 3007 3008 /* Initialize ports NUMA structures */ 3009 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3010 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3011 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3012 } 3013 3014 static void 3015 force_quit(void) 3016 { 3017 pmd_test_exit(); 3018 prompt_exit(); 3019 } 3020 3021 static void 3022 print_stats(void) 3023 { 3024 uint8_t i; 3025 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3026 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3027 3028 /* Clear screen and move to top left */ 3029 printf("%s%s", clr, top_left); 3030 3031 printf("\nPort statistics ===================================="); 3032 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3033 nic_stats_display(fwd_ports_ids[i]); 3034 3035 fflush(stdout); 3036 } 3037 3038 static void 3039 signal_handler(int signum) 3040 { 3041 if (signum == SIGINT || signum == SIGTERM) { 3042 printf("\nSignal %d received, preparing to exit...\n", 3043 signum); 3044 #ifdef RTE_LIBRTE_PDUMP 3045 /* uninitialize packet capture framework */ 3046 rte_pdump_uninit(); 3047 #endif 3048 #ifdef RTE_LIBRTE_LATENCY_STATS 3049 rte_latencystats_uninit(); 3050 #endif 3051 force_quit(); 3052 /* Set flag to indicate the force termination. */ 3053 f_quit = 1; 3054 /* exit with the expected status */ 3055 signal(signum, SIG_DFL); 3056 kill(getpid(), signum); 3057 } 3058 } 3059 3060 int 3061 main(int argc, char** argv) 3062 { 3063 int diag; 3064 portid_t port_id; 3065 uint16_t count; 3066 int ret; 3067 3068 signal(SIGINT, signal_handler); 3069 signal(SIGTERM, signal_handler); 3070 3071 diag = rte_eal_init(argc, argv); 3072 if (diag < 0) 3073 rte_panic("Cannot init EAL\n"); 3074 3075 testpmd_logtype = rte_log_register("testpmd"); 3076 if (testpmd_logtype < 0) 3077 rte_panic("Cannot register log type"); 3078 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3079 3080 ret = register_eth_event_callback(); 3081 if (ret != 0) 3082 rte_panic("Cannot register for ethdev events"); 3083 3084 #ifdef RTE_LIBRTE_PDUMP 3085 /* initialize packet capture framework */ 3086 rte_pdump_init(); 3087 #endif 3088 3089 count = 0; 3090 RTE_ETH_FOREACH_DEV(port_id) { 3091 ports_ids[count] = port_id; 3092 count++; 3093 } 3094 nb_ports = (portid_t) count; 3095 if (nb_ports == 0) 3096 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3097 3098 /* allocate port structures, and init them */ 3099 init_port(); 3100 3101 set_def_fwd_config(); 3102 if (nb_lcores == 0) 3103 rte_panic("Empty set of forwarding logical cores - check the " 3104 "core mask supplied in the command parameters\n"); 3105 3106 /* Bitrate/latency stats disabled by default */ 3107 #ifdef RTE_LIBRTE_BITRATE 3108 bitrate_enabled = 0; 3109 #endif 3110 #ifdef RTE_LIBRTE_LATENCY_STATS 3111 latencystats_enabled = 0; 3112 #endif 3113 3114 /* on FreeBSD, mlockall() is disabled by default */ 3115 #ifdef RTE_EXEC_ENV_FREEBSD 3116 do_mlockall = 0; 3117 #else 3118 do_mlockall = 1; 3119 #endif 3120 3121 argc -= diag; 3122 argv += diag; 3123 if (argc > 1) 3124 launch_args_parse(argc, argv); 3125 3126 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3127 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3128 strerror(errno)); 3129 } 3130 3131 if (tx_first && interactive) 3132 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3133 "interactive mode.\n"); 3134 3135 if (tx_first && lsc_interrupt) { 3136 printf("Warning: lsc_interrupt needs to be off when " 3137 " using tx_first. Disabling.\n"); 3138 lsc_interrupt = 0; 3139 } 3140 3141 if (!nb_rxq && !nb_txq) 3142 printf("Warning: Either rx or tx queues should be non-zero\n"); 3143 3144 if (nb_rxq > 1 && nb_rxq > nb_txq) 3145 printf("Warning: nb_rxq=%d enables RSS configuration, " 3146 "but nb_txq=%d will prevent to fully test it.\n", 3147 nb_rxq, nb_txq); 3148 3149 init_config(); 3150 3151 if (hot_plug) { 3152 ret = rte_dev_hotplug_handle_enable(); 3153 if (ret) { 3154 RTE_LOG(ERR, EAL, 3155 "fail to enable hotplug handling."); 3156 return -1; 3157 } 3158 3159 ret = rte_dev_event_monitor_start(); 3160 if (ret) { 3161 RTE_LOG(ERR, EAL, 3162 "fail to start device event monitoring."); 3163 return -1; 3164 } 3165 3166 ret = rte_dev_event_callback_register(NULL, 3167 dev_event_callback, NULL); 3168 if (ret) { 3169 RTE_LOG(ERR, EAL, 3170 "fail to register device event callback\n"); 3171 return -1; 3172 } 3173 } 3174 3175 if (start_port(RTE_PORT_ALL) != 0) 3176 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3177 3178 /* set all ports to promiscuous mode by default */ 3179 RTE_ETH_FOREACH_DEV(port_id) 3180 rte_eth_promiscuous_enable(port_id); 3181 3182 /* Init metrics library */ 3183 rte_metrics_init(rte_socket_id()); 3184 3185 #ifdef RTE_LIBRTE_LATENCY_STATS 3186 if (latencystats_enabled != 0) { 3187 int ret = rte_latencystats_init(1, NULL); 3188 if (ret) 3189 printf("Warning: latencystats init()" 3190 " returned error %d\n", ret); 3191 printf("Latencystats running on lcore %d\n", 3192 latencystats_lcore_id); 3193 } 3194 #endif 3195 3196 /* Setup bitrate stats */ 3197 #ifdef RTE_LIBRTE_BITRATE 3198 if (bitrate_enabled != 0) { 3199 bitrate_data = rte_stats_bitrate_create(); 3200 if (bitrate_data == NULL) 3201 rte_exit(EXIT_FAILURE, 3202 "Could not allocate bitrate data.\n"); 3203 rte_stats_bitrate_reg(bitrate_data); 3204 } 3205 #endif 3206 3207 #ifdef RTE_LIBRTE_CMDLINE 3208 if (strlen(cmdline_filename) != 0) 3209 cmdline_read_from_file(cmdline_filename); 3210 3211 if (interactive == 1) { 3212 if (auto_start) { 3213 printf("Start automatic packet forwarding\n"); 3214 start_packet_forwarding(0); 3215 } 3216 prompt(); 3217 pmd_test_exit(); 3218 } else 3219 #endif 3220 { 3221 char c; 3222 int rc; 3223 3224 f_quit = 0; 3225 3226 printf("No commandline core given, start packet forwarding\n"); 3227 start_packet_forwarding(tx_first); 3228 if (stats_period != 0) { 3229 uint64_t prev_time = 0, cur_time, diff_time = 0; 3230 uint64_t timer_period; 3231 3232 /* Convert to number of cycles */ 3233 timer_period = stats_period * rte_get_timer_hz(); 3234 3235 while (f_quit == 0) { 3236 cur_time = rte_get_timer_cycles(); 3237 diff_time += cur_time - prev_time; 3238 3239 if (diff_time >= timer_period) { 3240 print_stats(); 3241 /* Reset the timer */ 3242 diff_time = 0; 3243 } 3244 /* Sleep to avoid unnecessary checks */ 3245 prev_time = cur_time; 3246 sleep(1); 3247 } 3248 } 3249 3250 printf("Press enter to exit\n"); 3251 rc = read(0, &c, 1); 3252 pmd_test_exit(); 3253 if (rc < 0) 3254 return 1; 3255 } 3256 3257 return 0; 3258 } 3259