1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct fwd_config cur_fwd_config; 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 193 uint32_t retry_enabled; 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 196 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 198 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 199 * specified on command-line. */ 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 201 202 /* 203 * In container, it cannot terminate the process which running with 'stats-period' 204 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 205 */ 206 uint8_t f_quit; 207 208 /* 209 * Configuration of packet segments used by the "txonly" processing engine. 210 */ 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 213 TXONLY_DEF_PACKET_LEN, 214 }; 215 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 216 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 218 /**< Split policy for packets to TX. */ 219 220 uint8_t txonly_multi_flow; 221 /**< Whether multiple flows are generated in TXONLY mode. */ 222 223 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 224 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 225 226 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 227 uint8_t dcb_config = 0; 228 229 /* Whether the dcb is in testing status */ 230 uint8_t dcb_test = 0; 231 232 /* 233 * Configurable number of RX/TX queues. 234 */ 235 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 236 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 237 238 /* 239 * Configurable number of RX/TX ring descriptors. 240 * Defaults are supplied by drivers via ethdev. 241 */ 242 #define RTE_TEST_RX_DESC_DEFAULT 0 243 #define RTE_TEST_TX_DESC_DEFAULT 0 244 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 245 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 246 247 #define RTE_PMD_PARAM_UNSET -1 248 /* 249 * Configurable values of RX and TX ring threshold registers. 250 */ 251 252 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 253 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 254 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 255 256 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 257 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 258 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 259 260 /* 261 * Configurable value of RX free threshold. 262 */ 263 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 264 265 /* 266 * Configurable value of RX drop enable. 267 */ 268 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 269 270 /* 271 * Configurable value of TX free threshold. 272 */ 273 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 274 275 /* 276 * Configurable value of TX RS bit threshold. 277 */ 278 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 279 280 /* 281 * Configurable value of buffered packets before sending. 282 */ 283 uint16_t noisy_tx_sw_bufsz; 284 285 /* 286 * Configurable value of packet buffer timeout. 287 */ 288 uint16_t noisy_tx_sw_buf_flush_time; 289 290 /* 291 * Configurable value for size of VNF internal memory area 292 * used for simulating noisy neighbour behaviour 293 */ 294 uint64_t noisy_lkup_mem_sz; 295 296 /* 297 * Configurable value of number of random writes done in 298 * VNF simulation memory area. 299 */ 300 uint64_t noisy_lkup_num_writes; 301 302 /* 303 * Configurable value of number of random reads done in 304 * VNF simulation memory area. 305 */ 306 uint64_t noisy_lkup_num_reads; 307 308 /* 309 * Configurable value of number of random reads/writes done in 310 * VNF simulation memory area. 311 */ 312 uint64_t noisy_lkup_num_reads_writes; 313 314 /* 315 * Receive Side Scaling (RSS) configuration. 316 */ 317 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 318 319 /* 320 * Port topology configuration 321 */ 322 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 323 324 /* 325 * Avoids to flush all the RX streams before starts forwarding. 326 */ 327 uint8_t no_flush_rx = 0; /* flush by default */ 328 329 /* 330 * Flow API isolated mode. 331 */ 332 uint8_t flow_isolate_all; 333 334 /* 335 * Avoids to check link status when starting/stopping a port. 336 */ 337 uint8_t no_link_check = 0; /* check by default */ 338 339 /* 340 * Enable link status change notification 341 */ 342 uint8_t lsc_interrupt = 1; /* enabled by default */ 343 344 /* 345 * Enable device removal notification. 346 */ 347 uint8_t rmv_interrupt = 1; /* enabled by default */ 348 349 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 350 351 /* After attach, port setup is called on event or by iterator */ 352 bool setup_on_probe_event = true; 353 354 /* Pretty printing of ethdev events */ 355 static const char * const eth_event_desc[] = { 356 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 357 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 358 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 359 [RTE_ETH_EVENT_INTR_RESET] = "reset", 360 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 361 [RTE_ETH_EVENT_IPSEC] = "IPsec", 362 [RTE_ETH_EVENT_MACSEC] = "MACsec", 363 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 364 [RTE_ETH_EVENT_NEW] = "device probed", 365 [RTE_ETH_EVENT_DESTROY] = "device released", 366 [RTE_ETH_EVENT_MAX] = NULL, 367 }; 368 369 /* 370 * Display or mask ether events 371 * Default to all events except VF_MBOX 372 */ 373 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 374 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 375 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 376 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 377 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 378 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 379 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 380 /* 381 * Decide if all memory are locked for performance. 382 */ 383 int do_mlockall = 0; 384 385 /* 386 * NIC bypass mode configuration options. 387 */ 388 389 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 390 /* The NIC bypass watchdog timeout. */ 391 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 392 #endif 393 394 395 #ifdef RTE_LIBRTE_LATENCY_STATS 396 397 /* 398 * Set when latency stats is enabled in the commandline 399 */ 400 uint8_t latencystats_enabled; 401 402 /* 403 * Lcore ID to serive latency statistics. 404 */ 405 lcoreid_t latencystats_lcore_id = -1; 406 407 #endif 408 409 /* 410 * Ethernet device configuration. 411 */ 412 struct rte_eth_rxmode rx_mode = { 413 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 414 }; 415 416 struct rte_eth_txmode tx_mode = { 417 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 418 }; 419 420 struct rte_fdir_conf fdir_conf = { 421 .mode = RTE_FDIR_MODE_NONE, 422 .pballoc = RTE_FDIR_PBALLOC_64K, 423 .status = RTE_FDIR_REPORT_STATUS, 424 .mask = { 425 .vlan_tci_mask = 0xFFEF, 426 .ipv4_mask = { 427 .src_ip = 0xFFFFFFFF, 428 .dst_ip = 0xFFFFFFFF, 429 }, 430 .ipv6_mask = { 431 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 432 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 433 }, 434 .src_port_mask = 0xFFFF, 435 .dst_port_mask = 0xFFFF, 436 .mac_addr_byte_mask = 0xFF, 437 .tunnel_type_mask = 1, 438 .tunnel_id_mask = 0xFFFFFFFF, 439 }, 440 .drop_queue = 127, 441 }; 442 443 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 444 445 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 446 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 447 448 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 449 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 450 451 uint16_t nb_tx_queue_stats_mappings = 0; 452 uint16_t nb_rx_queue_stats_mappings = 0; 453 454 /* 455 * Display zero values by default for xstats 456 */ 457 uint8_t xstats_hide_zero; 458 459 unsigned int num_sockets = 0; 460 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 461 462 #ifdef RTE_LIBRTE_BITRATE 463 /* Bitrate statistics */ 464 struct rte_stats_bitrates *bitrate_data; 465 lcoreid_t bitrate_lcore_id; 466 uint8_t bitrate_enabled; 467 #endif 468 469 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 470 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 471 472 struct vxlan_encap_conf vxlan_encap_conf = { 473 .select_ipv4 = 1, 474 .select_vlan = 0, 475 .select_tos_ttl = 0, 476 .vni = "\x00\x00\x00", 477 .udp_src = 0, 478 .udp_dst = RTE_BE16(4789), 479 .ipv4_src = IPv4(127, 0, 0, 1), 480 .ipv4_dst = IPv4(255, 255, 255, 255), 481 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 482 "\x00\x00\x00\x00\x00\x00\x00\x01", 483 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 484 "\x00\x00\x00\x00\x00\x00\x11\x11", 485 .vlan_tci = 0, 486 .ip_tos = 0, 487 .ip_ttl = 255, 488 .eth_src = "\x00\x00\x00\x00\x00\x00", 489 .eth_dst = "\xff\xff\xff\xff\xff\xff", 490 }; 491 492 struct nvgre_encap_conf nvgre_encap_conf = { 493 .select_ipv4 = 1, 494 .select_vlan = 0, 495 .tni = "\x00\x00\x00", 496 .ipv4_src = IPv4(127, 0, 0, 1), 497 .ipv4_dst = IPv4(255, 255, 255, 255), 498 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 499 "\x00\x00\x00\x00\x00\x00\x00\x01", 500 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 501 "\x00\x00\x00\x00\x00\x00\x11\x11", 502 .vlan_tci = 0, 503 .eth_src = "\x00\x00\x00\x00\x00\x00", 504 .eth_dst = "\xff\xff\xff\xff\xff\xff", 505 }; 506 507 /* Forward function declarations */ 508 static void setup_attached_port(portid_t pi); 509 static void map_port_queue_stats_mapping_registers(portid_t pi, 510 struct rte_port *port); 511 static void check_all_ports_link_status(uint32_t port_mask); 512 static int eth_event_callback(portid_t port_id, 513 enum rte_eth_event_type type, 514 void *param, void *ret_param); 515 static void dev_event_callback(const char *device_name, 516 enum rte_dev_event_type type, 517 void *param); 518 519 /* 520 * Check if all the ports are started. 521 * If yes, return positive value. If not, return zero. 522 */ 523 static int all_ports_started(void); 524 525 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 526 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN; 527 528 /* 529 * Helper function to check if socket is already discovered. 530 * If yes, return positive value. If not, return zero. 531 */ 532 int 533 new_socket_id(unsigned int socket_id) 534 { 535 unsigned int i; 536 537 for (i = 0; i < num_sockets; i++) { 538 if (socket_ids[i] == socket_id) 539 return 0; 540 } 541 return 1; 542 } 543 544 /* 545 * Setup default configuration. 546 */ 547 static void 548 set_default_fwd_lcores_config(void) 549 { 550 unsigned int i; 551 unsigned int nb_lc; 552 unsigned int sock_num; 553 554 nb_lc = 0; 555 for (i = 0; i < RTE_MAX_LCORE; i++) { 556 if (!rte_lcore_is_enabled(i)) 557 continue; 558 sock_num = rte_lcore_to_socket_id(i); 559 if (new_socket_id(sock_num)) { 560 if (num_sockets >= RTE_MAX_NUMA_NODES) { 561 rte_exit(EXIT_FAILURE, 562 "Total sockets greater than %u\n", 563 RTE_MAX_NUMA_NODES); 564 } 565 socket_ids[num_sockets++] = sock_num; 566 } 567 if (i == rte_get_master_lcore()) 568 continue; 569 fwd_lcores_cpuids[nb_lc++] = i; 570 } 571 nb_lcores = (lcoreid_t) nb_lc; 572 nb_cfg_lcores = nb_lcores; 573 nb_fwd_lcores = 1; 574 } 575 576 static void 577 set_def_peer_eth_addrs(void) 578 { 579 portid_t i; 580 581 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 582 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 583 peer_eth_addrs[i].addr_bytes[5] = i; 584 } 585 } 586 587 static void 588 set_default_fwd_ports_config(void) 589 { 590 portid_t pt_id; 591 int i = 0; 592 593 RTE_ETH_FOREACH_DEV(pt_id) { 594 fwd_ports_ids[i++] = pt_id; 595 596 /* Update sockets info according to the attached device */ 597 int socket_id = rte_eth_dev_socket_id(pt_id); 598 if (socket_id >= 0 && new_socket_id(socket_id)) { 599 if (num_sockets >= RTE_MAX_NUMA_NODES) { 600 rte_exit(EXIT_FAILURE, 601 "Total sockets greater than %u\n", 602 RTE_MAX_NUMA_NODES); 603 } 604 socket_ids[num_sockets++] = socket_id; 605 } 606 } 607 608 nb_cfg_ports = nb_ports; 609 nb_fwd_ports = nb_ports; 610 } 611 612 void 613 set_def_fwd_config(void) 614 { 615 set_default_fwd_lcores_config(); 616 set_def_peer_eth_addrs(); 617 set_default_fwd_ports_config(); 618 } 619 620 /* extremely pessimistic estimation of memory required to create a mempool */ 621 static int 622 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 623 { 624 unsigned int n_pages, mbuf_per_pg, leftover; 625 uint64_t total_mem, mbuf_mem, obj_sz; 626 627 /* there is no good way to predict how much space the mempool will 628 * occupy because it will allocate chunks on the fly, and some of those 629 * will come from default DPDK memory while some will come from our 630 * external memory, so just assume 128MB will be enough for everyone. 631 */ 632 uint64_t hdr_mem = 128 << 20; 633 634 /* account for possible non-contiguousness */ 635 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 636 if (obj_sz > pgsz) { 637 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 638 return -1; 639 } 640 641 mbuf_per_pg = pgsz / obj_sz; 642 leftover = (nb_mbufs % mbuf_per_pg) > 0; 643 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 644 645 mbuf_mem = n_pages * pgsz; 646 647 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 648 649 if (total_mem > SIZE_MAX) { 650 TESTPMD_LOG(ERR, "Memory size too big\n"); 651 return -1; 652 } 653 *out = (size_t)total_mem; 654 655 return 0; 656 } 657 658 static int 659 pagesz_flags(uint64_t page_sz) 660 { 661 /* as per mmap() manpage, all page sizes are log2 of page size 662 * shifted by MAP_HUGE_SHIFT 663 */ 664 int log2 = rte_log2_u64(page_sz); 665 666 return (log2 << HUGE_SHIFT); 667 } 668 669 static void * 670 alloc_mem(size_t memsz, size_t pgsz, bool huge) 671 { 672 void *addr; 673 int flags; 674 675 /* allocate anonymous hugepages */ 676 flags = MAP_ANONYMOUS | MAP_PRIVATE; 677 if (huge) 678 flags |= HUGE_FLAG | pagesz_flags(pgsz); 679 680 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 681 if (addr == MAP_FAILED) 682 return NULL; 683 684 return addr; 685 } 686 687 struct extmem_param { 688 void *addr; 689 size_t len; 690 size_t pgsz; 691 rte_iova_t *iova_table; 692 unsigned int iova_table_len; 693 }; 694 695 static int 696 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 697 bool huge) 698 { 699 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 700 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 701 unsigned int cur_page, n_pages, pgsz_idx; 702 size_t mem_sz, cur_pgsz; 703 rte_iova_t *iovas = NULL; 704 void *addr; 705 int ret; 706 707 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 708 /* skip anything that is too big */ 709 if (pgsizes[pgsz_idx] > SIZE_MAX) 710 continue; 711 712 cur_pgsz = pgsizes[pgsz_idx]; 713 714 /* if we were told not to allocate hugepages, override */ 715 if (!huge) 716 cur_pgsz = sysconf(_SC_PAGESIZE); 717 718 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 719 if (ret < 0) { 720 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 721 return -1; 722 } 723 724 /* allocate our memory */ 725 addr = alloc_mem(mem_sz, cur_pgsz, huge); 726 727 /* if we couldn't allocate memory with a specified page size, 728 * that doesn't mean we can't do it with other page sizes, so 729 * try another one. 730 */ 731 if (addr == NULL) 732 continue; 733 734 /* store IOVA addresses for every page in this memory area */ 735 n_pages = mem_sz / cur_pgsz; 736 737 iovas = malloc(sizeof(*iovas) * n_pages); 738 739 if (iovas == NULL) { 740 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 741 goto fail; 742 } 743 /* lock memory if it's not huge pages */ 744 if (!huge) 745 mlock(addr, mem_sz); 746 747 /* populate IOVA addresses */ 748 for (cur_page = 0; cur_page < n_pages; cur_page++) { 749 rte_iova_t iova; 750 size_t offset; 751 void *cur; 752 753 offset = cur_pgsz * cur_page; 754 cur = RTE_PTR_ADD(addr, offset); 755 756 /* touch the page before getting its IOVA */ 757 *(volatile char *)cur = 0; 758 759 iova = rte_mem_virt2iova(cur); 760 761 iovas[cur_page] = iova; 762 } 763 764 break; 765 } 766 /* if we couldn't allocate anything */ 767 if (iovas == NULL) 768 return -1; 769 770 param->addr = addr; 771 param->len = mem_sz; 772 param->pgsz = cur_pgsz; 773 param->iova_table = iovas; 774 param->iova_table_len = n_pages; 775 776 return 0; 777 fail: 778 if (iovas) 779 free(iovas); 780 if (addr) 781 munmap(addr, mem_sz); 782 783 return -1; 784 } 785 786 static int 787 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 788 { 789 struct extmem_param param; 790 int socket_id, ret; 791 792 memset(¶m, 0, sizeof(param)); 793 794 /* check if our heap exists */ 795 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 796 if (socket_id < 0) { 797 /* create our heap */ 798 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 799 if (ret < 0) { 800 TESTPMD_LOG(ERR, "Cannot create heap\n"); 801 return -1; 802 } 803 } 804 805 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 806 if (ret < 0) { 807 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 808 return -1; 809 } 810 811 /* we now have a valid memory area, so add it to heap */ 812 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 813 param.addr, param.len, param.iova_table, 814 param.iova_table_len, param.pgsz); 815 816 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 817 818 /* not needed any more */ 819 free(param.iova_table); 820 821 if (ret < 0) { 822 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 823 munmap(param.addr, param.len); 824 return -1; 825 } 826 827 /* success */ 828 829 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 830 param.len >> 20); 831 832 return 0; 833 } 834 835 /* 836 * Configuration initialisation done once at init time. 837 */ 838 static void 839 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 840 unsigned int socket_id) 841 { 842 char pool_name[RTE_MEMPOOL_NAMESIZE]; 843 struct rte_mempool *rte_mp = NULL; 844 uint32_t mb_size; 845 846 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 847 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 848 849 TESTPMD_LOG(INFO, 850 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 851 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 852 853 switch (mp_alloc_type) { 854 case MP_ALLOC_NATIVE: 855 { 856 /* wrapper to rte_mempool_create() */ 857 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 858 rte_mbuf_best_mempool_ops()); 859 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 860 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 861 break; 862 } 863 case MP_ALLOC_ANON: 864 { 865 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 866 mb_size, (unsigned int) mb_mempool_cache, 867 sizeof(struct rte_pktmbuf_pool_private), 868 socket_id, 0); 869 if (rte_mp == NULL) 870 goto err; 871 872 if (rte_mempool_populate_anon(rte_mp) == 0) { 873 rte_mempool_free(rte_mp); 874 rte_mp = NULL; 875 goto err; 876 } 877 rte_pktmbuf_pool_init(rte_mp, NULL); 878 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 879 break; 880 } 881 case MP_ALLOC_XMEM: 882 case MP_ALLOC_XMEM_HUGE: 883 { 884 int heap_socket; 885 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 886 887 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 888 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 889 890 heap_socket = 891 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 892 if (heap_socket < 0) 893 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 894 895 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 896 rte_mbuf_best_mempool_ops()); 897 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 898 mb_mempool_cache, 0, mbuf_seg_size, 899 heap_socket); 900 break; 901 } 902 default: 903 { 904 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 905 } 906 } 907 908 err: 909 if (rte_mp == NULL) { 910 rte_exit(EXIT_FAILURE, 911 "Creation of mbuf pool for socket %u failed: %s\n", 912 socket_id, rte_strerror(rte_errno)); 913 } else if (verbose_level > 0) { 914 rte_mempool_dump(stdout, rte_mp); 915 } 916 } 917 918 /* 919 * Check given socket id is valid or not with NUMA mode, 920 * if valid, return 0, else return -1 921 */ 922 static int 923 check_socket_id(const unsigned int socket_id) 924 { 925 static int warning_once = 0; 926 927 if (new_socket_id(socket_id)) { 928 if (!warning_once && numa_support) 929 printf("Warning: NUMA should be configured manually by" 930 " using --port-numa-config and" 931 " --ring-numa-config parameters along with" 932 " --numa.\n"); 933 warning_once = 1; 934 return -1; 935 } 936 return 0; 937 } 938 939 /* 940 * Get the allowed maximum number of RX queues. 941 * *pid return the port id which has minimal value of 942 * max_rx_queues in all ports. 943 */ 944 queueid_t 945 get_allowed_max_nb_rxq(portid_t *pid) 946 { 947 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 948 portid_t pi; 949 struct rte_eth_dev_info dev_info; 950 951 RTE_ETH_FOREACH_DEV(pi) { 952 rte_eth_dev_info_get(pi, &dev_info); 953 if (dev_info.max_rx_queues < allowed_max_rxq) { 954 allowed_max_rxq = dev_info.max_rx_queues; 955 *pid = pi; 956 } 957 } 958 return allowed_max_rxq; 959 } 960 961 /* 962 * Check input rxq is valid or not. 963 * If input rxq is not greater than any of maximum number 964 * of RX queues of all ports, it is valid. 965 * if valid, return 0, else return -1 966 */ 967 int 968 check_nb_rxq(queueid_t rxq) 969 { 970 queueid_t allowed_max_rxq; 971 portid_t pid = 0; 972 973 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 974 if (rxq > allowed_max_rxq) { 975 printf("Fail: input rxq (%u) can't be greater " 976 "than max_rx_queues (%u) of port %u\n", 977 rxq, 978 allowed_max_rxq, 979 pid); 980 return -1; 981 } 982 return 0; 983 } 984 985 /* 986 * Get the allowed maximum number of TX queues. 987 * *pid return the port id which has minimal value of 988 * max_tx_queues in all ports. 989 */ 990 queueid_t 991 get_allowed_max_nb_txq(portid_t *pid) 992 { 993 queueid_t allowed_max_txq = MAX_QUEUE_ID; 994 portid_t pi; 995 struct rte_eth_dev_info dev_info; 996 997 RTE_ETH_FOREACH_DEV(pi) { 998 rte_eth_dev_info_get(pi, &dev_info); 999 if (dev_info.max_tx_queues < allowed_max_txq) { 1000 allowed_max_txq = dev_info.max_tx_queues; 1001 *pid = pi; 1002 } 1003 } 1004 return allowed_max_txq; 1005 } 1006 1007 /* 1008 * Check input txq is valid or not. 1009 * If input txq is not greater than any of maximum number 1010 * of TX queues of all ports, it is valid. 1011 * if valid, return 0, else return -1 1012 */ 1013 int 1014 check_nb_txq(queueid_t txq) 1015 { 1016 queueid_t allowed_max_txq; 1017 portid_t pid = 0; 1018 1019 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1020 if (txq > allowed_max_txq) { 1021 printf("Fail: input txq (%u) can't be greater " 1022 "than max_tx_queues (%u) of port %u\n", 1023 txq, 1024 allowed_max_txq, 1025 pid); 1026 return -1; 1027 } 1028 return 0; 1029 } 1030 1031 static void 1032 init_config(void) 1033 { 1034 portid_t pid; 1035 struct rte_port *port; 1036 struct rte_mempool *mbp; 1037 unsigned int nb_mbuf_per_pool; 1038 lcoreid_t lc_id; 1039 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1040 struct rte_gro_param gro_param; 1041 uint32_t gso_types; 1042 int k; 1043 1044 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1045 1046 /* Configuration of logical cores. */ 1047 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1048 sizeof(struct fwd_lcore *) * nb_lcores, 1049 RTE_CACHE_LINE_SIZE); 1050 if (fwd_lcores == NULL) { 1051 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1052 "failed\n", nb_lcores); 1053 } 1054 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1055 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1056 sizeof(struct fwd_lcore), 1057 RTE_CACHE_LINE_SIZE); 1058 if (fwd_lcores[lc_id] == NULL) { 1059 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1060 "failed\n"); 1061 } 1062 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1063 } 1064 1065 RTE_ETH_FOREACH_DEV(pid) { 1066 port = &ports[pid]; 1067 /* Apply default TxRx configuration for all ports */ 1068 port->dev_conf.txmode = tx_mode; 1069 port->dev_conf.rxmode = rx_mode; 1070 rte_eth_dev_info_get(pid, &port->dev_info); 1071 1072 if (!(port->dev_info.tx_offload_capa & 1073 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1074 port->dev_conf.txmode.offloads &= 1075 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1076 if (!(port->dev_info.tx_offload_capa & 1077 DEV_TX_OFFLOAD_MATCH_METADATA)) 1078 port->dev_conf.txmode.offloads &= 1079 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1080 if (numa_support) { 1081 if (port_numa[pid] != NUMA_NO_CONFIG) 1082 port_per_socket[port_numa[pid]]++; 1083 else { 1084 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1085 1086 /* 1087 * if socket_id is invalid, 1088 * set to the first available socket. 1089 */ 1090 if (check_socket_id(socket_id) < 0) 1091 socket_id = socket_ids[0]; 1092 port_per_socket[socket_id]++; 1093 } 1094 } 1095 1096 /* Apply Rx offloads configuration */ 1097 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1098 port->rx_conf[k].offloads = 1099 port->dev_conf.rxmode.offloads; 1100 /* Apply Tx offloads configuration */ 1101 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1102 port->tx_conf[k].offloads = 1103 port->dev_conf.txmode.offloads; 1104 1105 /* set flag to initialize port/queue */ 1106 port->need_reconfig = 1; 1107 port->need_reconfig_queues = 1; 1108 port->tx_metadata = 0; 1109 } 1110 1111 /* 1112 * Create pools of mbuf. 1113 * If NUMA support is disabled, create a single pool of mbuf in 1114 * socket 0 memory by default. 1115 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1116 * 1117 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1118 * nb_txd can be configured at run time. 1119 */ 1120 if (param_total_num_mbufs) 1121 nb_mbuf_per_pool = param_total_num_mbufs; 1122 else { 1123 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1124 (nb_lcores * mb_mempool_cache) + 1125 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1126 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1127 } 1128 1129 if (numa_support) { 1130 uint8_t i; 1131 1132 for (i = 0; i < num_sockets; i++) 1133 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1134 socket_ids[i]); 1135 } else { 1136 if (socket_num == UMA_NO_CONFIG) 1137 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 1138 else 1139 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1140 socket_num); 1141 } 1142 1143 init_port_config(); 1144 1145 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1146 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1147 /* 1148 * Records which Mbuf pool to use by each logical core, if needed. 1149 */ 1150 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1151 mbp = mbuf_pool_find( 1152 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1153 1154 if (mbp == NULL) 1155 mbp = mbuf_pool_find(0); 1156 fwd_lcores[lc_id]->mbp = mbp; 1157 /* initialize GSO context */ 1158 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1159 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1160 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1161 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN - 1162 ETHER_CRC_LEN; 1163 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1164 } 1165 1166 /* Configuration of packet forwarding streams. */ 1167 if (init_fwd_streams() < 0) 1168 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1169 1170 fwd_config_setup(); 1171 1172 /* create a gro context for each lcore */ 1173 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1174 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1175 gro_param.max_item_per_flow = MAX_PKT_BURST; 1176 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1177 gro_param.socket_id = rte_lcore_to_socket_id( 1178 fwd_lcores_cpuids[lc_id]); 1179 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1180 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1181 rte_exit(EXIT_FAILURE, 1182 "rte_gro_ctx_create() failed\n"); 1183 } 1184 } 1185 1186 #if defined RTE_LIBRTE_PMD_SOFTNIC 1187 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1188 RTE_ETH_FOREACH_DEV(pid) { 1189 port = &ports[pid]; 1190 const char *driver = port->dev_info.driver_name; 1191 1192 if (strcmp(driver, "net_softnic") == 0) 1193 port->softport.fwd_lcore_arg = fwd_lcores; 1194 } 1195 } 1196 #endif 1197 1198 } 1199 1200 1201 void 1202 reconfig(portid_t new_port_id, unsigned socket_id) 1203 { 1204 struct rte_port *port; 1205 1206 /* Reconfiguration of Ethernet ports. */ 1207 port = &ports[new_port_id]; 1208 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1209 1210 /* set flag to initialize port/queue */ 1211 port->need_reconfig = 1; 1212 port->need_reconfig_queues = 1; 1213 port->socket_id = socket_id; 1214 1215 init_port_config(); 1216 } 1217 1218 1219 int 1220 init_fwd_streams(void) 1221 { 1222 portid_t pid; 1223 struct rte_port *port; 1224 streamid_t sm_id, nb_fwd_streams_new; 1225 queueid_t q; 1226 1227 /* set socket id according to numa or not */ 1228 RTE_ETH_FOREACH_DEV(pid) { 1229 port = &ports[pid]; 1230 if (nb_rxq > port->dev_info.max_rx_queues) { 1231 printf("Fail: nb_rxq(%d) is greater than " 1232 "max_rx_queues(%d)\n", nb_rxq, 1233 port->dev_info.max_rx_queues); 1234 return -1; 1235 } 1236 if (nb_txq > port->dev_info.max_tx_queues) { 1237 printf("Fail: nb_txq(%d) is greater than " 1238 "max_tx_queues(%d)\n", nb_txq, 1239 port->dev_info.max_tx_queues); 1240 return -1; 1241 } 1242 if (numa_support) { 1243 if (port_numa[pid] != NUMA_NO_CONFIG) 1244 port->socket_id = port_numa[pid]; 1245 else { 1246 port->socket_id = rte_eth_dev_socket_id(pid); 1247 1248 /* 1249 * if socket_id is invalid, 1250 * set to the first available socket. 1251 */ 1252 if (check_socket_id(port->socket_id) < 0) 1253 port->socket_id = socket_ids[0]; 1254 } 1255 } 1256 else { 1257 if (socket_num == UMA_NO_CONFIG) 1258 port->socket_id = 0; 1259 else 1260 port->socket_id = socket_num; 1261 } 1262 } 1263 1264 q = RTE_MAX(nb_rxq, nb_txq); 1265 if (q == 0) { 1266 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1267 return -1; 1268 } 1269 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1270 if (nb_fwd_streams_new == nb_fwd_streams) 1271 return 0; 1272 /* clear the old */ 1273 if (fwd_streams != NULL) { 1274 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1275 if (fwd_streams[sm_id] == NULL) 1276 continue; 1277 rte_free(fwd_streams[sm_id]); 1278 fwd_streams[sm_id] = NULL; 1279 } 1280 rte_free(fwd_streams); 1281 fwd_streams = NULL; 1282 } 1283 1284 /* init new */ 1285 nb_fwd_streams = nb_fwd_streams_new; 1286 if (nb_fwd_streams) { 1287 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1288 sizeof(struct fwd_stream *) * nb_fwd_streams, 1289 RTE_CACHE_LINE_SIZE); 1290 if (fwd_streams == NULL) 1291 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1292 " (struct fwd_stream *)) failed\n", 1293 nb_fwd_streams); 1294 1295 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1296 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1297 " struct fwd_stream", sizeof(struct fwd_stream), 1298 RTE_CACHE_LINE_SIZE); 1299 if (fwd_streams[sm_id] == NULL) 1300 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1301 "(struct fwd_stream) failed\n"); 1302 } 1303 } 1304 1305 return 0; 1306 } 1307 1308 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1309 static void 1310 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1311 { 1312 unsigned int total_burst; 1313 unsigned int nb_burst; 1314 unsigned int burst_stats[3]; 1315 uint16_t pktnb_stats[3]; 1316 uint16_t nb_pkt; 1317 int burst_percent[3]; 1318 1319 /* 1320 * First compute the total number of packet bursts and the 1321 * two highest numbers of bursts of the same number of packets. 1322 */ 1323 total_burst = 0; 1324 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1325 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1326 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1327 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1328 if (nb_burst == 0) 1329 continue; 1330 total_burst += nb_burst; 1331 if (nb_burst > burst_stats[0]) { 1332 burst_stats[1] = burst_stats[0]; 1333 pktnb_stats[1] = pktnb_stats[0]; 1334 burst_stats[0] = nb_burst; 1335 pktnb_stats[0] = nb_pkt; 1336 } else if (nb_burst > burst_stats[1]) { 1337 burst_stats[1] = nb_burst; 1338 pktnb_stats[1] = nb_pkt; 1339 } 1340 } 1341 if (total_burst == 0) 1342 return; 1343 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1344 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1345 burst_percent[0], (int) pktnb_stats[0]); 1346 if (burst_stats[0] == total_burst) { 1347 printf("]\n"); 1348 return; 1349 } 1350 if (burst_stats[0] + burst_stats[1] == total_burst) { 1351 printf(" + %d%% of %d pkts]\n", 1352 100 - burst_percent[0], pktnb_stats[1]); 1353 return; 1354 } 1355 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1356 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1357 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1358 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1359 return; 1360 } 1361 printf(" + %d%% of %d pkts + %d%% of others]\n", 1362 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1363 } 1364 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1365 1366 static void 1367 fwd_stream_stats_display(streamid_t stream_id) 1368 { 1369 struct fwd_stream *fs; 1370 static const char *fwd_top_stats_border = "-------"; 1371 1372 fs = fwd_streams[stream_id]; 1373 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1374 (fs->fwd_dropped == 0)) 1375 return; 1376 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1377 "TX Port=%2d/Queue=%2d %s\n", 1378 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1379 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1380 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1381 " TX-dropped: %-14"PRIu64, 1382 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1383 1384 /* if checksum mode */ 1385 if (cur_fwd_eng == &csum_fwd_engine) { 1386 printf(" RX- bad IP checksum: %-14"PRIu64 1387 " Rx- bad L4 checksum: %-14"PRIu64 1388 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1389 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1390 fs->rx_bad_outer_l4_csum); 1391 } else { 1392 printf("\n"); 1393 } 1394 1395 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1396 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1397 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1398 #endif 1399 } 1400 1401 void 1402 fwd_stats_display(void) 1403 { 1404 static const char *fwd_stats_border = "----------------------"; 1405 static const char *acc_stats_border = "+++++++++++++++"; 1406 struct { 1407 struct fwd_stream *rx_stream; 1408 struct fwd_stream *tx_stream; 1409 uint64_t tx_dropped; 1410 uint64_t rx_bad_ip_csum; 1411 uint64_t rx_bad_l4_csum; 1412 uint64_t rx_bad_outer_l4_csum; 1413 } ports_stats[RTE_MAX_ETHPORTS]; 1414 uint64_t total_rx_dropped = 0; 1415 uint64_t total_tx_dropped = 0; 1416 uint64_t total_rx_nombuf = 0; 1417 struct rte_eth_stats stats; 1418 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1419 uint64_t fwd_cycles = 0; 1420 #endif 1421 uint64_t total_recv = 0; 1422 uint64_t total_xmit = 0; 1423 struct rte_port *port; 1424 streamid_t sm_id; 1425 portid_t pt_id; 1426 int i; 1427 1428 memset(ports_stats, 0, sizeof(ports_stats)); 1429 1430 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1431 struct fwd_stream *fs = fwd_streams[sm_id]; 1432 1433 if (cur_fwd_config.nb_fwd_streams > 1434 cur_fwd_config.nb_fwd_ports) { 1435 fwd_stream_stats_display(sm_id); 1436 } else { 1437 ports_stats[fs->tx_port].tx_stream = fs; 1438 ports_stats[fs->rx_port].rx_stream = fs; 1439 } 1440 1441 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1442 1443 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1444 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1445 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1446 fs->rx_bad_outer_l4_csum; 1447 1448 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1449 fwd_cycles += fs->core_cycles; 1450 #endif 1451 } 1452 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1453 uint8_t j; 1454 1455 pt_id = fwd_ports_ids[i]; 1456 port = &ports[pt_id]; 1457 1458 rte_eth_stats_get(pt_id, &stats); 1459 stats.ipackets -= port->stats.ipackets; 1460 stats.opackets -= port->stats.opackets; 1461 stats.ibytes -= port->stats.ibytes; 1462 stats.obytes -= port->stats.obytes; 1463 stats.imissed -= port->stats.imissed; 1464 stats.oerrors -= port->stats.oerrors; 1465 stats.rx_nombuf -= port->stats.rx_nombuf; 1466 1467 total_recv += stats.ipackets; 1468 total_xmit += stats.opackets; 1469 total_rx_dropped += stats.imissed; 1470 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1471 total_tx_dropped += stats.oerrors; 1472 total_rx_nombuf += stats.rx_nombuf; 1473 1474 printf("\n %s Forward statistics for port %-2d %s\n", 1475 fwd_stats_border, pt_id, fwd_stats_border); 1476 1477 if (!port->rx_queue_stats_mapping_enabled && 1478 !port->tx_queue_stats_mapping_enabled) { 1479 printf(" RX-packets: %-14"PRIu64 1480 " RX-dropped: %-14"PRIu64 1481 "RX-total: %-"PRIu64"\n", 1482 stats.ipackets, stats.imissed, 1483 stats.ipackets + stats.imissed); 1484 1485 if (cur_fwd_eng == &csum_fwd_engine) 1486 printf(" Bad-ipcsum: %-14"PRIu64 1487 " Bad-l4csum: %-14"PRIu64 1488 "Bad-outer-l4csum: %-14"PRIu64"\n", 1489 ports_stats[pt_id].rx_bad_ip_csum, 1490 ports_stats[pt_id].rx_bad_l4_csum, 1491 ports_stats[pt_id].rx_bad_outer_l4_csum); 1492 if (stats.ierrors + stats.rx_nombuf > 0) { 1493 printf(" RX-error: %-"PRIu64"\n", 1494 stats.ierrors); 1495 printf(" RX-nombufs: %-14"PRIu64"\n", 1496 stats.rx_nombuf); 1497 } 1498 1499 printf(" TX-packets: %-14"PRIu64 1500 " TX-dropped: %-14"PRIu64 1501 "TX-total: %-"PRIu64"\n", 1502 stats.opackets, ports_stats[pt_id].tx_dropped, 1503 stats.opackets + ports_stats[pt_id].tx_dropped); 1504 } else { 1505 printf(" RX-packets: %14"PRIu64 1506 " RX-dropped:%14"PRIu64 1507 " RX-total:%14"PRIu64"\n", 1508 stats.ipackets, stats.imissed, 1509 stats.ipackets + stats.imissed); 1510 1511 if (cur_fwd_eng == &csum_fwd_engine) 1512 printf(" Bad-ipcsum:%14"PRIu64 1513 " Bad-l4csum:%14"PRIu64 1514 " Bad-outer-l4csum: %-14"PRIu64"\n", 1515 ports_stats[pt_id].rx_bad_ip_csum, 1516 ports_stats[pt_id].rx_bad_l4_csum, 1517 ports_stats[pt_id].rx_bad_outer_l4_csum); 1518 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1519 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1520 printf(" RX-nombufs: %14"PRIu64"\n", 1521 stats.rx_nombuf); 1522 } 1523 1524 printf(" TX-packets: %14"PRIu64 1525 " TX-dropped:%14"PRIu64 1526 " TX-total:%14"PRIu64"\n", 1527 stats.opackets, ports_stats[pt_id].tx_dropped, 1528 stats.opackets + ports_stats[pt_id].tx_dropped); 1529 } 1530 1531 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1532 if (ports_stats[pt_id].rx_stream) 1533 pkt_burst_stats_display("RX", 1534 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1535 if (ports_stats[pt_id].tx_stream) 1536 pkt_burst_stats_display("TX", 1537 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1538 #endif 1539 1540 if (port->rx_queue_stats_mapping_enabled) { 1541 printf("\n"); 1542 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1543 printf(" Stats reg %2d RX-packets:%14"PRIu64 1544 " RX-errors:%14"PRIu64 1545 " RX-bytes:%14"PRIu64"\n", 1546 j, stats.q_ipackets[j], 1547 stats.q_errors[j], stats.q_ibytes[j]); 1548 } 1549 printf("\n"); 1550 } 1551 if (port->tx_queue_stats_mapping_enabled) { 1552 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1553 printf(" Stats reg %2d TX-packets:%14"PRIu64 1554 " TX-bytes:%14" 1555 PRIu64"\n", 1556 j, stats.q_opackets[j], 1557 stats.q_obytes[j]); 1558 } 1559 } 1560 1561 printf(" %s--------------------------------%s\n", 1562 fwd_stats_border, fwd_stats_border); 1563 } 1564 1565 printf("\n %s Accumulated forward statistics for all ports" 1566 "%s\n", 1567 acc_stats_border, acc_stats_border); 1568 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1569 "%-"PRIu64"\n" 1570 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1571 "%-"PRIu64"\n", 1572 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1573 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1574 if (total_rx_nombuf > 0) 1575 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1576 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1577 "%s\n", 1578 acc_stats_border, acc_stats_border); 1579 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1580 if (total_recv > 0) 1581 printf("\n CPU cycles/packet=%u (total cycles=" 1582 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1583 (unsigned int)(fwd_cycles / total_recv), 1584 fwd_cycles, total_recv); 1585 #endif 1586 } 1587 1588 void 1589 fwd_stats_reset(void) 1590 { 1591 streamid_t sm_id; 1592 portid_t pt_id; 1593 int i; 1594 1595 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1596 pt_id = fwd_ports_ids[i]; 1597 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1598 } 1599 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1600 struct fwd_stream *fs = fwd_streams[sm_id]; 1601 1602 fs->rx_packets = 0; 1603 fs->tx_packets = 0; 1604 fs->fwd_dropped = 0; 1605 fs->rx_bad_ip_csum = 0; 1606 fs->rx_bad_l4_csum = 0; 1607 fs->rx_bad_outer_l4_csum = 0; 1608 1609 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1610 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1611 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1612 #endif 1613 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1614 fs->core_cycles = 0; 1615 #endif 1616 } 1617 } 1618 1619 static void 1620 flush_fwd_rx_queues(void) 1621 { 1622 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1623 portid_t rxp; 1624 portid_t port_id; 1625 queueid_t rxq; 1626 uint16_t nb_rx; 1627 uint16_t i; 1628 uint8_t j; 1629 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1630 uint64_t timer_period; 1631 1632 /* convert to number of cycles */ 1633 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1634 1635 for (j = 0; j < 2; j++) { 1636 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1637 for (rxq = 0; rxq < nb_rxq; rxq++) { 1638 port_id = fwd_ports_ids[rxp]; 1639 /** 1640 * testpmd can stuck in the below do while loop 1641 * if rte_eth_rx_burst() always returns nonzero 1642 * packets. So timer is added to exit this loop 1643 * after 1sec timer expiry. 1644 */ 1645 prev_tsc = rte_rdtsc(); 1646 do { 1647 nb_rx = rte_eth_rx_burst(port_id, rxq, 1648 pkts_burst, MAX_PKT_BURST); 1649 for (i = 0; i < nb_rx; i++) 1650 rte_pktmbuf_free(pkts_burst[i]); 1651 1652 cur_tsc = rte_rdtsc(); 1653 diff_tsc = cur_tsc - prev_tsc; 1654 timer_tsc += diff_tsc; 1655 } while ((nb_rx > 0) && 1656 (timer_tsc < timer_period)); 1657 timer_tsc = 0; 1658 } 1659 } 1660 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1661 } 1662 } 1663 1664 static void 1665 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1666 { 1667 struct fwd_stream **fsm; 1668 streamid_t nb_fs; 1669 streamid_t sm_id; 1670 #ifdef RTE_LIBRTE_BITRATE 1671 uint64_t tics_per_1sec; 1672 uint64_t tics_datum; 1673 uint64_t tics_current; 1674 uint16_t i, cnt_ports; 1675 1676 cnt_ports = nb_ports; 1677 tics_datum = rte_rdtsc(); 1678 tics_per_1sec = rte_get_timer_hz(); 1679 #endif 1680 fsm = &fwd_streams[fc->stream_idx]; 1681 nb_fs = fc->stream_nb; 1682 do { 1683 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1684 (*pkt_fwd)(fsm[sm_id]); 1685 #ifdef RTE_LIBRTE_BITRATE 1686 if (bitrate_enabled != 0 && 1687 bitrate_lcore_id == rte_lcore_id()) { 1688 tics_current = rte_rdtsc(); 1689 if (tics_current - tics_datum >= tics_per_1sec) { 1690 /* Periodic bitrate calculation */ 1691 for (i = 0; i < cnt_ports; i++) 1692 rte_stats_bitrate_calc(bitrate_data, 1693 ports_ids[i]); 1694 tics_datum = tics_current; 1695 } 1696 } 1697 #endif 1698 #ifdef RTE_LIBRTE_LATENCY_STATS 1699 if (latencystats_enabled != 0 && 1700 latencystats_lcore_id == rte_lcore_id()) 1701 rte_latencystats_update(); 1702 #endif 1703 1704 } while (! fc->stopped); 1705 } 1706 1707 static int 1708 start_pkt_forward_on_core(void *fwd_arg) 1709 { 1710 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1711 cur_fwd_config.fwd_eng->packet_fwd); 1712 return 0; 1713 } 1714 1715 /* 1716 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1717 * Used to start communication flows in network loopback test configurations. 1718 */ 1719 static int 1720 run_one_txonly_burst_on_core(void *fwd_arg) 1721 { 1722 struct fwd_lcore *fwd_lc; 1723 struct fwd_lcore tmp_lcore; 1724 1725 fwd_lc = (struct fwd_lcore *) fwd_arg; 1726 tmp_lcore = *fwd_lc; 1727 tmp_lcore.stopped = 1; 1728 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1729 return 0; 1730 } 1731 1732 /* 1733 * Launch packet forwarding: 1734 * - Setup per-port forwarding context. 1735 * - launch logical cores with their forwarding configuration. 1736 */ 1737 static void 1738 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1739 { 1740 port_fwd_begin_t port_fwd_begin; 1741 unsigned int i; 1742 unsigned int lc_id; 1743 int diag; 1744 1745 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1746 if (port_fwd_begin != NULL) { 1747 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1748 (*port_fwd_begin)(fwd_ports_ids[i]); 1749 } 1750 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1751 lc_id = fwd_lcores_cpuids[i]; 1752 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1753 fwd_lcores[i]->stopped = 0; 1754 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1755 fwd_lcores[i], lc_id); 1756 if (diag != 0) 1757 printf("launch lcore %u failed - diag=%d\n", 1758 lc_id, diag); 1759 } 1760 } 1761 } 1762 1763 /* 1764 * Launch packet forwarding configuration. 1765 */ 1766 void 1767 start_packet_forwarding(int with_tx_first) 1768 { 1769 port_fwd_begin_t port_fwd_begin; 1770 port_fwd_end_t port_fwd_end; 1771 struct rte_port *port; 1772 unsigned int i; 1773 portid_t pt_id; 1774 1775 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1776 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1777 1778 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1779 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1780 1781 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1782 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1783 (!nb_rxq || !nb_txq)) 1784 rte_exit(EXIT_FAILURE, 1785 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1786 cur_fwd_eng->fwd_mode_name); 1787 1788 if (all_ports_started() == 0) { 1789 printf("Not all ports were started\n"); 1790 return; 1791 } 1792 if (test_done == 0) { 1793 printf("Packet forwarding already started\n"); 1794 return; 1795 } 1796 1797 1798 if(dcb_test) { 1799 for (i = 0; i < nb_fwd_ports; i++) { 1800 pt_id = fwd_ports_ids[i]; 1801 port = &ports[pt_id]; 1802 if (!port->dcb_flag) { 1803 printf("In DCB mode, all forwarding ports must " 1804 "be configured in this mode.\n"); 1805 return; 1806 } 1807 } 1808 if (nb_fwd_lcores == 1) { 1809 printf("In DCB mode,the nb forwarding cores " 1810 "should be larger than 1.\n"); 1811 return; 1812 } 1813 } 1814 test_done = 0; 1815 1816 fwd_config_setup(); 1817 1818 if(!no_flush_rx) 1819 flush_fwd_rx_queues(); 1820 1821 pkt_fwd_config_display(&cur_fwd_config); 1822 rxtx_config_display(); 1823 1824 fwd_stats_reset(); 1825 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1826 pt_id = fwd_ports_ids[i]; 1827 port = &ports[pt_id]; 1828 map_port_queue_stats_mapping_registers(pt_id, port); 1829 } 1830 if (with_tx_first) { 1831 port_fwd_begin = tx_only_engine.port_fwd_begin; 1832 if (port_fwd_begin != NULL) { 1833 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1834 (*port_fwd_begin)(fwd_ports_ids[i]); 1835 } 1836 while (with_tx_first--) { 1837 launch_packet_forwarding( 1838 run_one_txonly_burst_on_core); 1839 rte_eal_mp_wait_lcore(); 1840 } 1841 port_fwd_end = tx_only_engine.port_fwd_end; 1842 if (port_fwd_end != NULL) { 1843 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1844 (*port_fwd_end)(fwd_ports_ids[i]); 1845 } 1846 } 1847 launch_packet_forwarding(start_pkt_forward_on_core); 1848 } 1849 1850 void 1851 stop_packet_forwarding(void) 1852 { 1853 port_fwd_end_t port_fwd_end; 1854 lcoreid_t lc_id; 1855 portid_t pt_id; 1856 int i; 1857 1858 if (test_done) { 1859 printf("Packet forwarding not started\n"); 1860 return; 1861 } 1862 printf("Telling cores to stop..."); 1863 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1864 fwd_lcores[lc_id]->stopped = 1; 1865 printf("\nWaiting for lcores to finish...\n"); 1866 rte_eal_mp_wait_lcore(); 1867 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1868 if (port_fwd_end != NULL) { 1869 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1870 pt_id = fwd_ports_ids[i]; 1871 (*port_fwd_end)(pt_id); 1872 } 1873 } 1874 1875 fwd_stats_display(); 1876 1877 printf("\nDone.\n"); 1878 test_done = 1; 1879 } 1880 1881 void 1882 dev_set_link_up(portid_t pid) 1883 { 1884 if (rte_eth_dev_set_link_up(pid) < 0) 1885 printf("\nSet link up fail.\n"); 1886 } 1887 1888 void 1889 dev_set_link_down(portid_t pid) 1890 { 1891 if (rte_eth_dev_set_link_down(pid) < 0) 1892 printf("\nSet link down fail.\n"); 1893 } 1894 1895 static int 1896 all_ports_started(void) 1897 { 1898 portid_t pi; 1899 struct rte_port *port; 1900 1901 RTE_ETH_FOREACH_DEV(pi) { 1902 port = &ports[pi]; 1903 /* Check if there is a port which is not started */ 1904 if ((port->port_status != RTE_PORT_STARTED) && 1905 (port->slave_flag == 0)) 1906 return 0; 1907 } 1908 1909 /* No port is not started */ 1910 return 1; 1911 } 1912 1913 int 1914 port_is_stopped(portid_t port_id) 1915 { 1916 struct rte_port *port = &ports[port_id]; 1917 1918 if ((port->port_status != RTE_PORT_STOPPED) && 1919 (port->slave_flag == 0)) 1920 return 0; 1921 return 1; 1922 } 1923 1924 int 1925 all_ports_stopped(void) 1926 { 1927 portid_t pi; 1928 1929 RTE_ETH_FOREACH_DEV(pi) { 1930 if (!port_is_stopped(pi)) 1931 return 0; 1932 } 1933 1934 return 1; 1935 } 1936 1937 int 1938 port_is_started(portid_t port_id) 1939 { 1940 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1941 return 0; 1942 1943 if (ports[port_id].port_status != RTE_PORT_STARTED) 1944 return 0; 1945 1946 return 1; 1947 } 1948 1949 int 1950 start_port(portid_t pid) 1951 { 1952 int diag, need_check_link_status = -1; 1953 portid_t pi; 1954 queueid_t qi; 1955 struct rte_port *port; 1956 struct ether_addr mac_addr; 1957 1958 if (port_id_is_invalid(pid, ENABLED_WARN)) 1959 return 0; 1960 1961 if(dcb_config) 1962 dcb_test = 1; 1963 RTE_ETH_FOREACH_DEV(pi) { 1964 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1965 continue; 1966 1967 need_check_link_status = 0; 1968 port = &ports[pi]; 1969 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1970 RTE_PORT_HANDLING) == 0) { 1971 printf("Port %d is now not stopped\n", pi); 1972 continue; 1973 } 1974 1975 if (port->need_reconfig > 0) { 1976 port->need_reconfig = 0; 1977 1978 if (flow_isolate_all) { 1979 int ret = port_flow_isolate(pi, 1); 1980 if (ret) { 1981 printf("Failed to apply isolated" 1982 " mode on port %d\n", pi); 1983 return -1; 1984 } 1985 } 1986 configure_rxtx_dump_callbacks(0); 1987 printf("Configuring Port %d (socket %u)\n", pi, 1988 port->socket_id); 1989 /* configure port */ 1990 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1991 &(port->dev_conf)); 1992 if (diag != 0) { 1993 if (rte_atomic16_cmpset(&(port->port_status), 1994 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1995 printf("Port %d can not be set back " 1996 "to stopped\n", pi); 1997 printf("Fail to configure port %d\n", pi); 1998 /* try to reconfigure port next time */ 1999 port->need_reconfig = 1; 2000 return -1; 2001 } 2002 } 2003 if (port->need_reconfig_queues > 0) { 2004 port->need_reconfig_queues = 0; 2005 /* setup tx queues */ 2006 for (qi = 0; qi < nb_txq; qi++) { 2007 if ((numa_support) && 2008 (txring_numa[pi] != NUMA_NO_CONFIG)) 2009 diag = rte_eth_tx_queue_setup(pi, qi, 2010 port->nb_tx_desc[qi], 2011 txring_numa[pi], 2012 &(port->tx_conf[qi])); 2013 else 2014 diag = rte_eth_tx_queue_setup(pi, qi, 2015 port->nb_tx_desc[qi], 2016 port->socket_id, 2017 &(port->tx_conf[qi])); 2018 2019 if (diag == 0) 2020 continue; 2021 2022 /* Fail to setup tx queue, return */ 2023 if (rte_atomic16_cmpset(&(port->port_status), 2024 RTE_PORT_HANDLING, 2025 RTE_PORT_STOPPED) == 0) 2026 printf("Port %d can not be set back " 2027 "to stopped\n", pi); 2028 printf("Fail to configure port %d tx queues\n", 2029 pi); 2030 /* try to reconfigure queues next time */ 2031 port->need_reconfig_queues = 1; 2032 return -1; 2033 } 2034 for (qi = 0; qi < nb_rxq; qi++) { 2035 /* setup rx queues */ 2036 if ((numa_support) && 2037 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2038 struct rte_mempool * mp = 2039 mbuf_pool_find(rxring_numa[pi]); 2040 if (mp == NULL) { 2041 printf("Failed to setup RX queue:" 2042 "No mempool allocation" 2043 " on the socket %d\n", 2044 rxring_numa[pi]); 2045 return -1; 2046 } 2047 2048 diag = rte_eth_rx_queue_setup(pi, qi, 2049 port->nb_rx_desc[qi], 2050 rxring_numa[pi], 2051 &(port->rx_conf[qi]), 2052 mp); 2053 } else { 2054 struct rte_mempool *mp = 2055 mbuf_pool_find(port->socket_id); 2056 if (mp == NULL) { 2057 printf("Failed to setup RX queue:" 2058 "No mempool allocation" 2059 " on the socket %d\n", 2060 port->socket_id); 2061 return -1; 2062 } 2063 diag = rte_eth_rx_queue_setup(pi, qi, 2064 port->nb_rx_desc[qi], 2065 port->socket_id, 2066 &(port->rx_conf[qi]), 2067 mp); 2068 } 2069 if (diag == 0) 2070 continue; 2071 2072 /* Fail to setup rx queue, return */ 2073 if (rte_atomic16_cmpset(&(port->port_status), 2074 RTE_PORT_HANDLING, 2075 RTE_PORT_STOPPED) == 0) 2076 printf("Port %d can not be set back " 2077 "to stopped\n", pi); 2078 printf("Fail to configure port %d rx queues\n", 2079 pi); 2080 /* try to reconfigure queues next time */ 2081 port->need_reconfig_queues = 1; 2082 return -1; 2083 } 2084 } 2085 configure_rxtx_dump_callbacks(verbose_level); 2086 /* start port */ 2087 if (rte_eth_dev_start(pi) < 0) { 2088 printf("Fail to start port %d\n", pi); 2089 2090 /* Fail to setup rx queue, return */ 2091 if (rte_atomic16_cmpset(&(port->port_status), 2092 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2093 printf("Port %d can not be set back to " 2094 "stopped\n", pi); 2095 continue; 2096 } 2097 2098 if (rte_atomic16_cmpset(&(port->port_status), 2099 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2100 printf("Port %d can not be set into started\n", pi); 2101 2102 rte_eth_macaddr_get(pi, &mac_addr); 2103 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2104 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2105 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2106 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2107 2108 /* at least one port started, need checking link status */ 2109 need_check_link_status = 1; 2110 } 2111 2112 if (need_check_link_status == 1 && !no_link_check) 2113 check_all_ports_link_status(RTE_PORT_ALL); 2114 else if (need_check_link_status == 0) 2115 printf("Please stop the ports first\n"); 2116 2117 printf("Done\n"); 2118 return 0; 2119 } 2120 2121 void 2122 stop_port(portid_t pid) 2123 { 2124 portid_t pi; 2125 struct rte_port *port; 2126 int need_check_link_status = 0; 2127 2128 if (dcb_test) { 2129 dcb_test = 0; 2130 dcb_config = 0; 2131 } 2132 2133 if (port_id_is_invalid(pid, ENABLED_WARN)) 2134 return; 2135 2136 printf("Stopping ports...\n"); 2137 2138 RTE_ETH_FOREACH_DEV(pi) { 2139 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2140 continue; 2141 2142 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2143 printf("Please remove port %d from forwarding configuration.\n", pi); 2144 continue; 2145 } 2146 2147 if (port_is_bonding_slave(pi)) { 2148 printf("Please remove port %d from bonded device.\n", pi); 2149 continue; 2150 } 2151 2152 port = &ports[pi]; 2153 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2154 RTE_PORT_HANDLING) == 0) 2155 continue; 2156 2157 rte_eth_dev_stop(pi); 2158 2159 if (rte_atomic16_cmpset(&(port->port_status), 2160 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2161 printf("Port %d can not be set into stopped\n", pi); 2162 need_check_link_status = 1; 2163 } 2164 if (need_check_link_status && !no_link_check) 2165 check_all_ports_link_status(RTE_PORT_ALL); 2166 2167 printf("Done\n"); 2168 } 2169 2170 static void 2171 remove_invalid_ports_in(portid_t *array, portid_t *total) 2172 { 2173 portid_t i; 2174 portid_t new_total = 0; 2175 2176 for (i = 0; i < *total; i++) 2177 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2178 array[new_total] = array[i]; 2179 new_total++; 2180 } 2181 *total = new_total; 2182 } 2183 2184 static void 2185 remove_invalid_ports(void) 2186 { 2187 remove_invalid_ports_in(ports_ids, &nb_ports); 2188 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2189 nb_cfg_ports = nb_fwd_ports; 2190 } 2191 2192 void 2193 close_port(portid_t pid) 2194 { 2195 portid_t pi; 2196 struct rte_port *port; 2197 2198 if (port_id_is_invalid(pid, ENABLED_WARN)) 2199 return; 2200 2201 printf("Closing ports...\n"); 2202 2203 RTE_ETH_FOREACH_DEV(pi) { 2204 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2205 continue; 2206 2207 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2208 printf("Please remove port %d from forwarding configuration.\n", pi); 2209 continue; 2210 } 2211 2212 if (port_is_bonding_slave(pi)) { 2213 printf("Please remove port %d from bonded device.\n", pi); 2214 continue; 2215 } 2216 2217 port = &ports[pi]; 2218 if (rte_atomic16_cmpset(&(port->port_status), 2219 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2220 printf("Port %d is already closed\n", pi); 2221 continue; 2222 } 2223 2224 if (rte_atomic16_cmpset(&(port->port_status), 2225 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2226 printf("Port %d is now not stopped\n", pi); 2227 continue; 2228 } 2229 2230 if (port->flow_list) 2231 port_flow_flush(pi); 2232 rte_eth_dev_close(pi); 2233 2234 remove_invalid_ports(); 2235 2236 if (rte_atomic16_cmpset(&(port->port_status), 2237 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2238 printf("Port %d cannot be set to closed\n", pi); 2239 } 2240 2241 printf("Done\n"); 2242 } 2243 2244 void 2245 reset_port(portid_t pid) 2246 { 2247 int diag; 2248 portid_t pi; 2249 struct rte_port *port; 2250 2251 if (port_id_is_invalid(pid, ENABLED_WARN)) 2252 return; 2253 2254 printf("Resetting ports...\n"); 2255 2256 RTE_ETH_FOREACH_DEV(pi) { 2257 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2258 continue; 2259 2260 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2261 printf("Please remove port %d from forwarding " 2262 "configuration.\n", pi); 2263 continue; 2264 } 2265 2266 if (port_is_bonding_slave(pi)) { 2267 printf("Please remove port %d from bonded device.\n", 2268 pi); 2269 continue; 2270 } 2271 2272 diag = rte_eth_dev_reset(pi); 2273 if (diag == 0) { 2274 port = &ports[pi]; 2275 port->need_reconfig = 1; 2276 port->need_reconfig_queues = 1; 2277 } else { 2278 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2279 } 2280 } 2281 2282 printf("Done\n"); 2283 } 2284 2285 void 2286 attach_port(char *identifier) 2287 { 2288 portid_t pi; 2289 struct rte_dev_iterator iterator; 2290 2291 printf("Attaching a new port...\n"); 2292 2293 if (identifier == NULL) { 2294 printf("Invalid parameters are specified\n"); 2295 return; 2296 } 2297 2298 if (rte_dev_probe(identifier) != 0) { 2299 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2300 return; 2301 } 2302 2303 /* first attach mode: event */ 2304 if (setup_on_probe_event) { 2305 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2306 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2307 if (ports[pi].port_status == RTE_PORT_HANDLING && 2308 ports[pi].need_setup != 0) 2309 setup_attached_port(pi); 2310 return; 2311 } 2312 2313 /* second attach mode: iterator */ 2314 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2315 /* setup ports matching the devargs used for probing */ 2316 if (port_is_forwarding(pi)) 2317 continue; /* port was already attached before */ 2318 setup_attached_port(pi); 2319 } 2320 } 2321 2322 static void 2323 setup_attached_port(portid_t pi) 2324 { 2325 unsigned int socket_id; 2326 2327 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2328 /* if socket_id is invalid, set to the first available socket. */ 2329 if (check_socket_id(socket_id) < 0) 2330 socket_id = socket_ids[0]; 2331 reconfig(pi, socket_id); 2332 rte_eth_promiscuous_enable(pi); 2333 2334 ports_ids[nb_ports++] = pi; 2335 fwd_ports_ids[nb_fwd_ports++] = pi; 2336 nb_cfg_ports = nb_fwd_ports; 2337 ports[pi].need_setup = 0; 2338 ports[pi].port_status = RTE_PORT_STOPPED; 2339 2340 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2341 printf("Done\n"); 2342 } 2343 2344 void 2345 detach_port_device(portid_t port_id) 2346 { 2347 struct rte_device *dev; 2348 portid_t sibling; 2349 2350 printf("Removing a device...\n"); 2351 2352 dev = rte_eth_devices[port_id].device; 2353 if (dev == NULL) { 2354 printf("Device already removed\n"); 2355 return; 2356 } 2357 2358 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2359 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2360 printf("Port not stopped\n"); 2361 return; 2362 } 2363 printf("Port was not closed\n"); 2364 if (ports[port_id].flow_list) 2365 port_flow_flush(port_id); 2366 } 2367 2368 if (rte_dev_remove(dev) != 0) { 2369 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2370 return; 2371 } 2372 2373 RTE_ETH_FOREACH_DEV_SIBLING(sibling, port_id) { 2374 /* reset mapping between old ports and removed device */ 2375 rte_eth_devices[sibling].device = NULL; 2376 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2377 /* sibling ports are forced to be closed */ 2378 ports[sibling].port_status = RTE_PORT_CLOSED; 2379 printf("Port %u is closed\n", sibling); 2380 } 2381 } 2382 2383 remove_invalid_ports(); 2384 2385 printf("Device of port %u is detached\n", port_id); 2386 printf("Now total ports is %d\n", nb_ports); 2387 printf("Done\n"); 2388 return; 2389 } 2390 2391 void 2392 pmd_test_exit(void) 2393 { 2394 struct rte_device *device; 2395 portid_t pt_id; 2396 int ret; 2397 2398 if (test_done == 0) 2399 stop_packet_forwarding(); 2400 2401 if (ports != NULL) { 2402 no_link_check = 1; 2403 RTE_ETH_FOREACH_DEV(pt_id) { 2404 printf("\nStopping port %d...\n", pt_id); 2405 fflush(stdout); 2406 stop_port(pt_id); 2407 } 2408 RTE_ETH_FOREACH_DEV(pt_id) { 2409 printf("\nShutting down port %d...\n", pt_id); 2410 fflush(stdout); 2411 close_port(pt_id); 2412 2413 /* 2414 * This is a workaround to fix a virtio-user issue that 2415 * requires to call clean-up routine to remove existing 2416 * socket. 2417 * This workaround valid only for testpmd, needs a fix 2418 * valid for all applications. 2419 * TODO: Implement proper resource cleanup 2420 */ 2421 device = rte_eth_devices[pt_id].device; 2422 if (device && !strcmp(device->driver->name, "net_virtio_user")) 2423 detach_port_device(pt_id); 2424 } 2425 } 2426 2427 if (hot_plug) { 2428 ret = rte_dev_event_monitor_stop(); 2429 if (ret) { 2430 RTE_LOG(ERR, EAL, 2431 "fail to stop device event monitor."); 2432 return; 2433 } 2434 2435 ret = rte_dev_event_callback_unregister(NULL, 2436 dev_event_callback, NULL); 2437 if (ret < 0) { 2438 RTE_LOG(ERR, EAL, 2439 "fail to unregister device event callback.\n"); 2440 return; 2441 } 2442 2443 ret = rte_dev_hotplug_handle_disable(); 2444 if (ret) { 2445 RTE_LOG(ERR, EAL, 2446 "fail to disable hotplug handling.\n"); 2447 return; 2448 } 2449 } 2450 2451 printf("\nBye...\n"); 2452 } 2453 2454 typedef void (*cmd_func_t)(void); 2455 struct pmd_test_command { 2456 const char *cmd_name; 2457 cmd_func_t cmd_func; 2458 }; 2459 2460 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2461 2462 /* Check the link status of all ports in up to 9s, and print them finally */ 2463 static void 2464 check_all_ports_link_status(uint32_t port_mask) 2465 { 2466 #define CHECK_INTERVAL 100 /* 100ms */ 2467 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2468 portid_t portid; 2469 uint8_t count, all_ports_up, print_flag = 0; 2470 struct rte_eth_link link; 2471 2472 printf("Checking link statuses...\n"); 2473 fflush(stdout); 2474 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2475 all_ports_up = 1; 2476 RTE_ETH_FOREACH_DEV(portid) { 2477 if ((port_mask & (1 << portid)) == 0) 2478 continue; 2479 memset(&link, 0, sizeof(link)); 2480 rte_eth_link_get_nowait(portid, &link); 2481 /* print link status if flag set */ 2482 if (print_flag == 1) { 2483 if (link.link_status) 2484 printf( 2485 "Port%d Link Up. speed %u Mbps- %s\n", 2486 portid, link.link_speed, 2487 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2488 ("full-duplex") : ("half-duplex\n")); 2489 else 2490 printf("Port %d Link Down\n", portid); 2491 continue; 2492 } 2493 /* clear all_ports_up flag if any link down */ 2494 if (link.link_status == ETH_LINK_DOWN) { 2495 all_ports_up = 0; 2496 break; 2497 } 2498 } 2499 /* after finally printing all link status, get out */ 2500 if (print_flag == 1) 2501 break; 2502 2503 if (all_ports_up == 0) { 2504 fflush(stdout); 2505 rte_delay_ms(CHECK_INTERVAL); 2506 } 2507 2508 /* set the print_flag if all ports up or timeout */ 2509 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2510 print_flag = 1; 2511 } 2512 2513 if (lsc_interrupt) 2514 break; 2515 } 2516 } 2517 2518 /* 2519 * This callback is for remove a port for a device. It has limitation because 2520 * it is not for multiple port removal for a device. 2521 * TODO: the device detach invoke will plan to be removed from user side to 2522 * eal. And convert all PMDs to free port resources on ether device closing. 2523 */ 2524 static void 2525 rmv_port_callback(void *arg) 2526 { 2527 int need_to_start = 0; 2528 int org_no_link_check = no_link_check; 2529 portid_t port_id = (intptr_t)arg; 2530 2531 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2532 2533 if (!test_done && port_is_forwarding(port_id)) { 2534 need_to_start = 1; 2535 stop_packet_forwarding(); 2536 } 2537 no_link_check = 1; 2538 stop_port(port_id); 2539 no_link_check = org_no_link_check; 2540 close_port(port_id); 2541 detach_port_device(port_id); 2542 if (need_to_start) 2543 start_packet_forwarding(0); 2544 } 2545 2546 /* This function is used by the interrupt thread */ 2547 static int 2548 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2549 void *ret_param) 2550 { 2551 RTE_SET_USED(param); 2552 RTE_SET_USED(ret_param); 2553 2554 if (type >= RTE_ETH_EVENT_MAX) { 2555 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2556 port_id, __func__, type); 2557 fflush(stderr); 2558 } else if (event_print_mask & (UINT32_C(1) << type)) { 2559 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2560 eth_event_desc[type]); 2561 fflush(stdout); 2562 } 2563 2564 switch (type) { 2565 case RTE_ETH_EVENT_NEW: 2566 ports[port_id].need_setup = 1; 2567 ports[port_id].port_status = RTE_PORT_HANDLING; 2568 break; 2569 case RTE_ETH_EVENT_INTR_RMV: 2570 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2571 break; 2572 if (rte_eal_alarm_set(100000, 2573 rmv_port_callback, (void *)(intptr_t)port_id)) 2574 fprintf(stderr, "Could not set up deferred device removal\n"); 2575 break; 2576 default: 2577 break; 2578 } 2579 return 0; 2580 } 2581 2582 static int 2583 register_eth_event_callback(void) 2584 { 2585 int ret; 2586 enum rte_eth_event_type event; 2587 2588 for (event = RTE_ETH_EVENT_UNKNOWN; 2589 event < RTE_ETH_EVENT_MAX; event++) { 2590 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2591 event, 2592 eth_event_callback, 2593 NULL); 2594 if (ret != 0) { 2595 TESTPMD_LOG(ERR, "Failed to register callback for " 2596 "%s event\n", eth_event_desc[event]); 2597 return -1; 2598 } 2599 } 2600 2601 return 0; 2602 } 2603 2604 /* This function is used by the interrupt thread */ 2605 static void 2606 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2607 __rte_unused void *arg) 2608 { 2609 uint16_t port_id; 2610 int ret; 2611 2612 if (type >= RTE_DEV_EVENT_MAX) { 2613 fprintf(stderr, "%s called upon invalid event %d\n", 2614 __func__, type); 2615 fflush(stderr); 2616 } 2617 2618 switch (type) { 2619 case RTE_DEV_EVENT_REMOVE: 2620 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2621 device_name); 2622 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2623 if (ret) { 2624 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2625 device_name); 2626 return; 2627 } 2628 /* 2629 * Because the user's callback is invoked in eal interrupt 2630 * callback, the interrupt callback need to be finished before 2631 * it can be unregistered when detaching device. So finish 2632 * callback soon and use a deferred removal to detach device 2633 * is need. It is a workaround, once the device detaching be 2634 * moved into the eal in the future, the deferred removal could 2635 * be deleted. 2636 */ 2637 if (rte_eal_alarm_set(100000, 2638 rmv_port_callback, (void *)(intptr_t)port_id)) 2639 RTE_LOG(ERR, EAL, 2640 "Could not set up deferred device removal\n"); 2641 break; 2642 case RTE_DEV_EVENT_ADD: 2643 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2644 device_name); 2645 /* TODO: After finish kernel driver binding, 2646 * begin to attach port. 2647 */ 2648 break; 2649 default: 2650 break; 2651 } 2652 } 2653 2654 static int 2655 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2656 { 2657 uint16_t i; 2658 int diag; 2659 uint8_t mapping_found = 0; 2660 2661 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2662 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2663 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2664 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2665 tx_queue_stats_mappings[i].queue_id, 2666 tx_queue_stats_mappings[i].stats_counter_id); 2667 if (diag != 0) 2668 return diag; 2669 mapping_found = 1; 2670 } 2671 } 2672 if (mapping_found) 2673 port->tx_queue_stats_mapping_enabled = 1; 2674 return 0; 2675 } 2676 2677 static int 2678 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2679 { 2680 uint16_t i; 2681 int diag; 2682 uint8_t mapping_found = 0; 2683 2684 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2685 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2686 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2687 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2688 rx_queue_stats_mappings[i].queue_id, 2689 rx_queue_stats_mappings[i].stats_counter_id); 2690 if (diag != 0) 2691 return diag; 2692 mapping_found = 1; 2693 } 2694 } 2695 if (mapping_found) 2696 port->rx_queue_stats_mapping_enabled = 1; 2697 return 0; 2698 } 2699 2700 static void 2701 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2702 { 2703 int diag = 0; 2704 2705 diag = set_tx_queue_stats_mapping_registers(pi, port); 2706 if (diag != 0) { 2707 if (diag == -ENOTSUP) { 2708 port->tx_queue_stats_mapping_enabled = 0; 2709 printf("TX queue stats mapping not supported port id=%d\n", pi); 2710 } 2711 else 2712 rte_exit(EXIT_FAILURE, 2713 "set_tx_queue_stats_mapping_registers " 2714 "failed for port id=%d diag=%d\n", 2715 pi, diag); 2716 } 2717 2718 diag = set_rx_queue_stats_mapping_registers(pi, port); 2719 if (diag != 0) { 2720 if (diag == -ENOTSUP) { 2721 port->rx_queue_stats_mapping_enabled = 0; 2722 printf("RX queue stats mapping not supported port id=%d\n", pi); 2723 } 2724 else 2725 rte_exit(EXIT_FAILURE, 2726 "set_rx_queue_stats_mapping_registers " 2727 "failed for port id=%d diag=%d\n", 2728 pi, diag); 2729 } 2730 } 2731 2732 static void 2733 rxtx_port_config(struct rte_port *port) 2734 { 2735 uint16_t qid; 2736 2737 for (qid = 0; qid < nb_rxq; qid++) { 2738 port->rx_conf[qid] = port->dev_info.default_rxconf; 2739 2740 /* Check if any Rx parameters have been passed */ 2741 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2742 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2743 2744 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2745 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2746 2747 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2748 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2749 2750 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2751 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2752 2753 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2754 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2755 2756 port->nb_rx_desc[qid] = nb_rxd; 2757 } 2758 2759 for (qid = 0; qid < nb_txq; qid++) { 2760 port->tx_conf[qid] = port->dev_info.default_txconf; 2761 2762 /* Check if any Tx parameters have been passed */ 2763 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2764 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2765 2766 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2767 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2768 2769 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2770 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2771 2772 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2773 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2774 2775 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2776 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2777 2778 port->nb_tx_desc[qid] = nb_txd; 2779 } 2780 } 2781 2782 void 2783 init_port_config(void) 2784 { 2785 portid_t pid; 2786 struct rte_port *port; 2787 2788 RTE_ETH_FOREACH_DEV(pid) { 2789 port = &ports[pid]; 2790 port->dev_conf.fdir_conf = fdir_conf; 2791 rte_eth_dev_info_get(pid, &port->dev_info); 2792 if (nb_rxq > 1) { 2793 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2794 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2795 rss_hf & port->dev_info.flow_type_rss_offloads; 2796 } else { 2797 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2798 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2799 } 2800 2801 if (port->dcb_flag == 0) { 2802 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2803 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2804 else 2805 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2806 } 2807 2808 rxtx_port_config(port); 2809 2810 rte_eth_macaddr_get(pid, &port->eth_addr); 2811 2812 map_port_queue_stats_mapping_registers(pid, port); 2813 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2814 rte_pmd_ixgbe_bypass_init(pid); 2815 #endif 2816 2817 if (lsc_interrupt && 2818 (rte_eth_devices[pid].data->dev_flags & 2819 RTE_ETH_DEV_INTR_LSC)) 2820 port->dev_conf.intr_conf.lsc = 1; 2821 if (rmv_interrupt && 2822 (rte_eth_devices[pid].data->dev_flags & 2823 RTE_ETH_DEV_INTR_RMV)) 2824 port->dev_conf.intr_conf.rmv = 1; 2825 } 2826 } 2827 2828 void set_port_slave_flag(portid_t slave_pid) 2829 { 2830 struct rte_port *port; 2831 2832 port = &ports[slave_pid]; 2833 port->slave_flag = 1; 2834 } 2835 2836 void clear_port_slave_flag(portid_t slave_pid) 2837 { 2838 struct rte_port *port; 2839 2840 port = &ports[slave_pid]; 2841 port->slave_flag = 0; 2842 } 2843 2844 uint8_t port_is_bonding_slave(portid_t slave_pid) 2845 { 2846 struct rte_port *port; 2847 2848 port = &ports[slave_pid]; 2849 if ((rte_eth_devices[slave_pid].data->dev_flags & 2850 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2851 return 1; 2852 return 0; 2853 } 2854 2855 const uint16_t vlan_tags[] = { 2856 0, 1, 2, 3, 4, 5, 6, 7, 2857 8, 9, 10, 11, 12, 13, 14, 15, 2858 16, 17, 18, 19, 20, 21, 22, 23, 2859 24, 25, 26, 27, 28, 29, 30, 31 2860 }; 2861 2862 static int 2863 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 2864 enum dcb_mode_enable dcb_mode, 2865 enum rte_eth_nb_tcs num_tcs, 2866 uint8_t pfc_en) 2867 { 2868 uint8_t i; 2869 int32_t rc; 2870 struct rte_eth_rss_conf rss_conf; 2871 2872 /* 2873 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2874 * given above, and the number of traffic classes available for use. 2875 */ 2876 if (dcb_mode == DCB_VT_ENABLED) { 2877 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2878 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2879 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2880 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2881 2882 /* VMDQ+DCB RX and TX configurations */ 2883 vmdq_rx_conf->enable_default_pool = 0; 2884 vmdq_rx_conf->default_pool = 0; 2885 vmdq_rx_conf->nb_queue_pools = 2886 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2887 vmdq_tx_conf->nb_queue_pools = 2888 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2889 2890 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2891 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2892 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2893 vmdq_rx_conf->pool_map[i].pools = 2894 1 << (i % vmdq_rx_conf->nb_queue_pools); 2895 } 2896 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2897 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 2898 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 2899 } 2900 2901 /* set DCB mode of RX and TX of multiple queues */ 2902 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2903 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2904 } else { 2905 struct rte_eth_dcb_rx_conf *rx_conf = 2906 ð_conf->rx_adv_conf.dcb_rx_conf; 2907 struct rte_eth_dcb_tx_conf *tx_conf = 2908 ð_conf->tx_adv_conf.dcb_tx_conf; 2909 2910 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 2911 if (rc != 0) 2912 return rc; 2913 2914 rx_conf->nb_tcs = num_tcs; 2915 tx_conf->nb_tcs = num_tcs; 2916 2917 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2918 rx_conf->dcb_tc[i] = i % num_tcs; 2919 tx_conf->dcb_tc[i] = i % num_tcs; 2920 } 2921 2922 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2923 eth_conf->rx_adv_conf.rss_conf = rss_conf; 2924 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2925 } 2926 2927 if (pfc_en) 2928 eth_conf->dcb_capability_en = 2929 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2930 else 2931 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2932 2933 return 0; 2934 } 2935 2936 int 2937 init_port_dcb_config(portid_t pid, 2938 enum dcb_mode_enable dcb_mode, 2939 enum rte_eth_nb_tcs num_tcs, 2940 uint8_t pfc_en) 2941 { 2942 struct rte_eth_conf port_conf; 2943 struct rte_port *rte_port; 2944 int retval; 2945 uint16_t i; 2946 2947 rte_port = &ports[pid]; 2948 2949 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2950 /* Enter DCB configuration status */ 2951 dcb_config = 1; 2952 2953 port_conf.rxmode = rte_port->dev_conf.rxmode; 2954 port_conf.txmode = rte_port->dev_conf.txmode; 2955 2956 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2957 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 2958 if (retval < 0) 2959 return retval; 2960 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2961 2962 /* re-configure the device . */ 2963 rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 2964 2965 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2966 2967 /* If dev_info.vmdq_pool_base is greater than 0, 2968 * the queue id of vmdq pools is started after pf queues. 2969 */ 2970 if (dcb_mode == DCB_VT_ENABLED && 2971 rte_port->dev_info.vmdq_pool_base > 0) { 2972 printf("VMDQ_DCB multi-queue mode is nonsensical" 2973 " for port %d.", pid); 2974 return -1; 2975 } 2976 2977 /* Assume the ports in testpmd have the same dcb capability 2978 * and has the same number of rxq and txq in dcb mode 2979 */ 2980 if (dcb_mode == DCB_VT_ENABLED) { 2981 if (rte_port->dev_info.max_vfs > 0) { 2982 nb_rxq = rte_port->dev_info.nb_rx_queues; 2983 nb_txq = rte_port->dev_info.nb_tx_queues; 2984 } else { 2985 nb_rxq = rte_port->dev_info.max_rx_queues; 2986 nb_txq = rte_port->dev_info.max_tx_queues; 2987 } 2988 } else { 2989 /*if vt is disabled, use all pf queues */ 2990 if (rte_port->dev_info.vmdq_pool_base == 0) { 2991 nb_rxq = rte_port->dev_info.max_rx_queues; 2992 nb_txq = rte_port->dev_info.max_tx_queues; 2993 } else { 2994 nb_rxq = (queueid_t)num_tcs; 2995 nb_txq = (queueid_t)num_tcs; 2996 2997 } 2998 } 2999 rx_free_thresh = 64; 3000 3001 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3002 3003 rxtx_port_config(rte_port); 3004 /* VLAN filter */ 3005 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3006 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3007 rx_vft_set(pid, vlan_tags[i], 1); 3008 3009 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 3010 map_port_queue_stats_mapping_registers(pid, rte_port); 3011 3012 rte_port->dcb_flag = 1; 3013 3014 return 0; 3015 } 3016 3017 static void 3018 init_port(void) 3019 { 3020 /* Configuration of Ethernet ports. */ 3021 ports = rte_zmalloc("testpmd: ports", 3022 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3023 RTE_CACHE_LINE_SIZE); 3024 if (ports == NULL) { 3025 rte_exit(EXIT_FAILURE, 3026 "rte_zmalloc(%d struct rte_port) failed\n", 3027 RTE_MAX_ETHPORTS); 3028 } 3029 3030 /* Initialize ports NUMA structures */ 3031 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3032 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3033 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3034 } 3035 3036 static void 3037 force_quit(void) 3038 { 3039 pmd_test_exit(); 3040 prompt_exit(); 3041 } 3042 3043 static void 3044 print_stats(void) 3045 { 3046 uint8_t i; 3047 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3048 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3049 3050 /* Clear screen and move to top left */ 3051 printf("%s%s", clr, top_left); 3052 3053 printf("\nPort statistics ===================================="); 3054 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3055 nic_stats_display(fwd_ports_ids[i]); 3056 3057 fflush(stdout); 3058 } 3059 3060 static void 3061 signal_handler(int signum) 3062 { 3063 if (signum == SIGINT || signum == SIGTERM) { 3064 printf("\nSignal %d received, preparing to exit...\n", 3065 signum); 3066 #ifdef RTE_LIBRTE_PDUMP 3067 /* uninitialize packet capture framework */ 3068 rte_pdump_uninit(); 3069 #endif 3070 #ifdef RTE_LIBRTE_LATENCY_STATS 3071 rte_latencystats_uninit(); 3072 #endif 3073 force_quit(); 3074 /* Set flag to indicate the force termination. */ 3075 f_quit = 1; 3076 /* exit with the expected status */ 3077 signal(signum, SIG_DFL); 3078 kill(getpid(), signum); 3079 } 3080 } 3081 3082 int 3083 main(int argc, char** argv) 3084 { 3085 int diag; 3086 portid_t port_id; 3087 uint16_t count; 3088 int ret; 3089 3090 signal(SIGINT, signal_handler); 3091 signal(SIGTERM, signal_handler); 3092 3093 diag = rte_eal_init(argc, argv); 3094 if (diag < 0) 3095 rte_panic("Cannot init EAL\n"); 3096 3097 testpmd_logtype = rte_log_register("testpmd"); 3098 if (testpmd_logtype < 0) 3099 rte_panic("Cannot register log type"); 3100 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3101 3102 ret = register_eth_event_callback(); 3103 if (ret != 0) 3104 rte_panic("Cannot register for ethdev events"); 3105 3106 #ifdef RTE_LIBRTE_PDUMP 3107 /* initialize packet capture framework */ 3108 rte_pdump_init(); 3109 #endif 3110 3111 count = 0; 3112 RTE_ETH_FOREACH_DEV(port_id) { 3113 ports_ids[count] = port_id; 3114 count++; 3115 } 3116 nb_ports = (portid_t) count; 3117 if (nb_ports == 0) 3118 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3119 3120 /* allocate port structures, and init them */ 3121 init_port(); 3122 3123 set_def_fwd_config(); 3124 if (nb_lcores == 0) 3125 rte_panic("Empty set of forwarding logical cores - check the " 3126 "core mask supplied in the command parameters\n"); 3127 3128 /* Bitrate/latency stats disabled by default */ 3129 #ifdef RTE_LIBRTE_BITRATE 3130 bitrate_enabled = 0; 3131 #endif 3132 #ifdef RTE_LIBRTE_LATENCY_STATS 3133 latencystats_enabled = 0; 3134 #endif 3135 3136 /* on FreeBSD, mlockall() is disabled by default */ 3137 #ifdef RTE_EXEC_ENV_FREEBSD 3138 do_mlockall = 0; 3139 #else 3140 do_mlockall = 1; 3141 #endif 3142 3143 argc -= diag; 3144 argv += diag; 3145 if (argc > 1) 3146 launch_args_parse(argc, argv); 3147 3148 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3149 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3150 strerror(errno)); 3151 } 3152 3153 if (tx_first && interactive) 3154 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3155 "interactive mode.\n"); 3156 3157 if (tx_first && lsc_interrupt) { 3158 printf("Warning: lsc_interrupt needs to be off when " 3159 " using tx_first. Disabling.\n"); 3160 lsc_interrupt = 0; 3161 } 3162 3163 if (!nb_rxq && !nb_txq) 3164 printf("Warning: Either rx or tx queues should be non-zero\n"); 3165 3166 if (nb_rxq > 1 && nb_rxq > nb_txq) 3167 printf("Warning: nb_rxq=%d enables RSS configuration, " 3168 "but nb_txq=%d will prevent to fully test it.\n", 3169 nb_rxq, nb_txq); 3170 3171 init_config(); 3172 3173 if (hot_plug) { 3174 ret = rte_dev_hotplug_handle_enable(); 3175 if (ret) { 3176 RTE_LOG(ERR, EAL, 3177 "fail to enable hotplug handling."); 3178 return -1; 3179 } 3180 3181 ret = rte_dev_event_monitor_start(); 3182 if (ret) { 3183 RTE_LOG(ERR, EAL, 3184 "fail to start device event monitoring."); 3185 return -1; 3186 } 3187 3188 ret = rte_dev_event_callback_register(NULL, 3189 dev_event_callback, NULL); 3190 if (ret) { 3191 RTE_LOG(ERR, EAL, 3192 "fail to register device event callback\n"); 3193 return -1; 3194 } 3195 } 3196 3197 if (start_port(RTE_PORT_ALL) != 0) 3198 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3199 3200 /* set all ports to promiscuous mode by default */ 3201 RTE_ETH_FOREACH_DEV(port_id) 3202 rte_eth_promiscuous_enable(port_id); 3203 3204 /* Init metrics library */ 3205 rte_metrics_init(rte_socket_id()); 3206 3207 #ifdef RTE_LIBRTE_LATENCY_STATS 3208 if (latencystats_enabled != 0) { 3209 int ret = rte_latencystats_init(1, NULL); 3210 if (ret) 3211 printf("Warning: latencystats init()" 3212 " returned error %d\n", ret); 3213 printf("Latencystats running on lcore %d\n", 3214 latencystats_lcore_id); 3215 } 3216 #endif 3217 3218 /* Setup bitrate stats */ 3219 #ifdef RTE_LIBRTE_BITRATE 3220 if (bitrate_enabled != 0) { 3221 bitrate_data = rte_stats_bitrate_create(); 3222 if (bitrate_data == NULL) 3223 rte_exit(EXIT_FAILURE, 3224 "Could not allocate bitrate data.\n"); 3225 rte_stats_bitrate_reg(bitrate_data); 3226 } 3227 #endif 3228 3229 #ifdef RTE_LIBRTE_CMDLINE 3230 if (strlen(cmdline_filename) != 0) 3231 cmdline_read_from_file(cmdline_filename); 3232 3233 if (interactive == 1) { 3234 if (auto_start) { 3235 printf("Start automatic packet forwarding\n"); 3236 start_packet_forwarding(0); 3237 } 3238 prompt(); 3239 pmd_test_exit(); 3240 } else 3241 #endif 3242 { 3243 char c; 3244 int rc; 3245 3246 f_quit = 0; 3247 3248 printf("No commandline core given, start packet forwarding\n"); 3249 start_packet_forwarding(tx_first); 3250 if (stats_period != 0) { 3251 uint64_t prev_time = 0, cur_time, diff_time = 0; 3252 uint64_t timer_period; 3253 3254 /* Convert to number of cycles */ 3255 timer_period = stats_period * rte_get_timer_hz(); 3256 3257 while (f_quit == 0) { 3258 cur_time = rte_get_timer_cycles(); 3259 diff_time += cur_time - prev_time; 3260 3261 if (diff_time >= timer_period) { 3262 print_stats(); 3263 /* Reset the timer */ 3264 diff_time = 0; 3265 } 3266 /* Sleep to avoid unnecessary checks */ 3267 prev_time = cur_time; 3268 sleep(1); 3269 } 3270 } 3271 3272 printf("Press enter to exit\n"); 3273 rc = read(0, &c, 1); 3274 pmd_test_exit(); 3275 if (rc < 0) 3276 return 1; 3277 } 3278 3279 return 0; 3280 } 3281