1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct fwd_config cur_fwd_config; 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 193 uint32_t retry_enabled; 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 196 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 198 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 199 * specified on command-line. */ 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 201 202 /* 203 * In container, it cannot terminate the process which running with 'stats-period' 204 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 205 */ 206 uint8_t f_quit; 207 208 /* 209 * Configuration of packet segments used by the "txonly" processing engine. 210 */ 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 213 TXONLY_DEF_PACKET_LEN, 214 }; 215 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 216 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 218 /**< Split policy for packets to TX. */ 219 220 uint8_t txonly_multi_flow; 221 /**< Whether multiple flows are generated in TXONLY mode. */ 222 223 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 224 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 225 226 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 227 uint8_t dcb_config = 0; 228 229 /* Whether the dcb is in testing status */ 230 uint8_t dcb_test = 0; 231 232 /* 233 * Configurable number of RX/TX queues. 234 */ 235 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 236 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 237 238 /* 239 * Configurable number of RX/TX ring descriptors. 240 * Defaults are supplied by drivers via ethdev. 241 */ 242 #define RTE_TEST_RX_DESC_DEFAULT 0 243 #define RTE_TEST_TX_DESC_DEFAULT 0 244 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 245 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 246 247 #define RTE_PMD_PARAM_UNSET -1 248 /* 249 * Configurable values of RX and TX ring threshold registers. 250 */ 251 252 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 253 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 254 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 255 256 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 257 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 258 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 259 260 /* 261 * Configurable value of RX free threshold. 262 */ 263 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 264 265 /* 266 * Configurable value of RX drop enable. 267 */ 268 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 269 270 /* 271 * Configurable value of TX free threshold. 272 */ 273 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 274 275 /* 276 * Configurable value of TX RS bit threshold. 277 */ 278 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 279 280 /* 281 * Configurable value of buffered packets before sending. 282 */ 283 uint16_t noisy_tx_sw_bufsz; 284 285 /* 286 * Configurable value of packet buffer timeout. 287 */ 288 uint16_t noisy_tx_sw_buf_flush_time; 289 290 /* 291 * Configurable value for size of VNF internal memory area 292 * used for simulating noisy neighbour behaviour 293 */ 294 uint64_t noisy_lkup_mem_sz; 295 296 /* 297 * Configurable value of number of random writes done in 298 * VNF simulation memory area. 299 */ 300 uint64_t noisy_lkup_num_writes; 301 302 /* 303 * Configurable value of number of random reads done in 304 * VNF simulation memory area. 305 */ 306 uint64_t noisy_lkup_num_reads; 307 308 /* 309 * Configurable value of number of random reads/writes done in 310 * VNF simulation memory area. 311 */ 312 uint64_t noisy_lkup_num_reads_writes; 313 314 /* 315 * Receive Side Scaling (RSS) configuration. 316 */ 317 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 318 319 /* 320 * Port topology configuration 321 */ 322 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 323 324 /* 325 * Avoids to flush all the RX streams before starts forwarding. 326 */ 327 uint8_t no_flush_rx = 0; /* flush by default */ 328 329 /* 330 * Flow API isolated mode. 331 */ 332 uint8_t flow_isolate_all; 333 334 /* 335 * Avoids to check link status when starting/stopping a port. 336 */ 337 uint8_t no_link_check = 0; /* check by default */ 338 339 /* 340 * Enable link status change notification 341 */ 342 uint8_t lsc_interrupt = 1; /* enabled by default */ 343 344 /* 345 * Enable device removal notification. 346 */ 347 uint8_t rmv_interrupt = 1; /* enabled by default */ 348 349 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 350 351 /* After attach, port setup is called on event or by iterator */ 352 bool setup_on_probe_event = true; 353 354 /* Pretty printing of ethdev events */ 355 static const char * const eth_event_desc[] = { 356 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 357 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 358 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 359 [RTE_ETH_EVENT_INTR_RESET] = "reset", 360 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 361 [RTE_ETH_EVENT_IPSEC] = "IPsec", 362 [RTE_ETH_EVENT_MACSEC] = "MACsec", 363 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 364 [RTE_ETH_EVENT_NEW] = "device probed", 365 [RTE_ETH_EVENT_DESTROY] = "device released", 366 [RTE_ETH_EVENT_MAX] = NULL, 367 }; 368 369 /* 370 * Display or mask ether events 371 * Default to all events except VF_MBOX 372 */ 373 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 374 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 375 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 376 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 377 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 378 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 379 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 380 /* 381 * Decide if all memory are locked for performance. 382 */ 383 int do_mlockall = 0; 384 385 /* 386 * NIC bypass mode configuration options. 387 */ 388 389 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 390 /* The NIC bypass watchdog timeout. */ 391 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 392 #endif 393 394 395 #ifdef RTE_LIBRTE_LATENCY_STATS 396 397 /* 398 * Set when latency stats is enabled in the commandline 399 */ 400 uint8_t latencystats_enabled; 401 402 /* 403 * Lcore ID to serive latency statistics. 404 */ 405 lcoreid_t latencystats_lcore_id = -1; 406 407 #endif 408 409 /* 410 * Ethernet device configuration. 411 */ 412 struct rte_eth_rxmode rx_mode = { 413 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 414 }; 415 416 struct rte_eth_txmode tx_mode = { 417 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 418 }; 419 420 struct rte_fdir_conf fdir_conf = { 421 .mode = RTE_FDIR_MODE_NONE, 422 .pballoc = RTE_FDIR_PBALLOC_64K, 423 .status = RTE_FDIR_REPORT_STATUS, 424 .mask = { 425 .vlan_tci_mask = 0xFFEF, 426 .ipv4_mask = { 427 .src_ip = 0xFFFFFFFF, 428 .dst_ip = 0xFFFFFFFF, 429 }, 430 .ipv6_mask = { 431 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 432 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 433 }, 434 .src_port_mask = 0xFFFF, 435 .dst_port_mask = 0xFFFF, 436 .mac_addr_byte_mask = 0xFF, 437 .tunnel_type_mask = 1, 438 .tunnel_id_mask = 0xFFFFFFFF, 439 }, 440 .drop_queue = 127, 441 }; 442 443 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 444 445 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 446 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 447 448 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 449 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 450 451 uint16_t nb_tx_queue_stats_mappings = 0; 452 uint16_t nb_rx_queue_stats_mappings = 0; 453 454 /* 455 * Display zero values by default for xstats 456 */ 457 uint8_t xstats_hide_zero; 458 459 unsigned int num_sockets = 0; 460 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 461 462 #ifdef RTE_LIBRTE_BITRATE 463 /* Bitrate statistics */ 464 struct rte_stats_bitrates *bitrate_data; 465 lcoreid_t bitrate_lcore_id; 466 uint8_t bitrate_enabled; 467 #endif 468 469 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 470 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 471 472 struct vxlan_encap_conf vxlan_encap_conf = { 473 .select_ipv4 = 1, 474 .select_vlan = 0, 475 .select_tos_ttl = 0, 476 .vni = "\x00\x00\x00", 477 .udp_src = 0, 478 .udp_dst = RTE_BE16(4789), 479 .ipv4_src = IPv4(127, 0, 0, 1), 480 .ipv4_dst = IPv4(255, 255, 255, 255), 481 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 482 "\x00\x00\x00\x00\x00\x00\x00\x01", 483 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 484 "\x00\x00\x00\x00\x00\x00\x11\x11", 485 .vlan_tci = 0, 486 .ip_tos = 0, 487 .ip_ttl = 255, 488 .eth_src = "\x00\x00\x00\x00\x00\x00", 489 .eth_dst = "\xff\xff\xff\xff\xff\xff", 490 }; 491 492 struct nvgre_encap_conf nvgre_encap_conf = { 493 .select_ipv4 = 1, 494 .select_vlan = 0, 495 .tni = "\x00\x00\x00", 496 .ipv4_src = IPv4(127, 0, 0, 1), 497 .ipv4_dst = IPv4(255, 255, 255, 255), 498 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 499 "\x00\x00\x00\x00\x00\x00\x00\x01", 500 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 501 "\x00\x00\x00\x00\x00\x00\x11\x11", 502 .vlan_tci = 0, 503 .eth_src = "\x00\x00\x00\x00\x00\x00", 504 .eth_dst = "\xff\xff\xff\xff\xff\xff", 505 }; 506 507 /* Forward function declarations */ 508 static void setup_attached_port(portid_t pi); 509 static void map_port_queue_stats_mapping_registers(portid_t pi, 510 struct rte_port *port); 511 static void check_all_ports_link_status(uint32_t port_mask); 512 static int eth_event_callback(portid_t port_id, 513 enum rte_eth_event_type type, 514 void *param, void *ret_param); 515 static void dev_event_callback(const char *device_name, 516 enum rte_dev_event_type type, 517 void *param); 518 519 /* 520 * Check if all the ports are started. 521 * If yes, return positive value. If not, return zero. 522 */ 523 static int all_ports_started(void); 524 525 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 526 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN; 527 528 /* 529 * Helper function to check if socket is already discovered. 530 * If yes, return positive value. If not, return zero. 531 */ 532 int 533 new_socket_id(unsigned int socket_id) 534 { 535 unsigned int i; 536 537 for (i = 0; i < num_sockets; i++) { 538 if (socket_ids[i] == socket_id) 539 return 0; 540 } 541 return 1; 542 } 543 544 /* 545 * Setup default configuration. 546 */ 547 static void 548 set_default_fwd_lcores_config(void) 549 { 550 unsigned int i; 551 unsigned int nb_lc; 552 unsigned int sock_num; 553 554 nb_lc = 0; 555 for (i = 0; i < RTE_MAX_LCORE; i++) { 556 if (!rte_lcore_is_enabled(i)) 557 continue; 558 sock_num = rte_lcore_to_socket_id(i); 559 if (new_socket_id(sock_num)) { 560 if (num_sockets >= RTE_MAX_NUMA_NODES) { 561 rte_exit(EXIT_FAILURE, 562 "Total sockets greater than %u\n", 563 RTE_MAX_NUMA_NODES); 564 } 565 socket_ids[num_sockets++] = sock_num; 566 } 567 if (i == rte_get_master_lcore()) 568 continue; 569 fwd_lcores_cpuids[nb_lc++] = i; 570 } 571 nb_lcores = (lcoreid_t) nb_lc; 572 nb_cfg_lcores = nb_lcores; 573 nb_fwd_lcores = 1; 574 } 575 576 static void 577 set_def_peer_eth_addrs(void) 578 { 579 portid_t i; 580 581 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 582 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 583 peer_eth_addrs[i].addr_bytes[5] = i; 584 } 585 } 586 587 static void 588 set_default_fwd_ports_config(void) 589 { 590 portid_t pt_id; 591 int i = 0; 592 593 RTE_ETH_FOREACH_DEV(pt_id) { 594 fwd_ports_ids[i++] = pt_id; 595 596 /* Update sockets info according to the attached device */ 597 int socket_id = rte_eth_dev_socket_id(pt_id); 598 if (socket_id >= 0 && new_socket_id(socket_id)) { 599 if (num_sockets >= RTE_MAX_NUMA_NODES) { 600 rte_exit(EXIT_FAILURE, 601 "Total sockets greater than %u\n", 602 RTE_MAX_NUMA_NODES); 603 } 604 socket_ids[num_sockets++] = socket_id; 605 } 606 } 607 608 nb_cfg_ports = nb_ports; 609 nb_fwd_ports = nb_ports; 610 } 611 612 void 613 set_def_fwd_config(void) 614 { 615 set_default_fwd_lcores_config(); 616 set_def_peer_eth_addrs(); 617 set_default_fwd_ports_config(); 618 } 619 620 /* extremely pessimistic estimation of memory required to create a mempool */ 621 static int 622 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 623 { 624 unsigned int n_pages, mbuf_per_pg, leftover; 625 uint64_t total_mem, mbuf_mem, obj_sz; 626 627 /* there is no good way to predict how much space the mempool will 628 * occupy because it will allocate chunks on the fly, and some of those 629 * will come from default DPDK memory while some will come from our 630 * external memory, so just assume 128MB will be enough for everyone. 631 */ 632 uint64_t hdr_mem = 128 << 20; 633 634 /* account for possible non-contiguousness */ 635 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 636 if (obj_sz > pgsz) { 637 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 638 return -1; 639 } 640 641 mbuf_per_pg = pgsz / obj_sz; 642 leftover = (nb_mbufs % mbuf_per_pg) > 0; 643 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 644 645 mbuf_mem = n_pages * pgsz; 646 647 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 648 649 if (total_mem > SIZE_MAX) { 650 TESTPMD_LOG(ERR, "Memory size too big\n"); 651 return -1; 652 } 653 *out = (size_t)total_mem; 654 655 return 0; 656 } 657 658 static int 659 pagesz_flags(uint64_t page_sz) 660 { 661 /* as per mmap() manpage, all page sizes are log2 of page size 662 * shifted by MAP_HUGE_SHIFT 663 */ 664 int log2 = rte_log2_u64(page_sz); 665 666 return (log2 << HUGE_SHIFT); 667 } 668 669 static void * 670 alloc_mem(size_t memsz, size_t pgsz, bool huge) 671 { 672 void *addr; 673 int flags; 674 675 /* allocate anonymous hugepages */ 676 flags = MAP_ANONYMOUS | MAP_PRIVATE; 677 if (huge) 678 flags |= HUGE_FLAG | pagesz_flags(pgsz); 679 680 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 681 if (addr == MAP_FAILED) 682 return NULL; 683 684 return addr; 685 } 686 687 struct extmem_param { 688 void *addr; 689 size_t len; 690 size_t pgsz; 691 rte_iova_t *iova_table; 692 unsigned int iova_table_len; 693 }; 694 695 static int 696 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 697 bool huge) 698 { 699 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 700 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 701 unsigned int cur_page, n_pages, pgsz_idx; 702 size_t mem_sz, cur_pgsz; 703 rte_iova_t *iovas = NULL; 704 void *addr; 705 int ret; 706 707 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 708 /* skip anything that is too big */ 709 if (pgsizes[pgsz_idx] > SIZE_MAX) 710 continue; 711 712 cur_pgsz = pgsizes[pgsz_idx]; 713 714 /* if we were told not to allocate hugepages, override */ 715 if (!huge) 716 cur_pgsz = sysconf(_SC_PAGESIZE); 717 718 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 719 if (ret < 0) { 720 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 721 return -1; 722 } 723 724 /* allocate our memory */ 725 addr = alloc_mem(mem_sz, cur_pgsz, huge); 726 727 /* if we couldn't allocate memory with a specified page size, 728 * that doesn't mean we can't do it with other page sizes, so 729 * try another one. 730 */ 731 if (addr == NULL) 732 continue; 733 734 /* store IOVA addresses for every page in this memory area */ 735 n_pages = mem_sz / cur_pgsz; 736 737 iovas = malloc(sizeof(*iovas) * n_pages); 738 739 if (iovas == NULL) { 740 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 741 goto fail; 742 } 743 /* lock memory if it's not huge pages */ 744 if (!huge) 745 mlock(addr, mem_sz); 746 747 /* populate IOVA addresses */ 748 for (cur_page = 0; cur_page < n_pages; cur_page++) { 749 rte_iova_t iova; 750 size_t offset; 751 void *cur; 752 753 offset = cur_pgsz * cur_page; 754 cur = RTE_PTR_ADD(addr, offset); 755 756 /* touch the page before getting its IOVA */ 757 *(volatile char *)cur = 0; 758 759 iova = rte_mem_virt2iova(cur); 760 761 iovas[cur_page] = iova; 762 } 763 764 break; 765 } 766 /* if we couldn't allocate anything */ 767 if (iovas == NULL) 768 return -1; 769 770 param->addr = addr; 771 param->len = mem_sz; 772 param->pgsz = cur_pgsz; 773 param->iova_table = iovas; 774 param->iova_table_len = n_pages; 775 776 return 0; 777 fail: 778 if (iovas) 779 free(iovas); 780 if (addr) 781 munmap(addr, mem_sz); 782 783 return -1; 784 } 785 786 static int 787 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 788 { 789 struct extmem_param param; 790 int socket_id, ret; 791 792 memset(¶m, 0, sizeof(param)); 793 794 /* check if our heap exists */ 795 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 796 if (socket_id < 0) { 797 /* create our heap */ 798 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 799 if (ret < 0) { 800 TESTPMD_LOG(ERR, "Cannot create heap\n"); 801 return -1; 802 } 803 } 804 805 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 806 if (ret < 0) { 807 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 808 return -1; 809 } 810 811 /* we now have a valid memory area, so add it to heap */ 812 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 813 param.addr, param.len, param.iova_table, 814 param.iova_table_len, param.pgsz); 815 816 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 817 818 /* not needed any more */ 819 free(param.iova_table); 820 821 if (ret < 0) { 822 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 823 munmap(param.addr, param.len); 824 return -1; 825 } 826 827 /* success */ 828 829 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 830 param.len >> 20); 831 832 return 0; 833 } 834 835 /* 836 * Configuration initialisation done once at init time. 837 */ 838 static void 839 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 840 unsigned int socket_id) 841 { 842 char pool_name[RTE_MEMPOOL_NAMESIZE]; 843 struct rte_mempool *rte_mp = NULL; 844 uint32_t mb_size; 845 846 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 847 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 848 849 TESTPMD_LOG(INFO, 850 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 851 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 852 853 switch (mp_alloc_type) { 854 case MP_ALLOC_NATIVE: 855 { 856 /* wrapper to rte_mempool_create() */ 857 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 858 rte_mbuf_best_mempool_ops()); 859 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 860 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 861 break; 862 } 863 case MP_ALLOC_ANON: 864 { 865 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 866 mb_size, (unsigned int) mb_mempool_cache, 867 sizeof(struct rte_pktmbuf_pool_private), 868 socket_id, 0); 869 if (rte_mp == NULL) 870 goto err; 871 872 if (rte_mempool_populate_anon(rte_mp) == 0) { 873 rte_mempool_free(rte_mp); 874 rte_mp = NULL; 875 goto err; 876 } 877 rte_pktmbuf_pool_init(rte_mp, NULL); 878 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 879 break; 880 } 881 case MP_ALLOC_XMEM: 882 case MP_ALLOC_XMEM_HUGE: 883 { 884 int heap_socket; 885 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 886 887 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 888 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 889 890 heap_socket = 891 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 892 if (heap_socket < 0) 893 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 894 895 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 896 rte_mbuf_best_mempool_ops()); 897 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 898 mb_mempool_cache, 0, mbuf_seg_size, 899 heap_socket); 900 break; 901 } 902 default: 903 { 904 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 905 } 906 } 907 908 err: 909 if (rte_mp == NULL) { 910 rte_exit(EXIT_FAILURE, 911 "Creation of mbuf pool for socket %u failed: %s\n", 912 socket_id, rte_strerror(rte_errno)); 913 } else if (verbose_level > 0) { 914 rte_mempool_dump(stdout, rte_mp); 915 } 916 } 917 918 /* 919 * Check given socket id is valid or not with NUMA mode, 920 * if valid, return 0, else return -1 921 */ 922 static int 923 check_socket_id(const unsigned int socket_id) 924 { 925 static int warning_once = 0; 926 927 if (new_socket_id(socket_id)) { 928 if (!warning_once && numa_support) 929 printf("Warning: NUMA should be configured manually by" 930 " using --port-numa-config and" 931 " --ring-numa-config parameters along with" 932 " --numa.\n"); 933 warning_once = 1; 934 return -1; 935 } 936 return 0; 937 } 938 939 /* 940 * Get the allowed maximum number of RX queues. 941 * *pid return the port id which has minimal value of 942 * max_rx_queues in all ports. 943 */ 944 queueid_t 945 get_allowed_max_nb_rxq(portid_t *pid) 946 { 947 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 948 portid_t pi; 949 struct rte_eth_dev_info dev_info; 950 951 RTE_ETH_FOREACH_DEV(pi) { 952 rte_eth_dev_info_get(pi, &dev_info); 953 if (dev_info.max_rx_queues < allowed_max_rxq) { 954 allowed_max_rxq = dev_info.max_rx_queues; 955 *pid = pi; 956 } 957 } 958 return allowed_max_rxq; 959 } 960 961 /* 962 * Check input rxq is valid or not. 963 * If input rxq is not greater than any of maximum number 964 * of RX queues of all ports, it is valid. 965 * if valid, return 0, else return -1 966 */ 967 int 968 check_nb_rxq(queueid_t rxq) 969 { 970 queueid_t allowed_max_rxq; 971 portid_t pid = 0; 972 973 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 974 if (rxq > allowed_max_rxq) { 975 printf("Fail: input rxq (%u) can't be greater " 976 "than max_rx_queues (%u) of port %u\n", 977 rxq, 978 allowed_max_rxq, 979 pid); 980 return -1; 981 } 982 return 0; 983 } 984 985 /* 986 * Get the allowed maximum number of TX queues. 987 * *pid return the port id which has minimal value of 988 * max_tx_queues in all ports. 989 */ 990 queueid_t 991 get_allowed_max_nb_txq(portid_t *pid) 992 { 993 queueid_t allowed_max_txq = MAX_QUEUE_ID; 994 portid_t pi; 995 struct rte_eth_dev_info dev_info; 996 997 RTE_ETH_FOREACH_DEV(pi) { 998 rte_eth_dev_info_get(pi, &dev_info); 999 if (dev_info.max_tx_queues < allowed_max_txq) { 1000 allowed_max_txq = dev_info.max_tx_queues; 1001 *pid = pi; 1002 } 1003 } 1004 return allowed_max_txq; 1005 } 1006 1007 /* 1008 * Check input txq is valid or not. 1009 * If input txq is not greater than any of maximum number 1010 * of TX queues of all ports, it is valid. 1011 * if valid, return 0, else return -1 1012 */ 1013 int 1014 check_nb_txq(queueid_t txq) 1015 { 1016 queueid_t allowed_max_txq; 1017 portid_t pid = 0; 1018 1019 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1020 if (txq > allowed_max_txq) { 1021 printf("Fail: input txq (%u) can't be greater " 1022 "than max_tx_queues (%u) of port %u\n", 1023 txq, 1024 allowed_max_txq, 1025 pid); 1026 return -1; 1027 } 1028 return 0; 1029 } 1030 1031 static void 1032 init_config(void) 1033 { 1034 portid_t pid; 1035 struct rte_port *port; 1036 struct rte_mempool *mbp; 1037 unsigned int nb_mbuf_per_pool; 1038 lcoreid_t lc_id; 1039 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1040 struct rte_gro_param gro_param; 1041 uint32_t gso_types; 1042 int k; 1043 1044 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1045 1046 /* Configuration of logical cores. */ 1047 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1048 sizeof(struct fwd_lcore *) * nb_lcores, 1049 RTE_CACHE_LINE_SIZE); 1050 if (fwd_lcores == NULL) { 1051 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1052 "failed\n", nb_lcores); 1053 } 1054 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1055 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1056 sizeof(struct fwd_lcore), 1057 RTE_CACHE_LINE_SIZE); 1058 if (fwd_lcores[lc_id] == NULL) { 1059 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1060 "failed\n"); 1061 } 1062 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1063 } 1064 1065 RTE_ETH_FOREACH_DEV(pid) { 1066 port = &ports[pid]; 1067 /* Apply default TxRx configuration for all ports */ 1068 port->dev_conf.txmode = tx_mode; 1069 port->dev_conf.rxmode = rx_mode; 1070 rte_eth_dev_info_get(pid, &port->dev_info); 1071 1072 if (!(port->dev_info.tx_offload_capa & 1073 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1074 port->dev_conf.txmode.offloads &= 1075 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1076 if (!(port->dev_info.tx_offload_capa & 1077 DEV_TX_OFFLOAD_MATCH_METADATA)) 1078 port->dev_conf.txmode.offloads &= 1079 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1080 if (numa_support) { 1081 if (port_numa[pid] != NUMA_NO_CONFIG) 1082 port_per_socket[port_numa[pid]]++; 1083 else { 1084 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1085 1086 /* 1087 * if socket_id is invalid, 1088 * set to the first available socket. 1089 */ 1090 if (check_socket_id(socket_id) < 0) 1091 socket_id = socket_ids[0]; 1092 port_per_socket[socket_id]++; 1093 } 1094 } 1095 1096 /* Apply Rx offloads configuration */ 1097 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1098 port->rx_conf[k].offloads = 1099 port->dev_conf.rxmode.offloads; 1100 /* Apply Tx offloads configuration */ 1101 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1102 port->tx_conf[k].offloads = 1103 port->dev_conf.txmode.offloads; 1104 1105 /* set flag to initialize port/queue */ 1106 port->need_reconfig = 1; 1107 port->need_reconfig_queues = 1; 1108 port->tx_metadata = 0; 1109 } 1110 1111 /* 1112 * Create pools of mbuf. 1113 * If NUMA support is disabled, create a single pool of mbuf in 1114 * socket 0 memory by default. 1115 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1116 * 1117 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1118 * nb_txd can be configured at run time. 1119 */ 1120 if (param_total_num_mbufs) 1121 nb_mbuf_per_pool = param_total_num_mbufs; 1122 else { 1123 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1124 (nb_lcores * mb_mempool_cache) + 1125 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1126 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1127 } 1128 1129 if (numa_support) { 1130 uint8_t i; 1131 1132 for (i = 0; i < num_sockets; i++) 1133 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1134 socket_ids[i]); 1135 } else { 1136 if (socket_num == UMA_NO_CONFIG) 1137 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 1138 else 1139 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1140 socket_num); 1141 } 1142 1143 init_port_config(); 1144 1145 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1146 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1147 /* 1148 * Records which Mbuf pool to use by each logical core, if needed. 1149 */ 1150 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1151 mbp = mbuf_pool_find( 1152 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1153 1154 if (mbp == NULL) 1155 mbp = mbuf_pool_find(0); 1156 fwd_lcores[lc_id]->mbp = mbp; 1157 /* initialize GSO context */ 1158 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1159 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1160 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1161 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN - 1162 ETHER_CRC_LEN; 1163 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1164 } 1165 1166 /* Configuration of packet forwarding streams. */ 1167 if (init_fwd_streams() < 0) 1168 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1169 1170 fwd_config_setup(); 1171 1172 /* create a gro context for each lcore */ 1173 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1174 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1175 gro_param.max_item_per_flow = MAX_PKT_BURST; 1176 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1177 gro_param.socket_id = rte_lcore_to_socket_id( 1178 fwd_lcores_cpuids[lc_id]); 1179 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1180 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1181 rte_exit(EXIT_FAILURE, 1182 "rte_gro_ctx_create() failed\n"); 1183 } 1184 } 1185 1186 #if defined RTE_LIBRTE_PMD_SOFTNIC 1187 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1188 RTE_ETH_FOREACH_DEV(pid) { 1189 port = &ports[pid]; 1190 const char *driver = port->dev_info.driver_name; 1191 1192 if (strcmp(driver, "net_softnic") == 0) 1193 port->softport.fwd_lcore_arg = fwd_lcores; 1194 } 1195 } 1196 #endif 1197 1198 } 1199 1200 1201 void 1202 reconfig(portid_t new_port_id, unsigned socket_id) 1203 { 1204 struct rte_port *port; 1205 1206 /* Reconfiguration of Ethernet ports. */ 1207 port = &ports[new_port_id]; 1208 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1209 1210 /* set flag to initialize port/queue */ 1211 port->need_reconfig = 1; 1212 port->need_reconfig_queues = 1; 1213 port->socket_id = socket_id; 1214 1215 init_port_config(); 1216 } 1217 1218 1219 int 1220 init_fwd_streams(void) 1221 { 1222 portid_t pid; 1223 struct rte_port *port; 1224 streamid_t sm_id, nb_fwd_streams_new; 1225 queueid_t q; 1226 1227 /* set socket id according to numa or not */ 1228 RTE_ETH_FOREACH_DEV(pid) { 1229 port = &ports[pid]; 1230 if (nb_rxq > port->dev_info.max_rx_queues) { 1231 printf("Fail: nb_rxq(%d) is greater than " 1232 "max_rx_queues(%d)\n", nb_rxq, 1233 port->dev_info.max_rx_queues); 1234 return -1; 1235 } 1236 if (nb_txq > port->dev_info.max_tx_queues) { 1237 printf("Fail: nb_txq(%d) is greater than " 1238 "max_tx_queues(%d)\n", nb_txq, 1239 port->dev_info.max_tx_queues); 1240 return -1; 1241 } 1242 if (numa_support) { 1243 if (port_numa[pid] != NUMA_NO_CONFIG) 1244 port->socket_id = port_numa[pid]; 1245 else { 1246 port->socket_id = rte_eth_dev_socket_id(pid); 1247 1248 /* 1249 * if socket_id is invalid, 1250 * set to the first available socket. 1251 */ 1252 if (check_socket_id(port->socket_id) < 0) 1253 port->socket_id = socket_ids[0]; 1254 } 1255 } 1256 else { 1257 if (socket_num == UMA_NO_CONFIG) 1258 port->socket_id = 0; 1259 else 1260 port->socket_id = socket_num; 1261 } 1262 } 1263 1264 q = RTE_MAX(nb_rxq, nb_txq); 1265 if (q == 0) { 1266 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1267 return -1; 1268 } 1269 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1270 if (nb_fwd_streams_new == nb_fwd_streams) 1271 return 0; 1272 /* clear the old */ 1273 if (fwd_streams != NULL) { 1274 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1275 if (fwd_streams[sm_id] == NULL) 1276 continue; 1277 rte_free(fwd_streams[sm_id]); 1278 fwd_streams[sm_id] = NULL; 1279 } 1280 rte_free(fwd_streams); 1281 fwd_streams = NULL; 1282 } 1283 1284 /* init new */ 1285 nb_fwd_streams = nb_fwd_streams_new; 1286 if (nb_fwd_streams) { 1287 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1288 sizeof(struct fwd_stream *) * nb_fwd_streams, 1289 RTE_CACHE_LINE_SIZE); 1290 if (fwd_streams == NULL) 1291 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1292 " (struct fwd_stream *)) failed\n", 1293 nb_fwd_streams); 1294 1295 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1296 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1297 " struct fwd_stream", sizeof(struct fwd_stream), 1298 RTE_CACHE_LINE_SIZE); 1299 if (fwd_streams[sm_id] == NULL) 1300 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1301 "(struct fwd_stream) failed\n"); 1302 } 1303 } 1304 1305 return 0; 1306 } 1307 1308 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1309 static void 1310 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1311 { 1312 unsigned int total_burst; 1313 unsigned int nb_burst; 1314 unsigned int burst_stats[3]; 1315 uint16_t pktnb_stats[3]; 1316 uint16_t nb_pkt; 1317 int burst_percent[3]; 1318 1319 /* 1320 * First compute the total number of packet bursts and the 1321 * two highest numbers of bursts of the same number of packets. 1322 */ 1323 total_burst = 0; 1324 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1325 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1326 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1327 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1328 if (nb_burst == 0) 1329 continue; 1330 total_burst += nb_burst; 1331 if (nb_burst > burst_stats[0]) { 1332 burst_stats[1] = burst_stats[0]; 1333 pktnb_stats[1] = pktnb_stats[0]; 1334 burst_stats[0] = nb_burst; 1335 pktnb_stats[0] = nb_pkt; 1336 } else if (nb_burst > burst_stats[1]) { 1337 burst_stats[1] = nb_burst; 1338 pktnb_stats[1] = nb_pkt; 1339 } 1340 } 1341 if (total_burst == 0) 1342 return; 1343 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1344 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1345 burst_percent[0], (int) pktnb_stats[0]); 1346 if (burst_stats[0] == total_burst) { 1347 printf("]\n"); 1348 return; 1349 } 1350 if (burst_stats[0] + burst_stats[1] == total_burst) { 1351 printf(" + %d%% of %d pkts]\n", 1352 100 - burst_percent[0], pktnb_stats[1]); 1353 return; 1354 } 1355 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1356 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1357 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1358 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1359 return; 1360 } 1361 printf(" + %d%% of %d pkts + %d%% of others]\n", 1362 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1363 } 1364 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1365 1366 static void 1367 fwd_stream_stats_display(streamid_t stream_id) 1368 { 1369 struct fwd_stream *fs; 1370 static const char *fwd_top_stats_border = "-------"; 1371 1372 fs = fwd_streams[stream_id]; 1373 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1374 (fs->fwd_dropped == 0)) 1375 return; 1376 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1377 "TX Port=%2d/Queue=%2d %s\n", 1378 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1379 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1380 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1381 " TX-dropped: %-14"PRIu64, 1382 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1383 1384 /* if checksum mode */ 1385 if (cur_fwd_eng == &csum_fwd_engine) { 1386 printf(" RX- bad IP checksum: %-14"PRIu64 1387 " Rx- bad L4 checksum: %-14"PRIu64 1388 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1389 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1390 fs->rx_bad_outer_l4_csum); 1391 } else { 1392 printf("\n"); 1393 } 1394 1395 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1396 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1397 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1398 #endif 1399 } 1400 1401 void 1402 fwd_stats_display(void) 1403 { 1404 static const char *fwd_stats_border = "----------------------"; 1405 static const char *acc_stats_border = "+++++++++++++++"; 1406 struct { 1407 struct fwd_stream *rx_stream; 1408 struct fwd_stream *tx_stream; 1409 uint64_t tx_dropped; 1410 uint64_t rx_bad_ip_csum; 1411 uint64_t rx_bad_l4_csum; 1412 uint64_t rx_bad_outer_l4_csum; 1413 } ports_stats[RTE_MAX_ETHPORTS]; 1414 uint64_t total_rx_dropped = 0; 1415 uint64_t total_tx_dropped = 0; 1416 uint64_t total_rx_nombuf = 0; 1417 struct rte_eth_stats stats; 1418 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1419 uint64_t fwd_cycles = 0; 1420 #endif 1421 uint64_t total_recv = 0; 1422 uint64_t total_xmit = 0; 1423 struct rte_port *port; 1424 streamid_t sm_id; 1425 portid_t pt_id; 1426 int i; 1427 1428 memset(ports_stats, 0, sizeof(ports_stats)); 1429 1430 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1431 struct fwd_stream *fs = fwd_streams[sm_id]; 1432 1433 if (cur_fwd_config.nb_fwd_streams > 1434 cur_fwd_config.nb_fwd_ports) { 1435 fwd_stream_stats_display(sm_id); 1436 } else { 1437 ports_stats[fs->tx_port].tx_stream = fs; 1438 ports_stats[fs->rx_port].rx_stream = fs; 1439 } 1440 1441 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1442 1443 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1444 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1445 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1446 fs->rx_bad_outer_l4_csum; 1447 1448 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1449 fwd_cycles += fs->core_cycles; 1450 #endif 1451 } 1452 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1453 uint8_t j; 1454 1455 pt_id = fwd_ports_ids[i]; 1456 port = &ports[pt_id]; 1457 1458 rte_eth_stats_get(pt_id, &stats); 1459 stats.ipackets -= port->stats.ipackets; 1460 stats.opackets -= port->stats.opackets; 1461 stats.ibytes -= port->stats.ibytes; 1462 stats.obytes -= port->stats.obytes; 1463 stats.imissed -= port->stats.imissed; 1464 stats.oerrors -= port->stats.oerrors; 1465 stats.rx_nombuf -= port->stats.rx_nombuf; 1466 1467 total_recv += stats.ipackets; 1468 total_xmit += stats.opackets; 1469 total_rx_dropped += stats.imissed; 1470 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1471 total_tx_dropped += stats.oerrors; 1472 total_rx_nombuf += stats.rx_nombuf; 1473 1474 printf("\n %s Forward statistics for port %-2d %s\n", 1475 fwd_stats_border, pt_id, fwd_stats_border); 1476 1477 if (!port->rx_queue_stats_mapping_enabled && 1478 !port->tx_queue_stats_mapping_enabled) { 1479 printf(" RX-packets: %-14"PRIu64 1480 " RX-dropped: %-14"PRIu64 1481 "RX-total: %-"PRIu64"\n", 1482 stats.ipackets, stats.imissed, 1483 stats.ipackets + stats.imissed); 1484 1485 if (cur_fwd_eng == &csum_fwd_engine) 1486 printf(" Bad-ipcsum: %-14"PRIu64 1487 " Bad-l4csum: %-14"PRIu64 1488 "Bad-outer-l4csum: %-14"PRIu64"\n", 1489 ports_stats[pt_id].rx_bad_ip_csum, 1490 ports_stats[pt_id].rx_bad_l4_csum, 1491 ports_stats[pt_id].rx_bad_outer_l4_csum); 1492 if (stats.ierrors + stats.rx_nombuf > 0) { 1493 printf(" RX-error: %-"PRIu64"\n", 1494 stats.ierrors); 1495 printf(" RX-nombufs: %-14"PRIu64"\n", 1496 stats.rx_nombuf); 1497 } 1498 1499 printf(" TX-packets: %-14"PRIu64 1500 " TX-dropped: %-14"PRIu64 1501 "TX-total: %-"PRIu64"\n", 1502 stats.opackets, ports_stats[pt_id].tx_dropped, 1503 stats.opackets + ports_stats[pt_id].tx_dropped); 1504 } else { 1505 printf(" RX-packets: %14"PRIu64 1506 " RX-dropped:%14"PRIu64 1507 " RX-total:%14"PRIu64"\n", 1508 stats.ipackets, stats.imissed, 1509 stats.ipackets + stats.imissed); 1510 1511 if (cur_fwd_eng == &csum_fwd_engine) 1512 printf(" Bad-ipcsum:%14"PRIu64 1513 " Bad-l4csum:%14"PRIu64 1514 " Bad-outer-l4csum: %-14"PRIu64"\n", 1515 ports_stats[pt_id].rx_bad_ip_csum, 1516 ports_stats[pt_id].rx_bad_l4_csum, 1517 ports_stats[pt_id].rx_bad_outer_l4_csum); 1518 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1519 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1520 printf(" RX-nombufs: %14"PRIu64"\n", 1521 stats.rx_nombuf); 1522 } 1523 1524 printf(" TX-packets: %14"PRIu64 1525 " TX-dropped:%14"PRIu64 1526 " TX-total:%14"PRIu64"\n", 1527 stats.opackets, ports_stats[pt_id].tx_dropped, 1528 stats.opackets + ports_stats[pt_id].tx_dropped); 1529 } 1530 1531 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1532 if (ports_stats[pt_id].rx_stream) 1533 pkt_burst_stats_display("RX", 1534 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1535 if (ports_stats[pt_id].tx_stream) 1536 pkt_burst_stats_display("TX", 1537 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1538 #endif 1539 1540 if (port->rx_queue_stats_mapping_enabled) { 1541 printf("\n"); 1542 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1543 printf(" Stats reg %2d RX-packets:%14"PRIu64 1544 " RX-errors:%14"PRIu64 1545 " RX-bytes:%14"PRIu64"\n", 1546 j, stats.q_ipackets[j], 1547 stats.q_errors[j], stats.q_ibytes[j]); 1548 } 1549 printf("\n"); 1550 } 1551 if (port->tx_queue_stats_mapping_enabled) { 1552 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1553 printf(" Stats reg %2d TX-packets:%14"PRIu64 1554 " TX-bytes:%14" 1555 PRIu64"\n", 1556 j, stats.q_opackets[j], 1557 stats.q_obytes[j]); 1558 } 1559 } 1560 1561 printf(" %s--------------------------------%s\n", 1562 fwd_stats_border, fwd_stats_border); 1563 } 1564 1565 printf("\n %s Accumulated forward statistics for all ports" 1566 "%s\n", 1567 acc_stats_border, acc_stats_border); 1568 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1569 "%-"PRIu64"\n" 1570 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1571 "%-"PRIu64"\n", 1572 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1573 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1574 if (total_rx_nombuf > 0) 1575 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1576 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1577 "%s\n", 1578 acc_stats_border, acc_stats_border); 1579 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1580 if (total_recv > 0) 1581 printf("\n CPU cycles/packet=%u (total cycles=" 1582 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1583 (unsigned int)(fwd_cycles / total_recv), 1584 fwd_cycles, total_recv); 1585 #endif 1586 } 1587 1588 void 1589 fwd_stats_reset(void) 1590 { 1591 streamid_t sm_id; 1592 portid_t pt_id; 1593 int i; 1594 1595 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1596 pt_id = fwd_ports_ids[i]; 1597 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1598 } 1599 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1600 struct fwd_stream *fs = fwd_streams[sm_id]; 1601 1602 fs->rx_packets = 0; 1603 fs->tx_packets = 0; 1604 fs->fwd_dropped = 0; 1605 fs->rx_bad_ip_csum = 0; 1606 fs->rx_bad_l4_csum = 0; 1607 fs->rx_bad_outer_l4_csum = 0; 1608 1609 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1610 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1611 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1612 #endif 1613 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1614 fs->core_cycles = 0; 1615 #endif 1616 } 1617 } 1618 1619 static void 1620 flush_fwd_rx_queues(void) 1621 { 1622 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1623 portid_t rxp; 1624 portid_t port_id; 1625 queueid_t rxq; 1626 uint16_t nb_rx; 1627 uint16_t i; 1628 uint8_t j; 1629 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1630 uint64_t timer_period; 1631 1632 /* convert to number of cycles */ 1633 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1634 1635 for (j = 0; j < 2; j++) { 1636 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1637 for (rxq = 0; rxq < nb_rxq; rxq++) { 1638 port_id = fwd_ports_ids[rxp]; 1639 /** 1640 * testpmd can stuck in the below do while loop 1641 * if rte_eth_rx_burst() always returns nonzero 1642 * packets. So timer is added to exit this loop 1643 * after 1sec timer expiry. 1644 */ 1645 prev_tsc = rte_rdtsc(); 1646 do { 1647 nb_rx = rte_eth_rx_burst(port_id, rxq, 1648 pkts_burst, MAX_PKT_BURST); 1649 for (i = 0; i < nb_rx; i++) 1650 rte_pktmbuf_free(pkts_burst[i]); 1651 1652 cur_tsc = rte_rdtsc(); 1653 diff_tsc = cur_tsc - prev_tsc; 1654 timer_tsc += diff_tsc; 1655 } while ((nb_rx > 0) && 1656 (timer_tsc < timer_period)); 1657 timer_tsc = 0; 1658 } 1659 } 1660 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1661 } 1662 } 1663 1664 static void 1665 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1666 { 1667 struct fwd_stream **fsm; 1668 streamid_t nb_fs; 1669 streamid_t sm_id; 1670 #ifdef RTE_LIBRTE_BITRATE 1671 uint64_t tics_per_1sec; 1672 uint64_t tics_datum; 1673 uint64_t tics_current; 1674 uint16_t i, cnt_ports; 1675 1676 cnt_ports = nb_ports; 1677 tics_datum = rte_rdtsc(); 1678 tics_per_1sec = rte_get_timer_hz(); 1679 #endif 1680 fsm = &fwd_streams[fc->stream_idx]; 1681 nb_fs = fc->stream_nb; 1682 do { 1683 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1684 (*pkt_fwd)(fsm[sm_id]); 1685 #ifdef RTE_LIBRTE_BITRATE 1686 if (bitrate_enabled != 0 && 1687 bitrate_lcore_id == rte_lcore_id()) { 1688 tics_current = rte_rdtsc(); 1689 if (tics_current - tics_datum >= tics_per_1sec) { 1690 /* Periodic bitrate calculation */ 1691 for (i = 0; i < cnt_ports; i++) 1692 rte_stats_bitrate_calc(bitrate_data, 1693 ports_ids[i]); 1694 tics_datum = tics_current; 1695 } 1696 } 1697 #endif 1698 #ifdef RTE_LIBRTE_LATENCY_STATS 1699 if (latencystats_enabled != 0 && 1700 latencystats_lcore_id == rte_lcore_id()) 1701 rte_latencystats_update(); 1702 #endif 1703 1704 } while (! fc->stopped); 1705 } 1706 1707 static int 1708 start_pkt_forward_on_core(void *fwd_arg) 1709 { 1710 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1711 cur_fwd_config.fwd_eng->packet_fwd); 1712 return 0; 1713 } 1714 1715 /* 1716 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1717 * Used to start communication flows in network loopback test configurations. 1718 */ 1719 static int 1720 run_one_txonly_burst_on_core(void *fwd_arg) 1721 { 1722 struct fwd_lcore *fwd_lc; 1723 struct fwd_lcore tmp_lcore; 1724 1725 fwd_lc = (struct fwd_lcore *) fwd_arg; 1726 tmp_lcore = *fwd_lc; 1727 tmp_lcore.stopped = 1; 1728 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1729 return 0; 1730 } 1731 1732 /* 1733 * Launch packet forwarding: 1734 * - Setup per-port forwarding context. 1735 * - launch logical cores with their forwarding configuration. 1736 */ 1737 static void 1738 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1739 { 1740 port_fwd_begin_t port_fwd_begin; 1741 unsigned int i; 1742 unsigned int lc_id; 1743 int diag; 1744 1745 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1746 if (port_fwd_begin != NULL) { 1747 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1748 (*port_fwd_begin)(fwd_ports_ids[i]); 1749 } 1750 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1751 lc_id = fwd_lcores_cpuids[i]; 1752 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1753 fwd_lcores[i]->stopped = 0; 1754 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1755 fwd_lcores[i], lc_id); 1756 if (diag != 0) 1757 printf("launch lcore %u failed - diag=%d\n", 1758 lc_id, diag); 1759 } 1760 } 1761 } 1762 1763 /* 1764 * Launch packet forwarding configuration. 1765 */ 1766 void 1767 start_packet_forwarding(int with_tx_first) 1768 { 1769 port_fwd_begin_t port_fwd_begin; 1770 port_fwd_end_t port_fwd_end; 1771 struct rte_port *port; 1772 unsigned int i; 1773 portid_t pt_id; 1774 1775 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1776 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1777 1778 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1779 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1780 1781 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1782 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1783 (!nb_rxq || !nb_txq)) 1784 rte_exit(EXIT_FAILURE, 1785 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1786 cur_fwd_eng->fwd_mode_name); 1787 1788 if (all_ports_started() == 0) { 1789 printf("Not all ports were started\n"); 1790 return; 1791 } 1792 if (test_done == 0) { 1793 printf("Packet forwarding already started\n"); 1794 return; 1795 } 1796 1797 1798 if(dcb_test) { 1799 for (i = 0; i < nb_fwd_ports; i++) { 1800 pt_id = fwd_ports_ids[i]; 1801 port = &ports[pt_id]; 1802 if (!port->dcb_flag) { 1803 printf("In DCB mode, all forwarding ports must " 1804 "be configured in this mode.\n"); 1805 return; 1806 } 1807 } 1808 if (nb_fwd_lcores == 1) { 1809 printf("In DCB mode,the nb forwarding cores " 1810 "should be larger than 1.\n"); 1811 return; 1812 } 1813 } 1814 test_done = 0; 1815 1816 fwd_config_setup(); 1817 1818 if(!no_flush_rx) 1819 flush_fwd_rx_queues(); 1820 1821 pkt_fwd_config_display(&cur_fwd_config); 1822 rxtx_config_display(); 1823 1824 fwd_stats_reset(); 1825 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1826 pt_id = fwd_ports_ids[i]; 1827 port = &ports[pt_id]; 1828 map_port_queue_stats_mapping_registers(pt_id, port); 1829 } 1830 if (with_tx_first) { 1831 port_fwd_begin = tx_only_engine.port_fwd_begin; 1832 if (port_fwd_begin != NULL) { 1833 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1834 (*port_fwd_begin)(fwd_ports_ids[i]); 1835 } 1836 while (with_tx_first--) { 1837 launch_packet_forwarding( 1838 run_one_txonly_burst_on_core); 1839 rte_eal_mp_wait_lcore(); 1840 } 1841 port_fwd_end = tx_only_engine.port_fwd_end; 1842 if (port_fwd_end != NULL) { 1843 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1844 (*port_fwd_end)(fwd_ports_ids[i]); 1845 } 1846 } 1847 launch_packet_forwarding(start_pkt_forward_on_core); 1848 } 1849 1850 void 1851 stop_packet_forwarding(void) 1852 { 1853 port_fwd_end_t port_fwd_end; 1854 lcoreid_t lc_id; 1855 portid_t pt_id; 1856 int i; 1857 1858 if (test_done) { 1859 printf("Packet forwarding not started\n"); 1860 return; 1861 } 1862 printf("Telling cores to stop..."); 1863 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1864 fwd_lcores[lc_id]->stopped = 1; 1865 printf("\nWaiting for lcores to finish...\n"); 1866 rte_eal_mp_wait_lcore(); 1867 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1868 if (port_fwd_end != NULL) { 1869 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1870 pt_id = fwd_ports_ids[i]; 1871 (*port_fwd_end)(pt_id); 1872 } 1873 } 1874 1875 fwd_stats_display(); 1876 1877 printf("\nDone.\n"); 1878 test_done = 1; 1879 } 1880 1881 void 1882 dev_set_link_up(portid_t pid) 1883 { 1884 if (rte_eth_dev_set_link_up(pid) < 0) 1885 printf("\nSet link up fail.\n"); 1886 } 1887 1888 void 1889 dev_set_link_down(portid_t pid) 1890 { 1891 if (rte_eth_dev_set_link_down(pid) < 0) 1892 printf("\nSet link down fail.\n"); 1893 } 1894 1895 static int 1896 all_ports_started(void) 1897 { 1898 portid_t pi; 1899 struct rte_port *port; 1900 1901 RTE_ETH_FOREACH_DEV(pi) { 1902 port = &ports[pi]; 1903 /* Check if there is a port which is not started */ 1904 if ((port->port_status != RTE_PORT_STARTED) && 1905 (port->slave_flag == 0)) 1906 return 0; 1907 } 1908 1909 /* No port is not started */ 1910 return 1; 1911 } 1912 1913 int 1914 port_is_stopped(portid_t port_id) 1915 { 1916 struct rte_port *port = &ports[port_id]; 1917 1918 if ((port->port_status != RTE_PORT_STOPPED) && 1919 (port->slave_flag == 0)) 1920 return 0; 1921 return 1; 1922 } 1923 1924 int 1925 all_ports_stopped(void) 1926 { 1927 portid_t pi; 1928 1929 RTE_ETH_FOREACH_DEV(pi) { 1930 if (!port_is_stopped(pi)) 1931 return 0; 1932 } 1933 1934 return 1; 1935 } 1936 1937 int 1938 port_is_started(portid_t port_id) 1939 { 1940 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1941 return 0; 1942 1943 if (ports[port_id].port_status != RTE_PORT_STARTED) 1944 return 0; 1945 1946 return 1; 1947 } 1948 1949 int 1950 start_port(portid_t pid) 1951 { 1952 int diag, need_check_link_status = -1; 1953 portid_t pi; 1954 queueid_t qi; 1955 struct rte_port *port; 1956 struct ether_addr mac_addr; 1957 1958 if (port_id_is_invalid(pid, ENABLED_WARN)) 1959 return 0; 1960 1961 if(dcb_config) 1962 dcb_test = 1; 1963 RTE_ETH_FOREACH_DEV(pi) { 1964 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1965 continue; 1966 1967 need_check_link_status = 0; 1968 port = &ports[pi]; 1969 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1970 RTE_PORT_HANDLING) == 0) { 1971 printf("Port %d is now not stopped\n", pi); 1972 continue; 1973 } 1974 1975 if (port->need_reconfig > 0) { 1976 port->need_reconfig = 0; 1977 1978 if (flow_isolate_all) { 1979 int ret = port_flow_isolate(pi, 1); 1980 if (ret) { 1981 printf("Failed to apply isolated" 1982 " mode on port %d\n", pi); 1983 return -1; 1984 } 1985 } 1986 configure_rxtx_dump_callbacks(0); 1987 printf("Configuring Port %d (socket %u)\n", pi, 1988 port->socket_id); 1989 /* configure port */ 1990 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1991 &(port->dev_conf)); 1992 if (diag != 0) { 1993 if (rte_atomic16_cmpset(&(port->port_status), 1994 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1995 printf("Port %d can not be set back " 1996 "to stopped\n", pi); 1997 printf("Fail to configure port %d\n", pi); 1998 /* try to reconfigure port next time */ 1999 port->need_reconfig = 1; 2000 return -1; 2001 } 2002 } 2003 if (port->need_reconfig_queues > 0) { 2004 port->need_reconfig_queues = 0; 2005 /* setup tx queues */ 2006 for (qi = 0; qi < nb_txq; qi++) { 2007 if ((numa_support) && 2008 (txring_numa[pi] != NUMA_NO_CONFIG)) 2009 diag = rte_eth_tx_queue_setup(pi, qi, 2010 port->nb_tx_desc[qi], 2011 txring_numa[pi], 2012 &(port->tx_conf[qi])); 2013 else 2014 diag = rte_eth_tx_queue_setup(pi, qi, 2015 port->nb_tx_desc[qi], 2016 port->socket_id, 2017 &(port->tx_conf[qi])); 2018 2019 if (diag == 0) 2020 continue; 2021 2022 /* Fail to setup tx queue, return */ 2023 if (rte_atomic16_cmpset(&(port->port_status), 2024 RTE_PORT_HANDLING, 2025 RTE_PORT_STOPPED) == 0) 2026 printf("Port %d can not be set back " 2027 "to stopped\n", pi); 2028 printf("Fail to configure port %d tx queues\n", 2029 pi); 2030 /* try to reconfigure queues next time */ 2031 port->need_reconfig_queues = 1; 2032 return -1; 2033 } 2034 for (qi = 0; qi < nb_rxq; qi++) { 2035 /* setup rx queues */ 2036 if ((numa_support) && 2037 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2038 struct rte_mempool * mp = 2039 mbuf_pool_find(rxring_numa[pi]); 2040 if (mp == NULL) { 2041 printf("Failed to setup RX queue:" 2042 "No mempool allocation" 2043 " on the socket %d\n", 2044 rxring_numa[pi]); 2045 return -1; 2046 } 2047 2048 diag = rte_eth_rx_queue_setup(pi, qi, 2049 port->nb_rx_desc[qi], 2050 rxring_numa[pi], 2051 &(port->rx_conf[qi]), 2052 mp); 2053 } else { 2054 struct rte_mempool *mp = 2055 mbuf_pool_find(port->socket_id); 2056 if (mp == NULL) { 2057 printf("Failed to setup RX queue:" 2058 "No mempool allocation" 2059 " on the socket %d\n", 2060 port->socket_id); 2061 return -1; 2062 } 2063 diag = rte_eth_rx_queue_setup(pi, qi, 2064 port->nb_rx_desc[qi], 2065 port->socket_id, 2066 &(port->rx_conf[qi]), 2067 mp); 2068 } 2069 if (diag == 0) 2070 continue; 2071 2072 /* Fail to setup rx queue, return */ 2073 if (rte_atomic16_cmpset(&(port->port_status), 2074 RTE_PORT_HANDLING, 2075 RTE_PORT_STOPPED) == 0) 2076 printf("Port %d can not be set back " 2077 "to stopped\n", pi); 2078 printf("Fail to configure port %d rx queues\n", 2079 pi); 2080 /* try to reconfigure queues next time */ 2081 port->need_reconfig_queues = 1; 2082 return -1; 2083 } 2084 } 2085 configure_rxtx_dump_callbacks(verbose_level); 2086 /* start port */ 2087 if (rte_eth_dev_start(pi) < 0) { 2088 printf("Fail to start port %d\n", pi); 2089 2090 /* Fail to setup rx queue, return */ 2091 if (rte_atomic16_cmpset(&(port->port_status), 2092 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2093 printf("Port %d can not be set back to " 2094 "stopped\n", pi); 2095 continue; 2096 } 2097 2098 if (rte_atomic16_cmpset(&(port->port_status), 2099 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2100 printf("Port %d can not be set into started\n", pi); 2101 2102 rte_eth_macaddr_get(pi, &mac_addr); 2103 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2104 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2105 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2106 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2107 2108 /* at least one port started, need checking link status */ 2109 need_check_link_status = 1; 2110 } 2111 2112 if (need_check_link_status == 1 && !no_link_check) 2113 check_all_ports_link_status(RTE_PORT_ALL); 2114 else if (need_check_link_status == 0) 2115 printf("Please stop the ports first\n"); 2116 2117 printf("Done\n"); 2118 return 0; 2119 } 2120 2121 void 2122 stop_port(portid_t pid) 2123 { 2124 portid_t pi; 2125 struct rte_port *port; 2126 int need_check_link_status = 0; 2127 2128 if (dcb_test) { 2129 dcb_test = 0; 2130 dcb_config = 0; 2131 } 2132 2133 if (port_id_is_invalid(pid, ENABLED_WARN)) 2134 return; 2135 2136 printf("Stopping ports...\n"); 2137 2138 RTE_ETH_FOREACH_DEV(pi) { 2139 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2140 continue; 2141 2142 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2143 printf("Please remove port %d from forwarding configuration.\n", pi); 2144 continue; 2145 } 2146 2147 if (port_is_bonding_slave(pi)) { 2148 printf("Please remove port %d from bonded device.\n", pi); 2149 continue; 2150 } 2151 2152 port = &ports[pi]; 2153 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2154 RTE_PORT_HANDLING) == 0) 2155 continue; 2156 2157 rte_eth_dev_stop(pi); 2158 2159 if (rte_atomic16_cmpset(&(port->port_status), 2160 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2161 printf("Port %d can not be set into stopped\n", pi); 2162 need_check_link_status = 1; 2163 } 2164 if (need_check_link_status && !no_link_check) 2165 check_all_ports_link_status(RTE_PORT_ALL); 2166 2167 printf("Done\n"); 2168 } 2169 2170 static void 2171 remove_invalid_ports_in(portid_t *array, portid_t *total) 2172 { 2173 portid_t i; 2174 portid_t new_total = 0; 2175 2176 for (i = 0; i < *total; i++) 2177 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2178 array[new_total] = array[i]; 2179 new_total++; 2180 } 2181 *total = new_total; 2182 } 2183 2184 static void 2185 remove_invalid_ports(void) 2186 { 2187 remove_invalid_ports_in(ports_ids, &nb_ports); 2188 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2189 nb_cfg_ports = nb_fwd_ports; 2190 } 2191 2192 void 2193 close_port(portid_t pid) 2194 { 2195 portid_t pi; 2196 struct rte_port *port; 2197 2198 if (port_id_is_invalid(pid, ENABLED_WARN)) 2199 return; 2200 2201 printf("Closing ports...\n"); 2202 2203 RTE_ETH_FOREACH_DEV(pi) { 2204 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2205 continue; 2206 2207 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2208 printf("Please remove port %d from forwarding configuration.\n", pi); 2209 continue; 2210 } 2211 2212 if (port_is_bonding_slave(pi)) { 2213 printf("Please remove port %d from bonded device.\n", pi); 2214 continue; 2215 } 2216 2217 port = &ports[pi]; 2218 if (rte_atomic16_cmpset(&(port->port_status), 2219 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2220 printf("Port %d is already closed\n", pi); 2221 continue; 2222 } 2223 2224 if (rte_atomic16_cmpset(&(port->port_status), 2225 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2226 printf("Port %d is now not stopped\n", pi); 2227 continue; 2228 } 2229 2230 if (port->flow_list) 2231 port_flow_flush(pi); 2232 rte_eth_dev_close(pi); 2233 2234 remove_invalid_ports(); 2235 2236 if (rte_atomic16_cmpset(&(port->port_status), 2237 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2238 printf("Port %d cannot be set to closed\n", pi); 2239 } 2240 2241 printf("Done\n"); 2242 } 2243 2244 void 2245 reset_port(portid_t pid) 2246 { 2247 int diag; 2248 portid_t pi; 2249 struct rte_port *port; 2250 2251 if (port_id_is_invalid(pid, ENABLED_WARN)) 2252 return; 2253 2254 printf("Resetting ports...\n"); 2255 2256 RTE_ETH_FOREACH_DEV(pi) { 2257 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2258 continue; 2259 2260 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2261 printf("Please remove port %d from forwarding " 2262 "configuration.\n", pi); 2263 continue; 2264 } 2265 2266 if (port_is_bonding_slave(pi)) { 2267 printf("Please remove port %d from bonded device.\n", 2268 pi); 2269 continue; 2270 } 2271 2272 diag = rte_eth_dev_reset(pi); 2273 if (diag == 0) { 2274 port = &ports[pi]; 2275 port->need_reconfig = 1; 2276 port->need_reconfig_queues = 1; 2277 } else { 2278 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2279 } 2280 } 2281 2282 printf("Done\n"); 2283 } 2284 2285 void 2286 attach_port(char *identifier) 2287 { 2288 portid_t pi; 2289 struct rte_dev_iterator iterator; 2290 2291 printf("Attaching a new port...\n"); 2292 2293 if (identifier == NULL) { 2294 printf("Invalid parameters are specified\n"); 2295 return; 2296 } 2297 2298 if (rte_dev_probe(identifier) != 0) { 2299 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2300 return; 2301 } 2302 2303 /* first attach mode: event */ 2304 if (setup_on_probe_event) { 2305 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2306 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2307 if (ports[pi].port_status == RTE_PORT_HANDLING && 2308 ports[pi].need_setup != 0) 2309 setup_attached_port(pi); 2310 return; 2311 } 2312 2313 /* second attach mode: iterator */ 2314 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2315 /* setup ports matching the devargs used for probing */ 2316 if (port_is_forwarding(pi)) 2317 continue; /* port was already attached before */ 2318 setup_attached_port(pi); 2319 } 2320 } 2321 2322 static void 2323 setup_attached_port(portid_t pi) 2324 { 2325 unsigned int socket_id; 2326 2327 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2328 /* if socket_id is invalid, set to the first available socket. */ 2329 if (check_socket_id(socket_id) < 0) 2330 socket_id = socket_ids[0]; 2331 reconfig(pi, socket_id); 2332 rte_eth_promiscuous_enable(pi); 2333 2334 ports_ids[nb_ports++] = pi; 2335 fwd_ports_ids[nb_fwd_ports++] = pi; 2336 nb_cfg_ports = nb_fwd_ports; 2337 ports[pi].need_setup = 0; 2338 ports[pi].port_status = RTE_PORT_STOPPED; 2339 2340 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2341 printf("Done\n"); 2342 } 2343 2344 void 2345 detach_port_device(portid_t port_id) 2346 { 2347 struct rte_device *dev; 2348 portid_t sibling; 2349 2350 printf("Removing a device...\n"); 2351 2352 dev = rte_eth_devices[port_id].device; 2353 if (dev == NULL) { 2354 printf("Device already removed\n"); 2355 return; 2356 } 2357 2358 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2359 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2360 printf("Port not stopped\n"); 2361 return; 2362 } 2363 printf("Port was not closed\n"); 2364 if (ports[port_id].flow_list) 2365 port_flow_flush(port_id); 2366 } 2367 2368 if (rte_dev_remove(dev) != 0) { 2369 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2370 return; 2371 } 2372 2373 for (sibling = 0; sibling < RTE_MAX_ETHPORTS; sibling++) { 2374 if (rte_eth_devices[sibling].device != dev) 2375 continue; 2376 /* reset mapping between old ports and removed device */ 2377 rte_eth_devices[sibling].device = NULL; 2378 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2379 /* sibling ports are forced to be closed */ 2380 ports[sibling].port_status = RTE_PORT_CLOSED; 2381 printf("Port %u is closed\n", sibling); 2382 } 2383 } 2384 2385 remove_invalid_ports(); 2386 2387 printf("Device of port %u is detached\n", port_id); 2388 printf("Now total ports is %d\n", nb_ports); 2389 printf("Done\n"); 2390 return; 2391 } 2392 2393 void 2394 pmd_test_exit(void) 2395 { 2396 struct rte_device *device; 2397 portid_t pt_id; 2398 int ret; 2399 2400 if (test_done == 0) 2401 stop_packet_forwarding(); 2402 2403 if (ports != NULL) { 2404 no_link_check = 1; 2405 RTE_ETH_FOREACH_DEV(pt_id) { 2406 printf("\nStopping port %d...\n", pt_id); 2407 fflush(stdout); 2408 stop_port(pt_id); 2409 } 2410 RTE_ETH_FOREACH_DEV(pt_id) { 2411 printf("\nShutting down port %d...\n", pt_id); 2412 fflush(stdout); 2413 close_port(pt_id); 2414 2415 /* 2416 * This is a workaround to fix a virtio-user issue that 2417 * requires to call clean-up routine to remove existing 2418 * socket. 2419 * This workaround valid only for testpmd, needs a fix 2420 * valid for all applications. 2421 * TODO: Implement proper resource cleanup 2422 */ 2423 device = rte_eth_devices[pt_id].device; 2424 if (device && !strcmp(device->driver->name, "net_virtio_user")) 2425 detach_port_device(pt_id); 2426 } 2427 } 2428 2429 if (hot_plug) { 2430 ret = rte_dev_event_monitor_stop(); 2431 if (ret) { 2432 RTE_LOG(ERR, EAL, 2433 "fail to stop device event monitor."); 2434 return; 2435 } 2436 2437 ret = rte_dev_event_callback_unregister(NULL, 2438 dev_event_callback, NULL); 2439 if (ret < 0) { 2440 RTE_LOG(ERR, EAL, 2441 "fail to unregister device event callback.\n"); 2442 return; 2443 } 2444 2445 ret = rte_dev_hotplug_handle_disable(); 2446 if (ret) { 2447 RTE_LOG(ERR, EAL, 2448 "fail to disable hotplug handling.\n"); 2449 return; 2450 } 2451 } 2452 2453 printf("\nBye...\n"); 2454 } 2455 2456 typedef void (*cmd_func_t)(void); 2457 struct pmd_test_command { 2458 const char *cmd_name; 2459 cmd_func_t cmd_func; 2460 }; 2461 2462 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2463 2464 /* Check the link status of all ports in up to 9s, and print them finally */ 2465 static void 2466 check_all_ports_link_status(uint32_t port_mask) 2467 { 2468 #define CHECK_INTERVAL 100 /* 100ms */ 2469 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2470 portid_t portid; 2471 uint8_t count, all_ports_up, print_flag = 0; 2472 struct rte_eth_link link; 2473 2474 printf("Checking link statuses...\n"); 2475 fflush(stdout); 2476 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2477 all_ports_up = 1; 2478 RTE_ETH_FOREACH_DEV(portid) { 2479 if ((port_mask & (1 << portid)) == 0) 2480 continue; 2481 memset(&link, 0, sizeof(link)); 2482 rte_eth_link_get_nowait(portid, &link); 2483 /* print link status if flag set */ 2484 if (print_flag == 1) { 2485 if (link.link_status) 2486 printf( 2487 "Port%d Link Up. speed %u Mbps- %s\n", 2488 portid, link.link_speed, 2489 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2490 ("full-duplex") : ("half-duplex\n")); 2491 else 2492 printf("Port %d Link Down\n", portid); 2493 continue; 2494 } 2495 /* clear all_ports_up flag if any link down */ 2496 if (link.link_status == ETH_LINK_DOWN) { 2497 all_ports_up = 0; 2498 break; 2499 } 2500 } 2501 /* after finally printing all link status, get out */ 2502 if (print_flag == 1) 2503 break; 2504 2505 if (all_ports_up == 0) { 2506 fflush(stdout); 2507 rte_delay_ms(CHECK_INTERVAL); 2508 } 2509 2510 /* set the print_flag if all ports up or timeout */ 2511 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2512 print_flag = 1; 2513 } 2514 2515 if (lsc_interrupt) 2516 break; 2517 } 2518 } 2519 2520 /* 2521 * This callback is for remove a port for a device. It has limitation because 2522 * it is not for multiple port removal for a device. 2523 * TODO: the device detach invoke will plan to be removed from user side to 2524 * eal. And convert all PMDs to free port resources on ether device closing. 2525 */ 2526 static void 2527 rmv_port_callback(void *arg) 2528 { 2529 int need_to_start = 0; 2530 int org_no_link_check = no_link_check; 2531 portid_t port_id = (intptr_t)arg; 2532 2533 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2534 2535 if (!test_done && port_is_forwarding(port_id)) { 2536 need_to_start = 1; 2537 stop_packet_forwarding(); 2538 } 2539 no_link_check = 1; 2540 stop_port(port_id); 2541 no_link_check = org_no_link_check; 2542 close_port(port_id); 2543 detach_port_device(port_id); 2544 if (need_to_start) 2545 start_packet_forwarding(0); 2546 } 2547 2548 /* This function is used by the interrupt thread */ 2549 static int 2550 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2551 void *ret_param) 2552 { 2553 RTE_SET_USED(param); 2554 RTE_SET_USED(ret_param); 2555 2556 if (type >= RTE_ETH_EVENT_MAX) { 2557 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2558 port_id, __func__, type); 2559 fflush(stderr); 2560 } else if (event_print_mask & (UINT32_C(1) << type)) { 2561 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2562 eth_event_desc[type]); 2563 fflush(stdout); 2564 } 2565 2566 switch (type) { 2567 case RTE_ETH_EVENT_NEW: 2568 ports[port_id].need_setup = 1; 2569 ports[port_id].port_status = RTE_PORT_HANDLING; 2570 break; 2571 case RTE_ETH_EVENT_INTR_RMV: 2572 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2573 break; 2574 if (rte_eal_alarm_set(100000, 2575 rmv_port_callback, (void *)(intptr_t)port_id)) 2576 fprintf(stderr, "Could not set up deferred device removal\n"); 2577 break; 2578 default: 2579 break; 2580 } 2581 return 0; 2582 } 2583 2584 static int 2585 register_eth_event_callback(void) 2586 { 2587 int ret; 2588 enum rte_eth_event_type event; 2589 2590 for (event = RTE_ETH_EVENT_UNKNOWN; 2591 event < RTE_ETH_EVENT_MAX; event++) { 2592 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2593 event, 2594 eth_event_callback, 2595 NULL); 2596 if (ret != 0) { 2597 TESTPMD_LOG(ERR, "Failed to register callback for " 2598 "%s event\n", eth_event_desc[event]); 2599 return -1; 2600 } 2601 } 2602 2603 return 0; 2604 } 2605 2606 /* This function is used by the interrupt thread */ 2607 static void 2608 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2609 __rte_unused void *arg) 2610 { 2611 uint16_t port_id; 2612 int ret; 2613 2614 if (type >= RTE_DEV_EVENT_MAX) { 2615 fprintf(stderr, "%s called upon invalid event %d\n", 2616 __func__, type); 2617 fflush(stderr); 2618 } 2619 2620 switch (type) { 2621 case RTE_DEV_EVENT_REMOVE: 2622 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2623 device_name); 2624 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2625 if (ret) { 2626 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2627 device_name); 2628 return; 2629 } 2630 /* 2631 * Because the user's callback is invoked in eal interrupt 2632 * callback, the interrupt callback need to be finished before 2633 * it can be unregistered when detaching device. So finish 2634 * callback soon and use a deferred removal to detach device 2635 * is need. It is a workaround, once the device detaching be 2636 * moved into the eal in the future, the deferred removal could 2637 * be deleted. 2638 */ 2639 if (rte_eal_alarm_set(100000, 2640 rmv_port_callback, (void *)(intptr_t)port_id)) 2641 RTE_LOG(ERR, EAL, 2642 "Could not set up deferred device removal\n"); 2643 break; 2644 case RTE_DEV_EVENT_ADD: 2645 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2646 device_name); 2647 /* TODO: After finish kernel driver binding, 2648 * begin to attach port. 2649 */ 2650 break; 2651 default: 2652 break; 2653 } 2654 } 2655 2656 static int 2657 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2658 { 2659 uint16_t i; 2660 int diag; 2661 uint8_t mapping_found = 0; 2662 2663 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2664 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2665 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2666 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2667 tx_queue_stats_mappings[i].queue_id, 2668 tx_queue_stats_mappings[i].stats_counter_id); 2669 if (diag != 0) 2670 return diag; 2671 mapping_found = 1; 2672 } 2673 } 2674 if (mapping_found) 2675 port->tx_queue_stats_mapping_enabled = 1; 2676 return 0; 2677 } 2678 2679 static int 2680 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2681 { 2682 uint16_t i; 2683 int diag; 2684 uint8_t mapping_found = 0; 2685 2686 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2687 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2688 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2689 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2690 rx_queue_stats_mappings[i].queue_id, 2691 rx_queue_stats_mappings[i].stats_counter_id); 2692 if (diag != 0) 2693 return diag; 2694 mapping_found = 1; 2695 } 2696 } 2697 if (mapping_found) 2698 port->rx_queue_stats_mapping_enabled = 1; 2699 return 0; 2700 } 2701 2702 static void 2703 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2704 { 2705 int diag = 0; 2706 2707 diag = set_tx_queue_stats_mapping_registers(pi, port); 2708 if (diag != 0) { 2709 if (diag == -ENOTSUP) { 2710 port->tx_queue_stats_mapping_enabled = 0; 2711 printf("TX queue stats mapping not supported port id=%d\n", pi); 2712 } 2713 else 2714 rte_exit(EXIT_FAILURE, 2715 "set_tx_queue_stats_mapping_registers " 2716 "failed for port id=%d diag=%d\n", 2717 pi, diag); 2718 } 2719 2720 diag = set_rx_queue_stats_mapping_registers(pi, port); 2721 if (diag != 0) { 2722 if (diag == -ENOTSUP) { 2723 port->rx_queue_stats_mapping_enabled = 0; 2724 printf("RX queue stats mapping not supported port id=%d\n", pi); 2725 } 2726 else 2727 rte_exit(EXIT_FAILURE, 2728 "set_rx_queue_stats_mapping_registers " 2729 "failed for port id=%d diag=%d\n", 2730 pi, diag); 2731 } 2732 } 2733 2734 static void 2735 rxtx_port_config(struct rte_port *port) 2736 { 2737 uint16_t qid; 2738 2739 for (qid = 0; qid < nb_rxq; qid++) { 2740 port->rx_conf[qid] = port->dev_info.default_rxconf; 2741 2742 /* Check if any Rx parameters have been passed */ 2743 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2744 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2745 2746 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2747 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2748 2749 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2750 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2751 2752 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2753 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2754 2755 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2756 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2757 2758 port->nb_rx_desc[qid] = nb_rxd; 2759 } 2760 2761 for (qid = 0; qid < nb_txq; qid++) { 2762 port->tx_conf[qid] = port->dev_info.default_txconf; 2763 2764 /* Check if any Tx parameters have been passed */ 2765 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2766 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2767 2768 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2769 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2770 2771 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2772 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2773 2774 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2775 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2776 2777 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2778 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2779 2780 port->nb_tx_desc[qid] = nb_txd; 2781 } 2782 } 2783 2784 void 2785 init_port_config(void) 2786 { 2787 portid_t pid; 2788 struct rte_port *port; 2789 2790 RTE_ETH_FOREACH_DEV(pid) { 2791 port = &ports[pid]; 2792 port->dev_conf.fdir_conf = fdir_conf; 2793 rte_eth_dev_info_get(pid, &port->dev_info); 2794 if (nb_rxq > 1) { 2795 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2796 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2797 rss_hf & port->dev_info.flow_type_rss_offloads; 2798 } else { 2799 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2800 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2801 } 2802 2803 if (port->dcb_flag == 0) { 2804 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2805 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2806 else 2807 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2808 } 2809 2810 rxtx_port_config(port); 2811 2812 rte_eth_macaddr_get(pid, &port->eth_addr); 2813 2814 map_port_queue_stats_mapping_registers(pid, port); 2815 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2816 rte_pmd_ixgbe_bypass_init(pid); 2817 #endif 2818 2819 if (lsc_interrupt && 2820 (rte_eth_devices[pid].data->dev_flags & 2821 RTE_ETH_DEV_INTR_LSC)) 2822 port->dev_conf.intr_conf.lsc = 1; 2823 if (rmv_interrupt && 2824 (rte_eth_devices[pid].data->dev_flags & 2825 RTE_ETH_DEV_INTR_RMV)) 2826 port->dev_conf.intr_conf.rmv = 1; 2827 } 2828 } 2829 2830 void set_port_slave_flag(portid_t slave_pid) 2831 { 2832 struct rte_port *port; 2833 2834 port = &ports[slave_pid]; 2835 port->slave_flag = 1; 2836 } 2837 2838 void clear_port_slave_flag(portid_t slave_pid) 2839 { 2840 struct rte_port *port; 2841 2842 port = &ports[slave_pid]; 2843 port->slave_flag = 0; 2844 } 2845 2846 uint8_t port_is_bonding_slave(portid_t slave_pid) 2847 { 2848 struct rte_port *port; 2849 2850 port = &ports[slave_pid]; 2851 if ((rte_eth_devices[slave_pid].data->dev_flags & 2852 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2853 return 1; 2854 return 0; 2855 } 2856 2857 const uint16_t vlan_tags[] = { 2858 0, 1, 2, 3, 4, 5, 6, 7, 2859 8, 9, 10, 11, 12, 13, 14, 15, 2860 16, 17, 18, 19, 20, 21, 22, 23, 2861 24, 25, 26, 27, 28, 29, 30, 31 2862 }; 2863 2864 static int 2865 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 2866 enum dcb_mode_enable dcb_mode, 2867 enum rte_eth_nb_tcs num_tcs, 2868 uint8_t pfc_en) 2869 { 2870 uint8_t i; 2871 int32_t rc; 2872 struct rte_eth_rss_conf rss_conf; 2873 2874 /* 2875 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2876 * given above, and the number of traffic classes available for use. 2877 */ 2878 if (dcb_mode == DCB_VT_ENABLED) { 2879 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2880 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2881 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2882 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2883 2884 /* VMDQ+DCB RX and TX configurations */ 2885 vmdq_rx_conf->enable_default_pool = 0; 2886 vmdq_rx_conf->default_pool = 0; 2887 vmdq_rx_conf->nb_queue_pools = 2888 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2889 vmdq_tx_conf->nb_queue_pools = 2890 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2891 2892 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2893 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2894 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2895 vmdq_rx_conf->pool_map[i].pools = 2896 1 << (i % vmdq_rx_conf->nb_queue_pools); 2897 } 2898 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2899 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 2900 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 2901 } 2902 2903 /* set DCB mode of RX and TX of multiple queues */ 2904 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2905 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2906 } else { 2907 struct rte_eth_dcb_rx_conf *rx_conf = 2908 ð_conf->rx_adv_conf.dcb_rx_conf; 2909 struct rte_eth_dcb_tx_conf *tx_conf = 2910 ð_conf->tx_adv_conf.dcb_tx_conf; 2911 2912 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 2913 if (rc != 0) 2914 return rc; 2915 2916 rx_conf->nb_tcs = num_tcs; 2917 tx_conf->nb_tcs = num_tcs; 2918 2919 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2920 rx_conf->dcb_tc[i] = i % num_tcs; 2921 tx_conf->dcb_tc[i] = i % num_tcs; 2922 } 2923 2924 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2925 eth_conf->rx_adv_conf.rss_conf = rss_conf; 2926 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2927 } 2928 2929 if (pfc_en) 2930 eth_conf->dcb_capability_en = 2931 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2932 else 2933 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2934 2935 return 0; 2936 } 2937 2938 int 2939 init_port_dcb_config(portid_t pid, 2940 enum dcb_mode_enable dcb_mode, 2941 enum rte_eth_nb_tcs num_tcs, 2942 uint8_t pfc_en) 2943 { 2944 struct rte_eth_conf port_conf; 2945 struct rte_port *rte_port; 2946 int retval; 2947 uint16_t i; 2948 2949 rte_port = &ports[pid]; 2950 2951 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2952 /* Enter DCB configuration status */ 2953 dcb_config = 1; 2954 2955 port_conf.rxmode = rte_port->dev_conf.rxmode; 2956 port_conf.txmode = rte_port->dev_conf.txmode; 2957 2958 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2959 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 2960 if (retval < 0) 2961 return retval; 2962 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2963 2964 /* re-configure the device . */ 2965 rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 2966 2967 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2968 2969 /* If dev_info.vmdq_pool_base is greater than 0, 2970 * the queue id of vmdq pools is started after pf queues. 2971 */ 2972 if (dcb_mode == DCB_VT_ENABLED && 2973 rte_port->dev_info.vmdq_pool_base > 0) { 2974 printf("VMDQ_DCB multi-queue mode is nonsensical" 2975 " for port %d.", pid); 2976 return -1; 2977 } 2978 2979 /* Assume the ports in testpmd have the same dcb capability 2980 * and has the same number of rxq and txq in dcb mode 2981 */ 2982 if (dcb_mode == DCB_VT_ENABLED) { 2983 if (rte_port->dev_info.max_vfs > 0) { 2984 nb_rxq = rte_port->dev_info.nb_rx_queues; 2985 nb_txq = rte_port->dev_info.nb_tx_queues; 2986 } else { 2987 nb_rxq = rte_port->dev_info.max_rx_queues; 2988 nb_txq = rte_port->dev_info.max_tx_queues; 2989 } 2990 } else { 2991 /*if vt is disabled, use all pf queues */ 2992 if (rte_port->dev_info.vmdq_pool_base == 0) { 2993 nb_rxq = rte_port->dev_info.max_rx_queues; 2994 nb_txq = rte_port->dev_info.max_tx_queues; 2995 } else { 2996 nb_rxq = (queueid_t)num_tcs; 2997 nb_txq = (queueid_t)num_tcs; 2998 2999 } 3000 } 3001 rx_free_thresh = 64; 3002 3003 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3004 3005 rxtx_port_config(rte_port); 3006 /* VLAN filter */ 3007 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3008 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3009 rx_vft_set(pid, vlan_tags[i], 1); 3010 3011 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 3012 map_port_queue_stats_mapping_registers(pid, rte_port); 3013 3014 rte_port->dcb_flag = 1; 3015 3016 return 0; 3017 } 3018 3019 static void 3020 init_port(void) 3021 { 3022 /* Configuration of Ethernet ports. */ 3023 ports = rte_zmalloc("testpmd: ports", 3024 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3025 RTE_CACHE_LINE_SIZE); 3026 if (ports == NULL) { 3027 rte_exit(EXIT_FAILURE, 3028 "rte_zmalloc(%d struct rte_port) failed\n", 3029 RTE_MAX_ETHPORTS); 3030 } 3031 3032 /* Initialize ports NUMA structures */ 3033 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3034 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3035 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3036 } 3037 3038 static void 3039 force_quit(void) 3040 { 3041 pmd_test_exit(); 3042 prompt_exit(); 3043 } 3044 3045 static void 3046 print_stats(void) 3047 { 3048 uint8_t i; 3049 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3050 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3051 3052 /* Clear screen and move to top left */ 3053 printf("%s%s", clr, top_left); 3054 3055 printf("\nPort statistics ===================================="); 3056 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3057 nic_stats_display(fwd_ports_ids[i]); 3058 3059 fflush(stdout); 3060 } 3061 3062 static void 3063 signal_handler(int signum) 3064 { 3065 if (signum == SIGINT || signum == SIGTERM) { 3066 printf("\nSignal %d received, preparing to exit...\n", 3067 signum); 3068 #ifdef RTE_LIBRTE_PDUMP 3069 /* uninitialize packet capture framework */ 3070 rte_pdump_uninit(); 3071 #endif 3072 #ifdef RTE_LIBRTE_LATENCY_STATS 3073 rte_latencystats_uninit(); 3074 #endif 3075 force_quit(); 3076 /* Set flag to indicate the force termination. */ 3077 f_quit = 1; 3078 /* exit with the expected status */ 3079 signal(signum, SIG_DFL); 3080 kill(getpid(), signum); 3081 } 3082 } 3083 3084 int 3085 main(int argc, char** argv) 3086 { 3087 int diag; 3088 portid_t port_id; 3089 uint16_t count; 3090 int ret; 3091 3092 signal(SIGINT, signal_handler); 3093 signal(SIGTERM, signal_handler); 3094 3095 diag = rte_eal_init(argc, argv); 3096 if (diag < 0) 3097 rte_panic("Cannot init EAL\n"); 3098 3099 testpmd_logtype = rte_log_register("testpmd"); 3100 if (testpmd_logtype < 0) 3101 rte_panic("Cannot register log type"); 3102 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3103 3104 ret = register_eth_event_callback(); 3105 if (ret != 0) 3106 rte_panic("Cannot register for ethdev events"); 3107 3108 #ifdef RTE_LIBRTE_PDUMP 3109 /* initialize packet capture framework */ 3110 rte_pdump_init(); 3111 #endif 3112 3113 count = 0; 3114 RTE_ETH_FOREACH_DEV(port_id) { 3115 ports_ids[count] = port_id; 3116 count++; 3117 } 3118 nb_ports = (portid_t) count; 3119 if (nb_ports == 0) 3120 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3121 3122 /* allocate port structures, and init them */ 3123 init_port(); 3124 3125 set_def_fwd_config(); 3126 if (nb_lcores == 0) 3127 rte_panic("Empty set of forwarding logical cores - check the " 3128 "core mask supplied in the command parameters\n"); 3129 3130 /* Bitrate/latency stats disabled by default */ 3131 #ifdef RTE_LIBRTE_BITRATE 3132 bitrate_enabled = 0; 3133 #endif 3134 #ifdef RTE_LIBRTE_LATENCY_STATS 3135 latencystats_enabled = 0; 3136 #endif 3137 3138 /* on FreeBSD, mlockall() is disabled by default */ 3139 #ifdef RTE_EXEC_ENV_FREEBSD 3140 do_mlockall = 0; 3141 #else 3142 do_mlockall = 1; 3143 #endif 3144 3145 argc -= diag; 3146 argv += diag; 3147 if (argc > 1) 3148 launch_args_parse(argc, argv); 3149 3150 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3151 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3152 strerror(errno)); 3153 } 3154 3155 if (tx_first && interactive) 3156 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3157 "interactive mode.\n"); 3158 3159 if (tx_first && lsc_interrupt) { 3160 printf("Warning: lsc_interrupt needs to be off when " 3161 " using tx_first. Disabling.\n"); 3162 lsc_interrupt = 0; 3163 } 3164 3165 if (!nb_rxq && !nb_txq) 3166 printf("Warning: Either rx or tx queues should be non-zero\n"); 3167 3168 if (nb_rxq > 1 && nb_rxq > nb_txq) 3169 printf("Warning: nb_rxq=%d enables RSS configuration, " 3170 "but nb_txq=%d will prevent to fully test it.\n", 3171 nb_rxq, nb_txq); 3172 3173 init_config(); 3174 3175 if (hot_plug) { 3176 ret = rte_dev_hotplug_handle_enable(); 3177 if (ret) { 3178 RTE_LOG(ERR, EAL, 3179 "fail to enable hotplug handling."); 3180 return -1; 3181 } 3182 3183 ret = rte_dev_event_monitor_start(); 3184 if (ret) { 3185 RTE_LOG(ERR, EAL, 3186 "fail to start device event monitoring."); 3187 return -1; 3188 } 3189 3190 ret = rte_dev_event_callback_register(NULL, 3191 dev_event_callback, NULL); 3192 if (ret) { 3193 RTE_LOG(ERR, EAL, 3194 "fail to register device event callback\n"); 3195 return -1; 3196 } 3197 } 3198 3199 if (start_port(RTE_PORT_ALL) != 0) 3200 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3201 3202 /* set all ports to promiscuous mode by default */ 3203 RTE_ETH_FOREACH_DEV(port_id) 3204 rte_eth_promiscuous_enable(port_id); 3205 3206 /* Init metrics library */ 3207 rte_metrics_init(rte_socket_id()); 3208 3209 #ifdef RTE_LIBRTE_LATENCY_STATS 3210 if (latencystats_enabled != 0) { 3211 int ret = rte_latencystats_init(1, NULL); 3212 if (ret) 3213 printf("Warning: latencystats init()" 3214 " returned error %d\n", ret); 3215 printf("Latencystats running on lcore %d\n", 3216 latencystats_lcore_id); 3217 } 3218 #endif 3219 3220 /* Setup bitrate stats */ 3221 #ifdef RTE_LIBRTE_BITRATE 3222 if (bitrate_enabled != 0) { 3223 bitrate_data = rte_stats_bitrate_create(); 3224 if (bitrate_data == NULL) 3225 rte_exit(EXIT_FAILURE, 3226 "Could not allocate bitrate data.\n"); 3227 rte_stats_bitrate_reg(bitrate_data); 3228 } 3229 #endif 3230 3231 #ifdef RTE_LIBRTE_CMDLINE 3232 if (strlen(cmdline_filename) != 0) 3233 cmdline_read_from_file(cmdline_filename); 3234 3235 if (interactive == 1) { 3236 if (auto_start) { 3237 printf("Start automatic packet forwarding\n"); 3238 start_packet_forwarding(0); 3239 } 3240 prompt(); 3241 pmd_test_exit(); 3242 } else 3243 #endif 3244 { 3245 char c; 3246 int rc; 3247 3248 f_quit = 0; 3249 3250 printf("No commandline core given, start packet forwarding\n"); 3251 start_packet_forwarding(tx_first); 3252 if (stats_period != 0) { 3253 uint64_t prev_time = 0, cur_time, diff_time = 0; 3254 uint64_t timer_period; 3255 3256 /* Convert to number of cycles */ 3257 timer_period = stats_period * rte_get_timer_hz(); 3258 3259 while (f_quit == 0) { 3260 cur_time = rte_get_timer_cycles(); 3261 diff_time += cur_time - prev_time; 3262 3263 if (diff_time >= timer_period) { 3264 print_stats(); 3265 /* Reset the timer */ 3266 diff_time = 0; 3267 } 3268 /* Sleep to avoid unnecessary checks */ 3269 prev_time = cur_time; 3270 sleep(1); 3271 } 3272 } 3273 3274 printf("Press enter to exit\n"); 3275 rc = read(0, &c, 1); 3276 pmd_test_exit(); 3277 if (rc < 0) 3278 return 1; 3279 } 3280 3281 return 0; 3282 } 3283