1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 static volatile uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 /* 253 * Configuration of packet segments used by the "txonly" processing engine. 254 */ 255 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 256 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 257 TXONLY_DEF_PACKET_LEN, 258 }; 259 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 260 261 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 262 /**< Split policy for packets to TX. */ 263 264 uint8_t txonly_multi_flow; 265 /**< Whether multiple flows are generated in TXONLY mode. */ 266 267 uint32_t tx_pkt_times_inter; 268 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 269 270 uint32_t tx_pkt_times_intra; 271 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 272 273 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 274 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 275 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 276 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 277 278 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 279 uint8_t dcb_config = 0; 280 281 /* 282 * Configurable number of RX/TX queues. 283 */ 284 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 285 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 286 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 287 288 /* 289 * Configurable number of RX/TX ring descriptors. 290 * Defaults are supplied by drivers via ethdev. 291 */ 292 #define RX_DESC_DEFAULT 0 293 #define TX_DESC_DEFAULT 0 294 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 295 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 296 297 #define RTE_PMD_PARAM_UNSET -1 298 /* 299 * Configurable values of RX and TX ring threshold registers. 300 */ 301 302 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 303 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 304 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 305 306 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 307 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 308 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 309 310 /* 311 * Configurable value of RX free threshold. 312 */ 313 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 314 315 /* 316 * Configurable value of RX drop enable. 317 */ 318 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 319 320 /* 321 * Configurable value of TX free threshold. 322 */ 323 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 324 325 /* 326 * Configurable value of TX RS bit threshold. 327 */ 328 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 329 330 /* 331 * Configurable value of buffered packets before sending. 332 */ 333 uint16_t noisy_tx_sw_bufsz; 334 335 /* 336 * Configurable value of packet buffer timeout. 337 */ 338 uint16_t noisy_tx_sw_buf_flush_time; 339 340 /* 341 * Configurable value for size of VNF internal memory area 342 * used for simulating noisy neighbour behaviour 343 */ 344 uint64_t noisy_lkup_mem_sz; 345 346 /* 347 * Configurable value of number of random writes done in 348 * VNF simulation memory area. 349 */ 350 uint64_t noisy_lkup_num_writes; 351 352 /* 353 * Configurable value of number of random reads done in 354 * VNF simulation memory area. 355 */ 356 uint64_t noisy_lkup_num_reads; 357 358 /* 359 * Configurable value of number of random reads/writes done in 360 * VNF simulation memory area. 361 */ 362 uint64_t noisy_lkup_num_reads_writes; 363 364 /* 365 * Receive Side Scaling (RSS) configuration. 366 */ 367 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 368 369 /* 370 * Port topology configuration 371 */ 372 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 373 374 /* 375 * Avoids to flush all the RX streams before starts forwarding. 376 */ 377 uint8_t no_flush_rx = 0; /* flush by default */ 378 379 /* 380 * Flow API isolated mode. 381 */ 382 uint8_t flow_isolate_all; 383 384 /* 385 * Avoids to check link status when starting/stopping a port. 386 */ 387 uint8_t no_link_check = 0; /* check by default */ 388 389 /* 390 * Don't automatically start all ports in interactive mode. 391 */ 392 uint8_t no_device_start = 0; 393 394 /* 395 * Enable link status change notification 396 */ 397 uint8_t lsc_interrupt = 1; /* enabled by default */ 398 399 /* 400 * Enable device removal notification. 401 */ 402 uint8_t rmv_interrupt = 1; /* enabled by default */ 403 404 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 405 406 /* After attach, port setup is called on event or by iterator */ 407 bool setup_on_probe_event = true; 408 409 /* Clear ptypes on port initialization. */ 410 uint8_t clear_ptypes = true; 411 412 /* Hairpin ports configuration mode. */ 413 uint32_t hairpin_mode; 414 415 /* Pretty printing of ethdev events */ 416 static const char * const eth_event_desc[] = { 417 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 418 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 419 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 420 [RTE_ETH_EVENT_INTR_RESET] = "reset", 421 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 422 [RTE_ETH_EVENT_IPSEC] = "IPsec", 423 [RTE_ETH_EVENT_MACSEC] = "MACsec", 424 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 425 [RTE_ETH_EVENT_NEW] = "device probed", 426 [RTE_ETH_EVENT_DESTROY] = "device released", 427 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 428 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 429 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 430 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 431 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 432 [RTE_ETH_EVENT_MAX] = NULL, 433 }; 434 435 /* 436 * Display or mask ether events 437 * Default to all events except VF_MBOX 438 */ 439 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 440 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 441 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 442 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 443 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 444 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 445 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 446 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 447 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 448 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 449 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 450 /* 451 * Decide if all memory are locked for performance. 452 */ 453 int do_mlockall = 0; 454 455 #ifdef RTE_LIB_LATENCYSTATS 456 457 /* 458 * Set when latency stats is enabled in the commandline 459 */ 460 uint8_t latencystats_enabled; 461 462 /* 463 * Lcore ID to service latency statistics. 464 */ 465 lcoreid_t latencystats_lcore_id = -1; 466 467 #endif 468 469 /* 470 * Ethernet device configuration. 471 */ 472 struct rte_eth_rxmode rx_mode; 473 474 struct rte_eth_txmode tx_mode = { 475 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 476 }; 477 478 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 479 480 /* 481 * Display zero values by default for xstats 482 */ 483 uint8_t xstats_hide_zero; 484 485 /* 486 * Measure of CPU cycles disabled by default 487 */ 488 uint8_t record_core_cycles; 489 490 /* 491 * Display of RX and TX bursts disabled by default 492 */ 493 uint8_t record_burst_stats; 494 495 /* 496 * Number of ports per shared Rx queue group, 0 disable. 497 */ 498 uint32_t rxq_share; 499 500 unsigned int num_sockets = 0; 501 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 502 503 #ifdef RTE_LIB_BITRATESTATS 504 /* Bitrate statistics */ 505 struct rte_stats_bitrates *bitrate_data; 506 lcoreid_t bitrate_lcore_id; 507 uint8_t bitrate_enabled; 508 #endif 509 510 #ifdef RTE_LIB_GRO 511 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 512 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 513 #endif 514 515 /* 516 * hexadecimal bitmask of RX mq mode can be enabled. 517 */ 518 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 519 520 /* 521 * Used to set forced link speed 522 */ 523 uint32_t eth_link_speed; 524 525 /* 526 * ID of the current process in multi-process, used to 527 * configure the queues to be polled. 528 */ 529 int proc_id; 530 531 /* 532 * Number of processes in multi-process, used to 533 * configure the queues to be polled. 534 */ 535 unsigned int num_procs = 1; 536 537 static void 538 eth_rx_metadata_negotiate_mp(uint16_t port_id) 539 { 540 uint64_t rx_meta_features = 0; 541 int ret; 542 543 if (!is_proc_primary()) 544 return; 545 546 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 547 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 548 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 549 550 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 551 if (ret == 0) { 552 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 553 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 554 port_id); 555 } 556 557 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 558 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 559 port_id); 560 } 561 562 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 563 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 564 port_id); 565 } 566 } else if (ret != -ENOTSUP) { 567 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 568 port_id, rte_strerror(-ret)); 569 } 570 } 571 572 static int 573 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 574 const struct rte_eth_conf *dev_conf) 575 { 576 if (is_proc_primary()) 577 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 578 dev_conf); 579 return 0; 580 } 581 582 static int 583 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 584 { 585 #ifdef RTE_NET_BOND 586 587 portid_t slave_pids[RTE_MAX_ETHPORTS]; 588 struct rte_port *port; 589 int num_slaves; 590 portid_t slave_pid; 591 int i; 592 593 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 594 RTE_MAX_ETHPORTS); 595 if (num_slaves < 0) { 596 fprintf(stderr, "Failed to get slave list for port = %u\n", 597 bond_pid); 598 return num_slaves; 599 } 600 601 for (i = 0; i < num_slaves; i++) { 602 slave_pid = slave_pids[i]; 603 port = &ports[slave_pid]; 604 port->port_status = 605 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 606 } 607 #else 608 RTE_SET_USED(bond_pid); 609 RTE_SET_USED(is_stop); 610 #endif 611 return 0; 612 } 613 614 static int 615 eth_dev_start_mp(uint16_t port_id) 616 { 617 int ret; 618 619 if (is_proc_primary()) { 620 ret = rte_eth_dev_start(port_id); 621 if (ret != 0) 622 return ret; 623 624 struct rte_port *port = &ports[port_id]; 625 626 /* 627 * Starting a bonded port also starts all slaves under the bonded 628 * device. So if this port is bond device, we need to modify the 629 * port status of these slaves. 630 */ 631 if (port->bond_flag == 1) 632 return change_bonding_slave_port_status(port_id, false); 633 } 634 635 return 0; 636 } 637 638 static int 639 eth_dev_stop_mp(uint16_t port_id) 640 { 641 int ret; 642 643 if (is_proc_primary()) { 644 ret = rte_eth_dev_stop(port_id); 645 if (ret != 0) 646 return ret; 647 648 struct rte_port *port = &ports[port_id]; 649 650 /* 651 * Stopping a bonded port also stops all slaves under the bonded 652 * device. So if this port is bond device, we need to modify the 653 * port status of these slaves. 654 */ 655 if (port->bond_flag == 1) 656 return change_bonding_slave_port_status(port_id, true); 657 } 658 659 return 0; 660 } 661 662 static void 663 mempool_free_mp(struct rte_mempool *mp) 664 { 665 if (is_proc_primary()) 666 rte_mempool_free(mp); 667 } 668 669 static int 670 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 671 { 672 if (is_proc_primary()) 673 return rte_eth_dev_set_mtu(port_id, mtu); 674 675 return 0; 676 } 677 678 /* Forward function declarations */ 679 static void setup_attached_port(portid_t pi); 680 static void check_all_ports_link_status(uint32_t port_mask); 681 static int eth_event_callback(portid_t port_id, 682 enum rte_eth_event_type type, 683 void *param, void *ret_param); 684 static void dev_event_callback(const char *device_name, 685 enum rte_dev_event_type type, 686 void *param); 687 static void fill_xstats_display_info(void); 688 689 /* 690 * Check if all the ports are started. 691 * If yes, return positive value. If not, return zero. 692 */ 693 static int all_ports_started(void); 694 695 #ifdef RTE_LIB_GSO 696 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 697 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 698 #endif 699 700 /* Holds the registered mbuf dynamic flags names. */ 701 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 702 703 704 /* 705 * Helper function to check if socket is already discovered. 706 * If yes, return positive value. If not, return zero. 707 */ 708 int 709 new_socket_id(unsigned int socket_id) 710 { 711 unsigned int i; 712 713 for (i = 0; i < num_sockets; i++) { 714 if (socket_ids[i] == socket_id) 715 return 0; 716 } 717 return 1; 718 } 719 720 /* 721 * Setup default configuration. 722 */ 723 static void 724 set_default_fwd_lcores_config(void) 725 { 726 unsigned int i; 727 unsigned int nb_lc; 728 unsigned int sock_num; 729 730 nb_lc = 0; 731 for (i = 0; i < RTE_MAX_LCORE; i++) { 732 if (!rte_lcore_is_enabled(i)) 733 continue; 734 sock_num = rte_lcore_to_socket_id(i); 735 if (new_socket_id(sock_num)) { 736 if (num_sockets >= RTE_MAX_NUMA_NODES) { 737 rte_exit(EXIT_FAILURE, 738 "Total sockets greater than %u\n", 739 RTE_MAX_NUMA_NODES); 740 } 741 socket_ids[num_sockets++] = sock_num; 742 } 743 if (i == rte_get_main_lcore()) 744 continue; 745 fwd_lcores_cpuids[nb_lc++] = i; 746 } 747 nb_lcores = (lcoreid_t) nb_lc; 748 nb_cfg_lcores = nb_lcores; 749 nb_fwd_lcores = 1; 750 } 751 752 static void 753 set_def_peer_eth_addrs(void) 754 { 755 portid_t i; 756 757 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 758 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 759 peer_eth_addrs[i].addr_bytes[5] = i; 760 } 761 } 762 763 static void 764 set_default_fwd_ports_config(void) 765 { 766 portid_t pt_id; 767 int i = 0; 768 769 RTE_ETH_FOREACH_DEV(pt_id) { 770 fwd_ports_ids[i++] = pt_id; 771 772 /* Update sockets info according to the attached device */ 773 int socket_id = rte_eth_dev_socket_id(pt_id); 774 if (socket_id >= 0 && new_socket_id(socket_id)) { 775 if (num_sockets >= RTE_MAX_NUMA_NODES) { 776 rte_exit(EXIT_FAILURE, 777 "Total sockets greater than %u\n", 778 RTE_MAX_NUMA_NODES); 779 } 780 socket_ids[num_sockets++] = socket_id; 781 } 782 } 783 784 nb_cfg_ports = nb_ports; 785 nb_fwd_ports = nb_ports; 786 } 787 788 void 789 set_def_fwd_config(void) 790 { 791 set_default_fwd_lcores_config(); 792 set_def_peer_eth_addrs(); 793 set_default_fwd_ports_config(); 794 } 795 796 #ifndef RTE_EXEC_ENV_WINDOWS 797 /* extremely pessimistic estimation of memory required to create a mempool */ 798 static int 799 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 800 { 801 unsigned int n_pages, mbuf_per_pg, leftover; 802 uint64_t total_mem, mbuf_mem, obj_sz; 803 804 /* there is no good way to predict how much space the mempool will 805 * occupy because it will allocate chunks on the fly, and some of those 806 * will come from default DPDK memory while some will come from our 807 * external memory, so just assume 128MB will be enough for everyone. 808 */ 809 uint64_t hdr_mem = 128 << 20; 810 811 /* account for possible non-contiguousness */ 812 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 813 if (obj_sz > pgsz) { 814 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 815 return -1; 816 } 817 818 mbuf_per_pg = pgsz / obj_sz; 819 leftover = (nb_mbufs % mbuf_per_pg) > 0; 820 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 821 822 mbuf_mem = n_pages * pgsz; 823 824 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 825 826 if (total_mem > SIZE_MAX) { 827 TESTPMD_LOG(ERR, "Memory size too big\n"); 828 return -1; 829 } 830 *out = (size_t)total_mem; 831 832 return 0; 833 } 834 835 static int 836 pagesz_flags(uint64_t page_sz) 837 { 838 /* as per mmap() manpage, all page sizes are log2 of page size 839 * shifted by MAP_HUGE_SHIFT 840 */ 841 int log2 = rte_log2_u64(page_sz); 842 843 return (log2 << HUGE_SHIFT); 844 } 845 846 static void * 847 alloc_mem(size_t memsz, size_t pgsz, bool huge) 848 { 849 void *addr; 850 int flags; 851 852 /* allocate anonymous hugepages */ 853 flags = MAP_ANONYMOUS | MAP_PRIVATE; 854 if (huge) 855 flags |= HUGE_FLAG | pagesz_flags(pgsz); 856 857 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 858 if (addr == MAP_FAILED) 859 return NULL; 860 861 return addr; 862 } 863 864 struct extmem_param { 865 void *addr; 866 size_t len; 867 size_t pgsz; 868 rte_iova_t *iova_table; 869 unsigned int iova_table_len; 870 }; 871 872 static int 873 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 874 bool huge) 875 { 876 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 877 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 878 unsigned int cur_page, n_pages, pgsz_idx; 879 size_t mem_sz, cur_pgsz; 880 rte_iova_t *iovas = NULL; 881 void *addr; 882 int ret; 883 884 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 885 /* skip anything that is too big */ 886 if (pgsizes[pgsz_idx] > SIZE_MAX) 887 continue; 888 889 cur_pgsz = pgsizes[pgsz_idx]; 890 891 /* if we were told not to allocate hugepages, override */ 892 if (!huge) 893 cur_pgsz = sysconf(_SC_PAGESIZE); 894 895 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 896 if (ret < 0) { 897 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 898 return -1; 899 } 900 901 /* allocate our memory */ 902 addr = alloc_mem(mem_sz, cur_pgsz, huge); 903 904 /* if we couldn't allocate memory with a specified page size, 905 * that doesn't mean we can't do it with other page sizes, so 906 * try another one. 907 */ 908 if (addr == NULL) 909 continue; 910 911 /* store IOVA addresses for every page in this memory area */ 912 n_pages = mem_sz / cur_pgsz; 913 914 iovas = malloc(sizeof(*iovas) * n_pages); 915 916 if (iovas == NULL) { 917 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 918 goto fail; 919 } 920 /* lock memory if it's not huge pages */ 921 if (!huge) 922 mlock(addr, mem_sz); 923 924 /* populate IOVA addresses */ 925 for (cur_page = 0; cur_page < n_pages; cur_page++) { 926 rte_iova_t iova; 927 size_t offset; 928 void *cur; 929 930 offset = cur_pgsz * cur_page; 931 cur = RTE_PTR_ADD(addr, offset); 932 933 /* touch the page before getting its IOVA */ 934 *(volatile char *)cur = 0; 935 936 iova = rte_mem_virt2iova(cur); 937 938 iovas[cur_page] = iova; 939 } 940 941 break; 942 } 943 /* if we couldn't allocate anything */ 944 if (iovas == NULL) 945 return -1; 946 947 param->addr = addr; 948 param->len = mem_sz; 949 param->pgsz = cur_pgsz; 950 param->iova_table = iovas; 951 param->iova_table_len = n_pages; 952 953 return 0; 954 fail: 955 free(iovas); 956 if (addr) 957 munmap(addr, mem_sz); 958 959 return -1; 960 } 961 962 static int 963 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 964 { 965 struct extmem_param param; 966 int socket_id, ret; 967 968 memset(¶m, 0, sizeof(param)); 969 970 /* check if our heap exists */ 971 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 972 if (socket_id < 0) { 973 /* create our heap */ 974 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 975 if (ret < 0) { 976 TESTPMD_LOG(ERR, "Cannot create heap\n"); 977 return -1; 978 } 979 } 980 981 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 982 if (ret < 0) { 983 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 984 return -1; 985 } 986 987 /* we now have a valid memory area, so add it to heap */ 988 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 989 param.addr, param.len, param.iova_table, 990 param.iova_table_len, param.pgsz); 991 992 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 993 994 /* not needed any more */ 995 free(param.iova_table); 996 997 if (ret < 0) { 998 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 999 munmap(param.addr, param.len); 1000 return -1; 1001 } 1002 1003 /* success */ 1004 1005 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1006 param.len >> 20); 1007 1008 return 0; 1009 } 1010 static void 1011 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1012 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1013 { 1014 uint16_t pid = 0; 1015 int ret; 1016 1017 RTE_ETH_FOREACH_DEV(pid) { 1018 struct rte_eth_dev_info dev_info; 1019 1020 ret = eth_dev_info_get_print_err(pid, &dev_info); 1021 if (ret != 0) { 1022 TESTPMD_LOG(DEBUG, 1023 "unable to get device info for port %d on addr 0x%p," 1024 "mempool unmapping will not be performed\n", 1025 pid, memhdr->addr); 1026 continue; 1027 } 1028 1029 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1030 if (ret) { 1031 TESTPMD_LOG(DEBUG, 1032 "unable to DMA unmap addr 0x%p " 1033 "for device %s\n", 1034 memhdr->addr, rte_dev_name(dev_info.device)); 1035 } 1036 } 1037 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1038 if (ret) { 1039 TESTPMD_LOG(DEBUG, 1040 "unable to un-register addr 0x%p\n", memhdr->addr); 1041 } 1042 } 1043 1044 static void 1045 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1046 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1047 { 1048 uint16_t pid = 0; 1049 size_t page_size = sysconf(_SC_PAGESIZE); 1050 int ret; 1051 1052 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1053 page_size); 1054 if (ret) { 1055 TESTPMD_LOG(DEBUG, 1056 "unable to register addr 0x%p\n", memhdr->addr); 1057 return; 1058 } 1059 RTE_ETH_FOREACH_DEV(pid) { 1060 struct rte_eth_dev_info dev_info; 1061 1062 ret = eth_dev_info_get_print_err(pid, &dev_info); 1063 if (ret != 0) { 1064 TESTPMD_LOG(DEBUG, 1065 "unable to get device info for port %d on addr 0x%p," 1066 "mempool mapping will not be performed\n", 1067 pid, memhdr->addr); 1068 continue; 1069 } 1070 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1071 if (ret) { 1072 TESTPMD_LOG(DEBUG, 1073 "unable to DMA map addr 0x%p " 1074 "for device %s\n", 1075 memhdr->addr, rte_dev_name(dev_info.device)); 1076 } 1077 } 1078 } 1079 #endif 1080 1081 static unsigned int 1082 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1083 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1084 { 1085 struct rte_pktmbuf_extmem *xmem; 1086 unsigned int ext_num, zone_num, elt_num; 1087 uint16_t elt_size; 1088 1089 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1090 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1091 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1092 1093 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1094 if (xmem == NULL) { 1095 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1096 "external buffer descriptors\n"); 1097 *ext_mem = NULL; 1098 return 0; 1099 } 1100 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1101 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1102 const struct rte_memzone *mz; 1103 char mz_name[RTE_MEMZONE_NAMESIZE]; 1104 int ret; 1105 1106 ret = snprintf(mz_name, sizeof(mz_name), 1107 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1108 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1109 errno = ENAMETOOLONG; 1110 ext_num = 0; 1111 break; 1112 } 1113 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1114 socket_id, 1115 RTE_MEMZONE_IOVA_CONTIG | 1116 RTE_MEMZONE_1GB | 1117 RTE_MEMZONE_SIZE_HINT_ONLY); 1118 if (mz == NULL) { 1119 /* 1120 * The caller exits on external buffer creation 1121 * error, so there is no need to free memzones. 1122 */ 1123 errno = ENOMEM; 1124 ext_num = 0; 1125 break; 1126 } 1127 xseg->buf_ptr = mz->addr; 1128 xseg->buf_iova = mz->iova; 1129 xseg->buf_len = EXTBUF_ZONE_SIZE; 1130 xseg->elt_size = elt_size; 1131 } 1132 if (ext_num == 0 && xmem != NULL) { 1133 free(xmem); 1134 xmem = NULL; 1135 } 1136 *ext_mem = xmem; 1137 return ext_num; 1138 } 1139 1140 /* 1141 * Configuration initialisation done once at init time. 1142 */ 1143 static struct rte_mempool * 1144 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1145 unsigned int socket_id, uint16_t size_idx) 1146 { 1147 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1148 struct rte_mempool *rte_mp = NULL; 1149 #ifndef RTE_EXEC_ENV_WINDOWS 1150 uint32_t mb_size; 1151 1152 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1153 #endif 1154 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1155 if (!is_proc_primary()) { 1156 rte_mp = rte_mempool_lookup(pool_name); 1157 if (rte_mp == NULL) 1158 rte_exit(EXIT_FAILURE, 1159 "Get mbuf pool for socket %u failed: %s\n", 1160 socket_id, rte_strerror(rte_errno)); 1161 return rte_mp; 1162 } 1163 1164 TESTPMD_LOG(INFO, 1165 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1166 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1167 1168 switch (mp_alloc_type) { 1169 case MP_ALLOC_NATIVE: 1170 { 1171 /* wrapper to rte_mempool_create() */ 1172 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1173 rte_mbuf_best_mempool_ops()); 1174 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1175 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1176 break; 1177 } 1178 #ifndef RTE_EXEC_ENV_WINDOWS 1179 case MP_ALLOC_ANON: 1180 { 1181 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1182 mb_size, (unsigned int) mb_mempool_cache, 1183 sizeof(struct rte_pktmbuf_pool_private), 1184 socket_id, mempool_flags); 1185 if (rte_mp == NULL) 1186 goto err; 1187 1188 if (rte_mempool_populate_anon(rte_mp) == 0) { 1189 rte_mempool_free(rte_mp); 1190 rte_mp = NULL; 1191 goto err; 1192 } 1193 rte_pktmbuf_pool_init(rte_mp, NULL); 1194 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1195 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1196 break; 1197 } 1198 case MP_ALLOC_XMEM: 1199 case MP_ALLOC_XMEM_HUGE: 1200 { 1201 int heap_socket; 1202 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1203 1204 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1205 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1206 1207 heap_socket = 1208 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1209 if (heap_socket < 0) 1210 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1211 1212 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1213 rte_mbuf_best_mempool_ops()); 1214 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1215 mb_mempool_cache, 0, mbuf_seg_size, 1216 heap_socket); 1217 break; 1218 } 1219 #endif 1220 case MP_ALLOC_XBUF: 1221 { 1222 struct rte_pktmbuf_extmem *ext_mem; 1223 unsigned int ext_num; 1224 1225 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1226 socket_id, pool_name, &ext_mem); 1227 if (ext_num == 0) 1228 rte_exit(EXIT_FAILURE, 1229 "Can't create pinned data buffers\n"); 1230 1231 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1232 rte_mbuf_best_mempool_ops()); 1233 rte_mp = rte_pktmbuf_pool_create_extbuf 1234 (pool_name, nb_mbuf, mb_mempool_cache, 1235 0, mbuf_seg_size, socket_id, 1236 ext_mem, ext_num); 1237 free(ext_mem); 1238 break; 1239 } 1240 default: 1241 { 1242 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1243 } 1244 } 1245 1246 #ifndef RTE_EXEC_ENV_WINDOWS 1247 err: 1248 #endif 1249 if (rte_mp == NULL) { 1250 rte_exit(EXIT_FAILURE, 1251 "Creation of mbuf pool for socket %u failed: %s\n", 1252 socket_id, rte_strerror(rte_errno)); 1253 } else if (verbose_level > 0) { 1254 rte_mempool_dump(stdout, rte_mp); 1255 } 1256 return rte_mp; 1257 } 1258 1259 /* 1260 * Check given socket id is valid or not with NUMA mode, 1261 * if valid, return 0, else return -1 1262 */ 1263 static int 1264 check_socket_id(const unsigned int socket_id) 1265 { 1266 static int warning_once = 0; 1267 1268 if (new_socket_id(socket_id)) { 1269 if (!warning_once && numa_support) 1270 fprintf(stderr, 1271 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1272 warning_once = 1; 1273 return -1; 1274 } 1275 return 0; 1276 } 1277 1278 /* 1279 * Get the allowed maximum number of RX queues. 1280 * *pid return the port id which has minimal value of 1281 * max_rx_queues in all ports. 1282 */ 1283 queueid_t 1284 get_allowed_max_nb_rxq(portid_t *pid) 1285 { 1286 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1287 bool max_rxq_valid = false; 1288 portid_t pi; 1289 struct rte_eth_dev_info dev_info; 1290 1291 RTE_ETH_FOREACH_DEV(pi) { 1292 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1293 continue; 1294 1295 max_rxq_valid = true; 1296 if (dev_info.max_rx_queues < allowed_max_rxq) { 1297 allowed_max_rxq = dev_info.max_rx_queues; 1298 *pid = pi; 1299 } 1300 } 1301 return max_rxq_valid ? allowed_max_rxq : 0; 1302 } 1303 1304 /* 1305 * Check input rxq is valid or not. 1306 * If input rxq is not greater than any of maximum number 1307 * of RX queues of all ports, it is valid. 1308 * if valid, return 0, else return -1 1309 */ 1310 int 1311 check_nb_rxq(queueid_t rxq) 1312 { 1313 queueid_t allowed_max_rxq; 1314 portid_t pid = 0; 1315 1316 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1317 if (rxq > allowed_max_rxq) { 1318 fprintf(stderr, 1319 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1320 rxq, allowed_max_rxq, pid); 1321 return -1; 1322 } 1323 return 0; 1324 } 1325 1326 /* 1327 * Get the allowed maximum number of TX queues. 1328 * *pid return the port id which has minimal value of 1329 * max_tx_queues in all ports. 1330 */ 1331 queueid_t 1332 get_allowed_max_nb_txq(portid_t *pid) 1333 { 1334 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1335 bool max_txq_valid = false; 1336 portid_t pi; 1337 struct rte_eth_dev_info dev_info; 1338 1339 RTE_ETH_FOREACH_DEV(pi) { 1340 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1341 continue; 1342 1343 max_txq_valid = true; 1344 if (dev_info.max_tx_queues < allowed_max_txq) { 1345 allowed_max_txq = dev_info.max_tx_queues; 1346 *pid = pi; 1347 } 1348 } 1349 return max_txq_valid ? allowed_max_txq : 0; 1350 } 1351 1352 /* 1353 * Check input txq is valid or not. 1354 * If input txq is not greater than any of maximum number 1355 * of TX queues of all ports, it is valid. 1356 * if valid, return 0, else return -1 1357 */ 1358 int 1359 check_nb_txq(queueid_t txq) 1360 { 1361 queueid_t allowed_max_txq; 1362 portid_t pid = 0; 1363 1364 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1365 if (txq > allowed_max_txq) { 1366 fprintf(stderr, 1367 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1368 txq, allowed_max_txq, pid); 1369 return -1; 1370 } 1371 return 0; 1372 } 1373 1374 /* 1375 * Get the allowed maximum number of RXDs of every rx queue. 1376 * *pid return the port id which has minimal value of 1377 * max_rxd in all queues of all ports. 1378 */ 1379 static uint16_t 1380 get_allowed_max_nb_rxd(portid_t *pid) 1381 { 1382 uint16_t allowed_max_rxd = UINT16_MAX; 1383 portid_t pi; 1384 struct rte_eth_dev_info dev_info; 1385 1386 RTE_ETH_FOREACH_DEV(pi) { 1387 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1388 continue; 1389 1390 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1391 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1392 *pid = pi; 1393 } 1394 } 1395 return allowed_max_rxd; 1396 } 1397 1398 /* 1399 * Get the allowed minimal number of RXDs of every rx queue. 1400 * *pid return the port id which has minimal value of 1401 * min_rxd in all queues of all ports. 1402 */ 1403 static uint16_t 1404 get_allowed_min_nb_rxd(portid_t *pid) 1405 { 1406 uint16_t allowed_min_rxd = 0; 1407 portid_t pi; 1408 struct rte_eth_dev_info dev_info; 1409 1410 RTE_ETH_FOREACH_DEV(pi) { 1411 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1412 continue; 1413 1414 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1415 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1416 *pid = pi; 1417 } 1418 } 1419 1420 return allowed_min_rxd; 1421 } 1422 1423 /* 1424 * Check input rxd is valid or not. 1425 * If input rxd is not greater than any of maximum number 1426 * of RXDs of every Rx queues and is not less than any of 1427 * minimal number of RXDs of every Rx queues, it is valid. 1428 * if valid, return 0, else return -1 1429 */ 1430 int 1431 check_nb_rxd(queueid_t rxd) 1432 { 1433 uint16_t allowed_max_rxd; 1434 uint16_t allowed_min_rxd; 1435 portid_t pid = 0; 1436 1437 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1438 if (rxd > allowed_max_rxd) { 1439 fprintf(stderr, 1440 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1441 rxd, allowed_max_rxd, pid); 1442 return -1; 1443 } 1444 1445 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1446 if (rxd < allowed_min_rxd) { 1447 fprintf(stderr, 1448 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1449 rxd, allowed_min_rxd, pid); 1450 return -1; 1451 } 1452 1453 return 0; 1454 } 1455 1456 /* 1457 * Get the allowed maximum number of TXDs of every rx queues. 1458 * *pid return the port id which has minimal value of 1459 * max_txd in every tx queue. 1460 */ 1461 static uint16_t 1462 get_allowed_max_nb_txd(portid_t *pid) 1463 { 1464 uint16_t allowed_max_txd = UINT16_MAX; 1465 portid_t pi; 1466 struct rte_eth_dev_info dev_info; 1467 1468 RTE_ETH_FOREACH_DEV(pi) { 1469 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1470 continue; 1471 1472 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1473 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1474 *pid = pi; 1475 } 1476 } 1477 return allowed_max_txd; 1478 } 1479 1480 /* 1481 * Get the allowed maximum number of TXDs of every tx queues. 1482 * *pid return the port id which has minimal value of 1483 * min_txd in every tx queue. 1484 */ 1485 static uint16_t 1486 get_allowed_min_nb_txd(portid_t *pid) 1487 { 1488 uint16_t allowed_min_txd = 0; 1489 portid_t pi; 1490 struct rte_eth_dev_info dev_info; 1491 1492 RTE_ETH_FOREACH_DEV(pi) { 1493 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1494 continue; 1495 1496 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1497 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1498 *pid = pi; 1499 } 1500 } 1501 1502 return allowed_min_txd; 1503 } 1504 1505 /* 1506 * Check input txd is valid or not. 1507 * If input txd is not greater than any of maximum number 1508 * of TXDs of every Rx queues, it is valid. 1509 * if valid, return 0, else return -1 1510 */ 1511 int 1512 check_nb_txd(queueid_t txd) 1513 { 1514 uint16_t allowed_max_txd; 1515 uint16_t allowed_min_txd; 1516 portid_t pid = 0; 1517 1518 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1519 if (txd > allowed_max_txd) { 1520 fprintf(stderr, 1521 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1522 txd, allowed_max_txd, pid); 1523 return -1; 1524 } 1525 1526 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1527 if (txd < allowed_min_txd) { 1528 fprintf(stderr, 1529 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1530 txd, allowed_min_txd, pid); 1531 return -1; 1532 } 1533 return 0; 1534 } 1535 1536 1537 /* 1538 * Get the allowed maximum number of hairpin queues. 1539 * *pid return the port id which has minimal value of 1540 * max_hairpin_queues in all ports. 1541 */ 1542 queueid_t 1543 get_allowed_max_nb_hairpinq(portid_t *pid) 1544 { 1545 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1546 portid_t pi; 1547 struct rte_eth_hairpin_cap cap; 1548 1549 RTE_ETH_FOREACH_DEV(pi) { 1550 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1551 *pid = pi; 1552 return 0; 1553 } 1554 if (cap.max_nb_queues < allowed_max_hairpinq) { 1555 allowed_max_hairpinq = cap.max_nb_queues; 1556 *pid = pi; 1557 } 1558 } 1559 return allowed_max_hairpinq; 1560 } 1561 1562 /* 1563 * Check input hairpin is valid or not. 1564 * If input hairpin is not greater than any of maximum number 1565 * of hairpin queues of all ports, it is valid. 1566 * if valid, return 0, else return -1 1567 */ 1568 int 1569 check_nb_hairpinq(queueid_t hairpinq) 1570 { 1571 queueid_t allowed_max_hairpinq; 1572 portid_t pid = 0; 1573 1574 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1575 if (hairpinq > allowed_max_hairpinq) { 1576 fprintf(stderr, 1577 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1578 hairpinq, allowed_max_hairpinq, pid); 1579 return -1; 1580 } 1581 return 0; 1582 } 1583 1584 static int 1585 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1586 { 1587 uint32_t eth_overhead; 1588 1589 if (dev_info->max_mtu != UINT16_MAX && 1590 dev_info->max_rx_pktlen > dev_info->max_mtu) 1591 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1592 else 1593 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1594 1595 return eth_overhead; 1596 } 1597 1598 static void 1599 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1600 { 1601 struct rte_port *port = &ports[pid]; 1602 int ret; 1603 int i; 1604 1605 eth_rx_metadata_negotiate_mp(pid); 1606 1607 port->dev_conf.txmode = tx_mode; 1608 port->dev_conf.rxmode = rx_mode; 1609 1610 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1611 if (ret != 0) 1612 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1613 1614 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1615 port->dev_conf.txmode.offloads &= 1616 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1617 1618 /* Apply Rx offloads configuration */ 1619 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1620 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1621 /* Apply Tx offloads configuration */ 1622 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1623 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1624 1625 if (eth_link_speed) 1626 port->dev_conf.link_speeds = eth_link_speed; 1627 1628 if (max_rx_pkt_len) 1629 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1630 get_eth_overhead(&port->dev_info); 1631 1632 /* set flag to initialize port/queue */ 1633 port->need_reconfig = 1; 1634 port->need_reconfig_queues = 1; 1635 port->socket_id = socket_id; 1636 port->tx_metadata = 0; 1637 1638 /* 1639 * Check for maximum number of segments per MTU. 1640 * Accordingly update the mbuf data size. 1641 */ 1642 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1643 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1644 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1645 uint16_t mtu; 1646 1647 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1648 uint16_t data_size = (mtu + eth_overhead) / 1649 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1650 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1651 1652 if (buffer_size > mbuf_data_size[0]) { 1653 mbuf_data_size[0] = buffer_size; 1654 TESTPMD_LOG(WARNING, 1655 "Configured mbuf size of the first segment %hu\n", 1656 mbuf_data_size[0]); 1657 } 1658 } 1659 } 1660 } 1661 1662 static void 1663 init_config(void) 1664 { 1665 portid_t pid; 1666 struct rte_mempool *mbp; 1667 unsigned int nb_mbuf_per_pool; 1668 lcoreid_t lc_id; 1669 #ifdef RTE_LIB_GRO 1670 struct rte_gro_param gro_param; 1671 #endif 1672 #ifdef RTE_LIB_GSO 1673 uint32_t gso_types; 1674 #endif 1675 1676 /* Configuration of logical cores. */ 1677 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1678 sizeof(struct fwd_lcore *) * nb_lcores, 1679 RTE_CACHE_LINE_SIZE); 1680 if (fwd_lcores == NULL) { 1681 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1682 "failed\n", nb_lcores); 1683 } 1684 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1685 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1686 sizeof(struct fwd_lcore), 1687 RTE_CACHE_LINE_SIZE); 1688 if (fwd_lcores[lc_id] == NULL) { 1689 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1690 "failed\n"); 1691 } 1692 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1693 } 1694 1695 RTE_ETH_FOREACH_DEV(pid) { 1696 uint32_t socket_id; 1697 1698 if (numa_support) { 1699 socket_id = port_numa[pid]; 1700 if (port_numa[pid] == NUMA_NO_CONFIG) { 1701 socket_id = rte_eth_dev_socket_id(pid); 1702 1703 /* 1704 * if socket_id is invalid, 1705 * set to the first available socket. 1706 */ 1707 if (check_socket_id(socket_id) < 0) 1708 socket_id = socket_ids[0]; 1709 } 1710 } else { 1711 socket_id = (socket_num == UMA_NO_CONFIG) ? 1712 0 : socket_num; 1713 } 1714 /* Apply default TxRx configuration for all ports */ 1715 init_config_port_offloads(pid, socket_id); 1716 } 1717 /* 1718 * Create pools of mbuf. 1719 * If NUMA support is disabled, create a single pool of mbuf in 1720 * socket 0 memory by default. 1721 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1722 * 1723 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1724 * nb_txd can be configured at run time. 1725 */ 1726 if (param_total_num_mbufs) 1727 nb_mbuf_per_pool = param_total_num_mbufs; 1728 else { 1729 nb_mbuf_per_pool = RX_DESC_MAX + 1730 (nb_lcores * mb_mempool_cache) + 1731 TX_DESC_MAX + MAX_PKT_BURST; 1732 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1733 } 1734 1735 if (numa_support) { 1736 uint8_t i, j; 1737 1738 for (i = 0; i < num_sockets; i++) 1739 for (j = 0; j < mbuf_data_size_n; j++) 1740 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1741 mbuf_pool_create(mbuf_data_size[j], 1742 nb_mbuf_per_pool, 1743 socket_ids[i], j); 1744 } else { 1745 uint8_t i; 1746 1747 for (i = 0; i < mbuf_data_size_n; i++) 1748 mempools[i] = mbuf_pool_create 1749 (mbuf_data_size[i], 1750 nb_mbuf_per_pool, 1751 socket_num == UMA_NO_CONFIG ? 1752 0 : socket_num, i); 1753 } 1754 1755 init_port_config(); 1756 1757 #ifdef RTE_LIB_GSO 1758 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1759 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1760 #endif 1761 /* 1762 * Records which Mbuf pool to use by each logical core, if needed. 1763 */ 1764 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1765 mbp = mbuf_pool_find( 1766 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1767 1768 if (mbp == NULL) 1769 mbp = mbuf_pool_find(0, 0); 1770 fwd_lcores[lc_id]->mbp = mbp; 1771 #ifdef RTE_LIB_GSO 1772 /* initialize GSO context */ 1773 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1774 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1775 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1776 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1777 RTE_ETHER_CRC_LEN; 1778 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1779 #endif 1780 } 1781 1782 fwd_config_setup(); 1783 1784 #ifdef RTE_LIB_GRO 1785 /* create a gro context for each lcore */ 1786 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1787 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1788 gro_param.max_item_per_flow = MAX_PKT_BURST; 1789 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1790 gro_param.socket_id = rte_lcore_to_socket_id( 1791 fwd_lcores_cpuids[lc_id]); 1792 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1793 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1794 rte_exit(EXIT_FAILURE, 1795 "rte_gro_ctx_create() failed\n"); 1796 } 1797 } 1798 #endif 1799 } 1800 1801 1802 void 1803 reconfig(portid_t new_port_id, unsigned socket_id) 1804 { 1805 /* Reconfiguration of Ethernet ports. */ 1806 init_config_port_offloads(new_port_id, socket_id); 1807 init_port_config(); 1808 } 1809 1810 int 1811 init_fwd_streams(void) 1812 { 1813 portid_t pid; 1814 struct rte_port *port; 1815 streamid_t sm_id, nb_fwd_streams_new; 1816 queueid_t q; 1817 1818 /* set socket id according to numa or not */ 1819 RTE_ETH_FOREACH_DEV(pid) { 1820 port = &ports[pid]; 1821 if (nb_rxq > port->dev_info.max_rx_queues) { 1822 fprintf(stderr, 1823 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1824 nb_rxq, port->dev_info.max_rx_queues); 1825 return -1; 1826 } 1827 if (nb_txq > port->dev_info.max_tx_queues) { 1828 fprintf(stderr, 1829 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1830 nb_txq, port->dev_info.max_tx_queues); 1831 return -1; 1832 } 1833 if (numa_support) { 1834 if (port_numa[pid] != NUMA_NO_CONFIG) 1835 port->socket_id = port_numa[pid]; 1836 else { 1837 port->socket_id = rte_eth_dev_socket_id(pid); 1838 1839 /* 1840 * if socket_id is invalid, 1841 * set to the first available socket. 1842 */ 1843 if (check_socket_id(port->socket_id) < 0) 1844 port->socket_id = socket_ids[0]; 1845 } 1846 } 1847 else { 1848 if (socket_num == UMA_NO_CONFIG) 1849 port->socket_id = 0; 1850 else 1851 port->socket_id = socket_num; 1852 } 1853 } 1854 1855 q = RTE_MAX(nb_rxq, nb_txq); 1856 if (q == 0) { 1857 fprintf(stderr, 1858 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1859 return -1; 1860 } 1861 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1862 if (nb_fwd_streams_new == nb_fwd_streams) 1863 return 0; 1864 /* clear the old */ 1865 if (fwd_streams != NULL) { 1866 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1867 if (fwd_streams[sm_id] == NULL) 1868 continue; 1869 rte_free(fwd_streams[sm_id]); 1870 fwd_streams[sm_id] = NULL; 1871 } 1872 rte_free(fwd_streams); 1873 fwd_streams = NULL; 1874 } 1875 1876 /* init new */ 1877 nb_fwd_streams = nb_fwd_streams_new; 1878 if (nb_fwd_streams) { 1879 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1880 sizeof(struct fwd_stream *) * nb_fwd_streams, 1881 RTE_CACHE_LINE_SIZE); 1882 if (fwd_streams == NULL) 1883 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1884 " (struct fwd_stream *)) failed\n", 1885 nb_fwd_streams); 1886 1887 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1888 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1889 " struct fwd_stream", sizeof(struct fwd_stream), 1890 RTE_CACHE_LINE_SIZE); 1891 if (fwd_streams[sm_id] == NULL) 1892 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1893 "(struct fwd_stream) failed\n"); 1894 } 1895 } 1896 1897 return 0; 1898 } 1899 1900 static void 1901 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1902 { 1903 uint64_t total_burst, sburst; 1904 uint64_t nb_burst; 1905 uint64_t burst_stats[4]; 1906 uint16_t pktnb_stats[4]; 1907 uint16_t nb_pkt; 1908 int burst_percent[4], sburstp; 1909 int i; 1910 1911 /* 1912 * First compute the total number of packet bursts and the 1913 * two highest numbers of bursts of the same number of packets. 1914 */ 1915 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1916 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1917 1918 /* Show stats for 0 burst size always */ 1919 total_burst = pbs->pkt_burst_spread[0]; 1920 burst_stats[0] = pbs->pkt_burst_spread[0]; 1921 pktnb_stats[0] = 0; 1922 1923 /* Find the next 2 burst sizes with highest occurrences. */ 1924 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1925 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1926 1927 if (nb_burst == 0) 1928 continue; 1929 1930 total_burst += nb_burst; 1931 1932 if (nb_burst > burst_stats[1]) { 1933 burst_stats[2] = burst_stats[1]; 1934 pktnb_stats[2] = pktnb_stats[1]; 1935 burst_stats[1] = nb_burst; 1936 pktnb_stats[1] = nb_pkt; 1937 } else if (nb_burst > burst_stats[2]) { 1938 burst_stats[2] = nb_burst; 1939 pktnb_stats[2] = nb_pkt; 1940 } 1941 } 1942 if (total_burst == 0) 1943 return; 1944 1945 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1946 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1947 if (i == 3) { 1948 printf("%d%% of other]\n", 100 - sburstp); 1949 return; 1950 } 1951 1952 sburst += burst_stats[i]; 1953 if (sburst == total_burst) { 1954 printf("%d%% of %d pkts]\n", 1955 100 - sburstp, (int) pktnb_stats[i]); 1956 return; 1957 } 1958 1959 burst_percent[i] = 1960 (double)burst_stats[i] / total_burst * 100; 1961 printf("%d%% of %d pkts + ", 1962 burst_percent[i], (int) pktnb_stats[i]); 1963 sburstp += burst_percent[i]; 1964 } 1965 } 1966 1967 static void 1968 fwd_stream_stats_display(streamid_t stream_id) 1969 { 1970 struct fwd_stream *fs; 1971 static const char *fwd_top_stats_border = "-------"; 1972 1973 fs = fwd_streams[stream_id]; 1974 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1975 (fs->fwd_dropped == 0)) 1976 return; 1977 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1978 "TX Port=%2d/Queue=%2d %s\n", 1979 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1980 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1981 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1982 " TX-dropped: %-14"PRIu64, 1983 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1984 1985 /* if checksum mode */ 1986 if (cur_fwd_eng == &csum_fwd_engine) { 1987 printf(" RX- bad IP checksum: %-14"PRIu64 1988 " Rx- bad L4 checksum: %-14"PRIu64 1989 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1990 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1991 fs->rx_bad_outer_l4_csum); 1992 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1993 fs->rx_bad_outer_ip_csum); 1994 } else { 1995 printf("\n"); 1996 } 1997 1998 if (record_burst_stats) { 1999 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2000 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2001 } 2002 } 2003 2004 void 2005 fwd_stats_display(void) 2006 { 2007 static const char *fwd_stats_border = "----------------------"; 2008 static const char *acc_stats_border = "+++++++++++++++"; 2009 struct { 2010 struct fwd_stream *rx_stream; 2011 struct fwd_stream *tx_stream; 2012 uint64_t tx_dropped; 2013 uint64_t rx_bad_ip_csum; 2014 uint64_t rx_bad_l4_csum; 2015 uint64_t rx_bad_outer_l4_csum; 2016 uint64_t rx_bad_outer_ip_csum; 2017 } ports_stats[RTE_MAX_ETHPORTS]; 2018 uint64_t total_rx_dropped = 0; 2019 uint64_t total_tx_dropped = 0; 2020 uint64_t total_rx_nombuf = 0; 2021 struct rte_eth_stats stats; 2022 uint64_t fwd_cycles = 0; 2023 uint64_t total_recv = 0; 2024 uint64_t total_xmit = 0; 2025 struct rte_port *port; 2026 streamid_t sm_id; 2027 portid_t pt_id; 2028 int ret; 2029 int i; 2030 2031 memset(ports_stats, 0, sizeof(ports_stats)); 2032 2033 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2034 struct fwd_stream *fs = fwd_streams[sm_id]; 2035 2036 if (cur_fwd_config.nb_fwd_streams > 2037 cur_fwd_config.nb_fwd_ports) { 2038 fwd_stream_stats_display(sm_id); 2039 } else { 2040 ports_stats[fs->tx_port].tx_stream = fs; 2041 ports_stats[fs->rx_port].rx_stream = fs; 2042 } 2043 2044 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2045 2046 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2047 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2048 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2049 fs->rx_bad_outer_l4_csum; 2050 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2051 fs->rx_bad_outer_ip_csum; 2052 2053 if (record_core_cycles) 2054 fwd_cycles += fs->core_cycles; 2055 } 2056 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2057 pt_id = fwd_ports_ids[i]; 2058 port = &ports[pt_id]; 2059 2060 ret = rte_eth_stats_get(pt_id, &stats); 2061 if (ret != 0) { 2062 fprintf(stderr, 2063 "%s: Error: failed to get stats (port %u): %d", 2064 __func__, pt_id, ret); 2065 continue; 2066 } 2067 stats.ipackets -= port->stats.ipackets; 2068 stats.opackets -= port->stats.opackets; 2069 stats.ibytes -= port->stats.ibytes; 2070 stats.obytes -= port->stats.obytes; 2071 stats.imissed -= port->stats.imissed; 2072 stats.oerrors -= port->stats.oerrors; 2073 stats.rx_nombuf -= port->stats.rx_nombuf; 2074 2075 total_recv += stats.ipackets; 2076 total_xmit += stats.opackets; 2077 total_rx_dropped += stats.imissed; 2078 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2079 total_tx_dropped += stats.oerrors; 2080 total_rx_nombuf += stats.rx_nombuf; 2081 2082 printf("\n %s Forward statistics for port %-2d %s\n", 2083 fwd_stats_border, pt_id, fwd_stats_border); 2084 2085 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2086 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2087 stats.ipackets + stats.imissed); 2088 2089 if (cur_fwd_eng == &csum_fwd_engine) { 2090 printf(" Bad-ipcsum: %-14"PRIu64 2091 " Bad-l4csum: %-14"PRIu64 2092 "Bad-outer-l4csum: %-14"PRIu64"\n", 2093 ports_stats[pt_id].rx_bad_ip_csum, 2094 ports_stats[pt_id].rx_bad_l4_csum, 2095 ports_stats[pt_id].rx_bad_outer_l4_csum); 2096 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2097 ports_stats[pt_id].rx_bad_outer_ip_csum); 2098 } 2099 if (stats.ierrors + stats.rx_nombuf > 0) { 2100 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2101 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2102 } 2103 2104 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2105 "TX-total: %-"PRIu64"\n", 2106 stats.opackets, ports_stats[pt_id].tx_dropped, 2107 stats.opackets + ports_stats[pt_id].tx_dropped); 2108 2109 if (record_burst_stats) { 2110 if (ports_stats[pt_id].rx_stream) 2111 pkt_burst_stats_display("RX", 2112 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2113 if (ports_stats[pt_id].tx_stream) 2114 pkt_burst_stats_display("TX", 2115 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2116 } 2117 2118 printf(" %s--------------------------------%s\n", 2119 fwd_stats_border, fwd_stats_border); 2120 } 2121 2122 printf("\n %s Accumulated forward statistics for all ports" 2123 "%s\n", 2124 acc_stats_border, acc_stats_border); 2125 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2126 "%-"PRIu64"\n" 2127 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2128 "%-"PRIu64"\n", 2129 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2130 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2131 if (total_rx_nombuf > 0) 2132 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2133 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2134 "%s\n", 2135 acc_stats_border, acc_stats_border); 2136 if (record_core_cycles) { 2137 #define CYC_PER_MHZ 1E6 2138 if (total_recv > 0 || total_xmit > 0) { 2139 uint64_t total_pkts = 0; 2140 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2141 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2142 total_pkts = total_xmit; 2143 else 2144 total_pkts = total_recv; 2145 2146 printf("\n CPU cycles/packet=%.2F (total cycles=" 2147 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2148 " MHz Clock\n", 2149 (double) fwd_cycles / total_pkts, 2150 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2151 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2152 } 2153 } 2154 } 2155 2156 void 2157 fwd_stats_reset(void) 2158 { 2159 streamid_t sm_id; 2160 portid_t pt_id; 2161 int ret; 2162 int i; 2163 2164 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2165 pt_id = fwd_ports_ids[i]; 2166 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2167 if (ret != 0) 2168 fprintf(stderr, 2169 "%s: Error: failed to clear stats (port %u):%d", 2170 __func__, pt_id, ret); 2171 } 2172 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2173 struct fwd_stream *fs = fwd_streams[sm_id]; 2174 2175 fs->rx_packets = 0; 2176 fs->tx_packets = 0; 2177 fs->fwd_dropped = 0; 2178 fs->rx_bad_ip_csum = 0; 2179 fs->rx_bad_l4_csum = 0; 2180 fs->rx_bad_outer_l4_csum = 0; 2181 fs->rx_bad_outer_ip_csum = 0; 2182 2183 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2184 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2185 fs->core_cycles = 0; 2186 } 2187 } 2188 2189 static void 2190 flush_fwd_rx_queues(void) 2191 { 2192 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2193 portid_t rxp; 2194 portid_t port_id; 2195 queueid_t rxq; 2196 uint16_t nb_rx; 2197 uint16_t i; 2198 uint8_t j; 2199 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2200 uint64_t timer_period; 2201 2202 if (num_procs > 1) { 2203 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2204 return; 2205 } 2206 2207 /* convert to number of cycles */ 2208 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2209 2210 for (j = 0; j < 2; j++) { 2211 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2212 for (rxq = 0; rxq < nb_rxq; rxq++) { 2213 port_id = fwd_ports_ids[rxp]; 2214 2215 /* Polling stopped queues is prohibited. */ 2216 if (ports[port_id].rxq[rxq].state == 2217 RTE_ETH_QUEUE_STATE_STOPPED) 2218 continue; 2219 2220 /** 2221 * testpmd can stuck in the below do while loop 2222 * if rte_eth_rx_burst() always returns nonzero 2223 * packets. So timer is added to exit this loop 2224 * after 1sec timer expiry. 2225 */ 2226 prev_tsc = rte_rdtsc(); 2227 do { 2228 nb_rx = rte_eth_rx_burst(port_id, rxq, 2229 pkts_burst, MAX_PKT_BURST); 2230 for (i = 0; i < nb_rx; i++) 2231 rte_pktmbuf_free(pkts_burst[i]); 2232 2233 cur_tsc = rte_rdtsc(); 2234 diff_tsc = cur_tsc - prev_tsc; 2235 timer_tsc += diff_tsc; 2236 } while ((nb_rx > 0) && 2237 (timer_tsc < timer_period)); 2238 timer_tsc = 0; 2239 } 2240 } 2241 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2242 } 2243 } 2244 2245 static void 2246 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2247 { 2248 struct fwd_stream **fsm; 2249 streamid_t nb_fs; 2250 streamid_t sm_id; 2251 #ifdef RTE_LIB_BITRATESTATS 2252 uint64_t tics_per_1sec; 2253 uint64_t tics_datum; 2254 uint64_t tics_current; 2255 uint16_t i, cnt_ports; 2256 2257 cnt_ports = nb_ports; 2258 tics_datum = rte_rdtsc(); 2259 tics_per_1sec = rte_get_timer_hz(); 2260 #endif 2261 fsm = &fwd_streams[fc->stream_idx]; 2262 nb_fs = fc->stream_nb; 2263 do { 2264 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2265 if (!fsm[sm_id]->disabled) 2266 (*pkt_fwd)(fsm[sm_id]); 2267 #ifdef RTE_LIB_BITRATESTATS 2268 if (bitrate_enabled != 0 && 2269 bitrate_lcore_id == rte_lcore_id()) { 2270 tics_current = rte_rdtsc(); 2271 if (tics_current - tics_datum >= tics_per_1sec) { 2272 /* Periodic bitrate calculation */ 2273 for (i = 0; i < cnt_ports; i++) 2274 rte_stats_bitrate_calc(bitrate_data, 2275 ports_ids[i]); 2276 tics_datum = tics_current; 2277 } 2278 } 2279 #endif 2280 #ifdef RTE_LIB_LATENCYSTATS 2281 if (latencystats_enabled != 0 && 2282 latencystats_lcore_id == rte_lcore_id()) 2283 rte_latencystats_update(); 2284 #endif 2285 2286 } while (! fc->stopped); 2287 } 2288 2289 static int 2290 start_pkt_forward_on_core(void *fwd_arg) 2291 { 2292 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2293 cur_fwd_config.fwd_eng->packet_fwd); 2294 return 0; 2295 } 2296 2297 /* 2298 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2299 * Used to start communication flows in network loopback test configurations. 2300 */ 2301 static int 2302 run_one_txonly_burst_on_core(void *fwd_arg) 2303 { 2304 struct fwd_lcore *fwd_lc; 2305 struct fwd_lcore tmp_lcore; 2306 2307 fwd_lc = (struct fwd_lcore *) fwd_arg; 2308 tmp_lcore = *fwd_lc; 2309 tmp_lcore.stopped = 1; 2310 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2311 return 0; 2312 } 2313 2314 /* 2315 * Launch packet forwarding: 2316 * - Setup per-port forwarding context. 2317 * - launch logical cores with their forwarding configuration. 2318 */ 2319 static void 2320 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2321 { 2322 unsigned int i; 2323 unsigned int lc_id; 2324 int diag; 2325 2326 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2327 lc_id = fwd_lcores_cpuids[i]; 2328 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2329 fwd_lcores[i]->stopped = 0; 2330 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2331 fwd_lcores[i], lc_id); 2332 if (diag != 0) 2333 fprintf(stderr, 2334 "launch lcore %u failed - diag=%d\n", 2335 lc_id, diag); 2336 } 2337 } 2338 } 2339 2340 /* 2341 * Launch packet forwarding configuration. 2342 */ 2343 void 2344 start_packet_forwarding(int with_tx_first) 2345 { 2346 port_fwd_begin_t port_fwd_begin; 2347 port_fwd_end_t port_fwd_end; 2348 stream_init_t stream_init = cur_fwd_eng->stream_init; 2349 unsigned int i; 2350 2351 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2352 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2353 2354 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2355 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2356 2357 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2358 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2359 (!nb_rxq || !nb_txq)) 2360 rte_exit(EXIT_FAILURE, 2361 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2362 cur_fwd_eng->fwd_mode_name); 2363 2364 if (all_ports_started() == 0) { 2365 fprintf(stderr, "Not all ports were started\n"); 2366 return; 2367 } 2368 if (test_done == 0) { 2369 fprintf(stderr, "Packet forwarding already started\n"); 2370 return; 2371 } 2372 2373 fwd_config_setup(); 2374 2375 pkt_fwd_config_display(&cur_fwd_config); 2376 if (!pkt_fwd_shared_rxq_check()) 2377 return; 2378 2379 if (stream_init != NULL) 2380 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2381 stream_init(fwd_streams[i]); 2382 2383 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2384 if (port_fwd_begin != NULL) { 2385 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2386 if (port_fwd_begin(fwd_ports_ids[i])) { 2387 fprintf(stderr, 2388 "Packet forwarding is not ready\n"); 2389 return; 2390 } 2391 } 2392 } 2393 2394 if (with_tx_first) { 2395 port_fwd_begin = tx_only_engine.port_fwd_begin; 2396 if (port_fwd_begin != NULL) { 2397 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2398 if (port_fwd_begin(fwd_ports_ids[i])) { 2399 fprintf(stderr, 2400 "Packet forwarding is not ready\n"); 2401 return; 2402 } 2403 } 2404 } 2405 } 2406 2407 test_done = 0; 2408 2409 if(!no_flush_rx) 2410 flush_fwd_rx_queues(); 2411 2412 rxtx_config_display(); 2413 2414 fwd_stats_reset(); 2415 if (with_tx_first) { 2416 while (with_tx_first--) { 2417 launch_packet_forwarding( 2418 run_one_txonly_burst_on_core); 2419 rte_eal_mp_wait_lcore(); 2420 } 2421 port_fwd_end = tx_only_engine.port_fwd_end; 2422 if (port_fwd_end != NULL) { 2423 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2424 (*port_fwd_end)(fwd_ports_ids[i]); 2425 } 2426 } 2427 launch_packet_forwarding(start_pkt_forward_on_core); 2428 } 2429 2430 void 2431 stop_packet_forwarding(void) 2432 { 2433 port_fwd_end_t port_fwd_end; 2434 lcoreid_t lc_id; 2435 portid_t pt_id; 2436 int i; 2437 2438 if (test_done) { 2439 fprintf(stderr, "Packet forwarding not started\n"); 2440 return; 2441 } 2442 printf("Telling cores to stop..."); 2443 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2444 fwd_lcores[lc_id]->stopped = 1; 2445 printf("\nWaiting for lcores to finish...\n"); 2446 rte_eal_mp_wait_lcore(); 2447 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2448 if (port_fwd_end != NULL) { 2449 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2450 pt_id = fwd_ports_ids[i]; 2451 (*port_fwd_end)(pt_id); 2452 } 2453 } 2454 2455 fwd_stats_display(); 2456 2457 printf("\nDone.\n"); 2458 test_done = 1; 2459 } 2460 2461 void 2462 dev_set_link_up(portid_t pid) 2463 { 2464 if (rte_eth_dev_set_link_up(pid) < 0) 2465 fprintf(stderr, "\nSet link up fail.\n"); 2466 } 2467 2468 void 2469 dev_set_link_down(portid_t pid) 2470 { 2471 if (rte_eth_dev_set_link_down(pid) < 0) 2472 fprintf(stderr, "\nSet link down fail.\n"); 2473 } 2474 2475 static int 2476 all_ports_started(void) 2477 { 2478 portid_t pi; 2479 struct rte_port *port; 2480 2481 RTE_ETH_FOREACH_DEV(pi) { 2482 port = &ports[pi]; 2483 /* Check if there is a port which is not started */ 2484 if ((port->port_status != RTE_PORT_STARTED) && 2485 (port->slave_flag == 0)) 2486 return 0; 2487 } 2488 2489 /* No port is not started */ 2490 return 1; 2491 } 2492 2493 int 2494 port_is_stopped(portid_t port_id) 2495 { 2496 struct rte_port *port = &ports[port_id]; 2497 2498 if ((port->port_status != RTE_PORT_STOPPED) && 2499 (port->slave_flag == 0)) 2500 return 0; 2501 return 1; 2502 } 2503 2504 int 2505 all_ports_stopped(void) 2506 { 2507 portid_t pi; 2508 2509 RTE_ETH_FOREACH_DEV(pi) { 2510 if (!port_is_stopped(pi)) 2511 return 0; 2512 } 2513 2514 return 1; 2515 } 2516 2517 int 2518 port_is_started(portid_t port_id) 2519 { 2520 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2521 return 0; 2522 2523 if (ports[port_id].port_status != RTE_PORT_STARTED) 2524 return 0; 2525 2526 return 1; 2527 } 2528 2529 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2530 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2531 2532 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2533 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2534 2535 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2536 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2537 2538 2539 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2540 static int 2541 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2542 { 2543 queueid_t qi; 2544 struct rte_eth_hairpin_conf hairpin_conf = { 2545 .peer_count = 1, 2546 }; 2547 int i; 2548 int diag; 2549 struct rte_port *port = &ports[pi]; 2550 uint16_t peer_rx_port = pi; 2551 uint16_t peer_tx_port = pi; 2552 uint32_t manual = 1; 2553 uint32_t tx_exp = hairpin_mode & 0x10; 2554 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2555 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2556 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2557 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2558 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2559 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2560 2561 if (!(hairpin_mode & 0xf)) { 2562 peer_rx_port = pi; 2563 peer_tx_port = pi; 2564 manual = 0; 2565 } else if (hairpin_mode & 0x1) { 2566 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2567 RTE_ETH_DEV_NO_OWNER); 2568 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2569 peer_tx_port = rte_eth_find_next_owned_by(0, 2570 RTE_ETH_DEV_NO_OWNER); 2571 if (p_pi != RTE_MAX_ETHPORTS) { 2572 peer_rx_port = p_pi; 2573 } else { 2574 uint16_t next_pi; 2575 2576 /* Last port will be the peer RX port of the first. */ 2577 RTE_ETH_FOREACH_DEV(next_pi) 2578 peer_rx_port = next_pi; 2579 } 2580 manual = 1; 2581 } else if (hairpin_mode & 0x2) { 2582 if (cnt_pi & 0x1) { 2583 peer_rx_port = p_pi; 2584 } else { 2585 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2586 RTE_ETH_DEV_NO_OWNER); 2587 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2588 peer_rx_port = pi; 2589 } 2590 peer_tx_port = peer_rx_port; 2591 manual = 1; 2592 } 2593 2594 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2595 hairpin_conf.peers[0].port = peer_rx_port; 2596 hairpin_conf.peers[0].queue = i + nb_rxq; 2597 hairpin_conf.manual_bind = !!manual; 2598 hairpin_conf.tx_explicit = !!tx_exp; 2599 hairpin_conf.force_memory = !!tx_force_memory; 2600 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2601 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2602 diag = rte_eth_tx_hairpin_queue_setup 2603 (pi, qi, nb_txd, &hairpin_conf); 2604 i++; 2605 if (diag == 0) 2606 continue; 2607 2608 /* Fail to setup rx queue, return */ 2609 if (port->port_status == RTE_PORT_HANDLING) 2610 port->port_status = RTE_PORT_STOPPED; 2611 else 2612 fprintf(stderr, 2613 "Port %d can not be set back to stopped\n", pi); 2614 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2615 pi); 2616 /* try to reconfigure queues next time */ 2617 port->need_reconfig_queues = 1; 2618 return -1; 2619 } 2620 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2621 hairpin_conf.peers[0].port = peer_tx_port; 2622 hairpin_conf.peers[0].queue = i + nb_txq; 2623 hairpin_conf.manual_bind = !!manual; 2624 hairpin_conf.tx_explicit = !!tx_exp; 2625 hairpin_conf.force_memory = !!rx_force_memory; 2626 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2627 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2628 diag = rte_eth_rx_hairpin_queue_setup 2629 (pi, qi, nb_rxd, &hairpin_conf); 2630 i++; 2631 if (diag == 0) 2632 continue; 2633 2634 /* Fail to setup rx queue, return */ 2635 if (port->port_status == RTE_PORT_HANDLING) 2636 port->port_status = RTE_PORT_STOPPED; 2637 else 2638 fprintf(stderr, 2639 "Port %d can not be set back to stopped\n", pi); 2640 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2641 pi); 2642 /* try to reconfigure queues next time */ 2643 port->need_reconfig_queues = 1; 2644 return -1; 2645 } 2646 return 0; 2647 } 2648 2649 /* Configure the Rx with optional split. */ 2650 int 2651 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2652 uint16_t nb_rx_desc, unsigned int socket_id, 2653 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2654 { 2655 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2656 unsigned int i, mp_n; 2657 uint32_t prev_hdrs = 0; 2658 int ret; 2659 2660 if (rx_pkt_nb_segs <= 1 || 2661 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2662 rx_conf->rx_seg = NULL; 2663 rx_conf->rx_nseg = 0; 2664 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2665 nb_rx_desc, socket_id, 2666 rx_conf, mp); 2667 goto exit; 2668 } 2669 for (i = 0; i < rx_pkt_nb_segs; i++) { 2670 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2671 struct rte_mempool *mpx; 2672 /* 2673 * Use last valid pool for the segments with number 2674 * exceeding the pool index. 2675 */ 2676 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2677 mpx = mbuf_pool_find(socket_id, mp_n); 2678 /* Handle zero as mbuf data buffer size. */ 2679 rx_seg->offset = i < rx_pkt_nb_offs ? 2680 rx_pkt_seg_offsets[i] : 0; 2681 rx_seg->mp = mpx ? mpx : mp; 2682 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2683 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2684 prev_hdrs |= rx_seg->proto_hdr; 2685 } else { 2686 rx_seg->length = rx_pkt_seg_lengths[i] ? 2687 rx_pkt_seg_lengths[i] : 2688 mbuf_data_size[mp_n]; 2689 } 2690 } 2691 rx_conf->rx_nseg = rx_pkt_nb_segs; 2692 rx_conf->rx_seg = rx_useg; 2693 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2694 socket_id, rx_conf, NULL); 2695 rx_conf->rx_seg = NULL; 2696 rx_conf->rx_nseg = 0; 2697 exit: 2698 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2699 RTE_ETH_QUEUE_STATE_STOPPED : 2700 RTE_ETH_QUEUE_STATE_STARTED; 2701 return ret; 2702 } 2703 2704 static int 2705 alloc_xstats_display_info(portid_t pi) 2706 { 2707 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2708 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2709 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2710 2711 if (xstats_display_num == 0) 2712 return 0; 2713 2714 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2715 if (*ids_supp == NULL) 2716 goto fail_ids_supp; 2717 2718 *prev_values = calloc(xstats_display_num, 2719 sizeof(**prev_values)); 2720 if (*prev_values == NULL) 2721 goto fail_prev_values; 2722 2723 *curr_values = calloc(xstats_display_num, 2724 sizeof(**curr_values)); 2725 if (*curr_values == NULL) 2726 goto fail_curr_values; 2727 2728 ports[pi].xstats_info.allocated = true; 2729 2730 return 0; 2731 2732 fail_curr_values: 2733 free(*prev_values); 2734 fail_prev_values: 2735 free(*ids_supp); 2736 fail_ids_supp: 2737 return -ENOMEM; 2738 } 2739 2740 static void 2741 free_xstats_display_info(portid_t pi) 2742 { 2743 if (!ports[pi].xstats_info.allocated) 2744 return; 2745 free(ports[pi].xstats_info.ids_supp); 2746 free(ports[pi].xstats_info.prev_values); 2747 free(ports[pi].xstats_info.curr_values); 2748 ports[pi].xstats_info.allocated = false; 2749 } 2750 2751 /** Fill helper structures for specified port to show extended statistics. */ 2752 static void 2753 fill_xstats_display_info_for_port(portid_t pi) 2754 { 2755 unsigned int stat, stat_supp; 2756 const char *xstat_name; 2757 struct rte_port *port; 2758 uint64_t *ids_supp; 2759 int rc; 2760 2761 if (xstats_display_num == 0) 2762 return; 2763 2764 if (pi == (portid_t)RTE_PORT_ALL) { 2765 fill_xstats_display_info(); 2766 return; 2767 } 2768 2769 port = &ports[pi]; 2770 if (port->port_status != RTE_PORT_STARTED) 2771 return; 2772 2773 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2774 rte_exit(EXIT_FAILURE, 2775 "Failed to allocate xstats display memory\n"); 2776 2777 ids_supp = port->xstats_info.ids_supp; 2778 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2779 xstat_name = xstats_display[stat].name; 2780 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2781 ids_supp + stat_supp); 2782 if (rc != 0) { 2783 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2784 xstat_name, pi, stat); 2785 continue; 2786 } 2787 stat_supp++; 2788 } 2789 2790 port->xstats_info.ids_supp_sz = stat_supp; 2791 } 2792 2793 /** Fill helper structures for all ports to show extended statistics. */ 2794 static void 2795 fill_xstats_display_info(void) 2796 { 2797 portid_t pi; 2798 2799 if (xstats_display_num == 0) 2800 return; 2801 2802 RTE_ETH_FOREACH_DEV(pi) 2803 fill_xstats_display_info_for_port(pi); 2804 } 2805 2806 /* 2807 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2808 * device in dev_info is zero when no slave is added. And its capability 2809 * will be updated when add a new slave device. So adding a slave device need 2810 * to update the port configurations of bonding device. 2811 */ 2812 static void 2813 update_bonding_port_dev_conf(portid_t bond_pid) 2814 { 2815 #ifdef RTE_NET_BOND 2816 struct rte_port *port = &ports[bond_pid]; 2817 uint16_t i; 2818 int ret; 2819 2820 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2821 if (ret != 0) { 2822 fprintf(stderr, "Failed to get dev info for port = %u\n", 2823 bond_pid); 2824 return; 2825 } 2826 2827 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2828 port->dev_conf.txmode.offloads |= 2829 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2830 /* Apply Tx offloads configuration */ 2831 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2832 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2833 2834 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 2835 port->dev_info.flow_type_rss_offloads; 2836 #else 2837 RTE_SET_USED(bond_pid); 2838 #endif 2839 } 2840 2841 int 2842 start_port(portid_t pid) 2843 { 2844 int diag, need_check_link_status = -1; 2845 portid_t pi; 2846 portid_t p_pi = RTE_MAX_ETHPORTS; 2847 portid_t pl[RTE_MAX_ETHPORTS]; 2848 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2849 uint16_t cnt_pi = 0; 2850 uint16_t cfg_pi = 0; 2851 int peer_pi; 2852 queueid_t qi; 2853 struct rte_port *port; 2854 struct rte_eth_hairpin_cap cap; 2855 2856 if (port_id_is_invalid(pid, ENABLED_WARN)) 2857 return 0; 2858 2859 RTE_ETH_FOREACH_DEV(pi) { 2860 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2861 continue; 2862 2863 if (port_is_bonding_slave(pi)) { 2864 fprintf(stderr, 2865 "Please remove port %d from bonded device.\n", 2866 pi); 2867 continue; 2868 } 2869 2870 need_check_link_status = 0; 2871 port = &ports[pi]; 2872 if (port->port_status == RTE_PORT_STOPPED) 2873 port->port_status = RTE_PORT_HANDLING; 2874 else { 2875 fprintf(stderr, "Port %d is now not stopped\n", pi); 2876 continue; 2877 } 2878 2879 if (port->need_reconfig > 0) { 2880 struct rte_eth_conf dev_conf; 2881 int k; 2882 2883 port->need_reconfig = 0; 2884 2885 if (flow_isolate_all) { 2886 int ret = port_flow_isolate(pi, 1); 2887 if (ret) { 2888 fprintf(stderr, 2889 "Failed to apply isolated mode on port %d\n", 2890 pi); 2891 return -1; 2892 } 2893 } 2894 configure_rxtx_dump_callbacks(0); 2895 printf("Configuring Port %d (socket %u)\n", pi, 2896 port->socket_id); 2897 if (nb_hairpinq > 0 && 2898 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2899 fprintf(stderr, 2900 "Port %d doesn't support hairpin queues\n", 2901 pi); 2902 return -1; 2903 } 2904 2905 if (port->bond_flag == 1 && port->update_conf == 1) { 2906 update_bonding_port_dev_conf(pi); 2907 port->update_conf = 0; 2908 } 2909 2910 /* configure port */ 2911 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2912 nb_txq + nb_hairpinq, 2913 &(port->dev_conf)); 2914 if (diag != 0) { 2915 if (port->port_status == RTE_PORT_HANDLING) 2916 port->port_status = RTE_PORT_STOPPED; 2917 else 2918 fprintf(stderr, 2919 "Port %d can not be set back to stopped\n", 2920 pi); 2921 fprintf(stderr, "Fail to configure port %d\n", 2922 pi); 2923 /* try to reconfigure port next time */ 2924 port->need_reconfig = 1; 2925 return -1; 2926 } 2927 /* get device configuration*/ 2928 if (0 != 2929 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2930 fprintf(stderr, 2931 "port %d can not get device configuration\n", 2932 pi); 2933 return -1; 2934 } 2935 /* Apply Rx offloads configuration */ 2936 if (dev_conf.rxmode.offloads != 2937 port->dev_conf.rxmode.offloads) { 2938 port->dev_conf.rxmode.offloads |= 2939 dev_conf.rxmode.offloads; 2940 for (k = 0; 2941 k < port->dev_info.max_rx_queues; 2942 k++) 2943 port->rxq[k].conf.offloads |= 2944 dev_conf.rxmode.offloads; 2945 } 2946 /* Apply Tx offloads configuration */ 2947 if (dev_conf.txmode.offloads != 2948 port->dev_conf.txmode.offloads) { 2949 port->dev_conf.txmode.offloads |= 2950 dev_conf.txmode.offloads; 2951 for (k = 0; 2952 k < port->dev_info.max_tx_queues; 2953 k++) 2954 port->txq[k].conf.offloads |= 2955 dev_conf.txmode.offloads; 2956 } 2957 } 2958 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2959 port->need_reconfig_queues = 0; 2960 /* setup tx queues */ 2961 for (qi = 0; qi < nb_txq; qi++) { 2962 struct rte_eth_txconf *conf = 2963 &port->txq[qi].conf; 2964 2965 if ((numa_support) && 2966 (txring_numa[pi] != NUMA_NO_CONFIG)) 2967 diag = rte_eth_tx_queue_setup(pi, qi, 2968 port->nb_tx_desc[qi], 2969 txring_numa[pi], 2970 &(port->txq[qi].conf)); 2971 else 2972 diag = rte_eth_tx_queue_setup(pi, qi, 2973 port->nb_tx_desc[qi], 2974 port->socket_id, 2975 &(port->txq[qi].conf)); 2976 2977 if (diag == 0) { 2978 port->txq[qi].state = 2979 conf->tx_deferred_start ? 2980 RTE_ETH_QUEUE_STATE_STOPPED : 2981 RTE_ETH_QUEUE_STATE_STARTED; 2982 continue; 2983 } 2984 2985 /* Fail to setup tx queue, return */ 2986 if (port->port_status == RTE_PORT_HANDLING) 2987 port->port_status = RTE_PORT_STOPPED; 2988 else 2989 fprintf(stderr, 2990 "Port %d can not be set back to stopped\n", 2991 pi); 2992 fprintf(stderr, 2993 "Fail to configure port %d tx queues\n", 2994 pi); 2995 /* try to reconfigure queues next time */ 2996 port->need_reconfig_queues = 1; 2997 return -1; 2998 } 2999 for (qi = 0; qi < nb_rxq; qi++) { 3000 /* setup rx queues */ 3001 if ((numa_support) && 3002 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3003 struct rte_mempool * mp = 3004 mbuf_pool_find 3005 (rxring_numa[pi], 0); 3006 if (mp == NULL) { 3007 fprintf(stderr, 3008 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3009 rxring_numa[pi]); 3010 return -1; 3011 } 3012 3013 diag = rx_queue_setup(pi, qi, 3014 port->nb_rx_desc[qi], 3015 rxring_numa[pi], 3016 &(port->rxq[qi].conf), 3017 mp); 3018 } else { 3019 struct rte_mempool *mp = 3020 mbuf_pool_find 3021 (port->socket_id, 0); 3022 if (mp == NULL) { 3023 fprintf(stderr, 3024 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3025 port->socket_id); 3026 return -1; 3027 } 3028 diag = rx_queue_setup(pi, qi, 3029 port->nb_rx_desc[qi], 3030 port->socket_id, 3031 &(port->rxq[qi].conf), 3032 mp); 3033 } 3034 if (diag == 0) 3035 continue; 3036 3037 /* Fail to setup rx queue, return */ 3038 if (port->port_status == RTE_PORT_HANDLING) 3039 port->port_status = RTE_PORT_STOPPED; 3040 else 3041 fprintf(stderr, 3042 "Port %d can not be set back to stopped\n", 3043 pi); 3044 fprintf(stderr, 3045 "Fail to configure port %d rx queues\n", 3046 pi); 3047 /* try to reconfigure queues next time */ 3048 port->need_reconfig_queues = 1; 3049 return -1; 3050 } 3051 /* setup hairpin queues */ 3052 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3053 return -1; 3054 } 3055 configure_rxtx_dump_callbacks(verbose_level); 3056 if (clear_ptypes) { 3057 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3058 NULL, 0); 3059 if (diag < 0) 3060 fprintf(stderr, 3061 "Port %d: Failed to disable Ptype parsing\n", 3062 pi); 3063 } 3064 3065 p_pi = pi; 3066 cnt_pi++; 3067 3068 /* start port */ 3069 diag = eth_dev_start_mp(pi); 3070 if (diag < 0) { 3071 fprintf(stderr, "Fail to start port %d: %s\n", 3072 pi, rte_strerror(-diag)); 3073 3074 /* Fail to setup rx queue, return */ 3075 if (port->port_status == RTE_PORT_HANDLING) 3076 port->port_status = RTE_PORT_STOPPED; 3077 else 3078 fprintf(stderr, 3079 "Port %d can not be set back to stopped\n", 3080 pi); 3081 continue; 3082 } 3083 3084 if (port->port_status == RTE_PORT_HANDLING) 3085 port->port_status = RTE_PORT_STARTED; 3086 else 3087 fprintf(stderr, "Port %d can not be set into started\n", 3088 pi); 3089 3090 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3091 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3092 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3093 3094 /* at least one port started, need checking link status */ 3095 need_check_link_status = 1; 3096 3097 pl[cfg_pi++] = pi; 3098 } 3099 3100 if (need_check_link_status == 1 && !no_link_check) 3101 check_all_ports_link_status(RTE_PORT_ALL); 3102 else if (need_check_link_status == 0) 3103 fprintf(stderr, "Please stop the ports first\n"); 3104 3105 if (hairpin_mode & 0xf) { 3106 uint16_t i; 3107 int j; 3108 3109 /* bind all started hairpin ports */ 3110 for (i = 0; i < cfg_pi; i++) { 3111 pi = pl[i]; 3112 /* bind current Tx to all peer Rx */ 3113 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3114 RTE_MAX_ETHPORTS, 1); 3115 if (peer_pi < 0) 3116 return peer_pi; 3117 for (j = 0; j < peer_pi; j++) { 3118 if (!port_is_started(peer_pl[j])) 3119 continue; 3120 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3121 if (diag < 0) { 3122 fprintf(stderr, 3123 "Error during binding hairpin Tx port %u to %u: %s\n", 3124 pi, peer_pl[j], 3125 rte_strerror(-diag)); 3126 return -1; 3127 } 3128 } 3129 /* bind all peer Tx to current Rx */ 3130 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3131 RTE_MAX_ETHPORTS, 0); 3132 if (peer_pi < 0) 3133 return peer_pi; 3134 for (j = 0; j < peer_pi; j++) { 3135 if (!port_is_started(peer_pl[j])) 3136 continue; 3137 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3138 if (diag < 0) { 3139 fprintf(stderr, 3140 "Error during binding hairpin Tx port %u to %u: %s\n", 3141 peer_pl[j], pi, 3142 rte_strerror(-diag)); 3143 return -1; 3144 } 3145 } 3146 } 3147 } 3148 3149 fill_xstats_display_info_for_port(pid); 3150 3151 printf("Done\n"); 3152 return 0; 3153 } 3154 3155 void 3156 stop_port(portid_t pid) 3157 { 3158 portid_t pi; 3159 struct rte_port *port; 3160 int need_check_link_status = 0; 3161 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3162 int peer_pi; 3163 3164 if (port_id_is_invalid(pid, ENABLED_WARN)) 3165 return; 3166 3167 printf("Stopping ports...\n"); 3168 3169 RTE_ETH_FOREACH_DEV(pi) { 3170 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3171 continue; 3172 3173 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3174 fprintf(stderr, 3175 "Please remove port %d from forwarding configuration.\n", 3176 pi); 3177 continue; 3178 } 3179 3180 if (port_is_bonding_slave(pi)) { 3181 fprintf(stderr, 3182 "Please remove port %d from bonded device.\n", 3183 pi); 3184 continue; 3185 } 3186 3187 port = &ports[pi]; 3188 if (port->port_status == RTE_PORT_STARTED) 3189 port->port_status = RTE_PORT_HANDLING; 3190 else 3191 continue; 3192 3193 if (hairpin_mode & 0xf) { 3194 int j; 3195 3196 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3197 /* unbind all peer Tx from current Rx */ 3198 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3199 RTE_MAX_ETHPORTS, 0); 3200 if (peer_pi < 0) 3201 continue; 3202 for (j = 0; j < peer_pi; j++) { 3203 if (!port_is_started(peer_pl[j])) 3204 continue; 3205 rte_eth_hairpin_unbind(peer_pl[j], pi); 3206 } 3207 } 3208 3209 if (port->flow_list) 3210 port_flow_flush(pi); 3211 3212 if (eth_dev_stop_mp(pi) != 0) 3213 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3214 pi); 3215 3216 if (port->port_status == RTE_PORT_HANDLING) 3217 port->port_status = RTE_PORT_STOPPED; 3218 else 3219 fprintf(stderr, "Port %d can not be set into stopped\n", 3220 pi); 3221 need_check_link_status = 1; 3222 } 3223 if (need_check_link_status && !no_link_check) 3224 check_all_ports_link_status(RTE_PORT_ALL); 3225 3226 printf("Done\n"); 3227 } 3228 3229 static void 3230 remove_invalid_ports_in(portid_t *array, portid_t *total) 3231 { 3232 portid_t i; 3233 portid_t new_total = 0; 3234 3235 for (i = 0; i < *total; i++) 3236 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3237 array[new_total] = array[i]; 3238 new_total++; 3239 } 3240 *total = new_total; 3241 } 3242 3243 static void 3244 remove_invalid_ports(void) 3245 { 3246 remove_invalid_ports_in(ports_ids, &nb_ports); 3247 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3248 nb_cfg_ports = nb_fwd_ports; 3249 } 3250 3251 static void 3252 flush_port_owned_resources(portid_t pi) 3253 { 3254 mcast_addr_pool_destroy(pi); 3255 port_flow_flush(pi); 3256 port_flex_item_flush(pi); 3257 port_action_handle_flush(pi); 3258 } 3259 3260 static void 3261 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3262 { 3263 struct rte_port *port; 3264 portid_t slave_pid; 3265 uint16_t i; 3266 3267 for (i = 0; i < num_slaves; i++) { 3268 slave_pid = slave_pids[i]; 3269 if (port_is_started(slave_pid) == 1) { 3270 if (rte_eth_dev_stop(slave_pid) != 0) 3271 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3272 slave_pid); 3273 3274 port = &ports[slave_pid]; 3275 port->port_status = RTE_PORT_STOPPED; 3276 } 3277 3278 clear_port_slave_flag(slave_pid); 3279 3280 /* Close slave device when testpmd quit or is killed. */ 3281 if (cl_quit == 1 || f_quit == 1) 3282 rte_eth_dev_close(slave_pid); 3283 } 3284 } 3285 3286 void 3287 close_port(portid_t pid) 3288 { 3289 portid_t pi; 3290 struct rte_port *port; 3291 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3292 int num_slaves = 0; 3293 3294 if (port_id_is_invalid(pid, ENABLED_WARN)) 3295 return; 3296 3297 printf("Closing ports...\n"); 3298 3299 RTE_ETH_FOREACH_DEV(pi) { 3300 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3301 continue; 3302 3303 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3304 fprintf(stderr, 3305 "Please remove port %d from forwarding configuration.\n", 3306 pi); 3307 continue; 3308 } 3309 3310 if (port_is_bonding_slave(pi)) { 3311 fprintf(stderr, 3312 "Please remove port %d from bonded device.\n", 3313 pi); 3314 continue; 3315 } 3316 3317 port = &ports[pi]; 3318 if (port->port_status == RTE_PORT_CLOSED) { 3319 fprintf(stderr, "Port %d is already closed\n", pi); 3320 continue; 3321 } 3322 3323 if (is_proc_primary()) { 3324 flush_port_owned_resources(pi); 3325 #ifdef RTE_NET_BOND 3326 if (port->bond_flag == 1) 3327 num_slaves = rte_eth_bond_slaves_get(pi, 3328 slave_pids, RTE_MAX_ETHPORTS); 3329 #endif 3330 rte_eth_dev_close(pi); 3331 /* 3332 * If this port is bonded device, all slaves under the 3333 * device need to be removed or closed. 3334 */ 3335 if (port->bond_flag == 1 && num_slaves > 0) 3336 clear_bonding_slave_device(slave_pids, 3337 num_slaves); 3338 } 3339 3340 free_xstats_display_info(pi); 3341 } 3342 3343 remove_invalid_ports(); 3344 printf("Done\n"); 3345 } 3346 3347 void 3348 reset_port(portid_t pid) 3349 { 3350 int diag; 3351 portid_t pi; 3352 struct rte_port *port; 3353 3354 if (port_id_is_invalid(pid, ENABLED_WARN)) 3355 return; 3356 3357 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3358 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3359 fprintf(stderr, 3360 "Can not reset port(s), please stop port(s) first.\n"); 3361 return; 3362 } 3363 3364 printf("Resetting ports...\n"); 3365 3366 RTE_ETH_FOREACH_DEV(pi) { 3367 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3368 continue; 3369 3370 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3371 fprintf(stderr, 3372 "Please remove port %d from forwarding configuration.\n", 3373 pi); 3374 continue; 3375 } 3376 3377 if (port_is_bonding_slave(pi)) { 3378 fprintf(stderr, 3379 "Please remove port %d from bonded device.\n", 3380 pi); 3381 continue; 3382 } 3383 3384 if (is_proc_primary()) { 3385 diag = rte_eth_dev_reset(pi); 3386 if (diag == 0) { 3387 port = &ports[pi]; 3388 port->need_reconfig = 1; 3389 port->need_reconfig_queues = 1; 3390 } else { 3391 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3392 pi, diag); 3393 } 3394 } 3395 } 3396 3397 printf("Done\n"); 3398 } 3399 3400 void 3401 attach_port(char *identifier) 3402 { 3403 portid_t pi; 3404 struct rte_dev_iterator iterator; 3405 3406 printf("Attaching a new port...\n"); 3407 3408 if (identifier == NULL) { 3409 fprintf(stderr, "Invalid parameters are specified\n"); 3410 return; 3411 } 3412 3413 if (rte_dev_probe(identifier) < 0) { 3414 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3415 return; 3416 } 3417 3418 /* first attach mode: event */ 3419 if (setup_on_probe_event) { 3420 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3421 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3422 if (ports[pi].port_status == RTE_PORT_HANDLING && 3423 ports[pi].need_setup != 0) 3424 setup_attached_port(pi); 3425 return; 3426 } 3427 3428 /* second attach mode: iterator */ 3429 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3430 /* setup ports matching the devargs used for probing */ 3431 if (port_is_forwarding(pi)) 3432 continue; /* port was already attached before */ 3433 setup_attached_port(pi); 3434 } 3435 } 3436 3437 static void 3438 setup_attached_port(portid_t pi) 3439 { 3440 unsigned int socket_id; 3441 int ret; 3442 3443 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3444 /* if socket_id is invalid, set to the first available socket. */ 3445 if (check_socket_id(socket_id) < 0) 3446 socket_id = socket_ids[0]; 3447 reconfig(pi, socket_id); 3448 ret = rte_eth_promiscuous_enable(pi); 3449 if (ret != 0) 3450 fprintf(stderr, 3451 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3452 pi, rte_strerror(-ret)); 3453 3454 ports_ids[nb_ports++] = pi; 3455 fwd_ports_ids[nb_fwd_ports++] = pi; 3456 nb_cfg_ports = nb_fwd_ports; 3457 ports[pi].need_setup = 0; 3458 ports[pi].port_status = RTE_PORT_STOPPED; 3459 3460 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3461 printf("Done\n"); 3462 } 3463 3464 static void 3465 detach_device(struct rte_device *dev) 3466 { 3467 portid_t sibling; 3468 3469 if (dev == NULL) { 3470 fprintf(stderr, "Device already removed\n"); 3471 return; 3472 } 3473 3474 printf("Removing a device...\n"); 3475 3476 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3477 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3478 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3479 fprintf(stderr, "Port %u not stopped\n", 3480 sibling); 3481 return; 3482 } 3483 flush_port_owned_resources(sibling); 3484 } 3485 } 3486 3487 if (rte_dev_remove(dev) < 0) { 3488 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3489 return; 3490 } 3491 remove_invalid_ports(); 3492 3493 printf("Device is detached\n"); 3494 printf("Now total ports is %d\n", nb_ports); 3495 printf("Done\n"); 3496 return; 3497 } 3498 3499 void 3500 detach_port_device(portid_t port_id) 3501 { 3502 int ret; 3503 struct rte_eth_dev_info dev_info; 3504 3505 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3506 return; 3507 3508 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3509 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3510 fprintf(stderr, "Port not stopped\n"); 3511 return; 3512 } 3513 fprintf(stderr, "Port was not closed\n"); 3514 } 3515 3516 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3517 if (ret != 0) { 3518 TESTPMD_LOG(ERR, 3519 "Failed to get device info for port %d, not detaching\n", 3520 port_id); 3521 return; 3522 } 3523 detach_device(dev_info.device); 3524 } 3525 3526 void 3527 detach_devargs(char *identifier) 3528 { 3529 struct rte_dev_iterator iterator; 3530 struct rte_devargs da; 3531 portid_t port_id; 3532 3533 printf("Removing a device...\n"); 3534 3535 memset(&da, 0, sizeof(da)); 3536 if (rte_devargs_parsef(&da, "%s", identifier)) { 3537 fprintf(stderr, "cannot parse identifier\n"); 3538 return; 3539 } 3540 3541 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3542 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3543 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3544 fprintf(stderr, "Port %u not stopped\n", 3545 port_id); 3546 rte_eth_iterator_cleanup(&iterator); 3547 rte_devargs_reset(&da); 3548 return; 3549 } 3550 flush_port_owned_resources(port_id); 3551 } 3552 } 3553 3554 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3555 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3556 da.name, rte_bus_name(da.bus)); 3557 rte_devargs_reset(&da); 3558 return; 3559 } 3560 3561 remove_invalid_ports(); 3562 3563 printf("Device %s is detached\n", identifier); 3564 printf("Now total ports is %d\n", nb_ports); 3565 printf("Done\n"); 3566 rte_devargs_reset(&da); 3567 } 3568 3569 void 3570 pmd_test_exit(void) 3571 { 3572 portid_t pt_id; 3573 unsigned int i; 3574 int ret; 3575 3576 if (test_done == 0) 3577 stop_packet_forwarding(); 3578 3579 #ifndef RTE_EXEC_ENV_WINDOWS 3580 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3581 if (mempools[i]) { 3582 if (mp_alloc_type == MP_ALLOC_ANON) 3583 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3584 NULL); 3585 } 3586 } 3587 #endif 3588 if (ports != NULL) { 3589 no_link_check = 1; 3590 RTE_ETH_FOREACH_DEV(pt_id) { 3591 printf("\nStopping port %d...\n", pt_id); 3592 fflush(stdout); 3593 stop_port(pt_id); 3594 } 3595 RTE_ETH_FOREACH_DEV(pt_id) { 3596 printf("\nShutting down port %d...\n", pt_id); 3597 fflush(stdout); 3598 close_port(pt_id); 3599 } 3600 } 3601 3602 if (hot_plug) { 3603 ret = rte_dev_event_monitor_stop(); 3604 if (ret) { 3605 RTE_LOG(ERR, EAL, 3606 "fail to stop device event monitor."); 3607 return; 3608 } 3609 3610 ret = rte_dev_event_callback_unregister(NULL, 3611 dev_event_callback, NULL); 3612 if (ret < 0) { 3613 RTE_LOG(ERR, EAL, 3614 "fail to unregister device event callback.\n"); 3615 return; 3616 } 3617 3618 ret = rte_dev_hotplug_handle_disable(); 3619 if (ret) { 3620 RTE_LOG(ERR, EAL, 3621 "fail to disable hotplug handling.\n"); 3622 return; 3623 } 3624 } 3625 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3626 if (mempools[i]) 3627 mempool_free_mp(mempools[i]); 3628 } 3629 free(xstats_display); 3630 3631 printf("\nBye...\n"); 3632 } 3633 3634 typedef void (*cmd_func_t)(void); 3635 struct pmd_test_command { 3636 const char *cmd_name; 3637 cmd_func_t cmd_func; 3638 }; 3639 3640 /* Check the link status of all ports in up to 9s, and print them finally */ 3641 static void 3642 check_all_ports_link_status(uint32_t port_mask) 3643 { 3644 #define CHECK_INTERVAL 100 /* 100ms */ 3645 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3646 portid_t portid; 3647 uint8_t count, all_ports_up, print_flag = 0; 3648 struct rte_eth_link link; 3649 int ret; 3650 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3651 3652 printf("Checking link statuses...\n"); 3653 fflush(stdout); 3654 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3655 all_ports_up = 1; 3656 RTE_ETH_FOREACH_DEV(portid) { 3657 if ((port_mask & (1 << portid)) == 0) 3658 continue; 3659 memset(&link, 0, sizeof(link)); 3660 ret = rte_eth_link_get_nowait(portid, &link); 3661 if (ret < 0) { 3662 all_ports_up = 0; 3663 if (print_flag == 1) 3664 fprintf(stderr, 3665 "Port %u link get failed: %s\n", 3666 portid, rte_strerror(-ret)); 3667 continue; 3668 } 3669 /* print link status if flag set */ 3670 if (print_flag == 1) { 3671 rte_eth_link_to_str(link_status, 3672 sizeof(link_status), &link); 3673 printf("Port %d %s\n", portid, link_status); 3674 continue; 3675 } 3676 /* clear all_ports_up flag if any link down */ 3677 if (link.link_status == RTE_ETH_LINK_DOWN) { 3678 all_ports_up = 0; 3679 break; 3680 } 3681 } 3682 /* after finally printing all link status, get out */ 3683 if (print_flag == 1) 3684 break; 3685 3686 if (all_ports_up == 0) { 3687 fflush(stdout); 3688 rte_delay_ms(CHECK_INTERVAL); 3689 } 3690 3691 /* set the print_flag if all ports up or timeout */ 3692 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3693 print_flag = 1; 3694 } 3695 3696 if (lsc_interrupt) 3697 break; 3698 } 3699 } 3700 3701 static void 3702 rmv_port_callback(void *arg) 3703 { 3704 int need_to_start = 0; 3705 int org_no_link_check = no_link_check; 3706 portid_t port_id = (intptr_t)arg; 3707 struct rte_eth_dev_info dev_info; 3708 int ret; 3709 3710 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3711 3712 if (!test_done && port_is_forwarding(port_id)) { 3713 need_to_start = 1; 3714 stop_packet_forwarding(); 3715 } 3716 no_link_check = 1; 3717 stop_port(port_id); 3718 no_link_check = org_no_link_check; 3719 3720 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3721 if (ret != 0) 3722 TESTPMD_LOG(ERR, 3723 "Failed to get device info for port %d, not detaching\n", 3724 port_id); 3725 else { 3726 struct rte_device *device = dev_info.device; 3727 close_port(port_id); 3728 detach_device(device); /* might be already removed or have more ports */ 3729 } 3730 if (need_to_start) 3731 start_packet_forwarding(0); 3732 } 3733 3734 /* This function is used by the interrupt thread */ 3735 static int 3736 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3737 void *ret_param) 3738 { 3739 RTE_SET_USED(param); 3740 RTE_SET_USED(ret_param); 3741 3742 if (type >= RTE_ETH_EVENT_MAX) { 3743 fprintf(stderr, 3744 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3745 port_id, __func__, type); 3746 fflush(stderr); 3747 } else if (event_print_mask & (UINT32_C(1) << type)) { 3748 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3749 eth_event_desc[type]); 3750 fflush(stdout); 3751 } 3752 3753 switch (type) { 3754 case RTE_ETH_EVENT_NEW: 3755 ports[port_id].need_setup = 1; 3756 ports[port_id].port_status = RTE_PORT_HANDLING; 3757 break; 3758 case RTE_ETH_EVENT_INTR_RMV: 3759 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3760 break; 3761 if (rte_eal_alarm_set(100000, 3762 rmv_port_callback, (void *)(intptr_t)port_id)) 3763 fprintf(stderr, 3764 "Could not set up deferred device removal\n"); 3765 break; 3766 case RTE_ETH_EVENT_DESTROY: 3767 ports[port_id].port_status = RTE_PORT_CLOSED; 3768 printf("Port %u is closed\n", port_id); 3769 break; 3770 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3771 uint16_t rxq_id; 3772 int ret; 3773 3774 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3775 for (rxq_id = 0; ; rxq_id++) { 3776 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3777 NULL); 3778 if (ret <= 0) 3779 break; 3780 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3781 port_id, rxq_id); 3782 3783 #ifdef RTE_NET_MLX5 3784 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3785 #endif 3786 } 3787 break; 3788 } 3789 default: 3790 break; 3791 } 3792 return 0; 3793 } 3794 3795 static int 3796 register_eth_event_callback(void) 3797 { 3798 int ret; 3799 enum rte_eth_event_type event; 3800 3801 for (event = RTE_ETH_EVENT_UNKNOWN; 3802 event < RTE_ETH_EVENT_MAX; event++) { 3803 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3804 event, 3805 eth_event_callback, 3806 NULL); 3807 if (ret != 0) { 3808 TESTPMD_LOG(ERR, "Failed to register callback for " 3809 "%s event\n", eth_event_desc[event]); 3810 return -1; 3811 } 3812 } 3813 3814 return 0; 3815 } 3816 3817 /* This function is used by the interrupt thread */ 3818 static void 3819 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3820 __rte_unused void *arg) 3821 { 3822 uint16_t port_id; 3823 int ret; 3824 3825 if (type >= RTE_DEV_EVENT_MAX) { 3826 fprintf(stderr, "%s called upon invalid event %d\n", 3827 __func__, type); 3828 fflush(stderr); 3829 } 3830 3831 switch (type) { 3832 case RTE_DEV_EVENT_REMOVE: 3833 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3834 device_name); 3835 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3836 if (ret) { 3837 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3838 device_name); 3839 return; 3840 } 3841 /* 3842 * Because the user's callback is invoked in eal interrupt 3843 * callback, the interrupt callback need to be finished before 3844 * it can be unregistered when detaching device. So finish 3845 * callback soon and use a deferred removal to detach device 3846 * is need. It is a workaround, once the device detaching be 3847 * moved into the eal in the future, the deferred removal could 3848 * be deleted. 3849 */ 3850 if (rte_eal_alarm_set(100000, 3851 rmv_port_callback, (void *)(intptr_t)port_id)) 3852 RTE_LOG(ERR, EAL, 3853 "Could not set up deferred device removal\n"); 3854 break; 3855 case RTE_DEV_EVENT_ADD: 3856 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3857 device_name); 3858 /* TODO: After finish kernel driver binding, 3859 * begin to attach port. 3860 */ 3861 break; 3862 default: 3863 break; 3864 } 3865 } 3866 3867 static void 3868 rxtx_port_config(portid_t pid) 3869 { 3870 uint16_t qid; 3871 uint64_t offloads; 3872 struct rte_port *port = &ports[pid]; 3873 3874 for (qid = 0; qid < nb_rxq; qid++) { 3875 offloads = port->rxq[qid].conf.offloads; 3876 port->rxq[qid].conf = port->dev_info.default_rxconf; 3877 3878 if (rxq_share > 0 && 3879 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3880 /* Non-zero share group to enable RxQ share. */ 3881 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3882 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3883 } 3884 3885 if (offloads != 0) 3886 port->rxq[qid].conf.offloads = offloads; 3887 3888 /* Check if any Rx parameters have been passed */ 3889 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3890 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3891 3892 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3893 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3894 3895 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3896 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3897 3898 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3899 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3900 3901 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3902 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3903 3904 port->nb_rx_desc[qid] = nb_rxd; 3905 } 3906 3907 for (qid = 0; qid < nb_txq; qid++) { 3908 offloads = port->txq[qid].conf.offloads; 3909 port->txq[qid].conf = port->dev_info.default_txconf; 3910 if (offloads != 0) 3911 port->txq[qid].conf.offloads = offloads; 3912 3913 /* Check if any Tx parameters have been passed */ 3914 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3915 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3916 3917 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3918 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3919 3920 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3921 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3922 3923 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3924 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3925 3926 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3927 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3928 3929 port->nb_tx_desc[qid] = nb_txd; 3930 } 3931 } 3932 3933 /* 3934 * Helper function to set MTU from frame size 3935 * 3936 * port->dev_info should be set before calling this function. 3937 * 3938 * return 0 on success, negative on error 3939 */ 3940 int 3941 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3942 { 3943 struct rte_port *port = &ports[portid]; 3944 uint32_t eth_overhead; 3945 uint16_t mtu, new_mtu; 3946 3947 eth_overhead = get_eth_overhead(&port->dev_info); 3948 3949 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3950 printf("Failed to get MTU for port %u\n", portid); 3951 return -1; 3952 } 3953 3954 new_mtu = max_rx_pktlen - eth_overhead; 3955 3956 if (mtu == new_mtu) 3957 return 0; 3958 3959 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3960 fprintf(stderr, 3961 "Failed to set MTU to %u for port %u\n", 3962 new_mtu, portid); 3963 return -1; 3964 } 3965 3966 port->dev_conf.rxmode.mtu = new_mtu; 3967 3968 return 0; 3969 } 3970 3971 void 3972 init_port_config(void) 3973 { 3974 portid_t pid; 3975 struct rte_port *port; 3976 int ret, i; 3977 3978 RTE_ETH_FOREACH_DEV(pid) { 3979 port = &ports[pid]; 3980 3981 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3982 if (ret != 0) 3983 return; 3984 3985 if (nb_rxq > 1) { 3986 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3987 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3988 rss_hf & port->dev_info.flow_type_rss_offloads; 3989 } else { 3990 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3991 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3992 } 3993 3994 if (port->dcb_flag == 0) { 3995 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3996 port->dev_conf.rxmode.mq_mode = 3997 (enum rte_eth_rx_mq_mode) 3998 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3999 } else { 4000 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4001 port->dev_conf.rxmode.offloads &= 4002 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4003 4004 for (i = 0; 4005 i < port->dev_info.nb_rx_queues; 4006 i++) 4007 port->rxq[i].conf.offloads &= 4008 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4009 } 4010 } 4011 4012 rxtx_port_config(pid); 4013 4014 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4015 if (ret != 0) 4016 return; 4017 4018 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4019 port->dev_conf.intr_conf.lsc = 1; 4020 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4021 port->dev_conf.intr_conf.rmv = 1; 4022 } 4023 } 4024 4025 void set_port_slave_flag(portid_t slave_pid) 4026 { 4027 struct rte_port *port; 4028 4029 port = &ports[slave_pid]; 4030 port->slave_flag = 1; 4031 } 4032 4033 void clear_port_slave_flag(portid_t slave_pid) 4034 { 4035 struct rte_port *port; 4036 4037 port = &ports[slave_pid]; 4038 port->slave_flag = 0; 4039 } 4040 4041 uint8_t port_is_bonding_slave(portid_t slave_pid) 4042 { 4043 struct rte_port *port; 4044 struct rte_eth_dev_info dev_info; 4045 int ret; 4046 4047 port = &ports[slave_pid]; 4048 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4049 if (ret != 0) { 4050 TESTPMD_LOG(ERR, 4051 "Failed to get device info for port id %d," 4052 "cannot determine if the port is a bonded slave", 4053 slave_pid); 4054 return 0; 4055 } 4056 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4057 return 1; 4058 return 0; 4059 } 4060 4061 const uint16_t vlan_tags[] = { 4062 0, 1, 2, 3, 4, 5, 6, 7, 4063 8, 9, 10, 11, 12, 13, 14, 15, 4064 16, 17, 18, 19, 20, 21, 22, 23, 4065 24, 25, 26, 27, 28, 29, 30, 31 4066 }; 4067 4068 static int 4069 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4070 enum dcb_mode_enable dcb_mode, 4071 enum rte_eth_nb_tcs num_tcs, 4072 uint8_t pfc_en) 4073 { 4074 uint8_t i; 4075 int32_t rc; 4076 struct rte_eth_rss_conf rss_conf; 4077 4078 /* 4079 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4080 * given above, and the number of traffic classes available for use. 4081 */ 4082 if (dcb_mode == DCB_VT_ENABLED) { 4083 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4084 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4085 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4086 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4087 4088 /* VMDQ+DCB RX and TX configurations */ 4089 vmdq_rx_conf->enable_default_pool = 0; 4090 vmdq_rx_conf->default_pool = 0; 4091 vmdq_rx_conf->nb_queue_pools = 4092 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4093 vmdq_tx_conf->nb_queue_pools = 4094 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4095 4096 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4097 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4098 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4099 vmdq_rx_conf->pool_map[i].pools = 4100 1 << (i % vmdq_rx_conf->nb_queue_pools); 4101 } 4102 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4103 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4104 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4105 } 4106 4107 /* set DCB mode of RX and TX of multiple queues */ 4108 eth_conf->rxmode.mq_mode = 4109 (enum rte_eth_rx_mq_mode) 4110 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4111 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4112 } else { 4113 struct rte_eth_dcb_rx_conf *rx_conf = 4114 ð_conf->rx_adv_conf.dcb_rx_conf; 4115 struct rte_eth_dcb_tx_conf *tx_conf = 4116 ð_conf->tx_adv_conf.dcb_tx_conf; 4117 4118 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4119 4120 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4121 if (rc != 0) 4122 return rc; 4123 4124 rx_conf->nb_tcs = num_tcs; 4125 tx_conf->nb_tcs = num_tcs; 4126 4127 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4128 rx_conf->dcb_tc[i] = i % num_tcs; 4129 tx_conf->dcb_tc[i] = i % num_tcs; 4130 } 4131 4132 eth_conf->rxmode.mq_mode = 4133 (enum rte_eth_rx_mq_mode) 4134 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4135 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4136 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4137 } 4138 4139 if (pfc_en) 4140 eth_conf->dcb_capability_en = 4141 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4142 else 4143 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4144 4145 return 0; 4146 } 4147 4148 int 4149 init_port_dcb_config(portid_t pid, 4150 enum dcb_mode_enable dcb_mode, 4151 enum rte_eth_nb_tcs num_tcs, 4152 uint8_t pfc_en) 4153 { 4154 struct rte_eth_conf port_conf; 4155 struct rte_port *rte_port; 4156 int retval; 4157 uint16_t i; 4158 4159 if (num_procs > 1) { 4160 printf("The multi-process feature doesn't support dcb.\n"); 4161 return -ENOTSUP; 4162 } 4163 rte_port = &ports[pid]; 4164 4165 /* retain the original device configuration. */ 4166 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4167 4168 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4169 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4170 if (retval < 0) 4171 return retval; 4172 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4173 /* remove RSS HASH offload for DCB in vt mode */ 4174 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4175 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4176 for (i = 0; i < nb_rxq; i++) 4177 rte_port->rxq[i].conf.offloads &= 4178 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4179 } 4180 4181 /* re-configure the device . */ 4182 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4183 if (retval < 0) 4184 return retval; 4185 4186 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4187 if (retval != 0) 4188 return retval; 4189 4190 /* If dev_info.vmdq_pool_base is greater than 0, 4191 * the queue id of vmdq pools is started after pf queues. 4192 */ 4193 if (dcb_mode == DCB_VT_ENABLED && 4194 rte_port->dev_info.vmdq_pool_base > 0) { 4195 fprintf(stderr, 4196 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4197 pid); 4198 return -1; 4199 } 4200 4201 /* Assume the ports in testpmd have the same dcb capability 4202 * and has the same number of rxq and txq in dcb mode 4203 */ 4204 if (dcb_mode == DCB_VT_ENABLED) { 4205 if (rte_port->dev_info.max_vfs > 0) { 4206 nb_rxq = rte_port->dev_info.nb_rx_queues; 4207 nb_txq = rte_port->dev_info.nb_tx_queues; 4208 } else { 4209 nb_rxq = rte_port->dev_info.max_rx_queues; 4210 nb_txq = rte_port->dev_info.max_tx_queues; 4211 } 4212 } else { 4213 /*if vt is disabled, use all pf queues */ 4214 if (rte_port->dev_info.vmdq_pool_base == 0) { 4215 nb_rxq = rte_port->dev_info.max_rx_queues; 4216 nb_txq = rte_port->dev_info.max_tx_queues; 4217 } else { 4218 nb_rxq = (queueid_t)num_tcs; 4219 nb_txq = (queueid_t)num_tcs; 4220 4221 } 4222 } 4223 rx_free_thresh = 64; 4224 4225 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4226 4227 rxtx_port_config(pid); 4228 /* VLAN filter */ 4229 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4230 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4231 rx_vft_set(pid, vlan_tags[i], 1); 4232 4233 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4234 if (retval != 0) 4235 return retval; 4236 4237 rte_port->dcb_flag = 1; 4238 4239 /* Enter DCB configuration status */ 4240 dcb_config = 1; 4241 4242 return 0; 4243 } 4244 4245 static void 4246 init_port(void) 4247 { 4248 int i; 4249 4250 /* Configuration of Ethernet ports. */ 4251 ports = rte_zmalloc("testpmd: ports", 4252 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4253 RTE_CACHE_LINE_SIZE); 4254 if (ports == NULL) { 4255 rte_exit(EXIT_FAILURE, 4256 "rte_zmalloc(%d struct rte_port) failed\n", 4257 RTE_MAX_ETHPORTS); 4258 } 4259 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4260 ports[i].fwd_mac_swap = 1; 4261 ports[i].xstats_info.allocated = false; 4262 LIST_INIT(&ports[i].flow_tunnel_list); 4263 } 4264 /* Initialize ports NUMA structures */ 4265 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4266 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4267 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4268 } 4269 4270 static void 4271 force_quit(void) 4272 { 4273 pmd_test_exit(); 4274 prompt_exit(); 4275 } 4276 4277 static void 4278 print_stats(void) 4279 { 4280 uint8_t i; 4281 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4282 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4283 4284 /* Clear screen and move to top left */ 4285 printf("%s%s", clr, top_left); 4286 4287 printf("\nPort statistics ===================================="); 4288 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4289 nic_stats_display(fwd_ports_ids[i]); 4290 4291 fflush(stdout); 4292 } 4293 4294 static void 4295 signal_handler(int signum) 4296 { 4297 if (signum == SIGINT || signum == SIGTERM) { 4298 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4299 signum); 4300 #ifdef RTE_LIB_PDUMP 4301 /* uninitialize packet capture framework */ 4302 rte_pdump_uninit(); 4303 #endif 4304 #ifdef RTE_LIB_LATENCYSTATS 4305 if (latencystats_enabled != 0) 4306 rte_latencystats_uninit(); 4307 #endif 4308 force_quit(); 4309 /* Set flag to indicate the force termination. */ 4310 f_quit = 1; 4311 /* exit with the expected status */ 4312 #ifndef RTE_EXEC_ENV_WINDOWS 4313 signal(signum, SIG_DFL); 4314 kill(getpid(), signum); 4315 #endif 4316 } 4317 } 4318 4319 int 4320 main(int argc, char** argv) 4321 { 4322 int diag; 4323 portid_t port_id; 4324 uint16_t count; 4325 int ret; 4326 4327 signal(SIGINT, signal_handler); 4328 signal(SIGTERM, signal_handler); 4329 4330 testpmd_logtype = rte_log_register("testpmd"); 4331 if (testpmd_logtype < 0) 4332 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4333 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4334 4335 diag = rte_eal_init(argc, argv); 4336 if (diag < 0) 4337 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4338 rte_strerror(rte_errno)); 4339 4340 ret = register_eth_event_callback(); 4341 if (ret != 0) 4342 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4343 4344 #ifdef RTE_LIB_PDUMP 4345 /* initialize packet capture framework */ 4346 rte_pdump_init(); 4347 #endif 4348 4349 count = 0; 4350 RTE_ETH_FOREACH_DEV(port_id) { 4351 ports_ids[count] = port_id; 4352 count++; 4353 } 4354 nb_ports = (portid_t) count; 4355 if (nb_ports == 0) 4356 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4357 4358 /* allocate port structures, and init them */ 4359 init_port(); 4360 4361 set_def_fwd_config(); 4362 if (nb_lcores == 0) 4363 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4364 "Check the core mask argument\n"); 4365 4366 /* Bitrate/latency stats disabled by default */ 4367 #ifdef RTE_LIB_BITRATESTATS 4368 bitrate_enabled = 0; 4369 #endif 4370 #ifdef RTE_LIB_LATENCYSTATS 4371 latencystats_enabled = 0; 4372 #endif 4373 4374 /* on FreeBSD, mlockall() is disabled by default */ 4375 #ifdef RTE_EXEC_ENV_FREEBSD 4376 do_mlockall = 0; 4377 #else 4378 do_mlockall = 1; 4379 #endif 4380 4381 argc -= diag; 4382 argv += diag; 4383 if (argc > 1) 4384 launch_args_parse(argc, argv); 4385 4386 #ifndef RTE_EXEC_ENV_WINDOWS 4387 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4388 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4389 strerror(errno)); 4390 } 4391 #endif 4392 4393 if (tx_first && interactive) 4394 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4395 "interactive mode.\n"); 4396 4397 if (tx_first && lsc_interrupt) { 4398 fprintf(stderr, 4399 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4400 lsc_interrupt = 0; 4401 } 4402 4403 if (!nb_rxq && !nb_txq) 4404 fprintf(stderr, 4405 "Warning: Either rx or tx queues should be non-zero\n"); 4406 4407 if (nb_rxq > 1 && nb_rxq > nb_txq) 4408 fprintf(stderr, 4409 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4410 nb_rxq, nb_txq); 4411 4412 init_config(); 4413 4414 if (hot_plug) { 4415 ret = rte_dev_hotplug_handle_enable(); 4416 if (ret) { 4417 RTE_LOG(ERR, EAL, 4418 "fail to enable hotplug handling."); 4419 return -1; 4420 } 4421 4422 ret = rte_dev_event_monitor_start(); 4423 if (ret) { 4424 RTE_LOG(ERR, EAL, 4425 "fail to start device event monitoring."); 4426 return -1; 4427 } 4428 4429 ret = rte_dev_event_callback_register(NULL, 4430 dev_event_callback, NULL); 4431 if (ret) { 4432 RTE_LOG(ERR, EAL, 4433 "fail to register device event callback\n"); 4434 return -1; 4435 } 4436 } 4437 4438 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4439 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4440 4441 /* set all ports to promiscuous mode by default */ 4442 RTE_ETH_FOREACH_DEV(port_id) { 4443 ret = rte_eth_promiscuous_enable(port_id); 4444 if (ret != 0) 4445 fprintf(stderr, 4446 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4447 port_id, rte_strerror(-ret)); 4448 } 4449 4450 #ifdef RTE_LIB_METRICS 4451 /* Init metrics library */ 4452 rte_metrics_init(rte_socket_id()); 4453 #endif 4454 4455 #ifdef RTE_LIB_LATENCYSTATS 4456 if (latencystats_enabled != 0) { 4457 int ret = rte_latencystats_init(1, NULL); 4458 if (ret) 4459 fprintf(stderr, 4460 "Warning: latencystats init() returned error %d\n", 4461 ret); 4462 fprintf(stderr, "Latencystats running on lcore %d\n", 4463 latencystats_lcore_id); 4464 } 4465 #endif 4466 4467 /* Setup bitrate stats */ 4468 #ifdef RTE_LIB_BITRATESTATS 4469 if (bitrate_enabled != 0) { 4470 bitrate_data = rte_stats_bitrate_create(); 4471 if (bitrate_data == NULL) 4472 rte_exit(EXIT_FAILURE, 4473 "Could not allocate bitrate data.\n"); 4474 rte_stats_bitrate_reg(bitrate_data); 4475 } 4476 #endif 4477 #ifdef RTE_LIB_CMDLINE 4478 if (init_cmdline() != 0) 4479 rte_exit(EXIT_FAILURE, 4480 "Could not initialise cmdline context.\n"); 4481 4482 if (strlen(cmdline_filename) != 0) 4483 cmdline_read_from_file(cmdline_filename); 4484 4485 if (interactive == 1) { 4486 if (auto_start) { 4487 printf("Start automatic packet forwarding\n"); 4488 start_packet_forwarding(0); 4489 } 4490 prompt(); 4491 pmd_test_exit(); 4492 } else 4493 #endif 4494 { 4495 char c; 4496 int rc; 4497 4498 f_quit = 0; 4499 4500 printf("No commandline core given, start packet forwarding\n"); 4501 start_packet_forwarding(tx_first); 4502 if (stats_period != 0) { 4503 uint64_t prev_time = 0, cur_time, diff_time = 0; 4504 uint64_t timer_period; 4505 4506 /* Convert to number of cycles */ 4507 timer_period = stats_period * rte_get_timer_hz(); 4508 4509 while (f_quit == 0) { 4510 cur_time = rte_get_timer_cycles(); 4511 diff_time += cur_time - prev_time; 4512 4513 if (diff_time >= timer_period) { 4514 print_stats(); 4515 /* Reset the timer */ 4516 diff_time = 0; 4517 } 4518 /* Sleep to avoid unnecessary checks */ 4519 prev_time = cur_time; 4520 rte_delay_us_sleep(US_PER_S); 4521 } 4522 } 4523 4524 printf("Press enter to exit\n"); 4525 rc = read(0, &c, 1); 4526 pmd_test_exit(); 4527 if (rc < 0) 4528 return 1; 4529 } 4530 4531 ret = rte_eal_cleanup(); 4532 if (ret != 0) 4533 rte_exit(EXIT_FAILURE, 4534 "EAL cleanup failed: %s\n", strerror(-ret)); 4535 4536 return EXIT_SUCCESS; 4537 } 4538