1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 static volatile uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 /* 253 * Configuration of packet segments used by the "txonly" processing engine. 254 */ 255 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 256 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 257 TXONLY_DEF_PACKET_LEN, 258 }; 259 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 260 261 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 262 /**< Split policy for packets to TX. */ 263 264 uint8_t txonly_multi_flow; 265 /**< Whether multiple flows are generated in TXONLY mode. */ 266 267 uint32_t tx_pkt_times_inter; 268 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 269 270 uint32_t tx_pkt_times_intra; 271 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 272 273 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 274 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 275 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 276 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 277 278 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 279 uint8_t dcb_config = 0; 280 281 /* 282 * Configurable number of RX/TX queues. 283 */ 284 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 285 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 286 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 287 288 /* 289 * Configurable number of RX/TX ring descriptors. 290 * Defaults are supplied by drivers via ethdev. 291 */ 292 #define RX_DESC_DEFAULT 0 293 #define TX_DESC_DEFAULT 0 294 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 295 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 296 297 #define RTE_PMD_PARAM_UNSET -1 298 /* 299 * Configurable values of RX and TX ring threshold registers. 300 */ 301 302 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 303 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 304 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 305 306 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 307 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 308 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 309 310 /* 311 * Configurable value of RX free threshold. 312 */ 313 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 314 315 /* 316 * Configurable value of RX drop enable. 317 */ 318 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 319 320 /* 321 * Configurable value of TX free threshold. 322 */ 323 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 324 325 /* 326 * Configurable value of TX RS bit threshold. 327 */ 328 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 329 330 /* 331 * Configurable value of buffered packets before sending. 332 */ 333 uint16_t noisy_tx_sw_bufsz; 334 335 /* 336 * Configurable value of packet buffer timeout. 337 */ 338 uint16_t noisy_tx_sw_buf_flush_time; 339 340 /* 341 * Configurable value for size of VNF internal memory area 342 * used for simulating noisy neighbour behaviour 343 */ 344 uint64_t noisy_lkup_mem_sz; 345 346 /* 347 * Configurable value of number of random writes done in 348 * VNF simulation memory area. 349 */ 350 uint64_t noisy_lkup_num_writes; 351 352 /* 353 * Configurable value of number of random reads done in 354 * VNF simulation memory area. 355 */ 356 uint64_t noisy_lkup_num_reads; 357 358 /* 359 * Configurable value of number of random reads/writes done in 360 * VNF simulation memory area. 361 */ 362 uint64_t noisy_lkup_num_reads_writes; 363 364 /* 365 * Receive Side Scaling (RSS) configuration. 366 */ 367 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 368 369 /* 370 * Port topology configuration 371 */ 372 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 373 374 /* 375 * Avoids to flush all the RX streams before starts forwarding. 376 */ 377 uint8_t no_flush_rx = 0; /* flush by default */ 378 379 /* 380 * Flow API isolated mode. 381 */ 382 uint8_t flow_isolate_all; 383 384 /* 385 * Avoids to check link status when starting/stopping a port. 386 */ 387 uint8_t no_link_check = 0; /* check by default */ 388 389 /* 390 * Don't automatically start all ports in interactive mode. 391 */ 392 uint8_t no_device_start = 0; 393 394 /* 395 * Enable link status change notification 396 */ 397 uint8_t lsc_interrupt = 1; /* enabled by default */ 398 399 /* 400 * Enable device removal notification. 401 */ 402 uint8_t rmv_interrupt = 1; /* enabled by default */ 403 404 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 405 406 /* After attach, port setup is called on event or by iterator */ 407 bool setup_on_probe_event = true; 408 409 /* Clear ptypes on port initialization. */ 410 uint8_t clear_ptypes = true; 411 412 /* Hairpin ports configuration mode. */ 413 uint32_t hairpin_mode; 414 415 /* Pretty printing of ethdev events */ 416 static const char * const eth_event_desc[] = { 417 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 418 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 419 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 420 [RTE_ETH_EVENT_INTR_RESET] = "reset", 421 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 422 [RTE_ETH_EVENT_IPSEC] = "IPsec", 423 [RTE_ETH_EVENT_MACSEC] = "MACsec", 424 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 425 [RTE_ETH_EVENT_NEW] = "device probed", 426 [RTE_ETH_EVENT_DESTROY] = "device released", 427 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 428 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 429 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 430 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 431 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 432 [RTE_ETH_EVENT_MAX] = NULL, 433 }; 434 435 /* 436 * Display or mask ether events 437 * Default to all events except VF_MBOX 438 */ 439 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 440 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 441 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 442 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 443 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 444 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 445 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 446 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 447 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 448 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 449 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 450 /* 451 * Decide if all memory are locked for performance. 452 */ 453 int do_mlockall = 0; 454 455 #ifdef RTE_LIB_LATENCYSTATS 456 457 /* 458 * Set when latency stats is enabled in the commandline 459 */ 460 uint8_t latencystats_enabled; 461 462 /* 463 * Lcore ID to service latency statistics. 464 */ 465 lcoreid_t latencystats_lcore_id = -1; 466 467 #endif 468 469 /* 470 * Ethernet device configuration. 471 */ 472 struct rte_eth_rxmode rx_mode; 473 474 struct rte_eth_txmode tx_mode = { 475 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 476 }; 477 478 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 479 480 /* 481 * Display zero values by default for xstats 482 */ 483 uint8_t xstats_hide_zero; 484 485 /* 486 * Measure of CPU cycles disabled by default 487 */ 488 uint8_t record_core_cycles; 489 490 /* 491 * Display of RX and TX bursts disabled by default 492 */ 493 uint8_t record_burst_stats; 494 495 /* 496 * Number of ports per shared Rx queue group, 0 disable. 497 */ 498 uint32_t rxq_share; 499 500 unsigned int num_sockets = 0; 501 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 502 503 #ifdef RTE_LIB_BITRATESTATS 504 /* Bitrate statistics */ 505 struct rte_stats_bitrates *bitrate_data; 506 lcoreid_t bitrate_lcore_id; 507 uint8_t bitrate_enabled; 508 #endif 509 510 #ifdef RTE_LIB_GRO 511 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 512 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 513 #endif 514 515 /* 516 * hexadecimal bitmask of RX mq mode can be enabled. 517 */ 518 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 519 520 /* 521 * Used to set forced link speed 522 */ 523 uint32_t eth_link_speed; 524 525 /* 526 * ID of the current process in multi-process, used to 527 * configure the queues to be polled. 528 */ 529 int proc_id; 530 531 /* 532 * Number of processes in multi-process, used to 533 * configure the queues to be polled. 534 */ 535 unsigned int num_procs = 1; 536 537 static void 538 eth_rx_metadata_negotiate_mp(uint16_t port_id) 539 { 540 uint64_t rx_meta_features = 0; 541 int ret; 542 543 if (!is_proc_primary()) 544 return; 545 546 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 547 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 548 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 549 550 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 551 if (ret == 0) { 552 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 553 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 554 port_id); 555 } 556 557 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 558 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 559 port_id); 560 } 561 562 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 563 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 564 port_id); 565 } 566 } else if (ret != -ENOTSUP) { 567 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 568 port_id, rte_strerror(-ret)); 569 } 570 } 571 572 static int 573 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 574 const struct rte_eth_conf *dev_conf) 575 { 576 if (is_proc_primary()) 577 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 578 dev_conf); 579 return 0; 580 } 581 582 static int 583 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 584 { 585 #ifdef RTE_NET_BOND 586 587 portid_t slave_pids[RTE_MAX_ETHPORTS]; 588 struct rte_port *port; 589 int num_slaves; 590 portid_t slave_pid; 591 int i; 592 593 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 594 RTE_MAX_ETHPORTS); 595 if (num_slaves < 0) { 596 fprintf(stderr, "Failed to get slave list for port = %u\n", 597 bond_pid); 598 return num_slaves; 599 } 600 601 for (i = 0; i < num_slaves; i++) { 602 slave_pid = slave_pids[i]; 603 port = &ports[slave_pid]; 604 port->port_status = 605 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 606 } 607 #else 608 RTE_SET_USED(bond_pid); 609 RTE_SET_USED(is_stop); 610 #endif 611 return 0; 612 } 613 614 static int 615 eth_dev_start_mp(uint16_t port_id) 616 { 617 int ret; 618 619 if (is_proc_primary()) { 620 ret = rte_eth_dev_start(port_id); 621 if (ret != 0) 622 return ret; 623 624 struct rte_port *port = &ports[port_id]; 625 626 /* 627 * Starting a bonded port also starts all slaves under the bonded 628 * device. So if this port is bond device, we need to modify the 629 * port status of these slaves. 630 */ 631 if (port->bond_flag == 1) 632 return change_bonding_slave_port_status(port_id, false); 633 } 634 635 return 0; 636 } 637 638 static int 639 eth_dev_stop_mp(uint16_t port_id) 640 { 641 int ret; 642 643 if (is_proc_primary()) { 644 ret = rte_eth_dev_stop(port_id); 645 if (ret != 0) 646 return ret; 647 648 struct rte_port *port = &ports[port_id]; 649 650 /* 651 * Stopping a bonded port also stops all slaves under the bonded 652 * device. So if this port is bond device, we need to modify the 653 * port status of these slaves. 654 */ 655 if (port->bond_flag == 1) 656 return change_bonding_slave_port_status(port_id, true); 657 } 658 659 return 0; 660 } 661 662 static void 663 mempool_free_mp(struct rte_mempool *mp) 664 { 665 if (is_proc_primary()) 666 rte_mempool_free(mp); 667 } 668 669 static int 670 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 671 { 672 if (is_proc_primary()) 673 return rte_eth_dev_set_mtu(port_id, mtu); 674 675 return 0; 676 } 677 678 /* Forward function declarations */ 679 static void setup_attached_port(portid_t pi); 680 static void check_all_ports_link_status(uint32_t port_mask); 681 static int eth_event_callback(portid_t port_id, 682 enum rte_eth_event_type type, 683 void *param, void *ret_param); 684 static void dev_event_callback(const char *device_name, 685 enum rte_dev_event_type type, 686 void *param); 687 static void fill_xstats_display_info(void); 688 689 /* 690 * Check if all the ports are started. 691 * If yes, return positive value. If not, return zero. 692 */ 693 static int all_ports_started(void); 694 695 #ifdef RTE_LIB_GSO 696 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 697 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 698 #endif 699 700 /* Holds the registered mbuf dynamic flags names. */ 701 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 702 703 704 /* 705 * Helper function to check if socket is already discovered. 706 * If yes, return positive value. If not, return zero. 707 */ 708 int 709 new_socket_id(unsigned int socket_id) 710 { 711 unsigned int i; 712 713 for (i = 0; i < num_sockets; i++) { 714 if (socket_ids[i] == socket_id) 715 return 0; 716 } 717 return 1; 718 } 719 720 /* 721 * Setup default configuration. 722 */ 723 static void 724 set_default_fwd_lcores_config(void) 725 { 726 unsigned int i; 727 unsigned int nb_lc; 728 unsigned int sock_num; 729 730 nb_lc = 0; 731 for (i = 0; i < RTE_MAX_LCORE; i++) { 732 if (!rte_lcore_is_enabled(i)) 733 continue; 734 sock_num = rte_lcore_to_socket_id(i); 735 if (new_socket_id(sock_num)) { 736 if (num_sockets >= RTE_MAX_NUMA_NODES) { 737 rte_exit(EXIT_FAILURE, 738 "Total sockets greater than %u\n", 739 RTE_MAX_NUMA_NODES); 740 } 741 socket_ids[num_sockets++] = sock_num; 742 } 743 if (i == rte_get_main_lcore()) 744 continue; 745 fwd_lcores_cpuids[nb_lc++] = i; 746 } 747 nb_lcores = (lcoreid_t) nb_lc; 748 nb_cfg_lcores = nb_lcores; 749 nb_fwd_lcores = 1; 750 } 751 752 static void 753 set_def_peer_eth_addrs(void) 754 { 755 portid_t i; 756 757 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 758 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 759 peer_eth_addrs[i].addr_bytes[5] = i; 760 } 761 } 762 763 static void 764 set_default_fwd_ports_config(void) 765 { 766 portid_t pt_id; 767 int i = 0; 768 769 RTE_ETH_FOREACH_DEV(pt_id) { 770 fwd_ports_ids[i++] = pt_id; 771 772 /* Update sockets info according to the attached device */ 773 int socket_id = rte_eth_dev_socket_id(pt_id); 774 if (socket_id >= 0 && new_socket_id(socket_id)) { 775 if (num_sockets >= RTE_MAX_NUMA_NODES) { 776 rte_exit(EXIT_FAILURE, 777 "Total sockets greater than %u\n", 778 RTE_MAX_NUMA_NODES); 779 } 780 socket_ids[num_sockets++] = socket_id; 781 } 782 } 783 784 nb_cfg_ports = nb_ports; 785 nb_fwd_ports = nb_ports; 786 } 787 788 void 789 set_def_fwd_config(void) 790 { 791 set_default_fwd_lcores_config(); 792 set_def_peer_eth_addrs(); 793 set_default_fwd_ports_config(); 794 } 795 796 #ifndef RTE_EXEC_ENV_WINDOWS 797 /* extremely pessimistic estimation of memory required to create a mempool */ 798 static int 799 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 800 { 801 unsigned int n_pages, mbuf_per_pg, leftover; 802 uint64_t total_mem, mbuf_mem, obj_sz; 803 804 /* there is no good way to predict how much space the mempool will 805 * occupy because it will allocate chunks on the fly, and some of those 806 * will come from default DPDK memory while some will come from our 807 * external memory, so just assume 128MB will be enough for everyone. 808 */ 809 uint64_t hdr_mem = 128 << 20; 810 811 /* account for possible non-contiguousness */ 812 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 813 if (obj_sz > pgsz) { 814 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 815 return -1; 816 } 817 818 mbuf_per_pg = pgsz / obj_sz; 819 leftover = (nb_mbufs % mbuf_per_pg) > 0; 820 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 821 822 mbuf_mem = n_pages * pgsz; 823 824 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 825 826 if (total_mem > SIZE_MAX) { 827 TESTPMD_LOG(ERR, "Memory size too big\n"); 828 return -1; 829 } 830 *out = (size_t)total_mem; 831 832 return 0; 833 } 834 835 static int 836 pagesz_flags(uint64_t page_sz) 837 { 838 /* as per mmap() manpage, all page sizes are log2 of page size 839 * shifted by MAP_HUGE_SHIFT 840 */ 841 int log2 = rte_log2_u64(page_sz); 842 843 return (log2 << HUGE_SHIFT); 844 } 845 846 static void * 847 alloc_mem(size_t memsz, size_t pgsz, bool huge) 848 { 849 void *addr; 850 int flags; 851 852 /* allocate anonymous hugepages */ 853 flags = MAP_ANONYMOUS | MAP_PRIVATE; 854 if (huge) 855 flags |= HUGE_FLAG | pagesz_flags(pgsz); 856 857 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 858 if (addr == MAP_FAILED) 859 return NULL; 860 861 return addr; 862 } 863 864 struct extmem_param { 865 void *addr; 866 size_t len; 867 size_t pgsz; 868 rte_iova_t *iova_table; 869 unsigned int iova_table_len; 870 }; 871 872 static int 873 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 874 bool huge) 875 { 876 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 877 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 878 unsigned int cur_page, n_pages, pgsz_idx; 879 size_t mem_sz, cur_pgsz; 880 rte_iova_t *iovas = NULL; 881 void *addr; 882 int ret; 883 884 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 885 /* skip anything that is too big */ 886 if (pgsizes[pgsz_idx] > SIZE_MAX) 887 continue; 888 889 cur_pgsz = pgsizes[pgsz_idx]; 890 891 /* if we were told not to allocate hugepages, override */ 892 if (!huge) 893 cur_pgsz = sysconf(_SC_PAGESIZE); 894 895 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 896 if (ret < 0) { 897 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 898 return -1; 899 } 900 901 /* allocate our memory */ 902 addr = alloc_mem(mem_sz, cur_pgsz, huge); 903 904 /* if we couldn't allocate memory with a specified page size, 905 * that doesn't mean we can't do it with other page sizes, so 906 * try another one. 907 */ 908 if (addr == NULL) 909 continue; 910 911 /* store IOVA addresses for every page in this memory area */ 912 n_pages = mem_sz / cur_pgsz; 913 914 iovas = malloc(sizeof(*iovas) * n_pages); 915 916 if (iovas == NULL) { 917 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 918 goto fail; 919 } 920 /* lock memory if it's not huge pages */ 921 if (!huge) 922 mlock(addr, mem_sz); 923 924 /* populate IOVA addresses */ 925 for (cur_page = 0; cur_page < n_pages; cur_page++) { 926 rte_iova_t iova; 927 size_t offset; 928 void *cur; 929 930 offset = cur_pgsz * cur_page; 931 cur = RTE_PTR_ADD(addr, offset); 932 933 /* touch the page before getting its IOVA */ 934 *(volatile char *)cur = 0; 935 936 iova = rte_mem_virt2iova(cur); 937 938 iovas[cur_page] = iova; 939 } 940 941 break; 942 } 943 /* if we couldn't allocate anything */ 944 if (iovas == NULL) 945 return -1; 946 947 param->addr = addr; 948 param->len = mem_sz; 949 param->pgsz = cur_pgsz; 950 param->iova_table = iovas; 951 param->iova_table_len = n_pages; 952 953 return 0; 954 fail: 955 free(iovas); 956 if (addr) 957 munmap(addr, mem_sz); 958 959 return -1; 960 } 961 962 static int 963 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 964 { 965 struct extmem_param param; 966 int socket_id, ret; 967 968 memset(¶m, 0, sizeof(param)); 969 970 /* check if our heap exists */ 971 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 972 if (socket_id < 0) { 973 /* create our heap */ 974 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 975 if (ret < 0) { 976 TESTPMD_LOG(ERR, "Cannot create heap\n"); 977 return -1; 978 } 979 } 980 981 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 982 if (ret < 0) { 983 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 984 return -1; 985 } 986 987 /* we now have a valid memory area, so add it to heap */ 988 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 989 param.addr, param.len, param.iova_table, 990 param.iova_table_len, param.pgsz); 991 992 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 993 994 /* not needed any more */ 995 free(param.iova_table); 996 997 if (ret < 0) { 998 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 999 munmap(param.addr, param.len); 1000 return -1; 1001 } 1002 1003 /* success */ 1004 1005 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1006 param.len >> 20); 1007 1008 return 0; 1009 } 1010 static void 1011 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1012 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1013 { 1014 uint16_t pid = 0; 1015 int ret; 1016 1017 RTE_ETH_FOREACH_DEV(pid) { 1018 struct rte_eth_dev_info dev_info; 1019 1020 ret = eth_dev_info_get_print_err(pid, &dev_info); 1021 if (ret != 0) { 1022 TESTPMD_LOG(DEBUG, 1023 "unable to get device info for port %d on addr 0x%p," 1024 "mempool unmapping will not be performed\n", 1025 pid, memhdr->addr); 1026 continue; 1027 } 1028 1029 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1030 if (ret) { 1031 TESTPMD_LOG(DEBUG, 1032 "unable to DMA unmap addr 0x%p " 1033 "for device %s\n", 1034 memhdr->addr, rte_dev_name(dev_info.device)); 1035 } 1036 } 1037 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1038 if (ret) { 1039 TESTPMD_LOG(DEBUG, 1040 "unable to un-register addr 0x%p\n", memhdr->addr); 1041 } 1042 } 1043 1044 static void 1045 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1046 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1047 { 1048 uint16_t pid = 0; 1049 size_t page_size = sysconf(_SC_PAGESIZE); 1050 int ret; 1051 1052 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1053 page_size); 1054 if (ret) { 1055 TESTPMD_LOG(DEBUG, 1056 "unable to register addr 0x%p\n", memhdr->addr); 1057 return; 1058 } 1059 RTE_ETH_FOREACH_DEV(pid) { 1060 struct rte_eth_dev_info dev_info; 1061 1062 ret = eth_dev_info_get_print_err(pid, &dev_info); 1063 if (ret != 0) { 1064 TESTPMD_LOG(DEBUG, 1065 "unable to get device info for port %d on addr 0x%p," 1066 "mempool mapping will not be performed\n", 1067 pid, memhdr->addr); 1068 continue; 1069 } 1070 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1071 if (ret) { 1072 TESTPMD_LOG(DEBUG, 1073 "unable to DMA map addr 0x%p " 1074 "for device %s\n", 1075 memhdr->addr, rte_dev_name(dev_info.device)); 1076 } 1077 } 1078 } 1079 #endif 1080 1081 static unsigned int 1082 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1083 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1084 { 1085 struct rte_pktmbuf_extmem *xmem; 1086 unsigned int ext_num, zone_num, elt_num; 1087 uint16_t elt_size; 1088 1089 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1090 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1091 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1092 1093 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1094 if (xmem == NULL) { 1095 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1096 "external buffer descriptors\n"); 1097 *ext_mem = NULL; 1098 return 0; 1099 } 1100 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1101 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1102 const struct rte_memzone *mz; 1103 char mz_name[RTE_MEMZONE_NAMESIZE]; 1104 int ret; 1105 1106 ret = snprintf(mz_name, sizeof(mz_name), 1107 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1108 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1109 errno = ENAMETOOLONG; 1110 ext_num = 0; 1111 break; 1112 } 1113 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1114 socket_id, 1115 RTE_MEMZONE_IOVA_CONTIG | 1116 RTE_MEMZONE_1GB | 1117 RTE_MEMZONE_SIZE_HINT_ONLY); 1118 if (mz == NULL) { 1119 /* 1120 * The caller exits on external buffer creation 1121 * error, so there is no need to free memzones. 1122 */ 1123 errno = ENOMEM; 1124 ext_num = 0; 1125 break; 1126 } 1127 xseg->buf_ptr = mz->addr; 1128 xseg->buf_iova = mz->iova; 1129 xseg->buf_len = EXTBUF_ZONE_SIZE; 1130 xseg->elt_size = elt_size; 1131 } 1132 if (ext_num == 0 && xmem != NULL) { 1133 free(xmem); 1134 xmem = NULL; 1135 } 1136 *ext_mem = xmem; 1137 return ext_num; 1138 } 1139 1140 /* 1141 * Configuration initialisation done once at init time. 1142 */ 1143 static struct rte_mempool * 1144 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1145 unsigned int socket_id, uint16_t size_idx) 1146 { 1147 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1148 struct rte_mempool *rte_mp = NULL; 1149 #ifndef RTE_EXEC_ENV_WINDOWS 1150 uint32_t mb_size; 1151 1152 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1153 #endif 1154 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1155 if (!is_proc_primary()) { 1156 rte_mp = rte_mempool_lookup(pool_name); 1157 if (rte_mp == NULL) 1158 rte_exit(EXIT_FAILURE, 1159 "Get mbuf pool for socket %u failed: %s\n", 1160 socket_id, rte_strerror(rte_errno)); 1161 return rte_mp; 1162 } 1163 1164 TESTPMD_LOG(INFO, 1165 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1166 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1167 1168 switch (mp_alloc_type) { 1169 case MP_ALLOC_NATIVE: 1170 { 1171 /* wrapper to rte_mempool_create() */ 1172 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1173 rte_mbuf_best_mempool_ops()); 1174 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1175 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1176 break; 1177 } 1178 #ifndef RTE_EXEC_ENV_WINDOWS 1179 case MP_ALLOC_ANON: 1180 { 1181 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1182 mb_size, (unsigned int) mb_mempool_cache, 1183 sizeof(struct rte_pktmbuf_pool_private), 1184 socket_id, mempool_flags); 1185 if (rte_mp == NULL) 1186 goto err; 1187 1188 if (rte_mempool_populate_anon(rte_mp) == 0) { 1189 rte_mempool_free(rte_mp); 1190 rte_mp = NULL; 1191 goto err; 1192 } 1193 rte_pktmbuf_pool_init(rte_mp, NULL); 1194 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1195 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1196 break; 1197 } 1198 case MP_ALLOC_XMEM: 1199 case MP_ALLOC_XMEM_HUGE: 1200 { 1201 int heap_socket; 1202 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1203 1204 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1205 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1206 1207 heap_socket = 1208 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1209 if (heap_socket < 0) 1210 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1211 1212 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1213 rte_mbuf_best_mempool_ops()); 1214 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1215 mb_mempool_cache, 0, mbuf_seg_size, 1216 heap_socket); 1217 break; 1218 } 1219 #endif 1220 case MP_ALLOC_XBUF: 1221 { 1222 struct rte_pktmbuf_extmem *ext_mem; 1223 unsigned int ext_num; 1224 1225 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1226 socket_id, pool_name, &ext_mem); 1227 if (ext_num == 0) 1228 rte_exit(EXIT_FAILURE, 1229 "Can't create pinned data buffers\n"); 1230 1231 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1232 rte_mbuf_best_mempool_ops()); 1233 rte_mp = rte_pktmbuf_pool_create_extbuf 1234 (pool_name, nb_mbuf, mb_mempool_cache, 1235 0, mbuf_seg_size, socket_id, 1236 ext_mem, ext_num); 1237 free(ext_mem); 1238 break; 1239 } 1240 default: 1241 { 1242 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1243 } 1244 } 1245 1246 #ifndef RTE_EXEC_ENV_WINDOWS 1247 err: 1248 #endif 1249 if (rte_mp == NULL) { 1250 rte_exit(EXIT_FAILURE, 1251 "Creation of mbuf pool for socket %u failed: %s\n", 1252 socket_id, rte_strerror(rte_errno)); 1253 } else if (verbose_level > 0) { 1254 rte_mempool_dump(stdout, rte_mp); 1255 } 1256 return rte_mp; 1257 } 1258 1259 /* 1260 * Check given socket id is valid or not with NUMA mode, 1261 * if valid, return 0, else return -1 1262 */ 1263 static int 1264 check_socket_id(const unsigned int socket_id) 1265 { 1266 static int warning_once = 0; 1267 1268 if (new_socket_id(socket_id)) { 1269 if (!warning_once && numa_support) 1270 fprintf(stderr, 1271 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1272 warning_once = 1; 1273 return -1; 1274 } 1275 return 0; 1276 } 1277 1278 /* 1279 * Get the allowed maximum number of RX queues. 1280 * *pid return the port id which has minimal value of 1281 * max_rx_queues in all ports. 1282 */ 1283 queueid_t 1284 get_allowed_max_nb_rxq(portid_t *pid) 1285 { 1286 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1287 bool max_rxq_valid = false; 1288 portid_t pi; 1289 struct rte_eth_dev_info dev_info; 1290 1291 RTE_ETH_FOREACH_DEV(pi) { 1292 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1293 continue; 1294 1295 max_rxq_valid = true; 1296 if (dev_info.max_rx_queues < allowed_max_rxq) { 1297 allowed_max_rxq = dev_info.max_rx_queues; 1298 *pid = pi; 1299 } 1300 } 1301 return max_rxq_valid ? allowed_max_rxq : 0; 1302 } 1303 1304 /* 1305 * Check input rxq is valid or not. 1306 * If input rxq is not greater than any of maximum number 1307 * of RX queues of all ports, it is valid. 1308 * if valid, return 0, else return -1 1309 */ 1310 int 1311 check_nb_rxq(queueid_t rxq) 1312 { 1313 queueid_t allowed_max_rxq; 1314 portid_t pid = 0; 1315 1316 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1317 if (rxq > allowed_max_rxq) { 1318 fprintf(stderr, 1319 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1320 rxq, allowed_max_rxq, pid); 1321 return -1; 1322 } 1323 return 0; 1324 } 1325 1326 /* 1327 * Get the allowed maximum number of TX queues. 1328 * *pid return the port id which has minimal value of 1329 * max_tx_queues in all ports. 1330 */ 1331 queueid_t 1332 get_allowed_max_nb_txq(portid_t *pid) 1333 { 1334 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1335 bool max_txq_valid = false; 1336 portid_t pi; 1337 struct rte_eth_dev_info dev_info; 1338 1339 RTE_ETH_FOREACH_DEV(pi) { 1340 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1341 continue; 1342 1343 max_txq_valid = true; 1344 if (dev_info.max_tx_queues < allowed_max_txq) { 1345 allowed_max_txq = dev_info.max_tx_queues; 1346 *pid = pi; 1347 } 1348 } 1349 return max_txq_valid ? allowed_max_txq : 0; 1350 } 1351 1352 /* 1353 * Check input txq is valid or not. 1354 * If input txq is not greater than any of maximum number 1355 * of TX queues of all ports, it is valid. 1356 * if valid, return 0, else return -1 1357 */ 1358 int 1359 check_nb_txq(queueid_t txq) 1360 { 1361 queueid_t allowed_max_txq; 1362 portid_t pid = 0; 1363 1364 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1365 if (txq > allowed_max_txq) { 1366 fprintf(stderr, 1367 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1368 txq, allowed_max_txq, pid); 1369 return -1; 1370 } 1371 return 0; 1372 } 1373 1374 /* 1375 * Get the allowed maximum number of RXDs of every rx queue. 1376 * *pid return the port id which has minimal value of 1377 * max_rxd in all queues of all ports. 1378 */ 1379 static uint16_t 1380 get_allowed_max_nb_rxd(portid_t *pid) 1381 { 1382 uint16_t allowed_max_rxd = UINT16_MAX; 1383 portid_t pi; 1384 struct rte_eth_dev_info dev_info; 1385 1386 RTE_ETH_FOREACH_DEV(pi) { 1387 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1388 continue; 1389 1390 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1391 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1392 *pid = pi; 1393 } 1394 } 1395 return allowed_max_rxd; 1396 } 1397 1398 /* 1399 * Get the allowed minimal number of RXDs of every rx queue. 1400 * *pid return the port id which has minimal value of 1401 * min_rxd in all queues of all ports. 1402 */ 1403 static uint16_t 1404 get_allowed_min_nb_rxd(portid_t *pid) 1405 { 1406 uint16_t allowed_min_rxd = 0; 1407 portid_t pi; 1408 struct rte_eth_dev_info dev_info; 1409 1410 RTE_ETH_FOREACH_DEV(pi) { 1411 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1412 continue; 1413 1414 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1415 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1416 *pid = pi; 1417 } 1418 } 1419 1420 return allowed_min_rxd; 1421 } 1422 1423 /* 1424 * Check input rxd is valid or not. 1425 * If input rxd is not greater than any of maximum number 1426 * of RXDs of every Rx queues and is not less than any of 1427 * minimal number of RXDs of every Rx queues, it is valid. 1428 * if valid, return 0, else return -1 1429 */ 1430 int 1431 check_nb_rxd(queueid_t rxd) 1432 { 1433 uint16_t allowed_max_rxd; 1434 uint16_t allowed_min_rxd; 1435 portid_t pid = 0; 1436 1437 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1438 if (rxd > allowed_max_rxd) { 1439 fprintf(stderr, 1440 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1441 rxd, allowed_max_rxd, pid); 1442 return -1; 1443 } 1444 1445 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1446 if (rxd < allowed_min_rxd) { 1447 fprintf(stderr, 1448 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1449 rxd, allowed_min_rxd, pid); 1450 return -1; 1451 } 1452 1453 return 0; 1454 } 1455 1456 /* 1457 * Get the allowed maximum number of TXDs of every rx queues. 1458 * *pid return the port id which has minimal value of 1459 * max_txd in every tx queue. 1460 */ 1461 static uint16_t 1462 get_allowed_max_nb_txd(portid_t *pid) 1463 { 1464 uint16_t allowed_max_txd = UINT16_MAX; 1465 portid_t pi; 1466 struct rte_eth_dev_info dev_info; 1467 1468 RTE_ETH_FOREACH_DEV(pi) { 1469 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1470 continue; 1471 1472 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1473 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1474 *pid = pi; 1475 } 1476 } 1477 return allowed_max_txd; 1478 } 1479 1480 /* 1481 * Get the allowed maximum number of TXDs of every tx queues. 1482 * *pid return the port id which has minimal value of 1483 * min_txd in every tx queue. 1484 */ 1485 static uint16_t 1486 get_allowed_min_nb_txd(portid_t *pid) 1487 { 1488 uint16_t allowed_min_txd = 0; 1489 portid_t pi; 1490 struct rte_eth_dev_info dev_info; 1491 1492 RTE_ETH_FOREACH_DEV(pi) { 1493 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1494 continue; 1495 1496 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1497 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1498 *pid = pi; 1499 } 1500 } 1501 1502 return allowed_min_txd; 1503 } 1504 1505 /* 1506 * Check input txd is valid or not. 1507 * If input txd is not greater than any of maximum number 1508 * of TXDs of every Rx queues, it is valid. 1509 * if valid, return 0, else return -1 1510 */ 1511 int 1512 check_nb_txd(queueid_t txd) 1513 { 1514 uint16_t allowed_max_txd; 1515 uint16_t allowed_min_txd; 1516 portid_t pid = 0; 1517 1518 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1519 if (txd > allowed_max_txd) { 1520 fprintf(stderr, 1521 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1522 txd, allowed_max_txd, pid); 1523 return -1; 1524 } 1525 1526 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1527 if (txd < allowed_min_txd) { 1528 fprintf(stderr, 1529 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1530 txd, allowed_min_txd, pid); 1531 return -1; 1532 } 1533 return 0; 1534 } 1535 1536 1537 /* 1538 * Get the allowed maximum number of hairpin queues. 1539 * *pid return the port id which has minimal value of 1540 * max_hairpin_queues in all ports. 1541 */ 1542 queueid_t 1543 get_allowed_max_nb_hairpinq(portid_t *pid) 1544 { 1545 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1546 portid_t pi; 1547 struct rte_eth_hairpin_cap cap; 1548 1549 RTE_ETH_FOREACH_DEV(pi) { 1550 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1551 *pid = pi; 1552 return 0; 1553 } 1554 if (cap.max_nb_queues < allowed_max_hairpinq) { 1555 allowed_max_hairpinq = cap.max_nb_queues; 1556 *pid = pi; 1557 } 1558 } 1559 return allowed_max_hairpinq; 1560 } 1561 1562 /* 1563 * Check input hairpin is valid or not. 1564 * If input hairpin is not greater than any of maximum number 1565 * of hairpin queues of all ports, it is valid. 1566 * if valid, return 0, else return -1 1567 */ 1568 int 1569 check_nb_hairpinq(queueid_t hairpinq) 1570 { 1571 queueid_t allowed_max_hairpinq; 1572 portid_t pid = 0; 1573 1574 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1575 if (hairpinq > allowed_max_hairpinq) { 1576 fprintf(stderr, 1577 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1578 hairpinq, allowed_max_hairpinq, pid); 1579 return -1; 1580 } 1581 return 0; 1582 } 1583 1584 static int 1585 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1586 { 1587 uint32_t eth_overhead; 1588 1589 if (dev_info->max_mtu != UINT16_MAX && 1590 dev_info->max_rx_pktlen > dev_info->max_mtu) 1591 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1592 else 1593 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1594 1595 return eth_overhead; 1596 } 1597 1598 static void 1599 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1600 { 1601 struct rte_port *port = &ports[pid]; 1602 int ret; 1603 int i; 1604 1605 eth_rx_metadata_negotiate_mp(pid); 1606 1607 port->dev_conf.txmode = tx_mode; 1608 port->dev_conf.rxmode = rx_mode; 1609 1610 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1611 if (ret != 0) 1612 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1613 1614 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1615 port->dev_conf.txmode.offloads &= 1616 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1617 1618 /* Apply Rx offloads configuration */ 1619 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1620 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1621 /* Apply Tx offloads configuration */ 1622 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1623 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1624 1625 if (eth_link_speed) 1626 port->dev_conf.link_speeds = eth_link_speed; 1627 1628 if (max_rx_pkt_len) 1629 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1630 get_eth_overhead(&port->dev_info); 1631 1632 /* set flag to initialize port/queue */ 1633 port->need_reconfig = 1; 1634 port->need_reconfig_queues = 1; 1635 port->socket_id = socket_id; 1636 port->tx_metadata = 0; 1637 1638 /* 1639 * Check for maximum number of segments per MTU. 1640 * Accordingly update the mbuf data size. 1641 */ 1642 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1643 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1644 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1645 uint16_t mtu; 1646 1647 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1648 uint16_t data_size = (mtu + eth_overhead) / 1649 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1650 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1651 1652 if (buffer_size > mbuf_data_size[0]) { 1653 mbuf_data_size[0] = buffer_size; 1654 TESTPMD_LOG(WARNING, 1655 "Configured mbuf size of the first segment %hu\n", 1656 mbuf_data_size[0]); 1657 } 1658 } 1659 } 1660 } 1661 1662 static void 1663 init_config(void) 1664 { 1665 portid_t pid; 1666 struct rte_mempool *mbp; 1667 unsigned int nb_mbuf_per_pool; 1668 lcoreid_t lc_id; 1669 #ifdef RTE_LIB_GRO 1670 struct rte_gro_param gro_param; 1671 #endif 1672 #ifdef RTE_LIB_GSO 1673 uint32_t gso_types; 1674 #endif 1675 1676 /* Configuration of logical cores. */ 1677 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1678 sizeof(struct fwd_lcore *) * nb_lcores, 1679 RTE_CACHE_LINE_SIZE); 1680 if (fwd_lcores == NULL) { 1681 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1682 "failed\n", nb_lcores); 1683 } 1684 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1685 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1686 sizeof(struct fwd_lcore), 1687 RTE_CACHE_LINE_SIZE); 1688 if (fwd_lcores[lc_id] == NULL) { 1689 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1690 "failed\n"); 1691 } 1692 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1693 } 1694 1695 RTE_ETH_FOREACH_DEV(pid) { 1696 uint32_t socket_id; 1697 1698 if (numa_support) { 1699 socket_id = port_numa[pid]; 1700 if (port_numa[pid] == NUMA_NO_CONFIG) { 1701 socket_id = rte_eth_dev_socket_id(pid); 1702 1703 /* 1704 * if socket_id is invalid, 1705 * set to the first available socket. 1706 */ 1707 if (check_socket_id(socket_id) < 0) 1708 socket_id = socket_ids[0]; 1709 } 1710 } else { 1711 socket_id = (socket_num == UMA_NO_CONFIG) ? 1712 0 : socket_num; 1713 } 1714 /* Apply default TxRx configuration for all ports */ 1715 init_config_port_offloads(pid, socket_id); 1716 } 1717 /* 1718 * Create pools of mbuf. 1719 * If NUMA support is disabled, create a single pool of mbuf in 1720 * socket 0 memory by default. 1721 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1722 * 1723 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1724 * nb_txd can be configured at run time. 1725 */ 1726 if (param_total_num_mbufs) 1727 nb_mbuf_per_pool = param_total_num_mbufs; 1728 else { 1729 nb_mbuf_per_pool = RX_DESC_MAX + 1730 (nb_lcores * mb_mempool_cache) + 1731 TX_DESC_MAX + MAX_PKT_BURST; 1732 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1733 } 1734 1735 if (numa_support) { 1736 uint8_t i, j; 1737 1738 for (i = 0; i < num_sockets; i++) 1739 for (j = 0; j < mbuf_data_size_n; j++) 1740 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1741 mbuf_pool_create(mbuf_data_size[j], 1742 nb_mbuf_per_pool, 1743 socket_ids[i], j); 1744 } else { 1745 uint8_t i; 1746 1747 for (i = 0; i < mbuf_data_size_n; i++) 1748 mempools[i] = mbuf_pool_create 1749 (mbuf_data_size[i], 1750 nb_mbuf_per_pool, 1751 socket_num == UMA_NO_CONFIG ? 1752 0 : socket_num, i); 1753 } 1754 1755 init_port_config(); 1756 1757 #ifdef RTE_LIB_GSO 1758 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1759 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1760 #endif 1761 /* 1762 * Records which Mbuf pool to use by each logical core, if needed. 1763 */ 1764 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1765 mbp = mbuf_pool_find( 1766 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1767 1768 if (mbp == NULL) 1769 mbp = mbuf_pool_find(0, 0); 1770 fwd_lcores[lc_id]->mbp = mbp; 1771 #ifdef RTE_LIB_GSO 1772 /* initialize GSO context */ 1773 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1774 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1775 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1776 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1777 RTE_ETHER_CRC_LEN; 1778 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1779 #endif 1780 } 1781 1782 fwd_config_setup(); 1783 1784 #ifdef RTE_LIB_GRO 1785 /* create a gro context for each lcore */ 1786 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1787 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1788 gro_param.max_item_per_flow = MAX_PKT_BURST; 1789 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1790 gro_param.socket_id = rte_lcore_to_socket_id( 1791 fwd_lcores_cpuids[lc_id]); 1792 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1793 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1794 rte_exit(EXIT_FAILURE, 1795 "rte_gro_ctx_create() failed\n"); 1796 } 1797 } 1798 #endif 1799 } 1800 1801 1802 void 1803 reconfig(portid_t new_port_id, unsigned socket_id) 1804 { 1805 /* Reconfiguration of Ethernet ports. */ 1806 init_config_port_offloads(new_port_id, socket_id); 1807 init_port_config(); 1808 } 1809 1810 int 1811 init_fwd_streams(void) 1812 { 1813 portid_t pid; 1814 struct rte_port *port; 1815 streamid_t sm_id, nb_fwd_streams_new; 1816 queueid_t q; 1817 1818 /* set socket id according to numa or not */ 1819 RTE_ETH_FOREACH_DEV(pid) { 1820 port = &ports[pid]; 1821 if (nb_rxq > port->dev_info.max_rx_queues) { 1822 fprintf(stderr, 1823 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1824 nb_rxq, port->dev_info.max_rx_queues); 1825 return -1; 1826 } 1827 if (nb_txq > port->dev_info.max_tx_queues) { 1828 fprintf(stderr, 1829 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1830 nb_txq, port->dev_info.max_tx_queues); 1831 return -1; 1832 } 1833 if (numa_support) { 1834 if (port_numa[pid] != NUMA_NO_CONFIG) 1835 port->socket_id = port_numa[pid]; 1836 else { 1837 port->socket_id = rte_eth_dev_socket_id(pid); 1838 1839 /* 1840 * if socket_id is invalid, 1841 * set to the first available socket. 1842 */ 1843 if (check_socket_id(port->socket_id) < 0) 1844 port->socket_id = socket_ids[0]; 1845 } 1846 } 1847 else { 1848 if (socket_num == UMA_NO_CONFIG) 1849 port->socket_id = 0; 1850 else 1851 port->socket_id = socket_num; 1852 } 1853 } 1854 1855 q = RTE_MAX(nb_rxq, nb_txq); 1856 if (q == 0) { 1857 fprintf(stderr, 1858 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1859 return -1; 1860 } 1861 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1862 if (nb_fwd_streams_new == nb_fwd_streams) 1863 return 0; 1864 /* clear the old */ 1865 if (fwd_streams != NULL) { 1866 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1867 if (fwd_streams[sm_id] == NULL) 1868 continue; 1869 rte_free(fwd_streams[sm_id]); 1870 fwd_streams[sm_id] = NULL; 1871 } 1872 rte_free(fwd_streams); 1873 fwd_streams = NULL; 1874 } 1875 1876 /* init new */ 1877 nb_fwd_streams = nb_fwd_streams_new; 1878 if (nb_fwd_streams) { 1879 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1880 sizeof(struct fwd_stream *) * nb_fwd_streams, 1881 RTE_CACHE_LINE_SIZE); 1882 if (fwd_streams == NULL) 1883 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1884 " (struct fwd_stream *)) failed\n", 1885 nb_fwd_streams); 1886 1887 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1888 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1889 " struct fwd_stream", sizeof(struct fwd_stream), 1890 RTE_CACHE_LINE_SIZE); 1891 if (fwd_streams[sm_id] == NULL) 1892 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1893 "(struct fwd_stream) failed\n"); 1894 } 1895 } 1896 1897 return 0; 1898 } 1899 1900 static void 1901 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1902 { 1903 uint64_t total_burst, sburst; 1904 uint64_t nb_burst; 1905 uint64_t burst_stats[4]; 1906 uint16_t pktnb_stats[4]; 1907 uint16_t nb_pkt; 1908 int burst_percent[4], sburstp; 1909 int i; 1910 1911 /* 1912 * First compute the total number of packet bursts and the 1913 * two highest numbers of bursts of the same number of packets. 1914 */ 1915 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1916 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1917 1918 /* Show stats for 0 burst size always */ 1919 total_burst = pbs->pkt_burst_spread[0]; 1920 burst_stats[0] = pbs->pkt_burst_spread[0]; 1921 pktnb_stats[0] = 0; 1922 1923 /* Find the next 2 burst sizes with highest occurrences. */ 1924 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1925 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1926 1927 if (nb_burst == 0) 1928 continue; 1929 1930 total_burst += nb_burst; 1931 1932 if (nb_burst > burst_stats[1]) { 1933 burst_stats[2] = burst_stats[1]; 1934 pktnb_stats[2] = pktnb_stats[1]; 1935 burst_stats[1] = nb_burst; 1936 pktnb_stats[1] = nb_pkt; 1937 } else if (nb_burst > burst_stats[2]) { 1938 burst_stats[2] = nb_burst; 1939 pktnb_stats[2] = nb_pkt; 1940 } 1941 } 1942 if (total_burst == 0) 1943 return; 1944 1945 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1946 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1947 if (i == 3) { 1948 printf("%d%% of other]\n", 100 - sburstp); 1949 return; 1950 } 1951 1952 sburst += burst_stats[i]; 1953 if (sburst == total_burst) { 1954 printf("%d%% of %d pkts]\n", 1955 100 - sburstp, (int) pktnb_stats[i]); 1956 return; 1957 } 1958 1959 burst_percent[i] = 1960 (double)burst_stats[i] / total_burst * 100; 1961 printf("%d%% of %d pkts + ", 1962 burst_percent[i], (int) pktnb_stats[i]); 1963 sburstp += burst_percent[i]; 1964 } 1965 } 1966 1967 static void 1968 fwd_stream_stats_display(streamid_t stream_id) 1969 { 1970 struct fwd_stream *fs; 1971 static const char *fwd_top_stats_border = "-------"; 1972 1973 fs = fwd_streams[stream_id]; 1974 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1975 (fs->fwd_dropped == 0)) 1976 return; 1977 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1978 "TX Port=%2d/Queue=%2d %s\n", 1979 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1980 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1981 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1982 " TX-dropped: %-14"PRIu64, 1983 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1984 1985 /* if checksum mode */ 1986 if (cur_fwd_eng == &csum_fwd_engine) { 1987 printf(" RX- bad IP checksum: %-14"PRIu64 1988 " Rx- bad L4 checksum: %-14"PRIu64 1989 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1990 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1991 fs->rx_bad_outer_l4_csum); 1992 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1993 fs->rx_bad_outer_ip_csum); 1994 } else { 1995 printf("\n"); 1996 } 1997 1998 if (record_burst_stats) { 1999 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2000 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2001 } 2002 } 2003 2004 void 2005 fwd_stats_display(void) 2006 { 2007 static const char *fwd_stats_border = "----------------------"; 2008 static const char *acc_stats_border = "+++++++++++++++"; 2009 struct { 2010 struct fwd_stream *rx_stream; 2011 struct fwd_stream *tx_stream; 2012 uint64_t tx_dropped; 2013 uint64_t rx_bad_ip_csum; 2014 uint64_t rx_bad_l4_csum; 2015 uint64_t rx_bad_outer_l4_csum; 2016 uint64_t rx_bad_outer_ip_csum; 2017 } ports_stats[RTE_MAX_ETHPORTS]; 2018 uint64_t total_rx_dropped = 0; 2019 uint64_t total_tx_dropped = 0; 2020 uint64_t total_rx_nombuf = 0; 2021 struct rte_eth_stats stats; 2022 uint64_t fwd_cycles = 0; 2023 uint64_t total_recv = 0; 2024 uint64_t total_xmit = 0; 2025 struct rte_port *port; 2026 streamid_t sm_id; 2027 portid_t pt_id; 2028 int ret; 2029 int i; 2030 2031 memset(ports_stats, 0, sizeof(ports_stats)); 2032 2033 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2034 struct fwd_stream *fs = fwd_streams[sm_id]; 2035 2036 if (cur_fwd_config.nb_fwd_streams > 2037 cur_fwd_config.nb_fwd_ports) { 2038 fwd_stream_stats_display(sm_id); 2039 } else { 2040 ports_stats[fs->tx_port].tx_stream = fs; 2041 ports_stats[fs->rx_port].rx_stream = fs; 2042 } 2043 2044 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2045 2046 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2047 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2048 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2049 fs->rx_bad_outer_l4_csum; 2050 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2051 fs->rx_bad_outer_ip_csum; 2052 2053 if (record_core_cycles) 2054 fwd_cycles += fs->core_cycles; 2055 } 2056 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2057 pt_id = fwd_ports_ids[i]; 2058 port = &ports[pt_id]; 2059 2060 ret = rte_eth_stats_get(pt_id, &stats); 2061 if (ret != 0) { 2062 fprintf(stderr, 2063 "%s: Error: failed to get stats (port %u): %d", 2064 __func__, pt_id, ret); 2065 continue; 2066 } 2067 stats.ipackets -= port->stats.ipackets; 2068 stats.opackets -= port->stats.opackets; 2069 stats.ibytes -= port->stats.ibytes; 2070 stats.obytes -= port->stats.obytes; 2071 stats.imissed -= port->stats.imissed; 2072 stats.oerrors -= port->stats.oerrors; 2073 stats.rx_nombuf -= port->stats.rx_nombuf; 2074 2075 total_recv += stats.ipackets; 2076 total_xmit += stats.opackets; 2077 total_rx_dropped += stats.imissed; 2078 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2079 total_tx_dropped += stats.oerrors; 2080 total_rx_nombuf += stats.rx_nombuf; 2081 2082 printf("\n %s Forward statistics for port %-2d %s\n", 2083 fwd_stats_border, pt_id, fwd_stats_border); 2084 2085 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2086 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2087 stats.ipackets + stats.imissed); 2088 2089 if (cur_fwd_eng == &csum_fwd_engine) { 2090 printf(" Bad-ipcsum: %-14"PRIu64 2091 " Bad-l4csum: %-14"PRIu64 2092 "Bad-outer-l4csum: %-14"PRIu64"\n", 2093 ports_stats[pt_id].rx_bad_ip_csum, 2094 ports_stats[pt_id].rx_bad_l4_csum, 2095 ports_stats[pt_id].rx_bad_outer_l4_csum); 2096 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2097 ports_stats[pt_id].rx_bad_outer_ip_csum); 2098 } 2099 if (stats.ierrors + stats.rx_nombuf > 0) { 2100 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2101 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2102 } 2103 2104 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2105 "TX-total: %-"PRIu64"\n", 2106 stats.opackets, ports_stats[pt_id].tx_dropped, 2107 stats.opackets + ports_stats[pt_id].tx_dropped); 2108 2109 if (record_burst_stats) { 2110 if (ports_stats[pt_id].rx_stream) 2111 pkt_burst_stats_display("RX", 2112 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2113 if (ports_stats[pt_id].tx_stream) 2114 pkt_burst_stats_display("TX", 2115 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2116 } 2117 2118 printf(" %s--------------------------------%s\n", 2119 fwd_stats_border, fwd_stats_border); 2120 } 2121 2122 printf("\n %s Accumulated forward statistics for all ports" 2123 "%s\n", 2124 acc_stats_border, acc_stats_border); 2125 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2126 "%-"PRIu64"\n" 2127 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2128 "%-"PRIu64"\n", 2129 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2130 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2131 if (total_rx_nombuf > 0) 2132 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2133 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2134 "%s\n", 2135 acc_stats_border, acc_stats_border); 2136 if (record_core_cycles) { 2137 #define CYC_PER_MHZ 1E6 2138 if (total_recv > 0 || total_xmit > 0) { 2139 uint64_t total_pkts = 0; 2140 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2141 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2142 total_pkts = total_xmit; 2143 else 2144 total_pkts = total_recv; 2145 2146 printf("\n CPU cycles/packet=%.2F (total cycles=" 2147 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2148 " MHz Clock\n", 2149 (double) fwd_cycles / total_pkts, 2150 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2151 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2152 } 2153 } 2154 } 2155 2156 void 2157 fwd_stats_reset(void) 2158 { 2159 streamid_t sm_id; 2160 portid_t pt_id; 2161 int ret; 2162 int i; 2163 2164 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2165 pt_id = fwd_ports_ids[i]; 2166 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2167 if (ret != 0) 2168 fprintf(stderr, 2169 "%s: Error: failed to clear stats (port %u):%d", 2170 __func__, pt_id, ret); 2171 } 2172 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2173 struct fwd_stream *fs = fwd_streams[sm_id]; 2174 2175 fs->rx_packets = 0; 2176 fs->tx_packets = 0; 2177 fs->fwd_dropped = 0; 2178 fs->rx_bad_ip_csum = 0; 2179 fs->rx_bad_l4_csum = 0; 2180 fs->rx_bad_outer_l4_csum = 0; 2181 fs->rx_bad_outer_ip_csum = 0; 2182 2183 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2184 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2185 fs->core_cycles = 0; 2186 } 2187 } 2188 2189 static void 2190 flush_fwd_rx_queues(void) 2191 { 2192 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2193 portid_t rxp; 2194 portid_t port_id; 2195 queueid_t rxq; 2196 uint16_t nb_rx; 2197 uint16_t i; 2198 uint8_t j; 2199 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2200 uint64_t timer_period; 2201 2202 if (num_procs > 1) { 2203 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2204 return; 2205 } 2206 2207 /* convert to number of cycles */ 2208 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2209 2210 for (j = 0; j < 2; j++) { 2211 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2212 for (rxq = 0; rxq < nb_rxq; rxq++) { 2213 port_id = fwd_ports_ids[rxp]; 2214 2215 /* Polling stopped queues is prohibited. */ 2216 if (ports[port_id].rxq[rxq].state == 2217 RTE_ETH_QUEUE_STATE_STOPPED) 2218 continue; 2219 2220 /** 2221 * testpmd can stuck in the below do while loop 2222 * if rte_eth_rx_burst() always returns nonzero 2223 * packets. So timer is added to exit this loop 2224 * after 1sec timer expiry. 2225 */ 2226 prev_tsc = rte_rdtsc(); 2227 do { 2228 nb_rx = rte_eth_rx_burst(port_id, rxq, 2229 pkts_burst, MAX_PKT_BURST); 2230 for (i = 0; i < nb_rx; i++) 2231 rte_pktmbuf_free(pkts_burst[i]); 2232 2233 cur_tsc = rte_rdtsc(); 2234 diff_tsc = cur_tsc - prev_tsc; 2235 timer_tsc += diff_tsc; 2236 } while ((nb_rx > 0) && 2237 (timer_tsc < timer_period)); 2238 timer_tsc = 0; 2239 } 2240 } 2241 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2242 } 2243 } 2244 2245 static void 2246 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2247 { 2248 struct fwd_stream **fsm; 2249 streamid_t nb_fs; 2250 streamid_t sm_id; 2251 #ifdef RTE_LIB_BITRATESTATS 2252 uint64_t tics_per_1sec; 2253 uint64_t tics_datum; 2254 uint64_t tics_current; 2255 uint16_t i, cnt_ports; 2256 2257 cnt_ports = nb_ports; 2258 tics_datum = rte_rdtsc(); 2259 tics_per_1sec = rte_get_timer_hz(); 2260 #endif 2261 fsm = &fwd_streams[fc->stream_idx]; 2262 nb_fs = fc->stream_nb; 2263 do { 2264 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2265 if (!fsm[sm_id]->disabled) 2266 (*pkt_fwd)(fsm[sm_id]); 2267 #ifdef RTE_LIB_BITRATESTATS 2268 if (bitrate_enabled != 0 && 2269 bitrate_lcore_id == rte_lcore_id()) { 2270 tics_current = rte_rdtsc(); 2271 if (tics_current - tics_datum >= tics_per_1sec) { 2272 /* Periodic bitrate calculation */ 2273 for (i = 0; i < cnt_ports; i++) 2274 rte_stats_bitrate_calc(bitrate_data, 2275 ports_ids[i]); 2276 tics_datum = tics_current; 2277 } 2278 } 2279 #endif 2280 #ifdef RTE_LIB_LATENCYSTATS 2281 if (latencystats_enabled != 0 && 2282 latencystats_lcore_id == rte_lcore_id()) 2283 rte_latencystats_update(); 2284 #endif 2285 2286 } while (! fc->stopped); 2287 } 2288 2289 static int 2290 start_pkt_forward_on_core(void *fwd_arg) 2291 { 2292 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2293 cur_fwd_config.fwd_eng->packet_fwd); 2294 return 0; 2295 } 2296 2297 /* 2298 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2299 * Used to start communication flows in network loopback test configurations. 2300 */ 2301 static int 2302 run_one_txonly_burst_on_core(void *fwd_arg) 2303 { 2304 struct fwd_lcore *fwd_lc; 2305 struct fwd_lcore tmp_lcore; 2306 2307 fwd_lc = (struct fwd_lcore *) fwd_arg; 2308 tmp_lcore = *fwd_lc; 2309 tmp_lcore.stopped = 1; 2310 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2311 return 0; 2312 } 2313 2314 /* 2315 * Launch packet forwarding: 2316 * - Setup per-port forwarding context. 2317 * - launch logical cores with their forwarding configuration. 2318 */ 2319 static void 2320 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2321 { 2322 unsigned int i; 2323 unsigned int lc_id; 2324 int diag; 2325 2326 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2327 lc_id = fwd_lcores_cpuids[i]; 2328 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2329 fwd_lcores[i]->stopped = 0; 2330 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2331 fwd_lcores[i], lc_id); 2332 if (diag != 0) 2333 fprintf(stderr, 2334 "launch lcore %u failed - diag=%d\n", 2335 lc_id, diag); 2336 } 2337 } 2338 } 2339 2340 /* 2341 * Launch packet forwarding configuration. 2342 */ 2343 void 2344 start_packet_forwarding(int with_tx_first) 2345 { 2346 port_fwd_begin_t port_fwd_begin; 2347 port_fwd_end_t port_fwd_end; 2348 stream_init_t stream_init = cur_fwd_eng->stream_init; 2349 unsigned int i; 2350 2351 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2352 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2353 2354 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2355 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2356 2357 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2358 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2359 (!nb_rxq || !nb_txq)) 2360 rte_exit(EXIT_FAILURE, 2361 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2362 cur_fwd_eng->fwd_mode_name); 2363 2364 if (all_ports_started() == 0) { 2365 fprintf(stderr, "Not all ports were started\n"); 2366 return; 2367 } 2368 if (test_done == 0) { 2369 fprintf(stderr, "Packet forwarding already started\n"); 2370 return; 2371 } 2372 2373 fwd_config_setup(); 2374 2375 pkt_fwd_config_display(&cur_fwd_config); 2376 if (!pkt_fwd_shared_rxq_check()) 2377 return; 2378 2379 if (stream_init != NULL) 2380 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2381 stream_init(fwd_streams[i]); 2382 2383 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2384 if (port_fwd_begin != NULL) { 2385 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2386 if (port_fwd_begin(fwd_ports_ids[i])) { 2387 fprintf(stderr, 2388 "Packet forwarding is not ready\n"); 2389 return; 2390 } 2391 } 2392 } 2393 2394 if (with_tx_first) { 2395 port_fwd_begin = tx_only_engine.port_fwd_begin; 2396 if (port_fwd_begin != NULL) { 2397 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2398 if (port_fwd_begin(fwd_ports_ids[i])) { 2399 fprintf(stderr, 2400 "Packet forwarding is not ready\n"); 2401 return; 2402 } 2403 } 2404 } 2405 } 2406 2407 test_done = 0; 2408 2409 if(!no_flush_rx) 2410 flush_fwd_rx_queues(); 2411 2412 rxtx_config_display(); 2413 2414 fwd_stats_reset(); 2415 if (with_tx_first) { 2416 while (with_tx_first--) { 2417 launch_packet_forwarding( 2418 run_one_txonly_burst_on_core); 2419 rte_eal_mp_wait_lcore(); 2420 } 2421 port_fwd_end = tx_only_engine.port_fwd_end; 2422 if (port_fwd_end != NULL) { 2423 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2424 (*port_fwd_end)(fwd_ports_ids[i]); 2425 } 2426 } 2427 launch_packet_forwarding(start_pkt_forward_on_core); 2428 } 2429 2430 void 2431 stop_packet_forwarding(void) 2432 { 2433 port_fwd_end_t port_fwd_end; 2434 lcoreid_t lc_id; 2435 portid_t pt_id; 2436 int i; 2437 2438 if (test_done) { 2439 fprintf(stderr, "Packet forwarding not started\n"); 2440 return; 2441 } 2442 printf("Telling cores to stop..."); 2443 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2444 fwd_lcores[lc_id]->stopped = 1; 2445 printf("\nWaiting for lcores to finish...\n"); 2446 rte_eal_mp_wait_lcore(); 2447 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2448 if (port_fwd_end != NULL) { 2449 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2450 pt_id = fwd_ports_ids[i]; 2451 (*port_fwd_end)(pt_id); 2452 } 2453 } 2454 2455 fwd_stats_display(); 2456 2457 printf("\nDone.\n"); 2458 test_done = 1; 2459 } 2460 2461 void 2462 dev_set_link_up(portid_t pid) 2463 { 2464 if (rte_eth_dev_set_link_up(pid) < 0) 2465 fprintf(stderr, "\nSet link up fail.\n"); 2466 } 2467 2468 void 2469 dev_set_link_down(portid_t pid) 2470 { 2471 if (rte_eth_dev_set_link_down(pid) < 0) 2472 fprintf(stderr, "\nSet link down fail.\n"); 2473 } 2474 2475 static int 2476 all_ports_started(void) 2477 { 2478 portid_t pi; 2479 struct rte_port *port; 2480 2481 RTE_ETH_FOREACH_DEV(pi) { 2482 port = &ports[pi]; 2483 /* Check if there is a port which is not started */ 2484 if ((port->port_status != RTE_PORT_STARTED) && 2485 (port->slave_flag == 0)) 2486 return 0; 2487 } 2488 2489 /* No port is not started */ 2490 return 1; 2491 } 2492 2493 int 2494 port_is_stopped(portid_t port_id) 2495 { 2496 struct rte_port *port = &ports[port_id]; 2497 2498 if ((port->port_status != RTE_PORT_STOPPED) && 2499 (port->slave_flag == 0)) 2500 return 0; 2501 return 1; 2502 } 2503 2504 int 2505 all_ports_stopped(void) 2506 { 2507 portid_t pi; 2508 2509 RTE_ETH_FOREACH_DEV(pi) { 2510 if (!port_is_stopped(pi)) 2511 return 0; 2512 } 2513 2514 return 1; 2515 } 2516 2517 int 2518 port_is_started(portid_t port_id) 2519 { 2520 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2521 return 0; 2522 2523 if (ports[port_id].port_status != RTE_PORT_STARTED) 2524 return 0; 2525 2526 return 1; 2527 } 2528 2529 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2530 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2531 2532 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2533 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2534 2535 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2536 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2537 2538 2539 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2540 static int 2541 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2542 { 2543 queueid_t qi; 2544 struct rte_eth_hairpin_conf hairpin_conf = { 2545 .peer_count = 1, 2546 }; 2547 int i; 2548 int diag; 2549 struct rte_port *port = &ports[pi]; 2550 uint16_t peer_rx_port = pi; 2551 uint16_t peer_tx_port = pi; 2552 uint32_t manual = 1; 2553 uint32_t tx_exp = hairpin_mode & 0x10; 2554 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2555 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2556 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2557 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2558 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2559 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2560 2561 if (!(hairpin_mode & 0xf)) { 2562 peer_rx_port = pi; 2563 peer_tx_port = pi; 2564 manual = 0; 2565 } else if (hairpin_mode & 0x1) { 2566 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2567 RTE_ETH_DEV_NO_OWNER); 2568 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2569 peer_tx_port = rte_eth_find_next_owned_by(0, 2570 RTE_ETH_DEV_NO_OWNER); 2571 if (p_pi != RTE_MAX_ETHPORTS) { 2572 peer_rx_port = p_pi; 2573 } else { 2574 uint16_t next_pi; 2575 2576 /* Last port will be the peer RX port of the first. */ 2577 RTE_ETH_FOREACH_DEV(next_pi) 2578 peer_rx_port = next_pi; 2579 } 2580 manual = 1; 2581 } else if (hairpin_mode & 0x2) { 2582 if (cnt_pi & 0x1) { 2583 peer_rx_port = p_pi; 2584 } else { 2585 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2586 RTE_ETH_DEV_NO_OWNER); 2587 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2588 peer_rx_port = pi; 2589 } 2590 peer_tx_port = peer_rx_port; 2591 manual = 1; 2592 } 2593 2594 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2595 hairpin_conf.peers[0].port = peer_rx_port; 2596 hairpin_conf.peers[0].queue = i + nb_rxq; 2597 hairpin_conf.manual_bind = !!manual; 2598 hairpin_conf.tx_explicit = !!tx_exp; 2599 hairpin_conf.force_memory = !!tx_force_memory; 2600 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2601 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2602 diag = rte_eth_tx_hairpin_queue_setup 2603 (pi, qi, nb_txd, &hairpin_conf); 2604 i++; 2605 if (diag == 0) 2606 continue; 2607 2608 /* Fail to setup rx queue, return */ 2609 if (port->port_status == RTE_PORT_HANDLING) 2610 port->port_status = RTE_PORT_STOPPED; 2611 else 2612 fprintf(stderr, 2613 "Port %d can not be set back to stopped\n", pi); 2614 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2615 pi); 2616 /* try to reconfigure queues next time */ 2617 port->need_reconfig_queues = 1; 2618 return -1; 2619 } 2620 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2621 hairpin_conf.peers[0].port = peer_tx_port; 2622 hairpin_conf.peers[0].queue = i + nb_txq; 2623 hairpin_conf.manual_bind = !!manual; 2624 hairpin_conf.tx_explicit = !!tx_exp; 2625 hairpin_conf.force_memory = !!rx_force_memory; 2626 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2627 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2628 diag = rte_eth_rx_hairpin_queue_setup 2629 (pi, qi, nb_rxd, &hairpin_conf); 2630 i++; 2631 if (diag == 0) 2632 continue; 2633 2634 /* Fail to setup rx queue, return */ 2635 if (port->port_status == RTE_PORT_HANDLING) 2636 port->port_status = RTE_PORT_STOPPED; 2637 else 2638 fprintf(stderr, 2639 "Port %d can not be set back to stopped\n", pi); 2640 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2641 pi); 2642 /* try to reconfigure queues next time */ 2643 port->need_reconfig_queues = 1; 2644 return -1; 2645 } 2646 return 0; 2647 } 2648 2649 /* Configure the Rx with optional split. */ 2650 int 2651 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2652 uint16_t nb_rx_desc, unsigned int socket_id, 2653 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2654 { 2655 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2656 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2657 struct rte_mempool *mpx; 2658 unsigned int i, mp_n; 2659 uint32_t prev_hdrs = 0; 2660 int ret; 2661 2662 /* Verify Rx queue configuration is single pool and segment or 2663 * multiple pool/segment. 2664 * @see rte_eth_rxconf::rx_mempools 2665 * @see rte_eth_rxconf::rx_seg 2666 */ 2667 if (!(mbuf_data_size_n > 1) && !(rx_pkt_nb_segs > 1 || 2668 ((rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) != 0))) { 2669 /* Single pool/segment configuration */ 2670 rx_conf->rx_seg = NULL; 2671 rx_conf->rx_nseg = 0; 2672 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2673 nb_rx_desc, socket_id, 2674 rx_conf, mp); 2675 goto exit; 2676 } 2677 2678 if (rx_pkt_nb_segs > 1 || 2679 rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) { 2680 /* multi-segment configuration */ 2681 for (i = 0; i < rx_pkt_nb_segs; i++) { 2682 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2683 /* 2684 * Use last valid pool for the segments with number 2685 * exceeding the pool index. 2686 */ 2687 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2688 mpx = mbuf_pool_find(socket_id, mp_n); 2689 /* Handle zero as mbuf data buffer size. */ 2690 rx_seg->offset = i < rx_pkt_nb_offs ? 2691 rx_pkt_seg_offsets[i] : 0; 2692 rx_seg->mp = mpx ? mpx : mp; 2693 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2694 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2695 prev_hdrs |= rx_seg->proto_hdr; 2696 } else { 2697 rx_seg->length = rx_pkt_seg_lengths[i] ? 2698 rx_pkt_seg_lengths[i] : 2699 mbuf_data_size[mp_n]; 2700 } 2701 } 2702 rx_conf->rx_nseg = rx_pkt_nb_segs; 2703 rx_conf->rx_seg = rx_useg; 2704 } else { 2705 /* multi-pool configuration */ 2706 for (i = 0; i < mbuf_data_size_n; i++) { 2707 mpx = mbuf_pool_find(socket_id, i); 2708 rx_mempool[i] = mpx ? mpx : mp; 2709 } 2710 rx_conf->rx_mempools = rx_mempool; 2711 rx_conf->rx_nmempool = mbuf_data_size_n; 2712 } 2713 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2714 socket_id, rx_conf, NULL); 2715 rx_conf->rx_seg = NULL; 2716 rx_conf->rx_nseg = 0; 2717 rx_conf->rx_mempools = NULL; 2718 rx_conf->rx_nmempool = 0; 2719 exit: 2720 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2721 RTE_ETH_QUEUE_STATE_STOPPED : 2722 RTE_ETH_QUEUE_STATE_STARTED; 2723 return ret; 2724 } 2725 2726 static int 2727 alloc_xstats_display_info(portid_t pi) 2728 { 2729 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2730 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2731 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2732 2733 if (xstats_display_num == 0) 2734 return 0; 2735 2736 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2737 if (*ids_supp == NULL) 2738 goto fail_ids_supp; 2739 2740 *prev_values = calloc(xstats_display_num, 2741 sizeof(**prev_values)); 2742 if (*prev_values == NULL) 2743 goto fail_prev_values; 2744 2745 *curr_values = calloc(xstats_display_num, 2746 sizeof(**curr_values)); 2747 if (*curr_values == NULL) 2748 goto fail_curr_values; 2749 2750 ports[pi].xstats_info.allocated = true; 2751 2752 return 0; 2753 2754 fail_curr_values: 2755 free(*prev_values); 2756 fail_prev_values: 2757 free(*ids_supp); 2758 fail_ids_supp: 2759 return -ENOMEM; 2760 } 2761 2762 static void 2763 free_xstats_display_info(portid_t pi) 2764 { 2765 if (!ports[pi].xstats_info.allocated) 2766 return; 2767 free(ports[pi].xstats_info.ids_supp); 2768 free(ports[pi].xstats_info.prev_values); 2769 free(ports[pi].xstats_info.curr_values); 2770 ports[pi].xstats_info.allocated = false; 2771 } 2772 2773 /** Fill helper structures for specified port to show extended statistics. */ 2774 static void 2775 fill_xstats_display_info_for_port(portid_t pi) 2776 { 2777 unsigned int stat, stat_supp; 2778 const char *xstat_name; 2779 struct rte_port *port; 2780 uint64_t *ids_supp; 2781 int rc; 2782 2783 if (xstats_display_num == 0) 2784 return; 2785 2786 if (pi == (portid_t)RTE_PORT_ALL) { 2787 fill_xstats_display_info(); 2788 return; 2789 } 2790 2791 port = &ports[pi]; 2792 if (port->port_status != RTE_PORT_STARTED) 2793 return; 2794 2795 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2796 rte_exit(EXIT_FAILURE, 2797 "Failed to allocate xstats display memory\n"); 2798 2799 ids_supp = port->xstats_info.ids_supp; 2800 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2801 xstat_name = xstats_display[stat].name; 2802 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2803 ids_supp + stat_supp); 2804 if (rc != 0) { 2805 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2806 xstat_name, pi, stat); 2807 continue; 2808 } 2809 stat_supp++; 2810 } 2811 2812 port->xstats_info.ids_supp_sz = stat_supp; 2813 } 2814 2815 /** Fill helper structures for all ports to show extended statistics. */ 2816 static void 2817 fill_xstats_display_info(void) 2818 { 2819 portid_t pi; 2820 2821 if (xstats_display_num == 0) 2822 return; 2823 2824 RTE_ETH_FOREACH_DEV(pi) 2825 fill_xstats_display_info_for_port(pi); 2826 } 2827 2828 /* 2829 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2830 * device in dev_info is zero when no slave is added. And its capability 2831 * will be updated when add a new slave device. So adding a slave device need 2832 * to update the port configurations of bonding device. 2833 */ 2834 static void 2835 update_bonding_port_dev_conf(portid_t bond_pid) 2836 { 2837 #ifdef RTE_NET_BOND 2838 struct rte_port *port = &ports[bond_pid]; 2839 uint16_t i; 2840 int ret; 2841 2842 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2843 if (ret != 0) { 2844 fprintf(stderr, "Failed to get dev info for port = %u\n", 2845 bond_pid); 2846 return; 2847 } 2848 2849 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2850 port->dev_conf.txmode.offloads |= 2851 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2852 /* Apply Tx offloads configuration */ 2853 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2854 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2855 2856 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 2857 port->dev_info.flow_type_rss_offloads; 2858 #else 2859 RTE_SET_USED(bond_pid); 2860 #endif 2861 } 2862 2863 int 2864 start_port(portid_t pid) 2865 { 2866 int diag, need_check_link_status = -1; 2867 portid_t pi; 2868 portid_t p_pi = RTE_MAX_ETHPORTS; 2869 portid_t pl[RTE_MAX_ETHPORTS]; 2870 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2871 uint16_t cnt_pi = 0; 2872 uint16_t cfg_pi = 0; 2873 int peer_pi; 2874 queueid_t qi; 2875 struct rte_port *port; 2876 struct rte_eth_hairpin_cap cap; 2877 2878 if (port_id_is_invalid(pid, ENABLED_WARN)) 2879 return 0; 2880 2881 RTE_ETH_FOREACH_DEV(pi) { 2882 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2883 continue; 2884 2885 if (port_is_bonding_slave(pi)) { 2886 fprintf(stderr, 2887 "Please remove port %d from bonded device.\n", 2888 pi); 2889 continue; 2890 } 2891 2892 need_check_link_status = 0; 2893 port = &ports[pi]; 2894 if (port->port_status == RTE_PORT_STOPPED) 2895 port->port_status = RTE_PORT_HANDLING; 2896 else { 2897 fprintf(stderr, "Port %d is now not stopped\n", pi); 2898 continue; 2899 } 2900 2901 if (port->need_reconfig > 0) { 2902 struct rte_eth_conf dev_conf; 2903 int k; 2904 2905 port->need_reconfig = 0; 2906 2907 if (flow_isolate_all) { 2908 int ret = port_flow_isolate(pi, 1); 2909 if (ret) { 2910 fprintf(stderr, 2911 "Failed to apply isolated mode on port %d\n", 2912 pi); 2913 return -1; 2914 } 2915 } 2916 configure_rxtx_dump_callbacks(0); 2917 printf("Configuring Port %d (socket %u)\n", pi, 2918 port->socket_id); 2919 if (nb_hairpinq > 0 && 2920 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2921 fprintf(stderr, 2922 "Port %d doesn't support hairpin queues\n", 2923 pi); 2924 return -1; 2925 } 2926 2927 if (port->bond_flag == 1 && port->update_conf == 1) { 2928 update_bonding_port_dev_conf(pi); 2929 port->update_conf = 0; 2930 } 2931 2932 /* configure port */ 2933 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2934 nb_txq + nb_hairpinq, 2935 &(port->dev_conf)); 2936 if (diag != 0) { 2937 if (port->port_status == RTE_PORT_HANDLING) 2938 port->port_status = RTE_PORT_STOPPED; 2939 else 2940 fprintf(stderr, 2941 "Port %d can not be set back to stopped\n", 2942 pi); 2943 fprintf(stderr, "Fail to configure port %d\n", 2944 pi); 2945 /* try to reconfigure port next time */ 2946 port->need_reconfig = 1; 2947 return -1; 2948 } 2949 /* get device configuration*/ 2950 if (0 != 2951 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2952 fprintf(stderr, 2953 "port %d can not get device configuration\n", 2954 pi); 2955 return -1; 2956 } 2957 /* Apply Rx offloads configuration */ 2958 if (dev_conf.rxmode.offloads != 2959 port->dev_conf.rxmode.offloads) { 2960 port->dev_conf.rxmode.offloads |= 2961 dev_conf.rxmode.offloads; 2962 for (k = 0; 2963 k < port->dev_info.max_rx_queues; 2964 k++) 2965 port->rxq[k].conf.offloads |= 2966 dev_conf.rxmode.offloads; 2967 } 2968 /* Apply Tx offloads configuration */ 2969 if (dev_conf.txmode.offloads != 2970 port->dev_conf.txmode.offloads) { 2971 port->dev_conf.txmode.offloads |= 2972 dev_conf.txmode.offloads; 2973 for (k = 0; 2974 k < port->dev_info.max_tx_queues; 2975 k++) 2976 port->txq[k].conf.offloads |= 2977 dev_conf.txmode.offloads; 2978 } 2979 } 2980 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2981 port->need_reconfig_queues = 0; 2982 /* setup tx queues */ 2983 for (qi = 0; qi < nb_txq; qi++) { 2984 struct rte_eth_txconf *conf = 2985 &port->txq[qi].conf; 2986 2987 if ((numa_support) && 2988 (txring_numa[pi] != NUMA_NO_CONFIG)) 2989 diag = rte_eth_tx_queue_setup(pi, qi, 2990 port->nb_tx_desc[qi], 2991 txring_numa[pi], 2992 &(port->txq[qi].conf)); 2993 else 2994 diag = rte_eth_tx_queue_setup(pi, qi, 2995 port->nb_tx_desc[qi], 2996 port->socket_id, 2997 &(port->txq[qi].conf)); 2998 2999 if (diag == 0) { 3000 port->txq[qi].state = 3001 conf->tx_deferred_start ? 3002 RTE_ETH_QUEUE_STATE_STOPPED : 3003 RTE_ETH_QUEUE_STATE_STARTED; 3004 continue; 3005 } 3006 3007 /* Fail to setup tx queue, return */ 3008 if (port->port_status == RTE_PORT_HANDLING) 3009 port->port_status = RTE_PORT_STOPPED; 3010 else 3011 fprintf(stderr, 3012 "Port %d can not be set back to stopped\n", 3013 pi); 3014 fprintf(stderr, 3015 "Fail to configure port %d tx queues\n", 3016 pi); 3017 /* try to reconfigure queues next time */ 3018 port->need_reconfig_queues = 1; 3019 return -1; 3020 } 3021 for (qi = 0; qi < nb_rxq; qi++) { 3022 /* setup rx queues */ 3023 if ((numa_support) && 3024 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3025 struct rte_mempool * mp = 3026 mbuf_pool_find 3027 (rxring_numa[pi], 0); 3028 if (mp == NULL) { 3029 fprintf(stderr, 3030 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3031 rxring_numa[pi]); 3032 return -1; 3033 } 3034 3035 diag = rx_queue_setup(pi, qi, 3036 port->nb_rx_desc[qi], 3037 rxring_numa[pi], 3038 &(port->rxq[qi].conf), 3039 mp); 3040 } else { 3041 struct rte_mempool *mp = 3042 mbuf_pool_find 3043 (port->socket_id, 0); 3044 if (mp == NULL) { 3045 fprintf(stderr, 3046 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3047 port->socket_id); 3048 return -1; 3049 } 3050 diag = rx_queue_setup(pi, qi, 3051 port->nb_rx_desc[qi], 3052 port->socket_id, 3053 &(port->rxq[qi].conf), 3054 mp); 3055 } 3056 if (diag == 0) 3057 continue; 3058 3059 /* Fail to setup rx queue, return */ 3060 if (port->port_status == RTE_PORT_HANDLING) 3061 port->port_status = RTE_PORT_STOPPED; 3062 else 3063 fprintf(stderr, 3064 "Port %d can not be set back to stopped\n", 3065 pi); 3066 fprintf(stderr, 3067 "Fail to configure port %d rx queues\n", 3068 pi); 3069 /* try to reconfigure queues next time */ 3070 port->need_reconfig_queues = 1; 3071 return -1; 3072 } 3073 /* setup hairpin queues */ 3074 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3075 return -1; 3076 } 3077 configure_rxtx_dump_callbacks(verbose_level); 3078 if (clear_ptypes) { 3079 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3080 NULL, 0); 3081 if (diag < 0) 3082 fprintf(stderr, 3083 "Port %d: Failed to disable Ptype parsing\n", 3084 pi); 3085 } 3086 3087 p_pi = pi; 3088 cnt_pi++; 3089 3090 /* start port */ 3091 diag = eth_dev_start_mp(pi); 3092 if (diag < 0) { 3093 fprintf(stderr, "Fail to start port %d: %s\n", 3094 pi, rte_strerror(-diag)); 3095 3096 /* Fail to setup rx queue, return */ 3097 if (port->port_status == RTE_PORT_HANDLING) 3098 port->port_status = RTE_PORT_STOPPED; 3099 else 3100 fprintf(stderr, 3101 "Port %d can not be set back to stopped\n", 3102 pi); 3103 continue; 3104 } 3105 3106 if (port->port_status == RTE_PORT_HANDLING) 3107 port->port_status = RTE_PORT_STARTED; 3108 else 3109 fprintf(stderr, "Port %d can not be set into started\n", 3110 pi); 3111 3112 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3113 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3114 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3115 3116 /* at least one port started, need checking link status */ 3117 need_check_link_status = 1; 3118 3119 pl[cfg_pi++] = pi; 3120 } 3121 3122 if (need_check_link_status == 1 && !no_link_check) 3123 check_all_ports_link_status(RTE_PORT_ALL); 3124 else if (need_check_link_status == 0) 3125 fprintf(stderr, "Please stop the ports first\n"); 3126 3127 if (hairpin_mode & 0xf) { 3128 uint16_t i; 3129 int j; 3130 3131 /* bind all started hairpin ports */ 3132 for (i = 0; i < cfg_pi; i++) { 3133 pi = pl[i]; 3134 /* bind current Tx to all peer Rx */ 3135 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3136 RTE_MAX_ETHPORTS, 1); 3137 if (peer_pi < 0) 3138 return peer_pi; 3139 for (j = 0; j < peer_pi; j++) { 3140 if (!port_is_started(peer_pl[j])) 3141 continue; 3142 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3143 if (diag < 0) { 3144 fprintf(stderr, 3145 "Error during binding hairpin Tx port %u to %u: %s\n", 3146 pi, peer_pl[j], 3147 rte_strerror(-diag)); 3148 return -1; 3149 } 3150 } 3151 /* bind all peer Tx to current Rx */ 3152 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3153 RTE_MAX_ETHPORTS, 0); 3154 if (peer_pi < 0) 3155 return peer_pi; 3156 for (j = 0; j < peer_pi; j++) { 3157 if (!port_is_started(peer_pl[j])) 3158 continue; 3159 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3160 if (diag < 0) { 3161 fprintf(stderr, 3162 "Error during binding hairpin Tx port %u to %u: %s\n", 3163 peer_pl[j], pi, 3164 rte_strerror(-diag)); 3165 return -1; 3166 } 3167 } 3168 } 3169 } 3170 3171 fill_xstats_display_info_for_port(pid); 3172 3173 printf("Done\n"); 3174 return 0; 3175 } 3176 3177 void 3178 stop_port(portid_t pid) 3179 { 3180 portid_t pi; 3181 struct rte_port *port; 3182 int need_check_link_status = 0; 3183 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3184 int peer_pi; 3185 int ret; 3186 3187 if (port_id_is_invalid(pid, ENABLED_WARN)) 3188 return; 3189 3190 printf("Stopping ports...\n"); 3191 3192 RTE_ETH_FOREACH_DEV(pi) { 3193 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3194 continue; 3195 3196 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3197 fprintf(stderr, 3198 "Please remove port %d from forwarding configuration.\n", 3199 pi); 3200 continue; 3201 } 3202 3203 if (port_is_bonding_slave(pi)) { 3204 fprintf(stderr, 3205 "Please remove port %d from bonded device.\n", 3206 pi); 3207 continue; 3208 } 3209 3210 port = &ports[pi]; 3211 if (port->port_status == RTE_PORT_STARTED) 3212 port->port_status = RTE_PORT_HANDLING; 3213 else 3214 continue; 3215 3216 if (hairpin_mode & 0xf) { 3217 int j; 3218 3219 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3220 /* unbind all peer Tx from current Rx */ 3221 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3222 RTE_MAX_ETHPORTS, 0); 3223 if (peer_pi < 0) 3224 continue; 3225 for (j = 0; j < peer_pi; j++) { 3226 if (!port_is_started(peer_pl[j])) 3227 continue; 3228 rte_eth_hairpin_unbind(peer_pl[j], pi); 3229 } 3230 } 3231 3232 if (port->flow_list) 3233 port_flow_flush(pi); 3234 3235 ret = eth_dev_stop_mp(pi); 3236 if (ret != 0) { 3237 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3238 pi); 3239 /* Allow to retry stopping the port. */ 3240 port->port_status = RTE_PORT_STARTED; 3241 continue; 3242 } 3243 3244 if (port->port_status == RTE_PORT_HANDLING) 3245 port->port_status = RTE_PORT_STOPPED; 3246 else 3247 fprintf(stderr, "Port %d can not be set into stopped\n", 3248 pi); 3249 need_check_link_status = 1; 3250 } 3251 if (need_check_link_status && !no_link_check) 3252 check_all_ports_link_status(RTE_PORT_ALL); 3253 3254 printf("Done\n"); 3255 } 3256 3257 static void 3258 remove_invalid_ports_in(portid_t *array, portid_t *total) 3259 { 3260 portid_t i; 3261 portid_t new_total = 0; 3262 3263 for (i = 0; i < *total; i++) 3264 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3265 array[new_total] = array[i]; 3266 new_total++; 3267 } 3268 *total = new_total; 3269 } 3270 3271 static void 3272 remove_invalid_ports(void) 3273 { 3274 remove_invalid_ports_in(ports_ids, &nb_ports); 3275 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3276 nb_cfg_ports = nb_fwd_ports; 3277 } 3278 3279 static void 3280 flush_port_owned_resources(portid_t pi) 3281 { 3282 mcast_addr_pool_destroy(pi); 3283 port_flow_flush(pi); 3284 port_flex_item_flush(pi); 3285 port_flow_template_table_flush(pi); 3286 port_flow_pattern_template_flush(pi); 3287 port_flow_actions_template_flush(pi); 3288 port_action_handle_flush(pi); 3289 } 3290 3291 static void 3292 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3293 { 3294 struct rte_port *port; 3295 portid_t slave_pid; 3296 uint16_t i; 3297 3298 for (i = 0; i < num_slaves; i++) { 3299 slave_pid = slave_pids[i]; 3300 if (port_is_started(slave_pid) == 1) { 3301 if (rte_eth_dev_stop(slave_pid) != 0) 3302 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3303 slave_pid); 3304 3305 port = &ports[slave_pid]; 3306 port->port_status = RTE_PORT_STOPPED; 3307 } 3308 3309 clear_port_slave_flag(slave_pid); 3310 3311 /* Close slave device when testpmd quit or is killed. */ 3312 if (cl_quit == 1 || f_quit == 1) 3313 rte_eth_dev_close(slave_pid); 3314 } 3315 } 3316 3317 void 3318 close_port(portid_t pid) 3319 { 3320 portid_t pi; 3321 struct rte_port *port; 3322 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3323 int num_slaves = 0; 3324 3325 if (port_id_is_invalid(pid, ENABLED_WARN)) 3326 return; 3327 3328 printf("Closing ports...\n"); 3329 3330 RTE_ETH_FOREACH_DEV(pi) { 3331 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3332 continue; 3333 3334 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3335 fprintf(stderr, 3336 "Please remove port %d from forwarding configuration.\n", 3337 pi); 3338 continue; 3339 } 3340 3341 if (port_is_bonding_slave(pi)) { 3342 fprintf(stderr, 3343 "Please remove port %d from bonded device.\n", 3344 pi); 3345 continue; 3346 } 3347 3348 port = &ports[pi]; 3349 if (port->port_status == RTE_PORT_CLOSED) { 3350 fprintf(stderr, "Port %d is already closed\n", pi); 3351 continue; 3352 } 3353 3354 if (is_proc_primary()) { 3355 flush_port_owned_resources(pi); 3356 #ifdef RTE_NET_BOND 3357 if (port->bond_flag == 1) 3358 num_slaves = rte_eth_bond_slaves_get(pi, 3359 slave_pids, RTE_MAX_ETHPORTS); 3360 #endif 3361 rte_eth_dev_close(pi); 3362 /* 3363 * If this port is bonded device, all slaves under the 3364 * device need to be removed or closed. 3365 */ 3366 if (port->bond_flag == 1 && num_slaves > 0) 3367 clear_bonding_slave_device(slave_pids, 3368 num_slaves); 3369 } 3370 3371 free_xstats_display_info(pi); 3372 } 3373 3374 remove_invalid_ports(); 3375 printf("Done\n"); 3376 } 3377 3378 void 3379 reset_port(portid_t pid) 3380 { 3381 int diag; 3382 portid_t pi; 3383 struct rte_port *port; 3384 3385 if (port_id_is_invalid(pid, ENABLED_WARN)) 3386 return; 3387 3388 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3389 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3390 fprintf(stderr, 3391 "Can not reset port(s), please stop port(s) first.\n"); 3392 return; 3393 } 3394 3395 printf("Resetting ports...\n"); 3396 3397 RTE_ETH_FOREACH_DEV(pi) { 3398 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3399 continue; 3400 3401 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3402 fprintf(stderr, 3403 "Please remove port %d from forwarding configuration.\n", 3404 pi); 3405 continue; 3406 } 3407 3408 if (port_is_bonding_slave(pi)) { 3409 fprintf(stderr, 3410 "Please remove port %d from bonded device.\n", 3411 pi); 3412 continue; 3413 } 3414 3415 if (is_proc_primary()) { 3416 diag = rte_eth_dev_reset(pi); 3417 if (diag == 0) { 3418 port = &ports[pi]; 3419 port->need_reconfig = 1; 3420 port->need_reconfig_queues = 1; 3421 } else { 3422 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3423 pi, diag); 3424 } 3425 } 3426 } 3427 3428 printf("Done\n"); 3429 } 3430 3431 void 3432 attach_port(char *identifier) 3433 { 3434 portid_t pi; 3435 struct rte_dev_iterator iterator; 3436 3437 printf("Attaching a new port...\n"); 3438 3439 if (identifier == NULL) { 3440 fprintf(stderr, "Invalid parameters are specified\n"); 3441 return; 3442 } 3443 3444 if (rte_dev_probe(identifier) < 0) { 3445 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3446 return; 3447 } 3448 3449 /* first attach mode: event */ 3450 if (setup_on_probe_event) { 3451 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3452 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3453 if (ports[pi].port_status == RTE_PORT_HANDLING && 3454 ports[pi].need_setup != 0) 3455 setup_attached_port(pi); 3456 return; 3457 } 3458 3459 /* second attach mode: iterator */ 3460 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3461 /* setup ports matching the devargs used for probing */ 3462 if (port_is_forwarding(pi)) 3463 continue; /* port was already attached before */ 3464 setup_attached_port(pi); 3465 } 3466 } 3467 3468 static void 3469 setup_attached_port(portid_t pi) 3470 { 3471 unsigned int socket_id; 3472 int ret; 3473 3474 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3475 /* if socket_id is invalid, set to the first available socket. */ 3476 if (check_socket_id(socket_id) < 0) 3477 socket_id = socket_ids[0]; 3478 reconfig(pi, socket_id); 3479 ret = rte_eth_promiscuous_enable(pi); 3480 if (ret != 0) 3481 fprintf(stderr, 3482 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3483 pi, rte_strerror(-ret)); 3484 3485 ports_ids[nb_ports++] = pi; 3486 fwd_ports_ids[nb_fwd_ports++] = pi; 3487 nb_cfg_ports = nb_fwd_ports; 3488 ports[pi].need_setup = 0; 3489 ports[pi].port_status = RTE_PORT_STOPPED; 3490 3491 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3492 printf("Done\n"); 3493 } 3494 3495 static void 3496 detach_device(struct rte_device *dev) 3497 { 3498 portid_t sibling; 3499 3500 if (dev == NULL) { 3501 fprintf(stderr, "Device already removed\n"); 3502 return; 3503 } 3504 3505 printf("Removing a device...\n"); 3506 3507 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3508 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3509 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3510 fprintf(stderr, "Port %u not stopped\n", 3511 sibling); 3512 return; 3513 } 3514 flush_port_owned_resources(sibling); 3515 } 3516 } 3517 3518 if (rte_dev_remove(dev) < 0) { 3519 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3520 return; 3521 } 3522 remove_invalid_ports(); 3523 3524 printf("Device is detached\n"); 3525 printf("Now total ports is %d\n", nb_ports); 3526 printf("Done\n"); 3527 return; 3528 } 3529 3530 void 3531 detach_port_device(portid_t port_id) 3532 { 3533 int ret; 3534 struct rte_eth_dev_info dev_info; 3535 3536 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3537 return; 3538 3539 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3540 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3541 fprintf(stderr, "Port not stopped\n"); 3542 return; 3543 } 3544 fprintf(stderr, "Port was not closed\n"); 3545 } 3546 3547 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3548 if (ret != 0) { 3549 TESTPMD_LOG(ERR, 3550 "Failed to get device info for port %d, not detaching\n", 3551 port_id); 3552 return; 3553 } 3554 detach_device(dev_info.device); 3555 } 3556 3557 void 3558 detach_devargs(char *identifier) 3559 { 3560 struct rte_dev_iterator iterator; 3561 struct rte_devargs da; 3562 portid_t port_id; 3563 3564 printf("Removing a device...\n"); 3565 3566 memset(&da, 0, sizeof(da)); 3567 if (rte_devargs_parsef(&da, "%s", identifier)) { 3568 fprintf(stderr, "cannot parse identifier\n"); 3569 return; 3570 } 3571 3572 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3573 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3574 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3575 fprintf(stderr, "Port %u not stopped\n", 3576 port_id); 3577 rte_eth_iterator_cleanup(&iterator); 3578 rte_devargs_reset(&da); 3579 return; 3580 } 3581 flush_port_owned_resources(port_id); 3582 } 3583 } 3584 3585 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3586 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3587 da.name, rte_bus_name(da.bus)); 3588 rte_devargs_reset(&da); 3589 return; 3590 } 3591 3592 remove_invalid_ports(); 3593 3594 printf("Device %s is detached\n", identifier); 3595 printf("Now total ports is %d\n", nb_ports); 3596 printf("Done\n"); 3597 rte_devargs_reset(&da); 3598 } 3599 3600 void 3601 pmd_test_exit(void) 3602 { 3603 portid_t pt_id; 3604 unsigned int i; 3605 int ret; 3606 3607 if (test_done == 0) 3608 stop_packet_forwarding(); 3609 3610 #ifndef RTE_EXEC_ENV_WINDOWS 3611 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3612 if (mempools[i]) { 3613 if (mp_alloc_type == MP_ALLOC_ANON) 3614 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3615 NULL); 3616 } 3617 } 3618 #endif 3619 if (ports != NULL) { 3620 no_link_check = 1; 3621 RTE_ETH_FOREACH_DEV(pt_id) { 3622 printf("\nStopping port %d...\n", pt_id); 3623 fflush(stdout); 3624 stop_port(pt_id); 3625 } 3626 RTE_ETH_FOREACH_DEV(pt_id) { 3627 printf("\nShutting down port %d...\n", pt_id); 3628 fflush(stdout); 3629 close_port(pt_id); 3630 } 3631 } 3632 3633 if (hot_plug) { 3634 ret = rte_dev_event_monitor_stop(); 3635 if (ret) { 3636 RTE_LOG(ERR, EAL, 3637 "fail to stop device event monitor."); 3638 return; 3639 } 3640 3641 ret = rte_dev_event_callback_unregister(NULL, 3642 dev_event_callback, NULL); 3643 if (ret < 0) { 3644 RTE_LOG(ERR, EAL, 3645 "fail to unregister device event callback.\n"); 3646 return; 3647 } 3648 3649 ret = rte_dev_hotplug_handle_disable(); 3650 if (ret) { 3651 RTE_LOG(ERR, EAL, 3652 "fail to disable hotplug handling.\n"); 3653 return; 3654 } 3655 } 3656 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3657 if (mempools[i]) 3658 mempool_free_mp(mempools[i]); 3659 } 3660 free(xstats_display); 3661 3662 printf("\nBye...\n"); 3663 } 3664 3665 typedef void (*cmd_func_t)(void); 3666 struct pmd_test_command { 3667 const char *cmd_name; 3668 cmd_func_t cmd_func; 3669 }; 3670 3671 /* Check the link status of all ports in up to 9s, and print them finally */ 3672 static void 3673 check_all_ports_link_status(uint32_t port_mask) 3674 { 3675 #define CHECK_INTERVAL 100 /* 100ms */ 3676 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3677 portid_t portid; 3678 uint8_t count, all_ports_up, print_flag = 0; 3679 struct rte_eth_link link; 3680 int ret; 3681 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3682 3683 printf("Checking link statuses...\n"); 3684 fflush(stdout); 3685 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3686 all_ports_up = 1; 3687 RTE_ETH_FOREACH_DEV(portid) { 3688 if ((port_mask & (1 << portid)) == 0) 3689 continue; 3690 memset(&link, 0, sizeof(link)); 3691 ret = rte_eth_link_get_nowait(portid, &link); 3692 if (ret < 0) { 3693 all_ports_up = 0; 3694 if (print_flag == 1) 3695 fprintf(stderr, 3696 "Port %u link get failed: %s\n", 3697 portid, rte_strerror(-ret)); 3698 continue; 3699 } 3700 /* print link status if flag set */ 3701 if (print_flag == 1) { 3702 rte_eth_link_to_str(link_status, 3703 sizeof(link_status), &link); 3704 printf("Port %d %s\n", portid, link_status); 3705 continue; 3706 } 3707 /* clear all_ports_up flag if any link down */ 3708 if (link.link_status == RTE_ETH_LINK_DOWN) { 3709 all_ports_up = 0; 3710 break; 3711 } 3712 } 3713 /* after finally printing all link status, get out */ 3714 if (print_flag == 1) 3715 break; 3716 3717 if (all_ports_up == 0) { 3718 fflush(stdout); 3719 rte_delay_ms(CHECK_INTERVAL); 3720 } 3721 3722 /* set the print_flag if all ports up or timeout */ 3723 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3724 print_flag = 1; 3725 } 3726 3727 if (lsc_interrupt) 3728 break; 3729 } 3730 } 3731 3732 static void 3733 rmv_port_callback(void *arg) 3734 { 3735 int need_to_start = 0; 3736 int org_no_link_check = no_link_check; 3737 portid_t port_id = (intptr_t)arg; 3738 struct rte_eth_dev_info dev_info; 3739 int ret; 3740 3741 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3742 3743 if (!test_done && port_is_forwarding(port_id)) { 3744 need_to_start = 1; 3745 stop_packet_forwarding(); 3746 } 3747 no_link_check = 1; 3748 stop_port(port_id); 3749 no_link_check = org_no_link_check; 3750 3751 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3752 if (ret != 0) 3753 TESTPMD_LOG(ERR, 3754 "Failed to get device info for port %d, not detaching\n", 3755 port_id); 3756 else { 3757 struct rte_device *device = dev_info.device; 3758 close_port(port_id); 3759 detach_device(device); /* might be already removed or have more ports */ 3760 } 3761 if (need_to_start) 3762 start_packet_forwarding(0); 3763 } 3764 3765 /* This function is used by the interrupt thread */ 3766 static int 3767 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3768 void *ret_param) 3769 { 3770 RTE_SET_USED(param); 3771 RTE_SET_USED(ret_param); 3772 3773 if (type >= RTE_ETH_EVENT_MAX) { 3774 fprintf(stderr, 3775 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3776 port_id, __func__, type); 3777 fflush(stderr); 3778 } else if (event_print_mask & (UINT32_C(1) << type)) { 3779 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3780 eth_event_desc[type]); 3781 fflush(stdout); 3782 } 3783 3784 switch (type) { 3785 case RTE_ETH_EVENT_NEW: 3786 ports[port_id].need_setup = 1; 3787 ports[port_id].port_status = RTE_PORT_HANDLING; 3788 break; 3789 case RTE_ETH_EVENT_INTR_RMV: 3790 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3791 break; 3792 if (rte_eal_alarm_set(100000, 3793 rmv_port_callback, (void *)(intptr_t)port_id)) 3794 fprintf(stderr, 3795 "Could not set up deferred device removal\n"); 3796 break; 3797 case RTE_ETH_EVENT_DESTROY: 3798 ports[port_id].port_status = RTE_PORT_CLOSED; 3799 printf("Port %u is closed\n", port_id); 3800 break; 3801 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3802 uint16_t rxq_id; 3803 int ret; 3804 3805 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3806 for (rxq_id = 0; ; rxq_id++) { 3807 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3808 NULL); 3809 if (ret <= 0) 3810 break; 3811 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3812 port_id, rxq_id); 3813 3814 #ifdef RTE_NET_MLX5 3815 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3816 #endif 3817 } 3818 break; 3819 } 3820 default: 3821 break; 3822 } 3823 return 0; 3824 } 3825 3826 static int 3827 register_eth_event_callback(void) 3828 { 3829 int ret; 3830 enum rte_eth_event_type event; 3831 3832 for (event = RTE_ETH_EVENT_UNKNOWN; 3833 event < RTE_ETH_EVENT_MAX; event++) { 3834 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3835 event, 3836 eth_event_callback, 3837 NULL); 3838 if (ret != 0) { 3839 TESTPMD_LOG(ERR, "Failed to register callback for " 3840 "%s event\n", eth_event_desc[event]); 3841 return -1; 3842 } 3843 } 3844 3845 return 0; 3846 } 3847 3848 /* This function is used by the interrupt thread */ 3849 static void 3850 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3851 __rte_unused void *arg) 3852 { 3853 uint16_t port_id; 3854 int ret; 3855 3856 if (type >= RTE_DEV_EVENT_MAX) { 3857 fprintf(stderr, "%s called upon invalid event %d\n", 3858 __func__, type); 3859 fflush(stderr); 3860 } 3861 3862 switch (type) { 3863 case RTE_DEV_EVENT_REMOVE: 3864 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3865 device_name); 3866 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3867 if (ret) { 3868 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3869 device_name); 3870 return; 3871 } 3872 /* 3873 * Because the user's callback is invoked in eal interrupt 3874 * callback, the interrupt callback need to be finished before 3875 * it can be unregistered when detaching device. So finish 3876 * callback soon and use a deferred removal to detach device 3877 * is need. It is a workaround, once the device detaching be 3878 * moved into the eal in the future, the deferred removal could 3879 * be deleted. 3880 */ 3881 if (rte_eal_alarm_set(100000, 3882 rmv_port_callback, (void *)(intptr_t)port_id)) 3883 RTE_LOG(ERR, EAL, 3884 "Could not set up deferred device removal\n"); 3885 break; 3886 case RTE_DEV_EVENT_ADD: 3887 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3888 device_name); 3889 /* TODO: After finish kernel driver binding, 3890 * begin to attach port. 3891 */ 3892 break; 3893 default: 3894 break; 3895 } 3896 } 3897 3898 static void 3899 rxtx_port_config(portid_t pid) 3900 { 3901 uint16_t qid; 3902 uint64_t offloads; 3903 struct rte_port *port = &ports[pid]; 3904 3905 for (qid = 0; qid < nb_rxq; qid++) { 3906 offloads = port->rxq[qid].conf.offloads; 3907 port->rxq[qid].conf = port->dev_info.default_rxconf; 3908 3909 if (rxq_share > 0 && 3910 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3911 /* Non-zero share group to enable RxQ share. */ 3912 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3913 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3914 } 3915 3916 if (offloads != 0) 3917 port->rxq[qid].conf.offloads = offloads; 3918 3919 /* Check if any Rx parameters have been passed */ 3920 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3921 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3922 3923 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3924 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3925 3926 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3927 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3928 3929 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3930 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3931 3932 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3933 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3934 3935 port->nb_rx_desc[qid] = nb_rxd; 3936 } 3937 3938 for (qid = 0; qid < nb_txq; qid++) { 3939 offloads = port->txq[qid].conf.offloads; 3940 port->txq[qid].conf = port->dev_info.default_txconf; 3941 if (offloads != 0) 3942 port->txq[qid].conf.offloads = offloads; 3943 3944 /* Check if any Tx parameters have been passed */ 3945 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3946 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3947 3948 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3949 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3950 3951 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3952 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3953 3954 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3955 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3956 3957 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3958 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3959 3960 port->nb_tx_desc[qid] = nb_txd; 3961 } 3962 } 3963 3964 /* 3965 * Helper function to set MTU from frame size 3966 * 3967 * port->dev_info should be set before calling this function. 3968 * 3969 * return 0 on success, negative on error 3970 */ 3971 int 3972 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3973 { 3974 struct rte_port *port = &ports[portid]; 3975 uint32_t eth_overhead; 3976 uint16_t mtu, new_mtu; 3977 3978 eth_overhead = get_eth_overhead(&port->dev_info); 3979 3980 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3981 printf("Failed to get MTU for port %u\n", portid); 3982 return -1; 3983 } 3984 3985 new_mtu = max_rx_pktlen - eth_overhead; 3986 3987 if (mtu == new_mtu) 3988 return 0; 3989 3990 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3991 fprintf(stderr, 3992 "Failed to set MTU to %u for port %u\n", 3993 new_mtu, portid); 3994 return -1; 3995 } 3996 3997 port->dev_conf.rxmode.mtu = new_mtu; 3998 3999 return 0; 4000 } 4001 4002 void 4003 init_port_config(void) 4004 { 4005 portid_t pid; 4006 struct rte_port *port; 4007 int ret, i; 4008 4009 RTE_ETH_FOREACH_DEV(pid) { 4010 port = &ports[pid]; 4011 4012 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4013 if (ret != 0) 4014 return; 4015 4016 if (nb_rxq > 1) { 4017 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4018 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4019 rss_hf & port->dev_info.flow_type_rss_offloads; 4020 } else { 4021 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4022 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4023 } 4024 4025 if (port->dcb_flag == 0) { 4026 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4027 port->dev_conf.rxmode.mq_mode = 4028 (enum rte_eth_rx_mq_mode) 4029 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4030 } else { 4031 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4032 port->dev_conf.rxmode.offloads &= 4033 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4034 4035 for (i = 0; 4036 i < port->dev_info.nb_rx_queues; 4037 i++) 4038 port->rxq[i].conf.offloads &= 4039 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4040 } 4041 } 4042 4043 rxtx_port_config(pid); 4044 4045 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4046 if (ret != 0) 4047 return; 4048 4049 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4050 port->dev_conf.intr_conf.lsc = 1; 4051 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4052 port->dev_conf.intr_conf.rmv = 1; 4053 } 4054 } 4055 4056 void set_port_slave_flag(portid_t slave_pid) 4057 { 4058 struct rte_port *port; 4059 4060 port = &ports[slave_pid]; 4061 port->slave_flag = 1; 4062 } 4063 4064 void clear_port_slave_flag(portid_t slave_pid) 4065 { 4066 struct rte_port *port; 4067 4068 port = &ports[slave_pid]; 4069 port->slave_flag = 0; 4070 } 4071 4072 uint8_t port_is_bonding_slave(portid_t slave_pid) 4073 { 4074 struct rte_port *port; 4075 struct rte_eth_dev_info dev_info; 4076 int ret; 4077 4078 port = &ports[slave_pid]; 4079 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4080 if (ret != 0) { 4081 TESTPMD_LOG(ERR, 4082 "Failed to get device info for port id %d," 4083 "cannot determine if the port is a bonded slave", 4084 slave_pid); 4085 return 0; 4086 } 4087 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4088 return 1; 4089 return 0; 4090 } 4091 4092 const uint16_t vlan_tags[] = { 4093 0, 1, 2, 3, 4, 5, 6, 7, 4094 8, 9, 10, 11, 12, 13, 14, 15, 4095 16, 17, 18, 19, 20, 21, 22, 23, 4096 24, 25, 26, 27, 28, 29, 30, 31 4097 }; 4098 4099 static int 4100 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4101 enum dcb_mode_enable dcb_mode, 4102 enum rte_eth_nb_tcs num_tcs, 4103 uint8_t pfc_en) 4104 { 4105 uint8_t i; 4106 int32_t rc; 4107 struct rte_eth_rss_conf rss_conf; 4108 4109 /* 4110 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4111 * given above, and the number of traffic classes available for use. 4112 */ 4113 if (dcb_mode == DCB_VT_ENABLED) { 4114 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4115 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4116 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4117 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4118 4119 /* VMDQ+DCB RX and TX configurations */ 4120 vmdq_rx_conf->enable_default_pool = 0; 4121 vmdq_rx_conf->default_pool = 0; 4122 vmdq_rx_conf->nb_queue_pools = 4123 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4124 vmdq_tx_conf->nb_queue_pools = 4125 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4126 4127 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4128 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4129 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4130 vmdq_rx_conf->pool_map[i].pools = 4131 1 << (i % vmdq_rx_conf->nb_queue_pools); 4132 } 4133 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4134 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4135 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4136 } 4137 4138 /* set DCB mode of RX and TX of multiple queues */ 4139 eth_conf->rxmode.mq_mode = 4140 (enum rte_eth_rx_mq_mode) 4141 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4142 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4143 } else { 4144 struct rte_eth_dcb_rx_conf *rx_conf = 4145 ð_conf->rx_adv_conf.dcb_rx_conf; 4146 struct rte_eth_dcb_tx_conf *tx_conf = 4147 ð_conf->tx_adv_conf.dcb_tx_conf; 4148 4149 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4150 4151 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4152 if (rc != 0) 4153 return rc; 4154 4155 rx_conf->nb_tcs = num_tcs; 4156 tx_conf->nb_tcs = num_tcs; 4157 4158 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4159 rx_conf->dcb_tc[i] = i % num_tcs; 4160 tx_conf->dcb_tc[i] = i % num_tcs; 4161 } 4162 4163 eth_conf->rxmode.mq_mode = 4164 (enum rte_eth_rx_mq_mode) 4165 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4166 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4167 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4168 } 4169 4170 if (pfc_en) 4171 eth_conf->dcb_capability_en = 4172 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4173 else 4174 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4175 4176 return 0; 4177 } 4178 4179 int 4180 init_port_dcb_config(portid_t pid, 4181 enum dcb_mode_enable dcb_mode, 4182 enum rte_eth_nb_tcs num_tcs, 4183 uint8_t pfc_en) 4184 { 4185 struct rte_eth_conf port_conf; 4186 struct rte_port *rte_port; 4187 int retval; 4188 uint16_t i; 4189 4190 if (num_procs > 1) { 4191 printf("The multi-process feature doesn't support dcb.\n"); 4192 return -ENOTSUP; 4193 } 4194 rte_port = &ports[pid]; 4195 4196 /* retain the original device configuration. */ 4197 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4198 4199 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4200 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4201 if (retval < 0) 4202 return retval; 4203 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4204 /* remove RSS HASH offload for DCB in vt mode */ 4205 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4206 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4207 for (i = 0; i < nb_rxq; i++) 4208 rte_port->rxq[i].conf.offloads &= 4209 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4210 } 4211 4212 /* re-configure the device . */ 4213 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4214 if (retval < 0) 4215 return retval; 4216 4217 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4218 if (retval != 0) 4219 return retval; 4220 4221 /* If dev_info.vmdq_pool_base is greater than 0, 4222 * the queue id of vmdq pools is started after pf queues. 4223 */ 4224 if (dcb_mode == DCB_VT_ENABLED && 4225 rte_port->dev_info.vmdq_pool_base > 0) { 4226 fprintf(stderr, 4227 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4228 pid); 4229 return -1; 4230 } 4231 4232 /* Assume the ports in testpmd have the same dcb capability 4233 * and has the same number of rxq and txq in dcb mode 4234 */ 4235 if (dcb_mode == DCB_VT_ENABLED) { 4236 if (rte_port->dev_info.max_vfs > 0) { 4237 nb_rxq = rte_port->dev_info.nb_rx_queues; 4238 nb_txq = rte_port->dev_info.nb_tx_queues; 4239 } else { 4240 nb_rxq = rte_port->dev_info.max_rx_queues; 4241 nb_txq = rte_port->dev_info.max_tx_queues; 4242 } 4243 } else { 4244 /*if vt is disabled, use all pf queues */ 4245 if (rte_port->dev_info.vmdq_pool_base == 0) { 4246 nb_rxq = rte_port->dev_info.max_rx_queues; 4247 nb_txq = rte_port->dev_info.max_tx_queues; 4248 } else { 4249 nb_rxq = (queueid_t)num_tcs; 4250 nb_txq = (queueid_t)num_tcs; 4251 4252 } 4253 } 4254 rx_free_thresh = 64; 4255 4256 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4257 4258 rxtx_port_config(pid); 4259 /* VLAN filter */ 4260 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4261 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4262 rx_vft_set(pid, vlan_tags[i], 1); 4263 4264 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4265 if (retval != 0) 4266 return retval; 4267 4268 rte_port->dcb_flag = 1; 4269 4270 /* Enter DCB configuration status */ 4271 dcb_config = 1; 4272 4273 return 0; 4274 } 4275 4276 static void 4277 init_port(void) 4278 { 4279 int i; 4280 4281 /* Configuration of Ethernet ports. */ 4282 ports = rte_zmalloc("testpmd: ports", 4283 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4284 RTE_CACHE_LINE_SIZE); 4285 if (ports == NULL) { 4286 rte_exit(EXIT_FAILURE, 4287 "rte_zmalloc(%d struct rte_port) failed\n", 4288 RTE_MAX_ETHPORTS); 4289 } 4290 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4291 ports[i].fwd_mac_swap = 1; 4292 ports[i].xstats_info.allocated = false; 4293 LIST_INIT(&ports[i].flow_tunnel_list); 4294 } 4295 /* Initialize ports NUMA structures */ 4296 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4297 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4298 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4299 } 4300 4301 static void 4302 force_quit(void) 4303 { 4304 pmd_test_exit(); 4305 prompt_exit(); 4306 } 4307 4308 static void 4309 print_stats(void) 4310 { 4311 uint8_t i; 4312 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4313 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4314 4315 /* Clear screen and move to top left */ 4316 printf("%s%s", clr, top_left); 4317 4318 printf("\nPort statistics ===================================="); 4319 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4320 nic_stats_display(fwd_ports_ids[i]); 4321 4322 fflush(stdout); 4323 } 4324 4325 static void 4326 signal_handler(int signum) 4327 { 4328 if (signum == SIGINT || signum == SIGTERM) { 4329 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4330 signum); 4331 #ifdef RTE_LIB_PDUMP 4332 /* uninitialize packet capture framework */ 4333 rte_pdump_uninit(); 4334 #endif 4335 #ifdef RTE_LIB_LATENCYSTATS 4336 if (latencystats_enabled != 0) 4337 rte_latencystats_uninit(); 4338 #endif 4339 force_quit(); 4340 /* Set flag to indicate the force termination. */ 4341 f_quit = 1; 4342 /* exit with the expected status */ 4343 #ifndef RTE_EXEC_ENV_WINDOWS 4344 signal(signum, SIG_DFL); 4345 kill(getpid(), signum); 4346 #endif 4347 } 4348 } 4349 4350 int 4351 main(int argc, char** argv) 4352 { 4353 int diag; 4354 portid_t port_id; 4355 uint16_t count; 4356 int ret; 4357 4358 signal(SIGINT, signal_handler); 4359 signal(SIGTERM, signal_handler); 4360 4361 testpmd_logtype = rte_log_register("testpmd"); 4362 if (testpmd_logtype < 0) 4363 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4364 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4365 4366 diag = rte_eal_init(argc, argv); 4367 if (diag < 0) 4368 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4369 rte_strerror(rte_errno)); 4370 4371 ret = register_eth_event_callback(); 4372 if (ret != 0) 4373 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4374 4375 #ifdef RTE_LIB_PDUMP 4376 /* initialize packet capture framework */ 4377 rte_pdump_init(); 4378 #endif 4379 4380 count = 0; 4381 RTE_ETH_FOREACH_DEV(port_id) { 4382 ports_ids[count] = port_id; 4383 count++; 4384 } 4385 nb_ports = (portid_t) count; 4386 if (nb_ports == 0) 4387 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4388 4389 /* allocate port structures, and init them */ 4390 init_port(); 4391 4392 set_def_fwd_config(); 4393 if (nb_lcores == 0) 4394 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4395 "Check the core mask argument\n"); 4396 4397 /* Bitrate/latency stats disabled by default */ 4398 #ifdef RTE_LIB_BITRATESTATS 4399 bitrate_enabled = 0; 4400 #endif 4401 #ifdef RTE_LIB_LATENCYSTATS 4402 latencystats_enabled = 0; 4403 #endif 4404 4405 /* on FreeBSD, mlockall() is disabled by default */ 4406 #ifdef RTE_EXEC_ENV_FREEBSD 4407 do_mlockall = 0; 4408 #else 4409 do_mlockall = 1; 4410 #endif 4411 4412 argc -= diag; 4413 argv += diag; 4414 if (argc > 1) 4415 launch_args_parse(argc, argv); 4416 4417 #ifndef RTE_EXEC_ENV_WINDOWS 4418 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4419 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4420 strerror(errno)); 4421 } 4422 #endif 4423 4424 if (tx_first && interactive) 4425 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4426 "interactive mode.\n"); 4427 4428 if (tx_first && lsc_interrupt) { 4429 fprintf(stderr, 4430 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4431 lsc_interrupt = 0; 4432 } 4433 4434 if (!nb_rxq && !nb_txq) 4435 fprintf(stderr, 4436 "Warning: Either rx or tx queues should be non-zero\n"); 4437 4438 if (nb_rxq > 1 && nb_rxq > nb_txq) 4439 fprintf(stderr, 4440 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4441 nb_rxq, nb_txq); 4442 4443 init_config(); 4444 4445 if (hot_plug) { 4446 ret = rte_dev_hotplug_handle_enable(); 4447 if (ret) { 4448 RTE_LOG(ERR, EAL, 4449 "fail to enable hotplug handling."); 4450 return -1; 4451 } 4452 4453 ret = rte_dev_event_monitor_start(); 4454 if (ret) { 4455 RTE_LOG(ERR, EAL, 4456 "fail to start device event monitoring."); 4457 return -1; 4458 } 4459 4460 ret = rte_dev_event_callback_register(NULL, 4461 dev_event_callback, NULL); 4462 if (ret) { 4463 RTE_LOG(ERR, EAL, 4464 "fail to register device event callback\n"); 4465 return -1; 4466 } 4467 } 4468 4469 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4470 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4471 4472 /* set all ports to promiscuous mode by default */ 4473 RTE_ETH_FOREACH_DEV(port_id) { 4474 ret = rte_eth_promiscuous_enable(port_id); 4475 if (ret != 0) 4476 fprintf(stderr, 4477 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4478 port_id, rte_strerror(-ret)); 4479 } 4480 4481 #ifdef RTE_LIB_METRICS 4482 /* Init metrics library */ 4483 rte_metrics_init(rte_socket_id()); 4484 #endif 4485 4486 #ifdef RTE_LIB_LATENCYSTATS 4487 if (latencystats_enabled != 0) { 4488 int ret = rte_latencystats_init(1, NULL); 4489 if (ret) 4490 fprintf(stderr, 4491 "Warning: latencystats init() returned error %d\n", 4492 ret); 4493 fprintf(stderr, "Latencystats running on lcore %d\n", 4494 latencystats_lcore_id); 4495 } 4496 #endif 4497 4498 /* Setup bitrate stats */ 4499 #ifdef RTE_LIB_BITRATESTATS 4500 if (bitrate_enabled != 0) { 4501 bitrate_data = rte_stats_bitrate_create(); 4502 if (bitrate_data == NULL) 4503 rte_exit(EXIT_FAILURE, 4504 "Could not allocate bitrate data.\n"); 4505 rte_stats_bitrate_reg(bitrate_data); 4506 } 4507 #endif 4508 #ifdef RTE_LIB_CMDLINE 4509 if (init_cmdline() != 0) 4510 rte_exit(EXIT_FAILURE, 4511 "Could not initialise cmdline context.\n"); 4512 4513 if (strlen(cmdline_filename) != 0) 4514 cmdline_read_from_file(cmdline_filename); 4515 4516 if (interactive == 1) { 4517 if (auto_start) { 4518 printf("Start automatic packet forwarding\n"); 4519 start_packet_forwarding(0); 4520 } 4521 prompt(); 4522 pmd_test_exit(); 4523 } else 4524 #endif 4525 { 4526 char c; 4527 int rc; 4528 4529 f_quit = 0; 4530 4531 printf("No commandline core given, start packet forwarding\n"); 4532 start_packet_forwarding(tx_first); 4533 if (stats_period != 0) { 4534 uint64_t prev_time = 0, cur_time, diff_time = 0; 4535 uint64_t timer_period; 4536 4537 /* Convert to number of cycles */ 4538 timer_period = stats_period * rte_get_timer_hz(); 4539 4540 while (f_quit == 0) { 4541 cur_time = rte_get_timer_cycles(); 4542 diff_time += cur_time - prev_time; 4543 4544 if (diff_time >= timer_period) { 4545 print_stats(); 4546 /* Reset the timer */ 4547 diff_time = 0; 4548 } 4549 /* Sleep to avoid unnecessary checks */ 4550 prev_time = cur_time; 4551 rte_delay_us_sleep(US_PER_S); 4552 } 4553 } 4554 4555 printf("Press enter to exit\n"); 4556 rc = read(0, &c, 1); 4557 pmd_test_exit(); 4558 if (rc < 0) 4559 return 1; 4560 } 4561 4562 ret = rte_eal_cleanup(); 4563 if (ret != 0) 4564 rte_exit(EXIT_FAILURE, 4565 "EAL cleanup failed: %s\n", strerror(-ret)); 4566 4567 return EXIT_SUCCESS; 4568 } 4569