1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #include <sys/select.h> 15 #endif 16 #include <sys/types.h> 17 #include <errno.h> 18 #include <stdbool.h> 19 20 #include <sys/queue.h> 21 #include <sys/stat.h> 22 23 #include <stdint.h> 24 #include <unistd.h> 25 #include <inttypes.h> 26 27 #include <rte_common.h> 28 #include <rte_errno.h> 29 #include <rte_byteorder.h> 30 #include <rte_log.h> 31 #include <rte_debug.h> 32 #include <rte_cycles.h> 33 #include <rte_memory.h> 34 #include <rte_memcpy.h> 35 #include <rte_launch.h> 36 #include <rte_bus.h> 37 #include <rte_eal.h> 38 #include <rte_alarm.h> 39 #include <rte_per_lcore.h> 40 #include <rte_lcore.h> 41 #include <rte_branch_prediction.h> 42 #include <rte_mempool.h> 43 #include <rte_malloc.h> 44 #include <rte_mbuf.h> 45 #include <rte_mbuf_pool_ops.h> 46 #include <rte_interrupts.h> 47 #include <rte_ether.h> 48 #include <rte_ethdev.h> 49 #include <rte_dev.h> 50 #include <rte_string_fns.h> 51 #ifdef RTE_NET_IXGBE 52 #include <rte_pmd_ixgbe.h> 53 #endif 54 #ifdef RTE_LIB_PDUMP 55 #include <rte_pdump.h> 56 #endif 57 #include <rte_flow.h> 58 #ifdef RTE_LIB_METRICS 59 #include <rte_metrics.h> 60 #endif 61 #ifdef RTE_LIB_BITRATESTATS 62 #include <rte_bitrate.h> 63 #endif 64 #ifdef RTE_LIB_LATENCYSTATS 65 #include <rte_latencystats.h> 66 #endif 67 #ifdef RTE_EXEC_ENV_WINDOWS 68 #include <process.h> 69 #endif 70 #ifdef RTE_NET_BOND 71 #include <rte_eth_bond.h> 72 #endif 73 #ifdef RTE_NET_MLX5 74 #include "mlx5_testpmd.h" 75 #endif 76 77 #include "testpmd.h" 78 79 #ifndef MAP_HUGETLB 80 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 81 #define HUGE_FLAG (0x40000) 82 #else 83 #define HUGE_FLAG MAP_HUGETLB 84 #endif 85 86 #ifndef MAP_HUGE_SHIFT 87 /* older kernels (or FreeBSD) will not have this define */ 88 #define HUGE_SHIFT (26) 89 #else 90 #define HUGE_SHIFT MAP_HUGE_SHIFT 91 #endif 92 93 #define EXTMEM_HEAP_NAME "extmem" 94 /* 95 * Zone size with the malloc overhead (max of debug and release variants) 96 * must fit into the smallest supported hugepage size (2M), 97 * so that an IOVA-contiguous zone of this size can always be allocated 98 * if there are free 2M hugepages. 99 */ 100 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 101 102 uint16_t verbose_level = 0; /**< Silent by default. */ 103 int testpmd_logtype; /**< Log type for testpmd logs */ 104 105 /* use main core for command line ? */ 106 uint8_t interactive = 0; 107 uint8_t auto_start = 0; 108 uint8_t tx_first; 109 char cmdline_filename[PATH_MAX] = {0}; 110 111 /* 112 * NUMA support configuration. 113 * When set, the NUMA support attempts to dispatch the allocation of the 114 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 115 * probed ports among the CPU sockets 0 and 1. 116 * Otherwise, all memory is allocated from CPU socket 0. 117 */ 118 uint8_t numa_support = 1; /**< numa enabled by default */ 119 120 /* 121 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 122 * not configured. 123 */ 124 uint8_t socket_num = UMA_NO_CONFIG; 125 126 /* 127 * Select mempool allocation type: 128 * - native: use regular DPDK memory 129 * - anon: use regular DPDK memory to create mempool, but populate using 130 * anonymous memory (may not be IOVA-contiguous) 131 * - xmem: use externally allocated hugepage memory 132 */ 133 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 134 135 /* 136 * Store specified sockets on which memory pool to be used by ports 137 * is allocated. 138 */ 139 uint8_t port_numa[RTE_MAX_ETHPORTS]; 140 141 /* 142 * Store specified sockets on which RX ring to be used by ports 143 * is allocated. 144 */ 145 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 146 147 /* 148 * Store specified sockets on which TX ring to be used by ports 149 * is allocated. 150 */ 151 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 152 153 /* 154 * Record the Ethernet address of peer target ports to which packets are 155 * forwarded. 156 * Must be instantiated with the ethernet addresses of peer traffic generator 157 * ports. 158 */ 159 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 160 portid_t nb_peer_eth_addrs = 0; 161 162 /* 163 * Probed Target Environment. 164 */ 165 struct rte_port *ports; /**< For all probed ethernet ports. */ 166 portid_t nb_ports; /**< Number of probed ethernet ports. */ 167 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 168 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 169 170 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 171 172 /* 173 * Test Forwarding Configuration. 174 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 175 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 176 */ 177 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 178 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 179 portid_t nb_cfg_ports; /**< Number of configured ports. */ 180 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 181 182 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 183 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 184 185 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 186 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 187 188 /* 189 * Forwarding engines. 190 */ 191 struct fwd_engine * fwd_engines[] = { 192 &io_fwd_engine, 193 &mac_fwd_engine, 194 &mac_swap_engine, 195 &flow_gen_engine, 196 &rx_only_engine, 197 &tx_only_engine, 198 &csum_fwd_engine, 199 &icmp_echo_engine, 200 &noisy_vnf_engine, 201 &five_tuple_swap_fwd_engine, 202 &recycle_mbufs_engine, 203 #ifdef RTE_LIBRTE_IEEE1588 204 &ieee1588_fwd_engine, 205 #endif 206 &shared_rxq_engine, 207 NULL, 208 }; 209 210 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 211 uint16_t mempool_flags; 212 213 struct fwd_config cur_fwd_config; 214 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 215 uint32_t retry_enabled; 216 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 217 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 218 219 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 220 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 221 DEFAULT_MBUF_DATA_SIZE 222 }; /**< Mbuf data space size. */ 223 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 224 * specified on command-line. */ 225 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 226 227 /** Extended statistics to show. */ 228 struct rte_eth_xstat_name *xstats_display; 229 230 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 231 232 /* 233 * In container, it cannot terminate the process which running with 'stats-period' 234 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 235 */ 236 volatile uint8_t f_quit; 237 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 238 239 /* 240 * Max Rx frame size, set by '--max-pkt-len' parameter. 241 */ 242 uint32_t max_rx_pkt_len; 243 244 /* 245 * Configuration of packet segments used to scatter received packets 246 * if some of split features is configured. 247 */ 248 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 250 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 251 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 252 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 253 254 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 255 256 /* 257 * Configuration of packet segments used by the "txonly" processing engine. 258 */ 259 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 260 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 261 TXONLY_DEF_PACKET_LEN, 262 }; 263 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 264 265 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 266 /**< Split policy for packets to TX. */ 267 268 uint8_t txonly_multi_flow; 269 /**< Whether multiple flows are generated in TXONLY mode. */ 270 271 uint32_t tx_pkt_times_inter; 272 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 273 274 uint32_t tx_pkt_times_intra; 275 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 276 277 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 278 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 279 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 280 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 281 282 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 283 uint8_t dcb_config = 0; 284 285 /* 286 * Configurable number of RX/TX queues. 287 */ 288 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 289 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 290 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 291 292 /* 293 * Configurable number of RX/TX ring descriptors. 294 * Defaults are supplied by drivers via ethdev. 295 */ 296 #define RX_DESC_DEFAULT 0 297 #define TX_DESC_DEFAULT 0 298 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 299 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 300 301 #define RTE_PMD_PARAM_UNSET -1 302 /* 303 * Configurable values of RX and TX ring threshold registers. 304 */ 305 306 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 307 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 308 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 309 310 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 311 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 312 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 313 314 /* 315 * Configurable value of RX free threshold. 316 */ 317 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 318 319 /* 320 * Configurable value of RX drop enable. 321 */ 322 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 323 324 /* 325 * Configurable value of TX free threshold. 326 */ 327 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 328 329 /* 330 * Configurable value of TX RS bit threshold. 331 */ 332 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 333 334 /* 335 * Configurable sub-forwarding mode for the noisy_vnf forwarding mode. 336 */ 337 enum noisy_fwd_mode noisy_fwd_mode; 338 339 /* String version of enum noisy_fwd_mode */ 340 const char * const noisy_fwd_mode_desc[] = { 341 [NOISY_FWD_MODE_IO] = "io", 342 [NOISY_FWD_MODE_MAC] = "mac", 343 [NOISY_FWD_MODE_MACSWAP] = "macswap", 344 [NOISY_FWD_MODE_5TSWAP] = "5tswap", 345 [NOISY_FWD_MODE_MAX] = NULL, 346 }; 347 348 /* 349 * Configurable value of buffered packets before sending. 350 */ 351 uint16_t noisy_tx_sw_bufsz; 352 353 /* 354 * Configurable value of packet buffer timeout. 355 */ 356 uint16_t noisy_tx_sw_buf_flush_time; 357 358 /* 359 * Configurable value for size of VNF internal memory area 360 * used for simulating noisy neighbour behaviour 361 */ 362 uint64_t noisy_lkup_mem_sz; 363 364 /* 365 * Configurable value of number of random writes done in 366 * VNF simulation memory area. 367 */ 368 uint64_t noisy_lkup_num_writes; 369 370 /* 371 * Configurable value of number of random reads done in 372 * VNF simulation memory area. 373 */ 374 uint64_t noisy_lkup_num_reads; 375 376 /* 377 * Configurable value of number of random reads/writes done in 378 * VNF simulation memory area. 379 */ 380 uint64_t noisy_lkup_num_reads_writes; 381 382 /* 383 * Receive Side Scaling (RSS) configuration. 384 */ 385 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 386 387 /* 388 * Port topology configuration 389 */ 390 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 391 392 /* 393 * Avoids to flush all the RX streams before starts forwarding. 394 */ 395 uint8_t no_flush_rx = 0; /* flush by default */ 396 397 /* 398 * Flow API isolated mode. 399 */ 400 uint8_t flow_isolate_all; 401 402 /* 403 * Disable port flow flush when stop port. 404 */ 405 uint8_t no_flow_flush = 0; /* do flow flush by default */ 406 407 /* 408 * Avoids to check link status when starting/stopping a port. 409 */ 410 uint8_t no_link_check = 0; /* check by default */ 411 412 /* 413 * Don't automatically start all ports in interactive mode. 414 */ 415 uint8_t no_device_start = 0; 416 417 /* 418 * Enable link status change notification 419 */ 420 uint8_t lsc_interrupt = 1; /* enabled by default */ 421 422 /* 423 * Enable device removal notification. 424 */ 425 uint8_t rmv_interrupt = 1; /* enabled by default */ 426 427 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 428 429 /* After attach, port setup is called on event or by iterator */ 430 bool setup_on_probe_event = true; 431 432 /* Clear ptypes on port initialization. */ 433 uint8_t clear_ptypes = true; 434 435 /* Hairpin ports configuration mode. */ 436 uint32_t hairpin_mode; 437 438 /* Pretty printing of ethdev events */ 439 static const char * const eth_event_desc[] = { 440 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 441 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 442 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 443 [RTE_ETH_EVENT_INTR_RESET] = "reset", 444 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 445 [RTE_ETH_EVENT_IPSEC] = "IPsec", 446 [RTE_ETH_EVENT_MACSEC] = "MACsec", 447 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 448 [RTE_ETH_EVENT_NEW] = "device probed", 449 [RTE_ETH_EVENT_DESTROY] = "device released", 450 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 451 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 452 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 453 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 454 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 455 [RTE_ETH_EVENT_MAX] = NULL, 456 }; 457 458 /* 459 * Display or mask ether events 460 * Default to all events except VF_MBOX 461 */ 462 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 463 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 464 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 465 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 466 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 467 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 468 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 469 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 470 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 471 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 472 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 473 /* 474 * Decide if all memory are locked for performance. 475 */ 476 int do_mlockall = 0; 477 478 #ifdef RTE_LIB_LATENCYSTATS 479 480 /* 481 * Set when latency stats is enabled in the commandline 482 */ 483 uint8_t latencystats_enabled; 484 485 /* 486 * Lcore ID to service latency statistics. 487 */ 488 lcoreid_t latencystats_lcore_id = -1; 489 490 #endif 491 492 /* 493 * Ethernet device configuration. 494 */ 495 struct rte_eth_rxmode rx_mode; 496 497 struct rte_eth_txmode tx_mode = { 498 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 499 }; 500 501 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 502 503 /* 504 * Display zero values by default for xstats 505 */ 506 uint8_t xstats_hide_zero; 507 508 /* 509 * Measure of CPU cycles disabled by default 510 */ 511 uint8_t record_core_cycles; 512 513 /* 514 * Display of RX and TX bursts disabled by default 515 */ 516 uint8_t record_burst_stats; 517 518 /* 519 * Number of ports per shared Rx queue group, 0 disable. 520 */ 521 uint32_t rxq_share; 522 523 unsigned int num_sockets = 0; 524 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 525 526 #ifdef RTE_LIB_BITRATESTATS 527 /* Bitrate statistics */ 528 struct rte_stats_bitrates *bitrate_data; 529 lcoreid_t bitrate_lcore_id; 530 uint8_t bitrate_enabled; 531 #endif 532 533 #ifdef RTE_LIB_GRO 534 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 535 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 536 #endif 537 538 /* 539 * hexadecimal bitmask of RX mq mode can be enabled. 540 */ 541 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 542 543 /* 544 * Used to set forced link speed 545 */ 546 uint32_t eth_link_speed; 547 548 /* 549 * ID of the current process in multi-process, used to 550 * configure the queues to be polled. 551 */ 552 int proc_id; 553 554 /* 555 * Number of processes in multi-process, used to 556 * configure the queues to be polled. 557 */ 558 unsigned int num_procs = 1; 559 560 static void 561 eth_rx_metadata_negotiate_mp(uint16_t port_id) 562 { 563 uint64_t rx_meta_features = 0; 564 int ret; 565 566 if (!is_proc_primary()) 567 return; 568 569 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 570 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 571 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 572 573 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 574 if (ret == 0) { 575 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 576 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 577 port_id); 578 } 579 580 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 581 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 582 port_id); 583 } 584 585 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 586 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 587 port_id); 588 } 589 } else if (ret != -ENOTSUP) { 590 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 591 port_id, rte_strerror(-ret)); 592 } 593 } 594 595 static int 596 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 597 const struct rte_eth_conf *dev_conf) 598 { 599 if (is_proc_primary()) 600 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 601 dev_conf); 602 return 0; 603 } 604 605 static int 606 change_bonding_member_port_status(portid_t bond_pid, bool is_stop) 607 { 608 #ifdef RTE_NET_BOND 609 610 portid_t member_pids[RTE_MAX_ETHPORTS]; 611 struct rte_port *port; 612 int num_members; 613 portid_t member_pid; 614 int i; 615 616 num_members = rte_eth_bond_members_get(bond_pid, member_pids, 617 RTE_MAX_ETHPORTS); 618 if (num_members < 0) { 619 fprintf(stderr, "Failed to get member list for port = %u\n", 620 bond_pid); 621 return num_members; 622 } 623 624 for (i = 0; i < num_members; i++) { 625 member_pid = member_pids[i]; 626 port = &ports[member_pid]; 627 port->port_status = 628 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 629 } 630 #else 631 RTE_SET_USED(bond_pid); 632 RTE_SET_USED(is_stop); 633 #endif 634 return 0; 635 } 636 637 static int 638 eth_dev_start_mp(uint16_t port_id) 639 { 640 int ret; 641 642 if (is_proc_primary()) { 643 ret = rte_eth_dev_start(port_id); 644 if (ret != 0) 645 return ret; 646 647 struct rte_port *port = &ports[port_id]; 648 649 /* 650 * Starting a bonding port also starts all members under the bonding 651 * device. So if this port is bond device, we need to modify the 652 * port status of these members. 653 */ 654 if (port->bond_flag == 1) 655 return change_bonding_member_port_status(port_id, false); 656 } 657 658 return 0; 659 } 660 661 static int 662 eth_dev_stop_mp(uint16_t port_id) 663 { 664 int ret; 665 666 if (is_proc_primary()) { 667 ret = rte_eth_dev_stop(port_id); 668 if (ret != 0) 669 return ret; 670 671 struct rte_port *port = &ports[port_id]; 672 673 /* 674 * Stopping a bonding port also stops all members under the bonding 675 * device. So if this port is bond device, we need to modify the 676 * port status of these members. 677 */ 678 if (port->bond_flag == 1) 679 return change_bonding_member_port_status(port_id, true); 680 } 681 682 return 0; 683 } 684 685 static void 686 mempool_free_mp(struct rte_mempool *mp) 687 { 688 if (is_proc_primary()) 689 rte_mempool_free(mp); 690 } 691 692 static int 693 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 694 { 695 if (is_proc_primary()) 696 return rte_eth_dev_set_mtu(port_id, mtu); 697 698 return 0; 699 } 700 701 /* Forward function declarations */ 702 static void setup_attached_port(portid_t pi); 703 static void check_all_ports_link_status(uint32_t port_mask); 704 static int eth_event_callback(portid_t port_id, 705 enum rte_eth_event_type type, 706 void *param, void *ret_param); 707 static void dev_event_callback(const char *device_name, 708 enum rte_dev_event_type type, 709 void *param); 710 static void fill_xstats_display_info(void); 711 712 /* 713 * Check if all the ports are started. 714 * If yes, return positive value. If not, return zero. 715 */ 716 static int all_ports_started(void); 717 718 #ifdef RTE_LIB_GSO 719 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 720 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 721 #endif 722 723 /* Holds the registered mbuf dynamic flags names. */ 724 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 725 726 727 /* 728 * Helper function to check if socket is already discovered. 729 * If yes, return positive value. If not, return zero. 730 */ 731 int 732 new_socket_id(unsigned int socket_id) 733 { 734 unsigned int i; 735 736 for (i = 0; i < num_sockets; i++) { 737 if (socket_ids[i] == socket_id) 738 return 0; 739 } 740 return 1; 741 } 742 743 /* 744 * Setup default configuration. 745 */ 746 static void 747 set_default_fwd_lcores_config(void) 748 { 749 unsigned int i; 750 unsigned int nb_lc; 751 unsigned int sock_num; 752 753 nb_lc = 0; 754 for (i = 0; i < RTE_MAX_LCORE; i++) { 755 if (!rte_lcore_is_enabled(i)) 756 continue; 757 sock_num = rte_lcore_to_socket_id(i); 758 if (new_socket_id(sock_num)) { 759 if (num_sockets >= RTE_MAX_NUMA_NODES) { 760 rte_exit(EXIT_FAILURE, 761 "Total sockets greater than %u\n", 762 RTE_MAX_NUMA_NODES); 763 } 764 socket_ids[num_sockets++] = sock_num; 765 } 766 if (i == rte_get_main_lcore()) 767 continue; 768 fwd_lcores_cpuids[nb_lc++] = i; 769 } 770 nb_lcores = (lcoreid_t) nb_lc; 771 nb_cfg_lcores = nb_lcores; 772 nb_fwd_lcores = 1; 773 } 774 775 static void 776 set_def_peer_eth_addrs(void) 777 { 778 portid_t i; 779 780 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 781 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 782 peer_eth_addrs[i].addr_bytes[5] = i; 783 } 784 } 785 786 static void 787 set_default_fwd_ports_config(void) 788 { 789 portid_t pt_id; 790 int i = 0; 791 792 RTE_ETH_FOREACH_DEV(pt_id) { 793 fwd_ports_ids[i++] = pt_id; 794 795 /* Update sockets info according to the attached device */ 796 int socket_id = rte_eth_dev_socket_id(pt_id); 797 if (socket_id >= 0 && new_socket_id(socket_id)) { 798 if (num_sockets >= RTE_MAX_NUMA_NODES) { 799 rte_exit(EXIT_FAILURE, 800 "Total sockets greater than %u\n", 801 RTE_MAX_NUMA_NODES); 802 } 803 socket_ids[num_sockets++] = socket_id; 804 } 805 } 806 807 nb_cfg_ports = nb_ports; 808 nb_fwd_ports = nb_ports; 809 } 810 811 void 812 set_def_fwd_config(void) 813 { 814 set_default_fwd_lcores_config(); 815 set_def_peer_eth_addrs(); 816 set_default_fwd_ports_config(); 817 } 818 819 #ifndef RTE_EXEC_ENV_WINDOWS 820 /* extremely pessimistic estimation of memory required to create a mempool */ 821 static int 822 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 823 { 824 unsigned int n_pages, mbuf_per_pg, leftover; 825 uint64_t total_mem, mbuf_mem, obj_sz; 826 827 /* there is no good way to predict how much space the mempool will 828 * occupy because it will allocate chunks on the fly, and some of those 829 * will come from default DPDK memory while some will come from our 830 * external memory, so just assume 128MB will be enough for everyone. 831 */ 832 uint64_t hdr_mem = 128 << 20; 833 834 /* account for possible non-contiguousness */ 835 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 836 if (obj_sz > pgsz) { 837 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 838 return -1; 839 } 840 841 mbuf_per_pg = pgsz / obj_sz; 842 leftover = (nb_mbufs % mbuf_per_pg) > 0; 843 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 844 845 mbuf_mem = n_pages * pgsz; 846 847 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 848 849 if (total_mem > SIZE_MAX) { 850 TESTPMD_LOG(ERR, "Memory size too big\n"); 851 return -1; 852 } 853 *out = (size_t)total_mem; 854 855 return 0; 856 } 857 858 static int 859 pagesz_flags(uint64_t page_sz) 860 { 861 /* as per mmap() manpage, all page sizes are log2 of page size 862 * shifted by MAP_HUGE_SHIFT 863 */ 864 int log2 = rte_log2_u64(page_sz); 865 866 return (log2 << HUGE_SHIFT); 867 } 868 869 static void * 870 alloc_mem(size_t memsz, size_t pgsz, bool huge) 871 { 872 void *addr; 873 int flags; 874 875 /* allocate anonymous hugepages */ 876 flags = MAP_ANONYMOUS | MAP_PRIVATE; 877 if (huge) 878 flags |= HUGE_FLAG | pagesz_flags(pgsz); 879 880 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 881 if (addr == MAP_FAILED) 882 return NULL; 883 884 return addr; 885 } 886 887 struct extmem_param { 888 void *addr; 889 size_t len; 890 size_t pgsz; 891 rte_iova_t *iova_table; 892 unsigned int iova_table_len; 893 }; 894 895 static int 896 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 897 bool huge) 898 { 899 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 900 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 901 unsigned int cur_page, n_pages, pgsz_idx; 902 size_t mem_sz, cur_pgsz; 903 rte_iova_t *iovas = NULL; 904 void *addr; 905 int ret; 906 907 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 908 /* skip anything that is too big */ 909 if (pgsizes[pgsz_idx] > SIZE_MAX) 910 continue; 911 912 cur_pgsz = pgsizes[pgsz_idx]; 913 914 /* if we were told not to allocate hugepages, override */ 915 if (!huge) 916 cur_pgsz = sysconf(_SC_PAGESIZE); 917 918 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 919 if (ret < 0) { 920 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 921 return -1; 922 } 923 924 /* allocate our memory */ 925 addr = alloc_mem(mem_sz, cur_pgsz, huge); 926 927 /* if we couldn't allocate memory with a specified page size, 928 * that doesn't mean we can't do it with other page sizes, so 929 * try another one. 930 */ 931 if (addr == NULL) 932 continue; 933 934 /* store IOVA addresses for every page in this memory area */ 935 n_pages = mem_sz / cur_pgsz; 936 937 iovas = malloc(sizeof(*iovas) * n_pages); 938 939 if (iovas == NULL) { 940 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 941 goto fail; 942 } 943 /* lock memory if it's not huge pages */ 944 if (!huge) 945 mlock(addr, mem_sz); 946 947 /* populate IOVA addresses */ 948 for (cur_page = 0; cur_page < n_pages; cur_page++) { 949 rte_iova_t iova; 950 size_t offset; 951 void *cur; 952 953 offset = cur_pgsz * cur_page; 954 cur = RTE_PTR_ADD(addr, offset); 955 956 /* touch the page before getting its IOVA */ 957 *(volatile char *)cur = 0; 958 959 iova = rte_mem_virt2iova(cur); 960 961 iovas[cur_page] = iova; 962 } 963 964 break; 965 } 966 /* if we couldn't allocate anything */ 967 if (iovas == NULL) 968 return -1; 969 970 param->addr = addr; 971 param->len = mem_sz; 972 param->pgsz = cur_pgsz; 973 param->iova_table = iovas; 974 param->iova_table_len = n_pages; 975 976 return 0; 977 fail: 978 free(iovas); 979 if (addr) 980 munmap(addr, mem_sz); 981 982 return -1; 983 } 984 985 static int 986 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 987 { 988 struct extmem_param param; 989 int socket_id, ret; 990 991 memset(¶m, 0, sizeof(param)); 992 993 /* check if our heap exists */ 994 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 995 if (socket_id < 0) { 996 /* create our heap */ 997 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 998 if (ret < 0) { 999 TESTPMD_LOG(ERR, "Cannot create heap\n"); 1000 return -1; 1001 } 1002 } 1003 1004 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 1005 if (ret < 0) { 1006 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 1007 return -1; 1008 } 1009 1010 /* we now have a valid memory area, so add it to heap */ 1011 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 1012 param.addr, param.len, param.iova_table, 1013 param.iova_table_len, param.pgsz); 1014 1015 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1016 1017 /* not needed any more */ 1018 free(param.iova_table); 1019 1020 if (ret < 0) { 1021 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1022 munmap(param.addr, param.len); 1023 return -1; 1024 } 1025 1026 /* success */ 1027 1028 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1029 param.len >> 20); 1030 1031 return 0; 1032 } 1033 static void 1034 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1035 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1036 { 1037 uint16_t pid = 0; 1038 int ret; 1039 1040 RTE_ETH_FOREACH_DEV(pid) { 1041 struct rte_eth_dev_info dev_info; 1042 1043 ret = eth_dev_info_get_print_err(pid, &dev_info); 1044 if (ret != 0) { 1045 TESTPMD_LOG(DEBUG, 1046 "unable to get device info for port %d on addr 0x%p," 1047 "mempool unmapping will not be performed\n", 1048 pid, memhdr->addr); 1049 continue; 1050 } 1051 1052 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1053 if (ret) { 1054 TESTPMD_LOG(DEBUG, 1055 "unable to DMA unmap addr 0x%p " 1056 "for device %s\n", 1057 memhdr->addr, rte_dev_name(dev_info.device)); 1058 } 1059 } 1060 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1061 if (ret) { 1062 TESTPMD_LOG(DEBUG, 1063 "unable to un-register addr 0x%p\n", memhdr->addr); 1064 } 1065 } 1066 1067 static void 1068 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1069 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1070 { 1071 uint16_t pid = 0; 1072 size_t page_size = sysconf(_SC_PAGESIZE); 1073 int ret; 1074 1075 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1076 page_size); 1077 if (ret) { 1078 TESTPMD_LOG(DEBUG, 1079 "unable to register addr 0x%p\n", memhdr->addr); 1080 return; 1081 } 1082 RTE_ETH_FOREACH_DEV(pid) { 1083 struct rte_eth_dev_info dev_info; 1084 1085 ret = eth_dev_info_get_print_err(pid, &dev_info); 1086 if (ret != 0) { 1087 TESTPMD_LOG(DEBUG, 1088 "unable to get device info for port %d on addr 0x%p," 1089 "mempool mapping will not be performed\n", 1090 pid, memhdr->addr); 1091 continue; 1092 } 1093 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1094 if (ret) { 1095 TESTPMD_LOG(DEBUG, 1096 "unable to DMA map addr 0x%p " 1097 "for device %s\n", 1098 memhdr->addr, rte_dev_name(dev_info.device)); 1099 } 1100 } 1101 } 1102 #endif 1103 1104 static unsigned int 1105 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1106 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1107 { 1108 struct rte_pktmbuf_extmem *xmem; 1109 unsigned int ext_num, zone_num, elt_num; 1110 uint16_t elt_size; 1111 1112 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1113 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1114 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1115 1116 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1117 if (xmem == NULL) { 1118 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1119 "external buffer descriptors\n"); 1120 *ext_mem = NULL; 1121 return 0; 1122 } 1123 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1124 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1125 const struct rte_memzone *mz; 1126 char mz_name[RTE_MEMZONE_NAMESIZE]; 1127 int ret; 1128 1129 ret = snprintf(mz_name, sizeof(mz_name), 1130 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1131 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1132 errno = ENAMETOOLONG; 1133 ext_num = 0; 1134 break; 1135 } 1136 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1137 socket_id, 1138 RTE_MEMZONE_IOVA_CONTIG | 1139 RTE_MEMZONE_1GB | 1140 RTE_MEMZONE_SIZE_HINT_ONLY); 1141 if (mz == NULL) { 1142 /* 1143 * The caller exits on external buffer creation 1144 * error, so there is no need to free memzones. 1145 */ 1146 errno = ENOMEM; 1147 ext_num = 0; 1148 break; 1149 } 1150 xseg->buf_ptr = mz->addr; 1151 xseg->buf_iova = mz->iova; 1152 xseg->buf_len = EXTBUF_ZONE_SIZE; 1153 xseg->elt_size = elt_size; 1154 } 1155 if (ext_num == 0 && xmem != NULL) { 1156 free(xmem); 1157 xmem = NULL; 1158 } 1159 *ext_mem = xmem; 1160 return ext_num; 1161 } 1162 1163 /* 1164 * Configuration initialisation done once at init time. 1165 */ 1166 static struct rte_mempool * 1167 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1168 unsigned int socket_id, uint16_t size_idx) 1169 { 1170 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1171 struct rte_mempool *rte_mp = NULL; 1172 #ifndef RTE_EXEC_ENV_WINDOWS 1173 uint32_t mb_size; 1174 1175 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1176 #endif 1177 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1178 if (!is_proc_primary()) { 1179 rte_mp = rte_mempool_lookup(pool_name); 1180 if (rte_mp == NULL) 1181 rte_exit(EXIT_FAILURE, 1182 "Get mbuf pool for socket %u failed: %s\n", 1183 socket_id, rte_strerror(rte_errno)); 1184 return rte_mp; 1185 } 1186 1187 TESTPMD_LOG(INFO, 1188 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1189 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1190 1191 switch (mp_alloc_type) { 1192 case MP_ALLOC_NATIVE: 1193 { 1194 /* wrapper to rte_mempool_create() */ 1195 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1196 rte_mbuf_best_mempool_ops()); 1197 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1198 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1199 break; 1200 } 1201 #ifndef RTE_EXEC_ENV_WINDOWS 1202 case MP_ALLOC_ANON: 1203 { 1204 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1205 mb_size, (unsigned int) mb_mempool_cache, 1206 sizeof(struct rte_pktmbuf_pool_private), 1207 socket_id, mempool_flags); 1208 if (rte_mp == NULL) 1209 goto err; 1210 1211 if (rte_mempool_populate_anon(rte_mp) == 0) { 1212 rte_mempool_free(rte_mp); 1213 rte_mp = NULL; 1214 goto err; 1215 } 1216 rte_pktmbuf_pool_init(rte_mp, NULL); 1217 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1218 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1219 break; 1220 } 1221 case MP_ALLOC_XMEM: 1222 case MP_ALLOC_XMEM_HUGE: 1223 { 1224 int heap_socket; 1225 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1226 1227 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1228 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1229 1230 heap_socket = 1231 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1232 if (heap_socket < 0) 1233 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1234 1235 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1236 rte_mbuf_best_mempool_ops()); 1237 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1238 mb_mempool_cache, 0, mbuf_seg_size, 1239 heap_socket); 1240 break; 1241 } 1242 #endif 1243 case MP_ALLOC_XBUF: 1244 { 1245 struct rte_pktmbuf_extmem *ext_mem; 1246 unsigned int ext_num; 1247 1248 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1249 socket_id, pool_name, &ext_mem); 1250 if (ext_num == 0) 1251 rte_exit(EXIT_FAILURE, 1252 "Can't create pinned data buffers\n"); 1253 1254 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1255 rte_mbuf_best_mempool_ops()); 1256 rte_mp = rte_pktmbuf_pool_create_extbuf 1257 (pool_name, nb_mbuf, mb_mempool_cache, 1258 0, mbuf_seg_size, socket_id, 1259 ext_mem, ext_num); 1260 free(ext_mem); 1261 break; 1262 } 1263 default: 1264 { 1265 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1266 } 1267 } 1268 1269 #ifndef RTE_EXEC_ENV_WINDOWS 1270 err: 1271 #endif 1272 if (rte_mp == NULL) { 1273 rte_exit(EXIT_FAILURE, 1274 "Creation of mbuf pool for socket %u failed: %s\n", 1275 socket_id, rte_strerror(rte_errno)); 1276 } else if (verbose_level > 0) { 1277 rte_mempool_dump(stdout, rte_mp); 1278 } 1279 return rte_mp; 1280 } 1281 1282 /* 1283 * Check given socket id is valid or not with NUMA mode, 1284 * if valid, return 0, else return -1 1285 */ 1286 static int 1287 check_socket_id(const unsigned int socket_id) 1288 { 1289 static int warning_once = 0; 1290 1291 if (new_socket_id(socket_id)) { 1292 if (!warning_once && numa_support) 1293 fprintf(stderr, 1294 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1295 warning_once = 1; 1296 return -1; 1297 } 1298 return 0; 1299 } 1300 1301 /* 1302 * Get the allowed maximum number of RX queues. 1303 * *pid return the port id which has minimal value of 1304 * max_rx_queues in all ports. 1305 */ 1306 queueid_t 1307 get_allowed_max_nb_rxq(portid_t *pid) 1308 { 1309 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1310 bool max_rxq_valid = false; 1311 portid_t pi; 1312 struct rte_eth_dev_info dev_info; 1313 1314 RTE_ETH_FOREACH_DEV(pi) { 1315 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1316 continue; 1317 1318 max_rxq_valid = true; 1319 if (dev_info.max_rx_queues < allowed_max_rxq) { 1320 allowed_max_rxq = dev_info.max_rx_queues; 1321 *pid = pi; 1322 } 1323 } 1324 return max_rxq_valid ? allowed_max_rxq : 0; 1325 } 1326 1327 /* 1328 * Check input rxq is valid or not. 1329 * If input rxq is not greater than any of maximum number 1330 * of RX queues of all ports, it is valid. 1331 * if valid, return 0, else return -1 1332 */ 1333 int 1334 check_nb_rxq(queueid_t rxq) 1335 { 1336 queueid_t allowed_max_rxq; 1337 portid_t pid = 0; 1338 1339 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1340 if (rxq > allowed_max_rxq) { 1341 fprintf(stderr, 1342 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1343 rxq, allowed_max_rxq, pid); 1344 return -1; 1345 } 1346 return 0; 1347 } 1348 1349 /* 1350 * Get the allowed maximum number of TX queues. 1351 * *pid return the port id which has minimal value of 1352 * max_tx_queues in all ports. 1353 */ 1354 queueid_t 1355 get_allowed_max_nb_txq(portid_t *pid) 1356 { 1357 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1358 bool max_txq_valid = false; 1359 portid_t pi; 1360 struct rte_eth_dev_info dev_info; 1361 1362 RTE_ETH_FOREACH_DEV(pi) { 1363 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1364 continue; 1365 1366 max_txq_valid = true; 1367 if (dev_info.max_tx_queues < allowed_max_txq) { 1368 allowed_max_txq = dev_info.max_tx_queues; 1369 *pid = pi; 1370 } 1371 } 1372 return max_txq_valid ? allowed_max_txq : 0; 1373 } 1374 1375 /* 1376 * Check input txq is valid or not. 1377 * If input txq is not greater than any of maximum number 1378 * of TX queues of all ports, it is valid. 1379 * if valid, return 0, else return -1 1380 */ 1381 int 1382 check_nb_txq(queueid_t txq) 1383 { 1384 queueid_t allowed_max_txq; 1385 portid_t pid = 0; 1386 1387 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1388 if (txq > allowed_max_txq) { 1389 fprintf(stderr, 1390 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1391 txq, allowed_max_txq, pid); 1392 return -1; 1393 } 1394 return 0; 1395 } 1396 1397 /* 1398 * Get the allowed maximum number of RXDs of every rx queue. 1399 * *pid return the port id which has minimal value of 1400 * max_rxd in all queues of all ports. 1401 */ 1402 static uint16_t 1403 get_allowed_max_nb_rxd(portid_t *pid) 1404 { 1405 uint16_t allowed_max_rxd = UINT16_MAX; 1406 portid_t pi; 1407 struct rte_eth_dev_info dev_info; 1408 1409 RTE_ETH_FOREACH_DEV(pi) { 1410 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1411 continue; 1412 1413 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1414 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1415 *pid = pi; 1416 } 1417 } 1418 return allowed_max_rxd; 1419 } 1420 1421 /* 1422 * Get the allowed minimal number of RXDs of every rx queue. 1423 * *pid return the port id which has minimal value of 1424 * min_rxd in all queues of all ports. 1425 */ 1426 static uint16_t 1427 get_allowed_min_nb_rxd(portid_t *pid) 1428 { 1429 uint16_t allowed_min_rxd = 0; 1430 portid_t pi; 1431 struct rte_eth_dev_info dev_info; 1432 1433 RTE_ETH_FOREACH_DEV(pi) { 1434 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1435 continue; 1436 1437 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1438 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1439 *pid = pi; 1440 } 1441 } 1442 1443 return allowed_min_rxd; 1444 } 1445 1446 /* 1447 * Check input rxd is valid or not. 1448 * If input rxd is not greater than any of maximum number 1449 * of RXDs of every Rx queues and is not less than any of 1450 * minimal number of RXDs of every Rx queues, it is valid. 1451 * if valid, return 0, else return -1 1452 */ 1453 int 1454 check_nb_rxd(queueid_t rxd) 1455 { 1456 uint16_t allowed_max_rxd; 1457 uint16_t allowed_min_rxd; 1458 portid_t pid = 0; 1459 1460 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1461 if (rxd > allowed_max_rxd) { 1462 fprintf(stderr, 1463 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1464 rxd, allowed_max_rxd, pid); 1465 return -1; 1466 } 1467 1468 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1469 if (rxd < allowed_min_rxd) { 1470 fprintf(stderr, 1471 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1472 rxd, allowed_min_rxd, pid); 1473 return -1; 1474 } 1475 1476 return 0; 1477 } 1478 1479 /* 1480 * Get the allowed maximum number of TXDs of every rx queues. 1481 * *pid return the port id which has minimal value of 1482 * max_txd in every tx queue. 1483 */ 1484 static uint16_t 1485 get_allowed_max_nb_txd(portid_t *pid) 1486 { 1487 uint16_t allowed_max_txd = UINT16_MAX; 1488 portid_t pi; 1489 struct rte_eth_dev_info dev_info; 1490 1491 RTE_ETH_FOREACH_DEV(pi) { 1492 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1493 continue; 1494 1495 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1496 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1497 *pid = pi; 1498 } 1499 } 1500 return allowed_max_txd; 1501 } 1502 1503 /* 1504 * Get the allowed maximum number of TXDs of every tx queues. 1505 * *pid return the port id which has minimal value of 1506 * min_txd in every tx queue. 1507 */ 1508 static uint16_t 1509 get_allowed_min_nb_txd(portid_t *pid) 1510 { 1511 uint16_t allowed_min_txd = 0; 1512 portid_t pi; 1513 struct rte_eth_dev_info dev_info; 1514 1515 RTE_ETH_FOREACH_DEV(pi) { 1516 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1517 continue; 1518 1519 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1520 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1521 *pid = pi; 1522 } 1523 } 1524 1525 return allowed_min_txd; 1526 } 1527 1528 /* 1529 * Check input txd is valid or not. 1530 * If input txd is not greater than any of maximum number 1531 * of TXDs of every Rx queues, it is valid. 1532 * if valid, return 0, else return -1 1533 */ 1534 int 1535 check_nb_txd(queueid_t txd) 1536 { 1537 uint16_t allowed_max_txd; 1538 uint16_t allowed_min_txd; 1539 portid_t pid = 0; 1540 1541 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1542 if (txd > allowed_max_txd) { 1543 fprintf(stderr, 1544 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1545 txd, allowed_max_txd, pid); 1546 return -1; 1547 } 1548 1549 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1550 if (txd < allowed_min_txd) { 1551 fprintf(stderr, 1552 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1553 txd, allowed_min_txd, pid); 1554 return -1; 1555 } 1556 return 0; 1557 } 1558 1559 1560 /* 1561 * Get the allowed maximum number of hairpin queues. 1562 * *pid return the port id which has minimal value of 1563 * max_hairpin_queues in all ports. 1564 */ 1565 queueid_t 1566 get_allowed_max_nb_hairpinq(portid_t *pid) 1567 { 1568 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1569 portid_t pi; 1570 struct rte_eth_hairpin_cap cap; 1571 1572 RTE_ETH_FOREACH_DEV(pi) { 1573 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1574 *pid = pi; 1575 return 0; 1576 } 1577 if (cap.max_nb_queues < allowed_max_hairpinq) { 1578 allowed_max_hairpinq = cap.max_nb_queues; 1579 *pid = pi; 1580 } 1581 } 1582 return allowed_max_hairpinq; 1583 } 1584 1585 /* 1586 * Check input hairpin is valid or not. 1587 * If input hairpin is not greater than any of maximum number 1588 * of hairpin queues of all ports, it is valid. 1589 * if valid, return 0, else return -1 1590 */ 1591 int 1592 check_nb_hairpinq(queueid_t hairpinq) 1593 { 1594 queueid_t allowed_max_hairpinq; 1595 portid_t pid = 0; 1596 1597 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1598 if (hairpinq > allowed_max_hairpinq) { 1599 fprintf(stderr, 1600 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1601 hairpinq, allowed_max_hairpinq, pid); 1602 return -1; 1603 } 1604 return 0; 1605 } 1606 1607 static int 1608 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1609 { 1610 uint32_t eth_overhead; 1611 1612 if (dev_info->max_mtu != UINT16_MAX && 1613 dev_info->max_rx_pktlen > dev_info->max_mtu) 1614 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1615 else 1616 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1617 1618 return eth_overhead; 1619 } 1620 1621 static void 1622 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1623 { 1624 struct rte_port *port = &ports[pid]; 1625 int ret; 1626 int i; 1627 1628 eth_rx_metadata_negotiate_mp(pid); 1629 1630 port->dev_conf.txmode = tx_mode; 1631 port->dev_conf.rxmode = rx_mode; 1632 1633 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1634 if (ret != 0) 1635 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1636 1637 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1638 port->dev_conf.txmode.offloads &= 1639 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1640 1641 /* Apply Rx offloads configuration */ 1642 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1643 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1644 /* Apply Tx offloads configuration */ 1645 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1646 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1647 1648 if (eth_link_speed) 1649 port->dev_conf.link_speeds = eth_link_speed; 1650 1651 if (max_rx_pkt_len) 1652 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1653 get_eth_overhead(&port->dev_info); 1654 1655 /* set flag to initialize port/queue */ 1656 port->need_reconfig = 1; 1657 port->need_reconfig_queues = 1; 1658 port->socket_id = socket_id; 1659 port->tx_metadata = 0; 1660 1661 /* 1662 * Check for maximum number of segments per MTU. 1663 * Accordingly update the mbuf data size. 1664 */ 1665 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1666 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1667 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1668 uint16_t mtu; 1669 1670 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1671 uint16_t data_size = (mtu + eth_overhead) / 1672 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1673 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1674 1675 if (buffer_size > mbuf_data_size[0]) { 1676 mbuf_data_size[0] = buffer_size; 1677 TESTPMD_LOG(WARNING, 1678 "Configured mbuf size of the first segment %hu\n", 1679 mbuf_data_size[0]); 1680 } 1681 } 1682 } 1683 } 1684 1685 static void 1686 init_config(void) 1687 { 1688 portid_t pid; 1689 struct rte_mempool *mbp; 1690 unsigned int nb_mbuf_per_pool; 1691 lcoreid_t lc_id; 1692 #ifdef RTE_LIB_GRO 1693 struct rte_gro_param gro_param; 1694 #endif 1695 #ifdef RTE_LIB_GSO 1696 uint32_t gso_types; 1697 #endif 1698 1699 /* Configuration of logical cores. */ 1700 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1701 sizeof(struct fwd_lcore *) * nb_lcores, 1702 RTE_CACHE_LINE_SIZE); 1703 if (fwd_lcores == NULL) { 1704 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1705 "failed\n", nb_lcores); 1706 } 1707 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1708 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1709 sizeof(struct fwd_lcore), 1710 RTE_CACHE_LINE_SIZE); 1711 if (fwd_lcores[lc_id] == NULL) { 1712 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1713 "failed\n"); 1714 } 1715 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1716 } 1717 1718 RTE_ETH_FOREACH_DEV(pid) { 1719 uint32_t socket_id; 1720 1721 if (numa_support) { 1722 socket_id = port_numa[pid]; 1723 if (port_numa[pid] == NUMA_NO_CONFIG) { 1724 socket_id = rte_eth_dev_socket_id(pid); 1725 1726 /* 1727 * if socket_id is invalid, 1728 * set to the first available socket. 1729 */ 1730 if (check_socket_id(socket_id) < 0) 1731 socket_id = socket_ids[0]; 1732 } 1733 } else { 1734 socket_id = (socket_num == UMA_NO_CONFIG) ? 1735 0 : socket_num; 1736 } 1737 /* Apply default TxRx configuration for all ports */ 1738 init_config_port_offloads(pid, socket_id); 1739 } 1740 /* 1741 * Create pools of mbuf. 1742 * If NUMA support is disabled, create a single pool of mbuf in 1743 * socket 0 memory by default. 1744 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1745 * 1746 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1747 * nb_txd can be configured at run time. 1748 */ 1749 if (param_total_num_mbufs) 1750 nb_mbuf_per_pool = param_total_num_mbufs; 1751 else { 1752 nb_mbuf_per_pool = RX_DESC_MAX + 1753 (nb_lcores * mb_mempool_cache) + 1754 TX_DESC_MAX + MAX_PKT_BURST; 1755 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1756 } 1757 1758 if (numa_support) { 1759 uint8_t i, j; 1760 1761 for (i = 0; i < num_sockets; i++) 1762 for (j = 0; j < mbuf_data_size_n; j++) 1763 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1764 mbuf_pool_create(mbuf_data_size[j], 1765 nb_mbuf_per_pool, 1766 socket_ids[i], j); 1767 } else { 1768 uint8_t i; 1769 1770 for (i = 0; i < mbuf_data_size_n; i++) 1771 mempools[i] = mbuf_pool_create 1772 (mbuf_data_size[i], 1773 nb_mbuf_per_pool, 1774 socket_num == UMA_NO_CONFIG ? 1775 0 : socket_num, i); 1776 } 1777 1778 init_port_config(); 1779 1780 #ifdef RTE_LIB_GSO 1781 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1782 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1783 #endif 1784 /* 1785 * Records which Mbuf pool to use by each logical core, if needed. 1786 */ 1787 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1788 mbp = mbuf_pool_find( 1789 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1790 1791 if (mbp == NULL) 1792 mbp = mbuf_pool_find(0, 0); 1793 fwd_lcores[lc_id]->mbp = mbp; 1794 #ifdef RTE_LIB_GSO 1795 /* initialize GSO context */ 1796 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1797 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1798 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1799 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1800 RTE_ETHER_CRC_LEN; 1801 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1802 #endif 1803 } 1804 1805 fwd_config_setup(); 1806 1807 #ifdef RTE_LIB_GRO 1808 /* create a gro context for each lcore */ 1809 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1810 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1811 gro_param.max_item_per_flow = MAX_PKT_BURST; 1812 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1813 gro_param.socket_id = rte_lcore_to_socket_id( 1814 fwd_lcores_cpuids[lc_id]); 1815 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1816 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1817 rte_exit(EXIT_FAILURE, 1818 "rte_gro_ctx_create() failed\n"); 1819 } 1820 } 1821 #endif 1822 } 1823 1824 1825 void 1826 reconfig(portid_t new_port_id, unsigned socket_id) 1827 { 1828 /* Reconfiguration of Ethernet ports. */ 1829 init_config_port_offloads(new_port_id, socket_id); 1830 init_port_config(); 1831 } 1832 1833 int 1834 init_fwd_streams(void) 1835 { 1836 portid_t pid; 1837 struct rte_port *port; 1838 streamid_t sm_id, nb_fwd_streams_new; 1839 queueid_t q; 1840 1841 /* set socket id according to numa or not */ 1842 RTE_ETH_FOREACH_DEV(pid) { 1843 port = &ports[pid]; 1844 if (nb_rxq > port->dev_info.max_rx_queues) { 1845 fprintf(stderr, 1846 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1847 nb_rxq, port->dev_info.max_rx_queues); 1848 return -1; 1849 } 1850 if (nb_txq > port->dev_info.max_tx_queues) { 1851 fprintf(stderr, 1852 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1853 nb_txq, port->dev_info.max_tx_queues); 1854 return -1; 1855 } 1856 if (numa_support) { 1857 if (port_numa[pid] != NUMA_NO_CONFIG) 1858 port->socket_id = port_numa[pid]; 1859 else { 1860 port->socket_id = rte_eth_dev_socket_id(pid); 1861 1862 /* 1863 * if socket_id is invalid, 1864 * set to the first available socket. 1865 */ 1866 if (check_socket_id(port->socket_id) < 0) 1867 port->socket_id = socket_ids[0]; 1868 } 1869 } 1870 else { 1871 if (socket_num == UMA_NO_CONFIG) 1872 port->socket_id = 0; 1873 else 1874 port->socket_id = socket_num; 1875 } 1876 } 1877 1878 q = RTE_MAX(nb_rxq, nb_txq); 1879 if (q == 0) { 1880 fprintf(stderr, 1881 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1882 return -1; 1883 } 1884 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1885 if (nb_fwd_streams_new == nb_fwd_streams) 1886 return 0; 1887 /* clear the old */ 1888 if (fwd_streams != NULL) { 1889 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1890 if (fwd_streams[sm_id] == NULL) 1891 continue; 1892 rte_free(fwd_streams[sm_id]); 1893 fwd_streams[sm_id] = NULL; 1894 } 1895 rte_free(fwd_streams); 1896 fwd_streams = NULL; 1897 } 1898 1899 /* init new */ 1900 nb_fwd_streams = nb_fwd_streams_new; 1901 if (nb_fwd_streams) { 1902 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1903 sizeof(struct fwd_stream *) * nb_fwd_streams, 1904 RTE_CACHE_LINE_SIZE); 1905 if (fwd_streams == NULL) 1906 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1907 " (struct fwd_stream *)) failed\n", 1908 nb_fwd_streams); 1909 1910 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1911 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1912 " struct fwd_stream", sizeof(struct fwd_stream), 1913 RTE_CACHE_LINE_SIZE); 1914 if (fwd_streams[sm_id] == NULL) 1915 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1916 "(struct fwd_stream) failed\n"); 1917 } 1918 } 1919 1920 return 0; 1921 } 1922 1923 static void 1924 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1925 { 1926 uint64_t total_burst, sburst; 1927 uint64_t nb_burst; 1928 uint64_t burst_stats[4]; 1929 uint16_t pktnb_stats[4]; 1930 uint16_t nb_pkt; 1931 int burst_percent[4], sburstp; 1932 int i; 1933 1934 /* 1935 * First compute the total number of packet bursts and the 1936 * two highest numbers of bursts of the same number of packets. 1937 */ 1938 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1939 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1940 1941 /* Show stats for 0 burst size always */ 1942 total_burst = pbs->pkt_burst_spread[0]; 1943 burst_stats[0] = pbs->pkt_burst_spread[0]; 1944 pktnb_stats[0] = 0; 1945 1946 /* Find the next 2 burst sizes with highest occurrences. */ 1947 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1948 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1949 1950 if (nb_burst == 0) 1951 continue; 1952 1953 total_burst += nb_burst; 1954 1955 if (nb_burst > burst_stats[1]) { 1956 burst_stats[2] = burst_stats[1]; 1957 pktnb_stats[2] = pktnb_stats[1]; 1958 burst_stats[1] = nb_burst; 1959 pktnb_stats[1] = nb_pkt; 1960 } else if (nb_burst > burst_stats[2]) { 1961 burst_stats[2] = nb_burst; 1962 pktnb_stats[2] = nb_pkt; 1963 } 1964 } 1965 if (total_burst == 0) 1966 return; 1967 1968 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1969 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1970 if (i == 3) { 1971 printf("%d%% of other]\n", 100 - sburstp); 1972 return; 1973 } 1974 1975 sburst += burst_stats[i]; 1976 if (sburst == total_burst) { 1977 printf("%d%% of %d pkts]\n", 1978 100 - sburstp, (int) pktnb_stats[i]); 1979 return; 1980 } 1981 1982 burst_percent[i] = 1983 (double)burst_stats[i] / total_burst * 100; 1984 printf("%d%% of %d pkts + ", 1985 burst_percent[i], (int) pktnb_stats[i]); 1986 sburstp += burst_percent[i]; 1987 } 1988 } 1989 1990 static void 1991 fwd_stream_stats_display(streamid_t stream_id) 1992 { 1993 struct fwd_stream *fs; 1994 static const char *fwd_top_stats_border = "-------"; 1995 1996 fs = fwd_streams[stream_id]; 1997 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1998 (fs->fwd_dropped == 0)) 1999 return; 2000 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 2001 "TX Port=%2d/Queue=%2d %s\n", 2002 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 2003 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 2004 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 2005 " TX-dropped: %-14"PRIu64, 2006 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 2007 2008 /* if checksum mode */ 2009 if (cur_fwd_eng == &csum_fwd_engine) { 2010 printf(" RX- bad IP checksum: %-14"PRIu64 2011 " Rx- bad L4 checksum: %-14"PRIu64 2012 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 2013 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 2014 fs->rx_bad_outer_l4_csum); 2015 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2016 fs->rx_bad_outer_ip_csum); 2017 } else { 2018 printf("\n"); 2019 } 2020 2021 if (record_burst_stats) { 2022 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2023 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2024 } 2025 } 2026 2027 void 2028 fwd_stats_display(void) 2029 { 2030 static const char *fwd_stats_border = "----------------------"; 2031 static const char *acc_stats_border = "+++++++++++++++"; 2032 struct { 2033 struct fwd_stream *rx_stream; 2034 struct fwd_stream *tx_stream; 2035 uint64_t tx_dropped; 2036 uint64_t rx_bad_ip_csum; 2037 uint64_t rx_bad_l4_csum; 2038 uint64_t rx_bad_outer_l4_csum; 2039 uint64_t rx_bad_outer_ip_csum; 2040 } ports_stats[RTE_MAX_ETHPORTS]; 2041 uint64_t total_rx_dropped = 0; 2042 uint64_t total_tx_dropped = 0; 2043 uint64_t total_rx_nombuf = 0; 2044 struct rte_eth_stats stats; 2045 uint64_t fwd_cycles = 0; 2046 uint64_t total_recv = 0; 2047 uint64_t total_xmit = 0; 2048 struct rte_port *port; 2049 streamid_t sm_id; 2050 portid_t pt_id; 2051 int ret; 2052 int i; 2053 2054 memset(ports_stats, 0, sizeof(ports_stats)); 2055 2056 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2057 struct fwd_stream *fs = fwd_streams[sm_id]; 2058 2059 if (cur_fwd_config.nb_fwd_streams > 2060 cur_fwd_config.nb_fwd_ports) { 2061 fwd_stream_stats_display(sm_id); 2062 } else { 2063 ports_stats[fs->tx_port].tx_stream = fs; 2064 ports_stats[fs->rx_port].rx_stream = fs; 2065 } 2066 2067 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2068 2069 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2070 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2071 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2072 fs->rx_bad_outer_l4_csum; 2073 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2074 fs->rx_bad_outer_ip_csum; 2075 2076 if (record_core_cycles) 2077 fwd_cycles += fs->busy_cycles; 2078 } 2079 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2080 uint64_t tx_dropped = 0; 2081 2082 pt_id = fwd_ports_ids[i]; 2083 port = &ports[pt_id]; 2084 2085 ret = rte_eth_stats_get(pt_id, &stats); 2086 if (ret != 0) { 2087 fprintf(stderr, 2088 "%s: Error: failed to get stats (port %u): %d", 2089 __func__, pt_id, ret); 2090 continue; 2091 } 2092 stats.ipackets -= port->stats.ipackets; 2093 stats.opackets -= port->stats.opackets; 2094 stats.ibytes -= port->stats.ibytes; 2095 stats.obytes -= port->stats.obytes; 2096 stats.imissed -= port->stats.imissed; 2097 stats.oerrors -= port->stats.oerrors; 2098 stats.rx_nombuf -= port->stats.rx_nombuf; 2099 2100 total_recv += stats.ipackets; 2101 total_xmit += stats.opackets; 2102 total_rx_dropped += stats.imissed; 2103 tx_dropped += ports_stats[pt_id].tx_dropped; 2104 tx_dropped += stats.oerrors; 2105 total_tx_dropped += tx_dropped; 2106 total_rx_nombuf += stats.rx_nombuf; 2107 2108 printf("\n %s Forward statistics for port %-2d %s\n", 2109 fwd_stats_border, pt_id, fwd_stats_border); 2110 2111 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2112 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2113 stats.ipackets + stats.imissed); 2114 2115 if (cur_fwd_eng == &csum_fwd_engine) { 2116 printf(" Bad-ipcsum: %-14"PRIu64 2117 " Bad-l4csum: %-14"PRIu64 2118 "Bad-outer-l4csum: %-14"PRIu64"\n", 2119 ports_stats[pt_id].rx_bad_ip_csum, 2120 ports_stats[pt_id].rx_bad_l4_csum, 2121 ports_stats[pt_id].rx_bad_outer_l4_csum); 2122 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2123 ports_stats[pt_id].rx_bad_outer_ip_csum); 2124 } 2125 if (stats.ierrors + stats.rx_nombuf > 0) { 2126 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2127 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2128 } 2129 2130 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2131 "TX-total: %-"PRIu64"\n", 2132 stats.opackets, tx_dropped, 2133 stats.opackets + tx_dropped); 2134 2135 if (record_burst_stats) { 2136 if (ports_stats[pt_id].rx_stream) 2137 pkt_burst_stats_display("RX", 2138 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2139 if (ports_stats[pt_id].tx_stream) 2140 pkt_burst_stats_display("TX", 2141 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2142 } 2143 2144 printf(" %s--------------------------------%s\n", 2145 fwd_stats_border, fwd_stats_border); 2146 } 2147 2148 printf("\n %s Accumulated forward statistics for all ports" 2149 "%s\n", 2150 acc_stats_border, acc_stats_border); 2151 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2152 "%-"PRIu64"\n" 2153 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2154 "%-"PRIu64"\n", 2155 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2156 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2157 if (total_rx_nombuf > 0) 2158 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2159 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2160 "%s\n", 2161 acc_stats_border, acc_stats_border); 2162 if (record_core_cycles) { 2163 #define CYC_PER_MHZ 1E6 2164 if (total_recv > 0 || total_xmit > 0) { 2165 uint64_t total_pkts = 0; 2166 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2167 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2168 total_pkts = total_xmit; 2169 else 2170 total_pkts = total_recv; 2171 2172 printf("\n CPU cycles/packet=%.2F (busy cycles=" 2173 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2174 " MHz Clock\n", 2175 (double) fwd_cycles / total_pkts, 2176 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2177 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2178 } 2179 } 2180 } 2181 2182 void 2183 fwd_stats_reset(void) 2184 { 2185 streamid_t sm_id; 2186 portid_t pt_id; 2187 int ret; 2188 int i; 2189 2190 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2191 pt_id = fwd_ports_ids[i]; 2192 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2193 if (ret != 0) 2194 fprintf(stderr, 2195 "%s: Error: failed to clear stats (port %u):%d", 2196 __func__, pt_id, ret); 2197 } 2198 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2199 struct fwd_stream *fs = fwd_streams[sm_id]; 2200 2201 fs->rx_packets = 0; 2202 fs->tx_packets = 0; 2203 fs->fwd_dropped = 0; 2204 fs->rx_bad_ip_csum = 0; 2205 fs->rx_bad_l4_csum = 0; 2206 fs->rx_bad_outer_l4_csum = 0; 2207 fs->rx_bad_outer_ip_csum = 0; 2208 2209 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2210 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2211 fs->busy_cycles = 0; 2212 } 2213 } 2214 2215 static void 2216 flush_fwd_rx_queues(void) 2217 { 2218 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2219 portid_t rxp; 2220 portid_t port_id; 2221 queueid_t rxq; 2222 uint16_t nb_rx; 2223 uint8_t j; 2224 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2225 uint64_t timer_period; 2226 2227 if (num_procs > 1) { 2228 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2229 return; 2230 } 2231 2232 /* convert to number of cycles */ 2233 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2234 2235 for (j = 0; j < 2; j++) { 2236 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2237 for (rxq = 0; rxq < nb_rxq; rxq++) { 2238 port_id = fwd_ports_ids[rxp]; 2239 2240 /* Polling stopped queues is prohibited. */ 2241 if (ports[port_id].rxq[rxq].state == 2242 RTE_ETH_QUEUE_STATE_STOPPED) 2243 continue; 2244 2245 /** 2246 * testpmd can stuck in the below do while loop 2247 * if rte_eth_rx_burst() always returns nonzero 2248 * packets. So timer is added to exit this loop 2249 * after 1sec timer expiry. 2250 */ 2251 prev_tsc = rte_rdtsc(); 2252 do { 2253 nb_rx = rte_eth_rx_burst(port_id, rxq, 2254 pkts_burst, MAX_PKT_BURST); 2255 rte_pktmbuf_free_bulk(pkts_burst, nb_rx); 2256 2257 cur_tsc = rte_rdtsc(); 2258 diff_tsc = cur_tsc - prev_tsc; 2259 timer_tsc += diff_tsc; 2260 } while ((nb_rx > 0) && 2261 (timer_tsc < timer_period)); 2262 timer_tsc = 0; 2263 } 2264 } 2265 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2266 } 2267 } 2268 2269 static void 2270 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2271 { 2272 struct fwd_stream **fsm; 2273 uint64_t prev_tsc; 2274 streamid_t nb_fs; 2275 streamid_t sm_id; 2276 #ifdef RTE_LIB_BITRATESTATS 2277 uint64_t tics_per_1sec; 2278 uint64_t tics_datum; 2279 uint64_t tics_current; 2280 uint16_t i, cnt_ports; 2281 2282 cnt_ports = nb_ports; 2283 tics_datum = rte_rdtsc(); 2284 tics_per_1sec = rte_get_timer_hz(); 2285 #endif 2286 fsm = &fwd_streams[fc->stream_idx]; 2287 nb_fs = fc->stream_nb; 2288 prev_tsc = rte_rdtsc(); 2289 do { 2290 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2291 struct fwd_stream *fs = fsm[sm_id]; 2292 uint64_t start_fs_tsc = 0; 2293 bool busy; 2294 2295 if (fs->disabled) 2296 continue; 2297 if (record_core_cycles) 2298 start_fs_tsc = rte_rdtsc(); 2299 busy = (*pkt_fwd)(fs); 2300 if (record_core_cycles && busy) 2301 fs->busy_cycles += rte_rdtsc() - start_fs_tsc; 2302 } 2303 #ifdef RTE_LIB_BITRATESTATS 2304 if (bitrate_enabled != 0 && 2305 bitrate_lcore_id == rte_lcore_id()) { 2306 tics_current = rte_rdtsc(); 2307 if (tics_current - tics_datum >= tics_per_1sec) { 2308 /* Periodic bitrate calculation */ 2309 for (i = 0; i < cnt_ports; i++) 2310 rte_stats_bitrate_calc(bitrate_data, 2311 ports_ids[i]); 2312 tics_datum = tics_current; 2313 } 2314 } 2315 #endif 2316 #ifdef RTE_LIB_LATENCYSTATS 2317 if (latencystats_enabled != 0 && 2318 latencystats_lcore_id == rte_lcore_id()) 2319 rte_latencystats_update(); 2320 #endif 2321 if (record_core_cycles) { 2322 uint64_t tsc = rte_rdtsc(); 2323 2324 fc->total_cycles += tsc - prev_tsc; 2325 prev_tsc = tsc; 2326 } 2327 } while (! fc->stopped); 2328 } 2329 2330 static int 2331 lcore_usage_callback(unsigned int lcore_id, struct rte_lcore_usage *usage) 2332 { 2333 struct fwd_stream **fsm; 2334 struct fwd_lcore *fc; 2335 streamid_t nb_fs; 2336 streamid_t sm_id; 2337 2338 fc = lcore_to_fwd_lcore(lcore_id); 2339 if (fc == NULL) 2340 return -1; 2341 2342 fsm = &fwd_streams[fc->stream_idx]; 2343 nb_fs = fc->stream_nb; 2344 usage->busy_cycles = 0; 2345 usage->total_cycles = fc->total_cycles; 2346 2347 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2348 if (!fsm[sm_id]->disabled) 2349 usage->busy_cycles += fsm[sm_id]->busy_cycles; 2350 } 2351 2352 return 0; 2353 } 2354 2355 static int 2356 start_pkt_forward_on_core(void *fwd_arg) 2357 { 2358 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2359 cur_fwd_config.fwd_eng->packet_fwd); 2360 return 0; 2361 } 2362 2363 /* 2364 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2365 * Used to start communication flows in network loopback test configurations. 2366 */ 2367 static int 2368 run_one_txonly_burst_on_core(void *fwd_arg) 2369 { 2370 struct fwd_lcore *fwd_lc; 2371 struct fwd_lcore tmp_lcore; 2372 2373 fwd_lc = (struct fwd_lcore *) fwd_arg; 2374 tmp_lcore = *fwd_lc; 2375 tmp_lcore.stopped = 1; 2376 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2377 return 0; 2378 } 2379 2380 /* 2381 * Launch packet forwarding: 2382 * - Setup per-port forwarding context. 2383 * - launch logical cores with their forwarding configuration. 2384 */ 2385 static void 2386 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2387 { 2388 unsigned int i; 2389 unsigned int lc_id; 2390 int diag; 2391 2392 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2393 lc_id = fwd_lcores_cpuids[i]; 2394 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2395 fwd_lcores[i]->stopped = 0; 2396 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2397 fwd_lcores[i], lc_id); 2398 if (diag != 0) 2399 fprintf(stderr, 2400 "launch lcore %u failed - diag=%d\n", 2401 lc_id, diag); 2402 } 2403 } 2404 } 2405 2406 void 2407 common_fwd_stream_init(struct fwd_stream *fs) 2408 { 2409 bool rx_stopped, tx_stopped; 2410 2411 rx_stopped = (ports[fs->rx_port].rxq[fs->rx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2412 tx_stopped = (ports[fs->tx_port].txq[fs->tx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2413 fs->disabled = rx_stopped || tx_stopped; 2414 } 2415 2416 static void 2417 update_rx_queue_state(uint16_t port_id, uint16_t queue_id) 2418 { 2419 struct rte_eth_rxq_info rx_qinfo; 2420 int32_t rc; 2421 2422 rc = rte_eth_rx_queue_info_get(port_id, 2423 queue_id, &rx_qinfo); 2424 if (rc == 0) { 2425 ports[port_id].rxq[queue_id].state = 2426 rx_qinfo.queue_state; 2427 } else if (rc == -ENOTSUP) { 2428 /* 2429 * Set the rxq state to RTE_ETH_QUEUE_STATE_STARTED 2430 * to ensure that the PMDs do not implement 2431 * rte_eth_rx_queue_info_get can forward. 2432 */ 2433 ports[port_id].rxq[queue_id].state = 2434 RTE_ETH_QUEUE_STATE_STARTED; 2435 } else { 2436 TESTPMD_LOG(WARNING, 2437 "Failed to get rx queue info\n"); 2438 } 2439 } 2440 2441 static void 2442 update_tx_queue_state(uint16_t port_id, uint16_t queue_id) 2443 { 2444 struct rte_eth_txq_info tx_qinfo; 2445 int32_t rc; 2446 2447 rc = rte_eth_tx_queue_info_get(port_id, 2448 queue_id, &tx_qinfo); 2449 if (rc == 0) { 2450 ports[port_id].txq[queue_id].state = 2451 tx_qinfo.queue_state; 2452 } else if (rc == -ENOTSUP) { 2453 /* 2454 * Set the txq state to RTE_ETH_QUEUE_STATE_STARTED 2455 * to ensure that the PMDs do not implement 2456 * rte_eth_tx_queue_info_get can forward. 2457 */ 2458 ports[port_id].txq[queue_id].state = 2459 RTE_ETH_QUEUE_STATE_STARTED; 2460 } else { 2461 TESTPMD_LOG(WARNING, 2462 "Failed to get tx queue info\n"); 2463 } 2464 } 2465 2466 static void 2467 update_queue_state(void) 2468 { 2469 portid_t pi; 2470 queueid_t qi; 2471 2472 RTE_ETH_FOREACH_DEV(pi) { 2473 for (qi = 0; qi < nb_rxq; qi++) 2474 update_rx_queue_state(pi, qi); 2475 for (qi = 0; qi < nb_txq; qi++) 2476 update_tx_queue_state(pi, qi); 2477 } 2478 } 2479 2480 /* 2481 * Launch packet forwarding configuration. 2482 */ 2483 void 2484 start_packet_forwarding(int with_tx_first) 2485 { 2486 port_fwd_begin_t port_fwd_begin; 2487 port_fwd_end_t port_fwd_end; 2488 stream_init_t stream_init = cur_fwd_eng->stream_init; 2489 unsigned int i; 2490 2491 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2492 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2493 2494 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2495 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2496 2497 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2498 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2499 (!nb_rxq || !nb_txq)) 2500 rte_exit(EXIT_FAILURE, 2501 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2502 cur_fwd_eng->fwd_mode_name); 2503 2504 if (all_ports_started() == 0) { 2505 fprintf(stderr, "Not all ports were started\n"); 2506 return; 2507 } 2508 if (test_done == 0) { 2509 fprintf(stderr, "Packet forwarding already started\n"); 2510 return; 2511 } 2512 2513 fwd_config_setup(); 2514 2515 pkt_fwd_config_display(&cur_fwd_config); 2516 if (!pkt_fwd_shared_rxq_check()) 2517 return; 2518 2519 if (stream_init != NULL) { 2520 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 2521 update_queue_state(); 2522 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2523 stream_init(fwd_streams[i]); 2524 } 2525 2526 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2527 if (port_fwd_begin != NULL) { 2528 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2529 if (port_fwd_begin(fwd_ports_ids[i])) { 2530 fprintf(stderr, 2531 "Packet forwarding is not ready\n"); 2532 return; 2533 } 2534 } 2535 } 2536 2537 if (with_tx_first) { 2538 port_fwd_begin = tx_only_engine.port_fwd_begin; 2539 if (port_fwd_begin != NULL) { 2540 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2541 if (port_fwd_begin(fwd_ports_ids[i])) { 2542 fprintf(stderr, 2543 "Packet forwarding is not ready\n"); 2544 return; 2545 } 2546 } 2547 } 2548 } 2549 2550 test_done = 0; 2551 2552 if(!no_flush_rx) 2553 flush_fwd_rx_queues(); 2554 2555 rxtx_config_display(); 2556 2557 fwd_stats_reset(); 2558 if (with_tx_first) { 2559 while (with_tx_first--) { 2560 launch_packet_forwarding( 2561 run_one_txonly_burst_on_core); 2562 rte_eal_mp_wait_lcore(); 2563 } 2564 port_fwd_end = tx_only_engine.port_fwd_end; 2565 if (port_fwd_end != NULL) { 2566 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2567 (*port_fwd_end)(fwd_ports_ids[i]); 2568 } 2569 } 2570 launch_packet_forwarding(start_pkt_forward_on_core); 2571 } 2572 2573 void 2574 stop_packet_forwarding(void) 2575 { 2576 port_fwd_end_t port_fwd_end; 2577 lcoreid_t lc_id; 2578 portid_t pt_id; 2579 int i; 2580 2581 if (test_done) { 2582 fprintf(stderr, "Packet forwarding not started\n"); 2583 return; 2584 } 2585 printf("Telling cores to stop..."); 2586 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2587 fwd_lcores[lc_id]->stopped = 1; 2588 printf("\nWaiting for lcores to finish...\n"); 2589 rte_eal_mp_wait_lcore(); 2590 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2591 if (port_fwd_end != NULL) { 2592 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2593 pt_id = fwd_ports_ids[i]; 2594 (*port_fwd_end)(pt_id); 2595 } 2596 } 2597 2598 fwd_stats_display(); 2599 2600 printf("\nDone.\n"); 2601 test_done = 1; 2602 } 2603 2604 void 2605 dev_set_link_up(portid_t pid) 2606 { 2607 if (rte_eth_dev_set_link_up(pid) < 0) 2608 fprintf(stderr, "\nSet link up fail.\n"); 2609 } 2610 2611 void 2612 dev_set_link_down(portid_t pid) 2613 { 2614 if (rte_eth_dev_set_link_down(pid) < 0) 2615 fprintf(stderr, "\nSet link down fail.\n"); 2616 } 2617 2618 static int 2619 all_ports_started(void) 2620 { 2621 portid_t pi; 2622 struct rte_port *port; 2623 2624 RTE_ETH_FOREACH_DEV(pi) { 2625 port = &ports[pi]; 2626 /* Check if there is a port which is not started */ 2627 if ((port->port_status != RTE_PORT_STARTED) && 2628 (port->member_flag == 0)) 2629 return 0; 2630 } 2631 2632 /* No port is not started */ 2633 return 1; 2634 } 2635 2636 int 2637 port_is_stopped(portid_t port_id) 2638 { 2639 struct rte_port *port = &ports[port_id]; 2640 2641 if ((port->port_status != RTE_PORT_STOPPED) && 2642 (port->member_flag == 0)) 2643 return 0; 2644 return 1; 2645 } 2646 2647 int 2648 all_ports_stopped(void) 2649 { 2650 portid_t pi; 2651 2652 RTE_ETH_FOREACH_DEV(pi) { 2653 if (!port_is_stopped(pi)) 2654 return 0; 2655 } 2656 2657 return 1; 2658 } 2659 2660 int 2661 port_is_started(portid_t port_id) 2662 { 2663 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2664 return 0; 2665 2666 if (ports[port_id].port_status != RTE_PORT_STARTED) 2667 return 0; 2668 2669 return 1; 2670 } 2671 2672 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2673 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2674 2675 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2676 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2677 2678 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2679 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2680 2681 2682 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2683 static int 2684 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2685 { 2686 queueid_t qi; 2687 struct rte_eth_hairpin_conf hairpin_conf = { 2688 .peer_count = 1, 2689 }; 2690 int i; 2691 int diag; 2692 struct rte_port *port = &ports[pi]; 2693 uint16_t peer_rx_port = pi; 2694 uint16_t peer_tx_port = pi; 2695 uint32_t manual = 1; 2696 uint32_t tx_exp = hairpin_mode & 0x10; 2697 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2698 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2699 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2700 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2701 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2702 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2703 2704 if (!(hairpin_mode & 0xf)) { 2705 peer_rx_port = pi; 2706 peer_tx_port = pi; 2707 manual = 0; 2708 } else if (hairpin_mode & 0x1) { 2709 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2710 RTE_ETH_DEV_NO_OWNER); 2711 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2712 peer_tx_port = rte_eth_find_next_owned_by(0, 2713 RTE_ETH_DEV_NO_OWNER); 2714 if (p_pi != RTE_MAX_ETHPORTS) { 2715 peer_rx_port = p_pi; 2716 } else { 2717 uint16_t next_pi; 2718 2719 /* Last port will be the peer RX port of the first. */ 2720 RTE_ETH_FOREACH_DEV(next_pi) 2721 peer_rx_port = next_pi; 2722 } 2723 manual = 1; 2724 } else if (hairpin_mode & 0x2) { 2725 if (cnt_pi & 0x1) { 2726 peer_rx_port = p_pi; 2727 } else { 2728 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2729 RTE_ETH_DEV_NO_OWNER); 2730 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2731 peer_rx_port = pi; 2732 } 2733 peer_tx_port = peer_rx_port; 2734 manual = 1; 2735 } 2736 2737 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2738 hairpin_conf.peers[0].port = peer_rx_port; 2739 hairpin_conf.peers[0].queue = i + nb_rxq; 2740 hairpin_conf.manual_bind = !!manual; 2741 hairpin_conf.tx_explicit = !!tx_exp; 2742 hairpin_conf.force_memory = !!tx_force_memory; 2743 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2744 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2745 diag = rte_eth_tx_hairpin_queue_setup 2746 (pi, qi, nb_txd, &hairpin_conf); 2747 i++; 2748 if (diag == 0) 2749 continue; 2750 2751 /* Fail to setup rx queue, return */ 2752 if (port->port_status == RTE_PORT_HANDLING) 2753 port->port_status = RTE_PORT_STOPPED; 2754 else 2755 fprintf(stderr, 2756 "Port %d can not be set back to stopped\n", pi); 2757 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2758 pi); 2759 /* try to reconfigure queues next time */ 2760 port->need_reconfig_queues = 1; 2761 return -1; 2762 } 2763 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2764 hairpin_conf.peers[0].port = peer_tx_port; 2765 hairpin_conf.peers[0].queue = i + nb_txq; 2766 hairpin_conf.manual_bind = !!manual; 2767 hairpin_conf.tx_explicit = !!tx_exp; 2768 hairpin_conf.force_memory = !!rx_force_memory; 2769 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2770 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2771 diag = rte_eth_rx_hairpin_queue_setup 2772 (pi, qi, nb_rxd, &hairpin_conf); 2773 i++; 2774 if (diag == 0) 2775 continue; 2776 2777 /* Fail to setup rx queue, return */ 2778 if (port->port_status == RTE_PORT_HANDLING) 2779 port->port_status = RTE_PORT_STOPPED; 2780 else 2781 fprintf(stderr, 2782 "Port %d can not be set back to stopped\n", pi); 2783 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2784 pi); 2785 /* try to reconfigure queues next time */ 2786 port->need_reconfig_queues = 1; 2787 return -1; 2788 } 2789 return 0; 2790 } 2791 2792 /* Configure the Rx with optional split. */ 2793 int 2794 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2795 uint16_t nb_rx_desc, unsigned int socket_id, 2796 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2797 { 2798 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2799 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2800 struct rte_mempool *mpx; 2801 unsigned int i, mp_n; 2802 uint32_t prev_hdrs = 0; 2803 int ret; 2804 2805 2806 if ((rx_pkt_nb_segs > 1) && 2807 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2808 /* multi-segment configuration */ 2809 for (i = 0; i < rx_pkt_nb_segs; i++) { 2810 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2811 /* 2812 * Use last valid pool for the segments with number 2813 * exceeding the pool index. 2814 */ 2815 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2816 mpx = mbuf_pool_find(socket_id, mp_n); 2817 /* Handle zero as mbuf data buffer size. */ 2818 rx_seg->offset = i < rx_pkt_nb_offs ? 2819 rx_pkt_seg_offsets[i] : 0; 2820 rx_seg->mp = mpx ? mpx : mp; 2821 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2822 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2823 prev_hdrs |= rx_seg->proto_hdr; 2824 } else { 2825 rx_seg->length = rx_pkt_seg_lengths[i] ? 2826 rx_pkt_seg_lengths[i] : 2827 mbuf_data_size[mp_n]; 2828 } 2829 } 2830 rx_conf->rx_nseg = rx_pkt_nb_segs; 2831 rx_conf->rx_seg = rx_useg; 2832 rx_conf->rx_mempools = NULL; 2833 rx_conf->rx_nmempool = 0; 2834 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2835 socket_id, rx_conf, NULL); 2836 rx_conf->rx_seg = NULL; 2837 rx_conf->rx_nseg = 0; 2838 } else if (multi_rx_mempool == 1) { 2839 /* multi-pool configuration */ 2840 struct rte_eth_dev_info dev_info; 2841 2842 if (mbuf_data_size_n <= 1) { 2843 fprintf(stderr, "Invalid number of mempools %u\n", 2844 mbuf_data_size_n); 2845 return -EINVAL; 2846 } 2847 ret = rte_eth_dev_info_get(port_id, &dev_info); 2848 if (ret != 0) 2849 return ret; 2850 if (dev_info.max_rx_mempools == 0) { 2851 fprintf(stderr, 2852 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2853 port_id); 2854 return -ENOTSUP; 2855 } 2856 for (i = 0; i < mbuf_data_size_n; i++) { 2857 mpx = mbuf_pool_find(socket_id, i); 2858 rx_mempool[i] = mpx ? mpx : mp; 2859 } 2860 rx_conf->rx_mempools = rx_mempool; 2861 rx_conf->rx_nmempool = mbuf_data_size_n; 2862 rx_conf->rx_seg = NULL; 2863 rx_conf->rx_nseg = 0; 2864 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2865 socket_id, rx_conf, NULL); 2866 rx_conf->rx_mempools = NULL; 2867 rx_conf->rx_nmempool = 0; 2868 } else { 2869 /* Single pool/segment configuration */ 2870 rx_conf->rx_seg = NULL; 2871 rx_conf->rx_nseg = 0; 2872 rx_conf->rx_mempools = NULL; 2873 rx_conf->rx_nmempool = 0; 2874 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2875 socket_id, rx_conf, mp); 2876 } 2877 2878 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2879 RTE_ETH_QUEUE_STATE_STOPPED : 2880 RTE_ETH_QUEUE_STATE_STARTED; 2881 return ret; 2882 } 2883 2884 static int 2885 alloc_xstats_display_info(portid_t pi) 2886 { 2887 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2888 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2889 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2890 2891 if (xstats_display_num == 0) 2892 return 0; 2893 2894 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2895 if (*ids_supp == NULL) 2896 goto fail_ids_supp; 2897 2898 *prev_values = calloc(xstats_display_num, 2899 sizeof(**prev_values)); 2900 if (*prev_values == NULL) 2901 goto fail_prev_values; 2902 2903 *curr_values = calloc(xstats_display_num, 2904 sizeof(**curr_values)); 2905 if (*curr_values == NULL) 2906 goto fail_curr_values; 2907 2908 ports[pi].xstats_info.allocated = true; 2909 2910 return 0; 2911 2912 fail_curr_values: 2913 free(*prev_values); 2914 fail_prev_values: 2915 free(*ids_supp); 2916 fail_ids_supp: 2917 return -ENOMEM; 2918 } 2919 2920 static void 2921 free_xstats_display_info(portid_t pi) 2922 { 2923 if (!ports[pi].xstats_info.allocated) 2924 return; 2925 free(ports[pi].xstats_info.ids_supp); 2926 free(ports[pi].xstats_info.prev_values); 2927 free(ports[pi].xstats_info.curr_values); 2928 ports[pi].xstats_info.allocated = false; 2929 } 2930 2931 /** Fill helper structures for specified port to show extended statistics. */ 2932 static void 2933 fill_xstats_display_info_for_port(portid_t pi) 2934 { 2935 unsigned int stat, stat_supp; 2936 const char *xstat_name; 2937 struct rte_port *port; 2938 uint64_t *ids_supp; 2939 int rc; 2940 2941 if (xstats_display_num == 0) 2942 return; 2943 2944 if (pi == (portid_t)RTE_PORT_ALL) { 2945 fill_xstats_display_info(); 2946 return; 2947 } 2948 2949 port = &ports[pi]; 2950 if (port->port_status != RTE_PORT_STARTED) 2951 return; 2952 2953 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2954 rte_exit(EXIT_FAILURE, 2955 "Failed to allocate xstats display memory\n"); 2956 2957 ids_supp = port->xstats_info.ids_supp; 2958 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2959 xstat_name = xstats_display[stat].name; 2960 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2961 ids_supp + stat_supp); 2962 if (rc != 0) { 2963 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2964 xstat_name, pi, stat); 2965 continue; 2966 } 2967 stat_supp++; 2968 } 2969 2970 port->xstats_info.ids_supp_sz = stat_supp; 2971 } 2972 2973 /** Fill helper structures for all ports to show extended statistics. */ 2974 static void 2975 fill_xstats_display_info(void) 2976 { 2977 portid_t pi; 2978 2979 if (xstats_display_num == 0) 2980 return; 2981 2982 RTE_ETH_FOREACH_DEV(pi) 2983 fill_xstats_display_info_for_port(pi); 2984 } 2985 2986 /* 2987 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2988 * device in dev_info is zero when no member is added. And its capability 2989 * will be updated when add a new member device. So adding a member device need 2990 * to update the port configurations of bonding device. 2991 */ 2992 static void 2993 update_bonding_port_dev_conf(portid_t bond_pid) 2994 { 2995 #ifdef RTE_NET_BOND 2996 struct rte_port *port = &ports[bond_pid]; 2997 uint16_t i; 2998 int ret; 2999 3000 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 3001 if (ret != 0) { 3002 fprintf(stderr, "Failed to get dev info for port = %u\n", 3003 bond_pid); 3004 return; 3005 } 3006 3007 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 3008 port->dev_conf.txmode.offloads |= 3009 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 3010 /* Apply Tx offloads configuration */ 3011 for (i = 0; i < port->dev_info.max_tx_queues; i++) 3012 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 3013 3014 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 3015 port->dev_info.flow_type_rss_offloads; 3016 #else 3017 RTE_SET_USED(bond_pid); 3018 #endif 3019 } 3020 3021 int 3022 start_port(portid_t pid) 3023 { 3024 int diag; 3025 portid_t pi; 3026 portid_t p_pi = RTE_MAX_ETHPORTS; 3027 portid_t pl[RTE_MAX_ETHPORTS]; 3028 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3029 uint16_t cnt_pi = 0; 3030 uint16_t cfg_pi = 0; 3031 int peer_pi; 3032 queueid_t qi; 3033 struct rte_port *port; 3034 struct rte_eth_hairpin_cap cap; 3035 bool at_least_one_port_exist = false; 3036 bool all_ports_already_started = true; 3037 bool at_least_one_port_successfully_started = false; 3038 3039 if (port_id_is_invalid(pid, ENABLED_WARN)) 3040 return 0; 3041 3042 RTE_ETH_FOREACH_DEV(pi) { 3043 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3044 continue; 3045 3046 if (port_is_bonding_member(pi)) { 3047 fprintf(stderr, 3048 "Please remove port %d from bonding device.\n", 3049 pi); 3050 continue; 3051 } 3052 3053 at_least_one_port_exist = true; 3054 3055 port = &ports[pi]; 3056 if (port->port_status == RTE_PORT_STOPPED) { 3057 port->port_status = RTE_PORT_HANDLING; 3058 all_ports_already_started = false; 3059 } else { 3060 fprintf(stderr, "Port %d is now not stopped\n", pi); 3061 continue; 3062 } 3063 3064 if (port->need_reconfig > 0) { 3065 struct rte_eth_conf dev_conf; 3066 int k; 3067 3068 port->need_reconfig = 0; 3069 3070 if (flow_isolate_all) { 3071 int ret = port_flow_isolate(pi, 1); 3072 if (ret) { 3073 fprintf(stderr, 3074 "Failed to apply isolated mode on port %d\n", 3075 pi); 3076 return -1; 3077 } 3078 } 3079 configure_rxtx_dump_callbacks(0); 3080 printf("Configuring Port %d (socket %u)\n", pi, 3081 port->socket_id); 3082 if (nb_hairpinq > 0 && 3083 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 3084 fprintf(stderr, 3085 "Port %d doesn't support hairpin queues\n", 3086 pi); 3087 return -1; 3088 } 3089 3090 if (port->bond_flag == 1 && port->update_conf == 1) { 3091 update_bonding_port_dev_conf(pi); 3092 port->update_conf = 0; 3093 } 3094 3095 /* configure port */ 3096 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 3097 nb_txq + nb_hairpinq, 3098 &(port->dev_conf)); 3099 if (diag != 0) { 3100 if (port->port_status == RTE_PORT_HANDLING) 3101 port->port_status = RTE_PORT_STOPPED; 3102 else 3103 fprintf(stderr, 3104 "Port %d can not be set back to stopped\n", 3105 pi); 3106 fprintf(stderr, "Fail to configure port %d\n", 3107 pi); 3108 /* try to reconfigure port next time */ 3109 port->need_reconfig = 1; 3110 return -1; 3111 } 3112 /* get device configuration*/ 3113 if (0 != 3114 eth_dev_conf_get_print_err(pi, &dev_conf)) { 3115 fprintf(stderr, 3116 "port %d can not get device configuration\n", 3117 pi); 3118 return -1; 3119 } 3120 /* Apply Rx offloads configuration */ 3121 if (dev_conf.rxmode.offloads != 3122 port->dev_conf.rxmode.offloads) { 3123 port->dev_conf.rxmode.offloads |= 3124 dev_conf.rxmode.offloads; 3125 for (k = 0; 3126 k < port->dev_info.max_rx_queues; 3127 k++) 3128 port->rxq[k].conf.offloads |= 3129 dev_conf.rxmode.offloads; 3130 } 3131 /* Apply Tx offloads configuration */ 3132 if (dev_conf.txmode.offloads != 3133 port->dev_conf.txmode.offloads) { 3134 port->dev_conf.txmode.offloads |= 3135 dev_conf.txmode.offloads; 3136 for (k = 0; 3137 k < port->dev_info.max_tx_queues; 3138 k++) 3139 port->txq[k].conf.offloads |= 3140 dev_conf.txmode.offloads; 3141 } 3142 } 3143 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 3144 port->need_reconfig_queues = 0; 3145 /* setup tx queues */ 3146 for (qi = 0; qi < nb_txq; qi++) { 3147 struct rte_eth_txconf *conf = 3148 &port->txq[qi].conf; 3149 3150 if ((numa_support) && 3151 (txring_numa[pi] != NUMA_NO_CONFIG)) 3152 diag = rte_eth_tx_queue_setup(pi, qi, 3153 port->nb_tx_desc[qi], 3154 txring_numa[pi], 3155 &(port->txq[qi].conf)); 3156 else 3157 diag = rte_eth_tx_queue_setup(pi, qi, 3158 port->nb_tx_desc[qi], 3159 port->socket_id, 3160 &(port->txq[qi].conf)); 3161 3162 if (diag == 0) { 3163 port->txq[qi].state = 3164 conf->tx_deferred_start ? 3165 RTE_ETH_QUEUE_STATE_STOPPED : 3166 RTE_ETH_QUEUE_STATE_STARTED; 3167 continue; 3168 } 3169 3170 /* Fail to setup tx queue, return */ 3171 if (port->port_status == RTE_PORT_HANDLING) 3172 port->port_status = RTE_PORT_STOPPED; 3173 else 3174 fprintf(stderr, 3175 "Port %d can not be set back to stopped\n", 3176 pi); 3177 fprintf(stderr, 3178 "Fail to configure port %d tx queues\n", 3179 pi); 3180 /* try to reconfigure queues next time */ 3181 port->need_reconfig_queues = 1; 3182 return -1; 3183 } 3184 for (qi = 0; qi < nb_rxq; qi++) { 3185 /* setup rx queues */ 3186 if ((numa_support) && 3187 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3188 struct rte_mempool * mp = 3189 mbuf_pool_find 3190 (rxring_numa[pi], 0); 3191 if (mp == NULL) { 3192 fprintf(stderr, 3193 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3194 rxring_numa[pi]); 3195 return -1; 3196 } 3197 3198 diag = rx_queue_setup(pi, qi, 3199 port->nb_rx_desc[qi], 3200 rxring_numa[pi], 3201 &(port->rxq[qi].conf), 3202 mp); 3203 } else { 3204 struct rte_mempool *mp = 3205 mbuf_pool_find 3206 (port->socket_id, 0); 3207 if (mp == NULL) { 3208 fprintf(stderr, 3209 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3210 port->socket_id); 3211 return -1; 3212 } 3213 diag = rx_queue_setup(pi, qi, 3214 port->nb_rx_desc[qi], 3215 port->socket_id, 3216 &(port->rxq[qi].conf), 3217 mp); 3218 } 3219 if (diag == 0) 3220 continue; 3221 3222 /* Fail to setup rx queue, return */ 3223 if (port->port_status == RTE_PORT_HANDLING) 3224 port->port_status = RTE_PORT_STOPPED; 3225 else 3226 fprintf(stderr, 3227 "Port %d can not be set back to stopped\n", 3228 pi); 3229 fprintf(stderr, 3230 "Fail to configure port %d rx queues\n", 3231 pi); 3232 /* try to reconfigure queues next time */ 3233 port->need_reconfig_queues = 1; 3234 return -1; 3235 } 3236 /* setup hairpin queues */ 3237 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3238 return -1; 3239 } 3240 configure_rxtx_dump_callbacks(verbose_level); 3241 if (clear_ptypes) { 3242 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3243 NULL, 0); 3244 if (diag < 0) 3245 fprintf(stderr, 3246 "Port %d: Failed to disable Ptype parsing\n", 3247 pi); 3248 } 3249 3250 p_pi = pi; 3251 cnt_pi++; 3252 3253 /* start port */ 3254 diag = eth_dev_start_mp(pi); 3255 if (diag < 0) { 3256 fprintf(stderr, "Fail to start port %d: %s\n", 3257 pi, rte_strerror(-diag)); 3258 3259 /* Fail to setup rx queue, return */ 3260 if (port->port_status == RTE_PORT_HANDLING) 3261 port->port_status = RTE_PORT_STOPPED; 3262 else 3263 fprintf(stderr, 3264 "Port %d can not be set back to stopped\n", 3265 pi); 3266 continue; 3267 } 3268 3269 if (port->port_status == RTE_PORT_HANDLING) 3270 port->port_status = RTE_PORT_STARTED; 3271 else 3272 fprintf(stderr, "Port %d can not be set into started\n", 3273 pi); 3274 3275 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3276 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3277 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3278 3279 at_least_one_port_successfully_started = true; 3280 3281 pl[cfg_pi++] = pi; 3282 } 3283 3284 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3285 update_queue_state(); 3286 3287 if (at_least_one_port_successfully_started && !no_link_check) 3288 check_all_ports_link_status(RTE_PORT_ALL); 3289 else if (at_least_one_port_exist & all_ports_already_started) 3290 fprintf(stderr, "Please stop the ports first\n"); 3291 3292 if (hairpin_mode & 0xf) { 3293 uint16_t i; 3294 int j; 3295 3296 /* bind all started hairpin ports */ 3297 for (i = 0; i < cfg_pi; i++) { 3298 pi = pl[i]; 3299 /* bind current Tx to all peer Rx */ 3300 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3301 RTE_MAX_ETHPORTS, 1); 3302 if (peer_pi < 0) 3303 return peer_pi; 3304 for (j = 0; j < peer_pi; j++) { 3305 if (!port_is_started(peer_pl[j])) 3306 continue; 3307 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3308 if (diag < 0) { 3309 fprintf(stderr, 3310 "Error during binding hairpin Tx port %u to %u: %s\n", 3311 pi, peer_pl[j], 3312 rte_strerror(-diag)); 3313 return -1; 3314 } 3315 } 3316 /* bind all peer Tx to current Rx */ 3317 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3318 RTE_MAX_ETHPORTS, 0); 3319 if (peer_pi < 0) 3320 return peer_pi; 3321 for (j = 0; j < peer_pi; j++) { 3322 if (!port_is_started(peer_pl[j])) 3323 continue; 3324 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3325 if (diag < 0) { 3326 fprintf(stderr, 3327 "Error during binding hairpin Tx port %u to %u: %s\n", 3328 peer_pl[j], pi, 3329 rte_strerror(-diag)); 3330 return -1; 3331 } 3332 } 3333 } 3334 } 3335 3336 fill_xstats_display_info_for_port(pid); 3337 3338 printf("Done\n"); 3339 return 0; 3340 } 3341 3342 void 3343 stop_port(portid_t pid) 3344 { 3345 portid_t pi; 3346 struct rte_port *port; 3347 int need_check_link_status = 0; 3348 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3349 int peer_pi; 3350 int ret; 3351 3352 if (port_id_is_invalid(pid, ENABLED_WARN)) 3353 return; 3354 3355 printf("Stopping ports...\n"); 3356 3357 RTE_ETH_FOREACH_DEV(pi) { 3358 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3359 continue; 3360 3361 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3362 fprintf(stderr, 3363 "Please remove port %d from forwarding configuration.\n", 3364 pi); 3365 continue; 3366 } 3367 3368 if (port_is_bonding_member(pi)) { 3369 fprintf(stderr, 3370 "Please remove port %d from bonding device.\n", 3371 pi); 3372 continue; 3373 } 3374 3375 port = &ports[pi]; 3376 if (port->port_status == RTE_PORT_STARTED) 3377 port->port_status = RTE_PORT_HANDLING; 3378 else 3379 continue; 3380 3381 if (hairpin_mode & 0xf) { 3382 int j; 3383 3384 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3385 /* unbind all peer Tx from current Rx */ 3386 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3387 RTE_MAX_ETHPORTS, 0); 3388 if (peer_pi < 0) 3389 continue; 3390 for (j = 0; j < peer_pi; j++) { 3391 if (!port_is_started(peer_pl[j])) 3392 continue; 3393 rte_eth_hairpin_unbind(peer_pl[j], pi); 3394 } 3395 } 3396 3397 if (port->flow_list && !no_flow_flush) 3398 port_flow_flush(pi); 3399 3400 ret = eth_dev_stop_mp(pi); 3401 if (ret != 0) { 3402 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3403 pi); 3404 /* Allow to retry stopping the port. */ 3405 port->port_status = RTE_PORT_STARTED; 3406 continue; 3407 } 3408 3409 if (port->port_status == RTE_PORT_HANDLING) 3410 port->port_status = RTE_PORT_STOPPED; 3411 else 3412 fprintf(stderr, "Port %d can not be set into stopped\n", 3413 pi); 3414 need_check_link_status = 1; 3415 } 3416 if (need_check_link_status && !no_link_check) 3417 check_all_ports_link_status(RTE_PORT_ALL); 3418 3419 printf("Done\n"); 3420 } 3421 3422 static void 3423 remove_invalid_ports_in(portid_t *array, portid_t *total) 3424 { 3425 portid_t i; 3426 portid_t new_total = 0; 3427 3428 for (i = 0; i < *total; i++) 3429 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3430 array[new_total] = array[i]; 3431 new_total++; 3432 } 3433 *total = new_total; 3434 } 3435 3436 static void 3437 remove_invalid_ports(void) 3438 { 3439 remove_invalid_ports_in(ports_ids, &nb_ports); 3440 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3441 nb_cfg_ports = nb_fwd_ports; 3442 } 3443 3444 static void 3445 flush_port_owned_resources(portid_t pi) 3446 { 3447 mcast_addr_pool_destroy(pi); 3448 port_flow_flush(pi); 3449 port_flow_template_table_flush(pi); 3450 port_flow_pattern_template_flush(pi); 3451 port_flow_actions_template_flush(pi); 3452 port_flex_item_flush(pi); 3453 port_action_handle_flush(pi); 3454 } 3455 3456 static void 3457 clear_bonding_member_device(portid_t *member_pids, uint16_t num_members) 3458 { 3459 struct rte_port *port; 3460 portid_t member_pid; 3461 uint16_t i; 3462 3463 for (i = 0; i < num_members; i++) { 3464 member_pid = member_pids[i]; 3465 if (port_is_started(member_pid) == 1) { 3466 if (rte_eth_dev_stop(member_pid) != 0) 3467 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3468 member_pid); 3469 3470 port = &ports[member_pid]; 3471 port->port_status = RTE_PORT_STOPPED; 3472 } 3473 3474 clear_port_member_flag(member_pid); 3475 3476 /* Close member device when testpmd quit or is killed. */ 3477 if (cl_quit == 1 || f_quit == 1) 3478 rte_eth_dev_close(member_pid); 3479 } 3480 } 3481 3482 void 3483 close_port(portid_t pid) 3484 { 3485 portid_t pi; 3486 struct rte_port *port; 3487 portid_t member_pids[RTE_MAX_ETHPORTS]; 3488 int num_members = 0; 3489 3490 if (port_id_is_invalid(pid, ENABLED_WARN)) 3491 return; 3492 3493 printf("Closing ports...\n"); 3494 3495 RTE_ETH_FOREACH_DEV(pi) { 3496 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3497 continue; 3498 3499 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3500 fprintf(stderr, 3501 "Please remove port %d from forwarding configuration.\n", 3502 pi); 3503 continue; 3504 } 3505 3506 if (port_is_bonding_member(pi)) { 3507 fprintf(stderr, 3508 "Please remove port %d from bonding device.\n", 3509 pi); 3510 continue; 3511 } 3512 3513 port = &ports[pi]; 3514 if (port->port_status == RTE_PORT_CLOSED) { 3515 fprintf(stderr, "Port %d is already closed\n", pi); 3516 continue; 3517 } 3518 3519 if (is_proc_primary()) { 3520 flush_port_owned_resources(pi); 3521 #ifdef RTE_NET_BOND 3522 if (port->bond_flag == 1) 3523 num_members = rte_eth_bond_members_get(pi, 3524 member_pids, RTE_MAX_ETHPORTS); 3525 #endif 3526 rte_eth_dev_close(pi); 3527 /* 3528 * If this port is bonding device, all members under the 3529 * device need to be removed or closed. 3530 */ 3531 if (port->bond_flag == 1 && num_members > 0) 3532 clear_bonding_member_device(member_pids, 3533 num_members); 3534 } 3535 3536 free_xstats_display_info(pi); 3537 } 3538 3539 remove_invalid_ports(); 3540 printf("Done\n"); 3541 } 3542 3543 void 3544 reset_port(portid_t pid) 3545 { 3546 int diag; 3547 portid_t pi; 3548 struct rte_port *port; 3549 3550 if (port_id_is_invalid(pid, ENABLED_WARN)) 3551 return; 3552 3553 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3554 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3555 fprintf(stderr, 3556 "Can not reset port(s), please stop port(s) first.\n"); 3557 return; 3558 } 3559 3560 printf("Resetting ports...\n"); 3561 3562 RTE_ETH_FOREACH_DEV(pi) { 3563 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3564 continue; 3565 3566 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3567 fprintf(stderr, 3568 "Please remove port %d from forwarding configuration.\n", 3569 pi); 3570 continue; 3571 } 3572 3573 if (port_is_bonding_member(pi)) { 3574 fprintf(stderr, 3575 "Please remove port %d from bonding device.\n", 3576 pi); 3577 continue; 3578 } 3579 3580 if (is_proc_primary()) { 3581 diag = rte_eth_dev_reset(pi); 3582 if (diag == 0) { 3583 port = &ports[pi]; 3584 port->need_reconfig = 1; 3585 port->need_reconfig_queues = 1; 3586 } else { 3587 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3588 pi, diag); 3589 } 3590 } 3591 } 3592 3593 printf("Done\n"); 3594 } 3595 3596 void 3597 attach_port(char *identifier) 3598 { 3599 portid_t pi; 3600 struct rte_dev_iterator iterator; 3601 3602 printf("Attaching a new port...\n"); 3603 3604 if (identifier == NULL) { 3605 fprintf(stderr, "Invalid parameters are specified\n"); 3606 return; 3607 } 3608 3609 if (rte_dev_probe(identifier) < 0) { 3610 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3611 return; 3612 } 3613 3614 /* first attach mode: event */ 3615 if (setup_on_probe_event) { 3616 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3617 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3618 if (ports[pi].port_status == RTE_PORT_HANDLING && 3619 ports[pi].need_setup != 0) 3620 setup_attached_port(pi); 3621 return; 3622 } 3623 3624 /* second attach mode: iterator */ 3625 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3626 /* setup ports matching the devargs used for probing */ 3627 if (port_is_forwarding(pi)) 3628 continue; /* port was already attached before */ 3629 setup_attached_port(pi); 3630 } 3631 } 3632 3633 static void 3634 setup_attached_port(portid_t pi) 3635 { 3636 unsigned int socket_id; 3637 int ret; 3638 3639 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3640 /* if socket_id is invalid, set to the first available socket. */ 3641 if (check_socket_id(socket_id) < 0) 3642 socket_id = socket_ids[0]; 3643 reconfig(pi, socket_id); 3644 ret = rte_eth_promiscuous_enable(pi); 3645 if (ret != 0) 3646 fprintf(stderr, 3647 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3648 pi, rte_strerror(-ret)); 3649 3650 ports_ids[nb_ports++] = pi; 3651 fwd_ports_ids[nb_fwd_ports++] = pi; 3652 nb_cfg_ports = nb_fwd_ports; 3653 ports[pi].need_setup = 0; 3654 ports[pi].port_status = RTE_PORT_STOPPED; 3655 3656 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3657 printf("Done\n"); 3658 } 3659 3660 static void 3661 detach_device(struct rte_device *dev) 3662 { 3663 portid_t sibling; 3664 3665 if (dev == NULL) { 3666 fprintf(stderr, "Device already removed\n"); 3667 return; 3668 } 3669 3670 printf("Removing a device...\n"); 3671 3672 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3673 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3674 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3675 fprintf(stderr, "Port %u not stopped\n", 3676 sibling); 3677 return; 3678 } 3679 flush_port_owned_resources(sibling); 3680 } 3681 } 3682 3683 if (rte_dev_remove(dev) < 0) { 3684 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3685 return; 3686 } 3687 remove_invalid_ports(); 3688 3689 printf("Device is detached\n"); 3690 printf("Now total ports is %d\n", nb_ports); 3691 printf("Done\n"); 3692 return; 3693 } 3694 3695 void 3696 detach_port_device(portid_t port_id) 3697 { 3698 int ret; 3699 struct rte_eth_dev_info dev_info; 3700 3701 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3702 return; 3703 3704 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3705 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3706 fprintf(stderr, "Port not stopped\n"); 3707 return; 3708 } 3709 fprintf(stderr, "Port was not closed\n"); 3710 } 3711 3712 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3713 if (ret != 0) { 3714 TESTPMD_LOG(ERR, 3715 "Failed to get device info for port %d, not detaching\n", 3716 port_id); 3717 return; 3718 } 3719 detach_device(dev_info.device); 3720 } 3721 3722 void 3723 detach_devargs(char *identifier) 3724 { 3725 struct rte_dev_iterator iterator; 3726 struct rte_devargs da; 3727 portid_t port_id; 3728 3729 printf("Removing a device...\n"); 3730 3731 memset(&da, 0, sizeof(da)); 3732 if (rte_devargs_parsef(&da, "%s", identifier)) { 3733 fprintf(stderr, "cannot parse identifier\n"); 3734 return; 3735 } 3736 3737 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3738 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3739 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3740 fprintf(stderr, "Port %u not stopped\n", 3741 port_id); 3742 rte_eth_iterator_cleanup(&iterator); 3743 rte_devargs_reset(&da); 3744 return; 3745 } 3746 flush_port_owned_resources(port_id); 3747 } 3748 } 3749 3750 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3751 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3752 da.name, rte_bus_name(da.bus)); 3753 rte_devargs_reset(&da); 3754 return; 3755 } 3756 3757 remove_invalid_ports(); 3758 3759 printf("Device %s is detached\n", identifier); 3760 printf("Now total ports is %d\n", nb_ports); 3761 printf("Done\n"); 3762 rte_devargs_reset(&da); 3763 } 3764 3765 void 3766 pmd_test_exit(void) 3767 { 3768 portid_t pt_id; 3769 unsigned int i; 3770 int ret; 3771 3772 if (test_done == 0) 3773 stop_packet_forwarding(); 3774 3775 #ifndef RTE_EXEC_ENV_WINDOWS 3776 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3777 if (mempools[i]) { 3778 if (mp_alloc_type == MP_ALLOC_ANON) 3779 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3780 NULL); 3781 } 3782 } 3783 #endif 3784 if (ports != NULL) { 3785 no_link_check = 1; 3786 RTE_ETH_FOREACH_DEV(pt_id) { 3787 printf("\nStopping port %d...\n", pt_id); 3788 fflush(stdout); 3789 stop_port(pt_id); 3790 } 3791 RTE_ETH_FOREACH_DEV(pt_id) { 3792 printf("\nShutting down port %d...\n", pt_id); 3793 fflush(stdout); 3794 close_port(pt_id); 3795 } 3796 } 3797 3798 if (hot_plug) { 3799 ret = rte_dev_event_monitor_stop(); 3800 if (ret) { 3801 RTE_LOG(ERR, EAL, 3802 "fail to stop device event monitor."); 3803 return; 3804 } 3805 3806 ret = rte_dev_event_callback_unregister(NULL, 3807 dev_event_callback, NULL); 3808 if (ret < 0) { 3809 RTE_LOG(ERR, EAL, 3810 "fail to unregister device event callback.\n"); 3811 return; 3812 } 3813 3814 ret = rte_dev_hotplug_handle_disable(); 3815 if (ret) { 3816 RTE_LOG(ERR, EAL, 3817 "fail to disable hotplug handling.\n"); 3818 return; 3819 } 3820 } 3821 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3822 if (mempools[i]) 3823 mempool_free_mp(mempools[i]); 3824 } 3825 free(xstats_display); 3826 3827 printf("\nBye...\n"); 3828 } 3829 3830 typedef void (*cmd_func_t)(void); 3831 struct pmd_test_command { 3832 const char *cmd_name; 3833 cmd_func_t cmd_func; 3834 }; 3835 3836 /* Check the link status of all ports in up to 9s, and print them finally */ 3837 static void 3838 check_all_ports_link_status(uint32_t port_mask) 3839 { 3840 #define CHECK_INTERVAL 100 /* 100ms */ 3841 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3842 portid_t portid; 3843 uint8_t count, all_ports_up, print_flag = 0; 3844 struct rte_eth_link link; 3845 int ret; 3846 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3847 3848 printf("Checking link statuses...\n"); 3849 fflush(stdout); 3850 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3851 all_ports_up = 1; 3852 RTE_ETH_FOREACH_DEV(portid) { 3853 if ((port_mask & (1 << portid)) == 0) 3854 continue; 3855 memset(&link, 0, sizeof(link)); 3856 ret = rte_eth_link_get_nowait(portid, &link); 3857 if (ret < 0) { 3858 all_ports_up = 0; 3859 if (print_flag == 1) 3860 fprintf(stderr, 3861 "Port %u link get failed: %s\n", 3862 portid, rte_strerror(-ret)); 3863 continue; 3864 } 3865 /* print link status if flag set */ 3866 if (print_flag == 1) { 3867 rte_eth_link_to_str(link_status, 3868 sizeof(link_status), &link); 3869 printf("Port %d %s\n", portid, link_status); 3870 continue; 3871 } 3872 /* clear all_ports_up flag if any link down */ 3873 if (link.link_status == RTE_ETH_LINK_DOWN) { 3874 all_ports_up = 0; 3875 break; 3876 } 3877 } 3878 /* after finally printing all link status, get out */ 3879 if (print_flag == 1) 3880 break; 3881 3882 if (all_ports_up == 0) { 3883 fflush(stdout); 3884 rte_delay_ms(CHECK_INTERVAL); 3885 } 3886 3887 /* set the print_flag if all ports up or timeout */ 3888 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3889 print_flag = 1; 3890 } 3891 3892 if (lsc_interrupt) 3893 break; 3894 } 3895 } 3896 3897 static void 3898 rmv_port_callback(void *arg) 3899 { 3900 int need_to_start = 0; 3901 int org_no_link_check = no_link_check; 3902 portid_t port_id = (intptr_t)arg; 3903 struct rte_eth_dev_info dev_info; 3904 int ret; 3905 3906 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3907 3908 if (!test_done && port_is_forwarding(port_id)) { 3909 need_to_start = 1; 3910 stop_packet_forwarding(); 3911 } 3912 no_link_check = 1; 3913 stop_port(port_id); 3914 no_link_check = org_no_link_check; 3915 3916 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3917 if (ret != 0) 3918 TESTPMD_LOG(ERR, 3919 "Failed to get device info for port %d, not detaching\n", 3920 port_id); 3921 else { 3922 struct rte_device *device = dev_info.device; 3923 close_port(port_id); 3924 detach_device(device); /* might be already removed or have more ports */ 3925 } 3926 if (need_to_start) 3927 start_packet_forwarding(0); 3928 } 3929 3930 /* This function is used by the interrupt thread */ 3931 static int 3932 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3933 void *ret_param) 3934 { 3935 RTE_SET_USED(param); 3936 RTE_SET_USED(ret_param); 3937 3938 if (type >= RTE_ETH_EVENT_MAX) { 3939 fprintf(stderr, 3940 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3941 port_id, __func__, type); 3942 fflush(stderr); 3943 } else if (event_print_mask & (UINT32_C(1) << type)) { 3944 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3945 eth_event_desc[type]); 3946 fflush(stdout); 3947 } 3948 3949 switch (type) { 3950 case RTE_ETH_EVENT_NEW: 3951 ports[port_id].need_setup = 1; 3952 ports[port_id].port_status = RTE_PORT_HANDLING; 3953 break; 3954 case RTE_ETH_EVENT_INTR_RMV: 3955 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3956 break; 3957 if (rte_eal_alarm_set(100000, 3958 rmv_port_callback, (void *)(intptr_t)port_id)) 3959 fprintf(stderr, 3960 "Could not set up deferred device removal\n"); 3961 break; 3962 case RTE_ETH_EVENT_DESTROY: 3963 ports[port_id].port_status = RTE_PORT_CLOSED; 3964 printf("Port %u is closed\n", port_id); 3965 break; 3966 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3967 uint16_t rxq_id; 3968 int ret; 3969 3970 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3971 for (rxq_id = 0; ; rxq_id++) { 3972 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3973 NULL); 3974 if (ret <= 0) 3975 break; 3976 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3977 port_id, rxq_id); 3978 3979 #ifdef RTE_NET_MLX5 3980 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3981 #endif 3982 } 3983 break; 3984 } 3985 default: 3986 break; 3987 } 3988 return 0; 3989 } 3990 3991 static int 3992 register_eth_event_callback(void) 3993 { 3994 int ret; 3995 enum rte_eth_event_type event; 3996 3997 for (event = RTE_ETH_EVENT_UNKNOWN; 3998 event < RTE_ETH_EVENT_MAX; event++) { 3999 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 4000 event, 4001 eth_event_callback, 4002 NULL); 4003 if (ret != 0) { 4004 TESTPMD_LOG(ERR, "Failed to register callback for " 4005 "%s event\n", eth_event_desc[event]); 4006 return -1; 4007 } 4008 } 4009 4010 return 0; 4011 } 4012 4013 static int 4014 unregister_eth_event_callback(void) 4015 { 4016 int ret; 4017 enum rte_eth_event_type event; 4018 4019 for (event = RTE_ETH_EVENT_UNKNOWN; 4020 event < RTE_ETH_EVENT_MAX; event++) { 4021 ret = rte_eth_dev_callback_unregister(RTE_ETH_ALL, 4022 event, 4023 eth_event_callback, 4024 NULL); 4025 if (ret != 0) { 4026 TESTPMD_LOG(ERR, "Failed to unregister callback for " 4027 "%s event\n", eth_event_desc[event]); 4028 return -1; 4029 } 4030 } 4031 4032 return 0; 4033 } 4034 4035 /* This function is used by the interrupt thread */ 4036 static void 4037 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 4038 __rte_unused void *arg) 4039 { 4040 uint16_t port_id; 4041 int ret; 4042 4043 if (type >= RTE_DEV_EVENT_MAX) { 4044 fprintf(stderr, "%s called upon invalid event %d\n", 4045 __func__, type); 4046 fflush(stderr); 4047 } 4048 4049 switch (type) { 4050 case RTE_DEV_EVENT_REMOVE: 4051 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 4052 device_name); 4053 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 4054 if (ret) { 4055 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 4056 device_name); 4057 return; 4058 } 4059 /* 4060 * Because the user's callback is invoked in eal interrupt 4061 * callback, the interrupt callback need to be finished before 4062 * it can be unregistered when detaching device. So finish 4063 * callback soon and use a deferred removal to detach device 4064 * is need. It is a workaround, once the device detaching be 4065 * moved into the eal in the future, the deferred removal could 4066 * be deleted. 4067 */ 4068 if (rte_eal_alarm_set(100000, 4069 rmv_port_callback, (void *)(intptr_t)port_id)) 4070 RTE_LOG(ERR, EAL, 4071 "Could not set up deferred device removal\n"); 4072 break; 4073 case RTE_DEV_EVENT_ADD: 4074 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 4075 device_name); 4076 /* TODO: After finish kernel driver binding, 4077 * begin to attach port. 4078 */ 4079 break; 4080 default: 4081 break; 4082 } 4083 } 4084 4085 static void 4086 rxtx_port_config(portid_t pid) 4087 { 4088 uint16_t qid; 4089 uint64_t offloads; 4090 struct rte_port *port = &ports[pid]; 4091 4092 for (qid = 0; qid < nb_rxq; qid++) { 4093 offloads = port->rxq[qid].conf.offloads; 4094 port->rxq[qid].conf = port->dev_info.default_rxconf; 4095 4096 if (rxq_share > 0 && 4097 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 4098 /* Non-zero share group to enable RxQ share. */ 4099 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 4100 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 4101 } 4102 4103 if (offloads != 0) 4104 port->rxq[qid].conf.offloads = offloads; 4105 4106 /* Check if any Rx parameters have been passed */ 4107 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 4108 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 4109 4110 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 4111 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 4112 4113 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 4114 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 4115 4116 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 4117 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 4118 4119 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 4120 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 4121 4122 port->nb_rx_desc[qid] = nb_rxd; 4123 } 4124 4125 for (qid = 0; qid < nb_txq; qid++) { 4126 offloads = port->txq[qid].conf.offloads; 4127 port->txq[qid].conf = port->dev_info.default_txconf; 4128 if (offloads != 0) 4129 port->txq[qid].conf.offloads = offloads; 4130 4131 /* Check if any Tx parameters have been passed */ 4132 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 4133 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 4134 4135 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 4136 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 4137 4138 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 4139 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 4140 4141 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 4142 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 4143 4144 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 4145 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 4146 4147 port->nb_tx_desc[qid] = nb_txd; 4148 } 4149 } 4150 4151 /* 4152 * Helper function to set MTU from frame size 4153 * 4154 * port->dev_info should be set before calling this function. 4155 * 4156 * return 0 on success, negative on error 4157 */ 4158 int 4159 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 4160 { 4161 struct rte_port *port = &ports[portid]; 4162 uint32_t eth_overhead; 4163 uint16_t mtu, new_mtu; 4164 4165 eth_overhead = get_eth_overhead(&port->dev_info); 4166 4167 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 4168 printf("Failed to get MTU for port %u\n", portid); 4169 return -1; 4170 } 4171 4172 new_mtu = max_rx_pktlen - eth_overhead; 4173 4174 if (mtu == new_mtu) 4175 return 0; 4176 4177 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 4178 fprintf(stderr, 4179 "Failed to set MTU to %u for port %u\n", 4180 new_mtu, portid); 4181 return -1; 4182 } 4183 4184 port->dev_conf.rxmode.mtu = new_mtu; 4185 4186 return 0; 4187 } 4188 4189 void 4190 init_port_config(void) 4191 { 4192 portid_t pid; 4193 struct rte_port *port; 4194 int ret, i; 4195 4196 RTE_ETH_FOREACH_DEV(pid) { 4197 port = &ports[pid]; 4198 4199 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4200 if (ret != 0) 4201 return; 4202 4203 if (nb_rxq > 1) { 4204 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4205 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4206 rss_hf & port->dev_info.flow_type_rss_offloads; 4207 } else { 4208 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4209 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4210 } 4211 4212 if (port->dcb_flag == 0) { 4213 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4214 port->dev_conf.rxmode.mq_mode = 4215 (enum rte_eth_rx_mq_mode) 4216 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4217 } else { 4218 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4219 port->dev_conf.rxmode.offloads &= 4220 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4221 4222 for (i = 0; 4223 i < port->dev_info.nb_rx_queues; 4224 i++) 4225 port->rxq[i].conf.offloads &= 4226 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4227 } 4228 } 4229 4230 rxtx_port_config(pid); 4231 4232 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4233 if (ret != 0) 4234 return; 4235 4236 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4237 port->dev_conf.intr_conf.lsc = 1; 4238 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4239 port->dev_conf.intr_conf.rmv = 1; 4240 } 4241 } 4242 4243 void set_port_member_flag(portid_t member_pid) 4244 { 4245 struct rte_port *port; 4246 4247 port = &ports[member_pid]; 4248 port->member_flag = 1; 4249 } 4250 4251 void clear_port_member_flag(portid_t member_pid) 4252 { 4253 struct rte_port *port; 4254 4255 port = &ports[member_pid]; 4256 port->member_flag = 0; 4257 } 4258 4259 uint8_t port_is_bonding_member(portid_t member_pid) 4260 { 4261 struct rte_port *port; 4262 struct rte_eth_dev_info dev_info; 4263 int ret; 4264 4265 port = &ports[member_pid]; 4266 ret = eth_dev_info_get_print_err(member_pid, &dev_info); 4267 if (ret != 0) { 4268 TESTPMD_LOG(ERR, 4269 "Failed to get device info for port id %d," 4270 "cannot determine if the port is a bonding member", 4271 member_pid); 4272 return 0; 4273 } 4274 4275 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDING_MEMBER) || (port->member_flag == 1)) 4276 return 1; 4277 return 0; 4278 } 4279 4280 const uint16_t vlan_tags[] = { 4281 0, 1, 2, 3, 4, 5, 6, 7, 4282 8, 9, 10, 11, 12, 13, 14, 15, 4283 16, 17, 18, 19, 20, 21, 22, 23, 4284 24, 25, 26, 27, 28, 29, 30, 31 4285 }; 4286 4287 static int 4288 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4289 enum dcb_mode_enable dcb_mode, 4290 enum rte_eth_nb_tcs num_tcs, 4291 uint8_t pfc_en) 4292 { 4293 uint8_t i; 4294 int32_t rc; 4295 struct rte_eth_rss_conf rss_conf; 4296 4297 /* 4298 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4299 * given above, and the number of traffic classes available for use. 4300 */ 4301 if (dcb_mode == DCB_VT_ENABLED) { 4302 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4303 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4304 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4305 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4306 4307 /* VMDQ+DCB RX and TX configurations */ 4308 vmdq_rx_conf->enable_default_pool = 0; 4309 vmdq_rx_conf->default_pool = 0; 4310 vmdq_rx_conf->nb_queue_pools = 4311 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4312 vmdq_tx_conf->nb_queue_pools = 4313 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4314 4315 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4316 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4317 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4318 vmdq_rx_conf->pool_map[i].pools = 4319 1 << (i % vmdq_rx_conf->nb_queue_pools); 4320 } 4321 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4322 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4323 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4324 } 4325 4326 /* set DCB mode of RX and TX of multiple queues */ 4327 eth_conf->rxmode.mq_mode = 4328 (enum rte_eth_rx_mq_mode) 4329 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4330 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4331 } else { 4332 struct rte_eth_dcb_rx_conf *rx_conf = 4333 ð_conf->rx_adv_conf.dcb_rx_conf; 4334 struct rte_eth_dcb_tx_conf *tx_conf = 4335 ð_conf->tx_adv_conf.dcb_tx_conf; 4336 4337 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4338 4339 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4340 if (rc != 0) 4341 return rc; 4342 4343 rx_conf->nb_tcs = num_tcs; 4344 tx_conf->nb_tcs = num_tcs; 4345 4346 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4347 rx_conf->dcb_tc[i] = i % num_tcs; 4348 tx_conf->dcb_tc[i] = i % num_tcs; 4349 } 4350 4351 eth_conf->rxmode.mq_mode = 4352 (enum rte_eth_rx_mq_mode) 4353 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4354 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4355 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4356 } 4357 4358 if (pfc_en) 4359 eth_conf->dcb_capability_en = 4360 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4361 else 4362 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4363 4364 return 0; 4365 } 4366 4367 int 4368 init_port_dcb_config(portid_t pid, 4369 enum dcb_mode_enable dcb_mode, 4370 enum rte_eth_nb_tcs num_tcs, 4371 uint8_t pfc_en) 4372 { 4373 struct rte_eth_conf port_conf; 4374 struct rte_port *rte_port; 4375 int retval; 4376 uint16_t i; 4377 4378 if (num_procs > 1) { 4379 printf("The multi-process feature doesn't support dcb.\n"); 4380 return -ENOTSUP; 4381 } 4382 rte_port = &ports[pid]; 4383 4384 /* retain the original device configuration. */ 4385 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4386 4387 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4388 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4389 if (retval < 0) 4390 return retval; 4391 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4392 /* remove RSS HASH offload for DCB in vt mode */ 4393 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4394 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4395 for (i = 0; i < nb_rxq; i++) 4396 rte_port->rxq[i].conf.offloads &= 4397 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4398 } 4399 4400 /* re-configure the device . */ 4401 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4402 if (retval < 0) 4403 return retval; 4404 4405 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4406 if (retval != 0) 4407 return retval; 4408 4409 /* If dev_info.vmdq_pool_base is greater than 0, 4410 * the queue id of vmdq pools is started after pf queues. 4411 */ 4412 if (dcb_mode == DCB_VT_ENABLED && 4413 rte_port->dev_info.vmdq_pool_base > 0) { 4414 fprintf(stderr, 4415 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4416 pid); 4417 return -1; 4418 } 4419 4420 /* Assume the ports in testpmd have the same dcb capability 4421 * and has the same number of rxq and txq in dcb mode 4422 */ 4423 if (dcb_mode == DCB_VT_ENABLED) { 4424 if (rte_port->dev_info.max_vfs > 0) { 4425 nb_rxq = rte_port->dev_info.nb_rx_queues; 4426 nb_txq = rte_port->dev_info.nb_tx_queues; 4427 } else { 4428 nb_rxq = rte_port->dev_info.max_rx_queues; 4429 nb_txq = rte_port->dev_info.max_tx_queues; 4430 } 4431 } else { 4432 /*if vt is disabled, use all pf queues */ 4433 if (rte_port->dev_info.vmdq_pool_base == 0) { 4434 nb_rxq = rte_port->dev_info.max_rx_queues; 4435 nb_txq = rte_port->dev_info.max_tx_queues; 4436 } else { 4437 nb_rxq = (queueid_t)num_tcs; 4438 nb_txq = (queueid_t)num_tcs; 4439 4440 } 4441 } 4442 rx_free_thresh = 64; 4443 4444 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4445 4446 rxtx_port_config(pid); 4447 /* VLAN filter */ 4448 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4449 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4450 rx_vft_set(pid, vlan_tags[i], 1); 4451 4452 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4453 if (retval != 0) 4454 return retval; 4455 4456 rte_port->dcb_flag = 1; 4457 4458 /* Enter DCB configuration status */ 4459 dcb_config = 1; 4460 4461 return 0; 4462 } 4463 4464 static void 4465 init_port(void) 4466 { 4467 int i; 4468 4469 /* Configuration of Ethernet ports. */ 4470 ports = rte_zmalloc("testpmd: ports", 4471 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4472 RTE_CACHE_LINE_SIZE); 4473 if (ports == NULL) { 4474 rte_exit(EXIT_FAILURE, 4475 "rte_zmalloc(%d struct rte_port) failed\n", 4476 RTE_MAX_ETHPORTS); 4477 } 4478 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4479 ports[i].fwd_mac_swap = 1; 4480 ports[i].xstats_info.allocated = false; 4481 LIST_INIT(&ports[i].flow_tunnel_list); 4482 } 4483 /* Initialize ports NUMA structures */ 4484 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4485 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4486 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4487 } 4488 4489 static void 4490 print_stats(void) 4491 { 4492 uint8_t i; 4493 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4494 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4495 4496 /* Clear screen and move to top left */ 4497 printf("%s%s", clr, top_left); 4498 4499 printf("\nPort statistics ===================================="); 4500 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4501 nic_stats_display(fwd_ports_ids[i]); 4502 4503 fflush(stdout); 4504 } 4505 4506 static void 4507 signal_handler(int signum __rte_unused) 4508 { 4509 f_quit = 1; 4510 prompt_exit(); 4511 } 4512 4513 int 4514 main(int argc, char** argv) 4515 { 4516 int diag; 4517 portid_t port_id; 4518 uint16_t count; 4519 int ret; 4520 4521 #ifdef RTE_EXEC_ENV_WINDOWS 4522 signal(SIGINT, signal_handler); 4523 signal(SIGTERM, signal_handler); 4524 #else 4525 /* Want read() not to be restarted on signal */ 4526 struct sigaction action = { 4527 .sa_handler = signal_handler, 4528 }; 4529 4530 sigaction(SIGINT, &action, NULL); 4531 sigaction(SIGTERM, &action, NULL); 4532 #endif 4533 4534 testpmd_logtype = rte_log_register("testpmd"); 4535 if (testpmd_logtype < 0) 4536 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4537 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4538 4539 diag = rte_eal_init(argc, argv); 4540 if (diag < 0) 4541 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4542 rte_strerror(rte_errno)); 4543 4544 /* allocate port structures, and init them */ 4545 init_port(); 4546 4547 ret = register_eth_event_callback(); 4548 if (ret != 0) 4549 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4550 4551 #ifdef RTE_LIB_PDUMP 4552 /* initialize packet capture framework */ 4553 rte_pdump_init(); 4554 #endif 4555 4556 count = 0; 4557 RTE_ETH_FOREACH_DEV(port_id) { 4558 ports_ids[count] = port_id; 4559 count++; 4560 } 4561 nb_ports = (portid_t) count; 4562 if (nb_ports == 0) 4563 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4564 4565 set_def_fwd_config(); 4566 if (nb_lcores == 0) 4567 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4568 "Check the core mask argument\n"); 4569 4570 /* Bitrate/latency stats disabled by default */ 4571 #ifdef RTE_LIB_BITRATESTATS 4572 bitrate_enabled = 0; 4573 #endif 4574 #ifdef RTE_LIB_LATENCYSTATS 4575 latencystats_enabled = 0; 4576 #endif 4577 4578 /* on FreeBSD, mlockall() is disabled by default */ 4579 #ifdef RTE_EXEC_ENV_FREEBSD 4580 do_mlockall = 0; 4581 #else 4582 do_mlockall = 1; 4583 #endif 4584 4585 argc -= diag; 4586 argv += diag; 4587 if (argc > 1) 4588 launch_args_parse(argc, argv); 4589 4590 #ifndef RTE_EXEC_ENV_WINDOWS 4591 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4592 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4593 strerror(errno)); 4594 } 4595 #endif 4596 4597 if (tx_first && interactive) 4598 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4599 "interactive mode.\n"); 4600 4601 if (tx_first && lsc_interrupt) { 4602 fprintf(stderr, 4603 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4604 lsc_interrupt = 0; 4605 } 4606 4607 if (!nb_rxq && !nb_txq) 4608 fprintf(stderr, 4609 "Warning: Either rx or tx queues should be non-zero\n"); 4610 4611 if (nb_rxq > 1 && nb_rxq > nb_txq) 4612 fprintf(stderr, 4613 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4614 nb_rxq, nb_txq); 4615 4616 init_config(); 4617 4618 if (hot_plug) { 4619 ret = rte_dev_hotplug_handle_enable(); 4620 if (ret) { 4621 RTE_LOG(ERR, EAL, 4622 "fail to enable hotplug handling."); 4623 return -1; 4624 } 4625 4626 ret = rte_dev_event_monitor_start(); 4627 if (ret) { 4628 RTE_LOG(ERR, EAL, 4629 "fail to start device event monitoring."); 4630 return -1; 4631 } 4632 4633 ret = rte_dev_event_callback_register(NULL, 4634 dev_event_callback, NULL); 4635 if (ret) { 4636 RTE_LOG(ERR, EAL, 4637 "fail to register device event callback\n"); 4638 return -1; 4639 } 4640 } 4641 4642 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { 4643 if (!interactive) { 4644 rte_eal_cleanup(); 4645 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4646 } 4647 fprintf(stderr, "Start ports failed\n"); 4648 } 4649 4650 /* set all ports to promiscuous mode by default */ 4651 RTE_ETH_FOREACH_DEV(port_id) { 4652 ret = rte_eth_promiscuous_enable(port_id); 4653 if (ret != 0) 4654 fprintf(stderr, 4655 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4656 port_id, rte_strerror(-ret)); 4657 } 4658 4659 #ifdef RTE_LIB_METRICS 4660 /* Init metrics library */ 4661 rte_metrics_init(rte_socket_id()); 4662 #endif 4663 4664 #ifdef RTE_LIB_LATENCYSTATS 4665 if (latencystats_enabled != 0) { 4666 int ret = rte_latencystats_init(1, NULL); 4667 if (ret) 4668 fprintf(stderr, 4669 "Warning: latencystats init() returned error %d\n", 4670 ret); 4671 fprintf(stderr, "Latencystats running on lcore %d\n", 4672 latencystats_lcore_id); 4673 } 4674 #endif 4675 4676 /* Setup bitrate stats */ 4677 #ifdef RTE_LIB_BITRATESTATS 4678 if (bitrate_enabled != 0) { 4679 bitrate_data = rte_stats_bitrate_create(); 4680 if (bitrate_data == NULL) 4681 rte_exit(EXIT_FAILURE, 4682 "Could not allocate bitrate data.\n"); 4683 rte_stats_bitrate_reg(bitrate_data); 4684 } 4685 #endif 4686 4687 if (record_core_cycles) 4688 rte_lcore_register_usage_cb(lcore_usage_callback); 4689 4690 #ifdef RTE_LIB_CMDLINE 4691 if (init_cmdline() != 0) 4692 rte_exit(EXIT_FAILURE, 4693 "Could not initialise cmdline context.\n"); 4694 4695 if (strlen(cmdline_filename) != 0) 4696 cmdline_read_from_file(cmdline_filename); 4697 4698 if (interactive == 1) { 4699 if (auto_start) { 4700 printf("Start automatic packet forwarding\n"); 4701 start_packet_forwarding(0); 4702 } 4703 prompt(); 4704 } else 4705 #endif 4706 { 4707 printf("No commandline core given, start packet forwarding\n"); 4708 start_packet_forwarding(tx_first); 4709 if (stats_period != 0) { 4710 uint64_t prev_time = 0, cur_time, diff_time = 0; 4711 uint64_t timer_period; 4712 4713 /* Convert to number of cycles */ 4714 timer_period = stats_period * rte_get_timer_hz(); 4715 4716 while (f_quit == 0) { 4717 cur_time = rte_get_timer_cycles(); 4718 diff_time += cur_time - prev_time; 4719 4720 if (diff_time >= timer_period) { 4721 print_stats(); 4722 /* Reset the timer */ 4723 diff_time = 0; 4724 } 4725 /* Sleep to avoid unnecessary checks */ 4726 prev_time = cur_time; 4727 rte_delay_us_sleep(US_PER_S); 4728 } 4729 } else { 4730 char c; 4731 fd_set fds; 4732 4733 printf("Press enter to exit\n"); 4734 4735 FD_ZERO(&fds); 4736 FD_SET(0, &fds); 4737 4738 /* wait for signal or enter */ 4739 ret = select(1, &fds, NULL, NULL, NULL); 4740 if (ret < 0 && errno != EINTR) 4741 rte_exit(EXIT_FAILURE, 4742 "Select failed: %s\n", 4743 strerror(errno)); 4744 4745 /* if got enter then consume it */ 4746 if (ret == 1 && read(0, &c, 1) < 0) 4747 rte_exit(EXIT_FAILURE, 4748 "Read failed: %s\n", 4749 strerror(errno)); 4750 } 4751 } 4752 4753 pmd_test_exit(); 4754 4755 #ifdef RTE_LIB_PDUMP 4756 /* uninitialize packet capture framework */ 4757 rte_pdump_uninit(); 4758 #endif 4759 #ifdef RTE_LIB_LATENCYSTATS 4760 if (latencystats_enabled != 0) 4761 rte_latencystats_uninit(); 4762 #endif 4763 4764 ret = unregister_eth_event_callback(); 4765 if (ret != 0) 4766 rte_exit(EXIT_FAILURE, "Cannot unregister for ethdev events"); 4767 4768 4769 ret = rte_eal_cleanup(); 4770 if (ret != 0) 4771 rte_exit(EXIT_FAILURE, 4772 "EAL cleanup failed: %s\n", strerror(-ret)); 4773 4774 return EXIT_SUCCESS; 4775 } 4776