1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 &recycle_mbufs_engine, 202 #ifdef RTE_LIBRTE_IEEE1588 203 &ieee1588_fwd_engine, 204 #endif 205 &shared_rxq_engine, 206 NULL, 207 }; 208 209 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 210 uint16_t mempool_flags; 211 212 struct fwd_config cur_fwd_config; 213 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 214 uint32_t retry_enabled; 215 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 216 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 217 218 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 219 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 220 DEFAULT_MBUF_DATA_SIZE 221 }; /**< Mbuf data space size. */ 222 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 223 * specified on command-line. */ 224 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 225 226 /** Extended statistics to show. */ 227 struct rte_eth_xstat_name *xstats_display; 228 229 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 230 231 /* 232 * In container, it cannot terminate the process which running with 'stats-period' 233 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 234 */ 235 volatile uint8_t f_quit; 236 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 237 238 /* 239 * Max Rx frame size, set by '--max-pkt-len' parameter. 240 */ 241 uint32_t max_rx_pkt_len; 242 243 /* 244 * Configuration of packet segments used to scatter received packets 245 * if some of split features is configured. 246 */ 247 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 248 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 249 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 250 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 251 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 252 253 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 254 255 /* 256 * Configuration of packet segments used by the "txonly" processing engine. 257 */ 258 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 259 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 260 TXONLY_DEF_PACKET_LEN, 261 }; 262 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 263 264 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 265 /**< Split policy for packets to TX. */ 266 267 uint8_t txonly_multi_flow; 268 /**< Whether multiple flows are generated in TXONLY mode. */ 269 270 uint32_t tx_pkt_times_inter; 271 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 272 273 uint32_t tx_pkt_times_intra; 274 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 275 276 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 277 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 278 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 279 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 280 281 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 282 uint8_t dcb_config = 0; 283 284 /* 285 * Configurable number of RX/TX queues. 286 */ 287 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 288 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 289 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 290 291 /* 292 * Configurable number of RX/TX ring descriptors. 293 * Defaults are supplied by drivers via ethdev. 294 */ 295 #define RX_DESC_DEFAULT 0 296 #define TX_DESC_DEFAULT 0 297 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 298 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 299 300 #define RTE_PMD_PARAM_UNSET -1 301 /* 302 * Configurable values of RX and TX ring threshold registers. 303 */ 304 305 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 306 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 307 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 308 309 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 310 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 311 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 312 313 /* 314 * Configurable value of RX free threshold. 315 */ 316 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 317 318 /* 319 * Configurable value of RX drop enable. 320 */ 321 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 322 323 /* 324 * Configurable value of TX free threshold. 325 */ 326 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 327 328 /* 329 * Configurable value of TX RS bit threshold. 330 */ 331 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 332 333 /* 334 * Configurable sub-forwarding mode for the noisy_vnf forwarding mode. 335 */ 336 enum noisy_fwd_mode noisy_fwd_mode; 337 338 /* String version of enum noisy_fwd_mode */ 339 const char * const noisy_fwd_mode_desc[] = { 340 [NOISY_FWD_MODE_IO] = "io", 341 [NOISY_FWD_MODE_MAC] = "mac", 342 [NOISY_FWD_MODE_MACSWAP] = "macswap", 343 [NOISY_FWD_MODE_5TSWAP] = "5tswap", 344 [NOISY_FWD_MODE_MAX] = NULL, 345 }; 346 347 /* 348 * Configurable value of buffered packets before sending. 349 */ 350 uint16_t noisy_tx_sw_bufsz; 351 352 /* 353 * Configurable value of packet buffer timeout. 354 */ 355 uint16_t noisy_tx_sw_buf_flush_time; 356 357 /* 358 * Configurable value for size of VNF internal memory area 359 * used for simulating noisy neighbour behaviour 360 */ 361 uint64_t noisy_lkup_mem_sz; 362 363 /* 364 * Configurable value of number of random writes done in 365 * VNF simulation memory area. 366 */ 367 uint64_t noisy_lkup_num_writes; 368 369 /* 370 * Configurable value of number of random reads done in 371 * VNF simulation memory area. 372 */ 373 uint64_t noisy_lkup_num_reads; 374 375 /* 376 * Configurable value of number of random reads/writes done in 377 * VNF simulation memory area. 378 */ 379 uint64_t noisy_lkup_num_reads_writes; 380 381 /* 382 * Receive Side Scaling (RSS) configuration. 383 */ 384 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 385 386 /* 387 * Port topology configuration 388 */ 389 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 390 391 /* 392 * Avoids to flush all the RX streams before starts forwarding. 393 */ 394 uint8_t no_flush_rx = 0; /* flush by default */ 395 396 /* 397 * Flow API isolated mode. 398 */ 399 uint8_t flow_isolate_all; 400 401 /* 402 * Disable port flow flush when stop port. 403 */ 404 uint8_t no_flow_flush = 0; /* do flow flush by default */ 405 406 /* 407 * Avoids to check link status when starting/stopping a port. 408 */ 409 uint8_t no_link_check = 0; /* check by default */ 410 411 /* 412 * Don't automatically start all ports in interactive mode. 413 */ 414 uint8_t no_device_start = 0; 415 416 /* 417 * Enable link status change notification 418 */ 419 uint8_t lsc_interrupt = 1; /* enabled by default */ 420 421 /* 422 * Enable device removal notification. 423 */ 424 uint8_t rmv_interrupt = 1; /* enabled by default */ 425 426 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 427 428 /* After attach, port setup is called on event or by iterator */ 429 bool setup_on_probe_event = true; 430 431 /* Clear ptypes on port initialization. */ 432 uint8_t clear_ptypes = true; 433 434 /* Hairpin ports configuration mode. */ 435 uint32_t hairpin_mode; 436 437 /* Pretty printing of ethdev events */ 438 static const char * const eth_event_desc[] = { 439 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 440 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 441 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 442 [RTE_ETH_EVENT_INTR_RESET] = "reset", 443 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 444 [RTE_ETH_EVENT_IPSEC] = "IPsec", 445 [RTE_ETH_EVENT_MACSEC] = "MACsec", 446 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 447 [RTE_ETH_EVENT_NEW] = "device probed", 448 [RTE_ETH_EVENT_DESTROY] = "device released", 449 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 450 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 451 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 452 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 453 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 454 [RTE_ETH_EVENT_MAX] = NULL, 455 }; 456 457 /* 458 * Display or mask ether events 459 * Default to all events except VF_MBOX 460 */ 461 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 462 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 463 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 464 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 465 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 466 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 467 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 468 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 469 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 470 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 471 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 472 /* 473 * Decide if all memory are locked for performance. 474 */ 475 int do_mlockall = 0; 476 477 #ifdef RTE_LIB_LATENCYSTATS 478 479 /* 480 * Set when latency stats is enabled in the commandline 481 */ 482 uint8_t latencystats_enabled; 483 484 /* 485 * Lcore ID to service latency statistics. 486 */ 487 lcoreid_t latencystats_lcore_id = -1; 488 489 #endif 490 491 /* 492 * Ethernet device configuration. 493 */ 494 struct rte_eth_rxmode rx_mode; 495 496 struct rte_eth_txmode tx_mode = { 497 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 498 }; 499 500 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 501 502 /* 503 * Display zero values by default for xstats 504 */ 505 uint8_t xstats_hide_zero; 506 507 /* 508 * Measure of CPU cycles disabled by default 509 */ 510 uint8_t record_core_cycles; 511 512 /* 513 * Display of RX and TX bursts disabled by default 514 */ 515 uint8_t record_burst_stats; 516 517 /* 518 * Number of ports per shared Rx queue group, 0 disable. 519 */ 520 uint32_t rxq_share; 521 522 unsigned int num_sockets = 0; 523 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 524 525 #ifdef RTE_LIB_BITRATESTATS 526 /* Bitrate statistics */ 527 struct rte_stats_bitrates *bitrate_data; 528 lcoreid_t bitrate_lcore_id; 529 uint8_t bitrate_enabled; 530 #endif 531 532 #ifdef RTE_LIB_GRO 533 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 534 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 535 #endif 536 537 /* 538 * hexadecimal bitmask of RX mq mode can be enabled. 539 */ 540 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 541 542 /* 543 * Used to set forced link speed 544 */ 545 uint32_t eth_link_speed; 546 547 /* 548 * ID of the current process in multi-process, used to 549 * configure the queues to be polled. 550 */ 551 int proc_id; 552 553 /* 554 * Number of processes in multi-process, used to 555 * configure the queues to be polled. 556 */ 557 unsigned int num_procs = 1; 558 559 static void 560 eth_rx_metadata_negotiate_mp(uint16_t port_id) 561 { 562 uint64_t rx_meta_features = 0; 563 int ret; 564 565 if (!is_proc_primary()) 566 return; 567 568 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 569 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 570 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 571 572 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 573 if (ret == 0) { 574 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 575 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 576 port_id); 577 } 578 579 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 580 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 581 port_id); 582 } 583 584 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 585 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 586 port_id); 587 } 588 } else if (ret != -ENOTSUP) { 589 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 590 port_id, rte_strerror(-ret)); 591 } 592 } 593 594 static int 595 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 596 const struct rte_eth_conf *dev_conf) 597 { 598 if (is_proc_primary()) 599 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 600 dev_conf); 601 return 0; 602 } 603 604 static int 605 change_bonding_member_port_status(portid_t bond_pid, bool is_stop) 606 { 607 #ifdef RTE_NET_BOND 608 609 portid_t member_pids[RTE_MAX_ETHPORTS]; 610 struct rte_port *port; 611 int num_members; 612 portid_t member_pid; 613 int i; 614 615 num_members = rte_eth_bond_members_get(bond_pid, member_pids, 616 RTE_MAX_ETHPORTS); 617 if (num_members < 0) { 618 fprintf(stderr, "Failed to get member list for port = %u\n", 619 bond_pid); 620 return num_members; 621 } 622 623 for (i = 0; i < num_members; i++) { 624 member_pid = member_pids[i]; 625 port = &ports[member_pid]; 626 port->port_status = 627 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 628 } 629 #else 630 RTE_SET_USED(bond_pid); 631 RTE_SET_USED(is_stop); 632 #endif 633 return 0; 634 } 635 636 static int 637 eth_dev_start_mp(uint16_t port_id) 638 { 639 int ret; 640 641 if (is_proc_primary()) { 642 ret = rte_eth_dev_start(port_id); 643 if (ret != 0) 644 return ret; 645 646 struct rte_port *port = &ports[port_id]; 647 648 /* 649 * Starting a bonding port also starts all members under the bonding 650 * device. So if this port is bond device, we need to modify the 651 * port status of these members. 652 */ 653 if (port->bond_flag == 1) 654 return change_bonding_member_port_status(port_id, false); 655 } 656 657 return 0; 658 } 659 660 static int 661 eth_dev_stop_mp(uint16_t port_id) 662 { 663 int ret; 664 665 if (is_proc_primary()) { 666 ret = rte_eth_dev_stop(port_id); 667 if (ret != 0) 668 return ret; 669 670 struct rte_port *port = &ports[port_id]; 671 672 /* 673 * Stopping a bonding port also stops all members under the bonding 674 * device. So if this port is bond device, we need to modify the 675 * port status of these members. 676 */ 677 if (port->bond_flag == 1) 678 return change_bonding_member_port_status(port_id, true); 679 } 680 681 return 0; 682 } 683 684 static void 685 mempool_free_mp(struct rte_mempool *mp) 686 { 687 if (is_proc_primary()) 688 rte_mempool_free(mp); 689 } 690 691 static int 692 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 693 { 694 if (is_proc_primary()) 695 return rte_eth_dev_set_mtu(port_id, mtu); 696 697 return 0; 698 } 699 700 /* Forward function declarations */ 701 static void setup_attached_port(portid_t pi); 702 static void check_all_ports_link_status(uint32_t port_mask); 703 static int eth_event_callback(portid_t port_id, 704 enum rte_eth_event_type type, 705 void *param, void *ret_param); 706 static void dev_event_callback(const char *device_name, 707 enum rte_dev_event_type type, 708 void *param); 709 static void fill_xstats_display_info(void); 710 711 /* 712 * Check if all the ports are started. 713 * If yes, return positive value. If not, return zero. 714 */ 715 static int all_ports_started(void); 716 717 #ifdef RTE_LIB_GSO 718 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 719 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 720 #endif 721 722 /* Holds the registered mbuf dynamic flags names. */ 723 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 724 725 726 /* 727 * Helper function to check if socket is already discovered. 728 * If yes, return positive value. If not, return zero. 729 */ 730 int 731 new_socket_id(unsigned int socket_id) 732 { 733 unsigned int i; 734 735 for (i = 0; i < num_sockets; i++) { 736 if (socket_ids[i] == socket_id) 737 return 0; 738 } 739 return 1; 740 } 741 742 /* 743 * Setup default configuration. 744 */ 745 static void 746 set_default_fwd_lcores_config(void) 747 { 748 unsigned int i; 749 unsigned int nb_lc; 750 unsigned int sock_num; 751 752 nb_lc = 0; 753 for (i = 0; i < RTE_MAX_LCORE; i++) { 754 if (!rte_lcore_is_enabled(i)) 755 continue; 756 sock_num = rte_lcore_to_socket_id(i); 757 if (new_socket_id(sock_num)) { 758 if (num_sockets >= RTE_MAX_NUMA_NODES) { 759 rte_exit(EXIT_FAILURE, 760 "Total sockets greater than %u\n", 761 RTE_MAX_NUMA_NODES); 762 } 763 socket_ids[num_sockets++] = sock_num; 764 } 765 if (i == rte_get_main_lcore()) 766 continue; 767 fwd_lcores_cpuids[nb_lc++] = i; 768 } 769 nb_lcores = (lcoreid_t) nb_lc; 770 nb_cfg_lcores = nb_lcores; 771 nb_fwd_lcores = 1; 772 } 773 774 static void 775 set_def_peer_eth_addrs(void) 776 { 777 portid_t i; 778 779 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 780 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 781 peer_eth_addrs[i].addr_bytes[5] = i; 782 } 783 } 784 785 static void 786 set_default_fwd_ports_config(void) 787 { 788 portid_t pt_id; 789 int i = 0; 790 791 RTE_ETH_FOREACH_DEV(pt_id) { 792 fwd_ports_ids[i++] = pt_id; 793 794 /* Update sockets info according to the attached device */ 795 int socket_id = rte_eth_dev_socket_id(pt_id); 796 if (socket_id >= 0 && new_socket_id(socket_id)) { 797 if (num_sockets >= RTE_MAX_NUMA_NODES) { 798 rte_exit(EXIT_FAILURE, 799 "Total sockets greater than %u\n", 800 RTE_MAX_NUMA_NODES); 801 } 802 socket_ids[num_sockets++] = socket_id; 803 } 804 } 805 806 nb_cfg_ports = nb_ports; 807 nb_fwd_ports = nb_ports; 808 } 809 810 void 811 set_def_fwd_config(void) 812 { 813 set_default_fwd_lcores_config(); 814 set_def_peer_eth_addrs(); 815 set_default_fwd_ports_config(); 816 } 817 818 #ifndef RTE_EXEC_ENV_WINDOWS 819 /* extremely pessimistic estimation of memory required to create a mempool */ 820 static int 821 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 822 { 823 unsigned int n_pages, mbuf_per_pg, leftover; 824 uint64_t total_mem, mbuf_mem, obj_sz; 825 826 /* there is no good way to predict how much space the mempool will 827 * occupy because it will allocate chunks on the fly, and some of those 828 * will come from default DPDK memory while some will come from our 829 * external memory, so just assume 128MB will be enough for everyone. 830 */ 831 uint64_t hdr_mem = 128 << 20; 832 833 /* account for possible non-contiguousness */ 834 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 835 if (obj_sz > pgsz) { 836 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 837 return -1; 838 } 839 840 mbuf_per_pg = pgsz / obj_sz; 841 leftover = (nb_mbufs % mbuf_per_pg) > 0; 842 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 843 844 mbuf_mem = n_pages * pgsz; 845 846 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 847 848 if (total_mem > SIZE_MAX) { 849 TESTPMD_LOG(ERR, "Memory size too big\n"); 850 return -1; 851 } 852 *out = (size_t)total_mem; 853 854 return 0; 855 } 856 857 static int 858 pagesz_flags(uint64_t page_sz) 859 { 860 /* as per mmap() manpage, all page sizes are log2 of page size 861 * shifted by MAP_HUGE_SHIFT 862 */ 863 int log2 = rte_log2_u64(page_sz); 864 865 return (log2 << HUGE_SHIFT); 866 } 867 868 static void * 869 alloc_mem(size_t memsz, size_t pgsz, bool huge) 870 { 871 void *addr; 872 int flags; 873 874 /* allocate anonymous hugepages */ 875 flags = MAP_ANONYMOUS | MAP_PRIVATE; 876 if (huge) 877 flags |= HUGE_FLAG | pagesz_flags(pgsz); 878 879 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 880 if (addr == MAP_FAILED) 881 return NULL; 882 883 return addr; 884 } 885 886 struct extmem_param { 887 void *addr; 888 size_t len; 889 size_t pgsz; 890 rte_iova_t *iova_table; 891 unsigned int iova_table_len; 892 }; 893 894 static int 895 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 896 bool huge) 897 { 898 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 899 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 900 unsigned int cur_page, n_pages, pgsz_idx; 901 size_t mem_sz, cur_pgsz; 902 rte_iova_t *iovas = NULL; 903 void *addr; 904 int ret; 905 906 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 907 /* skip anything that is too big */ 908 if (pgsizes[pgsz_idx] > SIZE_MAX) 909 continue; 910 911 cur_pgsz = pgsizes[pgsz_idx]; 912 913 /* if we were told not to allocate hugepages, override */ 914 if (!huge) 915 cur_pgsz = sysconf(_SC_PAGESIZE); 916 917 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 918 if (ret < 0) { 919 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 920 return -1; 921 } 922 923 /* allocate our memory */ 924 addr = alloc_mem(mem_sz, cur_pgsz, huge); 925 926 /* if we couldn't allocate memory with a specified page size, 927 * that doesn't mean we can't do it with other page sizes, so 928 * try another one. 929 */ 930 if (addr == NULL) 931 continue; 932 933 /* store IOVA addresses for every page in this memory area */ 934 n_pages = mem_sz / cur_pgsz; 935 936 iovas = malloc(sizeof(*iovas) * n_pages); 937 938 if (iovas == NULL) { 939 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 940 goto fail; 941 } 942 /* lock memory if it's not huge pages */ 943 if (!huge) 944 mlock(addr, mem_sz); 945 946 /* populate IOVA addresses */ 947 for (cur_page = 0; cur_page < n_pages; cur_page++) { 948 rte_iova_t iova; 949 size_t offset; 950 void *cur; 951 952 offset = cur_pgsz * cur_page; 953 cur = RTE_PTR_ADD(addr, offset); 954 955 /* touch the page before getting its IOVA */ 956 *(volatile char *)cur = 0; 957 958 iova = rte_mem_virt2iova(cur); 959 960 iovas[cur_page] = iova; 961 } 962 963 break; 964 } 965 /* if we couldn't allocate anything */ 966 if (iovas == NULL) 967 return -1; 968 969 param->addr = addr; 970 param->len = mem_sz; 971 param->pgsz = cur_pgsz; 972 param->iova_table = iovas; 973 param->iova_table_len = n_pages; 974 975 return 0; 976 fail: 977 free(iovas); 978 if (addr) 979 munmap(addr, mem_sz); 980 981 return -1; 982 } 983 984 static int 985 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 986 { 987 struct extmem_param param; 988 int socket_id, ret; 989 990 memset(¶m, 0, sizeof(param)); 991 992 /* check if our heap exists */ 993 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 994 if (socket_id < 0) { 995 /* create our heap */ 996 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 997 if (ret < 0) { 998 TESTPMD_LOG(ERR, "Cannot create heap\n"); 999 return -1; 1000 } 1001 } 1002 1003 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 1004 if (ret < 0) { 1005 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 1006 return -1; 1007 } 1008 1009 /* we now have a valid memory area, so add it to heap */ 1010 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 1011 param.addr, param.len, param.iova_table, 1012 param.iova_table_len, param.pgsz); 1013 1014 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1015 1016 /* not needed any more */ 1017 free(param.iova_table); 1018 1019 if (ret < 0) { 1020 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1021 munmap(param.addr, param.len); 1022 return -1; 1023 } 1024 1025 /* success */ 1026 1027 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1028 param.len >> 20); 1029 1030 return 0; 1031 } 1032 static void 1033 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1034 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1035 { 1036 uint16_t pid = 0; 1037 int ret; 1038 1039 RTE_ETH_FOREACH_DEV(pid) { 1040 struct rte_eth_dev_info dev_info; 1041 1042 ret = eth_dev_info_get_print_err(pid, &dev_info); 1043 if (ret != 0) { 1044 TESTPMD_LOG(DEBUG, 1045 "unable to get device info for port %d on addr 0x%p," 1046 "mempool unmapping will not be performed\n", 1047 pid, memhdr->addr); 1048 continue; 1049 } 1050 1051 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1052 if (ret) { 1053 TESTPMD_LOG(DEBUG, 1054 "unable to DMA unmap addr 0x%p " 1055 "for device %s\n", 1056 memhdr->addr, rte_dev_name(dev_info.device)); 1057 } 1058 } 1059 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1060 if (ret) { 1061 TESTPMD_LOG(DEBUG, 1062 "unable to un-register addr 0x%p\n", memhdr->addr); 1063 } 1064 } 1065 1066 static void 1067 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1068 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1069 { 1070 uint16_t pid = 0; 1071 size_t page_size = sysconf(_SC_PAGESIZE); 1072 int ret; 1073 1074 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1075 page_size); 1076 if (ret) { 1077 TESTPMD_LOG(DEBUG, 1078 "unable to register addr 0x%p\n", memhdr->addr); 1079 return; 1080 } 1081 RTE_ETH_FOREACH_DEV(pid) { 1082 struct rte_eth_dev_info dev_info; 1083 1084 ret = eth_dev_info_get_print_err(pid, &dev_info); 1085 if (ret != 0) { 1086 TESTPMD_LOG(DEBUG, 1087 "unable to get device info for port %d on addr 0x%p," 1088 "mempool mapping will not be performed\n", 1089 pid, memhdr->addr); 1090 continue; 1091 } 1092 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1093 if (ret) { 1094 TESTPMD_LOG(DEBUG, 1095 "unable to DMA map addr 0x%p " 1096 "for device %s\n", 1097 memhdr->addr, rte_dev_name(dev_info.device)); 1098 } 1099 } 1100 } 1101 #endif 1102 1103 static unsigned int 1104 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1105 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1106 { 1107 struct rte_pktmbuf_extmem *xmem; 1108 unsigned int ext_num, zone_num, elt_num; 1109 uint16_t elt_size; 1110 1111 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1112 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1113 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1114 1115 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1116 if (xmem == NULL) { 1117 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1118 "external buffer descriptors\n"); 1119 *ext_mem = NULL; 1120 return 0; 1121 } 1122 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1123 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1124 const struct rte_memzone *mz; 1125 char mz_name[RTE_MEMZONE_NAMESIZE]; 1126 int ret; 1127 1128 ret = snprintf(mz_name, sizeof(mz_name), 1129 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1130 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1131 errno = ENAMETOOLONG; 1132 ext_num = 0; 1133 break; 1134 } 1135 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1136 socket_id, 1137 RTE_MEMZONE_IOVA_CONTIG | 1138 RTE_MEMZONE_1GB | 1139 RTE_MEMZONE_SIZE_HINT_ONLY); 1140 if (mz == NULL) { 1141 /* 1142 * The caller exits on external buffer creation 1143 * error, so there is no need to free memzones. 1144 */ 1145 errno = ENOMEM; 1146 ext_num = 0; 1147 break; 1148 } 1149 xseg->buf_ptr = mz->addr; 1150 xseg->buf_iova = mz->iova; 1151 xseg->buf_len = EXTBUF_ZONE_SIZE; 1152 xseg->elt_size = elt_size; 1153 } 1154 if (ext_num == 0 && xmem != NULL) { 1155 free(xmem); 1156 xmem = NULL; 1157 } 1158 *ext_mem = xmem; 1159 return ext_num; 1160 } 1161 1162 /* 1163 * Configuration initialisation done once at init time. 1164 */ 1165 static struct rte_mempool * 1166 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1167 unsigned int socket_id, uint16_t size_idx) 1168 { 1169 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1170 struct rte_mempool *rte_mp = NULL; 1171 #ifndef RTE_EXEC_ENV_WINDOWS 1172 uint32_t mb_size; 1173 1174 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1175 #endif 1176 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1177 if (!is_proc_primary()) { 1178 rte_mp = rte_mempool_lookup(pool_name); 1179 if (rte_mp == NULL) 1180 rte_exit(EXIT_FAILURE, 1181 "Get mbuf pool for socket %u failed: %s\n", 1182 socket_id, rte_strerror(rte_errno)); 1183 return rte_mp; 1184 } 1185 1186 TESTPMD_LOG(INFO, 1187 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1188 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1189 1190 switch (mp_alloc_type) { 1191 case MP_ALLOC_NATIVE: 1192 { 1193 /* wrapper to rte_mempool_create() */ 1194 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1195 rte_mbuf_best_mempool_ops()); 1196 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1197 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1198 break; 1199 } 1200 #ifndef RTE_EXEC_ENV_WINDOWS 1201 case MP_ALLOC_ANON: 1202 { 1203 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1204 mb_size, (unsigned int) mb_mempool_cache, 1205 sizeof(struct rte_pktmbuf_pool_private), 1206 socket_id, mempool_flags); 1207 if (rte_mp == NULL) 1208 goto err; 1209 1210 if (rte_mempool_populate_anon(rte_mp) == 0) { 1211 rte_mempool_free(rte_mp); 1212 rte_mp = NULL; 1213 goto err; 1214 } 1215 rte_pktmbuf_pool_init(rte_mp, NULL); 1216 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1217 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1218 break; 1219 } 1220 case MP_ALLOC_XMEM: 1221 case MP_ALLOC_XMEM_HUGE: 1222 { 1223 int heap_socket; 1224 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1225 1226 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1227 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1228 1229 heap_socket = 1230 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1231 if (heap_socket < 0) 1232 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1233 1234 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1235 rte_mbuf_best_mempool_ops()); 1236 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1237 mb_mempool_cache, 0, mbuf_seg_size, 1238 heap_socket); 1239 break; 1240 } 1241 #endif 1242 case MP_ALLOC_XBUF: 1243 { 1244 struct rte_pktmbuf_extmem *ext_mem; 1245 unsigned int ext_num; 1246 1247 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1248 socket_id, pool_name, &ext_mem); 1249 if (ext_num == 0) 1250 rte_exit(EXIT_FAILURE, 1251 "Can't create pinned data buffers\n"); 1252 1253 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1254 rte_mbuf_best_mempool_ops()); 1255 rte_mp = rte_pktmbuf_pool_create_extbuf 1256 (pool_name, nb_mbuf, mb_mempool_cache, 1257 0, mbuf_seg_size, socket_id, 1258 ext_mem, ext_num); 1259 free(ext_mem); 1260 break; 1261 } 1262 default: 1263 { 1264 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1265 } 1266 } 1267 1268 #ifndef RTE_EXEC_ENV_WINDOWS 1269 err: 1270 #endif 1271 if (rte_mp == NULL) { 1272 rte_exit(EXIT_FAILURE, 1273 "Creation of mbuf pool for socket %u failed: %s\n", 1274 socket_id, rte_strerror(rte_errno)); 1275 } else if (verbose_level > 0) { 1276 rte_mempool_dump(stdout, rte_mp); 1277 } 1278 return rte_mp; 1279 } 1280 1281 /* 1282 * Check given socket id is valid or not with NUMA mode, 1283 * if valid, return 0, else return -1 1284 */ 1285 static int 1286 check_socket_id(const unsigned int socket_id) 1287 { 1288 static int warning_once = 0; 1289 1290 if (new_socket_id(socket_id)) { 1291 if (!warning_once && numa_support) 1292 fprintf(stderr, 1293 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1294 warning_once = 1; 1295 return -1; 1296 } 1297 return 0; 1298 } 1299 1300 /* 1301 * Get the allowed maximum number of RX queues. 1302 * *pid return the port id which has minimal value of 1303 * max_rx_queues in all ports. 1304 */ 1305 queueid_t 1306 get_allowed_max_nb_rxq(portid_t *pid) 1307 { 1308 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1309 bool max_rxq_valid = false; 1310 portid_t pi; 1311 struct rte_eth_dev_info dev_info; 1312 1313 RTE_ETH_FOREACH_DEV(pi) { 1314 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1315 continue; 1316 1317 max_rxq_valid = true; 1318 if (dev_info.max_rx_queues < allowed_max_rxq) { 1319 allowed_max_rxq = dev_info.max_rx_queues; 1320 *pid = pi; 1321 } 1322 } 1323 return max_rxq_valid ? allowed_max_rxq : 0; 1324 } 1325 1326 /* 1327 * Check input rxq is valid or not. 1328 * If input rxq is not greater than any of maximum number 1329 * of RX queues of all ports, it is valid. 1330 * if valid, return 0, else return -1 1331 */ 1332 int 1333 check_nb_rxq(queueid_t rxq) 1334 { 1335 queueid_t allowed_max_rxq; 1336 portid_t pid = 0; 1337 1338 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1339 if (rxq > allowed_max_rxq) { 1340 fprintf(stderr, 1341 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1342 rxq, allowed_max_rxq, pid); 1343 return -1; 1344 } 1345 return 0; 1346 } 1347 1348 /* 1349 * Get the allowed maximum number of TX queues. 1350 * *pid return the port id which has minimal value of 1351 * max_tx_queues in all ports. 1352 */ 1353 queueid_t 1354 get_allowed_max_nb_txq(portid_t *pid) 1355 { 1356 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1357 bool max_txq_valid = false; 1358 portid_t pi; 1359 struct rte_eth_dev_info dev_info; 1360 1361 RTE_ETH_FOREACH_DEV(pi) { 1362 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1363 continue; 1364 1365 max_txq_valid = true; 1366 if (dev_info.max_tx_queues < allowed_max_txq) { 1367 allowed_max_txq = dev_info.max_tx_queues; 1368 *pid = pi; 1369 } 1370 } 1371 return max_txq_valid ? allowed_max_txq : 0; 1372 } 1373 1374 /* 1375 * Check input txq is valid or not. 1376 * If input txq is not greater than any of maximum number 1377 * of TX queues of all ports, it is valid. 1378 * if valid, return 0, else return -1 1379 */ 1380 int 1381 check_nb_txq(queueid_t txq) 1382 { 1383 queueid_t allowed_max_txq; 1384 portid_t pid = 0; 1385 1386 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1387 if (txq > allowed_max_txq) { 1388 fprintf(stderr, 1389 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1390 txq, allowed_max_txq, pid); 1391 return -1; 1392 } 1393 return 0; 1394 } 1395 1396 /* 1397 * Get the allowed maximum number of RXDs of every rx queue. 1398 * *pid return the port id which has minimal value of 1399 * max_rxd in all queues of all ports. 1400 */ 1401 static uint16_t 1402 get_allowed_max_nb_rxd(portid_t *pid) 1403 { 1404 uint16_t allowed_max_rxd = UINT16_MAX; 1405 portid_t pi; 1406 struct rte_eth_dev_info dev_info; 1407 1408 RTE_ETH_FOREACH_DEV(pi) { 1409 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1410 continue; 1411 1412 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1413 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1414 *pid = pi; 1415 } 1416 } 1417 return allowed_max_rxd; 1418 } 1419 1420 /* 1421 * Get the allowed minimal number of RXDs of every rx queue. 1422 * *pid return the port id which has minimal value of 1423 * min_rxd in all queues of all ports. 1424 */ 1425 static uint16_t 1426 get_allowed_min_nb_rxd(portid_t *pid) 1427 { 1428 uint16_t allowed_min_rxd = 0; 1429 portid_t pi; 1430 struct rte_eth_dev_info dev_info; 1431 1432 RTE_ETH_FOREACH_DEV(pi) { 1433 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1434 continue; 1435 1436 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1437 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1438 *pid = pi; 1439 } 1440 } 1441 1442 return allowed_min_rxd; 1443 } 1444 1445 /* 1446 * Check input rxd is valid or not. 1447 * If input rxd is not greater than any of maximum number 1448 * of RXDs of every Rx queues and is not less than any of 1449 * minimal number of RXDs of every Rx queues, it is valid. 1450 * if valid, return 0, else return -1 1451 */ 1452 int 1453 check_nb_rxd(queueid_t rxd) 1454 { 1455 uint16_t allowed_max_rxd; 1456 uint16_t allowed_min_rxd; 1457 portid_t pid = 0; 1458 1459 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1460 if (rxd > allowed_max_rxd) { 1461 fprintf(stderr, 1462 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1463 rxd, allowed_max_rxd, pid); 1464 return -1; 1465 } 1466 1467 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1468 if (rxd < allowed_min_rxd) { 1469 fprintf(stderr, 1470 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1471 rxd, allowed_min_rxd, pid); 1472 return -1; 1473 } 1474 1475 return 0; 1476 } 1477 1478 /* 1479 * Get the allowed maximum number of TXDs of every rx queues. 1480 * *pid return the port id which has minimal value of 1481 * max_txd in every tx queue. 1482 */ 1483 static uint16_t 1484 get_allowed_max_nb_txd(portid_t *pid) 1485 { 1486 uint16_t allowed_max_txd = UINT16_MAX; 1487 portid_t pi; 1488 struct rte_eth_dev_info dev_info; 1489 1490 RTE_ETH_FOREACH_DEV(pi) { 1491 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1492 continue; 1493 1494 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1495 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1496 *pid = pi; 1497 } 1498 } 1499 return allowed_max_txd; 1500 } 1501 1502 /* 1503 * Get the allowed maximum number of TXDs of every tx queues. 1504 * *pid return the port id which has minimal value of 1505 * min_txd in every tx queue. 1506 */ 1507 static uint16_t 1508 get_allowed_min_nb_txd(portid_t *pid) 1509 { 1510 uint16_t allowed_min_txd = 0; 1511 portid_t pi; 1512 struct rte_eth_dev_info dev_info; 1513 1514 RTE_ETH_FOREACH_DEV(pi) { 1515 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1516 continue; 1517 1518 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1519 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1520 *pid = pi; 1521 } 1522 } 1523 1524 return allowed_min_txd; 1525 } 1526 1527 /* 1528 * Check input txd is valid or not. 1529 * If input txd is not greater than any of maximum number 1530 * of TXDs of every Rx queues, it is valid. 1531 * if valid, return 0, else return -1 1532 */ 1533 int 1534 check_nb_txd(queueid_t txd) 1535 { 1536 uint16_t allowed_max_txd; 1537 uint16_t allowed_min_txd; 1538 portid_t pid = 0; 1539 1540 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1541 if (txd > allowed_max_txd) { 1542 fprintf(stderr, 1543 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1544 txd, allowed_max_txd, pid); 1545 return -1; 1546 } 1547 1548 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1549 if (txd < allowed_min_txd) { 1550 fprintf(stderr, 1551 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1552 txd, allowed_min_txd, pid); 1553 return -1; 1554 } 1555 return 0; 1556 } 1557 1558 1559 /* 1560 * Get the allowed maximum number of hairpin queues. 1561 * *pid return the port id which has minimal value of 1562 * max_hairpin_queues in all ports. 1563 */ 1564 queueid_t 1565 get_allowed_max_nb_hairpinq(portid_t *pid) 1566 { 1567 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1568 portid_t pi; 1569 struct rte_eth_hairpin_cap cap; 1570 1571 RTE_ETH_FOREACH_DEV(pi) { 1572 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1573 *pid = pi; 1574 return 0; 1575 } 1576 if (cap.max_nb_queues < allowed_max_hairpinq) { 1577 allowed_max_hairpinq = cap.max_nb_queues; 1578 *pid = pi; 1579 } 1580 } 1581 return allowed_max_hairpinq; 1582 } 1583 1584 /* 1585 * Check input hairpin is valid or not. 1586 * If input hairpin is not greater than any of maximum number 1587 * of hairpin queues of all ports, it is valid. 1588 * if valid, return 0, else return -1 1589 */ 1590 int 1591 check_nb_hairpinq(queueid_t hairpinq) 1592 { 1593 queueid_t allowed_max_hairpinq; 1594 portid_t pid = 0; 1595 1596 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1597 if (hairpinq > allowed_max_hairpinq) { 1598 fprintf(stderr, 1599 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1600 hairpinq, allowed_max_hairpinq, pid); 1601 return -1; 1602 } 1603 return 0; 1604 } 1605 1606 static int 1607 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1608 { 1609 uint32_t eth_overhead; 1610 1611 if (dev_info->max_mtu != UINT16_MAX && 1612 dev_info->max_rx_pktlen > dev_info->max_mtu) 1613 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1614 else 1615 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1616 1617 return eth_overhead; 1618 } 1619 1620 static void 1621 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1622 { 1623 struct rte_port *port = &ports[pid]; 1624 int ret; 1625 int i; 1626 1627 eth_rx_metadata_negotiate_mp(pid); 1628 1629 port->dev_conf.txmode = tx_mode; 1630 port->dev_conf.rxmode = rx_mode; 1631 1632 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1633 if (ret != 0) 1634 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1635 1636 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1637 port->dev_conf.txmode.offloads &= 1638 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1639 1640 /* Apply Rx offloads configuration */ 1641 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1642 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1643 /* Apply Tx offloads configuration */ 1644 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1645 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1646 1647 if (eth_link_speed) 1648 port->dev_conf.link_speeds = eth_link_speed; 1649 1650 if (max_rx_pkt_len) 1651 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1652 get_eth_overhead(&port->dev_info); 1653 1654 /* set flag to initialize port/queue */ 1655 port->need_reconfig = 1; 1656 port->need_reconfig_queues = 1; 1657 port->socket_id = socket_id; 1658 port->tx_metadata = 0; 1659 1660 /* 1661 * Check for maximum number of segments per MTU. 1662 * Accordingly update the mbuf data size. 1663 */ 1664 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1665 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1666 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1667 uint16_t mtu; 1668 1669 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1670 uint16_t data_size = (mtu + eth_overhead) / 1671 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1672 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1673 1674 if (buffer_size > mbuf_data_size[0]) { 1675 mbuf_data_size[0] = buffer_size; 1676 TESTPMD_LOG(WARNING, 1677 "Configured mbuf size of the first segment %hu\n", 1678 mbuf_data_size[0]); 1679 } 1680 } 1681 } 1682 } 1683 1684 static void 1685 init_config(void) 1686 { 1687 portid_t pid; 1688 struct rte_mempool *mbp; 1689 unsigned int nb_mbuf_per_pool; 1690 lcoreid_t lc_id; 1691 #ifdef RTE_LIB_GRO 1692 struct rte_gro_param gro_param; 1693 #endif 1694 #ifdef RTE_LIB_GSO 1695 uint32_t gso_types; 1696 #endif 1697 1698 /* Configuration of logical cores. */ 1699 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1700 sizeof(struct fwd_lcore *) * nb_lcores, 1701 RTE_CACHE_LINE_SIZE); 1702 if (fwd_lcores == NULL) { 1703 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1704 "failed\n", nb_lcores); 1705 } 1706 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1707 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1708 sizeof(struct fwd_lcore), 1709 RTE_CACHE_LINE_SIZE); 1710 if (fwd_lcores[lc_id] == NULL) { 1711 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1712 "failed\n"); 1713 } 1714 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1715 } 1716 1717 RTE_ETH_FOREACH_DEV(pid) { 1718 uint32_t socket_id; 1719 1720 if (numa_support) { 1721 socket_id = port_numa[pid]; 1722 if (port_numa[pid] == NUMA_NO_CONFIG) { 1723 socket_id = rte_eth_dev_socket_id(pid); 1724 1725 /* 1726 * if socket_id is invalid, 1727 * set to the first available socket. 1728 */ 1729 if (check_socket_id(socket_id) < 0) 1730 socket_id = socket_ids[0]; 1731 } 1732 } else { 1733 socket_id = (socket_num == UMA_NO_CONFIG) ? 1734 0 : socket_num; 1735 } 1736 /* Apply default TxRx configuration for all ports */ 1737 init_config_port_offloads(pid, socket_id); 1738 } 1739 /* 1740 * Create pools of mbuf. 1741 * If NUMA support is disabled, create a single pool of mbuf in 1742 * socket 0 memory by default. 1743 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1744 * 1745 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1746 * nb_txd can be configured at run time. 1747 */ 1748 if (param_total_num_mbufs) 1749 nb_mbuf_per_pool = param_total_num_mbufs; 1750 else { 1751 nb_mbuf_per_pool = RX_DESC_MAX + 1752 (nb_lcores * mb_mempool_cache) + 1753 TX_DESC_MAX + MAX_PKT_BURST; 1754 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1755 } 1756 1757 if (numa_support) { 1758 uint8_t i, j; 1759 1760 for (i = 0; i < num_sockets; i++) 1761 for (j = 0; j < mbuf_data_size_n; j++) 1762 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1763 mbuf_pool_create(mbuf_data_size[j], 1764 nb_mbuf_per_pool, 1765 socket_ids[i], j); 1766 } else { 1767 uint8_t i; 1768 1769 for (i = 0; i < mbuf_data_size_n; i++) 1770 mempools[i] = mbuf_pool_create 1771 (mbuf_data_size[i], 1772 nb_mbuf_per_pool, 1773 socket_num == UMA_NO_CONFIG ? 1774 0 : socket_num, i); 1775 } 1776 1777 init_port_config(); 1778 1779 #ifdef RTE_LIB_GSO 1780 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1781 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1782 #endif 1783 /* 1784 * Records which Mbuf pool to use by each logical core, if needed. 1785 */ 1786 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1787 mbp = mbuf_pool_find( 1788 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1789 1790 if (mbp == NULL) 1791 mbp = mbuf_pool_find(0, 0); 1792 fwd_lcores[lc_id]->mbp = mbp; 1793 #ifdef RTE_LIB_GSO 1794 /* initialize GSO context */ 1795 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1796 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1797 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1798 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1799 RTE_ETHER_CRC_LEN; 1800 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1801 #endif 1802 } 1803 1804 fwd_config_setup(); 1805 1806 #ifdef RTE_LIB_GRO 1807 /* create a gro context for each lcore */ 1808 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1809 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1810 gro_param.max_item_per_flow = MAX_PKT_BURST; 1811 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1812 gro_param.socket_id = rte_lcore_to_socket_id( 1813 fwd_lcores_cpuids[lc_id]); 1814 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1815 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1816 rte_exit(EXIT_FAILURE, 1817 "rte_gro_ctx_create() failed\n"); 1818 } 1819 } 1820 #endif 1821 } 1822 1823 1824 void 1825 reconfig(portid_t new_port_id, unsigned socket_id) 1826 { 1827 /* Reconfiguration of Ethernet ports. */ 1828 init_config_port_offloads(new_port_id, socket_id); 1829 init_port_config(); 1830 } 1831 1832 int 1833 init_fwd_streams(void) 1834 { 1835 portid_t pid; 1836 struct rte_port *port; 1837 streamid_t sm_id, nb_fwd_streams_new; 1838 queueid_t q; 1839 1840 /* set socket id according to numa or not */ 1841 RTE_ETH_FOREACH_DEV(pid) { 1842 port = &ports[pid]; 1843 if (nb_rxq > port->dev_info.max_rx_queues) { 1844 fprintf(stderr, 1845 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1846 nb_rxq, port->dev_info.max_rx_queues); 1847 return -1; 1848 } 1849 if (nb_txq > port->dev_info.max_tx_queues) { 1850 fprintf(stderr, 1851 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1852 nb_txq, port->dev_info.max_tx_queues); 1853 return -1; 1854 } 1855 if (numa_support) { 1856 if (port_numa[pid] != NUMA_NO_CONFIG) 1857 port->socket_id = port_numa[pid]; 1858 else { 1859 port->socket_id = rte_eth_dev_socket_id(pid); 1860 1861 /* 1862 * if socket_id is invalid, 1863 * set to the first available socket. 1864 */ 1865 if (check_socket_id(port->socket_id) < 0) 1866 port->socket_id = socket_ids[0]; 1867 } 1868 } 1869 else { 1870 if (socket_num == UMA_NO_CONFIG) 1871 port->socket_id = 0; 1872 else 1873 port->socket_id = socket_num; 1874 } 1875 } 1876 1877 q = RTE_MAX(nb_rxq, nb_txq); 1878 if (q == 0) { 1879 fprintf(stderr, 1880 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1881 return -1; 1882 } 1883 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1884 if (nb_fwd_streams_new == nb_fwd_streams) 1885 return 0; 1886 /* clear the old */ 1887 if (fwd_streams != NULL) { 1888 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1889 if (fwd_streams[sm_id] == NULL) 1890 continue; 1891 rte_free(fwd_streams[sm_id]); 1892 fwd_streams[sm_id] = NULL; 1893 } 1894 rte_free(fwd_streams); 1895 fwd_streams = NULL; 1896 } 1897 1898 /* init new */ 1899 nb_fwd_streams = nb_fwd_streams_new; 1900 if (nb_fwd_streams) { 1901 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1902 sizeof(struct fwd_stream *) * nb_fwd_streams, 1903 RTE_CACHE_LINE_SIZE); 1904 if (fwd_streams == NULL) 1905 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1906 " (struct fwd_stream *)) failed\n", 1907 nb_fwd_streams); 1908 1909 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1910 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1911 " struct fwd_stream", sizeof(struct fwd_stream), 1912 RTE_CACHE_LINE_SIZE); 1913 if (fwd_streams[sm_id] == NULL) 1914 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1915 "(struct fwd_stream) failed\n"); 1916 } 1917 } 1918 1919 return 0; 1920 } 1921 1922 static void 1923 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1924 { 1925 uint64_t total_burst, sburst; 1926 uint64_t nb_burst; 1927 uint64_t burst_stats[4]; 1928 uint16_t pktnb_stats[4]; 1929 uint16_t nb_pkt; 1930 int burst_percent[4], sburstp; 1931 int i; 1932 1933 /* 1934 * First compute the total number of packet bursts and the 1935 * two highest numbers of bursts of the same number of packets. 1936 */ 1937 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1938 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1939 1940 /* Show stats for 0 burst size always */ 1941 total_burst = pbs->pkt_burst_spread[0]; 1942 burst_stats[0] = pbs->pkt_burst_spread[0]; 1943 pktnb_stats[0] = 0; 1944 1945 /* Find the next 2 burst sizes with highest occurrences. */ 1946 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1947 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1948 1949 if (nb_burst == 0) 1950 continue; 1951 1952 total_burst += nb_burst; 1953 1954 if (nb_burst > burst_stats[1]) { 1955 burst_stats[2] = burst_stats[1]; 1956 pktnb_stats[2] = pktnb_stats[1]; 1957 burst_stats[1] = nb_burst; 1958 pktnb_stats[1] = nb_pkt; 1959 } else if (nb_burst > burst_stats[2]) { 1960 burst_stats[2] = nb_burst; 1961 pktnb_stats[2] = nb_pkt; 1962 } 1963 } 1964 if (total_burst == 0) 1965 return; 1966 1967 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1968 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1969 if (i == 3) { 1970 printf("%d%% of other]\n", 100 - sburstp); 1971 return; 1972 } 1973 1974 sburst += burst_stats[i]; 1975 if (sburst == total_burst) { 1976 printf("%d%% of %d pkts]\n", 1977 100 - sburstp, (int) pktnb_stats[i]); 1978 return; 1979 } 1980 1981 burst_percent[i] = 1982 (double)burst_stats[i] / total_burst * 100; 1983 printf("%d%% of %d pkts + ", 1984 burst_percent[i], (int) pktnb_stats[i]); 1985 sburstp += burst_percent[i]; 1986 } 1987 } 1988 1989 static void 1990 fwd_stream_stats_display(streamid_t stream_id) 1991 { 1992 struct fwd_stream *fs; 1993 static const char *fwd_top_stats_border = "-------"; 1994 1995 fs = fwd_streams[stream_id]; 1996 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1997 (fs->fwd_dropped == 0)) 1998 return; 1999 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 2000 "TX Port=%2d/Queue=%2d %s\n", 2001 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 2002 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 2003 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 2004 " TX-dropped: %-14"PRIu64, 2005 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 2006 2007 /* if checksum mode */ 2008 if (cur_fwd_eng == &csum_fwd_engine) { 2009 printf(" RX- bad IP checksum: %-14"PRIu64 2010 " Rx- bad L4 checksum: %-14"PRIu64 2011 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 2012 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 2013 fs->rx_bad_outer_l4_csum); 2014 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2015 fs->rx_bad_outer_ip_csum); 2016 } else { 2017 printf("\n"); 2018 } 2019 2020 if (record_burst_stats) { 2021 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2022 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2023 } 2024 } 2025 2026 void 2027 fwd_stats_display(void) 2028 { 2029 static const char *fwd_stats_border = "----------------------"; 2030 static const char *acc_stats_border = "+++++++++++++++"; 2031 struct { 2032 struct fwd_stream *rx_stream; 2033 struct fwd_stream *tx_stream; 2034 uint64_t tx_dropped; 2035 uint64_t rx_bad_ip_csum; 2036 uint64_t rx_bad_l4_csum; 2037 uint64_t rx_bad_outer_l4_csum; 2038 uint64_t rx_bad_outer_ip_csum; 2039 } ports_stats[RTE_MAX_ETHPORTS]; 2040 uint64_t total_rx_dropped = 0; 2041 uint64_t total_tx_dropped = 0; 2042 uint64_t total_rx_nombuf = 0; 2043 struct rte_eth_stats stats; 2044 uint64_t fwd_cycles = 0; 2045 uint64_t total_recv = 0; 2046 uint64_t total_xmit = 0; 2047 struct rte_port *port; 2048 streamid_t sm_id; 2049 portid_t pt_id; 2050 int ret; 2051 int i; 2052 2053 memset(ports_stats, 0, sizeof(ports_stats)); 2054 2055 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2056 struct fwd_stream *fs = fwd_streams[sm_id]; 2057 2058 if (cur_fwd_config.nb_fwd_streams > 2059 cur_fwd_config.nb_fwd_ports) { 2060 fwd_stream_stats_display(sm_id); 2061 } else { 2062 ports_stats[fs->tx_port].tx_stream = fs; 2063 ports_stats[fs->rx_port].rx_stream = fs; 2064 } 2065 2066 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2067 2068 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2069 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2070 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2071 fs->rx_bad_outer_l4_csum; 2072 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2073 fs->rx_bad_outer_ip_csum; 2074 2075 if (record_core_cycles) 2076 fwd_cycles += fs->busy_cycles; 2077 } 2078 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2079 uint64_t tx_dropped = 0; 2080 2081 pt_id = fwd_ports_ids[i]; 2082 port = &ports[pt_id]; 2083 2084 ret = rte_eth_stats_get(pt_id, &stats); 2085 if (ret != 0) { 2086 fprintf(stderr, 2087 "%s: Error: failed to get stats (port %u): %d", 2088 __func__, pt_id, ret); 2089 continue; 2090 } 2091 stats.ipackets -= port->stats.ipackets; 2092 stats.opackets -= port->stats.opackets; 2093 stats.ibytes -= port->stats.ibytes; 2094 stats.obytes -= port->stats.obytes; 2095 stats.imissed -= port->stats.imissed; 2096 stats.oerrors -= port->stats.oerrors; 2097 stats.rx_nombuf -= port->stats.rx_nombuf; 2098 2099 total_recv += stats.ipackets; 2100 total_xmit += stats.opackets; 2101 total_rx_dropped += stats.imissed; 2102 tx_dropped += ports_stats[pt_id].tx_dropped; 2103 tx_dropped += stats.oerrors; 2104 total_tx_dropped += tx_dropped; 2105 total_rx_nombuf += stats.rx_nombuf; 2106 2107 printf("\n %s Forward statistics for port %-2d %s\n", 2108 fwd_stats_border, pt_id, fwd_stats_border); 2109 2110 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2111 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2112 stats.ipackets + stats.imissed); 2113 2114 if (cur_fwd_eng == &csum_fwd_engine) { 2115 printf(" Bad-ipcsum: %-14"PRIu64 2116 " Bad-l4csum: %-14"PRIu64 2117 "Bad-outer-l4csum: %-14"PRIu64"\n", 2118 ports_stats[pt_id].rx_bad_ip_csum, 2119 ports_stats[pt_id].rx_bad_l4_csum, 2120 ports_stats[pt_id].rx_bad_outer_l4_csum); 2121 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2122 ports_stats[pt_id].rx_bad_outer_ip_csum); 2123 } 2124 if (stats.ierrors + stats.rx_nombuf > 0) { 2125 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2126 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2127 } 2128 2129 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2130 "TX-total: %-"PRIu64"\n", 2131 stats.opackets, tx_dropped, 2132 stats.opackets + tx_dropped); 2133 2134 if (record_burst_stats) { 2135 if (ports_stats[pt_id].rx_stream) 2136 pkt_burst_stats_display("RX", 2137 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2138 if (ports_stats[pt_id].tx_stream) 2139 pkt_burst_stats_display("TX", 2140 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2141 } 2142 2143 printf(" %s--------------------------------%s\n", 2144 fwd_stats_border, fwd_stats_border); 2145 } 2146 2147 printf("\n %s Accumulated forward statistics for all ports" 2148 "%s\n", 2149 acc_stats_border, acc_stats_border); 2150 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2151 "%-"PRIu64"\n" 2152 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2153 "%-"PRIu64"\n", 2154 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2155 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2156 if (total_rx_nombuf > 0) 2157 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2158 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2159 "%s\n", 2160 acc_stats_border, acc_stats_border); 2161 if (record_core_cycles) { 2162 #define CYC_PER_MHZ 1E6 2163 if (total_recv > 0 || total_xmit > 0) { 2164 uint64_t total_pkts = 0; 2165 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2166 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2167 total_pkts = total_xmit; 2168 else 2169 total_pkts = total_recv; 2170 2171 printf("\n CPU cycles/packet=%.2F (busy cycles=" 2172 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2173 " MHz Clock\n", 2174 (double) fwd_cycles / total_pkts, 2175 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2176 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2177 } 2178 } 2179 } 2180 2181 void 2182 fwd_stats_reset(void) 2183 { 2184 streamid_t sm_id; 2185 portid_t pt_id; 2186 int ret; 2187 int i; 2188 2189 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2190 pt_id = fwd_ports_ids[i]; 2191 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2192 if (ret != 0) 2193 fprintf(stderr, 2194 "%s: Error: failed to clear stats (port %u):%d", 2195 __func__, pt_id, ret); 2196 } 2197 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2198 struct fwd_stream *fs = fwd_streams[sm_id]; 2199 2200 fs->rx_packets = 0; 2201 fs->tx_packets = 0; 2202 fs->fwd_dropped = 0; 2203 fs->rx_bad_ip_csum = 0; 2204 fs->rx_bad_l4_csum = 0; 2205 fs->rx_bad_outer_l4_csum = 0; 2206 fs->rx_bad_outer_ip_csum = 0; 2207 2208 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2209 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2210 fs->busy_cycles = 0; 2211 } 2212 } 2213 2214 static void 2215 flush_fwd_rx_queues(void) 2216 { 2217 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2218 portid_t rxp; 2219 portid_t port_id; 2220 queueid_t rxq; 2221 uint16_t nb_rx; 2222 uint8_t j; 2223 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2224 uint64_t timer_period; 2225 2226 if (num_procs > 1) { 2227 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2228 return; 2229 } 2230 2231 /* convert to number of cycles */ 2232 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2233 2234 for (j = 0; j < 2; j++) { 2235 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2236 for (rxq = 0; rxq < nb_rxq; rxq++) { 2237 port_id = fwd_ports_ids[rxp]; 2238 2239 /* Polling stopped queues is prohibited. */ 2240 if (ports[port_id].rxq[rxq].state == 2241 RTE_ETH_QUEUE_STATE_STOPPED) 2242 continue; 2243 2244 /** 2245 * testpmd can stuck in the below do while loop 2246 * if rte_eth_rx_burst() always returns nonzero 2247 * packets. So timer is added to exit this loop 2248 * after 1sec timer expiry. 2249 */ 2250 prev_tsc = rte_rdtsc(); 2251 do { 2252 nb_rx = rte_eth_rx_burst(port_id, rxq, 2253 pkts_burst, MAX_PKT_BURST); 2254 rte_pktmbuf_free_bulk(pkts_burst, nb_rx); 2255 2256 cur_tsc = rte_rdtsc(); 2257 diff_tsc = cur_tsc - prev_tsc; 2258 timer_tsc += diff_tsc; 2259 } while ((nb_rx > 0) && 2260 (timer_tsc < timer_period)); 2261 timer_tsc = 0; 2262 } 2263 } 2264 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2265 } 2266 } 2267 2268 static void 2269 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2270 { 2271 struct fwd_stream **fsm; 2272 uint64_t prev_tsc; 2273 streamid_t nb_fs; 2274 streamid_t sm_id; 2275 #ifdef RTE_LIB_BITRATESTATS 2276 uint64_t tics_per_1sec; 2277 uint64_t tics_datum; 2278 uint64_t tics_current; 2279 uint16_t i, cnt_ports; 2280 2281 cnt_ports = nb_ports; 2282 tics_datum = rte_rdtsc(); 2283 tics_per_1sec = rte_get_timer_hz(); 2284 #endif 2285 fsm = &fwd_streams[fc->stream_idx]; 2286 nb_fs = fc->stream_nb; 2287 prev_tsc = rte_rdtsc(); 2288 do { 2289 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2290 struct fwd_stream *fs = fsm[sm_id]; 2291 uint64_t start_fs_tsc = 0; 2292 bool busy; 2293 2294 if (fs->disabled) 2295 continue; 2296 if (record_core_cycles) 2297 start_fs_tsc = rte_rdtsc(); 2298 busy = (*pkt_fwd)(fs); 2299 if (record_core_cycles && busy) 2300 fs->busy_cycles += rte_rdtsc() - start_fs_tsc; 2301 } 2302 #ifdef RTE_LIB_BITRATESTATS 2303 if (bitrate_enabled != 0 && 2304 bitrate_lcore_id == rte_lcore_id()) { 2305 tics_current = rte_rdtsc(); 2306 if (tics_current - tics_datum >= tics_per_1sec) { 2307 /* Periodic bitrate calculation */ 2308 for (i = 0; i < cnt_ports; i++) 2309 rte_stats_bitrate_calc(bitrate_data, 2310 ports_ids[i]); 2311 tics_datum = tics_current; 2312 } 2313 } 2314 #endif 2315 #ifdef RTE_LIB_LATENCYSTATS 2316 if (latencystats_enabled != 0 && 2317 latencystats_lcore_id == rte_lcore_id()) 2318 rte_latencystats_update(); 2319 #endif 2320 if (record_core_cycles) { 2321 uint64_t tsc = rte_rdtsc(); 2322 2323 fc->total_cycles += tsc - prev_tsc; 2324 prev_tsc = tsc; 2325 } 2326 } while (! fc->stopped); 2327 } 2328 2329 static int 2330 lcore_usage_callback(unsigned int lcore_id, struct rte_lcore_usage *usage) 2331 { 2332 struct fwd_stream **fsm; 2333 struct fwd_lcore *fc; 2334 streamid_t nb_fs; 2335 streamid_t sm_id; 2336 2337 fc = lcore_to_fwd_lcore(lcore_id); 2338 if (fc == NULL) 2339 return -1; 2340 2341 fsm = &fwd_streams[fc->stream_idx]; 2342 nb_fs = fc->stream_nb; 2343 usage->busy_cycles = 0; 2344 usage->total_cycles = fc->total_cycles; 2345 2346 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2347 if (!fsm[sm_id]->disabled) 2348 usage->busy_cycles += fsm[sm_id]->busy_cycles; 2349 } 2350 2351 return 0; 2352 } 2353 2354 static int 2355 start_pkt_forward_on_core(void *fwd_arg) 2356 { 2357 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2358 cur_fwd_config.fwd_eng->packet_fwd); 2359 return 0; 2360 } 2361 2362 /* 2363 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2364 * Used to start communication flows in network loopback test configurations. 2365 */ 2366 static int 2367 run_one_txonly_burst_on_core(void *fwd_arg) 2368 { 2369 struct fwd_lcore *fwd_lc; 2370 struct fwd_lcore tmp_lcore; 2371 2372 fwd_lc = (struct fwd_lcore *) fwd_arg; 2373 tmp_lcore = *fwd_lc; 2374 tmp_lcore.stopped = 1; 2375 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2376 return 0; 2377 } 2378 2379 /* 2380 * Launch packet forwarding: 2381 * - Setup per-port forwarding context. 2382 * - launch logical cores with their forwarding configuration. 2383 */ 2384 static void 2385 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2386 { 2387 unsigned int i; 2388 unsigned int lc_id; 2389 int diag; 2390 2391 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2392 lc_id = fwd_lcores_cpuids[i]; 2393 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2394 fwd_lcores[i]->stopped = 0; 2395 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2396 fwd_lcores[i], lc_id); 2397 if (diag != 0) 2398 fprintf(stderr, 2399 "launch lcore %u failed - diag=%d\n", 2400 lc_id, diag); 2401 } 2402 } 2403 } 2404 2405 void 2406 common_fwd_stream_init(struct fwd_stream *fs) 2407 { 2408 bool rx_stopped, tx_stopped; 2409 2410 rx_stopped = (ports[fs->rx_port].rxq[fs->rx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2411 tx_stopped = (ports[fs->tx_port].txq[fs->tx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2412 fs->disabled = rx_stopped || tx_stopped; 2413 } 2414 2415 static void 2416 update_rx_queue_state(uint16_t port_id, uint16_t queue_id) 2417 { 2418 struct rte_eth_rxq_info rx_qinfo; 2419 int32_t rc; 2420 2421 rc = rte_eth_rx_queue_info_get(port_id, 2422 queue_id, &rx_qinfo); 2423 if (rc == 0) { 2424 ports[port_id].rxq[queue_id].state = 2425 rx_qinfo.queue_state; 2426 } else if (rc == -ENOTSUP) { 2427 /* 2428 * Do not change the rxq state for primary process 2429 * to ensure that the PMDs do not implement 2430 * rte_eth_rx_queue_info_get can forward as before. 2431 */ 2432 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 2433 return; 2434 /* 2435 * Set the rxq state to RTE_ETH_QUEUE_STATE_STARTED 2436 * to ensure that the PMDs do not implement 2437 * rte_eth_rx_queue_info_get can forward. 2438 */ 2439 ports[port_id].rxq[queue_id].state = 2440 RTE_ETH_QUEUE_STATE_STARTED; 2441 } else { 2442 TESTPMD_LOG(WARNING, 2443 "Failed to get rx queue info\n"); 2444 } 2445 } 2446 2447 static void 2448 update_tx_queue_state(uint16_t port_id, uint16_t queue_id) 2449 { 2450 struct rte_eth_txq_info tx_qinfo; 2451 int32_t rc; 2452 2453 rc = rte_eth_tx_queue_info_get(port_id, 2454 queue_id, &tx_qinfo); 2455 if (rc == 0) { 2456 ports[port_id].txq[queue_id].state = 2457 tx_qinfo.queue_state; 2458 } else if (rc == -ENOTSUP) { 2459 /* 2460 * Do not change the txq state for primary process 2461 * to ensure that the PMDs do not implement 2462 * rte_eth_tx_queue_info_get can forward as before. 2463 */ 2464 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 2465 return; 2466 /* 2467 * Set the txq state to RTE_ETH_QUEUE_STATE_STARTED 2468 * to ensure that the PMDs do not implement 2469 * rte_eth_tx_queue_info_get can forward. 2470 */ 2471 ports[port_id].txq[queue_id].state = 2472 RTE_ETH_QUEUE_STATE_STARTED; 2473 } else { 2474 TESTPMD_LOG(WARNING, 2475 "Failed to get tx queue info\n"); 2476 } 2477 } 2478 2479 static void 2480 update_queue_state(portid_t pid) 2481 { 2482 portid_t pi; 2483 queueid_t qi; 2484 2485 RTE_ETH_FOREACH_DEV(pi) { 2486 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2487 continue; 2488 2489 for (qi = 0; qi < nb_rxq; qi++) 2490 update_rx_queue_state(pi, qi); 2491 for (qi = 0; qi < nb_txq; qi++) 2492 update_tx_queue_state(pi, qi); 2493 } 2494 } 2495 2496 /* 2497 * Launch packet forwarding configuration. 2498 */ 2499 void 2500 start_packet_forwarding(int with_tx_first) 2501 { 2502 port_fwd_begin_t port_fwd_begin; 2503 port_fwd_end_t port_fwd_end; 2504 stream_init_t stream_init = cur_fwd_eng->stream_init; 2505 unsigned int i; 2506 2507 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2508 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2509 2510 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2511 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2512 2513 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2514 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2515 (!nb_rxq || !nb_txq)) 2516 rte_exit(EXIT_FAILURE, 2517 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2518 cur_fwd_eng->fwd_mode_name); 2519 2520 if (all_ports_started() == 0) { 2521 fprintf(stderr, "Not all ports were started\n"); 2522 return; 2523 } 2524 if (test_done == 0) { 2525 fprintf(stderr, "Packet forwarding already started\n"); 2526 return; 2527 } 2528 2529 fwd_config_setup(); 2530 2531 pkt_fwd_config_display(&cur_fwd_config); 2532 if (!pkt_fwd_shared_rxq_check()) 2533 return; 2534 2535 if (stream_init != NULL) { 2536 update_queue_state(RTE_PORT_ALL); 2537 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2538 stream_init(fwd_streams[i]); 2539 } 2540 2541 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2542 if (port_fwd_begin != NULL) { 2543 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2544 if (port_fwd_begin(fwd_ports_ids[i])) { 2545 fprintf(stderr, 2546 "Packet forwarding is not ready\n"); 2547 return; 2548 } 2549 } 2550 } 2551 2552 if (with_tx_first) { 2553 port_fwd_begin = tx_only_engine.port_fwd_begin; 2554 if (port_fwd_begin != NULL) { 2555 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2556 if (port_fwd_begin(fwd_ports_ids[i])) { 2557 fprintf(stderr, 2558 "Packet forwarding is not ready\n"); 2559 return; 2560 } 2561 } 2562 } 2563 } 2564 2565 test_done = 0; 2566 2567 if(!no_flush_rx) 2568 flush_fwd_rx_queues(); 2569 2570 rxtx_config_display(); 2571 2572 fwd_stats_reset(); 2573 if (with_tx_first) { 2574 while (with_tx_first--) { 2575 launch_packet_forwarding( 2576 run_one_txonly_burst_on_core); 2577 rte_eal_mp_wait_lcore(); 2578 } 2579 port_fwd_end = tx_only_engine.port_fwd_end; 2580 if (port_fwd_end != NULL) { 2581 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2582 (*port_fwd_end)(fwd_ports_ids[i]); 2583 } 2584 } 2585 launch_packet_forwarding(start_pkt_forward_on_core); 2586 } 2587 2588 void 2589 stop_packet_forwarding(void) 2590 { 2591 port_fwd_end_t port_fwd_end; 2592 lcoreid_t lc_id; 2593 portid_t pt_id; 2594 int i; 2595 2596 if (test_done) { 2597 fprintf(stderr, "Packet forwarding not started\n"); 2598 return; 2599 } 2600 printf("Telling cores to stop..."); 2601 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2602 fwd_lcores[lc_id]->stopped = 1; 2603 printf("\nWaiting for lcores to finish...\n"); 2604 rte_eal_mp_wait_lcore(); 2605 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2606 if (port_fwd_end != NULL) { 2607 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2608 pt_id = fwd_ports_ids[i]; 2609 (*port_fwd_end)(pt_id); 2610 } 2611 } 2612 2613 fwd_stats_display(); 2614 2615 printf("\nDone.\n"); 2616 test_done = 1; 2617 } 2618 2619 void 2620 dev_set_link_up(portid_t pid) 2621 { 2622 if (rte_eth_dev_set_link_up(pid) < 0) 2623 fprintf(stderr, "\nSet link up fail.\n"); 2624 } 2625 2626 void 2627 dev_set_link_down(portid_t pid) 2628 { 2629 if (rte_eth_dev_set_link_down(pid) < 0) 2630 fprintf(stderr, "\nSet link down fail.\n"); 2631 } 2632 2633 static int 2634 all_ports_started(void) 2635 { 2636 portid_t pi; 2637 struct rte_port *port; 2638 2639 RTE_ETH_FOREACH_DEV(pi) { 2640 port = &ports[pi]; 2641 /* Check if there is a port which is not started */ 2642 if ((port->port_status != RTE_PORT_STARTED) && 2643 (port->member_flag == 0)) 2644 return 0; 2645 } 2646 2647 /* No port is not started */ 2648 return 1; 2649 } 2650 2651 int 2652 port_is_stopped(portid_t port_id) 2653 { 2654 struct rte_port *port = &ports[port_id]; 2655 2656 if ((port->port_status != RTE_PORT_STOPPED) && 2657 (port->member_flag == 0)) 2658 return 0; 2659 return 1; 2660 } 2661 2662 int 2663 all_ports_stopped(void) 2664 { 2665 portid_t pi; 2666 2667 RTE_ETH_FOREACH_DEV(pi) { 2668 if (!port_is_stopped(pi)) 2669 return 0; 2670 } 2671 2672 return 1; 2673 } 2674 2675 int 2676 port_is_started(portid_t port_id) 2677 { 2678 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2679 return 0; 2680 2681 if (ports[port_id].port_status != RTE_PORT_STARTED) 2682 return 0; 2683 2684 return 1; 2685 } 2686 2687 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2688 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2689 2690 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2691 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2692 2693 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2694 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2695 2696 2697 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2698 static int 2699 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2700 { 2701 queueid_t qi; 2702 struct rte_eth_hairpin_conf hairpin_conf = { 2703 .peer_count = 1, 2704 }; 2705 int i; 2706 int diag; 2707 struct rte_port *port = &ports[pi]; 2708 uint16_t peer_rx_port = pi; 2709 uint16_t peer_tx_port = pi; 2710 uint32_t manual = 1; 2711 uint32_t tx_exp = hairpin_mode & 0x10; 2712 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2713 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2714 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2715 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2716 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2717 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2718 2719 if (!(hairpin_mode & 0xf)) { 2720 peer_rx_port = pi; 2721 peer_tx_port = pi; 2722 manual = 0; 2723 } else if (hairpin_mode & 0x1) { 2724 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2725 RTE_ETH_DEV_NO_OWNER); 2726 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2727 peer_tx_port = rte_eth_find_next_owned_by(0, 2728 RTE_ETH_DEV_NO_OWNER); 2729 if (p_pi != RTE_MAX_ETHPORTS) { 2730 peer_rx_port = p_pi; 2731 } else { 2732 uint16_t next_pi; 2733 2734 /* Last port will be the peer RX port of the first. */ 2735 RTE_ETH_FOREACH_DEV(next_pi) 2736 peer_rx_port = next_pi; 2737 } 2738 manual = 1; 2739 } else if (hairpin_mode & 0x2) { 2740 if (cnt_pi & 0x1) { 2741 peer_rx_port = p_pi; 2742 } else { 2743 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2744 RTE_ETH_DEV_NO_OWNER); 2745 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2746 peer_rx_port = pi; 2747 } 2748 peer_tx_port = peer_rx_port; 2749 manual = 1; 2750 } 2751 2752 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2753 hairpin_conf.peers[0].port = peer_rx_port; 2754 hairpin_conf.peers[0].queue = i + nb_rxq; 2755 hairpin_conf.manual_bind = !!manual; 2756 hairpin_conf.tx_explicit = !!tx_exp; 2757 hairpin_conf.force_memory = !!tx_force_memory; 2758 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2759 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2760 diag = rte_eth_tx_hairpin_queue_setup 2761 (pi, qi, nb_txd, &hairpin_conf); 2762 i++; 2763 if (diag == 0) 2764 continue; 2765 2766 /* Fail to setup rx queue, return */ 2767 if (port->port_status == RTE_PORT_HANDLING) 2768 port->port_status = RTE_PORT_STOPPED; 2769 else 2770 fprintf(stderr, 2771 "Port %d can not be set back to stopped\n", pi); 2772 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2773 pi); 2774 /* try to reconfigure queues next time */ 2775 port->need_reconfig_queues = 1; 2776 return -1; 2777 } 2778 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2779 hairpin_conf.peers[0].port = peer_tx_port; 2780 hairpin_conf.peers[0].queue = i + nb_txq; 2781 hairpin_conf.manual_bind = !!manual; 2782 hairpin_conf.tx_explicit = !!tx_exp; 2783 hairpin_conf.force_memory = !!rx_force_memory; 2784 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2785 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2786 diag = rte_eth_rx_hairpin_queue_setup 2787 (pi, qi, nb_rxd, &hairpin_conf); 2788 i++; 2789 if (diag == 0) 2790 continue; 2791 2792 /* Fail to setup rx queue, return */ 2793 if (port->port_status == RTE_PORT_HANDLING) 2794 port->port_status = RTE_PORT_STOPPED; 2795 else 2796 fprintf(stderr, 2797 "Port %d can not be set back to stopped\n", pi); 2798 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2799 pi); 2800 /* try to reconfigure queues next time */ 2801 port->need_reconfig_queues = 1; 2802 return -1; 2803 } 2804 return 0; 2805 } 2806 2807 /* Configure the Rx with optional split. */ 2808 int 2809 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2810 uint16_t nb_rx_desc, unsigned int socket_id, 2811 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2812 { 2813 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2814 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2815 struct rte_mempool *mpx; 2816 unsigned int i, mp_n; 2817 uint32_t prev_hdrs = 0; 2818 int ret; 2819 2820 2821 if ((rx_pkt_nb_segs > 1) && 2822 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2823 /* multi-segment configuration */ 2824 for (i = 0; i < rx_pkt_nb_segs; i++) { 2825 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2826 /* 2827 * Use last valid pool for the segments with number 2828 * exceeding the pool index. 2829 */ 2830 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2831 mpx = mbuf_pool_find(socket_id, mp_n); 2832 /* Handle zero as mbuf data buffer size. */ 2833 rx_seg->offset = i < rx_pkt_nb_offs ? 2834 rx_pkt_seg_offsets[i] : 0; 2835 rx_seg->mp = mpx ? mpx : mp; 2836 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2837 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2838 prev_hdrs |= rx_seg->proto_hdr; 2839 } else { 2840 rx_seg->length = rx_pkt_seg_lengths[i] ? 2841 rx_pkt_seg_lengths[i] : 2842 mbuf_data_size[mp_n]; 2843 } 2844 } 2845 rx_conf->rx_nseg = rx_pkt_nb_segs; 2846 rx_conf->rx_seg = rx_useg; 2847 rx_conf->rx_mempools = NULL; 2848 rx_conf->rx_nmempool = 0; 2849 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2850 socket_id, rx_conf, NULL); 2851 rx_conf->rx_seg = NULL; 2852 rx_conf->rx_nseg = 0; 2853 } else if (multi_rx_mempool == 1) { 2854 /* multi-pool configuration */ 2855 struct rte_eth_dev_info dev_info; 2856 2857 if (mbuf_data_size_n <= 1) { 2858 fprintf(stderr, "Invalid number of mempools %u\n", 2859 mbuf_data_size_n); 2860 return -EINVAL; 2861 } 2862 ret = rte_eth_dev_info_get(port_id, &dev_info); 2863 if (ret != 0) 2864 return ret; 2865 if (dev_info.max_rx_mempools == 0) { 2866 fprintf(stderr, 2867 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2868 port_id); 2869 return -ENOTSUP; 2870 } 2871 for (i = 0; i < mbuf_data_size_n; i++) { 2872 mpx = mbuf_pool_find(socket_id, i); 2873 rx_mempool[i] = mpx ? mpx : mp; 2874 } 2875 rx_conf->rx_mempools = rx_mempool; 2876 rx_conf->rx_nmempool = mbuf_data_size_n; 2877 rx_conf->rx_seg = NULL; 2878 rx_conf->rx_nseg = 0; 2879 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2880 socket_id, rx_conf, NULL); 2881 rx_conf->rx_mempools = NULL; 2882 rx_conf->rx_nmempool = 0; 2883 } else { 2884 /* Single pool/segment configuration */ 2885 rx_conf->rx_seg = NULL; 2886 rx_conf->rx_nseg = 0; 2887 rx_conf->rx_mempools = NULL; 2888 rx_conf->rx_nmempool = 0; 2889 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2890 socket_id, rx_conf, mp); 2891 } 2892 2893 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2894 RTE_ETH_QUEUE_STATE_STOPPED : 2895 RTE_ETH_QUEUE_STATE_STARTED; 2896 return ret; 2897 } 2898 2899 static int 2900 alloc_xstats_display_info(portid_t pi) 2901 { 2902 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2903 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2904 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2905 2906 if (xstats_display_num == 0) 2907 return 0; 2908 2909 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2910 if (*ids_supp == NULL) 2911 goto fail_ids_supp; 2912 2913 *prev_values = calloc(xstats_display_num, 2914 sizeof(**prev_values)); 2915 if (*prev_values == NULL) 2916 goto fail_prev_values; 2917 2918 *curr_values = calloc(xstats_display_num, 2919 sizeof(**curr_values)); 2920 if (*curr_values == NULL) 2921 goto fail_curr_values; 2922 2923 ports[pi].xstats_info.allocated = true; 2924 2925 return 0; 2926 2927 fail_curr_values: 2928 free(*prev_values); 2929 fail_prev_values: 2930 free(*ids_supp); 2931 fail_ids_supp: 2932 return -ENOMEM; 2933 } 2934 2935 static void 2936 free_xstats_display_info(portid_t pi) 2937 { 2938 if (!ports[pi].xstats_info.allocated) 2939 return; 2940 free(ports[pi].xstats_info.ids_supp); 2941 free(ports[pi].xstats_info.prev_values); 2942 free(ports[pi].xstats_info.curr_values); 2943 ports[pi].xstats_info.allocated = false; 2944 } 2945 2946 /** Fill helper structures for specified port to show extended statistics. */ 2947 static void 2948 fill_xstats_display_info_for_port(portid_t pi) 2949 { 2950 unsigned int stat, stat_supp; 2951 const char *xstat_name; 2952 struct rte_port *port; 2953 uint64_t *ids_supp; 2954 int rc; 2955 2956 if (xstats_display_num == 0) 2957 return; 2958 2959 if (pi == (portid_t)RTE_PORT_ALL) { 2960 fill_xstats_display_info(); 2961 return; 2962 } 2963 2964 port = &ports[pi]; 2965 if (port->port_status != RTE_PORT_STARTED) 2966 return; 2967 2968 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2969 rte_exit(EXIT_FAILURE, 2970 "Failed to allocate xstats display memory\n"); 2971 2972 ids_supp = port->xstats_info.ids_supp; 2973 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2974 xstat_name = xstats_display[stat].name; 2975 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2976 ids_supp + stat_supp); 2977 if (rc != 0) { 2978 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2979 xstat_name, pi, stat); 2980 continue; 2981 } 2982 stat_supp++; 2983 } 2984 2985 port->xstats_info.ids_supp_sz = stat_supp; 2986 } 2987 2988 /** Fill helper structures for all ports to show extended statistics. */ 2989 static void 2990 fill_xstats_display_info(void) 2991 { 2992 portid_t pi; 2993 2994 if (xstats_display_num == 0) 2995 return; 2996 2997 RTE_ETH_FOREACH_DEV(pi) 2998 fill_xstats_display_info_for_port(pi); 2999 } 3000 3001 /* 3002 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 3003 * device in dev_info is zero when no member is added. And its capability 3004 * will be updated when add a new member device. So adding a member device need 3005 * to update the port configurations of bonding device. 3006 */ 3007 static void 3008 update_bonding_port_dev_conf(portid_t bond_pid) 3009 { 3010 #ifdef RTE_NET_BOND 3011 struct rte_port *port = &ports[bond_pid]; 3012 uint16_t i; 3013 int ret; 3014 3015 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 3016 if (ret != 0) { 3017 fprintf(stderr, "Failed to get dev info for port = %u\n", 3018 bond_pid); 3019 return; 3020 } 3021 3022 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 3023 port->dev_conf.txmode.offloads |= 3024 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 3025 /* Apply Tx offloads configuration */ 3026 for (i = 0; i < port->dev_info.max_tx_queues; i++) 3027 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 3028 3029 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 3030 port->dev_info.flow_type_rss_offloads; 3031 #else 3032 RTE_SET_USED(bond_pid); 3033 #endif 3034 } 3035 3036 int 3037 start_port(portid_t pid) 3038 { 3039 int diag; 3040 portid_t pi; 3041 portid_t p_pi = RTE_MAX_ETHPORTS; 3042 portid_t pl[RTE_MAX_ETHPORTS]; 3043 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3044 uint16_t cnt_pi = 0; 3045 uint16_t cfg_pi = 0; 3046 int peer_pi; 3047 queueid_t qi; 3048 struct rte_port *port; 3049 struct rte_eth_hairpin_cap cap; 3050 bool at_least_one_port_exist = false; 3051 bool all_ports_already_started = true; 3052 bool at_least_one_port_successfully_started = false; 3053 3054 if (port_id_is_invalid(pid, ENABLED_WARN)) 3055 return 0; 3056 3057 RTE_ETH_FOREACH_DEV(pi) { 3058 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3059 continue; 3060 3061 if (port_is_bonding_member(pi)) { 3062 fprintf(stderr, 3063 "Please remove port %d from bonding device.\n", 3064 pi); 3065 continue; 3066 } 3067 3068 at_least_one_port_exist = true; 3069 3070 port = &ports[pi]; 3071 if (port->port_status == RTE_PORT_STOPPED) { 3072 port->port_status = RTE_PORT_HANDLING; 3073 all_ports_already_started = false; 3074 } else { 3075 fprintf(stderr, "Port %d is now not stopped\n", pi); 3076 continue; 3077 } 3078 3079 if (port->need_reconfig > 0) { 3080 struct rte_eth_conf dev_conf; 3081 int k; 3082 3083 port->need_reconfig = 0; 3084 3085 if (flow_isolate_all) { 3086 int ret = port_flow_isolate(pi, 1); 3087 if (ret) { 3088 fprintf(stderr, 3089 "Failed to apply isolated mode on port %d\n", 3090 pi); 3091 return -1; 3092 } 3093 } 3094 configure_rxtx_dump_callbacks(0); 3095 printf("Configuring Port %d (socket %u)\n", pi, 3096 port->socket_id); 3097 if (nb_hairpinq > 0 && 3098 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 3099 fprintf(stderr, 3100 "Port %d doesn't support hairpin queues\n", 3101 pi); 3102 return -1; 3103 } 3104 3105 if (port->bond_flag == 1 && port->update_conf == 1) { 3106 update_bonding_port_dev_conf(pi); 3107 port->update_conf = 0; 3108 } 3109 3110 /* configure port */ 3111 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 3112 nb_txq + nb_hairpinq, 3113 &(port->dev_conf)); 3114 if (diag != 0) { 3115 if (port->port_status == RTE_PORT_HANDLING) 3116 port->port_status = RTE_PORT_STOPPED; 3117 else 3118 fprintf(stderr, 3119 "Port %d can not be set back to stopped\n", 3120 pi); 3121 fprintf(stderr, "Fail to configure port %d\n", 3122 pi); 3123 /* try to reconfigure port next time */ 3124 port->need_reconfig = 1; 3125 return -1; 3126 } 3127 /* get device configuration*/ 3128 if (0 != 3129 eth_dev_conf_get_print_err(pi, &dev_conf)) { 3130 fprintf(stderr, 3131 "port %d can not get device configuration\n", 3132 pi); 3133 return -1; 3134 } 3135 /* Apply Rx offloads configuration */ 3136 if (dev_conf.rxmode.offloads != 3137 port->dev_conf.rxmode.offloads) { 3138 port->dev_conf.rxmode.offloads |= 3139 dev_conf.rxmode.offloads; 3140 for (k = 0; 3141 k < port->dev_info.max_rx_queues; 3142 k++) 3143 port->rxq[k].conf.offloads |= 3144 dev_conf.rxmode.offloads; 3145 } 3146 /* Apply Tx offloads configuration */ 3147 if (dev_conf.txmode.offloads != 3148 port->dev_conf.txmode.offloads) { 3149 port->dev_conf.txmode.offloads |= 3150 dev_conf.txmode.offloads; 3151 for (k = 0; 3152 k < port->dev_info.max_tx_queues; 3153 k++) 3154 port->txq[k].conf.offloads |= 3155 dev_conf.txmode.offloads; 3156 } 3157 } 3158 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 3159 port->need_reconfig_queues = 0; 3160 /* setup tx queues */ 3161 for (qi = 0; qi < nb_txq; qi++) { 3162 struct rte_eth_txconf *conf = 3163 &port->txq[qi].conf; 3164 3165 if ((numa_support) && 3166 (txring_numa[pi] != NUMA_NO_CONFIG)) 3167 diag = rte_eth_tx_queue_setup(pi, qi, 3168 port->nb_tx_desc[qi], 3169 txring_numa[pi], 3170 &(port->txq[qi].conf)); 3171 else 3172 diag = rte_eth_tx_queue_setup(pi, qi, 3173 port->nb_tx_desc[qi], 3174 port->socket_id, 3175 &(port->txq[qi].conf)); 3176 3177 if (diag == 0) { 3178 port->txq[qi].state = 3179 conf->tx_deferred_start ? 3180 RTE_ETH_QUEUE_STATE_STOPPED : 3181 RTE_ETH_QUEUE_STATE_STARTED; 3182 continue; 3183 } 3184 3185 /* Fail to setup tx queue, return */ 3186 if (port->port_status == RTE_PORT_HANDLING) 3187 port->port_status = RTE_PORT_STOPPED; 3188 else 3189 fprintf(stderr, 3190 "Port %d can not be set back to stopped\n", 3191 pi); 3192 fprintf(stderr, 3193 "Fail to configure port %d tx queues\n", 3194 pi); 3195 /* try to reconfigure queues next time */ 3196 port->need_reconfig_queues = 1; 3197 return -1; 3198 } 3199 for (qi = 0; qi < nb_rxq; qi++) { 3200 /* setup rx queues */ 3201 if ((numa_support) && 3202 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3203 struct rte_mempool * mp = 3204 mbuf_pool_find 3205 (rxring_numa[pi], 0); 3206 if (mp == NULL) { 3207 fprintf(stderr, 3208 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3209 rxring_numa[pi]); 3210 return -1; 3211 } 3212 3213 diag = rx_queue_setup(pi, qi, 3214 port->nb_rx_desc[qi], 3215 rxring_numa[pi], 3216 &(port->rxq[qi].conf), 3217 mp); 3218 } else { 3219 struct rte_mempool *mp = 3220 mbuf_pool_find 3221 (port->socket_id, 0); 3222 if (mp == NULL) { 3223 fprintf(stderr, 3224 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3225 port->socket_id); 3226 return -1; 3227 } 3228 diag = rx_queue_setup(pi, qi, 3229 port->nb_rx_desc[qi], 3230 port->socket_id, 3231 &(port->rxq[qi].conf), 3232 mp); 3233 } 3234 if (diag == 0) 3235 continue; 3236 3237 /* Fail to setup rx queue, return */ 3238 if (port->port_status == RTE_PORT_HANDLING) 3239 port->port_status = RTE_PORT_STOPPED; 3240 else 3241 fprintf(stderr, 3242 "Port %d can not be set back to stopped\n", 3243 pi); 3244 fprintf(stderr, 3245 "Fail to configure port %d rx queues\n", 3246 pi); 3247 /* try to reconfigure queues next time */ 3248 port->need_reconfig_queues = 1; 3249 return -1; 3250 } 3251 /* setup hairpin queues */ 3252 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3253 return -1; 3254 } 3255 configure_rxtx_dump_callbacks(verbose_level); 3256 if (clear_ptypes) { 3257 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3258 NULL, 0); 3259 if (diag < 0) 3260 fprintf(stderr, 3261 "Port %d: Failed to disable Ptype parsing\n", 3262 pi); 3263 } 3264 3265 p_pi = pi; 3266 cnt_pi++; 3267 3268 /* start port */ 3269 diag = eth_dev_start_mp(pi); 3270 if (diag < 0) { 3271 fprintf(stderr, "Fail to start port %d: %s\n", 3272 pi, rte_strerror(-diag)); 3273 3274 /* Fail to setup rx queue, return */ 3275 if (port->port_status == RTE_PORT_HANDLING) 3276 port->port_status = RTE_PORT_STOPPED; 3277 else 3278 fprintf(stderr, 3279 "Port %d can not be set back to stopped\n", 3280 pi); 3281 continue; 3282 } 3283 3284 if (port->port_status == RTE_PORT_HANDLING) 3285 port->port_status = RTE_PORT_STARTED; 3286 else 3287 fprintf(stderr, "Port %d can not be set into started\n", 3288 pi); 3289 3290 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3291 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3292 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3293 3294 at_least_one_port_successfully_started = true; 3295 3296 pl[cfg_pi++] = pi; 3297 } 3298 3299 update_queue_state(pi); 3300 3301 if (at_least_one_port_successfully_started && !no_link_check) 3302 check_all_ports_link_status(RTE_PORT_ALL); 3303 else if (at_least_one_port_exist & all_ports_already_started) 3304 fprintf(stderr, "Please stop the ports first\n"); 3305 3306 if (hairpin_mode & 0xf) { 3307 uint16_t i; 3308 int j; 3309 3310 /* bind all started hairpin ports */ 3311 for (i = 0; i < cfg_pi; i++) { 3312 pi = pl[i]; 3313 /* bind current Tx to all peer Rx */ 3314 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3315 RTE_MAX_ETHPORTS, 1); 3316 if (peer_pi < 0) 3317 return peer_pi; 3318 for (j = 0; j < peer_pi; j++) { 3319 if (!port_is_started(peer_pl[j])) 3320 continue; 3321 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3322 if (diag < 0) { 3323 fprintf(stderr, 3324 "Error during binding hairpin Tx port %u to %u: %s\n", 3325 pi, peer_pl[j], 3326 rte_strerror(-diag)); 3327 return -1; 3328 } 3329 } 3330 /* bind all peer Tx to current Rx */ 3331 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3332 RTE_MAX_ETHPORTS, 0); 3333 if (peer_pi < 0) 3334 return peer_pi; 3335 for (j = 0; j < peer_pi; j++) { 3336 if (!port_is_started(peer_pl[j])) 3337 continue; 3338 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3339 if (diag < 0) { 3340 fprintf(stderr, 3341 "Error during binding hairpin Tx port %u to %u: %s\n", 3342 peer_pl[j], pi, 3343 rte_strerror(-diag)); 3344 return -1; 3345 } 3346 } 3347 } 3348 } 3349 3350 fill_xstats_display_info_for_port(pid); 3351 3352 printf("Done\n"); 3353 return 0; 3354 } 3355 3356 void 3357 stop_port(portid_t pid) 3358 { 3359 portid_t pi; 3360 struct rte_port *port; 3361 int need_check_link_status = 0; 3362 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3363 int peer_pi; 3364 int ret; 3365 3366 if (port_id_is_invalid(pid, ENABLED_WARN)) 3367 return; 3368 3369 printf("Stopping ports...\n"); 3370 3371 RTE_ETH_FOREACH_DEV(pi) { 3372 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3373 continue; 3374 3375 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3376 fprintf(stderr, 3377 "Please remove port %d from forwarding configuration.\n", 3378 pi); 3379 continue; 3380 } 3381 3382 if (port_is_bonding_member(pi)) { 3383 fprintf(stderr, 3384 "Please remove port %d from bonding device.\n", 3385 pi); 3386 continue; 3387 } 3388 3389 port = &ports[pi]; 3390 if (port->port_status == RTE_PORT_STARTED) 3391 port->port_status = RTE_PORT_HANDLING; 3392 else 3393 continue; 3394 3395 if (hairpin_mode & 0xf) { 3396 int j; 3397 3398 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3399 /* unbind all peer Tx from current Rx */ 3400 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3401 RTE_MAX_ETHPORTS, 0); 3402 if (peer_pi < 0) 3403 continue; 3404 for (j = 0; j < peer_pi; j++) { 3405 if (!port_is_started(peer_pl[j])) 3406 continue; 3407 rte_eth_hairpin_unbind(peer_pl[j], pi); 3408 } 3409 } 3410 3411 if (port->flow_list && !no_flow_flush) 3412 port_flow_flush(pi); 3413 3414 ret = eth_dev_stop_mp(pi); 3415 if (ret != 0) { 3416 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3417 pi); 3418 /* Allow to retry stopping the port. */ 3419 port->port_status = RTE_PORT_STARTED; 3420 continue; 3421 } 3422 3423 if (port->port_status == RTE_PORT_HANDLING) 3424 port->port_status = RTE_PORT_STOPPED; 3425 else 3426 fprintf(stderr, "Port %d can not be set into stopped\n", 3427 pi); 3428 need_check_link_status = 1; 3429 } 3430 if (need_check_link_status && !no_link_check) 3431 check_all_ports_link_status(RTE_PORT_ALL); 3432 3433 printf("Done\n"); 3434 } 3435 3436 static void 3437 remove_invalid_ports_in(portid_t *array, portid_t *total) 3438 { 3439 portid_t i; 3440 portid_t new_total = 0; 3441 3442 for (i = 0; i < *total; i++) 3443 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3444 array[new_total] = array[i]; 3445 new_total++; 3446 } 3447 *total = new_total; 3448 } 3449 3450 static void 3451 remove_invalid_ports(void) 3452 { 3453 remove_invalid_ports_in(ports_ids, &nb_ports); 3454 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3455 nb_cfg_ports = nb_fwd_ports; 3456 } 3457 3458 static void 3459 flush_port_owned_resources(portid_t pi) 3460 { 3461 mcast_addr_pool_destroy(pi); 3462 port_flow_flush(pi); 3463 port_flow_template_table_flush(pi); 3464 port_flow_pattern_template_flush(pi); 3465 port_flow_actions_template_flush(pi); 3466 port_flex_item_flush(pi); 3467 port_action_handle_flush(pi); 3468 } 3469 3470 static void 3471 clear_bonding_member_device(portid_t *member_pids, uint16_t num_members) 3472 { 3473 struct rte_port *port; 3474 portid_t member_pid; 3475 uint16_t i; 3476 3477 for (i = 0; i < num_members; i++) { 3478 member_pid = member_pids[i]; 3479 if (port_is_started(member_pid) == 1) { 3480 if (rte_eth_dev_stop(member_pid) != 0) 3481 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3482 member_pid); 3483 3484 port = &ports[member_pid]; 3485 port->port_status = RTE_PORT_STOPPED; 3486 } 3487 3488 clear_port_member_flag(member_pid); 3489 3490 /* Close member device when testpmd quit or is killed. */ 3491 if (cl_quit == 1 || f_quit == 1) 3492 rte_eth_dev_close(member_pid); 3493 } 3494 } 3495 3496 void 3497 close_port(portid_t pid) 3498 { 3499 portid_t pi; 3500 struct rte_port *port; 3501 portid_t member_pids[RTE_MAX_ETHPORTS]; 3502 int num_members = 0; 3503 3504 if (port_id_is_invalid(pid, ENABLED_WARN)) 3505 return; 3506 3507 printf("Closing ports...\n"); 3508 3509 RTE_ETH_FOREACH_DEV(pi) { 3510 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3511 continue; 3512 3513 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3514 fprintf(stderr, 3515 "Please remove port %d from forwarding configuration.\n", 3516 pi); 3517 continue; 3518 } 3519 3520 if (port_is_bonding_member(pi)) { 3521 fprintf(stderr, 3522 "Please remove port %d from bonding device.\n", 3523 pi); 3524 continue; 3525 } 3526 3527 port = &ports[pi]; 3528 if (port->port_status == RTE_PORT_CLOSED) { 3529 fprintf(stderr, "Port %d is already closed\n", pi); 3530 continue; 3531 } 3532 3533 if (is_proc_primary()) { 3534 flush_port_owned_resources(pi); 3535 #ifdef RTE_NET_BOND 3536 if (port->bond_flag == 1) 3537 num_members = rte_eth_bond_members_get(pi, 3538 member_pids, RTE_MAX_ETHPORTS); 3539 #endif 3540 rte_eth_dev_close(pi); 3541 /* 3542 * If this port is bonding device, all members under the 3543 * device need to be removed or closed. 3544 */ 3545 if (port->bond_flag == 1 && num_members > 0) 3546 clear_bonding_member_device(member_pids, 3547 num_members); 3548 } 3549 3550 free_xstats_display_info(pi); 3551 } 3552 3553 remove_invalid_ports(); 3554 printf("Done\n"); 3555 } 3556 3557 void 3558 reset_port(portid_t pid) 3559 { 3560 int diag; 3561 portid_t pi; 3562 struct rte_port *port; 3563 3564 if (port_id_is_invalid(pid, ENABLED_WARN)) 3565 return; 3566 3567 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3568 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3569 fprintf(stderr, 3570 "Can not reset port(s), please stop port(s) first.\n"); 3571 return; 3572 } 3573 3574 printf("Resetting ports...\n"); 3575 3576 RTE_ETH_FOREACH_DEV(pi) { 3577 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3578 continue; 3579 3580 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3581 fprintf(stderr, 3582 "Please remove port %d from forwarding configuration.\n", 3583 pi); 3584 continue; 3585 } 3586 3587 if (port_is_bonding_member(pi)) { 3588 fprintf(stderr, 3589 "Please remove port %d from bonding device.\n", 3590 pi); 3591 continue; 3592 } 3593 3594 if (is_proc_primary()) { 3595 diag = rte_eth_dev_reset(pi); 3596 if (diag == 0) { 3597 port = &ports[pi]; 3598 port->need_reconfig = 1; 3599 port->need_reconfig_queues = 1; 3600 } else { 3601 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3602 pi, diag); 3603 } 3604 } 3605 } 3606 3607 printf("Done\n"); 3608 } 3609 3610 void 3611 attach_port(char *identifier) 3612 { 3613 portid_t pi; 3614 struct rte_dev_iterator iterator; 3615 3616 printf("Attaching a new port...\n"); 3617 3618 if (identifier == NULL) { 3619 fprintf(stderr, "Invalid parameters are specified\n"); 3620 return; 3621 } 3622 3623 if (rte_dev_probe(identifier) < 0) { 3624 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3625 return; 3626 } 3627 3628 /* first attach mode: event */ 3629 if (setup_on_probe_event) { 3630 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3631 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3632 if (ports[pi].port_status == RTE_PORT_HANDLING && 3633 ports[pi].need_setup != 0) 3634 setup_attached_port(pi); 3635 return; 3636 } 3637 3638 /* second attach mode: iterator */ 3639 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3640 /* setup ports matching the devargs used for probing */ 3641 if (port_is_forwarding(pi)) 3642 continue; /* port was already attached before */ 3643 setup_attached_port(pi); 3644 } 3645 } 3646 3647 static void 3648 setup_attached_port(portid_t pi) 3649 { 3650 unsigned int socket_id; 3651 int ret; 3652 3653 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3654 /* if socket_id is invalid, set to the first available socket. */ 3655 if (check_socket_id(socket_id) < 0) 3656 socket_id = socket_ids[0]; 3657 reconfig(pi, socket_id); 3658 ret = rte_eth_promiscuous_enable(pi); 3659 if (ret != 0) 3660 fprintf(stderr, 3661 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3662 pi, rte_strerror(-ret)); 3663 3664 ports_ids[nb_ports++] = pi; 3665 fwd_ports_ids[nb_fwd_ports++] = pi; 3666 nb_cfg_ports = nb_fwd_ports; 3667 ports[pi].need_setup = 0; 3668 ports[pi].port_status = RTE_PORT_STOPPED; 3669 3670 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3671 printf("Done\n"); 3672 } 3673 3674 static void 3675 detach_device(struct rte_device *dev) 3676 { 3677 portid_t sibling; 3678 3679 if (dev == NULL) { 3680 fprintf(stderr, "Device already removed\n"); 3681 return; 3682 } 3683 3684 printf("Removing a device...\n"); 3685 3686 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3687 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3688 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3689 fprintf(stderr, "Port %u not stopped\n", 3690 sibling); 3691 return; 3692 } 3693 flush_port_owned_resources(sibling); 3694 } 3695 } 3696 3697 if (rte_dev_remove(dev) < 0) { 3698 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3699 return; 3700 } 3701 remove_invalid_ports(); 3702 3703 printf("Device is detached\n"); 3704 printf("Now total ports is %d\n", nb_ports); 3705 printf("Done\n"); 3706 return; 3707 } 3708 3709 void 3710 detach_port_device(portid_t port_id) 3711 { 3712 int ret; 3713 struct rte_eth_dev_info dev_info; 3714 3715 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3716 return; 3717 3718 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3719 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3720 fprintf(stderr, "Port not stopped\n"); 3721 return; 3722 } 3723 fprintf(stderr, "Port was not closed\n"); 3724 } 3725 3726 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3727 if (ret != 0) { 3728 TESTPMD_LOG(ERR, 3729 "Failed to get device info for port %d, not detaching\n", 3730 port_id); 3731 return; 3732 } 3733 detach_device(dev_info.device); 3734 } 3735 3736 void 3737 detach_devargs(char *identifier) 3738 { 3739 struct rte_dev_iterator iterator; 3740 struct rte_devargs da; 3741 portid_t port_id; 3742 3743 printf("Removing a device...\n"); 3744 3745 memset(&da, 0, sizeof(da)); 3746 if (rte_devargs_parsef(&da, "%s", identifier)) { 3747 fprintf(stderr, "cannot parse identifier\n"); 3748 return; 3749 } 3750 3751 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3752 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3753 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3754 fprintf(stderr, "Port %u not stopped\n", 3755 port_id); 3756 rte_eth_iterator_cleanup(&iterator); 3757 rte_devargs_reset(&da); 3758 return; 3759 } 3760 flush_port_owned_resources(port_id); 3761 } 3762 } 3763 3764 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3765 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3766 da.name, rte_bus_name(da.bus)); 3767 rte_devargs_reset(&da); 3768 return; 3769 } 3770 3771 remove_invalid_ports(); 3772 3773 printf("Device %s is detached\n", identifier); 3774 printf("Now total ports is %d\n", nb_ports); 3775 printf("Done\n"); 3776 rte_devargs_reset(&da); 3777 } 3778 3779 void 3780 pmd_test_exit(void) 3781 { 3782 portid_t pt_id; 3783 unsigned int i; 3784 int ret; 3785 3786 if (test_done == 0) 3787 stop_packet_forwarding(); 3788 3789 #ifndef RTE_EXEC_ENV_WINDOWS 3790 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3791 if (mempools[i]) { 3792 if (mp_alloc_type == MP_ALLOC_ANON) 3793 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3794 NULL); 3795 } 3796 } 3797 #endif 3798 if (ports != NULL) { 3799 no_link_check = 1; 3800 RTE_ETH_FOREACH_DEV(pt_id) { 3801 printf("\nStopping port %d...\n", pt_id); 3802 fflush(stdout); 3803 stop_port(pt_id); 3804 } 3805 RTE_ETH_FOREACH_DEV(pt_id) { 3806 printf("\nShutting down port %d...\n", pt_id); 3807 fflush(stdout); 3808 close_port(pt_id); 3809 } 3810 } 3811 3812 if (hot_plug) { 3813 ret = rte_dev_event_monitor_stop(); 3814 if (ret) { 3815 RTE_LOG(ERR, EAL, 3816 "fail to stop device event monitor."); 3817 return; 3818 } 3819 3820 ret = rte_dev_event_callback_unregister(NULL, 3821 dev_event_callback, NULL); 3822 if (ret < 0) { 3823 RTE_LOG(ERR, EAL, 3824 "fail to unregister device event callback.\n"); 3825 return; 3826 } 3827 3828 ret = rte_dev_hotplug_handle_disable(); 3829 if (ret) { 3830 RTE_LOG(ERR, EAL, 3831 "fail to disable hotplug handling.\n"); 3832 return; 3833 } 3834 } 3835 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3836 if (mempools[i]) 3837 mempool_free_mp(mempools[i]); 3838 } 3839 free(xstats_display); 3840 3841 printf("\nBye...\n"); 3842 } 3843 3844 typedef void (*cmd_func_t)(void); 3845 struct pmd_test_command { 3846 const char *cmd_name; 3847 cmd_func_t cmd_func; 3848 }; 3849 3850 /* Check the link status of all ports in up to 9s, and print them finally */ 3851 static void 3852 check_all_ports_link_status(uint32_t port_mask) 3853 { 3854 #define CHECK_INTERVAL 100 /* 100ms */ 3855 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3856 portid_t portid; 3857 uint8_t count, all_ports_up, print_flag = 0; 3858 struct rte_eth_link link; 3859 int ret; 3860 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3861 3862 printf("Checking link statuses...\n"); 3863 fflush(stdout); 3864 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3865 all_ports_up = 1; 3866 RTE_ETH_FOREACH_DEV(portid) { 3867 if ((port_mask & (1 << portid)) == 0) 3868 continue; 3869 memset(&link, 0, sizeof(link)); 3870 ret = rte_eth_link_get_nowait(portid, &link); 3871 if (ret < 0) { 3872 all_ports_up = 0; 3873 if (print_flag == 1) 3874 fprintf(stderr, 3875 "Port %u link get failed: %s\n", 3876 portid, rte_strerror(-ret)); 3877 continue; 3878 } 3879 /* print link status if flag set */ 3880 if (print_flag == 1) { 3881 rte_eth_link_to_str(link_status, 3882 sizeof(link_status), &link); 3883 printf("Port %d %s\n", portid, link_status); 3884 continue; 3885 } 3886 /* clear all_ports_up flag if any link down */ 3887 if (link.link_status == RTE_ETH_LINK_DOWN) { 3888 all_ports_up = 0; 3889 break; 3890 } 3891 } 3892 /* after finally printing all link status, get out */ 3893 if (print_flag == 1) 3894 break; 3895 3896 if (all_ports_up == 0) { 3897 fflush(stdout); 3898 rte_delay_ms(CHECK_INTERVAL); 3899 } 3900 3901 /* set the print_flag if all ports up or timeout */ 3902 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3903 print_flag = 1; 3904 } 3905 3906 if (lsc_interrupt) 3907 break; 3908 } 3909 } 3910 3911 static void 3912 rmv_port_callback(void *arg) 3913 { 3914 int need_to_start = 0; 3915 int org_no_link_check = no_link_check; 3916 portid_t port_id = (intptr_t)arg; 3917 struct rte_eth_dev_info dev_info; 3918 int ret; 3919 3920 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3921 3922 if (!test_done && port_is_forwarding(port_id)) { 3923 need_to_start = 1; 3924 stop_packet_forwarding(); 3925 } 3926 no_link_check = 1; 3927 stop_port(port_id); 3928 no_link_check = org_no_link_check; 3929 3930 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3931 if (ret != 0) 3932 TESTPMD_LOG(ERR, 3933 "Failed to get device info for port %d, not detaching\n", 3934 port_id); 3935 else { 3936 struct rte_device *device = dev_info.device; 3937 close_port(port_id); 3938 detach_device(device); /* might be already removed or have more ports */ 3939 } 3940 if (need_to_start) 3941 start_packet_forwarding(0); 3942 } 3943 3944 /* This function is used by the interrupt thread */ 3945 static int 3946 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3947 void *ret_param) 3948 { 3949 RTE_SET_USED(param); 3950 RTE_SET_USED(ret_param); 3951 3952 if (type >= RTE_ETH_EVENT_MAX) { 3953 fprintf(stderr, 3954 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3955 port_id, __func__, type); 3956 fflush(stderr); 3957 } else if (event_print_mask & (UINT32_C(1) << type)) { 3958 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3959 eth_event_desc[type]); 3960 fflush(stdout); 3961 } 3962 3963 switch (type) { 3964 case RTE_ETH_EVENT_NEW: 3965 ports[port_id].need_setup = 1; 3966 ports[port_id].port_status = RTE_PORT_HANDLING; 3967 break; 3968 case RTE_ETH_EVENT_INTR_RMV: 3969 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3970 break; 3971 if (rte_eal_alarm_set(100000, 3972 rmv_port_callback, (void *)(intptr_t)port_id)) 3973 fprintf(stderr, 3974 "Could not set up deferred device removal\n"); 3975 break; 3976 case RTE_ETH_EVENT_DESTROY: 3977 ports[port_id].port_status = RTE_PORT_CLOSED; 3978 printf("Port %u is closed\n", port_id); 3979 break; 3980 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3981 uint16_t rxq_id; 3982 int ret; 3983 3984 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3985 for (rxq_id = 0; ; rxq_id++) { 3986 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3987 NULL); 3988 if (ret <= 0) 3989 break; 3990 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3991 port_id, rxq_id); 3992 3993 #ifdef RTE_NET_MLX5 3994 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3995 #endif 3996 } 3997 break; 3998 } 3999 default: 4000 break; 4001 } 4002 return 0; 4003 } 4004 4005 static int 4006 register_eth_event_callback(void) 4007 { 4008 int ret; 4009 enum rte_eth_event_type event; 4010 4011 for (event = RTE_ETH_EVENT_UNKNOWN; 4012 event < RTE_ETH_EVENT_MAX; event++) { 4013 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 4014 event, 4015 eth_event_callback, 4016 NULL); 4017 if (ret != 0) { 4018 TESTPMD_LOG(ERR, "Failed to register callback for " 4019 "%s event\n", eth_event_desc[event]); 4020 return -1; 4021 } 4022 } 4023 4024 return 0; 4025 } 4026 4027 static int 4028 unregister_eth_event_callback(void) 4029 { 4030 int ret; 4031 enum rte_eth_event_type event; 4032 4033 for (event = RTE_ETH_EVENT_UNKNOWN; 4034 event < RTE_ETH_EVENT_MAX; event++) { 4035 ret = rte_eth_dev_callback_unregister(RTE_ETH_ALL, 4036 event, 4037 eth_event_callback, 4038 NULL); 4039 if (ret != 0) { 4040 TESTPMD_LOG(ERR, "Failed to unregister callback for " 4041 "%s event\n", eth_event_desc[event]); 4042 return -1; 4043 } 4044 } 4045 4046 return 0; 4047 } 4048 4049 /* This function is used by the interrupt thread */ 4050 static void 4051 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 4052 __rte_unused void *arg) 4053 { 4054 uint16_t port_id; 4055 int ret; 4056 4057 if (type >= RTE_DEV_EVENT_MAX) { 4058 fprintf(stderr, "%s called upon invalid event %d\n", 4059 __func__, type); 4060 fflush(stderr); 4061 } 4062 4063 switch (type) { 4064 case RTE_DEV_EVENT_REMOVE: 4065 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 4066 device_name); 4067 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 4068 if (ret) { 4069 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 4070 device_name); 4071 return; 4072 } 4073 /* 4074 * Because the user's callback is invoked in eal interrupt 4075 * callback, the interrupt callback need to be finished before 4076 * it can be unregistered when detaching device. So finish 4077 * callback soon and use a deferred removal to detach device 4078 * is need. It is a workaround, once the device detaching be 4079 * moved into the eal in the future, the deferred removal could 4080 * be deleted. 4081 */ 4082 if (rte_eal_alarm_set(100000, 4083 rmv_port_callback, (void *)(intptr_t)port_id)) 4084 RTE_LOG(ERR, EAL, 4085 "Could not set up deferred device removal\n"); 4086 break; 4087 case RTE_DEV_EVENT_ADD: 4088 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 4089 device_name); 4090 /* TODO: After finish kernel driver binding, 4091 * begin to attach port. 4092 */ 4093 break; 4094 default: 4095 break; 4096 } 4097 } 4098 4099 static void 4100 rxtx_port_config(portid_t pid) 4101 { 4102 uint16_t qid; 4103 uint64_t offloads; 4104 struct rte_port *port = &ports[pid]; 4105 4106 for (qid = 0; qid < nb_rxq; qid++) { 4107 offloads = port->rxq[qid].conf.offloads; 4108 port->rxq[qid].conf = port->dev_info.default_rxconf; 4109 4110 if (rxq_share > 0 && 4111 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 4112 /* Non-zero share group to enable RxQ share. */ 4113 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 4114 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 4115 } 4116 4117 if (offloads != 0) 4118 port->rxq[qid].conf.offloads = offloads; 4119 4120 /* Check if any Rx parameters have been passed */ 4121 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 4122 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 4123 4124 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 4125 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 4126 4127 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 4128 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 4129 4130 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 4131 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 4132 4133 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 4134 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 4135 4136 port->nb_rx_desc[qid] = nb_rxd; 4137 } 4138 4139 for (qid = 0; qid < nb_txq; qid++) { 4140 offloads = port->txq[qid].conf.offloads; 4141 port->txq[qid].conf = port->dev_info.default_txconf; 4142 if (offloads != 0) 4143 port->txq[qid].conf.offloads = offloads; 4144 4145 /* Check if any Tx parameters have been passed */ 4146 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 4147 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 4148 4149 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 4150 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 4151 4152 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 4153 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 4154 4155 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 4156 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 4157 4158 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 4159 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 4160 4161 port->nb_tx_desc[qid] = nb_txd; 4162 } 4163 } 4164 4165 /* 4166 * Helper function to set MTU from frame size 4167 * 4168 * port->dev_info should be set before calling this function. 4169 * 4170 * return 0 on success, negative on error 4171 */ 4172 int 4173 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 4174 { 4175 struct rte_port *port = &ports[portid]; 4176 uint32_t eth_overhead; 4177 uint16_t mtu, new_mtu; 4178 4179 eth_overhead = get_eth_overhead(&port->dev_info); 4180 4181 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 4182 printf("Failed to get MTU for port %u\n", portid); 4183 return -1; 4184 } 4185 4186 new_mtu = max_rx_pktlen - eth_overhead; 4187 4188 if (mtu == new_mtu) 4189 return 0; 4190 4191 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 4192 fprintf(stderr, 4193 "Failed to set MTU to %u for port %u\n", 4194 new_mtu, portid); 4195 return -1; 4196 } 4197 4198 port->dev_conf.rxmode.mtu = new_mtu; 4199 4200 return 0; 4201 } 4202 4203 void 4204 init_port_config(void) 4205 { 4206 portid_t pid; 4207 struct rte_port *port; 4208 int ret, i; 4209 4210 RTE_ETH_FOREACH_DEV(pid) { 4211 port = &ports[pid]; 4212 4213 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4214 if (ret != 0) 4215 return; 4216 4217 if (nb_rxq > 1) { 4218 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4219 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4220 rss_hf & port->dev_info.flow_type_rss_offloads; 4221 } else { 4222 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4223 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4224 } 4225 4226 if (port->dcb_flag == 0) { 4227 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4228 port->dev_conf.rxmode.mq_mode = 4229 (enum rte_eth_rx_mq_mode) 4230 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4231 } else { 4232 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4233 port->dev_conf.rxmode.offloads &= 4234 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4235 4236 for (i = 0; 4237 i < port->dev_info.nb_rx_queues; 4238 i++) 4239 port->rxq[i].conf.offloads &= 4240 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4241 } 4242 } 4243 4244 rxtx_port_config(pid); 4245 4246 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4247 if (ret != 0) 4248 return; 4249 4250 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4251 port->dev_conf.intr_conf.lsc = 1; 4252 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4253 port->dev_conf.intr_conf.rmv = 1; 4254 } 4255 } 4256 4257 void set_port_member_flag(portid_t member_pid) 4258 { 4259 struct rte_port *port; 4260 4261 port = &ports[member_pid]; 4262 port->member_flag = 1; 4263 } 4264 4265 void clear_port_member_flag(portid_t member_pid) 4266 { 4267 struct rte_port *port; 4268 4269 port = &ports[member_pid]; 4270 port->member_flag = 0; 4271 } 4272 4273 uint8_t port_is_bonding_member(portid_t member_pid) 4274 { 4275 struct rte_port *port; 4276 struct rte_eth_dev_info dev_info; 4277 int ret; 4278 4279 port = &ports[member_pid]; 4280 ret = eth_dev_info_get_print_err(member_pid, &dev_info); 4281 if (ret != 0) { 4282 TESTPMD_LOG(ERR, 4283 "Failed to get device info for port id %d," 4284 "cannot determine if the port is a bonding member", 4285 member_pid); 4286 return 0; 4287 } 4288 4289 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDING_MEMBER) || (port->member_flag == 1)) 4290 return 1; 4291 return 0; 4292 } 4293 4294 const uint16_t vlan_tags[] = { 4295 0, 1, 2, 3, 4, 5, 6, 7, 4296 8, 9, 10, 11, 12, 13, 14, 15, 4297 16, 17, 18, 19, 20, 21, 22, 23, 4298 24, 25, 26, 27, 28, 29, 30, 31 4299 }; 4300 4301 static int 4302 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4303 enum dcb_mode_enable dcb_mode, 4304 enum rte_eth_nb_tcs num_tcs, 4305 uint8_t pfc_en) 4306 { 4307 uint8_t i; 4308 int32_t rc; 4309 struct rte_eth_rss_conf rss_conf; 4310 4311 /* 4312 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4313 * given above, and the number of traffic classes available for use. 4314 */ 4315 if (dcb_mode == DCB_VT_ENABLED) { 4316 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4317 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4318 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4319 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4320 4321 /* VMDQ+DCB RX and TX configurations */ 4322 vmdq_rx_conf->enable_default_pool = 0; 4323 vmdq_rx_conf->default_pool = 0; 4324 vmdq_rx_conf->nb_queue_pools = 4325 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4326 vmdq_tx_conf->nb_queue_pools = 4327 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4328 4329 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4330 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4331 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4332 vmdq_rx_conf->pool_map[i].pools = 4333 1 << (i % vmdq_rx_conf->nb_queue_pools); 4334 } 4335 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4336 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4337 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4338 } 4339 4340 /* set DCB mode of RX and TX of multiple queues */ 4341 eth_conf->rxmode.mq_mode = 4342 (enum rte_eth_rx_mq_mode) 4343 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4344 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4345 } else { 4346 struct rte_eth_dcb_rx_conf *rx_conf = 4347 ð_conf->rx_adv_conf.dcb_rx_conf; 4348 struct rte_eth_dcb_tx_conf *tx_conf = 4349 ð_conf->tx_adv_conf.dcb_tx_conf; 4350 4351 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4352 4353 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4354 if (rc != 0) 4355 return rc; 4356 4357 rx_conf->nb_tcs = num_tcs; 4358 tx_conf->nb_tcs = num_tcs; 4359 4360 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4361 rx_conf->dcb_tc[i] = i % num_tcs; 4362 tx_conf->dcb_tc[i] = i % num_tcs; 4363 } 4364 4365 eth_conf->rxmode.mq_mode = 4366 (enum rte_eth_rx_mq_mode) 4367 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4368 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4369 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4370 } 4371 4372 if (pfc_en) 4373 eth_conf->dcb_capability_en = 4374 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4375 else 4376 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4377 4378 return 0; 4379 } 4380 4381 int 4382 init_port_dcb_config(portid_t pid, 4383 enum dcb_mode_enable dcb_mode, 4384 enum rte_eth_nb_tcs num_tcs, 4385 uint8_t pfc_en) 4386 { 4387 struct rte_eth_conf port_conf; 4388 struct rte_port *rte_port; 4389 int retval; 4390 uint16_t i; 4391 4392 if (num_procs > 1) { 4393 printf("The multi-process feature doesn't support dcb.\n"); 4394 return -ENOTSUP; 4395 } 4396 rte_port = &ports[pid]; 4397 4398 /* retain the original device configuration. */ 4399 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4400 4401 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4402 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4403 if (retval < 0) 4404 return retval; 4405 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4406 /* remove RSS HASH offload for DCB in vt mode */ 4407 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4408 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4409 for (i = 0; i < nb_rxq; i++) 4410 rte_port->rxq[i].conf.offloads &= 4411 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4412 } 4413 4414 /* re-configure the device . */ 4415 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4416 if (retval < 0) 4417 return retval; 4418 4419 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4420 if (retval != 0) 4421 return retval; 4422 4423 /* If dev_info.vmdq_pool_base is greater than 0, 4424 * the queue id of vmdq pools is started after pf queues. 4425 */ 4426 if (dcb_mode == DCB_VT_ENABLED && 4427 rte_port->dev_info.vmdq_pool_base > 0) { 4428 fprintf(stderr, 4429 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4430 pid); 4431 return -1; 4432 } 4433 4434 /* Assume the ports in testpmd have the same dcb capability 4435 * and has the same number of rxq and txq in dcb mode 4436 */ 4437 if (dcb_mode == DCB_VT_ENABLED) { 4438 if (rte_port->dev_info.max_vfs > 0) { 4439 nb_rxq = rte_port->dev_info.nb_rx_queues; 4440 nb_txq = rte_port->dev_info.nb_tx_queues; 4441 } else { 4442 nb_rxq = rte_port->dev_info.max_rx_queues; 4443 nb_txq = rte_port->dev_info.max_tx_queues; 4444 } 4445 } else { 4446 /*if vt is disabled, use all pf queues */ 4447 if (rte_port->dev_info.vmdq_pool_base == 0) { 4448 nb_rxq = rte_port->dev_info.max_rx_queues; 4449 nb_txq = rte_port->dev_info.max_tx_queues; 4450 } else { 4451 nb_rxq = (queueid_t)num_tcs; 4452 nb_txq = (queueid_t)num_tcs; 4453 4454 } 4455 } 4456 rx_free_thresh = 64; 4457 4458 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4459 4460 rxtx_port_config(pid); 4461 /* VLAN filter */ 4462 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4463 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4464 rx_vft_set(pid, vlan_tags[i], 1); 4465 4466 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4467 if (retval != 0) 4468 return retval; 4469 4470 rte_port->dcb_flag = 1; 4471 4472 /* Enter DCB configuration status */ 4473 dcb_config = 1; 4474 4475 return 0; 4476 } 4477 4478 static void 4479 init_port(void) 4480 { 4481 int i; 4482 4483 /* Configuration of Ethernet ports. */ 4484 ports = rte_zmalloc("testpmd: ports", 4485 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4486 RTE_CACHE_LINE_SIZE); 4487 if (ports == NULL) { 4488 rte_exit(EXIT_FAILURE, 4489 "rte_zmalloc(%d struct rte_port) failed\n", 4490 RTE_MAX_ETHPORTS); 4491 } 4492 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4493 ports[i].fwd_mac_swap = 1; 4494 ports[i].xstats_info.allocated = false; 4495 LIST_INIT(&ports[i].flow_tunnel_list); 4496 } 4497 /* Initialize ports NUMA structures */ 4498 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4499 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4500 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4501 } 4502 4503 static void 4504 print_stats(void) 4505 { 4506 uint8_t i; 4507 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4508 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4509 4510 /* Clear screen and move to top left */ 4511 printf("%s%s", clr, top_left); 4512 4513 printf("\nPort statistics ===================================="); 4514 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4515 nic_stats_display(fwd_ports_ids[i]); 4516 4517 fflush(stdout); 4518 } 4519 4520 static void 4521 signal_handler(int signum __rte_unused) 4522 { 4523 f_quit = 1; 4524 prompt_exit(); 4525 } 4526 4527 int 4528 main(int argc, char** argv) 4529 { 4530 int diag; 4531 portid_t port_id; 4532 uint16_t count; 4533 int ret; 4534 4535 #ifdef RTE_EXEC_ENV_WINDOWS 4536 signal(SIGINT, signal_handler); 4537 signal(SIGTERM, signal_handler); 4538 #else 4539 /* Want read() not to be restarted on signal */ 4540 struct sigaction action = { 4541 .sa_handler = signal_handler, 4542 }; 4543 4544 sigaction(SIGINT, &action, NULL); 4545 sigaction(SIGTERM, &action, NULL); 4546 #endif 4547 4548 testpmd_logtype = rte_log_register("testpmd"); 4549 if (testpmd_logtype < 0) 4550 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4551 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4552 4553 diag = rte_eal_init(argc, argv); 4554 if (diag < 0) 4555 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4556 rte_strerror(rte_errno)); 4557 4558 /* allocate port structures, and init them */ 4559 init_port(); 4560 4561 ret = register_eth_event_callback(); 4562 if (ret != 0) 4563 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4564 4565 #ifdef RTE_LIB_PDUMP 4566 /* initialize packet capture framework */ 4567 rte_pdump_init(); 4568 #endif 4569 4570 count = 0; 4571 RTE_ETH_FOREACH_DEV(port_id) { 4572 ports_ids[count] = port_id; 4573 count++; 4574 } 4575 nb_ports = (portid_t) count; 4576 if (nb_ports == 0) 4577 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4578 4579 set_def_fwd_config(); 4580 if (nb_lcores == 0) 4581 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4582 "Check the core mask argument\n"); 4583 4584 /* Bitrate/latency stats disabled by default */ 4585 #ifdef RTE_LIB_BITRATESTATS 4586 bitrate_enabled = 0; 4587 #endif 4588 #ifdef RTE_LIB_LATENCYSTATS 4589 latencystats_enabled = 0; 4590 #endif 4591 4592 /* on FreeBSD, mlockall() is disabled by default */ 4593 #ifdef RTE_EXEC_ENV_FREEBSD 4594 do_mlockall = 0; 4595 #else 4596 do_mlockall = 1; 4597 #endif 4598 4599 argc -= diag; 4600 argv += diag; 4601 if (argc > 1) 4602 launch_args_parse(argc, argv); 4603 4604 #ifndef RTE_EXEC_ENV_WINDOWS 4605 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4606 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4607 strerror(errno)); 4608 } 4609 #endif 4610 4611 if (tx_first && interactive) 4612 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4613 "interactive mode.\n"); 4614 4615 if (tx_first && lsc_interrupt) { 4616 fprintf(stderr, 4617 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4618 lsc_interrupt = 0; 4619 } 4620 4621 if (!nb_rxq && !nb_txq) 4622 rte_exit(EXIT_FAILURE, "Either rx or tx queues should be non-zero\n"); 4623 4624 if (nb_rxq > 1 && nb_rxq > nb_txq) 4625 fprintf(stderr, 4626 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4627 nb_rxq, nb_txq); 4628 4629 init_config(); 4630 4631 if (hot_plug) { 4632 ret = rte_dev_hotplug_handle_enable(); 4633 if (ret) { 4634 RTE_LOG(ERR, EAL, 4635 "fail to enable hotplug handling."); 4636 return -1; 4637 } 4638 4639 ret = rte_dev_event_monitor_start(); 4640 if (ret) { 4641 RTE_LOG(ERR, EAL, 4642 "fail to start device event monitoring."); 4643 return -1; 4644 } 4645 4646 ret = rte_dev_event_callback_register(NULL, 4647 dev_event_callback, NULL); 4648 if (ret) { 4649 RTE_LOG(ERR, EAL, 4650 "fail to register device event callback\n"); 4651 return -1; 4652 } 4653 } 4654 4655 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { 4656 if (!interactive) { 4657 rte_eal_cleanup(); 4658 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4659 } 4660 fprintf(stderr, "Start ports failed\n"); 4661 } 4662 4663 /* set all ports to promiscuous mode by default */ 4664 RTE_ETH_FOREACH_DEV(port_id) { 4665 ret = rte_eth_promiscuous_enable(port_id); 4666 if (ret != 0) 4667 fprintf(stderr, 4668 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4669 port_id, rte_strerror(-ret)); 4670 } 4671 4672 #ifdef RTE_LIB_METRICS 4673 /* Init metrics library */ 4674 rte_metrics_init(rte_socket_id()); 4675 #endif 4676 4677 #ifdef RTE_LIB_LATENCYSTATS 4678 if (latencystats_enabled != 0) { 4679 int ret = rte_latencystats_init(1, NULL); 4680 if (ret) 4681 fprintf(stderr, 4682 "Warning: latencystats init() returned error %d\n", 4683 ret); 4684 fprintf(stderr, "Latencystats running on lcore %d\n", 4685 latencystats_lcore_id); 4686 } 4687 #endif 4688 4689 /* Setup bitrate stats */ 4690 #ifdef RTE_LIB_BITRATESTATS 4691 if (bitrate_enabled != 0) { 4692 bitrate_data = rte_stats_bitrate_create(); 4693 if (bitrate_data == NULL) 4694 rte_exit(EXIT_FAILURE, 4695 "Could not allocate bitrate data.\n"); 4696 rte_stats_bitrate_reg(bitrate_data); 4697 } 4698 #endif 4699 4700 if (record_core_cycles) 4701 rte_lcore_register_usage_cb(lcore_usage_callback); 4702 4703 if (init_cmdline() != 0) 4704 rte_exit(EXIT_FAILURE, 4705 "Could not initialise cmdline context.\n"); 4706 4707 if (strlen(cmdline_filename) != 0) 4708 cmdline_read_from_file(cmdline_filename); 4709 4710 if (interactive == 1) { 4711 if (auto_start) { 4712 printf("Start automatic packet forwarding\n"); 4713 start_packet_forwarding(0); 4714 } 4715 prompt(); 4716 } else { 4717 printf("No commandline core given, start packet forwarding\n"); 4718 start_packet_forwarding(tx_first); 4719 if (stats_period != 0) { 4720 uint64_t prev_time = 0, cur_time, diff_time = 0; 4721 uint64_t timer_period; 4722 4723 /* Convert to number of cycles */ 4724 timer_period = stats_period * rte_get_timer_hz(); 4725 4726 while (f_quit == 0) { 4727 cur_time = rte_get_timer_cycles(); 4728 diff_time += cur_time - prev_time; 4729 4730 if (diff_time >= timer_period) { 4731 print_stats(); 4732 /* Reset the timer */ 4733 diff_time = 0; 4734 } 4735 /* Sleep to avoid unnecessary checks */ 4736 prev_time = cur_time; 4737 rte_delay_us_sleep(US_PER_S); 4738 } 4739 } else { 4740 char c; 4741 4742 printf("Press enter to exit\n"); 4743 while (f_quit == 0) { 4744 /* end-of-file or any character exits loop */ 4745 if (read(0, &c, 1) >= 0) 4746 break; 4747 if (errno == EINTR) 4748 continue; 4749 rte_exit(EXIT_FAILURE, "Read failed: %s\n", 4750 strerror(errno)); 4751 } 4752 } 4753 } 4754 4755 pmd_test_exit(); 4756 4757 #ifdef RTE_LIB_PDUMP 4758 /* uninitialize packet capture framework */ 4759 rte_pdump_uninit(); 4760 #endif 4761 #ifdef RTE_LIB_LATENCYSTATS 4762 if (latencystats_enabled != 0) 4763 rte_latencystats_uninit(); 4764 #endif 4765 4766 ret = unregister_eth_event_callback(); 4767 if (ret != 0) 4768 rte_exit(EXIT_FAILURE, "Cannot unregister for ethdev events"); 4769 4770 4771 ret = rte_eal_cleanup(); 4772 if (ret != 0) 4773 rte_exit(EXIT_FAILURE, 4774 "EAL cleanup failed: %s\n", strerror(-ret)); 4775 4776 return EXIT_SUCCESS; 4777 } 4778