1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 &recycle_mbufs_engine, 202 #ifdef RTE_LIBRTE_IEEE1588 203 &ieee1588_fwd_engine, 204 #endif 205 &shared_rxq_engine, 206 NULL, 207 }; 208 209 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 210 uint16_t mempool_flags; 211 212 struct fwd_config cur_fwd_config; 213 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 214 uint32_t retry_enabled; 215 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 216 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 217 218 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 219 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 220 DEFAULT_MBUF_DATA_SIZE 221 }; /**< Mbuf data space size. */ 222 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 223 * specified on command-line. */ 224 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 225 226 /** Extended statistics to show. */ 227 struct rte_eth_xstat_name *xstats_display; 228 229 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 230 231 /* 232 * In container, it cannot terminate the process which running with 'stats-period' 233 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 234 */ 235 volatile uint8_t f_quit; 236 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 237 238 /* 239 * Max Rx frame size, set by '--max-pkt-len' parameter. 240 */ 241 uint32_t max_rx_pkt_len; 242 243 /* 244 * Configuration of packet segments used to scatter received packets 245 * if some of split features is configured. 246 */ 247 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 248 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 249 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 250 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 251 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 252 253 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 254 255 /* 256 * Configuration of packet segments used by the "txonly" processing engine. 257 */ 258 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 259 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 260 TXONLY_DEF_PACKET_LEN, 261 }; 262 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 263 264 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 265 /**< Split policy for packets to TX. */ 266 267 uint8_t txonly_multi_flow; 268 /**< Whether multiple flows are generated in TXONLY mode. */ 269 270 uint32_t tx_pkt_times_inter; 271 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 272 273 uint32_t tx_pkt_times_intra; 274 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 275 276 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 277 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 278 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 279 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 280 281 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 282 uint8_t dcb_config = 0; 283 284 /* 285 * Configurable number of RX/TX queues. 286 */ 287 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 288 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 289 290 /* 291 * Configurable number of RX/TX ring descriptors. 292 * Defaults are supplied by drivers via ethdev. 293 */ 294 #define RX_DESC_DEFAULT 0 295 #define TX_DESC_DEFAULT 0 296 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 297 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 298 299 #define RTE_PMD_PARAM_UNSET -1 300 /* 301 * Configurable values of RX and TX ring threshold registers. 302 */ 303 304 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 305 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 306 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 307 308 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 309 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 310 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 311 312 /* 313 * Configurable value of RX free threshold. 314 */ 315 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 316 317 /* 318 * Configurable value of RX drop enable. 319 */ 320 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 321 322 /* 323 * Configurable value of TX free threshold. 324 */ 325 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 326 327 /* 328 * Configurable value of TX RS bit threshold. 329 */ 330 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 331 332 /* 333 * Configurable sub-forwarding mode for the noisy_vnf forwarding mode. 334 */ 335 enum noisy_fwd_mode noisy_fwd_mode; 336 337 /* String version of enum noisy_fwd_mode */ 338 const char * const noisy_fwd_mode_desc[] = { 339 [NOISY_FWD_MODE_IO] = "io", 340 [NOISY_FWD_MODE_MAC] = "mac", 341 [NOISY_FWD_MODE_MACSWAP] = "macswap", 342 [NOISY_FWD_MODE_5TSWAP] = "5tswap", 343 [NOISY_FWD_MODE_MAX] = NULL, 344 }; 345 346 /* 347 * Configurable value of buffered packets before sending. 348 */ 349 uint16_t noisy_tx_sw_bufsz; 350 351 /* 352 * Configurable value of packet buffer timeout. 353 */ 354 uint16_t noisy_tx_sw_buf_flush_time; 355 356 /* 357 * Configurable value for size of VNF internal memory area 358 * used for simulating noisy neighbour behaviour 359 */ 360 uint64_t noisy_lkup_mem_sz; 361 362 /* 363 * Configurable value of number of random writes done in 364 * VNF simulation memory area. 365 */ 366 uint64_t noisy_lkup_num_writes; 367 368 /* 369 * Configurable value of number of random reads done in 370 * VNF simulation memory area. 371 */ 372 uint64_t noisy_lkup_num_reads; 373 374 /* 375 * Configurable value of number of random reads/writes done in 376 * VNF simulation memory area. 377 */ 378 uint64_t noisy_lkup_num_reads_writes; 379 380 /* 381 * Receive Side Scaling (RSS) configuration. 382 */ 383 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 384 385 /* 386 * Port topology configuration 387 */ 388 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 389 390 /* 391 * Avoids to flush all the RX streams before starts forwarding. 392 */ 393 uint8_t no_flush_rx = 0; /* flush by default */ 394 395 /* 396 * Flow API isolated mode. 397 */ 398 uint8_t flow_isolate_all; 399 400 /* 401 * Disable port flow flush when stop port. 402 */ 403 uint8_t no_flow_flush = 0; /* do flow flush by default */ 404 405 /* 406 * Avoids to check link status when starting/stopping a port. 407 */ 408 uint8_t no_link_check = 0; /* check by default */ 409 410 /* 411 * Don't automatically start all ports in interactive mode. 412 */ 413 uint8_t no_device_start = 0; 414 415 /* 416 * Enable link status change notification 417 */ 418 uint8_t lsc_interrupt = 1; /* enabled by default */ 419 420 /* 421 * Enable device removal notification. 422 */ 423 uint8_t rmv_interrupt = 1; /* enabled by default */ 424 425 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 426 427 /* After attach, port setup is called on event or by iterator */ 428 bool setup_on_probe_event = true; 429 430 /* Clear ptypes on port initialization. */ 431 uint8_t clear_ptypes = true; 432 433 /* Pretty printing of ethdev events */ 434 static const char * const eth_event_desc[] = { 435 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 436 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 437 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 438 [RTE_ETH_EVENT_INTR_RESET] = "reset", 439 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 440 [RTE_ETH_EVENT_IPSEC] = "IPsec", 441 [RTE_ETH_EVENT_MACSEC] = "MACsec", 442 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 443 [RTE_ETH_EVENT_NEW] = "device probed", 444 [RTE_ETH_EVENT_DESTROY] = "device released", 445 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 446 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 447 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 448 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 449 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 450 [RTE_ETH_EVENT_MAX] = NULL, 451 }; 452 453 /* 454 * Display or mask ether events 455 * Default to all events except VF_MBOX 456 */ 457 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 458 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 459 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 460 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 461 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 462 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 463 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 464 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 465 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 466 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 467 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 468 /* 469 * Decide if all memory are locked for performance. 470 */ 471 int do_mlockall = 0; 472 473 #ifdef RTE_LIB_LATENCYSTATS 474 475 /* 476 * Set when latency stats is enabled in the commandline 477 */ 478 uint8_t latencystats_enabled; 479 480 /* 481 * Lcore ID to service latency statistics. 482 */ 483 lcoreid_t latencystats_lcore_id = -1; 484 485 #endif 486 487 /* 488 * Ethernet device configuration. 489 */ 490 struct rte_eth_rxmode rx_mode; 491 492 struct rte_eth_txmode tx_mode = { 493 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 494 }; 495 496 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 497 498 /* 499 * Display zero values by default for xstats 500 */ 501 uint8_t xstats_hide_zero; 502 503 /* 504 * Measure of CPU cycles disabled by default 505 */ 506 uint8_t record_core_cycles; 507 508 /* 509 * Display of RX and TX bursts disabled by default 510 */ 511 uint8_t record_burst_stats; 512 513 /* 514 * Number of ports per shared Rx queue group, 0 disable. 515 */ 516 uint32_t rxq_share; 517 518 unsigned int num_sockets = 0; 519 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 520 521 #ifdef RTE_LIB_BITRATESTATS 522 /* Bitrate statistics */ 523 struct rte_stats_bitrates *bitrate_data; 524 lcoreid_t bitrate_lcore_id; 525 uint8_t bitrate_enabled; 526 #endif 527 528 #ifdef RTE_LIB_GRO 529 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 530 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 531 #endif 532 533 /* 534 * hexadecimal bitmask of RX mq mode can be enabled. 535 */ 536 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 537 538 /* 539 * Used to set forced link speed 540 */ 541 uint32_t eth_link_speed; 542 543 /* 544 * ID of the current process in multi-process, used to 545 * configure the queues to be polled. 546 */ 547 int proc_id; 548 549 /* 550 * Number of processes in multi-process, used to 551 * configure the queues to be polled. 552 */ 553 unsigned int num_procs = 1; 554 555 static void 556 eth_rx_metadata_negotiate_mp(uint16_t port_id) 557 { 558 uint64_t rx_meta_features = 0; 559 int ret; 560 561 if (!is_proc_primary()) 562 return; 563 564 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 565 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 566 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 567 568 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 569 if (ret == 0) { 570 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 571 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 572 port_id); 573 } 574 575 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 576 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 577 port_id); 578 } 579 580 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 581 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 582 port_id); 583 } 584 } else if (ret != -ENOTSUP) { 585 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 586 port_id, rte_strerror(-ret)); 587 } 588 } 589 590 static int 591 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 592 const struct rte_eth_conf *dev_conf) 593 { 594 if (is_proc_primary()) 595 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 596 dev_conf); 597 return 0; 598 } 599 600 static int 601 change_bonding_member_port_status(portid_t bond_pid, bool is_stop) 602 { 603 #ifdef RTE_NET_BOND 604 605 portid_t member_pids[RTE_MAX_ETHPORTS]; 606 struct rte_port *port; 607 int num_members; 608 portid_t member_pid; 609 int i; 610 611 num_members = rte_eth_bond_members_get(bond_pid, member_pids, 612 RTE_MAX_ETHPORTS); 613 if (num_members < 0) { 614 fprintf(stderr, "Failed to get member list for port = %u\n", 615 bond_pid); 616 return num_members; 617 } 618 619 for (i = 0; i < num_members; i++) { 620 member_pid = member_pids[i]; 621 port = &ports[member_pid]; 622 port->port_status = 623 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 624 } 625 #else 626 RTE_SET_USED(bond_pid); 627 RTE_SET_USED(is_stop); 628 #endif 629 return 0; 630 } 631 632 static int 633 eth_dev_start_mp(uint16_t port_id) 634 { 635 int ret; 636 637 if (is_proc_primary()) { 638 ret = rte_eth_dev_start(port_id); 639 if (ret != 0) 640 return ret; 641 642 struct rte_port *port = &ports[port_id]; 643 644 /* 645 * Starting a bonding port also starts all members under the bonding 646 * device. So if this port is bond device, we need to modify the 647 * port status of these members. 648 */ 649 if (port->bond_flag == 1) 650 return change_bonding_member_port_status(port_id, false); 651 } 652 653 return 0; 654 } 655 656 static int 657 eth_dev_stop_mp(uint16_t port_id) 658 { 659 int ret; 660 661 if (is_proc_primary()) { 662 ret = rte_eth_dev_stop(port_id); 663 if (ret != 0) 664 return ret; 665 666 struct rte_port *port = &ports[port_id]; 667 668 /* 669 * Stopping a bonding port also stops all members under the bonding 670 * device. So if this port is bond device, we need to modify the 671 * port status of these members. 672 */ 673 if (port->bond_flag == 1) 674 return change_bonding_member_port_status(port_id, true); 675 } 676 677 return 0; 678 } 679 680 static void 681 mempool_free_mp(struct rte_mempool *mp) 682 { 683 if (is_proc_primary()) 684 rte_mempool_free(mp); 685 } 686 687 static int 688 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 689 { 690 if (is_proc_primary()) 691 return rte_eth_dev_set_mtu(port_id, mtu); 692 693 return 0; 694 } 695 696 /* Forward function declarations */ 697 static void setup_attached_port(portid_t pi); 698 static void check_all_ports_link_status(uint32_t port_mask); 699 static int eth_event_callback(portid_t port_id, 700 enum rte_eth_event_type type, 701 void *param, void *ret_param); 702 static void dev_event_callback(const char *device_name, 703 enum rte_dev_event_type type, 704 void *param); 705 static void fill_xstats_display_info(void); 706 707 /* 708 * Check if all the ports are started. 709 * If yes, return positive value. If not, return zero. 710 */ 711 static int all_ports_started(void); 712 713 #ifdef RTE_LIB_GSO 714 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 715 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 716 #endif 717 718 /* Holds the registered mbuf dynamic flags names. */ 719 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 720 721 722 /* 723 * Helper function to check if socket is already discovered. 724 * If yes, return positive value. If not, return zero. 725 */ 726 int 727 new_socket_id(unsigned int socket_id) 728 { 729 unsigned int i; 730 731 for (i = 0; i < num_sockets; i++) { 732 if (socket_ids[i] == socket_id) 733 return 0; 734 } 735 return 1; 736 } 737 738 /* 739 * Setup default configuration. 740 */ 741 static void 742 set_default_fwd_lcores_config(void) 743 { 744 unsigned int i; 745 unsigned int nb_lc; 746 unsigned int sock_num; 747 748 nb_lc = 0; 749 for (i = 0; i < RTE_MAX_LCORE; i++) { 750 if (!rte_lcore_is_enabled(i)) 751 continue; 752 sock_num = rte_lcore_to_socket_id(i); 753 if (new_socket_id(sock_num)) { 754 if (num_sockets >= RTE_MAX_NUMA_NODES) { 755 rte_exit(EXIT_FAILURE, 756 "Total sockets greater than %u\n", 757 RTE_MAX_NUMA_NODES); 758 } 759 socket_ids[num_sockets++] = sock_num; 760 } 761 if (i == rte_get_main_lcore()) 762 continue; 763 fwd_lcores_cpuids[nb_lc++] = i; 764 } 765 nb_lcores = (lcoreid_t) nb_lc; 766 nb_cfg_lcores = nb_lcores; 767 nb_fwd_lcores = 1; 768 } 769 770 static void 771 set_def_peer_eth_addrs(void) 772 { 773 portid_t i; 774 775 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 776 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 777 peer_eth_addrs[i].addr_bytes[5] = i; 778 } 779 } 780 781 static void 782 set_default_fwd_ports_config(void) 783 { 784 portid_t pt_id; 785 int i = 0; 786 787 RTE_ETH_FOREACH_DEV(pt_id) { 788 fwd_ports_ids[i++] = pt_id; 789 790 /* Update sockets info according to the attached device */ 791 int socket_id = rte_eth_dev_socket_id(pt_id); 792 if (socket_id >= 0 && new_socket_id(socket_id)) { 793 if (num_sockets >= RTE_MAX_NUMA_NODES) { 794 rte_exit(EXIT_FAILURE, 795 "Total sockets greater than %u\n", 796 RTE_MAX_NUMA_NODES); 797 } 798 socket_ids[num_sockets++] = socket_id; 799 } 800 } 801 802 nb_cfg_ports = nb_ports; 803 nb_fwd_ports = nb_ports; 804 } 805 806 void 807 set_def_fwd_config(void) 808 { 809 set_default_fwd_lcores_config(); 810 set_def_peer_eth_addrs(); 811 set_default_fwd_ports_config(); 812 } 813 814 #ifndef RTE_EXEC_ENV_WINDOWS 815 /* extremely pessimistic estimation of memory required to create a mempool */ 816 static int 817 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 818 { 819 unsigned int n_pages, mbuf_per_pg, leftover; 820 uint64_t total_mem, mbuf_mem, obj_sz; 821 822 /* there is no good way to predict how much space the mempool will 823 * occupy because it will allocate chunks on the fly, and some of those 824 * will come from default DPDK memory while some will come from our 825 * external memory, so just assume 128MB will be enough for everyone. 826 */ 827 uint64_t hdr_mem = 128 << 20; 828 829 /* account for possible non-contiguousness */ 830 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 831 if (obj_sz > pgsz) { 832 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 833 return -1; 834 } 835 836 mbuf_per_pg = pgsz / obj_sz; 837 leftover = (nb_mbufs % mbuf_per_pg) > 0; 838 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 839 840 mbuf_mem = n_pages * pgsz; 841 842 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 843 844 if (total_mem > SIZE_MAX) { 845 TESTPMD_LOG(ERR, "Memory size too big\n"); 846 return -1; 847 } 848 *out = (size_t)total_mem; 849 850 return 0; 851 } 852 853 static int 854 pagesz_flags(uint64_t page_sz) 855 { 856 /* as per mmap() manpage, all page sizes are log2 of page size 857 * shifted by MAP_HUGE_SHIFT 858 */ 859 int log2 = rte_log2_u64(page_sz); 860 861 return (log2 << HUGE_SHIFT); 862 } 863 864 static void * 865 alloc_mem(size_t memsz, size_t pgsz, bool huge) 866 { 867 void *addr; 868 int flags; 869 870 /* allocate anonymous hugepages */ 871 flags = MAP_ANONYMOUS | MAP_PRIVATE; 872 if (huge) 873 flags |= HUGE_FLAG | pagesz_flags(pgsz); 874 875 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 876 if (addr == MAP_FAILED) 877 return NULL; 878 879 return addr; 880 } 881 882 struct extmem_param { 883 void *addr; 884 size_t len; 885 size_t pgsz; 886 rte_iova_t *iova_table; 887 unsigned int iova_table_len; 888 }; 889 890 static int 891 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 892 bool huge) 893 { 894 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 895 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 896 unsigned int cur_page, n_pages, pgsz_idx; 897 size_t mem_sz, cur_pgsz; 898 rte_iova_t *iovas = NULL; 899 void *addr; 900 int ret; 901 902 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 903 /* skip anything that is too big */ 904 if (pgsizes[pgsz_idx] > SIZE_MAX) 905 continue; 906 907 cur_pgsz = pgsizes[pgsz_idx]; 908 909 /* if we were told not to allocate hugepages, override */ 910 if (!huge) 911 cur_pgsz = sysconf(_SC_PAGESIZE); 912 913 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 914 if (ret < 0) { 915 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 916 return -1; 917 } 918 919 /* allocate our memory */ 920 addr = alloc_mem(mem_sz, cur_pgsz, huge); 921 922 /* if we couldn't allocate memory with a specified page size, 923 * that doesn't mean we can't do it with other page sizes, so 924 * try another one. 925 */ 926 if (addr == NULL) 927 continue; 928 929 /* store IOVA addresses for every page in this memory area */ 930 n_pages = mem_sz / cur_pgsz; 931 932 iovas = malloc(sizeof(*iovas) * n_pages); 933 934 if (iovas == NULL) { 935 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 936 goto fail; 937 } 938 /* lock memory if it's not huge pages */ 939 if (!huge) 940 mlock(addr, mem_sz); 941 942 /* populate IOVA addresses */ 943 for (cur_page = 0; cur_page < n_pages; cur_page++) { 944 rte_iova_t iova; 945 size_t offset; 946 void *cur; 947 948 offset = cur_pgsz * cur_page; 949 cur = RTE_PTR_ADD(addr, offset); 950 951 /* touch the page before getting its IOVA */ 952 *(volatile char *)cur = 0; 953 954 iova = rte_mem_virt2iova(cur); 955 956 iovas[cur_page] = iova; 957 } 958 959 break; 960 } 961 /* if we couldn't allocate anything */ 962 if (iovas == NULL) 963 return -1; 964 965 param->addr = addr; 966 param->len = mem_sz; 967 param->pgsz = cur_pgsz; 968 param->iova_table = iovas; 969 param->iova_table_len = n_pages; 970 971 return 0; 972 fail: 973 free(iovas); 974 if (addr) 975 munmap(addr, mem_sz); 976 977 return -1; 978 } 979 980 static int 981 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 982 { 983 struct extmem_param param; 984 int socket_id, ret; 985 986 memset(¶m, 0, sizeof(param)); 987 988 /* check if our heap exists */ 989 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 990 if (socket_id < 0) { 991 /* create our heap */ 992 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 993 if (ret < 0) { 994 TESTPMD_LOG(ERR, "Cannot create heap\n"); 995 return -1; 996 } 997 } 998 999 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 1000 if (ret < 0) { 1001 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 1002 return -1; 1003 } 1004 1005 /* we now have a valid memory area, so add it to heap */ 1006 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 1007 param.addr, param.len, param.iova_table, 1008 param.iova_table_len, param.pgsz); 1009 1010 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1011 1012 /* not needed any more */ 1013 free(param.iova_table); 1014 1015 if (ret < 0) { 1016 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1017 munmap(param.addr, param.len); 1018 return -1; 1019 } 1020 1021 /* success */ 1022 1023 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1024 param.len >> 20); 1025 1026 return 0; 1027 } 1028 static void 1029 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1030 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1031 { 1032 uint16_t pid = 0; 1033 int ret; 1034 1035 RTE_ETH_FOREACH_DEV(pid) { 1036 struct rte_eth_dev_info dev_info; 1037 1038 ret = eth_dev_info_get_print_err(pid, &dev_info); 1039 if (ret != 0) { 1040 TESTPMD_LOG(DEBUG, 1041 "unable to get device info for port %d on addr 0x%p," 1042 "mempool unmapping will not be performed\n", 1043 pid, memhdr->addr); 1044 continue; 1045 } 1046 1047 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1048 if (ret) { 1049 TESTPMD_LOG(DEBUG, 1050 "unable to DMA unmap addr 0x%p " 1051 "for device %s\n", 1052 memhdr->addr, rte_dev_name(dev_info.device)); 1053 } 1054 } 1055 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1056 if (ret) { 1057 TESTPMD_LOG(DEBUG, 1058 "unable to un-register addr 0x%p\n", memhdr->addr); 1059 } 1060 } 1061 1062 static void 1063 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1064 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1065 { 1066 uint16_t pid = 0; 1067 size_t page_size = sysconf(_SC_PAGESIZE); 1068 int ret; 1069 1070 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1071 page_size); 1072 if (ret) { 1073 TESTPMD_LOG(DEBUG, 1074 "unable to register addr 0x%p\n", memhdr->addr); 1075 return; 1076 } 1077 RTE_ETH_FOREACH_DEV(pid) { 1078 struct rte_eth_dev_info dev_info; 1079 1080 ret = eth_dev_info_get_print_err(pid, &dev_info); 1081 if (ret != 0) { 1082 TESTPMD_LOG(DEBUG, 1083 "unable to get device info for port %d on addr 0x%p," 1084 "mempool mapping will not be performed\n", 1085 pid, memhdr->addr); 1086 continue; 1087 } 1088 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1089 if (ret) { 1090 TESTPMD_LOG(DEBUG, 1091 "unable to DMA map addr 0x%p " 1092 "for device %s\n", 1093 memhdr->addr, rte_dev_name(dev_info.device)); 1094 } 1095 } 1096 } 1097 #endif 1098 1099 static unsigned int 1100 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1101 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1102 { 1103 struct rte_pktmbuf_extmem *xmem; 1104 unsigned int ext_num, zone_num, elt_num; 1105 uint16_t elt_size; 1106 1107 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1108 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1109 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1110 1111 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1112 if (xmem == NULL) { 1113 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1114 "external buffer descriptors\n"); 1115 *ext_mem = NULL; 1116 return 0; 1117 } 1118 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1119 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1120 const struct rte_memzone *mz; 1121 char mz_name[RTE_MEMZONE_NAMESIZE]; 1122 int ret; 1123 1124 ret = snprintf(mz_name, sizeof(mz_name), 1125 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1126 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1127 errno = ENAMETOOLONG; 1128 ext_num = 0; 1129 break; 1130 } 1131 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1132 socket_id, 1133 RTE_MEMZONE_IOVA_CONTIG | 1134 RTE_MEMZONE_1GB | 1135 RTE_MEMZONE_SIZE_HINT_ONLY); 1136 if (mz == NULL) { 1137 /* 1138 * The caller exits on external buffer creation 1139 * error, so there is no need to free memzones. 1140 */ 1141 errno = ENOMEM; 1142 ext_num = 0; 1143 break; 1144 } 1145 xseg->buf_ptr = mz->addr; 1146 xseg->buf_iova = mz->iova; 1147 xseg->buf_len = EXTBUF_ZONE_SIZE; 1148 xseg->elt_size = elt_size; 1149 } 1150 if (ext_num == 0 && xmem != NULL) { 1151 free(xmem); 1152 xmem = NULL; 1153 } 1154 *ext_mem = xmem; 1155 return ext_num; 1156 } 1157 1158 /* 1159 * Configuration initialisation done once at init time. 1160 */ 1161 static struct rte_mempool * 1162 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1163 unsigned int socket_id, uint16_t size_idx) 1164 { 1165 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1166 struct rte_mempool *rte_mp = NULL; 1167 #ifndef RTE_EXEC_ENV_WINDOWS 1168 uint32_t mb_size; 1169 1170 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1171 #endif 1172 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1173 if (!is_proc_primary()) { 1174 rte_mp = rte_mempool_lookup(pool_name); 1175 if (rte_mp == NULL) 1176 rte_exit(EXIT_FAILURE, 1177 "Get mbuf pool for socket %u failed: %s\n", 1178 socket_id, rte_strerror(rte_errno)); 1179 return rte_mp; 1180 } 1181 1182 TESTPMD_LOG(INFO, 1183 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1184 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1185 1186 switch (mp_alloc_type) { 1187 case MP_ALLOC_NATIVE: 1188 { 1189 /* wrapper to rte_mempool_create() */ 1190 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1191 rte_mbuf_best_mempool_ops()); 1192 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1193 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1194 break; 1195 } 1196 #ifndef RTE_EXEC_ENV_WINDOWS 1197 case MP_ALLOC_ANON: 1198 { 1199 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1200 mb_size, (unsigned int) mb_mempool_cache, 1201 sizeof(struct rte_pktmbuf_pool_private), 1202 socket_id, mempool_flags); 1203 if (rte_mp == NULL) 1204 goto err; 1205 1206 if (rte_mempool_populate_anon(rte_mp) == 0) { 1207 rte_mempool_free(rte_mp); 1208 rte_mp = NULL; 1209 goto err; 1210 } 1211 rte_pktmbuf_pool_init(rte_mp, NULL); 1212 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1213 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1214 break; 1215 } 1216 case MP_ALLOC_XMEM: 1217 case MP_ALLOC_XMEM_HUGE: 1218 { 1219 int heap_socket; 1220 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1221 1222 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1223 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1224 1225 heap_socket = 1226 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1227 if (heap_socket < 0) 1228 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1229 1230 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1231 rte_mbuf_best_mempool_ops()); 1232 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1233 mb_mempool_cache, 0, mbuf_seg_size, 1234 heap_socket); 1235 break; 1236 } 1237 #endif 1238 case MP_ALLOC_XBUF: 1239 { 1240 struct rte_pktmbuf_extmem *ext_mem; 1241 unsigned int ext_num; 1242 1243 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1244 socket_id, pool_name, &ext_mem); 1245 if (ext_num == 0) 1246 rte_exit(EXIT_FAILURE, 1247 "Can't create pinned data buffers\n"); 1248 1249 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1250 rte_mbuf_best_mempool_ops()); 1251 rte_mp = rte_pktmbuf_pool_create_extbuf 1252 (pool_name, nb_mbuf, mb_mempool_cache, 1253 0, mbuf_seg_size, socket_id, 1254 ext_mem, ext_num); 1255 free(ext_mem); 1256 break; 1257 } 1258 default: 1259 { 1260 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1261 } 1262 } 1263 1264 #ifndef RTE_EXEC_ENV_WINDOWS 1265 err: 1266 #endif 1267 if (rte_mp == NULL) { 1268 rte_exit(EXIT_FAILURE, 1269 "Creation of mbuf pool for socket %u failed: %s\n", 1270 socket_id, rte_strerror(rte_errno)); 1271 } else if (verbose_level > 0) { 1272 rte_mempool_dump(stdout, rte_mp); 1273 } 1274 return rte_mp; 1275 } 1276 1277 /* 1278 * Check given socket id is valid or not with NUMA mode, 1279 * if valid, return 0, else return -1 1280 */ 1281 static int 1282 check_socket_id(const unsigned int socket_id) 1283 { 1284 static int warning_once = 0; 1285 1286 if (new_socket_id(socket_id)) { 1287 if (!warning_once && numa_support) 1288 fprintf(stderr, 1289 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1290 warning_once = 1; 1291 return -1; 1292 } 1293 return 0; 1294 } 1295 1296 /* 1297 * Get the allowed maximum number of RX queues. 1298 * *pid return the port id which has minimal value of 1299 * max_rx_queues in all ports. 1300 */ 1301 queueid_t 1302 get_allowed_max_nb_rxq(portid_t *pid) 1303 { 1304 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1305 bool max_rxq_valid = false; 1306 portid_t pi; 1307 struct rte_eth_dev_info dev_info; 1308 1309 RTE_ETH_FOREACH_DEV(pi) { 1310 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1311 continue; 1312 1313 max_rxq_valid = true; 1314 if (dev_info.max_rx_queues < allowed_max_rxq) { 1315 allowed_max_rxq = dev_info.max_rx_queues; 1316 *pid = pi; 1317 } 1318 } 1319 return max_rxq_valid ? allowed_max_rxq : 0; 1320 } 1321 1322 /* 1323 * Check input rxq is valid or not. 1324 * If input rxq is not greater than any of maximum number 1325 * of RX queues of all ports, it is valid. 1326 * if valid, return 0, else return -1 1327 */ 1328 int 1329 check_nb_rxq(queueid_t rxq) 1330 { 1331 queueid_t allowed_max_rxq; 1332 portid_t pid = 0; 1333 1334 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1335 if (rxq > allowed_max_rxq) { 1336 fprintf(stderr, 1337 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1338 rxq, allowed_max_rxq, pid); 1339 return -1; 1340 } 1341 return 0; 1342 } 1343 1344 /* 1345 * Get the allowed maximum number of TX queues. 1346 * *pid return the port id which has minimal value of 1347 * max_tx_queues in all ports. 1348 */ 1349 queueid_t 1350 get_allowed_max_nb_txq(portid_t *pid) 1351 { 1352 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1353 bool max_txq_valid = false; 1354 portid_t pi; 1355 struct rte_eth_dev_info dev_info; 1356 1357 RTE_ETH_FOREACH_DEV(pi) { 1358 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1359 continue; 1360 1361 max_txq_valid = true; 1362 if (dev_info.max_tx_queues < allowed_max_txq) { 1363 allowed_max_txq = dev_info.max_tx_queues; 1364 *pid = pi; 1365 } 1366 } 1367 return max_txq_valid ? allowed_max_txq : 0; 1368 } 1369 1370 /* 1371 * Check input txq is valid or not. 1372 * If input txq is not greater than any of maximum number 1373 * of TX queues of all ports, it is valid. 1374 * if valid, return 0, else return -1 1375 */ 1376 int 1377 check_nb_txq(queueid_t txq) 1378 { 1379 queueid_t allowed_max_txq; 1380 portid_t pid = 0; 1381 1382 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1383 if (txq > allowed_max_txq) { 1384 fprintf(stderr, 1385 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1386 txq, allowed_max_txq, pid); 1387 return -1; 1388 } 1389 return 0; 1390 } 1391 1392 /* 1393 * Get the allowed maximum number of RXDs of every rx queue. 1394 * *pid return the port id which has minimal value of 1395 * max_rxd in all queues of all ports. 1396 */ 1397 static uint16_t 1398 get_allowed_max_nb_rxd(portid_t *pid) 1399 { 1400 uint16_t allowed_max_rxd = UINT16_MAX; 1401 portid_t pi; 1402 struct rte_eth_dev_info dev_info; 1403 1404 RTE_ETH_FOREACH_DEV(pi) { 1405 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1406 continue; 1407 1408 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1409 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1410 *pid = pi; 1411 } 1412 } 1413 return allowed_max_rxd; 1414 } 1415 1416 /* 1417 * Get the allowed minimal number of RXDs of every rx queue. 1418 * *pid return the port id which has minimal value of 1419 * min_rxd in all queues of all ports. 1420 */ 1421 static uint16_t 1422 get_allowed_min_nb_rxd(portid_t *pid) 1423 { 1424 uint16_t allowed_min_rxd = 0; 1425 portid_t pi; 1426 struct rte_eth_dev_info dev_info; 1427 1428 RTE_ETH_FOREACH_DEV(pi) { 1429 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1430 continue; 1431 1432 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1433 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1434 *pid = pi; 1435 } 1436 } 1437 1438 return allowed_min_rxd; 1439 } 1440 1441 /* 1442 * Check input rxd is valid or not. 1443 * If input rxd is not greater than any of maximum number 1444 * of RXDs of every Rx queues and is not less than any of 1445 * minimal number of RXDs of every Rx queues, it is valid. 1446 * if valid, return 0, else return -1 1447 */ 1448 int 1449 check_nb_rxd(queueid_t rxd) 1450 { 1451 uint16_t allowed_max_rxd; 1452 uint16_t allowed_min_rxd; 1453 portid_t pid = 0; 1454 1455 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1456 if (rxd > allowed_max_rxd) { 1457 fprintf(stderr, 1458 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1459 rxd, allowed_max_rxd, pid); 1460 return -1; 1461 } 1462 1463 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1464 if (rxd < allowed_min_rxd) { 1465 fprintf(stderr, 1466 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1467 rxd, allowed_min_rxd, pid); 1468 return -1; 1469 } 1470 1471 return 0; 1472 } 1473 1474 /* 1475 * Get the allowed maximum number of TXDs of every rx queues. 1476 * *pid return the port id which has minimal value of 1477 * max_txd in every tx queue. 1478 */ 1479 static uint16_t 1480 get_allowed_max_nb_txd(portid_t *pid) 1481 { 1482 uint16_t allowed_max_txd = UINT16_MAX; 1483 portid_t pi; 1484 struct rte_eth_dev_info dev_info; 1485 1486 RTE_ETH_FOREACH_DEV(pi) { 1487 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1488 continue; 1489 1490 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1491 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1492 *pid = pi; 1493 } 1494 } 1495 return allowed_max_txd; 1496 } 1497 1498 /* 1499 * Get the allowed maximum number of TXDs of every tx queues. 1500 * *pid return the port id which has minimal value of 1501 * min_txd in every tx queue. 1502 */ 1503 static uint16_t 1504 get_allowed_min_nb_txd(portid_t *pid) 1505 { 1506 uint16_t allowed_min_txd = 0; 1507 portid_t pi; 1508 struct rte_eth_dev_info dev_info; 1509 1510 RTE_ETH_FOREACH_DEV(pi) { 1511 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1512 continue; 1513 1514 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1515 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1516 *pid = pi; 1517 } 1518 } 1519 1520 return allowed_min_txd; 1521 } 1522 1523 /* 1524 * Check input txd is valid or not. 1525 * If input txd is not greater than any of maximum number 1526 * of TXDs of every Rx queues, it is valid. 1527 * if valid, return 0, else return -1 1528 */ 1529 int 1530 check_nb_txd(queueid_t txd) 1531 { 1532 uint16_t allowed_max_txd; 1533 uint16_t allowed_min_txd; 1534 portid_t pid = 0; 1535 1536 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1537 if (txd > allowed_max_txd) { 1538 fprintf(stderr, 1539 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1540 txd, allowed_max_txd, pid); 1541 return -1; 1542 } 1543 1544 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1545 if (txd < allowed_min_txd) { 1546 fprintf(stderr, 1547 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1548 txd, allowed_min_txd, pid); 1549 return -1; 1550 } 1551 return 0; 1552 } 1553 1554 static int 1555 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1556 { 1557 uint32_t eth_overhead; 1558 1559 if (dev_info->max_mtu != UINT16_MAX && 1560 dev_info->max_rx_pktlen > dev_info->max_mtu) 1561 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1562 else 1563 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1564 1565 return eth_overhead; 1566 } 1567 1568 static void 1569 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1570 { 1571 struct rte_port *port = &ports[pid]; 1572 int ret; 1573 int i; 1574 1575 eth_rx_metadata_negotiate_mp(pid); 1576 1577 port->dev_conf.txmode = tx_mode; 1578 port->dev_conf.rxmode = rx_mode; 1579 1580 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1581 if (ret != 0) 1582 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1583 1584 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1585 port->dev_conf.txmode.offloads &= 1586 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1587 1588 /* Apply Rx offloads configuration */ 1589 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1590 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1591 /* Apply Tx offloads configuration */ 1592 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1593 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1594 1595 if (eth_link_speed) 1596 port->dev_conf.link_speeds = eth_link_speed; 1597 1598 if (max_rx_pkt_len) 1599 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1600 get_eth_overhead(&port->dev_info); 1601 1602 /* set flag to initialize port/queue */ 1603 port->need_reconfig = 1; 1604 port->need_reconfig_queues = 1; 1605 port->socket_id = socket_id; 1606 port->tx_metadata = 0; 1607 1608 /* 1609 * Check for maximum number of segments per MTU. 1610 * Accordingly update the mbuf data size. 1611 */ 1612 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1613 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1614 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1615 uint16_t mtu; 1616 1617 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1618 uint16_t data_size = (mtu + eth_overhead) / 1619 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1620 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1621 1622 if (buffer_size > mbuf_data_size[0]) { 1623 mbuf_data_size[0] = buffer_size; 1624 TESTPMD_LOG(WARNING, 1625 "Configured mbuf size of the first segment %hu\n", 1626 mbuf_data_size[0]); 1627 } 1628 } 1629 } 1630 } 1631 1632 static void 1633 init_config(void) 1634 { 1635 portid_t pid; 1636 struct rte_mempool *mbp; 1637 unsigned int nb_mbuf_per_pool; 1638 lcoreid_t lc_id; 1639 #ifdef RTE_LIB_GRO 1640 struct rte_gro_param gro_param; 1641 #endif 1642 #ifdef RTE_LIB_GSO 1643 uint32_t gso_types; 1644 #endif 1645 1646 /* Configuration of logical cores. */ 1647 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1648 sizeof(struct fwd_lcore *) * nb_lcores, 1649 RTE_CACHE_LINE_SIZE); 1650 if (fwd_lcores == NULL) { 1651 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1652 "failed\n", nb_lcores); 1653 } 1654 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1655 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1656 sizeof(struct fwd_lcore), 1657 RTE_CACHE_LINE_SIZE); 1658 if (fwd_lcores[lc_id] == NULL) { 1659 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1660 "failed\n"); 1661 } 1662 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1663 } 1664 1665 RTE_ETH_FOREACH_DEV(pid) { 1666 uint32_t socket_id; 1667 1668 if (numa_support) { 1669 socket_id = port_numa[pid]; 1670 if (port_numa[pid] == NUMA_NO_CONFIG) { 1671 socket_id = rte_eth_dev_socket_id(pid); 1672 1673 /* 1674 * if socket_id is invalid, 1675 * set to the first available socket. 1676 */ 1677 if (check_socket_id(socket_id) < 0) 1678 socket_id = socket_ids[0]; 1679 } 1680 } else { 1681 socket_id = (socket_num == UMA_NO_CONFIG) ? 1682 0 : socket_num; 1683 } 1684 /* Apply default TxRx configuration for all ports */ 1685 init_config_port_offloads(pid, socket_id); 1686 } 1687 /* 1688 * Create pools of mbuf. 1689 * If NUMA support is disabled, create a single pool of mbuf in 1690 * socket 0 memory by default. 1691 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1692 * 1693 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1694 * nb_txd can be configured at run time. 1695 */ 1696 if (param_total_num_mbufs) 1697 nb_mbuf_per_pool = param_total_num_mbufs; 1698 else { 1699 nb_mbuf_per_pool = RX_DESC_MAX + 1700 (nb_lcores * mb_mempool_cache) + 1701 TX_DESC_MAX + MAX_PKT_BURST; 1702 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1703 } 1704 1705 if (numa_support) { 1706 uint8_t i, j; 1707 1708 for (i = 0; i < num_sockets; i++) 1709 for (j = 0; j < mbuf_data_size_n; j++) 1710 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1711 mbuf_pool_create(mbuf_data_size[j], 1712 nb_mbuf_per_pool, 1713 socket_ids[i], j); 1714 } else { 1715 uint8_t i; 1716 1717 for (i = 0; i < mbuf_data_size_n; i++) 1718 mempools[i] = mbuf_pool_create 1719 (mbuf_data_size[i], 1720 nb_mbuf_per_pool, 1721 socket_num == UMA_NO_CONFIG ? 1722 0 : socket_num, i); 1723 } 1724 1725 init_port_config(); 1726 1727 #ifdef RTE_LIB_GSO 1728 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1729 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1730 #endif 1731 /* 1732 * Records which Mbuf pool to use by each logical core, if needed. 1733 */ 1734 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1735 mbp = mbuf_pool_find( 1736 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1737 1738 if (mbp == NULL) 1739 mbp = mbuf_pool_find(0, 0); 1740 fwd_lcores[lc_id]->mbp = mbp; 1741 #ifdef RTE_LIB_GSO 1742 /* initialize GSO context */ 1743 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1744 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1745 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1746 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1747 RTE_ETHER_CRC_LEN; 1748 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1749 #endif 1750 } 1751 1752 fwd_config_setup(); 1753 1754 #ifdef RTE_LIB_GRO 1755 /* create a gro context for each lcore */ 1756 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1757 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1758 gro_param.max_item_per_flow = MAX_PKT_BURST; 1759 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1760 gro_param.socket_id = rte_lcore_to_socket_id( 1761 fwd_lcores_cpuids[lc_id]); 1762 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1763 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1764 rte_exit(EXIT_FAILURE, 1765 "rte_gro_ctx_create() failed\n"); 1766 } 1767 } 1768 #endif 1769 } 1770 1771 1772 void 1773 reconfig(portid_t new_port_id, unsigned socket_id) 1774 { 1775 /* Reconfiguration of Ethernet ports. */ 1776 init_config_port_offloads(new_port_id, socket_id); 1777 init_port_config(); 1778 } 1779 1780 int 1781 init_fwd_streams(void) 1782 { 1783 portid_t pid; 1784 struct rte_port *port; 1785 streamid_t sm_id, nb_fwd_streams_new; 1786 queueid_t q; 1787 1788 /* set socket id according to numa or not */ 1789 RTE_ETH_FOREACH_DEV(pid) { 1790 port = &ports[pid]; 1791 if (nb_rxq > port->dev_info.max_rx_queues) { 1792 fprintf(stderr, 1793 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1794 nb_rxq, port->dev_info.max_rx_queues); 1795 return -1; 1796 } 1797 if (nb_txq > port->dev_info.max_tx_queues) { 1798 fprintf(stderr, 1799 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1800 nb_txq, port->dev_info.max_tx_queues); 1801 return -1; 1802 } 1803 if (numa_support) { 1804 if (port_numa[pid] != NUMA_NO_CONFIG) 1805 port->socket_id = port_numa[pid]; 1806 else { 1807 port->socket_id = rte_eth_dev_socket_id(pid); 1808 1809 /* 1810 * if socket_id is invalid, 1811 * set to the first available socket. 1812 */ 1813 if (check_socket_id(port->socket_id) < 0) 1814 port->socket_id = socket_ids[0]; 1815 } 1816 } 1817 else { 1818 if (socket_num == UMA_NO_CONFIG) 1819 port->socket_id = 0; 1820 else 1821 port->socket_id = socket_num; 1822 } 1823 } 1824 1825 q = RTE_MAX(nb_rxq, nb_txq); 1826 if (q == 0) { 1827 fprintf(stderr, 1828 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1829 return -1; 1830 } 1831 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1832 if (nb_fwd_streams_new == nb_fwd_streams) 1833 return 0; 1834 /* clear the old */ 1835 if (fwd_streams != NULL) { 1836 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1837 if (fwd_streams[sm_id] == NULL) 1838 continue; 1839 rte_free(fwd_streams[sm_id]); 1840 fwd_streams[sm_id] = NULL; 1841 } 1842 rte_free(fwd_streams); 1843 fwd_streams = NULL; 1844 } 1845 1846 /* init new */ 1847 nb_fwd_streams = nb_fwd_streams_new; 1848 if (nb_fwd_streams) { 1849 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1850 sizeof(struct fwd_stream *) * nb_fwd_streams, 1851 RTE_CACHE_LINE_SIZE); 1852 if (fwd_streams == NULL) 1853 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1854 " (struct fwd_stream *)) failed\n", 1855 nb_fwd_streams); 1856 1857 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1858 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1859 " struct fwd_stream", sizeof(struct fwd_stream), 1860 RTE_CACHE_LINE_SIZE); 1861 if (fwd_streams[sm_id] == NULL) 1862 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1863 "(struct fwd_stream) failed\n"); 1864 } 1865 } 1866 1867 return 0; 1868 } 1869 1870 static void 1871 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1872 { 1873 uint64_t total_burst, sburst; 1874 uint64_t nb_burst; 1875 uint64_t burst_stats[4]; 1876 uint16_t pktnb_stats[4]; 1877 uint16_t nb_pkt; 1878 int burst_percent[4], sburstp; 1879 int i; 1880 1881 /* 1882 * First compute the total number of packet bursts and the 1883 * two highest numbers of bursts of the same number of packets. 1884 */ 1885 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1886 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1887 1888 /* Show stats for 0 burst size always */ 1889 total_burst = pbs->pkt_burst_spread[0]; 1890 burst_stats[0] = pbs->pkt_burst_spread[0]; 1891 pktnb_stats[0] = 0; 1892 1893 /* Find the next 2 burst sizes with highest occurrences. */ 1894 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1895 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1896 1897 if (nb_burst == 0) 1898 continue; 1899 1900 total_burst += nb_burst; 1901 1902 if (nb_burst > burst_stats[1]) { 1903 burst_stats[2] = burst_stats[1]; 1904 pktnb_stats[2] = pktnb_stats[1]; 1905 burst_stats[1] = nb_burst; 1906 pktnb_stats[1] = nb_pkt; 1907 } else if (nb_burst > burst_stats[2]) { 1908 burst_stats[2] = nb_burst; 1909 pktnb_stats[2] = nb_pkt; 1910 } 1911 } 1912 if (total_burst == 0) 1913 return; 1914 1915 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1916 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1917 if (i == 3) { 1918 printf("%d%% of other]\n", 100 - sburstp); 1919 return; 1920 } 1921 1922 sburst += burst_stats[i]; 1923 if (sburst == total_burst) { 1924 printf("%d%% of %d pkts]\n", 1925 100 - sburstp, (int) pktnb_stats[i]); 1926 return; 1927 } 1928 1929 burst_percent[i] = 1930 (double)burst_stats[i] / total_burst * 100; 1931 printf("%d%% of %d pkts + ", 1932 burst_percent[i], (int) pktnb_stats[i]); 1933 sburstp += burst_percent[i]; 1934 } 1935 } 1936 1937 static void 1938 fwd_stream_stats_display(streamid_t stream_id) 1939 { 1940 struct fwd_stream *fs; 1941 static const char *fwd_top_stats_border = "-------"; 1942 1943 fs = fwd_streams[stream_id]; 1944 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1945 (fs->fwd_dropped == 0)) 1946 return; 1947 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1948 "TX Port=%2d/Queue=%2d %s\n", 1949 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1950 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1951 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1952 " TX-dropped: %-14"PRIu64, 1953 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1954 1955 /* if checksum mode */ 1956 if (cur_fwd_eng == &csum_fwd_engine) { 1957 printf(" RX- bad IP checksum: %-14"PRIu64 1958 " Rx- bad L4 checksum: %-14"PRIu64 1959 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1960 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1961 fs->rx_bad_outer_l4_csum); 1962 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1963 fs->rx_bad_outer_ip_csum); 1964 } else { 1965 printf("\n"); 1966 } 1967 1968 if (record_burst_stats) { 1969 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1970 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1971 } 1972 } 1973 1974 void 1975 fwd_stats_display(void) 1976 { 1977 static const char *fwd_stats_border = "----------------------"; 1978 static const char *acc_stats_border = "+++++++++++++++"; 1979 struct { 1980 struct fwd_stream *rx_stream; 1981 struct fwd_stream *tx_stream; 1982 uint64_t tx_dropped; 1983 uint64_t rx_bad_ip_csum; 1984 uint64_t rx_bad_l4_csum; 1985 uint64_t rx_bad_outer_l4_csum; 1986 uint64_t rx_bad_outer_ip_csum; 1987 } ports_stats[RTE_MAX_ETHPORTS]; 1988 uint64_t total_rx_dropped = 0; 1989 uint64_t total_tx_dropped = 0; 1990 uint64_t total_rx_nombuf = 0; 1991 struct rte_eth_stats stats; 1992 uint64_t fwd_cycles = 0; 1993 uint64_t total_recv = 0; 1994 uint64_t total_xmit = 0; 1995 struct rte_port *port; 1996 streamid_t sm_id; 1997 portid_t pt_id; 1998 int ret; 1999 int i; 2000 2001 memset(ports_stats, 0, sizeof(ports_stats)); 2002 2003 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2004 struct fwd_stream *fs = fwd_streams[sm_id]; 2005 2006 if (cur_fwd_config.nb_fwd_streams > 2007 cur_fwd_config.nb_fwd_ports) { 2008 fwd_stream_stats_display(sm_id); 2009 } else { 2010 ports_stats[fs->tx_port].tx_stream = fs; 2011 ports_stats[fs->rx_port].rx_stream = fs; 2012 } 2013 2014 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2015 2016 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2017 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2018 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2019 fs->rx_bad_outer_l4_csum; 2020 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2021 fs->rx_bad_outer_ip_csum; 2022 2023 if (record_core_cycles) 2024 fwd_cycles += fs->busy_cycles; 2025 } 2026 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2027 uint64_t tx_dropped = 0; 2028 2029 pt_id = fwd_ports_ids[i]; 2030 port = &ports[pt_id]; 2031 2032 ret = rte_eth_stats_get(pt_id, &stats); 2033 if (ret != 0) { 2034 fprintf(stderr, 2035 "%s: Error: failed to get stats (port %u): %d", 2036 __func__, pt_id, ret); 2037 continue; 2038 } 2039 stats.ipackets -= port->stats.ipackets; 2040 stats.opackets -= port->stats.opackets; 2041 stats.ibytes -= port->stats.ibytes; 2042 stats.obytes -= port->stats.obytes; 2043 stats.imissed -= port->stats.imissed; 2044 stats.oerrors -= port->stats.oerrors; 2045 stats.rx_nombuf -= port->stats.rx_nombuf; 2046 2047 total_recv += stats.ipackets; 2048 total_xmit += stats.opackets; 2049 total_rx_dropped += stats.imissed; 2050 tx_dropped += ports_stats[pt_id].tx_dropped; 2051 tx_dropped += stats.oerrors; 2052 total_tx_dropped += tx_dropped; 2053 total_rx_nombuf += stats.rx_nombuf; 2054 2055 printf("\n %s Forward statistics for port %-2d %s\n", 2056 fwd_stats_border, pt_id, fwd_stats_border); 2057 2058 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2059 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2060 stats.ipackets + stats.imissed); 2061 2062 if (cur_fwd_eng == &csum_fwd_engine) { 2063 printf(" Bad-ipcsum: %-14"PRIu64 2064 " Bad-l4csum: %-14"PRIu64 2065 "Bad-outer-l4csum: %-14"PRIu64"\n", 2066 ports_stats[pt_id].rx_bad_ip_csum, 2067 ports_stats[pt_id].rx_bad_l4_csum, 2068 ports_stats[pt_id].rx_bad_outer_l4_csum); 2069 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2070 ports_stats[pt_id].rx_bad_outer_ip_csum); 2071 } 2072 if (stats.ierrors + stats.rx_nombuf > 0) { 2073 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2074 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2075 } 2076 2077 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2078 "TX-total: %-"PRIu64"\n", 2079 stats.opackets, tx_dropped, 2080 stats.opackets + tx_dropped); 2081 2082 if (record_burst_stats) { 2083 if (ports_stats[pt_id].rx_stream) 2084 pkt_burst_stats_display("RX", 2085 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2086 if (ports_stats[pt_id].tx_stream) 2087 pkt_burst_stats_display("TX", 2088 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2089 } 2090 2091 printf(" %s--------------------------------%s\n", 2092 fwd_stats_border, fwd_stats_border); 2093 } 2094 2095 printf("\n %s Accumulated forward statistics for all ports" 2096 "%s\n", 2097 acc_stats_border, acc_stats_border); 2098 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2099 "%-"PRIu64"\n" 2100 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2101 "%-"PRIu64"\n", 2102 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2103 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2104 if (total_rx_nombuf > 0) 2105 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2106 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2107 "%s\n", 2108 acc_stats_border, acc_stats_border); 2109 if (record_core_cycles) { 2110 #define CYC_PER_MHZ 1E6 2111 if (total_recv > 0 || total_xmit > 0) { 2112 uint64_t total_pkts = 0; 2113 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2114 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2115 total_pkts = total_xmit; 2116 else 2117 total_pkts = total_recv; 2118 2119 printf("\n CPU cycles/packet=%.2F (busy cycles=" 2120 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2121 " MHz Clock\n", 2122 (double) fwd_cycles / total_pkts, 2123 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2124 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2125 } 2126 } 2127 } 2128 2129 void 2130 fwd_stats_reset(void) 2131 { 2132 streamid_t sm_id; 2133 portid_t pt_id; 2134 int ret; 2135 int i; 2136 2137 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2138 pt_id = fwd_ports_ids[i]; 2139 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2140 if (ret != 0) 2141 fprintf(stderr, 2142 "%s: Error: failed to clear stats (port %u):%d", 2143 __func__, pt_id, ret); 2144 } 2145 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2146 struct fwd_stream *fs = fwd_streams[sm_id]; 2147 2148 fs->rx_packets = 0; 2149 fs->tx_packets = 0; 2150 fs->fwd_dropped = 0; 2151 fs->rx_bad_ip_csum = 0; 2152 fs->rx_bad_l4_csum = 0; 2153 fs->rx_bad_outer_l4_csum = 0; 2154 fs->rx_bad_outer_ip_csum = 0; 2155 2156 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2157 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2158 fs->busy_cycles = 0; 2159 } 2160 } 2161 2162 static void 2163 flush_fwd_rx_queues(void) 2164 { 2165 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2166 portid_t rxp; 2167 portid_t port_id; 2168 queueid_t rxq; 2169 uint16_t nb_rx; 2170 uint8_t j; 2171 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2172 uint64_t timer_period; 2173 2174 if (num_procs > 1) { 2175 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2176 return; 2177 } 2178 2179 /* convert to number of cycles */ 2180 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2181 2182 for (j = 0; j < 2; j++) { 2183 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2184 for (rxq = 0; rxq < nb_rxq; rxq++) { 2185 port_id = fwd_ports_ids[rxp]; 2186 2187 /* Polling stopped queues is prohibited. */ 2188 if (ports[port_id].rxq[rxq].state == 2189 RTE_ETH_QUEUE_STATE_STOPPED) 2190 continue; 2191 2192 /** 2193 * testpmd can stuck in the below do while loop 2194 * if rte_eth_rx_burst() always returns nonzero 2195 * packets. So timer is added to exit this loop 2196 * after 1sec timer expiry. 2197 */ 2198 prev_tsc = rte_rdtsc(); 2199 do { 2200 nb_rx = rte_eth_rx_burst(port_id, rxq, 2201 pkts_burst, MAX_PKT_BURST); 2202 rte_pktmbuf_free_bulk(pkts_burst, nb_rx); 2203 2204 cur_tsc = rte_rdtsc(); 2205 diff_tsc = cur_tsc - prev_tsc; 2206 timer_tsc += diff_tsc; 2207 } while ((nb_rx > 0) && 2208 (timer_tsc < timer_period)); 2209 timer_tsc = 0; 2210 } 2211 } 2212 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2213 } 2214 } 2215 2216 static void 2217 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2218 { 2219 struct fwd_stream **fsm; 2220 uint64_t prev_tsc; 2221 streamid_t nb_fs; 2222 streamid_t sm_id; 2223 #ifdef RTE_LIB_BITRATESTATS 2224 uint64_t tics_per_1sec; 2225 uint64_t tics_datum; 2226 uint64_t tics_current; 2227 uint16_t i, cnt_ports; 2228 2229 cnt_ports = nb_ports; 2230 tics_datum = rte_rdtsc(); 2231 tics_per_1sec = rte_get_timer_hz(); 2232 #endif 2233 fsm = &fwd_streams[fc->stream_idx]; 2234 nb_fs = fc->stream_nb; 2235 prev_tsc = rte_rdtsc(); 2236 do { 2237 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2238 struct fwd_stream *fs = fsm[sm_id]; 2239 uint64_t start_fs_tsc = 0; 2240 bool busy; 2241 2242 if (fs->disabled) 2243 continue; 2244 if (record_core_cycles) 2245 start_fs_tsc = rte_rdtsc(); 2246 busy = (*pkt_fwd)(fs); 2247 if (record_core_cycles && busy) 2248 fs->busy_cycles += rte_rdtsc() - start_fs_tsc; 2249 } 2250 #ifdef RTE_LIB_BITRATESTATS 2251 if (bitrate_enabled != 0 && 2252 bitrate_lcore_id == rte_lcore_id()) { 2253 tics_current = rte_rdtsc(); 2254 if (tics_current - tics_datum >= tics_per_1sec) { 2255 /* Periodic bitrate calculation */ 2256 for (i = 0; i < cnt_ports; i++) 2257 rte_stats_bitrate_calc(bitrate_data, 2258 ports_ids[i]); 2259 tics_datum = tics_current; 2260 } 2261 } 2262 #endif 2263 #ifdef RTE_LIB_LATENCYSTATS 2264 if (latencystats_enabled != 0 && 2265 latencystats_lcore_id == rte_lcore_id()) 2266 rte_latencystats_update(); 2267 #endif 2268 if (record_core_cycles) { 2269 uint64_t tsc = rte_rdtsc(); 2270 2271 fc->total_cycles += tsc - prev_tsc; 2272 prev_tsc = tsc; 2273 } 2274 } while (! fc->stopped); 2275 } 2276 2277 static int 2278 lcore_usage_callback(unsigned int lcore_id, struct rte_lcore_usage *usage) 2279 { 2280 struct fwd_stream **fsm; 2281 struct fwd_lcore *fc; 2282 streamid_t nb_fs; 2283 streamid_t sm_id; 2284 2285 fc = lcore_to_fwd_lcore(lcore_id); 2286 if (fc == NULL) 2287 return -1; 2288 2289 fsm = &fwd_streams[fc->stream_idx]; 2290 nb_fs = fc->stream_nb; 2291 usage->busy_cycles = 0; 2292 usage->total_cycles = fc->total_cycles; 2293 2294 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2295 if (!fsm[sm_id]->disabled) 2296 usage->busy_cycles += fsm[sm_id]->busy_cycles; 2297 } 2298 2299 return 0; 2300 } 2301 2302 static int 2303 start_pkt_forward_on_core(void *fwd_arg) 2304 { 2305 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2306 cur_fwd_config.fwd_eng->packet_fwd); 2307 return 0; 2308 } 2309 2310 /* 2311 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2312 * Used to start communication flows in network loopback test configurations. 2313 */ 2314 static int 2315 run_one_txonly_burst_on_core(void *fwd_arg) 2316 { 2317 struct fwd_lcore *fwd_lc; 2318 struct fwd_lcore tmp_lcore; 2319 2320 fwd_lc = (struct fwd_lcore *) fwd_arg; 2321 tmp_lcore = *fwd_lc; 2322 tmp_lcore.stopped = 1; 2323 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2324 return 0; 2325 } 2326 2327 /* 2328 * Launch packet forwarding: 2329 * - Setup per-port forwarding context. 2330 * - launch logical cores with their forwarding configuration. 2331 */ 2332 static void 2333 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2334 { 2335 unsigned int i; 2336 unsigned int lc_id; 2337 int diag; 2338 2339 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2340 lc_id = fwd_lcores_cpuids[i]; 2341 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2342 fwd_lcores[i]->stopped = 0; 2343 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2344 fwd_lcores[i], lc_id); 2345 if (diag != 0) 2346 fprintf(stderr, 2347 "launch lcore %u failed - diag=%d\n", 2348 lc_id, diag); 2349 } 2350 } 2351 } 2352 2353 void 2354 common_fwd_stream_init(struct fwd_stream *fs) 2355 { 2356 bool rx_stopped, tx_stopped; 2357 2358 rx_stopped = (ports[fs->rx_port].rxq[fs->rx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2359 tx_stopped = (ports[fs->tx_port].txq[fs->tx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2360 fs->disabled = rx_stopped || tx_stopped; 2361 } 2362 2363 static void 2364 update_rx_queue_state(uint16_t port_id, uint16_t queue_id) 2365 { 2366 struct rte_eth_rxq_info rx_qinfo; 2367 int32_t rc; 2368 2369 rc = rte_eth_rx_queue_info_get(port_id, 2370 queue_id, &rx_qinfo); 2371 if (rc == 0) { 2372 ports[port_id].rxq[queue_id].state = 2373 rx_qinfo.queue_state; 2374 } else if (rc == -ENOTSUP) { 2375 /* 2376 * Do not change the rxq state for primary process 2377 * to ensure that the PMDs do not implement 2378 * rte_eth_rx_queue_info_get can forward as before. 2379 */ 2380 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 2381 return; 2382 /* 2383 * Set the rxq state to RTE_ETH_QUEUE_STATE_STARTED 2384 * to ensure that the PMDs do not implement 2385 * rte_eth_rx_queue_info_get can forward. 2386 */ 2387 ports[port_id].rxq[queue_id].state = 2388 RTE_ETH_QUEUE_STATE_STARTED; 2389 } else { 2390 TESTPMD_LOG(WARNING, 2391 "Failed to get rx queue info\n"); 2392 } 2393 } 2394 2395 static void 2396 update_tx_queue_state(uint16_t port_id, uint16_t queue_id) 2397 { 2398 struct rte_eth_txq_info tx_qinfo; 2399 int32_t rc; 2400 2401 rc = rte_eth_tx_queue_info_get(port_id, 2402 queue_id, &tx_qinfo); 2403 if (rc == 0) { 2404 ports[port_id].txq[queue_id].state = 2405 tx_qinfo.queue_state; 2406 } else if (rc == -ENOTSUP) { 2407 /* 2408 * Do not change the txq state for primary process 2409 * to ensure that the PMDs do not implement 2410 * rte_eth_tx_queue_info_get can forward as before. 2411 */ 2412 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 2413 return; 2414 /* 2415 * Set the txq state to RTE_ETH_QUEUE_STATE_STARTED 2416 * to ensure that the PMDs do not implement 2417 * rte_eth_tx_queue_info_get can forward. 2418 */ 2419 ports[port_id].txq[queue_id].state = 2420 RTE_ETH_QUEUE_STATE_STARTED; 2421 } else { 2422 TESTPMD_LOG(WARNING, 2423 "Failed to get tx queue info\n"); 2424 } 2425 } 2426 2427 static void 2428 update_queue_state(portid_t pid) 2429 { 2430 portid_t pi; 2431 queueid_t qi; 2432 2433 RTE_ETH_FOREACH_DEV(pi) { 2434 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2435 continue; 2436 2437 for (qi = 0; qi < nb_rxq; qi++) 2438 update_rx_queue_state(pi, qi); 2439 for (qi = 0; qi < nb_txq; qi++) 2440 update_tx_queue_state(pi, qi); 2441 } 2442 } 2443 2444 /* 2445 * Launch packet forwarding configuration. 2446 */ 2447 void 2448 start_packet_forwarding(int with_tx_first) 2449 { 2450 port_fwd_begin_t port_fwd_begin; 2451 port_fwd_end_t port_fwd_end; 2452 stream_init_t stream_init = cur_fwd_eng->stream_init; 2453 unsigned int i; 2454 2455 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2456 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2457 2458 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2459 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2460 2461 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2462 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2463 (!nb_rxq || !nb_txq)) 2464 rte_exit(EXIT_FAILURE, 2465 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2466 cur_fwd_eng->fwd_mode_name); 2467 2468 if (all_ports_started() == 0) { 2469 fprintf(stderr, "Not all ports were started\n"); 2470 return; 2471 } 2472 if (test_done == 0) { 2473 fprintf(stderr, "Packet forwarding already started\n"); 2474 return; 2475 } 2476 2477 fwd_config_setup(); 2478 2479 pkt_fwd_config_display(&cur_fwd_config); 2480 if (!pkt_fwd_shared_rxq_check()) 2481 return; 2482 2483 if (stream_init != NULL) { 2484 update_queue_state(RTE_PORT_ALL); 2485 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2486 stream_init(fwd_streams[i]); 2487 } 2488 2489 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2490 if (port_fwd_begin != NULL) { 2491 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2492 if (port_fwd_begin(fwd_ports_ids[i])) { 2493 fprintf(stderr, 2494 "Packet forwarding is not ready\n"); 2495 return; 2496 } 2497 } 2498 } 2499 2500 if (with_tx_first) { 2501 port_fwd_begin = tx_only_engine.port_fwd_begin; 2502 if (port_fwd_begin != NULL) { 2503 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2504 if (port_fwd_begin(fwd_ports_ids[i])) { 2505 fprintf(stderr, 2506 "Packet forwarding is not ready\n"); 2507 return; 2508 } 2509 } 2510 } 2511 } 2512 2513 test_done = 0; 2514 2515 if(!no_flush_rx) 2516 flush_fwd_rx_queues(); 2517 2518 rxtx_config_display(); 2519 2520 fwd_stats_reset(); 2521 if (with_tx_first) { 2522 while (with_tx_first--) { 2523 launch_packet_forwarding( 2524 run_one_txonly_burst_on_core); 2525 rte_eal_mp_wait_lcore(); 2526 } 2527 port_fwd_end = tx_only_engine.port_fwd_end; 2528 if (port_fwd_end != NULL) { 2529 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2530 (*port_fwd_end)(fwd_ports_ids[i]); 2531 } 2532 } 2533 launch_packet_forwarding(start_pkt_forward_on_core); 2534 } 2535 2536 void 2537 stop_packet_forwarding(void) 2538 { 2539 port_fwd_end_t port_fwd_end; 2540 lcoreid_t lc_id; 2541 portid_t pt_id; 2542 int i; 2543 2544 if (test_done) { 2545 fprintf(stderr, "Packet forwarding not started\n"); 2546 return; 2547 } 2548 printf("Telling cores to stop..."); 2549 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2550 fwd_lcores[lc_id]->stopped = 1; 2551 printf("\nWaiting for lcores to finish...\n"); 2552 rte_eal_mp_wait_lcore(); 2553 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2554 if (port_fwd_end != NULL) { 2555 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2556 pt_id = fwd_ports_ids[i]; 2557 (*port_fwd_end)(pt_id); 2558 } 2559 } 2560 2561 fwd_stats_display(); 2562 2563 printf("\nDone.\n"); 2564 test_done = 1; 2565 } 2566 2567 void 2568 dev_set_link_up(portid_t pid) 2569 { 2570 if (rte_eth_dev_set_link_up(pid) < 0) 2571 fprintf(stderr, "\nSet link up fail.\n"); 2572 } 2573 2574 void 2575 dev_set_link_down(portid_t pid) 2576 { 2577 if (rte_eth_dev_set_link_down(pid) < 0) 2578 fprintf(stderr, "\nSet link down fail.\n"); 2579 } 2580 2581 static int 2582 all_ports_started(void) 2583 { 2584 portid_t pi; 2585 struct rte_port *port; 2586 2587 RTE_ETH_FOREACH_DEV(pi) { 2588 port = &ports[pi]; 2589 /* Check if there is a port which is not started */ 2590 if ((port->port_status != RTE_PORT_STARTED) && 2591 (port->member_flag == 0)) 2592 return 0; 2593 } 2594 2595 /* No port is not started */ 2596 return 1; 2597 } 2598 2599 int 2600 port_is_stopped(portid_t port_id) 2601 { 2602 struct rte_port *port = &ports[port_id]; 2603 2604 if ((port->port_status != RTE_PORT_STOPPED) && 2605 (port->member_flag == 0)) 2606 return 0; 2607 return 1; 2608 } 2609 2610 int 2611 all_ports_stopped(void) 2612 { 2613 portid_t pi; 2614 2615 RTE_ETH_FOREACH_DEV(pi) { 2616 if (!port_is_stopped(pi)) 2617 return 0; 2618 } 2619 2620 return 1; 2621 } 2622 2623 int 2624 port_is_started(portid_t port_id) 2625 { 2626 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2627 return 0; 2628 2629 if (ports[port_id].port_status != RTE_PORT_STARTED) 2630 return 0; 2631 2632 return 1; 2633 } 2634 2635 /* Configure the Rx with optional split. */ 2636 int 2637 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2638 uint16_t nb_rx_desc, unsigned int socket_id, 2639 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2640 { 2641 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2642 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2643 struct rte_mempool *mpx; 2644 unsigned int i, mp_n; 2645 uint32_t prev_hdrs = 0; 2646 int ret; 2647 2648 2649 if ((rx_pkt_nb_segs > 1) && 2650 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2651 /* multi-segment configuration */ 2652 for (i = 0; i < rx_pkt_nb_segs; i++) { 2653 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2654 /* 2655 * Use last valid pool for the segments with number 2656 * exceeding the pool index. 2657 */ 2658 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2659 mpx = mbuf_pool_find(socket_id, mp_n); 2660 /* Handle zero as mbuf data buffer size. */ 2661 rx_seg->offset = i < rx_pkt_nb_offs ? 2662 rx_pkt_seg_offsets[i] : 0; 2663 rx_seg->mp = mpx ? mpx : mp; 2664 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2665 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2666 prev_hdrs |= rx_seg->proto_hdr; 2667 } else { 2668 rx_seg->length = rx_pkt_seg_lengths[i] ? 2669 rx_pkt_seg_lengths[i] : 2670 mbuf_data_size[mp_n]; 2671 } 2672 } 2673 rx_conf->rx_nseg = rx_pkt_nb_segs; 2674 rx_conf->rx_seg = rx_useg; 2675 rx_conf->rx_mempools = NULL; 2676 rx_conf->rx_nmempool = 0; 2677 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2678 socket_id, rx_conf, NULL); 2679 rx_conf->rx_seg = NULL; 2680 rx_conf->rx_nseg = 0; 2681 } else if (multi_rx_mempool == 1) { 2682 /* multi-pool configuration */ 2683 struct rte_eth_dev_info dev_info; 2684 2685 if (mbuf_data_size_n <= 1) { 2686 fprintf(stderr, "Invalid number of mempools %u\n", 2687 mbuf_data_size_n); 2688 return -EINVAL; 2689 } 2690 ret = rte_eth_dev_info_get(port_id, &dev_info); 2691 if (ret != 0) 2692 return ret; 2693 if (dev_info.max_rx_mempools == 0) { 2694 fprintf(stderr, 2695 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2696 port_id); 2697 return -ENOTSUP; 2698 } 2699 for (i = 0; i < mbuf_data_size_n; i++) { 2700 mpx = mbuf_pool_find(socket_id, i); 2701 rx_mempool[i] = mpx ? mpx : mp; 2702 } 2703 rx_conf->rx_mempools = rx_mempool; 2704 rx_conf->rx_nmempool = mbuf_data_size_n; 2705 rx_conf->rx_seg = NULL; 2706 rx_conf->rx_nseg = 0; 2707 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2708 socket_id, rx_conf, NULL); 2709 rx_conf->rx_mempools = NULL; 2710 rx_conf->rx_nmempool = 0; 2711 } else { 2712 /* Single pool/segment configuration */ 2713 rx_conf->rx_seg = NULL; 2714 rx_conf->rx_nseg = 0; 2715 rx_conf->rx_mempools = NULL; 2716 rx_conf->rx_nmempool = 0; 2717 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2718 socket_id, rx_conf, mp); 2719 } 2720 2721 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2722 RTE_ETH_QUEUE_STATE_STOPPED : 2723 RTE_ETH_QUEUE_STATE_STARTED; 2724 return ret; 2725 } 2726 2727 static int 2728 alloc_xstats_display_info(portid_t pi) 2729 { 2730 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2731 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2732 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2733 2734 if (xstats_display_num == 0) 2735 return 0; 2736 2737 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2738 if (*ids_supp == NULL) 2739 goto fail_ids_supp; 2740 2741 *prev_values = calloc(xstats_display_num, 2742 sizeof(**prev_values)); 2743 if (*prev_values == NULL) 2744 goto fail_prev_values; 2745 2746 *curr_values = calloc(xstats_display_num, 2747 sizeof(**curr_values)); 2748 if (*curr_values == NULL) 2749 goto fail_curr_values; 2750 2751 ports[pi].xstats_info.allocated = true; 2752 2753 return 0; 2754 2755 fail_curr_values: 2756 free(*prev_values); 2757 fail_prev_values: 2758 free(*ids_supp); 2759 fail_ids_supp: 2760 return -ENOMEM; 2761 } 2762 2763 static void 2764 free_xstats_display_info(portid_t pi) 2765 { 2766 if (!ports[pi].xstats_info.allocated) 2767 return; 2768 free(ports[pi].xstats_info.ids_supp); 2769 free(ports[pi].xstats_info.prev_values); 2770 free(ports[pi].xstats_info.curr_values); 2771 ports[pi].xstats_info.allocated = false; 2772 } 2773 2774 /** Fill helper structures for specified port to show extended statistics. */ 2775 static void 2776 fill_xstats_display_info_for_port(portid_t pi) 2777 { 2778 unsigned int stat, stat_supp; 2779 const char *xstat_name; 2780 struct rte_port *port; 2781 uint64_t *ids_supp; 2782 int rc; 2783 2784 if (xstats_display_num == 0) 2785 return; 2786 2787 if (pi == (portid_t)RTE_PORT_ALL) { 2788 fill_xstats_display_info(); 2789 return; 2790 } 2791 2792 port = &ports[pi]; 2793 if (port->port_status != RTE_PORT_STARTED) 2794 return; 2795 2796 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2797 rte_exit(EXIT_FAILURE, 2798 "Failed to allocate xstats display memory\n"); 2799 2800 ids_supp = port->xstats_info.ids_supp; 2801 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2802 xstat_name = xstats_display[stat].name; 2803 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2804 ids_supp + stat_supp); 2805 if (rc != 0) { 2806 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2807 xstat_name, pi, stat); 2808 continue; 2809 } 2810 stat_supp++; 2811 } 2812 2813 port->xstats_info.ids_supp_sz = stat_supp; 2814 } 2815 2816 /** Fill helper structures for all ports to show extended statistics. */ 2817 static void 2818 fill_xstats_display_info(void) 2819 { 2820 portid_t pi; 2821 2822 if (xstats_display_num == 0) 2823 return; 2824 2825 RTE_ETH_FOREACH_DEV(pi) 2826 fill_xstats_display_info_for_port(pi); 2827 } 2828 2829 /* 2830 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2831 * device in dev_info is zero when no member is added. And its capability 2832 * will be updated when add a new member device. So adding a member device need 2833 * to update the port configurations of bonding device. 2834 */ 2835 static void 2836 update_bonding_port_dev_conf(portid_t bond_pid) 2837 { 2838 #ifdef RTE_NET_BOND 2839 struct rte_port *port = &ports[bond_pid]; 2840 uint16_t i; 2841 int ret; 2842 2843 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2844 if (ret != 0) { 2845 fprintf(stderr, "Failed to get dev info for port = %u\n", 2846 bond_pid); 2847 return; 2848 } 2849 2850 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2851 port->dev_conf.txmode.offloads |= 2852 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2853 /* Apply Tx offloads configuration */ 2854 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2855 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2856 2857 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 2858 port->dev_info.flow_type_rss_offloads; 2859 #else 2860 RTE_SET_USED(bond_pid); 2861 #endif 2862 } 2863 2864 int 2865 start_port(portid_t pid) 2866 { 2867 int diag; 2868 portid_t pi; 2869 portid_t p_pi = RTE_MAX_ETHPORTS; 2870 portid_t pl[RTE_MAX_ETHPORTS]; 2871 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2872 uint16_t cnt_pi = 0; 2873 uint16_t cfg_pi = 0; 2874 queueid_t qi; 2875 struct rte_port *port; 2876 struct rte_eth_hairpin_cap cap; 2877 bool at_least_one_port_exist = false; 2878 bool all_ports_already_started = true; 2879 bool at_least_one_port_successfully_started = false; 2880 2881 if (port_id_is_invalid(pid, ENABLED_WARN)) 2882 return 0; 2883 2884 RTE_ETH_FOREACH_DEV(pi) { 2885 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2886 continue; 2887 2888 if (port_is_bonding_member(pi)) { 2889 fprintf(stderr, 2890 "Please remove port %d from bonding device.\n", 2891 pi); 2892 continue; 2893 } 2894 2895 at_least_one_port_exist = true; 2896 2897 port = &ports[pi]; 2898 if (port->port_status == RTE_PORT_STOPPED) { 2899 port->port_status = RTE_PORT_HANDLING; 2900 all_ports_already_started = false; 2901 } else { 2902 fprintf(stderr, "Port %d is now not stopped\n", pi); 2903 continue; 2904 } 2905 2906 if (port->need_reconfig > 0) { 2907 struct rte_eth_conf dev_conf; 2908 int k; 2909 2910 port->need_reconfig = 0; 2911 2912 if (flow_isolate_all) { 2913 int ret = port_flow_isolate(pi, 1); 2914 if (ret) { 2915 fprintf(stderr, 2916 "Failed to apply isolated mode on port %d\n", 2917 pi); 2918 return -1; 2919 } 2920 } 2921 configure_rxtx_dump_callbacks(0); 2922 printf("Configuring Port %d (socket %u)\n", pi, 2923 port->socket_id); 2924 if (nb_hairpinq > 0 && 2925 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2926 fprintf(stderr, 2927 "Port %d doesn't support hairpin queues\n", 2928 pi); 2929 return -1; 2930 } 2931 2932 if (port->bond_flag == 1 && port->update_conf == 1) { 2933 update_bonding_port_dev_conf(pi); 2934 port->update_conf = 0; 2935 } 2936 2937 /* configure port */ 2938 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2939 nb_txq + nb_hairpinq, 2940 &(port->dev_conf)); 2941 if (diag != 0) { 2942 if (port->port_status == RTE_PORT_HANDLING) 2943 port->port_status = RTE_PORT_STOPPED; 2944 else 2945 fprintf(stderr, 2946 "Port %d can not be set back to stopped\n", 2947 pi); 2948 fprintf(stderr, "Fail to configure port %d\n", 2949 pi); 2950 /* try to reconfigure port next time */ 2951 port->need_reconfig = 1; 2952 return -1; 2953 } 2954 /* get device configuration*/ 2955 if (0 != 2956 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2957 fprintf(stderr, 2958 "port %d can not get device configuration\n", 2959 pi); 2960 return -1; 2961 } 2962 /* Apply Rx offloads configuration */ 2963 if (dev_conf.rxmode.offloads != 2964 port->dev_conf.rxmode.offloads) { 2965 port->dev_conf.rxmode.offloads |= 2966 dev_conf.rxmode.offloads; 2967 for (k = 0; 2968 k < port->dev_info.max_rx_queues; 2969 k++) 2970 port->rxq[k].conf.offloads |= 2971 dev_conf.rxmode.offloads; 2972 } 2973 /* Apply Tx offloads configuration */ 2974 if (dev_conf.txmode.offloads != 2975 port->dev_conf.txmode.offloads) { 2976 port->dev_conf.txmode.offloads |= 2977 dev_conf.txmode.offloads; 2978 for (k = 0; 2979 k < port->dev_info.max_tx_queues; 2980 k++) 2981 port->txq[k].conf.offloads |= 2982 dev_conf.txmode.offloads; 2983 } 2984 } 2985 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2986 port->need_reconfig_queues = 0; 2987 /* setup tx queues */ 2988 for (qi = 0; qi < nb_txq; qi++) { 2989 struct rte_eth_txconf *conf = 2990 &port->txq[qi].conf; 2991 2992 if ((numa_support) && 2993 (txring_numa[pi] != NUMA_NO_CONFIG)) 2994 diag = rte_eth_tx_queue_setup(pi, qi, 2995 port->nb_tx_desc[qi], 2996 txring_numa[pi], 2997 &(port->txq[qi].conf)); 2998 else 2999 diag = rte_eth_tx_queue_setup(pi, qi, 3000 port->nb_tx_desc[qi], 3001 port->socket_id, 3002 &(port->txq[qi].conf)); 3003 3004 if (diag == 0) { 3005 port->txq[qi].state = 3006 conf->tx_deferred_start ? 3007 RTE_ETH_QUEUE_STATE_STOPPED : 3008 RTE_ETH_QUEUE_STATE_STARTED; 3009 continue; 3010 } 3011 3012 /* Fail to setup tx queue, return */ 3013 if (port->port_status == RTE_PORT_HANDLING) 3014 port->port_status = RTE_PORT_STOPPED; 3015 else 3016 fprintf(stderr, 3017 "Port %d can not be set back to stopped\n", 3018 pi); 3019 fprintf(stderr, 3020 "Fail to configure port %d tx queues\n", 3021 pi); 3022 /* try to reconfigure queues next time */ 3023 port->need_reconfig_queues = 1; 3024 return -1; 3025 } 3026 for (qi = 0; qi < nb_rxq; qi++) { 3027 /* setup rx queues */ 3028 if ((numa_support) && 3029 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3030 struct rte_mempool * mp = 3031 mbuf_pool_find 3032 (rxring_numa[pi], 0); 3033 if (mp == NULL) { 3034 fprintf(stderr, 3035 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3036 rxring_numa[pi]); 3037 return -1; 3038 } 3039 3040 diag = rx_queue_setup(pi, qi, 3041 port->nb_rx_desc[qi], 3042 rxring_numa[pi], 3043 &(port->rxq[qi].conf), 3044 mp); 3045 } else { 3046 struct rte_mempool *mp = 3047 mbuf_pool_find 3048 (port->socket_id, 0); 3049 if (mp == NULL) { 3050 fprintf(stderr, 3051 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3052 port->socket_id); 3053 return -1; 3054 } 3055 diag = rx_queue_setup(pi, qi, 3056 port->nb_rx_desc[qi], 3057 port->socket_id, 3058 &(port->rxq[qi].conf), 3059 mp); 3060 } 3061 if (diag == 0) 3062 continue; 3063 3064 /* Fail to setup rx queue, return */ 3065 if (port->port_status == RTE_PORT_HANDLING) 3066 port->port_status = RTE_PORT_STOPPED; 3067 else 3068 fprintf(stderr, 3069 "Port %d can not be set back to stopped\n", 3070 pi); 3071 fprintf(stderr, 3072 "Fail to configure port %d rx queues\n", 3073 pi); 3074 /* try to reconfigure queues next time */ 3075 port->need_reconfig_queues = 1; 3076 return -1; 3077 } 3078 /* setup hairpin queues */ 3079 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3080 return -1; 3081 } 3082 configure_rxtx_dump_callbacks(verbose_level); 3083 if (clear_ptypes) { 3084 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3085 NULL, 0); 3086 if (diag < 0) 3087 fprintf(stderr, 3088 "Port %d: Failed to disable Ptype parsing\n", 3089 pi); 3090 } 3091 3092 p_pi = pi; 3093 cnt_pi++; 3094 3095 /* start port */ 3096 diag = eth_dev_start_mp(pi); 3097 if (diag < 0) { 3098 fprintf(stderr, "Fail to start port %d: %s\n", 3099 pi, rte_strerror(-diag)); 3100 3101 /* Fail to setup rx queue, return */ 3102 if (port->port_status == RTE_PORT_HANDLING) 3103 port->port_status = RTE_PORT_STOPPED; 3104 else 3105 fprintf(stderr, 3106 "Port %d can not be set back to stopped\n", 3107 pi); 3108 continue; 3109 } 3110 3111 if (port->port_status == RTE_PORT_HANDLING) 3112 port->port_status = RTE_PORT_STARTED; 3113 else 3114 fprintf(stderr, "Port %d can not be set into started\n", 3115 pi); 3116 3117 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3118 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3119 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3120 3121 at_least_one_port_successfully_started = true; 3122 3123 pl[cfg_pi++] = pi; 3124 } 3125 3126 update_queue_state(pi); 3127 3128 if (at_least_one_port_successfully_started && !no_link_check) 3129 check_all_ports_link_status(RTE_PORT_ALL); 3130 else if (at_least_one_port_exist & all_ports_already_started) 3131 fprintf(stderr, "Please stop the ports first\n"); 3132 3133 if (hairpin_mode & 0xf) { 3134 diag = hairpin_bind(cfg_pi, pl, peer_pl); 3135 if (diag < 0) 3136 return -1; 3137 } 3138 3139 fill_xstats_display_info_for_port(pid); 3140 3141 printf("Done\n"); 3142 return 0; 3143 } 3144 3145 void 3146 stop_port(portid_t pid) 3147 { 3148 portid_t pi; 3149 struct rte_port *port; 3150 int need_check_link_status = 0; 3151 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3152 int peer_pi; 3153 int ret; 3154 3155 if (port_id_is_invalid(pid, ENABLED_WARN)) 3156 return; 3157 3158 printf("Stopping ports...\n"); 3159 3160 RTE_ETH_FOREACH_DEV(pi) { 3161 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3162 continue; 3163 3164 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3165 fprintf(stderr, 3166 "Please remove port %d from forwarding configuration.\n", 3167 pi); 3168 continue; 3169 } 3170 3171 if (port_is_bonding_member(pi)) { 3172 fprintf(stderr, 3173 "Please remove port %d from bonding device.\n", 3174 pi); 3175 continue; 3176 } 3177 3178 port = &ports[pi]; 3179 if (port->port_status == RTE_PORT_STARTED) 3180 port->port_status = RTE_PORT_HANDLING; 3181 else 3182 continue; 3183 3184 if (hairpin_mode & 0xf) { 3185 int j; 3186 3187 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3188 /* unbind all peer Tx from current Rx */ 3189 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3190 RTE_MAX_ETHPORTS, 0); 3191 if (peer_pi < 0) 3192 continue; 3193 for (j = 0; j < peer_pi; j++) { 3194 if (!port_is_started(peer_pl[j])) 3195 continue; 3196 rte_eth_hairpin_unbind(peer_pl[j], pi); 3197 } 3198 } 3199 3200 if (port->flow_list && !no_flow_flush) 3201 port_flow_flush(pi); 3202 3203 ret = eth_dev_stop_mp(pi); 3204 if (ret != 0) { 3205 TESTPMD_LOG(ERR, 3206 "rte_eth_dev_stop failed for port %u\n", pi); 3207 /* Allow to retry stopping the port. */ 3208 port->port_status = RTE_PORT_STARTED; 3209 continue; 3210 } 3211 3212 if (port->port_status == RTE_PORT_HANDLING) 3213 port->port_status = RTE_PORT_STOPPED; 3214 else 3215 fprintf(stderr, "Port %d can not be set into stopped\n", 3216 pi); 3217 need_check_link_status = 1; 3218 } 3219 if (need_check_link_status && !no_link_check) 3220 check_all_ports_link_status(RTE_PORT_ALL); 3221 3222 printf("Done\n"); 3223 } 3224 3225 static void 3226 remove_invalid_ports_in(portid_t *array, portid_t *total) 3227 { 3228 portid_t i; 3229 portid_t new_total = 0; 3230 3231 for (i = 0; i < *total; i++) 3232 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3233 array[new_total] = array[i]; 3234 new_total++; 3235 } 3236 *total = new_total; 3237 } 3238 3239 static void 3240 remove_invalid_ports(void) 3241 { 3242 remove_invalid_ports_in(ports_ids, &nb_ports); 3243 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3244 nb_cfg_ports = nb_fwd_ports; 3245 } 3246 3247 static void 3248 flush_port_owned_resources(portid_t pi) 3249 { 3250 mcast_addr_pool_destroy(pi); 3251 port_flow_flush(pi); 3252 port_flow_template_table_flush(pi); 3253 port_flow_pattern_template_flush(pi); 3254 port_flow_actions_template_flush(pi); 3255 port_flex_item_flush(pi); 3256 port_action_handle_flush(pi); 3257 } 3258 3259 static void 3260 clear_bonding_member_device(portid_t *member_pids, uint16_t num_members) 3261 { 3262 struct rte_port *port; 3263 portid_t member_pid; 3264 uint16_t i; 3265 3266 for (i = 0; i < num_members; i++) { 3267 member_pid = member_pids[i]; 3268 if (port_is_started(member_pid) == 1) { 3269 if (rte_eth_dev_stop(member_pid) != 0) 3270 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3271 member_pid); 3272 3273 port = &ports[member_pid]; 3274 port->port_status = RTE_PORT_STOPPED; 3275 } 3276 3277 clear_port_member_flag(member_pid); 3278 3279 /* Close member device when testpmd quit or is killed. */ 3280 if (cl_quit == 1 || f_quit == 1) 3281 rte_eth_dev_close(member_pid); 3282 } 3283 } 3284 3285 void 3286 close_port(portid_t pid) 3287 { 3288 portid_t pi; 3289 struct rte_port *port; 3290 portid_t member_pids[RTE_MAX_ETHPORTS]; 3291 int num_members = 0; 3292 3293 if (port_id_is_invalid(pid, ENABLED_WARN)) 3294 return; 3295 3296 printf("Closing ports...\n"); 3297 3298 RTE_ETH_FOREACH_DEV(pi) { 3299 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3300 continue; 3301 3302 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3303 fprintf(stderr, 3304 "Please remove port %d from forwarding configuration.\n", 3305 pi); 3306 continue; 3307 } 3308 3309 if (port_is_bonding_member(pi)) { 3310 fprintf(stderr, 3311 "Please remove port %d from bonding device.\n", 3312 pi); 3313 continue; 3314 } 3315 3316 port = &ports[pi]; 3317 if (port->port_status == RTE_PORT_CLOSED) { 3318 fprintf(stderr, "Port %d is already closed\n", pi); 3319 continue; 3320 } 3321 3322 if (is_proc_primary()) { 3323 flush_port_owned_resources(pi); 3324 #ifdef RTE_NET_BOND 3325 if (port->bond_flag == 1) 3326 num_members = rte_eth_bond_members_get(pi, 3327 member_pids, RTE_MAX_ETHPORTS); 3328 #endif 3329 rte_eth_dev_close(pi); 3330 /* 3331 * If this port is bonding device, all members under the 3332 * device need to be removed or closed. 3333 */ 3334 if (port->bond_flag == 1 && num_members > 0) 3335 clear_bonding_member_device(member_pids, 3336 num_members); 3337 } 3338 3339 free_xstats_display_info(pi); 3340 } 3341 3342 remove_invalid_ports(); 3343 printf("Done\n"); 3344 } 3345 3346 void 3347 reset_port(portid_t pid) 3348 { 3349 int diag; 3350 portid_t pi; 3351 struct rte_port *port; 3352 3353 if (port_id_is_invalid(pid, ENABLED_WARN)) 3354 return; 3355 3356 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3357 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3358 fprintf(stderr, 3359 "Can not reset port(s), please stop port(s) first.\n"); 3360 return; 3361 } 3362 3363 printf("Resetting ports...\n"); 3364 3365 RTE_ETH_FOREACH_DEV(pi) { 3366 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3367 continue; 3368 3369 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3370 fprintf(stderr, 3371 "Please remove port %d from forwarding configuration.\n", 3372 pi); 3373 continue; 3374 } 3375 3376 if (port_is_bonding_member(pi)) { 3377 fprintf(stderr, 3378 "Please remove port %d from bonding device.\n", 3379 pi); 3380 continue; 3381 } 3382 3383 if (is_proc_primary()) { 3384 diag = rte_eth_dev_reset(pi); 3385 if (diag == 0) { 3386 port = &ports[pi]; 3387 port->need_reconfig = 1; 3388 port->need_reconfig_queues = 1; 3389 } else { 3390 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3391 pi, diag); 3392 } 3393 } 3394 } 3395 3396 printf("Done\n"); 3397 } 3398 3399 void 3400 attach_port(char *identifier) 3401 { 3402 portid_t pi; 3403 struct rte_dev_iterator iterator; 3404 3405 printf("Attaching a new port...\n"); 3406 3407 if (identifier == NULL) { 3408 fprintf(stderr, "Invalid parameters are specified\n"); 3409 return; 3410 } 3411 3412 if (rte_dev_probe(identifier) < 0) { 3413 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3414 return; 3415 } 3416 3417 /* first attach mode: event */ 3418 if (setup_on_probe_event) { 3419 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3420 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3421 if (ports[pi].port_status == RTE_PORT_HANDLING && 3422 ports[pi].need_setup != 0) 3423 setup_attached_port(pi); 3424 return; 3425 } 3426 3427 /* second attach mode: iterator */ 3428 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3429 /* setup ports matching the devargs used for probing */ 3430 if (port_is_forwarding(pi)) 3431 continue; /* port was already attached before */ 3432 setup_attached_port(pi); 3433 } 3434 } 3435 3436 static void 3437 setup_attached_port(portid_t pi) 3438 { 3439 unsigned int socket_id; 3440 int ret; 3441 3442 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3443 /* if socket_id is invalid, set to the first available socket. */ 3444 if (check_socket_id(socket_id) < 0) 3445 socket_id = socket_ids[0]; 3446 reconfig(pi, socket_id); 3447 ret = rte_eth_promiscuous_enable(pi); 3448 if (ret != 0) 3449 fprintf(stderr, 3450 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3451 pi, rte_strerror(-ret)); 3452 3453 ports_ids[nb_ports++] = pi; 3454 fwd_ports_ids[nb_fwd_ports++] = pi; 3455 nb_cfg_ports = nb_fwd_ports; 3456 ports[pi].need_setup = 0; 3457 ports[pi].port_status = RTE_PORT_STOPPED; 3458 3459 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3460 printf("Done\n"); 3461 } 3462 3463 static void 3464 detach_device(struct rte_device *dev) 3465 { 3466 portid_t sibling; 3467 3468 if (dev == NULL) { 3469 fprintf(stderr, "Device already removed\n"); 3470 return; 3471 } 3472 3473 printf("Removing a device...\n"); 3474 3475 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3476 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3477 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3478 fprintf(stderr, "Port %u not stopped\n", 3479 sibling); 3480 return; 3481 } 3482 flush_port_owned_resources(sibling); 3483 } 3484 } 3485 3486 if (rte_dev_remove(dev) < 0) { 3487 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3488 return; 3489 } 3490 remove_invalid_ports(); 3491 3492 printf("Device is detached\n"); 3493 printf("Now total ports is %d\n", nb_ports); 3494 printf("Done\n"); 3495 return; 3496 } 3497 3498 void 3499 detach_port_device(portid_t port_id) 3500 { 3501 int ret; 3502 struct rte_eth_dev_info dev_info; 3503 3504 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3505 return; 3506 3507 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3508 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3509 fprintf(stderr, "Port not stopped\n"); 3510 return; 3511 } 3512 fprintf(stderr, "Port was not closed\n"); 3513 } 3514 3515 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3516 if (ret != 0) { 3517 TESTPMD_LOG(ERR, 3518 "Failed to get device info for port %d, not detaching\n", 3519 port_id); 3520 return; 3521 } 3522 detach_device(dev_info.device); 3523 } 3524 3525 void 3526 detach_devargs(char *identifier) 3527 { 3528 struct rte_dev_iterator iterator; 3529 struct rte_devargs da; 3530 portid_t port_id; 3531 3532 printf("Removing a device...\n"); 3533 3534 memset(&da, 0, sizeof(da)); 3535 if (rte_devargs_parsef(&da, "%s", identifier)) { 3536 fprintf(stderr, "cannot parse identifier\n"); 3537 return; 3538 } 3539 3540 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3541 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3542 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3543 fprintf(stderr, "Port %u not stopped\n", 3544 port_id); 3545 rte_eth_iterator_cleanup(&iterator); 3546 rte_devargs_reset(&da); 3547 return; 3548 } 3549 flush_port_owned_resources(port_id); 3550 } 3551 } 3552 3553 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3554 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3555 da.name, rte_bus_name(da.bus)); 3556 rte_devargs_reset(&da); 3557 return; 3558 } 3559 3560 remove_invalid_ports(); 3561 3562 printf("Device %s is detached\n", identifier); 3563 printf("Now total ports is %d\n", nb_ports); 3564 printf("Done\n"); 3565 rte_devargs_reset(&da); 3566 } 3567 3568 void 3569 pmd_test_exit(void) 3570 { 3571 portid_t pt_id; 3572 unsigned int i; 3573 int ret; 3574 3575 if (test_done == 0) 3576 stop_packet_forwarding(); 3577 3578 #ifndef RTE_EXEC_ENV_WINDOWS 3579 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3580 if (mempools[i]) { 3581 if (mp_alloc_type == MP_ALLOC_ANON) 3582 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3583 NULL); 3584 } 3585 } 3586 #endif 3587 if (ports != NULL) { 3588 no_link_check = 1; 3589 RTE_ETH_FOREACH_DEV(pt_id) { 3590 printf("\nStopping port %d...\n", pt_id); 3591 fflush(stdout); 3592 stop_port(pt_id); 3593 } 3594 RTE_ETH_FOREACH_DEV(pt_id) { 3595 printf("\nShutting down port %d...\n", pt_id); 3596 fflush(stdout); 3597 close_port(pt_id); 3598 } 3599 } 3600 3601 if (hot_plug) { 3602 ret = rte_dev_event_monitor_stop(); 3603 if (ret) { 3604 TESTPMD_LOG(ERR, "fail to stop device event monitor."); 3605 return; 3606 } 3607 3608 ret = rte_dev_event_callback_unregister(NULL, 3609 dev_event_callback, NULL); 3610 if (ret < 0) { 3611 TESTPMD_LOG(ERR, "fail to unregister device event callback.\n"); 3612 return; 3613 } 3614 3615 ret = rte_dev_hotplug_handle_disable(); 3616 if (ret) { 3617 TESTPMD_LOG(ERR, "fail to disable hotplug handling.\n"); 3618 return; 3619 } 3620 } 3621 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3622 if (mempools[i]) 3623 mempool_free_mp(mempools[i]); 3624 } 3625 free(xstats_display); 3626 3627 printf("\nBye...\n"); 3628 } 3629 3630 typedef void (*cmd_func_t)(void); 3631 struct pmd_test_command { 3632 const char *cmd_name; 3633 cmd_func_t cmd_func; 3634 }; 3635 3636 /* Check the link status of all ports in up to 9s, and print them finally */ 3637 static void 3638 check_all_ports_link_status(uint32_t port_mask) 3639 { 3640 #define CHECK_INTERVAL 100 /* 100ms */ 3641 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3642 portid_t portid; 3643 uint8_t count, all_ports_up, print_flag = 0; 3644 struct rte_eth_link link; 3645 int ret; 3646 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3647 3648 printf("Checking link statuses...\n"); 3649 fflush(stdout); 3650 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3651 all_ports_up = 1; 3652 RTE_ETH_FOREACH_DEV(portid) { 3653 if ((port_mask & (1 << portid)) == 0) 3654 continue; 3655 memset(&link, 0, sizeof(link)); 3656 ret = rte_eth_link_get_nowait(portid, &link); 3657 if (ret < 0) { 3658 all_ports_up = 0; 3659 if (print_flag == 1) 3660 fprintf(stderr, 3661 "Port %u link get failed: %s\n", 3662 portid, rte_strerror(-ret)); 3663 continue; 3664 } 3665 /* print link status if flag set */ 3666 if (print_flag == 1) { 3667 rte_eth_link_to_str(link_status, 3668 sizeof(link_status), &link); 3669 printf("Port %d %s\n", portid, link_status); 3670 continue; 3671 } 3672 /* clear all_ports_up flag if any link down */ 3673 if (link.link_status == RTE_ETH_LINK_DOWN) { 3674 all_ports_up = 0; 3675 break; 3676 } 3677 } 3678 /* after finally printing all link status, get out */ 3679 if (print_flag == 1) 3680 break; 3681 3682 if (all_ports_up == 0) { 3683 fflush(stdout); 3684 rte_delay_ms(CHECK_INTERVAL); 3685 } 3686 3687 /* set the print_flag if all ports up or timeout */ 3688 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3689 print_flag = 1; 3690 } 3691 3692 if (lsc_interrupt) 3693 break; 3694 } 3695 } 3696 3697 static void 3698 rmv_port_callback(void *arg) 3699 { 3700 int need_to_start = 0; 3701 int org_no_link_check = no_link_check; 3702 portid_t port_id = (intptr_t)arg; 3703 struct rte_eth_dev_info dev_info; 3704 int ret; 3705 3706 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3707 3708 if (!test_done && port_is_forwarding(port_id)) { 3709 need_to_start = 1; 3710 stop_packet_forwarding(); 3711 } 3712 no_link_check = 1; 3713 stop_port(port_id); 3714 no_link_check = org_no_link_check; 3715 3716 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3717 if (ret != 0) 3718 TESTPMD_LOG(ERR, 3719 "Failed to get device info for port %d, not detaching\n", 3720 port_id); 3721 else { 3722 struct rte_device *device = dev_info.device; 3723 close_port(port_id); 3724 detach_device(device); /* might be already removed or have more ports */ 3725 } 3726 if (need_to_start) 3727 start_packet_forwarding(0); 3728 } 3729 3730 /* This function is used by the interrupt thread */ 3731 static int 3732 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3733 void *ret_param) 3734 { 3735 RTE_SET_USED(param); 3736 RTE_SET_USED(ret_param); 3737 3738 if (type >= RTE_ETH_EVENT_MAX) { 3739 fprintf(stderr, 3740 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3741 port_id, __func__, type); 3742 fflush(stderr); 3743 } else if (event_print_mask & (UINT32_C(1) << type)) { 3744 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3745 eth_event_desc[type]); 3746 fflush(stdout); 3747 } 3748 3749 switch (type) { 3750 case RTE_ETH_EVENT_NEW: 3751 ports[port_id].need_setup = 1; 3752 ports[port_id].port_status = RTE_PORT_HANDLING; 3753 break; 3754 case RTE_ETH_EVENT_INTR_RMV: 3755 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3756 break; 3757 if (rte_eal_alarm_set(100000, 3758 rmv_port_callback, (void *)(intptr_t)port_id)) 3759 fprintf(stderr, 3760 "Could not set up deferred device removal\n"); 3761 break; 3762 case RTE_ETH_EVENT_DESTROY: 3763 ports[port_id].port_status = RTE_PORT_CLOSED; 3764 printf("Port %u is closed\n", port_id); 3765 break; 3766 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3767 uint16_t rxq_id; 3768 int ret; 3769 3770 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3771 for (rxq_id = 0; ; rxq_id++) { 3772 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3773 NULL); 3774 if (ret <= 0) 3775 break; 3776 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3777 port_id, rxq_id); 3778 3779 #ifdef RTE_NET_MLX5 3780 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3781 #endif 3782 } 3783 break; 3784 } 3785 default: 3786 break; 3787 } 3788 return 0; 3789 } 3790 3791 static int 3792 register_eth_event_callback(void) 3793 { 3794 int ret; 3795 enum rte_eth_event_type event; 3796 3797 for (event = RTE_ETH_EVENT_UNKNOWN; 3798 event < RTE_ETH_EVENT_MAX; event++) { 3799 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3800 event, 3801 eth_event_callback, 3802 NULL); 3803 if (ret != 0) { 3804 TESTPMD_LOG(ERR, "Failed to register callback for " 3805 "%s event\n", eth_event_desc[event]); 3806 return -1; 3807 } 3808 } 3809 3810 return 0; 3811 } 3812 3813 static int 3814 unregister_eth_event_callback(void) 3815 { 3816 int ret; 3817 enum rte_eth_event_type event; 3818 3819 for (event = RTE_ETH_EVENT_UNKNOWN; 3820 event < RTE_ETH_EVENT_MAX; event++) { 3821 ret = rte_eth_dev_callback_unregister(RTE_ETH_ALL, 3822 event, 3823 eth_event_callback, 3824 NULL); 3825 if (ret != 0) { 3826 TESTPMD_LOG(ERR, "Failed to unregister callback for " 3827 "%s event\n", eth_event_desc[event]); 3828 return -1; 3829 } 3830 } 3831 3832 return 0; 3833 } 3834 3835 /* This function is used by the interrupt thread */ 3836 static void 3837 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3838 __rte_unused void *arg) 3839 { 3840 uint16_t port_id; 3841 int ret; 3842 3843 switch (type) { 3844 case RTE_DEV_EVENT_REMOVE: 3845 TESTPMD_LOG(INFO, "The device: %s has been removed!\n", device_name); 3846 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3847 if (ret) { 3848 TESTPMD_LOG(ERR, 3849 "Can not get port for device %s!\n", device_name); 3850 return; 3851 } 3852 /* 3853 * Because the user's callback is invoked in eal interrupt 3854 * callback, the interrupt callback need to be finished before 3855 * it can be unregistered when detaching device. So finish 3856 * callback soon and use a deferred removal to detach device 3857 * is need. It is a workaround, once the device detaching be 3858 * moved into the eal in the future, the deferred removal could 3859 * be deleted. 3860 */ 3861 if (rte_eal_alarm_set(100000, 3862 rmv_port_callback, (void *)(intptr_t)port_id)) 3863 TESTPMD_LOG(ERR, "Could not set up deferred device removal\n"); 3864 break; 3865 3866 case RTE_DEV_EVENT_ADD: 3867 TESTPMD_LOG(INFO, "The device: %s has been added!\n", device_name); 3868 /* TODO: After finish kernel driver binding, 3869 * begin to attach port. 3870 */ 3871 break; 3872 3873 default: 3874 if (type >= RTE_DEV_EVENT_MAX) 3875 TESTPMD_LOG(ERR, "%s called upon invalid event %d\n", 3876 __func__, type); 3877 break; 3878 } 3879 } 3880 3881 static void 3882 rxtx_port_config(portid_t pid) 3883 { 3884 uint16_t qid; 3885 uint64_t offloads; 3886 struct rte_port *port = &ports[pid]; 3887 3888 for (qid = 0; qid < nb_rxq; qid++) { 3889 offloads = port->rxq[qid].conf.offloads; 3890 port->rxq[qid].conf = port->dev_info.default_rxconf; 3891 3892 if (rxq_share > 0 && 3893 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3894 /* Non-zero share group to enable RxQ share. */ 3895 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3896 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3897 } 3898 3899 if (offloads != 0) 3900 port->rxq[qid].conf.offloads = offloads; 3901 3902 /* Check if any Rx parameters have been passed */ 3903 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3904 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3905 3906 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3907 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3908 3909 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3910 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3911 3912 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3913 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3914 3915 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3916 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3917 3918 port->nb_rx_desc[qid] = nb_rxd; 3919 } 3920 3921 for (qid = 0; qid < nb_txq; qid++) { 3922 offloads = port->txq[qid].conf.offloads; 3923 port->txq[qid].conf = port->dev_info.default_txconf; 3924 if (offloads != 0) 3925 port->txq[qid].conf.offloads = offloads; 3926 3927 /* Check if any Tx parameters have been passed */ 3928 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3929 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3930 3931 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3932 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3933 3934 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3935 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3936 3937 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3938 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3939 3940 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3941 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3942 3943 port->nb_tx_desc[qid] = nb_txd; 3944 } 3945 } 3946 3947 /* 3948 * Helper function to set MTU from frame size 3949 * 3950 * port->dev_info should be set before calling this function. 3951 * 3952 * return 0 on success, negative on error 3953 */ 3954 int 3955 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3956 { 3957 struct rte_port *port = &ports[portid]; 3958 uint32_t eth_overhead; 3959 uint16_t mtu, new_mtu; 3960 3961 eth_overhead = get_eth_overhead(&port->dev_info); 3962 3963 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3964 printf("Failed to get MTU for port %u\n", portid); 3965 return -1; 3966 } 3967 3968 new_mtu = max_rx_pktlen - eth_overhead; 3969 3970 if (mtu == new_mtu) 3971 return 0; 3972 3973 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3974 fprintf(stderr, 3975 "Failed to set MTU to %u for port %u\n", 3976 new_mtu, portid); 3977 return -1; 3978 } 3979 3980 port->dev_conf.rxmode.mtu = new_mtu; 3981 3982 return 0; 3983 } 3984 3985 void 3986 init_port_config(void) 3987 { 3988 portid_t pid; 3989 struct rte_port *port; 3990 int ret, i; 3991 3992 RTE_ETH_FOREACH_DEV(pid) { 3993 port = &ports[pid]; 3994 3995 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3996 if (ret != 0) 3997 return; 3998 3999 if (nb_rxq > 1) { 4000 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4001 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4002 rss_hf & port->dev_info.flow_type_rss_offloads; 4003 } else { 4004 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4005 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4006 } 4007 4008 if (port->dcb_flag == 0) { 4009 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4010 port->dev_conf.rxmode.mq_mode = 4011 (enum rte_eth_rx_mq_mode) 4012 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4013 } else { 4014 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4015 port->dev_conf.rxmode.offloads &= 4016 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4017 4018 for (i = 0; 4019 i < port->dev_info.nb_rx_queues; 4020 i++) 4021 port->rxq[i].conf.offloads &= 4022 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4023 } 4024 } 4025 4026 rxtx_port_config(pid); 4027 4028 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4029 if (ret != 0) 4030 return; 4031 4032 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4033 port->dev_conf.intr_conf.lsc = 1; 4034 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4035 port->dev_conf.intr_conf.rmv = 1; 4036 } 4037 } 4038 4039 void set_port_member_flag(portid_t member_pid) 4040 { 4041 struct rte_port *port; 4042 4043 port = &ports[member_pid]; 4044 port->member_flag = 1; 4045 } 4046 4047 void clear_port_member_flag(portid_t member_pid) 4048 { 4049 struct rte_port *port; 4050 4051 port = &ports[member_pid]; 4052 port->member_flag = 0; 4053 } 4054 4055 uint8_t port_is_bonding_member(portid_t member_pid) 4056 { 4057 struct rte_port *port; 4058 struct rte_eth_dev_info dev_info; 4059 int ret; 4060 4061 port = &ports[member_pid]; 4062 ret = eth_dev_info_get_print_err(member_pid, &dev_info); 4063 if (ret != 0) { 4064 TESTPMD_LOG(ERR, 4065 "Failed to get device info for port id %d," 4066 "cannot determine if the port is a bonding member", 4067 member_pid); 4068 return 0; 4069 } 4070 4071 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDING_MEMBER) || (port->member_flag == 1)) 4072 return 1; 4073 return 0; 4074 } 4075 4076 const uint16_t vlan_tags[] = { 4077 0, 1, 2, 3, 4, 5, 6, 7, 4078 8, 9, 10, 11, 12, 13, 14, 15, 4079 16, 17, 18, 19, 20, 21, 22, 23, 4080 24, 25, 26, 27, 28, 29, 30, 31 4081 }; 4082 4083 static void 4084 get_eth_dcb_conf(struct rte_eth_conf *eth_conf, enum dcb_mode_enable dcb_mode, 4085 enum rte_eth_nb_tcs num_tcs, uint8_t pfc_en) 4086 { 4087 uint8_t i; 4088 4089 /* 4090 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4091 * given above, and the number of traffic classes available for use. 4092 */ 4093 if (dcb_mode == DCB_VT_ENABLED) { 4094 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4095 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4096 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4097 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4098 4099 /* VMDQ+DCB RX and TX configurations */ 4100 vmdq_rx_conf->enable_default_pool = 0; 4101 vmdq_rx_conf->default_pool = 0; 4102 vmdq_rx_conf->nb_queue_pools = 4103 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4104 vmdq_tx_conf->nb_queue_pools = 4105 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4106 4107 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4108 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4109 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4110 vmdq_rx_conf->pool_map[i].pools = 4111 1 << (i % vmdq_rx_conf->nb_queue_pools); 4112 } 4113 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4114 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4115 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4116 } 4117 4118 /* set DCB mode of RX and TX of multiple queues */ 4119 eth_conf->rxmode.mq_mode = 4120 (enum rte_eth_rx_mq_mode) 4121 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4122 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4123 } else { 4124 struct rte_eth_dcb_rx_conf *rx_conf = 4125 ð_conf->rx_adv_conf.dcb_rx_conf; 4126 struct rte_eth_dcb_tx_conf *tx_conf = 4127 ð_conf->tx_adv_conf.dcb_tx_conf; 4128 4129 rx_conf->nb_tcs = num_tcs; 4130 tx_conf->nb_tcs = num_tcs; 4131 4132 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4133 rx_conf->dcb_tc[i] = i % num_tcs; 4134 tx_conf->dcb_tc[i] = i % num_tcs; 4135 } 4136 4137 eth_conf->rxmode.mq_mode = 4138 (enum rte_eth_rx_mq_mode) 4139 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4140 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4141 } 4142 4143 if (pfc_en) 4144 eth_conf->dcb_capability_en = 4145 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4146 else 4147 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4148 } 4149 4150 int 4151 init_port_dcb_config(portid_t pid, 4152 enum dcb_mode_enable dcb_mode, 4153 enum rte_eth_nb_tcs num_tcs, 4154 uint8_t pfc_en) 4155 { 4156 struct rte_eth_conf port_conf; 4157 struct rte_port *rte_port; 4158 int retval; 4159 uint16_t i; 4160 4161 if (num_procs > 1) { 4162 printf("The multi-process feature doesn't support dcb.\n"); 4163 return -ENOTSUP; 4164 } 4165 rte_port = &ports[pid]; 4166 4167 /* retain the original device configuration. */ 4168 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4169 4170 /* set configuration of DCB in vt mode and DCB in non-vt mode */ 4171 get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en); 4172 4173 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4174 /* remove RSS HASH offload for DCB in vt mode */ 4175 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4176 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4177 for (i = 0; i < nb_rxq; i++) 4178 rte_port->rxq[i].conf.offloads &= 4179 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4180 } 4181 4182 /* re-configure the device . */ 4183 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4184 if (retval < 0) 4185 return retval; 4186 4187 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4188 if (retval != 0) 4189 return retval; 4190 4191 /* If dev_info.vmdq_pool_base is greater than 0, 4192 * the queue id of vmdq pools is started after pf queues. 4193 */ 4194 if (dcb_mode == DCB_VT_ENABLED && 4195 rte_port->dev_info.vmdq_pool_base > 0) { 4196 fprintf(stderr, 4197 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4198 pid); 4199 return -1; 4200 } 4201 4202 /* Assume the ports in testpmd have the same dcb capability 4203 * and has the same number of rxq and txq in dcb mode 4204 */ 4205 if (dcb_mode == DCB_VT_ENABLED) { 4206 if (rte_port->dev_info.max_vfs > 0) { 4207 nb_rxq = rte_port->dev_info.nb_rx_queues; 4208 nb_txq = rte_port->dev_info.nb_tx_queues; 4209 } else { 4210 nb_rxq = rte_port->dev_info.max_rx_queues; 4211 nb_txq = rte_port->dev_info.max_tx_queues; 4212 } 4213 } else { 4214 /*if vt is disabled, use all pf queues */ 4215 if (rte_port->dev_info.vmdq_pool_base == 0) { 4216 nb_rxq = rte_port->dev_info.max_rx_queues; 4217 nb_txq = rte_port->dev_info.max_tx_queues; 4218 } else { 4219 nb_rxq = (queueid_t)num_tcs; 4220 nb_txq = (queueid_t)num_tcs; 4221 4222 } 4223 } 4224 rx_free_thresh = 64; 4225 4226 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4227 4228 rxtx_port_config(pid); 4229 /* VLAN filter */ 4230 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4231 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4232 rx_vft_set(pid, vlan_tags[i], 1); 4233 4234 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4235 if (retval != 0) 4236 return retval; 4237 4238 rte_port->dcb_flag = 1; 4239 4240 /* Enter DCB configuration status */ 4241 dcb_config = 1; 4242 4243 return 0; 4244 } 4245 4246 static void 4247 init_port(void) 4248 { 4249 int i; 4250 4251 /* Configuration of Ethernet ports. */ 4252 ports = rte_zmalloc("testpmd: ports", 4253 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4254 RTE_CACHE_LINE_SIZE); 4255 if (ports == NULL) { 4256 rte_exit(EXIT_FAILURE, 4257 "rte_zmalloc(%d struct rte_port) failed\n", 4258 RTE_MAX_ETHPORTS); 4259 } 4260 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4261 ports[i].fwd_mac_swap = 1; 4262 ports[i].xstats_info.allocated = false; 4263 LIST_INIT(&ports[i].flow_tunnel_list); 4264 } 4265 /* Initialize ports NUMA structures */ 4266 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4267 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4268 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4269 } 4270 4271 static void 4272 print_stats(void) 4273 { 4274 uint8_t i; 4275 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4276 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4277 4278 /* Clear screen and move to top left */ 4279 printf("%s%s", clr, top_left); 4280 4281 printf("\nPort statistics ===================================="); 4282 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4283 nic_stats_display(fwd_ports_ids[i]); 4284 4285 fflush(stdout); 4286 } 4287 4288 static void 4289 signal_handler(int signum __rte_unused) 4290 { 4291 f_quit = 1; 4292 prompt_exit(); 4293 } 4294 4295 int 4296 main(int argc, char** argv) 4297 { 4298 int diag; 4299 portid_t port_id; 4300 uint16_t count; 4301 int ret; 4302 4303 #ifdef RTE_EXEC_ENV_WINDOWS 4304 signal(SIGINT, signal_handler); 4305 signal(SIGTERM, signal_handler); 4306 #else 4307 /* Want read() not to be restarted on signal */ 4308 struct sigaction action = { 4309 .sa_handler = signal_handler, 4310 }; 4311 4312 sigaction(SIGINT, &action, NULL); 4313 sigaction(SIGTERM, &action, NULL); 4314 #endif 4315 4316 testpmd_logtype = rte_log_register("testpmd"); 4317 if (testpmd_logtype < 0) 4318 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4319 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4320 4321 diag = rte_eal_init(argc, argv); 4322 if (diag < 0) 4323 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4324 rte_strerror(rte_errno)); 4325 4326 /* allocate port structures, and init them */ 4327 init_port(); 4328 4329 ret = register_eth_event_callback(); 4330 if (ret != 0) 4331 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4332 4333 #ifdef RTE_LIB_PDUMP 4334 /* initialize packet capture framework */ 4335 rte_pdump_init(); 4336 #endif 4337 4338 count = 0; 4339 RTE_ETH_FOREACH_DEV(port_id) { 4340 ports_ids[count] = port_id; 4341 count++; 4342 } 4343 nb_ports = (portid_t) count; 4344 if (nb_ports == 0) 4345 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4346 4347 set_def_fwd_config(); 4348 if (nb_lcores == 0) 4349 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4350 "Check the core mask argument\n"); 4351 4352 /* Bitrate/latency stats disabled by default */ 4353 #ifdef RTE_LIB_BITRATESTATS 4354 bitrate_enabled = 0; 4355 #endif 4356 #ifdef RTE_LIB_LATENCYSTATS 4357 latencystats_enabled = 0; 4358 #endif 4359 4360 /* on FreeBSD, mlockall() is disabled by default */ 4361 #ifdef RTE_EXEC_ENV_FREEBSD 4362 do_mlockall = 0; 4363 #else 4364 do_mlockall = 1; 4365 #endif 4366 4367 argc -= diag; 4368 argv += diag; 4369 if (argc > 1) 4370 launch_args_parse(argc, argv); 4371 4372 #ifndef RTE_EXEC_ENV_WINDOWS 4373 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4374 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4375 strerror(errno)); 4376 } 4377 #endif 4378 4379 if (tx_first && interactive) 4380 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4381 "interactive mode.\n"); 4382 4383 if (tx_first && lsc_interrupt) { 4384 fprintf(stderr, 4385 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4386 lsc_interrupt = 0; 4387 } 4388 4389 if (!nb_rxq && !nb_txq) 4390 rte_exit(EXIT_FAILURE, "Either rx or tx queues should be non-zero\n"); 4391 4392 if (nb_rxq > 1 && nb_rxq > nb_txq) 4393 fprintf(stderr, 4394 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4395 nb_rxq, nb_txq); 4396 4397 init_config(); 4398 4399 if (hot_plug) { 4400 ret = rte_dev_hotplug_handle_enable(); 4401 if (ret) { 4402 TESTPMD_LOG(ERR, "fail to enable hotplug handling."); 4403 return -1; 4404 } 4405 4406 ret = rte_dev_event_monitor_start(); 4407 if (ret) { 4408 TESTPMD_LOG(ERR, "fail to start device event monitoring."); 4409 return -1; 4410 } 4411 4412 ret = rte_dev_event_callback_register(NULL, dev_event_callback, NULL); 4413 if (ret) { 4414 TESTPMD_LOG(ERR, "fail to register device event callback\n"); 4415 return -1; 4416 } 4417 } 4418 4419 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { 4420 if (!interactive) { 4421 rte_eal_cleanup(); 4422 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4423 } 4424 fprintf(stderr, "Start ports failed\n"); 4425 } 4426 4427 /* set all ports to promiscuous mode by default */ 4428 RTE_ETH_FOREACH_DEV(port_id) { 4429 ret = rte_eth_promiscuous_enable(port_id); 4430 if (ret != 0) 4431 fprintf(stderr, 4432 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4433 port_id, rte_strerror(-ret)); 4434 } 4435 4436 #ifdef RTE_LIB_METRICS 4437 /* Init metrics library */ 4438 rte_metrics_init(rte_socket_id()); 4439 #endif 4440 4441 #ifdef RTE_LIB_LATENCYSTATS 4442 if (latencystats_enabled != 0) { 4443 int ret = rte_latencystats_init(1, NULL); 4444 if (ret) 4445 fprintf(stderr, 4446 "Warning: latencystats init() returned error %d\n", 4447 ret); 4448 fprintf(stderr, "Latencystats running on lcore %d\n", 4449 latencystats_lcore_id); 4450 } 4451 #endif 4452 4453 /* Setup bitrate stats */ 4454 #ifdef RTE_LIB_BITRATESTATS 4455 if (bitrate_enabled != 0) { 4456 bitrate_data = rte_stats_bitrate_create(); 4457 if (bitrate_data == NULL) 4458 rte_exit(EXIT_FAILURE, 4459 "Could not allocate bitrate data.\n"); 4460 rte_stats_bitrate_reg(bitrate_data); 4461 } 4462 #endif 4463 4464 if (record_core_cycles) 4465 rte_lcore_register_usage_cb(lcore_usage_callback); 4466 4467 if (init_cmdline() != 0) 4468 rte_exit(EXIT_FAILURE, 4469 "Could not initialise cmdline context.\n"); 4470 4471 if (strlen(cmdline_filename) != 0) 4472 cmdline_read_from_file(cmdline_filename); 4473 4474 if (interactive == 1) { 4475 if (auto_start) { 4476 printf("Start automatic packet forwarding\n"); 4477 start_packet_forwarding(0); 4478 } 4479 prompt(); 4480 } else { 4481 printf("No commandline core given, start packet forwarding\n"); 4482 start_packet_forwarding(tx_first); 4483 if (stats_period != 0) { 4484 uint64_t prev_time = 0, cur_time, diff_time = 0; 4485 uint64_t timer_period; 4486 4487 /* Convert to number of cycles */ 4488 timer_period = stats_period * rte_get_timer_hz(); 4489 4490 while (f_quit == 0) { 4491 cur_time = rte_get_timer_cycles(); 4492 diff_time += cur_time - prev_time; 4493 4494 if (diff_time >= timer_period) { 4495 print_stats(); 4496 /* Reset the timer */ 4497 diff_time = 0; 4498 } 4499 /* Sleep to avoid unnecessary checks */ 4500 prev_time = cur_time; 4501 rte_delay_us_sleep(US_PER_S); 4502 } 4503 } else { 4504 char c; 4505 4506 printf("Press enter to exit\n"); 4507 while (f_quit == 0) { 4508 /* end-of-file or any character exits loop */ 4509 if (read(0, &c, 1) >= 0) 4510 break; 4511 if (errno == EINTR) 4512 continue; 4513 rte_exit(EXIT_FAILURE, "Read failed: %s\n", 4514 strerror(errno)); 4515 } 4516 } 4517 } 4518 4519 pmd_test_exit(); 4520 4521 #ifdef RTE_LIB_PDUMP 4522 /* uninitialize packet capture framework */ 4523 rte_pdump_uninit(); 4524 #endif 4525 #ifdef RTE_LIB_LATENCYSTATS 4526 if (latencystats_enabled != 0) 4527 rte_latencystats_uninit(); 4528 #endif 4529 4530 ret = unregister_eth_event_callback(); 4531 if (ret != 0) 4532 rte_exit(EXIT_FAILURE, "Cannot unregister for ethdev events"); 4533 4534 4535 ret = rte_eal_cleanup(); 4536 if (ret != 0) 4537 rte_exit(EXIT_FAILURE, 4538 "EAL cleanup failed: %s\n", strerror(-ret)); 4539 4540 return EXIT_SUCCESS; 4541 } 4542