1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 static volatile uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 253 254 /* 255 * Configuration of packet segments used by the "txonly" processing engine. 256 */ 257 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 258 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 259 TXONLY_DEF_PACKET_LEN, 260 }; 261 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 262 263 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 264 /**< Split policy for packets to TX. */ 265 266 uint8_t txonly_multi_flow; 267 /**< Whether multiple flows are generated in TXONLY mode. */ 268 269 uint32_t tx_pkt_times_inter; 270 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 271 272 uint32_t tx_pkt_times_intra; 273 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 274 275 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 276 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 277 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 278 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 279 280 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 281 uint8_t dcb_config = 0; 282 283 /* 284 * Configurable number of RX/TX queues. 285 */ 286 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 287 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 288 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 289 290 /* 291 * Configurable number of RX/TX ring descriptors. 292 * Defaults are supplied by drivers via ethdev. 293 */ 294 #define RX_DESC_DEFAULT 0 295 #define TX_DESC_DEFAULT 0 296 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 297 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 298 299 #define RTE_PMD_PARAM_UNSET -1 300 /* 301 * Configurable values of RX and TX ring threshold registers. 302 */ 303 304 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 305 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 306 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 307 308 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 309 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 310 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 311 312 /* 313 * Configurable value of RX free threshold. 314 */ 315 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 316 317 /* 318 * Configurable value of RX drop enable. 319 */ 320 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 321 322 /* 323 * Configurable value of TX free threshold. 324 */ 325 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 326 327 /* 328 * Configurable value of TX RS bit threshold. 329 */ 330 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 331 332 /* 333 * Configurable value of buffered packets before sending. 334 */ 335 uint16_t noisy_tx_sw_bufsz; 336 337 /* 338 * Configurable value of packet buffer timeout. 339 */ 340 uint16_t noisy_tx_sw_buf_flush_time; 341 342 /* 343 * Configurable value for size of VNF internal memory area 344 * used for simulating noisy neighbour behaviour 345 */ 346 uint64_t noisy_lkup_mem_sz; 347 348 /* 349 * Configurable value of number of random writes done in 350 * VNF simulation memory area. 351 */ 352 uint64_t noisy_lkup_num_writes; 353 354 /* 355 * Configurable value of number of random reads done in 356 * VNF simulation memory area. 357 */ 358 uint64_t noisy_lkup_num_reads; 359 360 /* 361 * Configurable value of number of random reads/writes done in 362 * VNF simulation memory area. 363 */ 364 uint64_t noisy_lkup_num_reads_writes; 365 366 /* 367 * Receive Side Scaling (RSS) configuration. 368 */ 369 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 370 371 /* 372 * Port topology configuration 373 */ 374 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 375 376 /* 377 * Avoids to flush all the RX streams before starts forwarding. 378 */ 379 uint8_t no_flush_rx = 0; /* flush by default */ 380 381 /* 382 * Flow API isolated mode. 383 */ 384 uint8_t flow_isolate_all; 385 386 /* 387 * Avoids to check link status when starting/stopping a port. 388 */ 389 uint8_t no_link_check = 0; /* check by default */ 390 391 /* 392 * Don't automatically start all ports in interactive mode. 393 */ 394 uint8_t no_device_start = 0; 395 396 /* 397 * Enable link status change notification 398 */ 399 uint8_t lsc_interrupt = 1; /* enabled by default */ 400 401 /* 402 * Enable device removal notification. 403 */ 404 uint8_t rmv_interrupt = 1; /* enabled by default */ 405 406 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 407 408 /* After attach, port setup is called on event or by iterator */ 409 bool setup_on_probe_event = true; 410 411 /* Clear ptypes on port initialization. */ 412 uint8_t clear_ptypes = true; 413 414 /* Hairpin ports configuration mode. */ 415 uint32_t hairpin_mode; 416 417 /* Pretty printing of ethdev events */ 418 static const char * const eth_event_desc[] = { 419 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 420 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 421 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 422 [RTE_ETH_EVENT_INTR_RESET] = "reset", 423 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 424 [RTE_ETH_EVENT_IPSEC] = "IPsec", 425 [RTE_ETH_EVENT_MACSEC] = "MACsec", 426 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 427 [RTE_ETH_EVENT_NEW] = "device probed", 428 [RTE_ETH_EVENT_DESTROY] = "device released", 429 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 430 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 431 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 432 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 433 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 434 [RTE_ETH_EVENT_MAX] = NULL, 435 }; 436 437 /* 438 * Display or mask ether events 439 * Default to all events except VF_MBOX 440 */ 441 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 442 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 443 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 444 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 445 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 446 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 447 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 448 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 449 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 450 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 451 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 452 /* 453 * Decide if all memory are locked for performance. 454 */ 455 int do_mlockall = 0; 456 457 #ifdef RTE_LIB_LATENCYSTATS 458 459 /* 460 * Set when latency stats is enabled in the commandline 461 */ 462 uint8_t latencystats_enabled; 463 464 /* 465 * Lcore ID to service latency statistics. 466 */ 467 lcoreid_t latencystats_lcore_id = -1; 468 469 #endif 470 471 /* 472 * Ethernet device configuration. 473 */ 474 struct rte_eth_rxmode rx_mode; 475 476 struct rte_eth_txmode tx_mode = { 477 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 478 }; 479 480 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 481 482 /* 483 * Display zero values by default for xstats 484 */ 485 uint8_t xstats_hide_zero; 486 487 /* 488 * Measure of CPU cycles disabled by default 489 */ 490 uint8_t record_core_cycles; 491 492 /* 493 * Display of RX and TX bursts disabled by default 494 */ 495 uint8_t record_burst_stats; 496 497 /* 498 * Number of ports per shared Rx queue group, 0 disable. 499 */ 500 uint32_t rxq_share; 501 502 unsigned int num_sockets = 0; 503 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 504 505 #ifdef RTE_LIB_BITRATESTATS 506 /* Bitrate statistics */ 507 struct rte_stats_bitrates *bitrate_data; 508 lcoreid_t bitrate_lcore_id; 509 uint8_t bitrate_enabled; 510 #endif 511 512 #ifdef RTE_LIB_GRO 513 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 514 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 515 #endif 516 517 /* 518 * hexadecimal bitmask of RX mq mode can be enabled. 519 */ 520 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 521 522 /* 523 * Used to set forced link speed 524 */ 525 uint32_t eth_link_speed; 526 527 /* 528 * ID of the current process in multi-process, used to 529 * configure the queues to be polled. 530 */ 531 int proc_id; 532 533 /* 534 * Number of processes in multi-process, used to 535 * configure the queues to be polled. 536 */ 537 unsigned int num_procs = 1; 538 539 static void 540 eth_rx_metadata_negotiate_mp(uint16_t port_id) 541 { 542 uint64_t rx_meta_features = 0; 543 int ret; 544 545 if (!is_proc_primary()) 546 return; 547 548 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 549 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 550 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 551 552 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 553 if (ret == 0) { 554 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 555 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 556 port_id); 557 } 558 559 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 560 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 561 port_id); 562 } 563 564 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 565 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 566 port_id); 567 } 568 } else if (ret != -ENOTSUP) { 569 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 570 port_id, rte_strerror(-ret)); 571 } 572 } 573 574 static int 575 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 576 const struct rte_eth_conf *dev_conf) 577 { 578 if (is_proc_primary()) 579 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 580 dev_conf); 581 return 0; 582 } 583 584 static int 585 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 586 { 587 #ifdef RTE_NET_BOND 588 589 portid_t slave_pids[RTE_MAX_ETHPORTS]; 590 struct rte_port *port; 591 int num_slaves; 592 portid_t slave_pid; 593 int i; 594 595 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 596 RTE_MAX_ETHPORTS); 597 if (num_slaves < 0) { 598 fprintf(stderr, "Failed to get slave list for port = %u\n", 599 bond_pid); 600 return num_slaves; 601 } 602 603 for (i = 0; i < num_slaves; i++) { 604 slave_pid = slave_pids[i]; 605 port = &ports[slave_pid]; 606 port->port_status = 607 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 608 } 609 #else 610 RTE_SET_USED(bond_pid); 611 RTE_SET_USED(is_stop); 612 #endif 613 return 0; 614 } 615 616 static int 617 eth_dev_start_mp(uint16_t port_id) 618 { 619 int ret; 620 621 if (is_proc_primary()) { 622 ret = rte_eth_dev_start(port_id); 623 if (ret != 0) 624 return ret; 625 626 struct rte_port *port = &ports[port_id]; 627 628 /* 629 * Starting a bonded port also starts all slaves under the bonded 630 * device. So if this port is bond device, we need to modify the 631 * port status of these slaves. 632 */ 633 if (port->bond_flag == 1) 634 return change_bonding_slave_port_status(port_id, false); 635 } 636 637 return 0; 638 } 639 640 static int 641 eth_dev_stop_mp(uint16_t port_id) 642 { 643 int ret; 644 645 if (is_proc_primary()) { 646 ret = rte_eth_dev_stop(port_id); 647 if (ret != 0) 648 return ret; 649 650 struct rte_port *port = &ports[port_id]; 651 652 /* 653 * Stopping a bonded port also stops all slaves under the bonded 654 * device. So if this port is bond device, we need to modify the 655 * port status of these slaves. 656 */ 657 if (port->bond_flag == 1) 658 return change_bonding_slave_port_status(port_id, true); 659 } 660 661 return 0; 662 } 663 664 static void 665 mempool_free_mp(struct rte_mempool *mp) 666 { 667 if (is_proc_primary()) 668 rte_mempool_free(mp); 669 } 670 671 static int 672 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 673 { 674 if (is_proc_primary()) 675 return rte_eth_dev_set_mtu(port_id, mtu); 676 677 return 0; 678 } 679 680 /* Forward function declarations */ 681 static void setup_attached_port(portid_t pi); 682 static void check_all_ports_link_status(uint32_t port_mask); 683 static int eth_event_callback(portid_t port_id, 684 enum rte_eth_event_type type, 685 void *param, void *ret_param); 686 static void dev_event_callback(const char *device_name, 687 enum rte_dev_event_type type, 688 void *param); 689 static void fill_xstats_display_info(void); 690 691 /* 692 * Check if all the ports are started. 693 * If yes, return positive value. If not, return zero. 694 */ 695 static int all_ports_started(void); 696 697 #ifdef RTE_LIB_GSO 698 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 699 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 700 #endif 701 702 /* Holds the registered mbuf dynamic flags names. */ 703 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 704 705 706 /* 707 * Helper function to check if socket is already discovered. 708 * If yes, return positive value. If not, return zero. 709 */ 710 int 711 new_socket_id(unsigned int socket_id) 712 { 713 unsigned int i; 714 715 for (i = 0; i < num_sockets; i++) { 716 if (socket_ids[i] == socket_id) 717 return 0; 718 } 719 return 1; 720 } 721 722 /* 723 * Setup default configuration. 724 */ 725 static void 726 set_default_fwd_lcores_config(void) 727 { 728 unsigned int i; 729 unsigned int nb_lc; 730 unsigned int sock_num; 731 732 nb_lc = 0; 733 for (i = 0; i < RTE_MAX_LCORE; i++) { 734 if (!rte_lcore_is_enabled(i)) 735 continue; 736 sock_num = rte_lcore_to_socket_id(i); 737 if (new_socket_id(sock_num)) { 738 if (num_sockets >= RTE_MAX_NUMA_NODES) { 739 rte_exit(EXIT_FAILURE, 740 "Total sockets greater than %u\n", 741 RTE_MAX_NUMA_NODES); 742 } 743 socket_ids[num_sockets++] = sock_num; 744 } 745 if (i == rte_get_main_lcore()) 746 continue; 747 fwd_lcores_cpuids[nb_lc++] = i; 748 } 749 nb_lcores = (lcoreid_t) nb_lc; 750 nb_cfg_lcores = nb_lcores; 751 nb_fwd_lcores = 1; 752 } 753 754 static void 755 set_def_peer_eth_addrs(void) 756 { 757 portid_t i; 758 759 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 760 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 761 peer_eth_addrs[i].addr_bytes[5] = i; 762 } 763 } 764 765 static void 766 set_default_fwd_ports_config(void) 767 { 768 portid_t pt_id; 769 int i = 0; 770 771 RTE_ETH_FOREACH_DEV(pt_id) { 772 fwd_ports_ids[i++] = pt_id; 773 774 /* Update sockets info according to the attached device */ 775 int socket_id = rte_eth_dev_socket_id(pt_id); 776 if (socket_id >= 0 && new_socket_id(socket_id)) { 777 if (num_sockets >= RTE_MAX_NUMA_NODES) { 778 rte_exit(EXIT_FAILURE, 779 "Total sockets greater than %u\n", 780 RTE_MAX_NUMA_NODES); 781 } 782 socket_ids[num_sockets++] = socket_id; 783 } 784 } 785 786 nb_cfg_ports = nb_ports; 787 nb_fwd_ports = nb_ports; 788 } 789 790 void 791 set_def_fwd_config(void) 792 { 793 set_default_fwd_lcores_config(); 794 set_def_peer_eth_addrs(); 795 set_default_fwd_ports_config(); 796 } 797 798 #ifndef RTE_EXEC_ENV_WINDOWS 799 /* extremely pessimistic estimation of memory required to create a mempool */ 800 static int 801 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 802 { 803 unsigned int n_pages, mbuf_per_pg, leftover; 804 uint64_t total_mem, mbuf_mem, obj_sz; 805 806 /* there is no good way to predict how much space the mempool will 807 * occupy because it will allocate chunks on the fly, and some of those 808 * will come from default DPDK memory while some will come from our 809 * external memory, so just assume 128MB will be enough for everyone. 810 */ 811 uint64_t hdr_mem = 128 << 20; 812 813 /* account for possible non-contiguousness */ 814 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 815 if (obj_sz > pgsz) { 816 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 817 return -1; 818 } 819 820 mbuf_per_pg = pgsz / obj_sz; 821 leftover = (nb_mbufs % mbuf_per_pg) > 0; 822 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 823 824 mbuf_mem = n_pages * pgsz; 825 826 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 827 828 if (total_mem > SIZE_MAX) { 829 TESTPMD_LOG(ERR, "Memory size too big\n"); 830 return -1; 831 } 832 *out = (size_t)total_mem; 833 834 return 0; 835 } 836 837 static int 838 pagesz_flags(uint64_t page_sz) 839 { 840 /* as per mmap() manpage, all page sizes are log2 of page size 841 * shifted by MAP_HUGE_SHIFT 842 */ 843 int log2 = rte_log2_u64(page_sz); 844 845 return (log2 << HUGE_SHIFT); 846 } 847 848 static void * 849 alloc_mem(size_t memsz, size_t pgsz, bool huge) 850 { 851 void *addr; 852 int flags; 853 854 /* allocate anonymous hugepages */ 855 flags = MAP_ANONYMOUS | MAP_PRIVATE; 856 if (huge) 857 flags |= HUGE_FLAG | pagesz_flags(pgsz); 858 859 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 860 if (addr == MAP_FAILED) 861 return NULL; 862 863 return addr; 864 } 865 866 struct extmem_param { 867 void *addr; 868 size_t len; 869 size_t pgsz; 870 rte_iova_t *iova_table; 871 unsigned int iova_table_len; 872 }; 873 874 static int 875 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 876 bool huge) 877 { 878 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 879 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 880 unsigned int cur_page, n_pages, pgsz_idx; 881 size_t mem_sz, cur_pgsz; 882 rte_iova_t *iovas = NULL; 883 void *addr; 884 int ret; 885 886 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 887 /* skip anything that is too big */ 888 if (pgsizes[pgsz_idx] > SIZE_MAX) 889 continue; 890 891 cur_pgsz = pgsizes[pgsz_idx]; 892 893 /* if we were told not to allocate hugepages, override */ 894 if (!huge) 895 cur_pgsz = sysconf(_SC_PAGESIZE); 896 897 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 898 if (ret < 0) { 899 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 900 return -1; 901 } 902 903 /* allocate our memory */ 904 addr = alloc_mem(mem_sz, cur_pgsz, huge); 905 906 /* if we couldn't allocate memory with a specified page size, 907 * that doesn't mean we can't do it with other page sizes, so 908 * try another one. 909 */ 910 if (addr == NULL) 911 continue; 912 913 /* store IOVA addresses for every page in this memory area */ 914 n_pages = mem_sz / cur_pgsz; 915 916 iovas = malloc(sizeof(*iovas) * n_pages); 917 918 if (iovas == NULL) { 919 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 920 goto fail; 921 } 922 /* lock memory if it's not huge pages */ 923 if (!huge) 924 mlock(addr, mem_sz); 925 926 /* populate IOVA addresses */ 927 for (cur_page = 0; cur_page < n_pages; cur_page++) { 928 rte_iova_t iova; 929 size_t offset; 930 void *cur; 931 932 offset = cur_pgsz * cur_page; 933 cur = RTE_PTR_ADD(addr, offset); 934 935 /* touch the page before getting its IOVA */ 936 *(volatile char *)cur = 0; 937 938 iova = rte_mem_virt2iova(cur); 939 940 iovas[cur_page] = iova; 941 } 942 943 break; 944 } 945 /* if we couldn't allocate anything */ 946 if (iovas == NULL) 947 return -1; 948 949 param->addr = addr; 950 param->len = mem_sz; 951 param->pgsz = cur_pgsz; 952 param->iova_table = iovas; 953 param->iova_table_len = n_pages; 954 955 return 0; 956 fail: 957 free(iovas); 958 if (addr) 959 munmap(addr, mem_sz); 960 961 return -1; 962 } 963 964 static int 965 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 966 { 967 struct extmem_param param; 968 int socket_id, ret; 969 970 memset(¶m, 0, sizeof(param)); 971 972 /* check if our heap exists */ 973 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 974 if (socket_id < 0) { 975 /* create our heap */ 976 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 977 if (ret < 0) { 978 TESTPMD_LOG(ERR, "Cannot create heap\n"); 979 return -1; 980 } 981 } 982 983 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 984 if (ret < 0) { 985 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 986 return -1; 987 } 988 989 /* we now have a valid memory area, so add it to heap */ 990 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 991 param.addr, param.len, param.iova_table, 992 param.iova_table_len, param.pgsz); 993 994 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 995 996 /* not needed any more */ 997 free(param.iova_table); 998 999 if (ret < 0) { 1000 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1001 munmap(param.addr, param.len); 1002 return -1; 1003 } 1004 1005 /* success */ 1006 1007 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1008 param.len >> 20); 1009 1010 return 0; 1011 } 1012 static void 1013 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1014 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1015 { 1016 uint16_t pid = 0; 1017 int ret; 1018 1019 RTE_ETH_FOREACH_DEV(pid) { 1020 struct rte_eth_dev_info dev_info; 1021 1022 ret = eth_dev_info_get_print_err(pid, &dev_info); 1023 if (ret != 0) { 1024 TESTPMD_LOG(DEBUG, 1025 "unable to get device info for port %d on addr 0x%p," 1026 "mempool unmapping will not be performed\n", 1027 pid, memhdr->addr); 1028 continue; 1029 } 1030 1031 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1032 if (ret) { 1033 TESTPMD_LOG(DEBUG, 1034 "unable to DMA unmap addr 0x%p " 1035 "for device %s\n", 1036 memhdr->addr, rte_dev_name(dev_info.device)); 1037 } 1038 } 1039 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1040 if (ret) { 1041 TESTPMD_LOG(DEBUG, 1042 "unable to un-register addr 0x%p\n", memhdr->addr); 1043 } 1044 } 1045 1046 static void 1047 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1048 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1049 { 1050 uint16_t pid = 0; 1051 size_t page_size = sysconf(_SC_PAGESIZE); 1052 int ret; 1053 1054 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1055 page_size); 1056 if (ret) { 1057 TESTPMD_LOG(DEBUG, 1058 "unable to register addr 0x%p\n", memhdr->addr); 1059 return; 1060 } 1061 RTE_ETH_FOREACH_DEV(pid) { 1062 struct rte_eth_dev_info dev_info; 1063 1064 ret = eth_dev_info_get_print_err(pid, &dev_info); 1065 if (ret != 0) { 1066 TESTPMD_LOG(DEBUG, 1067 "unable to get device info for port %d on addr 0x%p," 1068 "mempool mapping will not be performed\n", 1069 pid, memhdr->addr); 1070 continue; 1071 } 1072 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1073 if (ret) { 1074 TESTPMD_LOG(DEBUG, 1075 "unable to DMA map addr 0x%p " 1076 "for device %s\n", 1077 memhdr->addr, rte_dev_name(dev_info.device)); 1078 } 1079 } 1080 } 1081 #endif 1082 1083 static unsigned int 1084 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1085 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1086 { 1087 struct rte_pktmbuf_extmem *xmem; 1088 unsigned int ext_num, zone_num, elt_num; 1089 uint16_t elt_size; 1090 1091 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1092 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1093 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1094 1095 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1096 if (xmem == NULL) { 1097 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1098 "external buffer descriptors\n"); 1099 *ext_mem = NULL; 1100 return 0; 1101 } 1102 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1103 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1104 const struct rte_memzone *mz; 1105 char mz_name[RTE_MEMZONE_NAMESIZE]; 1106 int ret; 1107 1108 ret = snprintf(mz_name, sizeof(mz_name), 1109 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1110 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1111 errno = ENAMETOOLONG; 1112 ext_num = 0; 1113 break; 1114 } 1115 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1116 socket_id, 1117 RTE_MEMZONE_IOVA_CONTIG | 1118 RTE_MEMZONE_1GB | 1119 RTE_MEMZONE_SIZE_HINT_ONLY); 1120 if (mz == NULL) { 1121 /* 1122 * The caller exits on external buffer creation 1123 * error, so there is no need to free memzones. 1124 */ 1125 errno = ENOMEM; 1126 ext_num = 0; 1127 break; 1128 } 1129 xseg->buf_ptr = mz->addr; 1130 xseg->buf_iova = mz->iova; 1131 xseg->buf_len = EXTBUF_ZONE_SIZE; 1132 xseg->elt_size = elt_size; 1133 } 1134 if (ext_num == 0 && xmem != NULL) { 1135 free(xmem); 1136 xmem = NULL; 1137 } 1138 *ext_mem = xmem; 1139 return ext_num; 1140 } 1141 1142 /* 1143 * Configuration initialisation done once at init time. 1144 */ 1145 static struct rte_mempool * 1146 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1147 unsigned int socket_id, uint16_t size_idx) 1148 { 1149 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1150 struct rte_mempool *rte_mp = NULL; 1151 #ifndef RTE_EXEC_ENV_WINDOWS 1152 uint32_t mb_size; 1153 1154 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1155 #endif 1156 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1157 if (!is_proc_primary()) { 1158 rte_mp = rte_mempool_lookup(pool_name); 1159 if (rte_mp == NULL) 1160 rte_exit(EXIT_FAILURE, 1161 "Get mbuf pool for socket %u failed: %s\n", 1162 socket_id, rte_strerror(rte_errno)); 1163 return rte_mp; 1164 } 1165 1166 TESTPMD_LOG(INFO, 1167 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1168 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1169 1170 switch (mp_alloc_type) { 1171 case MP_ALLOC_NATIVE: 1172 { 1173 /* wrapper to rte_mempool_create() */ 1174 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1175 rte_mbuf_best_mempool_ops()); 1176 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1177 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1178 break; 1179 } 1180 #ifndef RTE_EXEC_ENV_WINDOWS 1181 case MP_ALLOC_ANON: 1182 { 1183 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1184 mb_size, (unsigned int) mb_mempool_cache, 1185 sizeof(struct rte_pktmbuf_pool_private), 1186 socket_id, mempool_flags); 1187 if (rte_mp == NULL) 1188 goto err; 1189 1190 if (rte_mempool_populate_anon(rte_mp) == 0) { 1191 rte_mempool_free(rte_mp); 1192 rte_mp = NULL; 1193 goto err; 1194 } 1195 rte_pktmbuf_pool_init(rte_mp, NULL); 1196 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1197 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1198 break; 1199 } 1200 case MP_ALLOC_XMEM: 1201 case MP_ALLOC_XMEM_HUGE: 1202 { 1203 int heap_socket; 1204 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1205 1206 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1207 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1208 1209 heap_socket = 1210 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1211 if (heap_socket < 0) 1212 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1213 1214 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1215 rte_mbuf_best_mempool_ops()); 1216 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1217 mb_mempool_cache, 0, mbuf_seg_size, 1218 heap_socket); 1219 break; 1220 } 1221 #endif 1222 case MP_ALLOC_XBUF: 1223 { 1224 struct rte_pktmbuf_extmem *ext_mem; 1225 unsigned int ext_num; 1226 1227 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1228 socket_id, pool_name, &ext_mem); 1229 if (ext_num == 0) 1230 rte_exit(EXIT_FAILURE, 1231 "Can't create pinned data buffers\n"); 1232 1233 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1234 rte_mbuf_best_mempool_ops()); 1235 rte_mp = rte_pktmbuf_pool_create_extbuf 1236 (pool_name, nb_mbuf, mb_mempool_cache, 1237 0, mbuf_seg_size, socket_id, 1238 ext_mem, ext_num); 1239 free(ext_mem); 1240 break; 1241 } 1242 default: 1243 { 1244 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1245 } 1246 } 1247 1248 #ifndef RTE_EXEC_ENV_WINDOWS 1249 err: 1250 #endif 1251 if (rte_mp == NULL) { 1252 rte_exit(EXIT_FAILURE, 1253 "Creation of mbuf pool for socket %u failed: %s\n", 1254 socket_id, rte_strerror(rte_errno)); 1255 } else if (verbose_level > 0) { 1256 rte_mempool_dump(stdout, rte_mp); 1257 } 1258 return rte_mp; 1259 } 1260 1261 /* 1262 * Check given socket id is valid or not with NUMA mode, 1263 * if valid, return 0, else return -1 1264 */ 1265 static int 1266 check_socket_id(const unsigned int socket_id) 1267 { 1268 static int warning_once = 0; 1269 1270 if (new_socket_id(socket_id)) { 1271 if (!warning_once && numa_support) 1272 fprintf(stderr, 1273 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1274 warning_once = 1; 1275 return -1; 1276 } 1277 return 0; 1278 } 1279 1280 /* 1281 * Get the allowed maximum number of RX queues. 1282 * *pid return the port id which has minimal value of 1283 * max_rx_queues in all ports. 1284 */ 1285 queueid_t 1286 get_allowed_max_nb_rxq(portid_t *pid) 1287 { 1288 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1289 bool max_rxq_valid = false; 1290 portid_t pi; 1291 struct rte_eth_dev_info dev_info; 1292 1293 RTE_ETH_FOREACH_DEV(pi) { 1294 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1295 continue; 1296 1297 max_rxq_valid = true; 1298 if (dev_info.max_rx_queues < allowed_max_rxq) { 1299 allowed_max_rxq = dev_info.max_rx_queues; 1300 *pid = pi; 1301 } 1302 } 1303 return max_rxq_valid ? allowed_max_rxq : 0; 1304 } 1305 1306 /* 1307 * Check input rxq is valid or not. 1308 * If input rxq is not greater than any of maximum number 1309 * of RX queues of all ports, it is valid. 1310 * if valid, return 0, else return -1 1311 */ 1312 int 1313 check_nb_rxq(queueid_t rxq) 1314 { 1315 queueid_t allowed_max_rxq; 1316 portid_t pid = 0; 1317 1318 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1319 if (rxq > allowed_max_rxq) { 1320 fprintf(stderr, 1321 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1322 rxq, allowed_max_rxq, pid); 1323 return -1; 1324 } 1325 return 0; 1326 } 1327 1328 /* 1329 * Get the allowed maximum number of TX queues. 1330 * *pid return the port id which has minimal value of 1331 * max_tx_queues in all ports. 1332 */ 1333 queueid_t 1334 get_allowed_max_nb_txq(portid_t *pid) 1335 { 1336 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1337 bool max_txq_valid = false; 1338 portid_t pi; 1339 struct rte_eth_dev_info dev_info; 1340 1341 RTE_ETH_FOREACH_DEV(pi) { 1342 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1343 continue; 1344 1345 max_txq_valid = true; 1346 if (dev_info.max_tx_queues < allowed_max_txq) { 1347 allowed_max_txq = dev_info.max_tx_queues; 1348 *pid = pi; 1349 } 1350 } 1351 return max_txq_valid ? allowed_max_txq : 0; 1352 } 1353 1354 /* 1355 * Check input txq is valid or not. 1356 * If input txq is not greater than any of maximum number 1357 * of TX queues of all ports, it is valid. 1358 * if valid, return 0, else return -1 1359 */ 1360 int 1361 check_nb_txq(queueid_t txq) 1362 { 1363 queueid_t allowed_max_txq; 1364 portid_t pid = 0; 1365 1366 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1367 if (txq > allowed_max_txq) { 1368 fprintf(stderr, 1369 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1370 txq, allowed_max_txq, pid); 1371 return -1; 1372 } 1373 return 0; 1374 } 1375 1376 /* 1377 * Get the allowed maximum number of RXDs of every rx queue. 1378 * *pid return the port id which has minimal value of 1379 * max_rxd in all queues of all ports. 1380 */ 1381 static uint16_t 1382 get_allowed_max_nb_rxd(portid_t *pid) 1383 { 1384 uint16_t allowed_max_rxd = UINT16_MAX; 1385 portid_t pi; 1386 struct rte_eth_dev_info dev_info; 1387 1388 RTE_ETH_FOREACH_DEV(pi) { 1389 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1390 continue; 1391 1392 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1393 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1394 *pid = pi; 1395 } 1396 } 1397 return allowed_max_rxd; 1398 } 1399 1400 /* 1401 * Get the allowed minimal number of RXDs of every rx queue. 1402 * *pid return the port id which has minimal value of 1403 * min_rxd in all queues of all ports. 1404 */ 1405 static uint16_t 1406 get_allowed_min_nb_rxd(portid_t *pid) 1407 { 1408 uint16_t allowed_min_rxd = 0; 1409 portid_t pi; 1410 struct rte_eth_dev_info dev_info; 1411 1412 RTE_ETH_FOREACH_DEV(pi) { 1413 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1414 continue; 1415 1416 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1417 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1418 *pid = pi; 1419 } 1420 } 1421 1422 return allowed_min_rxd; 1423 } 1424 1425 /* 1426 * Check input rxd is valid or not. 1427 * If input rxd is not greater than any of maximum number 1428 * of RXDs of every Rx queues and is not less than any of 1429 * minimal number of RXDs of every Rx queues, it is valid. 1430 * if valid, return 0, else return -1 1431 */ 1432 int 1433 check_nb_rxd(queueid_t rxd) 1434 { 1435 uint16_t allowed_max_rxd; 1436 uint16_t allowed_min_rxd; 1437 portid_t pid = 0; 1438 1439 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1440 if (rxd > allowed_max_rxd) { 1441 fprintf(stderr, 1442 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1443 rxd, allowed_max_rxd, pid); 1444 return -1; 1445 } 1446 1447 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1448 if (rxd < allowed_min_rxd) { 1449 fprintf(stderr, 1450 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1451 rxd, allowed_min_rxd, pid); 1452 return -1; 1453 } 1454 1455 return 0; 1456 } 1457 1458 /* 1459 * Get the allowed maximum number of TXDs of every rx queues. 1460 * *pid return the port id which has minimal value of 1461 * max_txd in every tx queue. 1462 */ 1463 static uint16_t 1464 get_allowed_max_nb_txd(portid_t *pid) 1465 { 1466 uint16_t allowed_max_txd = UINT16_MAX; 1467 portid_t pi; 1468 struct rte_eth_dev_info dev_info; 1469 1470 RTE_ETH_FOREACH_DEV(pi) { 1471 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1472 continue; 1473 1474 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1475 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1476 *pid = pi; 1477 } 1478 } 1479 return allowed_max_txd; 1480 } 1481 1482 /* 1483 * Get the allowed maximum number of TXDs of every tx queues. 1484 * *pid return the port id which has minimal value of 1485 * min_txd in every tx queue. 1486 */ 1487 static uint16_t 1488 get_allowed_min_nb_txd(portid_t *pid) 1489 { 1490 uint16_t allowed_min_txd = 0; 1491 portid_t pi; 1492 struct rte_eth_dev_info dev_info; 1493 1494 RTE_ETH_FOREACH_DEV(pi) { 1495 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1496 continue; 1497 1498 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1499 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1500 *pid = pi; 1501 } 1502 } 1503 1504 return allowed_min_txd; 1505 } 1506 1507 /* 1508 * Check input txd is valid or not. 1509 * If input txd is not greater than any of maximum number 1510 * of TXDs of every Rx queues, it is valid. 1511 * if valid, return 0, else return -1 1512 */ 1513 int 1514 check_nb_txd(queueid_t txd) 1515 { 1516 uint16_t allowed_max_txd; 1517 uint16_t allowed_min_txd; 1518 portid_t pid = 0; 1519 1520 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1521 if (txd > allowed_max_txd) { 1522 fprintf(stderr, 1523 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1524 txd, allowed_max_txd, pid); 1525 return -1; 1526 } 1527 1528 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1529 if (txd < allowed_min_txd) { 1530 fprintf(stderr, 1531 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1532 txd, allowed_min_txd, pid); 1533 return -1; 1534 } 1535 return 0; 1536 } 1537 1538 1539 /* 1540 * Get the allowed maximum number of hairpin queues. 1541 * *pid return the port id which has minimal value of 1542 * max_hairpin_queues in all ports. 1543 */ 1544 queueid_t 1545 get_allowed_max_nb_hairpinq(portid_t *pid) 1546 { 1547 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1548 portid_t pi; 1549 struct rte_eth_hairpin_cap cap; 1550 1551 RTE_ETH_FOREACH_DEV(pi) { 1552 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1553 *pid = pi; 1554 return 0; 1555 } 1556 if (cap.max_nb_queues < allowed_max_hairpinq) { 1557 allowed_max_hairpinq = cap.max_nb_queues; 1558 *pid = pi; 1559 } 1560 } 1561 return allowed_max_hairpinq; 1562 } 1563 1564 /* 1565 * Check input hairpin is valid or not. 1566 * If input hairpin is not greater than any of maximum number 1567 * of hairpin queues of all ports, it is valid. 1568 * if valid, return 0, else return -1 1569 */ 1570 int 1571 check_nb_hairpinq(queueid_t hairpinq) 1572 { 1573 queueid_t allowed_max_hairpinq; 1574 portid_t pid = 0; 1575 1576 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1577 if (hairpinq > allowed_max_hairpinq) { 1578 fprintf(stderr, 1579 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1580 hairpinq, allowed_max_hairpinq, pid); 1581 return -1; 1582 } 1583 return 0; 1584 } 1585 1586 static int 1587 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1588 { 1589 uint32_t eth_overhead; 1590 1591 if (dev_info->max_mtu != UINT16_MAX && 1592 dev_info->max_rx_pktlen > dev_info->max_mtu) 1593 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1594 else 1595 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1596 1597 return eth_overhead; 1598 } 1599 1600 static void 1601 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1602 { 1603 struct rte_port *port = &ports[pid]; 1604 int ret; 1605 int i; 1606 1607 eth_rx_metadata_negotiate_mp(pid); 1608 1609 port->dev_conf.txmode = tx_mode; 1610 port->dev_conf.rxmode = rx_mode; 1611 1612 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1613 if (ret != 0) 1614 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1615 1616 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1617 port->dev_conf.txmode.offloads &= 1618 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1619 1620 /* Apply Rx offloads configuration */ 1621 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1622 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1623 /* Apply Tx offloads configuration */ 1624 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1625 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1626 1627 if (eth_link_speed) 1628 port->dev_conf.link_speeds = eth_link_speed; 1629 1630 if (max_rx_pkt_len) 1631 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1632 get_eth_overhead(&port->dev_info); 1633 1634 /* set flag to initialize port/queue */ 1635 port->need_reconfig = 1; 1636 port->need_reconfig_queues = 1; 1637 port->socket_id = socket_id; 1638 port->tx_metadata = 0; 1639 1640 /* 1641 * Check for maximum number of segments per MTU. 1642 * Accordingly update the mbuf data size. 1643 */ 1644 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1645 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1646 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1647 uint16_t mtu; 1648 1649 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1650 uint16_t data_size = (mtu + eth_overhead) / 1651 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1652 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1653 1654 if (buffer_size > mbuf_data_size[0]) { 1655 mbuf_data_size[0] = buffer_size; 1656 TESTPMD_LOG(WARNING, 1657 "Configured mbuf size of the first segment %hu\n", 1658 mbuf_data_size[0]); 1659 } 1660 } 1661 } 1662 } 1663 1664 static void 1665 init_config(void) 1666 { 1667 portid_t pid; 1668 struct rte_mempool *mbp; 1669 unsigned int nb_mbuf_per_pool; 1670 lcoreid_t lc_id; 1671 #ifdef RTE_LIB_GRO 1672 struct rte_gro_param gro_param; 1673 #endif 1674 #ifdef RTE_LIB_GSO 1675 uint32_t gso_types; 1676 #endif 1677 1678 /* Configuration of logical cores. */ 1679 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1680 sizeof(struct fwd_lcore *) * nb_lcores, 1681 RTE_CACHE_LINE_SIZE); 1682 if (fwd_lcores == NULL) { 1683 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1684 "failed\n", nb_lcores); 1685 } 1686 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1687 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1688 sizeof(struct fwd_lcore), 1689 RTE_CACHE_LINE_SIZE); 1690 if (fwd_lcores[lc_id] == NULL) { 1691 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1692 "failed\n"); 1693 } 1694 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1695 } 1696 1697 RTE_ETH_FOREACH_DEV(pid) { 1698 uint32_t socket_id; 1699 1700 if (numa_support) { 1701 socket_id = port_numa[pid]; 1702 if (port_numa[pid] == NUMA_NO_CONFIG) { 1703 socket_id = rte_eth_dev_socket_id(pid); 1704 1705 /* 1706 * if socket_id is invalid, 1707 * set to the first available socket. 1708 */ 1709 if (check_socket_id(socket_id) < 0) 1710 socket_id = socket_ids[0]; 1711 } 1712 } else { 1713 socket_id = (socket_num == UMA_NO_CONFIG) ? 1714 0 : socket_num; 1715 } 1716 /* Apply default TxRx configuration for all ports */ 1717 init_config_port_offloads(pid, socket_id); 1718 } 1719 /* 1720 * Create pools of mbuf. 1721 * If NUMA support is disabled, create a single pool of mbuf in 1722 * socket 0 memory by default. 1723 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1724 * 1725 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1726 * nb_txd can be configured at run time. 1727 */ 1728 if (param_total_num_mbufs) 1729 nb_mbuf_per_pool = param_total_num_mbufs; 1730 else { 1731 nb_mbuf_per_pool = RX_DESC_MAX + 1732 (nb_lcores * mb_mempool_cache) + 1733 TX_DESC_MAX + MAX_PKT_BURST; 1734 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1735 } 1736 1737 if (numa_support) { 1738 uint8_t i, j; 1739 1740 for (i = 0; i < num_sockets; i++) 1741 for (j = 0; j < mbuf_data_size_n; j++) 1742 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1743 mbuf_pool_create(mbuf_data_size[j], 1744 nb_mbuf_per_pool, 1745 socket_ids[i], j); 1746 } else { 1747 uint8_t i; 1748 1749 for (i = 0; i < mbuf_data_size_n; i++) 1750 mempools[i] = mbuf_pool_create 1751 (mbuf_data_size[i], 1752 nb_mbuf_per_pool, 1753 socket_num == UMA_NO_CONFIG ? 1754 0 : socket_num, i); 1755 } 1756 1757 init_port_config(); 1758 1759 #ifdef RTE_LIB_GSO 1760 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1761 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1762 #endif 1763 /* 1764 * Records which Mbuf pool to use by each logical core, if needed. 1765 */ 1766 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1767 mbp = mbuf_pool_find( 1768 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1769 1770 if (mbp == NULL) 1771 mbp = mbuf_pool_find(0, 0); 1772 fwd_lcores[lc_id]->mbp = mbp; 1773 #ifdef RTE_LIB_GSO 1774 /* initialize GSO context */ 1775 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1776 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1777 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1778 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1779 RTE_ETHER_CRC_LEN; 1780 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1781 #endif 1782 } 1783 1784 fwd_config_setup(); 1785 1786 #ifdef RTE_LIB_GRO 1787 /* create a gro context for each lcore */ 1788 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1789 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1790 gro_param.max_item_per_flow = MAX_PKT_BURST; 1791 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1792 gro_param.socket_id = rte_lcore_to_socket_id( 1793 fwd_lcores_cpuids[lc_id]); 1794 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1795 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1796 rte_exit(EXIT_FAILURE, 1797 "rte_gro_ctx_create() failed\n"); 1798 } 1799 } 1800 #endif 1801 } 1802 1803 1804 void 1805 reconfig(portid_t new_port_id, unsigned socket_id) 1806 { 1807 /* Reconfiguration of Ethernet ports. */ 1808 init_config_port_offloads(new_port_id, socket_id); 1809 init_port_config(); 1810 } 1811 1812 int 1813 init_fwd_streams(void) 1814 { 1815 portid_t pid; 1816 struct rte_port *port; 1817 streamid_t sm_id, nb_fwd_streams_new; 1818 queueid_t q; 1819 1820 /* set socket id according to numa or not */ 1821 RTE_ETH_FOREACH_DEV(pid) { 1822 port = &ports[pid]; 1823 if (nb_rxq > port->dev_info.max_rx_queues) { 1824 fprintf(stderr, 1825 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1826 nb_rxq, port->dev_info.max_rx_queues); 1827 return -1; 1828 } 1829 if (nb_txq > port->dev_info.max_tx_queues) { 1830 fprintf(stderr, 1831 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1832 nb_txq, port->dev_info.max_tx_queues); 1833 return -1; 1834 } 1835 if (numa_support) { 1836 if (port_numa[pid] != NUMA_NO_CONFIG) 1837 port->socket_id = port_numa[pid]; 1838 else { 1839 port->socket_id = rte_eth_dev_socket_id(pid); 1840 1841 /* 1842 * if socket_id is invalid, 1843 * set to the first available socket. 1844 */ 1845 if (check_socket_id(port->socket_id) < 0) 1846 port->socket_id = socket_ids[0]; 1847 } 1848 } 1849 else { 1850 if (socket_num == UMA_NO_CONFIG) 1851 port->socket_id = 0; 1852 else 1853 port->socket_id = socket_num; 1854 } 1855 } 1856 1857 q = RTE_MAX(nb_rxq, nb_txq); 1858 if (q == 0) { 1859 fprintf(stderr, 1860 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1861 return -1; 1862 } 1863 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1864 if (nb_fwd_streams_new == nb_fwd_streams) 1865 return 0; 1866 /* clear the old */ 1867 if (fwd_streams != NULL) { 1868 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1869 if (fwd_streams[sm_id] == NULL) 1870 continue; 1871 rte_free(fwd_streams[sm_id]); 1872 fwd_streams[sm_id] = NULL; 1873 } 1874 rte_free(fwd_streams); 1875 fwd_streams = NULL; 1876 } 1877 1878 /* init new */ 1879 nb_fwd_streams = nb_fwd_streams_new; 1880 if (nb_fwd_streams) { 1881 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1882 sizeof(struct fwd_stream *) * nb_fwd_streams, 1883 RTE_CACHE_LINE_SIZE); 1884 if (fwd_streams == NULL) 1885 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1886 " (struct fwd_stream *)) failed\n", 1887 nb_fwd_streams); 1888 1889 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1890 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1891 " struct fwd_stream", sizeof(struct fwd_stream), 1892 RTE_CACHE_LINE_SIZE); 1893 if (fwd_streams[sm_id] == NULL) 1894 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1895 "(struct fwd_stream) failed\n"); 1896 } 1897 } 1898 1899 return 0; 1900 } 1901 1902 static void 1903 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1904 { 1905 uint64_t total_burst, sburst; 1906 uint64_t nb_burst; 1907 uint64_t burst_stats[4]; 1908 uint16_t pktnb_stats[4]; 1909 uint16_t nb_pkt; 1910 int burst_percent[4], sburstp; 1911 int i; 1912 1913 /* 1914 * First compute the total number of packet bursts and the 1915 * two highest numbers of bursts of the same number of packets. 1916 */ 1917 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1918 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1919 1920 /* Show stats for 0 burst size always */ 1921 total_burst = pbs->pkt_burst_spread[0]; 1922 burst_stats[0] = pbs->pkt_burst_spread[0]; 1923 pktnb_stats[0] = 0; 1924 1925 /* Find the next 2 burst sizes with highest occurrences. */ 1926 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1927 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1928 1929 if (nb_burst == 0) 1930 continue; 1931 1932 total_burst += nb_burst; 1933 1934 if (nb_burst > burst_stats[1]) { 1935 burst_stats[2] = burst_stats[1]; 1936 pktnb_stats[2] = pktnb_stats[1]; 1937 burst_stats[1] = nb_burst; 1938 pktnb_stats[1] = nb_pkt; 1939 } else if (nb_burst > burst_stats[2]) { 1940 burst_stats[2] = nb_burst; 1941 pktnb_stats[2] = nb_pkt; 1942 } 1943 } 1944 if (total_burst == 0) 1945 return; 1946 1947 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1948 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1949 if (i == 3) { 1950 printf("%d%% of other]\n", 100 - sburstp); 1951 return; 1952 } 1953 1954 sburst += burst_stats[i]; 1955 if (sburst == total_burst) { 1956 printf("%d%% of %d pkts]\n", 1957 100 - sburstp, (int) pktnb_stats[i]); 1958 return; 1959 } 1960 1961 burst_percent[i] = 1962 (double)burst_stats[i] / total_burst * 100; 1963 printf("%d%% of %d pkts + ", 1964 burst_percent[i], (int) pktnb_stats[i]); 1965 sburstp += burst_percent[i]; 1966 } 1967 } 1968 1969 static void 1970 fwd_stream_stats_display(streamid_t stream_id) 1971 { 1972 struct fwd_stream *fs; 1973 static const char *fwd_top_stats_border = "-------"; 1974 1975 fs = fwd_streams[stream_id]; 1976 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1977 (fs->fwd_dropped == 0)) 1978 return; 1979 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1980 "TX Port=%2d/Queue=%2d %s\n", 1981 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1982 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1983 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1984 " TX-dropped: %-14"PRIu64, 1985 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1986 1987 /* if checksum mode */ 1988 if (cur_fwd_eng == &csum_fwd_engine) { 1989 printf(" RX- bad IP checksum: %-14"PRIu64 1990 " Rx- bad L4 checksum: %-14"PRIu64 1991 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1992 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1993 fs->rx_bad_outer_l4_csum); 1994 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1995 fs->rx_bad_outer_ip_csum); 1996 } else { 1997 printf("\n"); 1998 } 1999 2000 if (record_burst_stats) { 2001 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2002 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2003 } 2004 } 2005 2006 void 2007 fwd_stats_display(void) 2008 { 2009 static const char *fwd_stats_border = "----------------------"; 2010 static const char *acc_stats_border = "+++++++++++++++"; 2011 struct { 2012 struct fwd_stream *rx_stream; 2013 struct fwd_stream *tx_stream; 2014 uint64_t tx_dropped; 2015 uint64_t rx_bad_ip_csum; 2016 uint64_t rx_bad_l4_csum; 2017 uint64_t rx_bad_outer_l4_csum; 2018 uint64_t rx_bad_outer_ip_csum; 2019 } ports_stats[RTE_MAX_ETHPORTS]; 2020 uint64_t total_rx_dropped = 0; 2021 uint64_t total_tx_dropped = 0; 2022 uint64_t total_rx_nombuf = 0; 2023 struct rte_eth_stats stats; 2024 uint64_t fwd_cycles = 0; 2025 uint64_t total_recv = 0; 2026 uint64_t total_xmit = 0; 2027 struct rte_port *port; 2028 streamid_t sm_id; 2029 portid_t pt_id; 2030 int ret; 2031 int i; 2032 2033 memset(ports_stats, 0, sizeof(ports_stats)); 2034 2035 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2036 struct fwd_stream *fs = fwd_streams[sm_id]; 2037 2038 if (cur_fwd_config.nb_fwd_streams > 2039 cur_fwd_config.nb_fwd_ports) { 2040 fwd_stream_stats_display(sm_id); 2041 } else { 2042 ports_stats[fs->tx_port].tx_stream = fs; 2043 ports_stats[fs->rx_port].rx_stream = fs; 2044 } 2045 2046 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2047 2048 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2049 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2050 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2051 fs->rx_bad_outer_l4_csum; 2052 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2053 fs->rx_bad_outer_ip_csum; 2054 2055 if (record_core_cycles) 2056 fwd_cycles += fs->core_cycles; 2057 } 2058 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2059 pt_id = fwd_ports_ids[i]; 2060 port = &ports[pt_id]; 2061 2062 ret = rte_eth_stats_get(pt_id, &stats); 2063 if (ret != 0) { 2064 fprintf(stderr, 2065 "%s: Error: failed to get stats (port %u): %d", 2066 __func__, pt_id, ret); 2067 continue; 2068 } 2069 stats.ipackets -= port->stats.ipackets; 2070 stats.opackets -= port->stats.opackets; 2071 stats.ibytes -= port->stats.ibytes; 2072 stats.obytes -= port->stats.obytes; 2073 stats.imissed -= port->stats.imissed; 2074 stats.oerrors -= port->stats.oerrors; 2075 stats.rx_nombuf -= port->stats.rx_nombuf; 2076 2077 total_recv += stats.ipackets; 2078 total_xmit += stats.opackets; 2079 total_rx_dropped += stats.imissed; 2080 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2081 total_tx_dropped += stats.oerrors; 2082 total_rx_nombuf += stats.rx_nombuf; 2083 2084 printf("\n %s Forward statistics for port %-2d %s\n", 2085 fwd_stats_border, pt_id, fwd_stats_border); 2086 2087 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2088 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2089 stats.ipackets + stats.imissed); 2090 2091 if (cur_fwd_eng == &csum_fwd_engine) { 2092 printf(" Bad-ipcsum: %-14"PRIu64 2093 " Bad-l4csum: %-14"PRIu64 2094 "Bad-outer-l4csum: %-14"PRIu64"\n", 2095 ports_stats[pt_id].rx_bad_ip_csum, 2096 ports_stats[pt_id].rx_bad_l4_csum, 2097 ports_stats[pt_id].rx_bad_outer_l4_csum); 2098 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2099 ports_stats[pt_id].rx_bad_outer_ip_csum); 2100 } 2101 if (stats.ierrors + stats.rx_nombuf > 0) { 2102 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2103 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2104 } 2105 2106 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2107 "TX-total: %-"PRIu64"\n", 2108 stats.opackets, ports_stats[pt_id].tx_dropped, 2109 stats.opackets + ports_stats[pt_id].tx_dropped); 2110 2111 if (record_burst_stats) { 2112 if (ports_stats[pt_id].rx_stream) 2113 pkt_burst_stats_display("RX", 2114 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2115 if (ports_stats[pt_id].tx_stream) 2116 pkt_burst_stats_display("TX", 2117 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2118 } 2119 2120 printf(" %s--------------------------------%s\n", 2121 fwd_stats_border, fwd_stats_border); 2122 } 2123 2124 printf("\n %s Accumulated forward statistics for all ports" 2125 "%s\n", 2126 acc_stats_border, acc_stats_border); 2127 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2128 "%-"PRIu64"\n" 2129 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2130 "%-"PRIu64"\n", 2131 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2132 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2133 if (total_rx_nombuf > 0) 2134 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2135 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2136 "%s\n", 2137 acc_stats_border, acc_stats_border); 2138 if (record_core_cycles) { 2139 #define CYC_PER_MHZ 1E6 2140 if (total_recv > 0 || total_xmit > 0) { 2141 uint64_t total_pkts = 0; 2142 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2143 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2144 total_pkts = total_xmit; 2145 else 2146 total_pkts = total_recv; 2147 2148 printf("\n CPU cycles/packet=%.2F (total cycles=" 2149 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2150 " MHz Clock\n", 2151 (double) fwd_cycles / total_pkts, 2152 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2153 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2154 } 2155 } 2156 } 2157 2158 void 2159 fwd_stats_reset(void) 2160 { 2161 streamid_t sm_id; 2162 portid_t pt_id; 2163 int ret; 2164 int i; 2165 2166 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2167 pt_id = fwd_ports_ids[i]; 2168 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2169 if (ret != 0) 2170 fprintf(stderr, 2171 "%s: Error: failed to clear stats (port %u):%d", 2172 __func__, pt_id, ret); 2173 } 2174 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2175 struct fwd_stream *fs = fwd_streams[sm_id]; 2176 2177 fs->rx_packets = 0; 2178 fs->tx_packets = 0; 2179 fs->fwd_dropped = 0; 2180 fs->rx_bad_ip_csum = 0; 2181 fs->rx_bad_l4_csum = 0; 2182 fs->rx_bad_outer_l4_csum = 0; 2183 fs->rx_bad_outer_ip_csum = 0; 2184 2185 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2186 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2187 fs->core_cycles = 0; 2188 } 2189 } 2190 2191 static void 2192 flush_fwd_rx_queues(void) 2193 { 2194 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2195 portid_t rxp; 2196 portid_t port_id; 2197 queueid_t rxq; 2198 uint16_t nb_rx; 2199 uint16_t i; 2200 uint8_t j; 2201 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2202 uint64_t timer_period; 2203 2204 if (num_procs > 1) { 2205 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2206 return; 2207 } 2208 2209 /* convert to number of cycles */ 2210 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2211 2212 for (j = 0; j < 2; j++) { 2213 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2214 for (rxq = 0; rxq < nb_rxq; rxq++) { 2215 port_id = fwd_ports_ids[rxp]; 2216 2217 /* Polling stopped queues is prohibited. */ 2218 if (ports[port_id].rxq[rxq].state == 2219 RTE_ETH_QUEUE_STATE_STOPPED) 2220 continue; 2221 2222 /** 2223 * testpmd can stuck in the below do while loop 2224 * if rte_eth_rx_burst() always returns nonzero 2225 * packets. So timer is added to exit this loop 2226 * after 1sec timer expiry. 2227 */ 2228 prev_tsc = rte_rdtsc(); 2229 do { 2230 nb_rx = rte_eth_rx_burst(port_id, rxq, 2231 pkts_burst, MAX_PKT_BURST); 2232 for (i = 0; i < nb_rx; i++) 2233 rte_pktmbuf_free(pkts_burst[i]); 2234 2235 cur_tsc = rte_rdtsc(); 2236 diff_tsc = cur_tsc - prev_tsc; 2237 timer_tsc += diff_tsc; 2238 } while ((nb_rx > 0) && 2239 (timer_tsc < timer_period)); 2240 timer_tsc = 0; 2241 } 2242 } 2243 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2244 } 2245 } 2246 2247 static void 2248 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2249 { 2250 struct fwd_stream **fsm; 2251 streamid_t nb_fs; 2252 streamid_t sm_id; 2253 #ifdef RTE_LIB_BITRATESTATS 2254 uint64_t tics_per_1sec; 2255 uint64_t tics_datum; 2256 uint64_t tics_current; 2257 uint16_t i, cnt_ports; 2258 2259 cnt_ports = nb_ports; 2260 tics_datum = rte_rdtsc(); 2261 tics_per_1sec = rte_get_timer_hz(); 2262 #endif 2263 fsm = &fwd_streams[fc->stream_idx]; 2264 nb_fs = fc->stream_nb; 2265 do { 2266 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2267 if (!fsm[sm_id]->disabled) 2268 (*pkt_fwd)(fsm[sm_id]); 2269 #ifdef RTE_LIB_BITRATESTATS 2270 if (bitrate_enabled != 0 && 2271 bitrate_lcore_id == rte_lcore_id()) { 2272 tics_current = rte_rdtsc(); 2273 if (tics_current - tics_datum >= tics_per_1sec) { 2274 /* Periodic bitrate calculation */ 2275 for (i = 0; i < cnt_ports; i++) 2276 rte_stats_bitrate_calc(bitrate_data, 2277 ports_ids[i]); 2278 tics_datum = tics_current; 2279 } 2280 } 2281 #endif 2282 #ifdef RTE_LIB_LATENCYSTATS 2283 if (latencystats_enabled != 0 && 2284 latencystats_lcore_id == rte_lcore_id()) 2285 rte_latencystats_update(); 2286 #endif 2287 2288 } while (! fc->stopped); 2289 } 2290 2291 static int 2292 start_pkt_forward_on_core(void *fwd_arg) 2293 { 2294 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2295 cur_fwd_config.fwd_eng->packet_fwd); 2296 return 0; 2297 } 2298 2299 /* 2300 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2301 * Used to start communication flows in network loopback test configurations. 2302 */ 2303 static int 2304 run_one_txonly_burst_on_core(void *fwd_arg) 2305 { 2306 struct fwd_lcore *fwd_lc; 2307 struct fwd_lcore tmp_lcore; 2308 2309 fwd_lc = (struct fwd_lcore *) fwd_arg; 2310 tmp_lcore = *fwd_lc; 2311 tmp_lcore.stopped = 1; 2312 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2313 return 0; 2314 } 2315 2316 /* 2317 * Launch packet forwarding: 2318 * - Setup per-port forwarding context. 2319 * - launch logical cores with their forwarding configuration. 2320 */ 2321 static void 2322 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2323 { 2324 unsigned int i; 2325 unsigned int lc_id; 2326 int diag; 2327 2328 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2329 lc_id = fwd_lcores_cpuids[i]; 2330 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2331 fwd_lcores[i]->stopped = 0; 2332 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2333 fwd_lcores[i], lc_id); 2334 if (diag != 0) 2335 fprintf(stderr, 2336 "launch lcore %u failed - diag=%d\n", 2337 lc_id, diag); 2338 } 2339 } 2340 } 2341 2342 /* 2343 * Launch packet forwarding configuration. 2344 */ 2345 void 2346 start_packet_forwarding(int with_tx_first) 2347 { 2348 port_fwd_begin_t port_fwd_begin; 2349 port_fwd_end_t port_fwd_end; 2350 stream_init_t stream_init = cur_fwd_eng->stream_init; 2351 unsigned int i; 2352 2353 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2354 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2355 2356 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2357 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2358 2359 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2360 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2361 (!nb_rxq || !nb_txq)) 2362 rte_exit(EXIT_FAILURE, 2363 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2364 cur_fwd_eng->fwd_mode_name); 2365 2366 if (all_ports_started() == 0) { 2367 fprintf(stderr, "Not all ports were started\n"); 2368 return; 2369 } 2370 if (test_done == 0) { 2371 fprintf(stderr, "Packet forwarding already started\n"); 2372 return; 2373 } 2374 2375 fwd_config_setup(); 2376 2377 pkt_fwd_config_display(&cur_fwd_config); 2378 if (!pkt_fwd_shared_rxq_check()) 2379 return; 2380 2381 if (stream_init != NULL) 2382 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2383 stream_init(fwd_streams[i]); 2384 2385 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2386 if (port_fwd_begin != NULL) { 2387 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2388 if (port_fwd_begin(fwd_ports_ids[i])) { 2389 fprintf(stderr, 2390 "Packet forwarding is not ready\n"); 2391 return; 2392 } 2393 } 2394 } 2395 2396 if (with_tx_first) { 2397 port_fwd_begin = tx_only_engine.port_fwd_begin; 2398 if (port_fwd_begin != NULL) { 2399 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2400 if (port_fwd_begin(fwd_ports_ids[i])) { 2401 fprintf(stderr, 2402 "Packet forwarding is not ready\n"); 2403 return; 2404 } 2405 } 2406 } 2407 } 2408 2409 test_done = 0; 2410 2411 if(!no_flush_rx) 2412 flush_fwd_rx_queues(); 2413 2414 rxtx_config_display(); 2415 2416 fwd_stats_reset(); 2417 if (with_tx_first) { 2418 while (with_tx_first--) { 2419 launch_packet_forwarding( 2420 run_one_txonly_burst_on_core); 2421 rte_eal_mp_wait_lcore(); 2422 } 2423 port_fwd_end = tx_only_engine.port_fwd_end; 2424 if (port_fwd_end != NULL) { 2425 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2426 (*port_fwd_end)(fwd_ports_ids[i]); 2427 } 2428 } 2429 launch_packet_forwarding(start_pkt_forward_on_core); 2430 } 2431 2432 void 2433 stop_packet_forwarding(void) 2434 { 2435 port_fwd_end_t port_fwd_end; 2436 lcoreid_t lc_id; 2437 portid_t pt_id; 2438 int i; 2439 2440 if (test_done) { 2441 fprintf(stderr, "Packet forwarding not started\n"); 2442 return; 2443 } 2444 printf("Telling cores to stop..."); 2445 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2446 fwd_lcores[lc_id]->stopped = 1; 2447 printf("\nWaiting for lcores to finish...\n"); 2448 rte_eal_mp_wait_lcore(); 2449 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2450 if (port_fwd_end != NULL) { 2451 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2452 pt_id = fwd_ports_ids[i]; 2453 (*port_fwd_end)(pt_id); 2454 } 2455 } 2456 2457 fwd_stats_display(); 2458 2459 printf("\nDone.\n"); 2460 test_done = 1; 2461 } 2462 2463 void 2464 dev_set_link_up(portid_t pid) 2465 { 2466 if (rte_eth_dev_set_link_up(pid) < 0) 2467 fprintf(stderr, "\nSet link up fail.\n"); 2468 } 2469 2470 void 2471 dev_set_link_down(portid_t pid) 2472 { 2473 if (rte_eth_dev_set_link_down(pid) < 0) 2474 fprintf(stderr, "\nSet link down fail.\n"); 2475 } 2476 2477 static int 2478 all_ports_started(void) 2479 { 2480 portid_t pi; 2481 struct rte_port *port; 2482 2483 RTE_ETH_FOREACH_DEV(pi) { 2484 port = &ports[pi]; 2485 /* Check if there is a port which is not started */ 2486 if ((port->port_status != RTE_PORT_STARTED) && 2487 (port->slave_flag == 0)) 2488 return 0; 2489 } 2490 2491 /* No port is not started */ 2492 return 1; 2493 } 2494 2495 int 2496 port_is_stopped(portid_t port_id) 2497 { 2498 struct rte_port *port = &ports[port_id]; 2499 2500 if ((port->port_status != RTE_PORT_STOPPED) && 2501 (port->slave_flag == 0)) 2502 return 0; 2503 return 1; 2504 } 2505 2506 int 2507 all_ports_stopped(void) 2508 { 2509 portid_t pi; 2510 2511 RTE_ETH_FOREACH_DEV(pi) { 2512 if (!port_is_stopped(pi)) 2513 return 0; 2514 } 2515 2516 return 1; 2517 } 2518 2519 int 2520 port_is_started(portid_t port_id) 2521 { 2522 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2523 return 0; 2524 2525 if (ports[port_id].port_status != RTE_PORT_STARTED) 2526 return 0; 2527 2528 return 1; 2529 } 2530 2531 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2532 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2533 2534 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2535 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2536 2537 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2538 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2539 2540 2541 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2542 static int 2543 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2544 { 2545 queueid_t qi; 2546 struct rte_eth_hairpin_conf hairpin_conf = { 2547 .peer_count = 1, 2548 }; 2549 int i; 2550 int diag; 2551 struct rte_port *port = &ports[pi]; 2552 uint16_t peer_rx_port = pi; 2553 uint16_t peer_tx_port = pi; 2554 uint32_t manual = 1; 2555 uint32_t tx_exp = hairpin_mode & 0x10; 2556 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2557 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2558 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2559 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2560 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2561 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2562 2563 if (!(hairpin_mode & 0xf)) { 2564 peer_rx_port = pi; 2565 peer_tx_port = pi; 2566 manual = 0; 2567 } else if (hairpin_mode & 0x1) { 2568 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2569 RTE_ETH_DEV_NO_OWNER); 2570 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2571 peer_tx_port = rte_eth_find_next_owned_by(0, 2572 RTE_ETH_DEV_NO_OWNER); 2573 if (p_pi != RTE_MAX_ETHPORTS) { 2574 peer_rx_port = p_pi; 2575 } else { 2576 uint16_t next_pi; 2577 2578 /* Last port will be the peer RX port of the first. */ 2579 RTE_ETH_FOREACH_DEV(next_pi) 2580 peer_rx_port = next_pi; 2581 } 2582 manual = 1; 2583 } else if (hairpin_mode & 0x2) { 2584 if (cnt_pi & 0x1) { 2585 peer_rx_port = p_pi; 2586 } else { 2587 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2588 RTE_ETH_DEV_NO_OWNER); 2589 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2590 peer_rx_port = pi; 2591 } 2592 peer_tx_port = peer_rx_port; 2593 manual = 1; 2594 } 2595 2596 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2597 hairpin_conf.peers[0].port = peer_rx_port; 2598 hairpin_conf.peers[0].queue = i + nb_rxq; 2599 hairpin_conf.manual_bind = !!manual; 2600 hairpin_conf.tx_explicit = !!tx_exp; 2601 hairpin_conf.force_memory = !!tx_force_memory; 2602 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2603 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2604 diag = rte_eth_tx_hairpin_queue_setup 2605 (pi, qi, nb_txd, &hairpin_conf); 2606 i++; 2607 if (diag == 0) 2608 continue; 2609 2610 /* Fail to setup rx queue, return */ 2611 if (port->port_status == RTE_PORT_HANDLING) 2612 port->port_status = RTE_PORT_STOPPED; 2613 else 2614 fprintf(stderr, 2615 "Port %d can not be set back to stopped\n", pi); 2616 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2617 pi); 2618 /* try to reconfigure queues next time */ 2619 port->need_reconfig_queues = 1; 2620 return -1; 2621 } 2622 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2623 hairpin_conf.peers[0].port = peer_tx_port; 2624 hairpin_conf.peers[0].queue = i + nb_txq; 2625 hairpin_conf.manual_bind = !!manual; 2626 hairpin_conf.tx_explicit = !!tx_exp; 2627 hairpin_conf.force_memory = !!rx_force_memory; 2628 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2629 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2630 diag = rte_eth_rx_hairpin_queue_setup 2631 (pi, qi, nb_rxd, &hairpin_conf); 2632 i++; 2633 if (diag == 0) 2634 continue; 2635 2636 /* Fail to setup rx queue, return */ 2637 if (port->port_status == RTE_PORT_HANDLING) 2638 port->port_status = RTE_PORT_STOPPED; 2639 else 2640 fprintf(stderr, 2641 "Port %d can not be set back to stopped\n", pi); 2642 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2643 pi); 2644 /* try to reconfigure queues next time */ 2645 port->need_reconfig_queues = 1; 2646 return -1; 2647 } 2648 return 0; 2649 } 2650 2651 /* Configure the Rx with optional split. */ 2652 int 2653 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2654 uint16_t nb_rx_desc, unsigned int socket_id, 2655 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2656 { 2657 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2658 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2659 struct rte_mempool *mpx; 2660 unsigned int i, mp_n; 2661 uint32_t prev_hdrs = 0; 2662 int ret; 2663 2664 2665 if ((rx_pkt_nb_segs > 1) && 2666 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2667 /* multi-segment configuration */ 2668 for (i = 0; i < rx_pkt_nb_segs; i++) { 2669 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2670 /* 2671 * Use last valid pool for the segments with number 2672 * exceeding the pool index. 2673 */ 2674 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2675 mpx = mbuf_pool_find(socket_id, mp_n); 2676 /* Handle zero as mbuf data buffer size. */ 2677 rx_seg->offset = i < rx_pkt_nb_offs ? 2678 rx_pkt_seg_offsets[i] : 0; 2679 rx_seg->mp = mpx ? mpx : mp; 2680 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2681 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2682 prev_hdrs |= rx_seg->proto_hdr; 2683 } else { 2684 rx_seg->length = rx_pkt_seg_lengths[i] ? 2685 rx_pkt_seg_lengths[i] : 2686 mbuf_data_size[mp_n]; 2687 } 2688 } 2689 rx_conf->rx_nseg = rx_pkt_nb_segs; 2690 rx_conf->rx_seg = rx_useg; 2691 rx_conf->rx_mempools = NULL; 2692 rx_conf->rx_nmempool = 0; 2693 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2694 socket_id, rx_conf, NULL); 2695 rx_conf->rx_seg = NULL; 2696 rx_conf->rx_nseg = 0; 2697 } else if (multi_rx_mempool == 1) { 2698 /* multi-pool configuration */ 2699 struct rte_eth_dev_info dev_info; 2700 2701 if (mbuf_data_size_n <= 1) { 2702 fprintf(stderr, "Invalid number of mempools %u\n", 2703 mbuf_data_size_n); 2704 return -EINVAL; 2705 } 2706 ret = rte_eth_dev_info_get(port_id, &dev_info); 2707 if (ret != 0) 2708 return ret; 2709 if (dev_info.max_rx_mempools == 0) { 2710 fprintf(stderr, 2711 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2712 port_id); 2713 return -ENOTSUP; 2714 } 2715 for (i = 0; i < mbuf_data_size_n; i++) { 2716 mpx = mbuf_pool_find(socket_id, i); 2717 rx_mempool[i] = mpx ? mpx : mp; 2718 } 2719 rx_conf->rx_mempools = rx_mempool; 2720 rx_conf->rx_nmempool = mbuf_data_size_n; 2721 rx_conf->rx_seg = NULL; 2722 rx_conf->rx_nseg = 0; 2723 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2724 socket_id, rx_conf, NULL); 2725 rx_conf->rx_mempools = NULL; 2726 rx_conf->rx_nmempool = 0; 2727 } else { 2728 /* Single pool/segment configuration */ 2729 rx_conf->rx_seg = NULL; 2730 rx_conf->rx_nseg = 0; 2731 rx_conf->rx_mempools = NULL; 2732 rx_conf->rx_nmempool = 0; 2733 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2734 socket_id, rx_conf, mp); 2735 } 2736 2737 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2738 RTE_ETH_QUEUE_STATE_STOPPED : 2739 RTE_ETH_QUEUE_STATE_STARTED; 2740 return ret; 2741 } 2742 2743 static int 2744 alloc_xstats_display_info(portid_t pi) 2745 { 2746 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2747 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2748 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2749 2750 if (xstats_display_num == 0) 2751 return 0; 2752 2753 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2754 if (*ids_supp == NULL) 2755 goto fail_ids_supp; 2756 2757 *prev_values = calloc(xstats_display_num, 2758 sizeof(**prev_values)); 2759 if (*prev_values == NULL) 2760 goto fail_prev_values; 2761 2762 *curr_values = calloc(xstats_display_num, 2763 sizeof(**curr_values)); 2764 if (*curr_values == NULL) 2765 goto fail_curr_values; 2766 2767 ports[pi].xstats_info.allocated = true; 2768 2769 return 0; 2770 2771 fail_curr_values: 2772 free(*prev_values); 2773 fail_prev_values: 2774 free(*ids_supp); 2775 fail_ids_supp: 2776 return -ENOMEM; 2777 } 2778 2779 static void 2780 free_xstats_display_info(portid_t pi) 2781 { 2782 if (!ports[pi].xstats_info.allocated) 2783 return; 2784 free(ports[pi].xstats_info.ids_supp); 2785 free(ports[pi].xstats_info.prev_values); 2786 free(ports[pi].xstats_info.curr_values); 2787 ports[pi].xstats_info.allocated = false; 2788 } 2789 2790 /** Fill helper structures for specified port to show extended statistics. */ 2791 static void 2792 fill_xstats_display_info_for_port(portid_t pi) 2793 { 2794 unsigned int stat, stat_supp; 2795 const char *xstat_name; 2796 struct rte_port *port; 2797 uint64_t *ids_supp; 2798 int rc; 2799 2800 if (xstats_display_num == 0) 2801 return; 2802 2803 if (pi == (portid_t)RTE_PORT_ALL) { 2804 fill_xstats_display_info(); 2805 return; 2806 } 2807 2808 port = &ports[pi]; 2809 if (port->port_status != RTE_PORT_STARTED) 2810 return; 2811 2812 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2813 rte_exit(EXIT_FAILURE, 2814 "Failed to allocate xstats display memory\n"); 2815 2816 ids_supp = port->xstats_info.ids_supp; 2817 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2818 xstat_name = xstats_display[stat].name; 2819 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2820 ids_supp + stat_supp); 2821 if (rc != 0) { 2822 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2823 xstat_name, pi, stat); 2824 continue; 2825 } 2826 stat_supp++; 2827 } 2828 2829 port->xstats_info.ids_supp_sz = stat_supp; 2830 } 2831 2832 /** Fill helper structures for all ports to show extended statistics. */ 2833 static void 2834 fill_xstats_display_info(void) 2835 { 2836 portid_t pi; 2837 2838 if (xstats_display_num == 0) 2839 return; 2840 2841 RTE_ETH_FOREACH_DEV(pi) 2842 fill_xstats_display_info_for_port(pi); 2843 } 2844 2845 /* 2846 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2847 * device in dev_info is zero when no slave is added. And its capability 2848 * will be updated when add a new slave device. So adding a slave device need 2849 * to update the port configurations of bonding device. 2850 */ 2851 static void 2852 update_bonding_port_dev_conf(portid_t bond_pid) 2853 { 2854 #ifdef RTE_NET_BOND 2855 struct rte_port *port = &ports[bond_pid]; 2856 uint16_t i; 2857 int ret; 2858 2859 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2860 if (ret != 0) { 2861 fprintf(stderr, "Failed to get dev info for port = %u\n", 2862 bond_pid); 2863 return; 2864 } 2865 2866 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2867 port->dev_conf.txmode.offloads |= 2868 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2869 /* Apply Tx offloads configuration */ 2870 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2871 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2872 2873 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 2874 port->dev_info.flow_type_rss_offloads; 2875 #else 2876 RTE_SET_USED(bond_pid); 2877 #endif 2878 } 2879 2880 int 2881 start_port(portid_t pid) 2882 { 2883 int diag, need_check_link_status = -1; 2884 portid_t pi; 2885 portid_t p_pi = RTE_MAX_ETHPORTS; 2886 portid_t pl[RTE_MAX_ETHPORTS]; 2887 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2888 uint16_t cnt_pi = 0; 2889 uint16_t cfg_pi = 0; 2890 int peer_pi; 2891 queueid_t qi; 2892 struct rte_port *port; 2893 struct rte_eth_hairpin_cap cap; 2894 2895 if (port_id_is_invalid(pid, ENABLED_WARN)) 2896 return 0; 2897 2898 RTE_ETH_FOREACH_DEV(pi) { 2899 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2900 continue; 2901 2902 if (port_is_bonding_slave(pi)) { 2903 fprintf(stderr, 2904 "Please remove port %d from bonded device.\n", 2905 pi); 2906 continue; 2907 } 2908 2909 need_check_link_status = 0; 2910 port = &ports[pi]; 2911 if (port->port_status == RTE_PORT_STOPPED) 2912 port->port_status = RTE_PORT_HANDLING; 2913 else { 2914 fprintf(stderr, "Port %d is now not stopped\n", pi); 2915 continue; 2916 } 2917 2918 if (port->need_reconfig > 0) { 2919 struct rte_eth_conf dev_conf; 2920 int k; 2921 2922 port->need_reconfig = 0; 2923 2924 if (flow_isolate_all) { 2925 int ret = port_flow_isolate(pi, 1); 2926 if (ret) { 2927 fprintf(stderr, 2928 "Failed to apply isolated mode on port %d\n", 2929 pi); 2930 return -1; 2931 } 2932 } 2933 configure_rxtx_dump_callbacks(0); 2934 printf("Configuring Port %d (socket %u)\n", pi, 2935 port->socket_id); 2936 if (nb_hairpinq > 0 && 2937 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2938 fprintf(stderr, 2939 "Port %d doesn't support hairpin queues\n", 2940 pi); 2941 return -1; 2942 } 2943 2944 if (port->bond_flag == 1 && port->update_conf == 1) { 2945 update_bonding_port_dev_conf(pi); 2946 port->update_conf = 0; 2947 } 2948 2949 /* configure port */ 2950 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2951 nb_txq + nb_hairpinq, 2952 &(port->dev_conf)); 2953 if (diag != 0) { 2954 if (port->port_status == RTE_PORT_HANDLING) 2955 port->port_status = RTE_PORT_STOPPED; 2956 else 2957 fprintf(stderr, 2958 "Port %d can not be set back to stopped\n", 2959 pi); 2960 fprintf(stderr, "Fail to configure port %d\n", 2961 pi); 2962 /* try to reconfigure port next time */ 2963 port->need_reconfig = 1; 2964 return -1; 2965 } 2966 /* get device configuration*/ 2967 if (0 != 2968 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2969 fprintf(stderr, 2970 "port %d can not get device configuration\n", 2971 pi); 2972 return -1; 2973 } 2974 /* Apply Rx offloads configuration */ 2975 if (dev_conf.rxmode.offloads != 2976 port->dev_conf.rxmode.offloads) { 2977 port->dev_conf.rxmode.offloads |= 2978 dev_conf.rxmode.offloads; 2979 for (k = 0; 2980 k < port->dev_info.max_rx_queues; 2981 k++) 2982 port->rxq[k].conf.offloads |= 2983 dev_conf.rxmode.offloads; 2984 } 2985 /* Apply Tx offloads configuration */ 2986 if (dev_conf.txmode.offloads != 2987 port->dev_conf.txmode.offloads) { 2988 port->dev_conf.txmode.offloads |= 2989 dev_conf.txmode.offloads; 2990 for (k = 0; 2991 k < port->dev_info.max_tx_queues; 2992 k++) 2993 port->txq[k].conf.offloads |= 2994 dev_conf.txmode.offloads; 2995 } 2996 } 2997 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2998 port->need_reconfig_queues = 0; 2999 /* setup tx queues */ 3000 for (qi = 0; qi < nb_txq; qi++) { 3001 struct rte_eth_txconf *conf = 3002 &port->txq[qi].conf; 3003 3004 if ((numa_support) && 3005 (txring_numa[pi] != NUMA_NO_CONFIG)) 3006 diag = rte_eth_tx_queue_setup(pi, qi, 3007 port->nb_tx_desc[qi], 3008 txring_numa[pi], 3009 &(port->txq[qi].conf)); 3010 else 3011 diag = rte_eth_tx_queue_setup(pi, qi, 3012 port->nb_tx_desc[qi], 3013 port->socket_id, 3014 &(port->txq[qi].conf)); 3015 3016 if (diag == 0) { 3017 port->txq[qi].state = 3018 conf->tx_deferred_start ? 3019 RTE_ETH_QUEUE_STATE_STOPPED : 3020 RTE_ETH_QUEUE_STATE_STARTED; 3021 continue; 3022 } 3023 3024 /* Fail to setup tx queue, return */ 3025 if (port->port_status == RTE_PORT_HANDLING) 3026 port->port_status = RTE_PORT_STOPPED; 3027 else 3028 fprintf(stderr, 3029 "Port %d can not be set back to stopped\n", 3030 pi); 3031 fprintf(stderr, 3032 "Fail to configure port %d tx queues\n", 3033 pi); 3034 /* try to reconfigure queues next time */ 3035 port->need_reconfig_queues = 1; 3036 return -1; 3037 } 3038 for (qi = 0; qi < nb_rxq; qi++) { 3039 /* setup rx queues */ 3040 if ((numa_support) && 3041 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3042 struct rte_mempool * mp = 3043 mbuf_pool_find 3044 (rxring_numa[pi], 0); 3045 if (mp == NULL) { 3046 fprintf(stderr, 3047 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3048 rxring_numa[pi]); 3049 return -1; 3050 } 3051 3052 diag = rx_queue_setup(pi, qi, 3053 port->nb_rx_desc[qi], 3054 rxring_numa[pi], 3055 &(port->rxq[qi].conf), 3056 mp); 3057 } else { 3058 struct rte_mempool *mp = 3059 mbuf_pool_find 3060 (port->socket_id, 0); 3061 if (mp == NULL) { 3062 fprintf(stderr, 3063 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3064 port->socket_id); 3065 return -1; 3066 } 3067 diag = rx_queue_setup(pi, qi, 3068 port->nb_rx_desc[qi], 3069 port->socket_id, 3070 &(port->rxq[qi].conf), 3071 mp); 3072 } 3073 if (diag == 0) 3074 continue; 3075 3076 /* Fail to setup rx queue, return */ 3077 if (port->port_status == RTE_PORT_HANDLING) 3078 port->port_status = RTE_PORT_STOPPED; 3079 else 3080 fprintf(stderr, 3081 "Port %d can not be set back to stopped\n", 3082 pi); 3083 fprintf(stderr, 3084 "Fail to configure port %d rx queues\n", 3085 pi); 3086 /* try to reconfigure queues next time */ 3087 port->need_reconfig_queues = 1; 3088 return -1; 3089 } 3090 /* setup hairpin queues */ 3091 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3092 return -1; 3093 } 3094 configure_rxtx_dump_callbacks(verbose_level); 3095 if (clear_ptypes) { 3096 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3097 NULL, 0); 3098 if (diag < 0) 3099 fprintf(stderr, 3100 "Port %d: Failed to disable Ptype parsing\n", 3101 pi); 3102 } 3103 3104 p_pi = pi; 3105 cnt_pi++; 3106 3107 /* start port */ 3108 diag = eth_dev_start_mp(pi); 3109 if (diag < 0) { 3110 fprintf(stderr, "Fail to start port %d: %s\n", 3111 pi, rte_strerror(-diag)); 3112 3113 /* Fail to setup rx queue, return */ 3114 if (port->port_status == RTE_PORT_HANDLING) 3115 port->port_status = RTE_PORT_STOPPED; 3116 else 3117 fprintf(stderr, 3118 "Port %d can not be set back to stopped\n", 3119 pi); 3120 continue; 3121 } 3122 3123 if (port->port_status == RTE_PORT_HANDLING) 3124 port->port_status = RTE_PORT_STARTED; 3125 else 3126 fprintf(stderr, "Port %d can not be set into started\n", 3127 pi); 3128 3129 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3130 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3131 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3132 3133 /* at least one port started, need checking link status */ 3134 need_check_link_status = 1; 3135 3136 pl[cfg_pi++] = pi; 3137 } 3138 3139 if (need_check_link_status == 1 && !no_link_check) 3140 check_all_ports_link_status(RTE_PORT_ALL); 3141 else if (need_check_link_status == 0) 3142 fprintf(stderr, "Please stop the ports first\n"); 3143 3144 if (hairpin_mode & 0xf) { 3145 uint16_t i; 3146 int j; 3147 3148 /* bind all started hairpin ports */ 3149 for (i = 0; i < cfg_pi; i++) { 3150 pi = pl[i]; 3151 /* bind current Tx to all peer Rx */ 3152 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3153 RTE_MAX_ETHPORTS, 1); 3154 if (peer_pi < 0) 3155 return peer_pi; 3156 for (j = 0; j < peer_pi; j++) { 3157 if (!port_is_started(peer_pl[j])) 3158 continue; 3159 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3160 if (diag < 0) { 3161 fprintf(stderr, 3162 "Error during binding hairpin Tx port %u to %u: %s\n", 3163 pi, peer_pl[j], 3164 rte_strerror(-diag)); 3165 return -1; 3166 } 3167 } 3168 /* bind all peer Tx to current Rx */ 3169 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3170 RTE_MAX_ETHPORTS, 0); 3171 if (peer_pi < 0) 3172 return peer_pi; 3173 for (j = 0; j < peer_pi; j++) { 3174 if (!port_is_started(peer_pl[j])) 3175 continue; 3176 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3177 if (diag < 0) { 3178 fprintf(stderr, 3179 "Error during binding hairpin Tx port %u to %u: %s\n", 3180 peer_pl[j], pi, 3181 rte_strerror(-diag)); 3182 return -1; 3183 } 3184 } 3185 } 3186 } 3187 3188 fill_xstats_display_info_for_port(pid); 3189 3190 printf("Done\n"); 3191 return 0; 3192 } 3193 3194 void 3195 stop_port(portid_t pid) 3196 { 3197 portid_t pi; 3198 struct rte_port *port; 3199 int need_check_link_status = 0; 3200 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3201 int peer_pi; 3202 int ret; 3203 3204 if (port_id_is_invalid(pid, ENABLED_WARN)) 3205 return; 3206 3207 printf("Stopping ports...\n"); 3208 3209 RTE_ETH_FOREACH_DEV(pi) { 3210 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3211 continue; 3212 3213 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3214 fprintf(stderr, 3215 "Please remove port %d from forwarding configuration.\n", 3216 pi); 3217 continue; 3218 } 3219 3220 if (port_is_bonding_slave(pi)) { 3221 fprintf(stderr, 3222 "Please remove port %d from bonded device.\n", 3223 pi); 3224 continue; 3225 } 3226 3227 port = &ports[pi]; 3228 if (port->port_status == RTE_PORT_STARTED) 3229 port->port_status = RTE_PORT_HANDLING; 3230 else 3231 continue; 3232 3233 if (hairpin_mode & 0xf) { 3234 int j; 3235 3236 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3237 /* unbind all peer Tx from current Rx */ 3238 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3239 RTE_MAX_ETHPORTS, 0); 3240 if (peer_pi < 0) 3241 continue; 3242 for (j = 0; j < peer_pi; j++) { 3243 if (!port_is_started(peer_pl[j])) 3244 continue; 3245 rte_eth_hairpin_unbind(peer_pl[j], pi); 3246 } 3247 } 3248 3249 if (port->flow_list) 3250 port_flow_flush(pi); 3251 3252 ret = eth_dev_stop_mp(pi); 3253 if (ret != 0) { 3254 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3255 pi); 3256 /* Allow to retry stopping the port. */ 3257 port->port_status = RTE_PORT_STARTED; 3258 continue; 3259 } 3260 3261 if (port->port_status == RTE_PORT_HANDLING) 3262 port->port_status = RTE_PORT_STOPPED; 3263 else 3264 fprintf(stderr, "Port %d can not be set into stopped\n", 3265 pi); 3266 need_check_link_status = 1; 3267 } 3268 if (need_check_link_status && !no_link_check) 3269 check_all_ports_link_status(RTE_PORT_ALL); 3270 3271 printf("Done\n"); 3272 } 3273 3274 static void 3275 remove_invalid_ports_in(portid_t *array, portid_t *total) 3276 { 3277 portid_t i; 3278 portid_t new_total = 0; 3279 3280 for (i = 0; i < *total; i++) 3281 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3282 array[new_total] = array[i]; 3283 new_total++; 3284 } 3285 *total = new_total; 3286 } 3287 3288 static void 3289 remove_invalid_ports(void) 3290 { 3291 remove_invalid_ports_in(ports_ids, &nb_ports); 3292 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3293 nb_cfg_ports = nb_fwd_ports; 3294 } 3295 3296 static void 3297 flush_port_owned_resources(portid_t pi) 3298 { 3299 mcast_addr_pool_destroy(pi); 3300 port_flow_flush(pi); 3301 port_flex_item_flush(pi); 3302 port_flow_template_table_flush(pi); 3303 port_flow_pattern_template_flush(pi); 3304 port_flow_actions_template_flush(pi); 3305 port_action_handle_flush(pi); 3306 } 3307 3308 static void 3309 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3310 { 3311 struct rte_port *port; 3312 portid_t slave_pid; 3313 uint16_t i; 3314 3315 for (i = 0; i < num_slaves; i++) { 3316 slave_pid = slave_pids[i]; 3317 if (port_is_started(slave_pid) == 1) { 3318 if (rte_eth_dev_stop(slave_pid) != 0) 3319 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3320 slave_pid); 3321 3322 port = &ports[slave_pid]; 3323 port->port_status = RTE_PORT_STOPPED; 3324 } 3325 3326 clear_port_slave_flag(slave_pid); 3327 3328 /* Close slave device when testpmd quit or is killed. */ 3329 if (cl_quit == 1 || f_quit == 1) 3330 rte_eth_dev_close(slave_pid); 3331 } 3332 } 3333 3334 void 3335 close_port(portid_t pid) 3336 { 3337 portid_t pi; 3338 struct rte_port *port; 3339 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3340 int num_slaves = 0; 3341 3342 if (port_id_is_invalid(pid, ENABLED_WARN)) 3343 return; 3344 3345 printf("Closing ports...\n"); 3346 3347 RTE_ETH_FOREACH_DEV(pi) { 3348 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3349 continue; 3350 3351 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3352 fprintf(stderr, 3353 "Please remove port %d from forwarding configuration.\n", 3354 pi); 3355 continue; 3356 } 3357 3358 if (port_is_bonding_slave(pi)) { 3359 fprintf(stderr, 3360 "Please remove port %d from bonded device.\n", 3361 pi); 3362 continue; 3363 } 3364 3365 port = &ports[pi]; 3366 if (port->port_status == RTE_PORT_CLOSED) { 3367 fprintf(stderr, "Port %d is already closed\n", pi); 3368 continue; 3369 } 3370 3371 if (is_proc_primary()) { 3372 flush_port_owned_resources(pi); 3373 #ifdef RTE_NET_BOND 3374 if (port->bond_flag == 1) 3375 num_slaves = rte_eth_bond_slaves_get(pi, 3376 slave_pids, RTE_MAX_ETHPORTS); 3377 #endif 3378 rte_eth_dev_close(pi); 3379 /* 3380 * If this port is bonded device, all slaves under the 3381 * device need to be removed or closed. 3382 */ 3383 if (port->bond_flag == 1 && num_slaves > 0) 3384 clear_bonding_slave_device(slave_pids, 3385 num_slaves); 3386 } 3387 3388 free_xstats_display_info(pi); 3389 } 3390 3391 remove_invalid_ports(); 3392 printf("Done\n"); 3393 } 3394 3395 void 3396 reset_port(portid_t pid) 3397 { 3398 int diag; 3399 portid_t pi; 3400 struct rte_port *port; 3401 3402 if (port_id_is_invalid(pid, ENABLED_WARN)) 3403 return; 3404 3405 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3406 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3407 fprintf(stderr, 3408 "Can not reset port(s), please stop port(s) first.\n"); 3409 return; 3410 } 3411 3412 printf("Resetting ports...\n"); 3413 3414 RTE_ETH_FOREACH_DEV(pi) { 3415 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3416 continue; 3417 3418 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3419 fprintf(stderr, 3420 "Please remove port %d from forwarding configuration.\n", 3421 pi); 3422 continue; 3423 } 3424 3425 if (port_is_bonding_slave(pi)) { 3426 fprintf(stderr, 3427 "Please remove port %d from bonded device.\n", 3428 pi); 3429 continue; 3430 } 3431 3432 if (is_proc_primary()) { 3433 diag = rte_eth_dev_reset(pi); 3434 if (diag == 0) { 3435 port = &ports[pi]; 3436 port->need_reconfig = 1; 3437 port->need_reconfig_queues = 1; 3438 } else { 3439 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3440 pi, diag); 3441 } 3442 } 3443 } 3444 3445 printf("Done\n"); 3446 } 3447 3448 void 3449 attach_port(char *identifier) 3450 { 3451 portid_t pi; 3452 struct rte_dev_iterator iterator; 3453 3454 printf("Attaching a new port...\n"); 3455 3456 if (identifier == NULL) { 3457 fprintf(stderr, "Invalid parameters are specified\n"); 3458 return; 3459 } 3460 3461 if (rte_dev_probe(identifier) < 0) { 3462 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3463 return; 3464 } 3465 3466 /* first attach mode: event */ 3467 if (setup_on_probe_event) { 3468 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3469 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3470 if (ports[pi].port_status == RTE_PORT_HANDLING && 3471 ports[pi].need_setup != 0) 3472 setup_attached_port(pi); 3473 return; 3474 } 3475 3476 /* second attach mode: iterator */ 3477 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3478 /* setup ports matching the devargs used for probing */ 3479 if (port_is_forwarding(pi)) 3480 continue; /* port was already attached before */ 3481 setup_attached_port(pi); 3482 } 3483 } 3484 3485 static void 3486 setup_attached_port(portid_t pi) 3487 { 3488 unsigned int socket_id; 3489 int ret; 3490 3491 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3492 /* if socket_id is invalid, set to the first available socket. */ 3493 if (check_socket_id(socket_id) < 0) 3494 socket_id = socket_ids[0]; 3495 reconfig(pi, socket_id); 3496 ret = rte_eth_promiscuous_enable(pi); 3497 if (ret != 0) 3498 fprintf(stderr, 3499 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3500 pi, rte_strerror(-ret)); 3501 3502 ports_ids[nb_ports++] = pi; 3503 fwd_ports_ids[nb_fwd_ports++] = pi; 3504 nb_cfg_ports = nb_fwd_ports; 3505 ports[pi].need_setup = 0; 3506 ports[pi].port_status = RTE_PORT_STOPPED; 3507 3508 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3509 printf("Done\n"); 3510 } 3511 3512 static void 3513 detach_device(struct rte_device *dev) 3514 { 3515 portid_t sibling; 3516 3517 if (dev == NULL) { 3518 fprintf(stderr, "Device already removed\n"); 3519 return; 3520 } 3521 3522 printf("Removing a device...\n"); 3523 3524 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3525 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3526 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3527 fprintf(stderr, "Port %u not stopped\n", 3528 sibling); 3529 return; 3530 } 3531 flush_port_owned_resources(sibling); 3532 } 3533 } 3534 3535 if (rte_dev_remove(dev) < 0) { 3536 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3537 return; 3538 } 3539 remove_invalid_ports(); 3540 3541 printf("Device is detached\n"); 3542 printf("Now total ports is %d\n", nb_ports); 3543 printf("Done\n"); 3544 return; 3545 } 3546 3547 void 3548 detach_port_device(portid_t port_id) 3549 { 3550 int ret; 3551 struct rte_eth_dev_info dev_info; 3552 3553 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3554 return; 3555 3556 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3557 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3558 fprintf(stderr, "Port not stopped\n"); 3559 return; 3560 } 3561 fprintf(stderr, "Port was not closed\n"); 3562 } 3563 3564 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3565 if (ret != 0) { 3566 TESTPMD_LOG(ERR, 3567 "Failed to get device info for port %d, not detaching\n", 3568 port_id); 3569 return; 3570 } 3571 detach_device(dev_info.device); 3572 } 3573 3574 void 3575 detach_devargs(char *identifier) 3576 { 3577 struct rte_dev_iterator iterator; 3578 struct rte_devargs da; 3579 portid_t port_id; 3580 3581 printf("Removing a device...\n"); 3582 3583 memset(&da, 0, sizeof(da)); 3584 if (rte_devargs_parsef(&da, "%s", identifier)) { 3585 fprintf(stderr, "cannot parse identifier\n"); 3586 return; 3587 } 3588 3589 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3590 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3591 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3592 fprintf(stderr, "Port %u not stopped\n", 3593 port_id); 3594 rte_eth_iterator_cleanup(&iterator); 3595 rte_devargs_reset(&da); 3596 return; 3597 } 3598 flush_port_owned_resources(port_id); 3599 } 3600 } 3601 3602 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3603 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3604 da.name, rte_bus_name(da.bus)); 3605 rte_devargs_reset(&da); 3606 return; 3607 } 3608 3609 remove_invalid_ports(); 3610 3611 printf("Device %s is detached\n", identifier); 3612 printf("Now total ports is %d\n", nb_ports); 3613 printf("Done\n"); 3614 rte_devargs_reset(&da); 3615 } 3616 3617 void 3618 pmd_test_exit(void) 3619 { 3620 portid_t pt_id; 3621 unsigned int i; 3622 int ret; 3623 3624 if (test_done == 0) 3625 stop_packet_forwarding(); 3626 3627 #ifndef RTE_EXEC_ENV_WINDOWS 3628 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3629 if (mempools[i]) { 3630 if (mp_alloc_type == MP_ALLOC_ANON) 3631 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3632 NULL); 3633 } 3634 } 3635 #endif 3636 if (ports != NULL) { 3637 no_link_check = 1; 3638 RTE_ETH_FOREACH_DEV(pt_id) { 3639 printf("\nStopping port %d...\n", pt_id); 3640 fflush(stdout); 3641 stop_port(pt_id); 3642 } 3643 RTE_ETH_FOREACH_DEV(pt_id) { 3644 printf("\nShutting down port %d...\n", pt_id); 3645 fflush(stdout); 3646 close_port(pt_id); 3647 } 3648 } 3649 3650 if (hot_plug) { 3651 ret = rte_dev_event_monitor_stop(); 3652 if (ret) { 3653 RTE_LOG(ERR, EAL, 3654 "fail to stop device event monitor."); 3655 return; 3656 } 3657 3658 ret = rte_dev_event_callback_unregister(NULL, 3659 dev_event_callback, NULL); 3660 if (ret < 0) { 3661 RTE_LOG(ERR, EAL, 3662 "fail to unregister device event callback.\n"); 3663 return; 3664 } 3665 3666 ret = rte_dev_hotplug_handle_disable(); 3667 if (ret) { 3668 RTE_LOG(ERR, EAL, 3669 "fail to disable hotplug handling.\n"); 3670 return; 3671 } 3672 } 3673 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3674 if (mempools[i]) 3675 mempool_free_mp(mempools[i]); 3676 } 3677 free(xstats_display); 3678 3679 printf("\nBye...\n"); 3680 } 3681 3682 typedef void (*cmd_func_t)(void); 3683 struct pmd_test_command { 3684 const char *cmd_name; 3685 cmd_func_t cmd_func; 3686 }; 3687 3688 /* Check the link status of all ports in up to 9s, and print them finally */ 3689 static void 3690 check_all_ports_link_status(uint32_t port_mask) 3691 { 3692 #define CHECK_INTERVAL 100 /* 100ms */ 3693 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3694 portid_t portid; 3695 uint8_t count, all_ports_up, print_flag = 0; 3696 struct rte_eth_link link; 3697 int ret; 3698 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3699 3700 printf("Checking link statuses...\n"); 3701 fflush(stdout); 3702 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3703 all_ports_up = 1; 3704 RTE_ETH_FOREACH_DEV(portid) { 3705 if ((port_mask & (1 << portid)) == 0) 3706 continue; 3707 memset(&link, 0, sizeof(link)); 3708 ret = rte_eth_link_get_nowait(portid, &link); 3709 if (ret < 0) { 3710 all_ports_up = 0; 3711 if (print_flag == 1) 3712 fprintf(stderr, 3713 "Port %u link get failed: %s\n", 3714 portid, rte_strerror(-ret)); 3715 continue; 3716 } 3717 /* print link status if flag set */ 3718 if (print_flag == 1) { 3719 rte_eth_link_to_str(link_status, 3720 sizeof(link_status), &link); 3721 printf("Port %d %s\n", portid, link_status); 3722 continue; 3723 } 3724 /* clear all_ports_up flag if any link down */ 3725 if (link.link_status == RTE_ETH_LINK_DOWN) { 3726 all_ports_up = 0; 3727 break; 3728 } 3729 } 3730 /* after finally printing all link status, get out */ 3731 if (print_flag == 1) 3732 break; 3733 3734 if (all_ports_up == 0) { 3735 fflush(stdout); 3736 rte_delay_ms(CHECK_INTERVAL); 3737 } 3738 3739 /* set the print_flag if all ports up or timeout */ 3740 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3741 print_flag = 1; 3742 } 3743 3744 if (lsc_interrupt) 3745 break; 3746 } 3747 } 3748 3749 static void 3750 rmv_port_callback(void *arg) 3751 { 3752 int need_to_start = 0; 3753 int org_no_link_check = no_link_check; 3754 portid_t port_id = (intptr_t)arg; 3755 struct rte_eth_dev_info dev_info; 3756 int ret; 3757 3758 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3759 3760 if (!test_done && port_is_forwarding(port_id)) { 3761 need_to_start = 1; 3762 stop_packet_forwarding(); 3763 } 3764 no_link_check = 1; 3765 stop_port(port_id); 3766 no_link_check = org_no_link_check; 3767 3768 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3769 if (ret != 0) 3770 TESTPMD_LOG(ERR, 3771 "Failed to get device info for port %d, not detaching\n", 3772 port_id); 3773 else { 3774 struct rte_device *device = dev_info.device; 3775 close_port(port_id); 3776 detach_device(device); /* might be already removed or have more ports */ 3777 } 3778 if (need_to_start) 3779 start_packet_forwarding(0); 3780 } 3781 3782 /* This function is used by the interrupt thread */ 3783 static int 3784 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3785 void *ret_param) 3786 { 3787 RTE_SET_USED(param); 3788 RTE_SET_USED(ret_param); 3789 3790 if (type >= RTE_ETH_EVENT_MAX) { 3791 fprintf(stderr, 3792 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3793 port_id, __func__, type); 3794 fflush(stderr); 3795 } else if (event_print_mask & (UINT32_C(1) << type)) { 3796 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3797 eth_event_desc[type]); 3798 fflush(stdout); 3799 } 3800 3801 switch (type) { 3802 case RTE_ETH_EVENT_NEW: 3803 ports[port_id].need_setup = 1; 3804 ports[port_id].port_status = RTE_PORT_HANDLING; 3805 break; 3806 case RTE_ETH_EVENT_INTR_RMV: 3807 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3808 break; 3809 if (rte_eal_alarm_set(100000, 3810 rmv_port_callback, (void *)(intptr_t)port_id)) 3811 fprintf(stderr, 3812 "Could not set up deferred device removal\n"); 3813 break; 3814 case RTE_ETH_EVENT_DESTROY: 3815 ports[port_id].port_status = RTE_PORT_CLOSED; 3816 printf("Port %u is closed\n", port_id); 3817 break; 3818 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3819 uint16_t rxq_id; 3820 int ret; 3821 3822 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3823 for (rxq_id = 0; ; rxq_id++) { 3824 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3825 NULL); 3826 if (ret <= 0) 3827 break; 3828 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3829 port_id, rxq_id); 3830 3831 #ifdef RTE_NET_MLX5 3832 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3833 #endif 3834 } 3835 break; 3836 } 3837 default: 3838 break; 3839 } 3840 return 0; 3841 } 3842 3843 static int 3844 register_eth_event_callback(void) 3845 { 3846 int ret; 3847 enum rte_eth_event_type event; 3848 3849 for (event = RTE_ETH_EVENT_UNKNOWN; 3850 event < RTE_ETH_EVENT_MAX; event++) { 3851 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3852 event, 3853 eth_event_callback, 3854 NULL); 3855 if (ret != 0) { 3856 TESTPMD_LOG(ERR, "Failed to register callback for " 3857 "%s event\n", eth_event_desc[event]); 3858 return -1; 3859 } 3860 } 3861 3862 return 0; 3863 } 3864 3865 /* This function is used by the interrupt thread */ 3866 static void 3867 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3868 __rte_unused void *arg) 3869 { 3870 uint16_t port_id; 3871 int ret; 3872 3873 if (type >= RTE_DEV_EVENT_MAX) { 3874 fprintf(stderr, "%s called upon invalid event %d\n", 3875 __func__, type); 3876 fflush(stderr); 3877 } 3878 3879 switch (type) { 3880 case RTE_DEV_EVENT_REMOVE: 3881 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3882 device_name); 3883 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3884 if (ret) { 3885 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3886 device_name); 3887 return; 3888 } 3889 /* 3890 * Because the user's callback is invoked in eal interrupt 3891 * callback, the interrupt callback need to be finished before 3892 * it can be unregistered when detaching device. So finish 3893 * callback soon and use a deferred removal to detach device 3894 * is need. It is a workaround, once the device detaching be 3895 * moved into the eal in the future, the deferred removal could 3896 * be deleted. 3897 */ 3898 if (rte_eal_alarm_set(100000, 3899 rmv_port_callback, (void *)(intptr_t)port_id)) 3900 RTE_LOG(ERR, EAL, 3901 "Could not set up deferred device removal\n"); 3902 break; 3903 case RTE_DEV_EVENT_ADD: 3904 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3905 device_name); 3906 /* TODO: After finish kernel driver binding, 3907 * begin to attach port. 3908 */ 3909 break; 3910 default: 3911 break; 3912 } 3913 } 3914 3915 static void 3916 rxtx_port_config(portid_t pid) 3917 { 3918 uint16_t qid; 3919 uint64_t offloads; 3920 struct rte_port *port = &ports[pid]; 3921 3922 for (qid = 0; qid < nb_rxq; qid++) { 3923 offloads = port->rxq[qid].conf.offloads; 3924 port->rxq[qid].conf = port->dev_info.default_rxconf; 3925 3926 if (rxq_share > 0 && 3927 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3928 /* Non-zero share group to enable RxQ share. */ 3929 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3930 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3931 } 3932 3933 if (offloads != 0) 3934 port->rxq[qid].conf.offloads = offloads; 3935 3936 /* Check if any Rx parameters have been passed */ 3937 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3938 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3939 3940 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3941 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3942 3943 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3944 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3945 3946 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3947 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3948 3949 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3950 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3951 3952 port->nb_rx_desc[qid] = nb_rxd; 3953 } 3954 3955 for (qid = 0; qid < nb_txq; qid++) { 3956 offloads = port->txq[qid].conf.offloads; 3957 port->txq[qid].conf = port->dev_info.default_txconf; 3958 if (offloads != 0) 3959 port->txq[qid].conf.offloads = offloads; 3960 3961 /* Check if any Tx parameters have been passed */ 3962 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3963 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3964 3965 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3966 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3967 3968 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3969 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3970 3971 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3972 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3973 3974 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3975 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3976 3977 port->nb_tx_desc[qid] = nb_txd; 3978 } 3979 } 3980 3981 /* 3982 * Helper function to set MTU from frame size 3983 * 3984 * port->dev_info should be set before calling this function. 3985 * 3986 * return 0 on success, negative on error 3987 */ 3988 int 3989 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3990 { 3991 struct rte_port *port = &ports[portid]; 3992 uint32_t eth_overhead; 3993 uint16_t mtu, new_mtu; 3994 3995 eth_overhead = get_eth_overhead(&port->dev_info); 3996 3997 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3998 printf("Failed to get MTU for port %u\n", portid); 3999 return -1; 4000 } 4001 4002 new_mtu = max_rx_pktlen - eth_overhead; 4003 4004 if (mtu == new_mtu) 4005 return 0; 4006 4007 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 4008 fprintf(stderr, 4009 "Failed to set MTU to %u for port %u\n", 4010 new_mtu, portid); 4011 return -1; 4012 } 4013 4014 port->dev_conf.rxmode.mtu = new_mtu; 4015 4016 return 0; 4017 } 4018 4019 void 4020 init_port_config(void) 4021 { 4022 portid_t pid; 4023 struct rte_port *port; 4024 int ret, i; 4025 4026 RTE_ETH_FOREACH_DEV(pid) { 4027 port = &ports[pid]; 4028 4029 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4030 if (ret != 0) 4031 return; 4032 4033 if (nb_rxq > 1) { 4034 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4035 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4036 rss_hf & port->dev_info.flow_type_rss_offloads; 4037 } else { 4038 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4039 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4040 } 4041 4042 if (port->dcb_flag == 0) { 4043 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4044 port->dev_conf.rxmode.mq_mode = 4045 (enum rte_eth_rx_mq_mode) 4046 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4047 } else { 4048 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4049 port->dev_conf.rxmode.offloads &= 4050 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4051 4052 for (i = 0; 4053 i < port->dev_info.nb_rx_queues; 4054 i++) 4055 port->rxq[i].conf.offloads &= 4056 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4057 } 4058 } 4059 4060 rxtx_port_config(pid); 4061 4062 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4063 if (ret != 0) 4064 return; 4065 4066 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4067 port->dev_conf.intr_conf.lsc = 1; 4068 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4069 port->dev_conf.intr_conf.rmv = 1; 4070 } 4071 } 4072 4073 void set_port_slave_flag(portid_t slave_pid) 4074 { 4075 struct rte_port *port; 4076 4077 port = &ports[slave_pid]; 4078 port->slave_flag = 1; 4079 } 4080 4081 void clear_port_slave_flag(portid_t slave_pid) 4082 { 4083 struct rte_port *port; 4084 4085 port = &ports[slave_pid]; 4086 port->slave_flag = 0; 4087 } 4088 4089 uint8_t port_is_bonding_slave(portid_t slave_pid) 4090 { 4091 struct rte_port *port; 4092 struct rte_eth_dev_info dev_info; 4093 int ret; 4094 4095 port = &ports[slave_pid]; 4096 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4097 if (ret != 0) { 4098 TESTPMD_LOG(ERR, 4099 "Failed to get device info for port id %d," 4100 "cannot determine if the port is a bonded slave", 4101 slave_pid); 4102 return 0; 4103 } 4104 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4105 return 1; 4106 return 0; 4107 } 4108 4109 const uint16_t vlan_tags[] = { 4110 0, 1, 2, 3, 4, 5, 6, 7, 4111 8, 9, 10, 11, 12, 13, 14, 15, 4112 16, 17, 18, 19, 20, 21, 22, 23, 4113 24, 25, 26, 27, 28, 29, 30, 31 4114 }; 4115 4116 static int 4117 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4118 enum dcb_mode_enable dcb_mode, 4119 enum rte_eth_nb_tcs num_tcs, 4120 uint8_t pfc_en) 4121 { 4122 uint8_t i; 4123 int32_t rc; 4124 struct rte_eth_rss_conf rss_conf; 4125 4126 /* 4127 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4128 * given above, and the number of traffic classes available for use. 4129 */ 4130 if (dcb_mode == DCB_VT_ENABLED) { 4131 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4132 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4133 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4134 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4135 4136 /* VMDQ+DCB RX and TX configurations */ 4137 vmdq_rx_conf->enable_default_pool = 0; 4138 vmdq_rx_conf->default_pool = 0; 4139 vmdq_rx_conf->nb_queue_pools = 4140 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4141 vmdq_tx_conf->nb_queue_pools = 4142 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4143 4144 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4145 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4146 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4147 vmdq_rx_conf->pool_map[i].pools = 4148 1 << (i % vmdq_rx_conf->nb_queue_pools); 4149 } 4150 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4151 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4152 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4153 } 4154 4155 /* set DCB mode of RX and TX of multiple queues */ 4156 eth_conf->rxmode.mq_mode = 4157 (enum rte_eth_rx_mq_mode) 4158 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4159 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4160 } else { 4161 struct rte_eth_dcb_rx_conf *rx_conf = 4162 ð_conf->rx_adv_conf.dcb_rx_conf; 4163 struct rte_eth_dcb_tx_conf *tx_conf = 4164 ð_conf->tx_adv_conf.dcb_tx_conf; 4165 4166 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4167 4168 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4169 if (rc != 0) 4170 return rc; 4171 4172 rx_conf->nb_tcs = num_tcs; 4173 tx_conf->nb_tcs = num_tcs; 4174 4175 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4176 rx_conf->dcb_tc[i] = i % num_tcs; 4177 tx_conf->dcb_tc[i] = i % num_tcs; 4178 } 4179 4180 eth_conf->rxmode.mq_mode = 4181 (enum rte_eth_rx_mq_mode) 4182 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4183 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4184 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4185 } 4186 4187 if (pfc_en) 4188 eth_conf->dcb_capability_en = 4189 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4190 else 4191 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4192 4193 return 0; 4194 } 4195 4196 int 4197 init_port_dcb_config(portid_t pid, 4198 enum dcb_mode_enable dcb_mode, 4199 enum rte_eth_nb_tcs num_tcs, 4200 uint8_t pfc_en) 4201 { 4202 struct rte_eth_conf port_conf; 4203 struct rte_port *rte_port; 4204 int retval; 4205 uint16_t i; 4206 4207 if (num_procs > 1) { 4208 printf("The multi-process feature doesn't support dcb.\n"); 4209 return -ENOTSUP; 4210 } 4211 rte_port = &ports[pid]; 4212 4213 /* retain the original device configuration. */ 4214 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4215 4216 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4217 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4218 if (retval < 0) 4219 return retval; 4220 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4221 /* remove RSS HASH offload for DCB in vt mode */ 4222 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4223 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4224 for (i = 0; i < nb_rxq; i++) 4225 rte_port->rxq[i].conf.offloads &= 4226 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4227 } 4228 4229 /* re-configure the device . */ 4230 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4231 if (retval < 0) 4232 return retval; 4233 4234 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4235 if (retval != 0) 4236 return retval; 4237 4238 /* If dev_info.vmdq_pool_base is greater than 0, 4239 * the queue id of vmdq pools is started after pf queues. 4240 */ 4241 if (dcb_mode == DCB_VT_ENABLED && 4242 rte_port->dev_info.vmdq_pool_base > 0) { 4243 fprintf(stderr, 4244 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4245 pid); 4246 return -1; 4247 } 4248 4249 /* Assume the ports in testpmd have the same dcb capability 4250 * and has the same number of rxq and txq in dcb mode 4251 */ 4252 if (dcb_mode == DCB_VT_ENABLED) { 4253 if (rte_port->dev_info.max_vfs > 0) { 4254 nb_rxq = rte_port->dev_info.nb_rx_queues; 4255 nb_txq = rte_port->dev_info.nb_tx_queues; 4256 } else { 4257 nb_rxq = rte_port->dev_info.max_rx_queues; 4258 nb_txq = rte_port->dev_info.max_tx_queues; 4259 } 4260 } else { 4261 /*if vt is disabled, use all pf queues */ 4262 if (rte_port->dev_info.vmdq_pool_base == 0) { 4263 nb_rxq = rte_port->dev_info.max_rx_queues; 4264 nb_txq = rte_port->dev_info.max_tx_queues; 4265 } else { 4266 nb_rxq = (queueid_t)num_tcs; 4267 nb_txq = (queueid_t)num_tcs; 4268 4269 } 4270 } 4271 rx_free_thresh = 64; 4272 4273 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4274 4275 rxtx_port_config(pid); 4276 /* VLAN filter */ 4277 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4278 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4279 rx_vft_set(pid, vlan_tags[i], 1); 4280 4281 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4282 if (retval != 0) 4283 return retval; 4284 4285 rte_port->dcb_flag = 1; 4286 4287 /* Enter DCB configuration status */ 4288 dcb_config = 1; 4289 4290 return 0; 4291 } 4292 4293 static void 4294 init_port(void) 4295 { 4296 int i; 4297 4298 /* Configuration of Ethernet ports. */ 4299 ports = rte_zmalloc("testpmd: ports", 4300 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4301 RTE_CACHE_LINE_SIZE); 4302 if (ports == NULL) { 4303 rte_exit(EXIT_FAILURE, 4304 "rte_zmalloc(%d struct rte_port) failed\n", 4305 RTE_MAX_ETHPORTS); 4306 } 4307 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4308 ports[i].fwd_mac_swap = 1; 4309 ports[i].xstats_info.allocated = false; 4310 LIST_INIT(&ports[i].flow_tunnel_list); 4311 } 4312 /* Initialize ports NUMA structures */ 4313 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4314 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4315 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4316 } 4317 4318 static void 4319 force_quit(void) 4320 { 4321 pmd_test_exit(); 4322 prompt_exit(); 4323 } 4324 4325 static void 4326 print_stats(void) 4327 { 4328 uint8_t i; 4329 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4330 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4331 4332 /* Clear screen and move to top left */ 4333 printf("%s%s", clr, top_left); 4334 4335 printf("\nPort statistics ===================================="); 4336 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4337 nic_stats_display(fwd_ports_ids[i]); 4338 4339 fflush(stdout); 4340 } 4341 4342 static void 4343 signal_handler(int signum) 4344 { 4345 if (signum == SIGINT || signum == SIGTERM) { 4346 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4347 signum); 4348 #ifdef RTE_LIB_PDUMP 4349 /* uninitialize packet capture framework */ 4350 rte_pdump_uninit(); 4351 #endif 4352 #ifdef RTE_LIB_LATENCYSTATS 4353 if (latencystats_enabled != 0) 4354 rte_latencystats_uninit(); 4355 #endif 4356 force_quit(); 4357 /* Set flag to indicate the force termination. */ 4358 f_quit = 1; 4359 /* exit with the expected status */ 4360 #ifndef RTE_EXEC_ENV_WINDOWS 4361 signal(signum, SIG_DFL); 4362 kill(getpid(), signum); 4363 #endif 4364 } 4365 } 4366 4367 int 4368 main(int argc, char** argv) 4369 { 4370 int diag; 4371 portid_t port_id; 4372 uint16_t count; 4373 int ret; 4374 4375 signal(SIGINT, signal_handler); 4376 signal(SIGTERM, signal_handler); 4377 4378 testpmd_logtype = rte_log_register("testpmd"); 4379 if (testpmd_logtype < 0) 4380 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4381 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4382 4383 diag = rte_eal_init(argc, argv); 4384 if (diag < 0) 4385 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4386 rte_strerror(rte_errno)); 4387 4388 ret = register_eth_event_callback(); 4389 if (ret != 0) 4390 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4391 4392 #ifdef RTE_LIB_PDUMP 4393 /* initialize packet capture framework */ 4394 rte_pdump_init(); 4395 #endif 4396 4397 count = 0; 4398 RTE_ETH_FOREACH_DEV(port_id) { 4399 ports_ids[count] = port_id; 4400 count++; 4401 } 4402 nb_ports = (portid_t) count; 4403 if (nb_ports == 0) 4404 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4405 4406 /* allocate port structures, and init them */ 4407 init_port(); 4408 4409 set_def_fwd_config(); 4410 if (nb_lcores == 0) 4411 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4412 "Check the core mask argument\n"); 4413 4414 /* Bitrate/latency stats disabled by default */ 4415 #ifdef RTE_LIB_BITRATESTATS 4416 bitrate_enabled = 0; 4417 #endif 4418 #ifdef RTE_LIB_LATENCYSTATS 4419 latencystats_enabled = 0; 4420 #endif 4421 4422 /* on FreeBSD, mlockall() is disabled by default */ 4423 #ifdef RTE_EXEC_ENV_FREEBSD 4424 do_mlockall = 0; 4425 #else 4426 do_mlockall = 1; 4427 #endif 4428 4429 argc -= diag; 4430 argv += diag; 4431 if (argc > 1) 4432 launch_args_parse(argc, argv); 4433 4434 #ifndef RTE_EXEC_ENV_WINDOWS 4435 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4436 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4437 strerror(errno)); 4438 } 4439 #endif 4440 4441 if (tx_first && interactive) 4442 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4443 "interactive mode.\n"); 4444 4445 if (tx_first && lsc_interrupt) { 4446 fprintf(stderr, 4447 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4448 lsc_interrupt = 0; 4449 } 4450 4451 if (!nb_rxq && !nb_txq) 4452 fprintf(stderr, 4453 "Warning: Either rx or tx queues should be non-zero\n"); 4454 4455 if (nb_rxq > 1 && nb_rxq > nb_txq) 4456 fprintf(stderr, 4457 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4458 nb_rxq, nb_txq); 4459 4460 init_config(); 4461 4462 if (hot_plug) { 4463 ret = rte_dev_hotplug_handle_enable(); 4464 if (ret) { 4465 RTE_LOG(ERR, EAL, 4466 "fail to enable hotplug handling."); 4467 return -1; 4468 } 4469 4470 ret = rte_dev_event_monitor_start(); 4471 if (ret) { 4472 RTE_LOG(ERR, EAL, 4473 "fail to start device event monitoring."); 4474 return -1; 4475 } 4476 4477 ret = rte_dev_event_callback_register(NULL, 4478 dev_event_callback, NULL); 4479 if (ret) { 4480 RTE_LOG(ERR, EAL, 4481 "fail to register device event callback\n"); 4482 return -1; 4483 } 4484 } 4485 4486 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4487 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4488 4489 /* set all ports to promiscuous mode by default */ 4490 RTE_ETH_FOREACH_DEV(port_id) { 4491 ret = rte_eth_promiscuous_enable(port_id); 4492 if (ret != 0) 4493 fprintf(stderr, 4494 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4495 port_id, rte_strerror(-ret)); 4496 } 4497 4498 #ifdef RTE_LIB_METRICS 4499 /* Init metrics library */ 4500 rte_metrics_init(rte_socket_id()); 4501 #endif 4502 4503 #ifdef RTE_LIB_LATENCYSTATS 4504 if (latencystats_enabled != 0) { 4505 int ret = rte_latencystats_init(1, NULL); 4506 if (ret) 4507 fprintf(stderr, 4508 "Warning: latencystats init() returned error %d\n", 4509 ret); 4510 fprintf(stderr, "Latencystats running on lcore %d\n", 4511 latencystats_lcore_id); 4512 } 4513 #endif 4514 4515 /* Setup bitrate stats */ 4516 #ifdef RTE_LIB_BITRATESTATS 4517 if (bitrate_enabled != 0) { 4518 bitrate_data = rte_stats_bitrate_create(); 4519 if (bitrate_data == NULL) 4520 rte_exit(EXIT_FAILURE, 4521 "Could not allocate bitrate data.\n"); 4522 rte_stats_bitrate_reg(bitrate_data); 4523 } 4524 #endif 4525 #ifdef RTE_LIB_CMDLINE 4526 if (init_cmdline() != 0) 4527 rte_exit(EXIT_FAILURE, 4528 "Could not initialise cmdline context.\n"); 4529 4530 if (strlen(cmdline_filename) != 0) 4531 cmdline_read_from_file(cmdline_filename); 4532 4533 if (interactive == 1) { 4534 if (auto_start) { 4535 printf("Start automatic packet forwarding\n"); 4536 start_packet_forwarding(0); 4537 } 4538 prompt(); 4539 pmd_test_exit(); 4540 } else 4541 #endif 4542 { 4543 char c; 4544 int rc; 4545 4546 f_quit = 0; 4547 4548 printf("No commandline core given, start packet forwarding\n"); 4549 start_packet_forwarding(tx_first); 4550 if (stats_period != 0) { 4551 uint64_t prev_time = 0, cur_time, diff_time = 0; 4552 uint64_t timer_period; 4553 4554 /* Convert to number of cycles */ 4555 timer_period = stats_period * rte_get_timer_hz(); 4556 4557 while (f_quit == 0) { 4558 cur_time = rte_get_timer_cycles(); 4559 diff_time += cur_time - prev_time; 4560 4561 if (diff_time >= timer_period) { 4562 print_stats(); 4563 /* Reset the timer */ 4564 diff_time = 0; 4565 } 4566 /* Sleep to avoid unnecessary checks */ 4567 prev_time = cur_time; 4568 rte_delay_us_sleep(US_PER_S); 4569 } 4570 } 4571 4572 printf("Press enter to exit\n"); 4573 rc = read(0, &c, 1); 4574 pmd_test_exit(); 4575 if (rc < 0) 4576 return 1; 4577 } 4578 4579 ret = rte_eal_cleanup(); 4580 if (ret != 0) 4581 rte_exit(EXIT_FAILURE, 4582 "EAL cleanup failed: %s\n", strerror(-ret)); 4583 4584 return EXIT_SUCCESS; 4585 } 4586