1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 static volatile uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 253 254 /* 255 * Configuration of packet segments used by the "txonly" processing engine. 256 */ 257 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 258 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 259 TXONLY_DEF_PACKET_LEN, 260 }; 261 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 262 263 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 264 /**< Split policy for packets to TX. */ 265 266 uint8_t txonly_multi_flow; 267 /**< Whether multiple flows are generated in TXONLY mode. */ 268 269 uint32_t tx_pkt_times_inter; 270 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 271 272 uint32_t tx_pkt_times_intra; 273 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 274 275 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 276 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 277 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 278 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 279 280 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 281 uint8_t dcb_config = 0; 282 283 /* 284 * Configurable number of RX/TX queues. 285 */ 286 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 287 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 288 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 289 290 /* 291 * Configurable number of RX/TX ring descriptors. 292 * Defaults are supplied by drivers via ethdev. 293 */ 294 #define RX_DESC_DEFAULT 0 295 #define TX_DESC_DEFAULT 0 296 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 297 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 298 299 #define RTE_PMD_PARAM_UNSET -1 300 /* 301 * Configurable values of RX and TX ring threshold registers. 302 */ 303 304 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 305 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 306 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 307 308 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 309 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 310 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 311 312 /* 313 * Configurable value of RX free threshold. 314 */ 315 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 316 317 /* 318 * Configurable value of RX drop enable. 319 */ 320 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 321 322 /* 323 * Configurable value of TX free threshold. 324 */ 325 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 326 327 /* 328 * Configurable value of TX RS bit threshold. 329 */ 330 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 331 332 /* 333 * Configurable value of buffered packets before sending. 334 */ 335 uint16_t noisy_tx_sw_bufsz; 336 337 /* 338 * Configurable value of packet buffer timeout. 339 */ 340 uint16_t noisy_tx_sw_buf_flush_time; 341 342 /* 343 * Configurable value for size of VNF internal memory area 344 * used for simulating noisy neighbour behaviour 345 */ 346 uint64_t noisy_lkup_mem_sz; 347 348 /* 349 * Configurable value of number of random writes done in 350 * VNF simulation memory area. 351 */ 352 uint64_t noisy_lkup_num_writes; 353 354 /* 355 * Configurable value of number of random reads done in 356 * VNF simulation memory area. 357 */ 358 uint64_t noisy_lkup_num_reads; 359 360 /* 361 * Configurable value of number of random reads/writes done in 362 * VNF simulation memory area. 363 */ 364 uint64_t noisy_lkup_num_reads_writes; 365 366 /* 367 * Receive Side Scaling (RSS) configuration. 368 */ 369 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 370 371 /* 372 * Port topology configuration 373 */ 374 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 375 376 /* 377 * Avoids to flush all the RX streams before starts forwarding. 378 */ 379 uint8_t no_flush_rx = 0; /* flush by default */ 380 381 /* 382 * Flow API isolated mode. 383 */ 384 uint8_t flow_isolate_all; 385 386 /* 387 * Avoids to check link status when starting/stopping a port. 388 */ 389 uint8_t no_link_check = 0; /* check by default */ 390 391 /* 392 * Don't automatically start all ports in interactive mode. 393 */ 394 uint8_t no_device_start = 0; 395 396 /* 397 * Enable link status change notification 398 */ 399 uint8_t lsc_interrupt = 1; /* enabled by default */ 400 401 /* 402 * Enable device removal notification. 403 */ 404 uint8_t rmv_interrupt = 1; /* enabled by default */ 405 406 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 407 408 /* After attach, port setup is called on event or by iterator */ 409 bool setup_on_probe_event = true; 410 411 /* Clear ptypes on port initialization. */ 412 uint8_t clear_ptypes = true; 413 414 /* Hairpin ports configuration mode. */ 415 uint32_t hairpin_mode; 416 417 /* Pretty printing of ethdev events */ 418 static const char * const eth_event_desc[] = { 419 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 420 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 421 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 422 [RTE_ETH_EVENT_INTR_RESET] = "reset", 423 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 424 [RTE_ETH_EVENT_IPSEC] = "IPsec", 425 [RTE_ETH_EVENT_MACSEC] = "MACsec", 426 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 427 [RTE_ETH_EVENT_NEW] = "device probed", 428 [RTE_ETH_EVENT_DESTROY] = "device released", 429 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 430 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 431 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 432 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 433 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 434 [RTE_ETH_EVENT_MAX] = NULL, 435 }; 436 437 /* 438 * Display or mask ether events 439 * Default to all events except VF_MBOX 440 */ 441 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 442 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 443 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 444 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 445 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 446 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 447 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 448 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 449 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 450 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 451 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 452 /* 453 * Decide if all memory are locked for performance. 454 */ 455 int do_mlockall = 0; 456 457 #ifdef RTE_LIB_LATENCYSTATS 458 459 /* 460 * Set when latency stats is enabled in the commandline 461 */ 462 uint8_t latencystats_enabled; 463 464 /* 465 * Lcore ID to service latency statistics. 466 */ 467 lcoreid_t latencystats_lcore_id = -1; 468 469 #endif 470 471 /* 472 * Ethernet device configuration. 473 */ 474 struct rte_eth_rxmode rx_mode; 475 476 struct rte_eth_txmode tx_mode = { 477 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 478 }; 479 480 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 481 482 /* 483 * Display zero values by default for xstats 484 */ 485 uint8_t xstats_hide_zero; 486 487 /* 488 * Measure of CPU cycles disabled by default 489 */ 490 uint8_t record_core_cycles; 491 492 /* 493 * Display of RX and TX bursts disabled by default 494 */ 495 uint8_t record_burst_stats; 496 497 /* 498 * Number of ports per shared Rx queue group, 0 disable. 499 */ 500 uint32_t rxq_share; 501 502 unsigned int num_sockets = 0; 503 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 504 505 #ifdef RTE_LIB_BITRATESTATS 506 /* Bitrate statistics */ 507 struct rte_stats_bitrates *bitrate_data; 508 lcoreid_t bitrate_lcore_id; 509 uint8_t bitrate_enabled; 510 #endif 511 512 #ifdef RTE_LIB_GRO 513 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 514 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 515 #endif 516 517 /* 518 * hexadecimal bitmask of RX mq mode can be enabled. 519 */ 520 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 521 522 /* 523 * Used to set forced link speed 524 */ 525 uint32_t eth_link_speed; 526 527 /* 528 * ID of the current process in multi-process, used to 529 * configure the queues to be polled. 530 */ 531 int proc_id; 532 533 /* 534 * Number of processes in multi-process, used to 535 * configure the queues to be polled. 536 */ 537 unsigned int num_procs = 1; 538 539 static void 540 eth_rx_metadata_negotiate_mp(uint16_t port_id) 541 { 542 uint64_t rx_meta_features = 0; 543 int ret; 544 545 if (!is_proc_primary()) 546 return; 547 548 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 549 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 550 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 551 552 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 553 if (ret == 0) { 554 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 555 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 556 port_id); 557 } 558 559 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 560 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 561 port_id); 562 } 563 564 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 565 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 566 port_id); 567 } 568 } else if (ret != -ENOTSUP) { 569 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 570 port_id, rte_strerror(-ret)); 571 } 572 } 573 574 static int 575 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 576 const struct rte_eth_conf *dev_conf) 577 { 578 if (is_proc_primary()) 579 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 580 dev_conf); 581 return 0; 582 } 583 584 static int 585 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 586 { 587 #ifdef RTE_NET_BOND 588 589 portid_t slave_pids[RTE_MAX_ETHPORTS]; 590 struct rte_port *port; 591 int num_slaves; 592 portid_t slave_pid; 593 int i; 594 595 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 596 RTE_MAX_ETHPORTS); 597 if (num_slaves < 0) { 598 fprintf(stderr, "Failed to get slave list for port = %u\n", 599 bond_pid); 600 return num_slaves; 601 } 602 603 for (i = 0; i < num_slaves; i++) { 604 slave_pid = slave_pids[i]; 605 port = &ports[slave_pid]; 606 port->port_status = 607 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 608 } 609 #else 610 RTE_SET_USED(bond_pid); 611 RTE_SET_USED(is_stop); 612 #endif 613 return 0; 614 } 615 616 static int 617 eth_dev_start_mp(uint16_t port_id) 618 { 619 int ret; 620 621 if (is_proc_primary()) { 622 ret = rte_eth_dev_start(port_id); 623 if (ret != 0) 624 return ret; 625 626 struct rte_port *port = &ports[port_id]; 627 628 /* 629 * Starting a bonded port also starts all slaves under the bonded 630 * device. So if this port is bond device, we need to modify the 631 * port status of these slaves. 632 */ 633 if (port->bond_flag == 1) 634 return change_bonding_slave_port_status(port_id, false); 635 } 636 637 return 0; 638 } 639 640 static int 641 eth_dev_stop_mp(uint16_t port_id) 642 { 643 int ret; 644 645 if (is_proc_primary()) { 646 ret = rte_eth_dev_stop(port_id); 647 if (ret != 0) 648 return ret; 649 650 struct rte_port *port = &ports[port_id]; 651 652 /* 653 * Stopping a bonded port also stops all slaves under the bonded 654 * device. So if this port is bond device, we need to modify the 655 * port status of these slaves. 656 */ 657 if (port->bond_flag == 1) 658 return change_bonding_slave_port_status(port_id, true); 659 } 660 661 return 0; 662 } 663 664 static void 665 mempool_free_mp(struct rte_mempool *mp) 666 { 667 if (is_proc_primary()) 668 rte_mempool_free(mp); 669 } 670 671 static int 672 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 673 { 674 if (is_proc_primary()) 675 return rte_eth_dev_set_mtu(port_id, mtu); 676 677 return 0; 678 } 679 680 /* Forward function declarations */ 681 static void setup_attached_port(portid_t pi); 682 static void check_all_ports_link_status(uint32_t port_mask); 683 static int eth_event_callback(portid_t port_id, 684 enum rte_eth_event_type type, 685 void *param, void *ret_param); 686 static void dev_event_callback(const char *device_name, 687 enum rte_dev_event_type type, 688 void *param); 689 static void fill_xstats_display_info(void); 690 691 /* 692 * Check if all the ports are started. 693 * If yes, return positive value. If not, return zero. 694 */ 695 static int all_ports_started(void); 696 697 #ifdef RTE_LIB_GSO 698 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 699 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 700 #endif 701 702 /* Holds the registered mbuf dynamic flags names. */ 703 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 704 705 706 /* 707 * Helper function to check if socket is already discovered. 708 * If yes, return positive value. If not, return zero. 709 */ 710 int 711 new_socket_id(unsigned int socket_id) 712 { 713 unsigned int i; 714 715 for (i = 0; i < num_sockets; i++) { 716 if (socket_ids[i] == socket_id) 717 return 0; 718 } 719 return 1; 720 } 721 722 /* 723 * Setup default configuration. 724 */ 725 static void 726 set_default_fwd_lcores_config(void) 727 { 728 unsigned int i; 729 unsigned int nb_lc; 730 unsigned int sock_num; 731 732 nb_lc = 0; 733 for (i = 0; i < RTE_MAX_LCORE; i++) { 734 if (!rte_lcore_is_enabled(i)) 735 continue; 736 sock_num = rte_lcore_to_socket_id(i); 737 if (new_socket_id(sock_num)) { 738 if (num_sockets >= RTE_MAX_NUMA_NODES) { 739 rte_exit(EXIT_FAILURE, 740 "Total sockets greater than %u\n", 741 RTE_MAX_NUMA_NODES); 742 } 743 socket_ids[num_sockets++] = sock_num; 744 } 745 if (i == rte_get_main_lcore()) 746 continue; 747 fwd_lcores_cpuids[nb_lc++] = i; 748 } 749 nb_lcores = (lcoreid_t) nb_lc; 750 nb_cfg_lcores = nb_lcores; 751 nb_fwd_lcores = 1; 752 } 753 754 static void 755 set_def_peer_eth_addrs(void) 756 { 757 portid_t i; 758 759 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 760 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 761 peer_eth_addrs[i].addr_bytes[5] = i; 762 } 763 } 764 765 static void 766 set_default_fwd_ports_config(void) 767 { 768 portid_t pt_id; 769 int i = 0; 770 771 RTE_ETH_FOREACH_DEV(pt_id) { 772 fwd_ports_ids[i++] = pt_id; 773 774 /* Update sockets info according to the attached device */ 775 int socket_id = rte_eth_dev_socket_id(pt_id); 776 if (socket_id >= 0 && new_socket_id(socket_id)) { 777 if (num_sockets >= RTE_MAX_NUMA_NODES) { 778 rte_exit(EXIT_FAILURE, 779 "Total sockets greater than %u\n", 780 RTE_MAX_NUMA_NODES); 781 } 782 socket_ids[num_sockets++] = socket_id; 783 } 784 } 785 786 nb_cfg_ports = nb_ports; 787 nb_fwd_ports = nb_ports; 788 } 789 790 void 791 set_def_fwd_config(void) 792 { 793 set_default_fwd_lcores_config(); 794 set_def_peer_eth_addrs(); 795 set_default_fwd_ports_config(); 796 } 797 798 #ifndef RTE_EXEC_ENV_WINDOWS 799 /* extremely pessimistic estimation of memory required to create a mempool */ 800 static int 801 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 802 { 803 unsigned int n_pages, mbuf_per_pg, leftover; 804 uint64_t total_mem, mbuf_mem, obj_sz; 805 806 /* there is no good way to predict how much space the mempool will 807 * occupy because it will allocate chunks on the fly, and some of those 808 * will come from default DPDK memory while some will come from our 809 * external memory, so just assume 128MB will be enough for everyone. 810 */ 811 uint64_t hdr_mem = 128 << 20; 812 813 /* account for possible non-contiguousness */ 814 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 815 if (obj_sz > pgsz) { 816 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 817 return -1; 818 } 819 820 mbuf_per_pg = pgsz / obj_sz; 821 leftover = (nb_mbufs % mbuf_per_pg) > 0; 822 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 823 824 mbuf_mem = n_pages * pgsz; 825 826 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 827 828 if (total_mem > SIZE_MAX) { 829 TESTPMD_LOG(ERR, "Memory size too big\n"); 830 return -1; 831 } 832 *out = (size_t)total_mem; 833 834 return 0; 835 } 836 837 static int 838 pagesz_flags(uint64_t page_sz) 839 { 840 /* as per mmap() manpage, all page sizes are log2 of page size 841 * shifted by MAP_HUGE_SHIFT 842 */ 843 int log2 = rte_log2_u64(page_sz); 844 845 return (log2 << HUGE_SHIFT); 846 } 847 848 static void * 849 alloc_mem(size_t memsz, size_t pgsz, bool huge) 850 { 851 void *addr; 852 int flags; 853 854 /* allocate anonymous hugepages */ 855 flags = MAP_ANONYMOUS | MAP_PRIVATE; 856 if (huge) 857 flags |= HUGE_FLAG | pagesz_flags(pgsz); 858 859 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 860 if (addr == MAP_FAILED) 861 return NULL; 862 863 return addr; 864 } 865 866 struct extmem_param { 867 void *addr; 868 size_t len; 869 size_t pgsz; 870 rte_iova_t *iova_table; 871 unsigned int iova_table_len; 872 }; 873 874 static int 875 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 876 bool huge) 877 { 878 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 879 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 880 unsigned int cur_page, n_pages, pgsz_idx; 881 size_t mem_sz, cur_pgsz; 882 rte_iova_t *iovas = NULL; 883 void *addr; 884 int ret; 885 886 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 887 /* skip anything that is too big */ 888 if (pgsizes[pgsz_idx] > SIZE_MAX) 889 continue; 890 891 cur_pgsz = pgsizes[pgsz_idx]; 892 893 /* if we were told not to allocate hugepages, override */ 894 if (!huge) 895 cur_pgsz = sysconf(_SC_PAGESIZE); 896 897 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 898 if (ret < 0) { 899 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 900 return -1; 901 } 902 903 /* allocate our memory */ 904 addr = alloc_mem(mem_sz, cur_pgsz, huge); 905 906 /* if we couldn't allocate memory with a specified page size, 907 * that doesn't mean we can't do it with other page sizes, so 908 * try another one. 909 */ 910 if (addr == NULL) 911 continue; 912 913 /* store IOVA addresses for every page in this memory area */ 914 n_pages = mem_sz / cur_pgsz; 915 916 iovas = malloc(sizeof(*iovas) * n_pages); 917 918 if (iovas == NULL) { 919 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 920 goto fail; 921 } 922 /* lock memory if it's not huge pages */ 923 if (!huge) 924 mlock(addr, mem_sz); 925 926 /* populate IOVA addresses */ 927 for (cur_page = 0; cur_page < n_pages; cur_page++) { 928 rte_iova_t iova; 929 size_t offset; 930 void *cur; 931 932 offset = cur_pgsz * cur_page; 933 cur = RTE_PTR_ADD(addr, offset); 934 935 /* touch the page before getting its IOVA */ 936 *(volatile char *)cur = 0; 937 938 iova = rte_mem_virt2iova(cur); 939 940 iovas[cur_page] = iova; 941 } 942 943 break; 944 } 945 /* if we couldn't allocate anything */ 946 if (iovas == NULL) 947 return -1; 948 949 param->addr = addr; 950 param->len = mem_sz; 951 param->pgsz = cur_pgsz; 952 param->iova_table = iovas; 953 param->iova_table_len = n_pages; 954 955 return 0; 956 fail: 957 free(iovas); 958 if (addr) 959 munmap(addr, mem_sz); 960 961 return -1; 962 } 963 964 static int 965 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 966 { 967 struct extmem_param param; 968 int socket_id, ret; 969 970 memset(¶m, 0, sizeof(param)); 971 972 /* check if our heap exists */ 973 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 974 if (socket_id < 0) { 975 /* create our heap */ 976 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 977 if (ret < 0) { 978 TESTPMD_LOG(ERR, "Cannot create heap\n"); 979 return -1; 980 } 981 } 982 983 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 984 if (ret < 0) { 985 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 986 return -1; 987 } 988 989 /* we now have a valid memory area, so add it to heap */ 990 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 991 param.addr, param.len, param.iova_table, 992 param.iova_table_len, param.pgsz); 993 994 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 995 996 /* not needed any more */ 997 free(param.iova_table); 998 999 if (ret < 0) { 1000 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1001 munmap(param.addr, param.len); 1002 return -1; 1003 } 1004 1005 /* success */ 1006 1007 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1008 param.len >> 20); 1009 1010 return 0; 1011 } 1012 static void 1013 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1014 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1015 { 1016 uint16_t pid = 0; 1017 int ret; 1018 1019 RTE_ETH_FOREACH_DEV(pid) { 1020 struct rte_eth_dev_info dev_info; 1021 1022 ret = eth_dev_info_get_print_err(pid, &dev_info); 1023 if (ret != 0) { 1024 TESTPMD_LOG(DEBUG, 1025 "unable to get device info for port %d on addr 0x%p," 1026 "mempool unmapping will not be performed\n", 1027 pid, memhdr->addr); 1028 continue; 1029 } 1030 1031 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1032 if (ret) { 1033 TESTPMD_LOG(DEBUG, 1034 "unable to DMA unmap addr 0x%p " 1035 "for device %s\n", 1036 memhdr->addr, rte_dev_name(dev_info.device)); 1037 } 1038 } 1039 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1040 if (ret) { 1041 TESTPMD_LOG(DEBUG, 1042 "unable to un-register addr 0x%p\n", memhdr->addr); 1043 } 1044 } 1045 1046 static void 1047 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1048 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1049 { 1050 uint16_t pid = 0; 1051 size_t page_size = sysconf(_SC_PAGESIZE); 1052 int ret; 1053 1054 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1055 page_size); 1056 if (ret) { 1057 TESTPMD_LOG(DEBUG, 1058 "unable to register addr 0x%p\n", memhdr->addr); 1059 return; 1060 } 1061 RTE_ETH_FOREACH_DEV(pid) { 1062 struct rte_eth_dev_info dev_info; 1063 1064 ret = eth_dev_info_get_print_err(pid, &dev_info); 1065 if (ret != 0) { 1066 TESTPMD_LOG(DEBUG, 1067 "unable to get device info for port %d on addr 0x%p," 1068 "mempool mapping will not be performed\n", 1069 pid, memhdr->addr); 1070 continue; 1071 } 1072 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1073 if (ret) { 1074 TESTPMD_LOG(DEBUG, 1075 "unable to DMA map addr 0x%p " 1076 "for device %s\n", 1077 memhdr->addr, rte_dev_name(dev_info.device)); 1078 } 1079 } 1080 } 1081 #endif 1082 1083 static unsigned int 1084 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1085 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1086 { 1087 struct rte_pktmbuf_extmem *xmem; 1088 unsigned int ext_num, zone_num, elt_num; 1089 uint16_t elt_size; 1090 1091 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1092 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1093 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1094 1095 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1096 if (xmem == NULL) { 1097 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1098 "external buffer descriptors\n"); 1099 *ext_mem = NULL; 1100 return 0; 1101 } 1102 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1103 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1104 const struct rte_memzone *mz; 1105 char mz_name[RTE_MEMZONE_NAMESIZE]; 1106 int ret; 1107 1108 ret = snprintf(mz_name, sizeof(mz_name), 1109 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1110 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1111 errno = ENAMETOOLONG; 1112 ext_num = 0; 1113 break; 1114 } 1115 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1116 socket_id, 1117 RTE_MEMZONE_IOVA_CONTIG | 1118 RTE_MEMZONE_1GB | 1119 RTE_MEMZONE_SIZE_HINT_ONLY); 1120 if (mz == NULL) { 1121 /* 1122 * The caller exits on external buffer creation 1123 * error, so there is no need to free memzones. 1124 */ 1125 errno = ENOMEM; 1126 ext_num = 0; 1127 break; 1128 } 1129 xseg->buf_ptr = mz->addr; 1130 xseg->buf_iova = mz->iova; 1131 xseg->buf_len = EXTBUF_ZONE_SIZE; 1132 xseg->elt_size = elt_size; 1133 } 1134 if (ext_num == 0 && xmem != NULL) { 1135 free(xmem); 1136 xmem = NULL; 1137 } 1138 *ext_mem = xmem; 1139 return ext_num; 1140 } 1141 1142 /* 1143 * Configuration initialisation done once at init time. 1144 */ 1145 static struct rte_mempool * 1146 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1147 unsigned int socket_id, uint16_t size_idx) 1148 { 1149 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1150 struct rte_mempool *rte_mp = NULL; 1151 #ifndef RTE_EXEC_ENV_WINDOWS 1152 uint32_t mb_size; 1153 1154 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1155 #endif 1156 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1157 if (!is_proc_primary()) { 1158 rte_mp = rte_mempool_lookup(pool_name); 1159 if (rte_mp == NULL) 1160 rte_exit(EXIT_FAILURE, 1161 "Get mbuf pool for socket %u failed: %s\n", 1162 socket_id, rte_strerror(rte_errno)); 1163 return rte_mp; 1164 } 1165 1166 TESTPMD_LOG(INFO, 1167 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1168 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1169 1170 switch (mp_alloc_type) { 1171 case MP_ALLOC_NATIVE: 1172 { 1173 /* wrapper to rte_mempool_create() */ 1174 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1175 rte_mbuf_best_mempool_ops()); 1176 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1177 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1178 break; 1179 } 1180 #ifndef RTE_EXEC_ENV_WINDOWS 1181 case MP_ALLOC_ANON: 1182 { 1183 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1184 mb_size, (unsigned int) mb_mempool_cache, 1185 sizeof(struct rte_pktmbuf_pool_private), 1186 socket_id, mempool_flags); 1187 if (rte_mp == NULL) 1188 goto err; 1189 1190 if (rte_mempool_populate_anon(rte_mp) == 0) { 1191 rte_mempool_free(rte_mp); 1192 rte_mp = NULL; 1193 goto err; 1194 } 1195 rte_pktmbuf_pool_init(rte_mp, NULL); 1196 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1197 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1198 break; 1199 } 1200 case MP_ALLOC_XMEM: 1201 case MP_ALLOC_XMEM_HUGE: 1202 { 1203 int heap_socket; 1204 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1205 1206 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1207 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1208 1209 heap_socket = 1210 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1211 if (heap_socket < 0) 1212 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1213 1214 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1215 rte_mbuf_best_mempool_ops()); 1216 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1217 mb_mempool_cache, 0, mbuf_seg_size, 1218 heap_socket); 1219 break; 1220 } 1221 #endif 1222 case MP_ALLOC_XBUF: 1223 { 1224 struct rte_pktmbuf_extmem *ext_mem; 1225 unsigned int ext_num; 1226 1227 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1228 socket_id, pool_name, &ext_mem); 1229 if (ext_num == 0) 1230 rte_exit(EXIT_FAILURE, 1231 "Can't create pinned data buffers\n"); 1232 1233 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1234 rte_mbuf_best_mempool_ops()); 1235 rte_mp = rte_pktmbuf_pool_create_extbuf 1236 (pool_name, nb_mbuf, mb_mempool_cache, 1237 0, mbuf_seg_size, socket_id, 1238 ext_mem, ext_num); 1239 free(ext_mem); 1240 break; 1241 } 1242 default: 1243 { 1244 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1245 } 1246 } 1247 1248 #ifndef RTE_EXEC_ENV_WINDOWS 1249 err: 1250 #endif 1251 if (rte_mp == NULL) { 1252 rte_exit(EXIT_FAILURE, 1253 "Creation of mbuf pool for socket %u failed: %s\n", 1254 socket_id, rte_strerror(rte_errno)); 1255 } else if (verbose_level > 0) { 1256 rte_mempool_dump(stdout, rte_mp); 1257 } 1258 return rte_mp; 1259 } 1260 1261 /* 1262 * Check given socket id is valid or not with NUMA mode, 1263 * if valid, return 0, else return -1 1264 */ 1265 static int 1266 check_socket_id(const unsigned int socket_id) 1267 { 1268 static int warning_once = 0; 1269 1270 if (new_socket_id(socket_id)) { 1271 if (!warning_once && numa_support) 1272 fprintf(stderr, 1273 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1274 warning_once = 1; 1275 return -1; 1276 } 1277 return 0; 1278 } 1279 1280 /* 1281 * Get the allowed maximum number of RX queues. 1282 * *pid return the port id which has minimal value of 1283 * max_rx_queues in all ports. 1284 */ 1285 queueid_t 1286 get_allowed_max_nb_rxq(portid_t *pid) 1287 { 1288 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1289 bool max_rxq_valid = false; 1290 portid_t pi; 1291 struct rte_eth_dev_info dev_info; 1292 1293 RTE_ETH_FOREACH_DEV(pi) { 1294 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1295 continue; 1296 1297 max_rxq_valid = true; 1298 if (dev_info.max_rx_queues < allowed_max_rxq) { 1299 allowed_max_rxq = dev_info.max_rx_queues; 1300 *pid = pi; 1301 } 1302 } 1303 return max_rxq_valid ? allowed_max_rxq : 0; 1304 } 1305 1306 /* 1307 * Check input rxq is valid or not. 1308 * If input rxq is not greater than any of maximum number 1309 * of RX queues of all ports, it is valid. 1310 * if valid, return 0, else return -1 1311 */ 1312 int 1313 check_nb_rxq(queueid_t rxq) 1314 { 1315 queueid_t allowed_max_rxq; 1316 portid_t pid = 0; 1317 1318 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1319 if (rxq > allowed_max_rxq) { 1320 fprintf(stderr, 1321 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1322 rxq, allowed_max_rxq, pid); 1323 return -1; 1324 } 1325 return 0; 1326 } 1327 1328 /* 1329 * Get the allowed maximum number of TX queues. 1330 * *pid return the port id which has minimal value of 1331 * max_tx_queues in all ports. 1332 */ 1333 queueid_t 1334 get_allowed_max_nb_txq(portid_t *pid) 1335 { 1336 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1337 bool max_txq_valid = false; 1338 portid_t pi; 1339 struct rte_eth_dev_info dev_info; 1340 1341 RTE_ETH_FOREACH_DEV(pi) { 1342 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1343 continue; 1344 1345 max_txq_valid = true; 1346 if (dev_info.max_tx_queues < allowed_max_txq) { 1347 allowed_max_txq = dev_info.max_tx_queues; 1348 *pid = pi; 1349 } 1350 } 1351 return max_txq_valid ? allowed_max_txq : 0; 1352 } 1353 1354 /* 1355 * Check input txq is valid or not. 1356 * If input txq is not greater than any of maximum number 1357 * of TX queues of all ports, it is valid. 1358 * if valid, return 0, else return -1 1359 */ 1360 int 1361 check_nb_txq(queueid_t txq) 1362 { 1363 queueid_t allowed_max_txq; 1364 portid_t pid = 0; 1365 1366 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1367 if (txq > allowed_max_txq) { 1368 fprintf(stderr, 1369 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1370 txq, allowed_max_txq, pid); 1371 return -1; 1372 } 1373 return 0; 1374 } 1375 1376 /* 1377 * Get the allowed maximum number of RXDs of every rx queue. 1378 * *pid return the port id which has minimal value of 1379 * max_rxd in all queues of all ports. 1380 */ 1381 static uint16_t 1382 get_allowed_max_nb_rxd(portid_t *pid) 1383 { 1384 uint16_t allowed_max_rxd = UINT16_MAX; 1385 portid_t pi; 1386 struct rte_eth_dev_info dev_info; 1387 1388 RTE_ETH_FOREACH_DEV(pi) { 1389 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1390 continue; 1391 1392 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1393 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1394 *pid = pi; 1395 } 1396 } 1397 return allowed_max_rxd; 1398 } 1399 1400 /* 1401 * Get the allowed minimal number of RXDs of every rx queue. 1402 * *pid return the port id which has minimal value of 1403 * min_rxd in all queues of all ports. 1404 */ 1405 static uint16_t 1406 get_allowed_min_nb_rxd(portid_t *pid) 1407 { 1408 uint16_t allowed_min_rxd = 0; 1409 portid_t pi; 1410 struct rte_eth_dev_info dev_info; 1411 1412 RTE_ETH_FOREACH_DEV(pi) { 1413 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1414 continue; 1415 1416 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1417 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1418 *pid = pi; 1419 } 1420 } 1421 1422 return allowed_min_rxd; 1423 } 1424 1425 /* 1426 * Check input rxd is valid or not. 1427 * If input rxd is not greater than any of maximum number 1428 * of RXDs of every Rx queues and is not less than any of 1429 * minimal number of RXDs of every Rx queues, it is valid. 1430 * if valid, return 0, else return -1 1431 */ 1432 int 1433 check_nb_rxd(queueid_t rxd) 1434 { 1435 uint16_t allowed_max_rxd; 1436 uint16_t allowed_min_rxd; 1437 portid_t pid = 0; 1438 1439 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1440 if (rxd > allowed_max_rxd) { 1441 fprintf(stderr, 1442 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1443 rxd, allowed_max_rxd, pid); 1444 return -1; 1445 } 1446 1447 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1448 if (rxd < allowed_min_rxd) { 1449 fprintf(stderr, 1450 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1451 rxd, allowed_min_rxd, pid); 1452 return -1; 1453 } 1454 1455 return 0; 1456 } 1457 1458 /* 1459 * Get the allowed maximum number of TXDs of every rx queues. 1460 * *pid return the port id which has minimal value of 1461 * max_txd in every tx queue. 1462 */ 1463 static uint16_t 1464 get_allowed_max_nb_txd(portid_t *pid) 1465 { 1466 uint16_t allowed_max_txd = UINT16_MAX; 1467 portid_t pi; 1468 struct rte_eth_dev_info dev_info; 1469 1470 RTE_ETH_FOREACH_DEV(pi) { 1471 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1472 continue; 1473 1474 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1475 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1476 *pid = pi; 1477 } 1478 } 1479 return allowed_max_txd; 1480 } 1481 1482 /* 1483 * Get the allowed maximum number of TXDs of every tx queues. 1484 * *pid return the port id which has minimal value of 1485 * min_txd in every tx queue. 1486 */ 1487 static uint16_t 1488 get_allowed_min_nb_txd(portid_t *pid) 1489 { 1490 uint16_t allowed_min_txd = 0; 1491 portid_t pi; 1492 struct rte_eth_dev_info dev_info; 1493 1494 RTE_ETH_FOREACH_DEV(pi) { 1495 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1496 continue; 1497 1498 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1499 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1500 *pid = pi; 1501 } 1502 } 1503 1504 return allowed_min_txd; 1505 } 1506 1507 /* 1508 * Check input txd is valid or not. 1509 * If input txd is not greater than any of maximum number 1510 * of TXDs of every Rx queues, it is valid. 1511 * if valid, return 0, else return -1 1512 */ 1513 int 1514 check_nb_txd(queueid_t txd) 1515 { 1516 uint16_t allowed_max_txd; 1517 uint16_t allowed_min_txd; 1518 portid_t pid = 0; 1519 1520 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1521 if (txd > allowed_max_txd) { 1522 fprintf(stderr, 1523 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1524 txd, allowed_max_txd, pid); 1525 return -1; 1526 } 1527 1528 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1529 if (txd < allowed_min_txd) { 1530 fprintf(stderr, 1531 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1532 txd, allowed_min_txd, pid); 1533 return -1; 1534 } 1535 return 0; 1536 } 1537 1538 1539 /* 1540 * Get the allowed maximum number of hairpin queues. 1541 * *pid return the port id which has minimal value of 1542 * max_hairpin_queues in all ports. 1543 */ 1544 queueid_t 1545 get_allowed_max_nb_hairpinq(portid_t *pid) 1546 { 1547 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1548 portid_t pi; 1549 struct rte_eth_hairpin_cap cap; 1550 1551 RTE_ETH_FOREACH_DEV(pi) { 1552 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1553 *pid = pi; 1554 return 0; 1555 } 1556 if (cap.max_nb_queues < allowed_max_hairpinq) { 1557 allowed_max_hairpinq = cap.max_nb_queues; 1558 *pid = pi; 1559 } 1560 } 1561 return allowed_max_hairpinq; 1562 } 1563 1564 /* 1565 * Check input hairpin is valid or not. 1566 * If input hairpin is not greater than any of maximum number 1567 * of hairpin queues of all ports, it is valid. 1568 * if valid, return 0, else return -1 1569 */ 1570 int 1571 check_nb_hairpinq(queueid_t hairpinq) 1572 { 1573 queueid_t allowed_max_hairpinq; 1574 portid_t pid = 0; 1575 1576 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1577 if (hairpinq > allowed_max_hairpinq) { 1578 fprintf(stderr, 1579 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1580 hairpinq, allowed_max_hairpinq, pid); 1581 return -1; 1582 } 1583 return 0; 1584 } 1585 1586 static int 1587 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1588 { 1589 uint32_t eth_overhead; 1590 1591 if (dev_info->max_mtu != UINT16_MAX && 1592 dev_info->max_rx_pktlen > dev_info->max_mtu) 1593 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1594 else 1595 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1596 1597 return eth_overhead; 1598 } 1599 1600 static void 1601 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1602 { 1603 struct rte_port *port = &ports[pid]; 1604 int ret; 1605 int i; 1606 1607 eth_rx_metadata_negotiate_mp(pid); 1608 1609 port->dev_conf.txmode = tx_mode; 1610 port->dev_conf.rxmode = rx_mode; 1611 1612 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1613 if (ret != 0) 1614 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1615 1616 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1617 port->dev_conf.txmode.offloads &= 1618 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1619 1620 /* Apply Rx offloads configuration */ 1621 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1622 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1623 /* Apply Tx offloads configuration */ 1624 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1625 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1626 1627 if (eth_link_speed) 1628 port->dev_conf.link_speeds = eth_link_speed; 1629 1630 if (max_rx_pkt_len) 1631 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1632 get_eth_overhead(&port->dev_info); 1633 1634 /* set flag to initialize port/queue */ 1635 port->need_reconfig = 1; 1636 port->need_reconfig_queues = 1; 1637 port->socket_id = socket_id; 1638 port->tx_metadata = 0; 1639 1640 /* 1641 * Check for maximum number of segments per MTU. 1642 * Accordingly update the mbuf data size. 1643 */ 1644 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1645 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1646 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1647 uint16_t mtu; 1648 1649 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1650 uint16_t data_size = (mtu + eth_overhead) / 1651 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1652 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1653 1654 if (buffer_size > mbuf_data_size[0]) { 1655 mbuf_data_size[0] = buffer_size; 1656 TESTPMD_LOG(WARNING, 1657 "Configured mbuf size of the first segment %hu\n", 1658 mbuf_data_size[0]); 1659 } 1660 } 1661 } 1662 } 1663 1664 static void 1665 init_config(void) 1666 { 1667 portid_t pid; 1668 struct rte_mempool *mbp; 1669 unsigned int nb_mbuf_per_pool; 1670 lcoreid_t lc_id; 1671 #ifdef RTE_LIB_GRO 1672 struct rte_gro_param gro_param; 1673 #endif 1674 #ifdef RTE_LIB_GSO 1675 uint32_t gso_types; 1676 #endif 1677 1678 /* Configuration of logical cores. */ 1679 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1680 sizeof(struct fwd_lcore *) * nb_lcores, 1681 RTE_CACHE_LINE_SIZE); 1682 if (fwd_lcores == NULL) { 1683 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1684 "failed\n", nb_lcores); 1685 } 1686 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1687 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1688 sizeof(struct fwd_lcore), 1689 RTE_CACHE_LINE_SIZE); 1690 if (fwd_lcores[lc_id] == NULL) { 1691 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1692 "failed\n"); 1693 } 1694 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1695 } 1696 1697 RTE_ETH_FOREACH_DEV(pid) { 1698 uint32_t socket_id; 1699 1700 if (numa_support) { 1701 socket_id = port_numa[pid]; 1702 if (port_numa[pid] == NUMA_NO_CONFIG) { 1703 socket_id = rte_eth_dev_socket_id(pid); 1704 1705 /* 1706 * if socket_id is invalid, 1707 * set to the first available socket. 1708 */ 1709 if (check_socket_id(socket_id) < 0) 1710 socket_id = socket_ids[0]; 1711 } 1712 } else { 1713 socket_id = (socket_num == UMA_NO_CONFIG) ? 1714 0 : socket_num; 1715 } 1716 /* Apply default TxRx configuration for all ports */ 1717 init_config_port_offloads(pid, socket_id); 1718 } 1719 /* 1720 * Create pools of mbuf. 1721 * If NUMA support is disabled, create a single pool of mbuf in 1722 * socket 0 memory by default. 1723 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1724 * 1725 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1726 * nb_txd can be configured at run time. 1727 */ 1728 if (param_total_num_mbufs) 1729 nb_mbuf_per_pool = param_total_num_mbufs; 1730 else { 1731 nb_mbuf_per_pool = RX_DESC_MAX + 1732 (nb_lcores * mb_mempool_cache) + 1733 TX_DESC_MAX + MAX_PKT_BURST; 1734 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1735 } 1736 1737 if (numa_support) { 1738 uint8_t i, j; 1739 1740 for (i = 0; i < num_sockets; i++) 1741 for (j = 0; j < mbuf_data_size_n; j++) 1742 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1743 mbuf_pool_create(mbuf_data_size[j], 1744 nb_mbuf_per_pool, 1745 socket_ids[i], j); 1746 } else { 1747 uint8_t i; 1748 1749 for (i = 0; i < mbuf_data_size_n; i++) 1750 mempools[i] = mbuf_pool_create 1751 (mbuf_data_size[i], 1752 nb_mbuf_per_pool, 1753 socket_num == UMA_NO_CONFIG ? 1754 0 : socket_num, i); 1755 } 1756 1757 init_port_config(); 1758 1759 #ifdef RTE_LIB_GSO 1760 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1761 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1762 #endif 1763 /* 1764 * Records which Mbuf pool to use by each logical core, if needed. 1765 */ 1766 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1767 mbp = mbuf_pool_find( 1768 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1769 1770 if (mbp == NULL) 1771 mbp = mbuf_pool_find(0, 0); 1772 fwd_lcores[lc_id]->mbp = mbp; 1773 #ifdef RTE_LIB_GSO 1774 /* initialize GSO context */ 1775 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1776 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1777 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1778 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1779 RTE_ETHER_CRC_LEN; 1780 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1781 #endif 1782 } 1783 1784 fwd_config_setup(); 1785 1786 #ifdef RTE_LIB_GRO 1787 /* create a gro context for each lcore */ 1788 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1789 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1790 gro_param.max_item_per_flow = MAX_PKT_BURST; 1791 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1792 gro_param.socket_id = rte_lcore_to_socket_id( 1793 fwd_lcores_cpuids[lc_id]); 1794 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1795 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1796 rte_exit(EXIT_FAILURE, 1797 "rte_gro_ctx_create() failed\n"); 1798 } 1799 } 1800 #endif 1801 } 1802 1803 1804 void 1805 reconfig(portid_t new_port_id, unsigned socket_id) 1806 { 1807 /* Reconfiguration of Ethernet ports. */ 1808 init_config_port_offloads(new_port_id, socket_id); 1809 init_port_config(); 1810 } 1811 1812 int 1813 init_fwd_streams(void) 1814 { 1815 portid_t pid; 1816 struct rte_port *port; 1817 streamid_t sm_id, nb_fwd_streams_new; 1818 queueid_t q; 1819 1820 /* set socket id according to numa or not */ 1821 RTE_ETH_FOREACH_DEV(pid) { 1822 port = &ports[pid]; 1823 if (nb_rxq > port->dev_info.max_rx_queues) { 1824 fprintf(stderr, 1825 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1826 nb_rxq, port->dev_info.max_rx_queues); 1827 return -1; 1828 } 1829 if (nb_txq > port->dev_info.max_tx_queues) { 1830 fprintf(stderr, 1831 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1832 nb_txq, port->dev_info.max_tx_queues); 1833 return -1; 1834 } 1835 if (numa_support) { 1836 if (port_numa[pid] != NUMA_NO_CONFIG) 1837 port->socket_id = port_numa[pid]; 1838 else { 1839 port->socket_id = rte_eth_dev_socket_id(pid); 1840 1841 /* 1842 * if socket_id is invalid, 1843 * set to the first available socket. 1844 */ 1845 if (check_socket_id(port->socket_id) < 0) 1846 port->socket_id = socket_ids[0]; 1847 } 1848 } 1849 else { 1850 if (socket_num == UMA_NO_CONFIG) 1851 port->socket_id = 0; 1852 else 1853 port->socket_id = socket_num; 1854 } 1855 } 1856 1857 q = RTE_MAX(nb_rxq, nb_txq); 1858 if (q == 0) { 1859 fprintf(stderr, 1860 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1861 return -1; 1862 } 1863 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1864 if (nb_fwd_streams_new == nb_fwd_streams) 1865 return 0; 1866 /* clear the old */ 1867 if (fwd_streams != NULL) { 1868 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1869 if (fwd_streams[sm_id] == NULL) 1870 continue; 1871 rte_free(fwd_streams[sm_id]); 1872 fwd_streams[sm_id] = NULL; 1873 } 1874 rte_free(fwd_streams); 1875 fwd_streams = NULL; 1876 } 1877 1878 /* init new */ 1879 nb_fwd_streams = nb_fwd_streams_new; 1880 if (nb_fwd_streams) { 1881 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1882 sizeof(struct fwd_stream *) * nb_fwd_streams, 1883 RTE_CACHE_LINE_SIZE); 1884 if (fwd_streams == NULL) 1885 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1886 " (struct fwd_stream *)) failed\n", 1887 nb_fwd_streams); 1888 1889 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1890 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1891 " struct fwd_stream", sizeof(struct fwd_stream), 1892 RTE_CACHE_LINE_SIZE); 1893 if (fwd_streams[sm_id] == NULL) 1894 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1895 "(struct fwd_stream) failed\n"); 1896 } 1897 } 1898 1899 return 0; 1900 } 1901 1902 static void 1903 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1904 { 1905 uint64_t total_burst, sburst; 1906 uint64_t nb_burst; 1907 uint64_t burst_stats[4]; 1908 uint16_t pktnb_stats[4]; 1909 uint16_t nb_pkt; 1910 int burst_percent[4], sburstp; 1911 int i; 1912 1913 /* 1914 * First compute the total number of packet bursts and the 1915 * two highest numbers of bursts of the same number of packets. 1916 */ 1917 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1918 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1919 1920 /* Show stats for 0 burst size always */ 1921 total_burst = pbs->pkt_burst_spread[0]; 1922 burst_stats[0] = pbs->pkt_burst_spread[0]; 1923 pktnb_stats[0] = 0; 1924 1925 /* Find the next 2 burst sizes with highest occurrences. */ 1926 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1927 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1928 1929 if (nb_burst == 0) 1930 continue; 1931 1932 total_burst += nb_burst; 1933 1934 if (nb_burst > burst_stats[1]) { 1935 burst_stats[2] = burst_stats[1]; 1936 pktnb_stats[2] = pktnb_stats[1]; 1937 burst_stats[1] = nb_burst; 1938 pktnb_stats[1] = nb_pkt; 1939 } else if (nb_burst > burst_stats[2]) { 1940 burst_stats[2] = nb_burst; 1941 pktnb_stats[2] = nb_pkt; 1942 } 1943 } 1944 if (total_burst == 0) 1945 return; 1946 1947 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1948 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1949 if (i == 3) { 1950 printf("%d%% of other]\n", 100 - sburstp); 1951 return; 1952 } 1953 1954 sburst += burst_stats[i]; 1955 if (sburst == total_burst) { 1956 printf("%d%% of %d pkts]\n", 1957 100 - sburstp, (int) pktnb_stats[i]); 1958 return; 1959 } 1960 1961 burst_percent[i] = 1962 (double)burst_stats[i] / total_burst * 100; 1963 printf("%d%% of %d pkts + ", 1964 burst_percent[i], (int) pktnb_stats[i]); 1965 sburstp += burst_percent[i]; 1966 } 1967 } 1968 1969 static void 1970 fwd_stream_stats_display(streamid_t stream_id) 1971 { 1972 struct fwd_stream *fs; 1973 static const char *fwd_top_stats_border = "-------"; 1974 1975 fs = fwd_streams[stream_id]; 1976 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1977 (fs->fwd_dropped == 0)) 1978 return; 1979 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1980 "TX Port=%2d/Queue=%2d %s\n", 1981 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1982 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1983 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1984 " TX-dropped: %-14"PRIu64, 1985 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1986 1987 /* if checksum mode */ 1988 if (cur_fwd_eng == &csum_fwd_engine) { 1989 printf(" RX- bad IP checksum: %-14"PRIu64 1990 " Rx- bad L4 checksum: %-14"PRIu64 1991 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1992 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1993 fs->rx_bad_outer_l4_csum); 1994 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1995 fs->rx_bad_outer_ip_csum); 1996 } else { 1997 printf("\n"); 1998 } 1999 2000 if (record_burst_stats) { 2001 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2002 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2003 } 2004 } 2005 2006 void 2007 fwd_stats_display(void) 2008 { 2009 static const char *fwd_stats_border = "----------------------"; 2010 static const char *acc_stats_border = "+++++++++++++++"; 2011 struct { 2012 struct fwd_stream *rx_stream; 2013 struct fwd_stream *tx_stream; 2014 uint64_t tx_dropped; 2015 uint64_t rx_bad_ip_csum; 2016 uint64_t rx_bad_l4_csum; 2017 uint64_t rx_bad_outer_l4_csum; 2018 uint64_t rx_bad_outer_ip_csum; 2019 } ports_stats[RTE_MAX_ETHPORTS]; 2020 uint64_t total_rx_dropped = 0; 2021 uint64_t total_tx_dropped = 0; 2022 uint64_t total_rx_nombuf = 0; 2023 struct rte_eth_stats stats; 2024 uint64_t fwd_cycles = 0; 2025 uint64_t total_recv = 0; 2026 uint64_t total_xmit = 0; 2027 struct rte_port *port; 2028 streamid_t sm_id; 2029 portid_t pt_id; 2030 int ret; 2031 int i; 2032 2033 memset(ports_stats, 0, sizeof(ports_stats)); 2034 2035 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2036 struct fwd_stream *fs = fwd_streams[sm_id]; 2037 2038 if (cur_fwd_config.nb_fwd_streams > 2039 cur_fwd_config.nb_fwd_ports) { 2040 fwd_stream_stats_display(sm_id); 2041 } else { 2042 ports_stats[fs->tx_port].tx_stream = fs; 2043 ports_stats[fs->rx_port].rx_stream = fs; 2044 } 2045 2046 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2047 2048 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2049 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2050 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2051 fs->rx_bad_outer_l4_csum; 2052 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2053 fs->rx_bad_outer_ip_csum; 2054 2055 if (record_core_cycles) 2056 fwd_cycles += fs->busy_cycles; 2057 } 2058 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2059 pt_id = fwd_ports_ids[i]; 2060 port = &ports[pt_id]; 2061 2062 ret = rte_eth_stats_get(pt_id, &stats); 2063 if (ret != 0) { 2064 fprintf(stderr, 2065 "%s: Error: failed to get stats (port %u): %d", 2066 __func__, pt_id, ret); 2067 continue; 2068 } 2069 stats.ipackets -= port->stats.ipackets; 2070 stats.opackets -= port->stats.opackets; 2071 stats.ibytes -= port->stats.ibytes; 2072 stats.obytes -= port->stats.obytes; 2073 stats.imissed -= port->stats.imissed; 2074 stats.oerrors -= port->stats.oerrors; 2075 stats.rx_nombuf -= port->stats.rx_nombuf; 2076 2077 total_recv += stats.ipackets; 2078 total_xmit += stats.opackets; 2079 total_rx_dropped += stats.imissed; 2080 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2081 total_tx_dropped += stats.oerrors; 2082 total_rx_nombuf += stats.rx_nombuf; 2083 2084 printf("\n %s Forward statistics for port %-2d %s\n", 2085 fwd_stats_border, pt_id, fwd_stats_border); 2086 2087 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2088 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2089 stats.ipackets + stats.imissed); 2090 2091 if (cur_fwd_eng == &csum_fwd_engine) { 2092 printf(" Bad-ipcsum: %-14"PRIu64 2093 " Bad-l4csum: %-14"PRIu64 2094 "Bad-outer-l4csum: %-14"PRIu64"\n", 2095 ports_stats[pt_id].rx_bad_ip_csum, 2096 ports_stats[pt_id].rx_bad_l4_csum, 2097 ports_stats[pt_id].rx_bad_outer_l4_csum); 2098 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2099 ports_stats[pt_id].rx_bad_outer_ip_csum); 2100 } 2101 if (stats.ierrors + stats.rx_nombuf > 0) { 2102 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2103 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2104 } 2105 2106 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2107 "TX-total: %-"PRIu64"\n", 2108 stats.opackets, ports_stats[pt_id].tx_dropped, 2109 stats.opackets + ports_stats[pt_id].tx_dropped); 2110 2111 if (record_burst_stats) { 2112 if (ports_stats[pt_id].rx_stream) 2113 pkt_burst_stats_display("RX", 2114 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2115 if (ports_stats[pt_id].tx_stream) 2116 pkt_burst_stats_display("TX", 2117 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2118 } 2119 2120 printf(" %s--------------------------------%s\n", 2121 fwd_stats_border, fwd_stats_border); 2122 } 2123 2124 printf("\n %s Accumulated forward statistics for all ports" 2125 "%s\n", 2126 acc_stats_border, acc_stats_border); 2127 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2128 "%-"PRIu64"\n" 2129 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2130 "%-"PRIu64"\n", 2131 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2132 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2133 if (total_rx_nombuf > 0) 2134 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2135 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2136 "%s\n", 2137 acc_stats_border, acc_stats_border); 2138 if (record_core_cycles) { 2139 #define CYC_PER_MHZ 1E6 2140 if (total_recv > 0 || total_xmit > 0) { 2141 uint64_t total_pkts = 0; 2142 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2143 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2144 total_pkts = total_xmit; 2145 else 2146 total_pkts = total_recv; 2147 2148 printf("\n CPU cycles/packet=%.2F (busy cycles=" 2149 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2150 " MHz Clock\n", 2151 (double) fwd_cycles / total_pkts, 2152 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2153 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2154 } 2155 } 2156 } 2157 2158 void 2159 fwd_stats_reset(void) 2160 { 2161 streamid_t sm_id; 2162 portid_t pt_id; 2163 int ret; 2164 int i; 2165 2166 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2167 pt_id = fwd_ports_ids[i]; 2168 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2169 if (ret != 0) 2170 fprintf(stderr, 2171 "%s: Error: failed to clear stats (port %u):%d", 2172 __func__, pt_id, ret); 2173 } 2174 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2175 struct fwd_stream *fs = fwd_streams[sm_id]; 2176 2177 fs->rx_packets = 0; 2178 fs->tx_packets = 0; 2179 fs->fwd_dropped = 0; 2180 fs->rx_bad_ip_csum = 0; 2181 fs->rx_bad_l4_csum = 0; 2182 fs->rx_bad_outer_l4_csum = 0; 2183 fs->rx_bad_outer_ip_csum = 0; 2184 2185 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2186 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2187 fs->busy_cycles = 0; 2188 } 2189 } 2190 2191 static void 2192 flush_fwd_rx_queues(void) 2193 { 2194 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2195 portid_t rxp; 2196 portid_t port_id; 2197 queueid_t rxq; 2198 uint16_t nb_rx; 2199 uint16_t i; 2200 uint8_t j; 2201 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2202 uint64_t timer_period; 2203 2204 if (num_procs > 1) { 2205 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2206 return; 2207 } 2208 2209 /* convert to number of cycles */ 2210 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2211 2212 for (j = 0; j < 2; j++) { 2213 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2214 for (rxq = 0; rxq < nb_rxq; rxq++) { 2215 port_id = fwd_ports_ids[rxp]; 2216 2217 /* Polling stopped queues is prohibited. */ 2218 if (ports[port_id].rxq[rxq].state == 2219 RTE_ETH_QUEUE_STATE_STOPPED) 2220 continue; 2221 2222 /** 2223 * testpmd can stuck in the below do while loop 2224 * if rte_eth_rx_burst() always returns nonzero 2225 * packets. So timer is added to exit this loop 2226 * after 1sec timer expiry. 2227 */ 2228 prev_tsc = rte_rdtsc(); 2229 do { 2230 nb_rx = rte_eth_rx_burst(port_id, rxq, 2231 pkts_burst, MAX_PKT_BURST); 2232 for (i = 0; i < nb_rx; i++) 2233 rte_pktmbuf_free(pkts_burst[i]); 2234 2235 cur_tsc = rte_rdtsc(); 2236 diff_tsc = cur_tsc - prev_tsc; 2237 timer_tsc += diff_tsc; 2238 } while ((nb_rx > 0) && 2239 (timer_tsc < timer_period)); 2240 timer_tsc = 0; 2241 } 2242 } 2243 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2244 } 2245 } 2246 2247 static void 2248 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2249 { 2250 struct fwd_stream **fsm; 2251 uint64_t prev_tsc; 2252 streamid_t nb_fs; 2253 streamid_t sm_id; 2254 #ifdef RTE_LIB_BITRATESTATS 2255 uint64_t tics_per_1sec; 2256 uint64_t tics_datum; 2257 uint64_t tics_current; 2258 uint16_t i, cnt_ports; 2259 2260 cnt_ports = nb_ports; 2261 tics_datum = rte_rdtsc(); 2262 tics_per_1sec = rte_get_timer_hz(); 2263 #endif 2264 fsm = &fwd_streams[fc->stream_idx]; 2265 nb_fs = fc->stream_nb; 2266 prev_tsc = rte_rdtsc(); 2267 do { 2268 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2269 if (!fsm[sm_id]->disabled) 2270 (*pkt_fwd)(fsm[sm_id]); 2271 #ifdef RTE_LIB_BITRATESTATS 2272 if (bitrate_enabled != 0 && 2273 bitrate_lcore_id == rte_lcore_id()) { 2274 tics_current = rte_rdtsc(); 2275 if (tics_current - tics_datum >= tics_per_1sec) { 2276 /* Periodic bitrate calculation */ 2277 for (i = 0; i < cnt_ports; i++) 2278 rte_stats_bitrate_calc(bitrate_data, 2279 ports_ids[i]); 2280 tics_datum = tics_current; 2281 } 2282 } 2283 #endif 2284 #ifdef RTE_LIB_LATENCYSTATS 2285 if (latencystats_enabled != 0 && 2286 latencystats_lcore_id == rte_lcore_id()) 2287 rte_latencystats_update(); 2288 #endif 2289 if (record_core_cycles) { 2290 uint64_t tsc = rte_rdtsc(); 2291 2292 fc->total_cycles += tsc - prev_tsc; 2293 prev_tsc = tsc; 2294 } 2295 } while (! fc->stopped); 2296 } 2297 2298 static int 2299 lcore_usage_callback(unsigned int lcore_id, struct rte_lcore_usage *usage) 2300 { 2301 struct fwd_stream **fsm; 2302 struct fwd_lcore *fc; 2303 streamid_t nb_fs; 2304 streamid_t sm_id; 2305 2306 fc = lcore_to_fwd_lcore(lcore_id); 2307 if (fc == NULL) 2308 return -1; 2309 2310 fsm = &fwd_streams[fc->stream_idx]; 2311 nb_fs = fc->stream_nb; 2312 usage->busy_cycles = 0; 2313 usage->total_cycles = fc->total_cycles; 2314 2315 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2316 if (!fsm[sm_id]->disabled) 2317 usage->busy_cycles += fsm[sm_id]->busy_cycles; 2318 } 2319 2320 return 0; 2321 } 2322 2323 static int 2324 start_pkt_forward_on_core(void *fwd_arg) 2325 { 2326 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2327 cur_fwd_config.fwd_eng->packet_fwd); 2328 return 0; 2329 } 2330 2331 /* 2332 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2333 * Used to start communication flows in network loopback test configurations. 2334 */ 2335 static int 2336 run_one_txonly_burst_on_core(void *fwd_arg) 2337 { 2338 struct fwd_lcore *fwd_lc; 2339 struct fwd_lcore tmp_lcore; 2340 2341 fwd_lc = (struct fwd_lcore *) fwd_arg; 2342 tmp_lcore = *fwd_lc; 2343 tmp_lcore.stopped = 1; 2344 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2345 return 0; 2346 } 2347 2348 /* 2349 * Launch packet forwarding: 2350 * - Setup per-port forwarding context. 2351 * - launch logical cores with their forwarding configuration. 2352 */ 2353 static void 2354 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2355 { 2356 unsigned int i; 2357 unsigned int lc_id; 2358 int diag; 2359 2360 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2361 lc_id = fwd_lcores_cpuids[i]; 2362 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2363 fwd_lcores[i]->stopped = 0; 2364 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2365 fwd_lcores[i], lc_id); 2366 if (diag != 0) 2367 fprintf(stderr, 2368 "launch lcore %u failed - diag=%d\n", 2369 lc_id, diag); 2370 } 2371 } 2372 } 2373 2374 /* 2375 * Launch packet forwarding configuration. 2376 */ 2377 void 2378 start_packet_forwarding(int with_tx_first) 2379 { 2380 port_fwd_begin_t port_fwd_begin; 2381 port_fwd_end_t port_fwd_end; 2382 stream_init_t stream_init = cur_fwd_eng->stream_init; 2383 unsigned int i; 2384 2385 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2386 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2387 2388 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2389 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2390 2391 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2392 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2393 (!nb_rxq || !nb_txq)) 2394 rte_exit(EXIT_FAILURE, 2395 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2396 cur_fwd_eng->fwd_mode_name); 2397 2398 if (all_ports_started() == 0) { 2399 fprintf(stderr, "Not all ports were started\n"); 2400 return; 2401 } 2402 if (test_done == 0) { 2403 fprintf(stderr, "Packet forwarding already started\n"); 2404 return; 2405 } 2406 2407 fwd_config_setup(); 2408 2409 pkt_fwd_config_display(&cur_fwd_config); 2410 if (!pkt_fwd_shared_rxq_check()) 2411 return; 2412 2413 if (stream_init != NULL) 2414 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2415 stream_init(fwd_streams[i]); 2416 2417 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2418 if (port_fwd_begin != NULL) { 2419 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2420 if (port_fwd_begin(fwd_ports_ids[i])) { 2421 fprintf(stderr, 2422 "Packet forwarding is not ready\n"); 2423 return; 2424 } 2425 } 2426 } 2427 2428 if (with_tx_first) { 2429 port_fwd_begin = tx_only_engine.port_fwd_begin; 2430 if (port_fwd_begin != NULL) { 2431 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2432 if (port_fwd_begin(fwd_ports_ids[i])) { 2433 fprintf(stderr, 2434 "Packet forwarding is not ready\n"); 2435 return; 2436 } 2437 } 2438 } 2439 } 2440 2441 test_done = 0; 2442 2443 if(!no_flush_rx) 2444 flush_fwd_rx_queues(); 2445 2446 rxtx_config_display(); 2447 2448 fwd_stats_reset(); 2449 if (with_tx_first) { 2450 while (with_tx_first--) { 2451 launch_packet_forwarding( 2452 run_one_txonly_burst_on_core); 2453 rte_eal_mp_wait_lcore(); 2454 } 2455 port_fwd_end = tx_only_engine.port_fwd_end; 2456 if (port_fwd_end != NULL) { 2457 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2458 (*port_fwd_end)(fwd_ports_ids[i]); 2459 } 2460 } 2461 launch_packet_forwarding(start_pkt_forward_on_core); 2462 } 2463 2464 void 2465 stop_packet_forwarding(void) 2466 { 2467 port_fwd_end_t port_fwd_end; 2468 lcoreid_t lc_id; 2469 portid_t pt_id; 2470 int i; 2471 2472 if (test_done) { 2473 fprintf(stderr, "Packet forwarding not started\n"); 2474 return; 2475 } 2476 printf("Telling cores to stop..."); 2477 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2478 fwd_lcores[lc_id]->stopped = 1; 2479 printf("\nWaiting for lcores to finish...\n"); 2480 rte_eal_mp_wait_lcore(); 2481 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2482 if (port_fwd_end != NULL) { 2483 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2484 pt_id = fwd_ports_ids[i]; 2485 (*port_fwd_end)(pt_id); 2486 } 2487 } 2488 2489 fwd_stats_display(); 2490 2491 printf("\nDone.\n"); 2492 test_done = 1; 2493 } 2494 2495 void 2496 dev_set_link_up(portid_t pid) 2497 { 2498 if (rte_eth_dev_set_link_up(pid) < 0) 2499 fprintf(stderr, "\nSet link up fail.\n"); 2500 } 2501 2502 void 2503 dev_set_link_down(portid_t pid) 2504 { 2505 if (rte_eth_dev_set_link_down(pid) < 0) 2506 fprintf(stderr, "\nSet link down fail.\n"); 2507 } 2508 2509 static int 2510 all_ports_started(void) 2511 { 2512 portid_t pi; 2513 struct rte_port *port; 2514 2515 RTE_ETH_FOREACH_DEV(pi) { 2516 port = &ports[pi]; 2517 /* Check if there is a port which is not started */ 2518 if ((port->port_status != RTE_PORT_STARTED) && 2519 (port->slave_flag == 0)) 2520 return 0; 2521 } 2522 2523 /* No port is not started */ 2524 return 1; 2525 } 2526 2527 int 2528 port_is_stopped(portid_t port_id) 2529 { 2530 struct rte_port *port = &ports[port_id]; 2531 2532 if ((port->port_status != RTE_PORT_STOPPED) && 2533 (port->slave_flag == 0)) 2534 return 0; 2535 return 1; 2536 } 2537 2538 int 2539 all_ports_stopped(void) 2540 { 2541 portid_t pi; 2542 2543 RTE_ETH_FOREACH_DEV(pi) { 2544 if (!port_is_stopped(pi)) 2545 return 0; 2546 } 2547 2548 return 1; 2549 } 2550 2551 int 2552 port_is_started(portid_t port_id) 2553 { 2554 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2555 return 0; 2556 2557 if (ports[port_id].port_status != RTE_PORT_STARTED) 2558 return 0; 2559 2560 return 1; 2561 } 2562 2563 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2564 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2565 2566 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2567 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2568 2569 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2570 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2571 2572 2573 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2574 static int 2575 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2576 { 2577 queueid_t qi; 2578 struct rte_eth_hairpin_conf hairpin_conf = { 2579 .peer_count = 1, 2580 }; 2581 int i; 2582 int diag; 2583 struct rte_port *port = &ports[pi]; 2584 uint16_t peer_rx_port = pi; 2585 uint16_t peer_tx_port = pi; 2586 uint32_t manual = 1; 2587 uint32_t tx_exp = hairpin_mode & 0x10; 2588 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2589 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2590 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2591 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2592 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2593 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2594 2595 if (!(hairpin_mode & 0xf)) { 2596 peer_rx_port = pi; 2597 peer_tx_port = pi; 2598 manual = 0; 2599 } else if (hairpin_mode & 0x1) { 2600 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2601 RTE_ETH_DEV_NO_OWNER); 2602 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2603 peer_tx_port = rte_eth_find_next_owned_by(0, 2604 RTE_ETH_DEV_NO_OWNER); 2605 if (p_pi != RTE_MAX_ETHPORTS) { 2606 peer_rx_port = p_pi; 2607 } else { 2608 uint16_t next_pi; 2609 2610 /* Last port will be the peer RX port of the first. */ 2611 RTE_ETH_FOREACH_DEV(next_pi) 2612 peer_rx_port = next_pi; 2613 } 2614 manual = 1; 2615 } else if (hairpin_mode & 0x2) { 2616 if (cnt_pi & 0x1) { 2617 peer_rx_port = p_pi; 2618 } else { 2619 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2620 RTE_ETH_DEV_NO_OWNER); 2621 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2622 peer_rx_port = pi; 2623 } 2624 peer_tx_port = peer_rx_port; 2625 manual = 1; 2626 } 2627 2628 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2629 hairpin_conf.peers[0].port = peer_rx_port; 2630 hairpin_conf.peers[0].queue = i + nb_rxq; 2631 hairpin_conf.manual_bind = !!manual; 2632 hairpin_conf.tx_explicit = !!tx_exp; 2633 hairpin_conf.force_memory = !!tx_force_memory; 2634 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2635 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2636 diag = rte_eth_tx_hairpin_queue_setup 2637 (pi, qi, nb_txd, &hairpin_conf); 2638 i++; 2639 if (diag == 0) 2640 continue; 2641 2642 /* Fail to setup rx queue, return */ 2643 if (port->port_status == RTE_PORT_HANDLING) 2644 port->port_status = RTE_PORT_STOPPED; 2645 else 2646 fprintf(stderr, 2647 "Port %d can not be set back to stopped\n", pi); 2648 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2649 pi); 2650 /* try to reconfigure queues next time */ 2651 port->need_reconfig_queues = 1; 2652 return -1; 2653 } 2654 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2655 hairpin_conf.peers[0].port = peer_tx_port; 2656 hairpin_conf.peers[0].queue = i + nb_txq; 2657 hairpin_conf.manual_bind = !!manual; 2658 hairpin_conf.tx_explicit = !!tx_exp; 2659 hairpin_conf.force_memory = !!rx_force_memory; 2660 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2661 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2662 diag = rte_eth_rx_hairpin_queue_setup 2663 (pi, qi, nb_rxd, &hairpin_conf); 2664 i++; 2665 if (diag == 0) 2666 continue; 2667 2668 /* Fail to setup rx queue, return */ 2669 if (port->port_status == RTE_PORT_HANDLING) 2670 port->port_status = RTE_PORT_STOPPED; 2671 else 2672 fprintf(stderr, 2673 "Port %d can not be set back to stopped\n", pi); 2674 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2675 pi); 2676 /* try to reconfigure queues next time */ 2677 port->need_reconfig_queues = 1; 2678 return -1; 2679 } 2680 return 0; 2681 } 2682 2683 /* Configure the Rx with optional split. */ 2684 int 2685 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2686 uint16_t nb_rx_desc, unsigned int socket_id, 2687 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2688 { 2689 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2690 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2691 struct rte_mempool *mpx; 2692 unsigned int i, mp_n; 2693 uint32_t prev_hdrs = 0; 2694 int ret; 2695 2696 2697 if ((rx_pkt_nb_segs > 1) && 2698 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2699 /* multi-segment configuration */ 2700 for (i = 0; i < rx_pkt_nb_segs; i++) { 2701 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2702 /* 2703 * Use last valid pool for the segments with number 2704 * exceeding the pool index. 2705 */ 2706 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2707 mpx = mbuf_pool_find(socket_id, mp_n); 2708 /* Handle zero as mbuf data buffer size. */ 2709 rx_seg->offset = i < rx_pkt_nb_offs ? 2710 rx_pkt_seg_offsets[i] : 0; 2711 rx_seg->mp = mpx ? mpx : mp; 2712 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2713 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2714 prev_hdrs |= rx_seg->proto_hdr; 2715 } else { 2716 rx_seg->length = rx_pkt_seg_lengths[i] ? 2717 rx_pkt_seg_lengths[i] : 2718 mbuf_data_size[mp_n]; 2719 } 2720 } 2721 rx_conf->rx_nseg = rx_pkt_nb_segs; 2722 rx_conf->rx_seg = rx_useg; 2723 rx_conf->rx_mempools = NULL; 2724 rx_conf->rx_nmempool = 0; 2725 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2726 socket_id, rx_conf, NULL); 2727 rx_conf->rx_seg = NULL; 2728 rx_conf->rx_nseg = 0; 2729 } else if (multi_rx_mempool == 1) { 2730 /* multi-pool configuration */ 2731 struct rte_eth_dev_info dev_info; 2732 2733 if (mbuf_data_size_n <= 1) { 2734 fprintf(stderr, "Invalid number of mempools %u\n", 2735 mbuf_data_size_n); 2736 return -EINVAL; 2737 } 2738 ret = rte_eth_dev_info_get(port_id, &dev_info); 2739 if (ret != 0) 2740 return ret; 2741 if (dev_info.max_rx_mempools == 0) { 2742 fprintf(stderr, 2743 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2744 port_id); 2745 return -ENOTSUP; 2746 } 2747 for (i = 0; i < mbuf_data_size_n; i++) { 2748 mpx = mbuf_pool_find(socket_id, i); 2749 rx_mempool[i] = mpx ? mpx : mp; 2750 } 2751 rx_conf->rx_mempools = rx_mempool; 2752 rx_conf->rx_nmempool = mbuf_data_size_n; 2753 rx_conf->rx_seg = NULL; 2754 rx_conf->rx_nseg = 0; 2755 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2756 socket_id, rx_conf, NULL); 2757 rx_conf->rx_mempools = NULL; 2758 rx_conf->rx_nmempool = 0; 2759 } else { 2760 /* Single pool/segment configuration */ 2761 rx_conf->rx_seg = NULL; 2762 rx_conf->rx_nseg = 0; 2763 rx_conf->rx_mempools = NULL; 2764 rx_conf->rx_nmempool = 0; 2765 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2766 socket_id, rx_conf, mp); 2767 } 2768 2769 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2770 RTE_ETH_QUEUE_STATE_STOPPED : 2771 RTE_ETH_QUEUE_STATE_STARTED; 2772 return ret; 2773 } 2774 2775 static int 2776 alloc_xstats_display_info(portid_t pi) 2777 { 2778 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2779 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2780 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2781 2782 if (xstats_display_num == 0) 2783 return 0; 2784 2785 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2786 if (*ids_supp == NULL) 2787 goto fail_ids_supp; 2788 2789 *prev_values = calloc(xstats_display_num, 2790 sizeof(**prev_values)); 2791 if (*prev_values == NULL) 2792 goto fail_prev_values; 2793 2794 *curr_values = calloc(xstats_display_num, 2795 sizeof(**curr_values)); 2796 if (*curr_values == NULL) 2797 goto fail_curr_values; 2798 2799 ports[pi].xstats_info.allocated = true; 2800 2801 return 0; 2802 2803 fail_curr_values: 2804 free(*prev_values); 2805 fail_prev_values: 2806 free(*ids_supp); 2807 fail_ids_supp: 2808 return -ENOMEM; 2809 } 2810 2811 static void 2812 free_xstats_display_info(portid_t pi) 2813 { 2814 if (!ports[pi].xstats_info.allocated) 2815 return; 2816 free(ports[pi].xstats_info.ids_supp); 2817 free(ports[pi].xstats_info.prev_values); 2818 free(ports[pi].xstats_info.curr_values); 2819 ports[pi].xstats_info.allocated = false; 2820 } 2821 2822 /** Fill helper structures for specified port to show extended statistics. */ 2823 static void 2824 fill_xstats_display_info_for_port(portid_t pi) 2825 { 2826 unsigned int stat, stat_supp; 2827 const char *xstat_name; 2828 struct rte_port *port; 2829 uint64_t *ids_supp; 2830 int rc; 2831 2832 if (xstats_display_num == 0) 2833 return; 2834 2835 if (pi == (portid_t)RTE_PORT_ALL) { 2836 fill_xstats_display_info(); 2837 return; 2838 } 2839 2840 port = &ports[pi]; 2841 if (port->port_status != RTE_PORT_STARTED) 2842 return; 2843 2844 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2845 rte_exit(EXIT_FAILURE, 2846 "Failed to allocate xstats display memory\n"); 2847 2848 ids_supp = port->xstats_info.ids_supp; 2849 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2850 xstat_name = xstats_display[stat].name; 2851 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2852 ids_supp + stat_supp); 2853 if (rc != 0) { 2854 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2855 xstat_name, pi, stat); 2856 continue; 2857 } 2858 stat_supp++; 2859 } 2860 2861 port->xstats_info.ids_supp_sz = stat_supp; 2862 } 2863 2864 /** Fill helper structures for all ports to show extended statistics. */ 2865 static void 2866 fill_xstats_display_info(void) 2867 { 2868 portid_t pi; 2869 2870 if (xstats_display_num == 0) 2871 return; 2872 2873 RTE_ETH_FOREACH_DEV(pi) 2874 fill_xstats_display_info_for_port(pi); 2875 } 2876 2877 /* 2878 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2879 * device in dev_info is zero when no slave is added. And its capability 2880 * will be updated when add a new slave device. So adding a slave device need 2881 * to update the port configurations of bonding device. 2882 */ 2883 static void 2884 update_bonding_port_dev_conf(portid_t bond_pid) 2885 { 2886 #ifdef RTE_NET_BOND 2887 struct rte_port *port = &ports[bond_pid]; 2888 uint16_t i; 2889 int ret; 2890 2891 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2892 if (ret != 0) { 2893 fprintf(stderr, "Failed to get dev info for port = %u\n", 2894 bond_pid); 2895 return; 2896 } 2897 2898 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2899 port->dev_conf.txmode.offloads |= 2900 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2901 /* Apply Tx offloads configuration */ 2902 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2903 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2904 2905 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 2906 port->dev_info.flow_type_rss_offloads; 2907 #else 2908 RTE_SET_USED(bond_pid); 2909 #endif 2910 } 2911 2912 int 2913 start_port(portid_t pid) 2914 { 2915 int diag, need_check_link_status = -1; 2916 portid_t pi; 2917 portid_t p_pi = RTE_MAX_ETHPORTS; 2918 portid_t pl[RTE_MAX_ETHPORTS]; 2919 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2920 uint16_t cnt_pi = 0; 2921 uint16_t cfg_pi = 0; 2922 int peer_pi; 2923 queueid_t qi; 2924 struct rte_port *port; 2925 struct rte_eth_hairpin_cap cap; 2926 2927 if (port_id_is_invalid(pid, ENABLED_WARN)) 2928 return 0; 2929 2930 RTE_ETH_FOREACH_DEV(pi) { 2931 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2932 continue; 2933 2934 if (port_is_bonding_slave(pi)) { 2935 fprintf(stderr, 2936 "Please remove port %d from bonded device.\n", 2937 pi); 2938 continue; 2939 } 2940 2941 need_check_link_status = 0; 2942 port = &ports[pi]; 2943 if (port->port_status == RTE_PORT_STOPPED) 2944 port->port_status = RTE_PORT_HANDLING; 2945 else { 2946 fprintf(stderr, "Port %d is now not stopped\n", pi); 2947 continue; 2948 } 2949 2950 if (port->need_reconfig > 0) { 2951 struct rte_eth_conf dev_conf; 2952 int k; 2953 2954 port->need_reconfig = 0; 2955 2956 if (flow_isolate_all) { 2957 int ret = port_flow_isolate(pi, 1); 2958 if (ret) { 2959 fprintf(stderr, 2960 "Failed to apply isolated mode on port %d\n", 2961 pi); 2962 return -1; 2963 } 2964 } 2965 configure_rxtx_dump_callbacks(0); 2966 printf("Configuring Port %d (socket %u)\n", pi, 2967 port->socket_id); 2968 if (nb_hairpinq > 0 && 2969 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2970 fprintf(stderr, 2971 "Port %d doesn't support hairpin queues\n", 2972 pi); 2973 return -1; 2974 } 2975 2976 if (port->bond_flag == 1 && port->update_conf == 1) { 2977 update_bonding_port_dev_conf(pi); 2978 port->update_conf = 0; 2979 } 2980 2981 /* configure port */ 2982 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2983 nb_txq + nb_hairpinq, 2984 &(port->dev_conf)); 2985 if (diag != 0) { 2986 if (port->port_status == RTE_PORT_HANDLING) 2987 port->port_status = RTE_PORT_STOPPED; 2988 else 2989 fprintf(stderr, 2990 "Port %d can not be set back to stopped\n", 2991 pi); 2992 fprintf(stderr, "Fail to configure port %d\n", 2993 pi); 2994 /* try to reconfigure port next time */ 2995 port->need_reconfig = 1; 2996 return -1; 2997 } 2998 /* get device configuration*/ 2999 if (0 != 3000 eth_dev_conf_get_print_err(pi, &dev_conf)) { 3001 fprintf(stderr, 3002 "port %d can not get device configuration\n", 3003 pi); 3004 return -1; 3005 } 3006 /* Apply Rx offloads configuration */ 3007 if (dev_conf.rxmode.offloads != 3008 port->dev_conf.rxmode.offloads) { 3009 port->dev_conf.rxmode.offloads |= 3010 dev_conf.rxmode.offloads; 3011 for (k = 0; 3012 k < port->dev_info.max_rx_queues; 3013 k++) 3014 port->rxq[k].conf.offloads |= 3015 dev_conf.rxmode.offloads; 3016 } 3017 /* Apply Tx offloads configuration */ 3018 if (dev_conf.txmode.offloads != 3019 port->dev_conf.txmode.offloads) { 3020 port->dev_conf.txmode.offloads |= 3021 dev_conf.txmode.offloads; 3022 for (k = 0; 3023 k < port->dev_info.max_tx_queues; 3024 k++) 3025 port->txq[k].conf.offloads |= 3026 dev_conf.txmode.offloads; 3027 } 3028 } 3029 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 3030 port->need_reconfig_queues = 0; 3031 /* setup tx queues */ 3032 for (qi = 0; qi < nb_txq; qi++) { 3033 struct rte_eth_txconf *conf = 3034 &port->txq[qi].conf; 3035 3036 if ((numa_support) && 3037 (txring_numa[pi] != NUMA_NO_CONFIG)) 3038 diag = rte_eth_tx_queue_setup(pi, qi, 3039 port->nb_tx_desc[qi], 3040 txring_numa[pi], 3041 &(port->txq[qi].conf)); 3042 else 3043 diag = rte_eth_tx_queue_setup(pi, qi, 3044 port->nb_tx_desc[qi], 3045 port->socket_id, 3046 &(port->txq[qi].conf)); 3047 3048 if (diag == 0) { 3049 port->txq[qi].state = 3050 conf->tx_deferred_start ? 3051 RTE_ETH_QUEUE_STATE_STOPPED : 3052 RTE_ETH_QUEUE_STATE_STARTED; 3053 continue; 3054 } 3055 3056 /* Fail to setup tx queue, return */ 3057 if (port->port_status == RTE_PORT_HANDLING) 3058 port->port_status = RTE_PORT_STOPPED; 3059 else 3060 fprintf(stderr, 3061 "Port %d can not be set back to stopped\n", 3062 pi); 3063 fprintf(stderr, 3064 "Fail to configure port %d tx queues\n", 3065 pi); 3066 /* try to reconfigure queues next time */ 3067 port->need_reconfig_queues = 1; 3068 return -1; 3069 } 3070 for (qi = 0; qi < nb_rxq; qi++) { 3071 /* setup rx queues */ 3072 if ((numa_support) && 3073 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3074 struct rte_mempool * mp = 3075 mbuf_pool_find 3076 (rxring_numa[pi], 0); 3077 if (mp == NULL) { 3078 fprintf(stderr, 3079 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3080 rxring_numa[pi]); 3081 return -1; 3082 } 3083 3084 diag = rx_queue_setup(pi, qi, 3085 port->nb_rx_desc[qi], 3086 rxring_numa[pi], 3087 &(port->rxq[qi].conf), 3088 mp); 3089 } else { 3090 struct rte_mempool *mp = 3091 mbuf_pool_find 3092 (port->socket_id, 0); 3093 if (mp == NULL) { 3094 fprintf(stderr, 3095 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3096 port->socket_id); 3097 return -1; 3098 } 3099 diag = rx_queue_setup(pi, qi, 3100 port->nb_rx_desc[qi], 3101 port->socket_id, 3102 &(port->rxq[qi].conf), 3103 mp); 3104 } 3105 if (diag == 0) 3106 continue; 3107 3108 /* Fail to setup rx queue, return */ 3109 if (port->port_status == RTE_PORT_HANDLING) 3110 port->port_status = RTE_PORT_STOPPED; 3111 else 3112 fprintf(stderr, 3113 "Port %d can not be set back to stopped\n", 3114 pi); 3115 fprintf(stderr, 3116 "Fail to configure port %d rx queues\n", 3117 pi); 3118 /* try to reconfigure queues next time */ 3119 port->need_reconfig_queues = 1; 3120 return -1; 3121 } 3122 /* setup hairpin queues */ 3123 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3124 return -1; 3125 } 3126 configure_rxtx_dump_callbacks(verbose_level); 3127 if (clear_ptypes) { 3128 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3129 NULL, 0); 3130 if (diag < 0) 3131 fprintf(stderr, 3132 "Port %d: Failed to disable Ptype parsing\n", 3133 pi); 3134 } 3135 3136 p_pi = pi; 3137 cnt_pi++; 3138 3139 /* start port */ 3140 diag = eth_dev_start_mp(pi); 3141 if (diag < 0) { 3142 fprintf(stderr, "Fail to start port %d: %s\n", 3143 pi, rte_strerror(-diag)); 3144 3145 /* Fail to setup rx queue, return */ 3146 if (port->port_status == RTE_PORT_HANDLING) 3147 port->port_status = RTE_PORT_STOPPED; 3148 else 3149 fprintf(stderr, 3150 "Port %d can not be set back to stopped\n", 3151 pi); 3152 continue; 3153 } 3154 3155 if (port->port_status == RTE_PORT_HANDLING) 3156 port->port_status = RTE_PORT_STARTED; 3157 else 3158 fprintf(stderr, "Port %d can not be set into started\n", 3159 pi); 3160 3161 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3162 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3163 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3164 3165 /* at least one port started, need checking link status */ 3166 need_check_link_status = 1; 3167 3168 pl[cfg_pi++] = pi; 3169 } 3170 3171 if (need_check_link_status == 1 && !no_link_check) 3172 check_all_ports_link_status(RTE_PORT_ALL); 3173 else if (need_check_link_status == 0) 3174 fprintf(stderr, "Please stop the ports first\n"); 3175 3176 if (hairpin_mode & 0xf) { 3177 uint16_t i; 3178 int j; 3179 3180 /* bind all started hairpin ports */ 3181 for (i = 0; i < cfg_pi; i++) { 3182 pi = pl[i]; 3183 /* bind current Tx to all peer Rx */ 3184 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3185 RTE_MAX_ETHPORTS, 1); 3186 if (peer_pi < 0) 3187 return peer_pi; 3188 for (j = 0; j < peer_pi; j++) { 3189 if (!port_is_started(peer_pl[j])) 3190 continue; 3191 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3192 if (diag < 0) { 3193 fprintf(stderr, 3194 "Error during binding hairpin Tx port %u to %u: %s\n", 3195 pi, peer_pl[j], 3196 rte_strerror(-diag)); 3197 return -1; 3198 } 3199 } 3200 /* bind all peer Tx to current Rx */ 3201 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3202 RTE_MAX_ETHPORTS, 0); 3203 if (peer_pi < 0) 3204 return peer_pi; 3205 for (j = 0; j < peer_pi; j++) { 3206 if (!port_is_started(peer_pl[j])) 3207 continue; 3208 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3209 if (diag < 0) { 3210 fprintf(stderr, 3211 "Error during binding hairpin Tx port %u to %u: %s\n", 3212 peer_pl[j], pi, 3213 rte_strerror(-diag)); 3214 return -1; 3215 } 3216 } 3217 } 3218 } 3219 3220 fill_xstats_display_info_for_port(pid); 3221 3222 printf("Done\n"); 3223 return 0; 3224 } 3225 3226 void 3227 stop_port(portid_t pid) 3228 { 3229 portid_t pi; 3230 struct rte_port *port; 3231 int need_check_link_status = 0; 3232 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3233 int peer_pi; 3234 int ret; 3235 3236 if (port_id_is_invalid(pid, ENABLED_WARN)) 3237 return; 3238 3239 printf("Stopping ports...\n"); 3240 3241 RTE_ETH_FOREACH_DEV(pi) { 3242 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3243 continue; 3244 3245 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3246 fprintf(stderr, 3247 "Please remove port %d from forwarding configuration.\n", 3248 pi); 3249 continue; 3250 } 3251 3252 if (port_is_bonding_slave(pi)) { 3253 fprintf(stderr, 3254 "Please remove port %d from bonded device.\n", 3255 pi); 3256 continue; 3257 } 3258 3259 port = &ports[pi]; 3260 if (port->port_status == RTE_PORT_STARTED) 3261 port->port_status = RTE_PORT_HANDLING; 3262 else 3263 continue; 3264 3265 if (hairpin_mode & 0xf) { 3266 int j; 3267 3268 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3269 /* unbind all peer Tx from current Rx */ 3270 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3271 RTE_MAX_ETHPORTS, 0); 3272 if (peer_pi < 0) 3273 continue; 3274 for (j = 0; j < peer_pi; j++) { 3275 if (!port_is_started(peer_pl[j])) 3276 continue; 3277 rte_eth_hairpin_unbind(peer_pl[j], pi); 3278 } 3279 } 3280 3281 if (port->flow_list) 3282 port_flow_flush(pi); 3283 3284 ret = eth_dev_stop_mp(pi); 3285 if (ret != 0) { 3286 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3287 pi); 3288 /* Allow to retry stopping the port. */ 3289 port->port_status = RTE_PORT_STARTED; 3290 continue; 3291 } 3292 3293 if (port->port_status == RTE_PORT_HANDLING) 3294 port->port_status = RTE_PORT_STOPPED; 3295 else 3296 fprintf(stderr, "Port %d can not be set into stopped\n", 3297 pi); 3298 need_check_link_status = 1; 3299 } 3300 if (need_check_link_status && !no_link_check) 3301 check_all_ports_link_status(RTE_PORT_ALL); 3302 3303 printf("Done\n"); 3304 } 3305 3306 static void 3307 remove_invalid_ports_in(portid_t *array, portid_t *total) 3308 { 3309 portid_t i; 3310 portid_t new_total = 0; 3311 3312 for (i = 0; i < *total; i++) 3313 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3314 array[new_total] = array[i]; 3315 new_total++; 3316 } 3317 *total = new_total; 3318 } 3319 3320 static void 3321 remove_invalid_ports(void) 3322 { 3323 remove_invalid_ports_in(ports_ids, &nb_ports); 3324 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3325 nb_cfg_ports = nb_fwd_ports; 3326 } 3327 3328 static void 3329 flush_port_owned_resources(portid_t pi) 3330 { 3331 mcast_addr_pool_destroy(pi); 3332 port_flow_flush(pi); 3333 port_flex_item_flush(pi); 3334 port_flow_template_table_flush(pi); 3335 port_flow_pattern_template_flush(pi); 3336 port_flow_actions_template_flush(pi); 3337 port_action_handle_flush(pi); 3338 } 3339 3340 static void 3341 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3342 { 3343 struct rte_port *port; 3344 portid_t slave_pid; 3345 uint16_t i; 3346 3347 for (i = 0; i < num_slaves; i++) { 3348 slave_pid = slave_pids[i]; 3349 if (port_is_started(slave_pid) == 1) { 3350 if (rte_eth_dev_stop(slave_pid) != 0) 3351 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3352 slave_pid); 3353 3354 port = &ports[slave_pid]; 3355 port->port_status = RTE_PORT_STOPPED; 3356 } 3357 3358 clear_port_slave_flag(slave_pid); 3359 3360 /* Close slave device when testpmd quit or is killed. */ 3361 if (cl_quit == 1 || f_quit == 1) 3362 rte_eth_dev_close(slave_pid); 3363 } 3364 } 3365 3366 void 3367 close_port(portid_t pid) 3368 { 3369 portid_t pi; 3370 struct rte_port *port; 3371 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3372 int num_slaves = 0; 3373 3374 if (port_id_is_invalid(pid, ENABLED_WARN)) 3375 return; 3376 3377 printf("Closing ports...\n"); 3378 3379 RTE_ETH_FOREACH_DEV(pi) { 3380 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3381 continue; 3382 3383 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3384 fprintf(stderr, 3385 "Please remove port %d from forwarding configuration.\n", 3386 pi); 3387 continue; 3388 } 3389 3390 if (port_is_bonding_slave(pi)) { 3391 fprintf(stderr, 3392 "Please remove port %d from bonded device.\n", 3393 pi); 3394 continue; 3395 } 3396 3397 port = &ports[pi]; 3398 if (port->port_status == RTE_PORT_CLOSED) { 3399 fprintf(stderr, "Port %d is already closed\n", pi); 3400 continue; 3401 } 3402 3403 if (is_proc_primary()) { 3404 flush_port_owned_resources(pi); 3405 #ifdef RTE_NET_BOND 3406 if (port->bond_flag == 1) 3407 num_slaves = rte_eth_bond_slaves_get(pi, 3408 slave_pids, RTE_MAX_ETHPORTS); 3409 #endif 3410 rte_eth_dev_close(pi); 3411 /* 3412 * If this port is bonded device, all slaves under the 3413 * device need to be removed or closed. 3414 */ 3415 if (port->bond_flag == 1 && num_slaves > 0) 3416 clear_bonding_slave_device(slave_pids, 3417 num_slaves); 3418 } 3419 3420 free_xstats_display_info(pi); 3421 } 3422 3423 remove_invalid_ports(); 3424 printf("Done\n"); 3425 } 3426 3427 void 3428 reset_port(portid_t pid) 3429 { 3430 int diag; 3431 portid_t pi; 3432 struct rte_port *port; 3433 3434 if (port_id_is_invalid(pid, ENABLED_WARN)) 3435 return; 3436 3437 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3438 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3439 fprintf(stderr, 3440 "Can not reset port(s), please stop port(s) first.\n"); 3441 return; 3442 } 3443 3444 printf("Resetting ports...\n"); 3445 3446 RTE_ETH_FOREACH_DEV(pi) { 3447 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3448 continue; 3449 3450 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3451 fprintf(stderr, 3452 "Please remove port %d from forwarding configuration.\n", 3453 pi); 3454 continue; 3455 } 3456 3457 if (port_is_bonding_slave(pi)) { 3458 fprintf(stderr, 3459 "Please remove port %d from bonded device.\n", 3460 pi); 3461 continue; 3462 } 3463 3464 if (is_proc_primary()) { 3465 diag = rte_eth_dev_reset(pi); 3466 if (diag == 0) { 3467 port = &ports[pi]; 3468 port->need_reconfig = 1; 3469 port->need_reconfig_queues = 1; 3470 } else { 3471 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3472 pi, diag); 3473 } 3474 } 3475 } 3476 3477 printf("Done\n"); 3478 } 3479 3480 void 3481 attach_port(char *identifier) 3482 { 3483 portid_t pi; 3484 struct rte_dev_iterator iterator; 3485 3486 printf("Attaching a new port...\n"); 3487 3488 if (identifier == NULL) { 3489 fprintf(stderr, "Invalid parameters are specified\n"); 3490 return; 3491 } 3492 3493 if (rte_dev_probe(identifier) < 0) { 3494 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3495 return; 3496 } 3497 3498 /* first attach mode: event */ 3499 if (setup_on_probe_event) { 3500 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3501 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3502 if (ports[pi].port_status == RTE_PORT_HANDLING && 3503 ports[pi].need_setup != 0) 3504 setup_attached_port(pi); 3505 return; 3506 } 3507 3508 /* second attach mode: iterator */ 3509 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3510 /* setup ports matching the devargs used for probing */ 3511 if (port_is_forwarding(pi)) 3512 continue; /* port was already attached before */ 3513 setup_attached_port(pi); 3514 } 3515 } 3516 3517 static void 3518 setup_attached_port(portid_t pi) 3519 { 3520 unsigned int socket_id; 3521 int ret; 3522 3523 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3524 /* if socket_id is invalid, set to the first available socket. */ 3525 if (check_socket_id(socket_id) < 0) 3526 socket_id = socket_ids[0]; 3527 reconfig(pi, socket_id); 3528 ret = rte_eth_promiscuous_enable(pi); 3529 if (ret != 0) 3530 fprintf(stderr, 3531 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3532 pi, rte_strerror(-ret)); 3533 3534 ports_ids[nb_ports++] = pi; 3535 fwd_ports_ids[nb_fwd_ports++] = pi; 3536 nb_cfg_ports = nb_fwd_ports; 3537 ports[pi].need_setup = 0; 3538 ports[pi].port_status = RTE_PORT_STOPPED; 3539 3540 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3541 printf("Done\n"); 3542 } 3543 3544 static void 3545 detach_device(struct rte_device *dev) 3546 { 3547 portid_t sibling; 3548 3549 if (dev == NULL) { 3550 fprintf(stderr, "Device already removed\n"); 3551 return; 3552 } 3553 3554 printf("Removing a device...\n"); 3555 3556 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3557 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3558 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3559 fprintf(stderr, "Port %u not stopped\n", 3560 sibling); 3561 return; 3562 } 3563 flush_port_owned_resources(sibling); 3564 } 3565 } 3566 3567 if (rte_dev_remove(dev) < 0) { 3568 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3569 return; 3570 } 3571 remove_invalid_ports(); 3572 3573 printf("Device is detached\n"); 3574 printf("Now total ports is %d\n", nb_ports); 3575 printf("Done\n"); 3576 return; 3577 } 3578 3579 void 3580 detach_port_device(portid_t port_id) 3581 { 3582 int ret; 3583 struct rte_eth_dev_info dev_info; 3584 3585 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3586 return; 3587 3588 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3589 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3590 fprintf(stderr, "Port not stopped\n"); 3591 return; 3592 } 3593 fprintf(stderr, "Port was not closed\n"); 3594 } 3595 3596 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3597 if (ret != 0) { 3598 TESTPMD_LOG(ERR, 3599 "Failed to get device info for port %d, not detaching\n", 3600 port_id); 3601 return; 3602 } 3603 detach_device(dev_info.device); 3604 } 3605 3606 void 3607 detach_devargs(char *identifier) 3608 { 3609 struct rte_dev_iterator iterator; 3610 struct rte_devargs da; 3611 portid_t port_id; 3612 3613 printf("Removing a device...\n"); 3614 3615 memset(&da, 0, sizeof(da)); 3616 if (rte_devargs_parsef(&da, "%s", identifier)) { 3617 fprintf(stderr, "cannot parse identifier\n"); 3618 return; 3619 } 3620 3621 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3622 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3623 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3624 fprintf(stderr, "Port %u not stopped\n", 3625 port_id); 3626 rte_eth_iterator_cleanup(&iterator); 3627 rte_devargs_reset(&da); 3628 return; 3629 } 3630 flush_port_owned_resources(port_id); 3631 } 3632 } 3633 3634 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3635 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3636 da.name, rte_bus_name(da.bus)); 3637 rte_devargs_reset(&da); 3638 return; 3639 } 3640 3641 remove_invalid_ports(); 3642 3643 printf("Device %s is detached\n", identifier); 3644 printf("Now total ports is %d\n", nb_ports); 3645 printf("Done\n"); 3646 rte_devargs_reset(&da); 3647 } 3648 3649 void 3650 pmd_test_exit(void) 3651 { 3652 portid_t pt_id; 3653 unsigned int i; 3654 int ret; 3655 3656 if (test_done == 0) 3657 stop_packet_forwarding(); 3658 3659 #ifndef RTE_EXEC_ENV_WINDOWS 3660 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3661 if (mempools[i]) { 3662 if (mp_alloc_type == MP_ALLOC_ANON) 3663 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3664 NULL); 3665 } 3666 } 3667 #endif 3668 if (ports != NULL) { 3669 no_link_check = 1; 3670 RTE_ETH_FOREACH_DEV(pt_id) { 3671 printf("\nStopping port %d...\n", pt_id); 3672 fflush(stdout); 3673 stop_port(pt_id); 3674 } 3675 RTE_ETH_FOREACH_DEV(pt_id) { 3676 printf("\nShutting down port %d...\n", pt_id); 3677 fflush(stdout); 3678 close_port(pt_id); 3679 } 3680 } 3681 3682 if (hot_plug) { 3683 ret = rte_dev_event_monitor_stop(); 3684 if (ret) { 3685 RTE_LOG(ERR, EAL, 3686 "fail to stop device event monitor."); 3687 return; 3688 } 3689 3690 ret = rte_dev_event_callback_unregister(NULL, 3691 dev_event_callback, NULL); 3692 if (ret < 0) { 3693 RTE_LOG(ERR, EAL, 3694 "fail to unregister device event callback.\n"); 3695 return; 3696 } 3697 3698 ret = rte_dev_hotplug_handle_disable(); 3699 if (ret) { 3700 RTE_LOG(ERR, EAL, 3701 "fail to disable hotplug handling.\n"); 3702 return; 3703 } 3704 } 3705 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3706 if (mempools[i]) 3707 mempool_free_mp(mempools[i]); 3708 } 3709 free(xstats_display); 3710 3711 printf("\nBye...\n"); 3712 } 3713 3714 typedef void (*cmd_func_t)(void); 3715 struct pmd_test_command { 3716 const char *cmd_name; 3717 cmd_func_t cmd_func; 3718 }; 3719 3720 /* Check the link status of all ports in up to 9s, and print them finally */ 3721 static void 3722 check_all_ports_link_status(uint32_t port_mask) 3723 { 3724 #define CHECK_INTERVAL 100 /* 100ms */ 3725 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3726 portid_t portid; 3727 uint8_t count, all_ports_up, print_flag = 0; 3728 struct rte_eth_link link; 3729 int ret; 3730 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3731 3732 printf("Checking link statuses...\n"); 3733 fflush(stdout); 3734 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3735 all_ports_up = 1; 3736 RTE_ETH_FOREACH_DEV(portid) { 3737 if ((port_mask & (1 << portid)) == 0) 3738 continue; 3739 memset(&link, 0, sizeof(link)); 3740 ret = rte_eth_link_get_nowait(portid, &link); 3741 if (ret < 0) { 3742 all_ports_up = 0; 3743 if (print_flag == 1) 3744 fprintf(stderr, 3745 "Port %u link get failed: %s\n", 3746 portid, rte_strerror(-ret)); 3747 continue; 3748 } 3749 /* print link status if flag set */ 3750 if (print_flag == 1) { 3751 rte_eth_link_to_str(link_status, 3752 sizeof(link_status), &link); 3753 printf("Port %d %s\n", portid, link_status); 3754 continue; 3755 } 3756 /* clear all_ports_up flag if any link down */ 3757 if (link.link_status == RTE_ETH_LINK_DOWN) { 3758 all_ports_up = 0; 3759 break; 3760 } 3761 } 3762 /* after finally printing all link status, get out */ 3763 if (print_flag == 1) 3764 break; 3765 3766 if (all_ports_up == 0) { 3767 fflush(stdout); 3768 rte_delay_ms(CHECK_INTERVAL); 3769 } 3770 3771 /* set the print_flag if all ports up or timeout */ 3772 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3773 print_flag = 1; 3774 } 3775 3776 if (lsc_interrupt) 3777 break; 3778 } 3779 } 3780 3781 static void 3782 rmv_port_callback(void *arg) 3783 { 3784 int need_to_start = 0; 3785 int org_no_link_check = no_link_check; 3786 portid_t port_id = (intptr_t)arg; 3787 struct rte_eth_dev_info dev_info; 3788 int ret; 3789 3790 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3791 3792 if (!test_done && port_is_forwarding(port_id)) { 3793 need_to_start = 1; 3794 stop_packet_forwarding(); 3795 } 3796 no_link_check = 1; 3797 stop_port(port_id); 3798 no_link_check = org_no_link_check; 3799 3800 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3801 if (ret != 0) 3802 TESTPMD_LOG(ERR, 3803 "Failed to get device info for port %d, not detaching\n", 3804 port_id); 3805 else { 3806 struct rte_device *device = dev_info.device; 3807 close_port(port_id); 3808 detach_device(device); /* might be already removed or have more ports */ 3809 } 3810 if (need_to_start) 3811 start_packet_forwarding(0); 3812 } 3813 3814 /* This function is used by the interrupt thread */ 3815 static int 3816 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3817 void *ret_param) 3818 { 3819 RTE_SET_USED(param); 3820 RTE_SET_USED(ret_param); 3821 3822 if (type >= RTE_ETH_EVENT_MAX) { 3823 fprintf(stderr, 3824 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3825 port_id, __func__, type); 3826 fflush(stderr); 3827 } else if (event_print_mask & (UINT32_C(1) << type)) { 3828 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3829 eth_event_desc[type]); 3830 fflush(stdout); 3831 } 3832 3833 switch (type) { 3834 case RTE_ETH_EVENT_NEW: 3835 ports[port_id].need_setup = 1; 3836 ports[port_id].port_status = RTE_PORT_HANDLING; 3837 break; 3838 case RTE_ETH_EVENT_INTR_RMV: 3839 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3840 break; 3841 if (rte_eal_alarm_set(100000, 3842 rmv_port_callback, (void *)(intptr_t)port_id)) 3843 fprintf(stderr, 3844 "Could not set up deferred device removal\n"); 3845 break; 3846 case RTE_ETH_EVENT_DESTROY: 3847 ports[port_id].port_status = RTE_PORT_CLOSED; 3848 printf("Port %u is closed\n", port_id); 3849 break; 3850 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3851 uint16_t rxq_id; 3852 int ret; 3853 3854 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3855 for (rxq_id = 0; ; rxq_id++) { 3856 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3857 NULL); 3858 if (ret <= 0) 3859 break; 3860 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3861 port_id, rxq_id); 3862 3863 #ifdef RTE_NET_MLX5 3864 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3865 #endif 3866 } 3867 break; 3868 } 3869 default: 3870 break; 3871 } 3872 return 0; 3873 } 3874 3875 static int 3876 register_eth_event_callback(void) 3877 { 3878 int ret; 3879 enum rte_eth_event_type event; 3880 3881 for (event = RTE_ETH_EVENT_UNKNOWN; 3882 event < RTE_ETH_EVENT_MAX; event++) { 3883 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3884 event, 3885 eth_event_callback, 3886 NULL); 3887 if (ret != 0) { 3888 TESTPMD_LOG(ERR, "Failed to register callback for " 3889 "%s event\n", eth_event_desc[event]); 3890 return -1; 3891 } 3892 } 3893 3894 return 0; 3895 } 3896 3897 /* This function is used by the interrupt thread */ 3898 static void 3899 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3900 __rte_unused void *arg) 3901 { 3902 uint16_t port_id; 3903 int ret; 3904 3905 if (type >= RTE_DEV_EVENT_MAX) { 3906 fprintf(stderr, "%s called upon invalid event %d\n", 3907 __func__, type); 3908 fflush(stderr); 3909 } 3910 3911 switch (type) { 3912 case RTE_DEV_EVENT_REMOVE: 3913 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3914 device_name); 3915 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3916 if (ret) { 3917 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3918 device_name); 3919 return; 3920 } 3921 /* 3922 * Because the user's callback is invoked in eal interrupt 3923 * callback, the interrupt callback need to be finished before 3924 * it can be unregistered when detaching device. So finish 3925 * callback soon and use a deferred removal to detach device 3926 * is need. It is a workaround, once the device detaching be 3927 * moved into the eal in the future, the deferred removal could 3928 * be deleted. 3929 */ 3930 if (rte_eal_alarm_set(100000, 3931 rmv_port_callback, (void *)(intptr_t)port_id)) 3932 RTE_LOG(ERR, EAL, 3933 "Could not set up deferred device removal\n"); 3934 break; 3935 case RTE_DEV_EVENT_ADD: 3936 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3937 device_name); 3938 /* TODO: After finish kernel driver binding, 3939 * begin to attach port. 3940 */ 3941 break; 3942 default: 3943 break; 3944 } 3945 } 3946 3947 static void 3948 rxtx_port_config(portid_t pid) 3949 { 3950 uint16_t qid; 3951 uint64_t offloads; 3952 struct rte_port *port = &ports[pid]; 3953 3954 for (qid = 0; qid < nb_rxq; qid++) { 3955 offloads = port->rxq[qid].conf.offloads; 3956 port->rxq[qid].conf = port->dev_info.default_rxconf; 3957 3958 if (rxq_share > 0 && 3959 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3960 /* Non-zero share group to enable RxQ share. */ 3961 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3962 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3963 } 3964 3965 if (offloads != 0) 3966 port->rxq[qid].conf.offloads = offloads; 3967 3968 /* Check if any Rx parameters have been passed */ 3969 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3970 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3971 3972 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3973 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3974 3975 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3976 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3977 3978 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3979 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3980 3981 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3982 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3983 3984 port->nb_rx_desc[qid] = nb_rxd; 3985 } 3986 3987 for (qid = 0; qid < nb_txq; qid++) { 3988 offloads = port->txq[qid].conf.offloads; 3989 port->txq[qid].conf = port->dev_info.default_txconf; 3990 if (offloads != 0) 3991 port->txq[qid].conf.offloads = offloads; 3992 3993 /* Check if any Tx parameters have been passed */ 3994 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3995 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3996 3997 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3998 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3999 4000 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 4001 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 4002 4003 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 4004 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 4005 4006 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 4007 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 4008 4009 port->nb_tx_desc[qid] = nb_txd; 4010 } 4011 } 4012 4013 /* 4014 * Helper function to set MTU from frame size 4015 * 4016 * port->dev_info should be set before calling this function. 4017 * 4018 * return 0 on success, negative on error 4019 */ 4020 int 4021 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 4022 { 4023 struct rte_port *port = &ports[portid]; 4024 uint32_t eth_overhead; 4025 uint16_t mtu, new_mtu; 4026 4027 eth_overhead = get_eth_overhead(&port->dev_info); 4028 4029 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 4030 printf("Failed to get MTU for port %u\n", portid); 4031 return -1; 4032 } 4033 4034 new_mtu = max_rx_pktlen - eth_overhead; 4035 4036 if (mtu == new_mtu) 4037 return 0; 4038 4039 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 4040 fprintf(stderr, 4041 "Failed to set MTU to %u for port %u\n", 4042 new_mtu, portid); 4043 return -1; 4044 } 4045 4046 port->dev_conf.rxmode.mtu = new_mtu; 4047 4048 return 0; 4049 } 4050 4051 void 4052 init_port_config(void) 4053 { 4054 portid_t pid; 4055 struct rte_port *port; 4056 int ret, i; 4057 4058 RTE_ETH_FOREACH_DEV(pid) { 4059 port = &ports[pid]; 4060 4061 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4062 if (ret != 0) 4063 return; 4064 4065 if (nb_rxq > 1) { 4066 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4067 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4068 rss_hf & port->dev_info.flow_type_rss_offloads; 4069 } else { 4070 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4071 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4072 } 4073 4074 if (port->dcb_flag == 0) { 4075 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4076 port->dev_conf.rxmode.mq_mode = 4077 (enum rte_eth_rx_mq_mode) 4078 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4079 } else { 4080 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4081 port->dev_conf.rxmode.offloads &= 4082 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4083 4084 for (i = 0; 4085 i < port->dev_info.nb_rx_queues; 4086 i++) 4087 port->rxq[i].conf.offloads &= 4088 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4089 } 4090 } 4091 4092 rxtx_port_config(pid); 4093 4094 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4095 if (ret != 0) 4096 return; 4097 4098 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4099 port->dev_conf.intr_conf.lsc = 1; 4100 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4101 port->dev_conf.intr_conf.rmv = 1; 4102 } 4103 } 4104 4105 void set_port_slave_flag(portid_t slave_pid) 4106 { 4107 struct rte_port *port; 4108 4109 port = &ports[slave_pid]; 4110 port->slave_flag = 1; 4111 } 4112 4113 void clear_port_slave_flag(portid_t slave_pid) 4114 { 4115 struct rte_port *port; 4116 4117 port = &ports[slave_pid]; 4118 port->slave_flag = 0; 4119 } 4120 4121 uint8_t port_is_bonding_slave(portid_t slave_pid) 4122 { 4123 struct rte_port *port; 4124 struct rte_eth_dev_info dev_info; 4125 int ret; 4126 4127 port = &ports[slave_pid]; 4128 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4129 if (ret != 0) { 4130 TESTPMD_LOG(ERR, 4131 "Failed to get device info for port id %d," 4132 "cannot determine if the port is a bonded slave", 4133 slave_pid); 4134 return 0; 4135 } 4136 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4137 return 1; 4138 return 0; 4139 } 4140 4141 const uint16_t vlan_tags[] = { 4142 0, 1, 2, 3, 4, 5, 6, 7, 4143 8, 9, 10, 11, 12, 13, 14, 15, 4144 16, 17, 18, 19, 20, 21, 22, 23, 4145 24, 25, 26, 27, 28, 29, 30, 31 4146 }; 4147 4148 static int 4149 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4150 enum dcb_mode_enable dcb_mode, 4151 enum rte_eth_nb_tcs num_tcs, 4152 uint8_t pfc_en) 4153 { 4154 uint8_t i; 4155 int32_t rc; 4156 struct rte_eth_rss_conf rss_conf; 4157 4158 /* 4159 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4160 * given above, and the number of traffic classes available for use. 4161 */ 4162 if (dcb_mode == DCB_VT_ENABLED) { 4163 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4164 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4165 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4166 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4167 4168 /* VMDQ+DCB RX and TX configurations */ 4169 vmdq_rx_conf->enable_default_pool = 0; 4170 vmdq_rx_conf->default_pool = 0; 4171 vmdq_rx_conf->nb_queue_pools = 4172 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4173 vmdq_tx_conf->nb_queue_pools = 4174 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4175 4176 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4177 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4178 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4179 vmdq_rx_conf->pool_map[i].pools = 4180 1 << (i % vmdq_rx_conf->nb_queue_pools); 4181 } 4182 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4183 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4184 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4185 } 4186 4187 /* set DCB mode of RX and TX of multiple queues */ 4188 eth_conf->rxmode.mq_mode = 4189 (enum rte_eth_rx_mq_mode) 4190 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4191 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4192 } else { 4193 struct rte_eth_dcb_rx_conf *rx_conf = 4194 ð_conf->rx_adv_conf.dcb_rx_conf; 4195 struct rte_eth_dcb_tx_conf *tx_conf = 4196 ð_conf->tx_adv_conf.dcb_tx_conf; 4197 4198 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4199 4200 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4201 if (rc != 0) 4202 return rc; 4203 4204 rx_conf->nb_tcs = num_tcs; 4205 tx_conf->nb_tcs = num_tcs; 4206 4207 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4208 rx_conf->dcb_tc[i] = i % num_tcs; 4209 tx_conf->dcb_tc[i] = i % num_tcs; 4210 } 4211 4212 eth_conf->rxmode.mq_mode = 4213 (enum rte_eth_rx_mq_mode) 4214 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4215 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4216 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4217 } 4218 4219 if (pfc_en) 4220 eth_conf->dcb_capability_en = 4221 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4222 else 4223 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4224 4225 return 0; 4226 } 4227 4228 int 4229 init_port_dcb_config(portid_t pid, 4230 enum dcb_mode_enable dcb_mode, 4231 enum rte_eth_nb_tcs num_tcs, 4232 uint8_t pfc_en) 4233 { 4234 struct rte_eth_conf port_conf; 4235 struct rte_port *rte_port; 4236 int retval; 4237 uint16_t i; 4238 4239 if (num_procs > 1) { 4240 printf("The multi-process feature doesn't support dcb.\n"); 4241 return -ENOTSUP; 4242 } 4243 rte_port = &ports[pid]; 4244 4245 /* retain the original device configuration. */ 4246 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4247 4248 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4249 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4250 if (retval < 0) 4251 return retval; 4252 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4253 /* remove RSS HASH offload for DCB in vt mode */ 4254 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4255 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4256 for (i = 0; i < nb_rxq; i++) 4257 rte_port->rxq[i].conf.offloads &= 4258 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4259 } 4260 4261 /* re-configure the device . */ 4262 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4263 if (retval < 0) 4264 return retval; 4265 4266 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4267 if (retval != 0) 4268 return retval; 4269 4270 /* If dev_info.vmdq_pool_base is greater than 0, 4271 * the queue id of vmdq pools is started after pf queues. 4272 */ 4273 if (dcb_mode == DCB_VT_ENABLED && 4274 rte_port->dev_info.vmdq_pool_base > 0) { 4275 fprintf(stderr, 4276 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4277 pid); 4278 return -1; 4279 } 4280 4281 /* Assume the ports in testpmd have the same dcb capability 4282 * and has the same number of rxq and txq in dcb mode 4283 */ 4284 if (dcb_mode == DCB_VT_ENABLED) { 4285 if (rte_port->dev_info.max_vfs > 0) { 4286 nb_rxq = rte_port->dev_info.nb_rx_queues; 4287 nb_txq = rte_port->dev_info.nb_tx_queues; 4288 } else { 4289 nb_rxq = rte_port->dev_info.max_rx_queues; 4290 nb_txq = rte_port->dev_info.max_tx_queues; 4291 } 4292 } else { 4293 /*if vt is disabled, use all pf queues */ 4294 if (rte_port->dev_info.vmdq_pool_base == 0) { 4295 nb_rxq = rte_port->dev_info.max_rx_queues; 4296 nb_txq = rte_port->dev_info.max_tx_queues; 4297 } else { 4298 nb_rxq = (queueid_t)num_tcs; 4299 nb_txq = (queueid_t)num_tcs; 4300 4301 } 4302 } 4303 rx_free_thresh = 64; 4304 4305 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4306 4307 rxtx_port_config(pid); 4308 /* VLAN filter */ 4309 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4310 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4311 rx_vft_set(pid, vlan_tags[i], 1); 4312 4313 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4314 if (retval != 0) 4315 return retval; 4316 4317 rte_port->dcb_flag = 1; 4318 4319 /* Enter DCB configuration status */ 4320 dcb_config = 1; 4321 4322 return 0; 4323 } 4324 4325 static void 4326 init_port(void) 4327 { 4328 int i; 4329 4330 /* Configuration of Ethernet ports. */ 4331 ports = rte_zmalloc("testpmd: ports", 4332 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4333 RTE_CACHE_LINE_SIZE); 4334 if (ports == NULL) { 4335 rte_exit(EXIT_FAILURE, 4336 "rte_zmalloc(%d struct rte_port) failed\n", 4337 RTE_MAX_ETHPORTS); 4338 } 4339 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4340 ports[i].fwd_mac_swap = 1; 4341 ports[i].xstats_info.allocated = false; 4342 LIST_INIT(&ports[i].flow_tunnel_list); 4343 } 4344 /* Initialize ports NUMA structures */ 4345 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4346 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4347 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4348 } 4349 4350 static void 4351 force_quit(void) 4352 { 4353 pmd_test_exit(); 4354 prompt_exit(); 4355 } 4356 4357 static void 4358 print_stats(void) 4359 { 4360 uint8_t i; 4361 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4362 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4363 4364 /* Clear screen and move to top left */ 4365 printf("%s%s", clr, top_left); 4366 4367 printf("\nPort statistics ===================================="); 4368 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4369 nic_stats_display(fwd_ports_ids[i]); 4370 4371 fflush(stdout); 4372 } 4373 4374 static void 4375 signal_handler(int signum) 4376 { 4377 if (signum == SIGINT || signum == SIGTERM) { 4378 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4379 signum); 4380 #ifdef RTE_LIB_PDUMP 4381 /* uninitialize packet capture framework */ 4382 rte_pdump_uninit(); 4383 #endif 4384 #ifdef RTE_LIB_LATENCYSTATS 4385 if (latencystats_enabled != 0) 4386 rte_latencystats_uninit(); 4387 #endif 4388 force_quit(); 4389 /* Set flag to indicate the force termination. */ 4390 f_quit = 1; 4391 /* exit with the expected status */ 4392 #ifndef RTE_EXEC_ENV_WINDOWS 4393 signal(signum, SIG_DFL); 4394 kill(getpid(), signum); 4395 #endif 4396 } 4397 } 4398 4399 int 4400 main(int argc, char** argv) 4401 { 4402 int diag; 4403 portid_t port_id; 4404 uint16_t count; 4405 int ret; 4406 4407 signal(SIGINT, signal_handler); 4408 signal(SIGTERM, signal_handler); 4409 4410 testpmd_logtype = rte_log_register("testpmd"); 4411 if (testpmd_logtype < 0) 4412 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4413 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4414 4415 diag = rte_eal_init(argc, argv); 4416 if (diag < 0) 4417 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4418 rte_strerror(rte_errno)); 4419 4420 ret = register_eth_event_callback(); 4421 if (ret != 0) 4422 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4423 4424 #ifdef RTE_LIB_PDUMP 4425 /* initialize packet capture framework */ 4426 rte_pdump_init(); 4427 #endif 4428 4429 count = 0; 4430 RTE_ETH_FOREACH_DEV(port_id) { 4431 ports_ids[count] = port_id; 4432 count++; 4433 } 4434 nb_ports = (portid_t) count; 4435 if (nb_ports == 0) 4436 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4437 4438 /* allocate port structures, and init them */ 4439 init_port(); 4440 4441 set_def_fwd_config(); 4442 if (nb_lcores == 0) 4443 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4444 "Check the core mask argument\n"); 4445 4446 /* Bitrate/latency stats disabled by default */ 4447 #ifdef RTE_LIB_BITRATESTATS 4448 bitrate_enabled = 0; 4449 #endif 4450 #ifdef RTE_LIB_LATENCYSTATS 4451 latencystats_enabled = 0; 4452 #endif 4453 4454 /* on FreeBSD, mlockall() is disabled by default */ 4455 #ifdef RTE_EXEC_ENV_FREEBSD 4456 do_mlockall = 0; 4457 #else 4458 do_mlockall = 1; 4459 #endif 4460 4461 argc -= diag; 4462 argv += diag; 4463 if (argc > 1) 4464 launch_args_parse(argc, argv); 4465 4466 #ifndef RTE_EXEC_ENV_WINDOWS 4467 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4468 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4469 strerror(errno)); 4470 } 4471 #endif 4472 4473 if (tx_first && interactive) 4474 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4475 "interactive mode.\n"); 4476 4477 if (tx_first && lsc_interrupt) { 4478 fprintf(stderr, 4479 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4480 lsc_interrupt = 0; 4481 } 4482 4483 if (!nb_rxq && !nb_txq) 4484 fprintf(stderr, 4485 "Warning: Either rx or tx queues should be non-zero\n"); 4486 4487 if (nb_rxq > 1 && nb_rxq > nb_txq) 4488 fprintf(stderr, 4489 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4490 nb_rxq, nb_txq); 4491 4492 init_config(); 4493 4494 if (hot_plug) { 4495 ret = rte_dev_hotplug_handle_enable(); 4496 if (ret) { 4497 RTE_LOG(ERR, EAL, 4498 "fail to enable hotplug handling."); 4499 return -1; 4500 } 4501 4502 ret = rte_dev_event_monitor_start(); 4503 if (ret) { 4504 RTE_LOG(ERR, EAL, 4505 "fail to start device event monitoring."); 4506 return -1; 4507 } 4508 4509 ret = rte_dev_event_callback_register(NULL, 4510 dev_event_callback, NULL); 4511 if (ret) { 4512 RTE_LOG(ERR, EAL, 4513 "fail to register device event callback\n"); 4514 return -1; 4515 } 4516 } 4517 4518 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { 4519 if (!interactive) { 4520 rte_eal_cleanup(); 4521 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4522 } 4523 fprintf(stderr, "Start ports failed\n"); 4524 } 4525 4526 /* set all ports to promiscuous mode by default */ 4527 RTE_ETH_FOREACH_DEV(port_id) { 4528 ret = rte_eth_promiscuous_enable(port_id); 4529 if (ret != 0) 4530 fprintf(stderr, 4531 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4532 port_id, rte_strerror(-ret)); 4533 } 4534 4535 #ifdef RTE_LIB_METRICS 4536 /* Init metrics library */ 4537 rte_metrics_init(rte_socket_id()); 4538 #endif 4539 4540 #ifdef RTE_LIB_LATENCYSTATS 4541 if (latencystats_enabled != 0) { 4542 int ret = rte_latencystats_init(1, NULL); 4543 if (ret) 4544 fprintf(stderr, 4545 "Warning: latencystats init() returned error %d\n", 4546 ret); 4547 fprintf(stderr, "Latencystats running on lcore %d\n", 4548 latencystats_lcore_id); 4549 } 4550 #endif 4551 4552 /* Setup bitrate stats */ 4553 #ifdef RTE_LIB_BITRATESTATS 4554 if (bitrate_enabled != 0) { 4555 bitrate_data = rte_stats_bitrate_create(); 4556 if (bitrate_data == NULL) 4557 rte_exit(EXIT_FAILURE, 4558 "Could not allocate bitrate data.\n"); 4559 rte_stats_bitrate_reg(bitrate_data); 4560 } 4561 #endif 4562 4563 if (record_core_cycles) 4564 rte_lcore_register_usage_cb(lcore_usage_callback); 4565 4566 #ifdef RTE_LIB_CMDLINE 4567 if (init_cmdline() != 0) 4568 rte_exit(EXIT_FAILURE, 4569 "Could not initialise cmdline context.\n"); 4570 4571 if (strlen(cmdline_filename) != 0) 4572 cmdline_read_from_file(cmdline_filename); 4573 4574 if (interactive == 1) { 4575 if (auto_start) { 4576 printf("Start automatic packet forwarding\n"); 4577 start_packet_forwarding(0); 4578 } 4579 prompt(); 4580 pmd_test_exit(); 4581 } else 4582 #endif 4583 { 4584 char c; 4585 int rc; 4586 4587 f_quit = 0; 4588 4589 printf("No commandline core given, start packet forwarding\n"); 4590 start_packet_forwarding(tx_first); 4591 if (stats_period != 0) { 4592 uint64_t prev_time = 0, cur_time, diff_time = 0; 4593 uint64_t timer_period; 4594 4595 /* Convert to number of cycles */ 4596 timer_period = stats_period * rte_get_timer_hz(); 4597 4598 while (f_quit == 0) { 4599 cur_time = rte_get_timer_cycles(); 4600 diff_time += cur_time - prev_time; 4601 4602 if (diff_time >= timer_period) { 4603 print_stats(); 4604 /* Reset the timer */ 4605 diff_time = 0; 4606 } 4607 /* Sleep to avoid unnecessary checks */ 4608 prev_time = cur_time; 4609 rte_delay_us_sleep(US_PER_S); 4610 } 4611 } 4612 4613 printf("Press enter to exit\n"); 4614 rc = read(0, &c, 1); 4615 pmd_test_exit(); 4616 if (rc < 0) 4617 return 1; 4618 } 4619 4620 ret = rte_eal_cleanup(); 4621 if (ret != 0) 4622 rte_exit(EXIT_FAILURE, 4623 "EAL cleanup failed: %s\n", strerror(-ret)); 4624 4625 return EXIT_SUCCESS; 4626 } 4627