1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 static volatile uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 253 254 /* 255 * Configuration of packet segments used by the "txonly" processing engine. 256 */ 257 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 258 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 259 TXONLY_DEF_PACKET_LEN, 260 }; 261 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 262 263 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 264 /**< Split policy for packets to TX. */ 265 266 uint8_t txonly_multi_flow; 267 /**< Whether multiple flows are generated in TXONLY mode. */ 268 269 uint32_t tx_pkt_times_inter; 270 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 271 272 uint32_t tx_pkt_times_intra; 273 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 274 275 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 276 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 277 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 278 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 279 280 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 281 uint8_t dcb_config = 0; 282 283 /* 284 * Configurable number of RX/TX queues. 285 */ 286 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 287 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 288 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 289 290 /* 291 * Configurable number of RX/TX ring descriptors. 292 * Defaults are supplied by drivers via ethdev. 293 */ 294 #define RX_DESC_DEFAULT 0 295 #define TX_DESC_DEFAULT 0 296 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 297 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 298 299 #define RTE_PMD_PARAM_UNSET -1 300 /* 301 * Configurable values of RX and TX ring threshold registers. 302 */ 303 304 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 305 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 306 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 307 308 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 309 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 310 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 311 312 /* 313 * Configurable value of RX free threshold. 314 */ 315 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 316 317 /* 318 * Configurable value of RX drop enable. 319 */ 320 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 321 322 /* 323 * Configurable value of TX free threshold. 324 */ 325 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 326 327 /* 328 * Configurable value of TX RS bit threshold. 329 */ 330 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 331 332 /* 333 * Configurable value of buffered packets before sending. 334 */ 335 uint16_t noisy_tx_sw_bufsz; 336 337 /* 338 * Configurable value of packet buffer timeout. 339 */ 340 uint16_t noisy_tx_sw_buf_flush_time; 341 342 /* 343 * Configurable value for size of VNF internal memory area 344 * used for simulating noisy neighbour behaviour 345 */ 346 uint64_t noisy_lkup_mem_sz; 347 348 /* 349 * Configurable value of number of random writes done in 350 * VNF simulation memory area. 351 */ 352 uint64_t noisy_lkup_num_writes; 353 354 /* 355 * Configurable value of number of random reads done in 356 * VNF simulation memory area. 357 */ 358 uint64_t noisy_lkup_num_reads; 359 360 /* 361 * Configurable value of number of random reads/writes done in 362 * VNF simulation memory area. 363 */ 364 uint64_t noisy_lkup_num_reads_writes; 365 366 /* 367 * Receive Side Scaling (RSS) configuration. 368 */ 369 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 370 371 /* 372 * Port topology configuration 373 */ 374 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 375 376 /* 377 * Avoids to flush all the RX streams before starts forwarding. 378 */ 379 uint8_t no_flush_rx = 0; /* flush by default */ 380 381 /* 382 * Flow API isolated mode. 383 */ 384 uint8_t flow_isolate_all; 385 386 /* 387 * Disable port flow flush when stop port. 388 */ 389 uint8_t no_flow_flush = 0; /* do flow flush by default */ 390 391 /* 392 * Avoids to check link status when starting/stopping a port. 393 */ 394 uint8_t no_link_check = 0; /* check by default */ 395 396 /* 397 * Don't automatically start all ports in interactive mode. 398 */ 399 uint8_t no_device_start = 0; 400 401 /* 402 * Enable link status change notification 403 */ 404 uint8_t lsc_interrupt = 1; /* enabled by default */ 405 406 /* 407 * Enable device removal notification. 408 */ 409 uint8_t rmv_interrupt = 1; /* enabled by default */ 410 411 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 412 413 /* After attach, port setup is called on event or by iterator */ 414 bool setup_on_probe_event = true; 415 416 /* Clear ptypes on port initialization. */ 417 uint8_t clear_ptypes = true; 418 419 /* Hairpin ports configuration mode. */ 420 uint32_t hairpin_mode; 421 422 /* Pretty printing of ethdev events */ 423 static const char * const eth_event_desc[] = { 424 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 425 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 426 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 427 [RTE_ETH_EVENT_INTR_RESET] = "reset", 428 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 429 [RTE_ETH_EVENT_IPSEC] = "IPsec", 430 [RTE_ETH_EVENT_MACSEC] = "MACsec", 431 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 432 [RTE_ETH_EVENT_NEW] = "device probed", 433 [RTE_ETH_EVENT_DESTROY] = "device released", 434 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 435 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 436 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 437 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 438 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 439 [RTE_ETH_EVENT_MAX] = NULL, 440 }; 441 442 /* 443 * Display or mask ether events 444 * Default to all events except VF_MBOX 445 */ 446 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 447 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 448 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 449 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 450 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 451 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 452 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 453 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 454 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 455 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 456 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 457 /* 458 * Decide if all memory are locked for performance. 459 */ 460 int do_mlockall = 0; 461 462 #ifdef RTE_LIB_LATENCYSTATS 463 464 /* 465 * Set when latency stats is enabled in the commandline 466 */ 467 uint8_t latencystats_enabled; 468 469 /* 470 * Lcore ID to service latency statistics. 471 */ 472 lcoreid_t latencystats_lcore_id = -1; 473 474 #endif 475 476 /* 477 * Ethernet device configuration. 478 */ 479 struct rte_eth_rxmode rx_mode; 480 481 struct rte_eth_txmode tx_mode = { 482 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 483 }; 484 485 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 486 487 /* 488 * Display zero values by default for xstats 489 */ 490 uint8_t xstats_hide_zero; 491 492 /* 493 * Measure of CPU cycles disabled by default 494 */ 495 uint8_t record_core_cycles; 496 497 /* 498 * Display of RX and TX bursts disabled by default 499 */ 500 uint8_t record_burst_stats; 501 502 /* 503 * Number of ports per shared Rx queue group, 0 disable. 504 */ 505 uint32_t rxq_share; 506 507 unsigned int num_sockets = 0; 508 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 509 510 #ifdef RTE_LIB_BITRATESTATS 511 /* Bitrate statistics */ 512 struct rte_stats_bitrates *bitrate_data; 513 lcoreid_t bitrate_lcore_id; 514 uint8_t bitrate_enabled; 515 #endif 516 517 #ifdef RTE_LIB_GRO 518 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 519 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 520 #endif 521 522 /* 523 * hexadecimal bitmask of RX mq mode can be enabled. 524 */ 525 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 526 527 /* 528 * Used to set forced link speed 529 */ 530 uint32_t eth_link_speed; 531 532 /* 533 * ID of the current process in multi-process, used to 534 * configure the queues to be polled. 535 */ 536 int proc_id; 537 538 /* 539 * Number of processes in multi-process, used to 540 * configure the queues to be polled. 541 */ 542 unsigned int num_procs = 1; 543 544 static void 545 eth_rx_metadata_negotiate_mp(uint16_t port_id) 546 { 547 uint64_t rx_meta_features = 0; 548 int ret; 549 550 if (!is_proc_primary()) 551 return; 552 553 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 554 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 555 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 556 557 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 558 if (ret == 0) { 559 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 560 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 561 port_id); 562 } 563 564 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 565 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 566 port_id); 567 } 568 569 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 570 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 571 port_id); 572 } 573 } else if (ret != -ENOTSUP) { 574 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 575 port_id, rte_strerror(-ret)); 576 } 577 } 578 579 static int 580 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 581 const struct rte_eth_conf *dev_conf) 582 { 583 if (is_proc_primary()) 584 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 585 dev_conf); 586 return 0; 587 } 588 589 static int 590 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 591 { 592 #ifdef RTE_NET_BOND 593 594 portid_t slave_pids[RTE_MAX_ETHPORTS]; 595 struct rte_port *port; 596 int num_slaves; 597 portid_t slave_pid; 598 int i; 599 600 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 601 RTE_MAX_ETHPORTS); 602 if (num_slaves < 0) { 603 fprintf(stderr, "Failed to get slave list for port = %u\n", 604 bond_pid); 605 return num_slaves; 606 } 607 608 for (i = 0; i < num_slaves; i++) { 609 slave_pid = slave_pids[i]; 610 port = &ports[slave_pid]; 611 port->port_status = 612 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 613 } 614 #else 615 RTE_SET_USED(bond_pid); 616 RTE_SET_USED(is_stop); 617 #endif 618 return 0; 619 } 620 621 static int 622 eth_dev_start_mp(uint16_t port_id) 623 { 624 int ret; 625 626 if (is_proc_primary()) { 627 ret = rte_eth_dev_start(port_id); 628 if (ret != 0) 629 return ret; 630 631 struct rte_port *port = &ports[port_id]; 632 633 /* 634 * Starting a bonded port also starts all slaves under the bonded 635 * device. So if this port is bond device, we need to modify the 636 * port status of these slaves. 637 */ 638 if (port->bond_flag == 1) 639 return change_bonding_slave_port_status(port_id, false); 640 } 641 642 return 0; 643 } 644 645 static int 646 eth_dev_stop_mp(uint16_t port_id) 647 { 648 int ret; 649 650 if (is_proc_primary()) { 651 ret = rte_eth_dev_stop(port_id); 652 if (ret != 0) 653 return ret; 654 655 struct rte_port *port = &ports[port_id]; 656 657 /* 658 * Stopping a bonded port also stops all slaves under the bonded 659 * device. So if this port is bond device, we need to modify the 660 * port status of these slaves. 661 */ 662 if (port->bond_flag == 1) 663 return change_bonding_slave_port_status(port_id, true); 664 } 665 666 return 0; 667 } 668 669 static void 670 mempool_free_mp(struct rte_mempool *mp) 671 { 672 if (is_proc_primary()) 673 rte_mempool_free(mp); 674 } 675 676 static int 677 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 678 { 679 if (is_proc_primary()) 680 return rte_eth_dev_set_mtu(port_id, mtu); 681 682 return 0; 683 } 684 685 /* Forward function declarations */ 686 static void setup_attached_port(portid_t pi); 687 static void check_all_ports_link_status(uint32_t port_mask); 688 static int eth_event_callback(portid_t port_id, 689 enum rte_eth_event_type type, 690 void *param, void *ret_param); 691 static void dev_event_callback(const char *device_name, 692 enum rte_dev_event_type type, 693 void *param); 694 static void fill_xstats_display_info(void); 695 696 /* 697 * Check if all the ports are started. 698 * If yes, return positive value. If not, return zero. 699 */ 700 static int all_ports_started(void); 701 702 #ifdef RTE_LIB_GSO 703 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 704 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 705 #endif 706 707 /* Holds the registered mbuf dynamic flags names. */ 708 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 709 710 711 /* 712 * Helper function to check if socket is already discovered. 713 * If yes, return positive value. If not, return zero. 714 */ 715 int 716 new_socket_id(unsigned int socket_id) 717 { 718 unsigned int i; 719 720 for (i = 0; i < num_sockets; i++) { 721 if (socket_ids[i] == socket_id) 722 return 0; 723 } 724 return 1; 725 } 726 727 /* 728 * Setup default configuration. 729 */ 730 static void 731 set_default_fwd_lcores_config(void) 732 { 733 unsigned int i; 734 unsigned int nb_lc; 735 unsigned int sock_num; 736 737 nb_lc = 0; 738 for (i = 0; i < RTE_MAX_LCORE; i++) { 739 if (!rte_lcore_is_enabled(i)) 740 continue; 741 sock_num = rte_lcore_to_socket_id(i); 742 if (new_socket_id(sock_num)) { 743 if (num_sockets >= RTE_MAX_NUMA_NODES) { 744 rte_exit(EXIT_FAILURE, 745 "Total sockets greater than %u\n", 746 RTE_MAX_NUMA_NODES); 747 } 748 socket_ids[num_sockets++] = sock_num; 749 } 750 if (i == rte_get_main_lcore()) 751 continue; 752 fwd_lcores_cpuids[nb_lc++] = i; 753 } 754 nb_lcores = (lcoreid_t) nb_lc; 755 nb_cfg_lcores = nb_lcores; 756 nb_fwd_lcores = 1; 757 } 758 759 static void 760 set_def_peer_eth_addrs(void) 761 { 762 portid_t i; 763 764 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 765 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 766 peer_eth_addrs[i].addr_bytes[5] = i; 767 } 768 } 769 770 static void 771 set_default_fwd_ports_config(void) 772 { 773 portid_t pt_id; 774 int i = 0; 775 776 RTE_ETH_FOREACH_DEV(pt_id) { 777 fwd_ports_ids[i++] = pt_id; 778 779 /* Update sockets info according to the attached device */ 780 int socket_id = rte_eth_dev_socket_id(pt_id); 781 if (socket_id >= 0 && new_socket_id(socket_id)) { 782 if (num_sockets >= RTE_MAX_NUMA_NODES) { 783 rte_exit(EXIT_FAILURE, 784 "Total sockets greater than %u\n", 785 RTE_MAX_NUMA_NODES); 786 } 787 socket_ids[num_sockets++] = socket_id; 788 } 789 } 790 791 nb_cfg_ports = nb_ports; 792 nb_fwd_ports = nb_ports; 793 } 794 795 void 796 set_def_fwd_config(void) 797 { 798 set_default_fwd_lcores_config(); 799 set_def_peer_eth_addrs(); 800 set_default_fwd_ports_config(); 801 } 802 803 #ifndef RTE_EXEC_ENV_WINDOWS 804 /* extremely pessimistic estimation of memory required to create a mempool */ 805 static int 806 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 807 { 808 unsigned int n_pages, mbuf_per_pg, leftover; 809 uint64_t total_mem, mbuf_mem, obj_sz; 810 811 /* there is no good way to predict how much space the mempool will 812 * occupy because it will allocate chunks on the fly, and some of those 813 * will come from default DPDK memory while some will come from our 814 * external memory, so just assume 128MB will be enough for everyone. 815 */ 816 uint64_t hdr_mem = 128 << 20; 817 818 /* account for possible non-contiguousness */ 819 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 820 if (obj_sz > pgsz) { 821 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 822 return -1; 823 } 824 825 mbuf_per_pg = pgsz / obj_sz; 826 leftover = (nb_mbufs % mbuf_per_pg) > 0; 827 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 828 829 mbuf_mem = n_pages * pgsz; 830 831 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 832 833 if (total_mem > SIZE_MAX) { 834 TESTPMD_LOG(ERR, "Memory size too big\n"); 835 return -1; 836 } 837 *out = (size_t)total_mem; 838 839 return 0; 840 } 841 842 static int 843 pagesz_flags(uint64_t page_sz) 844 { 845 /* as per mmap() manpage, all page sizes are log2 of page size 846 * shifted by MAP_HUGE_SHIFT 847 */ 848 int log2 = rte_log2_u64(page_sz); 849 850 return (log2 << HUGE_SHIFT); 851 } 852 853 static void * 854 alloc_mem(size_t memsz, size_t pgsz, bool huge) 855 { 856 void *addr; 857 int flags; 858 859 /* allocate anonymous hugepages */ 860 flags = MAP_ANONYMOUS | MAP_PRIVATE; 861 if (huge) 862 flags |= HUGE_FLAG | pagesz_flags(pgsz); 863 864 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 865 if (addr == MAP_FAILED) 866 return NULL; 867 868 return addr; 869 } 870 871 struct extmem_param { 872 void *addr; 873 size_t len; 874 size_t pgsz; 875 rte_iova_t *iova_table; 876 unsigned int iova_table_len; 877 }; 878 879 static int 880 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 881 bool huge) 882 { 883 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 884 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 885 unsigned int cur_page, n_pages, pgsz_idx; 886 size_t mem_sz, cur_pgsz; 887 rte_iova_t *iovas = NULL; 888 void *addr; 889 int ret; 890 891 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 892 /* skip anything that is too big */ 893 if (pgsizes[pgsz_idx] > SIZE_MAX) 894 continue; 895 896 cur_pgsz = pgsizes[pgsz_idx]; 897 898 /* if we were told not to allocate hugepages, override */ 899 if (!huge) 900 cur_pgsz = sysconf(_SC_PAGESIZE); 901 902 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 903 if (ret < 0) { 904 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 905 return -1; 906 } 907 908 /* allocate our memory */ 909 addr = alloc_mem(mem_sz, cur_pgsz, huge); 910 911 /* if we couldn't allocate memory with a specified page size, 912 * that doesn't mean we can't do it with other page sizes, so 913 * try another one. 914 */ 915 if (addr == NULL) 916 continue; 917 918 /* store IOVA addresses for every page in this memory area */ 919 n_pages = mem_sz / cur_pgsz; 920 921 iovas = malloc(sizeof(*iovas) * n_pages); 922 923 if (iovas == NULL) { 924 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 925 goto fail; 926 } 927 /* lock memory if it's not huge pages */ 928 if (!huge) 929 mlock(addr, mem_sz); 930 931 /* populate IOVA addresses */ 932 for (cur_page = 0; cur_page < n_pages; cur_page++) { 933 rte_iova_t iova; 934 size_t offset; 935 void *cur; 936 937 offset = cur_pgsz * cur_page; 938 cur = RTE_PTR_ADD(addr, offset); 939 940 /* touch the page before getting its IOVA */ 941 *(volatile char *)cur = 0; 942 943 iova = rte_mem_virt2iova(cur); 944 945 iovas[cur_page] = iova; 946 } 947 948 break; 949 } 950 /* if we couldn't allocate anything */ 951 if (iovas == NULL) 952 return -1; 953 954 param->addr = addr; 955 param->len = mem_sz; 956 param->pgsz = cur_pgsz; 957 param->iova_table = iovas; 958 param->iova_table_len = n_pages; 959 960 return 0; 961 fail: 962 free(iovas); 963 if (addr) 964 munmap(addr, mem_sz); 965 966 return -1; 967 } 968 969 static int 970 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 971 { 972 struct extmem_param param; 973 int socket_id, ret; 974 975 memset(¶m, 0, sizeof(param)); 976 977 /* check if our heap exists */ 978 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 979 if (socket_id < 0) { 980 /* create our heap */ 981 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 982 if (ret < 0) { 983 TESTPMD_LOG(ERR, "Cannot create heap\n"); 984 return -1; 985 } 986 } 987 988 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 989 if (ret < 0) { 990 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 991 return -1; 992 } 993 994 /* we now have a valid memory area, so add it to heap */ 995 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 996 param.addr, param.len, param.iova_table, 997 param.iova_table_len, param.pgsz); 998 999 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1000 1001 /* not needed any more */ 1002 free(param.iova_table); 1003 1004 if (ret < 0) { 1005 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1006 munmap(param.addr, param.len); 1007 return -1; 1008 } 1009 1010 /* success */ 1011 1012 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1013 param.len >> 20); 1014 1015 return 0; 1016 } 1017 static void 1018 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1019 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1020 { 1021 uint16_t pid = 0; 1022 int ret; 1023 1024 RTE_ETH_FOREACH_DEV(pid) { 1025 struct rte_eth_dev_info dev_info; 1026 1027 ret = eth_dev_info_get_print_err(pid, &dev_info); 1028 if (ret != 0) { 1029 TESTPMD_LOG(DEBUG, 1030 "unable to get device info for port %d on addr 0x%p," 1031 "mempool unmapping will not be performed\n", 1032 pid, memhdr->addr); 1033 continue; 1034 } 1035 1036 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1037 if (ret) { 1038 TESTPMD_LOG(DEBUG, 1039 "unable to DMA unmap addr 0x%p " 1040 "for device %s\n", 1041 memhdr->addr, rte_dev_name(dev_info.device)); 1042 } 1043 } 1044 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1045 if (ret) { 1046 TESTPMD_LOG(DEBUG, 1047 "unable to un-register addr 0x%p\n", memhdr->addr); 1048 } 1049 } 1050 1051 static void 1052 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1053 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1054 { 1055 uint16_t pid = 0; 1056 size_t page_size = sysconf(_SC_PAGESIZE); 1057 int ret; 1058 1059 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1060 page_size); 1061 if (ret) { 1062 TESTPMD_LOG(DEBUG, 1063 "unable to register addr 0x%p\n", memhdr->addr); 1064 return; 1065 } 1066 RTE_ETH_FOREACH_DEV(pid) { 1067 struct rte_eth_dev_info dev_info; 1068 1069 ret = eth_dev_info_get_print_err(pid, &dev_info); 1070 if (ret != 0) { 1071 TESTPMD_LOG(DEBUG, 1072 "unable to get device info for port %d on addr 0x%p," 1073 "mempool mapping will not be performed\n", 1074 pid, memhdr->addr); 1075 continue; 1076 } 1077 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1078 if (ret) { 1079 TESTPMD_LOG(DEBUG, 1080 "unable to DMA map addr 0x%p " 1081 "for device %s\n", 1082 memhdr->addr, rte_dev_name(dev_info.device)); 1083 } 1084 } 1085 } 1086 #endif 1087 1088 static unsigned int 1089 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1090 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1091 { 1092 struct rte_pktmbuf_extmem *xmem; 1093 unsigned int ext_num, zone_num, elt_num; 1094 uint16_t elt_size; 1095 1096 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1097 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1098 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1099 1100 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1101 if (xmem == NULL) { 1102 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1103 "external buffer descriptors\n"); 1104 *ext_mem = NULL; 1105 return 0; 1106 } 1107 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1108 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1109 const struct rte_memzone *mz; 1110 char mz_name[RTE_MEMZONE_NAMESIZE]; 1111 int ret; 1112 1113 ret = snprintf(mz_name, sizeof(mz_name), 1114 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1115 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1116 errno = ENAMETOOLONG; 1117 ext_num = 0; 1118 break; 1119 } 1120 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1121 socket_id, 1122 RTE_MEMZONE_IOVA_CONTIG | 1123 RTE_MEMZONE_1GB | 1124 RTE_MEMZONE_SIZE_HINT_ONLY); 1125 if (mz == NULL) { 1126 /* 1127 * The caller exits on external buffer creation 1128 * error, so there is no need to free memzones. 1129 */ 1130 errno = ENOMEM; 1131 ext_num = 0; 1132 break; 1133 } 1134 xseg->buf_ptr = mz->addr; 1135 xseg->buf_iova = mz->iova; 1136 xseg->buf_len = EXTBUF_ZONE_SIZE; 1137 xseg->elt_size = elt_size; 1138 } 1139 if (ext_num == 0 && xmem != NULL) { 1140 free(xmem); 1141 xmem = NULL; 1142 } 1143 *ext_mem = xmem; 1144 return ext_num; 1145 } 1146 1147 /* 1148 * Configuration initialisation done once at init time. 1149 */ 1150 static struct rte_mempool * 1151 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1152 unsigned int socket_id, uint16_t size_idx) 1153 { 1154 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1155 struct rte_mempool *rte_mp = NULL; 1156 #ifndef RTE_EXEC_ENV_WINDOWS 1157 uint32_t mb_size; 1158 1159 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1160 #endif 1161 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1162 if (!is_proc_primary()) { 1163 rte_mp = rte_mempool_lookup(pool_name); 1164 if (rte_mp == NULL) 1165 rte_exit(EXIT_FAILURE, 1166 "Get mbuf pool for socket %u failed: %s\n", 1167 socket_id, rte_strerror(rte_errno)); 1168 return rte_mp; 1169 } 1170 1171 TESTPMD_LOG(INFO, 1172 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1173 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1174 1175 switch (mp_alloc_type) { 1176 case MP_ALLOC_NATIVE: 1177 { 1178 /* wrapper to rte_mempool_create() */ 1179 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1180 rte_mbuf_best_mempool_ops()); 1181 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1182 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1183 break; 1184 } 1185 #ifndef RTE_EXEC_ENV_WINDOWS 1186 case MP_ALLOC_ANON: 1187 { 1188 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1189 mb_size, (unsigned int) mb_mempool_cache, 1190 sizeof(struct rte_pktmbuf_pool_private), 1191 socket_id, mempool_flags); 1192 if (rte_mp == NULL) 1193 goto err; 1194 1195 if (rte_mempool_populate_anon(rte_mp) == 0) { 1196 rte_mempool_free(rte_mp); 1197 rte_mp = NULL; 1198 goto err; 1199 } 1200 rte_pktmbuf_pool_init(rte_mp, NULL); 1201 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1202 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1203 break; 1204 } 1205 case MP_ALLOC_XMEM: 1206 case MP_ALLOC_XMEM_HUGE: 1207 { 1208 int heap_socket; 1209 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1210 1211 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1212 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1213 1214 heap_socket = 1215 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1216 if (heap_socket < 0) 1217 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1218 1219 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1220 rte_mbuf_best_mempool_ops()); 1221 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1222 mb_mempool_cache, 0, mbuf_seg_size, 1223 heap_socket); 1224 break; 1225 } 1226 #endif 1227 case MP_ALLOC_XBUF: 1228 { 1229 struct rte_pktmbuf_extmem *ext_mem; 1230 unsigned int ext_num; 1231 1232 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1233 socket_id, pool_name, &ext_mem); 1234 if (ext_num == 0) 1235 rte_exit(EXIT_FAILURE, 1236 "Can't create pinned data buffers\n"); 1237 1238 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1239 rte_mbuf_best_mempool_ops()); 1240 rte_mp = rte_pktmbuf_pool_create_extbuf 1241 (pool_name, nb_mbuf, mb_mempool_cache, 1242 0, mbuf_seg_size, socket_id, 1243 ext_mem, ext_num); 1244 free(ext_mem); 1245 break; 1246 } 1247 default: 1248 { 1249 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1250 } 1251 } 1252 1253 #ifndef RTE_EXEC_ENV_WINDOWS 1254 err: 1255 #endif 1256 if (rte_mp == NULL) { 1257 rte_exit(EXIT_FAILURE, 1258 "Creation of mbuf pool for socket %u failed: %s\n", 1259 socket_id, rte_strerror(rte_errno)); 1260 } else if (verbose_level > 0) { 1261 rte_mempool_dump(stdout, rte_mp); 1262 } 1263 return rte_mp; 1264 } 1265 1266 /* 1267 * Check given socket id is valid or not with NUMA mode, 1268 * if valid, return 0, else return -1 1269 */ 1270 static int 1271 check_socket_id(const unsigned int socket_id) 1272 { 1273 static int warning_once = 0; 1274 1275 if (new_socket_id(socket_id)) { 1276 if (!warning_once && numa_support) 1277 fprintf(stderr, 1278 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1279 warning_once = 1; 1280 return -1; 1281 } 1282 return 0; 1283 } 1284 1285 /* 1286 * Get the allowed maximum number of RX queues. 1287 * *pid return the port id which has minimal value of 1288 * max_rx_queues in all ports. 1289 */ 1290 queueid_t 1291 get_allowed_max_nb_rxq(portid_t *pid) 1292 { 1293 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1294 bool max_rxq_valid = false; 1295 portid_t pi; 1296 struct rte_eth_dev_info dev_info; 1297 1298 RTE_ETH_FOREACH_DEV(pi) { 1299 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1300 continue; 1301 1302 max_rxq_valid = true; 1303 if (dev_info.max_rx_queues < allowed_max_rxq) { 1304 allowed_max_rxq = dev_info.max_rx_queues; 1305 *pid = pi; 1306 } 1307 } 1308 return max_rxq_valid ? allowed_max_rxq : 0; 1309 } 1310 1311 /* 1312 * Check input rxq is valid or not. 1313 * If input rxq is not greater than any of maximum number 1314 * of RX queues of all ports, it is valid. 1315 * if valid, return 0, else return -1 1316 */ 1317 int 1318 check_nb_rxq(queueid_t rxq) 1319 { 1320 queueid_t allowed_max_rxq; 1321 portid_t pid = 0; 1322 1323 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1324 if (rxq > allowed_max_rxq) { 1325 fprintf(stderr, 1326 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1327 rxq, allowed_max_rxq, pid); 1328 return -1; 1329 } 1330 return 0; 1331 } 1332 1333 /* 1334 * Get the allowed maximum number of TX queues. 1335 * *pid return the port id which has minimal value of 1336 * max_tx_queues in all ports. 1337 */ 1338 queueid_t 1339 get_allowed_max_nb_txq(portid_t *pid) 1340 { 1341 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1342 bool max_txq_valid = false; 1343 portid_t pi; 1344 struct rte_eth_dev_info dev_info; 1345 1346 RTE_ETH_FOREACH_DEV(pi) { 1347 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1348 continue; 1349 1350 max_txq_valid = true; 1351 if (dev_info.max_tx_queues < allowed_max_txq) { 1352 allowed_max_txq = dev_info.max_tx_queues; 1353 *pid = pi; 1354 } 1355 } 1356 return max_txq_valid ? allowed_max_txq : 0; 1357 } 1358 1359 /* 1360 * Check input txq is valid or not. 1361 * If input txq is not greater than any of maximum number 1362 * of TX queues of all ports, it is valid. 1363 * if valid, return 0, else return -1 1364 */ 1365 int 1366 check_nb_txq(queueid_t txq) 1367 { 1368 queueid_t allowed_max_txq; 1369 portid_t pid = 0; 1370 1371 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1372 if (txq > allowed_max_txq) { 1373 fprintf(stderr, 1374 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1375 txq, allowed_max_txq, pid); 1376 return -1; 1377 } 1378 return 0; 1379 } 1380 1381 /* 1382 * Get the allowed maximum number of RXDs of every rx queue. 1383 * *pid return the port id which has minimal value of 1384 * max_rxd in all queues of all ports. 1385 */ 1386 static uint16_t 1387 get_allowed_max_nb_rxd(portid_t *pid) 1388 { 1389 uint16_t allowed_max_rxd = UINT16_MAX; 1390 portid_t pi; 1391 struct rte_eth_dev_info dev_info; 1392 1393 RTE_ETH_FOREACH_DEV(pi) { 1394 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1395 continue; 1396 1397 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1398 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1399 *pid = pi; 1400 } 1401 } 1402 return allowed_max_rxd; 1403 } 1404 1405 /* 1406 * Get the allowed minimal number of RXDs of every rx queue. 1407 * *pid return the port id which has minimal value of 1408 * min_rxd in all queues of all ports. 1409 */ 1410 static uint16_t 1411 get_allowed_min_nb_rxd(portid_t *pid) 1412 { 1413 uint16_t allowed_min_rxd = 0; 1414 portid_t pi; 1415 struct rte_eth_dev_info dev_info; 1416 1417 RTE_ETH_FOREACH_DEV(pi) { 1418 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1419 continue; 1420 1421 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1422 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1423 *pid = pi; 1424 } 1425 } 1426 1427 return allowed_min_rxd; 1428 } 1429 1430 /* 1431 * Check input rxd is valid or not. 1432 * If input rxd is not greater than any of maximum number 1433 * of RXDs of every Rx queues and is not less than any of 1434 * minimal number of RXDs of every Rx queues, it is valid. 1435 * if valid, return 0, else return -1 1436 */ 1437 int 1438 check_nb_rxd(queueid_t rxd) 1439 { 1440 uint16_t allowed_max_rxd; 1441 uint16_t allowed_min_rxd; 1442 portid_t pid = 0; 1443 1444 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1445 if (rxd > allowed_max_rxd) { 1446 fprintf(stderr, 1447 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1448 rxd, allowed_max_rxd, pid); 1449 return -1; 1450 } 1451 1452 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1453 if (rxd < allowed_min_rxd) { 1454 fprintf(stderr, 1455 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1456 rxd, allowed_min_rxd, pid); 1457 return -1; 1458 } 1459 1460 return 0; 1461 } 1462 1463 /* 1464 * Get the allowed maximum number of TXDs of every rx queues. 1465 * *pid return the port id which has minimal value of 1466 * max_txd in every tx queue. 1467 */ 1468 static uint16_t 1469 get_allowed_max_nb_txd(portid_t *pid) 1470 { 1471 uint16_t allowed_max_txd = UINT16_MAX; 1472 portid_t pi; 1473 struct rte_eth_dev_info dev_info; 1474 1475 RTE_ETH_FOREACH_DEV(pi) { 1476 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1477 continue; 1478 1479 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1480 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1481 *pid = pi; 1482 } 1483 } 1484 return allowed_max_txd; 1485 } 1486 1487 /* 1488 * Get the allowed maximum number of TXDs of every tx queues. 1489 * *pid return the port id which has minimal value of 1490 * min_txd in every tx queue. 1491 */ 1492 static uint16_t 1493 get_allowed_min_nb_txd(portid_t *pid) 1494 { 1495 uint16_t allowed_min_txd = 0; 1496 portid_t pi; 1497 struct rte_eth_dev_info dev_info; 1498 1499 RTE_ETH_FOREACH_DEV(pi) { 1500 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1501 continue; 1502 1503 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1504 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1505 *pid = pi; 1506 } 1507 } 1508 1509 return allowed_min_txd; 1510 } 1511 1512 /* 1513 * Check input txd is valid or not. 1514 * If input txd is not greater than any of maximum number 1515 * of TXDs of every Rx queues, it is valid. 1516 * if valid, return 0, else return -1 1517 */ 1518 int 1519 check_nb_txd(queueid_t txd) 1520 { 1521 uint16_t allowed_max_txd; 1522 uint16_t allowed_min_txd; 1523 portid_t pid = 0; 1524 1525 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1526 if (txd > allowed_max_txd) { 1527 fprintf(stderr, 1528 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1529 txd, allowed_max_txd, pid); 1530 return -1; 1531 } 1532 1533 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1534 if (txd < allowed_min_txd) { 1535 fprintf(stderr, 1536 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1537 txd, allowed_min_txd, pid); 1538 return -1; 1539 } 1540 return 0; 1541 } 1542 1543 1544 /* 1545 * Get the allowed maximum number of hairpin queues. 1546 * *pid return the port id which has minimal value of 1547 * max_hairpin_queues in all ports. 1548 */ 1549 queueid_t 1550 get_allowed_max_nb_hairpinq(portid_t *pid) 1551 { 1552 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1553 portid_t pi; 1554 struct rte_eth_hairpin_cap cap; 1555 1556 RTE_ETH_FOREACH_DEV(pi) { 1557 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1558 *pid = pi; 1559 return 0; 1560 } 1561 if (cap.max_nb_queues < allowed_max_hairpinq) { 1562 allowed_max_hairpinq = cap.max_nb_queues; 1563 *pid = pi; 1564 } 1565 } 1566 return allowed_max_hairpinq; 1567 } 1568 1569 /* 1570 * Check input hairpin is valid or not. 1571 * If input hairpin is not greater than any of maximum number 1572 * of hairpin queues of all ports, it is valid. 1573 * if valid, return 0, else return -1 1574 */ 1575 int 1576 check_nb_hairpinq(queueid_t hairpinq) 1577 { 1578 queueid_t allowed_max_hairpinq; 1579 portid_t pid = 0; 1580 1581 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1582 if (hairpinq > allowed_max_hairpinq) { 1583 fprintf(stderr, 1584 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1585 hairpinq, allowed_max_hairpinq, pid); 1586 return -1; 1587 } 1588 return 0; 1589 } 1590 1591 static int 1592 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1593 { 1594 uint32_t eth_overhead; 1595 1596 if (dev_info->max_mtu != UINT16_MAX && 1597 dev_info->max_rx_pktlen > dev_info->max_mtu) 1598 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1599 else 1600 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1601 1602 return eth_overhead; 1603 } 1604 1605 static void 1606 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1607 { 1608 struct rte_port *port = &ports[pid]; 1609 int ret; 1610 int i; 1611 1612 eth_rx_metadata_negotiate_mp(pid); 1613 1614 port->dev_conf.txmode = tx_mode; 1615 port->dev_conf.rxmode = rx_mode; 1616 1617 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1618 if (ret != 0) 1619 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1620 1621 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1622 port->dev_conf.txmode.offloads &= 1623 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1624 1625 /* Apply Rx offloads configuration */ 1626 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1627 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1628 /* Apply Tx offloads configuration */ 1629 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1630 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1631 1632 if (eth_link_speed) 1633 port->dev_conf.link_speeds = eth_link_speed; 1634 1635 if (max_rx_pkt_len) 1636 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1637 get_eth_overhead(&port->dev_info); 1638 1639 /* set flag to initialize port/queue */ 1640 port->need_reconfig = 1; 1641 port->need_reconfig_queues = 1; 1642 port->socket_id = socket_id; 1643 port->tx_metadata = 0; 1644 1645 /* 1646 * Check for maximum number of segments per MTU. 1647 * Accordingly update the mbuf data size. 1648 */ 1649 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1650 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1651 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1652 uint16_t mtu; 1653 1654 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1655 uint16_t data_size = (mtu + eth_overhead) / 1656 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1657 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1658 1659 if (buffer_size > mbuf_data_size[0]) { 1660 mbuf_data_size[0] = buffer_size; 1661 TESTPMD_LOG(WARNING, 1662 "Configured mbuf size of the first segment %hu\n", 1663 mbuf_data_size[0]); 1664 } 1665 } 1666 } 1667 } 1668 1669 static void 1670 init_config(void) 1671 { 1672 portid_t pid; 1673 struct rte_mempool *mbp; 1674 unsigned int nb_mbuf_per_pool; 1675 lcoreid_t lc_id; 1676 #ifdef RTE_LIB_GRO 1677 struct rte_gro_param gro_param; 1678 #endif 1679 #ifdef RTE_LIB_GSO 1680 uint32_t gso_types; 1681 #endif 1682 1683 /* Configuration of logical cores. */ 1684 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1685 sizeof(struct fwd_lcore *) * nb_lcores, 1686 RTE_CACHE_LINE_SIZE); 1687 if (fwd_lcores == NULL) { 1688 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1689 "failed\n", nb_lcores); 1690 } 1691 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1692 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1693 sizeof(struct fwd_lcore), 1694 RTE_CACHE_LINE_SIZE); 1695 if (fwd_lcores[lc_id] == NULL) { 1696 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1697 "failed\n"); 1698 } 1699 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1700 } 1701 1702 RTE_ETH_FOREACH_DEV(pid) { 1703 uint32_t socket_id; 1704 1705 if (numa_support) { 1706 socket_id = port_numa[pid]; 1707 if (port_numa[pid] == NUMA_NO_CONFIG) { 1708 socket_id = rte_eth_dev_socket_id(pid); 1709 1710 /* 1711 * if socket_id is invalid, 1712 * set to the first available socket. 1713 */ 1714 if (check_socket_id(socket_id) < 0) 1715 socket_id = socket_ids[0]; 1716 } 1717 } else { 1718 socket_id = (socket_num == UMA_NO_CONFIG) ? 1719 0 : socket_num; 1720 } 1721 /* Apply default TxRx configuration for all ports */ 1722 init_config_port_offloads(pid, socket_id); 1723 } 1724 /* 1725 * Create pools of mbuf. 1726 * If NUMA support is disabled, create a single pool of mbuf in 1727 * socket 0 memory by default. 1728 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1729 * 1730 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1731 * nb_txd can be configured at run time. 1732 */ 1733 if (param_total_num_mbufs) 1734 nb_mbuf_per_pool = param_total_num_mbufs; 1735 else { 1736 nb_mbuf_per_pool = RX_DESC_MAX + 1737 (nb_lcores * mb_mempool_cache) + 1738 TX_DESC_MAX + MAX_PKT_BURST; 1739 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1740 } 1741 1742 if (numa_support) { 1743 uint8_t i, j; 1744 1745 for (i = 0; i < num_sockets; i++) 1746 for (j = 0; j < mbuf_data_size_n; j++) 1747 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1748 mbuf_pool_create(mbuf_data_size[j], 1749 nb_mbuf_per_pool, 1750 socket_ids[i], j); 1751 } else { 1752 uint8_t i; 1753 1754 for (i = 0; i < mbuf_data_size_n; i++) 1755 mempools[i] = mbuf_pool_create 1756 (mbuf_data_size[i], 1757 nb_mbuf_per_pool, 1758 socket_num == UMA_NO_CONFIG ? 1759 0 : socket_num, i); 1760 } 1761 1762 init_port_config(); 1763 1764 #ifdef RTE_LIB_GSO 1765 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1766 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1767 #endif 1768 /* 1769 * Records which Mbuf pool to use by each logical core, if needed. 1770 */ 1771 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1772 mbp = mbuf_pool_find( 1773 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1774 1775 if (mbp == NULL) 1776 mbp = mbuf_pool_find(0, 0); 1777 fwd_lcores[lc_id]->mbp = mbp; 1778 #ifdef RTE_LIB_GSO 1779 /* initialize GSO context */ 1780 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1781 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1782 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1783 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1784 RTE_ETHER_CRC_LEN; 1785 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1786 #endif 1787 } 1788 1789 fwd_config_setup(); 1790 1791 #ifdef RTE_LIB_GRO 1792 /* create a gro context for each lcore */ 1793 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1794 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1795 gro_param.max_item_per_flow = MAX_PKT_BURST; 1796 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1797 gro_param.socket_id = rte_lcore_to_socket_id( 1798 fwd_lcores_cpuids[lc_id]); 1799 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1800 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1801 rte_exit(EXIT_FAILURE, 1802 "rte_gro_ctx_create() failed\n"); 1803 } 1804 } 1805 #endif 1806 } 1807 1808 1809 void 1810 reconfig(portid_t new_port_id, unsigned socket_id) 1811 { 1812 /* Reconfiguration of Ethernet ports. */ 1813 init_config_port_offloads(new_port_id, socket_id); 1814 init_port_config(); 1815 } 1816 1817 int 1818 init_fwd_streams(void) 1819 { 1820 portid_t pid; 1821 struct rte_port *port; 1822 streamid_t sm_id, nb_fwd_streams_new; 1823 queueid_t q; 1824 1825 /* set socket id according to numa or not */ 1826 RTE_ETH_FOREACH_DEV(pid) { 1827 port = &ports[pid]; 1828 if (nb_rxq > port->dev_info.max_rx_queues) { 1829 fprintf(stderr, 1830 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1831 nb_rxq, port->dev_info.max_rx_queues); 1832 return -1; 1833 } 1834 if (nb_txq > port->dev_info.max_tx_queues) { 1835 fprintf(stderr, 1836 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1837 nb_txq, port->dev_info.max_tx_queues); 1838 return -1; 1839 } 1840 if (numa_support) { 1841 if (port_numa[pid] != NUMA_NO_CONFIG) 1842 port->socket_id = port_numa[pid]; 1843 else { 1844 port->socket_id = rte_eth_dev_socket_id(pid); 1845 1846 /* 1847 * if socket_id is invalid, 1848 * set to the first available socket. 1849 */ 1850 if (check_socket_id(port->socket_id) < 0) 1851 port->socket_id = socket_ids[0]; 1852 } 1853 } 1854 else { 1855 if (socket_num == UMA_NO_CONFIG) 1856 port->socket_id = 0; 1857 else 1858 port->socket_id = socket_num; 1859 } 1860 } 1861 1862 q = RTE_MAX(nb_rxq, nb_txq); 1863 if (q == 0) { 1864 fprintf(stderr, 1865 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1866 return -1; 1867 } 1868 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1869 if (nb_fwd_streams_new == nb_fwd_streams) 1870 return 0; 1871 /* clear the old */ 1872 if (fwd_streams != NULL) { 1873 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1874 if (fwd_streams[sm_id] == NULL) 1875 continue; 1876 rte_free(fwd_streams[sm_id]); 1877 fwd_streams[sm_id] = NULL; 1878 } 1879 rte_free(fwd_streams); 1880 fwd_streams = NULL; 1881 } 1882 1883 /* init new */ 1884 nb_fwd_streams = nb_fwd_streams_new; 1885 if (nb_fwd_streams) { 1886 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1887 sizeof(struct fwd_stream *) * nb_fwd_streams, 1888 RTE_CACHE_LINE_SIZE); 1889 if (fwd_streams == NULL) 1890 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1891 " (struct fwd_stream *)) failed\n", 1892 nb_fwd_streams); 1893 1894 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1895 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1896 " struct fwd_stream", sizeof(struct fwd_stream), 1897 RTE_CACHE_LINE_SIZE); 1898 if (fwd_streams[sm_id] == NULL) 1899 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1900 "(struct fwd_stream) failed\n"); 1901 } 1902 } 1903 1904 return 0; 1905 } 1906 1907 static void 1908 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1909 { 1910 uint64_t total_burst, sburst; 1911 uint64_t nb_burst; 1912 uint64_t burst_stats[4]; 1913 uint16_t pktnb_stats[4]; 1914 uint16_t nb_pkt; 1915 int burst_percent[4], sburstp; 1916 int i; 1917 1918 /* 1919 * First compute the total number of packet bursts and the 1920 * two highest numbers of bursts of the same number of packets. 1921 */ 1922 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1923 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1924 1925 /* Show stats for 0 burst size always */ 1926 total_burst = pbs->pkt_burst_spread[0]; 1927 burst_stats[0] = pbs->pkt_burst_spread[0]; 1928 pktnb_stats[0] = 0; 1929 1930 /* Find the next 2 burst sizes with highest occurrences. */ 1931 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1932 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1933 1934 if (nb_burst == 0) 1935 continue; 1936 1937 total_burst += nb_burst; 1938 1939 if (nb_burst > burst_stats[1]) { 1940 burst_stats[2] = burst_stats[1]; 1941 pktnb_stats[2] = pktnb_stats[1]; 1942 burst_stats[1] = nb_burst; 1943 pktnb_stats[1] = nb_pkt; 1944 } else if (nb_burst > burst_stats[2]) { 1945 burst_stats[2] = nb_burst; 1946 pktnb_stats[2] = nb_pkt; 1947 } 1948 } 1949 if (total_burst == 0) 1950 return; 1951 1952 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1953 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1954 if (i == 3) { 1955 printf("%d%% of other]\n", 100 - sburstp); 1956 return; 1957 } 1958 1959 sburst += burst_stats[i]; 1960 if (sburst == total_burst) { 1961 printf("%d%% of %d pkts]\n", 1962 100 - sburstp, (int) pktnb_stats[i]); 1963 return; 1964 } 1965 1966 burst_percent[i] = 1967 (double)burst_stats[i] / total_burst * 100; 1968 printf("%d%% of %d pkts + ", 1969 burst_percent[i], (int) pktnb_stats[i]); 1970 sburstp += burst_percent[i]; 1971 } 1972 } 1973 1974 static void 1975 fwd_stream_stats_display(streamid_t stream_id) 1976 { 1977 struct fwd_stream *fs; 1978 static const char *fwd_top_stats_border = "-------"; 1979 1980 fs = fwd_streams[stream_id]; 1981 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1982 (fs->fwd_dropped == 0)) 1983 return; 1984 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1985 "TX Port=%2d/Queue=%2d %s\n", 1986 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1987 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1988 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1989 " TX-dropped: %-14"PRIu64, 1990 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1991 1992 /* if checksum mode */ 1993 if (cur_fwd_eng == &csum_fwd_engine) { 1994 printf(" RX- bad IP checksum: %-14"PRIu64 1995 " Rx- bad L4 checksum: %-14"PRIu64 1996 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1997 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1998 fs->rx_bad_outer_l4_csum); 1999 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2000 fs->rx_bad_outer_ip_csum); 2001 } else { 2002 printf("\n"); 2003 } 2004 2005 if (record_burst_stats) { 2006 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2007 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2008 } 2009 } 2010 2011 void 2012 fwd_stats_display(void) 2013 { 2014 static const char *fwd_stats_border = "----------------------"; 2015 static const char *acc_stats_border = "+++++++++++++++"; 2016 struct { 2017 struct fwd_stream *rx_stream; 2018 struct fwd_stream *tx_stream; 2019 uint64_t tx_dropped; 2020 uint64_t rx_bad_ip_csum; 2021 uint64_t rx_bad_l4_csum; 2022 uint64_t rx_bad_outer_l4_csum; 2023 uint64_t rx_bad_outer_ip_csum; 2024 } ports_stats[RTE_MAX_ETHPORTS]; 2025 uint64_t total_rx_dropped = 0; 2026 uint64_t total_tx_dropped = 0; 2027 uint64_t total_rx_nombuf = 0; 2028 struct rte_eth_stats stats; 2029 uint64_t fwd_cycles = 0; 2030 uint64_t total_recv = 0; 2031 uint64_t total_xmit = 0; 2032 struct rte_port *port; 2033 streamid_t sm_id; 2034 portid_t pt_id; 2035 int ret; 2036 int i; 2037 2038 memset(ports_stats, 0, sizeof(ports_stats)); 2039 2040 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2041 struct fwd_stream *fs = fwd_streams[sm_id]; 2042 2043 if (cur_fwd_config.nb_fwd_streams > 2044 cur_fwd_config.nb_fwd_ports) { 2045 fwd_stream_stats_display(sm_id); 2046 } else { 2047 ports_stats[fs->tx_port].tx_stream = fs; 2048 ports_stats[fs->rx_port].rx_stream = fs; 2049 } 2050 2051 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2052 2053 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2054 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2055 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2056 fs->rx_bad_outer_l4_csum; 2057 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2058 fs->rx_bad_outer_ip_csum; 2059 2060 if (record_core_cycles) 2061 fwd_cycles += fs->busy_cycles; 2062 } 2063 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2064 uint64_t tx_dropped = 0; 2065 2066 pt_id = fwd_ports_ids[i]; 2067 port = &ports[pt_id]; 2068 2069 ret = rte_eth_stats_get(pt_id, &stats); 2070 if (ret != 0) { 2071 fprintf(stderr, 2072 "%s: Error: failed to get stats (port %u): %d", 2073 __func__, pt_id, ret); 2074 continue; 2075 } 2076 stats.ipackets -= port->stats.ipackets; 2077 stats.opackets -= port->stats.opackets; 2078 stats.ibytes -= port->stats.ibytes; 2079 stats.obytes -= port->stats.obytes; 2080 stats.imissed -= port->stats.imissed; 2081 stats.oerrors -= port->stats.oerrors; 2082 stats.rx_nombuf -= port->stats.rx_nombuf; 2083 2084 total_recv += stats.ipackets; 2085 total_xmit += stats.opackets; 2086 total_rx_dropped += stats.imissed; 2087 tx_dropped += ports_stats[pt_id].tx_dropped; 2088 tx_dropped += stats.oerrors; 2089 total_tx_dropped += tx_dropped; 2090 total_rx_nombuf += stats.rx_nombuf; 2091 2092 printf("\n %s Forward statistics for port %-2d %s\n", 2093 fwd_stats_border, pt_id, fwd_stats_border); 2094 2095 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2096 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2097 stats.ipackets + stats.imissed); 2098 2099 if (cur_fwd_eng == &csum_fwd_engine) { 2100 printf(" Bad-ipcsum: %-14"PRIu64 2101 " Bad-l4csum: %-14"PRIu64 2102 "Bad-outer-l4csum: %-14"PRIu64"\n", 2103 ports_stats[pt_id].rx_bad_ip_csum, 2104 ports_stats[pt_id].rx_bad_l4_csum, 2105 ports_stats[pt_id].rx_bad_outer_l4_csum); 2106 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2107 ports_stats[pt_id].rx_bad_outer_ip_csum); 2108 } 2109 if (stats.ierrors + stats.rx_nombuf > 0) { 2110 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2111 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2112 } 2113 2114 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2115 "TX-total: %-"PRIu64"\n", 2116 stats.opackets, tx_dropped, 2117 stats.opackets + tx_dropped); 2118 2119 if (record_burst_stats) { 2120 if (ports_stats[pt_id].rx_stream) 2121 pkt_burst_stats_display("RX", 2122 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2123 if (ports_stats[pt_id].tx_stream) 2124 pkt_burst_stats_display("TX", 2125 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2126 } 2127 2128 printf(" %s--------------------------------%s\n", 2129 fwd_stats_border, fwd_stats_border); 2130 } 2131 2132 printf("\n %s Accumulated forward statistics for all ports" 2133 "%s\n", 2134 acc_stats_border, acc_stats_border); 2135 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2136 "%-"PRIu64"\n" 2137 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2138 "%-"PRIu64"\n", 2139 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2140 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2141 if (total_rx_nombuf > 0) 2142 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2143 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2144 "%s\n", 2145 acc_stats_border, acc_stats_border); 2146 if (record_core_cycles) { 2147 #define CYC_PER_MHZ 1E6 2148 if (total_recv > 0 || total_xmit > 0) { 2149 uint64_t total_pkts = 0; 2150 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2151 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2152 total_pkts = total_xmit; 2153 else 2154 total_pkts = total_recv; 2155 2156 printf("\n CPU cycles/packet=%.2F (busy cycles=" 2157 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2158 " MHz Clock\n", 2159 (double) fwd_cycles / total_pkts, 2160 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2161 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2162 } 2163 } 2164 } 2165 2166 void 2167 fwd_stats_reset(void) 2168 { 2169 streamid_t sm_id; 2170 portid_t pt_id; 2171 int ret; 2172 int i; 2173 2174 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2175 pt_id = fwd_ports_ids[i]; 2176 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2177 if (ret != 0) 2178 fprintf(stderr, 2179 "%s: Error: failed to clear stats (port %u):%d", 2180 __func__, pt_id, ret); 2181 } 2182 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2183 struct fwd_stream *fs = fwd_streams[sm_id]; 2184 2185 fs->rx_packets = 0; 2186 fs->tx_packets = 0; 2187 fs->fwd_dropped = 0; 2188 fs->rx_bad_ip_csum = 0; 2189 fs->rx_bad_l4_csum = 0; 2190 fs->rx_bad_outer_l4_csum = 0; 2191 fs->rx_bad_outer_ip_csum = 0; 2192 2193 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2194 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2195 fs->busy_cycles = 0; 2196 } 2197 } 2198 2199 static void 2200 flush_fwd_rx_queues(void) 2201 { 2202 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2203 portid_t rxp; 2204 portid_t port_id; 2205 queueid_t rxq; 2206 uint16_t nb_rx; 2207 uint16_t i; 2208 uint8_t j; 2209 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2210 uint64_t timer_period; 2211 2212 if (num_procs > 1) { 2213 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2214 return; 2215 } 2216 2217 /* convert to number of cycles */ 2218 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2219 2220 for (j = 0; j < 2; j++) { 2221 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2222 for (rxq = 0; rxq < nb_rxq; rxq++) { 2223 port_id = fwd_ports_ids[rxp]; 2224 2225 /* Polling stopped queues is prohibited. */ 2226 if (ports[port_id].rxq[rxq].state == 2227 RTE_ETH_QUEUE_STATE_STOPPED) 2228 continue; 2229 2230 /** 2231 * testpmd can stuck in the below do while loop 2232 * if rte_eth_rx_burst() always returns nonzero 2233 * packets. So timer is added to exit this loop 2234 * after 1sec timer expiry. 2235 */ 2236 prev_tsc = rte_rdtsc(); 2237 do { 2238 nb_rx = rte_eth_rx_burst(port_id, rxq, 2239 pkts_burst, MAX_PKT_BURST); 2240 for (i = 0; i < nb_rx; i++) 2241 rte_pktmbuf_free(pkts_burst[i]); 2242 2243 cur_tsc = rte_rdtsc(); 2244 diff_tsc = cur_tsc - prev_tsc; 2245 timer_tsc += diff_tsc; 2246 } while ((nb_rx > 0) && 2247 (timer_tsc < timer_period)); 2248 timer_tsc = 0; 2249 } 2250 } 2251 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2252 } 2253 } 2254 2255 static void 2256 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2257 { 2258 struct fwd_stream **fsm; 2259 uint64_t prev_tsc; 2260 streamid_t nb_fs; 2261 streamid_t sm_id; 2262 #ifdef RTE_LIB_BITRATESTATS 2263 uint64_t tics_per_1sec; 2264 uint64_t tics_datum; 2265 uint64_t tics_current; 2266 uint16_t i, cnt_ports; 2267 2268 cnt_ports = nb_ports; 2269 tics_datum = rte_rdtsc(); 2270 tics_per_1sec = rte_get_timer_hz(); 2271 #endif 2272 fsm = &fwd_streams[fc->stream_idx]; 2273 nb_fs = fc->stream_nb; 2274 prev_tsc = rte_rdtsc(); 2275 do { 2276 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2277 if (!fsm[sm_id]->disabled) 2278 (*pkt_fwd)(fsm[sm_id]); 2279 #ifdef RTE_LIB_BITRATESTATS 2280 if (bitrate_enabled != 0 && 2281 bitrate_lcore_id == rte_lcore_id()) { 2282 tics_current = rte_rdtsc(); 2283 if (tics_current - tics_datum >= tics_per_1sec) { 2284 /* Periodic bitrate calculation */ 2285 for (i = 0; i < cnt_ports; i++) 2286 rte_stats_bitrate_calc(bitrate_data, 2287 ports_ids[i]); 2288 tics_datum = tics_current; 2289 } 2290 } 2291 #endif 2292 #ifdef RTE_LIB_LATENCYSTATS 2293 if (latencystats_enabled != 0 && 2294 latencystats_lcore_id == rte_lcore_id()) 2295 rte_latencystats_update(); 2296 #endif 2297 if (record_core_cycles) { 2298 uint64_t tsc = rte_rdtsc(); 2299 2300 fc->total_cycles += tsc - prev_tsc; 2301 prev_tsc = tsc; 2302 } 2303 } while (! fc->stopped); 2304 } 2305 2306 static int 2307 lcore_usage_callback(unsigned int lcore_id, struct rte_lcore_usage *usage) 2308 { 2309 struct fwd_stream **fsm; 2310 struct fwd_lcore *fc; 2311 streamid_t nb_fs; 2312 streamid_t sm_id; 2313 2314 fc = lcore_to_fwd_lcore(lcore_id); 2315 if (fc == NULL) 2316 return -1; 2317 2318 fsm = &fwd_streams[fc->stream_idx]; 2319 nb_fs = fc->stream_nb; 2320 usage->busy_cycles = 0; 2321 usage->total_cycles = fc->total_cycles; 2322 2323 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2324 if (!fsm[sm_id]->disabled) 2325 usage->busy_cycles += fsm[sm_id]->busy_cycles; 2326 } 2327 2328 return 0; 2329 } 2330 2331 static int 2332 start_pkt_forward_on_core(void *fwd_arg) 2333 { 2334 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2335 cur_fwd_config.fwd_eng->packet_fwd); 2336 return 0; 2337 } 2338 2339 /* 2340 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2341 * Used to start communication flows in network loopback test configurations. 2342 */ 2343 static int 2344 run_one_txonly_burst_on_core(void *fwd_arg) 2345 { 2346 struct fwd_lcore *fwd_lc; 2347 struct fwd_lcore tmp_lcore; 2348 2349 fwd_lc = (struct fwd_lcore *) fwd_arg; 2350 tmp_lcore = *fwd_lc; 2351 tmp_lcore.stopped = 1; 2352 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2353 return 0; 2354 } 2355 2356 /* 2357 * Launch packet forwarding: 2358 * - Setup per-port forwarding context. 2359 * - launch logical cores with their forwarding configuration. 2360 */ 2361 static void 2362 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2363 { 2364 unsigned int i; 2365 unsigned int lc_id; 2366 int diag; 2367 2368 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2369 lc_id = fwd_lcores_cpuids[i]; 2370 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2371 fwd_lcores[i]->stopped = 0; 2372 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2373 fwd_lcores[i], lc_id); 2374 if (diag != 0) 2375 fprintf(stderr, 2376 "launch lcore %u failed - diag=%d\n", 2377 lc_id, diag); 2378 } 2379 } 2380 } 2381 2382 /* 2383 * Launch packet forwarding configuration. 2384 */ 2385 void 2386 start_packet_forwarding(int with_tx_first) 2387 { 2388 port_fwd_begin_t port_fwd_begin; 2389 port_fwd_end_t port_fwd_end; 2390 stream_init_t stream_init = cur_fwd_eng->stream_init; 2391 unsigned int i; 2392 2393 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2394 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2395 2396 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2397 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2398 2399 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2400 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2401 (!nb_rxq || !nb_txq)) 2402 rte_exit(EXIT_FAILURE, 2403 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2404 cur_fwd_eng->fwd_mode_name); 2405 2406 if (all_ports_started() == 0) { 2407 fprintf(stderr, "Not all ports were started\n"); 2408 return; 2409 } 2410 if (test_done == 0) { 2411 fprintf(stderr, "Packet forwarding already started\n"); 2412 return; 2413 } 2414 2415 fwd_config_setup(); 2416 2417 pkt_fwd_config_display(&cur_fwd_config); 2418 if (!pkt_fwd_shared_rxq_check()) 2419 return; 2420 2421 if (stream_init != NULL) 2422 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2423 stream_init(fwd_streams[i]); 2424 2425 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2426 if (port_fwd_begin != NULL) { 2427 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2428 if (port_fwd_begin(fwd_ports_ids[i])) { 2429 fprintf(stderr, 2430 "Packet forwarding is not ready\n"); 2431 return; 2432 } 2433 } 2434 } 2435 2436 if (with_tx_first) { 2437 port_fwd_begin = tx_only_engine.port_fwd_begin; 2438 if (port_fwd_begin != NULL) { 2439 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2440 if (port_fwd_begin(fwd_ports_ids[i])) { 2441 fprintf(stderr, 2442 "Packet forwarding is not ready\n"); 2443 return; 2444 } 2445 } 2446 } 2447 } 2448 2449 test_done = 0; 2450 2451 if(!no_flush_rx) 2452 flush_fwd_rx_queues(); 2453 2454 rxtx_config_display(); 2455 2456 fwd_stats_reset(); 2457 if (with_tx_first) { 2458 while (with_tx_first--) { 2459 launch_packet_forwarding( 2460 run_one_txonly_burst_on_core); 2461 rte_eal_mp_wait_lcore(); 2462 } 2463 port_fwd_end = tx_only_engine.port_fwd_end; 2464 if (port_fwd_end != NULL) { 2465 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2466 (*port_fwd_end)(fwd_ports_ids[i]); 2467 } 2468 } 2469 launch_packet_forwarding(start_pkt_forward_on_core); 2470 } 2471 2472 void 2473 stop_packet_forwarding(void) 2474 { 2475 port_fwd_end_t port_fwd_end; 2476 lcoreid_t lc_id; 2477 portid_t pt_id; 2478 int i; 2479 2480 if (test_done) { 2481 fprintf(stderr, "Packet forwarding not started\n"); 2482 return; 2483 } 2484 printf("Telling cores to stop..."); 2485 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2486 fwd_lcores[lc_id]->stopped = 1; 2487 printf("\nWaiting for lcores to finish...\n"); 2488 rte_eal_mp_wait_lcore(); 2489 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2490 if (port_fwd_end != NULL) { 2491 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2492 pt_id = fwd_ports_ids[i]; 2493 (*port_fwd_end)(pt_id); 2494 } 2495 } 2496 2497 fwd_stats_display(); 2498 2499 printf("\nDone.\n"); 2500 test_done = 1; 2501 } 2502 2503 void 2504 dev_set_link_up(portid_t pid) 2505 { 2506 if (rte_eth_dev_set_link_up(pid) < 0) 2507 fprintf(stderr, "\nSet link up fail.\n"); 2508 } 2509 2510 void 2511 dev_set_link_down(portid_t pid) 2512 { 2513 if (rte_eth_dev_set_link_down(pid) < 0) 2514 fprintf(stderr, "\nSet link down fail.\n"); 2515 } 2516 2517 static int 2518 all_ports_started(void) 2519 { 2520 portid_t pi; 2521 struct rte_port *port; 2522 2523 RTE_ETH_FOREACH_DEV(pi) { 2524 port = &ports[pi]; 2525 /* Check if there is a port which is not started */ 2526 if ((port->port_status != RTE_PORT_STARTED) && 2527 (port->slave_flag == 0)) 2528 return 0; 2529 } 2530 2531 /* No port is not started */ 2532 return 1; 2533 } 2534 2535 int 2536 port_is_stopped(portid_t port_id) 2537 { 2538 struct rte_port *port = &ports[port_id]; 2539 2540 if ((port->port_status != RTE_PORT_STOPPED) && 2541 (port->slave_flag == 0)) 2542 return 0; 2543 return 1; 2544 } 2545 2546 int 2547 all_ports_stopped(void) 2548 { 2549 portid_t pi; 2550 2551 RTE_ETH_FOREACH_DEV(pi) { 2552 if (!port_is_stopped(pi)) 2553 return 0; 2554 } 2555 2556 return 1; 2557 } 2558 2559 int 2560 port_is_started(portid_t port_id) 2561 { 2562 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2563 return 0; 2564 2565 if (ports[port_id].port_status != RTE_PORT_STARTED) 2566 return 0; 2567 2568 return 1; 2569 } 2570 2571 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2572 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2573 2574 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2575 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2576 2577 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2578 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2579 2580 2581 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2582 static int 2583 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2584 { 2585 queueid_t qi; 2586 struct rte_eth_hairpin_conf hairpin_conf = { 2587 .peer_count = 1, 2588 }; 2589 int i; 2590 int diag; 2591 struct rte_port *port = &ports[pi]; 2592 uint16_t peer_rx_port = pi; 2593 uint16_t peer_tx_port = pi; 2594 uint32_t manual = 1; 2595 uint32_t tx_exp = hairpin_mode & 0x10; 2596 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2597 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2598 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2599 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2600 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2601 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2602 2603 if (!(hairpin_mode & 0xf)) { 2604 peer_rx_port = pi; 2605 peer_tx_port = pi; 2606 manual = 0; 2607 } else if (hairpin_mode & 0x1) { 2608 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2609 RTE_ETH_DEV_NO_OWNER); 2610 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2611 peer_tx_port = rte_eth_find_next_owned_by(0, 2612 RTE_ETH_DEV_NO_OWNER); 2613 if (p_pi != RTE_MAX_ETHPORTS) { 2614 peer_rx_port = p_pi; 2615 } else { 2616 uint16_t next_pi; 2617 2618 /* Last port will be the peer RX port of the first. */ 2619 RTE_ETH_FOREACH_DEV(next_pi) 2620 peer_rx_port = next_pi; 2621 } 2622 manual = 1; 2623 } else if (hairpin_mode & 0x2) { 2624 if (cnt_pi & 0x1) { 2625 peer_rx_port = p_pi; 2626 } else { 2627 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2628 RTE_ETH_DEV_NO_OWNER); 2629 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2630 peer_rx_port = pi; 2631 } 2632 peer_tx_port = peer_rx_port; 2633 manual = 1; 2634 } 2635 2636 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2637 hairpin_conf.peers[0].port = peer_rx_port; 2638 hairpin_conf.peers[0].queue = i + nb_rxq; 2639 hairpin_conf.manual_bind = !!manual; 2640 hairpin_conf.tx_explicit = !!tx_exp; 2641 hairpin_conf.force_memory = !!tx_force_memory; 2642 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2643 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2644 diag = rte_eth_tx_hairpin_queue_setup 2645 (pi, qi, nb_txd, &hairpin_conf); 2646 i++; 2647 if (diag == 0) 2648 continue; 2649 2650 /* Fail to setup rx queue, return */ 2651 if (port->port_status == RTE_PORT_HANDLING) 2652 port->port_status = RTE_PORT_STOPPED; 2653 else 2654 fprintf(stderr, 2655 "Port %d can not be set back to stopped\n", pi); 2656 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2657 pi); 2658 /* try to reconfigure queues next time */ 2659 port->need_reconfig_queues = 1; 2660 return -1; 2661 } 2662 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2663 hairpin_conf.peers[0].port = peer_tx_port; 2664 hairpin_conf.peers[0].queue = i + nb_txq; 2665 hairpin_conf.manual_bind = !!manual; 2666 hairpin_conf.tx_explicit = !!tx_exp; 2667 hairpin_conf.force_memory = !!rx_force_memory; 2668 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2669 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2670 diag = rte_eth_rx_hairpin_queue_setup 2671 (pi, qi, nb_rxd, &hairpin_conf); 2672 i++; 2673 if (diag == 0) 2674 continue; 2675 2676 /* Fail to setup rx queue, return */ 2677 if (port->port_status == RTE_PORT_HANDLING) 2678 port->port_status = RTE_PORT_STOPPED; 2679 else 2680 fprintf(stderr, 2681 "Port %d can not be set back to stopped\n", pi); 2682 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2683 pi); 2684 /* try to reconfigure queues next time */ 2685 port->need_reconfig_queues = 1; 2686 return -1; 2687 } 2688 return 0; 2689 } 2690 2691 /* Configure the Rx with optional split. */ 2692 int 2693 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2694 uint16_t nb_rx_desc, unsigned int socket_id, 2695 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2696 { 2697 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2698 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2699 struct rte_mempool *mpx; 2700 unsigned int i, mp_n; 2701 uint32_t prev_hdrs = 0; 2702 int ret; 2703 2704 2705 if ((rx_pkt_nb_segs > 1) && 2706 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2707 /* multi-segment configuration */ 2708 for (i = 0; i < rx_pkt_nb_segs; i++) { 2709 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2710 /* 2711 * Use last valid pool for the segments with number 2712 * exceeding the pool index. 2713 */ 2714 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2715 mpx = mbuf_pool_find(socket_id, mp_n); 2716 /* Handle zero as mbuf data buffer size. */ 2717 rx_seg->offset = i < rx_pkt_nb_offs ? 2718 rx_pkt_seg_offsets[i] : 0; 2719 rx_seg->mp = mpx ? mpx : mp; 2720 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2721 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2722 prev_hdrs |= rx_seg->proto_hdr; 2723 } else { 2724 rx_seg->length = rx_pkt_seg_lengths[i] ? 2725 rx_pkt_seg_lengths[i] : 2726 mbuf_data_size[mp_n]; 2727 } 2728 } 2729 rx_conf->rx_nseg = rx_pkt_nb_segs; 2730 rx_conf->rx_seg = rx_useg; 2731 rx_conf->rx_mempools = NULL; 2732 rx_conf->rx_nmempool = 0; 2733 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2734 socket_id, rx_conf, NULL); 2735 rx_conf->rx_seg = NULL; 2736 rx_conf->rx_nseg = 0; 2737 } else if (multi_rx_mempool == 1) { 2738 /* multi-pool configuration */ 2739 struct rte_eth_dev_info dev_info; 2740 2741 if (mbuf_data_size_n <= 1) { 2742 fprintf(stderr, "Invalid number of mempools %u\n", 2743 mbuf_data_size_n); 2744 return -EINVAL; 2745 } 2746 ret = rte_eth_dev_info_get(port_id, &dev_info); 2747 if (ret != 0) 2748 return ret; 2749 if (dev_info.max_rx_mempools == 0) { 2750 fprintf(stderr, 2751 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2752 port_id); 2753 return -ENOTSUP; 2754 } 2755 for (i = 0; i < mbuf_data_size_n; i++) { 2756 mpx = mbuf_pool_find(socket_id, i); 2757 rx_mempool[i] = mpx ? mpx : mp; 2758 } 2759 rx_conf->rx_mempools = rx_mempool; 2760 rx_conf->rx_nmempool = mbuf_data_size_n; 2761 rx_conf->rx_seg = NULL; 2762 rx_conf->rx_nseg = 0; 2763 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2764 socket_id, rx_conf, NULL); 2765 rx_conf->rx_mempools = NULL; 2766 rx_conf->rx_nmempool = 0; 2767 } else { 2768 /* Single pool/segment configuration */ 2769 rx_conf->rx_seg = NULL; 2770 rx_conf->rx_nseg = 0; 2771 rx_conf->rx_mempools = NULL; 2772 rx_conf->rx_nmempool = 0; 2773 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2774 socket_id, rx_conf, mp); 2775 } 2776 2777 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2778 RTE_ETH_QUEUE_STATE_STOPPED : 2779 RTE_ETH_QUEUE_STATE_STARTED; 2780 return ret; 2781 } 2782 2783 static int 2784 alloc_xstats_display_info(portid_t pi) 2785 { 2786 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2787 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2788 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2789 2790 if (xstats_display_num == 0) 2791 return 0; 2792 2793 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2794 if (*ids_supp == NULL) 2795 goto fail_ids_supp; 2796 2797 *prev_values = calloc(xstats_display_num, 2798 sizeof(**prev_values)); 2799 if (*prev_values == NULL) 2800 goto fail_prev_values; 2801 2802 *curr_values = calloc(xstats_display_num, 2803 sizeof(**curr_values)); 2804 if (*curr_values == NULL) 2805 goto fail_curr_values; 2806 2807 ports[pi].xstats_info.allocated = true; 2808 2809 return 0; 2810 2811 fail_curr_values: 2812 free(*prev_values); 2813 fail_prev_values: 2814 free(*ids_supp); 2815 fail_ids_supp: 2816 return -ENOMEM; 2817 } 2818 2819 static void 2820 free_xstats_display_info(portid_t pi) 2821 { 2822 if (!ports[pi].xstats_info.allocated) 2823 return; 2824 free(ports[pi].xstats_info.ids_supp); 2825 free(ports[pi].xstats_info.prev_values); 2826 free(ports[pi].xstats_info.curr_values); 2827 ports[pi].xstats_info.allocated = false; 2828 } 2829 2830 /** Fill helper structures for specified port to show extended statistics. */ 2831 static void 2832 fill_xstats_display_info_for_port(portid_t pi) 2833 { 2834 unsigned int stat, stat_supp; 2835 const char *xstat_name; 2836 struct rte_port *port; 2837 uint64_t *ids_supp; 2838 int rc; 2839 2840 if (xstats_display_num == 0) 2841 return; 2842 2843 if (pi == (portid_t)RTE_PORT_ALL) { 2844 fill_xstats_display_info(); 2845 return; 2846 } 2847 2848 port = &ports[pi]; 2849 if (port->port_status != RTE_PORT_STARTED) 2850 return; 2851 2852 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2853 rte_exit(EXIT_FAILURE, 2854 "Failed to allocate xstats display memory\n"); 2855 2856 ids_supp = port->xstats_info.ids_supp; 2857 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2858 xstat_name = xstats_display[stat].name; 2859 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2860 ids_supp + stat_supp); 2861 if (rc != 0) { 2862 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2863 xstat_name, pi, stat); 2864 continue; 2865 } 2866 stat_supp++; 2867 } 2868 2869 port->xstats_info.ids_supp_sz = stat_supp; 2870 } 2871 2872 /** Fill helper structures for all ports to show extended statistics. */ 2873 static void 2874 fill_xstats_display_info(void) 2875 { 2876 portid_t pi; 2877 2878 if (xstats_display_num == 0) 2879 return; 2880 2881 RTE_ETH_FOREACH_DEV(pi) 2882 fill_xstats_display_info_for_port(pi); 2883 } 2884 2885 /* 2886 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2887 * device in dev_info is zero when no slave is added. And its capability 2888 * will be updated when add a new slave device. So adding a slave device need 2889 * to update the port configurations of bonding device. 2890 */ 2891 static void 2892 update_bonding_port_dev_conf(portid_t bond_pid) 2893 { 2894 #ifdef RTE_NET_BOND 2895 struct rte_port *port = &ports[bond_pid]; 2896 uint16_t i; 2897 int ret; 2898 2899 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2900 if (ret != 0) { 2901 fprintf(stderr, "Failed to get dev info for port = %u\n", 2902 bond_pid); 2903 return; 2904 } 2905 2906 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2907 port->dev_conf.txmode.offloads |= 2908 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2909 /* Apply Tx offloads configuration */ 2910 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2911 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2912 2913 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 2914 port->dev_info.flow_type_rss_offloads; 2915 #else 2916 RTE_SET_USED(bond_pid); 2917 #endif 2918 } 2919 2920 int 2921 start_port(portid_t pid) 2922 { 2923 int diag, need_check_link_status = -1; 2924 portid_t pi; 2925 portid_t p_pi = RTE_MAX_ETHPORTS; 2926 portid_t pl[RTE_MAX_ETHPORTS]; 2927 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2928 uint16_t cnt_pi = 0; 2929 uint16_t cfg_pi = 0; 2930 int peer_pi; 2931 queueid_t qi; 2932 struct rte_port *port; 2933 struct rte_eth_hairpin_cap cap; 2934 2935 if (port_id_is_invalid(pid, ENABLED_WARN)) 2936 return 0; 2937 2938 RTE_ETH_FOREACH_DEV(pi) { 2939 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2940 continue; 2941 2942 if (port_is_bonding_slave(pi)) { 2943 fprintf(stderr, 2944 "Please remove port %d from bonded device.\n", 2945 pi); 2946 continue; 2947 } 2948 2949 need_check_link_status = 0; 2950 port = &ports[pi]; 2951 if (port->port_status == RTE_PORT_STOPPED) 2952 port->port_status = RTE_PORT_HANDLING; 2953 else { 2954 fprintf(stderr, "Port %d is now not stopped\n", pi); 2955 continue; 2956 } 2957 2958 if (port->need_reconfig > 0) { 2959 struct rte_eth_conf dev_conf; 2960 int k; 2961 2962 port->need_reconfig = 0; 2963 2964 if (flow_isolate_all) { 2965 int ret = port_flow_isolate(pi, 1); 2966 if (ret) { 2967 fprintf(stderr, 2968 "Failed to apply isolated mode on port %d\n", 2969 pi); 2970 return -1; 2971 } 2972 } 2973 configure_rxtx_dump_callbacks(0); 2974 printf("Configuring Port %d (socket %u)\n", pi, 2975 port->socket_id); 2976 if (nb_hairpinq > 0 && 2977 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2978 fprintf(stderr, 2979 "Port %d doesn't support hairpin queues\n", 2980 pi); 2981 return -1; 2982 } 2983 2984 if (port->bond_flag == 1 && port->update_conf == 1) { 2985 update_bonding_port_dev_conf(pi); 2986 port->update_conf = 0; 2987 } 2988 2989 /* configure port */ 2990 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2991 nb_txq + nb_hairpinq, 2992 &(port->dev_conf)); 2993 if (diag != 0) { 2994 if (port->port_status == RTE_PORT_HANDLING) 2995 port->port_status = RTE_PORT_STOPPED; 2996 else 2997 fprintf(stderr, 2998 "Port %d can not be set back to stopped\n", 2999 pi); 3000 fprintf(stderr, "Fail to configure port %d\n", 3001 pi); 3002 /* try to reconfigure port next time */ 3003 port->need_reconfig = 1; 3004 return -1; 3005 } 3006 /* get device configuration*/ 3007 if (0 != 3008 eth_dev_conf_get_print_err(pi, &dev_conf)) { 3009 fprintf(stderr, 3010 "port %d can not get device configuration\n", 3011 pi); 3012 return -1; 3013 } 3014 /* Apply Rx offloads configuration */ 3015 if (dev_conf.rxmode.offloads != 3016 port->dev_conf.rxmode.offloads) { 3017 port->dev_conf.rxmode.offloads |= 3018 dev_conf.rxmode.offloads; 3019 for (k = 0; 3020 k < port->dev_info.max_rx_queues; 3021 k++) 3022 port->rxq[k].conf.offloads |= 3023 dev_conf.rxmode.offloads; 3024 } 3025 /* Apply Tx offloads configuration */ 3026 if (dev_conf.txmode.offloads != 3027 port->dev_conf.txmode.offloads) { 3028 port->dev_conf.txmode.offloads |= 3029 dev_conf.txmode.offloads; 3030 for (k = 0; 3031 k < port->dev_info.max_tx_queues; 3032 k++) 3033 port->txq[k].conf.offloads |= 3034 dev_conf.txmode.offloads; 3035 } 3036 } 3037 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 3038 port->need_reconfig_queues = 0; 3039 /* setup tx queues */ 3040 for (qi = 0; qi < nb_txq; qi++) { 3041 struct rte_eth_txconf *conf = 3042 &port->txq[qi].conf; 3043 3044 if ((numa_support) && 3045 (txring_numa[pi] != NUMA_NO_CONFIG)) 3046 diag = rte_eth_tx_queue_setup(pi, qi, 3047 port->nb_tx_desc[qi], 3048 txring_numa[pi], 3049 &(port->txq[qi].conf)); 3050 else 3051 diag = rte_eth_tx_queue_setup(pi, qi, 3052 port->nb_tx_desc[qi], 3053 port->socket_id, 3054 &(port->txq[qi].conf)); 3055 3056 if (diag == 0) { 3057 port->txq[qi].state = 3058 conf->tx_deferred_start ? 3059 RTE_ETH_QUEUE_STATE_STOPPED : 3060 RTE_ETH_QUEUE_STATE_STARTED; 3061 continue; 3062 } 3063 3064 /* Fail to setup tx queue, return */ 3065 if (port->port_status == RTE_PORT_HANDLING) 3066 port->port_status = RTE_PORT_STOPPED; 3067 else 3068 fprintf(stderr, 3069 "Port %d can not be set back to stopped\n", 3070 pi); 3071 fprintf(stderr, 3072 "Fail to configure port %d tx queues\n", 3073 pi); 3074 /* try to reconfigure queues next time */ 3075 port->need_reconfig_queues = 1; 3076 return -1; 3077 } 3078 for (qi = 0; qi < nb_rxq; qi++) { 3079 /* setup rx queues */ 3080 if ((numa_support) && 3081 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3082 struct rte_mempool * mp = 3083 mbuf_pool_find 3084 (rxring_numa[pi], 0); 3085 if (mp == NULL) { 3086 fprintf(stderr, 3087 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3088 rxring_numa[pi]); 3089 return -1; 3090 } 3091 3092 diag = rx_queue_setup(pi, qi, 3093 port->nb_rx_desc[qi], 3094 rxring_numa[pi], 3095 &(port->rxq[qi].conf), 3096 mp); 3097 } else { 3098 struct rte_mempool *mp = 3099 mbuf_pool_find 3100 (port->socket_id, 0); 3101 if (mp == NULL) { 3102 fprintf(stderr, 3103 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3104 port->socket_id); 3105 return -1; 3106 } 3107 diag = rx_queue_setup(pi, qi, 3108 port->nb_rx_desc[qi], 3109 port->socket_id, 3110 &(port->rxq[qi].conf), 3111 mp); 3112 } 3113 if (diag == 0) 3114 continue; 3115 3116 /* Fail to setup rx queue, return */ 3117 if (port->port_status == RTE_PORT_HANDLING) 3118 port->port_status = RTE_PORT_STOPPED; 3119 else 3120 fprintf(stderr, 3121 "Port %d can not be set back to stopped\n", 3122 pi); 3123 fprintf(stderr, 3124 "Fail to configure port %d rx queues\n", 3125 pi); 3126 /* try to reconfigure queues next time */ 3127 port->need_reconfig_queues = 1; 3128 return -1; 3129 } 3130 /* setup hairpin queues */ 3131 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3132 return -1; 3133 } 3134 configure_rxtx_dump_callbacks(verbose_level); 3135 if (clear_ptypes) { 3136 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3137 NULL, 0); 3138 if (diag < 0) 3139 fprintf(stderr, 3140 "Port %d: Failed to disable Ptype parsing\n", 3141 pi); 3142 } 3143 3144 p_pi = pi; 3145 cnt_pi++; 3146 3147 /* start port */ 3148 diag = eth_dev_start_mp(pi); 3149 if (diag < 0) { 3150 fprintf(stderr, "Fail to start port %d: %s\n", 3151 pi, rte_strerror(-diag)); 3152 3153 /* Fail to setup rx queue, return */ 3154 if (port->port_status == RTE_PORT_HANDLING) 3155 port->port_status = RTE_PORT_STOPPED; 3156 else 3157 fprintf(stderr, 3158 "Port %d can not be set back to stopped\n", 3159 pi); 3160 continue; 3161 } 3162 3163 if (port->port_status == RTE_PORT_HANDLING) 3164 port->port_status = RTE_PORT_STARTED; 3165 else 3166 fprintf(stderr, "Port %d can not be set into started\n", 3167 pi); 3168 3169 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3170 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3171 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3172 3173 /* at least one port started, need checking link status */ 3174 need_check_link_status = 1; 3175 3176 pl[cfg_pi++] = pi; 3177 } 3178 3179 if (need_check_link_status == 1 && !no_link_check) 3180 check_all_ports_link_status(RTE_PORT_ALL); 3181 else if (need_check_link_status == 0) 3182 fprintf(stderr, "Please stop the ports first\n"); 3183 3184 if (hairpin_mode & 0xf) { 3185 uint16_t i; 3186 int j; 3187 3188 /* bind all started hairpin ports */ 3189 for (i = 0; i < cfg_pi; i++) { 3190 pi = pl[i]; 3191 /* bind current Tx to all peer Rx */ 3192 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3193 RTE_MAX_ETHPORTS, 1); 3194 if (peer_pi < 0) 3195 return peer_pi; 3196 for (j = 0; j < peer_pi; j++) { 3197 if (!port_is_started(peer_pl[j])) 3198 continue; 3199 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3200 if (diag < 0) { 3201 fprintf(stderr, 3202 "Error during binding hairpin Tx port %u to %u: %s\n", 3203 pi, peer_pl[j], 3204 rte_strerror(-diag)); 3205 return -1; 3206 } 3207 } 3208 /* bind all peer Tx to current Rx */ 3209 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3210 RTE_MAX_ETHPORTS, 0); 3211 if (peer_pi < 0) 3212 return peer_pi; 3213 for (j = 0; j < peer_pi; j++) { 3214 if (!port_is_started(peer_pl[j])) 3215 continue; 3216 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3217 if (diag < 0) { 3218 fprintf(stderr, 3219 "Error during binding hairpin Tx port %u to %u: %s\n", 3220 peer_pl[j], pi, 3221 rte_strerror(-diag)); 3222 return -1; 3223 } 3224 } 3225 } 3226 } 3227 3228 fill_xstats_display_info_for_port(pid); 3229 3230 printf("Done\n"); 3231 return 0; 3232 } 3233 3234 void 3235 stop_port(portid_t pid) 3236 { 3237 portid_t pi; 3238 struct rte_port *port; 3239 int need_check_link_status = 0; 3240 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3241 int peer_pi; 3242 int ret; 3243 3244 if (port_id_is_invalid(pid, ENABLED_WARN)) 3245 return; 3246 3247 printf("Stopping ports...\n"); 3248 3249 RTE_ETH_FOREACH_DEV(pi) { 3250 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3251 continue; 3252 3253 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3254 fprintf(stderr, 3255 "Please remove port %d from forwarding configuration.\n", 3256 pi); 3257 continue; 3258 } 3259 3260 if (port_is_bonding_slave(pi)) { 3261 fprintf(stderr, 3262 "Please remove port %d from bonded device.\n", 3263 pi); 3264 continue; 3265 } 3266 3267 port = &ports[pi]; 3268 if (port->port_status == RTE_PORT_STARTED) 3269 port->port_status = RTE_PORT_HANDLING; 3270 else 3271 continue; 3272 3273 if (hairpin_mode & 0xf) { 3274 int j; 3275 3276 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3277 /* unbind all peer Tx from current Rx */ 3278 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3279 RTE_MAX_ETHPORTS, 0); 3280 if (peer_pi < 0) 3281 continue; 3282 for (j = 0; j < peer_pi; j++) { 3283 if (!port_is_started(peer_pl[j])) 3284 continue; 3285 rte_eth_hairpin_unbind(peer_pl[j], pi); 3286 } 3287 } 3288 3289 if (port->flow_list && !no_flow_flush) 3290 port_flow_flush(pi); 3291 3292 ret = eth_dev_stop_mp(pi); 3293 if (ret != 0) { 3294 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3295 pi); 3296 /* Allow to retry stopping the port. */ 3297 port->port_status = RTE_PORT_STARTED; 3298 continue; 3299 } 3300 3301 if (port->port_status == RTE_PORT_HANDLING) 3302 port->port_status = RTE_PORT_STOPPED; 3303 else 3304 fprintf(stderr, "Port %d can not be set into stopped\n", 3305 pi); 3306 need_check_link_status = 1; 3307 } 3308 if (need_check_link_status && !no_link_check) 3309 check_all_ports_link_status(RTE_PORT_ALL); 3310 3311 printf("Done\n"); 3312 } 3313 3314 static void 3315 remove_invalid_ports_in(portid_t *array, portid_t *total) 3316 { 3317 portid_t i; 3318 portid_t new_total = 0; 3319 3320 for (i = 0; i < *total; i++) 3321 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3322 array[new_total] = array[i]; 3323 new_total++; 3324 } 3325 *total = new_total; 3326 } 3327 3328 static void 3329 remove_invalid_ports(void) 3330 { 3331 remove_invalid_ports_in(ports_ids, &nb_ports); 3332 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3333 nb_cfg_ports = nb_fwd_ports; 3334 } 3335 3336 static void 3337 flush_port_owned_resources(portid_t pi) 3338 { 3339 mcast_addr_pool_destroy(pi); 3340 port_flow_flush(pi); 3341 port_flex_item_flush(pi); 3342 port_flow_template_table_flush(pi); 3343 port_flow_pattern_template_flush(pi); 3344 port_flow_actions_template_flush(pi); 3345 port_action_handle_flush(pi); 3346 } 3347 3348 static void 3349 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3350 { 3351 struct rte_port *port; 3352 portid_t slave_pid; 3353 uint16_t i; 3354 3355 for (i = 0; i < num_slaves; i++) { 3356 slave_pid = slave_pids[i]; 3357 if (port_is_started(slave_pid) == 1) { 3358 if (rte_eth_dev_stop(slave_pid) != 0) 3359 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3360 slave_pid); 3361 3362 port = &ports[slave_pid]; 3363 port->port_status = RTE_PORT_STOPPED; 3364 } 3365 3366 clear_port_slave_flag(slave_pid); 3367 3368 /* Close slave device when testpmd quit or is killed. */ 3369 if (cl_quit == 1 || f_quit == 1) 3370 rte_eth_dev_close(slave_pid); 3371 } 3372 } 3373 3374 void 3375 close_port(portid_t pid) 3376 { 3377 portid_t pi; 3378 struct rte_port *port; 3379 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3380 int num_slaves = 0; 3381 3382 if (port_id_is_invalid(pid, ENABLED_WARN)) 3383 return; 3384 3385 printf("Closing ports...\n"); 3386 3387 RTE_ETH_FOREACH_DEV(pi) { 3388 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3389 continue; 3390 3391 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3392 fprintf(stderr, 3393 "Please remove port %d from forwarding configuration.\n", 3394 pi); 3395 continue; 3396 } 3397 3398 if (port_is_bonding_slave(pi)) { 3399 fprintf(stderr, 3400 "Please remove port %d from bonded device.\n", 3401 pi); 3402 continue; 3403 } 3404 3405 port = &ports[pi]; 3406 if (port->port_status == RTE_PORT_CLOSED) { 3407 fprintf(stderr, "Port %d is already closed\n", pi); 3408 continue; 3409 } 3410 3411 if (is_proc_primary()) { 3412 flush_port_owned_resources(pi); 3413 #ifdef RTE_NET_BOND 3414 if (port->bond_flag == 1) 3415 num_slaves = rte_eth_bond_slaves_get(pi, 3416 slave_pids, RTE_MAX_ETHPORTS); 3417 #endif 3418 rte_eth_dev_close(pi); 3419 /* 3420 * If this port is bonded device, all slaves under the 3421 * device need to be removed or closed. 3422 */ 3423 if (port->bond_flag == 1 && num_slaves > 0) 3424 clear_bonding_slave_device(slave_pids, 3425 num_slaves); 3426 } 3427 3428 free_xstats_display_info(pi); 3429 } 3430 3431 remove_invalid_ports(); 3432 printf("Done\n"); 3433 } 3434 3435 void 3436 reset_port(portid_t pid) 3437 { 3438 int diag; 3439 portid_t pi; 3440 struct rte_port *port; 3441 3442 if (port_id_is_invalid(pid, ENABLED_WARN)) 3443 return; 3444 3445 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3446 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3447 fprintf(stderr, 3448 "Can not reset port(s), please stop port(s) first.\n"); 3449 return; 3450 } 3451 3452 printf("Resetting ports...\n"); 3453 3454 RTE_ETH_FOREACH_DEV(pi) { 3455 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3456 continue; 3457 3458 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3459 fprintf(stderr, 3460 "Please remove port %d from forwarding configuration.\n", 3461 pi); 3462 continue; 3463 } 3464 3465 if (port_is_bonding_slave(pi)) { 3466 fprintf(stderr, 3467 "Please remove port %d from bonded device.\n", 3468 pi); 3469 continue; 3470 } 3471 3472 if (is_proc_primary()) { 3473 diag = rte_eth_dev_reset(pi); 3474 if (diag == 0) { 3475 port = &ports[pi]; 3476 port->need_reconfig = 1; 3477 port->need_reconfig_queues = 1; 3478 } else { 3479 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3480 pi, diag); 3481 } 3482 } 3483 } 3484 3485 printf("Done\n"); 3486 } 3487 3488 void 3489 attach_port(char *identifier) 3490 { 3491 portid_t pi; 3492 struct rte_dev_iterator iterator; 3493 3494 printf("Attaching a new port...\n"); 3495 3496 if (identifier == NULL) { 3497 fprintf(stderr, "Invalid parameters are specified\n"); 3498 return; 3499 } 3500 3501 if (rte_dev_probe(identifier) < 0) { 3502 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3503 return; 3504 } 3505 3506 /* first attach mode: event */ 3507 if (setup_on_probe_event) { 3508 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3509 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3510 if (ports[pi].port_status == RTE_PORT_HANDLING && 3511 ports[pi].need_setup != 0) 3512 setup_attached_port(pi); 3513 return; 3514 } 3515 3516 /* second attach mode: iterator */ 3517 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3518 /* setup ports matching the devargs used for probing */ 3519 if (port_is_forwarding(pi)) 3520 continue; /* port was already attached before */ 3521 setup_attached_port(pi); 3522 } 3523 } 3524 3525 static void 3526 setup_attached_port(portid_t pi) 3527 { 3528 unsigned int socket_id; 3529 int ret; 3530 3531 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3532 /* if socket_id is invalid, set to the first available socket. */ 3533 if (check_socket_id(socket_id) < 0) 3534 socket_id = socket_ids[0]; 3535 reconfig(pi, socket_id); 3536 ret = rte_eth_promiscuous_enable(pi); 3537 if (ret != 0) 3538 fprintf(stderr, 3539 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3540 pi, rte_strerror(-ret)); 3541 3542 ports_ids[nb_ports++] = pi; 3543 fwd_ports_ids[nb_fwd_ports++] = pi; 3544 nb_cfg_ports = nb_fwd_ports; 3545 ports[pi].need_setup = 0; 3546 ports[pi].port_status = RTE_PORT_STOPPED; 3547 3548 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3549 printf("Done\n"); 3550 } 3551 3552 static void 3553 detach_device(struct rte_device *dev) 3554 { 3555 portid_t sibling; 3556 3557 if (dev == NULL) { 3558 fprintf(stderr, "Device already removed\n"); 3559 return; 3560 } 3561 3562 printf("Removing a device...\n"); 3563 3564 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3565 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3566 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3567 fprintf(stderr, "Port %u not stopped\n", 3568 sibling); 3569 return; 3570 } 3571 flush_port_owned_resources(sibling); 3572 } 3573 } 3574 3575 if (rte_dev_remove(dev) < 0) { 3576 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3577 return; 3578 } 3579 remove_invalid_ports(); 3580 3581 printf("Device is detached\n"); 3582 printf("Now total ports is %d\n", nb_ports); 3583 printf("Done\n"); 3584 return; 3585 } 3586 3587 void 3588 detach_port_device(portid_t port_id) 3589 { 3590 int ret; 3591 struct rte_eth_dev_info dev_info; 3592 3593 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3594 return; 3595 3596 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3597 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3598 fprintf(stderr, "Port not stopped\n"); 3599 return; 3600 } 3601 fprintf(stderr, "Port was not closed\n"); 3602 } 3603 3604 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3605 if (ret != 0) { 3606 TESTPMD_LOG(ERR, 3607 "Failed to get device info for port %d, not detaching\n", 3608 port_id); 3609 return; 3610 } 3611 detach_device(dev_info.device); 3612 } 3613 3614 void 3615 detach_devargs(char *identifier) 3616 { 3617 struct rte_dev_iterator iterator; 3618 struct rte_devargs da; 3619 portid_t port_id; 3620 3621 printf("Removing a device...\n"); 3622 3623 memset(&da, 0, sizeof(da)); 3624 if (rte_devargs_parsef(&da, "%s", identifier)) { 3625 fprintf(stderr, "cannot parse identifier\n"); 3626 return; 3627 } 3628 3629 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3630 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3631 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3632 fprintf(stderr, "Port %u not stopped\n", 3633 port_id); 3634 rte_eth_iterator_cleanup(&iterator); 3635 rte_devargs_reset(&da); 3636 return; 3637 } 3638 flush_port_owned_resources(port_id); 3639 } 3640 } 3641 3642 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3643 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3644 da.name, rte_bus_name(da.bus)); 3645 rte_devargs_reset(&da); 3646 return; 3647 } 3648 3649 remove_invalid_ports(); 3650 3651 printf("Device %s is detached\n", identifier); 3652 printf("Now total ports is %d\n", nb_ports); 3653 printf("Done\n"); 3654 rte_devargs_reset(&da); 3655 } 3656 3657 void 3658 pmd_test_exit(void) 3659 { 3660 portid_t pt_id; 3661 unsigned int i; 3662 int ret; 3663 3664 if (test_done == 0) 3665 stop_packet_forwarding(); 3666 3667 #ifndef RTE_EXEC_ENV_WINDOWS 3668 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3669 if (mempools[i]) { 3670 if (mp_alloc_type == MP_ALLOC_ANON) 3671 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3672 NULL); 3673 } 3674 } 3675 #endif 3676 if (ports != NULL) { 3677 no_link_check = 1; 3678 RTE_ETH_FOREACH_DEV(pt_id) { 3679 printf("\nStopping port %d...\n", pt_id); 3680 fflush(stdout); 3681 stop_port(pt_id); 3682 } 3683 RTE_ETH_FOREACH_DEV(pt_id) { 3684 printf("\nShutting down port %d...\n", pt_id); 3685 fflush(stdout); 3686 close_port(pt_id); 3687 } 3688 } 3689 3690 if (hot_plug) { 3691 ret = rte_dev_event_monitor_stop(); 3692 if (ret) { 3693 RTE_LOG(ERR, EAL, 3694 "fail to stop device event monitor."); 3695 return; 3696 } 3697 3698 ret = rte_dev_event_callback_unregister(NULL, 3699 dev_event_callback, NULL); 3700 if (ret < 0) { 3701 RTE_LOG(ERR, EAL, 3702 "fail to unregister device event callback.\n"); 3703 return; 3704 } 3705 3706 ret = rte_dev_hotplug_handle_disable(); 3707 if (ret) { 3708 RTE_LOG(ERR, EAL, 3709 "fail to disable hotplug handling.\n"); 3710 return; 3711 } 3712 } 3713 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3714 if (mempools[i]) 3715 mempool_free_mp(mempools[i]); 3716 } 3717 free(xstats_display); 3718 3719 printf("\nBye...\n"); 3720 } 3721 3722 typedef void (*cmd_func_t)(void); 3723 struct pmd_test_command { 3724 const char *cmd_name; 3725 cmd_func_t cmd_func; 3726 }; 3727 3728 /* Check the link status of all ports in up to 9s, and print them finally */ 3729 static void 3730 check_all_ports_link_status(uint32_t port_mask) 3731 { 3732 #define CHECK_INTERVAL 100 /* 100ms */ 3733 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3734 portid_t portid; 3735 uint8_t count, all_ports_up, print_flag = 0; 3736 struct rte_eth_link link; 3737 int ret; 3738 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3739 3740 printf("Checking link statuses...\n"); 3741 fflush(stdout); 3742 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3743 all_ports_up = 1; 3744 RTE_ETH_FOREACH_DEV(portid) { 3745 if ((port_mask & (1 << portid)) == 0) 3746 continue; 3747 memset(&link, 0, sizeof(link)); 3748 ret = rte_eth_link_get_nowait(portid, &link); 3749 if (ret < 0) { 3750 all_ports_up = 0; 3751 if (print_flag == 1) 3752 fprintf(stderr, 3753 "Port %u link get failed: %s\n", 3754 portid, rte_strerror(-ret)); 3755 continue; 3756 } 3757 /* print link status if flag set */ 3758 if (print_flag == 1) { 3759 rte_eth_link_to_str(link_status, 3760 sizeof(link_status), &link); 3761 printf("Port %d %s\n", portid, link_status); 3762 continue; 3763 } 3764 /* clear all_ports_up flag if any link down */ 3765 if (link.link_status == RTE_ETH_LINK_DOWN) { 3766 all_ports_up = 0; 3767 break; 3768 } 3769 } 3770 /* after finally printing all link status, get out */ 3771 if (print_flag == 1) 3772 break; 3773 3774 if (all_ports_up == 0) { 3775 fflush(stdout); 3776 rte_delay_ms(CHECK_INTERVAL); 3777 } 3778 3779 /* set the print_flag if all ports up or timeout */ 3780 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3781 print_flag = 1; 3782 } 3783 3784 if (lsc_interrupt) 3785 break; 3786 } 3787 } 3788 3789 static void 3790 rmv_port_callback(void *arg) 3791 { 3792 int need_to_start = 0; 3793 int org_no_link_check = no_link_check; 3794 portid_t port_id = (intptr_t)arg; 3795 struct rte_eth_dev_info dev_info; 3796 int ret; 3797 3798 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3799 3800 if (!test_done && port_is_forwarding(port_id)) { 3801 need_to_start = 1; 3802 stop_packet_forwarding(); 3803 } 3804 no_link_check = 1; 3805 stop_port(port_id); 3806 no_link_check = org_no_link_check; 3807 3808 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3809 if (ret != 0) 3810 TESTPMD_LOG(ERR, 3811 "Failed to get device info for port %d, not detaching\n", 3812 port_id); 3813 else { 3814 struct rte_device *device = dev_info.device; 3815 close_port(port_id); 3816 detach_device(device); /* might be already removed or have more ports */ 3817 } 3818 if (need_to_start) 3819 start_packet_forwarding(0); 3820 } 3821 3822 /* This function is used by the interrupt thread */ 3823 static int 3824 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3825 void *ret_param) 3826 { 3827 RTE_SET_USED(param); 3828 RTE_SET_USED(ret_param); 3829 3830 if (type >= RTE_ETH_EVENT_MAX) { 3831 fprintf(stderr, 3832 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3833 port_id, __func__, type); 3834 fflush(stderr); 3835 } else if (event_print_mask & (UINT32_C(1) << type)) { 3836 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3837 eth_event_desc[type]); 3838 fflush(stdout); 3839 } 3840 3841 switch (type) { 3842 case RTE_ETH_EVENT_NEW: 3843 ports[port_id].need_setup = 1; 3844 ports[port_id].port_status = RTE_PORT_HANDLING; 3845 break; 3846 case RTE_ETH_EVENT_INTR_RMV: 3847 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3848 break; 3849 if (rte_eal_alarm_set(100000, 3850 rmv_port_callback, (void *)(intptr_t)port_id)) 3851 fprintf(stderr, 3852 "Could not set up deferred device removal\n"); 3853 break; 3854 case RTE_ETH_EVENT_DESTROY: 3855 ports[port_id].port_status = RTE_PORT_CLOSED; 3856 printf("Port %u is closed\n", port_id); 3857 break; 3858 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3859 uint16_t rxq_id; 3860 int ret; 3861 3862 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3863 for (rxq_id = 0; ; rxq_id++) { 3864 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3865 NULL); 3866 if (ret <= 0) 3867 break; 3868 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3869 port_id, rxq_id); 3870 3871 #ifdef RTE_NET_MLX5 3872 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3873 #endif 3874 } 3875 break; 3876 } 3877 default: 3878 break; 3879 } 3880 return 0; 3881 } 3882 3883 static int 3884 register_eth_event_callback(void) 3885 { 3886 int ret; 3887 enum rte_eth_event_type event; 3888 3889 for (event = RTE_ETH_EVENT_UNKNOWN; 3890 event < RTE_ETH_EVENT_MAX; event++) { 3891 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3892 event, 3893 eth_event_callback, 3894 NULL); 3895 if (ret != 0) { 3896 TESTPMD_LOG(ERR, "Failed to register callback for " 3897 "%s event\n", eth_event_desc[event]); 3898 return -1; 3899 } 3900 } 3901 3902 return 0; 3903 } 3904 3905 /* This function is used by the interrupt thread */ 3906 static void 3907 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3908 __rte_unused void *arg) 3909 { 3910 uint16_t port_id; 3911 int ret; 3912 3913 if (type >= RTE_DEV_EVENT_MAX) { 3914 fprintf(stderr, "%s called upon invalid event %d\n", 3915 __func__, type); 3916 fflush(stderr); 3917 } 3918 3919 switch (type) { 3920 case RTE_DEV_EVENT_REMOVE: 3921 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3922 device_name); 3923 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3924 if (ret) { 3925 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3926 device_name); 3927 return; 3928 } 3929 /* 3930 * Because the user's callback is invoked in eal interrupt 3931 * callback, the interrupt callback need to be finished before 3932 * it can be unregistered when detaching device. So finish 3933 * callback soon and use a deferred removal to detach device 3934 * is need. It is a workaround, once the device detaching be 3935 * moved into the eal in the future, the deferred removal could 3936 * be deleted. 3937 */ 3938 if (rte_eal_alarm_set(100000, 3939 rmv_port_callback, (void *)(intptr_t)port_id)) 3940 RTE_LOG(ERR, EAL, 3941 "Could not set up deferred device removal\n"); 3942 break; 3943 case RTE_DEV_EVENT_ADD: 3944 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3945 device_name); 3946 /* TODO: After finish kernel driver binding, 3947 * begin to attach port. 3948 */ 3949 break; 3950 default: 3951 break; 3952 } 3953 } 3954 3955 static void 3956 rxtx_port_config(portid_t pid) 3957 { 3958 uint16_t qid; 3959 uint64_t offloads; 3960 struct rte_port *port = &ports[pid]; 3961 3962 for (qid = 0; qid < nb_rxq; qid++) { 3963 offloads = port->rxq[qid].conf.offloads; 3964 port->rxq[qid].conf = port->dev_info.default_rxconf; 3965 3966 if (rxq_share > 0 && 3967 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3968 /* Non-zero share group to enable RxQ share. */ 3969 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3970 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3971 } 3972 3973 if (offloads != 0) 3974 port->rxq[qid].conf.offloads = offloads; 3975 3976 /* Check if any Rx parameters have been passed */ 3977 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3978 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3979 3980 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3981 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3982 3983 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3984 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3985 3986 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3987 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3988 3989 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3990 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3991 3992 port->nb_rx_desc[qid] = nb_rxd; 3993 } 3994 3995 for (qid = 0; qid < nb_txq; qid++) { 3996 offloads = port->txq[qid].conf.offloads; 3997 port->txq[qid].conf = port->dev_info.default_txconf; 3998 if (offloads != 0) 3999 port->txq[qid].conf.offloads = offloads; 4000 4001 /* Check if any Tx parameters have been passed */ 4002 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 4003 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 4004 4005 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 4006 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 4007 4008 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 4009 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 4010 4011 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 4012 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 4013 4014 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 4015 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 4016 4017 port->nb_tx_desc[qid] = nb_txd; 4018 } 4019 } 4020 4021 /* 4022 * Helper function to set MTU from frame size 4023 * 4024 * port->dev_info should be set before calling this function. 4025 * 4026 * return 0 on success, negative on error 4027 */ 4028 int 4029 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 4030 { 4031 struct rte_port *port = &ports[portid]; 4032 uint32_t eth_overhead; 4033 uint16_t mtu, new_mtu; 4034 4035 eth_overhead = get_eth_overhead(&port->dev_info); 4036 4037 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 4038 printf("Failed to get MTU for port %u\n", portid); 4039 return -1; 4040 } 4041 4042 new_mtu = max_rx_pktlen - eth_overhead; 4043 4044 if (mtu == new_mtu) 4045 return 0; 4046 4047 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 4048 fprintf(stderr, 4049 "Failed to set MTU to %u for port %u\n", 4050 new_mtu, portid); 4051 return -1; 4052 } 4053 4054 port->dev_conf.rxmode.mtu = new_mtu; 4055 4056 return 0; 4057 } 4058 4059 void 4060 init_port_config(void) 4061 { 4062 portid_t pid; 4063 struct rte_port *port; 4064 int ret, i; 4065 4066 RTE_ETH_FOREACH_DEV(pid) { 4067 port = &ports[pid]; 4068 4069 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4070 if (ret != 0) 4071 return; 4072 4073 if (nb_rxq > 1) { 4074 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4075 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4076 rss_hf & port->dev_info.flow_type_rss_offloads; 4077 } else { 4078 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4079 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4080 } 4081 4082 if (port->dcb_flag == 0) { 4083 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4084 port->dev_conf.rxmode.mq_mode = 4085 (enum rte_eth_rx_mq_mode) 4086 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4087 } else { 4088 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4089 port->dev_conf.rxmode.offloads &= 4090 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4091 4092 for (i = 0; 4093 i < port->dev_info.nb_rx_queues; 4094 i++) 4095 port->rxq[i].conf.offloads &= 4096 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4097 } 4098 } 4099 4100 rxtx_port_config(pid); 4101 4102 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4103 if (ret != 0) 4104 return; 4105 4106 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4107 port->dev_conf.intr_conf.lsc = 1; 4108 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4109 port->dev_conf.intr_conf.rmv = 1; 4110 } 4111 } 4112 4113 void set_port_slave_flag(portid_t slave_pid) 4114 { 4115 struct rte_port *port; 4116 4117 port = &ports[slave_pid]; 4118 port->slave_flag = 1; 4119 } 4120 4121 void clear_port_slave_flag(portid_t slave_pid) 4122 { 4123 struct rte_port *port; 4124 4125 port = &ports[slave_pid]; 4126 port->slave_flag = 0; 4127 } 4128 4129 uint8_t port_is_bonding_slave(portid_t slave_pid) 4130 { 4131 struct rte_port *port; 4132 struct rte_eth_dev_info dev_info; 4133 int ret; 4134 4135 port = &ports[slave_pid]; 4136 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4137 if (ret != 0) { 4138 TESTPMD_LOG(ERR, 4139 "Failed to get device info for port id %d," 4140 "cannot determine if the port is a bonded slave", 4141 slave_pid); 4142 return 0; 4143 } 4144 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4145 return 1; 4146 return 0; 4147 } 4148 4149 const uint16_t vlan_tags[] = { 4150 0, 1, 2, 3, 4, 5, 6, 7, 4151 8, 9, 10, 11, 12, 13, 14, 15, 4152 16, 17, 18, 19, 20, 21, 22, 23, 4153 24, 25, 26, 27, 28, 29, 30, 31 4154 }; 4155 4156 static int 4157 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4158 enum dcb_mode_enable dcb_mode, 4159 enum rte_eth_nb_tcs num_tcs, 4160 uint8_t pfc_en) 4161 { 4162 uint8_t i; 4163 int32_t rc; 4164 struct rte_eth_rss_conf rss_conf; 4165 4166 /* 4167 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4168 * given above, and the number of traffic classes available for use. 4169 */ 4170 if (dcb_mode == DCB_VT_ENABLED) { 4171 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4172 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4173 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4174 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4175 4176 /* VMDQ+DCB RX and TX configurations */ 4177 vmdq_rx_conf->enable_default_pool = 0; 4178 vmdq_rx_conf->default_pool = 0; 4179 vmdq_rx_conf->nb_queue_pools = 4180 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4181 vmdq_tx_conf->nb_queue_pools = 4182 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4183 4184 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4185 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4186 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4187 vmdq_rx_conf->pool_map[i].pools = 4188 1 << (i % vmdq_rx_conf->nb_queue_pools); 4189 } 4190 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4191 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4192 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4193 } 4194 4195 /* set DCB mode of RX and TX of multiple queues */ 4196 eth_conf->rxmode.mq_mode = 4197 (enum rte_eth_rx_mq_mode) 4198 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4199 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4200 } else { 4201 struct rte_eth_dcb_rx_conf *rx_conf = 4202 ð_conf->rx_adv_conf.dcb_rx_conf; 4203 struct rte_eth_dcb_tx_conf *tx_conf = 4204 ð_conf->tx_adv_conf.dcb_tx_conf; 4205 4206 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4207 4208 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4209 if (rc != 0) 4210 return rc; 4211 4212 rx_conf->nb_tcs = num_tcs; 4213 tx_conf->nb_tcs = num_tcs; 4214 4215 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4216 rx_conf->dcb_tc[i] = i % num_tcs; 4217 tx_conf->dcb_tc[i] = i % num_tcs; 4218 } 4219 4220 eth_conf->rxmode.mq_mode = 4221 (enum rte_eth_rx_mq_mode) 4222 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4223 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4224 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4225 } 4226 4227 if (pfc_en) 4228 eth_conf->dcb_capability_en = 4229 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4230 else 4231 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4232 4233 return 0; 4234 } 4235 4236 int 4237 init_port_dcb_config(portid_t pid, 4238 enum dcb_mode_enable dcb_mode, 4239 enum rte_eth_nb_tcs num_tcs, 4240 uint8_t pfc_en) 4241 { 4242 struct rte_eth_conf port_conf; 4243 struct rte_port *rte_port; 4244 int retval; 4245 uint16_t i; 4246 4247 if (num_procs > 1) { 4248 printf("The multi-process feature doesn't support dcb.\n"); 4249 return -ENOTSUP; 4250 } 4251 rte_port = &ports[pid]; 4252 4253 /* retain the original device configuration. */ 4254 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4255 4256 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4257 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4258 if (retval < 0) 4259 return retval; 4260 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4261 /* remove RSS HASH offload for DCB in vt mode */ 4262 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4263 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4264 for (i = 0; i < nb_rxq; i++) 4265 rte_port->rxq[i].conf.offloads &= 4266 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4267 } 4268 4269 /* re-configure the device . */ 4270 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4271 if (retval < 0) 4272 return retval; 4273 4274 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4275 if (retval != 0) 4276 return retval; 4277 4278 /* If dev_info.vmdq_pool_base is greater than 0, 4279 * the queue id of vmdq pools is started after pf queues. 4280 */ 4281 if (dcb_mode == DCB_VT_ENABLED && 4282 rte_port->dev_info.vmdq_pool_base > 0) { 4283 fprintf(stderr, 4284 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4285 pid); 4286 return -1; 4287 } 4288 4289 /* Assume the ports in testpmd have the same dcb capability 4290 * and has the same number of rxq and txq in dcb mode 4291 */ 4292 if (dcb_mode == DCB_VT_ENABLED) { 4293 if (rte_port->dev_info.max_vfs > 0) { 4294 nb_rxq = rte_port->dev_info.nb_rx_queues; 4295 nb_txq = rte_port->dev_info.nb_tx_queues; 4296 } else { 4297 nb_rxq = rte_port->dev_info.max_rx_queues; 4298 nb_txq = rte_port->dev_info.max_tx_queues; 4299 } 4300 } else { 4301 /*if vt is disabled, use all pf queues */ 4302 if (rte_port->dev_info.vmdq_pool_base == 0) { 4303 nb_rxq = rte_port->dev_info.max_rx_queues; 4304 nb_txq = rte_port->dev_info.max_tx_queues; 4305 } else { 4306 nb_rxq = (queueid_t)num_tcs; 4307 nb_txq = (queueid_t)num_tcs; 4308 4309 } 4310 } 4311 rx_free_thresh = 64; 4312 4313 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4314 4315 rxtx_port_config(pid); 4316 /* VLAN filter */ 4317 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4318 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4319 rx_vft_set(pid, vlan_tags[i], 1); 4320 4321 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4322 if (retval != 0) 4323 return retval; 4324 4325 rte_port->dcb_flag = 1; 4326 4327 /* Enter DCB configuration status */ 4328 dcb_config = 1; 4329 4330 return 0; 4331 } 4332 4333 static void 4334 init_port(void) 4335 { 4336 int i; 4337 4338 /* Configuration of Ethernet ports. */ 4339 ports = rte_zmalloc("testpmd: ports", 4340 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4341 RTE_CACHE_LINE_SIZE); 4342 if (ports == NULL) { 4343 rte_exit(EXIT_FAILURE, 4344 "rte_zmalloc(%d struct rte_port) failed\n", 4345 RTE_MAX_ETHPORTS); 4346 } 4347 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4348 ports[i].fwd_mac_swap = 1; 4349 ports[i].xstats_info.allocated = false; 4350 LIST_INIT(&ports[i].flow_tunnel_list); 4351 } 4352 /* Initialize ports NUMA structures */ 4353 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4354 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4355 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4356 } 4357 4358 static void 4359 force_quit(void) 4360 { 4361 pmd_test_exit(); 4362 prompt_exit(); 4363 } 4364 4365 static void 4366 print_stats(void) 4367 { 4368 uint8_t i; 4369 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4370 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4371 4372 /* Clear screen and move to top left */ 4373 printf("%s%s", clr, top_left); 4374 4375 printf("\nPort statistics ===================================="); 4376 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4377 nic_stats_display(fwd_ports_ids[i]); 4378 4379 fflush(stdout); 4380 } 4381 4382 static void 4383 signal_handler(int signum) 4384 { 4385 if (signum == SIGINT || signum == SIGTERM) { 4386 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4387 signum); 4388 #ifdef RTE_LIB_PDUMP 4389 /* uninitialize packet capture framework */ 4390 rte_pdump_uninit(); 4391 #endif 4392 #ifdef RTE_LIB_LATENCYSTATS 4393 if (latencystats_enabled != 0) 4394 rte_latencystats_uninit(); 4395 #endif 4396 force_quit(); 4397 /* Set flag to indicate the force termination. */ 4398 f_quit = 1; 4399 /* exit with the expected status */ 4400 #ifndef RTE_EXEC_ENV_WINDOWS 4401 signal(signum, SIG_DFL); 4402 kill(getpid(), signum); 4403 #endif 4404 } 4405 } 4406 4407 int 4408 main(int argc, char** argv) 4409 { 4410 int diag; 4411 portid_t port_id; 4412 uint16_t count; 4413 int ret; 4414 4415 signal(SIGINT, signal_handler); 4416 signal(SIGTERM, signal_handler); 4417 4418 testpmd_logtype = rte_log_register("testpmd"); 4419 if (testpmd_logtype < 0) 4420 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4421 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4422 4423 diag = rte_eal_init(argc, argv); 4424 if (diag < 0) 4425 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4426 rte_strerror(rte_errno)); 4427 4428 ret = register_eth_event_callback(); 4429 if (ret != 0) 4430 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4431 4432 #ifdef RTE_LIB_PDUMP 4433 /* initialize packet capture framework */ 4434 rte_pdump_init(); 4435 #endif 4436 4437 count = 0; 4438 RTE_ETH_FOREACH_DEV(port_id) { 4439 ports_ids[count] = port_id; 4440 count++; 4441 } 4442 nb_ports = (portid_t) count; 4443 if (nb_ports == 0) 4444 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4445 4446 /* allocate port structures, and init them */ 4447 init_port(); 4448 4449 set_def_fwd_config(); 4450 if (nb_lcores == 0) 4451 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4452 "Check the core mask argument\n"); 4453 4454 /* Bitrate/latency stats disabled by default */ 4455 #ifdef RTE_LIB_BITRATESTATS 4456 bitrate_enabled = 0; 4457 #endif 4458 #ifdef RTE_LIB_LATENCYSTATS 4459 latencystats_enabled = 0; 4460 #endif 4461 4462 /* on FreeBSD, mlockall() is disabled by default */ 4463 #ifdef RTE_EXEC_ENV_FREEBSD 4464 do_mlockall = 0; 4465 #else 4466 do_mlockall = 1; 4467 #endif 4468 4469 argc -= diag; 4470 argv += diag; 4471 if (argc > 1) 4472 launch_args_parse(argc, argv); 4473 4474 #ifndef RTE_EXEC_ENV_WINDOWS 4475 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4476 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4477 strerror(errno)); 4478 } 4479 #endif 4480 4481 if (tx_first && interactive) 4482 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4483 "interactive mode.\n"); 4484 4485 if (tx_first && lsc_interrupt) { 4486 fprintf(stderr, 4487 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4488 lsc_interrupt = 0; 4489 } 4490 4491 if (!nb_rxq && !nb_txq) 4492 fprintf(stderr, 4493 "Warning: Either rx or tx queues should be non-zero\n"); 4494 4495 if (nb_rxq > 1 && nb_rxq > nb_txq) 4496 fprintf(stderr, 4497 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4498 nb_rxq, nb_txq); 4499 4500 init_config(); 4501 4502 if (hot_plug) { 4503 ret = rte_dev_hotplug_handle_enable(); 4504 if (ret) { 4505 RTE_LOG(ERR, EAL, 4506 "fail to enable hotplug handling."); 4507 return -1; 4508 } 4509 4510 ret = rte_dev_event_monitor_start(); 4511 if (ret) { 4512 RTE_LOG(ERR, EAL, 4513 "fail to start device event monitoring."); 4514 return -1; 4515 } 4516 4517 ret = rte_dev_event_callback_register(NULL, 4518 dev_event_callback, NULL); 4519 if (ret) { 4520 RTE_LOG(ERR, EAL, 4521 "fail to register device event callback\n"); 4522 return -1; 4523 } 4524 } 4525 4526 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { 4527 if (!interactive) { 4528 rte_eal_cleanup(); 4529 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4530 } 4531 fprintf(stderr, "Start ports failed\n"); 4532 } 4533 4534 /* set all ports to promiscuous mode by default */ 4535 RTE_ETH_FOREACH_DEV(port_id) { 4536 ret = rte_eth_promiscuous_enable(port_id); 4537 if (ret != 0) 4538 fprintf(stderr, 4539 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4540 port_id, rte_strerror(-ret)); 4541 } 4542 4543 #ifdef RTE_LIB_METRICS 4544 /* Init metrics library */ 4545 rte_metrics_init(rte_socket_id()); 4546 #endif 4547 4548 #ifdef RTE_LIB_LATENCYSTATS 4549 if (latencystats_enabled != 0) { 4550 int ret = rte_latencystats_init(1, NULL); 4551 if (ret) 4552 fprintf(stderr, 4553 "Warning: latencystats init() returned error %d\n", 4554 ret); 4555 fprintf(stderr, "Latencystats running on lcore %d\n", 4556 latencystats_lcore_id); 4557 } 4558 #endif 4559 4560 /* Setup bitrate stats */ 4561 #ifdef RTE_LIB_BITRATESTATS 4562 if (bitrate_enabled != 0) { 4563 bitrate_data = rte_stats_bitrate_create(); 4564 if (bitrate_data == NULL) 4565 rte_exit(EXIT_FAILURE, 4566 "Could not allocate bitrate data.\n"); 4567 rte_stats_bitrate_reg(bitrate_data); 4568 } 4569 #endif 4570 4571 if (record_core_cycles) 4572 rte_lcore_register_usage_cb(lcore_usage_callback); 4573 4574 #ifdef RTE_LIB_CMDLINE 4575 if (init_cmdline() != 0) 4576 rte_exit(EXIT_FAILURE, 4577 "Could not initialise cmdline context.\n"); 4578 4579 if (strlen(cmdline_filename) != 0) 4580 cmdline_read_from_file(cmdline_filename); 4581 4582 if (interactive == 1) { 4583 if (auto_start) { 4584 printf("Start automatic packet forwarding\n"); 4585 start_packet_forwarding(0); 4586 } 4587 prompt(); 4588 pmd_test_exit(); 4589 } else 4590 #endif 4591 { 4592 char c; 4593 int rc; 4594 4595 f_quit = 0; 4596 4597 printf("No commandline core given, start packet forwarding\n"); 4598 start_packet_forwarding(tx_first); 4599 if (stats_period != 0) { 4600 uint64_t prev_time = 0, cur_time, diff_time = 0; 4601 uint64_t timer_period; 4602 4603 /* Convert to number of cycles */ 4604 timer_period = stats_period * rte_get_timer_hz(); 4605 4606 while (f_quit == 0) { 4607 cur_time = rte_get_timer_cycles(); 4608 diff_time += cur_time - prev_time; 4609 4610 if (diff_time >= timer_period) { 4611 print_stats(); 4612 /* Reset the timer */ 4613 diff_time = 0; 4614 } 4615 /* Sleep to avoid unnecessary checks */ 4616 prev_time = cur_time; 4617 rte_delay_us_sleep(US_PER_S); 4618 } 4619 } 4620 4621 printf("Press enter to exit\n"); 4622 rc = read(0, &c, 1); 4623 pmd_test_exit(); 4624 if (rc < 0) 4625 return 1; 4626 } 4627 4628 ret = rte_eal_cleanup(); 4629 if (ret != 0) 4630 rte_exit(EXIT_FAILURE, 4631 "EAL cleanup failed: %s\n", strerror(-ret)); 4632 4633 return EXIT_SUCCESS; 4634 } 4635