1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 static volatile uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 253 254 /* 255 * Configuration of packet segments used by the "txonly" processing engine. 256 */ 257 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 258 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 259 TXONLY_DEF_PACKET_LEN, 260 }; 261 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 262 263 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 264 /**< Split policy for packets to TX. */ 265 266 uint8_t txonly_multi_flow; 267 /**< Whether multiple flows are generated in TXONLY mode. */ 268 269 uint32_t tx_pkt_times_inter; 270 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 271 272 uint32_t tx_pkt_times_intra; 273 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 274 275 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 276 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 277 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 278 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 279 280 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 281 uint8_t dcb_config = 0; 282 283 /* 284 * Configurable number of RX/TX queues. 285 */ 286 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 287 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 288 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 289 290 /* 291 * Configurable number of RX/TX ring descriptors. 292 * Defaults are supplied by drivers via ethdev. 293 */ 294 #define RX_DESC_DEFAULT 0 295 #define TX_DESC_DEFAULT 0 296 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 297 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 298 299 #define RTE_PMD_PARAM_UNSET -1 300 /* 301 * Configurable values of RX and TX ring threshold registers. 302 */ 303 304 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 305 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 306 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 307 308 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 309 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 310 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 311 312 /* 313 * Configurable value of RX free threshold. 314 */ 315 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 316 317 /* 318 * Configurable value of RX drop enable. 319 */ 320 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 321 322 /* 323 * Configurable value of TX free threshold. 324 */ 325 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 326 327 /* 328 * Configurable value of TX RS bit threshold. 329 */ 330 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 331 332 /* 333 * Configurable value of buffered packets before sending. 334 */ 335 uint16_t noisy_tx_sw_bufsz; 336 337 /* 338 * Configurable value of packet buffer timeout. 339 */ 340 uint16_t noisy_tx_sw_buf_flush_time; 341 342 /* 343 * Configurable value for size of VNF internal memory area 344 * used for simulating noisy neighbour behaviour 345 */ 346 uint64_t noisy_lkup_mem_sz; 347 348 /* 349 * Configurable value of number of random writes done in 350 * VNF simulation memory area. 351 */ 352 uint64_t noisy_lkup_num_writes; 353 354 /* 355 * Configurable value of number of random reads done in 356 * VNF simulation memory area. 357 */ 358 uint64_t noisy_lkup_num_reads; 359 360 /* 361 * Configurable value of number of random reads/writes done in 362 * VNF simulation memory area. 363 */ 364 uint64_t noisy_lkup_num_reads_writes; 365 366 /* 367 * Receive Side Scaling (RSS) configuration. 368 */ 369 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 370 371 /* 372 * Port topology configuration 373 */ 374 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 375 376 /* 377 * Avoids to flush all the RX streams before starts forwarding. 378 */ 379 uint8_t no_flush_rx = 0; /* flush by default */ 380 381 /* 382 * Flow API isolated mode. 383 */ 384 uint8_t flow_isolate_all; 385 386 /* 387 * Disable port flow flush when stop port. 388 */ 389 uint8_t no_flow_flush = 0; /* do flow flush by default */ 390 391 /* 392 * Avoids to check link status when starting/stopping a port. 393 */ 394 uint8_t no_link_check = 0; /* check by default */ 395 396 /* 397 * Don't automatically start all ports in interactive mode. 398 */ 399 uint8_t no_device_start = 0; 400 401 /* 402 * Enable link status change notification 403 */ 404 uint8_t lsc_interrupt = 1; /* enabled by default */ 405 406 /* 407 * Enable device removal notification. 408 */ 409 uint8_t rmv_interrupt = 1; /* enabled by default */ 410 411 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 412 413 /* After attach, port setup is called on event or by iterator */ 414 bool setup_on_probe_event = true; 415 416 /* Clear ptypes on port initialization. */ 417 uint8_t clear_ptypes = true; 418 419 /* Hairpin ports configuration mode. */ 420 uint32_t hairpin_mode; 421 422 /* Pretty printing of ethdev events */ 423 static const char * const eth_event_desc[] = { 424 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 425 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 426 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 427 [RTE_ETH_EVENT_INTR_RESET] = "reset", 428 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 429 [RTE_ETH_EVENT_IPSEC] = "IPsec", 430 [RTE_ETH_EVENT_MACSEC] = "MACsec", 431 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 432 [RTE_ETH_EVENT_NEW] = "device probed", 433 [RTE_ETH_EVENT_DESTROY] = "device released", 434 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 435 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 436 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 437 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 438 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 439 [RTE_ETH_EVENT_MAX] = NULL, 440 }; 441 442 /* 443 * Display or mask ether events 444 * Default to all events except VF_MBOX 445 */ 446 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 447 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 448 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 449 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 450 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 451 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 452 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 453 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 454 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 455 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 456 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 457 /* 458 * Decide if all memory are locked for performance. 459 */ 460 int do_mlockall = 0; 461 462 #ifdef RTE_LIB_LATENCYSTATS 463 464 /* 465 * Set when latency stats is enabled in the commandline 466 */ 467 uint8_t latencystats_enabled; 468 469 /* 470 * Lcore ID to service latency statistics. 471 */ 472 lcoreid_t latencystats_lcore_id = -1; 473 474 #endif 475 476 /* 477 * Ethernet device configuration. 478 */ 479 struct rte_eth_rxmode rx_mode; 480 481 struct rte_eth_txmode tx_mode = { 482 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 483 }; 484 485 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 486 487 /* 488 * Display zero values by default for xstats 489 */ 490 uint8_t xstats_hide_zero; 491 492 /* 493 * Measure of CPU cycles disabled by default 494 */ 495 uint8_t record_core_cycles; 496 497 /* 498 * Display of RX and TX bursts disabled by default 499 */ 500 uint8_t record_burst_stats; 501 502 /* 503 * Number of ports per shared Rx queue group, 0 disable. 504 */ 505 uint32_t rxq_share; 506 507 unsigned int num_sockets = 0; 508 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 509 510 #ifdef RTE_LIB_BITRATESTATS 511 /* Bitrate statistics */ 512 struct rte_stats_bitrates *bitrate_data; 513 lcoreid_t bitrate_lcore_id; 514 uint8_t bitrate_enabled; 515 #endif 516 517 #ifdef RTE_LIB_GRO 518 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 519 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 520 #endif 521 522 /* 523 * hexadecimal bitmask of RX mq mode can be enabled. 524 */ 525 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 526 527 /* 528 * Used to set forced link speed 529 */ 530 uint32_t eth_link_speed; 531 532 /* 533 * ID of the current process in multi-process, used to 534 * configure the queues to be polled. 535 */ 536 int proc_id; 537 538 /* 539 * Number of processes in multi-process, used to 540 * configure the queues to be polled. 541 */ 542 unsigned int num_procs = 1; 543 544 static void 545 eth_rx_metadata_negotiate_mp(uint16_t port_id) 546 { 547 uint64_t rx_meta_features = 0; 548 int ret; 549 550 if (!is_proc_primary()) 551 return; 552 553 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 554 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 555 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 556 557 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 558 if (ret == 0) { 559 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 560 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 561 port_id); 562 } 563 564 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 565 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 566 port_id); 567 } 568 569 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 570 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 571 port_id); 572 } 573 } else if (ret != -ENOTSUP) { 574 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 575 port_id, rte_strerror(-ret)); 576 } 577 } 578 579 static int 580 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 581 const struct rte_eth_conf *dev_conf) 582 { 583 if (is_proc_primary()) 584 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 585 dev_conf); 586 return 0; 587 } 588 589 static int 590 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 591 { 592 #ifdef RTE_NET_BOND 593 594 portid_t slave_pids[RTE_MAX_ETHPORTS]; 595 struct rte_port *port; 596 int num_slaves; 597 portid_t slave_pid; 598 int i; 599 600 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 601 RTE_MAX_ETHPORTS); 602 if (num_slaves < 0) { 603 fprintf(stderr, "Failed to get slave list for port = %u\n", 604 bond_pid); 605 return num_slaves; 606 } 607 608 for (i = 0; i < num_slaves; i++) { 609 slave_pid = slave_pids[i]; 610 port = &ports[slave_pid]; 611 port->port_status = 612 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 613 } 614 #else 615 RTE_SET_USED(bond_pid); 616 RTE_SET_USED(is_stop); 617 #endif 618 return 0; 619 } 620 621 static int 622 eth_dev_start_mp(uint16_t port_id) 623 { 624 int ret; 625 626 if (is_proc_primary()) { 627 ret = rte_eth_dev_start(port_id); 628 if (ret != 0) 629 return ret; 630 631 struct rte_port *port = &ports[port_id]; 632 633 /* 634 * Starting a bonded port also starts all slaves under the bonded 635 * device. So if this port is bond device, we need to modify the 636 * port status of these slaves. 637 */ 638 if (port->bond_flag == 1) 639 return change_bonding_slave_port_status(port_id, false); 640 } 641 642 return 0; 643 } 644 645 static int 646 eth_dev_stop_mp(uint16_t port_id) 647 { 648 int ret; 649 650 if (is_proc_primary()) { 651 ret = rte_eth_dev_stop(port_id); 652 if (ret != 0) 653 return ret; 654 655 struct rte_port *port = &ports[port_id]; 656 657 /* 658 * Stopping a bonded port also stops all slaves under the bonded 659 * device. So if this port is bond device, we need to modify the 660 * port status of these slaves. 661 */ 662 if (port->bond_flag == 1) 663 return change_bonding_slave_port_status(port_id, true); 664 } 665 666 return 0; 667 } 668 669 static void 670 mempool_free_mp(struct rte_mempool *mp) 671 { 672 if (is_proc_primary()) 673 rte_mempool_free(mp); 674 } 675 676 static int 677 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 678 { 679 if (is_proc_primary()) 680 return rte_eth_dev_set_mtu(port_id, mtu); 681 682 return 0; 683 } 684 685 /* Forward function declarations */ 686 static void setup_attached_port(portid_t pi); 687 static void check_all_ports_link_status(uint32_t port_mask); 688 static int eth_event_callback(portid_t port_id, 689 enum rte_eth_event_type type, 690 void *param, void *ret_param); 691 static void dev_event_callback(const char *device_name, 692 enum rte_dev_event_type type, 693 void *param); 694 static void fill_xstats_display_info(void); 695 696 /* 697 * Check if all the ports are started. 698 * If yes, return positive value. If not, return zero. 699 */ 700 static int all_ports_started(void); 701 702 #ifdef RTE_LIB_GSO 703 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 704 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 705 #endif 706 707 /* Holds the registered mbuf dynamic flags names. */ 708 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 709 710 711 /* 712 * Helper function to check if socket is already discovered. 713 * If yes, return positive value. If not, return zero. 714 */ 715 int 716 new_socket_id(unsigned int socket_id) 717 { 718 unsigned int i; 719 720 for (i = 0; i < num_sockets; i++) { 721 if (socket_ids[i] == socket_id) 722 return 0; 723 } 724 return 1; 725 } 726 727 /* 728 * Setup default configuration. 729 */ 730 static void 731 set_default_fwd_lcores_config(void) 732 { 733 unsigned int i; 734 unsigned int nb_lc; 735 unsigned int sock_num; 736 737 nb_lc = 0; 738 for (i = 0; i < RTE_MAX_LCORE; i++) { 739 if (!rte_lcore_is_enabled(i)) 740 continue; 741 sock_num = rte_lcore_to_socket_id(i); 742 if (new_socket_id(sock_num)) { 743 if (num_sockets >= RTE_MAX_NUMA_NODES) { 744 rte_exit(EXIT_FAILURE, 745 "Total sockets greater than %u\n", 746 RTE_MAX_NUMA_NODES); 747 } 748 socket_ids[num_sockets++] = sock_num; 749 } 750 if (i == rte_get_main_lcore()) 751 continue; 752 fwd_lcores_cpuids[nb_lc++] = i; 753 } 754 nb_lcores = (lcoreid_t) nb_lc; 755 nb_cfg_lcores = nb_lcores; 756 nb_fwd_lcores = 1; 757 } 758 759 static void 760 set_def_peer_eth_addrs(void) 761 { 762 portid_t i; 763 764 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 765 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 766 peer_eth_addrs[i].addr_bytes[5] = i; 767 } 768 } 769 770 static void 771 set_default_fwd_ports_config(void) 772 { 773 portid_t pt_id; 774 int i = 0; 775 776 RTE_ETH_FOREACH_DEV(pt_id) { 777 fwd_ports_ids[i++] = pt_id; 778 779 /* Update sockets info according to the attached device */ 780 int socket_id = rte_eth_dev_socket_id(pt_id); 781 if (socket_id >= 0 && new_socket_id(socket_id)) { 782 if (num_sockets >= RTE_MAX_NUMA_NODES) { 783 rte_exit(EXIT_FAILURE, 784 "Total sockets greater than %u\n", 785 RTE_MAX_NUMA_NODES); 786 } 787 socket_ids[num_sockets++] = socket_id; 788 } 789 } 790 791 nb_cfg_ports = nb_ports; 792 nb_fwd_ports = nb_ports; 793 } 794 795 void 796 set_def_fwd_config(void) 797 { 798 set_default_fwd_lcores_config(); 799 set_def_peer_eth_addrs(); 800 set_default_fwd_ports_config(); 801 } 802 803 #ifndef RTE_EXEC_ENV_WINDOWS 804 /* extremely pessimistic estimation of memory required to create a mempool */ 805 static int 806 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 807 { 808 unsigned int n_pages, mbuf_per_pg, leftover; 809 uint64_t total_mem, mbuf_mem, obj_sz; 810 811 /* there is no good way to predict how much space the mempool will 812 * occupy because it will allocate chunks on the fly, and some of those 813 * will come from default DPDK memory while some will come from our 814 * external memory, so just assume 128MB will be enough for everyone. 815 */ 816 uint64_t hdr_mem = 128 << 20; 817 818 /* account for possible non-contiguousness */ 819 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 820 if (obj_sz > pgsz) { 821 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 822 return -1; 823 } 824 825 mbuf_per_pg = pgsz / obj_sz; 826 leftover = (nb_mbufs % mbuf_per_pg) > 0; 827 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 828 829 mbuf_mem = n_pages * pgsz; 830 831 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 832 833 if (total_mem > SIZE_MAX) { 834 TESTPMD_LOG(ERR, "Memory size too big\n"); 835 return -1; 836 } 837 *out = (size_t)total_mem; 838 839 return 0; 840 } 841 842 static int 843 pagesz_flags(uint64_t page_sz) 844 { 845 /* as per mmap() manpage, all page sizes are log2 of page size 846 * shifted by MAP_HUGE_SHIFT 847 */ 848 int log2 = rte_log2_u64(page_sz); 849 850 return (log2 << HUGE_SHIFT); 851 } 852 853 static void * 854 alloc_mem(size_t memsz, size_t pgsz, bool huge) 855 { 856 void *addr; 857 int flags; 858 859 /* allocate anonymous hugepages */ 860 flags = MAP_ANONYMOUS | MAP_PRIVATE; 861 if (huge) 862 flags |= HUGE_FLAG | pagesz_flags(pgsz); 863 864 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 865 if (addr == MAP_FAILED) 866 return NULL; 867 868 return addr; 869 } 870 871 struct extmem_param { 872 void *addr; 873 size_t len; 874 size_t pgsz; 875 rte_iova_t *iova_table; 876 unsigned int iova_table_len; 877 }; 878 879 static int 880 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 881 bool huge) 882 { 883 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 884 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 885 unsigned int cur_page, n_pages, pgsz_idx; 886 size_t mem_sz, cur_pgsz; 887 rte_iova_t *iovas = NULL; 888 void *addr; 889 int ret; 890 891 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 892 /* skip anything that is too big */ 893 if (pgsizes[pgsz_idx] > SIZE_MAX) 894 continue; 895 896 cur_pgsz = pgsizes[pgsz_idx]; 897 898 /* if we were told not to allocate hugepages, override */ 899 if (!huge) 900 cur_pgsz = sysconf(_SC_PAGESIZE); 901 902 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 903 if (ret < 0) { 904 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 905 return -1; 906 } 907 908 /* allocate our memory */ 909 addr = alloc_mem(mem_sz, cur_pgsz, huge); 910 911 /* if we couldn't allocate memory with a specified page size, 912 * that doesn't mean we can't do it with other page sizes, so 913 * try another one. 914 */ 915 if (addr == NULL) 916 continue; 917 918 /* store IOVA addresses for every page in this memory area */ 919 n_pages = mem_sz / cur_pgsz; 920 921 iovas = malloc(sizeof(*iovas) * n_pages); 922 923 if (iovas == NULL) { 924 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 925 goto fail; 926 } 927 /* lock memory if it's not huge pages */ 928 if (!huge) 929 mlock(addr, mem_sz); 930 931 /* populate IOVA addresses */ 932 for (cur_page = 0; cur_page < n_pages; cur_page++) { 933 rte_iova_t iova; 934 size_t offset; 935 void *cur; 936 937 offset = cur_pgsz * cur_page; 938 cur = RTE_PTR_ADD(addr, offset); 939 940 /* touch the page before getting its IOVA */ 941 *(volatile char *)cur = 0; 942 943 iova = rte_mem_virt2iova(cur); 944 945 iovas[cur_page] = iova; 946 } 947 948 break; 949 } 950 /* if we couldn't allocate anything */ 951 if (iovas == NULL) 952 return -1; 953 954 param->addr = addr; 955 param->len = mem_sz; 956 param->pgsz = cur_pgsz; 957 param->iova_table = iovas; 958 param->iova_table_len = n_pages; 959 960 return 0; 961 fail: 962 free(iovas); 963 if (addr) 964 munmap(addr, mem_sz); 965 966 return -1; 967 } 968 969 static int 970 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 971 { 972 struct extmem_param param; 973 int socket_id, ret; 974 975 memset(¶m, 0, sizeof(param)); 976 977 /* check if our heap exists */ 978 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 979 if (socket_id < 0) { 980 /* create our heap */ 981 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 982 if (ret < 0) { 983 TESTPMD_LOG(ERR, "Cannot create heap\n"); 984 return -1; 985 } 986 } 987 988 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 989 if (ret < 0) { 990 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 991 return -1; 992 } 993 994 /* we now have a valid memory area, so add it to heap */ 995 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 996 param.addr, param.len, param.iova_table, 997 param.iova_table_len, param.pgsz); 998 999 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1000 1001 /* not needed any more */ 1002 free(param.iova_table); 1003 1004 if (ret < 0) { 1005 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1006 munmap(param.addr, param.len); 1007 return -1; 1008 } 1009 1010 /* success */ 1011 1012 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1013 param.len >> 20); 1014 1015 return 0; 1016 } 1017 static void 1018 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1019 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1020 { 1021 uint16_t pid = 0; 1022 int ret; 1023 1024 RTE_ETH_FOREACH_DEV(pid) { 1025 struct rte_eth_dev_info dev_info; 1026 1027 ret = eth_dev_info_get_print_err(pid, &dev_info); 1028 if (ret != 0) { 1029 TESTPMD_LOG(DEBUG, 1030 "unable to get device info for port %d on addr 0x%p," 1031 "mempool unmapping will not be performed\n", 1032 pid, memhdr->addr); 1033 continue; 1034 } 1035 1036 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1037 if (ret) { 1038 TESTPMD_LOG(DEBUG, 1039 "unable to DMA unmap addr 0x%p " 1040 "for device %s\n", 1041 memhdr->addr, rte_dev_name(dev_info.device)); 1042 } 1043 } 1044 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1045 if (ret) { 1046 TESTPMD_LOG(DEBUG, 1047 "unable to un-register addr 0x%p\n", memhdr->addr); 1048 } 1049 } 1050 1051 static void 1052 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1053 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1054 { 1055 uint16_t pid = 0; 1056 size_t page_size = sysconf(_SC_PAGESIZE); 1057 int ret; 1058 1059 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1060 page_size); 1061 if (ret) { 1062 TESTPMD_LOG(DEBUG, 1063 "unable to register addr 0x%p\n", memhdr->addr); 1064 return; 1065 } 1066 RTE_ETH_FOREACH_DEV(pid) { 1067 struct rte_eth_dev_info dev_info; 1068 1069 ret = eth_dev_info_get_print_err(pid, &dev_info); 1070 if (ret != 0) { 1071 TESTPMD_LOG(DEBUG, 1072 "unable to get device info for port %d on addr 0x%p," 1073 "mempool mapping will not be performed\n", 1074 pid, memhdr->addr); 1075 continue; 1076 } 1077 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1078 if (ret) { 1079 TESTPMD_LOG(DEBUG, 1080 "unable to DMA map addr 0x%p " 1081 "for device %s\n", 1082 memhdr->addr, rte_dev_name(dev_info.device)); 1083 } 1084 } 1085 } 1086 #endif 1087 1088 static unsigned int 1089 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1090 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1091 { 1092 struct rte_pktmbuf_extmem *xmem; 1093 unsigned int ext_num, zone_num, elt_num; 1094 uint16_t elt_size; 1095 1096 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1097 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1098 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1099 1100 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1101 if (xmem == NULL) { 1102 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1103 "external buffer descriptors\n"); 1104 *ext_mem = NULL; 1105 return 0; 1106 } 1107 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1108 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1109 const struct rte_memzone *mz; 1110 char mz_name[RTE_MEMZONE_NAMESIZE]; 1111 int ret; 1112 1113 ret = snprintf(mz_name, sizeof(mz_name), 1114 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1115 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1116 errno = ENAMETOOLONG; 1117 ext_num = 0; 1118 break; 1119 } 1120 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1121 socket_id, 1122 RTE_MEMZONE_IOVA_CONTIG | 1123 RTE_MEMZONE_1GB | 1124 RTE_MEMZONE_SIZE_HINT_ONLY); 1125 if (mz == NULL) { 1126 /* 1127 * The caller exits on external buffer creation 1128 * error, so there is no need to free memzones. 1129 */ 1130 errno = ENOMEM; 1131 ext_num = 0; 1132 break; 1133 } 1134 xseg->buf_ptr = mz->addr; 1135 xseg->buf_iova = mz->iova; 1136 xseg->buf_len = EXTBUF_ZONE_SIZE; 1137 xseg->elt_size = elt_size; 1138 } 1139 if (ext_num == 0 && xmem != NULL) { 1140 free(xmem); 1141 xmem = NULL; 1142 } 1143 *ext_mem = xmem; 1144 return ext_num; 1145 } 1146 1147 /* 1148 * Configuration initialisation done once at init time. 1149 */ 1150 static struct rte_mempool * 1151 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1152 unsigned int socket_id, uint16_t size_idx) 1153 { 1154 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1155 struct rte_mempool *rte_mp = NULL; 1156 #ifndef RTE_EXEC_ENV_WINDOWS 1157 uint32_t mb_size; 1158 1159 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1160 #endif 1161 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1162 if (!is_proc_primary()) { 1163 rte_mp = rte_mempool_lookup(pool_name); 1164 if (rte_mp == NULL) 1165 rte_exit(EXIT_FAILURE, 1166 "Get mbuf pool for socket %u failed: %s\n", 1167 socket_id, rte_strerror(rte_errno)); 1168 return rte_mp; 1169 } 1170 1171 TESTPMD_LOG(INFO, 1172 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1173 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1174 1175 switch (mp_alloc_type) { 1176 case MP_ALLOC_NATIVE: 1177 { 1178 /* wrapper to rte_mempool_create() */ 1179 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1180 rte_mbuf_best_mempool_ops()); 1181 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1182 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1183 break; 1184 } 1185 #ifndef RTE_EXEC_ENV_WINDOWS 1186 case MP_ALLOC_ANON: 1187 { 1188 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1189 mb_size, (unsigned int) mb_mempool_cache, 1190 sizeof(struct rte_pktmbuf_pool_private), 1191 socket_id, mempool_flags); 1192 if (rte_mp == NULL) 1193 goto err; 1194 1195 if (rte_mempool_populate_anon(rte_mp) == 0) { 1196 rte_mempool_free(rte_mp); 1197 rte_mp = NULL; 1198 goto err; 1199 } 1200 rte_pktmbuf_pool_init(rte_mp, NULL); 1201 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1202 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1203 break; 1204 } 1205 case MP_ALLOC_XMEM: 1206 case MP_ALLOC_XMEM_HUGE: 1207 { 1208 int heap_socket; 1209 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1210 1211 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1212 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1213 1214 heap_socket = 1215 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1216 if (heap_socket < 0) 1217 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1218 1219 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1220 rte_mbuf_best_mempool_ops()); 1221 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1222 mb_mempool_cache, 0, mbuf_seg_size, 1223 heap_socket); 1224 break; 1225 } 1226 #endif 1227 case MP_ALLOC_XBUF: 1228 { 1229 struct rte_pktmbuf_extmem *ext_mem; 1230 unsigned int ext_num; 1231 1232 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1233 socket_id, pool_name, &ext_mem); 1234 if (ext_num == 0) 1235 rte_exit(EXIT_FAILURE, 1236 "Can't create pinned data buffers\n"); 1237 1238 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1239 rte_mbuf_best_mempool_ops()); 1240 rte_mp = rte_pktmbuf_pool_create_extbuf 1241 (pool_name, nb_mbuf, mb_mempool_cache, 1242 0, mbuf_seg_size, socket_id, 1243 ext_mem, ext_num); 1244 free(ext_mem); 1245 break; 1246 } 1247 default: 1248 { 1249 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1250 } 1251 } 1252 1253 #ifndef RTE_EXEC_ENV_WINDOWS 1254 err: 1255 #endif 1256 if (rte_mp == NULL) { 1257 rte_exit(EXIT_FAILURE, 1258 "Creation of mbuf pool for socket %u failed: %s\n", 1259 socket_id, rte_strerror(rte_errno)); 1260 } else if (verbose_level > 0) { 1261 rte_mempool_dump(stdout, rte_mp); 1262 } 1263 return rte_mp; 1264 } 1265 1266 /* 1267 * Check given socket id is valid or not with NUMA mode, 1268 * if valid, return 0, else return -1 1269 */ 1270 static int 1271 check_socket_id(const unsigned int socket_id) 1272 { 1273 static int warning_once = 0; 1274 1275 if (new_socket_id(socket_id)) { 1276 if (!warning_once && numa_support) 1277 fprintf(stderr, 1278 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1279 warning_once = 1; 1280 return -1; 1281 } 1282 return 0; 1283 } 1284 1285 /* 1286 * Get the allowed maximum number of RX queues. 1287 * *pid return the port id which has minimal value of 1288 * max_rx_queues in all ports. 1289 */ 1290 queueid_t 1291 get_allowed_max_nb_rxq(portid_t *pid) 1292 { 1293 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1294 bool max_rxq_valid = false; 1295 portid_t pi; 1296 struct rte_eth_dev_info dev_info; 1297 1298 RTE_ETH_FOREACH_DEV(pi) { 1299 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1300 continue; 1301 1302 max_rxq_valid = true; 1303 if (dev_info.max_rx_queues < allowed_max_rxq) { 1304 allowed_max_rxq = dev_info.max_rx_queues; 1305 *pid = pi; 1306 } 1307 } 1308 return max_rxq_valid ? allowed_max_rxq : 0; 1309 } 1310 1311 /* 1312 * Check input rxq is valid or not. 1313 * If input rxq is not greater than any of maximum number 1314 * of RX queues of all ports, it is valid. 1315 * if valid, return 0, else return -1 1316 */ 1317 int 1318 check_nb_rxq(queueid_t rxq) 1319 { 1320 queueid_t allowed_max_rxq; 1321 portid_t pid = 0; 1322 1323 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1324 if (rxq > allowed_max_rxq) { 1325 fprintf(stderr, 1326 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1327 rxq, allowed_max_rxq, pid); 1328 return -1; 1329 } 1330 return 0; 1331 } 1332 1333 /* 1334 * Get the allowed maximum number of TX queues. 1335 * *pid return the port id which has minimal value of 1336 * max_tx_queues in all ports. 1337 */ 1338 queueid_t 1339 get_allowed_max_nb_txq(portid_t *pid) 1340 { 1341 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1342 bool max_txq_valid = false; 1343 portid_t pi; 1344 struct rte_eth_dev_info dev_info; 1345 1346 RTE_ETH_FOREACH_DEV(pi) { 1347 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1348 continue; 1349 1350 max_txq_valid = true; 1351 if (dev_info.max_tx_queues < allowed_max_txq) { 1352 allowed_max_txq = dev_info.max_tx_queues; 1353 *pid = pi; 1354 } 1355 } 1356 return max_txq_valid ? allowed_max_txq : 0; 1357 } 1358 1359 /* 1360 * Check input txq is valid or not. 1361 * If input txq is not greater than any of maximum number 1362 * of TX queues of all ports, it is valid. 1363 * if valid, return 0, else return -1 1364 */ 1365 int 1366 check_nb_txq(queueid_t txq) 1367 { 1368 queueid_t allowed_max_txq; 1369 portid_t pid = 0; 1370 1371 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1372 if (txq > allowed_max_txq) { 1373 fprintf(stderr, 1374 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1375 txq, allowed_max_txq, pid); 1376 return -1; 1377 } 1378 return 0; 1379 } 1380 1381 /* 1382 * Get the allowed maximum number of RXDs of every rx queue. 1383 * *pid return the port id which has minimal value of 1384 * max_rxd in all queues of all ports. 1385 */ 1386 static uint16_t 1387 get_allowed_max_nb_rxd(portid_t *pid) 1388 { 1389 uint16_t allowed_max_rxd = UINT16_MAX; 1390 portid_t pi; 1391 struct rte_eth_dev_info dev_info; 1392 1393 RTE_ETH_FOREACH_DEV(pi) { 1394 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1395 continue; 1396 1397 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1398 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1399 *pid = pi; 1400 } 1401 } 1402 return allowed_max_rxd; 1403 } 1404 1405 /* 1406 * Get the allowed minimal number of RXDs of every rx queue. 1407 * *pid return the port id which has minimal value of 1408 * min_rxd in all queues of all ports. 1409 */ 1410 static uint16_t 1411 get_allowed_min_nb_rxd(portid_t *pid) 1412 { 1413 uint16_t allowed_min_rxd = 0; 1414 portid_t pi; 1415 struct rte_eth_dev_info dev_info; 1416 1417 RTE_ETH_FOREACH_DEV(pi) { 1418 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1419 continue; 1420 1421 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1422 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1423 *pid = pi; 1424 } 1425 } 1426 1427 return allowed_min_rxd; 1428 } 1429 1430 /* 1431 * Check input rxd is valid or not. 1432 * If input rxd is not greater than any of maximum number 1433 * of RXDs of every Rx queues and is not less than any of 1434 * minimal number of RXDs of every Rx queues, it is valid. 1435 * if valid, return 0, else return -1 1436 */ 1437 int 1438 check_nb_rxd(queueid_t rxd) 1439 { 1440 uint16_t allowed_max_rxd; 1441 uint16_t allowed_min_rxd; 1442 portid_t pid = 0; 1443 1444 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1445 if (rxd > allowed_max_rxd) { 1446 fprintf(stderr, 1447 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1448 rxd, allowed_max_rxd, pid); 1449 return -1; 1450 } 1451 1452 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1453 if (rxd < allowed_min_rxd) { 1454 fprintf(stderr, 1455 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1456 rxd, allowed_min_rxd, pid); 1457 return -1; 1458 } 1459 1460 return 0; 1461 } 1462 1463 /* 1464 * Get the allowed maximum number of TXDs of every rx queues. 1465 * *pid return the port id which has minimal value of 1466 * max_txd in every tx queue. 1467 */ 1468 static uint16_t 1469 get_allowed_max_nb_txd(portid_t *pid) 1470 { 1471 uint16_t allowed_max_txd = UINT16_MAX; 1472 portid_t pi; 1473 struct rte_eth_dev_info dev_info; 1474 1475 RTE_ETH_FOREACH_DEV(pi) { 1476 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1477 continue; 1478 1479 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1480 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1481 *pid = pi; 1482 } 1483 } 1484 return allowed_max_txd; 1485 } 1486 1487 /* 1488 * Get the allowed maximum number of TXDs of every tx queues. 1489 * *pid return the port id which has minimal value of 1490 * min_txd in every tx queue. 1491 */ 1492 static uint16_t 1493 get_allowed_min_nb_txd(portid_t *pid) 1494 { 1495 uint16_t allowed_min_txd = 0; 1496 portid_t pi; 1497 struct rte_eth_dev_info dev_info; 1498 1499 RTE_ETH_FOREACH_DEV(pi) { 1500 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1501 continue; 1502 1503 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1504 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1505 *pid = pi; 1506 } 1507 } 1508 1509 return allowed_min_txd; 1510 } 1511 1512 /* 1513 * Check input txd is valid or not. 1514 * If input txd is not greater than any of maximum number 1515 * of TXDs of every Rx queues, it is valid. 1516 * if valid, return 0, else return -1 1517 */ 1518 int 1519 check_nb_txd(queueid_t txd) 1520 { 1521 uint16_t allowed_max_txd; 1522 uint16_t allowed_min_txd; 1523 portid_t pid = 0; 1524 1525 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1526 if (txd > allowed_max_txd) { 1527 fprintf(stderr, 1528 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1529 txd, allowed_max_txd, pid); 1530 return -1; 1531 } 1532 1533 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1534 if (txd < allowed_min_txd) { 1535 fprintf(stderr, 1536 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1537 txd, allowed_min_txd, pid); 1538 return -1; 1539 } 1540 return 0; 1541 } 1542 1543 1544 /* 1545 * Get the allowed maximum number of hairpin queues. 1546 * *pid return the port id which has minimal value of 1547 * max_hairpin_queues in all ports. 1548 */ 1549 queueid_t 1550 get_allowed_max_nb_hairpinq(portid_t *pid) 1551 { 1552 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1553 portid_t pi; 1554 struct rte_eth_hairpin_cap cap; 1555 1556 RTE_ETH_FOREACH_DEV(pi) { 1557 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1558 *pid = pi; 1559 return 0; 1560 } 1561 if (cap.max_nb_queues < allowed_max_hairpinq) { 1562 allowed_max_hairpinq = cap.max_nb_queues; 1563 *pid = pi; 1564 } 1565 } 1566 return allowed_max_hairpinq; 1567 } 1568 1569 /* 1570 * Check input hairpin is valid or not. 1571 * If input hairpin is not greater than any of maximum number 1572 * of hairpin queues of all ports, it is valid. 1573 * if valid, return 0, else return -1 1574 */ 1575 int 1576 check_nb_hairpinq(queueid_t hairpinq) 1577 { 1578 queueid_t allowed_max_hairpinq; 1579 portid_t pid = 0; 1580 1581 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1582 if (hairpinq > allowed_max_hairpinq) { 1583 fprintf(stderr, 1584 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1585 hairpinq, allowed_max_hairpinq, pid); 1586 return -1; 1587 } 1588 return 0; 1589 } 1590 1591 static int 1592 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1593 { 1594 uint32_t eth_overhead; 1595 1596 if (dev_info->max_mtu != UINT16_MAX && 1597 dev_info->max_rx_pktlen > dev_info->max_mtu) 1598 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1599 else 1600 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1601 1602 return eth_overhead; 1603 } 1604 1605 static void 1606 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1607 { 1608 struct rte_port *port = &ports[pid]; 1609 int ret; 1610 int i; 1611 1612 eth_rx_metadata_negotiate_mp(pid); 1613 1614 port->dev_conf.txmode = tx_mode; 1615 port->dev_conf.rxmode = rx_mode; 1616 1617 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1618 if (ret != 0) 1619 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1620 1621 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1622 port->dev_conf.txmode.offloads &= 1623 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1624 1625 /* Apply Rx offloads configuration */ 1626 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1627 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1628 /* Apply Tx offloads configuration */ 1629 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1630 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1631 1632 if (eth_link_speed) 1633 port->dev_conf.link_speeds = eth_link_speed; 1634 1635 if (max_rx_pkt_len) 1636 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1637 get_eth_overhead(&port->dev_info); 1638 1639 /* set flag to initialize port/queue */ 1640 port->need_reconfig = 1; 1641 port->need_reconfig_queues = 1; 1642 port->socket_id = socket_id; 1643 port->tx_metadata = 0; 1644 1645 /* 1646 * Check for maximum number of segments per MTU. 1647 * Accordingly update the mbuf data size. 1648 */ 1649 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1650 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1651 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1652 uint16_t mtu; 1653 1654 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1655 uint16_t data_size = (mtu + eth_overhead) / 1656 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1657 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1658 1659 if (buffer_size > mbuf_data_size[0]) { 1660 mbuf_data_size[0] = buffer_size; 1661 TESTPMD_LOG(WARNING, 1662 "Configured mbuf size of the first segment %hu\n", 1663 mbuf_data_size[0]); 1664 } 1665 } 1666 } 1667 } 1668 1669 static void 1670 init_config(void) 1671 { 1672 portid_t pid; 1673 struct rte_mempool *mbp; 1674 unsigned int nb_mbuf_per_pool; 1675 lcoreid_t lc_id; 1676 #ifdef RTE_LIB_GRO 1677 struct rte_gro_param gro_param; 1678 #endif 1679 #ifdef RTE_LIB_GSO 1680 uint32_t gso_types; 1681 #endif 1682 1683 /* Configuration of logical cores. */ 1684 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1685 sizeof(struct fwd_lcore *) * nb_lcores, 1686 RTE_CACHE_LINE_SIZE); 1687 if (fwd_lcores == NULL) { 1688 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1689 "failed\n", nb_lcores); 1690 } 1691 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1692 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1693 sizeof(struct fwd_lcore), 1694 RTE_CACHE_LINE_SIZE); 1695 if (fwd_lcores[lc_id] == NULL) { 1696 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1697 "failed\n"); 1698 } 1699 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1700 } 1701 1702 RTE_ETH_FOREACH_DEV(pid) { 1703 uint32_t socket_id; 1704 1705 if (numa_support) { 1706 socket_id = port_numa[pid]; 1707 if (port_numa[pid] == NUMA_NO_CONFIG) { 1708 socket_id = rte_eth_dev_socket_id(pid); 1709 1710 /* 1711 * if socket_id is invalid, 1712 * set to the first available socket. 1713 */ 1714 if (check_socket_id(socket_id) < 0) 1715 socket_id = socket_ids[0]; 1716 } 1717 } else { 1718 socket_id = (socket_num == UMA_NO_CONFIG) ? 1719 0 : socket_num; 1720 } 1721 /* Apply default TxRx configuration for all ports */ 1722 init_config_port_offloads(pid, socket_id); 1723 } 1724 /* 1725 * Create pools of mbuf. 1726 * If NUMA support is disabled, create a single pool of mbuf in 1727 * socket 0 memory by default. 1728 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1729 * 1730 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1731 * nb_txd can be configured at run time. 1732 */ 1733 if (param_total_num_mbufs) 1734 nb_mbuf_per_pool = param_total_num_mbufs; 1735 else { 1736 nb_mbuf_per_pool = RX_DESC_MAX + 1737 (nb_lcores * mb_mempool_cache) + 1738 TX_DESC_MAX + MAX_PKT_BURST; 1739 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1740 } 1741 1742 if (numa_support) { 1743 uint8_t i, j; 1744 1745 for (i = 0; i < num_sockets; i++) 1746 for (j = 0; j < mbuf_data_size_n; j++) 1747 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1748 mbuf_pool_create(mbuf_data_size[j], 1749 nb_mbuf_per_pool, 1750 socket_ids[i], j); 1751 } else { 1752 uint8_t i; 1753 1754 for (i = 0; i < mbuf_data_size_n; i++) 1755 mempools[i] = mbuf_pool_create 1756 (mbuf_data_size[i], 1757 nb_mbuf_per_pool, 1758 socket_num == UMA_NO_CONFIG ? 1759 0 : socket_num, i); 1760 } 1761 1762 init_port_config(); 1763 1764 #ifdef RTE_LIB_GSO 1765 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1766 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1767 #endif 1768 /* 1769 * Records which Mbuf pool to use by each logical core, if needed. 1770 */ 1771 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1772 mbp = mbuf_pool_find( 1773 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1774 1775 if (mbp == NULL) 1776 mbp = mbuf_pool_find(0, 0); 1777 fwd_lcores[lc_id]->mbp = mbp; 1778 #ifdef RTE_LIB_GSO 1779 /* initialize GSO context */ 1780 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1781 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1782 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1783 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1784 RTE_ETHER_CRC_LEN; 1785 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1786 #endif 1787 } 1788 1789 fwd_config_setup(); 1790 1791 #ifdef RTE_LIB_GRO 1792 /* create a gro context for each lcore */ 1793 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1794 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1795 gro_param.max_item_per_flow = MAX_PKT_BURST; 1796 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1797 gro_param.socket_id = rte_lcore_to_socket_id( 1798 fwd_lcores_cpuids[lc_id]); 1799 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1800 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1801 rte_exit(EXIT_FAILURE, 1802 "rte_gro_ctx_create() failed\n"); 1803 } 1804 } 1805 #endif 1806 } 1807 1808 1809 void 1810 reconfig(portid_t new_port_id, unsigned socket_id) 1811 { 1812 /* Reconfiguration of Ethernet ports. */ 1813 init_config_port_offloads(new_port_id, socket_id); 1814 init_port_config(); 1815 } 1816 1817 int 1818 init_fwd_streams(void) 1819 { 1820 portid_t pid; 1821 struct rte_port *port; 1822 streamid_t sm_id, nb_fwd_streams_new; 1823 queueid_t q; 1824 1825 /* set socket id according to numa or not */ 1826 RTE_ETH_FOREACH_DEV(pid) { 1827 port = &ports[pid]; 1828 if (nb_rxq > port->dev_info.max_rx_queues) { 1829 fprintf(stderr, 1830 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1831 nb_rxq, port->dev_info.max_rx_queues); 1832 return -1; 1833 } 1834 if (nb_txq > port->dev_info.max_tx_queues) { 1835 fprintf(stderr, 1836 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1837 nb_txq, port->dev_info.max_tx_queues); 1838 return -1; 1839 } 1840 if (numa_support) { 1841 if (port_numa[pid] != NUMA_NO_CONFIG) 1842 port->socket_id = port_numa[pid]; 1843 else { 1844 port->socket_id = rte_eth_dev_socket_id(pid); 1845 1846 /* 1847 * if socket_id is invalid, 1848 * set to the first available socket. 1849 */ 1850 if (check_socket_id(port->socket_id) < 0) 1851 port->socket_id = socket_ids[0]; 1852 } 1853 } 1854 else { 1855 if (socket_num == UMA_NO_CONFIG) 1856 port->socket_id = 0; 1857 else 1858 port->socket_id = socket_num; 1859 } 1860 } 1861 1862 q = RTE_MAX(nb_rxq, nb_txq); 1863 if (q == 0) { 1864 fprintf(stderr, 1865 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1866 return -1; 1867 } 1868 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1869 if (nb_fwd_streams_new == nb_fwd_streams) 1870 return 0; 1871 /* clear the old */ 1872 if (fwd_streams != NULL) { 1873 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1874 if (fwd_streams[sm_id] == NULL) 1875 continue; 1876 rte_free(fwd_streams[sm_id]); 1877 fwd_streams[sm_id] = NULL; 1878 } 1879 rte_free(fwd_streams); 1880 fwd_streams = NULL; 1881 } 1882 1883 /* init new */ 1884 nb_fwd_streams = nb_fwd_streams_new; 1885 if (nb_fwd_streams) { 1886 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1887 sizeof(struct fwd_stream *) * nb_fwd_streams, 1888 RTE_CACHE_LINE_SIZE); 1889 if (fwd_streams == NULL) 1890 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1891 " (struct fwd_stream *)) failed\n", 1892 nb_fwd_streams); 1893 1894 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1895 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1896 " struct fwd_stream", sizeof(struct fwd_stream), 1897 RTE_CACHE_LINE_SIZE); 1898 if (fwd_streams[sm_id] == NULL) 1899 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1900 "(struct fwd_stream) failed\n"); 1901 } 1902 } 1903 1904 return 0; 1905 } 1906 1907 static void 1908 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1909 { 1910 uint64_t total_burst, sburst; 1911 uint64_t nb_burst; 1912 uint64_t burst_stats[4]; 1913 uint16_t pktnb_stats[4]; 1914 uint16_t nb_pkt; 1915 int burst_percent[4], sburstp; 1916 int i; 1917 1918 /* 1919 * First compute the total number of packet bursts and the 1920 * two highest numbers of bursts of the same number of packets. 1921 */ 1922 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1923 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1924 1925 /* Show stats for 0 burst size always */ 1926 total_burst = pbs->pkt_burst_spread[0]; 1927 burst_stats[0] = pbs->pkt_burst_spread[0]; 1928 pktnb_stats[0] = 0; 1929 1930 /* Find the next 2 burst sizes with highest occurrences. */ 1931 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1932 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1933 1934 if (nb_burst == 0) 1935 continue; 1936 1937 total_burst += nb_burst; 1938 1939 if (nb_burst > burst_stats[1]) { 1940 burst_stats[2] = burst_stats[1]; 1941 pktnb_stats[2] = pktnb_stats[1]; 1942 burst_stats[1] = nb_burst; 1943 pktnb_stats[1] = nb_pkt; 1944 } else if (nb_burst > burst_stats[2]) { 1945 burst_stats[2] = nb_burst; 1946 pktnb_stats[2] = nb_pkt; 1947 } 1948 } 1949 if (total_burst == 0) 1950 return; 1951 1952 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1953 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1954 if (i == 3) { 1955 printf("%d%% of other]\n", 100 - sburstp); 1956 return; 1957 } 1958 1959 sburst += burst_stats[i]; 1960 if (sburst == total_burst) { 1961 printf("%d%% of %d pkts]\n", 1962 100 - sburstp, (int) pktnb_stats[i]); 1963 return; 1964 } 1965 1966 burst_percent[i] = 1967 (double)burst_stats[i] / total_burst * 100; 1968 printf("%d%% of %d pkts + ", 1969 burst_percent[i], (int) pktnb_stats[i]); 1970 sburstp += burst_percent[i]; 1971 } 1972 } 1973 1974 static void 1975 fwd_stream_stats_display(streamid_t stream_id) 1976 { 1977 struct fwd_stream *fs; 1978 static const char *fwd_top_stats_border = "-------"; 1979 1980 fs = fwd_streams[stream_id]; 1981 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1982 (fs->fwd_dropped == 0)) 1983 return; 1984 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1985 "TX Port=%2d/Queue=%2d %s\n", 1986 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1987 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1988 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1989 " TX-dropped: %-14"PRIu64, 1990 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1991 1992 /* if checksum mode */ 1993 if (cur_fwd_eng == &csum_fwd_engine) { 1994 printf(" RX- bad IP checksum: %-14"PRIu64 1995 " Rx- bad L4 checksum: %-14"PRIu64 1996 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1997 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1998 fs->rx_bad_outer_l4_csum); 1999 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2000 fs->rx_bad_outer_ip_csum); 2001 } else { 2002 printf("\n"); 2003 } 2004 2005 if (record_burst_stats) { 2006 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2007 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2008 } 2009 } 2010 2011 void 2012 fwd_stats_display(void) 2013 { 2014 static const char *fwd_stats_border = "----------------------"; 2015 static const char *acc_stats_border = "+++++++++++++++"; 2016 struct { 2017 struct fwd_stream *rx_stream; 2018 struct fwd_stream *tx_stream; 2019 uint64_t tx_dropped; 2020 uint64_t rx_bad_ip_csum; 2021 uint64_t rx_bad_l4_csum; 2022 uint64_t rx_bad_outer_l4_csum; 2023 uint64_t rx_bad_outer_ip_csum; 2024 } ports_stats[RTE_MAX_ETHPORTS]; 2025 uint64_t total_rx_dropped = 0; 2026 uint64_t total_tx_dropped = 0; 2027 uint64_t total_rx_nombuf = 0; 2028 struct rte_eth_stats stats; 2029 uint64_t fwd_cycles = 0; 2030 uint64_t total_recv = 0; 2031 uint64_t total_xmit = 0; 2032 struct rte_port *port; 2033 streamid_t sm_id; 2034 portid_t pt_id; 2035 int ret; 2036 int i; 2037 2038 memset(ports_stats, 0, sizeof(ports_stats)); 2039 2040 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2041 struct fwd_stream *fs = fwd_streams[sm_id]; 2042 2043 if (cur_fwd_config.nb_fwd_streams > 2044 cur_fwd_config.nb_fwd_ports) { 2045 fwd_stream_stats_display(sm_id); 2046 } else { 2047 ports_stats[fs->tx_port].tx_stream = fs; 2048 ports_stats[fs->rx_port].rx_stream = fs; 2049 } 2050 2051 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2052 2053 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2054 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2055 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2056 fs->rx_bad_outer_l4_csum; 2057 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2058 fs->rx_bad_outer_ip_csum; 2059 2060 if (record_core_cycles) 2061 fwd_cycles += fs->busy_cycles; 2062 } 2063 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2064 uint64_t tx_dropped = 0; 2065 2066 pt_id = fwd_ports_ids[i]; 2067 port = &ports[pt_id]; 2068 2069 ret = rte_eth_stats_get(pt_id, &stats); 2070 if (ret != 0) { 2071 fprintf(stderr, 2072 "%s: Error: failed to get stats (port %u): %d", 2073 __func__, pt_id, ret); 2074 continue; 2075 } 2076 stats.ipackets -= port->stats.ipackets; 2077 stats.opackets -= port->stats.opackets; 2078 stats.ibytes -= port->stats.ibytes; 2079 stats.obytes -= port->stats.obytes; 2080 stats.imissed -= port->stats.imissed; 2081 stats.oerrors -= port->stats.oerrors; 2082 stats.rx_nombuf -= port->stats.rx_nombuf; 2083 2084 total_recv += stats.ipackets; 2085 total_xmit += stats.opackets; 2086 total_rx_dropped += stats.imissed; 2087 tx_dropped += ports_stats[pt_id].tx_dropped; 2088 tx_dropped += stats.oerrors; 2089 total_tx_dropped += tx_dropped; 2090 total_rx_nombuf += stats.rx_nombuf; 2091 2092 printf("\n %s Forward statistics for port %-2d %s\n", 2093 fwd_stats_border, pt_id, fwd_stats_border); 2094 2095 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2096 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2097 stats.ipackets + stats.imissed); 2098 2099 if (cur_fwd_eng == &csum_fwd_engine) { 2100 printf(" Bad-ipcsum: %-14"PRIu64 2101 " Bad-l4csum: %-14"PRIu64 2102 "Bad-outer-l4csum: %-14"PRIu64"\n", 2103 ports_stats[pt_id].rx_bad_ip_csum, 2104 ports_stats[pt_id].rx_bad_l4_csum, 2105 ports_stats[pt_id].rx_bad_outer_l4_csum); 2106 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2107 ports_stats[pt_id].rx_bad_outer_ip_csum); 2108 } 2109 if (stats.ierrors + stats.rx_nombuf > 0) { 2110 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2111 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2112 } 2113 2114 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2115 "TX-total: %-"PRIu64"\n", 2116 stats.opackets, tx_dropped, 2117 stats.opackets + tx_dropped); 2118 2119 if (record_burst_stats) { 2120 if (ports_stats[pt_id].rx_stream) 2121 pkt_burst_stats_display("RX", 2122 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2123 if (ports_stats[pt_id].tx_stream) 2124 pkt_burst_stats_display("TX", 2125 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2126 } 2127 2128 printf(" %s--------------------------------%s\n", 2129 fwd_stats_border, fwd_stats_border); 2130 } 2131 2132 printf("\n %s Accumulated forward statistics for all ports" 2133 "%s\n", 2134 acc_stats_border, acc_stats_border); 2135 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2136 "%-"PRIu64"\n" 2137 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2138 "%-"PRIu64"\n", 2139 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2140 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2141 if (total_rx_nombuf > 0) 2142 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2143 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2144 "%s\n", 2145 acc_stats_border, acc_stats_border); 2146 if (record_core_cycles) { 2147 #define CYC_PER_MHZ 1E6 2148 if (total_recv > 0 || total_xmit > 0) { 2149 uint64_t total_pkts = 0; 2150 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2151 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2152 total_pkts = total_xmit; 2153 else 2154 total_pkts = total_recv; 2155 2156 printf("\n CPU cycles/packet=%.2F (busy cycles=" 2157 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2158 " MHz Clock\n", 2159 (double) fwd_cycles / total_pkts, 2160 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2161 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2162 } 2163 } 2164 } 2165 2166 void 2167 fwd_stats_reset(void) 2168 { 2169 streamid_t sm_id; 2170 portid_t pt_id; 2171 int ret; 2172 int i; 2173 2174 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2175 pt_id = fwd_ports_ids[i]; 2176 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2177 if (ret != 0) 2178 fprintf(stderr, 2179 "%s: Error: failed to clear stats (port %u):%d", 2180 __func__, pt_id, ret); 2181 } 2182 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2183 struct fwd_stream *fs = fwd_streams[sm_id]; 2184 2185 fs->rx_packets = 0; 2186 fs->tx_packets = 0; 2187 fs->fwd_dropped = 0; 2188 fs->rx_bad_ip_csum = 0; 2189 fs->rx_bad_l4_csum = 0; 2190 fs->rx_bad_outer_l4_csum = 0; 2191 fs->rx_bad_outer_ip_csum = 0; 2192 2193 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2194 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2195 fs->busy_cycles = 0; 2196 } 2197 } 2198 2199 static void 2200 flush_fwd_rx_queues(void) 2201 { 2202 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2203 portid_t rxp; 2204 portid_t port_id; 2205 queueid_t rxq; 2206 uint16_t nb_rx; 2207 uint16_t i; 2208 uint8_t j; 2209 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2210 uint64_t timer_period; 2211 2212 if (num_procs > 1) { 2213 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2214 return; 2215 } 2216 2217 /* convert to number of cycles */ 2218 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2219 2220 for (j = 0; j < 2; j++) { 2221 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2222 for (rxq = 0; rxq < nb_rxq; rxq++) { 2223 port_id = fwd_ports_ids[rxp]; 2224 2225 /* Polling stopped queues is prohibited. */ 2226 if (ports[port_id].rxq[rxq].state == 2227 RTE_ETH_QUEUE_STATE_STOPPED) 2228 continue; 2229 2230 /** 2231 * testpmd can stuck in the below do while loop 2232 * if rte_eth_rx_burst() always returns nonzero 2233 * packets. So timer is added to exit this loop 2234 * after 1sec timer expiry. 2235 */ 2236 prev_tsc = rte_rdtsc(); 2237 do { 2238 nb_rx = rte_eth_rx_burst(port_id, rxq, 2239 pkts_burst, MAX_PKT_BURST); 2240 for (i = 0; i < nb_rx; i++) 2241 rte_pktmbuf_free(pkts_burst[i]); 2242 2243 cur_tsc = rte_rdtsc(); 2244 diff_tsc = cur_tsc - prev_tsc; 2245 timer_tsc += diff_tsc; 2246 } while ((nb_rx > 0) && 2247 (timer_tsc < timer_period)); 2248 timer_tsc = 0; 2249 } 2250 } 2251 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2252 } 2253 } 2254 2255 static void 2256 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2257 { 2258 struct fwd_stream **fsm; 2259 uint64_t prev_tsc; 2260 streamid_t nb_fs; 2261 streamid_t sm_id; 2262 #ifdef RTE_LIB_BITRATESTATS 2263 uint64_t tics_per_1sec; 2264 uint64_t tics_datum; 2265 uint64_t tics_current; 2266 uint16_t i, cnt_ports; 2267 2268 cnt_ports = nb_ports; 2269 tics_datum = rte_rdtsc(); 2270 tics_per_1sec = rte_get_timer_hz(); 2271 #endif 2272 fsm = &fwd_streams[fc->stream_idx]; 2273 nb_fs = fc->stream_nb; 2274 prev_tsc = rte_rdtsc(); 2275 do { 2276 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2277 if (!fsm[sm_id]->disabled) 2278 (*pkt_fwd)(fsm[sm_id]); 2279 #ifdef RTE_LIB_BITRATESTATS 2280 if (bitrate_enabled != 0 && 2281 bitrate_lcore_id == rte_lcore_id()) { 2282 tics_current = rte_rdtsc(); 2283 if (tics_current - tics_datum >= tics_per_1sec) { 2284 /* Periodic bitrate calculation */ 2285 for (i = 0; i < cnt_ports; i++) 2286 rte_stats_bitrate_calc(bitrate_data, 2287 ports_ids[i]); 2288 tics_datum = tics_current; 2289 } 2290 } 2291 #endif 2292 #ifdef RTE_LIB_LATENCYSTATS 2293 if (latencystats_enabled != 0 && 2294 latencystats_lcore_id == rte_lcore_id()) 2295 rte_latencystats_update(); 2296 #endif 2297 if (record_core_cycles) { 2298 uint64_t tsc = rte_rdtsc(); 2299 2300 fc->total_cycles += tsc - prev_tsc; 2301 prev_tsc = tsc; 2302 } 2303 } while (! fc->stopped); 2304 } 2305 2306 static int 2307 lcore_usage_callback(unsigned int lcore_id, struct rte_lcore_usage *usage) 2308 { 2309 struct fwd_stream **fsm; 2310 struct fwd_lcore *fc; 2311 streamid_t nb_fs; 2312 streamid_t sm_id; 2313 2314 fc = lcore_to_fwd_lcore(lcore_id); 2315 if (fc == NULL) 2316 return -1; 2317 2318 fsm = &fwd_streams[fc->stream_idx]; 2319 nb_fs = fc->stream_nb; 2320 usage->busy_cycles = 0; 2321 usage->total_cycles = fc->total_cycles; 2322 2323 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2324 if (!fsm[sm_id]->disabled) 2325 usage->busy_cycles += fsm[sm_id]->busy_cycles; 2326 } 2327 2328 return 0; 2329 } 2330 2331 static int 2332 start_pkt_forward_on_core(void *fwd_arg) 2333 { 2334 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2335 cur_fwd_config.fwd_eng->packet_fwd); 2336 return 0; 2337 } 2338 2339 /* 2340 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2341 * Used to start communication flows in network loopback test configurations. 2342 */ 2343 static int 2344 run_one_txonly_burst_on_core(void *fwd_arg) 2345 { 2346 struct fwd_lcore *fwd_lc; 2347 struct fwd_lcore tmp_lcore; 2348 2349 fwd_lc = (struct fwd_lcore *) fwd_arg; 2350 tmp_lcore = *fwd_lc; 2351 tmp_lcore.stopped = 1; 2352 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2353 return 0; 2354 } 2355 2356 /* 2357 * Launch packet forwarding: 2358 * - Setup per-port forwarding context. 2359 * - launch logical cores with their forwarding configuration. 2360 */ 2361 static void 2362 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2363 { 2364 unsigned int i; 2365 unsigned int lc_id; 2366 int diag; 2367 2368 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2369 lc_id = fwd_lcores_cpuids[i]; 2370 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2371 fwd_lcores[i]->stopped = 0; 2372 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2373 fwd_lcores[i], lc_id); 2374 if (diag != 0) 2375 fprintf(stderr, 2376 "launch lcore %u failed - diag=%d\n", 2377 lc_id, diag); 2378 } 2379 } 2380 } 2381 2382 /* 2383 * Launch packet forwarding configuration. 2384 */ 2385 void 2386 start_packet_forwarding(int with_tx_first) 2387 { 2388 port_fwd_begin_t port_fwd_begin; 2389 port_fwd_end_t port_fwd_end; 2390 stream_init_t stream_init = cur_fwd_eng->stream_init; 2391 unsigned int i; 2392 2393 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2394 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2395 2396 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2397 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2398 2399 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2400 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2401 (!nb_rxq || !nb_txq)) 2402 rte_exit(EXIT_FAILURE, 2403 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2404 cur_fwd_eng->fwd_mode_name); 2405 2406 if (all_ports_started() == 0) { 2407 fprintf(stderr, "Not all ports were started\n"); 2408 return; 2409 } 2410 if (test_done == 0) { 2411 fprintf(stderr, "Packet forwarding already started\n"); 2412 return; 2413 } 2414 2415 fwd_config_setup(); 2416 2417 pkt_fwd_config_display(&cur_fwd_config); 2418 if (!pkt_fwd_shared_rxq_check()) 2419 return; 2420 2421 if (stream_init != NULL) 2422 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2423 stream_init(fwd_streams[i]); 2424 2425 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2426 if (port_fwd_begin != NULL) { 2427 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2428 if (port_fwd_begin(fwd_ports_ids[i])) { 2429 fprintf(stderr, 2430 "Packet forwarding is not ready\n"); 2431 return; 2432 } 2433 } 2434 } 2435 2436 if (with_tx_first) { 2437 port_fwd_begin = tx_only_engine.port_fwd_begin; 2438 if (port_fwd_begin != NULL) { 2439 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2440 if (port_fwd_begin(fwd_ports_ids[i])) { 2441 fprintf(stderr, 2442 "Packet forwarding is not ready\n"); 2443 return; 2444 } 2445 } 2446 } 2447 } 2448 2449 test_done = 0; 2450 2451 if(!no_flush_rx) 2452 flush_fwd_rx_queues(); 2453 2454 rxtx_config_display(); 2455 2456 fwd_stats_reset(); 2457 if (with_tx_first) { 2458 while (with_tx_first--) { 2459 launch_packet_forwarding( 2460 run_one_txonly_burst_on_core); 2461 rte_eal_mp_wait_lcore(); 2462 } 2463 port_fwd_end = tx_only_engine.port_fwd_end; 2464 if (port_fwd_end != NULL) { 2465 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2466 (*port_fwd_end)(fwd_ports_ids[i]); 2467 } 2468 } 2469 launch_packet_forwarding(start_pkt_forward_on_core); 2470 } 2471 2472 void 2473 stop_packet_forwarding(void) 2474 { 2475 port_fwd_end_t port_fwd_end; 2476 lcoreid_t lc_id; 2477 portid_t pt_id; 2478 int i; 2479 2480 if (test_done) { 2481 fprintf(stderr, "Packet forwarding not started\n"); 2482 return; 2483 } 2484 printf("Telling cores to stop..."); 2485 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2486 fwd_lcores[lc_id]->stopped = 1; 2487 printf("\nWaiting for lcores to finish...\n"); 2488 rte_eal_mp_wait_lcore(); 2489 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2490 if (port_fwd_end != NULL) { 2491 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2492 pt_id = fwd_ports_ids[i]; 2493 (*port_fwd_end)(pt_id); 2494 } 2495 } 2496 2497 fwd_stats_display(); 2498 2499 printf("\nDone.\n"); 2500 test_done = 1; 2501 } 2502 2503 void 2504 dev_set_link_up(portid_t pid) 2505 { 2506 if (rte_eth_dev_set_link_up(pid) < 0) 2507 fprintf(stderr, "\nSet link up fail.\n"); 2508 } 2509 2510 void 2511 dev_set_link_down(portid_t pid) 2512 { 2513 if (rte_eth_dev_set_link_down(pid) < 0) 2514 fprintf(stderr, "\nSet link down fail.\n"); 2515 } 2516 2517 static int 2518 all_ports_started(void) 2519 { 2520 portid_t pi; 2521 struct rte_port *port; 2522 2523 RTE_ETH_FOREACH_DEV(pi) { 2524 port = &ports[pi]; 2525 /* Check if there is a port which is not started */ 2526 if ((port->port_status != RTE_PORT_STARTED) && 2527 (port->slave_flag == 0)) 2528 return 0; 2529 } 2530 2531 /* No port is not started */ 2532 return 1; 2533 } 2534 2535 int 2536 port_is_stopped(portid_t port_id) 2537 { 2538 struct rte_port *port = &ports[port_id]; 2539 2540 if ((port->port_status != RTE_PORT_STOPPED) && 2541 (port->slave_flag == 0)) 2542 return 0; 2543 return 1; 2544 } 2545 2546 int 2547 all_ports_stopped(void) 2548 { 2549 portid_t pi; 2550 2551 RTE_ETH_FOREACH_DEV(pi) { 2552 if (!port_is_stopped(pi)) 2553 return 0; 2554 } 2555 2556 return 1; 2557 } 2558 2559 int 2560 port_is_started(portid_t port_id) 2561 { 2562 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2563 return 0; 2564 2565 if (ports[port_id].port_status != RTE_PORT_STARTED) 2566 return 0; 2567 2568 return 1; 2569 } 2570 2571 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2572 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2573 2574 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2575 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2576 2577 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2578 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2579 2580 2581 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2582 static int 2583 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2584 { 2585 queueid_t qi; 2586 struct rte_eth_hairpin_conf hairpin_conf = { 2587 .peer_count = 1, 2588 }; 2589 int i; 2590 int diag; 2591 struct rte_port *port = &ports[pi]; 2592 uint16_t peer_rx_port = pi; 2593 uint16_t peer_tx_port = pi; 2594 uint32_t manual = 1; 2595 uint32_t tx_exp = hairpin_mode & 0x10; 2596 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2597 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2598 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2599 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2600 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2601 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2602 2603 if (!(hairpin_mode & 0xf)) { 2604 peer_rx_port = pi; 2605 peer_tx_port = pi; 2606 manual = 0; 2607 } else if (hairpin_mode & 0x1) { 2608 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2609 RTE_ETH_DEV_NO_OWNER); 2610 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2611 peer_tx_port = rte_eth_find_next_owned_by(0, 2612 RTE_ETH_DEV_NO_OWNER); 2613 if (p_pi != RTE_MAX_ETHPORTS) { 2614 peer_rx_port = p_pi; 2615 } else { 2616 uint16_t next_pi; 2617 2618 /* Last port will be the peer RX port of the first. */ 2619 RTE_ETH_FOREACH_DEV(next_pi) 2620 peer_rx_port = next_pi; 2621 } 2622 manual = 1; 2623 } else if (hairpin_mode & 0x2) { 2624 if (cnt_pi & 0x1) { 2625 peer_rx_port = p_pi; 2626 } else { 2627 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2628 RTE_ETH_DEV_NO_OWNER); 2629 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2630 peer_rx_port = pi; 2631 } 2632 peer_tx_port = peer_rx_port; 2633 manual = 1; 2634 } 2635 2636 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2637 hairpin_conf.peers[0].port = peer_rx_port; 2638 hairpin_conf.peers[0].queue = i + nb_rxq; 2639 hairpin_conf.manual_bind = !!manual; 2640 hairpin_conf.tx_explicit = !!tx_exp; 2641 hairpin_conf.force_memory = !!tx_force_memory; 2642 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2643 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2644 diag = rte_eth_tx_hairpin_queue_setup 2645 (pi, qi, nb_txd, &hairpin_conf); 2646 i++; 2647 if (diag == 0) 2648 continue; 2649 2650 /* Fail to setup rx queue, return */ 2651 if (port->port_status == RTE_PORT_HANDLING) 2652 port->port_status = RTE_PORT_STOPPED; 2653 else 2654 fprintf(stderr, 2655 "Port %d can not be set back to stopped\n", pi); 2656 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2657 pi); 2658 /* try to reconfigure queues next time */ 2659 port->need_reconfig_queues = 1; 2660 return -1; 2661 } 2662 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2663 hairpin_conf.peers[0].port = peer_tx_port; 2664 hairpin_conf.peers[0].queue = i + nb_txq; 2665 hairpin_conf.manual_bind = !!manual; 2666 hairpin_conf.tx_explicit = !!tx_exp; 2667 hairpin_conf.force_memory = !!rx_force_memory; 2668 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2669 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2670 diag = rte_eth_rx_hairpin_queue_setup 2671 (pi, qi, nb_rxd, &hairpin_conf); 2672 i++; 2673 if (diag == 0) 2674 continue; 2675 2676 /* Fail to setup rx queue, return */ 2677 if (port->port_status == RTE_PORT_HANDLING) 2678 port->port_status = RTE_PORT_STOPPED; 2679 else 2680 fprintf(stderr, 2681 "Port %d can not be set back to stopped\n", pi); 2682 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2683 pi); 2684 /* try to reconfigure queues next time */ 2685 port->need_reconfig_queues = 1; 2686 return -1; 2687 } 2688 return 0; 2689 } 2690 2691 /* Configure the Rx with optional split. */ 2692 int 2693 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2694 uint16_t nb_rx_desc, unsigned int socket_id, 2695 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2696 { 2697 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2698 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2699 struct rte_mempool *mpx; 2700 unsigned int i, mp_n; 2701 uint32_t prev_hdrs = 0; 2702 int ret; 2703 2704 2705 if ((rx_pkt_nb_segs > 1) && 2706 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2707 /* multi-segment configuration */ 2708 for (i = 0; i < rx_pkt_nb_segs; i++) { 2709 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2710 /* 2711 * Use last valid pool for the segments with number 2712 * exceeding the pool index. 2713 */ 2714 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2715 mpx = mbuf_pool_find(socket_id, mp_n); 2716 /* Handle zero as mbuf data buffer size. */ 2717 rx_seg->offset = i < rx_pkt_nb_offs ? 2718 rx_pkt_seg_offsets[i] : 0; 2719 rx_seg->mp = mpx ? mpx : mp; 2720 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2721 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2722 prev_hdrs |= rx_seg->proto_hdr; 2723 } else { 2724 rx_seg->length = rx_pkt_seg_lengths[i] ? 2725 rx_pkt_seg_lengths[i] : 2726 mbuf_data_size[mp_n]; 2727 } 2728 } 2729 rx_conf->rx_nseg = rx_pkt_nb_segs; 2730 rx_conf->rx_seg = rx_useg; 2731 rx_conf->rx_mempools = NULL; 2732 rx_conf->rx_nmempool = 0; 2733 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2734 socket_id, rx_conf, NULL); 2735 rx_conf->rx_seg = NULL; 2736 rx_conf->rx_nseg = 0; 2737 } else if (multi_rx_mempool == 1) { 2738 /* multi-pool configuration */ 2739 struct rte_eth_dev_info dev_info; 2740 2741 if (mbuf_data_size_n <= 1) { 2742 fprintf(stderr, "Invalid number of mempools %u\n", 2743 mbuf_data_size_n); 2744 return -EINVAL; 2745 } 2746 ret = rte_eth_dev_info_get(port_id, &dev_info); 2747 if (ret != 0) 2748 return ret; 2749 if (dev_info.max_rx_mempools == 0) { 2750 fprintf(stderr, 2751 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2752 port_id); 2753 return -ENOTSUP; 2754 } 2755 for (i = 0; i < mbuf_data_size_n; i++) { 2756 mpx = mbuf_pool_find(socket_id, i); 2757 rx_mempool[i] = mpx ? mpx : mp; 2758 } 2759 rx_conf->rx_mempools = rx_mempool; 2760 rx_conf->rx_nmempool = mbuf_data_size_n; 2761 rx_conf->rx_seg = NULL; 2762 rx_conf->rx_nseg = 0; 2763 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2764 socket_id, rx_conf, NULL); 2765 rx_conf->rx_mempools = NULL; 2766 rx_conf->rx_nmempool = 0; 2767 } else { 2768 /* Single pool/segment configuration */ 2769 rx_conf->rx_seg = NULL; 2770 rx_conf->rx_nseg = 0; 2771 rx_conf->rx_mempools = NULL; 2772 rx_conf->rx_nmempool = 0; 2773 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2774 socket_id, rx_conf, mp); 2775 } 2776 2777 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2778 RTE_ETH_QUEUE_STATE_STOPPED : 2779 RTE_ETH_QUEUE_STATE_STARTED; 2780 return ret; 2781 } 2782 2783 static int 2784 alloc_xstats_display_info(portid_t pi) 2785 { 2786 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2787 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2788 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2789 2790 if (xstats_display_num == 0) 2791 return 0; 2792 2793 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2794 if (*ids_supp == NULL) 2795 goto fail_ids_supp; 2796 2797 *prev_values = calloc(xstats_display_num, 2798 sizeof(**prev_values)); 2799 if (*prev_values == NULL) 2800 goto fail_prev_values; 2801 2802 *curr_values = calloc(xstats_display_num, 2803 sizeof(**curr_values)); 2804 if (*curr_values == NULL) 2805 goto fail_curr_values; 2806 2807 ports[pi].xstats_info.allocated = true; 2808 2809 return 0; 2810 2811 fail_curr_values: 2812 free(*prev_values); 2813 fail_prev_values: 2814 free(*ids_supp); 2815 fail_ids_supp: 2816 return -ENOMEM; 2817 } 2818 2819 static void 2820 free_xstats_display_info(portid_t pi) 2821 { 2822 if (!ports[pi].xstats_info.allocated) 2823 return; 2824 free(ports[pi].xstats_info.ids_supp); 2825 free(ports[pi].xstats_info.prev_values); 2826 free(ports[pi].xstats_info.curr_values); 2827 ports[pi].xstats_info.allocated = false; 2828 } 2829 2830 /** Fill helper structures for specified port to show extended statistics. */ 2831 static void 2832 fill_xstats_display_info_for_port(portid_t pi) 2833 { 2834 unsigned int stat, stat_supp; 2835 const char *xstat_name; 2836 struct rte_port *port; 2837 uint64_t *ids_supp; 2838 int rc; 2839 2840 if (xstats_display_num == 0) 2841 return; 2842 2843 if (pi == (portid_t)RTE_PORT_ALL) { 2844 fill_xstats_display_info(); 2845 return; 2846 } 2847 2848 port = &ports[pi]; 2849 if (port->port_status != RTE_PORT_STARTED) 2850 return; 2851 2852 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2853 rte_exit(EXIT_FAILURE, 2854 "Failed to allocate xstats display memory\n"); 2855 2856 ids_supp = port->xstats_info.ids_supp; 2857 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2858 xstat_name = xstats_display[stat].name; 2859 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2860 ids_supp + stat_supp); 2861 if (rc != 0) { 2862 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2863 xstat_name, pi, stat); 2864 continue; 2865 } 2866 stat_supp++; 2867 } 2868 2869 port->xstats_info.ids_supp_sz = stat_supp; 2870 } 2871 2872 /** Fill helper structures for all ports to show extended statistics. */ 2873 static void 2874 fill_xstats_display_info(void) 2875 { 2876 portid_t pi; 2877 2878 if (xstats_display_num == 0) 2879 return; 2880 2881 RTE_ETH_FOREACH_DEV(pi) 2882 fill_xstats_display_info_for_port(pi); 2883 } 2884 2885 /* 2886 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2887 * device in dev_info is zero when no slave is added. And its capability 2888 * will be updated when add a new slave device. So adding a slave device need 2889 * to update the port configurations of bonding device. 2890 */ 2891 static void 2892 update_bonding_port_dev_conf(portid_t bond_pid) 2893 { 2894 #ifdef RTE_NET_BOND 2895 struct rte_port *port = &ports[bond_pid]; 2896 uint16_t i; 2897 int ret; 2898 2899 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2900 if (ret != 0) { 2901 fprintf(stderr, "Failed to get dev info for port = %u\n", 2902 bond_pid); 2903 return; 2904 } 2905 2906 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2907 port->dev_conf.txmode.offloads |= 2908 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2909 /* Apply Tx offloads configuration */ 2910 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2911 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2912 2913 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 2914 port->dev_info.flow_type_rss_offloads; 2915 #else 2916 RTE_SET_USED(bond_pid); 2917 #endif 2918 } 2919 2920 int 2921 start_port(portid_t pid) 2922 { 2923 int diag; 2924 portid_t pi; 2925 portid_t p_pi = RTE_MAX_ETHPORTS; 2926 portid_t pl[RTE_MAX_ETHPORTS]; 2927 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2928 uint16_t cnt_pi = 0; 2929 uint16_t cfg_pi = 0; 2930 int peer_pi; 2931 queueid_t qi; 2932 struct rte_port *port; 2933 struct rte_eth_hairpin_cap cap; 2934 bool at_least_one_port_exist = false; 2935 bool all_ports_already_started = true; 2936 bool at_least_one_port_successfully_started = false; 2937 2938 if (port_id_is_invalid(pid, ENABLED_WARN)) 2939 return 0; 2940 2941 RTE_ETH_FOREACH_DEV(pi) { 2942 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2943 continue; 2944 2945 if (port_is_bonding_slave(pi)) { 2946 fprintf(stderr, 2947 "Please remove port %d from bonded device.\n", 2948 pi); 2949 continue; 2950 } 2951 2952 at_least_one_port_exist = true; 2953 2954 port = &ports[pi]; 2955 if (port->port_status == RTE_PORT_STOPPED) { 2956 port->port_status = RTE_PORT_HANDLING; 2957 all_ports_already_started = false; 2958 } else { 2959 fprintf(stderr, "Port %d is now not stopped\n", pi); 2960 continue; 2961 } 2962 2963 if (port->need_reconfig > 0) { 2964 struct rte_eth_conf dev_conf; 2965 int k; 2966 2967 port->need_reconfig = 0; 2968 2969 if (flow_isolate_all) { 2970 int ret = port_flow_isolate(pi, 1); 2971 if (ret) { 2972 fprintf(stderr, 2973 "Failed to apply isolated mode on port %d\n", 2974 pi); 2975 return -1; 2976 } 2977 } 2978 configure_rxtx_dump_callbacks(0); 2979 printf("Configuring Port %d (socket %u)\n", pi, 2980 port->socket_id); 2981 if (nb_hairpinq > 0 && 2982 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2983 fprintf(stderr, 2984 "Port %d doesn't support hairpin queues\n", 2985 pi); 2986 return -1; 2987 } 2988 2989 if (port->bond_flag == 1 && port->update_conf == 1) { 2990 update_bonding_port_dev_conf(pi); 2991 port->update_conf = 0; 2992 } 2993 2994 /* configure port */ 2995 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2996 nb_txq + nb_hairpinq, 2997 &(port->dev_conf)); 2998 if (diag != 0) { 2999 if (port->port_status == RTE_PORT_HANDLING) 3000 port->port_status = RTE_PORT_STOPPED; 3001 else 3002 fprintf(stderr, 3003 "Port %d can not be set back to stopped\n", 3004 pi); 3005 fprintf(stderr, "Fail to configure port %d\n", 3006 pi); 3007 /* try to reconfigure port next time */ 3008 port->need_reconfig = 1; 3009 return -1; 3010 } 3011 /* get device configuration*/ 3012 if (0 != 3013 eth_dev_conf_get_print_err(pi, &dev_conf)) { 3014 fprintf(stderr, 3015 "port %d can not get device configuration\n", 3016 pi); 3017 return -1; 3018 } 3019 /* Apply Rx offloads configuration */ 3020 if (dev_conf.rxmode.offloads != 3021 port->dev_conf.rxmode.offloads) { 3022 port->dev_conf.rxmode.offloads |= 3023 dev_conf.rxmode.offloads; 3024 for (k = 0; 3025 k < port->dev_info.max_rx_queues; 3026 k++) 3027 port->rxq[k].conf.offloads |= 3028 dev_conf.rxmode.offloads; 3029 } 3030 /* Apply Tx offloads configuration */ 3031 if (dev_conf.txmode.offloads != 3032 port->dev_conf.txmode.offloads) { 3033 port->dev_conf.txmode.offloads |= 3034 dev_conf.txmode.offloads; 3035 for (k = 0; 3036 k < port->dev_info.max_tx_queues; 3037 k++) 3038 port->txq[k].conf.offloads |= 3039 dev_conf.txmode.offloads; 3040 } 3041 } 3042 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 3043 port->need_reconfig_queues = 0; 3044 /* setup tx queues */ 3045 for (qi = 0; qi < nb_txq; qi++) { 3046 struct rte_eth_txconf *conf = 3047 &port->txq[qi].conf; 3048 3049 if ((numa_support) && 3050 (txring_numa[pi] != NUMA_NO_CONFIG)) 3051 diag = rte_eth_tx_queue_setup(pi, qi, 3052 port->nb_tx_desc[qi], 3053 txring_numa[pi], 3054 &(port->txq[qi].conf)); 3055 else 3056 diag = rte_eth_tx_queue_setup(pi, qi, 3057 port->nb_tx_desc[qi], 3058 port->socket_id, 3059 &(port->txq[qi].conf)); 3060 3061 if (diag == 0) { 3062 port->txq[qi].state = 3063 conf->tx_deferred_start ? 3064 RTE_ETH_QUEUE_STATE_STOPPED : 3065 RTE_ETH_QUEUE_STATE_STARTED; 3066 continue; 3067 } 3068 3069 /* Fail to setup tx queue, return */ 3070 if (port->port_status == RTE_PORT_HANDLING) 3071 port->port_status = RTE_PORT_STOPPED; 3072 else 3073 fprintf(stderr, 3074 "Port %d can not be set back to stopped\n", 3075 pi); 3076 fprintf(stderr, 3077 "Fail to configure port %d tx queues\n", 3078 pi); 3079 /* try to reconfigure queues next time */ 3080 port->need_reconfig_queues = 1; 3081 return -1; 3082 } 3083 for (qi = 0; qi < nb_rxq; qi++) { 3084 /* setup rx queues */ 3085 if ((numa_support) && 3086 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3087 struct rte_mempool * mp = 3088 mbuf_pool_find 3089 (rxring_numa[pi], 0); 3090 if (mp == NULL) { 3091 fprintf(stderr, 3092 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3093 rxring_numa[pi]); 3094 return -1; 3095 } 3096 3097 diag = rx_queue_setup(pi, qi, 3098 port->nb_rx_desc[qi], 3099 rxring_numa[pi], 3100 &(port->rxq[qi].conf), 3101 mp); 3102 } else { 3103 struct rte_mempool *mp = 3104 mbuf_pool_find 3105 (port->socket_id, 0); 3106 if (mp == NULL) { 3107 fprintf(stderr, 3108 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3109 port->socket_id); 3110 return -1; 3111 } 3112 diag = rx_queue_setup(pi, qi, 3113 port->nb_rx_desc[qi], 3114 port->socket_id, 3115 &(port->rxq[qi].conf), 3116 mp); 3117 } 3118 if (diag == 0) 3119 continue; 3120 3121 /* Fail to setup rx queue, return */ 3122 if (port->port_status == RTE_PORT_HANDLING) 3123 port->port_status = RTE_PORT_STOPPED; 3124 else 3125 fprintf(stderr, 3126 "Port %d can not be set back to stopped\n", 3127 pi); 3128 fprintf(stderr, 3129 "Fail to configure port %d rx queues\n", 3130 pi); 3131 /* try to reconfigure queues next time */ 3132 port->need_reconfig_queues = 1; 3133 return -1; 3134 } 3135 /* setup hairpin queues */ 3136 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3137 return -1; 3138 } 3139 configure_rxtx_dump_callbacks(verbose_level); 3140 if (clear_ptypes) { 3141 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3142 NULL, 0); 3143 if (diag < 0) 3144 fprintf(stderr, 3145 "Port %d: Failed to disable Ptype parsing\n", 3146 pi); 3147 } 3148 3149 p_pi = pi; 3150 cnt_pi++; 3151 3152 /* start port */ 3153 diag = eth_dev_start_mp(pi); 3154 if (diag < 0) { 3155 fprintf(stderr, "Fail to start port %d: %s\n", 3156 pi, rte_strerror(-diag)); 3157 3158 /* Fail to setup rx queue, return */ 3159 if (port->port_status == RTE_PORT_HANDLING) 3160 port->port_status = RTE_PORT_STOPPED; 3161 else 3162 fprintf(stderr, 3163 "Port %d can not be set back to stopped\n", 3164 pi); 3165 continue; 3166 } 3167 3168 if (port->port_status == RTE_PORT_HANDLING) 3169 port->port_status = RTE_PORT_STARTED; 3170 else 3171 fprintf(stderr, "Port %d can not be set into started\n", 3172 pi); 3173 3174 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3175 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3176 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3177 3178 at_least_one_port_successfully_started = true; 3179 3180 pl[cfg_pi++] = pi; 3181 } 3182 3183 if (at_least_one_port_successfully_started && !no_link_check) 3184 check_all_ports_link_status(RTE_PORT_ALL); 3185 else if (at_least_one_port_exist & all_ports_already_started) 3186 fprintf(stderr, "Please stop the ports first\n"); 3187 3188 if (hairpin_mode & 0xf) { 3189 uint16_t i; 3190 int j; 3191 3192 /* bind all started hairpin ports */ 3193 for (i = 0; i < cfg_pi; i++) { 3194 pi = pl[i]; 3195 /* bind current Tx to all peer Rx */ 3196 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3197 RTE_MAX_ETHPORTS, 1); 3198 if (peer_pi < 0) 3199 return peer_pi; 3200 for (j = 0; j < peer_pi; j++) { 3201 if (!port_is_started(peer_pl[j])) 3202 continue; 3203 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3204 if (diag < 0) { 3205 fprintf(stderr, 3206 "Error during binding hairpin Tx port %u to %u: %s\n", 3207 pi, peer_pl[j], 3208 rte_strerror(-diag)); 3209 return -1; 3210 } 3211 } 3212 /* bind all peer Tx to current Rx */ 3213 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3214 RTE_MAX_ETHPORTS, 0); 3215 if (peer_pi < 0) 3216 return peer_pi; 3217 for (j = 0; j < peer_pi; j++) { 3218 if (!port_is_started(peer_pl[j])) 3219 continue; 3220 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3221 if (diag < 0) { 3222 fprintf(stderr, 3223 "Error during binding hairpin Tx port %u to %u: %s\n", 3224 peer_pl[j], pi, 3225 rte_strerror(-diag)); 3226 return -1; 3227 } 3228 } 3229 } 3230 } 3231 3232 fill_xstats_display_info_for_port(pid); 3233 3234 printf("Done\n"); 3235 return 0; 3236 } 3237 3238 void 3239 stop_port(portid_t pid) 3240 { 3241 portid_t pi; 3242 struct rte_port *port; 3243 int need_check_link_status = 0; 3244 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3245 int peer_pi; 3246 int ret; 3247 3248 if (port_id_is_invalid(pid, ENABLED_WARN)) 3249 return; 3250 3251 printf("Stopping ports...\n"); 3252 3253 RTE_ETH_FOREACH_DEV(pi) { 3254 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3255 continue; 3256 3257 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3258 fprintf(stderr, 3259 "Please remove port %d from forwarding configuration.\n", 3260 pi); 3261 continue; 3262 } 3263 3264 if (port_is_bonding_slave(pi)) { 3265 fprintf(stderr, 3266 "Please remove port %d from bonded device.\n", 3267 pi); 3268 continue; 3269 } 3270 3271 port = &ports[pi]; 3272 if (port->port_status == RTE_PORT_STARTED) 3273 port->port_status = RTE_PORT_HANDLING; 3274 else 3275 continue; 3276 3277 if (hairpin_mode & 0xf) { 3278 int j; 3279 3280 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3281 /* unbind all peer Tx from current Rx */ 3282 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3283 RTE_MAX_ETHPORTS, 0); 3284 if (peer_pi < 0) 3285 continue; 3286 for (j = 0; j < peer_pi; j++) { 3287 if (!port_is_started(peer_pl[j])) 3288 continue; 3289 rte_eth_hairpin_unbind(peer_pl[j], pi); 3290 } 3291 } 3292 3293 if (port->flow_list && !no_flow_flush) 3294 port_flow_flush(pi); 3295 3296 ret = eth_dev_stop_mp(pi); 3297 if (ret != 0) { 3298 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3299 pi); 3300 /* Allow to retry stopping the port. */ 3301 port->port_status = RTE_PORT_STARTED; 3302 continue; 3303 } 3304 3305 if (port->port_status == RTE_PORT_HANDLING) 3306 port->port_status = RTE_PORT_STOPPED; 3307 else 3308 fprintf(stderr, "Port %d can not be set into stopped\n", 3309 pi); 3310 need_check_link_status = 1; 3311 } 3312 if (need_check_link_status && !no_link_check) 3313 check_all_ports_link_status(RTE_PORT_ALL); 3314 3315 printf("Done\n"); 3316 } 3317 3318 static void 3319 remove_invalid_ports_in(portid_t *array, portid_t *total) 3320 { 3321 portid_t i; 3322 portid_t new_total = 0; 3323 3324 for (i = 0; i < *total; i++) 3325 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3326 array[new_total] = array[i]; 3327 new_total++; 3328 } 3329 *total = new_total; 3330 } 3331 3332 static void 3333 remove_invalid_ports(void) 3334 { 3335 remove_invalid_ports_in(ports_ids, &nb_ports); 3336 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3337 nb_cfg_ports = nb_fwd_ports; 3338 } 3339 3340 static void 3341 flush_port_owned_resources(portid_t pi) 3342 { 3343 mcast_addr_pool_destroy(pi); 3344 port_flow_flush(pi); 3345 port_flow_template_table_flush(pi); 3346 port_flow_pattern_template_flush(pi); 3347 port_flow_actions_template_flush(pi); 3348 port_flex_item_flush(pi); 3349 port_action_handle_flush(pi); 3350 } 3351 3352 static void 3353 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3354 { 3355 struct rte_port *port; 3356 portid_t slave_pid; 3357 uint16_t i; 3358 3359 for (i = 0; i < num_slaves; i++) { 3360 slave_pid = slave_pids[i]; 3361 if (port_is_started(slave_pid) == 1) { 3362 if (rte_eth_dev_stop(slave_pid) != 0) 3363 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3364 slave_pid); 3365 3366 port = &ports[slave_pid]; 3367 port->port_status = RTE_PORT_STOPPED; 3368 } 3369 3370 clear_port_slave_flag(slave_pid); 3371 3372 /* Close slave device when testpmd quit or is killed. */ 3373 if (cl_quit == 1 || f_quit == 1) 3374 rte_eth_dev_close(slave_pid); 3375 } 3376 } 3377 3378 void 3379 close_port(portid_t pid) 3380 { 3381 portid_t pi; 3382 struct rte_port *port; 3383 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3384 int num_slaves = 0; 3385 3386 if (port_id_is_invalid(pid, ENABLED_WARN)) 3387 return; 3388 3389 printf("Closing ports...\n"); 3390 3391 RTE_ETH_FOREACH_DEV(pi) { 3392 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3393 continue; 3394 3395 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3396 fprintf(stderr, 3397 "Please remove port %d from forwarding configuration.\n", 3398 pi); 3399 continue; 3400 } 3401 3402 if (port_is_bonding_slave(pi)) { 3403 fprintf(stderr, 3404 "Please remove port %d from bonded device.\n", 3405 pi); 3406 continue; 3407 } 3408 3409 port = &ports[pi]; 3410 if (port->port_status == RTE_PORT_CLOSED) { 3411 fprintf(stderr, "Port %d is already closed\n", pi); 3412 continue; 3413 } 3414 3415 if (is_proc_primary()) { 3416 flush_port_owned_resources(pi); 3417 #ifdef RTE_NET_BOND 3418 if (port->bond_flag == 1) 3419 num_slaves = rte_eth_bond_slaves_get(pi, 3420 slave_pids, RTE_MAX_ETHPORTS); 3421 #endif 3422 rte_eth_dev_close(pi); 3423 /* 3424 * If this port is bonded device, all slaves under the 3425 * device need to be removed or closed. 3426 */ 3427 if (port->bond_flag == 1 && num_slaves > 0) 3428 clear_bonding_slave_device(slave_pids, 3429 num_slaves); 3430 } 3431 3432 free_xstats_display_info(pi); 3433 } 3434 3435 remove_invalid_ports(); 3436 printf("Done\n"); 3437 } 3438 3439 void 3440 reset_port(portid_t pid) 3441 { 3442 int diag; 3443 portid_t pi; 3444 struct rte_port *port; 3445 3446 if (port_id_is_invalid(pid, ENABLED_WARN)) 3447 return; 3448 3449 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3450 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3451 fprintf(stderr, 3452 "Can not reset port(s), please stop port(s) first.\n"); 3453 return; 3454 } 3455 3456 printf("Resetting ports...\n"); 3457 3458 RTE_ETH_FOREACH_DEV(pi) { 3459 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3460 continue; 3461 3462 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3463 fprintf(stderr, 3464 "Please remove port %d from forwarding configuration.\n", 3465 pi); 3466 continue; 3467 } 3468 3469 if (port_is_bonding_slave(pi)) { 3470 fprintf(stderr, 3471 "Please remove port %d from bonded device.\n", 3472 pi); 3473 continue; 3474 } 3475 3476 if (is_proc_primary()) { 3477 diag = rte_eth_dev_reset(pi); 3478 if (diag == 0) { 3479 port = &ports[pi]; 3480 port->need_reconfig = 1; 3481 port->need_reconfig_queues = 1; 3482 } else { 3483 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3484 pi, diag); 3485 } 3486 } 3487 } 3488 3489 printf("Done\n"); 3490 } 3491 3492 void 3493 attach_port(char *identifier) 3494 { 3495 portid_t pi; 3496 struct rte_dev_iterator iterator; 3497 3498 printf("Attaching a new port...\n"); 3499 3500 if (identifier == NULL) { 3501 fprintf(stderr, "Invalid parameters are specified\n"); 3502 return; 3503 } 3504 3505 if (rte_dev_probe(identifier) < 0) { 3506 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3507 return; 3508 } 3509 3510 /* first attach mode: event */ 3511 if (setup_on_probe_event) { 3512 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3513 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3514 if (ports[pi].port_status == RTE_PORT_HANDLING && 3515 ports[pi].need_setup != 0) 3516 setup_attached_port(pi); 3517 return; 3518 } 3519 3520 /* second attach mode: iterator */ 3521 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3522 /* setup ports matching the devargs used for probing */ 3523 if (port_is_forwarding(pi)) 3524 continue; /* port was already attached before */ 3525 setup_attached_port(pi); 3526 } 3527 } 3528 3529 static void 3530 setup_attached_port(portid_t pi) 3531 { 3532 unsigned int socket_id; 3533 int ret; 3534 3535 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3536 /* if socket_id is invalid, set to the first available socket. */ 3537 if (check_socket_id(socket_id) < 0) 3538 socket_id = socket_ids[0]; 3539 reconfig(pi, socket_id); 3540 ret = rte_eth_promiscuous_enable(pi); 3541 if (ret != 0) 3542 fprintf(stderr, 3543 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3544 pi, rte_strerror(-ret)); 3545 3546 ports_ids[nb_ports++] = pi; 3547 fwd_ports_ids[nb_fwd_ports++] = pi; 3548 nb_cfg_ports = nb_fwd_ports; 3549 ports[pi].need_setup = 0; 3550 ports[pi].port_status = RTE_PORT_STOPPED; 3551 3552 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3553 printf("Done\n"); 3554 } 3555 3556 static void 3557 detach_device(struct rte_device *dev) 3558 { 3559 portid_t sibling; 3560 3561 if (dev == NULL) { 3562 fprintf(stderr, "Device already removed\n"); 3563 return; 3564 } 3565 3566 printf("Removing a device...\n"); 3567 3568 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3569 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3570 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3571 fprintf(stderr, "Port %u not stopped\n", 3572 sibling); 3573 return; 3574 } 3575 flush_port_owned_resources(sibling); 3576 } 3577 } 3578 3579 if (rte_dev_remove(dev) < 0) { 3580 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3581 return; 3582 } 3583 remove_invalid_ports(); 3584 3585 printf("Device is detached\n"); 3586 printf("Now total ports is %d\n", nb_ports); 3587 printf("Done\n"); 3588 return; 3589 } 3590 3591 void 3592 detach_port_device(portid_t port_id) 3593 { 3594 int ret; 3595 struct rte_eth_dev_info dev_info; 3596 3597 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3598 return; 3599 3600 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3601 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3602 fprintf(stderr, "Port not stopped\n"); 3603 return; 3604 } 3605 fprintf(stderr, "Port was not closed\n"); 3606 } 3607 3608 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3609 if (ret != 0) { 3610 TESTPMD_LOG(ERR, 3611 "Failed to get device info for port %d, not detaching\n", 3612 port_id); 3613 return; 3614 } 3615 detach_device(dev_info.device); 3616 } 3617 3618 void 3619 detach_devargs(char *identifier) 3620 { 3621 struct rte_dev_iterator iterator; 3622 struct rte_devargs da; 3623 portid_t port_id; 3624 3625 printf("Removing a device...\n"); 3626 3627 memset(&da, 0, sizeof(da)); 3628 if (rte_devargs_parsef(&da, "%s", identifier)) { 3629 fprintf(stderr, "cannot parse identifier\n"); 3630 return; 3631 } 3632 3633 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3634 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3635 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3636 fprintf(stderr, "Port %u not stopped\n", 3637 port_id); 3638 rte_eth_iterator_cleanup(&iterator); 3639 rte_devargs_reset(&da); 3640 return; 3641 } 3642 flush_port_owned_resources(port_id); 3643 } 3644 } 3645 3646 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3647 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3648 da.name, rte_bus_name(da.bus)); 3649 rte_devargs_reset(&da); 3650 return; 3651 } 3652 3653 remove_invalid_ports(); 3654 3655 printf("Device %s is detached\n", identifier); 3656 printf("Now total ports is %d\n", nb_ports); 3657 printf("Done\n"); 3658 rte_devargs_reset(&da); 3659 } 3660 3661 void 3662 pmd_test_exit(void) 3663 { 3664 portid_t pt_id; 3665 unsigned int i; 3666 int ret; 3667 3668 if (test_done == 0) 3669 stop_packet_forwarding(); 3670 3671 #ifndef RTE_EXEC_ENV_WINDOWS 3672 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3673 if (mempools[i]) { 3674 if (mp_alloc_type == MP_ALLOC_ANON) 3675 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3676 NULL); 3677 } 3678 } 3679 #endif 3680 if (ports != NULL) { 3681 no_link_check = 1; 3682 RTE_ETH_FOREACH_DEV(pt_id) { 3683 printf("\nStopping port %d...\n", pt_id); 3684 fflush(stdout); 3685 stop_port(pt_id); 3686 } 3687 RTE_ETH_FOREACH_DEV(pt_id) { 3688 printf("\nShutting down port %d...\n", pt_id); 3689 fflush(stdout); 3690 close_port(pt_id); 3691 } 3692 } 3693 3694 if (hot_plug) { 3695 ret = rte_dev_event_monitor_stop(); 3696 if (ret) { 3697 RTE_LOG(ERR, EAL, 3698 "fail to stop device event monitor."); 3699 return; 3700 } 3701 3702 ret = rte_dev_event_callback_unregister(NULL, 3703 dev_event_callback, NULL); 3704 if (ret < 0) { 3705 RTE_LOG(ERR, EAL, 3706 "fail to unregister device event callback.\n"); 3707 return; 3708 } 3709 3710 ret = rte_dev_hotplug_handle_disable(); 3711 if (ret) { 3712 RTE_LOG(ERR, EAL, 3713 "fail to disable hotplug handling.\n"); 3714 return; 3715 } 3716 } 3717 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3718 if (mempools[i]) 3719 mempool_free_mp(mempools[i]); 3720 } 3721 free(xstats_display); 3722 3723 printf("\nBye...\n"); 3724 } 3725 3726 typedef void (*cmd_func_t)(void); 3727 struct pmd_test_command { 3728 const char *cmd_name; 3729 cmd_func_t cmd_func; 3730 }; 3731 3732 /* Check the link status of all ports in up to 9s, and print them finally */ 3733 static void 3734 check_all_ports_link_status(uint32_t port_mask) 3735 { 3736 #define CHECK_INTERVAL 100 /* 100ms */ 3737 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3738 portid_t portid; 3739 uint8_t count, all_ports_up, print_flag = 0; 3740 struct rte_eth_link link; 3741 int ret; 3742 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3743 3744 printf("Checking link statuses...\n"); 3745 fflush(stdout); 3746 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3747 all_ports_up = 1; 3748 RTE_ETH_FOREACH_DEV(portid) { 3749 if ((port_mask & (1 << portid)) == 0) 3750 continue; 3751 memset(&link, 0, sizeof(link)); 3752 ret = rte_eth_link_get_nowait(portid, &link); 3753 if (ret < 0) { 3754 all_ports_up = 0; 3755 if (print_flag == 1) 3756 fprintf(stderr, 3757 "Port %u link get failed: %s\n", 3758 portid, rte_strerror(-ret)); 3759 continue; 3760 } 3761 /* print link status if flag set */ 3762 if (print_flag == 1) { 3763 rte_eth_link_to_str(link_status, 3764 sizeof(link_status), &link); 3765 printf("Port %d %s\n", portid, link_status); 3766 continue; 3767 } 3768 /* clear all_ports_up flag if any link down */ 3769 if (link.link_status == RTE_ETH_LINK_DOWN) { 3770 all_ports_up = 0; 3771 break; 3772 } 3773 } 3774 /* after finally printing all link status, get out */ 3775 if (print_flag == 1) 3776 break; 3777 3778 if (all_ports_up == 0) { 3779 fflush(stdout); 3780 rte_delay_ms(CHECK_INTERVAL); 3781 } 3782 3783 /* set the print_flag if all ports up or timeout */ 3784 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3785 print_flag = 1; 3786 } 3787 3788 if (lsc_interrupt) 3789 break; 3790 } 3791 } 3792 3793 static void 3794 rmv_port_callback(void *arg) 3795 { 3796 int need_to_start = 0; 3797 int org_no_link_check = no_link_check; 3798 portid_t port_id = (intptr_t)arg; 3799 struct rte_eth_dev_info dev_info; 3800 int ret; 3801 3802 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3803 3804 if (!test_done && port_is_forwarding(port_id)) { 3805 need_to_start = 1; 3806 stop_packet_forwarding(); 3807 } 3808 no_link_check = 1; 3809 stop_port(port_id); 3810 no_link_check = org_no_link_check; 3811 3812 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3813 if (ret != 0) 3814 TESTPMD_LOG(ERR, 3815 "Failed to get device info for port %d, not detaching\n", 3816 port_id); 3817 else { 3818 struct rte_device *device = dev_info.device; 3819 close_port(port_id); 3820 detach_device(device); /* might be already removed or have more ports */ 3821 } 3822 if (need_to_start) 3823 start_packet_forwarding(0); 3824 } 3825 3826 /* This function is used by the interrupt thread */ 3827 static int 3828 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3829 void *ret_param) 3830 { 3831 RTE_SET_USED(param); 3832 RTE_SET_USED(ret_param); 3833 3834 if (type >= RTE_ETH_EVENT_MAX) { 3835 fprintf(stderr, 3836 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3837 port_id, __func__, type); 3838 fflush(stderr); 3839 } else if (event_print_mask & (UINT32_C(1) << type)) { 3840 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3841 eth_event_desc[type]); 3842 fflush(stdout); 3843 } 3844 3845 switch (type) { 3846 case RTE_ETH_EVENT_NEW: 3847 ports[port_id].need_setup = 1; 3848 ports[port_id].port_status = RTE_PORT_HANDLING; 3849 break; 3850 case RTE_ETH_EVENT_INTR_RMV: 3851 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3852 break; 3853 if (rte_eal_alarm_set(100000, 3854 rmv_port_callback, (void *)(intptr_t)port_id)) 3855 fprintf(stderr, 3856 "Could not set up deferred device removal\n"); 3857 break; 3858 case RTE_ETH_EVENT_DESTROY: 3859 ports[port_id].port_status = RTE_PORT_CLOSED; 3860 printf("Port %u is closed\n", port_id); 3861 break; 3862 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3863 uint16_t rxq_id; 3864 int ret; 3865 3866 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3867 for (rxq_id = 0; ; rxq_id++) { 3868 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3869 NULL); 3870 if (ret <= 0) 3871 break; 3872 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3873 port_id, rxq_id); 3874 3875 #ifdef RTE_NET_MLX5 3876 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3877 #endif 3878 } 3879 break; 3880 } 3881 default: 3882 break; 3883 } 3884 return 0; 3885 } 3886 3887 static int 3888 register_eth_event_callback(void) 3889 { 3890 int ret; 3891 enum rte_eth_event_type event; 3892 3893 for (event = RTE_ETH_EVENT_UNKNOWN; 3894 event < RTE_ETH_EVENT_MAX; event++) { 3895 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3896 event, 3897 eth_event_callback, 3898 NULL); 3899 if (ret != 0) { 3900 TESTPMD_LOG(ERR, "Failed to register callback for " 3901 "%s event\n", eth_event_desc[event]); 3902 return -1; 3903 } 3904 } 3905 3906 return 0; 3907 } 3908 3909 /* This function is used by the interrupt thread */ 3910 static void 3911 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3912 __rte_unused void *arg) 3913 { 3914 uint16_t port_id; 3915 int ret; 3916 3917 if (type >= RTE_DEV_EVENT_MAX) { 3918 fprintf(stderr, "%s called upon invalid event %d\n", 3919 __func__, type); 3920 fflush(stderr); 3921 } 3922 3923 switch (type) { 3924 case RTE_DEV_EVENT_REMOVE: 3925 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3926 device_name); 3927 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3928 if (ret) { 3929 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3930 device_name); 3931 return; 3932 } 3933 /* 3934 * Because the user's callback is invoked in eal interrupt 3935 * callback, the interrupt callback need to be finished before 3936 * it can be unregistered when detaching device. So finish 3937 * callback soon and use a deferred removal to detach device 3938 * is need. It is a workaround, once the device detaching be 3939 * moved into the eal in the future, the deferred removal could 3940 * be deleted. 3941 */ 3942 if (rte_eal_alarm_set(100000, 3943 rmv_port_callback, (void *)(intptr_t)port_id)) 3944 RTE_LOG(ERR, EAL, 3945 "Could not set up deferred device removal\n"); 3946 break; 3947 case RTE_DEV_EVENT_ADD: 3948 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3949 device_name); 3950 /* TODO: After finish kernel driver binding, 3951 * begin to attach port. 3952 */ 3953 break; 3954 default: 3955 break; 3956 } 3957 } 3958 3959 static void 3960 rxtx_port_config(portid_t pid) 3961 { 3962 uint16_t qid; 3963 uint64_t offloads; 3964 struct rte_port *port = &ports[pid]; 3965 3966 for (qid = 0; qid < nb_rxq; qid++) { 3967 offloads = port->rxq[qid].conf.offloads; 3968 port->rxq[qid].conf = port->dev_info.default_rxconf; 3969 3970 if (rxq_share > 0 && 3971 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3972 /* Non-zero share group to enable RxQ share. */ 3973 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3974 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3975 } 3976 3977 if (offloads != 0) 3978 port->rxq[qid].conf.offloads = offloads; 3979 3980 /* Check if any Rx parameters have been passed */ 3981 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3982 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3983 3984 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3985 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3986 3987 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3988 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3989 3990 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3991 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3992 3993 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3994 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3995 3996 port->nb_rx_desc[qid] = nb_rxd; 3997 } 3998 3999 for (qid = 0; qid < nb_txq; qid++) { 4000 offloads = port->txq[qid].conf.offloads; 4001 port->txq[qid].conf = port->dev_info.default_txconf; 4002 if (offloads != 0) 4003 port->txq[qid].conf.offloads = offloads; 4004 4005 /* Check if any Tx parameters have been passed */ 4006 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 4007 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 4008 4009 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 4010 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 4011 4012 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 4013 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 4014 4015 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 4016 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 4017 4018 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 4019 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 4020 4021 port->nb_tx_desc[qid] = nb_txd; 4022 } 4023 } 4024 4025 /* 4026 * Helper function to set MTU from frame size 4027 * 4028 * port->dev_info should be set before calling this function. 4029 * 4030 * return 0 on success, negative on error 4031 */ 4032 int 4033 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 4034 { 4035 struct rte_port *port = &ports[portid]; 4036 uint32_t eth_overhead; 4037 uint16_t mtu, new_mtu; 4038 4039 eth_overhead = get_eth_overhead(&port->dev_info); 4040 4041 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 4042 printf("Failed to get MTU for port %u\n", portid); 4043 return -1; 4044 } 4045 4046 new_mtu = max_rx_pktlen - eth_overhead; 4047 4048 if (mtu == new_mtu) 4049 return 0; 4050 4051 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 4052 fprintf(stderr, 4053 "Failed to set MTU to %u for port %u\n", 4054 new_mtu, portid); 4055 return -1; 4056 } 4057 4058 port->dev_conf.rxmode.mtu = new_mtu; 4059 4060 return 0; 4061 } 4062 4063 void 4064 init_port_config(void) 4065 { 4066 portid_t pid; 4067 struct rte_port *port; 4068 int ret, i; 4069 4070 RTE_ETH_FOREACH_DEV(pid) { 4071 port = &ports[pid]; 4072 4073 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4074 if (ret != 0) 4075 return; 4076 4077 if (nb_rxq > 1) { 4078 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4079 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4080 rss_hf & port->dev_info.flow_type_rss_offloads; 4081 } else { 4082 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4083 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4084 } 4085 4086 if (port->dcb_flag == 0) { 4087 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4088 port->dev_conf.rxmode.mq_mode = 4089 (enum rte_eth_rx_mq_mode) 4090 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4091 } else { 4092 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4093 port->dev_conf.rxmode.offloads &= 4094 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4095 4096 for (i = 0; 4097 i < port->dev_info.nb_rx_queues; 4098 i++) 4099 port->rxq[i].conf.offloads &= 4100 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4101 } 4102 } 4103 4104 rxtx_port_config(pid); 4105 4106 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4107 if (ret != 0) 4108 return; 4109 4110 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4111 port->dev_conf.intr_conf.lsc = 1; 4112 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4113 port->dev_conf.intr_conf.rmv = 1; 4114 } 4115 } 4116 4117 void set_port_slave_flag(portid_t slave_pid) 4118 { 4119 struct rte_port *port; 4120 4121 port = &ports[slave_pid]; 4122 port->slave_flag = 1; 4123 } 4124 4125 void clear_port_slave_flag(portid_t slave_pid) 4126 { 4127 struct rte_port *port; 4128 4129 port = &ports[slave_pid]; 4130 port->slave_flag = 0; 4131 } 4132 4133 uint8_t port_is_bonding_slave(portid_t slave_pid) 4134 { 4135 struct rte_port *port; 4136 struct rte_eth_dev_info dev_info; 4137 int ret; 4138 4139 port = &ports[slave_pid]; 4140 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4141 if (ret != 0) { 4142 TESTPMD_LOG(ERR, 4143 "Failed to get device info for port id %d," 4144 "cannot determine if the port is a bonded slave", 4145 slave_pid); 4146 return 0; 4147 } 4148 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4149 return 1; 4150 return 0; 4151 } 4152 4153 const uint16_t vlan_tags[] = { 4154 0, 1, 2, 3, 4, 5, 6, 7, 4155 8, 9, 10, 11, 12, 13, 14, 15, 4156 16, 17, 18, 19, 20, 21, 22, 23, 4157 24, 25, 26, 27, 28, 29, 30, 31 4158 }; 4159 4160 static int 4161 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4162 enum dcb_mode_enable dcb_mode, 4163 enum rte_eth_nb_tcs num_tcs, 4164 uint8_t pfc_en) 4165 { 4166 uint8_t i; 4167 int32_t rc; 4168 struct rte_eth_rss_conf rss_conf; 4169 4170 /* 4171 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4172 * given above, and the number of traffic classes available for use. 4173 */ 4174 if (dcb_mode == DCB_VT_ENABLED) { 4175 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4176 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4177 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4178 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4179 4180 /* VMDQ+DCB RX and TX configurations */ 4181 vmdq_rx_conf->enable_default_pool = 0; 4182 vmdq_rx_conf->default_pool = 0; 4183 vmdq_rx_conf->nb_queue_pools = 4184 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4185 vmdq_tx_conf->nb_queue_pools = 4186 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4187 4188 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4189 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4190 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4191 vmdq_rx_conf->pool_map[i].pools = 4192 1 << (i % vmdq_rx_conf->nb_queue_pools); 4193 } 4194 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4195 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4196 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4197 } 4198 4199 /* set DCB mode of RX and TX of multiple queues */ 4200 eth_conf->rxmode.mq_mode = 4201 (enum rte_eth_rx_mq_mode) 4202 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4203 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4204 } else { 4205 struct rte_eth_dcb_rx_conf *rx_conf = 4206 ð_conf->rx_adv_conf.dcb_rx_conf; 4207 struct rte_eth_dcb_tx_conf *tx_conf = 4208 ð_conf->tx_adv_conf.dcb_tx_conf; 4209 4210 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4211 4212 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4213 if (rc != 0) 4214 return rc; 4215 4216 rx_conf->nb_tcs = num_tcs; 4217 tx_conf->nb_tcs = num_tcs; 4218 4219 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4220 rx_conf->dcb_tc[i] = i % num_tcs; 4221 tx_conf->dcb_tc[i] = i % num_tcs; 4222 } 4223 4224 eth_conf->rxmode.mq_mode = 4225 (enum rte_eth_rx_mq_mode) 4226 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4227 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4228 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4229 } 4230 4231 if (pfc_en) 4232 eth_conf->dcb_capability_en = 4233 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4234 else 4235 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4236 4237 return 0; 4238 } 4239 4240 int 4241 init_port_dcb_config(portid_t pid, 4242 enum dcb_mode_enable dcb_mode, 4243 enum rte_eth_nb_tcs num_tcs, 4244 uint8_t pfc_en) 4245 { 4246 struct rte_eth_conf port_conf; 4247 struct rte_port *rte_port; 4248 int retval; 4249 uint16_t i; 4250 4251 if (num_procs > 1) { 4252 printf("The multi-process feature doesn't support dcb.\n"); 4253 return -ENOTSUP; 4254 } 4255 rte_port = &ports[pid]; 4256 4257 /* retain the original device configuration. */ 4258 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4259 4260 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4261 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4262 if (retval < 0) 4263 return retval; 4264 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4265 /* remove RSS HASH offload for DCB in vt mode */ 4266 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4267 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4268 for (i = 0; i < nb_rxq; i++) 4269 rte_port->rxq[i].conf.offloads &= 4270 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4271 } 4272 4273 /* re-configure the device . */ 4274 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4275 if (retval < 0) 4276 return retval; 4277 4278 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4279 if (retval != 0) 4280 return retval; 4281 4282 /* If dev_info.vmdq_pool_base is greater than 0, 4283 * the queue id of vmdq pools is started after pf queues. 4284 */ 4285 if (dcb_mode == DCB_VT_ENABLED && 4286 rte_port->dev_info.vmdq_pool_base > 0) { 4287 fprintf(stderr, 4288 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4289 pid); 4290 return -1; 4291 } 4292 4293 /* Assume the ports in testpmd have the same dcb capability 4294 * and has the same number of rxq and txq in dcb mode 4295 */ 4296 if (dcb_mode == DCB_VT_ENABLED) { 4297 if (rte_port->dev_info.max_vfs > 0) { 4298 nb_rxq = rte_port->dev_info.nb_rx_queues; 4299 nb_txq = rte_port->dev_info.nb_tx_queues; 4300 } else { 4301 nb_rxq = rte_port->dev_info.max_rx_queues; 4302 nb_txq = rte_port->dev_info.max_tx_queues; 4303 } 4304 } else { 4305 /*if vt is disabled, use all pf queues */ 4306 if (rte_port->dev_info.vmdq_pool_base == 0) { 4307 nb_rxq = rte_port->dev_info.max_rx_queues; 4308 nb_txq = rte_port->dev_info.max_tx_queues; 4309 } else { 4310 nb_rxq = (queueid_t)num_tcs; 4311 nb_txq = (queueid_t)num_tcs; 4312 4313 } 4314 } 4315 rx_free_thresh = 64; 4316 4317 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4318 4319 rxtx_port_config(pid); 4320 /* VLAN filter */ 4321 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4322 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4323 rx_vft_set(pid, vlan_tags[i], 1); 4324 4325 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4326 if (retval != 0) 4327 return retval; 4328 4329 rte_port->dcb_flag = 1; 4330 4331 /* Enter DCB configuration status */ 4332 dcb_config = 1; 4333 4334 return 0; 4335 } 4336 4337 static void 4338 init_port(void) 4339 { 4340 int i; 4341 4342 /* Configuration of Ethernet ports. */ 4343 ports = rte_zmalloc("testpmd: ports", 4344 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4345 RTE_CACHE_LINE_SIZE); 4346 if (ports == NULL) { 4347 rte_exit(EXIT_FAILURE, 4348 "rte_zmalloc(%d struct rte_port) failed\n", 4349 RTE_MAX_ETHPORTS); 4350 } 4351 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4352 ports[i].fwd_mac_swap = 1; 4353 ports[i].xstats_info.allocated = false; 4354 LIST_INIT(&ports[i].flow_tunnel_list); 4355 } 4356 /* Initialize ports NUMA structures */ 4357 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4358 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4359 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4360 } 4361 4362 static void 4363 force_quit(void) 4364 { 4365 pmd_test_exit(); 4366 prompt_exit(); 4367 } 4368 4369 static void 4370 print_stats(void) 4371 { 4372 uint8_t i; 4373 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4374 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4375 4376 /* Clear screen and move to top left */ 4377 printf("%s%s", clr, top_left); 4378 4379 printf("\nPort statistics ===================================="); 4380 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4381 nic_stats_display(fwd_ports_ids[i]); 4382 4383 fflush(stdout); 4384 } 4385 4386 static void 4387 signal_handler(int signum) 4388 { 4389 if (signum == SIGINT || signum == SIGTERM) { 4390 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4391 signum); 4392 #ifdef RTE_LIB_PDUMP 4393 /* uninitialize packet capture framework */ 4394 rte_pdump_uninit(); 4395 #endif 4396 #ifdef RTE_LIB_LATENCYSTATS 4397 if (latencystats_enabled != 0) 4398 rte_latencystats_uninit(); 4399 #endif 4400 force_quit(); 4401 /* Set flag to indicate the force termination. */ 4402 f_quit = 1; 4403 /* exit with the expected status */ 4404 #ifndef RTE_EXEC_ENV_WINDOWS 4405 signal(signum, SIG_DFL); 4406 kill(getpid(), signum); 4407 #endif 4408 } 4409 } 4410 4411 int 4412 main(int argc, char** argv) 4413 { 4414 int diag; 4415 portid_t port_id; 4416 uint16_t count; 4417 int ret; 4418 4419 signal(SIGINT, signal_handler); 4420 signal(SIGTERM, signal_handler); 4421 4422 testpmd_logtype = rte_log_register("testpmd"); 4423 if (testpmd_logtype < 0) 4424 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4425 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4426 4427 diag = rte_eal_init(argc, argv); 4428 if (diag < 0) 4429 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4430 rte_strerror(rte_errno)); 4431 4432 /* allocate port structures, and init them */ 4433 init_port(); 4434 4435 ret = register_eth_event_callback(); 4436 if (ret != 0) 4437 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4438 4439 #ifdef RTE_LIB_PDUMP 4440 /* initialize packet capture framework */ 4441 rte_pdump_init(); 4442 #endif 4443 4444 count = 0; 4445 RTE_ETH_FOREACH_DEV(port_id) { 4446 ports_ids[count] = port_id; 4447 count++; 4448 } 4449 nb_ports = (portid_t) count; 4450 if (nb_ports == 0) 4451 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4452 4453 set_def_fwd_config(); 4454 if (nb_lcores == 0) 4455 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4456 "Check the core mask argument\n"); 4457 4458 /* Bitrate/latency stats disabled by default */ 4459 #ifdef RTE_LIB_BITRATESTATS 4460 bitrate_enabled = 0; 4461 #endif 4462 #ifdef RTE_LIB_LATENCYSTATS 4463 latencystats_enabled = 0; 4464 #endif 4465 4466 /* on FreeBSD, mlockall() is disabled by default */ 4467 #ifdef RTE_EXEC_ENV_FREEBSD 4468 do_mlockall = 0; 4469 #else 4470 do_mlockall = 1; 4471 #endif 4472 4473 argc -= diag; 4474 argv += diag; 4475 if (argc > 1) 4476 launch_args_parse(argc, argv); 4477 4478 #ifndef RTE_EXEC_ENV_WINDOWS 4479 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4480 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4481 strerror(errno)); 4482 } 4483 #endif 4484 4485 if (tx_first && interactive) 4486 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4487 "interactive mode.\n"); 4488 4489 if (tx_first && lsc_interrupt) { 4490 fprintf(stderr, 4491 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4492 lsc_interrupt = 0; 4493 } 4494 4495 if (!nb_rxq && !nb_txq) 4496 fprintf(stderr, 4497 "Warning: Either rx or tx queues should be non-zero\n"); 4498 4499 if (nb_rxq > 1 && nb_rxq > nb_txq) 4500 fprintf(stderr, 4501 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4502 nb_rxq, nb_txq); 4503 4504 init_config(); 4505 4506 if (hot_plug) { 4507 ret = rte_dev_hotplug_handle_enable(); 4508 if (ret) { 4509 RTE_LOG(ERR, EAL, 4510 "fail to enable hotplug handling."); 4511 return -1; 4512 } 4513 4514 ret = rte_dev_event_monitor_start(); 4515 if (ret) { 4516 RTE_LOG(ERR, EAL, 4517 "fail to start device event monitoring."); 4518 return -1; 4519 } 4520 4521 ret = rte_dev_event_callback_register(NULL, 4522 dev_event_callback, NULL); 4523 if (ret) { 4524 RTE_LOG(ERR, EAL, 4525 "fail to register device event callback\n"); 4526 return -1; 4527 } 4528 } 4529 4530 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { 4531 if (!interactive) { 4532 rte_eal_cleanup(); 4533 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4534 } 4535 fprintf(stderr, "Start ports failed\n"); 4536 } 4537 4538 /* set all ports to promiscuous mode by default */ 4539 RTE_ETH_FOREACH_DEV(port_id) { 4540 ret = rte_eth_promiscuous_enable(port_id); 4541 if (ret != 0) 4542 fprintf(stderr, 4543 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4544 port_id, rte_strerror(-ret)); 4545 } 4546 4547 #ifdef RTE_LIB_METRICS 4548 /* Init metrics library */ 4549 rte_metrics_init(rte_socket_id()); 4550 #endif 4551 4552 #ifdef RTE_LIB_LATENCYSTATS 4553 if (latencystats_enabled != 0) { 4554 int ret = rte_latencystats_init(1, NULL); 4555 if (ret) 4556 fprintf(stderr, 4557 "Warning: latencystats init() returned error %d\n", 4558 ret); 4559 fprintf(stderr, "Latencystats running on lcore %d\n", 4560 latencystats_lcore_id); 4561 } 4562 #endif 4563 4564 /* Setup bitrate stats */ 4565 #ifdef RTE_LIB_BITRATESTATS 4566 if (bitrate_enabled != 0) { 4567 bitrate_data = rte_stats_bitrate_create(); 4568 if (bitrate_data == NULL) 4569 rte_exit(EXIT_FAILURE, 4570 "Could not allocate bitrate data.\n"); 4571 rte_stats_bitrate_reg(bitrate_data); 4572 } 4573 #endif 4574 4575 if (record_core_cycles) 4576 rte_lcore_register_usage_cb(lcore_usage_callback); 4577 4578 #ifdef RTE_LIB_CMDLINE 4579 if (init_cmdline() != 0) 4580 rte_exit(EXIT_FAILURE, 4581 "Could not initialise cmdline context.\n"); 4582 4583 if (strlen(cmdline_filename) != 0) 4584 cmdline_read_from_file(cmdline_filename); 4585 4586 if (interactive == 1) { 4587 if (auto_start) { 4588 printf("Start automatic packet forwarding\n"); 4589 start_packet_forwarding(0); 4590 } 4591 prompt(); 4592 pmd_test_exit(); 4593 } else 4594 #endif 4595 { 4596 char c; 4597 int rc; 4598 4599 f_quit = 0; 4600 4601 printf("No commandline core given, start packet forwarding\n"); 4602 start_packet_forwarding(tx_first); 4603 if (stats_period != 0) { 4604 uint64_t prev_time = 0, cur_time, diff_time = 0; 4605 uint64_t timer_period; 4606 4607 /* Convert to number of cycles */ 4608 timer_period = stats_period * rte_get_timer_hz(); 4609 4610 while (f_quit == 0) { 4611 cur_time = rte_get_timer_cycles(); 4612 diff_time += cur_time - prev_time; 4613 4614 if (diff_time >= timer_period) { 4615 print_stats(); 4616 /* Reset the timer */ 4617 diff_time = 0; 4618 } 4619 /* Sleep to avoid unnecessary checks */ 4620 prev_time = cur_time; 4621 rte_delay_us_sleep(US_PER_S); 4622 } 4623 } 4624 4625 printf("Press enter to exit\n"); 4626 rc = read(0, &c, 1); 4627 pmd_test_exit(); 4628 if (rc < 0) 4629 return 1; 4630 } 4631 4632 ret = rte_eal_cleanup(); 4633 if (ret != 0) 4634 rte_exit(EXIT_FAILURE, 4635 "EAL cleanup failed: %s\n", strerror(-ret)); 4636 4637 return EXIT_SUCCESS; 4638 } 4639