1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #include <sys/select.h> 15 #endif 16 #include <sys/types.h> 17 #include <errno.h> 18 #include <stdbool.h> 19 20 #include <sys/queue.h> 21 #include <sys/stat.h> 22 23 #include <stdint.h> 24 #include <unistd.h> 25 #include <inttypes.h> 26 27 #include <rte_common.h> 28 #include <rte_errno.h> 29 #include <rte_byteorder.h> 30 #include <rte_log.h> 31 #include <rte_debug.h> 32 #include <rte_cycles.h> 33 #include <rte_memory.h> 34 #include <rte_memcpy.h> 35 #include <rte_launch.h> 36 #include <rte_bus.h> 37 #include <rte_eal.h> 38 #include <rte_alarm.h> 39 #include <rte_per_lcore.h> 40 #include <rte_lcore.h> 41 #include <rte_branch_prediction.h> 42 #include <rte_mempool.h> 43 #include <rte_malloc.h> 44 #include <rte_mbuf.h> 45 #include <rte_mbuf_pool_ops.h> 46 #include <rte_interrupts.h> 47 #include <rte_ether.h> 48 #include <rte_ethdev.h> 49 #include <rte_dev.h> 50 #include <rte_string_fns.h> 51 #ifdef RTE_NET_IXGBE 52 #include <rte_pmd_ixgbe.h> 53 #endif 54 #ifdef RTE_LIB_PDUMP 55 #include <rte_pdump.h> 56 #endif 57 #include <rte_flow.h> 58 #ifdef RTE_LIB_METRICS 59 #include <rte_metrics.h> 60 #endif 61 #ifdef RTE_LIB_BITRATESTATS 62 #include <rte_bitrate.h> 63 #endif 64 #ifdef RTE_LIB_LATENCYSTATS 65 #include <rte_latencystats.h> 66 #endif 67 #ifdef RTE_EXEC_ENV_WINDOWS 68 #include <process.h> 69 #endif 70 #ifdef RTE_NET_BOND 71 #include <rte_eth_bond.h> 72 #endif 73 #ifdef RTE_NET_MLX5 74 #include "mlx5_testpmd.h" 75 #endif 76 77 #include "testpmd.h" 78 79 #ifndef MAP_HUGETLB 80 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 81 #define HUGE_FLAG (0x40000) 82 #else 83 #define HUGE_FLAG MAP_HUGETLB 84 #endif 85 86 #ifndef MAP_HUGE_SHIFT 87 /* older kernels (or FreeBSD) will not have this define */ 88 #define HUGE_SHIFT (26) 89 #else 90 #define HUGE_SHIFT MAP_HUGE_SHIFT 91 #endif 92 93 #define EXTMEM_HEAP_NAME "extmem" 94 /* 95 * Zone size with the malloc overhead (max of debug and release variants) 96 * must fit into the smallest supported hugepage size (2M), 97 * so that an IOVA-contiguous zone of this size can always be allocated 98 * if there are free 2M hugepages. 99 */ 100 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 101 102 uint16_t verbose_level = 0; /**< Silent by default. */ 103 int testpmd_logtype; /**< Log type for testpmd logs */ 104 105 /* use main core for command line ? */ 106 uint8_t interactive = 0; 107 uint8_t auto_start = 0; 108 uint8_t tx_first; 109 char cmdline_filename[PATH_MAX] = {0}; 110 111 /* 112 * NUMA support configuration. 113 * When set, the NUMA support attempts to dispatch the allocation of the 114 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 115 * probed ports among the CPU sockets 0 and 1. 116 * Otherwise, all memory is allocated from CPU socket 0. 117 */ 118 uint8_t numa_support = 1; /**< numa enabled by default */ 119 120 /* 121 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 122 * not configured. 123 */ 124 uint8_t socket_num = UMA_NO_CONFIG; 125 126 /* 127 * Select mempool allocation type: 128 * - native: use regular DPDK memory 129 * - anon: use regular DPDK memory to create mempool, but populate using 130 * anonymous memory (may not be IOVA-contiguous) 131 * - xmem: use externally allocated hugepage memory 132 */ 133 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 134 135 /* 136 * Store specified sockets on which memory pool to be used by ports 137 * is allocated. 138 */ 139 uint8_t port_numa[RTE_MAX_ETHPORTS]; 140 141 /* 142 * Store specified sockets on which RX ring to be used by ports 143 * is allocated. 144 */ 145 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 146 147 /* 148 * Store specified sockets on which TX ring to be used by ports 149 * is allocated. 150 */ 151 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 152 153 /* 154 * Record the Ethernet address of peer target ports to which packets are 155 * forwarded. 156 * Must be instantiated with the ethernet addresses of peer traffic generator 157 * ports. 158 */ 159 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 160 portid_t nb_peer_eth_addrs = 0; 161 162 /* 163 * Probed Target Environment. 164 */ 165 struct rte_port *ports; /**< For all probed ethernet ports. */ 166 portid_t nb_ports; /**< Number of probed ethernet ports. */ 167 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 168 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 169 170 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 171 172 /* 173 * Test Forwarding Configuration. 174 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 175 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 176 */ 177 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 178 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 179 portid_t nb_cfg_ports; /**< Number of configured ports. */ 180 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 181 182 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 183 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 184 185 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 186 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 187 188 /* 189 * Forwarding engines. 190 */ 191 struct fwd_engine * fwd_engines[] = { 192 &io_fwd_engine, 193 &mac_fwd_engine, 194 &mac_swap_engine, 195 &flow_gen_engine, 196 &rx_only_engine, 197 &tx_only_engine, 198 &csum_fwd_engine, 199 &icmp_echo_engine, 200 &noisy_vnf_engine, 201 &five_tuple_swap_fwd_engine, 202 #ifdef RTE_LIBRTE_IEEE1588 203 &ieee1588_fwd_engine, 204 #endif 205 &shared_rxq_engine, 206 NULL, 207 }; 208 209 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 210 uint16_t mempool_flags; 211 212 struct fwd_config cur_fwd_config; 213 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 214 uint32_t retry_enabled; 215 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 216 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 217 218 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 219 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 220 DEFAULT_MBUF_DATA_SIZE 221 }; /**< Mbuf data space size. */ 222 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 223 * specified on command-line. */ 224 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 225 226 /** Extended statistics to show. */ 227 struct rte_eth_xstat_name *xstats_display; 228 229 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 230 231 /* 232 * In container, it cannot terminate the process which running with 'stats-period' 233 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 234 */ 235 volatile uint8_t f_quit; 236 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 237 238 /* 239 * Max Rx frame size, set by '--max-pkt-len' parameter. 240 */ 241 uint32_t max_rx_pkt_len; 242 243 /* 244 * Configuration of packet segments used to scatter received packets 245 * if some of split features is configured. 246 */ 247 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 248 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 249 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 250 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 251 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 252 253 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 254 255 /* 256 * Configuration of packet segments used by the "txonly" processing engine. 257 */ 258 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 259 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 260 TXONLY_DEF_PACKET_LEN, 261 }; 262 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 263 264 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 265 /**< Split policy for packets to TX. */ 266 267 uint8_t txonly_multi_flow; 268 /**< Whether multiple flows are generated in TXONLY mode. */ 269 270 uint32_t tx_pkt_times_inter; 271 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 272 273 uint32_t tx_pkt_times_intra; 274 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 275 276 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 277 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 278 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 279 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 280 281 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 282 uint8_t dcb_config = 0; 283 284 /* 285 * Configurable number of RX/TX queues. 286 */ 287 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 288 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 289 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 290 291 /* 292 * Configurable number of RX/TX ring descriptors. 293 * Defaults are supplied by drivers via ethdev. 294 */ 295 #define RX_DESC_DEFAULT 0 296 #define TX_DESC_DEFAULT 0 297 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 298 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 299 300 #define RTE_PMD_PARAM_UNSET -1 301 /* 302 * Configurable values of RX and TX ring threshold registers. 303 */ 304 305 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 306 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 307 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 308 309 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 310 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 311 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 312 313 /* 314 * Configurable value of RX free threshold. 315 */ 316 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 317 318 /* 319 * Configurable value of RX drop enable. 320 */ 321 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 322 323 /* 324 * Configurable value of TX free threshold. 325 */ 326 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 327 328 /* 329 * Configurable value of TX RS bit threshold. 330 */ 331 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 332 333 /* 334 * Configurable value of buffered packets before sending. 335 */ 336 uint16_t noisy_tx_sw_bufsz; 337 338 /* 339 * Configurable value of packet buffer timeout. 340 */ 341 uint16_t noisy_tx_sw_buf_flush_time; 342 343 /* 344 * Configurable value for size of VNF internal memory area 345 * used for simulating noisy neighbour behaviour 346 */ 347 uint64_t noisy_lkup_mem_sz; 348 349 /* 350 * Configurable value of number of random writes done in 351 * VNF simulation memory area. 352 */ 353 uint64_t noisy_lkup_num_writes; 354 355 /* 356 * Configurable value of number of random reads done in 357 * VNF simulation memory area. 358 */ 359 uint64_t noisy_lkup_num_reads; 360 361 /* 362 * Configurable value of number of random reads/writes done in 363 * VNF simulation memory area. 364 */ 365 uint64_t noisy_lkup_num_reads_writes; 366 367 /* 368 * Receive Side Scaling (RSS) configuration. 369 */ 370 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 371 372 /* 373 * Port topology configuration 374 */ 375 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 376 377 /* 378 * Avoids to flush all the RX streams before starts forwarding. 379 */ 380 uint8_t no_flush_rx = 0; /* flush by default */ 381 382 /* 383 * Flow API isolated mode. 384 */ 385 uint8_t flow_isolate_all; 386 387 /* 388 * Disable port flow flush when stop port. 389 */ 390 uint8_t no_flow_flush = 0; /* do flow flush by default */ 391 392 /* 393 * Avoids to check link status when starting/stopping a port. 394 */ 395 uint8_t no_link_check = 0; /* check by default */ 396 397 /* 398 * Don't automatically start all ports in interactive mode. 399 */ 400 uint8_t no_device_start = 0; 401 402 /* 403 * Enable link status change notification 404 */ 405 uint8_t lsc_interrupt = 1; /* enabled by default */ 406 407 /* 408 * Enable device removal notification. 409 */ 410 uint8_t rmv_interrupt = 1; /* enabled by default */ 411 412 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 413 414 /* After attach, port setup is called on event or by iterator */ 415 bool setup_on_probe_event = true; 416 417 /* Clear ptypes on port initialization. */ 418 uint8_t clear_ptypes = true; 419 420 /* Hairpin ports configuration mode. */ 421 uint32_t hairpin_mode; 422 423 /* Pretty printing of ethdev events */ 424 static const char * const eth_event_desc[] = { 425 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 426 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 427 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 428 [RTE_ETH_EVENT_INTR_RESET] = "reset", 429 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 430 [RTE_ETH_EVENT_IPSEC] = "IPsec", 431 [RTE_ETH_EVENT_MACSEC] = "MACsec", 432 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 433 [RTE_ETH_EVENT_NEW] = "device probed", 434 [RTE_ETH_EVENT_DESTROY] = "device released", 435 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 436 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 437 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 438 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 439 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 440 [RTE_ETH_EVENT_MAX] = NULL, 441 }; 442 443 /* 444 * Display or mask ether events 445 * Default to all events except VF_MBOX 446 */ 447 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 448 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 449 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 450 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 451 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 452 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 453 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 454 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 455 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 456 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 457 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 458 /* 459 * Decide if all memory are locked for performance. 460 */ 461 int do_mlockall = 0; 462 463 #ifdef RTE_LIB_LATENCYSTATS 464 465 /* 466 * Set when latency stats is enabled in the commandline 467 */ 468 uint8_t latencystats_enabled; 469 470 /* 471 * Lcore ID to service latency statistics. 472 */ 473 lcoreid_t latencystats_lcore_id = -1; 474 475 #endif 476 477 /* 478 * Ethernet device configuration. 479 */ 480 struct rte_eth_rxmode rx_mode; 481 482 struct rte_eth_txmode tx_mode = { 483 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 484 }; 485 486 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 487 488 /* 489 * Display zero values by default for xstats 490 */ 491 uint8_t xstats_hide_zero; 492 493 /* 494 * Measure of CPU cycles disabled by default 495 */ 496 uint8_t record_core_cycles; 497 498 /* 499 * Display of RX and TX bursts disabled by default 500 */ 501 uint8_t record_burst_stats; 502 503 /* 504 * Number of ports per shared Rx queue group, 0 disable. 505 */ 506 uint32_t rxq_share; 507 508 unsigned int num_sockets = 0; 509 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 510 511 #ifdef RTE_LIB_BITRATESTATS 512 /* Bitrate statistics */ 513 struct rte_stats_bitrates *bitrate_data; 514 lcoreid_t bitrate_lcore_id; 515 uint8_t bitrate_enabled; 516 #endif 517 518 #ifdef RTE_LIB_GRO 519 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 520 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 521 #endif 522 523 /* 524 * hexadecimal bitmask of RX mq mode can be enabled. 525 */ 526 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 527 528 /* 529 * Used to set forced link speed 530 */ 531 uint32_t eth_link_speed; 532 533 /* 534 * ID of the current process in multi-process, used to 535 * configure the queues to be polled. 536 */ 537 int proc_id; 538 539 /* 540 * Number of processes in multi-process, used to 541 * configure the queues to be polled. 542 */ 543 unsigned int num_procs = 1; 544 545 static void 546 eth_rx_metadata_negotiate_mp(uint16_t port_id) 547 { 548 uint64_t rx_meta_features = 0; 549 int ret; 550 551 if (!is_proc_primary()) 552 return; 553 554 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 555 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 556 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 557 558 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 559 if (ret == 0) { 560 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 561 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 562 port_id); 563 } 564 565 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 566 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 567 port_id); 568 } 569 570 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 571 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 572 port_id); 573 } 574 } else if (ret != -ENOTSUP) { 575 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 576 port_id, rte_strerror(-ret)); 577 } 578 } 579 580 static int 581 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 582 const struct rte_eth_conf *dev_conf) 583 { 584 if (is_proc_primary()) 585 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 586 dev_conf); 587 return 0; 588 } 589 590 static int 591 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 592 { 593 #ifdef RTE_NET_BOND 594 595 portid_t slave_pids[RTE_MAX_ETHPORTS]; 596 struct rte_port *port; 597 int num_slaves; 598 portid_t slave_pid; 599 int i; 600 601 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 602 RTE_MAX_ETHPORTS); 603 if (num_slaves < 0) { 604 fprintf(stderr, "Failed to get slave list for port = %u\n", 605 bond_pid); 606 return num_slaves; 607 } 608 609 for (i = 0; i < num_slaves; i++) { 610 slave_pid = slave_pids[i]; 611 port = &ports[slave_pid]; 612 port->port_status = 613 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 614 } 615 #else 616 RTE_SET_USED(bond_pid); 617 RTE_SET_USED(is_stop); 618 #endif 619 return 0; 620 } 621 622 static int 623 eth_dev_start_mp(uint16_t port_id) 624 { 625 int ret; 626 627 if (is_proc_primary()) { 628 ret = rte_eth_dev_start(port_id); 629 if (ret != 0) 630 return ret; 631 632 struct rte_port *port = &ports[port_id]; 633 634 /* 635 * Starting a bonded port also starts all slaves under the bonded 636 * device. So if this port is bond device, we need to modify the 637 * port status of these slaves. 638 */ 639 if (port->bond_flag == 1) 640 return change_bonding_slave_port_status(port_id, false); 641 } 642 643 return 0; 644 } 645 646 static int 647 eth_dev_stop_mp(uint16_t port_id) 648 { 649 int ret; 650 651 if (is_proc_primary()) { 652 ret = rte_eth_dev_stop(port_id); 653 if (ret != 0) 654 return ret; 655 656 struct rte_port *port = &ports[port_id]; 657 658 /* 659 * Stopping a bonded port also stops all slaves under the bonded 660 * device. So if this port is bond device, we need to modify the 661 * port status of these slaves. 662 */ 663 if (port->bond_flag == 1) 664 return change_bonding_slave_port_status(port_id, true); 665 } 666 667 return 0; 668 } 669 670 static void 671 mempool_free_mp(struct rte_mempool *mp) 672 { 673 if (is_proc_primary()) 674 rte_mempool_free(mp); 675 } 676 677 static int 678 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 679 { 680 if (is_proc_primary()) 681 return rte_eth_dev_set_mtu(port_id, mtu); 682 683 return 0; 684 } 685 686 /* Forward function declarations */ 687 static void setup_attached_port(portid_t pi); 688 static void check_all_ports_link_status(uint32_t port_mask); 689 static int eth_event_callback(portid_t port_id, 690 enum rte_eth_event_type type, 691 void *param, void *ret_param); 692 static void dev_event_callback(const char *device_name, 693 enum rte_dev_event_type type, 694 void *param); 695 static void fill_xstats_display_info(void); 696 697 /* 698 * Check if all the ports are started. 699 * If yes, return positive value. If not, return zero. 700 */ 701 static int all_ports_started(void); 702 703 #ifdef RTE_LIB_GSO 704 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 705 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 706 #endif 707 708 /* Holds the registered mbuf dynamic flags names. */ 709 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 710 711 712 /* 713 * Helper function to check if socket is already discovered. 714 * If yes, return positive value. If not, return zero. 715 */ 716 int 717 new_socket_id(unsigned int socket_id) 718 { 719 unsigned int i; 720 721 for (i = 0; i < num_sockets; i++) { 722 if (socket_ids[i] == socket_id) 723 return 0; 724 } 725 return 1; 726 } 727 728 /* 729 * Setup default configuration. 730 */ 731 static void 732 set_default_fwd_lcores_config(void) 733 { 734 unsigned int i; 735 unsigned int nb_lc; 736 unsigned int sock_num; 737 738 nb_lc = 0; 739 for (i = 0; i < RTE_MAX_LCORE; i++) { 740 if (!rte_lcore_is_enabled(i)) 741 continue; 742 sock_num = rte_lcore_to_socket_id(i); 743 if (new_socket_id(sock_num)) { 744 if (num_sockets >= RTE_MAX_NUMA_NODES) { 745 rte_exit(EXIT_FAILURE, 746 "Total sockets greater than %u\n", 747 RTE_MAX_NUMA_NODES); 748 } 749 socket_ids[num_sockets++] = sock_num; 750 } 751 if (i == rte_get_main_lcore()) 752 continue; 753 fwd_lcores_cpuids[nb_lc++] = i; 754 } 755 nb_lcores = (lcoreid_t) nb_lc; 756 nb_cfg_lcores = nb_lcores; 757 nb_fwd_lcores = 1; 758 } 759 760 static void 761 set_def_peer_eth_addrs(void) 762 { 763 portid_t i; 764 765 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 766 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 767 peer_eth_addrs[i].addr_bytes[5] = i; 768 } 769 } 770 771 static void 772 set_default_fwd_ports_config(void) 773 { 774 portid_t pt_id; 775 int i = 0; 776 777 RTE_ETH_FOREACH_DEV(pt_id) { 778 fwd_ports_ids[i++] = pt_id; 779 780 /* Update sockets info according to the attached device */ 781 int socket_id = rte_eth_dev_socket_id(pt_id); 782 if (socket_id >= 0 && new_socket_id(socket_id)) { 783 if (num_sockets >= RTE_MAX_NUMA_NODES) { 784 rte_exit(EXIT_FAILURE, 785 "Total sockets greater than %u\n", 786 RTE_MAX_NUMA_NODES); 787 } 788 socket_ids[num_sockets++] = socket_id; 789 } 790 } 791 792 nb_cfg_ports = nb_ports; 793 nb_fwd_ports = nb_ports; 794 } 795 796 void 797 set_def_fwd_config(void) 798 { 799 set_default_fwd_lcores_config(); 800 set_def_peer_eth_addrs(); 801 set_default_fwd_ports_config(); 802 } 803 804 #ifndef RTE_EXEC_ENV_WINDOWS 805 /* extremely pessimistic estimation of memory required to create a mempool */ 806 static int 807 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 808 { 809 unsigned int n_pages, mbuf_per_pg, leftover; 810 uint64_t total_mem, mbuf_mem, obj_sz; 811 812 /* there is no good way to predict how much space the mempool will 813 * occupy because it will allocate chunks on the fly, and some of those 814 * will come from default DPDK memory while some will come from our 815 * external memory, so just assume 128MB will be enough for everyone. 816 */ 817 uint64_t hdr_mem = 128 << 20; 818 819 /* account for possible non-contiguousness */ 820 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 821 if (obj_sz > pgsz) { 822 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 823 return -1; 824 } 825 826 mbuf_per_pg = pgsz / obj_sz; 827 leftover = (nb_mbufs % mbuf_per_pg) > 0; 828 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 829 830 mbuf_mem = n_pages * pgsz; 831 832 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 833 834 if (total_mem > SIZE_MAX) { 835 TESTPMD_LOG(ERR, "Memory size too big\n"); 836 return -1; 837 } 838 *out = (size_t)total_mem; 839 840 return 0; 841 } 842 843 static int 844 pagesz_flags(uint64_t page_sz) 845 { 846 /* as per mmap() manpage, all page sizes are log2 of page size 847 * shifted by MAP_HUGE_SHIFT 848 */ 849 int log2 = rte_log2_u64(page_sz); 850 851 return (log2 << HUGE_SHIFT); 852 } 853 854 static void * 855 alloc_mem(size_t memsz, size_t pgsz, bool huge) 856 { 857 void *addr; 858 int flags; 859 860 /* allocate anonymous hugepages */ 861 flags = MAP_ANONYMOUS | MAP_PRIVATE; 862 if (huge) 863 flags |= HUGE_FLAG | pagesz_flags(pgsz); 864 865 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 866 if (addr == MAP_FAILED) 867 return NULL; 868 869 return addr; 870 } 871 872 struct extmem_param { 873 void *addr; 874 size_t len; 875 size_t pgsz; 876 rte_iova_t *iova_table; 877 unsigned int iova_table_len; 878 }; 879 880 static int 881 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 882 bool huge) 883 { 884 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 885 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 886 unsigned int cur_page, n_pages, pgsz_idx; 887 size_t mem_sz, cur_pgsz; 888 rte_iova_t *iovas = NULL; 889 void *addr; 890 int ret; 891 892 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 893 /* skip anything that is too big */ 894 if (pgsizes[pgsz_idx] > SIZE_MAX) 895 continue; 896 897 cur_pgsz = pgsizes[pgsz_idx]; 898 899 /* if we were told not to allocate hugepages, override */ 900 if (!huge) 901 cur_pgsz = sysconf(_SC_PAGESIZE); 902 903 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 904 if (ret < 0) { 905 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 906 return -1; 907 } 908 909 /* allocate our memory */ 910 addr = alloc_mem(mem_sz, cur_pgsz, huge); 911 912 /* if we couldn't allocate memory with a specified page size, 913 * that doesn't mean we can't do it with other page sizes, so 914 * try another one. 915 */ 916 if (addr == NULL) 917 continue; 918 919 /* store IOVA addresses for every page in this memory area */ 920 n_pages = mem_sz / cur_pgsz; 921 922 iovas = malloc(sizeof(*iovas) * n_pages); 923 924 if (iovas == NULL) { 925 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 926 goto fail; 927 } 928 /* lock memory if it's not huge pages */ 929 if (!huge) 930 mlock(addr, mem_sz); 931 932 /* populate IOVA addresses */ 933 for (cur_page = 0; cur_page < n_pages; cur_page++) { 934 rte_iova_t iova; 935 size_t offset; 936 void *cur; 937 938 offset = cur_pgsz * cur_page; 939 cur = RTE_PTR_ADD(addr, offset); 940 941 /* touch the page before getting its IOVA */ 942 *(volatile char *)cur = 0; 943 944 iova = rte_mem_virt2iova(cur); 945 946 iovas[cur_page] = iova; 947 } 948 949 break; 950 } 951 /* if we couldn't allocate anything */ 952 if (iovas == NULL) 953 return -1; 954 955 param->addr = addr; 956 param->len = mem_sz; 957 param->pgsz = cur_pgsz; 958 param->iova_table = iovas; 959 param->iova_table_len = n_pages; 960 961 return 0; 962 fail: 963 free(iovas); 964 if (addr) 965 munmap(addr, mem_sz); 966 967 return -1; 968 } 969 970 static int 971 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 972 { 973 struct extmem_param param; 974 int socket_id, ret; 975 976 memset(¶m, 0, sizeof(param)); 977 978 /* check if our heap exists */ 979 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 980 if (socket_id < 0) { 981 /* create our heap */ 982 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 983 if (ret < 0) { 984 TESTPMD_LOG(ERR, "Cannot create heap\n"); 985 return -1; 986 } 987 } 988 989 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 990 if (ret < 0) { 991 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 992 return -1; 993 } 994 995 /* we now have a valid memory area, so add it to heap */ 996 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 997 param.addr, param.len, param.iova_table, 998 param.iova_table_len, param.pgsz); 999 1000 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1001 1002 /* not needed any more */ 1003 free(param.iova_table); 1004 1005 if (ret < 0) { 1006 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1007 munmap(param.addr, param.len); 1008 return -1; 1009 } 1010 1011 /* success */ 1012 1013 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1014 param.len >> 20); 1015 1016 return 0; 1017 } 1018 static void 1019 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1020 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1021 { 1022 uint16_t pid = 0; 1023 int ret; 1024 1025 RTE_ETH_FOREACH_DEV(pid) { 1026 struct rte_eth_dev_info dev_info; 1027 1028 ret = eth_dev_info_get_print_err(pid, &dev_info); 1029 if (ret != 0) { 1030 TESTPMD_LOG(DEBUG, 1031 "unable to get device info for port %d on addr 0x%p," 1032 "mempool unmapping will not be performed\n", 1033 pid, memhdr->addr); 1034 continue; 1035 } 1036 1037 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1038 if (ret) { 1039 TESTPMD_LOG(DEBUG, 1040 "unable to DMA unmap addr 0x%p " 1041 "for device %s\n", 1042 memhdr->addr, rte_dev_name(dev_info.device)); 1043 } 1044 } 1045 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1046 if (ret) { 1047 TESTPMD_LOG(DEBUG, 1048 "unable to un-register addr 0x%p\n", memhdr->addr); 1049 } 1050 } 1051 1052 static void 1053 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1054 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1055 { 1056 uint16_t pid = 0; 1057 size_t page_size = sysconf(_SC_PAGESIZE); 1058 int ret; 1059 1060 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1061 page_size); 1062 if (ret) { 1063 TESTPMD_LOG(DEBUG, 1064 "unable to register addr 0x%p\n", memhdr->addr); 1065 return; 1066 } 1067 RTE_ETH_FOREACH_DEV(pid) { 1068 struct rte_eth_dev_info dev_info; 1069 1070 ret = eth_dev_info_get_print_err(pid, &dev_info); 1071 if (ret != 0) { 1072 TESTPMD_LOG(DEBUG, 1073 "unable to get device info for port %d on addr 0x%p," 1074 "mempool mapping will not be performed\n", 1075 pid, memhdr->addr); 1076 continue; 1077 } 1078 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1079 if (ret) { 1080 TESTPMD_LOG(DEBUG, 1081 "unable to DMA map addr 0x%p " 1082 "for device %s\n", 1083 memhdr->addr, rte_dev_name(dev_info.device)); 1084 } 1085 } 1086 } 1087 #endif 1088 1089 static unsigned int 1090 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1091 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1092 { 1093 struct rte_pktmbuf_extmem *xmem; 1094 unsigned int ext_num, zone_num, elt_num; 1095 uint16_t elt_size; 1096 1097 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1098 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1099 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1100 1101 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1102 if (xmem == NULL) { 1103 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1104 "external buffer descriptors\n"); 1105 *ext_mem = NULL; 1106 return 0; 1107 } 1108 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1109 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1110 const struct rte_memzone *mz; 1111 char mz_name[RTE_MEMZONE_NAMESIZE]; 1112 int ret; 1113 1114 ret = snprintf(mz_name, sizeof(mz_name), 1115 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1116 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1117 errno = ENAMETOOLONG; 1118 ext_num = 0; 1119 break; 1120 } 1121 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1122 socket_id, 1123 RTE_MEMZONE_IOVA_CONTIG | 1124 RTE_MEMZONE_1GB | 1125 RTE_MEMZONE_SIZE_HINT_ONLY); 1126 if (mz == NULL) { 1127 /* 1128 * The caller exits on external buffer creation 1129 * error, so there is no need to free memzones. 1130 */ 1131 errno = ENOMEM; 1132 ext_num = 0; 1133 break; 1134 } 1135 xseg->buf_ptr = mz->addr; 1136 xseg->buf_iova = mz->iova; 1137 xseg->buf_len = EXTBUF_ZONE_SIZE; 1138 xseg->elt_size = elt_size; 1139 } 1140 if (ext_num == 0 && xmem != NULL) { 1141 free(xmem); 1142 xmem = NULL; 1143 } 1144 *ext_mem = xmem; 1145 return ext_num; 1146 } 1147 1148 /* 1149 * Configuration initialisation done once at init time. 1150 */ 1151 static struct rte_mempool * 1152 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1153 unsigned int socket_id, uint16_t size_idx) 1154 { 1155 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1156 struct rte_mempool *rte_mp = NULL; 1157 #ifndef RTE_EXEC_ENV_WINDOWS 1158 uint32_t mb_size; 1159 1160 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1161 #endif 1162 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1163 if (!is_proc_primary()) { 1164 rte_mp = rte_mempool_lookup(pool_name); 1165 if (rte_mp == NULL) 1166 rte_exit(EXIT_FAILURE, 1167 "Get mbuf pool for socket %u failed: %s\n", 1168 socket_id, rte_strerror(rte_errno)); 1169 return rte_mp; 1170 } 1171 1172 TESTPMD_LOG(INFO, 1173 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1174 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1175 1176 switch (mp_alloc_type) { 1177 case MP_ALLOC_NATIVE: 1178 { 1179 /* wrapper to rte_mempool_create() */ 1180 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1181 rte_mbuf_best_mempool_ops()); 1182 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1183 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1184 break; 1185 } 1186 #ifndef RTE_EXEC_ENV_WINDOWS 1187 case MP_ALLOC_ANON: 1188 { 1189 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1190 mb_size, (unsigned int) mb_mempool_cache, 1191 sizeof(struct rte_pktmbuf_pool_private), 1192 socket_id, mempool_flags); 1193 if (rte_mp == NULL) 1194 goto err; 1195 1196 if (rte_mempool_populate_anon(rte_mp) == 0) { 1197 rte_mempool_free(rte_mp); 1198 rte_mp = NULL; 1199 goto err; 1200 } 1201 rte_pktmbuf_pool_init(rte_mp, NULL); 1202 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1203 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1204 break; 1205 } 1206 case MP_ALLOC_XMEM: 1207 case MP_ALLOC_XMEM_HUGE: 1208 { 1209 int heap_socket; 1210 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1211 1212 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1213 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1214 1215 heap_socket = 1216 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1217 if (heap_socket < 0) 1218 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1219 1220 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1221 rte_mbuf_best_mempool_ops()); 1222 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1223 mb_mempool_cache, 0, mbuf_seg_size, 1224 heap_socket); 1225 break; 1226 } 1227 #endif 1228 case MP_ALLOC_XBUF: 1229 { 1230 struct rte_pktmbuf_extmem *ext_mem; 1231 unsigned int ext_num; 1232 1233 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1234 socket_id, pool_name, &ext_mem); 1235 if (ext_num == 0) 1236 rte_exit(EXIT_FAILURE, 1237 "Can't create pinned data buffers\n"); 1238 1239 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1240 rte_mbuf_best_mempool_ops()); 1241 rte_mp = rte_pktmbuf_pool_create_extbuf 1242 (pool_name, nb_mbuf, mb_mempool_cache, 1243 0, mbuf_seg_size, socket_id, 1244 ext_mem, ext_num); 1245 free(ext_mem); 1246 break; 1247 } 1248 default: 1249 { 1250 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1251 } 1252 } 1253 1254 #ifndef RTE_EXEC_ENV_WINDOWS 1255 err: 1256 #endif 1257 if (rte_mp == NULL) { 1258 rte_exit(EXIT_FAILURE, 1259 "Creation of mbuf pool for socket %u failed: %s\n", 1260 socket_id, rte_strerror(rte_errno)); 1261 } else if (verbose_level > 0) { 1262 rte_mempool_dump(stdout, rte_mp); 1263 } 1264 return rte_mp; 1265 } 1266 1267 /* 1268 * Check given socket id is valid or not with NUMA mode, 1269 * if valid, return 0, else return -1 1270 */ 1271 static int 1272 check_socket_id(const unsigned int socket_id) 1273 { 1274 static int warning_once = 0; 1275 1276 if (new_socket_id(socket_id)) { 1277 if (!warning_once && numa_support) 1278 fprintf(stderr, 1279 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1280 warning_once = 1; 1281 return -1; 1282 } 1283 return 0; 1284 } 1285 1286 /* 1287 * Get the allowed maximum number of RX queues. 1288 * *pid return the port id which has minimal value of 1289 * max_rx_queues in all ports. 1290 */ 1291 queueid_t 1292 get_allowed_max_nb_rxq(portid_t *pid) 1293 { 1294 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1295 bool max_rxq_valid = false; 1296 portid_t pi; 1297 struct rte_eth_dev_info dev_info; 1298 1299 RTE_ETH_FOREACH_DEV(pi) { 1300 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1301 continue; 1302 1303 max_rxq_valid = true; 1304 if (dev_info.max_rx_queues < allowed_max_rxq) { 1305 allowed_max_rxq = dev_info.max_rx_queues; 1306 *pid = pi; 1307 } 1308 } 1309 return max_rxq_valid ? allowed_max_rxq : 0; 1310 } 1311 1312 /* 1313 * Check input rxq is valid or not. 1314 * If input rxq is not greater than any of maximum number 1315 * of RX queues of all ports, it is valid. 1316 * if valid, return 0, else return -1 1317 */ 1318 int 1319 check_nb_rxq(queueid_t rxq) 1320 { 1321 queueid_t allowed_max_rxq; 1322 portid_t pid = 0; 1323 1324 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1325 if (rxq > allowed_max_rxq) { 1326 fprintf(stderr, 1327 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1328 rxq, allowed_max_rxq, pid); 1329 return -1; 1330 } 1331 return 0; 1332 } 1333 1334 /* 1335 * Get the allowed maximum number of TX queues. 1336 * *pid return the port id which has minimal value of 1337 * max_tx_queues in all ports. 1338 */ 1339 queueid_t 1340 get_allowed_max_nb_txq(portid_t *pid) 1341 { 1342 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1343 bool max_txq_valid = false; 1344 portid_t pi; 1345 struct rte_eth_dev_info dev_info; 1346 1347 RTE_ETH_FOREACH_DEV(pi) { 1348 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1349 continue; 1350 1351 max_txq_valid = true; 1352 if (dev_info.max_tx_queues < allowed_max_txq) { 1353 allowed_max_txq = dev_info.max_tx_queues; 1354 *pid = pi; 1355 } 1356 } 1357 return max_txq_valid ? allowed_max_txq : 0; 1358 } 1359 1360 /* 1361 * Check input txq is valid or not. 1362 * If input txq is not greater than any of maximum number 1363 * of TX queues of all ports, it is valid. 1364 * if valid, return 0, else return -1 1365 */ 1366 int 1367 check_nb_txq(queueid_t txq) 1368 { 1369 queueid_t allowed_max_txq; 1370 portid_t pid = 0; 1371 1372 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1373 if (txq > allowed_max_txq) { 1374 fprintf(stderr, 1375 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1376 txq, allowed_max_txq, pid); 1377 return -1; 1378 } 1379 return 0; 1380 } 1381 1382 /* 1383 * Get the allowed maximum number of RXDs of every rx queue. 1384 * *pid return the port id which has minimal value of 1385 * max_rxd in all queues of all ports. 1386 */ 1387 static uint16_t 1388 get_allowed_max_nb_rxd(portid_t *pid) 1389 { 1390 uint16_t allowed_max_rxd = UINT16_MAX; 1391 portid_t pi; 1392 struct rte_eth_dev_info dev_info; 1393 1394 RTE_ETH_FOREACH_DEV(pi) { 1395 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1396 continue; 1397 1398 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1399 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1400 *pid = pi; 1401 } 1402 } 1403 return allowed_max_rxd; 1404 } 1405 1406 /* 1407 * Get the allowed minimal number of RXDs of every rx queue. 1408 * *pid return the port id which has minimal value of 1409 * min_rxd in all queues of all ports. 1410 */ 1411 static uint16_t 1412 get_allowed_min_nb_rxd(portid_t *pid) 1413 { 1414 uint16_t allowed_min_rxd = 0; 1415 portid_t pi; 1416 struct rte_eth_dev_info dev_info; 1417 1418 RTE_ETH_FOREACH_DEV(pi) { 1419 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1420 continue; 1421 1422 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1423 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1424 *pid = pi; 1425 } 1426 } 1427 1428 return allowed_min_rxd; 1429 } 1430 1431 /* 1432 * Check input rxd is valid or not. 1433 * If input rxd is not greater than any of maximum number 1434 * of RXDs of every Rx queues and is not less than any of 1435 * minimal number of RXDs of every Rx queues, it is valid. 1436 * if valid, return 0, else return -1 1437 */ 1438 int 1439 check_nb_rxd(queueid_t rxd) 1440 { 1441 uint16_t allowed_max_rxd; 1442 uint16_t allowed_min_rxd; 1443 portid_t pid = 0; 1444 1445 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1446 if (rxd > allowed_max_rxd) { 1447 fprintf(stderr, 1448 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1449 rxd, allowed_max_rxd, pid); 1450 return -1; 1451 } 1452 1453 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1454 if (rxd < allowed_min_rxd) { 1455 fprintf(stderr, 1456 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1457 rxd, allowed_min_rxd, pid); 1458 return -1; 1459 } 1460 1461 return 0; 1462 } 1463 1464 /* 1465 * Get the allowed maximum number of TXDs of every rx queues. 1466 * *pid return the port id which has minimal value of 1467 * max_txd in every tx queue. 1468 */ 1469 static uint16_t 1470 get_allowed_max_nb_txd(portid_t *pid) 1471 { 1472 uint16_t allowed_max_txd = UINT16_MAX; 1473 portid_t pi; 1474 struct rte_eth_dev_info dev_info; 1475 1476 RTE_ETH_FOREACH_DEV(pi) { 1477 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1478 continue; 1479 1480 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1481 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1482 *pid = pi; 1483 } 1484 } 1485 return allowed_max_txd; 1486 } 1487 1488 /* 1489 * Get the allowed maximum number of TXDs of every tx queues. 1490 * *pid return the port id which has minimal value of 1491 * min_txd in every tx queue. 1492 */ 1493 static uint16_t 1494 get_allowed_min_nb_txd(portid_t *pid) 1495 { 1496 uint16_t allowed_min_txd = 0; 1497 portid_t pi; 1498 struct rte_eth_dev_info dev_info; 1499 1500 RTE_ETH_FOREACH_DEV(pi) { 1501 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1502 continue; 1503 1504 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1505 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1506 *pid = pi; 1507 } 1508 } 1509 1510 return allowed_min_txd; 1511 } 1512 1513 /* 1514 * Check input txd is valid or not. 1515 * If input txd is not greater than any of maximum number 1516 * of TXDs of every Rx queues, it is valid. 1517 * if valid, return 0, else return -1 1518 */ 1519 int 1520 check_nb_txd(queueid_t txd) 1521 { 1522 uint16_t allowed_max_txd; 1523 uint16_t allowed_min_txd; 1524 portid_t pid = 0; 1525 1526 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1527 if (txd > allowed_max_txd) { 1528 fprintf(stderr, 1529 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1530 txd, allowed_max_txd, pid); 1531 return -1; 1532 } 1533 1534 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1535 if (txd < allowed_min_txd) { 1536 fprintf(stderr, 1537 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1538 txd, allowed_min_txd, pid); 1539 return -1; 1540 } 1541 return 0; 1542 } 1543 1544 1545 /* 1546 * Get the allowed maximum number of hairpin queues. 1547 * *pid return the port id which has minimal value of 1548 * max_hairpin_queues in all ports. 1549 */ 1550 queueid_t 1551 get_allowed_max_nb_hairpinq(portid_t *pid) 1552 { 1553 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1554 portid_t pi; 1555 struct rte_eth_hairpin_cap cap; 1556 1557 RTE_ETH_FOREACH_DEV(pi) { 1558 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1559 *pid = pi; 1560 return 0; 1561 } 1562 if (cap.max_nb_queues < allowed_max_hairpinq) { 1563 allowed_max_hairpinq = cap.max_nb_queues; 1564 *pid = pi; 1565 } 1566 } 1567 return allowed_max_hairpinq; 1568 } 1569 1570 /* 1571 * Check input hairpin is valid or not. 1572 * If input hairpin is not greater than any of maximum number 1573 * of hairpin queues of all ports, it is valid. 1574 * if valid, return 0, else return -1 1575 */ 1576 int 1577 check_nb_hairpinq(queueid_t hairpinq) 1578 { 1579 queueid_t allowed_max_hairpinq; 1580 portid_t pid = 0; 1581 1582 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1583 if (hairpinq > allowed_max_hairpinq) { 1584 fprintf(stderr, 1585 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1586 hairpinq, allowed_max_hairpinq, pid); 1587 return -1; 1588 } 1589 return 0; 1590 } 1591 1592 static int 1593 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1594 { 1595 uint32_t eth_overhead; 1596 1597 if (dev_info->max_mtu != UINT16_MAX && 1598 dev_info->max_rx_pktlen > dev_info->max_mtu) 1599 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1600 else 1601 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1602 1603 return eth_overhead; 1604 } 1605 1606 static void 1607 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1608 { 1609 struct rte_port *port = &ports[pid]; 1610 int ret; 1611 int i; 1612 1613 eth_rx_metadata_negotiate_mp(pid); 1614 1615 port->dev_conf.txmode = tx_mode; 1616 port->dev_conf.rxmode = rx_mode; 1617 1618 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1619 if (ret != 0) 1620 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1621 1622 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1623 port->dev_conf.txmode.offloads &= 1624 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1625 1626 /* Apply Rx offloads configuration */ 1627 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1628 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1629 /* Apply Tx offloads configuration */ 1630 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1631 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1632 1633 if (eth_link_speed) 1634 port->dev_conf.link_speeds = eth_link_speed; 1635 1636 if (max_rx_pkt_len) 1637 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1638 get_eth_overhead(&port->dev_info); 1639 1640 /* set flag to initialize port/queue */ 1641 port->need_reconfig = 1; 1642 port->need_reconfig_queues = 1; 1643 port->socket_id = socket_id; 1644 port->tx_metadata = 0; 1645 1646 /* 1647 * Check for maximum number of segments per MTU. 1648 * Accordingly update the mbuf data size. 1649 */ 1650 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1651 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1652 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1653 uint16_t mtu; 1654 1655 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1656 uint16_t data_size = (mtu + eth_overhead) / 1657 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1658 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1659 1660 if (buffer_size > mbuf_data_size[0]) { 1661 mbuf_data_size[0] = buffer_size; 1662 TESTPMD_LOG(WARNING, 1663 "Configured mbuf size of the first segment %hu\n", 1664 mbuf_data_size[0]); 1665 } 1666 } 1667 } 1668 } 1669 1670 static void 1671 init_config(void) 1672 { 1673 portid_t pid; 1674 struct rte_mempool *mbp; 1675 unsigned int nb_mbuf_per_pool; 1676 lcoreid_t lc_id; 1677 #ifdef RTE_LIB_GRO 1678 struct rte_gro_param gro_param; 1679 #endif 1680 #ifdef RTE_LIB_GSO 1681 uint32_t gso_types; 1682 #endif 1683 1684 /* Configuration of logical cores. */ 1685 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1686 sizeof(struct fwd_lcore *) * nb_lcores, 1687 RTE_CACHE_LINE_SIZE); 1688 if (fwd_lcores == NULL) { 1689 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1690 "failed\n", nb_lcores); 1691 } 1692 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1693 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1694 sizeof(struct fwd_lcore), 1695 RTE_CACHE_LINE_SIZE); 1696 if (fwd_lcores[lc_id] == NULL) { 1697 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1698 "failed\n"); 1699 } 1700 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1701 } 1702 1703 RTE_ETH_FOREACH_DEV(pid) { 1704 uint32_t socket_id; 1705 1706 if (numa_support) { 1707 socket_id = port_numa[pid]; 1708 if (port_numa[pid] == NUMA_NO_CONFIG) { 1709 socket_id = rte_eth_dev_socket_id(pid); 1710 1711 /* 1712 * if socket_id is invalid, 1713 * set to the first available socket. 1714 */ 1715 if (check_socket_id(socket_id) < 0) 1716 socket_id = socket_ids[0]; 1717 } 1718 } else { 1719 socket_id = (socket_num == UMA_NO_CONFIG) ? 1720 0 : socket_num; 1721 } 1722 /* Apply default TxRx configuration for all ports */ 1723 init_config_port_offloads(pid, socket_id); 1724 } 1725 /* 1726 * Create pools of mbuf. 1727 * If NUMA support is disabled, create a single pool of mbuf in 1728 * socket 0 memory by default. 1729 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1730 * 1731 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1732 * nb_txd can be configured at run time. 1733 */ 1734 if (param_total_num_mbufs) 1735 nb_mbuf_per_pool = param_total_num_mbufs; 1736 else { 1737 nb_mbuf_per_pool = RX_DESC_MAX + 1738 (nb_lcores * mb_mempool_cache) + 1739 TX_DESC_MAX + MAX_PKT_BURST; 1740 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1741 } 1742 1743 if (numa_support) { 1744 uint8_t i, j; 1745 1746 for (i = 0; i < num_sockets; i++) 1747 for (j = 0; j < mbuf_data_size_n; j++) 1748 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1749 mbuf_pool_create(mbuf_data_size[j], 1750 nb_mbuf_per_pool, 1751 socket_ids[i], j); 1752 } else { 1753 uint8_t i; 1754 1755 for (i = 0; i < mbuf_data_size_n; i++) 1756 mempools[i] = mbuf_pool_create 1757 (mbuf_data_size[i], 1758 nb_mbuf_per_pool, 1759 socket_num == UMA_NO_CONFIG ? 1760 0 : socket_num, i); 1761 } 1762 1763 init_port_config(); 1764 1765 #ifdef RTE_LIB_GSO 1766 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1767 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1768 #endif 1769 /* 1770 * Records which Mbuf pool to use by each logical core, if needed. 1771 */ 1772 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1773 mbp = mbuf_pool_find( 1774 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1775 1776 if (mbp == NULL) 1777 mbp = mbuf_pool_find(0, 0); 1778 fwd_lcores[lc_id]->mbp = mbp; 1779 #ifdef RTE_LIB_GSO 1780 /* initialize GSO context */ 1781 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1782 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1783 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1784 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1785 RTE_ETHER_CRC_LEN; 1786 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1787 #endif 1788 } 1789 1790 fwd_config_setup(); 1791 1792 #ifdef RTE_LIB_GRO 1793 /* create a gro context for each lcore */ 1794 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1795 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1796 gro_param.max_item_per_flow = MAX_PKT_BURST; 1797 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1798 gro_param.socket_id = rte_lcore_to_socket_id( 1799 fwd_lcores_cpuids[lc_id]); 1800 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1801 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1802 rte_exit(EXIT_FAILURE, 1803 "rte_gro_ctx_create() failed\n"); 1804 } 1805 } 1806 #endif 1807 } 1808 1809 1810 void 1811 reconfig(portid_t new_port_id, unsigned socket_id) 1812 { 1813 /* Reconfiguration of Ethernet ports. */ 1814 init_config_port_offloads(new_port_id, socket_id); 1815 init_port_config(); 1816 } 1817 1818 int 1819 init_fwd_streams(void) 1820 { 1821 portid_t pid; 1822 struct rte_port *port; 1823 streamid_t sm_id, nb_fwd_streams_new; 1824 queueid_t q; 1825 1826 /* set socket id according to numa or not */ 1827 RTE_ETH_FOREACH_DEV(pid) { 1828 port = &ports[pid]; 1829 if (nb_rxq > port->dev_info.max_rx_queues) { 1830 fprintf(stderr, 1831 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1832 nb_rxq, port->dev_info.max_rx_queues); 1833 return -1; 1834 } 1835 if (nb_txq > port->dev_info.max_tx_queues) { 1836 fprintf(stderr, 1837 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1838 nb_txq, port->dev_info.max_tx_queues); 1839 return -1; 1840 } 1841 if (numa_support) { 1842 if (port_numa[pid] != NUMA_NO_CONFIG) 1843 port->socket_id = port_numa[pid]; 1844 else { 1845 port->socket_id = rte_eth_dev_socket_id(pid); 1846 1847 /* 1848 * if socket_id is invalid, 1849 * set to the first available socket. 1850 */ 1851 if (check_socket_id(port->socket_id) < 0) 1852 port->socket_id = socket_ids[0]; 1853 } 1854 } 1855 else { 1856 if (socket_num == UMA_NO_CONFIG) 1857 port->socket_id = 0; 1858 else 1859 port->socket_id = socket_num; 1860 } 1861 } 1862 1863 q = RTE_MAX(nb_rxq, nb_txq); 1864 if (q == 0) { 1865 fprintf(stderr, 1866 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1867 return -1; 1868 } 1869 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1870 if (nb_fwd_streams_new == nb_fwd_streams) 1871 return 0; 1872 /* clear the old */ 1873 if (fwd_streams != NULL) { 1874 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1875 if (fwd_streams[sm_id] == NULL) 1876 continue; 1877 rte_free(fwd_streams[sm_id]); 1878 fwd_streams[sm_id] = NULL; 1879 } 1880 rte_free(fwd_streams); 1881 fwd_streams = NULL; 1882 } 1883 1884 /* init new */ 1885 nb_fwd_streams = nb_fwd_streams_new; 1886 if (nb_fwd_streams) { 1887 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1888 sizeof(struct fwd_stream *) * nb_fwd_streams, 1889 RTE_CACHE_LINE_SIZE); 1890 if (fwd_streams == NULL) 1891 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1892 " (struct fwd_stream *)) failed\n", 1893 nb_fwd_streams); 1894 1895 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1896 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1897 " struct fwd_stream", sizeof(struct fwd_stream), 1898 RTE_CACHE_LINE_SIZE); 1899 if (fwd_streams[sm_id] == NULL) 1900 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1901 "(struct fwd_stream) failed\n"); 1902 } 1903 } 1904 1905 return 0; 1906 } 1907 1908 static void 1909 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1910 { 1911 uint64_t total_burst, sburst; 1912 uint64_t nb_burst; 1913 uint64_t burst_stats[4]; 1914 uint16_t pktnb_stats[4]; 1915 uint16_t nb_pkt; 1916 int burst_percent[4], sburstp; 1917 int i; 1918 1919 /* 1920 * First compute the total number of packet bursts and the 1921 * two highest numbers of bursts of the same number of packets. 1922 */ 1923 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1924 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1925 1926 /* Show stats for 0 burst size always */ 1927 total_burst = pbs->pkt_burst_spread[0]; 1928 burst_stats[0] = pbs->pkt_burst_spread[0]; 1929 pktnb_stats[0] = 0; 1930 1931 /* Find the next 2 burst sizes with highest occurrences. */ 1932 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1933 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1934 1935 if (nb_burst == 0) 1936 continue; 1937 1938 total_burst += nb_burst; 1939 1940 if (nb_burst > burst_stats[1]) { 1941 burst_stats[2] = burst_stats[1]; 1942 pktnb_stats[2] = pktnb_stats[1]; 1943 burst_stats[1] = nb_burst; 1944 pktnb_stats[1] = nb_pkt; 1945 } else if (nb_burst > burst_stats[2]) { 1946 burst_stats[2] = nb_burst; 1947 pktnb_stats[2] = nb_pkt; 1948 } 1949 } 1950 if (total_burst == 0) 1951 return; 1952 1953 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1954 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1955 if (i == 3) { 1956 printf("%d%% of other]\n", 100 - sburstp); 1957 return; 1958 } 1959 1960 sburst += burst_stats[i]; 1961 if (sburst == total_burst) { 1962 printf("%d%% of %d pkts]\n", 1963 100 - sburstp, (int) pktnb_stats[i]); 1964 return; 1965 } 1966 1967 burst_percent[i] = 1968 (double)burst_stats[i] / total_burst * 100; 1969 printf("%d%% of %d pkts + ", 1970 burst_percent[i], (int) pktnb_stats[i]); 1971 sburstp += burst_percent[i]; 1972 } 1973 } 1974 1975 static void 1976 fwd_stream_stats_display(streamid_t stream_id) 1977 { 1978 struct fwd_stream *fs; 1979 static const char *fwd_top_stats_border = "-------"; 1980 1981 fs = fwd_streams[stream_id]; 1982 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1983 (fs->fwd_dropped == 0)) 1984 return; 1985 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1986 "TX Port=%2d/Queue=%2d %s\n", 1987 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1988 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1989 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1990 " TX-dropped: %-14"PRIu64, 1991 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1992 1993 /* if checksum mode */ 1994 if (cur_fwd_eng == &csum_fwd_engine) { 1995 printf(" RX- bad IP checksum: %-14"PRIu64 1996 " Rx- bad L4 checksum: %-14"PRIu64 1997 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1998 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1999 fs->rx_bad_outer_l4_csum); 2000 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2001 fs->rx_bad_outer_ip_csum); 2002 } else { 2003 printf("\n"); 2004 } 2005 2006 if (record_burst_stats) { 2007 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2008 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2009 } 2010 } 2011 2012 void 2013 fwd_stats_display(void) 2014 { 2015 static const char *fwd_stats_border = "----------------------"; 2016 static const char *acc_stats_border = "+++++++++++++++"; 2017 struct { 2018 struct fwd_stream *rx_stream; 2019 struct fwd_stream *tx_stream; 2020 uint64_t tx_dropped; 2021 uint64_t rx_bad_ip_csum; 2022 uint64_t rx_bad_l4_csum; 2023 uint64_t rx_bad_outer_l4_csum; 2024 uint64_t rx_bad_outer_ip_csum; 2025 } ports_stats[RTE_MAX_ETHPORTS]; 2026 uint64_t total_rx_dropped = 0; 2027 uint64_t total_tx_dropped = 0; 2028 uint64_t total_rx_nombuf = 0; 2029 struct rte_eth_stats stats; 2030 uint64_t fwd_cycles = 0; 2031 uint64_t total_recv = 0; 2032 uint64_t total_xmit = 0; 2033 struct rte_port *port; 2034 streamid_t sm_id; 2035 portid_t pt_id; 2036 int ret; 2037 int i; 2038 2039 memset(ports_stats, 0, sizeof(ports_stats)); 2040 2041 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2042 struct fwd_stream *fs = fwd_streams[sm_id]; 2043 2044 if (cur_fwd_config.nb_fwd_streams > 2045 cur_fwd_config.nb_fwd_ports) { 2046 fwd_stream_stats_display(sm_id); 2047 } else { 2048 ports_stats[fs->tx_port].tx_stream = fs; 2049 ports_stats[fs->rx_port].rx_stream = fs; 2050 } 2051 2052 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2053 2054 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2055 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2056 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2057 fs->rx_bad_outer_l4_csum; 2058 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2059 fs->rx_bad_outer_ip_csum; 2060 2061 if (record_core_cycles) 2062 fwd_cycles += fs->busy_cycles; 2063 } 2064 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2065 uint64_t tx_dropped = 0; 2066 2067 pt_id = fwd_ports_ids[i]; 2068 port = &ports[pt_id]; 2069 2070 ret = rte_eth_stats_get(pt_id, &stats); 2071 if (ret != 0) { 2072 fprintf(stderr, 2073 "%s: Error: failed to get stats (port %u): %d", 2074 __func__, pt_id, ret); 2075 continue; 2076 } 2077 stats.ipackets -= port->stats.ipackets; 2078 stats.opackets -= port->stats.opackets; 2079 stats.ibytes -= port->stats.ibytes; 2080 stats.obytes -= port->stats.obytes; 2081 stats.imissed -= port->stats.imissed; 2082 stats.oerrors -= port->stats.oerrors; 2083 stats.rx_nombuf -= port->stats.rx_nombuf; 2084 2085 total_recv += stats.ipackets; 2086 total_xmit += stats.opackets; 2087 total_rx_dropped += stats.imissed; 2088 tx_dropped += ports_stats[pt_id].tx_dropped; 2089 tx_dropped += stats.oerrors; 2090 total_tx_dropped += tx_dropped; 2091 total_rx_nombuf += stats.rx_nombuf; 2092 2093 printf("\n %s Forward statistics for port %-2d %s\n", 2094 fwd_stats_border, pt_id, fwd_stats_border); 2095 2096 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2097 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2098 stats.ipackets + stats.imissed); 2099 2100 if (cur_fwd_eng == &csum_fwd_engine) { 2101 printf(" Bad-ipcsum: %-14"PRIu64 2102 " Bad-l4csum: %-14"PRIu64 2103 "Bad-outer-l4csum: %-14"PRIu64"\n", 2104 ports_stats[pt_id].rx_bad_ip_csum, 2105 ports_stats[pt_id].rx_bad_l4_csum, 2106 ports_stats[pt_id].rx_bad_outer_l4_csum); 2107 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2108 ports_stats[pt_id].rx_bad_outer_ip_csum); 2109 } 2110 if (stats.ierrors + stats.rx_nombuf > 0) { 2111 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2112 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2113 } 2114 2115 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2116 "TX-total: %-"PRIu64"\n", 2117 stats.opackets, tx_dropped, 2118 stats.opackets + tx_dropped); 2119 2120 if (record_burst_stats) { 2121 if (ports_stats[pt_id].rx_stream) 2122 pkt_burst_stats_display("RX", 2123 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2124 if (ports_stats[pt_id].tx_stream) 2125 pkt_burst_stats_display("TX", 2126 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2127 } 2128 2129 printf(" %s--------------------------------%s\n", 2130 fwd_stats_border, fwd_stats_border); 2131 } 2132 2133 printf("\n %s Accumulated forward statistics for all ports" 2134 "%s\n", 2135 acc_stats_border, acc_stats_border); 2136 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2137 "%-"PRIu64"\n" 2138 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2139 "%-"PRIu64"\n", 2140 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2141 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2142 if (total_rx_nombuf > 0) 2143 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2144 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2145 "%s\n", 2146 acc_stats_border, acc_stats_border); 2147 if (record_core_cycles) { 2148 #define CYC_PER_MHZ 1E6 2149 if (total_recv > 0 || total_xmit > 0) { 2150 uint64_t total_pkts = 0; 2151 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2152 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2153 total_pkts = total_xmit; 2154 else 2155 total_pkts = total_recv; 2156 2157 printf("\n CPU cycles/packet=%.2F (busy cycles=" 2158 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2159 " MHz Clock\n", 2160 (double) fwd_cycles / total_pkts, 2161 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2162 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2163 } 2164 } 2165 } 2166 2167 void 2168 fwd_stats_reset(void) 2169 { 2170 streamid_t sm_id; 2171 portid_t pt_id; 2172 int ret; 2173 int i; 2174 2175 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2176 pt_id = fwd_ports_ids[i]; 2177 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2178 if (ret != 0) 2179 fprintf(stderr, 2180 "%s: Error: failed to clear stats (port %u):%d", 2181 __func__, pt_id, ret); 2182 } 2183 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2184 struct fwd_stream *fs = fwd_streams[sm_id]; 2185 2186 fs->rx_packets = 0; 2187 fs->tx_packets = 0; 2188 fs->fwd_dropped = 0; 2189 fs->rx_bad_ip_csum = 0; 2190 fs->rx_bad_l4_csum = 0; 2191 fs->rx_bad_outer_l4_csum = 0; 2192 fs->rx_bad_outer_ip_csum = 0; 2193 2194 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2195 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2196 fs->busy_cycles = 0; 2197 } 2198 } 2199 2200 static void 2201 flush_fwd_rx_queues(void) 2202 { 2203 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2204 portid_t rxp; 2205 portid_t port_id; 2206 queueid_t rxq; 2207 uint16_t nb_rx; 2208 uint8_t j; 2209 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2210 uint64_t timer_period; 2211 2212 if (num_procs > 1) { 2213 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2214 return; 2215 } 2216 2217 /* convert to number of cycles */ 2218 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2219 2220 for (j = 0; j < 2; j++) { 2221 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2222 for (rxq = 0; rxq < nb_rxq; rxq++) { 2223 port_id = fwd_ports_ids[rxp]; 2224 2225 /* Polling stopped queues is prohibited. */ 2226 if (ports[port_id].rxq[rxq].state == 2227 RTE_ETH_QUEUE_STATE_STOPPED) 2228 continue; 2229 2230 /** 2231 * testpmd can stuck in the below do while loop 2232 * if rte_eth_rx_burst() always returns nonzero 2233 * packets. So timer is added to exit this loop 2234 * after 1sec timer expiry. 2235 */ 2236 prev_tsc = rte_rdtsc(); 2237 do { 2238 nb_rx = rte_eth_rx_burst(port_id, rxq, 2239 pkts_burst, MAX_PKT_BURST); 2240 rte_pktmbuf_free_bulk(pkts_burst, nb_rx); 2241 2242 cur_tsc = rte_rdtsc(); 2243 diff_tsc = cur_tsc - prev_tsc; 2244 timer_tsc += diff_tsc; 2245 } while ((nb_rx > 0) && 2246 (timer_tsc < timer_period)); 2247 timer_tsc = 0; 2248 } 2249 } 2250 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2251 } 2252 } 2253 2254 static void 2255 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2256 { 2257 struct fwd_stream **fsm; 2258 uint64_t prev_tsc; 2259 streamid_t nb_fs; 2260 streamid_t sm_id; 2261 #ifdef RTE_LIB_BITRATESTATS 2262 uint64_t tics_per_1sec; 2263 uint64_t tics_datum; 2264 uint64_t tics_current; 2265 uint16_t i, cnt_ports; 2266 2267 cnt_ports = nb_ports; 2268 tics_datum = rte_rdtsc(); 2269 tics_per_1sec = rte_get_timer_hz(); 2270 #endif 2271 fsm = &fwd_streams[fc->stream_idx]; 2272 nb_fs = fc->stream_nb; 2273 prev_tsc = rte_rdtsc(); 2274 do { 2275 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2276 struct fwd_stream *fs = fsm[sm_id]; 2277 uint64_t start_fs_tsc = 0; 2278 bool busy; 2279 2280 if (fs->disabled) 2281 continue; 2282 if (record_core_cycles) 2283 start_fs_tsc = rte_rdtsc(); 2284 busy = (*pkt_fwd)(fs); 2285 if (record_core_cycles && busy) 2286 fs->busy_cycles += rte_rdtsc() - start_fs_tsc; 2287 } 2288 #ifdef RTE_LIB_BITRATESTATS 2289 if (bitrate_enabled != 0 && 2290 bitrate_lcore_id == rte_lcore_id()) { 2291 tics_current = rte_rdtsc(); 2292 if (tics_current - tics_datum >= tics_per_1sec) { 2293 /* Periodic bitrate calculation */ 2294 for (i = 0; i < cnt_ports; i++) 2295 rte_stats_bitrate_calc(bitrate_data, 2296 ports_ids[i]); 2297 tics_datum = tics_current; 2298 } 2299 } 2300 #endif 2301 #ifdef RTE_LIB_LATENCYSTATS 2302 if (latencystats_enabled != 0 && 2303 latencystats_lcore_id == rte_lcore_id()) 2304 rte_latencystats_update(); 2305 #endif 2306 if (record_core_cycles) { 2307 uint64_t tsc = rte_rdtsc(); 2308 2309 fc->total_cycles += tsc - prev_tsc; 2310 prev_tsc = tsc; 2311 } 2312 } while (! fc->stopped); 2313 } 2314 2315 static int 2316 lcore_usage_callback(unsigned int lcore_id, struct rte_lcore_usage *usage) 2317 { 2318 struct fwd_stream **fsm; 2319 struct fwd_lcore *fc; 2320 streamid_t nb_fs; 2321 streamid_t sm_id; 2322 2323 fc = lcore_to_fwd_lcore(lcore_id); 2324 if (fc == NULL) 2325 return -1; 2326 2327 fsm = &fwd_streams[fc->stream_idx]; 2328 nb_fs = fc->stream_nb; 2329 usage->busy_cycles = 0; 2330 usage->total_cycles = fc->total_cycles; 2331 2332 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2333 if (!fsm[sm_id]->disabled) 2334 usage->busy_cycles += fsm[sm_id]->busy_cycles; 2335 } 2336 2337 return 0; 2338 } 2339 2340 static int 2341 start_pkt_forward_on_core(void *fwd_arg) 2342 { 2343 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2344 cur_fwd_config.fwd_eng->packet_fwd); 2345 return 0; 2346 } 2347 2348 /* 2349 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2350 * Used to start communication flows in network loopback test configurations. 2351 */ 2352 static int 2353 run_one_txonly_burst_on_core(void *fwd_arg) 2354 { 2355 struct fwd_lcore *fwd_lc; 2356 struct fwd_lcore tmp_lcore; 2357 2358 fwd_lc = (struct fwd_lcore *) fwd_arg; 2359 tmp_lcore = *fwd_lc; 2360 tmp_lcore.stopped = 1; 2361 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2362 return 0; 2363 } 2364 2365 /* 2366 * Launch packet forwarding: 2367 * - Setup per-port forwarding context. 2368 * - launch logical cores with their forwarding configuration. 2369 */ 2370 static void 2371 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2372 { 2373 unsigned int i; 2374 unsigned int lc_id; 2375 int diag; 2376 2377 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2378 lc_id = fwd_lcores_cpuids[i]; 2379 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2380 fwd_lcores[i]->stopped = 0; 2381 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2382 fwd_lcores[i], lc_id); 2383 if (diag != 0) 2384 fprintf(stderr, 2385 "launch lcore %u failed - diag=%d\n", 2386 lc_id, diag); 2387 } 2388 } 2389 } 2390 2391 void 2392 common_fwd_stream_init(struct fwd_stream *fs) 2393 { 2394 bool rx_stopped, tx_stopped; 2395 2396 rx_stopped = (ports[fs->rx_port].rxq[fs->rx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2397 tx_stopped = (ports[fs->tx_port].txq[fs->tx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2398 fs->disabled = rx_stopped || tx_stopped; 2399 } 2400 2401 static void 2402 update_rx_queue_state(uint16_t port_id, uint16_t queue_id) 2403 { 2404 struct rte_eth_rxq_info rx_qinfo; 2405 int32_t rc; 2406 2407 rc = rte_eth_rx_queue_info_get(port_id, 2408 queue_id, &rx_qinfo); 2409 if (rc == 0) { 2410 ports[port_id].rxq[queue_id].state = 2411 rx_qinfo.queue_state; 2412 } else if (rc == -ENOTSUP) { 2413 /* 2414 * Set the rxq state to RTE_ETH_QUEUE_STATE_STARTED 2415 * to ensure that the PMDs do not implement 2416 * rte_eth_rx_queue_info_get can forward. 2417 */ 2418 ports[port_id].rxq[queue_id].state = 2419 RTE_ETH_QUEUE_STATE_STARTED; 2420 } else { 2421 TESTPMD_LOG(WARNING, 2422 "Failed to get rx queue info\n"); 2423 } 2424 } 2425 2426 static void 2427 update_tx_queue_state(uint16_t port_id, uint16_t queue_id) 2428 { 2429 struct rte_eth_txq_info tx_qinfo; 2430 int32_t rc; 2431 2432 rc = rte_eth_tx_queue_info_get(port_id, 2433 queue_id, &tx_qinfo); 2434 if (rc == 0) { 2435 ports[port_id].txq[queue_id].state = 2436 tx_qinfo.queue_state; 2437 } else if (rc == -ENOTSUP) { 2438 /* 2439 * Set the txq state to RTE_ETH_QUEUE_STATE_STARTED 2440 * to ensure that the PMDs do not implement 2441 * rte_eth_tx_queue_info_get can forward. 2442 */ 2443 ports[port_id].txq[queue_id].state = 2444 RTE_ETH_QUEUE_STATE_STARTED; 2445 } else { 2446 TESTPMD_LOG(WARNING, 2447 "Failed to get tx queue info\n"); 2448 } 2449 } 2450 2451 static void 2452 update_queue_state(void) 2453 { 2454 portid_t pi; 2455 queueid_t qi; 2456 2457 RTE_ETH_FOREACH_DEV(pi) { 2458 for (qi = 0; qi < nb_rxq; qi++) 2459 update_rx_queue_state(pi, qi); 2460 for (qi = 0; qi < nb_txq; qi++) 2461 update_tx_queue_state(pi, qi); 2462 } 2463 } 2464 2465 /* 2466 * Launch packet forwarding configuration. 2467 */ 2468 void 2469 start_packet_forwarding(int with_tx_first) 2470 { 2471 port_fwd_begin_t port_fwd_begin; 2472 port_fwd_end_t port_fwd_end; 2473 stream_init_t stream_init = cur_fwd_eng->stream_init; 2474 unsigned int i; 2475 2476 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2477 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2478 2479 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2480 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2481 2482 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2483 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2484 (!nb_rxq || !nb_txq)) 2485 rte_exit(EXIT_FAILURE, 2486 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2487 cur_fwd_eng->fwd_mode_name); 2488 2489 if (all_ports_started() == 0) { 2490 fprintf(stderr, "Not all ports were started\n"); 2491 return; 2492 } 2493 if (test_done == 0) { 2494 fprintf(stderr, "Packet forwarding already started\n"); 2495 return; 2496 } 2497 2498 fwd_config_setup(); 2499 2500 pkt_fwd_config_display(&cur_fwd_config); 2501 if (!pkt_fwd_shared_rxq_check()) 2502 return; 2503 2504 if (stream_init != NULL) { 2505 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 2506 update_queue_state(); 2507 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2508 stream_init(fwd_streams[i]); 2509 } 2510 2511 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2512 if (port_fwd_begin != NULL) { 2513 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2514 if (port_fwd_begin(fwd_ports_ids[i])) { 2515 fprintf(stderr, 2516 "Packet forwarding is not ready\n"); 2517 return; 2518 } 2519 } 2520 } 2521 2522 if (with_tx_first) { 2523 port_fwd_begin = tx_only_engine.port_fwd_begin; 2524 if (port_fwd_begin != NULL) { 2525 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2526 if (port_fwd_begin(fwd_ports_ids[i])) { 2527 fprintf(stderr, 2528 "Packet forwarding is not ready\n"); 2529 return; 2530 } 2531 } 2532 } 2533 } 2534 2535 test_done = 0; 2536 2537 if(!no_flush_rx) 2538 flush_fwd_rx_queues(); 2539 2540 rxtx_config_display(); 2541 2542 fwd_stats_reset(); 2543 if (with_tx_first) { 2544 while (with_tx_first--) { 2545 launch_packet_forwarding( 2546 run_one_txonly_burst_on_core); 2547 rte_eal_mp_wait_lcore(); 2548 } 2549 port_fwd_end = tx_only_engine.port_fwd_end; 2550 if (port_fwd_end != NULL) { 2551 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2552 (*port_fwd_end)(fwd_ports_ids[i]); 2553 } 2554 } 2555 launch_packet_forwarding(start_pkt_forward_on_core); 2556 } 2557 2558 void 2559 stop_packet_forwarding(void) 2560 { 2561 port_fwd_end_t port_fwd_end; 2562 lcoreid_t lc_id; 2563 portid_t pt_id; 2564 int i; 2565 2566 if (test_done) { 2567 fprintf(stderr, "Packet forwarding not started\n"); 2568 return; 2569 } 2570 printf("Telling cores to stop..."); 2571 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2572 fwd_lcores[lc_id]->stopped = 1; 2573 printf("\nWaiting for lcores to finish...\n"); 2574 rte_eal_mp_wait_lcore(); 2575 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2576 if (port_fwd_end != NULL) { 2577 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2578 pt_id = fwd_ports_ids[i]; 2579 (*port_fwd_end)(pt_id); 2580 } 2581 } 2582 2583 fwd_stats_display(); 2584 2585 printf("\nDone.\n"); 2586 test_done = 1; 2587 } 2588 2589 void 2590 dev_set_link_up(portid_t pid) 2591 { 2592 if (rte_eth_dev_set_link_up(pid) < 0) 2593 fprintf(stderr, "\nSet link up fail.\n"); 2594 } 2595 2596 void 2597 dev_set_link_down(portid_t pid) 2598 { 2599 if (rte_eth_dev_set_link_down(pid) < 0) 2600 fprintf(stderr, "\nSet link down fail.\n"); 2601 } 2602 2603 static int 2604 all_ports_started(void) 2605 { 2606 portid_t pi; 2607 struct rte_port *port; 2608 2609 RTE_ETH_FOREACH_DEV(pi) { 2610 port = &ports[pi]; 2611 /* Check if there is a port which is not started */ 2612 if ((port->port_status != RTE_PORT_STARTED) && 2613 (port->slave_flag == 0)) 2614 return 0; 2615 } 2616 2617 /* No port is not started */ 2618 return 1; 2619 } 2620 2621 int 2622 port_is_stopped(portid_t port_id) 2623 { 2624 struct rte_port *port = &ports[port_id]; 2625 2626 if ((port->port_status != RTE_PORT_STOPPED) && 2627 (port->slave_flag == 0)) 2628 return 0; 2629 return 1; 2630 } 2631 2632 int 2633 all_ports_stopped(void) 2634 { 2635 portid_t pi; 2636 2637 RTE_ETH_FOREACH_DEV(pi) { 2638 if (!port_is_stopped(pi)) 2639 return 0; 2640 } 2641 2642 return 1; 2643 } 2644 2645 int 2646 port_is_started(portid_t port_id) 2647 { 2648 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2649 return 0; 2650 2651 if (ports[port_id].port_status != RTE_PORT_STARTED) 2652 return 0; 2653 2654 return 1; 2655 } 2656 2657 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2658 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2659 2660 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2661 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2662 2663 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2664 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2665 2666 2667 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2668 static int 2669 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2670 { 2671 queueid_t qi; 2672 struct rte_eth_hairpin_conf hairpin_conf = { 2673 .peer_count = 1, 2674 }; 2675 int i; 2676 int diag; 2677 struct rte_port *port = &ports[pi]; 2678 uint16_t peer_rx_port = pi; 2679 uint16_t peer_tx_port = pi; 2680 uint32_t manual = 1; 2681 uint32_t tx_exp = hairpin_mode & 0x10; 2682 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2683 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2684 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2685 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2686 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2687 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2688 2689 if (!(hairpin_mode & 0xf)) { 2690 peer_rx_port = pi; 2691 peer_tx_port = pi; 2692 manual = 0; 2693 } else if (hairpin_mode & 0x1) { 2694 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2695 RTE_ETH_DEV_NO_OWNER); 2696 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2697 peer_tx_port = rte_eth_find_next_owned_by(0, 2698 RTE_ETH_DEV_NO_OWNER); 2699 if (p_pi != RTE_MAX_ETHPORTS) { 2700 peer_rx_port = p_pi; 2701 } else { 2702 uint16_t next_pi; 2703 2704 /* Last port will be the peer RX port of the first. */ 2705 RTE_ETH_FOREACH_DEV(next_pi) 2706 peer_rx_port = next_pi; 2707 } 2708 manual = 1; 2709 } else if (hairpin_mode & 0x2) { 2710 if (cnt_pi & 0x1) { 2711 peer_rx_port = p_pi; 2712 } else { 2713 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2714 RTE_ETH_DEV_NO_OWNER); 2715 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2716 peer_rx_port = pi; 2717 } 2718 peer_tx_port = peer_rx_port; 2719 manual = 1; 2720 } 2721 2722 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2723 hairpin_conf.peers[0].port = peer_rx_port; 2724 hairpin_conf.peers[0].queue = i + nb_rxq; 2725 hairpin_conf.manual_bind = !!manual; 2726 hairpin_conf.tx_explicit = !!tx_exp; 2727 hairpin_conf.force_memory = !!tx_force_memory; 2728 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2729 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2730 diag = rte_eth_tx_hairpin_queue_setup 2731 (pi, qi, nb_txd, &hairpin_conf); 2732 i++; 2733 if (diag == 0) 2734 continue; 2735 2736 /* Fail to setup rx queue, return */ 2737 if (port->port_status == RTE_PORT_HANDLING) 2738 port->port_status = RTE_PORT_STOPPED; 2739 else 2740 fprintf(stderr, 2741 "Port %d can not be set back to stopped\n", pi); 2742 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2743 pi); 2744 /* try to reconfigure queues next time */ 2745 port->need_reconfig_queues = 1; 2746 return -1; 2747 } 2748 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2749 hairpin_conf.peers[0].port = peer_tx_port; 2750 hairpin_conf.peers[0].queue = i + nb_txq; 2751 hairpin_conf.manual_bind = !!manual; 2752 hairpin_conf.tx_explicit = !!tx_exp; 2753 hairpin_conf.force_memory = !!rx_force_memory; 2754 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2755 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2756 diag = rte_eth_rx_hairpin_queue_setup 2757 (pi, qi, nb_rxd, &hairpin_conf); 2758 i++; 2759 if (diag == 0) 2760 continue; 2761 2762 /* Fail to setup rx queue, return */ 2763 if (port->port_status == RTE_PORT_HANDLING) 2764 port->port_status = RTE_PORT_STOPPED; 2765 else 2766 fprintf(stderr, 2767 "Port %d can not be set back to stopped\n", pi); 2768 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2769 pi); 2770 /* try to reconfigure queues next time */ 2771 port->need_reconfig_queues = 1; 2772 return -1; 2773 } 2774 return 0; 2775 } 2776 2777 /* Configure the Rx with optional split. */ 2778 int 2779 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2780 uint16_t nb_rx_desc, unsigned int socket_id, 2781 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2782 { 2783 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2784 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2785 struct rte_mempool *mpx; 2786 unsigned int i, mp_n; 2787 uint32_t prev_hdrs = 0; 2788 int ret; 2789 2790 2791 if ((rx_pkt_nb_segs > 1) && 2792 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2793 /* multi-segment configuration */ 2794 for (i = 0; i < rx_pkt_nb_segs; i++) { 2795 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2796 /* 2797 * Use last valid pool for the segments with number 2798 * exceeding the pool index. 2799 */ 2800 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2801 mpx = mbuf_pool_find(socket_id, mp_n); 2802 /* Handle zero as mbuf data buffer size. */ 2803 rx_seg->offset = i < rx_pkt_nb_offs ? 2804 rx_pkt_seg_offsets[i] : 0; 2805 rx_seg->mp = mpx ? mpx : mp; 2806 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2807 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2808 prev_hdrs |= rx_seg->proto_hdr; 2809 } else { 2810 rx_seg->length = rx_pkt_seg_lengths[i] ? 2811 rx_pkt_seg_lengths[i] : 2812 mbuf_data_size[mp_n]; 2813 } 2814 } 2815 rx_conf->rx_nseg = rx_pkt_nb_segs; 2816 rx_conf->rx_seg = rx_useg; 2817 rx_conf->rx_mempools = NULL; 2818 rx_conf->rx_nmempool = 0; 2819 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2820 socket_id, rx_conf, NULL); 2821 rx_conf->rx_seg = NULL; 2822 rx_conf->rx_nseg = 0; 2823 } else if (multi_rx_mempool == 1) { 2824 /* multi-pool configuration */ 2825 struct rte_eth_dev_info dev_info; 2826 2827 if (mbuf_data_size_n <= 1) { 2828 fprintf(stderr, "Invalid number of mempools %u\n", 2829 mbuf_data_size_n); 2830 return -EINVAL; 2831 } 2832 ret = rte_eth_dev_info_get(port_id, &dev_info); 2833 if (ret != 0) 2834 return ret; 2835 if (dev_info.max_rx_mempools == 0) { 2836 fprintf(stderr, 2837 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2838 port_id); 2839 return -ENOTSUP; 2840 } 2841 for (i = 0; i < mbuf_data_size_n; i++) { 2842 mpx = mbuf_pool_find(socket_id, i); 2843 rx_mempool[i] = mpx ? mpx : mp; 2844 } 2845 rx_conf->rx_mempools = rx_mempool; 2846 rx_conf->rx_nmempool = mbuf_data_size_n; 2847 rx_conf->rx_seg = NULL; 2848 rx_conf->rx_nseg = 0; 2849 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2850 socket_id, rx_conf, NULL); 2851 rx_conf->rx_mempools = NULL; 2852 rx_conf->rx_nmempool = 0; 2853 } else { 2854 /* Single pool/segment configuration */ 2855 rx_conf->rx_seg = NULL; 2856 rx_conf->rx_nseg = 0; 2857 rx_conf->rx_mempools = NULL; 2858 rx_conf->rx_nmempool = 0; 2859 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2860 socket_id, rx_conf, mp); 2861 } 2862 2863 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2864 RTE_ETH_QUEUE_STATE_STOPPED : 2865 RTE_ETH_QUEUE_STATE_STARTED; 2866 return ret; 2867 } 2868 2869 static int 2870 alloc_xstats_display_info(portid_t pi) 2871 { 2872 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2873 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2874 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2875 2876 if (xstats_display_num == 0) 2877 return 0; 2878 2879 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2880 if (*ids_supp == NULL) 2881 goto fail_ids_supp; 2882 2883 *prev_values = calloc(xstats_display_num, 2884 sizeof(**prev_values)); 2885 if (*prev_values == NULL) 2886 goto fail_prev_values; 2887 2888 *curr_values = calloc(xstats_display_num, 2889 sizeof(**curr_values)); 2890 if (*curr_values == NULL) 2891 goto fail_curr_values; 2892 2893 ports[pi].xstats_info.allocated = true; 2894 2895 return 0; 2896 2897 fail_curr_values: 2898 free(*prev_values); 2899 fail_prev_values: 2900 free(*ids_supp); 2901 fail_ids_supp: 2902 return -ENOMEM; 2903 } 2904 2905 static void 2906 free_xstats_display_info(portid_t pi) 2907 { 2908 if (!ports[pi].xstats_info.allocated) 2909 return; 2910 free(ports[pi].xstats_info.ids_supp); 2911 free(ports[pi].xstats_info.prev_values); 2912 free(ports[pi].xstats_info.curr_values); 2913 ports[pi].xstats_info.allocated = false; 2914 } 2915 2916 /** Fill helper structures for specified port to show extended statistics. */ 2917 static void 2918 fill_xstats_display_info_for_port(portid_t pi) 2919 { 2920 unsigned int stat, stat_supp; 2921 const char *xstat_name; 2922 struct rte_port *port; 2923 uint64_t *ids_supp; 2924 int rc; 2925 2926 if (xstats_display_num == 0) 2927 return; 2928 2929 if (pi == (portid_t)RTE_PORT_ALL) { 2930 fill_xstats_display_info(); 2931 return; 2932 } 2933 2934 port = &ports[pi]; 2935 if (port->port_status != RTE_PORT_STARTED) 2936 return; 2937 2938 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2939 rte_exit(EXIT_FAILURE, 2940 "Failed to allocate xstats display memory\n"); 2941 2942 ids_supp = port->xstats_info.ids_supp; 2943 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2944 xstat_name = xstats_display[stat].name; 2945 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2946 ids_supp + stat_supp); 2947 if (rc != 0) { 2948 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2949 xstat_name, pi, stat); 2950 continue; 2951 } 2952 stat_supp++; 2953 } 2954 2955 port->xstats_info.ids_supp_sz = stat_supp; 2956 } 2957 2958 /** Fill helper structures for all ports to show extended statistics. */ 2959 static void 2960 fill_xstats_display_info(void) 2961 { 2962 portid_t pi; 2963 2964 if (xstats_display_num == 0) 2965 return; 2966 2967 RTE_ETH_FOREACH_DEV(pi) 2968 fill_xstats_display_info_for_port(pi); 2969 } 2970 2971 /* 2972 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2973 * device in dev_info is zero when no slave is added. And its capability 2974 * will be updated when add a new slave device. So adding a slave device need 2975 * to update the port configurations of bonding device. 2976 */ 2977 static void 2978 update_bonding_port_dev_conf(portid_t bond_pid) 2979 { 2980 #ifdef RTE_NET_BOND 2981 struct rte_port *port = &ports[bond_pid]; 2982 uint16_t i; 2983 int ret; 2984 2985 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2986 if (ret != 0) { 2987 fprintf(stderr, "Failed to get dev info for port = %u\n", 2988 bond_pid); 2989 return; 2990 } 2991 2992 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2993 port->dev_conf.txmode.offloads |= 2994 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2995 /* Apply Tx offloads configuration */ 2996 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2997 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2998 2999 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 3000 port->dev_info.flow_type_rss_offloads; 3001 #else 3002 RTE_SET_USED(bond_pid); 3003 #endif 3004 } 3005 3006 int 3007 start_port(portid_t pid) 3008 { 3009 int diag; 3010 portid_t pi; 3011 portid_t p_pi = RTE_MAX_ETHPORTS; 3012 portid_t pl[RTE_MAX_ETHPORTS]; 3013 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3014 uint16_t cnt_pi = 0; 3015 uint16_t cfg_pi = 0; 3016 int peer_pi; 3017 queueid_t qi; 3018 struct rte_port *port; 3019 struct rte_eth_hairpin_cap cap; 3020 bool at_least_one_port_exist = false; 3021 bool all_ports_already_started = true; 3022 bool at_least_one_port_successfully_started = false; 3023 3024 if (port_id_is_invalid(pid, ENABLED_WARN)) 3025 return 0; 3026 3027 RTE_ETH_FOREACH_DEV(pi) { 3028 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3029 continue; 3030 3031 if (port_is_bonding_slave(pi)) { 3032 fprintf(stderr, 3033 "Please remove port %d from bonded device.\n", 3034 pi); 3035 continue; 3036 } 3037 3038 at_least_one_port_exist = true; 3039 3040 port = &ports[pi]; 3041 if (port->port_status == RTE_PORT_STOPPED) { 3042 port->port_status = RTE_PORT_HANDLING; 3043 all_ports_already_started = false; 3044 } else { 3045 fprintf(stderr, "Port %d is now not stopped\n", pi); 3046 continue; 3047 } 3048 3049 if (port->need_reconfig > 0) { 3050 struct rte_eth_conf dev_conf; 3051 int k; 3052 3053 port->need_reconfig = 0; 3054 3055 if (flow_isolate_all) { 3056 int ret = port_flow_isolate(pi, 1); 3057 if (ret) { 3058 fprintf(stderr, 3059 "Failed to apply isolated mode on port %d\n", 3060 pi); 3061 return -1; 3062 } 3063 } 3064 configure_rxtx_dump_callbacks(0); 3065 printf("Configuring Port %d (socket %u)\n", pi, 3066 port->socket_id); 3067 if (nb_hairpinq > 0 && 3068 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 3069 fprintf(stderr, 3070 "Port %d doesn't support hairpin queues\n", 3071 pi); 3072 return -1; 3073 } 3074 3075 if (port->bond_flag == 1 && port->update_conf == 1) { 3076 update_bonding_port_dev_conf(pi); 3077 port->update_conf = 0; 3078 } 3079 3080 /* configure port */ 3081 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 3082 nb_txq + nb_hairpinq, 3083 &(port->dev_conf)); 3084 if (diag != 0) { 3085 if (port->port_status == RTE_PORT_HANDLING) 3086 port->port_status = RTE_PORT_STOPPED; 3087 else 3088 fprintf(stderr, 3089 "Port %d can not be set back to stopped\n", 3090 pi); 3091 fprintf(stderr, "Fail to configure port %d\n", 3092 pi); 3093 /* try to reconfigure port next time */ 3094 port->need_reconfig = 1; 3095 return -1; 3096 } 3097 /* get device configuration*/ 3098 if (0 != 3099 eth_dev_conf_get_print_err(pi, &dev_conf)) { 3100 fprintf(stderr, 3101 "port %d can not get device configuration\n", 3102 pi); 3103 return -1; 3104 } 3105 /* Apply Rx offloads configuration */ 3106 if (dev_conf.rxmode.offloads != 3107 port->dev_conf.rxmode.offloads) { 3108 port->dev_conf.rxmode.offloads |= 3109 dev_conf.rxmode.offloads; 3110 for (k = 0; 3111 k < port->dev_info.max_rx_queues; 3112 k++) 3113 port->rxq[k].conf.offloads |= 3114 dev_conf.rxmode.offloads; 3115 } 3116 /* Apply Tx offloads configuration */ 3117 if (dev_conf.txmode.offloads != 3118 port->dev_conf.txmode.offloads) { 3119 port->dev_conf.txmode.offloads |= 3120 dev_conf.txmode.offloads; 3121 for (k = 0; 3122 k < port->dev_info.max_tx_queues; 3123 k++) 3124 port->txq[k].conf.offloads |= 3125 dev_conf.txmode.offloads; 3126 } 3127 } 3128 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 3129 port->need_reconfig_queues = 0; 3130 /* setup tx queues */ 3131 for (qi = 0; qi < nb_txq; qi++) { 3132 struct rte_eth_txconf *conf = 3133 &port->txq[qi].conf; 3134 3135 if ((numa_support) && 3136 (txring_numa[pi] != NUMA_NO_CONFIG)) 3137 diag = rte_eth_tx_queue_setup(pi, qi, 3138 port->nb_tx_desc[qi], 3139 txring_numa[pi], 3140 &(port->txq[qi].conf)); 3141 else 3142 diag = rte_eth_tx_queue_setup(pi, qi, 3143 port->nb_tx_desc[qi], 3144 port->socket_id, 3145 &(port->txq[qi].conf)); 3146 3147 if (diag == 0) { 3148 port->txq[qi].state = 3149 conf->tx_deferred_start ? 3150 RTE_ETH_QUEUE_STATE_STOPPED : 3151 RTE_ETH_QUEUE_STATE_STARTED; 3152 continue; 3153 } 3154 3155 /* Fail to setup tx queue, return */ 3156 if (port->port_status == RTE_PORT_HANDLING) 3157 port->port_status = RTE_PORT_STOPPED; 3158 else 3159 fprintf(stderr, 3160 "Port %d can not be set back to stopped\n", 3161 pi); 3162 fprintf(stderr, 3163 "Fail to configure port %d tx queues\n", 3164 pi); 3165 /* try to reconfigure queues next time */ 3166 port->need_reconfig_queues = 1; 3167 return -1; 3168 } 3169 for (qi = 0; qi < nb_rxq; qi++) { 3170 /* setup rx queues */ 3171 if ((numa_support) && 3172 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3173 struct rte_mempool * mp = 3174 mbuf_pool_find 3175 (rxring_numa[pi], 0); 3176 if (mp == NULL) { 3177 fprintf(stderr, 3178 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3179 rxring_numa[pi]); 3180 return -1; 3181 } 3182 3183 diag = rx_queue_setup(pi, qi, 3184 port->nb_rx_desc[qi], 3185 rxring_numa[pi], 3186 &(port->rxq[qi].conf), 3187 mp); 3188 } else { 3189 struct rte_mempool *mp = 3190 mbuf_pool_find 3191 (port->socket_id, 0); 3192 if (mp == NULL) { 3193 fprintf(stderr, 3194 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3195 port->socket_id); 3196 return -1; 3197 } 3198 diag = rx_queue_setup(pi, qi, 3199 port->nb_rx_desc[qi], 3200 port->socket_id, 3201 &(port->rxq[qi].conf), 3202 mp); 3203 } 3204 if (diag == 0) 3205 continue; 3206 3207 /* Fail to setup rx queue, return */ 3208 if (port->port_status == RTE_PORT_HANDLING) 3209 port->port_status = RTE_PORT_STOPPED; 3210 else 3211 fprintf(stderr, 3212 "Port %d can not be set back to stopped\n", 3213 pi); 3214 fprintf(stderr, 3215 "Fail to configure port %d rx queues\n", 3216 pi); 3217 /* try to reconfigure queues next time */ 3218 port->need_reconfig_queues = 1; 3219 return -1; 3220 } 3221 /* setup hairpin queues */ 3222 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3223 return -1; 3224 } 3225 configure_rxtx_dump_callbacks(verbose_level); 3226 if (clear_ptypes) { 3227 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3228 NULL, 0); 3229 if (diag < 0) 3230 fprintf(stderr, 3231 "Port %d: Failed to disable Ptype parsing\n", 3232 pi); 3233 } 3234 3235 p_pi = pi; 3236 cnt_pi++; 3237 3238 /* start port */ 3239 diag = eth_dev_start_mp(pi); 3240 if (diag < 0) { 3241 fprintf(stderr, "Fail to start port %d: %s\n", 3242 pi, rte_strerror(-diag)); 3243 3244 /* Fail to setup rx queue, return */ 3245 if (port->port_status == RTE_PORT_HANDLING) 3246 port->port_status = RTE_PORT_STOPPED; 3247 else 3248 fprintf(stderr, 3249 "Port %d can not be set back to stopped\n", 3250 pi); 3251 continue; 3252 } 3253 3254 if (port->port_status == RTE_PORT_HANDLING) 3255 port->port_status = RTE_PORT_STARTED; 3256 else 3257 fprintf(stderr, "Port %d can not be set into started\n", 3258 pi); 3259 3260 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3261 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3262 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3263 3264 at_least_one_port_successfully_started = true; 3265 3266 pl[cfg_pi++] = pi; 3267 } 3268 3269 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3270 update_queue_state(); 3271 3272 if (at_least_one_port_successfully_started && !no_link_check) 3273 check_all_ports_link_status(RTE_PORT_ALL); 3274 else if (at_least_one_port_exist & all_ports_already_started) 3275 fprintf(stderr, "Please stop the ports first\n"); 3276 3277 if (hairpin_mode & 0xf) { 3278 uint16_t i; 3279 int j; 3280 3281 /* bind all started hairpin ports */ 3282 for (i = 0; i < cfg_pi; i++) { 3283 pi = pl[i]; 3284 /* bind current Tx to all peer Rx */ 3285 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3286 RTE_MAX_ETHPORTS, 1); 3287 if (peer_pi < 0) 3288 return peer_pi; 3289 for (j = 0; j < peer_pi; j++) { 3290 if (!port_is_started(peer_pl[j])) 3291 continue; 3292 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3293 if (diag < 0) { 3294 fprintf(stderr, 3295 "Error during binding hairpin Tx port %u to %u: %s\n", 3296 pi, peer_pl[j], 3297 rte_strerror(-diag)); 3298 return -1; 3299 } 3300 } 3301 /* bind all peer Tx to current Rx */ 3302 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3303 RTE_MAX_ETHPORTS, 0); 3304 if (peer_pi < 0) 3305 return peer_pi; 3306 for (j = 0; j < peer_pi; j++) { 3307 if (!port_is_started(peer_pl[j])) 3308 continue; 3309 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3310 if (diag < 0) { 3311 fprintf(stderr, 3312 "Error during binding hairpin Tx port %u to %u: %s\n", 3313 peer_pl[j], pi, 3314 rte_strerror(-diag)); 3315 return -1; 3316 } 3317 } 3318 } 3319 } 3320 3321 fill_xstats_display_info_for_port(pid); 3322 3323 printf("Done\n"); 3324 return 0; 3325 } 3326 3327 void 3328 stop_port(portid_t pid) 3329 { 3330 portid_t pi; 3331 struct rte_port *port; 3332 int need_check_link_status = 0; 3333 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3334 int peer_pi; 3335 int ret; 3336 3337 if (port_id_is_invalid(pid, ENABLED_WARN)) 3338 return; 3339 3340 printf("Stopping ports...\n"); 3341 3342 RTE_ETH_FOREACH_DEV(pi) { 3343 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3344 continue; 3345 3346 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3347 fprintf(stderr, 3348 "Please remove port %d from forwarding configuration.\n", 3349 pi); 3350 continue; 3351 } 3352 3353 if (port_is_bonding_slave(pi)) { 3354 fprintf(stderr, 3355 "Please remove port %d from bonded device.\n", 3356 pi); 3357 continue; 3358 } 3359 3360 port = &ports[pi]; 3361 if (port->port_status == RTE_PORT_STARTED) 3362 port->port_status = RTE_PORT_HANDLING; 3363 else 3364 continue; 3365 3366 if (hairpin_mode & 0xf) { 3367 int j; 3368 3369 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3370 /* unbind all peer Tx from current Rx */ 3371 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3372 RTE_MAX_ETHPORTS, 0); 3373 if (peer_pi < 0) 3374 continue; 3375 for (j = 0; j < peer_pi; j++) { 3376 if (!port_is_started(peer_pl[j])) 3377 continue; 3378 rte_eth_hairpin_unbind(peer_pl[j], pi); 3379 } 3380 } 3381 3382 if (port->flow_list && !no_flow_flush) 3383 port_flow_flush(pi); 3384 3385 ret = eth_dev_stop_mp(pi); 3386 if (ret != 0) { 3387 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3388 pi); 3389 /* Allow to retry stopping the port. */ 3390 port->port_status = RTE_PORT_STARTED; 3391 continue; 3392 } 3393 3394 if (port->port_status == RTE_PORT_HANDLING) 3395 port->port_status = RTE_PORT_STOPPED; 3396 else 3397 fprintf(stderr, "Port %d can not be set into stopped\n", 3398 pi); 3399 need_check_link_status = 1; 3400 } 3401 if (need_check_link_status && !no_link_check) 3402 check_all_ports_link_status(RTE_PORT_ALL); 3403 3404 printf("Done\n"); 3405 } 3406 3407 static void 3408 remove_invalid_ports_in(portid_t *array, portid_t *total) 3409 { 3410 portid_t i; 3411 portid_t new_total = 0; 3412 3413 for (i = 0; i < *total; i++) 3414 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3415 array[new_total] = array[i]; 3416 new_total++; 3417 } 3418 *total = new_total; 3419 } 3420 3421 static void 3422 remove_invalid_ports(void) 3423 { 3424 remove_invalid_ports_in(ports_ids, &nb_ports); 3425 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3426 nb_cfg_ports = nb_fwd_ports; 3427 } 3428 3429 static void 3430 flush_port_owned_resources(portid_t pi) 3431 { 3432 mcast_addr_pool_destroy(pi); 3433 port_flow_flush(pi); 3434 port_flow_template_table_flush(pi); 3435 port_flow_pattern_template_flush(pi); 3436 port_flow_actions_template_flush(pi); 3437 port_flex_item_flush(pi); 3438 port_action_handle_flush(pi); 3439 } 3440 3441 static void 3442 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3443 { 3444 struct rte_port *port; 3445 portid_t slave_pid; 3446 uint16_t i; 3447 3448 for (i = 0; i < num_slaves; i++) { 3449 slave_pid = slave_pids[i]; 3450 if (port_is_started(slave_pid) == 1) { 3451 if (rte_eth_dev_stop(slave_pid) != 0) 3452 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3453 slave_pid); 3454 3455 port = &ports[slave_pid]; 3456 port->port_status = RTE_PORT_STOPPED; 3457 } 3458 3459 clear_port_slave_flag(slave_pid); 3460 3461 /* Close slave device when testpmd quit or is killed. */ 3462 if (cl_quit == 1 || f_quit == 1) 3463 rte_eth_dev_close(slave_pid); 3464 } 3465 } 3466 3467 void 3468 close_port(portid_t pid) 3469 { 3470 portid_t pi; 3471 struct rte_port *port; 3472 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3473 int num_slaves = 0; 3474 3475 if (port_id_is_invalid(pid, ENABLED_WARN)) 3476 return; 3477 3478 printf("Closing ports...\n"); 3479 3480 RTE_ETH_FOREACH_DEV(pi) { 3481 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3482 continue; 3483 3484 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3485 fprintf(stderr, 3486 "Please remove port %d from forwarding configuration.\n", 3487 pi); 3488 continue; 3489 } 3490 3491 if (port_is_bonding_slave(pi)) { 3492 fprintf(stderr, 3493 "Please remove port %d from bonded device.\n", 3494 pi); 3495 continue; 3496 } 3497 3498 port = &ports[pi]; 3499 if (port->port_status == RTE_PORT_CLOSED) { 3500 fprintf(stderr, "Port %d is already closed\n", pi); 3501 continue; 3502 } 3503 3504 if (is_proc_primary()) { 3505 flush_port_owned_resources(pi); 3506 #ifdef RTE_NET_BOND 3507 if (port->bond_flag == 1) 3508 num_slaves = rte_eth_bond_slaves_get(pi, 3509 slave_pids, RTE_MAX_ETHPORTS); 3510 #endif 3511 rte_eth_dev_close(pi); 3512 /* 3513 * If this port is bonded device, all slaves under the 3514 * device need to be removed or closed. 3515 */ 3516 if (port->bond_flag == 1 && num_slaves > 0) 3517 clear_bonding_slave_device(slave_pids, 3518 num_slaves); 3519 } 3520 3521 free_xstats_display_info(pi); 3522 } 3523 3524 remove_invalid_ports(); 3525 printf("Done\n"); 3526 } 3527 3528 void 3529 reset_port(portid_t pid) 3530 { 3531 int diag; 3532 portid_t pi; 3533 struct rte_port *port; 3534 3535 if (port_id_is_invalid(pid, ENABLED_WARN)) 3536 return; 3537 3538 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3539 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3540 fprintf(stderr, 3541 "Can not reset port(s), please stop port(s) first.\n"); 3542 return; 3543 } 3544 3545 printf("Resetting ports...\n"); 3546 3547 RTE_ETH_FOREACH_DEV(pi) { 3548 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3549 continue; 3550 3551 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3552 fprintf(stderr, 3553 "Please remove port %d from forwarding configuration.\n", 3554 pi); 3555 continue; 3556 } 3557 3558 if (port_is_bonding_slave(pi)) { 3559 fprintf(stderr, 3560 "Please remove port %d from bonded device.\n", 3561 pi); 3562 continue; 3563 } 3564 3565 if (is_proc_primary()) { 3566 diag = rte_eth_dev_reset(pi); 3567 if (diag == 0) { 3568 port = &ports[pi]; 3569 port->need_reconfig = 1; 3570 port->need_reconfig_queues = 1; 3571 } else { 3572 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3573 pi, diag); 3574 } 3575 } 3576 } 3577 3578 printf("Done\n"); 3579 } 3580 3581 void 3582 attach_port(char *identifier) 3583 { 3584 portid_t pi; 3585 struct rte_dev_iterator iterator; 3586 3587 printf("Attaching a new port...\n"); 3588 3589 if (identifier == NULL) { 3590 fprintf(stderr, "Invalid parameters are specified\n"); 3591 return; 3592 } 3593 3594 if (rte_dev_probe(identifier) < 0) { 3595 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3596 return; 3597 } 3598 3599 /* first attach mode: event */ 3600 if (setup_on_probe_event) { 3601 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3602 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3603 if (ports[pi].port_status == RTE_PORT_HANDLING && 3604 ports[pi].need_setup != 0) 3605 setup_attached_port(pi); 3606 return; 3607 } 3608 3609 /* second attach mode: iterator */ 3610 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3611 /* setup ports matching the devargs used for probing */ 3612 if (port_is_forwarding(pi)) 3613 continue; /* port was already attached before */ 3614 setup_attached_port(pi); 3615 } 3616 } 3617 3618 static void 3619 setup_attached_port(portid_t pi) 3620 { 3621 unsigned int socket_id; 3622 int ret; 3623 3624 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3625 /* if socket_id is invalid, set to the first available socket. */ 3626 if (check_socket_id(socket_id) < 0) 3627 socket_id = socket_ids[0]; 3628 reconfig(pi, socket_id); 3629 ret = rte_eth_promiscuous_enable(pi); 3630 if (ret != 0) 3631 fprintf(stderr, 3632 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3633 pi, rte_strerror(-ret)); 3634 3635 ports_ids[nb_ports++] = pi; 3636 fwd_ports_ids[nb_fwd_ports++] = pi; 3637 nb_cfg_ports = nb_fwd_ports; 3638 ports[pi].need_setup = 0; 3639 ports[pi].port_status = RTE_PORT_STOPPED; 3640 3641 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3642 printf("Done\n"); 3643 } 3644 3645 static void 3646 detach_device(struct rte_device *dev) 3647 { 3648 portid_t sibling; 3649 3650 if (dev == NULL) { 3651 fprintf(stderr, "Device already removed\n"); 3652 return; 3653 } 3654 3655 printf("Removing a device...\n"); 3656 3657 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3658 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3659 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3660 fprintf(stderr, "Port %u not stopped\n", 3661 sibling); 3662 return; 3663 } 3664 flush_port_owned_resources(sibling); 3665 } 3666 } 3667 3668 if (rte_dev_remove(dev) < 0) { 3669 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3670 return; 3671 } 3672 remove_invalid_ports(); 3673 3674 printf("Device is detached\n"); 3675 printf("Now total ports is %d\n", nb_ports); 3676 printf("Done\n"); 3677 return; 3678 } 3679 3680 void 3681 detach_port_device(portid_t port_id) 3682 { 3683 int ret; 3684 struct rte_eth_dev_info dev_info; 3685 3686 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3687 return; 3688 3689 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3690 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3691 fprintf(stderr, "Port not stopped\n"); 3692 return; 3693 } 3694 fprintf(stderr, "Port was not closed\n"); 3695 } 3696 3697 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3698 if (ret != 0) { 3699 TESTPMD_LOG(ERR, 3700 "Failed to get device info for port %d, not detaching\n", 3701 port_id); 3702 return; 3703 } 3704 detach_device(dev_info.device); 3705 } 3706 3707 void 3708 detach_devargs(char *identifier) 3709 { 3710 struct rte_dev_iterator iterator; 3711 struct rte_devargs da; 3712 portid_t port_id; 3713 3714 printf("Removing a device...\n"); 3715 3716 memset(&da, 0, sizeof(da)); 3717 if (rte_devargs_parsef(&da, "%s", identifier)) { 3718 fprintf(stderr, "cannot parse identifier\n"); 3719 return; 3720 } 3721 3722 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3723 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3724 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3725 fprintf(stderr, "Port %u not stopped\n", 3726 port_id); 3727 rte_eth_iterator_cleanup(&iterator); 3728 rte_devargs_reset(&da); 3729 return; 3730 } 3731 flush_port_owned_resources(port_id); 3732 } 3733 } 3734 3735 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3736 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3737 da.name, rte_bus_name(da.bus)); 3738 rte_devargs_reset(&da); 3739 return; 3740 } 3741 3742 remove_invalid_ports(); 3743 3744 printf("Device %s is detached\n", identifier); 3745 printf("Now total ports is %d\n", nb_ports); 3746 printf("Done\n"); 3747 rte_devargs_reset(&da); 3748 } 3749 3750 void 3751 pmd_test_exit(void) 3752 { 3753 portid_t pt_id; 3754 unsigned int i; 3755 int ret; 3756 3757 if (test_done == 0) 3758 stop_packet_forwarding(); 3759 3760 #ifndef RTE_EXEC_ENV_WINDOWS 3761 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3762 if (mempools[i]) { 3763 if (mp_alloc_type == MP_ALLOC_ANON) 3764 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3765 NULL); 3766 } 3767 } 3768 #endif 3769 if (ports != NULL) { 3770 no_link_check = 1; 3771 RTE_ETH_FOREACH_DEV(pt_id) { 3772 printf("\nStopping port %d...\n", pt_id); 3773 fflush(stdout); 3774 stop_port(pt_id); 3775 } 3776 RTE_ETH_FOREACH_DEV(pt_id) { 3777 printf("\nShutting down port %d...\n", pt_id); 3778 fflush(stdout); 3779 close_port(pt_id); 3780 } 3781 } 3782 3783 if (hot_plug) { 3784 ret = rte_dev_event_monitor_stop(); 3785 if (ret) { 3786 RTE_LOG(ERR, EAL, 3787 "fail to stop device event monitor."); 3788 return; 3789 } 3790 3791 ret = rte_dev_event_callback_unregister(NULL, 3792 dev_event_callback, NULL); 3793 if (ret < 0) { 3794 RTE_LOG(ERR, EAL, 3795 "fail to unregister device event callback.\n"); 3796 return; 3797 } 3798 3799 ret = rte_dev_hotplug_handle_disable(); 3800 if (ret) { 3801 RTE_LOG(ERR, EAL, 3802 "fail to disable hotplug handling.\n"); 3803 return; 3804 } 3805 } 3806 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3807 if (mempools[i]) 3808 mempool_free_mp(mempools[i]); 3809 } 3810 free(xstats_display); 3811 3812 printf("\nBye...\n"); 3813 } 3814 3815 typedef void (*cmd_func_t)(void); 3816 struct pmd_test_command { 3817 const char *cmd_name; 3818 cmd_func_t cmd_func; 3819 }; 3820 3821 /* Check the link status of all ports in up to 9s, and print them finally */ 3822 static void 3823 check_all_ports_link_status(uint32_t port_mask) 3824 { 3825 #define CHECK_INTERVAL 100 /* 100ms */ 3826 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3827 portid_t portid; 3828 uint8_t count, all_ports_up, print_flag = 0; 3829 struct rte_eth_link link; 3830 int ret; 3831 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3832 3833 printf("Checking link statuses...\n"); 3834 fflush(stdout); 3835 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3836 all_ports_up = 1; 3837 RTE_ETH_FOREACH_DEV(portid) { 3838 if ((port_mask & (1 << portid)) == 0) 3839 continue; 3840 memset(&link, 0, sizeof(link)); 3841 ret = rte_eth_link_get_nowait(portid, &link); 3842 if (ret < 0) { 3843 all_ports_up = 0; 3844 if (print_flag == 1) 3845 fprintf(stderr, 3846 "Port %u link get failed: %s\n", 3847 portid, rte_strerror(-ret)); 3848 continue; 3849 } 3850 /* print link status if flag set */ 3851 if (print_flag == 1) { 3852 rte_eth_link_to_str(link_status, 3853 sizeof(link_status), &link); 3854 printf("Port %d %s\n", portid, link_status); 3855 continue; 3856 } 3857 /* clear all_ports_up flag if any link down */ 3858 if (link.link_status == RTE_ETH_LINK_DOWN) { 3859 all_ports_up = 0; 3860 break; 3861 } 3862 } 3863 /* after finally printing all link status, get out */ 3864 if (print_flag == 1) 3865 break; 3866 3867 if (all_ports_up == 0) { 3868 fflush(stdout); 3869 rte_delay_ms(CHECK_INTERVAL); 3870 } 3871 3872 /* set the print_flag if all ports up or timeout */ 3873 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3874 print_flag = 1; 3875 } 3876 3877 if (lsc_interrupt) 3878 break; 3879 } 3880 } 3881 3882 static void 3883 rmv_port_callback(void *arg) 3884 { 3885 int need_to_start = 0; 3886 int org_no_link_check = no_link_check; 3887 portid_t port_id = (intptr_t)arg; 3888 struct rte_eth_dev_info dev_info; 3889 int ret; 3890 3891 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3892 3893 if (!test_done && port_is_forwarding(port_id)) { 3894 need_to_start = 1; 3895 stop_packet_forwarding(); 3896 } 3897 no_link_check = 1; 3898 stop_port(port_id); 3899 no_link_check = org_no_link_check; 3900 3901 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3902 if (ret != 0) 3903 TESTPMD_LOG(ERR, 3904 "Failed to get device info for port %d, not detaching\n", 3905 port_id); 3906 else { 3907 struct rte_device *device = dev_info.device; 3908 close_port(port_id); 3909 detach_device(device); /* might be already removed or have more ports */ 3910 } 3911 if (need_to_start) 3912 start_packet_forwarding(0); 3913 } 3914 3915 /* This function is used by the interrupt thread */ 3916 static int 3917 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3918 void *ret_param) 3919 { 3920 RTE_SET_USED(param); 3921 RTE_SET_USED(ret_param); 3922 3923 if (type >= RTE_ETH_EVENT_MAX) { 3924 fprintf(stderr, 3925 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3926 port_id, __func__, type); 3927 fflush(stderr); 3928 } else if (event_print_mask & (UINT32_C(1) << type)) { 3929 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3930 eth_event_desc[type]); 3931 fflush(stdout); 3932 } 3933 3934 switch (type) { 3935 case RTE_ETH_EVENT_NEW: 3936 ports[port_id].need_setup = 1; 3937 ports[port_id].port_status = RTE_PORT_HANDLING; 3938 break; 3939 case RTE_ETH_EVENT_INTR_RMV: 3940 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3941 break; 3942 if (rte_eal_alarm_set(100000, 3943 rmv_port_callback, (void *)(intptr_t)port_id)) 3944 fprintf(stderr, 3945 "Could not set up deferred device removal\n"); 3946 break; 3947 case RTE_ETH_EVENT_DESTROY: 3948 ports[port_id].port_status = RTE_PORT_CLOSED; 3949 printf("Port %u is closed\n", port_id); 3950 break; 3951 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3952 uint16_t rxq_id; 3953 int ret; 3954 3955 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3956 for (rxq_id = 0; ; rxq_id++) { 3957 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3958 NULL); 3959 if (ret <= 0) 3960 break; 3961 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3962 port_id, rxq_id); 3963 3964 #ifdef RTE_NET_MLX5 3965 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3966 #endif 3967 } 3968 break; 3969 } 3970 default: 3971 break; 3972 } 3973 return 0; 3974 } 3975 3976 static int 3977 register_eth_event_callback(void) 3978 { 3979 int ret; 3980 enum rte_eth_event_type event; 3981 3982 for (event = RTE_ETH_EVENT_UNKNOWN; 3983 event < RTE_ETH_EVENT_MAX; event++) { 3984 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3985 event, 3986 eth_event_callback, 3987 NULL); 3988 if (ret != 0) { 3989 TESTPMD_LOG(ERR, "Failed to register callback for " 3990 "%s event\n", eth_event_desc[event]); 3991 return -1; 3992 } 3993 } 3994 3995 return 0; 3996 } 3997 3998 /* This function is used by the interrupt thread */ 3999 static void 4000 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 4001 __rte_unused void *arg) 4002 { 4003 uint16_t port_id; 4004 int ret; 4005 4006 if (type >= RTE_DEV_EVENT_MAX) { 4007 fprintf(stderr, "%s called upon invalid event %d\n", 4008 __func__, type); 4009 fflush(stderr); 4010 } 4011 4012 switch (type) { 4013 case RTE_DEV_EVENT_REMOVE: 4014 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 4015 device_name); 4016 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 4017 if (ret) { 4018 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 4019 device_name); 4020 return; 4021 } 4022 /* 4023 * Because the user's callback is invoked in eal interrupt 4024 * callback, the interrupt callback need to be finished before 4025 * it can be unregistered when detaching device. So finish 4026 * callback soon and use a deferred removal to detach device 4027 * is need. It is a workaround, once the device detaching be 4028 * moved into the eal in the future, the deferred removal could 4029 * be deleted. 4030 */ 4031 if (rte_eal_alarm_set(100000, 4032 rmv_port_callback, (void *)(intptr_t)port_id)) 4033 RTE_LOG(ERR, EAL, 4034 "Could not set up deferred device removal\n"); 4035 break; 4036 case RTE_DEV_EVENT_ADD: 4037 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 4038 device_name); 4039 /* TODO: After finish kernel driver binding, 4040 * begin to attach port. 4041 */ 4042 break; 4043 default: 4044 break; 4045 } 4046 } 4047 4048 static void 4049 rxtx_port_config(portid_t pid) 4050 { 4051 uint16_t qid; 4052 uint64_t offloads; 4053 struct rte_port *port = &ports[pid]; 4054 4055 for (qid = 0; qid < nb_rxq; qid++) { 4056 offloads = port->rxq[qid].conf.offloads; 4057 port->rxq[qid].conf = port->dev_info.default_rxconf; 4058 4059 if (rxq_share > 0 && 4060 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 4061 /* Non-zero share group to enable RxQ share. */ 4062 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 4063 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 4064 } 4065 4066 if (offloads != 0) 4067 port->rxq[qid].conf.offloads = offloads; 4068 4069 /* Check if any Rx parameters have been passed */ 4070 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 4071 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 4072 4073 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 4074 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 4075 4076 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 4077 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 4078 4079 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 4080 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 4081 4082 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 4083 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 4084 4085 port->nb_rx_desc[qid] = nb_rxd; 4086 } 4087 4088 for (qid = 0; qid < nb_txq; qid++) { 4089 offloads = port->txq[qid].conf.offloads; 4090 port->txq[qid].conf = port->dev_info.default_txconf; 4091 if (offloads != 0) 4092 port->txq[qid].conf.offloads = offloads; 4093 4094 /* Check if any Tx parameters have been passed */ 4095 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 4096 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 4097 4098 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 4099 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 4100 4101 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 4102 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 4103 4104 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 4105 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 4106 4107 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 4108 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 4109 4110 port->nb_tx_desc[qid] = nb_txd; 4111 } 4112 } 4113 4114 /* 4115 * Helper function to set MTU from frame size 4116 * 4117 * port->dev_info should be set before calling this function. 4118 * 4119 * return 0 on success, negative on error 4120 */ 4121 int 4122 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 4123 { 4124 struct rte_port *port = &ports[portid]; 4125 uint32_t eth_overhead; 4126 uint16_t mtu, new_mtu; 4127 4128 eth_overhead = get_eth_overhead(&port->dev_info); 4129 4130 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 4131 printf("Failed to get MTU for port %u\n", portid); 4132 return -1; 4133 } 4134 4135 new_mtu = max_rx_pktlen - eth_overhead; 4136 4137 if (mtu == new_mtu) 4138 return 0; 4139 4140 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 4141 fprintf(stderr, 4142 "Failed to set MTU to %u for port %u\n", 4143 new_mtu, portid); 4144 return -1; 4145 } 4146 4147 port->dev_conf.rxmode.mtu = new_mtu; 4148 4149 return 0; 4150 } 4151 4152 void 4153 init_port_config(void) 4154 { 4155 portid_t pid; 4156 struct rte_port *port; 4157 int ret, i; 4158 4159 RTE_ETH_FOREACH_DEV(pid) { 4160 port = &ports[pid]; 4161 4162 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4163 if (ret != 0) 4164 return; 4165 4166 if (nb_rxq > 1) { 4167 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4168 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4169 rss_hf & port->dev_info.flow_type_rss_offloads; 4170 } else { 4171 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4172 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4173 } 4174 4175 if (port->dcb_flag == 0) { 4176 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4177 port->dev_conf.rxmode.mq_mode = 4178 (enum rte_eth_rx_mq_mode) 4179 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4180 } else { 4181 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4182 port->dev_conf.rxmode.offloads &= 4183 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4184 4185 for (i = 0; 4186 i < port->dev_info.nb_rx_queues; 4187 i++) 4188 port->rxq[i].conf.offloads &= 4189 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4190 } 4191 } 4192 4193 rxtx_port_config(pid); 4194 4195 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4196 if (ret != 0) 4197 return; 4198 4199 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4200 port->dev_conf.intr_conf.lsc = 1; 4201 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4202 port->dev_conf.intr_conf.rmv = 1; 4203 } 4204 } 4205 4206 void set_port_slave_flag(portid_t slave_pid) 4207 { 4208 struct rte_port *port; 4209 4210 port = &ports[slave_pid]; 4211 port->slave_flag = 1; 4212 } 4213 4214 void clear_port_slave_flag(portid_t slave_pid) 4215 { 4216 struct rte_port *port; 4217 4218 port = &ports[slave_pid]; 4219 port->slave_flag = 0; 4220 } 4221 4222 uint8_t port_is_bonding_slave(portid_t slave_pid) 4223 { 4224 struct rte_port *port; 4225 struct rte_eth_dev_info dev_info; 4226 int ret; 4227 4228 port = &ports[slave_pid]; 4229 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4230 if (ret != 0) { 4231 TESTPMD_LOG(ERR, 4232 "Failed to get device info for port id %d," 4233 "cannot determine if the port is a bonded slave", 4234 slave_pid); 4235 return 0; 4236 } 4237 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4238 return 1; 4239 return 0; 4240 } 4241 4242 const uint16_t vlan_tags[] = { 4243 0, 1, 2, 3, 4, 5, 6, 7, 4244 8, 9, 10, 11, 12, 13, 14, 15, 4245 16, 17, 18, 19, 20, 21, 22, 23, 4246 24, 25, 26, 27, 28, 29, 30, 31 4247 }; 4248 4249 static int 4250 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4251 enum dcb_mode_enable dcb_mode, 4252 enum rte_eth_nb_tcs num_tcs, 4253 uint8_t pfc_en) 4254 { 4255 uint8_t i; 4256 int32_t rc; 4257 struct rte_eth_rss_conf rss_conf; 4258 4259 /* 4260 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4261 * given above, and the number of traffic classes available for use. 4262 */ 4263 if (dcb_mode == DCB_VT_ENABLED) { 4264 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4265 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4266 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4267 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4268 4269 /* VMDQ+DCB RX and TX configurations */ 4270 vmdq_rx_conf->enable_default_pool = 0; 4271 vmdq_rx_conf->default_pool = 0; 4272 vmdq_rx_conf->nb_queue_pools = 4273 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4274 vmdq_tx_conf->nb_queue_pools = 4275 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4276 4277 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4278 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4279 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4280 vmdq_rx_conf->pool_map[i].pools = 4281 1 << (i % vmdq_rx_conf->nb_queue_pools); 4282 } 4283 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4284 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4285 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4286 } 4287 4288 /* set DCB mode of RX and TX of multiple queues */ 4289 eth_conf->rxmode.mq_mode = 4290 (enum rte_eth_rx_mq_mode) 4291 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4292 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4293 } else { 4294 struct rte_eth_dcb_rx_conf *rx_conf = 4295 ð_conf->rx_adv_conf.dcb_rx_conf; 4296 struct rte_eth_dcb_tx_conf *tx_conf = 4297 ð_conf->tx_adv_conf.dcb_tx_conf; 4298 4299 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4300 4301 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4302 if (rc != 0) 4303 return rc; 4304 4305 rx_conf->nb_tcs = num_tcs; 4306 tx_conf->nb_tcs = num_tcs; 4307 4308 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4309 rx_conf->dcb_tc[i] = i % num_tcs; 4310 tx_conf->dcb_tc[i] = i % num_tcs; 4311 } 4312 4313 eth_conf->rxmode.mq_mode = 4314 (enum rte_eth_rx_mq_mode) 4315 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4316 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4317 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4318 } 4319 4320 if (pfc_en) 4321 eth_conf->dcb_capability_en = 4322 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4323 else 4324 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4325 4326 return 0; 4327 } 4328 4329 int 4330 init_port_dcb_config(portid_t pid, 4331 enum dcb_mode_enable dcb_mode, 4332 enum rte_eth_nb_tcs num_tcs, 4333 uint8_t pfc_en) 4334 { 4335 struct rte_eth_conf port_conf; 4336 struct rte_port *rte_port; 4337 int retval; 4338 uint16_t i; 4339 4340 if (num_procs > 1) { 4341 printf("The multi-process feature doesn't support dcb.\n"); 4342 return -ENOTSUP; 4343 } 4344 rte_port = &ports[pid]; 4345 4346 /* retain the original device configuration. */ 4347 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4348 4349 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4350 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4351 if (retval < 0) 4352 return retval; 4353 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4354 /* remove RSS HASH offload for DCB in vt mode */ 4355 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4356 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4357 for (i = 0; i < nb_rxq; i++) 4358 rte_port->rxq[i].conf.offloads &= 4359 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4360 } 4361 4362 /* re-configure the device . */ 4363 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4364 if (retval < 0) 4365 return retval; 4366 4367 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4368 if (retval != 0) 4369 return retval; 4370 4371 /* If dev_info.vmdq_pool_base is greater than 0, 4372 * the queue id of vmdq pools is started after pf queues. 4373 */ 4374 if (dcb_mode == DCB_VT_ENABLED && 4375 rte_port->dev_info.vmdq_pool_base > 0) { 4376 fprintf(stderr, 4377 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4378 pid); 4379 return -1; 4380 } 4381 4382 /* Assume the ports in testpmd have the same dcb capability 4383 * and has the same number of rxq and txq in dcb mode 4384 */ 4385 if (dcb_mode == DCB_VT_ENABLED) { 4386 if (rte_port->dev_info.max_vfs > 0) { 4387 nb_rxq = rte_port->dev_info.nb_rx_queues; 4388 nb_txq = rte_port->dev_info.nb_tx_queues; 4389 } else { 4390 nb_rxq = rte_port->dev_info.max_rx_queues; 4391 nb_txq = rte_port->dev_info.max_tx_queues; 4392 } 4393 } else { 4394 /*if vt is disabled, use all pf queues */ 4395 if (rte_port->dev_info.vmdq_pool_base == 0) { 4396 nb_rxq = rte_port->dev_info.max_rx_queues; 4397 nb_txq = rte_port->dev_info.max_tx_queues; 4398 } else { 4399 nb_rxq = (queueid_t)num_tcs; 4400 nb_txq = (queueid_t)num_tcs; 4401 4402 } 4403 } 4404 rx_free_thresh = 64; 4405 4406 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4407 4408 rxtx_port_config(pid); 4409 /* VLAN filter */ 4410 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4411 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4412 rx_vft_set(pid, vlan_tags[i], 1); 4413 4414 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4415 if (retval != 0) 4416 return retval; 4417 4418 rte_port->dcb_flag = 1; 4419 4420 /* Enter DCB configuration status */ 4421 dcb_config = 1; 4422 4423 return 0; 4424 } 4425 4426 static void 4427 init_port(void) 4428 { 4429 int i; 4430 4431 /* Configuration of Ethernet ports. */ 4432 ports = rte_zmalloc("testpmd: ports", 4433 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4434 RTE_CACHE_LINE_SIZE); 4435 if (ports == NULL) { 4436 rte_exit(EXIT_FAILURE, 4437 "rte_zmalloc(%d struct rte_port) failed\n", 4438 RTE_MAX_ETHPORTS); 4439 } 4440 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4441 ports[i].fwd_mac_swap = 1; 4442 ports[i].xstats_info.allocated = false; 4443 LIST_INIT(&ports[i].flow_tunnel_list); 4444 } 4445 /* Initialize ports NUMA structures */ 4446 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4447 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4448 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4449 } 4450 4451 static void 4452 print_stats(void) 4453 { 4454 uint8_t i; 4455 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4456 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4457 4458 /* Clear screen and move to top left */ 4459 printf("%s%s", clr, top_left); 4460 4461 printf("\nPort statistics ===================================="); 4462 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4463 nic_stats_display(fwd_ports_ids[i]); 4464 4465 fflush(stdout); 4466 } 4467 4468 static void 4469 signal_handler(int signum __rte_unused) 4470 { 4471 f_quit = 1; 4472 prompt_exit(); 4473 } 4474 4475 int 4476 main(int argc, char** argv) 4477 { 4478 int diag; 4479 portid_t port_id; 4480 uint16_t count; 4481 int ret; 4482 4483 #ifdef RTE_EXEC_ENV_WINDOWS 4484 signal(SIGINT, signal_handler); 4485 signal(SIGTERM, signal_handler); 4486 #else 4487 /* Want read() not to be restarted on signal */ 4488 struct sigaction action = { 4489 .sa_handler = signal_handler, 4490 }; 4491 4492 sigaction(SIGINT, &action, NULL); 4493 sigaction(SIGTERM, &action, NULL); 4494 #endif 4495 4496 testpmd_logtype = rte_log_register("testpmd"); 4497 if (testpmd_logtype < 0) 4498 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4499 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4500 4501 diag = rte_eal_init(argc, argv); 4502 if (diag < 0) 4503 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4504 rte_strerror(rte_errno)); 4505 4506 /* allocate port structures, and init them */ 4507 init_port(); 4508 4509 ret = register_eth_event_callback(); 4510 if (ret != 0) 4511 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4512 4513 #ifdef RTE_LIB_PDUMP 4514 /* initialize packet capture framework */ 4515 rte_pdump_init(); 4516 #endif 4517 4518 count = 0; 4519 RTE_ETH_FOREACH_DEV(port_id) { 4520 ports_ids[count] = port_id; 4521 count++; 4522 } 4523 nb_ports = (portid_t) count; 4524 if (nb_ports == 0) 4525 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4526 4527 set_def_fwd_config(); 4528 if (nb_lcores == 0) 4529 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4530 "Check the core mask argument\n"); 4531 4532 /* Bitrate/latency stats disabled by default */ 4533 #ifdef RTE_LIB_BITRATESTATS 4534 bitrate_enabled = 0; 4535 #endif 4536 #ifdef RTE_LIB_LATENCYSTATS 4537 latencystats_enabled = 0; 4538 #endif 4539 4540 /* on FreeBSD, mlockall() is disabled by default */ 4541 #ifdef RTE_EXEC_ENV_FREEBSD 4542 do_mlockall = 0; 4543 #else 4544 do_mlockall = 1; 4545 #endif 4546 4547 argc -= diag; 4548 argv += diag; 4549 if (argc > 1) 4550 launch_args_parse(argc, argv); 4551 4552 #ifndef RTE_EXEC_ENV_WINDOWS 4553 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4554 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4555 strerror(errno)); 4556 } 4557 #endif 4558 4559 if (tx_first && interactive) 4560 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4561 "interactive mode.\n"); 4562 4563 if (tx_first && lsc_interrupt) { 4564 fprintf(stderr, 4565 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4566 lsc_interrupt = 0; 4567 } 4568 4569 if (!nb_rxq && !nb_txq) 4570 fprintf(stderr, 4571 "Warning: Either rx or tx queues should be non-zero\n"); 4572 4573 if (nb_rxq > 1 && nb_rxq > nb_txq) 4574 fprintf(stderr, 4575 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4576 nb_rxq, nb_txq); 4577 4578 init_config(); 4579 4580 if (hot_plug) { 4581 ret = rte_dev_hotplug_handle_enable(); 4582 if (ret) { 4583 RTE_LOG(ERR, EAL, 4584 "fail to enable hotplug handling."); 4585 return -1; 4586 } 4587 4588 ret = rte_dev_event_monitor_start(); 4589 if (ret) { 4590 RTE_LOG(ERR, EAL, 4591 "fail to start device event monitoring."); 4592 return -1; 4593 } 4594 4595 ret = rte_dev_event_callback_register(NULL, 4596 dev_event_callback, NULL); 4597 if (ret) { 4598 RTE_LOG(ERR, EAL, 4599 "fail to register device event callback\n"); 4600 return -1; 4601 } 4602 } 4603 4604 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { 4605 if (!interactive) { 4606 rte_eal_cleanup(); 4607 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4608 } 4609 fprintf(stderr, "Start ports failed\n"); 4610 } 4611 4612 /* set all ports to promiscuous mode by default */ 4613 RTE_ETH_FOREACH_DEV(port_id) { 4614 ret = rte_eth_promiscuous_enable(port_id); 4615 if (ret != 0) 4616 fprintf(stderr, 4617 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4618 port_id, rte_strerror(-ret)); 4619 } 4620 4621 #ifdef RTE_LIB_METRICS 4622 /* Init metrics library */ 4623 rte_metrics_init(rte_socket_id()); 4624 #endif 4625 4626 #ifdef RTE_LIB_LATENCYSTATS 4627 if (latencystats_enabled != 0) { 4628 int ret = rte_latencystats_init(1, NULL); 4629 if (ret) 4630 fprintf(stderr, 4631 "Warning: latencystats init() returned error %d\n", 4632 ret); 4633 fprintf(stderr, "Latencystats running on lcore %d\n", 4634 latencystats_lcore_id); 4635 } 4636 #endif 4637 4638 /* Setup bitrate stats */ 4639 #ifdef RTE_LIB_BITRATESTATS 4640 if (bitrate_enabled != 0) { 4641 bitrate_data = rte_stats_bitrate_create(); 4642 if (bitrate_data == NULL) 4643 rte_exit(EXIT_FAILURE, 4644 "Could not allocate bitrate data.\n"); 4645 rte_stats_bitrate_reg(bitrate_data); 4646 } 4647 #endif 4648 4649 if (record_core_cycles) 4650 rte_lcore_register_usage_cb(lcore_usage_callback); 4651 4652 #ifdef RTE_LIB_CMDLINE 4653 if (init_cmdline() != 0) 4654 rte_exit(EXIT_FAILURE, 4655 "Could not initialise cmdline context.\n"); 4656 4657 if (strlen(cmdline_filename) != 0) 4658 cmdline_read_from_file(cmdline_filename); 4659 4660 if (interactive == 1) { 4661 if (auto_start) { 4662 printf("Start automatic packet forwarding\n"); 4663 start_packet_forwarding(0); 4664 } 4665 prompt(); 4666 } else 4667 #endif 4668 { 4669 printf("No commandline core given, start packet forwarding\n"); 4670 start_packet_forwarding(tx_first); 4671 if (stats_period != 0) { 4672 uint64_t prev_time = 0, cur_time, diff_time = 0; 4673 uint64_t timer_period; 4674 4675 /* Convert to number of cycles */ 4676 timer_period = stats_period * rte_get_timer_hz(); 4677 4678 while (f_quit == 0) { 4679 cur_time = rte_get_timer_cycles(); 4680 diff_time += cur_time - prev_time; 4681 4682 if (diff_time >= timer_period) { 4683 print_stats(); 4684 /* Reset the timer */ 4685 diff_time = 0; 4686 } 4687 /* Sleep to avoid unnecessary checks */ 4688 prev_time = cur_time; 4689 rte_delay_us_sleep(US_PER_S); 4690 } 4691 } else { 4692 char c; 4693 fd_set fds; 4694 4695 printf("Press enter to exit\n"); 4696 4697 FD_ZERO(&fds); 4698 FD_SET(0, &fds); 4699 4700 /* wait for signal or enter */ 4701 ret = select(1, &fds, NULL, NULL, NULL); 4702 if (ret < 0 && errno != EINTR) 4703 rte_exit(EXIT_FAILURE, 4704 "Select failed: %s\n", 4705 strerror(errno)); 4706 4707 /* if got enter then consume it */ 4708 if (ret == 1 && read(0, &c, 1) < 0) 4709 rte_exit(EXIT_FAILURE, 4710 "Read failed: %s\n", 4711 strerror(errno)); 4712 } 4713 } 4714 4715 pmd_test_exit(); 4716 4717 #ifdef RTE_LIB_PDUMP 4718 /* uninitialize packet capture framework */ 4719 rte_pdump_uninit(); 4720 #endif 4721 #ifdef RTE_LIB_LATENCYSTATS 4722 if (latencystats_enabled != 0) 4723 rte_latencystats_uninit(); 4724 #endif 4725 4726 ret = rte_eal_cleanup(); 4727 if (ret != 0) 4728 rte_exit(EXIT_FAILURE, 4729 "EAL cleanup failed: %s\n", strerror(-ret)); 4730 4731 return EXIT_SUCCESS; 4732 } 4733