1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 static volatile uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 uint8_t multi_rx_mempool; /**< Enables multi-rx-mempool feature */ 253 254 /* 255 * Configuration of packet segments used by the "txonly" processing engine. 256 */ 257 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 258 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 259 TXONLY_DEF_PACKET_LEN, 260 }; 261 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 262 263 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 264 /**< Split policy for packets to TX. */ 265 266 uint8_t txonly_multi_flow; 267 /**< Whether multiple flows are generated in TXONLY mode. */ 268 269 uint32_t tx_pkt_times_inter; 270 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 271 272 uint32_t tx_pkt_times_intra; 273 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 274 275 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 276 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 277 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 278 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 279 280 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 281 uint8_t dcb_config = 0; 282 283 /* 284 * Configurable number of RX/TX queues. 285 */ 286 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 287 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 288 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 289 290 /* 291 * Configurable number of RX/TX ring descriptors. 292 * Defaults are supplied by drivers via ethdev. 293 */ 294 #define RX_DESC_DEFAULT 0 295 #define TX_DESC_DEFAULT 0 296 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 297 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 298 299 #define RTE_PMD_PARAM_UNSET -1 300 /* 301 * Configurable values of RX and TX ring threshold registers. 302 */ 303 304 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 305 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 306 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 307 308 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 309 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 310 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 311 312 /* 313 * Configurable value of RX free threshold. 314 */ 315 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 316 317 /* 318 * Configurable value of RX drop enable. 319 */ 320 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 321 322 /* 323 * Configurable value of TX free threshold. 324 */ 325 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 326 327 /* 328 * Configurable value of TX RS bit threshold. 329 */ 330 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 331 332 /* 333 * Configurable value of buffered packets before sending. 334 */ 335 uint16_t noisy_tx_sw_bufsz; 336 337 /* 338 * Configurable value of packet buffer timeout. 339 */ 340 uint16_t noisy_tx_sw_buf_flush_time; 341 342 /* 343 * Configurable value for size of VNF internal memory area 344 * used for simulating noisy neighbour behaviour 345 */ 346 uint64_t noisy_lkup_mem_sz; 347 348 /* 349 * Configurable value of number of random writes done in 350 * VNF simulation memory area. 351 */ 352 uint64_t noisy_lkup_num_writes; 353 354 /* 355 * Configurable value of number of random reads done in 356 * VNF simulation memory area. 357 */ 358 uint64_t noisy_lkup_num_reads; 359 360 /* 361 * Configurable value of number of random reads/writes done in 362 * VNF simulation memory area. 363 */ 364 uint64_t noisy_lkup_num_reads_writes; 365 366 /* 367 * Receive Side Scaling (RSS) configuration. 368 */ 369 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 370 371 /* 372 * Port topology configuration 373 */ 374 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 375 376 /* 377 * Avoids to flush all the RX streams before starts forwarding. 378 */ 379 uint8_t no_flush_rx = 0; /* flush by default */ 380 381 /* 382 * Flow API isolated mode. 383 */ 384 uint8_t flow_isolate_all; 385 386 /* 387 * Disable port flow flush when stop port. 388 */ 389 uint8_t no_flow_flush = 0; /* do flow flush by default */ 390 391 /* 392 * Avoids to check link status when starting/stopping a port. 393 */ 394 uint8_t no_link_check = 0; /* check by default */ 395 396 /* 397 * Don't automatically start all ports in interactive mode. 398 */ 399 uint8_t no_device_start = 0; 400 401 /* 402 * Enable link status change notification 403 */ 404 uint8_t lsc_interrupt = 1; /* enabled by default */ 405 406 /* 407 * Enable device removal notification. 408 */ 409 uint8_t rmv_interrupt = 1; /* enabled by default */ 410 411 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 412 413 /* After attach, port setup is called on event or by iterator */ 414 bool setup_on_probe_event = true; 415 416 /* Clear ptypes on port initialization. */ 417 uint8_t clear_ptypes = true; 418 419 /* Hairpin ports configuration mode. */ 420 uint32_t hairpin_mode; 421 422 /* Pretty printing of ethdev events */ 423 static const char * const eth_event_desc[] = { 424 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 425 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 426 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 427 [RTE_ETH_EVENT_INTR_RESET] = "reset", 428 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 429 [RTE_ETH_EVENT_IPSEC] = "IPsec", 430 [RTE_ETH_EVENT_MACSEC] = "MACsec", 431 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 432 [RTE_ETH_EVENT_NEW] = "device probed", 433 [RTE_ETH_EVENT_DESTROY] = "device released", 434 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 435 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 436 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 437 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 438 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 439 [RTE_ETH_EVENT_MAX] = NULL, 440 }; 441 442 /* 443 * Display or mask ether events 444 * Default to all events except VF_MBOX 445 */ 446 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 447 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 448 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 449 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 450 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 451 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 452 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 453 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 454 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 455 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 456 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 457 /* 458 * Decide if all memory are locked for performance. 459 */ 460 int do_mlockall = 0; 461 462 #ifdef RTE_LIB_LATENCYSTATS 463 464 /* 465 * Set when latency stats is enabled in the commandline 466 */ 467 uint8_t latencystats_enabled; 468 469 /* 470 * Lcore ID to service latency statistics. 471 */ 472 lcoreid_t latencystats_lcore_id = -1; 473 474 #endif 475 476 /* 477 * Ethernet device configuration. 478 */ 479 struct rte_eth_rxmode rx_mode; 480 481 struct rte_eth_txmode tx_mode = { 482 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 483 }; 484 485 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 486 487 /* 488 * Display zero values by default for xstats 489 */ 490 uint8_t xstats_hide_zero; 491 492 /* 493 * Measure of CPU cycles disabled by default 494 */ 495 uint8_t record_core_cycles; 496 497 /* 498 * Display of RX and TX bursts disabled by default 499 */ 500 uint8_t record_burst_stats; 501 502 /* 503 * Number of ports per shared Rx queue group, 0 disable. 504 */ 505 uint32_t rxq_share; 506 507 unsigned int num_sockets = 0; 508 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 509 510 #ifdef RTE_LIB_BITRATESTATS 511 /* Bitrate statistics */ 512 struct rte_stats_bitrates *bitrate_data; 513 lcoreid_t bitrate_lcore_id; 514 uint8_t bitrate_enabled; 515 #endif 516 517 #ifdef RTE_LIB_GRO 518 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 519 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 520 #endif 521 522 /* 523 * hexadecimal bitmask of RX mq mode can be enabled. 524 */ 525 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 526 527 /* 528 * Used to set forced link speed 529 */ 530 uint32_t eth_link_speed; 531 532 /* 533 * ID of the current process in multi-process, used to 534 * configure the queues to be polled. 535 */ 536 int proc_id; 537 538 /* 539 * Number of processes in multi-process, used to 540 * configure the queues to be polled. 541 */ 542 unsigned int num_procs = 1; 543 544 static void 545 eth_rx_metadata_negotiate_mp(uint16_t port_id) 546 { 547 uint64_t rx_meta_features = 0; 548 int ret; 549 550 if (!is_proc_primary()) 551 return; 552 553 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 554 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 555 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 556 557 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 558 if (ret == 0) { 559 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 560 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 561 port_id); 562 } 563 564 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 565 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 566 port_id); 567 } 568 569 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 570 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 571 port_id); 572 } 573 } else if (ret != -ENOTSUP) { 574 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 575 port_id, rte_strerror(-ret)); 576 } 577 } 578 579 static int 580 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 581 const struct rte_eth_conf *dev_conf) 582 { 583 if (is_proc_primary()) 584 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 585 dev_conf); 586 return 0; 587 } 588 589 static int 590 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 591 { 592 #ifdef RTE_NET_BOND 593 594 portid_t slave_pids[RTE_MAX_ETHPORTS]; 595 struct rte_port *port; 596 int num_slaves; 597 portid_t slave_pid; 598 int i; 599 600 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 601 RTE_MAX_ETHPORTS); 602 if (num_slaves < 0) { 603 fprintf(stderr, "Failed to get slave list for port = %u\n", 604 bond_pid); 605 return num_slaves; 606 } 607 608 for (i = 0; i < num_slaves; i++) { 609 slave_pid = slave_pids[i]; 610 port = &ports[slave_pid]; 611 port->port_status = 612 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 613 } 614 #else 615 RTE_SET_USED(bond_pid); 616 RTE_SET_USED(is_stop); 617 #endif 618 return 0; 619 } 620 621 static int 622 eth_dev_start_mp(uint16_t port_id) 623 { 624 int ret; 625 626 if (is_proc_primary()) { 627 ret = rte_eth_dev_start(port_id); 628 if (ret != 0) 629 return ret; 630 631 struct rte_port *port = &ports[port_id]; 632 633 /* 634 * Starting a bonded port also starts all slaves under the bonded 635 * device. So if this port is bond device, we need to modify the 636 * port status of these slaves. 637 */ 638 if (port->bond_flag == 1) 639 return change_bonding_slave_port_status(port_id, false); 640 } 641 642 return 0; 643 } 644 645 static int 646 eth_dev_stop_mp(uint16_t port_id) 647 { 648 int ret; 649 650 if (is_proc_primary()) { 651 ret = rte_eth_dev_stop(port_id); 652 if (ret != 0) 653 return ret; 654 655 struct rte_port *port = &ports[port_id]; 656 657 /* 658 * Stopping a bonded port also stops all slaves under the bonded 659 * device. So if this port is bond device, we need to modify the 660 * port status of these slaves. 661 */ 662 if (port->bond_flag == 1) 663 return change_bonding_slave_port_status(port_id, true); 664 } 665 666 return 0; 667 } 668 669 static void 670 mempool_free_mp(struct rte_mempool *mp) 671 { 672 if (is_proc_primary()) 673 rte_mempool_free(mp); 674 } 675 676 static int 677 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 678 { 679 if (is_proc_primary()) 680 return rte_eth_dev_set_mtu(port_id, mtu); 681 682 return 0; 683 } 684 685 /* Forward function declarations */ 686 static void setup_attached_port(portid_t pi); 687 static void check_all_ports_link_status(uint32_t port_mask); 688 static int eth_event_callback(portid_t port_id, 689 enum rte_eth_event_type type, 690 void *param, void *ret_param); 691 static void dev_event_callback(const char *device_name, 692 enum rte_dev_event_type type, 693 void *param); 694 static void fill_xstats_display_info(void); 695 696 /* 697 * Check if all the ports are started. 698 * If yes, return positive value. If not, return zero. 699 */ 700 static int all_ports_started(void); 701 702 #ifdef RTE_LIB_GSO 703 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 704 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 705 #endif 706 707 /* Holds the registered mbuf dynamic flags names. */ 708 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 709 710 711 /* 712 * Helper function to check if socket is already discovered. 713 * If yes, return positive value. If not, return zero. 714 */ 715 int 716 new_socket_id(unsigned int socket_id) 717 { 718 unsigned int i; 719 720 for (i = 0; i < num_sockets; i++) { 721 if (socket_ids[i] == socket_id) 722 return 0; 723 } 724 return 1; 725 } 726 727 /* 728 * Setup default configuration. 729 */ 730 static void 731 set_default_fwd_lcores_config(void) 732 { 733 unsigned int i; 734 unsigned int nb_lc; 735 unsigned int sock_num; 736 737 nb_lc = 0; 738 for (i = 0; i < RTE_MAX_LCORE; i++) { 739 if (!rte_lcore_is_enabled(i)) 740 continue; 741 sock_num = rte_lcore_to_socket_id(i); 742 if (new_socket_id(sock_num)) { 743 if (num_sockets >= RTE_MAX_NUMA_NODES) { 744 rte_exit(EXIT_FAILURE, 745 "Total sockets greater than %u\n", 746 RTE_MAX_NUMA_NODES); 747 } 748 socket_ids[num_sockets++] = sock_num; 749 } 750 if (i == rte_get_main_lcore()) 751 continue; 752 fwd_lcores_cpuids[nb_lc++] = i; 753 } 754 nb_lcores = (lcoreid_t) nb_lc; 755 nb_cfg_lcores = nb_lcores; 756 nb_fwd_lcores = 1; 757 } 758 759 static void 760 set_def_peer_eth_addrs(void) 761 { 762 portid_t i; 763 764 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 765 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 766 peer_eth_addrs[i].addr_bytes[5] = i; 767 } 768 } 769 770 static void 771 set_default_fwd_ports_config(void) 772 { 773 portid_t pt_id; 774 int i = 0; 775 776 RTE_ETH_FOREACH_DEV(pt_id) { 777 fwd_ports_ids[i++] = pt_id; 778 779 /* Update sockets info according to the attached device */ 780 int socket_id = rte_eth_dev_socket_id(pt_id); 781 if (socket_id >= 0 && new_socket_id(socket_id)) { 782 if (num_sockets >= RTE_MAX_NUMA_NODES) { 783 rte_exit(EXIT_FAILURE, 784 "Total sockets greater than %u\n", 785 RTE_MAX_NUMA_NODES); 786 } 787 socket_ids[num_sockets++] = socket_id; 788 } 789 } 790 791 nb_cfg_ports = nb_ports; 792 nb_fwd_ports = nb_ports; 793 } 794 795 void 796 set_def_fwd_config(void) 797 { 798 set_default_fwd_lcores_config(); 799 set_def_peer_eth_addrs(); 800 set_default_fwd_ports_config(); 801 } 802 803 #ifndef RTE_EXEC_ENV_WINDOWS 804 /* extremely pessimistic estimation of memory required to create a mempool */ 805 static int 806 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 807 { 808 unsigned int n_pages, mbuf_per_pg, leftover; 809 uint64_t total_mem, mbuf_mem, obj_sz; 810 811 /* there is no good way to predict how much space the mempool will 812 * occupy because it will allocate chunks on the fly, and some of those 813 * will come from default DPDK memory while some will come from our 814 * external memory, so just assume 128MB will be enough for everyone. 815 */ 816 uint64_t hdr_mem = 128 << 20; 817 818 /* account for possible non-contiguousness */ 819 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 820 if (obj_sz > pgsz) { 821 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 822 return -1; 823 } 824 825 mbuf_per_pg = pgsz / obj_sz; 826 leftover = (nb_mbufs % mbuf_per_pg) > 0; 827 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 828 829 mbuf_mem = n_pages * pgsz; 830 831 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 832 833 if (total_mem > SIZE_MAX) { 834 TESTPMD_LOG(ERR, "Memory size too big\n"); 835 return -1; 836 } 837 *out = (size_t)total_mem; 838 839 return 0; 840 } 841 842 static int 843 pagesz_flags(uint64_t page_sz) 844 { 845 /* as per mmap() manpage, all page sizes are log2 of page size 846 * shifted by MAP_HUGE_SHIFT 847 */ 848 int log2 = rte_log2_u64(page_sz); 849 850 return (log2 << HUGE_SHIFT); 851 } 852 853 static void * 854 alloc_mem(size_t memsz, size_t pgsz, bool huge) 855 { 856 void *addr; 857 int flags; 858 859 /* allocate anonymous hugepages */ 860 flags = MAP_ANONYMOUS | MAP_PRIVATE; 861 if (huge) 862 flags |= HUGE_FLAG | pagesz_flags(pgsz); 863 864 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 865 if (addr == MAP_FAILED) 866 return NULL; 867 868 return addr; 869 } 870 871 struct extmem_param { 872 void *addr; 873 size_t len; 874 size_t pgsz; 875 rte_iova_t *iova_table; 876 unsigned int iova_table_len; 877 }; 878 879 static int 880 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 881 bool huge) 882 { 883 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 884 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 885 unsigned int cur_page, n_pages, pgsz_idx; 886 size_t mem_sz, cur_pgsz; 887 rte_iova_t *iovas = NULL; 888 void *addr; 889 int ret; 890 891 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 892 /* skip anything that is too big */ 893 if (pgsizes[pgsz_idx] > SIZE_MAX) 894 continue; 895 896 cur_pgsz = pgsizes[pgsz_idx]; 897 898 /* if we were told not to allocate hugepages, override */ 899 if (!huge) 900 cur_pgsz = sysconf(_SC_PAGESIZE); 901 902 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 903 if (ret < 0) { 904 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 905 return -1; 906 } 907 908 /* allocate our memory */ 909 addr = alloc_mem(mem_sz, cur_pgsz, huge); 910 911 /* if we couldn't allocate memory with a specified page size, 912 * that doesn't mean we can't do it with other page sizes, so 913 * try another one. 914 */ 915 if (addr == NULL) 916 continue; 917 918 /* store IOVA addresses for every page in this memory area */ 919 n_pages = mem_sz / cur_pgsz; 920 921 iovas = malloc(sizeof(*iovas) * n_pages); 922 923 if (iovas == NULL) { 924 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 925 goto fail; 926 } 927 /* lock memory if it's not huge pages */ 928 if (!huge) 929 mlock(addr, mem_sz); 930 931 /* populate IOVA addresses */ 932 for (cur_page = 0; cur_page < n_pages; cur_page++) { 933 rte_iova_t iova; 934 size_t offset; 935 void *cur; 936 937 offset = cur_pgsz * cur_page; 938 cur = RTE_PTR_ADD(addr, offset); 939 940 /* touch the page before getting its IOVA */ 941 *(volatile char *)cur = 0; 942 943 iova = rte_mem_virt2iova(cur); 944 945 iovas[cur_page] = iova; 946 } 947 948 break; 949 } 950 /* if we couldn't allocate anything */ 951 if (iovas == NULL) 952 return -1; 953 954 param->addr = addr; 955 param->len = mem_sz; 956 param->pgsz = cur_pgsz; 957 param->iova_table = iovas; 958 param->iova_table_len = n_pages; 959 960 return 0; 961 fail: 962 free(iovas); 963 if (addr) 964 munmap(addr, mem_sz); 965 966 return -1; 967 } 968 969 static int 970 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 971 { 972 struct extmem_param param; 973 int socket_id, ret; 974 975 memset(¶m, 0, sizeof(param)); 976 977 /* check if our heap exists */ 978 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 979 if (socket_id < 0) { 980 /* create our heap */ 981 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 982 if (ret < 0) { 983 TESTPMD_LOG(ERR, "Cannot create heap\n"); 984 return -1; 985 } 986 } 987 988 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 989 if (ret < 0) { 990 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 991 return -1; 992 } 993 994 /* we now have a valid memory area, so add it to heap */ 995 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 996 param.addr, param.len, param.iova_table, 997 param.iova_table_len, param.pgsz); 998 999 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1000 1001 /* not needed any more */ 1002 free(param.iova_table); 1003 1004 if (ret < 0) { 1005 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1006 munmap(param.addr, param.len); 1007 return -1; 1008 } 1009 1010 /* success */ 1011 1012 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1013 param.len >> 20); 1014 1015 return 0; 1016 } 1017 static void 1018 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1019 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1020 { 1021 uint16_t pid = 0; 1022 int ret; 1023 1024 RTE_ETH_FOREACH_DEV(pid) { 1025 struct rte_eth_dev_info dev_info; 1026 1027 ret = eth_dev_info_get_print_err(pid, &dev_info); 1028 if (ret != 0) { 1029 TESTPMD_LOG(DEBUG, 1030 "unable to get device info for port %d on addr 0x%p," 1031 "mempool unmapping will not be performed\n", 1032 pid, memhdr->addr); 1033 continue; 1034 } 1035 1036 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1037 if (ret) { 1038 TESTPMD_LOG(DEBUG, 1039 "unable to DMA unmap addr 0x%p " 1040 "for device %s\n", 1041 memhdr->addr, rte_dev_name(dev_info.device)); 1042 } 1043 } 1044 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1045 if (ret) { 1046 TESTPMD_LOG(DEBUG, 1047 "unable to un-register addr 0x%p\n", memhdr->addr); 1048 } 1049 } 1050 1051 static void 1052 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1053 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1054 { 1055 uint16_t pid = 0; 1056 size_t page_size = sysconf(_SC_PAGESIZE); 1057 int ret; 1058 1059 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1060 page_size); 1061 if (ret) { 1062 TESTPMD_LOG(DEBUG, 1063 "unable to register addr 0x%p\n", memhdr->addr); 1064 return; 1065 } 1066 RTE_ETH_FOREACH_DEV(pid) { 1067 struct rte_eth_dev_info dev_info; 1068 1069 ret = eth_dev_info_get_print_err(pid, &dev_info); 1070 if (ret != 0) { 1071 TESTPMD_LOG(DEBUG, 1072 "unable to get device info for port %d on addr 0x%p," 1073 "mempool mapping will not be performed\n", 1074 pid, memhdr->addr); 1075 continue; 1076 } 1077 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1078 if (ret) { 1079 TESTPMD_LOG(DEBUG, 1080 "unable to DMA map addr 0x%p " 1081 "for device %s\n", 1082 memhdr->addr, rte_dev_name(dev_info.device)); 1083 } 1084 } 1085 } 1086 #endif 1087 1088 static unsigned int 1089 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1090 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1091 { 1092 struct rte_pktmbuf_extmem *xmem; 1093 unsigned int ext_num, zone_num, elt_num; 1094 uint16_t elt_size; 1095 1096 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1097 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1098 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1099 1100 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1101 if (xmem == NULL) { 1102 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1103 "external buffer descriptors\n"); 1104 *ext_mem = NULL; 1105 return 0; 1106 } 1107 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1108 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1109 const struct rte_memzone *mz; 1110 char mz_name[RTE_MEMZONE_NAMESIZE]; 1111 int ret; 1112 1113 ret = snprintf(mz_name, sizeof(mz_name), 1114 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1115 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1116 errno = ENAMETOOLONG; 1117 ext_num = 0; 1118 break; 1119 } 1120 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1121 socket_id, 1122 RTE_MEMZONE_IOVA_CONTIG | 1123 RTE_MEMZONE_1GB | 1124 RTE_MEMZONE_SIZE_HINT_ONLY); 1125 if (mz == NULL) { 1126 /* 1127 * The caller exits on external buffer creation 1128 * error, so there is no need to free memzones. 1129 */ 1130 errno = ENOMEM; 1131 ext_num = 0; 1132 break; 1133 } 1134 xseg->buf_ptr = mz->addr; 1135 xseg->buf_iova = mz->iova; 1136 xseg->buf_len = EXTBUF_ZONE_SIZE; 1137 xseg->elt_size = elt_size; 1138 } 1139 if (ext_num == 0 && xmem != NULL) { 1140 free(xmem); 1141 xmem = NULL; 1142 } 1143 *ext_mem = xmem; 1144 return ext_num; 1145 } 1146 1147 /* 1148 * Configuration initialisation done once at init time. 1149 */ 1150 static struct rte_mempool * 1151 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1152 unsigned int socket_id, uint16_t size_idx) 1153 { 1154 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1155 struct rte_mempool *rte_mp = NULL; 1156 #ifndef RTE_EXEC_ENV_WINDOWS 1157 uint32_t mb_size; 1158 1159 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1160 #endif 1161 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1162 if (!is_proc_primary()) { 1163 rte_mp = rte_mempool_lookup(pool_name); 1164 if (rte_mp == NULL) 1165 rte_exit(EXIT_FAILURE, 1166 "Get mbuf pool for socket %u failed: %s\n", 1167 socket_id, rte_strerror(rte_errno)); 1168 return rte_mp; 1169 } 1170 1171 TESTPMD_LOG(INFO, 1172 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1173 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1174 1175 switch (mp_alloc_type) { 1176 case MP_ALLOC_NATIVE: 1177 { 1178 /* wrapper to rte_mempool_create() */ 1179 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1180 rte_mbuf_best_mempool_ops()); 1181 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1182 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1183 break; 1184 } 1185 #ifndef RTE_EXEC_ENV_WINDOWS 1186 case MP_ALLOC_ANON: 1187 { 1188 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1189 mb_size, (unsigned int) mb_mempool_cache, 1190 sizeof(struct rte_pktmbuf_pool_private), 1191 socket_id, mempool_flags); 1192 if (rte_mp == NULL) 1193 goto err; 1194 1195 if (rte_mempool_populate_anon(rte_mp) == 0) { 1196 rte_mempool_free(rte_mp); 1197 rte_mp = NULL; 1198 goto err; 1199 } 1200 rte_pktmbuf_pool_init(rte_mp, NULL); 1201 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1202 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1203 break; 1204 } 1205 case MP_ALLOC_XMEM: 1206 case MP_ALLOC_XMEM_HUGE: 1207 { 1208 int heap_socket; 1209 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1210 1211 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1212 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1213 1214 heap_socket = 1215 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1216 if (heap_socket < 0) 1217 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1218 1219 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1220 rte_mbuf_best_mempool_ops()); 1221 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1222 mb_mempool_cache, 0, mbuf_seg_size, 1223 heap_socket); 1224 break; 1225 } 1226 #endif 1227 case MP_ALLOC_XBUF: 1228 { 1229 struct rte_pktmbuf_extmem *ext_mem; 1230 unsigned int ext_num; 1231 1232 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1233 socket_id, pool_name, &ext_mem); 1234 if (ext_num == 0) 1235 rte_exit(EXIT_FAILURE, 1236 "Can't create pinned data buffers\n"); 1237 1238 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1239 rte_mbuf_best_mempool_ops()); 1240 rte_mp = rte_pktmbuf_pool_create_extbuf 1241 (pool_name, nb_mbuf, mb_mempool_cache, 1242 0, mbuf_seg_size, socket_id, 1243 ext_mem, ext_num); 1244 free(ext_mem); 1245 break; 1246 } 1247 default: 1248 { 1249 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1250 } 1251 } 1252 1253 #ifndef RTE_EXEC_ENV_WINDOWS 1254 err: 1255 #endif 1256 if (rte_mp == NULL) { 1257 rte_exit(EXIT_FAILURE, 1258 "Creation of mbuf pool for socket %u failed: %s\n", 1259 socket_id, rte_strerror(rte_errno)); 1260 } else if (verbose_level > 0) { 1261 rte_mempool_dump(stdout, rte_mp); 1262 } 1263 return rte_mp; 1264 } 1265 1266 /* 1267 * Check given socket id is valid or not with NUMA mode, 1268 * if valid, return 0, else return -1 1269 */ 1270 static int 1271 check_socket_id(const unsigned int socket_id) 1272 { 1273 static int warning_once = 0; 1274 1275 if (new_socket_id(socket_id)) { 1276 if (!warning_once && numa_support) 1277 fprintf(stderr, 1278 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1279 warning_once = 1; 1280 return -1; 1281 } 1282 return 0; 1283 } 1284 1285 /* 1286 * Get the allowed maximum number of RX queues. 1287 * *pid return the port id which has minimal value of 1288 * max_rx_queues in all ports. 1289 */ 1290 queueid_t 1291 get_allowed_max_nb_rxq(portid_t *pid) 1292 { 1293 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1294 bool max_rxq_valid = false; 1295 portid_t pi; 1296 struct rte_eth_dev_info dev_info; 1297 1298 RTE_ETH_FOREACH_DEV(pi) { 1299 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1300 continue; 1301 1302 max_rxq_valid = true; 1303 if (dev_info.max_rx_queues < allowed_max_rxq) { 1304 allowed_max_rxq = dev_info.max_rx_queues; 1305 *pid = pi; 1306 } 1307 } 1308 return max_rxq_valid ? allowed_max_rxq : 0; 1309 } 1310 1311 /* 1312 * Check input rxq is valid or not. 1313 * If input rxq is not greater than any of maximum number 1314 * of RX queues of all ports, it is valid. 1315 * if valid, return 0, else return -1 1316 */ 1317 int 1318 check_nb_rxq(queueid_t rxq) 1319 { 1320 queueid_t allowed_max_rxq; 1321 portid_t pid = 0; 1322 1323 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1324 if (rxq > allowed_max_rxq) { 1325 fprintf(stderr, 1326 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1327 rxq, allowed_max_rxq, pid); 1328 return -1; 1329 } 1330 return 0; 1331 } 1332 1333 /* 1334 * Get the allowed maximum number of TX queues. 1335 * *pid return the port id which has minimal value of 1336 * max_tx_queues in all ports. 1337 */ 1338 queueid_t 1339 get_allowed_max_nb_txq(portid_t *pid) 1340 { 1341 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1342 bool max_txq_valid = false; 1343 portid_t pi; 1344 struct rte_eth_dev_info dev_info; 1345 1346 RTE_ETH_FOREACH_DEV(pi) { 1347 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1348 continue; 1349 1350 max_txq_valid = true; 1351 if (dev_info.max_tx_queues < allowed_max_txq) { 1352 allowed_max_txq = dev_info.max_tx_queues; 1353 *pid = pi; 1354 } 1355 } 1356 return max_txq_valid ? allowed_max_txq : 0; 1357 } 1358 1359 /* 1360 * Check input txq is valid or not. 1361 * If input txq is not greater than any of maximum number 1362 * of TX queues of all ports, it is valid. 1363 * if valid, return 0, else return -1 1364 */ 1365 int 1366 check_nb_txq(queueid_t txq) 1367 { 1368 queueid_t allowed_max_txq; 1369 portid_t pid = 0; 1370 1371 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1372 if (txq > allowed_max_txq) { 1373 fprintf(stderr, 1374 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1375 txq, allowed_max_txq, pid); 1376 return -1; 1377 } 1378 return 0; 1379 } 1380 1381 /* 1382 * Get the allowed maximum number of RXDs of every rx queue. 1383 * *pid return the port id which has minimal value of 1384 * max_rxd in all queues of all ports. 1385 */ 1386 static uint16_t 1387 get_allowed_max_nb_rxd(portid_t *pid) 1388 { 1389 uint16_t allowed_max_rxd = UINT16_MAX; 1390 portid_t pi; 1391 struct rte_eth_dev_info dev_info; 1392 1393 RTE_ETH_FOREACH_DEV(pi) { 1394 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1395 continue; 1396 1397 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1398 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1399 *pid = pi; 1400 } 1401 } 1402 return allowed_max_rxd; 1403 } 1404 1405 /* 1406 * Get the allowed minimal number of RXDs of every rx queue. 1407 * *pid return the port id which has minimal value of 1408 * min_rxd in all queues of all ports. 1409 */ 1410 static uint16_t 1411 get_allowed_min_nb_rxd(portid_t *pid) 1412 { 1413 uint16_t allowed_min_rxd = 0; 1414 portid_t pi; 1415 struct rte_eth_dev_info dev_info; 1416 1417 RTE_ETH_FOREACH_DEV(pi) { 1418 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1419 continue; 1420 1421 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1422 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1423 *pid = pi; 1424 } 1425 } 1426 1427 return allowed_min_rxd; 1428 } 1429 1430 /* 1431 * Check input rxd is valid or not. 1432 * If input rxd is not greater than any of maximum number 1433 * of RXDs of every Rx queues and is not less than any of 1434 * minimal number of RXDs of every Rx queues, it is valid. 1435 * if valid, return 0, else return -1 1436 */ 1437 int 1438 check_nb_rxd(queueid_t rxd) 1439 { 1440 uint16_t allowed_max_rxd; 1441 uint16_t allowed_min_rxd; 1442 portid_t pid = 0; 1443 1444 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1445 if (rxd > allowed_max_rxd) { 1446 fprintf(stderr, 1447 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1448 rxd, allowed_max_rxd, pid); 1449 return -1; 1450 } 1451 1452 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1453 if (rxd < allowed_min_rxd) { 1454 fprintf(stderr, 1455 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1456 rxd, allowed_min_rxd, pid); 1457 return -1; 1458 } 1459 1460 return 0; 1461 } 1462 1463 /* 1464 * Get the allowed maximum number of TXDs of every rx queues. 1465 * *pid return the port id which has minimal value of 1466 * max_txd in every tx queue. 1467 */ 1468 static uint16_t 1469 get_allowed_max_nb_txd(portid_t *pid) 1470 { 1471 uint16_t allowed_max_txd = UINT16_MAX; 1472 portid_t pi; 1473 struct rte_eth_dev_info dev_info; 1474 1475 RTE_ETH_FOREACH_DEV(pi) { 1476 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1477 continue; 1478 1479 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1480 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1481 *pid = pi; 1482 } 1483 } 1484 return allowed_max_txd; 1485 } 1486 1487 /* 1488 * Get the allowed maximum number of TXDs of every tx queues. 1489 * *pid return the port id which has minimal value of 1490 * min_txd in every tx queue. 1491 */ 1492 static uint16_t 1493 get_allowed_min_nb_txd(portid_t *pid) 1494 { 1495 uint16_t allowed_min_txd = 0; 1496 portid_t pi; 1497 struct rte_eth_dev_info dev_info; 1498 1499 RTE_ETH_FOREACH_DEV(pi) { 1500 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1501 continue; 1502 1503 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1504 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1505 *pid = pi; 1506 } 1507 } 1508 1509 return allowed_min_txd; 1510 } 1511 1512 /* 1513 * Check input txd is valid or not. 1514 * If input txd is not greater than any of maximum number 1515 * of TXDs of every Rx queues, it is valid. 1516 * if valid, return 0, else return -1 1517 */ 1518 int 1519 check_nb_txd(queueid_t txd) 1520 { 1521 uint16_t allowed_max_txd; 1522 uint16_t allowed_min_txd; 1523 portid_t pid = 0; 1524 1525 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1526 if (txd > allowed_max_txd) { 1527 fprintf(stderr, 1528 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1529 txd, allowed_max_txd, pid); 1530 return -1; 1531 } 1532 1533 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1534 if (txd < allowed_min_txd) { 1535 fprintf(stderr, 1536 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1537 txd, allowed_min_txd, pid); 1538 return -1; 1539 } 1540 return 0; 1541 } 1542 1543 1544 /* 1545 * Get the allowed maximum number of hairpin queues. 1546 * *pid return the port id which has minimal value of 1547 * max_hairpin_queues in all ports. 1548 */ 1549 queueid_t 1550 get_allowed_max_nb_hairpinq(portid_t *pid) 1551 { 1552 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1553 portid_t pi; 1554 struct rte_eth_hairpin_cap cap; 1555 1556 RTE_ETH_FOREACH_DEV(pi) { 1557 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1558 *pid = pi; 1559 return 0; 1560 } 1561 if (cap.max_nb_queues < allowed_max_hairpinq) { 1562 allowed_max_hairpinq = cap.max_nb_queues; 1563 *pid = pi; 1564 } 1565 } 1566 return allowed_max_hairpinq; 1567 } 1568 1569 /* 1570 * Check input hairpin is valid or not. 1571 * If input hairpin is not greater than any of maximum number 1572 * of hairpin queues of all ports, it is valid. 1573 * if valid, return 0, else return -1 1574 */ 1575 int 1576 check_nb_hairpinq(queueid_t hairpinq) 1577 { 1578 queueid_t allowed_max_hairpinq; 1579 portid_t pid = 0; 1580 1581 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1582 if (hairpinq > allowed_max_hairpinq) { 1583 fprintf(stderr, 1584 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1585 hairpinq, allowed_max_hairpinq, pid); 1586 return -1; 1587 } 1588 return 0; 1589 } 1590 1591 static int 1592 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1593 { 1594 uint32_t eth_overhead; 1595 1596 if (dev_info->max_mtu != UINT16_MAX && 1597 dev_info->max_rx_pktlen > dev_info->max_mtu) 1598 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1599 else 1600 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1601 1602 return eth_overhead; 1603 } 1604 1605 static void 1606 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1607 { 1608 struct rte_port *port = &ports[pid]; 1609 int ret; 1610 int i; 1611 1612 eth_rx_metadata_negotiate_mp(pid); 1613 1614 port->dev_conf.txmode = tx_mode; 1615 port->dev_conf.rxmode = rx_mode; 1616 1617 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1618 if (ret != 0) 1619 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1620 1621 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1622 port->dev_conf.txmode.offloads &= 1623 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1624 1625 /* Apply Rx offloads configuration */ 1626 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1627 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1628 /* Apply Tx offloads configuration */ 1629 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1630 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1631 1632 if (eth_link_speed) 1633 port->dev_conf.link_speeds = eth_link_speed; 1634 1635 if (max_rx_pkt_len) 1636 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1637 get_eth_overhead(&port->dev_info); 1638 1639 /* set flag to initialize port/queue */ 1640 port->need_reconfig = 1; 1641 port->need_reconfig_queues = 1; 1642 port->socket_id = socket_id; 1643 port->tx_metadata = 0; 1644 1645 /* 1646 * Check for maximum number of segments per MTU. 1647 * Accordingly update the mbuf data size. 1648 */ 1649 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1650 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1651 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1652 uint16_t mtu; 1653 1654 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1655 uint16_t data_size = (mtu + eth_overhead) / 1656 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1657 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1658 1659 if (buffer_size > mbuf_data_size[0]) { 1660 mbuf_data_size[0] = buffer_size; 1661 TESTPMD_LOG(WARNING, 1662 "Configured mbuf size of the first segment %hu\n", 1663 mbuf_data_size[0]); 1664 } 1665 } 1666 } 1667 } 1668 1669 static void 1670 init_config(void) 1671 { 1672 portid_t pid; 1673 struct rte_mempool *mbp; 1674 unsigned int nb_mbuf_per_pool; 1675 lcoreid_t lc_id; 1676 #ifdef RTE_LIB_GRO 1677 struct rte_gro_param gro_param; 1678 #endif 1679 #ifdef RTE_LIB_GSO 1680 uint32_t gso_types; 1681 #endif 1682 1683 /* Configuration of logical cores. */ 1684 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1685 sizeof(struct fwd_lcore *) * nb_lcores, 1686 RTE_CACHE_LINE_SIZE); 1687 if (fwd_lcores == NULL) { 1688 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1689 "failed\n", nb_lcores); 1690 } 1691 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1692 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1693 sizeof(struct fwd_lcore), 1694 RTE_CACHE_LINE_SIZE); 1695 if (fwd_lcores[lc_id] == NULL) { 1696 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1697 "failed\n"); 1698 } 1699 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1700 } 1701 1702 RTE_ETH_FOREACH_DEV(pid) { 1703 uint32_t socket_id; 1704 1705 if (numa_support) { 1706 socket_id = port_numa[pid]; 1707 if (port_numa[pid] == NUMA_NO_CONFIG) { 1708 socket_id = rte_eth_dev_socket_id(pid); 1709 1710 /* 1711 * if socket_id is invalid, 1712 * set to the first available socket. 1713 */ 1714 if (check_socket_id(socket_id) < 0) 1715 socket_id = socket_ids[0]; 1716 } 1717 } else { 1718 socket_id = (socket_num == UMA_NO_CONFIG) ? 1719 0 : socket_num; 1720 } 1721 /* Apply default TxRx configuration for all ports */ 1722 init_config_port_offloads(pid, socket_id); 1723 } 1724 /* 1725 * Create pools of mbuf. 1726 * If NUMA support is disabled, create a single pool of mbuf in 1727 * socket 0 memory by default. 1728 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1729 * 1730 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1731 * nb_txd can be configured at run time. 1732 */ 1733 if (param_total_num_mbufs) 1734 nb_mbuf_per_pool = param_total_num_mbufs; 1735 else { 1736 nb_mbuf_per_pool = RX_DESC_MAX + 1737 (nb_lcores * mb_mempool_cache) + 1738 TX_DESC_MAX + MAX_PKT_BURST; 1739 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1740 } 1741 1742 if (numa_support) { 1743 uint8_t i, j; 1744 1745 for (i = 0; i < num_sockets; i++) 1746 for (j = 0; j < mbuf_data_size_n; j++) 1747 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1748 mbuf_pool_create(mbuf_data_size[j], 1749 nb_mbuf_per_pool, 1750 socket_ids[i], j); 1751 } else { 1752 uint8_t i; 1753 1754 for (i = 0; i < mbuf_data_size_n; i++) 1755 mempools[i] = mbuf_pool_create 1756 (mbuf_data_size[i], 1757 nb_mbuf_per_pool, 1758 socket_num == UMA_NO_CONFIG ? 1759 0 : socket_num, i); 1760 } 1761 1762 init_port_config(); 1763 1764 #ifdef RTE_LIB_GSO 1765 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1766 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1767 #endif 1768 /* 1769 * Records which Mbuf pool to use by each logical core, if needed. 1770 */ 1771 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1772 mbp = mbuf_pool_find( 1773 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1774 1775 if (mbp == NULL) 1776 mbp = mbuf_pool_find(0, 0); 1777 fwd_lcores[lc_id]->mbp = mbp; 1778 #ifdef RTE_LIB_GSO 1779 /* initialize GSO context */ 1780 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1781 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1782 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1783 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1784 RTE_ETHER_CRC_LEN; 1785 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1786 #endif 1787 } 1788 1789 fwd_config_setup(); 1790 1791 #ifdef RTE_LIB_GRO 1792 /* create a gro context for each lcore */ 1793 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1794 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1795 gro_param.max_item_per_flow = MAX_PKT_BURST; 1796 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1797 gro_param.socket_id = rte_lcore_to_socket_id( 1798 fwd_lcores_cpuids[lc_id]); 1799 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1800 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1801 rte_exit(EXIT_FAILURE, 1802 "rte_gro_ctx_create() failed\n"); 1803 } 1804 } 1805 #endif 1806 } 1807 1808 1809 void 1810 reconfig(portid_t new_port_id, unsigned socket_id) 1811 { 1812 /* Reconfiguration of Ethernet ports. */ 1813 init_config_port_offloads(new_port_id, socket_id); 1814 init_port_config(); 1815 } 1816 1817 int 1818 init_fwd_streams(void) 1819 { 1820 portid_t pid; 1821 struct rte_port *port; 1822 streamid_t sm_id, nb_fwd_streams_new; 1823 queueid_t q; 1824 1825 /* set socket id according to numa or not */ 1826 RTE_ETH_FOREACH_DEV(pid) { 1827 port = &ports[pid]; 1828 if (nb_rxq > port->dev_info.max_rx_queues) { 1829 fprintf(stderr, 1830 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1831 nb_rxq, port->dev_info.max_rx_queues); 1832 return -1; 1833 } 1834 if (nb_txq > port->dev_info.max_tx_queues) { 1835 fprintf(stderr, 1836 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1837 nb_txq, port->dev_info.max_tx_queues); 1838 return -1; 1839 } 1840 if (numa_support) { 1841 if (port_numa[pid] != NUMA_NO_CONFIG) 1842 port->socket_id = port_numa[pid]; 1843 else { 1844 port->socket_id = rte_eth_dev_socket_id(pid); 1845 1846 /* 1847 * if socket_id is invalid, 1848 * set to the first available socket. 1849 */ 1850 if (check_socket_id(port->socket_id) < 0) 1851 port->socket_id = socket_ids[0]; 1852 } 1853 } 1854 else { 1855 if (socket_num == UMA_NO_CONFIG) 1856 port->socket_id = 0; 1857 else 1858 port->socket_id = socket_num; 1859 } 1860 } 1861 1862 q = RTE_MAX(nb_rxq, nb_txq); 1863 if (q == 0) { 1864 fprintf(stderr, 1865 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1866 return -1; 1867 } 1868 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1869 if (nb_fwd_streams_new == nb_fwd_streams) 1870 return 0; 1871 /* clear the old */ 1872 if (fwd_streams != NULL) { 1873 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1874 if (fwd_streams[sm_id] == NULL) 1875 continue; 1876 rte_free(fwd_streams[sm_id]); 1877 fwd_streams[sm_id] = NULL; 1878 } 1879 rte_free(fwd_streams); 1880 fwd_streams = NULL; 1881 } 1882 1883 /* init new */ 1884 nb_fwd_streams = nb_fwd_streams_new; 1885 if (nb_fwd_streams) { 1886 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1887 sizeof(struct fwd_stream *) * nb_fwd_streams, 1888 RTE_CACHE_LINE_SIZE); 1889 if (fwd_streams == NULL) 1890 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1891 " (struct fwd_stream *)) failed\n", 1892 nb_fwd_streams); 1893 1894 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1895 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1896 " struct fwd_stream", sizeof(struct fwd_stream), 1897 RTE_CACHE_LINE_SIZE); 1898 if (fwd_streams[sm_id] == NULL) 1899 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1900 "(struct fwd_stream) failed\n"); 1901 } 1902 } 1903 1904 return 0; 1905 } 1906 1907 static void 1908 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1909 { 1910 uint64_t total_burst, sburst; 1911 uint64_t nb_burst; 1912 uint64_t burst_stats[4]; 1913 uint16_t pktnb_stats[4]; 1914 uint16_t nb_pkt; 1915 int burst_percent[4], sburstp; 1916 int i; 1917 1918 /* 1919 * First compute the total number of packet bursts and the 1920 * two highest numbers of bursts of the same number of packets. 1921 */ 1922 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1923 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1924 1925 /* Show stats for 0 burst size always */ 1926 total_burst = pbs->pkt_burst_spread[0]; 1927 burst_stats[0] = pbs->pkt_burst_spread[0]; 1928 pktnb_stats[0] = 0; 1929 1930 /* Find the next 2 burst sizes with highest occurrences. */ 1931 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1932 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1933 1934 if (nb_burst == 0) 1935 continue; 1936 1937 total_burst += nb_burst; 1938 1939 if (nb_burst > burst_stats[1]) { 1940 burst_stats[2] = burst_stats[1]; 1941 pktnb_stats[2] = pktnb_stats[1]; 1942 burst_stats[1] = nb_burst; 1943 pktnb_stats[1] = nb_pkt; 1944 } else if (nb_burst > burst_stats[2]) { 1945 burst_stats[2] = nb_burst; 1946 pktnb_stats[2] = nb_pkt; 1947 } 1948 } 1949 if (total_burst == 0) 1950 return; 1951 1952 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1953 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1954 if (i == 3) { 1955 printf("%d%% of other]\n", 100 - sburstp); 1956 return; 1957 } 1958 1959 sburst += burst_stats[i]; 1960 if (sburst == total_burst) { 1961 printf("%d%% of %d pkts]\n", 1962 100 - sburstp, (int) pktnb_stats[i]); 1963 return; 1964 } 1965 1966 burst_percent[i] = 1967 (double)burst_stats[i] / total_burst * 100; 1968 printf("%d%% of %d pkts + ", 1969 burst_percent[i], (int) pktnb_stats[i]); 1970 sburstp += burst_percent[i]; 1971 } 1972 } 1973 1974 static void 1975 fwd_stream_stats_display(streamid_t stream_id) 1976 { 1977 struct fwd_stream *fs; 1978 static const char *fwd_top_stats_border = "-------"; 1979 1980 fs = fwd_streams[stream_id]; 1981 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1982 (fs->fwd_dropped == 0)) 1983 return; 1984 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1985 "TX Port=%2d/Queue=%2d %s\n", 1986 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1987 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1988 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1989 " TX-dropped: %-14"PRIu64, 1990 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1991 1992 /* if checksum mode */ 1993 if (cur_fwd_eng == &csum_fwd_engine) { 1994 printf(" RX- bad IP checksum: %-14"PRIu64 1995 " Rx- bad L4 checksum: %-14"PRIu64 1996 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1997 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1998 fs->rx_bad_outer_l4_csum); 1999 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2000 fs->rx_bad_outer_ip_csum); 2001 } else { 2002 printf("\n"); 2003 } 2004 2005 if (record_burst_stats) { 2006 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2007 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2008 } 2009 } 2010 2011 void 2012 fwd_stats_display(void) 2013 { 2014 static const char *fwd_stats_border = "----------------------"; 2015 static const char *acc_stats_border = "+++++++++++++++"; 2016 struct { 2017 struct fwd_stream *rx_stream; 2018 struct fwd_stream *tx_stream; 2019 uint64_t tx_dropped; 2020 uint64_t rx_bad_ip_csum; 2021 uint64_t rx_bad_l4_csum; 2022 uint64_t rx_bad_outer_l4_csum; 2023 uint64_t rx_bad_outer_ip_csum; 2024 } ports_stats[RTE_MAX_ETHPORTS]; 2025 uint64_t total_rx_dropped = 0; 2026 uint64_t total_tx_dropped = 0; 2027 uint64_t total_rx_nombuf = 0; 2028 struct rte_eth_stats stats; 2029 uint64_t fwd_cycles = 0; 2030 uint64_t total_recv = 0; 2031 uint64_t total_xmit = 0; 2032 struct rte_port *port; 2033 streamid_t sm_id; 2034 portid_t pt_id; 2035 int ret; 2036 int i; 2037 2038 memset(ports_stats, 0, sizeof(ports_stats)); 2039 2040 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2041 struct fwd_stream *fs = fwd_streams[sm_id]; 2042 2043 if (cur_fwd_config.nb_fwd_streams > 2044 cur_fwd_config.nb_fwd_ports) { 2045 fwd_stream_stats_display(sm_id); 2046 } else { 2047 ports_stats[fs->tx_port].tx_stream = fs; 2048 ports_stats[fs->rx_port].rx_stream = fs; 2049 } 2050 2051 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2052 2053 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2054 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2055 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2056 fs->rx_bad_outer_l4_csum; 2057 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2058 fs->rx_bad_outer_ip_csum; 2059 2060 if (record_core_cycles) 2061 fwd_cycles += fs->busy_cycles; 2062 } 2063 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2064 uint64_t tx_dropped = 0; 2065 2066 pt_id = fwd_ports_ids[i]; 2067 port = &ports[pt_id]; 2068 2069 ret = rte_eth_stats_get(pt_id, &stats); 2070 if (ret != 0) { 2071 fprintf(stderr, 2072 "%s: Error: failed to get stats (port %u): %d", 2073 __func__, pt_id, ret); 2074 continue; 2075 } 2076 stats.ipackets -= port->stats.ipackets; 2077 stats.opackets -= port->stats.opackets; 2078 stats.ibytes -= port->stats.ibytes; 2079 stats.obytes -= port->stats.obytes; 2080 stats.imissed -= port->stats.imissed; 2081 stats.oerrors -= port->stats.oerrors; 2082 stats.rx_nombuf -= port->stats.rx_nombuf; 2083 2084 total_recv += stats.ipackets; 2085 total_xmit += stats.opackets; 2086 total_rx_dropped += stats.imissed; 2087 tx_dropped += ports_stats[pt_id].tx_dropped; 2088 tx_dropped += stats.oerrors; 2089 total_tx_dropped += tx_dropped; 2090 total_rx_nombuf += stats.rx_nombuf; 2091 2092 printf("\n %s Forward statistics for port %-2d %s\n", 2093 fwd_stats_border, pt_id, fwd_stats_border); 2094 2095 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2096 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2097 stats.ipackets + stats.imissed); 2098 2099 if (cur_fwd_eng == &csum_fwd_engine) { 2100 printf(" Bad-ipcsum: %-14"PRIu64 2101 " Bad-l4csum: %-14"PRIu64 2102 "Bad-outer-l4csum: %-14"PRIu64"\n", 2103 ports_stats[pt_id].rx_bad_ip_csum, 2104 ports_stats[pt_id].rx_bad_l4_csum, 2105 ports_stats[pt_id].rx_bad_outer_l4_csum); 2106 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2107 ports_stats[pt_id].rx_bad_outer_ip_csum); 2108 } 2109 if (stats.ierrors + stats.rx_nombuf > 0) { 2110 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2111 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2112 } 2113 2114 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2115 "TX-total: %-"PRIu64"\n", 2116 stats.opackets, tx_dropped, 2117 stats.opackets + tx_dropped); 2118 2119 if (record_burst_stats) { 2120 if (ports_stats[pt_id].rx_stream) 2121 pkt_burst_stats_display("RX", 2122 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2123 if (ports_stats[pt_id].tx_stream) 2124 pkt_burst_stats_display("TX", 2125 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2126 } 2127 2128 printf(" %s--------------------------------%s\n", 2129 fwd_stats_border, fwd_stats_border); 2130 } 2131 2132 printf("\n %s Accumulated forward statistics for all ports" 2133 "%s\n", 2134 acc_stats_border, acc_stats_border); 2135 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2136 "%-"PRIu64"\n" 2137 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2138 "%-"PRIu64"\n", 2139 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2140 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2141 if (total_rx_nombuf > 0) 2142 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2143 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2144 "%s\n", 2145 acc_stats_border, acc_stats_border); 2146 if (record_core_cycles) { 2147 #define CYC_PER_MHZ 1E6 2148 if (total_recv > 0 || total_xmit > 0) { 2149 uint64_t total_pkts = 0; 2150 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2151 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2152 total_pkts = total_xmit; 2153 else 2154 total_pkts = total_recv; 2155 2156 printf("\n CPU cycles/packet=%.2F (busy cycles=" 2157 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2158 " MHz Clock\n", 2159 (double) fwd_cycles / total_pkts, 2160 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2161 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2162 } 2163 } 2164 } 2165 2166 void 2167 fwd_stats_reset(void) 2168 { 2169 streamid_t sm_id; 2170 portid_t pt_id; 2171 int ret; 2172 int i; 2173 2174 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2175 pt_id = fwd_ports_ids[i]; 2176 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2177 if (ret != 0) 2178 fprintf(stderr, 2179 "%s: Error: failed to clear stats (port %u):%d", 2180 __func__, pt_id, ret); 2181 } 2182 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2183 struct fwd_stream *fs = fwd_streams[sm_id]; 2184 2185 fs->rx_packets = 0; 2186 fs->tx_packets = 0; 2187 fs->fwd_dropped = 0; 2188 fs->rx_bad_ip_csum = 0; 2189 fs->rx_bad_l4_csum = 0; 2190 fs->rx_bad_outer_l4_csum = 0; 2191 fs->rx_bad_outer_ip_csum = 0; 2192 2193 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2194 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2195 fs->busy_cycles = 0; 2196 } 2197 } 2198 2199 static void 2200 flush_fwd_rx_queues(void) 2201 { 2202 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2203 portid_t rxp; 2204 portid_t port_id; 2205 queueid_t rxq; 2206 uint16_t nb_rx; 2207 uint8_t j; 2208 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2209 uint64_t timer_period; 2210 2211 if (num_procs > 1) { 2212 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2213 return; 2214 } 2215 2216 /* convert to number of cycles */ 2217 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2218 2219 for (j = 0; j < 2; j++) { 2220 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2221 for (rxq = 0; rxq < nb_rxq; rxq++) { 2222 port_id = fwd_ports_ids[rxp]; 2223 2224 /* Polling stopped queues is prohibited. */ 2225 if (ports[port_id].rxq[rxq].state == 2226 RTE_ETH_QUEUE_STATE_STOPPED) 2227 continue; 2228 2229 /** 2230 * testpmd can stuck in the below do while loop 2231 * if rte_eth_rx_burst() always returns nonzero 2232 * packets. So timer is added to exit this loop 2233 * after 1sec timer expiry. 2234 */ 2235 prev_tsc = rte_rdtsc(); 2236 do { 2237 nb_rx = rte_eth_rx_burst(port_id, rxq, 2238 pkts_burst, MAX_PKT_BURST); 2239 rte_pktmbuf_free_bulk(pkts_burst, nb_rx); 2240 2241 cur_tsc = rte_rdtsc(); 2242 diff_tsc = cur_tsc - prev_tsc; 2243 timer_tsc += diff_tsc; 2244 } while ((nb_rx > 0) && 2245 (timer_tsc < timer_period)); 2246 timer_tsc = 0; 2247 } 2248 } 2249 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2250 } 2251 } 2252 2253 static void 2254 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2255 { 2256 struct fwd_stream **fsm; 2257 uint64_t prev_tsc; 2258 streamid_t nb_fs; 2259 streamid_t sm_id; 2260 #ifdef RTE_LIB_BITRATESTATS 2261 uint64_t tics_per_1sec; 2262 uint64_t tics_datum; 2263 uint64_t tics_current; 2264 uint16_t i, cnt_ports; 2265 2266 cnt_ports = nb_ports; 2267 tics_datum = rte_rdtsc(); 2268 tics_per_1sec = rte_get_timer_hz(); 2269 #endif 2270 fsm = &fwd_streams[fc->stream_idx]; 2271 nb_fs = fc->stream_nb; 2272 prev_tsc = rte_rdtsc(); 2273 do { 2274 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2275 struct fwd_stream *fs = fsm[sm_id]; 2276 uint64_t start_fs_tsc = 0; 2277 bool busy; 2278 2279 if (fs->disabled) 2280 continue; 2281 if (record_core_cycles) 2282 start_fs_tsc = rte_rdtsc(); 2283 busy = (*pkt_fwd)(fs); 2284 if (record_core_cycles && busy) 2285 fs->busy_cycles += rte_rdtsc() - start_fs_tsc; 2286 } 2287 #ifdef RTE_LIB_BITRATESTATS 2288 if (bitrate_enabled != 0 && 2289 bitrate_lcore_id == rte_lcore_id()) { 2290 tics_current = rte_rdtsc(); 2291 if (tics_current - tics_datum >= tics_per_1sec) { 2292 /* Periodic bitrate calculation */ 2293 for (i = 0; i < cnt_ports; i++) 2294 rte_stats_bitrate_calc(bitrate_data, 2295 ports_ids[i]); 2296 tics_datum = tics_current; 2297 } 2298 } 2299 #endif 2300 #ifdef RTE_LIB_LATENCYSTATS 2301 if (latencystats_enabled != 0 && 2302 latencystats_lcore_id == rte_lcore_id()) 2303 rte_latencystats_update(); 2304 #endif 2305 if (record_core_cycles) { 2306 uint64_t tsc = rte_rdtsc(); 2307 2308 fc->total_cycles += tsc - prev_tsc; 2309 prev_tsc = tsc; 2310 } 2311 } while (! fc->stopped); 2312 } 2313 2314 static int 2315 lcore_usage_callback(unsigned int lcore_id, struct rte_lcore_usage *usage) 2316 { 2317 struct fwd_stream **fsm; 2318 struct fwd_lcore *fc; 2319 streamid_t nb_fs; 2320 streamid_t sm_id; 2321 2322 fc = lcore_to_fwd_lcore(lcore_id); 2323 if (fc == NULL) 2324 return -1; 2325 2326 fsm = &fwd_streams[fc->stream_idx]; 2327 nb_fs = fc->stream_nb; 2328 usage->busy_cycles = 0; 2329 usage->total_cycles = fc->total_cycles; 2330 2331 for (sm_id = 0; sm_id < nb_fs; sm_id++) { 2332 if (!fsm[sm_id]->disabled) 2333 usage->busy_cycles += fsm[sm_id]->busy_cycles; 2334 } 2335 2336 return 0; 2337 } 2338 2339 static int 2340 start_pkt_forward_on_core(void *fwd_arg) 2341 { 2342 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2343 cur_fwd_config.fwd_eng->packet_fwd); 2344 return 0; 2345 } 2346 2347 /* 2348 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2349 * Used to start communication flows in network loopback test configurations. 2350 */ 2351 static int 2352 run_one_txonly_burst_on_core(void *fwd_arg) 2353 { 2354 struct fwd_lcore *fwd_lc; 2355 struct fwd_lcore tmp_lcore; 2356 2357 fwd_lc = (struct fwd_lcore *) fwd_arg; 2358 tmp_lcore = *fwd_lc; 2359 tmp_lcore.stopped = 1; 2360 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2361 return 0; 2362 } 2363 2364 /* 2365 * Launch packet forwarding: 2366 * - Setup per-port forwarding context. 2367 * - launch logical cores with their forwarding configuration. 2368 */ 2369 static void 2370 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2371 { 2372 unsigned int i; 2373 unsigned int lc_id; 2374 int diag; 2375 2376 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2377 lc_id = fwd_lcores_cpuids[i]; 2378 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2379 fwd_lcores[i]->stopped = 0; 2380 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2381 fwd_lcores[i], lc_id); 2382 if (diag != 0) 2383 fprintf(stderr, 2384 "launch lcore %u failed - diag=%d\n", 2385 lc_id, diag); 2386 } 2387 } 2388 } 2389 2390 void 2391 common_fwd_stream_init(struct fwd_stream *fs) 2392 { 2393 bool rx_stopped, tx_stopped; 2394 2395 rx_stopped = (ports[fs->rx_port].rxq[fs->rx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2396 tx_stopped = (ports[fs->tx_port].txq[fs->tx_queue].state == RTE_ETH_QUEUE_STATE_STOPPED); 2397 fs->disabled = rx_stopped || tx_stopped; 2398 } 2399 2400 /* 2401 * Launch packet forwarding configuration. 2402 */ 2403 void 2404 start_packet_forwarding(int with_tx_first) 2405 { 2406 port_fwd_begin_t port_fwd_begin; 2407 port_fwd_end_t port_fwd_end; 2408 stream_init_t stream_init = cur_fwd_eng->stream_init; 2409 unsigned int i; 2410 2411 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2412 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2413 2414 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2415 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2416 2417 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2418 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2419 (!nb_rxq || !nb_txq)) 2420 rte_exit(EXIT_FAILURE, 2421 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2422 cur_fwd_eng->fwd_mode_name); 2423 2424 if (all_ports_started() == 0) { 2425 fprintf(stderr, "Not all ports were started\n"); 2426 return; 2427 } 2428 if (test_done == 0) { 2429 fprintf(stderr, "Packet forwarding already started\n"); 2430 return; 2431 } 2432 2433 fwd_config_setup(); 2434 2435 pkt_fwd_config_display(&cur_fwd_config); 2436 if (!pkt_fwd_shared_rxq_check()) 2437 return; 2438 2439 if (stream_init != NULL) 2440 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2441 stream_init(fwd_streams[i]); 2442 2443 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2444 if (port_fwd_begin != NULL) { 2445 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2446 if (port_fwd_begin(fwd_ports_ids[i])) { 2447 fprintf(stderr, 2448 "Packet forwarding is not ready\n"); 2449 return; 2450 } 2451 } 2452 } 2453 2454 if (with_tx_first) { 2455 port_fwd_begin = tx_only_engine.port_fwd_begin; 2456 if (port_fwd_begin != NULL) { 2457 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2458 if (port_fwd_begin(fwd_ports_ids[i])) { 2459 fprintf(stderr, 2460 "Packet forwarding is not ready\n"); 2461 return; 2462 } 2463 } 2464 } 2465 } 2466 2467 test_done = 0; 2468 2469 if(!no_flush_rx) 2470 flush_fwd_rx_queues(); 2471 2472 rxtx_config_display(); 2473 2474 fwd_stats_reset(); 2475 if (with_tx_first) { 2476 while (with_tx_first--) { 2477 launch_packet_forwarding( 2478 run_one_txonly_burst_on_core); 2479 rte_eal_mp_wait_lcore(); 2480 } 2481 port_fwd_end = tx_only_engine.port_fwd_end; 2482 if (port_fwd_end != NULL) { 2483 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2484 (*port_fwd_end)(fwd_ports_ids[i]); 2485 } 2486 } 2487 launch_packet_forwarding(start_pkt_forward_on_core); 2488 } 2489 2490 void 2491 stop_packet_forwarding(void) 2492 { 2493 port_fwd_end_t port_fwd_end; 2494 lcoreid_t lc_id; 2495 portid_t pt_id; 2496 int i; 2497 2498 if (test_done) { 2499 fprintf(stderr, "Packet forwarding not started\n"); 2500 return; 2501 } 2502 printf("Telling cores to stop..."); 2503 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2504 fwd_lcores[lc_id]->stopped = 1; 2505 printf("\nWaiting for lcores to finish...\n"); 2506 rte_eal_mp_wait_lcore(); 2507 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2508 if (port_fwd_end != NULL) { 2509 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2510 pt_id = fwd_ports_ids[i]; 2511 (*port_fwd_end)(pt_id); 2512 } 2513 } 2514 2515 fwd_stats_display(); 2516 2517 printf("\nDone.\n"); 2518 test_done = 1; 2519 } 2520 2521 void 2522 dev_set_link_up(portid_t pid) 2523 { 2524 if (rte_eth_dev_set_link_up(pid) < 0) 2525 fprintf(stderr, "\nSet link up fail.\n"); 2526 } 2527 2528 void 2529 dev_set_link_down(portid_t pid) 2530 { 2531 if (rte_eth_dev_set_link_down(pid) < 0) 2532 fprintf(stderr, "\nSet link down fail.\n"); 2533 } 2534 2535 static int 2536 all_ports_started(void) 2537 { 2538 portid_t pi; 2539 struct rte_port *port; 2540 2541 RTE_ETH_FOREACH_DEV(pi) { 2542 port = &ports[pi]; 2543 /* Check if there is a port which is not started */ 2544 if ((port->port_status != RTE_PORT_STARTED) && 2545 (port->slave_flag == 0)) 2546 return 0; 2547 } 2548 2549 /* No port is not started */ 2550 return 1; 2551 } 2552 2553 int 2554 port_is_stopped(portid_t port_id) 2555 { 2556 struct rte_port *port = &ports[port_id]; 2557 2558 if ((port->port_status != RTE_PORT_STOPPED) && 2559 (port->slave_flag == 0)) 2560 return 0; 2561 return 1; 2562 } 2563 2564 int 2565 all_ports_stopped(void) 2566 { 2567 portid_t pi; 2568 2569 RTE_ETH_FOREACH_DEV(pi) { 2570 if (!port_is_stopped(pi)) 2571 return 0; 2572 } 2573 2574 return 1; 2575 } 2576 2577 int 2578 port_is_started(portid_t port_id) 2579 { 2580 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2581 return 0; 2582 2583 if (ports[port_id].port_status != RTE_PORT_STARTED) 2584 return 0; 2585 2586 return 1; 2587 } 2588 2589 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2590 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2591 2592 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2593 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2594 2595 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2596 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2597 2598 2599 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2600 static int 2601 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2602 { 2603 queueid_t qi; 2604 struct rte_eth_hairpin_conf hairpin_conf = { 2605 .peer_count = 1, 2606 }; 2607 int i; 2608 int diag; 2609 struct rte_port *port = &ports[pi]; 2610 uint16_t peer_rx_port = pi; 2611 uint16_t peer_tx_port = pi; 2612 uint32_t manual = 1; 2613 uint32_t tx_exp = hairpin_mode & 0x10; 2614 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2615 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2616 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2617 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2618 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2619 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2620 2621 if (!(hairpin_mode & 0xf)) { 2622 peer_rx_port = pi; 2623 peer_tx_port = pi; 2624 manual = 0; 2625 } else if (hairpin_mode & 0x1) { 2626 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2627 RTE_ETH_DEV_NO_OWNER); 2628 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2629 peer_tx_port = rte_eth_find_next_owned_by(0, 2630 RTE_ETH_DEV_NO_OWNER); 2631 if (p_pi != RTE_MAX_ETHPORTS) { 2632 peer_rx_port = p_pi; 2633 } else { 2634 uint16_t next_pi; 2635 2636 /* Last port will be the peer RX port of the first. */ 2637 RTE_ETH_FOREACH_DEV(next_pi) 2638 peer_rx_port = next_pi; 2639 } 2640 manual = 1; 2641 } else if (hairpin_mode & 0x2) { 2642 if (cnt_pi & 0x1) { 2643 peer_rx_port = p_pi; 2644 } else { 2645 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2646 RTE_ETH_DEV_NO_OWNER); 2647 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2648 peer_rx_port = pi; 2649 } 2650 peer_tx_port = peer_rx_port; 2651 manual = 1; 2652 } 2653 2654 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2655 hairpin_conf.peers[0].port = peer_rx_port; 2656 hairpin_conf.peers[0].queue = i + nb_rxq; 2657 hairpin_conf.manual_bind = !!manual; 2658 hairpin_conf.tx_explicit = !!tx_exp; 2659 hairpin_conf.force_memory = !!tx_force_memory; 2660 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2661 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2662 diag = rte_eth_tx_hairpin_queue_setup 2663 (pi, qi, nb_txd, &hairpin_conf); 2664 i++; 2665 if (diag == 0) 2666 continue; 2667 2668 /* Fail to setup rx queue, return */ 2669 if (port->port_status == RTE_PORT_HANDLING) 2670 port->port_status = RTE_PORT_STOPPED; 2671 else 2672 fprintf(stderr, 2673 "Port %d can not be set back to stopped\n", pi); 2674 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2675 pi); 2676 /* try to reconfigure queues next time */ 2677 port->need_reconfig_queues = 1; 2678 return -1; 2679 } 2680 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2681 hairpin_conf.peers[0].port = peer_tx_port; 2682 hairpin_conf.peers[0].queue = i + nb_txq; 2683 hairpin_conf.manual_bind = !!manual; 2684 hairpin_conf.tx_explicit = !!tx_exp; 2685 hairpin_conf.force_memory = !!rx_force_memory; 2686 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2687 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2688 diag = rte_eth_rx_hairpin_queue_setup 2689 (pi, qi, nb_rxd, &hairpin_conf); 2690 i++; 2691 if (diag == 0) 2692 continue; 2693 2694 /* Fail to setup rx queue, return */ 2695 if (port->port_status == RTE_PORT_HANDLING) 2696 port->port_status = RTE_PORT_STOPPED; 2697 else 2698 fprintf(stderr, 2699 "Port %d can not be set back to stopped\n", pi); 2700 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2701 pi); 2702 /* try to reconfigure queues next time */ 2703 port->need_reconfig_queues = 1; 2704 return -1; 2705 } 2706 return 0; 2707 } 2708 2709 /* Configure the Rx with optional split. */ 2710 int 2711 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2712 uint16_t nb_rx_desc, unsigned int socket_id, 2713 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2714 { 2715 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2716 struct rte_mempool *rx_mempool[MAX_MEMPOOL] = {}; 2717 struct rte_mempool *mpx; 2718 unsigned int i, mp_n; 2719 uint32_t prev_hdrs = 0; 2720 int ret; 2721 2722 2723 if ((rx_pkt_nb_segs > 1) && 2724 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2725 /* multi-segment configuration */ 2726 for (i = 0; i < rx_pkt_nb_segs; i++) { 2727 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2728 /* 2729 * Use last valid pool for the segments with number 2730 * exceeding the pool index. 2731 */ 2732 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2733 mpx = mbuf_pool_find(socket_id, mp_n); 2734 /* Handle zero as mbuf data buffer size. */ 2735 rx_seg->offset = i < rx_pkt_nb_offs ? 2736 rx_pkt_seg_offsets[i] : 0; 2737 rx_seg->mp = mpx ? mpx : mp; 2738 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2739 rx_seg->proto_hdr = rx_pkt_hdr_protos[i] & ~prev_hdrs; 2740 prev_hdrs |= rx_seg->proto_hdr; 2741 } else { 2742 rx_seg->length = rx_pkt_seg_lengths[i] ? 2743 rx_pkt_seg_lengths[i] : 2744 mbuf_data_size[mp_n]; 2745 } 2746 } 2747 rx_conf->rx_nseg = rx_pkt_nb_segs; 2748 rx_conf->rx_seg = rx_useg; 2749 rx_conf->rx_mempools = NULL; 2750 rx_conf->rx_nmempool = 0; 2751 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2752 socket_id, rx_conf, NULL); 2753 rx_conf->rx_seg = NULL; 2754 rx_conf->rx_nseg = 0; 2755 } else if (multi_rx_mempool == 1) { 2756 /* multi-pool configuration */ 2757 struct rte_eth_dev_info dev_info; 2758 2759 if (mbuf_data_size_n <= 1) { 2760 fprintf(stderr, "Invalid number of mempools %u\n", 2761 mbuf_data_size_n); 2762 return -EINVAL; 2763 } 2764 ret = rte_eth_dev_info_get(port_id, &dev_info); 2765 if (ret != 0) 2766 return ret; 2767 if (dev_info.max_rx_mempools == 0) { 2768 fprintf(stderr, 2769 "Port %u doesn't support requested multi-rx-mempool configuration.\n", 2770 port_id); 2771 return -ENOTSUP; 2772 } 2773 for (i = 0; i < mbuf_data_size_n; i++) { 2774 mpx = mbuf_pool_find(socket_id, i); 2775 rx_mempool[i] = mpx ? mpx : mp; 2776 } 2777 rx_conf->rx_mempools = rx_mempool; 2778 rx_conf->rx_nmempool = mbuf_data_size_n; 2779 rx_conf->rx_seg = NULL; 2780 rx_conf->rx_nseg = 0; 2781 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2782 socket_id, rx_conf, NULL); 2783 rx_conf->rx_mempools = NULL; 2784 rx_conf->rx_nmempool = 0; 2785 } else { 2786 /* Single pool/segment configuration */ 2787 rx_conf->rx_seg = NULL; 2788 rx_conf->rx_nseg = 0; 2789 rx_conf->rx_mempools = NULL; 2790 rx_conf->rx_nmempool = 0; 2791 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2792 socket_id, rx_conf, mp); 2793 } 2794 2795 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2796 RTE_ETH_QUEUE_STATE_STOPPED : 2797 RTE_ETH_QUEUE_STATE_STARTED; 2798 return ret; 2799 } 2800 2801 static int 2802 alloc_xstats_display_info(portid_t pi) 2803 { 2804 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2805 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2806 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2807 2808 if (xstats_display_num == 0) 2809 return 0; 2810 2811 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2812 if (*ids_supp == NULL) 2813 goto fail_ids_supp; 2814 2815 *prev_values = calloc(xstats_display_num, 2816 sizeof(**prev_values)); 2817 if (*prev_values == NULL) 2818 goto fail_prev_values; 2819 2820 *curr_values = calloc(xstats_display_num, 2821 sizeof(**curr_values)); 2822 if (*curr_values == NULL) 2823 goto fail_curr_values; 2824 2825 ports[pi].xstats_info.allocated = true; 2826 2827 return 0; 2828 2829 fail_curr_values: 2830 free(*prev_values); 2831 fail_prev_values: 2832 free(*ids_supp); 2833 fail_ids_supp: 2834 return -ENOMEM; 2835 } 2836 2837 static void 2838 free_xstats_display_info(portid_t pi) 2839 { 2840 if (!ports[pi].xstats_info.allocated) 2841 return; 2842 free(ports[pi].xstats_info.ids_supp); 2843 free(ports[pi].xstats_info.prev_values); 2844 free(ports[pi].xstats_info.curr_values); 2845 ports[pi].xstats_info.allocated = false; 2846 } 2847 2848 /** Fill helper structures for specified port to show extended statistics. */ 2849 static void 2850 fill_xstats_display_info_for_port(portid_t pi) 2851 { 2852 unsigned int stat, stat_supp; 2853 const char *xstat_name; 2854 struct rte_port *port; 2855 uint64_t *ids_supp; 2856 int rc; 2857 2858 if (xstats_display_num == 0) 2859 return; 2860 2861 if (pi == (portid_t)RTE_PORT_ALL) { 2862 fill_xstats_display_info(); 2863 return; 2864 } 2865 2866 port = &ports[pi]; 2867 if (port->port_status != RTE_PORT_STARTED) 2868 return; 2869 2870 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2871 rte_exit(EXIT_FAILURE, 2872 "Failed to allocate xstats display memory\n"); 2873 2874 ids_supp = port->xstats_info.ids_supp; 2875 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2876 xstat_name = xstats_display[stat].name; 2877 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2878 ids_supp + stat_supp); 2879 if (rc != 0) { 2880 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2881 xstat_name, pi, stat); 2882 continue; 2883 } 2884 stat_supp++; 2885 } 2886 2887 port->xstats_info.ids_supp_sz = stat_supp; 2888 } 2889 2890 /** Fill helper structures for all ports to show extended statistics. */ 2891 static void 2892 fill_xstats_display_info(void) 2893 { 2894 portid_t pi; 2895 2896 if (xstats_display_num == 0) 2897 return; 2898 2899 RTE_ETH_FOREACH_DEV(pi) 2900 fill_xstats_display_info_for_port(pi); 2901 } 2902 2903 /* 2904 * Some capabilities (like, rx_offload_capa and tx_offload_capa) of bonding 2905 * device in dev_info is zero when no slave is added. And its capability 2906 * will be updated when add a new slave device. So adding a slave device need 2907 * to update the port configurations of bonding device. 2908 */ 2909 static void 2910 update_bonding_port_dev_conf(portid_t bond_pid) 2911 { 2912 #ifdef RTE_NET_BOND 2913 struct rte_port *port = &ports[bond_pid]; 2914 uint16_t i; 2915 int ret; 2916 2917 ret = eth_dev_info_get_print_err(bond_pid, &port->dev_info); 2918 if (ret != 0) { 2919 fprintf(stderr, "Failed to get dev info for port = %u\n", 2920 bond_pid); 2921 return; 2922 } 2923 2924 if (port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 2925 port->dev_conf.txmode.offloads |= 2926 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 2927 /* Apply Tx offloads configuration */ 2928 for (i = 0; i < port->dev_info.max_tx_queues; i++) 2929 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 2930 2931 port->dev_conf.rx_adv_conf.rss_conf.rss_hf &= 2932 port->dev_info.flow_type_rss_offloads; 2933 #else 2934 RTE_SET_USED(bond_pid); 2935 #endif 2936 } 2937 2938 int 2939 start_port(portid_t pid) 2940 { 2941 int diag; 2942 portid_t pi; 2943 portid_t p_pi = RTE_MAX_ETHPORTS; 2944 portid_t pl[RTE_MAX_ETHPORTS]; 2945 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2946 uint16_t cnt_pi = 0; 2947 uint16_t cfg_pi = 0; 2948 int peer_pi; 2949 queueid_t qi; 2950 struct rte_port *port; 2951 struct rte_eth_hairpin_cap cap; 2952 bool at_least_one_port_exist = false; 2953 bool all_ports_already_started = true; 2954 bool at_least_one_port_successfully_started = false; 2955 2956 if (port_id_is_invalid(pid, ENABLED_WARN)) 2957 return 0; 2958 2959 RTE_ETH_FOREACH_DEV(pi) { 2960 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2961 continue; 2962 2963 if (port_is_bonding_slave(pi)) { 2964 fprintf(stderr, 2965 "Please remove port %d from bonded device.\n", 2966 pi); 2967 continue; 2968 } 2969 2970 at_least_one_port_exist = true; 2971 2972 port = &ports[pi]; 2973 if (port->port_status == RTE_PORT_STOPPED) { 2974 port->port_status = RTE_PORT_HANDLING; 2975 all_ports_already_started = false; 2976 } else { 2977 fprintf(stderr, "Port %d is now not stopped\n", pi); 2978 continue; 2979 } 2980 2981 if (port->need_reconfig > 0) { 2982 struct rte_eth_conf dev_conf; 2983 int k; 2984 2985 port->need_reconfig = 0; 2986 2987 if (flow_isolate_all) { 2988 int ret = port_flow_isolate(pi, 1); 2989 if (ret) { 2990 fprintf(stderr, 2991 "Failed to apply isolated mode on port %d\n", 2992 pi); 2993 return -1; 2994 } 2995 } 2996 configure_rxtx_dump_callbacks(0); 2997 printf("Configuring Port %d (socket %u)\n", pi, 2998 port->socket_id); 2999 if (nb_hairpinq > 0 && 3000 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 3001 fprintf(stderr, 3002 "Port %d doesn't support hairpin queues\n", 3003 pi); 3004 return -1; 3005 } 3006 3007 if (port->bond_flag == 1 && port->update_conf == 1) { 3008 update_bonding_port_dev_conf(pi); 3009 port->update_conf = 0; 3010 } 3011 3012 /* configure port */ 3013 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 3014 nb_txq + nb_hairpinq, 3015 &(port->dev_conf)); 3016 if (diag != 0) { 3017 if (port->port_status == RTE_PORT_HANDLING) 3018 port->port_status = RTE_PORT_STOPPED; 3019 else 3020 fprintf(stderr, 3021 "Port %d can not be set back to stopped\n", 3022 pi); 3023 fprintf(stderr, "Fail to configure port %d\n", 3024 pi); 3025 /* try to reconfigure port next time */ 3026 port->need_reconfig = 1; 3027 return -1; 3028 } 3029 /* get device configuration*/ 3030 if (0 != 3031 eth_dev_conf_get_print_err(pi, &dev_conf)) { 3032 fprintf(stderr, 3033 "port %d can not get device configuration\n", 3034 pi); 3035 return -1; 3036 } 3037 /* Apply Rx offloads configuration */ 3038 if (dev_conf.rxmode.offloads != 3039 port->dev_conf.rxmode.offloads) { 3040 port->dev_conf.rxmode.offloads |= 3041 dev_conf.rxmode.offloads; 3042 for (k = 0; 3043 k < port->dev_info.max_rx_queues; 3044 k++) 3045 port->rxq[k].conf.offloads |= 3046 dev_conf.rxmode.offloads; 3047 } 3048 /* Apply Tx offloads configuration */ 3049 if (dev_conf.txmode.offloads != 3050 port->dev_conf.txmode.offloads) { 3051 port->dev_conf.txmode.offloads |= 3052 dev_conf.txmode.offloads; 3053 for (k = 0; 3054 k < port->dev_info.max_tx_queues; 3055 k++) 3056 port->txq[k].conf.offloads |= 3057 dev_conf.txmode.offloads; 3058 } 3059 } 3060 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 3061 port->need_reconfig_queues = 0; 3062 /* setup tx queues */ 3063 for (qi = 0; qi < nb_txq; qi++) { 3064 struct rte_eth_txconf *conf = 3065 &port->txq[qi].conf; 3066 3067 if ((numa_support) && 3068 (txring_numa[pi] != NUMA_NO_CONFIG)) 3069 diag = rte_eth_tx_queue_setup(pi, qi, 3070 port->nb_tx_desc[qi], 3071 txring_numa[pi], 3072 &(port->txq[qi].conf)); 3073 else 3074 diag = rte_eth_tx_queue_setup(pi, qi, 3075 port->nb_tx_desc[qi], 3076 port->socket_id, 3077 &(port->txq[qi].conf)); 3078 3079 if (diag == 0) { 3080 port->txq[qi].state = 3081 conf->tx_deferred_start ? 3082 RTE_ETH_QUEUE_STATE_STOPPED : 3083 RTE_ETH_QUEUE_STATE_STARTED; 3084 continue; 3085 } 3086 3087 /* Fail to setup tx queue, return */ 3088 if (port->port_status == RTE_PORT_HANDLING) 3089 port->port_status = RTE_PORT_STOPPED; 3090 else 3091 fprintf(stderr, 3092 "Port %d can not be set back to stopped\n", 3093 pi); 3094 fprintf(stderr, 3095 "Fail to configure port %d tx queues\n", 3096 pi); 3097 /* try to reconfigure queues next time */ 3098 port->need_reconfig_queues = 1; 3099 return -1; 3100 } 3101 for (qi = 0; qi < nb_rxq; qi++) { 3102 /* setup rx queues */ 3103 if ((numa_support) && 3104 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 3105 struct rte_mempool * mp = 3106 mbuf_pool_find 3107 (rxring_numa[pi], 0); 3108 if (mp == NULL) { 3109 fprintf(stderr, 3110 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3111 rxring_numa[pi]); 3112 return -1; 3113 } 3114 3115 diag = rx_queue_setup(pi, qi, 3116 port->nb_rx_desc[qi], 3117 rxring_numa[pi], 3118 &(port->rxq[qi].conf), 3119 mp); 3120 } else { 3121 struct rte_mempool *mp = 3122 mbuf_pool_find 3123 (port->socket_id, 0); 3124 if (mp == NULL) { 3125 fprintf(stderr, 3126 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 3127 port->socket_id); 3128 return -1; 3129 } 3130 diag = rx_queue_setup(pi, qi, 3131 port->nb_rx_desc[qi], 3132 port->socket_id, 3133 &(port->rxq[qi].conf), 3134 mp); 3135 } 3136 if (diag == 0) 3137 continue; 3138 3139 /* Fail to setup rx queue, return */ 3140 if (port->port_status == RTE_PORT_HANDLING) 3141 port->port_status = RTE_PORT_STOPPED; 3142 else 3143 fprintf(stderr, 3144 "Port %d can not be set back to stopped\n", 3145 pi); 3146 fprintf(stderr, 3147 "Fail to configure port %d rx queues\n", 3148 pi); 3149 /* try to reconfigure queues next time */ 3150 port->need_reconfig_queues = 1; 3151 return -1; 3152 } 3153 /* setup hairpin queues */ 3154 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3155 return -1; 3156 } 3157 configure_rxtx_dump_callbacks(verbose_level); 3158 if (clear_ptypes) { 3159 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3160 NULL, 0); 3161 if (diag < 0) 3162 fprintf(stderr, 3163 "Port %d: Failed to disable Ptype parsing\n", 3164 pi); 3165 } 3166 3167 p_pi = pi; 3168 cnt_pi++; 3169 3170 /* start port */ 3171 diag = eth_dev_start_mp(pi); 3172 if (diag < 0) { 3173 fprintf(stderr, "Fail to start port %d: %s\n", 3174 pi, rte_strerror(-diag)); 3175 3176 /* Fail to setup rx queue, return */ 3177 if (port->port_status == RTE_PORT_HANDLING) 3178 port->port_status = RTE_PORT_STOPPED; 3179 else 3180 fprintf(stderr, 3181 "Port %d can not be set back to stopped\n", 3182 pi); 3183 continue; 3184 } 3185 3186 if (port->port_status == RTE_PORT_HANDLING) 3187 port->port_status = RTE_PORT_STARTED; 3188 else 3189 fprintf(stderr, "Port %d can not be set into started\n", 3190 pi); 3191 3192 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3193 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3194 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3195 3196 at_least_one_port_successfully_started = true; 3197 3198 pl[cfg_pi++] = pi; 3199 } 3200 3201 if (at_least_one_port_successfully_started && !no_link_check) 3202 check_all_ports_link_status(RTE_PORT_ALL); 3203 else if (at_least_one_port_exist & all_ports_already_started) 3204 fprintf(stderr, "Please stop the ports first\n"); 3205 3206 if (hairpin_mode & 0xf) { 3207 uint16_t i; 3208 int j; 3209 3210 /* bind all started hairpin ports */ 3211 for (i = 0; i < cfg_pi; i++) { 3212 pi = pl[i]; 3213 /* bind current Tx to all peer Rx */ 3214 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3215 RTE_MAX_ETHPORTS, 1); 3216 if (peer_pi < 0) 3217 return peer_pi; 3218 for (j = 0; j < peer_pi; j++) { 3219 if (!port_is_started(peer_pl[j])) 3220 continue; 3221 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3222 if (diag < 0) { 3223 fprintf(stderr, 3224 "Error during binding hairpin Tx port %u to %u: %s\n", 3225 pi, peer_pl[j], 3226 rte_strerror(-diag)); 3227 return -1; 3228 } 3229 } 3230 /* bind all peer Tx to current Rx */ 3231 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3232 RTE_MAX_ETHPORTS, 0); 3233 if (peer_pi < 0) 3234 return peer_pi; 3235 for (j = 0; j < peer_pi; j++) { 3236 if (!port_is_started(peer_pl[j])) 3237 continue; 3238 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3239 if (diag < 0) { 3240 fprintf(stderr, 3241 "Error during binding hairpin Tx port %u to %u: %s\n", 3242 peer_pl[j], pi, 3243 rte_strerror(-diag)); 3244 return -1; 3245 } 3246 } 3247 } 3248 } 3249 3250 fill_xstats_display_info_for_port(pid); 3251 3252 printf("Done\n"); 3253 return 0; 3254 } 3255 3256 void 3257 stop_port(portid_t pid) 3258 { 3259 portid_t pi; 3260 struct rte_port *port; 3261 int need_check_link_status = 0; 3262 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3263 int peer_pi; 3264 int ret; 3265 3266 if (port_id_is_invalid(pid, ENABLED_WARN)) 3267 return; 3268 3269 printf("Stopping ports...\n"); 3270 3271 RTE_ETH_FOREACH_DEV(pi) { 3272 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3273 continue; 3274 3275 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3276 fprintf(stderr, 3277 "Please remove port %d from forwarding configuration.\n", 3278 pi); 3279 continue; 3280 } 3281 3282 if (port_is_bonding_slave(pi)) { 3283 fprintf(stderr, 3284 "Please remove port %d from bonded device.\n", 3285 pi); 3286 continue; 3287 } 3288 3289 port = &ports[pi]; 3290 if (port->port_status == RTE_PORT_STARTED) 3291 port->port_status = RTE_PORT_HANDLING; 3292 else 3293 continue; 3294 3295 if (hairpin_mode & 0xf) { 3296 int j; 3297 3298 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3299 /* unbind all peer Tx from current Rx */ 3300 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3301 RTE_MAX_ETHPORTS, 0); 3302 if (peer_pi < 0) 3303 continue; 3304 for (j = 0; j < peer_pi; j++) { 3305 if (!port_is_started(peer_pl[j])) 3306 continue; 3307 rte_eth_hairpin_unbind(peer_pl[j], pi); 3308 } 3309 } 3310 3311 if (port->flow_list && !no_flow_flush) 3312 port_flow_flush(pi); 3313 3314 ret = eth_dev_stop_mp(pi); 3315 if (ret != 0) { 3316 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3317 pi); 3318 /* Allow to retry stopping the port. */ 3319 port->port_status = RTE_PORT_STARTED; 3320 continue; 3321 } 3322 3323 if (port->port_status == RTE_PORT_HANDLING) 3324 port->port_status = RTE_PORT_STOPPED; 3325 else 3326 fprintf(stderr, "Port %d can not be set into stopped\n", 3327 pi); 3328 need_check_link_status = 1; 3329 } 3330 if (need_check_link_status && !no_link_check) 3331 check_all_ports_link_status(RTE_PORT_ALL); 3332 3333 printf("Done\n"); 3334 } 3335 3336 static void 3337 remove_invalid_ports_in(portid_t *array, portid_t *total) 3338 { 3339 portid_t i; 3340 portid_t new_total = 0; 3341 3342 for (i = 0; i < *total; i++) 3343 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3344 array[new_total] = array[i]; 3345 new_total++; 3346 } 3347 *total = new_total; 3348 } 3349 3350 static void 3351 remove_invalid_ports(void) 3352 { 3353 remove_invalid_ports_in(ports_ids, &nb_ports); 3354 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3355 nb_cfg_ports = nb_fwd_ports; 3356 } 3357 3358 static void 3359 flush_port_owned_resources(portid_t pi) 3360 { 3361 mcast_addr_pool_destroy(pi); 3362 port_flow_flush(pi); 3363 port_flow_template_table_flush(pi); 3364 port_flow_pattern_template_flush(pi); 3365 port_flow_actions_template_flush(pi); 3366 port_flex_item_flush(pi); 3367 port_action_handle_flush(pi); 3368 } 3369 3370 static void 3371 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3372 { 3373 struct rte_port *port; 3374 portid_t slave_pid; 3375 uint16_t i; 3376 3377 for (i = 0; i < num_slaves; i++) { 3378 slave_pid = slave_pids[i]; 3379 if (port_is_started(slave_pid) == 1) { 3380 if (rte_eth_dev_stop(slave_pid) != 0) 3381 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3382 slave_pid); 3383 3384 port = &ports[slave_pid]; 3385 port->port_status = RTE_PORT_STOPPED; 3386 } 3387 3388 clear_port_slave_flag(slave_pid); 3389 3390 /* Close slave device when testpmd quit or is killed. */ 3391 if (cl_quit == 1 || f_quit == 1) 3392 rte_eth_dev_close(slave_pid); 3393 } 3394 } 3395 3396 void 3397 close_port(portid_t pid) 3398 { 3399 portid_t pi; 3400 struct rte_port *port; 3401 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3402 int num_slaves = 0; 3403 3404 if (port_id_is_invalid(pid, ENABLED_WARN)) 3405 return; 3406 3407 printf("Closing ports...\n"); 3408 3409 RTE_ETH_FOREACH_DEV(pi) { 3410 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3411 continue; 3412 3413 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3414 fprintf(stderr, 3415 "Please remove port %d from forwarding configuration.\n", 3416 pi); 3417 continue; 3418 } 3419 3420 if (port_is_bonding_slave(pi)) { 3421 fprintf(stderr, 3422 "Please remove port %d from bonded device.\n", 3423 pi); 3424 continue; 3425 } 3426 3427 port = &ports[pi]; 3428 if (port->port_status == RTE_PORT_CLOSED) { 3429 fprintf(stderr, "Port %d is already closed\n", pi); 3430 continue; 3431 } 3432 3433 if (is_proc_primary()) { 3434 flush_port_owned_resources(pi); 3435 #ifdef RTE_NET_BOND 3436 if (port->bond_flag == 1) 3437 num_slaves = rte_eth_bond_slaves_get(pi, 3438 slave_pids, RTE_MAX_ETHPORTS); 3439 #endif 3440 rte_eth_dev_close(pi); 3441 /* 3442 * If this port is bonded device, all slaves under the 3443 * device need to be removed or closed. 3444 */ 3445 if (port->bond_flag == 1 && num_slaves > 0) 3446 clear_bonding_slave_device(slave_pids, 3447 num_slaves); 3448 } 3449 3450 free_xstats_display_info(pi); 3451 } 3452 3453 remove_invalid_ports(); 3454 printf("Done\n"); 3455 } 3456 3457 void 3458 reset_port(portid_t pid) 3459 { 3460 int diag; 3461 portid_t pi; 3462 struct rte_port *port; 3463 3464 if (port_id_is_invalid(pid, ENABLED_WARN)) 3465 return; 3466 3467 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3468 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3469 fprintf(stderr, 3470 "Can not reset port(s), please stop port(s) first.\n"); 3471 return; 3472 } 3473 3474 printf("Resetting ports...\n"); 3475 3476 RTE_ETH_FOREACH_DEV(pi) { 3477 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3478 continue; 3479 3480 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3481 fprintf(stderr, 3482 "Please remove port %d from forwarding configuration.\n", 3483 pi); 3484 continue; 3485 } 3486 3487 if (port_is_bonding_slave(pi)) { 3488 fprintf(stderr, 3489 "Please remove port %d from bonded device.\n", 3490 pi); 3491 continue; 3492 } 3493 3494 if (is_proc_primary()) { 3495 diag = rte_eth_dev_reset(pi); 3496 if (diag == 0) { 3497 port = &ports[pi]; 3498 port->need_reconfig = 1; 3499 port->need_reconfig_queues = 1; 3500 } else { 3501 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3502 pi, diag); 3503 } 3504 } 3505 } 3506 3507 printf("Done\n"); 3508 } 3509 3510 void 3511 attach_port(char *identifier) 3512 { 3513 portid_t pi; 3514 struct rte_dev_iterator iterator; 3515 3516 printf("Attaching a new port...\n"); 3517 3518 if (identifier == NULL) { 3519 fprintf(stderr, "Invalid parameters are specified\n"); 3520 return; 3521 } 3522 3523 if (rte_dev_probe(identifier) < 0) { 3524 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3525 return; 3526 } 3527 3528 /* first attach mode: event */ 3529 if (setup_on_probe_event) { 3530 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3531 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3532 if (ports[pi].port_status == RTE_PORT_HANDLING && 3533 ports[pi].need_setup != 0) 3534 setup_attached_port(pi); 3535 return; 3536 } 3537 3538 /* second attach mode: iterator */ 3539 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3540 /* setup ports matching the devargs used for probing */ 3541 if (port_is_forwarding(pi)) 3542 continue; /* port was already attached before */ 3543 setup_attached_port(pi); 3544 } 3545 } 3546 3547 static void 3548 setup_attached_port(portid_t pi) 3549 { 3550 unsigned int socket_id; 3551 int ret; 3552 3553 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3554 /* if socket_id is invalid, set to the first available socket. */ 3555 if (check_socket_id(socket_id) < 0) 3556 socket_id = socket_ids[0]; 3557 reconfig(pi, socket_id); 3558 ret = rte_eth_promiscuous_enable(pi); 3559 if (ret != 0) 3560 fprintf(stderr, 3561 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3562 pi, rte_strerror(-ret)); 3563 3564 ports_ids[nb_ports++] = pi; 3565 fwd_ports_ids[nb_fwd_ports++] = pi; 3566 nb_cfg_ports = nb_fwd_ports; 3567 ports[pi].need_setup = 0; 3568 ports[pi].port_status = RTE_PORT_STOPPED; 3569 3570 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3571 printf("Done\n"); 3572 } 3573 3574 static void 3575 detach_device(struct rte_device *dev) 3576 { 3577 portid_t sibling; 3578 3579 if (dev == NULL) { 3580 fprintf(stderr, "Device already removed\n"); 3581 return; 3582 } 3583 3584 printf("Removing a device...\n"); 3585 3586 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3587 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3588 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3589 fprintf(stderr, "Port %u not stopped\n", 3590 sibling); 3591 return; 3592 } 3593 flush_port_owned_resources(sibling); 3594 } 3595 } 3596 3597 if (rte_dev_remove(dev) < 0) { 3598 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3599 return; 3600 } 3601 remove_invalid_ports(); 3602 3603 printf("Device is detached\n"); 3604 printf("Now total ports is %d\n", nb_ports); 3605 printf("Done\n"); 3606 return; 3607 } 3608 3609 void 3610 detach_port_device(portid_t port_id) 3611 { 3612 int ret; 3613 struct rte_eth_dev_info dev_info; 3614 3615 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3616 return; 3617 3618 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3619 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3620 fprintf(stderr, "Port not stopped\n"); 3621 return; 3622 } 3623 fprintf(stderr, "Port was not closed\n"); 3624 } 3625 3626 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3627 if (ret != 0) { 3628 TESTPMD_LOG(ERR, 3629 "Failed to get device info for port %d, not detaching\n", 3630 port_id); 3631 return; 3632 } 3633 detach_device(dev_info.device); 3634 } 3635 3636 void 3637 detach_devargs(char *identifier) 3638 { 3639 struct rte_dev_iterator iterator; 3640 struct rte_devargs da; 3641 portid_t port_id; 3642 3643 printf("Removing a device...\n"); 3644 3645 memset(&da, 0, sizeof(da)); 3646 if (rte_devargs_parsef(&da, "%s", identifier)) { 3647 fprintf(stderr, "cannot parse identifier\n"); 3648 return; 3649 } 3650 3651 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3652 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3653 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3654 fprintf(stderr, "Port %u not stopped\n", 3655 port_id); 3656 rte_eth_iterator_cleanup(&iterator); 3657 rte_devargs_reset(&da); 3658 return; 3659 } 3660 flush_port_owned_resources(port_id); 3661 } 3662 } 3663 3664 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3665 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3666 da.name, rte_bus_name(da.bus)); 3667 rte_devargs_reset(&da); 3668 return; 3669 } 3670 3671 remove_invalid_ports(); 3672 3673 printf("Device %s is detached\n", identifier); 3674 printf("Now total ports is %d\n", nb_ports); 3675 printf("Done\n"); 3676 rte_devargs_reset(&da); 3677 } 3678 3679 void 3680 pmd_test_exit(void) 3681 { 3682 portid_t pt_id; 3683 unsigned int i; 3684 int ret; 3685 3686 if (test_done == 0) 3687 stop_packet_forwarding(); 3688 3689 #ifndef RTE_EXEC_ENV_WINDOWS 3690 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3691 if (mempools[i]) { 3692 if (mp_alloc_type == MP_ALLOC_ANON) 3693 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3694 NULL); 3695 } 3696 } 3697 #endif 3698 if (ports != NULL) { 3699 no_link_check = 1; 3700 RTE_ETH_FOREACH_DEV(pt_id) { 3701 printf("\nStopping port %d...\n", pt_id); 3702 fflush(stdout); 3703 stop_port(pt_id); 3704 } 3705 RTE_ETH_FOREACH_DEV(pt_id) { 3706 printf("\nShutting down port %d...\n", pt_id); 3707 fflush(stdout); 3708 close_port(pt_id); 3709 } 3710 } 3711 3712 if (hot_plug) { 3713 ret = rte_dev_event_monitor_stop(); 3714 if (ret) { 3715 RTE_LOG(ERR, EAL, 3716 "fail to stop device event monitor."); 3717 return; 3718 } 3719 3720 ret = rte_dev_event_callback_unregister(NULL, 3721 dev_event_callback, NULL); 3722 if (ret < 0) { 3723 RTE_LOG(ERR, EAL, 3724 "fail to unregister device event callback.\n"); 3725 return; 3726 } 3727 3728 ret = rte_dev_hotplug_handle_disable(); 3729 if (ret) { 3730 RTE_LOG(ERR, EAL, 3731 "fail to disable hotplug handling.\n"); 3732 return; 3733 } 3734 } 3735 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3736 if (mempools[i]) 3737 mempool_free_mp(mempools[i]); 3738 } 3739 free(xstats_display); 3740 3741 printf("\nBye...\n"); 3742 } 3743 3744 typedef void (*cmd_func_t)(void); 3745 struct pmd_test_command { 3746 const char *cmd_name; 3747 cmd_func_t cmd_func; 3748 }; 3749 3750 /* Check the link status of all ports in up to 9s, and print them finally */ 3751 static void 3752 check_all_ports_link_status(uint32_t port_mask) 3753 { 3754 #define CHECK_INTERVAL 100 /* 100ms */ 3755 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3756 portid_t portid; 3757 uint8_t count, all_ports_up, print_flag = 0; 3758 struct rte_eth_link link; 3759 int ret; 3760 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3761 3762 printf("Checking link statuses...\n"); 3763 fflush(stdout); 3764 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3765 all_ports_up = 1; 3766 RTE_ETH_FOREACH_DEV(portid) { 3767 if ((port_mask & (1 << portid)) == 0) 3768 continue; 3769 memset(&link, 0, sizeof(link)); 3770 ret = rte_eth_link_get_nowait(portid, &link); 3771 if (ret < 0) { 3772 all_ports_up = 0; 3773 if (print_flag == 1) 3774 fprintf(stderr, 3775 "Port %u link get failed: %s\n", 3776 portid, rte_strerror(-ret)); 3777 continue; 3778 } 3779 /* print link status if flag set */ 3780 if (print_flag == 1) { 3781 rte_eth_link_to_str(link_status, 3782 sizeof(link_status), &link); 3783 printf("Port %d %s\n", portid, link_status); 3784 continue; 3785 } 3786 /* clear all_ports_up flag if any link down */ 3787 if (link.link_status == RTE_ETH_LINK_DOWN) { 3788 all_ports_up = 0; 3789 break; 3790 } 3791 } 3792 /* after finally printing all link status, get out */ 3793 if (print_flag == 1) 3794 break; 3795 3796 if (all_ports_up == 0) { 3797 fflush(stdout); 3798 rte_delay_ms(CHECK_INTERVAL); 3799 } 3800 3801 /* set the print_flag if all ports up or timeout */ 3802 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3803 print_flag = 1; 3804 } 3805 3806 if (lsc_interrupt) 3807 break; 3808 } 3809 } 3810 3811 static void 3812 rmv_port_callback(void *arg) 3813 { 3814 int need_to_start = 0; 3815 int org_no_link_check = no_link_check; 3816 portid_t port_id = (intptr_t)arg; 3817 struct rte_eth_dev_info dev_info; 3818 int ret; 3819 3820 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3821 3822 if (!test_done && port_is_forwarding(port_id)) { 3823 need_to_start = 1; 3824 stop_packet_forwarding(); 3825 } 3826 no_link_check = 1; 3827 stop_port(port_id); 3828 no_link_check = org_no_link_check; 3829 3830 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3831 if (ret != 0) 3832 TESTPMD_LOG(ERR, 3833 "Failed to get device info for port %d, not detaching\n", 3834 port_id); 3835 else { 3836 struct rte_device *device = dev_info.device; 3837 close_port(port_id); 3838 detach_device(device); /* might be already removed or have more ports */ 3839 } 3840 if (need_to_start) 3841 start_packet_forwarding(0); 3842 } 3843 3844 /* This function is used by the interrupt thread */ 3845 static int 3846 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3847 void *ret_param) 3848 { 3849 RTE_SET_USED(param); 3850 RTE_SET_USED(ret_param); 3851 3852 if (type >= RTE_ETH_EVENT_MAX) { 3853 fprintf(stderr, 3854 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3855 port_id, __func__, type); 3856 fflush(stderr); 3857 } else if (event_print_mask & (UINT32_C(1) << type)) { 3858 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3859 eth_event_desc[type]); 3860 fflush(stdout); 3861 } 3862 3863 switch (type) { 3864 case RTE_ETH_EVENT_NEW: 3865 ports[port_id].need_setup = 1; 3866 ports[port_id].port_status = RTE_PORT_HANDLING; 3867 break; 3868 case RTE_ETH_EVENT_INTR_RMV: 3869 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3870 break; 3871 if (rte_eal_alarm_set(100000, 3872 rmv_port_callback, (void *)(intptr_t)port_id)) 3873 fprintf(stderr, 3874 "Could not set up deferred device removal\n"); 3875 break; 3876 case RTE_ETH_EVENT_DESTROY: 3877 ports[port_id].port_status = RTE_PORT_CLOSED; 3878 printf("Port %u is closed\n", port_id); 3879 break; 3880 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3881 uint16_t rxq_id; 3882 int ret; 3883 3884 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3885 for (rxq_id = 0; ; rxq_id++) { 3886 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3887 NULL); 3888 if (ret <= 0) 3889 break; 3890 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3891 port_id, rxq_id); 3892 3893 #ifdef RTE_NET_MLX5 3894 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3895 #endif 3896 } 3897 break; 3898 } 3899 default: 3900 break; 3901 } 3902 return 0; 3903 } 3904 3905 static int 3906 register_eth_event_callback(void) 3907 { 3908 int ret; 3909 enum rte_eth_event_type event; 3910 3911 for (event = RTE_ETH_EVENT_UNKNOWN; 3912 event < RTE_ETH_EVENT_MAX; event++) { 3913 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3914 event, 3915 eth_event_callback, 3916 NULL); 3917 if (ret != 0) { 3918 TESTPMD_LOG(ERR, "Failed to register callback for " 3919 "%s event\n", eth_event_desc[event]); 3920 return -1; 3921 } 3922 } 3923 3924 return 0; 3925 } 3926 3927 /* This function is used by the interrupt thread */ 3928 static void 3929 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3930 __rte_unused void *arg) 3931 { 3932 uint16_t port_id; 3933 int ret; 3934 3935 if (type >= RTE_DEV_EVENT_MAX) { 3936 fprintf(stderr, "%s called upon invalid event %d\n", 3937 __func__, type); 3938 fflush(stderr); 3939 } 3940 3941 switch (type) { 3942 case RTE_DEV_EVENT_REMOVE: 3943 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3944 device_name); 3945 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3946 if (ret) { 3947 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3948 device_name); 3949 return; 3950 } 3951 /* 3952 * Because the user's callback is invoked in eal interrupt 3953 * callback, the interrupt callback need to be finished before 3954 * it can be unregistered when detaching device. So finish 3955 * callback soon and use a deferred removal to detach device 3956 * is need. It is a workaround, once the device detaching be 3957 * moved into the eal in the future, the deferred removal could 3958 * be deleted. 3959 */ 3960 if (rte_eal_alarm_set(100000, 3961 rmv_port_callback, (void *)(intptr_t)port_id)) 3962 RTE_LOG(ERR, EAL, 3963 "Could not set up deferred device removal\n"); 3964 break; 3965 case RTE_DEV_EVENT_ADD: 3966 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3967 device_name); 3968 /* TODO: After finish kernel driver binding, 3969 * begin to attach port. 3970 */ 3971 break; 3972 default: 3973 break; 3974 } 3975 } 3976 3977 static void 3978 rxtx_port_config(portid_t pid) 3979 { 3980 uint16_t qid; 3981 uint64_t offloads; 3982 struct rte_port *port = &ports[pid]; 3983 3984 for (qid = 0; qid < nb_rxq; qid++) { 3985 offloads = port->rxq[qid].conf.offloads; 3986 port->rxq[qid].conf = port->dev_info.default_rxconf; 3987 3988 if (rxq_share > 0 && 3989 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3990 /* Non-zero share group to enable RxQ share. */ 3991 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3992 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3993 } 3994 3995 if (offloads != 0) 3996 port->rxq[qid].conf.offloads = offloads; 3997 3998 /* Check if any Rx parameters have been passed */ 3999 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 4000 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 4001 4002 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 4003 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 4004 4005 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 4006 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 4007 4008 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 4009 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 4010 4011 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 4012 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 4013 4014 port->nb_rx_desc[qid] = nb_rxd; 4015 } 4016 4017 for (qid = 0; qid < nb_txq; qid++) { 4018 offloads = port->txq[qid].conf.offloads; 4019 port->txq[qid].conf = port->dev_info.default_txconf; 4020 if (offloads != 0) 4021 port->txq[qid].conf.offloads = offloads; 4022 4023 /* Check if any Tx parameters have been passed */ 4024 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 4025 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 4026 4027 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 4028 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 4029 4030 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 4031 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 4032 4033 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 4034 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 4035 4036 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 4037 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 4038 4039 port->nb_tx_desc[qid] = nb_txd; 4040 } 4041 } 4042 4043 /* 4044 * Helper function to set MTU from frame size 4045 * 4046 * port->dev_info should be set before calling this function. 4047 * 4048 * return 0 on success, negative on error 4049 */ 4050 int 4051 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 4052 { 4053 struct rte_port *port = &ports[portid]; 4054 uint32_t eth_overhead; 4055 uint16_t mtu, new_mtu; 4056 4057 eth_overhead = get_eth_overhead(&port->dev_info); 4058 4059 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 4060 printf("Failed to get MTU for port %u\n", portid); 4061 return -1; 4062 } 4063 4064 new_mtu = max_rx_pktlen - eth_overhead; 4065 4066 if (mtu == new_mtu) 4067 return 0; 4068 4069 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 4070 fprintf(stderr, 4071 "Failed to set MTU to %u for port %u\n", 4072 new_mtu, portid); 4073 return -1; 4074 } 4075 4076 port->dev_conf.rxmode.mtu = new_mtu; 4077 4078 return 0; 4079 } 4080 4081 void 4082 init_port_config(void) 4083 { 4084 portid_t pid; 4085 struct rte_port *port; 4086 int ret, i; 4087 4088 RTE_ETH_FOREACH_DEV(pid) { 4089 port = &ports[pid]; 4090 4091 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 4092 if (ret != 0) 4093 return; 4094 4095 if (nb_rxq > 1) { 4096 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4097 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 4098 rss_hf & port->dev_info.flow_type_rss_offloads; 4099 } else { 4100 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 4101 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 4102 } 4103 4104 if (port->dcb_flag == 0) { 4105 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 4106 port->dev_conf.rxmode.mq_mode = 4107 (enum rte_eth_rx_mq_mode) 4108 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 4109 } else { 4110 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 4111 port->dev_conf.rxmode.offloads &= 4112 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4113 4114 for (i = 0; 4115 i < port->dev_info.nb_rx_queues; 4116 i++) 4117 port->rxq[i].conf.offloads &= 4118 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4119 } 4120 } 4121 4122 rxtx_port_config(pid); 4123 4124 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 4125 if (ret != 0) 4126 return; 4127 4128 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 4129 port->dev_conf.intr_conf.lsc = 1; 4130 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 4131 port->dev_conf.intr_conf.rmv = 1; 4132 } 4133 } 4134 4135 void set_port_slave_flag(portid_t slave_pid) 4136 { 4137 struct rte_port *port; 4138 4139 port = &ports[slave_pid]; 4140 port->slave_flag = 1; 4141 } 4142 4143 void clear_port_slave_flag(portid_t slave_pid) 4144 { 4145 struct rte_port *port; 4146 4147 port = &ports[slave_pid]; 4148 port->slave_flag = 0; 4149 } 4150 4151 uint8_t port_is_bonding_slave(portid_t slave_pid) 4152 { 4153 struct rte_port *port; 4154 struct rte_eth_dev_info dev_info; 4155 int ret; 4156 4157 port = &ports[slave_pid]; 4158 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4159 if (ret != 0) { 4160 TESTPMD_LOG(ERR, 4161 "Failed to get device info for port id %d," 4162 "cannot determine if the port is a bonded slave", 4163 slave_pid); 4164 return 0; 4165 } 4166 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4167 return 1; 4168 return 0; 4169 } 4170 4171 const uint16_t vlan_tags[] = { 4172 0, 1, 2, 3, 4, 5, 6, 7, 4173 8, 9, 10, 11, 12, 13, 14, 15, 4174 16, 17, 18, 19, 20, 21, 22, 23, 4175 24, 25, 26, 27, 28, 29, 30, 31 4176 }; 4177 4178 static int 4179 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4180 enum dcb_mode_enable dcb_mode, 4181 enum rte_eth_nb_tcs num_tcs, 4182 uint8_t pfc_en) 4183 { 4184 uint8_t i; 4185 int32_t rc; 4186 struct rte_eth_rss_conf rss_conf; 4187 4188 /* 4189 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4190 * given above, and the number of traffic classes available for use. 4191 */ 4192 if (dcb_mode == DCB_VT_ENABLED) { 4193 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4194 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4195 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4196 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4197 4198 /* VMDQ+DCB RX and TX configurations */ 4199 vmdq_rx_conf->enable_default_pool = 0; 4200 vmdq_rx_conf->default_pool = 0; 4201 vmdq_rx_conf->nb_queue_pools = 4202 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4203 vmdq_tx_conf->nb_queue_pools = 4204 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4205 4206 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4207 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4208 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4209 vmdq_rx_conf->pool_map[i].pools = 4210 1 << (i % vmdq_rx_conf->nb_queue_pools); 4211 } 4212 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4213 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4214 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4215 } 4216 4217 /* set DCB mode of RX and TX of multiple queues */ 4218 eth_conf->rxmode.mq_mode = 4219 (enum rte_eth_rx_mq_mode) 4220 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4221 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4222 } else { 4223 struct rte_eth_dcb_rx_conf *rx_conf = 4224 ð_conf->rx_adv_conf.dcb_rx_conf; 4225 struct rte_eth_dcb_tx_conf *tx_conf = 4226 ð_conf->tx_adv_conf.dcb_tx_conf; 4227 4228 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4229 4230 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4231 if (rc != 0) 4232 return rc; 4233 4234 rx_conf->nb_tcs = num_tcs; 4235 tx_conf->nb_tcs = num_tcs; 4236 4237 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4238 rx_conf->dcb_tc[i] = i % num_tcs; 4239 tx_conf->dcb_tc[i] = i % num_tcs; 4240 } 4241 4242 eth_conf->rxmode.mq_mode = 4243 (enum rte_eth_rx_mq_mode) 4244 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4245 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4246 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4247 } 4248 4249 if (pfc_en) 4250 eth_conf->dcb_capability_en = 4251 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4252 else 4253 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4254 4255 return 0; 4256 } 4257 4258 int 4259 init_port_dcb_config(portid_t pid, 4260 enum dcb_mode_enable dcb_mode, 4261 enum rte_eth_nb_tcs num_tcs, 4262 uint8_t pfc_en) 4263 { 4264 struct rte_eth_conf port_conf; 4265 struct rte_port *rte_port; 4266 int retval; 4267 uint16_t i; 4268 4269 if (num_procs > 1) { 4270 printf("The multi-process feature doesn't support dcb.\n"); 4271 return -ENOTSUP; 4272 } 4273 rte_port = &ports[pid]; 4274 4275 /* retain the original device configuration. */ 4276 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4277 4278 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4279 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4280 if (retval < 0) 4281 return retval; 4282 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4283 /* remove RSS HASH offload for DCB in vt mode */ 4284 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4285 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4286 for (i = 0; i < nb_rxq; i++) 4287 rte_port->rxq[i].conf.offloads &= 4288 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4289 } 4290 4291 /* re-configure the device . */ 4292 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4293 if (retval < 0) 4294 return retval; 4295 4296 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4297 if (retval != 0) 4298 return retval; 4299 4300 /* If dev_info.vmdq_pool_base is greater than 0, 4301 * the queue id of vmdq pools is started after pf queues. 4302 */ 4303 if (dcb_mode == DCB_VT_ENABLED && 4304 rte_port->dev_info.vmdq_pool_base > 0) { 4305 fprintf(stderr, 4306 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4307 pid); 4308 return -1; 4309 } 4310 4311 /* Assume the ports in testpmd have the same dcb capability 4312 * and has the same number of rxq and txq in dcb mode 4313 */ 4314 if (dcb_mode == DCB_VT_ENABLED) { 4315 if (rte_port->dev_info.max_vfs > 0) { 4316 nb_rxq = rte_port->dev_info.nb_rx_queues; 4317 nb_txq = rte_port->dev_info.nb_tx_queues; 4318 } else { 4319 nb_rxq = rte_port->dev_info.max_rx_queues; 4320 nb_txq = rte_port->dev_info.max_tx_queues; 4321 } 4322 } else { 4323 /*if vt is disabled, use all pf queues */ 4324 if (rte_port->dev_info.vmdq_pool_base == 0) { 4325 nb_rxq = rte_port->dev_info.max_rx_queues; 4326 nb_txq = rte_port->dev_info.max_tx_queues; 4327 } else { 4328 nb_rxq = (queueid_t)num_tcs; 4329 nb_txq = (queueid_t)num_tcs; 4330 4331 } 4332 } 4333 rx_free_thresh = 64; 4334 4335 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4336 4337 rxtx_port_config(pid); 4338 /* VLAN filter */ 4339 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4340 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4341 rx_vft_set(pid, vlan_tags[i], 1); 4342 4343 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4344 if (retval != 0) 4345 return retval; 4346 4347 rte_port->dcb_flag = 1; 4348 4349 /* Enter DCB configuration status */ 4350 dcb_config = 1; 4351 4352 return 0; 4353 } 4354 4355 static void 4356 init_port(void) 4357 { 4358 int i; 4359 4360 /* Configuration of Ethernet ports. */ 4361 ports = rte_zmalloc("testpmd: ports", 4362 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4363 RTE_CACHE_LINE_SIZE); 4364 if (ports == NULL) { 4365 rte_exit(EXIT_FAILURE, 4366 "rte_zmalloc(%d struct rte_port) failed\n", 4367 RTE_MAX_ETHPORTS); 4368 } 4369 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4370 ports[i].fwd_mac_swap = 1; 4371 ports[i].xstats_info.allocated = false; 4372 LIST_INIT(&ports[i].flow_tunnel_list); 4373 } 4374 /* Initialize ports NUMA structures */ 4375 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4376 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4377 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4378 } 4379 4380 static void 4381 force_quit(void) 4382 { 4383 pmd_test_exit(); 4384 prompt_exit(); 4385 } 4386 4387 static void 4388 print_stats(void) 4389 { 4390 uint8_t i; 4391 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4392 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4393 4394 /* Clear screen and move to top left */ 4395 printf("%s%s", clr, top_left); 4396 4397 printf("\nPort statistics ===================================="); 4398 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4399 nic_stats_display(fwd_ports_ids[i]); 4400 4401 fflush(stdout); 4402 } 4403 4404 static void 4405 signal_handler(int signum) 4406 { 4407 if (signum == SIGINT || signum == SIGTERM) { 4408 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4409 signum); 4410 #ifdef RTE_LIB_PDUMP 4411 /* uninitialize packet capture framework */ 4412 rte_pdump_uninit(); 4413 #endif 4414 #ifdef RTE_LIB_LATENCYSTATS 4415 if (latencystats_enabled != 0) 4416 rte_latencystats_uninit(); 4417 #endif 4418 force_quit(); 4419 /* Set flag to indicate the force termination. */ 4420 f_quit = 1; 4421 /* exit with the expected status */ 4422 #ifndef RTE_EXEC_ENV_WINDOWS 4423 signal(signum, SIG_DFL); 4424 kill(getpid(), signum); 4425 #endif 4426 } 4427 } 4428 4429 int 4430 main(int argc, char** argv) 4431 { 4432 int diag; 4433 portid_t port_id; 4434 uint16_t count; 4435 int ret; 4436 4437 signal(SIGINT, signal_handler); 4438 signal(SIGTERM, signal_handler); 4439 4440 testpmd_logtype = rte_log_register("testpmd"); 4441 if (testpmd_logtype < 0) 4442 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4443 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4444 4445 diag = rte_eal_init(argc, argv); 4446 if (diag < 0) 4447 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4448 rte_strerror(rte_errno)); 4449 4450 /* allocate port structures, and init them */ 4451 init_port(); 4452 4453 ret = register_eth_event_callback(); 4454 if (ret != 0) 4455 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4456 4457 #ifdef RTE_LIB_PDUMP 4458 /* initialize packet capture framework */ 4459 rte_pdump_init(); 4460 #endif 4461 4462 count = 0; 4463 RTE_ETH_FOREACH_DEV(port_id) { 4464 ports_ids[count] = port_id; 4465 count++; 4466 } 4467 nb_ports = (portid_t) count; 4468 if (nb_ports == 0) 4469 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4470 4471 set_def_fwd_config(); 4472 if (nb_lcores == 0) 4473 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4474 "Check the core mask argument\n"); 4475 4476 /* Bitrate/latency stats disabled by default */ 4477 #ifdef RTE_LIB_BITRATESTATS 4478 bitrate_enabled = 0; 4479 #endif 4480 #ifdef RTE_LIB_LATENCYSTATS 4481 latencystats_enabled = 0; 4482 #endif 4483 4484 /* on FreeBSD, mlockall() is disabled by default */ 4485 #ifdef RTE_EXEC_ENV_FREEBSD 4486 do_mlockall = 0; 4487 #else 4488 do_mlockall = 1; 4489 #endif 4490 4491 argc -= diag; 4492 argv += diag; 4493 if (argc > 1) 4494 launch_args_parse(argc, argv); 4495 4496 #ifndef RTE_EXEC_ENV_WINDOWS 4497 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4498 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4499 strerror(errno)); 4500 } 4501 #endif 4502 4503 if (tx_first && interactive) 4504 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4505 "interactive mode.\n"); 4506 4507 if (tx_first && lsc_interrupt) { 4508 fprintf(stderr, 4509 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4510 lsc_interrupt = 0; 4511 } 4512 4513 if (!nb_rxq && !nb_txq) 4514 fprintf(stderr, 4515 "Warning: Either rx or tx queues should be non-zero\n"); 4516 4517 if (nb_rxq > 1 && nb_rxq > nb_txq) 4518 fprintf(stderr, 4519 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4520 nb_rxq, nb_txq); 4521 4522 init_config(); 4523 4524 if (hot_plug) { 4525 ret = rte_dev_hotplug_handle_enable(); 4526 if (ret) { 4527 RTE_LOG(ERR, EAL, 4528 "fail to enable hotplug handling."); 4529 return -1; 4530 } 4531 4532 ret = rte_dev_event_monitor_start(); 4533 if (ret) { 4534 RTE_LOG(ERR, EAL, 4535 "fail to start device event monitoring."); 4536 return -1; 4537 } 4538 4539 ret = rte_dev_event_callback_register(NULL, 4540 dev_event_callback, NULL); 4541 if (ret) { 4542 RTE_LOG(ERR, EAL, 4543 "fail to register device event callback\n"); 4544 return -1; 4545 } 4546 } 4547 4548 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { 4549 if (!interactive) { 4550 rte_eal_cleanup(); 4551 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4552 } 4553 fprintf(stderr, "Start ports failed\n"); 4554 } 4555 4556 /* set all ports to promiscuous mode by default */ 4557 RTE_ETH_FOREACH_DEV(port_id) { 4558 ret = rte_eth_promiscuous_enable(port_id); 4559 if (ret != 0) 4560 fprintf(stderr, 4561 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4562 port_id, rte_strerror(-ret)); 4563 } 4564 4565 #ifdef RTE_LIB_METRICS 4566 /* Init metrics library */ 4567 rte_metrics_init(rte_socket_id()); 4568 #endif 4569 4570 #ifdef RTE_LIB_LATENCYSTATS 4571 if (latencystats_enabled != 0) { 4572 int ret = rte_latencystats_init(1, NULL); 4573 if (ret) 4574 fprintf(stderr, 4575 "Warning: latencystats init() returned error %d\n", 4576 ret); 4577 fprintf(stderr, "Latencystats running on lcore %d\n", 4578 latencystats_lcore_id); 4579 } 4580 #endif 4581 4582 /* Setup bitrate stats */ 4583 #ifdef RTE_LIB_BITRATESTATS 4584 if (bitrate_enabled != 0) { 4585 bitrate_data = rte_stats_bitrate_create(); 4586 if (bitrate_data == NULL) 4587 rte_exit(EXIT_FAILURE, 4588 "Could not allocate bitrate data.\n"); 4589 rte_stats_bitrate_reg(bitrate_data); 4590 } 4591 #endif 4592 4593 if (record_core_cycles) 4594 rte_lcore_register_usage_cb(lcore_usage_callback); 4595 4596 #ifdef RTE_LIB_CMDLINE 4597 if (init_cmdline() != 0) 4598 rte_exit(EXIT_FAILURE, 4599 "Could not initialise cmdline context.\n"); 4600 4601 if (strlen(cmdline_filename) != 0) 4602 cmdline_read_from_file(cmdline_filename); 4603 4604 if (interactive == 1) { 4605 if (auto_start) { 4606 printf("Start automatic packet forwarding\n"); 4607 start_packet_forwarding(0); 4608 } 4609 prompt(); 4610 pmd_test_exit(); 4611 } else 4612 #endif 4613 { 4614 char c; 4615 int rc; 4616 4617 f_quit = 0; 4618 4619 printf("No commandline core given, start packet forwarding\n"); 4620 start_packet_forwarding(tx_first); 4621 if (stats_period != 0) { 4622 uint64_t prev_time = 0, cur_time, diff_time = 0; 4623 uint64_t timer_period; 4624 4625 /* Convert to number of cycles */ 4626 timer_period = stats_period * rte_get_timer_hz(); 4627 4628 while (f_quit == 0) { 4629 cur_time = rte_get_timer_cycles(); 4630 diff_time += cur_time - prev_time; 4631 4632 if (diff_time >= timer_period) { 4633 print_stats(); 4634 /* Reset the timer */ 4635 diff_time = 0; 4636 } 4637 /* Sleep to avoid unnecessary checks */ 4638 prev_time = cur_time; 4639 rte_delay_us_sleep(US_PER_S); 4640 } 4641 } 4642 4643 printf("Press enter to exit\n"); 4644 rc = read(0, &c, 1); 4645 pmd_test_exit(); 4646 if (rc < 0) 4647 return 1; 4648 } 4649 4650 ret = rte_eal_cleanup(); 4651 if (ret != 0) 4652 rte_exit(EXIT_FAILURE, 4653 "EAL cleanup failed: %s\n", strerror(-ret)); 4654 4655 return EXIT_SUCCESS; 4656 } 4657