1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #ifdef RTE_LIB_METRICS 57 #include <rte_metrics.h> 58 #endif 59 #ifdef RTE_LIB_BITRATESTATS 60 #include <rte_bitrate.h> 61 #endif 62 #ifdef RTE_LIB_LATENCYSTATS 63 #include <rte_latencystats.h> 64 #endif 65 #ifdef RTE_EXEC_ENV_WINDOWS 66 #include <process.h> 67 #endif 68 #ifdef RTE_NET_BOND 69 #include <rte_eth_bond.h> 70 #endif 71 #ifdef RTE_NET_MLX5 72 #include "mlx5_testpmd.h" 73 #endif 74 75 #include "testpmd.h" 76 77 #ifndef MAP_HUGETLB 78 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 79 #define HUGE_FLAG (0x40000) 80 #else 81 #define HUGE_FLAG MAP_HUGETLB 82 #endif 83 84 #ifndef MAP_HUGE_SHIFT 85 /* older kernels (or FreeBSD) will not have this define */ 86 #define HUGE_SHIFT (26) 87 #else 88 #define HUGE_SHIFT MAP_HUGE_SHIFT 89 #endif 90 91 #define EXTMEM_HEAP_NAME "extmem" 92 /* 93 * Zone size with the malloc overhead (max of debug and release variants) 94 * must fit into the smallest supported hugepage size (2M), 95 * so that an IOVA-contiguous zone of this size can always be allocated 96 * if there are free 2M hugepages. 97 */ 98 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 99 100 uint16_t verbose_level = 0; /**< Silent by default. */ 101 int testpmd_logtype; /**< Log type for testpmd logs */ 102 103 /* use main core for command line ? */ 104 uint8_t interactive = 0; 105 uint8_t auto_start = 0; 106 uint8_t tx_first; 107 char cmdline_filename[PATH_MAX] = {0}; 108 109 /* 110 * NUMA support configuration. 111 * When set, the NUMA support attempts to dispatch the allocation of the 112 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 113 * probed ports among the CPU sockets 0 and 1. 114 * Otherwise, all memory is allocated from CPU socket 0. 115 */ 116 uint8_t numa_support = 1; /**< numa enabled by default */ 117 118 /* 119 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 120 * not configured. 121 */ 122 uint8_t socket_num = UMA_NO_CONFIG; 123 124 /* 125 * Select mempool allocation type: 126 * - native: use regular DPDK memory 127 * - anon: use regular DPDK memory to create mempool, but populate using 128 * anonymous memory (may not be IOVA-contiguous) 129 * - xmem: use externally allocated hugepage memory 130 */ 131 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 132 133 /* 134 * Store specified sockets on which memory pool to be used by ports 135 * is allocated. 136 */ 137 uint8_t port_numa[RTE_MAX_ETHPORTS]; 138 139 /* 140 * Store specified sockets on which RX ring to be used by ports 141 * is allocated. 142 */ 143 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 144 145 /* 146 * Store specified sockets on which TX ring to be used by ports 147 * is allocated. 148 */ 149 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 150 151 /* 152 * Record the Ethernet address of peer target ports to which packets are 153 * forwarded. 154 * Must be instantiated with the ethernet addresses of peer traffic generator 155 * ports. 156 */ 157 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 158 portid_t nb_peer_eth_addrs = 0; 159 160 /* 161 * Probed Target Environment. 162 */ 163 struct rte_port *ports; /**< For all probed ethernet ports. */ 164 portid_t nb_ports; /**< Number of probed ethernet ports. */ 165 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 166 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 167 168 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 169 170 /* 171 * Test Forwarding Configuration. 172 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 173 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 174 */ 175 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 176 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 177 portid_t nb_cfg_ports; /**< Number of configured ports. */ 178 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 179 180 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 181 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 182 183 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 184 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 185 186 /* 187 * Forwarding engines. 188 */ 189 struct fwd_engine * fwd_engines[] = { 190 &io_fwd_engine, 191 &mac_fwd_engine, 192 &mac_swap_engine, 193 &flow_gen_engine, 194 &rx_only_engine, 195 &tx_only_engine, 196 &csum_fwd_engine, 197 &icmp_echo_engine, 198 &noisy_vnf_engine, 199 &five_tuple_swap_fwd_engine, 200 #ifdef RTE_LIBRTE_IEEE1588 201 &ieee1588_fwd_engine, 202 #endif 203 &shared_rxq_engine, 204 NULL, 205 }; 206 207 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 208 uint16_t mempool_flags; 209 210 struct fwd_config cur_fwd_config; 211 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 212 uint32_t retry_enabled; 213 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 214 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 215 216 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 217 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 218 DEFAULT_MBUF_DATA_SIZE 219 }; /**< Mbuf data space size. */ 220 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 221 * specified on command-line. */ 222 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 223 224 /** Extended statistics to show. */ 225 struct rte_eth_xstat_name *xstats_display; 226 227 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 228 229 /* 230 * In container, it cannot terminate the process which running with 'stats-period' 231 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 232 */ 233 uint8_t f_quit; 234 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 235 236 /* 237 * Max Rx frame size, set by '--max-pkt-len' parameter. 238 */ 239 uint32_t max_rx_pkt_len; 240 241 /* 242 * Configuration of packet segments used to scatter received packets 243 * if some of split features is configured. 244 */ 245 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 246 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 247 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 248 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 249 250 /* 251 * Configuration of packet segments used by the "txonly" processing engine. 252 */ 253 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 254 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 255 TXONLY_DEF_PACKET_LEN, 256 }; 257 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 258 259 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 260 /**< Split policy for packets to TX. */ 261 262 uint8_t txonly_multi_flow; 263 /**< Whether multiple flows are generated in TXONLY mode. */ 264 265 uint32_t tx_pkt_times_inter; 266 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 267 268 uint32_t tx_pkt_times_intra; 269 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 270 271 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 272 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 273 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 274 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 275 276 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 277 uint8_t dcb_config = 0; 278 279 /* 280 * Configurable number of RX/TX queues. 281 */ 282 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 283 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 284 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 285 286 /* 287 * Configurable number of RX/TX ring descriptors. 288 * Defaults are supplied by drivers via ethdev. 289 */ 290 #define RTE_TEST_RX_DESC_DEFAULT 0 291 #define RTE_TEST_TX_DESC_DEFAULT 0 292 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 293 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 294 295 #define RTE_PMD_PARAM_UNSET -1 296 /* 297 * Configurable values of RX and TX ring threshold registers. 298 */ 299 300 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 301 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 302 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 303 304 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 305 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 306 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 307 308 /* 309 * Configurable value of RX free threshold. 310 */ 311 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 312 313 /* 314 * Configurable value of RX drop enable. 315 */ 316 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 317 318 /* 319 * Configurable value of TX free threshold. 320 */ 321 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 322 323 /* 324 * Configurable value of TX RS bit threshold. 325 */ 326 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 327 328 /* 329 * Configurable value of buffered packets before sending. 330 */ 331 uint16_t noisy_tx_sw_bufsz; 332 333 /* 334 * Configurable value of packet buffer timeout. 335 */ 336 uint16_t noisy_tx_sw_buf_flush_time; 337 338 /* 339 * Configurable value for size of VNF internal memory area 340 * used for simulating noisy neighbour behaviour 341 */ 342 uint64_t noisy_lkup_mem_sz; 343 344 /* 345 * Configurable value of number of random writes done in 346 * VNF simulation memory area. 347 */ 348 uint64_t noisy_lkup_num_writes; 349 350 /* 351 * Configurable value of number of random reads done in 352 * VNF simulation memory area. 353 */ 354 uint64_t noisy_lkup_num_reads; 355 356 /* 357 * Configurable value of number of random reads/writes done in 358 * VNF simulation memory area. 359 */ 360 uint64_t noisy_lkup_num_reads_writes; 361 362 /* 363 * Receive Side Scaling (RSS) configuration. 364 */ 365 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 366 367 /* 368 * Port topology configuration 369 */ 370 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 371 372 /* 373 * Avoids to flush all the RX streams before starts forwarding. 374 */ 375 uint8_t no_flush_rx = 0; /* flush by default */ 376 377 /* 378 * Flow API isolated mode. 379 */ 380 uint8_t flow_isolate_all; 381 382 /* 383 * Avoids to check link status when starting/stopping a port. 384 */ 385 uint8_t no_link_check = 0; /* check by default */ 386 387 /* 388 * Don't automatically start all ports in interactive mode. 389 */ 390 uint8_t no_device_start = 0; 391 392 /* 393 * Enable link status change notification 394 */ 395 uint8_t lsc_interrupt = 1; /* enabled by default */ 396 397 /* 398 * Enable device removal notification. 399 */ 400 uint8_t rmv_interrupt = 1; /* enabled by default */ 401 402 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 403 404 /* After attach, port setup is called on event or by iterator */ 405 bool setup_on_probe_event = true; 406 407 /* Clear ptypes on port initialization. */ 408 uint8_t clear_ptypes = true; 409 410 /* Hairpin ports configuration mode. */ 411 uint16_t hairpin_mode; 412 413 /* Pretty printing of ethdev events */ 414 static const char * const eth_event_desc[] = { 415 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 416 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 417 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 418 [RTE_ETH_EVENT_INTR_RESET] = "reset", 419 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 420 [RTE_ETH_EVENT_IPSEC] = "IPsec", 421 [RTE_ETH_EVENT_MACSEC] = "MACsec", 422 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 423 [RTE_ETH_EVENT_NEW] = "device probed", 424 [RTE_ETH_EVENT_DESTROY] = "device released", 425 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 426 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 427 [RTE_ETH_EVENT_MAX] = NULL, 428 }; 429 430 /* 431 * Display or mask ether events 432 * Default to all events except VF_MBOX 433 */ 434 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 435 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 436 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 437 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 438 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 439 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 440 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 441 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 442 /* 443 * Decide if all memory are locked for performance. 444 */ 445 int do_mlockall = 0; 446 447 /* 448 * NIC bypass mode configuration options. 449 */ 450 451 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 452 /* The NIC bypass watchdog timeout. */ 453 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 454 #endif 455 456 457 #ifdef RTE_LIB_LATENCYSTATS 458 459 /* 460 * Set when latency stats is enabled in the commandline 461 */ 462 uint8_t latencystats_enabled; 463 464 /* 465 * Lcore ID to service latency statistics. 466 */ 467 lcoreid_t latencystats_lcore_id = -1; 468 469 #endif 470 471 /* 472 * Ethernet device configuration. 473 */ 474 struct rte_eth_rxmode rx_mode; 475 476 struct rte_eth_txmode tx_mode = { 477 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 478 }; 479 480 struct rte_eth_fdir_conf fdir_conf = { 481 .mode = RTE_FDIR_MODE_NONE, 482 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 483 .status = RTE_FDIR_REPORT_STATUS, 484 .mask = { 485 .vlan_tci_mask = 0xFFEF, 486 .ipv4_mask = { 487 .src_ip = 0xFFFFFFFF, 488 .dst_ip = 0xFFFFFFFF, 489 }, 490 .ipv6_mask = { 491 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 492 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 493 }, 494 .src_port_mask = 0xFFFF, 495 .dst_port_mask = 0xFFFF, 496 .mac_addr_byte_mask = 0xFF, 497 .tunnel_type_mask = 1, 498 .tunnel_id_mask = 0xFFFFFFFF, 499 }, 500 .drop_queue = 127, 501 }; 502 503 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 504 505 /* 506 * Display zero values by default for xstats 507 */ 508 uint8_t xstats_hide_zero; 509 510 /* 511 * Measure of CPU cycles disabled by default 512 */ 513 uint8_t record_core_cycles; 514 515 /* 516 * Display of RX and TX bursts disabled by default 517 */ 518 uint8_t record_burst_stats; 519 520 /* 521 * Number of ports per shared Rx queue group, 0 disable. 522 */ 523 uint32_t rxq_share; 524 525 unsigned int num_sockets = 0; 526 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 527 528 #ifdef RTE_LIB_BITRATESTATS 529 /* Bitrate statistics */ 530 struct rte_stats_bitrates *bitrate_data; 531 lcoreid_t bitrate_lcore_id; 532 uint8_t bitrate_enabled; 533 #endif 534 535 #ifdef RTE_LIB_GRO 536 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 537 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 538 #endif 539 540 /* 541 * hexadecimal bitmask of RX mq mode can be enabled. 542 */ 543 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 544 545 /* 546 * Used to set forced link speed 547 */ 548 uint32_t eth_link_speed; 549 550 /* 551 * ID of the current process in multi-process, used to 552 * configure the queues to be polled. 553 */ 554 int proc_id; 555 556 /* 557 * Number of processes in multi-process, used to 558 * configure the queues to be polled. 559 */ 560 unsigned int num_procs = 1; 561 562 static void 563 eth_rx_metadata_negotiate_mp(uint16_t port_id) 564 { 565 uint64_t rx_meta_features = 0; 566 int ret; 567 568 if (!is_proc_primary()) 569 return; 570 571 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 572 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 573 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 574 575 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 576 if (ret == 0) { 577 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 578 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 579 port_id); 580 } 581 582 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 583 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 584 port_id); 585 } 586 587 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 588 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 589 port_id); 590 } 591 } else if (ret != -ENOTSUP) { 592 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 593 port_id, rte_strerror(-ret)); 594 } 595 } 596 597 static int 598 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 599 const struct rte_eth_conf *dev_conf) 600 { 601 if (is_proc_primary()) 602 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 603 dev_conf); 604 return 0; 605 } 606 607 static int 608 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 609 { 610 #ifdef RTE_NET_BOND 611 612 portid_t slave_pids[RTE_MAX_ETHPORTS]; 613 struct rte_port *port; 614 int num_slaves; 615 portid_t slave_pid; 616 int i; 617 618 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 619 RTE_MAX_ETHPORTS); 620 if (num_slaves < 0) { 621 fprintf(stderr, "Failed to get slave list for port = %u\n", 622 bond_pid); 623 return num_slaves; 624 } 625 626 for (i = 0; i < num_slaves; i++) { 627 slave_pid = slave_pids[i]; 628 port = &ports[slave_pid]; 629 port->port_status = 630 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 631 } 632 #else 633 RTE_SET_USED(bond_pid); 634 RTE_SET_USED(is_stop); 635 #endif 636 return 0; 637 } 638 639 static int 640 eth_dev_start_mp(uint16_t port_id) 641 { 642 int ret; 643 644 if (is_proc_primary()) { 645 ret = rte_eth_dev_start(port_id); 646 if (ret != 0) 647 return ret; 648 649 struct rte_port *port = &ports[port_id]; 650 651 /* 652 * Starting a bonded port also starts all slaves under the bonded 653 * device. So if this port is bond device, we need to modify the 654 * port status of these slaves. 655 */ 656 if (port->bond_flag == 1) 657 return change_bonding_slave_port_status(port_id, false); 658 } 659 660 return 0; 661 } 662 663 static int 664 eth_dev_stop_mp(uint16_t port_id) 665 { 666 int ret; 667 668 if (is_proc_primary()) { 669 ret = rte_eth_dev_stop(port_id); 670 if (ret != 0) 671 return ret; 672 673 struct rte_port *port = &ports[port_id]; 674 675 /* 676 * Stopping a bonded port also stops all slaves under the bonded 677 * device. So if this port is bond device, we need to modify the 678 * port status of these slaves. 679 */ 680 if (port->bond_flag == 1) 681 return change_bonding_slave_port_status(port_id, true); 682 } 683 684 return 0; 685 } 686 687 static void 688 mempool_free_mp(struct rte_mempool *mp) 689 { 690 if (is_proc_primary()) 691 rte_mempool_free(mp); 692 } 693 694 static int 695 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 696 { 697 if (is_proc_primary()) 698 return rte_eth_dev_set_mtu(port_id, mtu); 699 700 return 0; 701 } 702 703 /* Forward function declarations */ 704 static void setup_attached_port(portid_t pi); 705 static void check_all_ports_link_status(uint32_t port_mask); 706 static int eth_event_callback(portid_t port_id, 707 enum rte_eth_event_type type, 708 void *param, void *ret_param); 709 static void dev_event_callback(const char *device_name, 710 enum rte_dev_event_type type, 711 void *param); 712 static void fill_xstats_display_info(void); 713 714 /* 715 * Check if all the ports are started. 716 * If yes, return positive value. If not, return zero. 717 */ 718 static int all_ports_started(void); 719 720 #ifdef RTE_LIB_GSO 721 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 722 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 723 #endif 724 725 /* Holds the registered mbuf dynamic flags names. */ 726 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 727 728 729 /* 730 * Helper function to check if socket is already discovered. 731 * If yes, return positive value. If not, return zero. 732 */ 733 int 734 new_socket_id(unsigned int socket_id) 735 { 736 unsigned int i; 737 738 for (i = 0; i < num_sockets; i++) { 739 if (socket_ids[i] == socket_id) 740 return 0; 741 } 742 return 1; 743 } 744 745 /* 746 * Setup default configuration. 747 */ 748 static void 749 set_default_fwd_lcores_config(void) 750 { 751 unsigned int i; 752 unsigned int nb_lc; 753 unsigned int sock_num; 754 755 nb_lc = 0; 756 for (i = 0; i < RTE_MAX_LCORE; i++) { 757 if (!rte_lcore_is_enabled(i)) 758 continue; 759 sock_num = rte_lcore_to_socket_id(i); 760 if (new_socket_id(sock_num)) { 761 if (num_sockets >= RTE_MAX_NUMA_NODES) { 762 rte_exit(EXIT_FAILURE, 763 "Total sockets greater than %u\n", 764 RTE_MAX_NUMA_NODES); 765 } 766 socket_ids[num_sockets++] = sock_num; 767 } 768 if (i == rte_get_main_lcore()) 769 continue; 770 fwd_lcores_cpuids[nb_lc++] = i; 771 } 772 nb_lcores = (lcoreid_t) nb_lc; 773 nb_cfg_lcores = nb_lcores; 774 nb_fwd_lcores = 1; 775 } 776 777 static void 778 set_def_peer_eth_addrs(void) 779 { 780 portid_t i; 781 782 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 783 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 784 peer_eth_addrs[i].addr_bytes[5] = i; 785 } 786 } 787 788 static void 789 set_default_fwd_ports_config(void) 790 { 791 portid_t pt_id; 792 int i = 0; 793 794 RTE_ETH_FOREACH_DEV(pt_id) { 795 fwd_ports_ids[i++] = pt_id; 796 797 /* Update sockets info according to the attached device */ 798 int socket_id = rte_eth_dev_socket_id(pt_id); 799 if (socket_id >= 0 && new_socket_id(socket_id)) { 800 if (num_sockets >= RTE_MAX_NUMA_NODES) { 801 rte_exit(EXIT_FAILURE, 802 "Total sockets greater than %u\n", 803 RTE_MAX_NUMA_NODES); 804 } 805 socket_ids[num_sockets++] = socket_id; 806 } 807 } 808 809 nb_cfg_ports = nb_ports; 810 nb_fwd_ports = nb_ports; 811 } 812 813 void 814 set_def_fwd_config(void) 815 { 816 set_default_fwd_lcores_config(); 817 set_def_peer_eth_addrs(); 818 set_default_fwd_ports_config(); 819 } 820 821 #ifndef RTE_EXEC_ENV_WINDOWS 822 /* extremely pessimistic estimation of memory required to create a mempool */ 823 static int 824 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 825 { 826 unsigned int n_pages, mbuf_per_pg, leftover; 827 uint64_t total_mem, mbuf_mem, obj_sz; 828 829 /* there is no good way to predict how much space the mempool will 830 * occupy because it will allocate chunks on the fly, and some of those 831 * will come from default DPDK memory while some will come from our 832 * external memory, so just assume 128MB will be enough for everyone. 833 */ 834 uint64_t hdr_mem = 128 << 20; 835 836 /* account for possible non-contiguousness */ 837 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 838 if (obj_sz > pgsz) { 839 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 840 return -1; 841 } 842 843 mbuf_per_pg = pgsz / obj_sz; 844 leftover = (nb_mbufs % mbuf_per_pg) > 0; 845 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 846 847 mbuf_mem = n_pages * pgsz; 848 849 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 850 851 if (total_mem > SIZE_MAX) { 852 TESTPMD_LOG(ERR, "Memory size too big\n"); 853 return -1; 854 } 855 *out = (size_t)total_mem; 856 857 return 0; 858 } 859 860 static int 861 pagesz_flags(uint64_t page_sz) 862 { 863 /* as per mmap() manpage, all page sizes are log2 of page size 864 * shifted by MAP_HUGE_SHIFT 865 */ 866 int log2 = rte_log2_u64(page_sz); 867 868 return (log2 << HUGE_SHIFT); 869 } 870 871 static void * 872 alloc_mem(size_t memsz, size_t pgsz, bool huge) 873 { 874 void *addr; 875 int flags; 876 877 /* allocate anonymous hugepages */ 878 flags = MAP_ANONYMOUS | MAP_PRIVATE; 879 if (huge) 880 flags |= HUGE_FLAG | pagesz_flags(pgsz); 881 882 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 883 if (addr == MAP_FAILED) 884 return NULL; 885 886 return addr; 887 } 888 889 struct extmem_param { 890 void *addr; 891 size_t len; 892 size_t pgsz; 893 rte_iova_t *iova_table; 894 unsigned int iova_table_len; 895 }; 896 897 static int 898 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 899 bool huge) 900 { 901 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 902 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 903 unsigned int cur_page, n_pages, pgsz_idx; 904 size_t mem_sz, cur_pgsz; 905 rte_iova_t *iovas = NULL; 906 void *addr; 907 int ret; 908 909 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 910 /* skip anything that is too big */ 911 if (pgsizes[pgsz_idx] > SIZE_MAX) 912 continue; 913 914 cur_pgsz = pgsizes[pgsz_idx]; 915 916 /* if we were told not to allocate hugepages, override */ 917 if (!huge) 918 cur_pgsz = sysconf(_SC_PAGESIZE); 919 920 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 921 if (ret < 0) { 922 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 923 return -1; 924 } 925 926 /* allocate our memory */ 927 addr = alloc_mem(mem_sz, cur_pgsz, huge); 928 929 /* if we couldn't allocate memory with a specified page size, 930 * that doesn't mean we can't do it with other page sizes, so 931 * try another one. 932 */ 933 if (addr == NULL) 934 continue; 935 936 /* store IOVA addresses for every page in this memory area */ 937 n_pages = mem_sz / cur_pgsz; 938 939 iovas = malloc(sizeof(*iovas) * n_pages); 940 941 if (iovas == NULL) { 942 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 943 goto fail; 944 } 945 /* lock memory if it's not huge pages */ 946 if (!huge) 947 mlock(addr, mem_sz); 948 949 /* populate IOVA addresses */ 950 for (cur_page = 0; cur_page < n_pages; cur_page++) { 951 rte_iova_t iova; 952 size_t offset; 953 void *cur; 954 955 offset = cur_pgsz * cur_page; 956 cur = RTE_PTR_ADD(addr, offset); 957 958 /* touch the page before getting its IOVA */ 959 *(volatile char *)cur = 0; 960 961 iova = rte_mem_virt2iova(cur); 962 963 iovas[cur_page] = iova; 964 } 965 966 break; 967 } 968 /* if we couldn't allocate anything */ 969 if (iovas == NULL) 970 return -1; 971 972 param->addr = addr; 973 param->len = mem_sz; 974 param->pgsz = cur_pgsz; 975 param->iova_table = iovas; 976 param->iova_table_len = n_pages; 977 978 return 0; 979 fail: 980 free(iovas); 981 if (addr) 982 munmap(addr, mem_sz); 983 984 return -1; 985 } 986 987 static int 988 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 989 { 990 struct extmem_param param; 991 int socket_id, ret; 992 993 memset(¶m, 0, sizeof(param)); 994 995 /* check if our heap exists */ 996 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 997 if (socket_id < 0) { 998 /* create our heap */ 999 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 1000 if (ret < 0) { 1001 TESTPMD_LOG(ERR, "Cannot create heap\n"); 1002 return -1; 1003 } 1004 } 1005 1006 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 1007 if (ret < 0) { 1008 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 1009 return -1; 1010 } 1011 1012 /* we now have a valid memory area, so add it to heap */ 1013 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 1014 param.addr, param.len, param.iova_table, 1015 param.iova_table_len, param.pgsz); 1016 1017 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1018 1019 /* not needed any more */ 1020 free(param.iova_table); 1021 1022 if (ret < 0) { 1023 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1024 munmap(param.addr, param.len); 1025 return -1; 1026 } 1027 1028 /* success */ 1029 1030 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1031 param.len >> 20); 1032 1033 return 0; 1034 } 1035 static void 1036 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1037 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1038 { 1039 uint16_t pid = 0; 1040 int ret; 1041 1042 RTE_ETH_FOREACH_DEV(pid) { 1043 struct rte_eth_dev_info dev_info; 1044 1045 ret = eth_dev_info_get_print_err(pid, &dev_info); 1046 if (ret != 0) { 1047 TESTPMD_LOG(DEBUG, 1048 "unable to get device info for port %d on addr 0x%p," 1049 "mempool unmapping will not be performed\n", 1050 pid, memhdr->addr); 1051 continue; 1052 } 1053 1054 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1055 if (ret) { 1056 TESTPMD_LOG(DEBUG, 1057 "unable to DMA unmap addr 0x%p " 1058 "for device %s\n", 1059 memhdr->addr, dev_info.device->name); 1060 } 1061 } 1062 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1063 if (ret) { 1064 TESTPMD_LOG(DEBUG, 1065 "unable to un-register addr 0x%p\n", memhdr->addr); 1066 } 1067 } 1068 1069 static void 1070 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1071 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1072 { 1073 uint16_t pid = 0; 1074 size_t page_size = sysconf(_SC_PAGESIZE); 1075 int ret; 1076 1077 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1078 page_size); 1079 if (ret) { 1080 TESTPMD_LOG(DEBUG, 1081 "unable to register addr 0x%p\n", memhdr->addr); 1082 return; 1083 } 1084 RTE_ETH_FOREACH_DEV(pid) { 1085 struct rte_eth_dev_info dev_info; 1086 1087 ret = eth_dev_info_get_print_err(pid, &dev_info); 1088 if (ret != 0) { 1089 TESTPMD_LOG(DEBUG, 1090 "unable to get device info for port %d on addr 0x%p," 1091 "mempool mapping will not be performed\n", 1092 pid, memhdr->addr); 1093 continue; 1094 } 1095 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1096 if (ret) { 1097 TESTPMD_LOG(DEBUG, 1098 "unable to DMA map addr 0x%p " 1099 "for device %s\n", 1100 memhdr->addr, dev_info.device->name); 1101 } 1102 } 1103 } 1104 #endif 1105 1106 static unsigned int 1107 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1108 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1109 { 1110 struct rte_pktmbuf_extmem *xmem; 1111 unsigned int ext_num, zone_num, elt_num; 1112 uint16_t elt_size; 1113 1114 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1115 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1116 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1117 1118 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1119 if (xmem == NULL) { 1120 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1121 "external buffer descriptors\n"); 1122 *ext_mem = NULL; 1123 return 0; 1124 } 1125 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1126 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1127 const struct rte_memzone *mz; 1128 char mz_name[RTE_MEMZONE_NAMESIZE]; 1129 int ret; 1130 1131 ret = snprintf(mz_name, sizeof(mz_name), 1132 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1133 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1134 errno = ENAMETOOLONG; 1135 ext_num = 0; 1136 break; 1137 } 1138 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1139 socket_id, 1140 RTE_MEMZONE_IOVA_CONTIG | 1141 RTE_MEMZONE_1GB | 1142 RTE_MEMZONE_SIZE_HINT_ONLY); 1143 if (mz == NULL) { 1144 /* 1145 * The caller exits on external buffer creation 1146 * error, so there is no need to free memzones. 1147 */ 1148 errno = ENOMEM; 1149 ext_num = 0; 1150 break; 1151 } 1152 xseg->buf_ptr = mz->addr; 1153 xseg->buf_iova = mz->iova; 1154 xseg->buf_len = EXTBUF_ZONE_SIZE; 1155 xseg->elt_size = elt_size; 1156 } 1157 if (ext_num == 0 && xmem != NULL) { 1158 free(xmem); 1159 xmem = NULL; 1160 } 1161 *ext_mem = xmem; 1162 return ext_num; 1163 } 1164 1165 /* 1166 * Configuration initialisation done once at init time. 1167 */ 1168 static struct rte_mempool * 1169 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1170 unsigned int socket_id, uint16_t size_idx) 1171 { 1172 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1173 struct rte_mempool *rte_mp = NULL; 1174 #ifndef RTE_EXEC_ENV_WINDOWS 1175 uint32_t mb_size; 1176 1177 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1178 #endif 1179 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1180 if (!is_proc_primary()) { 1181 rte_mp = rte_mempool_lookup(pool_name); 1182 if (rte_mp == NULL) 1183 rte_exit(EXIT_FAILURE, 1184 "Get mbuf pool for socket %u failed: %s\n", 1185 socket_id, rte_strerror(rte_errno)); 1186 return rte_mp; 1187 } 1188 1189 TESTPMD_LOG(INFO, 1190 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1191 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1192 1193 switch (mp_alloc_type) { 1194 case MP_ALLOC_NATIVE: 1195 { 1196 /* wrapper to rte_mempool_create() */ 1197 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1198 rte_mbuf_best_mempool_ops()); 1199 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1200 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1201 break; 1202 } 1203 #ifndef RTE_EXEC_ENV_WINDOWS 1204 case MP_ALLOC_ANON: 1205 { 1206 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1207 mb_size, (unsigned int) mb_mempool_cache, 1208 sizeof(struct rte_pktmbuf_pool_private), 1209 socket_id, mempool_flags); 1210 if (rte_mp == NULL) 1211 goto err; 1212 1213 if (rte_mempool_populate_anon(rte_mp) == 0) { 1214 rte_mempool_free(rte_mp); 1215 rte_mp = NULL; 1216 goto err; 1217 } 1218 rte_pktmbuf_pool_init(rte_mp, NULL); 1219 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1220 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1221 break; 1222 } 1223 case MP_ALLOC_XMEM: 1224 case MP_ALLOC_XMEM_HUGE: 1225 { 1226 int heap_socket; 1227 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1228 1229 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1230 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1231 1232 heap_socket = 1233 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1234 if (heap_socket < 0) 1235 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1236 1237 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1238 rte_mbuf_best_mempool_ops()); 1239 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1240 mb_mempool_cache, 0, mbuf_seg_size, 1241 heap_socket); 1242 break; 1243 } 1244 #endif 1245 case MP_ALLOC_XBUF: 1246 { 1247 struct rte_pktmbuf_extmem *ext_mem; 1248 unsigned int ext_num; 1249 1250 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1251 socket_id, pool_name, &ext_mem); 1252 if (ext_num == 0) 1253 rte_exit(EXIT_FAILURE, 1254 "Can't create pinned data buffers\n"); 1255 1256 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1257 rte_mbuf_best_mempool_ops()); 1258 rte_mp = rte_pktmbuf_pool_create_extbuf 1259 (pool_name, nb_mbuf, mb_mempool_cache, 1260 0, mbuf_seg_size, socket_id, 1261 ext_mem, ext_num); 1262 free(ext_mem); 1263 break; 1264 } 1265 default: 1266 { 1267 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1268 } 1269 } 1270 1271 #ifndef RTE_EXEC_ENV_WINDOWS 1272 err: 1273 #endif 1274 if (rte_mp == NULL) { 1275 rte_exit(EXIT_FAILURE, 1276 "Creation of mbuf pool for socket %u failed: %s\n", 1277 socket_id, rte_strerror(rte_errno)); 1278 } else if (verbose_level > 0) { 1279 rte_mempool_dump(stdout, rte_mp); 1280 } 1281 return rte_mp; 1282 } 1283 1284 /* 1285 * Check given socket id is valid or not with NUMA mode, 1286 * if valid, return 0, else return -1 1287 */ 1288 static int 1289 check_socket_id(const unsigned int socket_id) 1290 { 1291 static int warning_once = 0; 1292 1293 if (new_socket_id(socket_id)) { 1294 if (!warning_once && numa_support) 1295 fprintf(stderr, 1296 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1297 warning_once = 1; 1298 return -1; 1299 } 1300 return 0; 1301 } 1302 1303 /* 1304 * Get the allowed maximum number of RX queues. 1305 * *pid return the port id which has minimal value of 1306 * max_rx_queues in all ports. 1307 */ 1308 queueid_t 1309 get_allowed_max_nb_rxq(portid_t *pid) 1310 { 1311 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1312 bool max_rxq_valid = false; 1313 portid_t pi; 1314 struct rte_eth_dev_info dev_info; 1315 1316 RTE_ETH_FOREACH_DEV(pi) { 1317 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1318 continue; 1319 1320 max_rxq_valid = true; 1321 if (dev_info.max_rx_queues < allowed_max_rxq) { 1322 allowed_max_rxq = dev_info.max_rx_queues; 1323 *pid = pi; 1324 } 1325 } 1326 return max_rxq_valid ? allowed_max_rxq : 0; 1327 } 1328 1329 /* 1330 * Check input rxq is valid or not. 1331 * If input rxq is not greater than any of maximum number 1332 * of RX queues of all ports, it is valid. 1333 * if valid, return 0, else return -1 1334 */ 1335 int 1336 check_nb_rxq(queueid_t rxq) 1337 { 1338 queueid_t allowed_max_rxq; 1339 portid_t pid = 0; 1340 1341 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1342 if (rxq > allowed_max_rxq) { 1343 fprintf(stderr, 1344 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1345 rxq, allowed_max_rxq, pid); 1346 return -1; 1347 } 1348 return 0; 1349 } 1350 1351 /* 1352 * Get the allowed maximum number of TX queues. 1353 * *pid return the port id which has minimal value of 1354 * max_tx_queues in all ports. 1355 */ 1356 queueid_t 1357 get_allowed_max_nb_txq(portid_t *pid) 1358 { 1359 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1360 bool max_txq_valid = false; 1361 portid_t pi; 1362 struct rte_eth_dev_info dev_info; 1363 1364 RTE_ETH_FOREACH_DEV(pi) { 1365 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1366 continue; 1367 1368 max_txq_valid = true; 1369 if (dev_info.max_tx_queues < allowed_max_txq) { 1370 allowed_max_txq = dev_info.max_tx_queues; 1371 *pid = pi; 1372 } 1373 } 1374 return max_txq_valid ? allowed_max_txq : 0; 1375 } 1376 1377 /* 1378 * Check input txq is valid or not. 1379 * If input txq is not greater than any of maximum number 1380 * of TX queues of all ports, it is valid. 1381 * if valid, return 0, else return -1 1382 */ 1383 int 1384 check_nb_txq(queueid_t txq) 1385 { 1386 queueid_t allowed_max_txq; 1387 portid_t pid = 0; 1388 1389 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1390 if (txq > allowed_max_txq) { 1391 fprintf(stderr, 1392 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1393 txq, allowed_max_txq, pid); 1394 return -1; 1395 } 1396 return 0; 1397 } 1398 1399 /* 1400 * Get the allowed maximum number of RXDs of every rx queue. 1401 * *pid return the port id which has minimal value of 1402 * max_rxd in all queues of all ports. 1403 */ 1404 static uint16_t 1405 get_allowed_max_nb_rxd(portid_t *pid) 1406 { 1407 uint16_t allowed_max_rxd = UINT16_MAX; 1408 portid_t pi; 1409 struct rte_eth_dev_info dev_info; 1410 1411 RTE_ETH_FOREACH_DEV(pi) { 1412 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1413 continue; 1414 1415 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1416 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1417 *pid = pi; 1418 } 1419 } 1420 return allowed_max_rxd; 1421 } 1422 1423 /* 1424 * Get the allowed minimal number of RXDs of every rx queue. 1425 * *pid return the port id which has minimal value of 1426 * min_rxd in all queues of all ports. 1427 */ 1428 static uint16_t 1429 get_allowed_min_nb_rxd(portid_t *pid) 1430 { 1431 uint16_t allowed_min_rxd = 0; 1432 portid_t pi; 1433 struct rte_eth_dev_info dev_info; 1434 1435 RTE_ETH_FOREACH_DEV(pi) { 1436 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1437 continue; 1438 1439 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1440 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1441 *pid = pi; 1442 } 1443 } 1444 1445 return allowed_min_rxd; 1446 } 1447 1448 /* 1449 * Check input rxd is valid or not. 1450 * If input rxd is not greater than any of maximum number 1451 * of RXDs of every Rx queues and is not less than any of 1452 * minimal number of RXDs of every Rx queues, it is valid. 1453 * if valid, return 0, else return -1 1454 */ 1455 int 1456 check_nb_rxd(queueid_t rxd) 1457 { 1458 uint16_t allowed_max_rxd; 1459 uint16_t allowed_min_rxd; 1460 portid_t pid = 0; 1461 1462 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1463 if (rxd > allowed_max_rxd) { 1464 fprintf(stderr, 1465 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1466 rxd, allowed_max_rxd, pid); 1467 return -1; 1468 } 1469 1470 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1471 if (rxd < allowed_min_rxd) { 1472 fprintf(stderr, 1473 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1474 rxd, allowed_min_rxd, pid); 1475 return -1; 1476 } 1477 1478 return 0; 1479 } 1480 1481 /* 1482 * Get the allowed maximum number of TXDs of every rx queues. 1483 * *pid return the port id which has minimal value of 1484 * max_txd in every tx queue. 1485 */ 1486 static uint16_t 1487 get_allowed_max_nb_txd(portid_t *pid) 1488 { 1489 uint16_t allowed_max_txd = UINT16_MAX; 1490 portid_t pi; 1491 struct rte_eth_dev_info dev_info; 1492 1493 RTE_ETH_FOREACH_DEV(pi) { 1494 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1495 continue; 1496 1497 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1498 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1499 *pid = pi; 1500 } 1501 } 1502 return allowed_max_txd; 1503 } 1504 1505 /* 1506 * Get the allowed maximum number of TXDs of every tx queues. 1507 * *pid return the port id which has minimal value of 1508 * min_txd in every tx queue. 1509 */ 1510 static uint16_t 1511 get_allowed_min_nb_txd(portid_t *pid) 1512 { 1513 uint16_t allowed_min_txd = 0; 1514 portid_t pi; 1515 struct rte_eth_dev_info dev_info; 1516 1517 RTE_ETH_FOREACH_DEV(pi) { 1518 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1519 continue; 1520 1521 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1522 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1523 *pid = pi; 1524 } 1525 } 1526 1527 return allowed_min_txd; 1528 } 1529 1530 /* 1531 * Check input txd is valid or not. 1532 * If input txd is not greater than any of maximum number 1533 * of TXDs of every Rx queues, it is valid. 1534 * if valid, return 0, else return -1 1535 */ 1536 int 1537 check_nb_txd(queueid_t txd) 1538 { 1539 uint16_t allowed_max_txd; 1540 uint16_t allowed_min_txd; 1541 portid_t pid = 0; 1542 1543 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1544 if (txd > allowed_max_txd) { 1545 fprintf(stderr, 1546 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1547 txd, allowed_max_txd, pid); 1548 return -1; 1549 } 1550 1551 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1552 if (txd < allowed_min_txd) { 1553 fprintf(stderr, 1554 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1555 txd, allowed_min_txd, pid); 1556 return -1; 1557 } 1558 return 0; 1559 } 1560 1561 1562 /* 1563 * Get the allowed maximum number of hairpin queues. 1564 * *pid return the port id which has minimal value of 1565 * max_hairpin_queues in all ports. 1566 */ 1567 queueid_t 1568 get_allowed_max_nb_hairpinq(portid_t *pid) 1569 { 1570 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1571 portid_t pi; 1572 struct rte_eth_hairpin_cap cap; 1573 1574 RTE_ETH_FOREACH_DEV(pi) { 1575 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1576 *pid = pi; 1577 return 0; 1578 } 1579 if (cap.max_nb_queues < allowed_max_hairpinq) { 1580 allowed_max_hairpinq = cap.max_nb_queues; 1581 *pid = pi; 1582 } 1583 } 1584 return allowed_max_hairpinq; 1585 } 1586 1587 /* 1588 * Check input hairpin is valid or not. 1589 * If input hairpin is not greater than any of maximum number 1590 * of hairpin queues of all ports, it is valid. 1591 * if valid, return 0, else return -1 1592 */ 1593 int 1594 check_nb_hairpinq(queueid_t hairpinq) 1595 { 1596 queueid_t allowed_max_hairpinq; 1597 portid_t pid = 0; 1598 1599 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1600 if (hairpinq > allowed_max_hairpinq) { 1601 fprintf(stderr, 1602 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1603 hairpinq, allowed_max_hairpinq, pid); 1604 return -1; 1605 } 1606 return 0; 1607 } 1608 1609 static int 1610 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1611 { 1612 uint32_t eth_overhead; 1613 1614 if (dev_info->max_mtu != UINT16_MAX && 1615 dev_info->max_rx_pktlen > dev_info->max_mtu) 1616 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1617 else 1618 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1619 1620 return eth_overhead; 1621 } 1622 1623 static void 1624 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1625 { 1626 struct rte_port *port = &ports[pid]; 1627 int ret; 1628 int i; 1629 1630 eth_rx_metadata_negotiate_mp(pid); 1631 1632 port->dev_conf.txmode = tx_mode; 1633 port->dev_conf.rxmode = rx_mode; 1634 1635 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1636 if (ret != 0) 1637 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1638 1639 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1640 port->dev_conf.txmode.offloads &= 1641 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1642 1643 /* Apply Rx offloads configuration */ 1644 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1645 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1646 /* Apply Tx offloads configuration */ 1647 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1648 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1649 1650 if (eth_link_speed) 1651 port->dev_conf.link_speeds = eth_link_speed; 1652 1653 if (max_rx_pkt_len) 1654 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1655 get_eth_overhead(&port->dev_info); 1656 1657 /* set flag to initialize port/queue */ 1658 port->need_reconfig = 1; 1659 port->need_reconfig_queues = 1; 1660 port->socket_id = socket_id; 1661 port->tx_metadata = 0; 1662 1663 /* 1664 * Check for maximum number of segments per MTU. 1665 * Accordingly update the mbuf data size. 1666 */ 1667 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1668 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1669 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1670 uint16_t mtu; 1671 1672 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1673 uint16_t data_size = (mtu + eth_overhead) / 1674 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1675 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1676 1677 if (buffer_size > mbuf_data_size[0]) { 1678 mbuf_data_size[0] = buffer_size; 1679 TESTPMD_LOG(WARNING, 1680 "Configured mbuf size of the first segment %hu\n", 1681 mbuf_data_size[0]); 1682 } 1683 } 1684 } 1685 } 1686 1687 static void 1688 init_config(void) 1689 { 1690 portid_t pid; 1691 struct rte_mempool *mbp; 1692 unsigned int nb_mbuf_per_pool; 1693 lcoreid_t lc_id; 1694 #ifdef RTE_LIB_GRO 1695 struct rte_gro_param gro_param; 1696 #endif 1697 #ifdef RTE_LIB_GSO 1698 uint32_t gso_types; 1699 #endif 1700 1701 /* Configuration of logical cores. */ 1702 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1703 sizeof(struct fwd_lcore *) * nb_lcores, 1704 RTE_CACHE_LINE_SIZE); 1705 if (fwd_lcores == NULL) { 1706 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1707 "failed\n", nb_lcores); 1708 } 1709 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1710 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1711 sizeof(struct fwd_lcore), 1712 RTE_CACHE_LINE_SIZE); 1713 if (fwd_lcores[lc_id] == NULL) { 1714 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1715 "failed\n"); 1716 } 1717 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1718 } 1719 1720 RTE_ETH_FOREACH_DEV(pid) { 1721 uint32_t socket_id; 1722 1723 if (numa_support) { 1724 socket_id = port_numa[pid]; 1725 if (port_numa[pid] == NUMA_NO_CONFIG) { 1726 socket_id = rte_eth_dev_socket_id(pid); 1727 1728 /* 1729 * if socket_id is invalid, 1730 * set to the first available socket. 1731 */ 1732 if (check_socket_id(socket_id) < 0) 1733 socket_id = socket_ids[0]; 1734 } 1735 } else { 1736 socket_id = (socket_num == UMA_NO_CONFIG) ? 1737 0 : socket_num; 1738 } 1739 /* Apply default TxRx configuration for all ports */ 1740 init_config_port_offloads(pid, socket_id); 1741 } 1742 /* 1743 * Create pools of mbuf. 1744 * If NUMA support is disabled, create a single pool of mbuf in 1745 * socket 0 memory by default. 1746 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1747 * 1748 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1749 * nb_txd can be configured at run time. 1750 */ 1751 if (param_total_num_mbufs) 1752 nb_mbuf_per_pool = param_total_num_mbufs; 1753 else { 1754 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1755 (nb_lcores * mb_mempool_cache) + 1756 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1757 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1758 } 1759 1760 if (numa_support) { 1761 uint8_t i, j; 1762 1763 for (i = 0; i < num_sockets; i++) 1764 for (j = 0; j < mbuf_data_size_n; j++) 1765 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1766 mbuf_pool_create(mbuf_data_size[j], 1767 nb_mbuf_per_pool, 1768 socket_ids[i], j); 1769 } else { 1770 uint8_t i; 1771 1772 for (i = 0; i < mbuf_data_size_n; i++) 1773 mempools[i] = mbuf_pool_create 1774 (mbuf_data_size[i], 1775 nb_mbuf_per_pool, 1776 socket_num == UMA_NO_CONFIG ? 1777 0 : socket_num, i); 1778 } 1779 1780 init_port_config(); 1781 1782 #ifdef RTE_LIB_GSO 1783 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1784 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1785 #endif 1786 /* 1787 * Records which Mbuf pool to use by each logical core, if needed. 1788 */ 1789 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1790 mbp = mbuf_pool_find( 1791 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1792 1793 if (mbp == NULL) 1794 mbp = mbuf_pool_find(0, 0); 1795 fwd_lcores[lc_id]->mbp = mbp; 1796 #ifdef RTE_LIB_GSO 1797 /* initialize GSO context */ 1798 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1799 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1800 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1801 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1802 RTE_ETHER_CRC_LEN; 1803 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1804 #endif 1805 } 1806 1807 fwd_config_setup(); 1808 1809 #ifdef RTE_LIB_GRO 1810 /* create a gro context for each lcore */ 1811 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1812 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1813 gro_param.max_item_per_flow = MAX_PKT_BURST; 1814 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1815 gro_param.socket_id = rte_lcore_to_socket_id( 1816 fwd_lcores_cpuids[lc_id]); 1817 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1818 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1819 rte_exit(EXIT_FAILURE, 1820 "rte_gro_ctx_create() failed\n"); 1821 } 1822 } 1823 #endif 1824 } 1825 1826 1827 void 1828 reconfig(portid_t new_port_id, unsigned socket_id) 1829 { 1830 /* Reconfiguration of Ethernet ports. */ 1831 init_config_port_offloads(new_port_id, socket_id); 1832 init_port_config(); 1833 } 1834 1835 int 1836 init_fwd_streams(void) 1837 { 1838 portid_t pid; 1839 struct rte_port *port; 1840 streamid_t sm_id, nb_fwd_streams_new; 1841 queueid_t q; 1842 1843 /* set socket id according to numa or not */ 1844 RTE_ETH_FOREACH_DEV(pid) { 1845 port = &ports[pid]; 1846 if (nb_rxq > port->dev_info.max_rx_queues) { 1847 fprintf(stderr, 1848 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1849 nb_rxq, port->dev_info.max_rx_queues); 1850 return -1; 1851 } 1852 if (nb_txq > port->dev_info.max_tx_queues) { 1853 fprintf(stderr, 1854 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1855 nb_txq, port->dev_info.max_tx_queues); 1856 return -1; 1857 } 1858 if (numa_support) { 1859 if (port_numa[pid] != NUMA_NO_CONFIG) 1860 port->socket_id = port_numa[pid]; 1861 else { 1862 port->socket_id = rte_eth_dev_socket_id(pid); 1863 1864 /* 1865 * if socket_id is invalid, 1866 * set to the first available socket. 1867 */ 1868 if (check_socket_id(port->socket_id) < 0) 1869 port->socket_id = socket_ids[0]; 1870 } 1871 } 1872 else { 1873 if (socket_num == UMA_NO_CONFIG) 1874 port->socket_id = 0; 1875 else 1876 port->socket_id = socket_num; 1877 } 1878 } 1879 1880 q = RTE_MAX(nb_rxq, nb_txq); 1881 if (q == 0) { 1882 fprintf(stderr, 1883 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1884 return -1; 1885 } 1886 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1887 if (nb_fwd_streams_new == nb_fwd_streams) 1888 return 0; 1889 /* clear the old */ 1890 if (fwd_streams != NULL) { 1891 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1892 if (fwd_streams[sm_id] == NULL) 1893 continue; 1894 rte_free(fwd_streams[sm_id]); 1895 fwd_streams[sm_id] = NULL; 1896 } 1897 rte_free(fwd_streams); 1898 fwd_streams = NULL; 1899 } 1900 1901 /* init new */ 1902 nb_fwd_streams = nb_fwd_streams_new; 1903 if (nb_fwd_streams) { 1904 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1905 sizeof(struct fwd_stream *) * nb_fwd_streams, 1906 RTE_CACHE_LINE_SIZE); 1907 if (fwd_streams == NULL) 1908 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1909 " (struct fwd_stream *)) failed\n", 1910 nb_fwd_streams); 1911 1912 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1913 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1914 " struct fwd_stream", sizeof(struct fwd_stream), 1915 RTE_CACHE_LINE_SIZE); 1916 if (fwd_streams[sm_id] == NULL) 1917 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1918 "(struct fwd_stream) failed\n"); 1919 } 1920 } 1921 1922 return 0; 1923 } 1924 1925 static void 1926 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1927 { 1928 uint64_t total_burst, sburst; 1929 uint64_t nb_burst; 1930 uint64_t burst_stats[4]; 1931 uint16_t pktnb_stats[4]; 1932 uint16_t nb_pkt; 1933 int burst_percent[4], sburstp; 1934 int i; 1935 1936 /* 1937 * First compute the total number of packet bursts and the 1938 * two highest numbers of bursts of the same number of packets. 1939 */ 1940 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1941 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1942 1943 /* Show stats for 0 burst size always */ 1944 total_burst = pbs->pkt_burst_spread[0]; 1945 burst_stats[0] = pbs->pkt_burst_spread[0]; 1946 pktnb_stats[0] = 0; 1947 1948 /* Find the next 2 burst sizes with highest occurrences. */ 1949 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1950 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1951 1952 if (nb_burst == 0) 1953 continue; 1954 1955 total_burst += nb_burst; 1956 1957 if (nb_burst > burst_stats[1]) { 1958 burst_stats[2] = burst_stats[1]; 1959 pktnb_stats[2] = pktnb_stats[1]; 1960 burst_stats[1] = nb_burst; 1961 pktnb_stats[1] = nb_pkt; 1962 } else if (nb_burst > burst_stats[2]) { 1963 burst_stats[2] = nb_burst; 1964 pktnb_stats[2] = nb_pkt; 1965 } 1966 } 1967 if (total_burst == 0) 1968 return; 1969 1970 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1971 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1972 if (i == 3) { 1973 printf("%d%% of other]\n", 100 - sburstp); 1974 return; 1975 } 1976 1977 sburst += burst_stats[i]; 1978 if (sburst == total_burst) { 1979 printf("%d%% of %d pkts]\n", 1980 100 - sburstp, (int) pktnb_stats[i]); 1981 return; 1982 } 1983 1984 burst_percent[i] = 1985 (double)burst_stats[i] / total_burst * 100; 1986 printf("%d%% of %d pkts + ", 1987 burst_percent[i], (int) pktnb_stats[i]); 1988 sburstp += burst_percent[i]; 1989 } 1990 } 1991 1992 static void 1993 fwd_stream_stats_display(streamid_t stream_id) 1994 { 1995 struct fwd_stream *fs; 1996 static const char *fwd_top_stats_border = "-------"; 1997 1998 fs = fwd_streams[stream_id]; 1999 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 2000 (fs->fwd_dropped == 0)) 2001 return; 2002 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 2003 "TX Port=%2d/Queue=%2d %s\n", 2004 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 2005 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 2006 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 2007 " TX-dropped: %-14"PRIu64, 2008 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 2009 2010 /* if checksum mode */ 2011 if (cur_fwd_eng == &csum_fwd_engine) { 2012 printf(" RX- bad IP checksum: %-14"PRIu64 2013 " Rx- bad L4 checksum: %-14"PRIu64 2014 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 2015 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 2016 fs->rx_bad_outer_l4_csum); 2017 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2018 fs->rx_bad_outer_ip_csum); 2019 } else { 2020 printf("\n"); 2021 } 2022 2023 if (record_burst_stats) { 2024 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2025 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2026 } 2027 } 2028 2029 void 2030 fwd_stats_display(void) 2031 { 2032 static const char *fwd_stats_border = "----------------------"; 2033 static const char *acc_stats_border = "+++++++++++++++"; 2034 struct { 2035 struct fwd_stream *rx_stream; 2036 struct fwd_stream *tx_stream; 2037 uint64_t tx_dropped; 2038 uint64_t rx_bad_ip_csum; 2039 uint64_t rx_bad_l4_csum; 2040 uint64_t rx_bad_outer_l4_csum; 2041 uint64_t rx_bad_outer_ip_csum; 2042 } ports_stats[RTE_MAX_ETHPORTS]; 2043 uint64_t total_rx_dropped = 0; 2044 uint64_t total_tx_dropped = 0; 2045 uint64_t total_rx_nombuf = 0; 2046 struct rte_eth_stats stats; 2047 uint64_t fwd_cycles = 0; 2048 uint64_t total_recv = 0; 2049 uint64_t total_xmit = 0; 2050 struct rte_port *port; 2051 streamid_t sm_id; 2052 portid_t pt_id; 2053 int ret; 2054 int i; 2055 2056 memset(ports_stats, 0, sizeof(ports_stats)); 2057 2058 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2059 struct fwd_stream *fs = fwd_streams[sm_id]; 2060 2061 if (cur_fwd_config.nb_fwd_streams > 2062 cur_fwd_config.nb_fwd_ports) { 2063 fwd_stream_stats_display(sm_id); 2064 } else { 2065 ports_stats[fs->tx_port].tx_stream = fs; 2066 ports_stats[fs->rx_port].rx_stream = fs; 2067 } 2068 2069 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2070 2071 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2072 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2073 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2074 fs->rx_bad_outer_l4_csum; 2075 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2076 fs->rx_bad_outer_ip_csum; 2077 2078 if (record_core_cycles) 2079 fwd_cycles += fs->core_cycles; 2080 } 2081 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2082 pt_id = fwd_ports_ids[i]; 2083 port = &ports[pt_id]; 2084 2085 ret = rte_eth_stats_get(pt_id, &stats); 2086 if (ret != 0) { 2087 fprintf(stderr, 2088 "%s: Error: failed to get stats (port %u): %d", 2089 __func__, pt_id, ret); 2090 continue; 2091 } 2092 stats.ipackets -= port->stats.ipackets; 2093 stats.opackets -= port->stats.opackets; 2094 stats.ibytes -= port->stats.ibytes; 2095 stats.obytes -= port->stats.obytes; 2096 stats.imissed -= port->stats.imissed; 2097 stats.oerrors -= port->stats.oerrors; 2098 stats.rx_nombuf -= port->stats.rx_nombuf; 2099 2100 total_recv += stats.ipackets; 2101 total_xmit += stats.opackets; 2102 total_rx_dropped += stats.imissed; 2103 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2104 total_tx_dropped += stats.oerrors; 2105 total_rx_nombuf += stats.rx_nombuf; 2106 2107 printf("\n %s Forward statistics for port %-2d %s\n", 2108 fwd_stats_border, pt_id, fwd_stats_border); 2109 2110 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2111 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2112 stats.ipackets + stats.imissed); 2113 2114 if (cur_fwd_eng == &csum_fwd_engine) { 2115 printf(" Bad-ipcsum: %-14"PRIu64 2116 " Bad-l4csum: %-14"PRIu64 2117 "Bad-outer-l4csum: %-14"PRIu64"\n", 2118 ports_stats[pt_id].rx_bad_ip_csum, 2119 ports_stats[pt_id].rx_bad_l4_csum, 2120 ports_stats[pt_id].rx_bad_outer_l4_csum); 2121 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2122 ports_stats[pt_id].rx_bad_outer_ip_csum); 2123 } 2124 if (stats.ierrors + stats.rx_nombuf > 0) { 2125 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2126 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2127 } 2128 2129 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2130 "TX-total: %-"PRIu64"\n", 2131 stats.opackets, ports_stats[pt_id].tx_dropped, 2132 stats.opackets + ports_stats[pt_id].tx_dropped); 2133 2134 if (record_burst_stats) { 2135 if (ports_stats[pt_id].rx_stream) 2136 pkt_burst_stats_display("RX", 2137 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2138 if (ports_stats[pt_id].tx_stream) 2139 pkt_burst_stats_display("TX", 2140 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2141 } 2142 2143 printf(" %s--------------------------------%s\n", 2144 fwd_stats_border, fwd_stats_border); 2145 } 2146 2147 printf("\n %s Accumulated forward statistics for all ports" 2148 "%s\n", 2149 acc_stats_border, acc_stats_border); 2150 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2151 "%-"PRIu64"\n" 2152 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2153 "%-"PRIu64"\n", 2154 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2155 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2156 if (total_rx_nombuf > 0) 2157 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2158 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2159 "%s\n", 2160 acc_stats_border, acc_stats_border); 2161 if (record_core_cycles) { 2162 #define CYC_PER_MHZ 1E6 2163 if (total_recv > 0 || total_xmit > 0) { 2164 uint64_t total_pkts = 0; 2165 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2166 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2167 total_pkts = total_xmit; 2168 else 2169 total_pkts = total_recv; 2170 2171 printf("\n CPU cycles/packet=%.2F (total cycles=" 2172 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2173 " MHz Clock\n", 2174 (double) fwd_cycles / total_pkts, 2175 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2176 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2177 } 2178 } 2179 } 2180 2181 void 2182 fwd_stats_reset(void) 2183 { 2184 streamid_t sm_id; 2185 portid_t pt_id; 2186 int ret; 2187 int i; 2188 2189 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2190 pt_id = fwd_ports_ids[i]; 2191 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2192 if (ret != 0) 2193 fprintf(stderr, 2194 "%s: Error: failed to clear stats (port %u):%d", 2195 __func__, pt_id, ret); 2196 } 2197 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2198 struct fwd_stream *fs = fwd_streams[sm_id]; 2199 2200 fs->rx_packets = 0; 2201 fs->tx_packets = 0; 2202 fs->fwd_dropped = 0; 2203 fs->rx_bad_ip_csum = 0; 2204 fs->rx_bad_l4_csum = 0; 2205 fs->rx_bad_outer_l4_csum = 0; 2206 fs->rx_bad_outer_ip_csum = 0; 2207 2208 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2209 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2210 fs->core_cycles = 0; 2211 } 2212 } 2213 2214 static void 2215 flush_fwd_rx_queues(void) 2216 { 2217 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2218 portid_t rxp; 2219 portid_t port_id; 2220 queueid_t rxq; 2221 uint16_t nb_rx; 2222 uint16_t i; 2223 uint8_t j; 2224 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2225 uint64_t timer_period; 2226 2227 if (num_procs > 1) { 2228 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2229 return; 2230 } 2231 2232 /* convert to number of cycles */ 2233 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2234 2235 for (j = 0; j < 2; j++) { 2236 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2237 for (rxq = 0; rxq < nb_rxq; rxq++) { 2238 port_id = fwd_ports_ids[rxp]; 2239 2240 /* Polling stopped queues is prohibited. */ 2241 if (ports[port_id].rxq[rxq].state == 2242 RTE_ETH_QUEUE_STATE_STOPPED) 2243 continue; 2244 2245 /** 2246 * testpmd can stuck in the below do while loop 2247 * if rte_eth_rx_burst() always returns nonzero 2248 * packets. So timer is added to exit this loop 2249 * after 1sec timer expiry. 2250 */ 2251 prev_tsc = rte_rdtsc(); 2252 do { 2253 nb_rx = rte_eth_rx_burst(port_id, rxq, 2254 pkts_burst, MAX_PKT_BURST); 2255 for (i = 0; i < nb_rx; i++) 2256 rte_pktmbuf_free(pkts_burst[i]); 2257 2258 cur_tsc = rte_rdtsc(); 2259 diff_tsc = cur_tsc - prev_tsc; 2260 timer_tsc += diff_tsc; 2261 } while ((nb_rx > 0) && 2262 (timer_tsc < timer_period)); 2263 timer_tsc = 0; 2264 } 2265 } 2266 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2267 } 2268 } 2269 2270 static void 2271 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2272 { 2273 struct fwd_stream **fsm; 2274 streamid_t nb_fs; 2275 streamid_t sm_id; 2276 #ifdef RTE_LIB_BITRATESTATS 2277 uint64_t tics_per_1sec; 2278 uint64_t tics_datum; 2279 uint64_t tics_current; 2280 uint16_t i, cnt_ports; 2281 2282 cnt_ports = nb_ports; 2283 tics_datum = rte_rdtsc(); 2284 tics_per_1sec = rte_get_timer_hz(); 2285 #endif 2286 fsm = &fwd_streams[fc->stream_idx]; 2287 nb_fs = fc->stream_nb; 2288 do { 2289 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2290 if (!fsm[sm_id]->disabled) 2291 (*pkt_fwd)(fsm[sm_id]); 2292 #ifdef RTE_LIB_BITRATESTATS 2293 if (bitrate_enabled != 0 && 2294 bitrate_lcore_id == rte_lcore_id()) { 2295 tics_current = rte_rdtsc(); 2296 if (tics_current - tics_datum >= tics_per_1sec) { 2297 /* Periodic bitrate calculation */ 2298 for (i = 0; i < cnt_ports; i++) 2299 rte_stats_bitrate_calc(bitrate_data, 2300 ports_ids[i]); 2301 tics_datum = tics_current; 2302 } 2303 } 2304 #endif 2305 #ifdef RTE_LIB_LATENCYSTATS 2306 if (latencystats_enabled != 0 && 2307 latencystats_lcore_id == rte_lcore_id()) 2308 rte_latencystats_update(); 2309 #endif 2310 2311 } while (! fc->stopped); 2312 } 2313 2314 static int 2315 start_pkt_forward_on_core(void *fwd_arg) 2316 { 2317 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2318 cur_fwd_config.fwd_eng->packet_fwd); 2319 return 0; 2320 } 2321 2322 /* 2323 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2324 * Used to start communication flows in network loopback test configurations. 2325 */ 2326 static int 2327 run_one_txonly_burst_on_core(void *fwd_arg) 2328 { 2329 struct fwd_lcore *fwd_lc; 2330 struct fwd_lcore tmp_lcore; 2331 2332 fwd_lc = (struct fwd_lcore *) fwd_arg; 2333 tmp_lcore = *fwd_lc; 2334 tmp_lcore.stopped = 1; 2335 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2336 return 0; 2337 } 2338 2339 /* 2340 * Launch packet forwarding: 2341 * - Setup per-port forwarding context. 2342 * - launch logical cores with their forwarding configuration. 2343 */ 2344 static void 2345 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2346 { 2347 unsigned int i; 2348 unsigned int lc_id; 2349 int diag; 2350 2351 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2352 lc_id = fwd_lcores_cpuids[i]; 2353 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2354 fwd_lcores[i]->stopped = 0; 2355 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2356 fwd_lcores[i], lc_id); 2357 if (diag != 0) 2358 fprintf(stderr, 2359 "launch lcore %u failed - diag=%d\n", 2360 lc_id, diag); 2361 } 2362 } 2363 } 2364 2365 /* 2366 * Launch packet forwarding configuration. 2367 */ 2368 void 2369 start_packet_forwarding(int with_tx_first) 2370 { 2371 port_fwd_begin_t port_fwd_begin; 2372 port_fwd_end_t port_fwd_end; 2373 stream_init_t stream_init = cur_fwd_eng->stream_init; 2374 unsigned int i; 2375 2376 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2377 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2378 2379 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2380 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2381 2382 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2383 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2384 (!nb_rxq || !nb_txq)) 2385 rte_exit(EXIT_FAILURE, 2386 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2387 cur_fwd_eng->fwd_mode_name); 2388 2389 if (all_ports_started() == 0) { 2390 fprintf(stderr, "Not all ports were started\n"); 2391 return; 2392 } 2393 if (test_done == 0) { 2394 fprintf(stderr, "Packet forwarding already started\n"); 2395 return; 2396 } 2397 2398 fwd_config_setup(); 2399 2400 pkt_fwd_config_display(&cur_fwd_config); 2401 if (!pkt_fwd_shared_rxq_check()) 2402 return; 2403 2404 if (stream_init != NULL) 2405 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2406 stream_init(fwd_streams[i]); 2407 2408 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2409 if (port_fwd_begin != NULL) { 2410 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2411 if (port_fwd_begin(fwd_ports_ids[i])) { 2412 fprintf(stderr, 2413 "Packet forwarding is not ready\n"); 2414 return; 2415 } 2416 } 2417 } 2418 2419 if (with_tx_first) { 2420 port_fwd_begin = tx_only_engine.port_fwd_begin; 2421 if (port_fwd_begin != NULL) { 2422 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2423 if (port_fwd_begin(fwd_ports_ids[i])) { 2424 fprintf(stderr, 2425 "Packet forwarding is not ready\n"); 2426 return; 2427 } 2428 } 2429 } 2430 } 2431 2432 test_done = 0; 2433 2434 if(!no_flush_rx) 2435 flush_fwd_rx_queues(); 2436 2437 rxtx_config_display(); 2438 2439 fwd_stats_reset(); 2440 if (with_tx_first) { 2441 while (with_tx_first--) { 2442 launch_packet_forwarding( 2443 run_one_txonly_burst_on_core); 2444 rte_eal_mp_wait_lcore(); 2445 } 2446 port_fwd_end = tx_only_engine.port_fwd_end; 2447 if (port_fwd_end != NULL) { 2448 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2449 (*port_fwd_end)(fwd_ports_ids[i]); 2450 } 2451 } 2452 launch_packet_forwarding(start_pkt_forward_on_core); 2453 } 2454 2455 void 2456 stop_packet_forwarding(void) 2457 { 2458 port_fwd_end_t port_fwd_end; 2459 lcoreid_t lc_id; 2460 portid_t pt_id; 2461 int i; 2462 2463 if (test_done) { 2464 fprintf(stderr, "Packet forwarding not started\n"); 2465 return; 2466 } 2467 printf("Telling cores to stop..."); 2468 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2469 fwd_lcores[lc_id]->stopped = 1; 2470 printf("\nWaiting for lcores to finish...\n"); 2471 rte_eal_mp_wait_lcore(); 2472 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2473 if (port_fwd_end != NULL) { 2474 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2475 pt_id = fwd_ports_ids[i]; 2476 (*port_fwd_end)(pt_id); 2477 } 2478 } 2479 2480 fwd_stats_display(); 2481 2482 printf("\nDone.\n"); 2483 test_done = 1; 2484 } 2485 2486 void 2487 dev_set_link_up(portid_t pid) 2488 { 2489 if (rte_eth_dev_set_link_up(pid) < 0) 2490 fprintf(stderr, "\nSet link up fail.\n"); 2491 } 2492 2493 void 2494 dev_set_link_down(portid_t pid) 2495 { 2496 if (rte_eth_dev_set_link_down(pid) < 0) 2497 fprintf(stderr, "\nSet link down fail.\n"); 2498 } 2499 2500 static int 2501 all_ports_started(void) 2502 { 2503 portid_t pi; 2504 struct rte_port *port; 2505 2506 RTE_ETH_FOREACH_DEV(pi) { 2507 port = &ports[pi]; 2508 /* Check if there is a port which is not started */ 2509 if ((port->port_status != RTE_PORT_STARTED) && 2510 (port->slave_flag == 0)) 2511 return 0; 2512 } 2513 2514 /* No port is not started */ 2515 return 1; 2516 } 2517 2518 int 2519 port_is_stopped(portid_t port_id) 2520 { 2521 struct rte_port *port = &ports[port_id]; 2522 2523 if ((port->port_status != RTE_PORT_STOPPED) && 2524 (port->slave_flag == 0)) 2525 return 0; 2526 return 1; 2527 } 2528 2529 int 2530 all_ports_stopped(void) 2531 { 2532 portid_t pi; 2533 2534 RTE_ETH_FOREACH_DEV(pi) { 2535 if (!port_is_stopped(pi)) 2536 return 0; 2537 } 2538 2539 return 1; 2540 } 2541 2542 int 2543 port_is_started(portid_t port_id) 2544 { 2545 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2546 return 0; 2547 2548 if (ports[port_id].port_status != RTE_PORT_STARTED) 2549 return 0; 2550 2551 return 1; 2552 } 2553 2554 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2555 static int 2556 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2557 { 2558 queueid_t qi; 2559 struct rte_eth_hairpin_conf hairpin_conf = { 2560 .peer_count = 1, 2561 }; 2562 int i; 2563 int diag; 2564 struct rte_port *port = &ports[pi]; 2565 uint16_t peer_rx_port = pi; 2566 uint16_t peer_tx_port = pi; 2567 uint32_t manual = 1; 2568 uint32_t tx_exp = hairpin_mode & 0x10; 2569 2570 if (!(hairpin_mode & 0xf)) { 2571 peer_rx_port = pi; 2572 peer_tx_port = pi; 2573 manual = 0; 2574 } else if (hairpin_mode & 0x1) { 2575 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2576 RTE_ETH_DEV_NO_OWNER); 2577 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2578 peer_tx_port = rte_eth_find_next_owned_by(0, 2579 RTE_ETH_DEV_NO_OWNER); 2580 if (p_pi != RTE_MAX_ETHPORTS) { 2581 peer_rx_port = p_pi; 2582 } else { 2583 uint16_t next_pi; 2584 2585 /* Last port will be the peer RX port of the first. */ 2586 RTE_ETH_FOREACH_DEV(next_pi) 2587 peer_rx_port = next_pi; 2588 } 2589 manual = 1; 2590 } else if (hairpin_mode & 0x2) { 2591 if (cnt_pi & 0x1) { 2592 peer_rx_port = p_pi; 2593 } else { 2594 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2595 RTE_ETH_DEV_NO_OWNER); 2596 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2597 peer_rx_port = pi; 2598 } 2599 peer_tx_port = peer_rx_port; 2600 manual = 1; 2601 } 2602 2603 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2604 hairpin_conf.peers[0].port = peer_rx_port; 2605 hairpin_conf.peers[0].queue = i + nb_rxq; 2606 hairpin_conf.manual_bind = !!manual; 2607 hairpin_conf.tx_explicit = !!tx_exp; 2608 diag = rte_eth_tx_hairpin_queue_setup 2609 (pi, qi, nb_txd, &hairpin_conf); 2610 i++; 2611 if (diag == 0) 2612 continue; 2613 2614 /* Fail to setup rx queue, return */ 2615 if (port->port_status == RTE_PORT_HANDLING) 2616 port->port_status = RTE_PORT_STOPPED; 2617 else 2618 fprintf(stderr, 2619 "Port %d can not be set back to stopped\n", pi); 2620 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2621 pi); 2622 /* try to reconfigure queues next time */ 2623 port->need_reconfig_queues = 1; 2624 return -1; 2625 } 2626 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2627 hairpin_conf.peers[0].port = peer_tx_port; 2628 hairpin_conf.peers[0].queue = i + nb_txq; 2629 hairpin_conf.manual_bind = !!manual; 2630 hairpin_conf.tx_explicit = !!tx_exp; 2631 diag = rte_eth_rx_hairpin_queue_setup 2632 (pi, qi, nb_rxd, &hairpin_conf); 2633 i++; 2634 if (diag == 0) 2635 continue; 2636 2637 /* Fail to setup rx queue, return */ 2638 if (port->port_status == RTE_PORT_HANDLING) 2639 port->port_status = RTE_PORT_STOPPED; 2640 else 2641 fprintf(stderr, 2642 "Port %d can not be set back to stopped\n", pi); 2643 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2644 pi); 2645 /* try to reconfigure queues next time */ 2646 port->need_reconfig_queues = 1; 2647 return -1; 2648 } 2649 return 0; 2650 } 2651 2652 /* Configure the Rx with optional split. */ 2653 int 2654 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2655 uint16_t nb_rx_desc, unsigned int socket_id, 2656 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2657 { 2658 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2659 unsigned int i, mp_n; 2660 int ret; 2661 2662 if (rx_pkt_nb_segs <= 1 || 2663 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2664 rx_conf->rx_seg = NULL; 2665 rx_conf->rx_nseg = 0; 2666 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2667 nb_rx_desc, socket_id, 2668 rx_conf, mp); 2669 goto exit; 2670 } 2671 for (i = 0; i < rx_pkt_nb_segs; i++) { 2672 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2673 struct rte_mempool *mpx; 2674 /* 2675 * Use last valid pool for the segments with number 2676 * exceeding the pool index. 2677 */ 2678 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2679 mpx = mbuf_pool_find(socket_id, mp_n); 2680 /* Handle zero as mbuf data buffer size. */ 2681 rx_seg->length = rx_pkt_seg_lengths[i] ? 2682 rx_pkt_seg_lengths[i] : 2683 mbuf_data_size[mp_n]; 2684 rx_seg->offset = i < rx_pkt_nb_offs ? 2685 rx_pkt_seg_offsets[i] : 0; 2686 rx_seg->mp = mpx ? mpx : mp; 2687 } 2688 rx_conf->rx_nseg = rx_pkt_nb_segs; 2689 rx_conf->rx_seg = rx_useg; 2690 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2691 socket_id, rx_conf, NULL); 2692 rx_conf->rx_seg = NULL; 2693 rx_conf->rx_nseg = 0; 2694 exit: 2695 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2696 RTE_ETH_QUEUE_STATE_STOPPED : 2697 RTE_ETH_QUEUE_STATE_STARTED; 2698 return ret; 2699 } 2700 2701 static int 2702 alloc_xstats_display_info(portid_t pi) 2703 { 2704 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2705 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2706 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2707 2708 if (xstats_display_num == 0) 2709 return 0; 2710 2711 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2712 if (*ids_supp == NULL) 2713 goto fail_ids_supp; 2714 2715 *prev_values = calloc(xstats_display_num, 2716 sizeof(**prev_values)); 2717 if (*prev_values == NULL) 2718 goto fail_prev_values; 2719 2720 *curr_values = calloc(xstats_display_num, 2721 sizeof(**curr_values)); 2722 if (*curr_values == NULL) 2723 goto fail_curr_values; 2724 2725 ports[pi].xstats_info.allocated = true; 2726 2727 return 0; 2728 2729 fail_curr_values: 2730 free(*prev_values); 2731 fail_prev_values: 2732 free(*ids_supp); 2733 fail_ids_supp: 2734 return -ENOMEM; 2735 } 2736 2737 static void 2738 free_xstats_display_info(portid_t pi) 2739 { 2740 if (!ports[pi].xstats_info.allocated) 2741 return; 2742 free(ports[pi].xstats_info.ids_supp); 2743 free(ports[pi].xstats_info.prev_values); 2744 free(ports[pi].xstats_info.curr_values); 2745 ports[pi].xstats_info.allocated = false; 2746 } 2747 2748 /** Fill helper structures for specified port to show extended statistics. */ 2749 static void 2750 fill_xstats_display_info_for_port(portid_t pi) 2751 { 2752 unsigned int stat, stat_supp; 2753 const char *xstat_name; 2754 struct rte_port *port; 2755 uint64_t *ids_supp; 2756 int rc; 2757 2758 if (xstats_display_num == 0) 2759 return; 2760 2761 if (pi == (portid_t)RTE_PORT_ALL) { 2762 fill_xstats_display_info(); 2763 return; 2764 } 2765 2766 port = &ports[pi]; 2767 if (port->port_status != RTE_PORT_STARTED) 2768 return; 2769 2770 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2771 rte_exit(EXIT_FAILURE, 2772 "Failed to allocate xstats display memory\n"); 2773 2774 ids_supp = port->xstats_info.ids_supp; 2775 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2776 xstat_name = xstats_display[stat].name; 2777 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2778 ids_supp + stat_supp); 2779 if (rc != 0) { 2780 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2781 xstat_name, pi, stat); 2782 continue; 2783 } 2784 stat_supp++; 2785 } 2786 2787 port->xstats_info.ids_supp_sz = stat_supp; 2788 } 2789 2790 /** Fill helper structures for all ports to show extended statistics. */ 2791 static void 2792 fill_xstats_display_info(void) 2793 { 2794 portid_t pi; 2795 2796 if (xstats_display_num == 0) 2797 return; 2798 2799 RTE_ETH_FOREACH_DEV(pi) 2800 fill_xstats_display_info_for_port(pi); 2801 } 2802 2803 int 2804 start_port(portid_t pid) 2805 { 2806 int diag, need_check_link_status = -1; 2807 portid_t pi; 2808 portid_t p_pi = RTE_MAX_ETHPORTS; 2809 portid_t pl[RTE_MAX_ETHPORTS]; 2810 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2811 uint16_t cnt_pi = 0; 2812 uint16_t cfg_pi = 0; 2813 int peer_pi; 2814 queueid_t qi; 2815 struct rte_port *port; 2816 struct rte_eth_hairpin_cap cap; 2817 2818 if (port_id_is_invalid(pid, ENABLED_WARN)) 2819 return 0; 2820 2821 RTE_ETH_FOREACH_DEV(pi) { 2822 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2823 continue; 2824 2825 if (port_is_bonding_slave(pi)) { 2826 fprintf(stderr, 2827 "Please remove port %d from bonded device.\n", 2828 pi); 2829 continue; 2830 } 2831 2832 need_check_link_status = 0; 2833 port = &ports[pi]; 2834 if (port->port_status == RTE_PORT_STOPPED) 2835 port->port_status = RTE_PORT_HANDLING; 2836 else { 2837 fprintf(stderr, "Port %d is now not stopped\n", pi); 2838 continue; 2839 } 2840 2841 if (port->need_reconfig > 0) { 2842 struct rte_eth_conf dev_conf; 2843 int k; 2844 2845 port->need_reconfig = 0; 2846 2847 if (flow_isolate_all) { 2848 int ret = port_flow_isolate(pi, 1); 2849 if (ret) { 2850 fprintf(stderr, 2851 "Failed to apply isolated mode on port %d\n", 2852 pi); 2853 return -1; 2854 } 2855 } 2856 configure_rxtx_dump_callbacks(0); 2857 printf("Configuring Port %d (socket %u)\n", pi, 2858 port->socket_id); 2859 if (nb_hairpinq > 0 && 2860 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2861 fprintf(stderr, 2862 "Port %d doesn't support hairpin queues\n", 2863 pi); 2864 return -1; 2865 } 2866 2867 /* configure port */ 2868 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2869 nb_txq + nb_hairpinq, 2870 &(port->dev_conf)); 2871 if (diag != 0) { 2872 if (port->port_status == RTE_PORT_HANDLING) 2873 port->port_status = RTE_PORT_STOPPED; 2874 else 2875 fprintf(stderr, 2876 "Port %d can not be set back to stopped\n", 2877 pi); 2878 fprintf(stderr, "Fail to configure port %d\n", 2879 pi); 2880 /* try to reconfigure port next time */ 2881 port->need_reconfig = 1; 2882 return -1; 2883 } 2884 /* get device configuration*/ 2885 if (0 != 2886 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2887 fprintf(stderr, 2888 "port %d can not get device configuration\n", 2889 pi); 2890 return -1; 2891 } 2892 /* Apply Rx offloads configuration */ 2893 if (dev_conf.rxmode.offloads != 2894 port->dev_conf.rxmode.offloads) { 2895 port->dev_conf.rxmode.offloads |= 2896 dev_conf.rxmode.offloads; 2897 for (k = 0; 2898 k < port->dev_info.max_rx_queues; 2899 k++) 2900 port->rxq[k].conf.offloads |= 2901 dev_conf.rxmode.offloads; 2902 } 2903 /* Apply Tx offloads configuration */ 2904 if (dev_conf.txmode.offloads != 2905 port->dev_conf.txmode.offloads) { 2906 port->dev_conf.txmode.offloads |= 2907 dev_conf.txmode.offloads; 2908 for (k = 0; 2909 k < port->dev_info.max_tx_queues; 2910 k++) 2911 port->txq[k].conf.offloads |= 2912 dev_conf.txmode.offloads; 2913 } 2914 } 2915 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2916 port->need_reconfig_queues = 0; 2917 /* setup tx queues */ 2918 for (qi = 0; qi < nb_txq; qi++) { 2919 struct rte_eth_txconf *conf = 2920 &port->txq[qi].conf; 2921 2922 if ((numa_support) && 2923 (txring_numa[pi] != NUMA_NO_CONFIG)) 2924 diag = rte_eth_tx_queue_setup(pi, qi, 2925 port->nb_tx_desc[qi], 2926 txring_numa[pi], 2927 &(port->txq[qi].conf)); 2928 else 2929 diag = rte_eth_tx_queue_setup(pi, qi, 2930 port->nb_tx_desc[qi], 2931 port->socket_id, 2932 &(port->txq[qi].conf)); 2933 2934 if (diag == 0) { 2935 port->txq[qi].state = 2936 conf->tx_deferred_start ? 2937 RTE_ETH_QUEUE_STATE_STOPPED : 2938 RTE_ETH_QUEUE_STATE_STARTED; 2939 continue; 2940 } 2941 2942 /* Fail to setup tx queue, return */ 2943 if (port->port_status == RTE_PORT_HANDLING) 2944 port->port_status = RTE_PORT_STOPPED; 2945 else 2946 fprintf(stderr, 2947 "Port %d can not be set back to stopped\n", 2948 pi); 2949 fprintf(stderr, 2950 "Fail to configure port %d tx queues\n", 2951 pi); 2952 /* try to reconfigure queues next time */ 2953 port->need_reconfig_queues = 1; 2954 return -1; 2955 } 2956 for (qi = 0; qi < nb_rxq; qi++) { 2957 /* setup rx queues */ 2958 if ((numa_support) && 2959 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2960 struct rte_mempool * mp = 2961 mbuf_pool_find 2962 (rxring_numa[pi], 0); 2963 if (mp == NULL) { 2964 fprintf(stderr, 2965 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2966 rxring_numa[pi]); 2967 return -1; 2968 } 2969 2970 diag = rx_queue_setup(pi, qi, 2971 port->nb_rx_desc[qi], 2972 rxring_numa[pi], 2973 &(port->rxq[qi].conf), 2974 mp); 2975 } else { 2976 struct rte_mempool *mp = 2977 mbuf_pool_find 2978 (port->socket_id, 0); 2979 if (mp == NULL) { 2980 fprintf(stderr, 2981 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2982 port->socket_id); 2983 return -1; 2984 } 2985 diag = rx_queue_setup(pi, qi, 2986 port->nb_rx_desc[qi], 2987 port->socket_id, 2988 &(port->rxq[qi].conf), 2989 mp); 2990 } 2991 if (diag == 0) 2992 continue; 2993 2994 /* Fail to setup rx queue, return */ 2995 if (port->port_status == RTE_PORT_HANDLING) 2996 port->port_status = RTE_PORT_STOPPED; 2997 else 2998 fprintf(stderr, 2999 "Port %d can not be set back to stopped\n", 3000 pi); 3001 fprintf(stderr, 3002 "Fail to configure port %d rx queues\n", 3003 pi); 3004 /* try to reconfigure queues next time */ 3005 port->need_reconfig_queues = 1; 3006 return -1; 3007 } 3008 /* setup hairpin queues */ 3009 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3010 return -1; 3011 } 3012 configure_rxtx_dump_callbacks(verbose_level); 3013 if (clear_ptypes) { 3014 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3015 NULL, 0); 3016 if (diag < 0) 3017 fprintf(stderr, 3018 "Port %d: Failed to disable Ptype parsing\n", 3019 pi); 3020 } 3021 3022 p_pi = pi; 3023 cnt_pi++; 3024 3025 /* start port */ 3026 diag = eth_dev_start_mp(pi); 3027 if (diag < 0) { 3028 fprintf(stderr, "Fail to start port %d: %s\n", 3029 pi, rte_strerror(-diag)); 3030 3031 /* Fail to setup rx queue, return */ 3032 if (port->port_status == RTE_PORT_HANDLING) 3033 port->port_status = RTE_PORT_STOPPED; 3034 else 3035 fprintf(stderr, 3036 "Port %d can not be set back to stopped\n", 3037 pi); 3038 continue; 3039 } 3040 3041 if (port->port_status == RTE_PORT_HANDLING) 3042 port->port_status = RTE_PORT_STARTED; 3043 else 3044 fprintf(stderr, "Port %d can not be set into started\n", 3045 pi); 3046 3047 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3048 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3049 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3050 3051 /* at least one port started, need checking link status */ 3052 need_check_link_status = 1; 3053 3054 pl[cfg_pi++] = pi; 3055 } 3056 3057 if (need_check_link_status == 1 && !no_link_check) 3058 check_all_ports_link_status(RTE_PORT_ALL); 3059 else if (need_check_link_status == 0) 3060 fprintf(stderr, "Please stop the ports first\n"); 3061 3062 if (hairpin_mode & 0xf) { 3063 uint16_t i; 3064 int j; 3065 3066 /* bind all started hairpin ports */ 3067 for (i = 0; i < cfg_pi; i++) { 3068 pi = pl[i]; 3069 /* bind current Tx to all peer Rx */ 3070 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3071 RTE_MAX_ETHPORTS, 1); 3072 if (peer_pi < 0) 3073 return peer_pi; 3074 for (j = 0; j < peer_pi; j++) { 3075 if (!port_is_started(peer_pl[j])) 3076 continue; 3077 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3078 if (diag < 0) { 3079 fprintf(stderr, 3080 "Error during binding hairpin Tx port %u to %u: %s\n", 3081 pi, peer_pl[j], 3082 rte_strerror(-diag)); 3083 return -1; 3084 } 3085 } 3086 /* bind all peer Tx to current Rx */ 3087 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3088 RTE_MAX_ETHPORTS, 0); 3089 if (peer_pi < 0) 3090 return peer_pi; 3091 for (j = 0; j < peer_pi; j++) { 3092 if (!port_is_started(peer_pl[j])) 3093 continue; 3094 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3095 if (diag < 0) { 3096 fprintf(stderr, 3097 "Error during binding hairpin Tx port %u to %u: %s\n", 3098 peer_pl[j], pi, 3099 rte_strerror(-diag)); 3100 return -1; 3101 } 3102 } 3103 } 3104 } 3105 3106 fill_xstats_display_info_for_port(pid); 3107 3108 printf("Done\n"); 3109 return 0; 3110 } 3111 3112 void 3113 stop_port(portid_t pid) 3114 { 3115 portid_t pi; 3116 struct rte_port *port; 3117 int need_check_link_status = 0; 3118 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3119 int peer_pi; 3120 3121 if (port_id_is_invalid(pid, ENABLED_WARN)) 3122 return; 3123 3124 printf("Stopping ports...\n"); 3125 3126 RTE_ETH_FOREACH_DEV(pi) { 3127 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3128 continue; 3129 3130 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3131 fprintf(stderr, 3132 "Please remove port %d from forwarding configuration.\n", 3133 pi); 3134 continue; 3135 } 3136 3137 if (port_is_bonding_slave(pi)) { 3138 fprintf(stderr, 3139 "Please remove port %d from bonded device.\n", 3140 pi); 3141 continue; 3142 } 3143 3144 port = &ports[pi]; 3145 if (port->port_status == RTE_PORT_STARTED) 3146 port->port_status = RTE_PORT_HANDLING; 3147 else 3148 continue; 3149 3150 if (hairpin_mode & 0xf) { 3151 int j; 3152 3153 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3154 /* unbind all peer Tx from current Rx */ 3155 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3156 RTE_MAX_ETHPORTS, 0); 3157 if (peer_pi < 0) 3158 continue; 3159 for (j = 0; j < peer_pi; j++) { 3160 if (!port_is_started(peer_pl[j])) 3161 continue; 3162 rte_eth_hairpin_unbind(peer_pl[j], pi); 3163 } 3164 } 3165 3166 if (port->flow_list) 3167 port_flow_flush(pi); 3168 3169 if (eth_dev_stop_mp(pi) != 0) 3170 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3171 pi); 3172 3173 if (port->port_status == RTE_PORT_HANDLING) 3174 port->port_status = RTE_PORT_STOPPED; 3175 else 3176 fprintf(stderr, "Port %d can not be set into stopped\n", 3177 pi); 3178 need_check_link_status = 1; 3179 } 3180 if (need_check_link_status && !no_link_check) 3181 check_all_ports_link_status(RTE_PORT_ALL); 3182 3183 printf("Done\n"); 3184 } 3185 3186 static void 3187 remove_invalid_ports_in(portid_t *array, portid_t *total) 3188 { 3189 portid_t i; 3190 portid_t new_total = 0; 3191 3192 for (i = 0; i < *total; i++) 3193 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3194 array[new_total] = array[i]; 3195 new_total++; 3196 } 3197 *total = new_total; 3198 } 3199 3200 static void 3201 remove_invalid_ports(void) 3202 { 3203 remove_invalid_ports_in(ports_ids, &nb_ports); 3204 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3205 nb_cfg_ports = nb_fwd_ports; 3206 } 3207 3208 static void 3209 flush_port_owned_resources(portid_t pi) 3210 { 3211 mcast_addr_pool_destroy(pi); 3212 port_flow_flush(pi); 3213 port_flex_item_flush(pi); 3214 port_action_handle_flush(pi); 3215 } 3216 3217 static void 3218 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3219 { 3220 struct rte_port *port; 3221 portid_t slave_pid; 3222 uint16_t i; 3223 3224 for (i = 0; i < num_slaves; i++) { 3225 slave_pid = slave_pids[i]; 3226 if (port_is_started(slave_pid) == 1) { 3227 if (rte_eth_dev_stop(slave_pid) != 0) 3228 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3229 slave_pid); 3230 3231 port = &ports[slave_pid]; 3232 port->port_status = RTE_PORT_STOPPED; 3233 } 3234 3235 clear_port_slave_flag(slave_pid); 3236 3237 /* Close slave device when testpmd quit or is killed. */ 3238 if (cl_quit == 1 || f_quit == 1) 3239 rte_eth_dev_close(slave_pid); 3240 } 3241 } 3242 3243 void 3244 close_port(portid_t pid) 3245 { 3246 portid_t pi; 3247 struct rte_port *port; 3248 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3249 int num_slaves = 0; 3250 3251 if (port_id_is_invalid(pid, ENABLED_WARN)) 3252 return; 3253 3254 printf("Closing ports...\n"); 3255 3256 RTE_ETH_FOREACH_DEV(pi) { 3257 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3258 continue; 3259 3260 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3261 fprintf(stderr, 3262 "Please remove port %d from forwarding configuration.\n", 3263 pi); 3264 continue; 3265 } 3266 3267 if (port_is_bonding_slave(pi)) { 3268 fprintf(stderr, 3269 "Please remove port %d from bonded device.\n", 3270 pi); 3271 continue; 3272 } 3273 3274 port = &ports[pi]; 3275 if (port->port_status == RTE_PORT_CLOSED) { 3276 fprintf(stderr, "Port %d is already closed\n", pi); 3277 continue; 3278 } 3279 3280 if (is_proc_primary()) { 3281 flush_port_owned_resources(pi); 3282 #ifdef RTE_NET_BOND 3283 if (port->bond_flag == 1) 3284 num_slaves = rte_eth_bond_slaves_get(pi, 3285 slave_pids, RTE_MAX_ETHPORTS); 3286 #endif 3287 rte_eth_dev_close(pi); 3288 /* 3289 * If this port is bonded device, all slaves under the 3290 * device need to be removed or closed. 3291 */ 3292 if (port->bond_flag == 1 && num_slaves > 0) 3293 clear_bonding_slave_device(slave_pids, 3294 num_slaves); 3295 } 3296 3297 free_xstats_display_info(pi); 3298 } 3299 3300 remove_invalid_ports(); 3301 printf("Done\n"); 3302 } 3303 3304 void 3305 reset_port(portid_t pid) 3306 { 3307 int diag; 3308 portid_t pi; 3309 struct rte_port *port; 3310 3311 if (port_id_is_invalid(pid, ENABLED_WARN)) 3312 return; 3313 3314 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3315 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3316 fprintf(stderr, 3317 "Can not reset port(s), please stop port(s) first.\n"); 3318 return; 3319 } 3320 3321 printf("Resetting ports...\n"); 3322 3323 RTE_ETH_FOREACH_DEV(pi) { 3324 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3325 continue; 3326 3327 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3328 fprintf(stderr, 3329 "Please remove port %d from forwarding configuration.\n", 3330 pi); 3331 continue; 3332 } 3333 3334 if (port_is_bonding_slave(pi)) { 3335 fprintf(stderr, 3336 "Please remove port %d from bonded device.\n", 3337 pi); 3338 continue; 3339 } 3340 3341 diag = rte_eth_dev_reset(pi); 3342 if (diag == 0) { 3343 port = &ports[pi]; 3344 port->need_reconfig = 1; 3345 port->need_reconfig_queues = 1; 3346 } else { 3347 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3348 pi, diag); 3349 } 3350 } 3351 3352 printf("Done\n"); 3353 } 3354 3355 void 3356 attach_port(char *identifier) 3357 { 3358 portid_t pi; 3359 struct rte_dev_iterator iterator; 3360 3361 printf("Attaching a new port...\n"); 3362 3363 if (identifier == NULL) { 3364 fprintf(stderr, "Invalid parameters are specified\n"); 3365 return; 3366 } 3367 3368 if (rte_dev_probe(identifier) < 0) { 3369 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3370 return; 3371 } 3372 3373 /* first attach mode: event */ 3374 if (setup_on_probe_event) { 3375 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3376 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3377 if (ports[pi].port_status == RTE_PORT_HANDLING && 3378 ports[pi].need_setup != 0) 3379 setup_attached_port(pi); 3380 return; 3381 } 3382 3383 /* second attach mode: iterator */ 3384 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3385 /* setup ports matching the devargs used for probing */ 3386 if (port_is_forwarding(pi)) 3387 continue; /* port was already attached before */ 3388 setup_attached_port(pi); 3389 } 3390 } 3391 3392 static void 3393 setup_attached_port(portid_t pi) 3394 { 3395 unsigned int socket_id; 3396 int ret; 3397 3398 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3399 /* if socket_id is invalid, set to the first available socket. */ 3400 if (check_socket_id(socket_id) < 0) 3401 socket_id = socket_ids[0]; 3402 reconfig(pi, socket_id); 3403 ret = rte_eth_promiscuous_enable(pi); 3404 if (ret != 0) 3405 fprintf(stderr, 3406 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3407 pi, rte_strerror(-ret)); 3408 3409 ports_ids[nb_ports++] = pi; 3410 fwd_ports_ids[nb_fwd_ports++] = pi; 3411 nb_cfg_ports = nb_fwd_ports; 3412 ports[pi].need_setup = 0; 3413 ports[pi].port_status = RTE_PORT_STOPPED; 3414 3415 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3416 printf("Done\n"); 3417 } 3418 3419 static void 3420 detach_device(struct rte_device *dev) 3421 { 3422 portid_t sibling; 3423 3424 if (dev == NULL) { 3425 fprintf(stderr, "Device already removed\n"); 3426 return; 3427 } 3428 3429 printf("Removing a device...\n"); 3430 3431 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3432 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3433 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3434 fprintf(stderr, "Port %u not stopped\n", 3435 sibling); 3436 return; 3437 } 3438 flush_port_owned_resources(sibling); 3439 } 3440 } 3441 3442 if (rte_dev_remove(dev) < 0) { 3443 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3444 return; 3445 } 3446 remove_invalid_ports(); 3447 3448 printf("Device is detached\n"); 3449 printf("Now total ports is %d\n", nb_ports); 3450 printf("Done\n"); 3451 return; 3452 } 3453 3454 void 3455 detach_port_device(portid_t port_id) 3456 { 3457 int ret; 3458 struct rte_eth_dev_info dev_info; 3459 3460 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3461 return; 3462 3463 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3464 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3465 fprintf(stderr, "Port not stopped\n"); 3466 return; 3467 } 3468 fprintf(stderr, "Port was not closed\n"); 3469 } 3470 3471 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3472 if (ret != 0) { 3473 TESTPMD_LOG(ERR, 3474 "Failed to get device info for port %d, not detaching\n", 3475 port_id); 3476 return; 3477 } 3478 detach_device(dev_info.device); 3479 } 3480 3481 void 3482 detach_devargs(char *identifier) 3483 { 3484 struct rte_dev_iterator iterator; 3485 struct rte_devargs da; 3486 portid_t port_id; 3487 3488 printf("Removing a device...\n"); 3489 3490 memset(&da, 0, sizeof(da)); 3491 if (rte_devargs_parsef(&da, "%s", identifier)) { 3492 fprintf(stderr, "cannot parse identifier\n"); 3493 return; 3494 } 3495 3496 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3497 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3498 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3499 fprintf(stderr, "Port %u not stopped\n", 3500 port_id); 3501 rte_eth_iterator_cleanup(&iterator); 3502 rte_devargs_reset(&da); 3503 return; 3504 } 3505 flush_port_owned_resources(port_id); 3506 } 3507 } 3508 3509 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3510 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3511 da.name, da.bus->name); 3512 rte_devargs_reset(&da); 3513 return; 3514 } 3515 3516 remove_invalid_ports(); 3517 3518 printf("Device %s is detached\n", identifier); 3519 printf("Now total ports is %d\n", nb_ports); 3520 printf("Done\n"); 3521 rte_devargs_reset(&da); 3522 } 3523 3524 void 3525 pmd_test_exit(void) 3526 { 3527 portid_t pt_id; 3528 unsigned int i; 3529 int ret; 3530 3531 if (test_done == 0) 3532 stop_packet_forwarding(); 3533 3534 #ifndef RTE_EXEC_ENV_WINDOWS 3535 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3536 if (mempools[i]) { 3537 if (mp_alloc_type == MP_ALLOC_ANON) 3538 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3539 NULL); 3540 } 3541 } 3542 #endif 3543 if (ports != NULL) { 3544 no_link_check = 1; 3545 RTE_ETH_FOREACH_DEV(pt_id) { 3546 printf("\nStopping port %d...\n", pt_id); 3547 fflush(stdout); 3548 stop_port(pt_id); 3549 } 3550 RTE_ETH_FOREACH_DEV(pt_id) { 3551 printf("\nShutting down port %d...\n", pt_id); 3552 fflush(stdout); 3553 close_port(pt_id); 3554 } 3555 } 3556 3557 if (hot_plug) { 3558 ret = rte_dev_event_monitor_stop(); 3559 if (ret) { 3560 RTE_LOG(ERR, EAL, 3561 "fail to stop device event monitor."); 3562 return; 3563 } 3564 3565 ret = rte_dev_event_callback_unregister(NULL, 3566 dev_event_callback, NULL); 3567 if (ret < 0) { 3568 RTE_LOG(ERR, EAL, 3569 "fail to unregister device event callback.\n"); 3570 return; 3571 } 3572 3573 ret = rte_dev_hotplug_handle_disable(); 3574 if (ret) { 3575 RTE_LOG(ERR, EAL, 3576 "fail to disable hotplug handling.\n"); 3577 return; 3578 } 3579 } 3580 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3581 if (mempools[i]) 3582 mempool_free_mp(mempools[i]); 3583 } 3584 free(xstats_display); 3585 3586 printf("\nBye...\n"); 3587 } 3588 3589 typedef void (*cmd_func_t)(void); 3590 struct pmd_test_command { 3591 const char *cmd_name; 3592 cmd_func_t cmd_func; 3593 }; 3594 3595 /* Check the link status of all ports in up to 9s, and print them finally */ 3596 static void 3597 check_all_ports_link_status(uint32_t port_mask) 3598 { 3599 #define CHECK_INTERVAL 100 /* 100ms */ 3600 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3601 portid_t portid; 3602 uint8_t count, all_ports_up, print_flag = 0; 3603 struct rte_eth_link link; 3604 int ret; 3605 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3606 3607 printf("Checking link statuses...\n"); 3608 fflush(stdout); 3609 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3610 all_ports_up = 1; 3611 RTE_ETH_FOREACH_DEV(portid) { 3612 if ((port_mask & (1 << portid)) == 0) 3613 continue; 3614 memset(&link, 0, sizeof(link)); 3615 ret = rte_eth_link_get_nowait(portid, &link); 3616 if (ret < 0) { 3617 all_ports_up = 0; 3618 if (print_flag == 1) 3619 fprintf(stderr, 3620 "Port %u link get failed: %s\n", 3621 portid, rte_strerror(-ret)); 3622 continue; 3623 } 3624 /* print link status if flag set */ 3625 if (print_flag == 1) { 3626 rte_eth_link_to_str(link_status, 3627 sizeof(link_status), &link); 3628 printf("Port %d %s\n", portid, link_status); 3629 continue; 3630 } 3631 /* clear all_ports_up flag if any link down */ 3632 if (link.link_status == RTE_ETH_LINK_DOWN) { 3633 all_ports_up = 0; 3634 break; 3635 } 3636 } 3637 /* after finally printing all link status, get out */ 3638 if (print_flag == 1) 3639 break; 3640 3641 if (all_ports_up == 0) { 3642 fflush(stdout); 3643 rte_delay_ms(CHECK_INTERVAL); 3644 } 3645 3646 /* set the print_flag if all ports up or timeout */ 3647 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3648 print_flag = 1; 3649 } 3650 3651 if (lsc_interrupt) 3652 break; 3653 } 3654 } 3655 3656 static void 3657 rmv_port_callback(void *arg) 3658 { 3659 int need_to_start = 0; 3660 int org_no_link_check = no_link_check; 3661 portid_t port_id = (intptr_t)arg; 3662 struct rte_eth_dev_info dev_info; 3663 int ret; 3664 3665 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3666 3667 if (!test_done && port_is_forwarding(port_id)) { 3668 need_to_start = 1; 3669 stop_packet_forwarding(); 3670 } 3671 no_link_check = 1; 3672 stop_port(port_id); 3673 no_link_check = org_no_link_check; 3674 3675 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3676 if (ret != 0) 3677 TESTPMD_LOG(ERR, 3678 "Failed to get device info for port %d, not detaching\n", 3679 port_id); 3680 else { 3681 struct rte_device *device = dev_info.device; 3682 close_port(port_id); 3683 detach_device(device); /* might be already removed or have more ports */ 3684 } 3685 if (need_to_start) 3686 start_packet_forwarding(0); 3687 } 3688 3689 /* This function is used by the interrupt thread */ 3690 static int 3691 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3692 void *ret_param) 3693 { 3694 RTE_SET_USED(param); 3695 RTE_SET_USED(ret_param); 3696 3697 if (type >= RTE_ETH_EVENT_MAX) { 3698 fprintf(stderr, 3699 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3700 port_id, __func__, type); 3701 fflush(stderr); 3702 } else if (event_print_mask & (UINT32_C(1) << type)) { 3703 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3704 eth_event_desc[type]); 3705 fflush(stdout); 3706 } 3707 3708 switch (type) { 3709 case RTE_ETH_EVENT_NEW: 3710 ports[port_id].need_setup = 1; 3711 ports[port_id].port_status = RTE_PORT_HANDLING; 3712 break; 3713 case RTE_ETH_EVENT_INTR_RMV: 3714 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3715 break; 3716 if (rte_eal_alarm_set(100000, 3717 rmv_port_callback, (void *)(intptr_t)port_id)) 3718 fprintf(stderr, 3719 "Could not set up deferred device removal\n"); 3720 break; 3721 case RTE_ETH_EVENT_DESTROY: 3722 ports[port_id].port_status = RTE_PORT_CLOSED; 3723 printf("Port %u is closed\n", port_id); 3724 break; 3725 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3726 uint16_t rxq_id; 3727 int ret; 3728 3729 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3730 for (rxq_id = 0; ; rxq_id++) { 3731 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3732 NULL); 3733 if (ret <= 0) 3734 break; 3735 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3736 port_id, rxq_id); 3737 3738 #ifdef RTE_NET_MLX5 3739 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3740 #endif 3741 } 3742 break; 3743 } 3744 default: 3745 break; 3746 } 3747 return 0; 3748 } 3749 3750 static int 3751 register_eth_event_callback(void) 3752 { 3753 int ret; 3754 enum rte_eth_event_type event; 3755 3756 for (event = RTE_ETH_EVENT_UNKNOWN; 3757 event < RTE_ETH_EVENT_MAX; event++) { 3758 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3759 event, 3760 eth_event_callback, 3761 NULL); 3762 if (ret != 0) { 3763 TESTPMD_LOG(ERR, "Failed to register callback for " 3764 "%s event\n", eth_event_desc[event]); 3765 return -1; 3766 } 3767 } 3768 3769 return 0; 3770 } 3771 3772 /* This function is used by the interrupt thread */ 3773 static void 3774 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3775 __rte_unused void *arg) 3776 { 3777 uint16_t port_id; 3778 int ret; 3779 3780 if (type >= RTE_DEV_EVENT_MAX) { 3781 fprintf(stderr, "%s called upon invalid event %d\n", 3782 __func__, type); 3783 fflush(stderr); 3784 } 3785 3786 switch (type) { 3787 case RTE_DEV_EVENT_REMOVE: 3788 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3789 device_name); 3790 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3791 if (ret) { 3792 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3793 device_name); 3794 return; 3795 } 3796 /* 3797 * Because the user's callback is invoked in eal interrupt 3798 * callback, the interrupt callback need to be finished before 3799 * it can be unregistered when detaching device. So finish 3800 * callback soon and use a deferred removal to detach device 3801 * is need. It is a workaround, once the device detaching be 3802 * moved into the eal in the future, the deferred removal could 3803 * be deleted. 3804 */ 3805 if (rte_eal_alarm_set(100000, 3806 rmv_port_callback, (void *)(intptr_t)port_id)) 3807 RTE_LOG(ERR, EAL, 3808 "Could not set up deferred device removal\n"); 3809 break; 3810 case RTE_DEV_EVENT_ADD: 3811 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3812 device_name); 3813 /* TODO: After finish kernel driver binding, 3814 * begin to attach port. 3815 */ 3816 break; 3817 default: 3818 break; 3819 } 3820 } 3821 3822 static void 3823 rxtx_port_config(portid_t pid) 3824 { 3825 uint16_t qid; 3826 uint64_t offloads; 3827 struct rte_port *port = &ports[pid]; 3828 3829 for (qid = 0; qid < nb_rxq; qid++) { 3830 offloads = port->rxq[qid].conf.offloads; 3831 port->rxq[qid].conf = port->dev_info.default_rxconf; 3832 3833 if (rxq_share > 0 && 3834 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3835 /* Non-zero share group to enable RxQ share. */ 3836 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3837 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3838 } 3839 3840 if (offloads != 0) 3841 port->rxq[qid].conf.offloads = offloads; 3842 3843 /* Check if any Rx parameters have been passed */ 3844 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3845 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3846 3847 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3848 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3849 3850 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3851 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3852 3853 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3854 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3855 3856 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3857 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3858 3859 port->nb_rx_desc[qid] = nb_rxd; 3860 } 3861 3862 for (qid = 0; qid < nb_txq; qid++) { 3863 offloads = port->txq[qid].conf.offloads; 3864 port->txq[qid].conf = port->dev_info.default_txconf; 3865 if (offloads != 0) 3866 port->txq[qid].conf.offloads = offloads; 3867 3868 /* Check if any Tx parameters have been passed */ 3869 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3870 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3871 3872 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3873 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3874 3875 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3876 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3877 3878 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3879 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3880 3881 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3882 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3883 3884 port->nb_tx_desc[qid] = nb_txd; 3885 } 3886 } 3887 3888 /* 3889 * Helper function to set MTU from frame size 3890 * 3891 * port->dev_info should be set before calling this function. 3892 * 3893 * return 0 on success, negative on error 3894 */ 3895 int 3896 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3897 { 3898 struct rte_port *port = &ports[portid]; 3899 uint32_t eth_overhead; 3900 uint16_t mtu, new_mtu; 3901 3902 eth_overhead = get_eth_overhead(&port->dev_info); 3903 3904 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3905 printf("Failed to get MTU for port %u\n", portid); 3906 return -1; 3907 } 3908 3909 new_mtu = max_rx_pktlen - eth_overhead; 3910 3911 if (mtu == new_mtu) 3912 return 0; 3913 3914 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3915 fprintf(stderr, 3916 "Failed to set MTU to %u for port %u\n", 3917 new_mtu, portid); 3918 return -1; 3919 } 3920 3921 port->dev_conf.rxmode.mtu = new_mtu; 3922 3923 return 0; 3924 } 3925 3926 void 3927 init_port_config(void) 3928 { 3929 portid_t pid; 3930 struct rte_port *port; 3931 int ret, i; 3932 3933 RTE_ETH_FOREACH_DEV(pid) { 3934 port = &ports[pid]; 3935 port->dev_conf.fdir_conf = fdir_conf; 3936 3937 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3938 if (ret != 0) 3939 return; 3940 3941 if (nb_rxq > 1) { 3942 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3943 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3944 rss_hf & port->dev_info.flow_type_rss_offloads; 3945 } else { 3946 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3947 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3948 } 3949 3950 if (port->dcb_flag == 0) { 3951 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3952 port->dev_conf.rxmode.mq_mode = 3953 (enum rte_eth_rx_mq_mode) 3954 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3955 } else { 3956 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3957 port->dev_conf.rxmode.offloads &= 3958 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3959 3960 for (i = 0; 3961 i < port->dev_info.nb_rx_queues; 3962 i++) 3963 port->rxq[i].conf.offloads &= 3964 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3965 } 3966 } 3967 3968 rxtx_port_config(pid); 3969 3970 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3971 if (ret != 0) 3972 return; 3973 3974 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3975 rte_pmd_ixgbe_bypass_init(pid); 3976 #endif 3977 3978 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3979 port->dev_conf.intr_conf.lsc = 1; 3980 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3981 port->dev_conf.intr_conf.rmv = 1; 3982 } 3983 } 3984 3985 void set_port_slave_flag(portid_t slave_pid) 3986 { 3987 struct rte_port *port; 3988 3989 port = &ports[slave_pid]; 3990 port->slave_flag = 1; 3991 } 3992 3993 void clear_port_slave_flag(portid_t slave_pid) 3994 { 3995 struct rte_port *port; 3996 3997 port = &ports[slave_pid]; 3998 port->slave_flag = 0; 3999 } 4000 4001 uint8_t port_is_bonding_slave(portid_t slave_pid) 4002 { 4003 struct rte_port *port; 4004 struct rte_eth_dev_info dev_info; 4005 int ret; 4006 4007 port = &ports[slave_pid]; 4008 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4009 if (ret != 0) { 4010 TESTPMD_LOG(ERR, 4011 "Failed to get device info for port id %d," 4012 "cannot determine if the port is a bonded slave", 4013 slave_pid); 4014 return 0; 4015 } 4016 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4017 return 1; 4018 return 0; 4019 } 4020 4021 const uint16_t vlan_tags[] = { 4022 0, 1, 2, 3, 4, 5, 6, 7, 4023 8, 9, 10, 11, 12, 13, 14, 15, 4024 16, 17, 18, 19, 20, 21, 22, 23, 4025 24, 25, 26, 27, 28, 29, 30, 31 4026 }; 4027 4028 static int 4029 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4030 enum dcb_mode_enable dcb_mode, 4031 enum rte_eth_nb_tcs num_tcs, 4032 uint8_t pfc_en) 4033 { 4034 uint8_t i; 4035 int32_t rc; 4036 struct rte_eth_rss_conf rss_conf; 4037 4038 /* 4039 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4040 * given above, and the number of traffic classes available for use. 4041 */ 4042 if (dcb_mode == DCB_VT_ENABLED) { 4043 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4044 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4045 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4046 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4047 4048 /* VMDQ+DCB RX and TX configurations */ 4049 vmdq_rx_conf->enable_default_pool = 0; 4050 vmdq_rx_conf->default_pool = 0; 4051 vmdq_rx_conf->nb_queue_pools = 4052 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4053 vmdq_tx_conf->nb_queue_pools = 4054 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4055 4056 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4057 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4058 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4059 vmdq_rx_conf->pool_map[i].pools = 4060 1 << (i % vmdq_rx_conf->nb_queue_pools); 4061 } 4062 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4063 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4064 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4065 } 4066 4067 /* set DCB mode of RX and TX of multiple queues */ 4068 eth_conf->rxmode.mq_mode = 4069 (enum rte_eth_rx_mq_mode) 4070 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4071 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4072 } else { 4073 struct rte_eth_dcb_rx_conf *rx_conf = 4074 ð_conf->rx_adv_conf.dcb_rx_conf; 4075 struct rte_eth_dcb_tx_conf *tx_conf = 4076 ð_conf->tx_adv_conf.dcb_tx_conf; 4077 4078 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4079 4080 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4081 if (rc != 0) 4082 return rc; 4083 4084 rx_conf->nb_tcs = num_tcs; 4085 tx_conf->nb_tcs = num_tcs; 4086 4087 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4088 rx_conf->dcb_tc[i] = i % num_tcs; 4089 tx_conf->dcb_tc[i] = i % num_tcs; 4090 } 4091 4092 eth_conf->rxmode.mq_mode = 4093 (enum rte_eth_rx_mq_mode) 4094 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4095 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4096 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4097 } 4098 4099 if (pfc_en) 4100 eth_conf->dcb_capability_en = 4101 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4102 else 4103 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4104 4105 return 0; 4106 } 4107 4108 int 4109 init_port_dcb_config(portid_t pid, 4110 enum dcb_mode_enable dcb_mode, 4111 enum rte_eth_nb_tcs num_tcs, 4112 uint8_t pfc_en) 4113 { 4114 struct rte_eth_conf port_conf; 4115 struct rte_port *rte_port; 4116 int retval; 4117 uint16_t i; 4118 4119 if (num_procs > 1) { 4120 printf("The multi-process feature doesn't support dcb.\n"); 4121 return -ENOTSUP; 4122 } 4123 rte_port = &ports[pid]; 4124 4125 /* retain the original device configuration. */ 4126 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4127 4128 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4129 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4130 if (retval < 0) 4131 return retval; 4132 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4133 /* remove RSS HASH offload for DCB in vt mode */ 4134 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4135 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4136 for (i = 0; i < nb_rxq; i++) 4137 rte_port->rxq[i].conf.offloads &= 4138 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4139 } 4140 4141 /* re-configure the device . */ 4142 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4143 if (retval < 0) 4144 return retval; 4145 4146 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4147 if (retval != 0) 4148 return retval; 4149 4150 /* If dev_info.vmdq_pool_base is greater than 0, 4151 * the queue id of vmdq pools is started after pf queues. 4152 */ 4153 if (dcb_mode == DCB_VT_ENABLED && 4154 rte_port->dev_info.vmdq_pool_base > 0) { 4155 fprintf(stderr, 4156 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4157 pid); 4158 return -1; 4159 } 4160 4161 /* Assume the ports in testpmd have the same dcb capability 4162 * and has the same number of rxq and txq in dcb mode 4163 */ 4164 if (dcb_mode == DCB_VT_ENABLED) { 4165 if (rte_port->dev_info.max_vfs > 0) { 4166 nb_rxq = rte_port->dev_info.nb_rx_queues; 4167 nb_txq = rte_port->dev_info.nb_tx_queues; 4168 } else { 4169 nb_rxq = rte_port->dev_info.max_rx_queues; 4170 nb_txq = rte_port->dev_info.max_tx_queues; 4171 } 4172 } else { 4173 /*if vt is disabled, use all pf queues */ 4174 if (rte_port->dev_info.vmdq_pool_base == 0) { 4175 nb_rxq = rte_port->dev_info.max_rx_queues; 4176 nb_txq = rte_port->dev_info.max_tx_queues; 4177 } else { 4178 nb_rxq = (queueid_t)num_tcs; 4179 nb_txq = (queueid_t)num_tcs; 4180 4181 } 4182 } 4183 rx_free_thresh = 64; 4184 4185 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4186 4187 rxtx_port_config(pid); 4188 /* VLAN filter */ 4189 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4190 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4191 rx_vft_set(pid, vlan_tags[i], 1); 4192 4193 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4194 if (retval != 0) 4195 return retval; 4196 4197 rte_port->dcb_flag = 1; 4198 4199 /* Enter DCB configuration status */ 4200 dcb_config = 1; 4201 4202 return 0; 4203 } 4204 4205 static void 4206 init_port(void) 4207 { 4208 int i; 4209 4210 /* Configuration of Ethernet ports. */ 4211 ports = rte_zmalloc("testpmd: ports", 4212 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4213 RTE_CACHE_LINE_SIZE); 4214 if (ports == NULL) { 4215 rte_exit(EXIT_FAILURE, 4216 "rte_zmalloc(%d struct rte_port) failed\n", 4217 RTE_MAX_ETHPORTS); 4218 } 4219 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4220 ports[i].xstats_info.allocated = false; 4221 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4222 LIST_INIT(&ports[i].flow_tunnel_list); 4223 /* Initialize ports NUMA structures */ 4224 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4225 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4226 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4227 } 4228 4229 static void 4230 force_quit(void) 4231 { 4232 pmd_test_exit(); 4233 prompt_exit(); 4234 } 4235 4236 static void 4237 print_stats(void) 4238 { 4239 uint8_t i; 4240 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4241 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4242 4243 /* Clear screen and move to top left */ 4244 printf("%s%s", clr, top_left); 4245 4246 printf("\nPort statistics ===================================="); 4247 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4248 nic_stats_display(fwd_ports_ids[i]); 4249 4250 fflush(stdout); 4251 } 4252 4253 static void 4254 signal_handler(int signum) 4255 { 4256 if (signum == SIGINT || signum == SIGTERM) { 4257 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4258 signum); 4259 #ifdef RTE_LIB_PDUMP 4260 /* uninitialize packet capture framework */ 4261 rte_pdump_uninit(); 4262 #endif 4263 #ifdef RTE_LIB_LATENCYSTATS 4264 if (latencystats_enabled != 0) 4265 rte_latencystats_uninit(); 4266 #endif 4267 force_quit(); 4268 /* Set flag to indicate the force termination. */ 4269 f_quit = 1; 4270 /* exit with the expected status */ 4271 #ifndef RTE_EXEC_ENV_WINDOWS 4272 signal(signum, SIG_DFL); 4273 kill(getpid(), signum); 4274 #endif 4275 } 4276 } 4277 4278 int 4279 main(int argc, char** argv) 4280 { 4281 int diag; 4282 portid_t port_id; 4283 uint16_t count; 4284 int ret; 4285 4286 signal(SIGINT, signal_handler); 4287 signal(SIGTERM, signal_handler); 4288 4289 testpmd_logtype = rte_log_register("testpmd"); 4290 if (testpmd_logtype < 0) 4291 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4292 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4293 4294 diag = rte_eal_init(argc, argv); 4295 if (diag < 0) 4296 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4297 rte_strerror(rte_errno)); 4298 4299 ret = register_eth_event_callback(); 4300 if (ret != 0) 4301 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4302 4303 #ifdef RTE_LIB_PDUMP 4304 /* initialize packet capture framework */ 4305 rte_pdump_init(); 4306 #endif 4307 4308 count = 0; 4309 RTE_ETH_FOREACH_DEV(port_id) { 4310 ports_ids[count] = port_id; 4311 count++; 4312 } 4313 nb_ports = (portid_t) count; 4314 if (nb_ports == 0) 4315 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4316 4317 /* allocate port structures, and init them */ 4318 init_port(); 4319 4320 set_def_fwd_config(); 4321 if (nb_lcores == 0) 4322 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4323 "Check the core mask argument\n"); 4324 4325 /* Bitrate/latency stats disabled by default */ 4326 #ifdef RTE_LIB_BITRATESTATS 4327 bitrate_enabled = 0; 4328 #endif 4329 #ifdef RTE_LIB_LATENCYSTATS 4330 latencystats_enabled = 0; 4331 #endif 4332 4333 /* on FreeBSD, mlockall() is disabled by default */ 4334 #ifdef RTE_EXEC_ENV_FREEBSD 4335 do_mlockall = 0; 4336 #else 4337 do_mlockall = 1; 4338 #endif 4339 4340 argc -= diag; 4341 argv += diag; 4342 if (argc > 1) 4343 launch_args_parse(argc, argv); 4344 4345 #ifndef RTE_EXEC_ENV_WINDOWS 4346 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4347 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4348 strerror(errno)); 4349 } 4350 #endif 4351 4352 if (tx_first && interactive) 4353 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4354 "interactive mode.\n"); 4355 4356 if (tx_first && lsc_interrupt) { 4357 fprintf(stderr, 4358 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4359 lsc_interrupt = 0; 4360 } 4361 4362 if (!nb_rxq && !nb_txq) 4363 fprintf(stderr, 4364 "Warning: Either rx or tx queues should be non-zero\n"); 4365 4366 if (nb_rxq > 1 && nb_rxq > nb_txq) 4367 fprintf(stderr, 4368 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4369 nb_rxq, nb_txq); 4370 4371 init_config(); 4372 4373 if (hot_plug) { 4374 ret = rte_dev_hotplug_handle_enable(); 4375 if (ret) { 4376 RTE_LOG(ERR, EAL, 4377 "fail to enable hotplug handling."); 4378 return -1; 4379 } 4380 4381 ret = rte_dev_event_monitor_start(); 4382 if (ret) { 4383 RTE_LOG(ERR, EAL, 4384 "fail to start device event monitoring."); 4385 return -1; 4386 } 4387 4388 ret = rte_dev_event_callback_register(NULL, 4389 dev_event_callback, NULL); 4390 if (ret) { 4391 RTE_LOG(ERR, EAL, 4392 "fail to register device event callback\n"); 4393 return -1; 4394 } 4395 } 4396 4397 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4398 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4399 4400 /* set all ports to promiscuous mode by default */ 4401 RTE_ETH_FOREACH_DEV(port_id) { 4402 ret = rte_eth_promiscuous_enable(port_id); 4403 if (ret != 0) 4404 fprintf(stderr, 4405 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4406 port_id, rte_strerror(-ret)); 4407 } 4408 4409 #ifdef RTE_LIB_METRICS 4410 /* Init metrics library */ 4411 rte_metrics_init(rte_socket_id()); 4412 #endif 4413 4414 #ifdef RTE_LIB_LATENCYSTATS 4415 if (latencystats_enabled != 0) { 4416 int ret = rte_latencystats_init(1, NULL); 4417 if (ret) 4418 fprintf(stderr, 4419 "Warning: latencystats init() returned error %d\n", 4420 ret); 4421 fprintf(stderr, "Latencystats running on lcore %d\n", 4422 latencystats_lcore_id); 4423 } 4424 #endif 4425 4426 /* Setup bitrate stats */ 4427 #ifdef RTE_LIB_BITRATESTATS 4428 if (bitrate_enabled != 0) { 4429 bitrate_data = rte_stats_bitrate_create(); 4430 if (bitrate_data == NULL) 4431 rte_exit(EXIT_FAILURE, 4432 "Could not allocate bitrate data.\n"); 4433 rte_stats_bitrate_reg(bitrate_data); 4434 } 4435 #endif 4436 #ifdef RTE_LIB_CMDLINE 4437 if (init_cmdline() != 0) 4438 rte_exit(EXIT_FAILURE, 4439 "Could not initialise cmdline context.\n"); 4440 4441 if (strlen(cmdline_filename) != 0) 4442 cmdline_read_from_file(cmdline_filename); 4443 4444 if (interactive == 1) { 4445 if (auto_start) { 4446 printf("Start automatic packet forwarding\n"); 4447 start_packet_forwarding(0); 4448 } 4449 prompt(); 4450 pmd_test_exit(); 4451 } else 4452 #endif 4453 { 4454 char c; 4455 int rc; 4456 4457 f_quit = 0; 4458 4459 printf("No commandline core given, start packet forwarding\n"); 4460 start_packet_forwarding(tx_first); 4461 if (stats_period != 0) { 4462 uint64_t prev_time = 0, cur_time, diff_time = 0; 4463 uint64_t timer_period; 4464 4465 /* Convert to number of cycles */ 4466 timer_period = stats_period * rte_get_timer_hz(); 4467 4468 while (f_quit == 0) { 4469 cur_time = rte_get_timer_cycles(); 4470 diff_time += cur_time - prev_time; 4471 4472 if (diff_time >= timer_period) { 4473 print_stats(); 4474 /* Reset the timer */ 4475 diff_time = 0; 4476 } 4477 /* Sleep to avoid unnecessary checks */ 4478 prev_time = cur_time; 4479 rte_delay_us_sleep(US_PER_S); 4480 } 4481 } 4482 4483 printf("Press enter to exit\n"); 4484 rc = read(0, &c, 1); 4485 pmd_test_exit(); 4486 if (rc < 0) 4487 return 1; 4488 } 4489 4490 ret = rte_eal_cleanup(); 4491 if (ret != 0) 4492 rte_exit(EXIT_FAILURE, 4493 "EAL cleanup failed: %s\n", strerror(-ret)); 4494 4495 return EXIT_SUCCESS; 4496 } 4497