1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 251 /* 252 * Configuration of packet segments used by the "txonly" processing engine. 253 */ 254 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 255 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 256 TXONLY_DEF_PACKET_LEN, 257 }; 258 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 259 260 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 261 /**< Split policy for packets to TX. */ 262 263 uint8_t txonly_multi_flow; 264 /**< Whether multiple flows are generated in TXONLY mode. */ 265 266 uint32_t tx_pkt_times_inter; 267 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 268 269 uint32_t tx_pkt_times_intra; 270 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 271 272 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 273 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 274 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 275 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 276 277 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 278 uint8_t dcb_config = 0; 279 280 /* 281 * Configurable number of RX/TX queues. 282 */ 283 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 284 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 285 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 286 287 /* 288 * Configurable number of RX/TX ring descriptors. 289 * Defaults are supplied by drivers via ethdev. 290 */ 291 #define RX_DESC_DEFAULT 0 292 #define TX_DESC_DEFAULT 0 293 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 294 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 295 296 #define RTE_PMD_PARAM_UNSET -1 297 /* 298 * Configurable values of RX and TX ring threshold registers. 299 */ 300 301 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 302 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 303 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 304 305 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 306 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 307 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 308 309 /* 310 * Configurable value of RX free threshold. 311 */ 312 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 313 314 /* 315 * Configurable value of RX drop enable. 316 */ 317 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 318 319 /* 320 * Configurable value of TX free threshold. 321 */ 322 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 323 324 /* 325 * Configurable value of TX RS bit threshold. 326 */ 327 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 328 329 /* 330 * Configurable value of buffered packets before sending. 331 */ 332 uint16_t noisy_tx_sw_bufsz; 333 334 /* 335 * Configurable value of packet buffer timeout. 336 */ 337 uint16_t noisy_tx_sw_buf_flush_time; 338 339 /* 340 * Configurable value for size of VNF internal memory area 341 * used for simulating noisy neighbour behaviour 342 */ 343 uint64_t noisy_lkup_mem_sz; 344 345 /* 346 * Configurable value of number of random writes done in 347 * VNF simulation memory area. 348 */ 349 uint64_t noisy_lkup_num_writes; 350 351 /* 352 * Configurable value of number of random reads done in 353 * VNF simulation memory area. 354 */ 355 uint64_t noisy_lkup_num_reads; 356 357 /* 358 * Configurable value of number of random reads/writes done in 359 * VNF simulation memory area. 360 */ 361 uint64_t noisy_lkup_num_reads_writes; 362 363 /* 364 * Receive Side Scaling (RSS) configuration. 365 */ 366 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 367 368 /* 369 * Port topology configuration 370 */ 371 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 372 373 /* 374 * Avoids to flush all the RX streams before starts forwarding. 375 */ 376 uint8_t no_flush_rx = 0; /* flush by default */ 377 378 /* 379 * Flow API isolated mode. 380 */ 381 uint8_t flow_isolate_all; 382 383 /* 384 * Avoids to check link status when starting/stopping a port. 385 */ 386 uint8_t no_link_check = 0; /* check by default */ 387 388 /* 389 * Don't automatically start all ports in interactive mode. 390 */ 391 uint8_t no_device_start = 0; 392 393 /* 394 * Enable link status change notification 395 */ 396 uint8_t lsc_interrupt = 1; /* enabled by default */ 397 398 /* 399 * Enable device removal notification. 400 */ 401 uint8_t rmv_interrupt = 1; /* enabled by default */ 402 403 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 404 405 /* After attach, port setup is called on event or by iterator */ 406 bool setup_on_probe_event = true; 407 408 /* Clear ptypes on port initialization. */ 409 uint8_t clear_ptypes = true; 410 411 /* Hairpin ports configuration mode. */ 412 uint32_t hairpin_mode; 413 414 /* Pretty printing of ethdev events */ 415 static const char * const eth_event_desc[] = { 416 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 417 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 418 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 419 [RTE_ETH_EVENT_INTR_RESET] = "reset", 420 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 421 [RTE_ETH_EVENT_IPSEC] = "IPsec", 422 [RTE_ETH_EVENT_MACSEC] = "MACsec", 423 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 424 [RTE_ETH_EVENT_NEW] = "device probed", 425 [RTE_ETH_EVENT_DESTROY] = "device released", 426 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 427 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 428 [RTE_ETH_EVENT_MAX] = NULL, 429 }; 430 431 /* 432 * Display or mask ether events 433 * Default to all events except VF_MBOX 434 */ 435 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 436 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 437 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 438 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 439 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 440 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 441 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 442 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 443 /* 444 * Decide if all memory are locked for performance. 445 */ 446 int do_mlockall = 0; 447 448 #ifdef RTE_LIB_LATENCYSTATS 449 450 /* 451 * Set when latency stats is enabled in the commandline 452 */ 453 uint8_t latencystats_enabled; 454 455 /* 456 * Lcore ID to service latency statistics. 457 */ 458 lcoreid_t latencystats_lcore_id = -1; 459 460 #endif 461 462 /* 463 * Ethernet device configuration. 464 */ 465 struct rte_eth_rxmode rx_mode; 466 467 struct rte_eth_txmode tx_mode = { 468 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 469 }; 470 471 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 472 473 /* 474 * Display zero values by default for xstats 475 */ 476 uint8_t xstats_hide_zero; 477 478 /* 479 * Measure of CPU cycles disabled by default 480 */ 481 uint8_t record_core_cycles; 482 483 /* 484 * Display of RX and TX bursts disabled by default 485 */ 486 uint8_t record_burst_stats; 487 488 /* 489 * Number of ports per shared Rx queue group, 0 disable. 490 */ 491 uint32_t rxq_share; 492 493 unsigned int num_sockets = 0; 494 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 495 496 #ifdef RTE_LIB_BITRATESTATS 497 /* Bitrate statistics */ 498 struct rte_stats_bitrates *bitrate_data; 499 lcoreid_t bitrate_lcore_id; 500 uint8_t bitrate_enabled; 501 #endif 502 503 #ifdef RTE_LIB_GRO 504 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 505 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 506 #endif 507 508 /* 509 * hexadecimal bitmask of RX mq mode can be enabled. 510 */ 511 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 512 513 /* 514 * Used to set forced link speed 515 */ 516 uint32_t eth_link_speed; 517 518 /* 519 * ID of the current process in multi-process, used to 520 * configure the queues to be polled. 521 */ 522 int proc_id; 523 524 /* 525 * Number of processes in multi-process, used to 526 * configure the queues to be polled. 527 */ 528 unsigned int num_procs = 1; 529 530 static void 531 eth_rx_metadata_negotiate_mp(uint16_t port_id) 532 { 533 uint64_t rx_meta_features = 0; 534 int ret; 535 536 if (!is_proc_primary()) 537 return; 538 539 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 540 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 541 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 542 543 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 544 if (ret == 0) { 545 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 546 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 547 port_id); 548 } 549 550 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 551 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 552 port_id); 553 } 554 555 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 556 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 557 port_id); 558 } 559 } else if (ret != -ENOTSUP) { 560 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 561 port_id, rte_strerror(-ret)); 562 } 563 } 564 565 static int 566 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 567 const struct rte_eth_conf *dev_conf) 568 { 569 if (is_proc_primary()) 570 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 571 dev_conf); 572 return 0; 573 } 574 575 static int 576 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 577 { 578 #ifdef RTE_NET_BOND 579 580 portid_t slave_pids[RTE_MAX_ETHPORTS]; 581 struct rte_port *port; 582 int num_slaves; 583 portid_t slave_pid; 584 int i; 585 586 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 587 RTE_MAX_ETHPORTS); 588 if (num_slaves < 0) { 589 fprintf(stderr, "Failed to get slave list for port = %u\n", 590 bond_pid); 591 return num_slaves; 592 } 593 594 for (i = 0; i < num_slaves; i++) { 595 slave_pid = slave_pids[i]; 596 port = &ports[slave_pid]; 597 port->port_status = 598 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 599 } 600 #else 601 RTE_SET_USED(bond_pid); 602 RTE_SET_USED(is_stop); 603 #endif 604 return 0; 605 } 606 607 static int 608 eth_dev_start_mp(uint16_t port_id) 609 { 610 int ret; 611 612 if (is_proc_primary()) { 613 ret = rte_eth_dev_start(port_id); 614 if (ret != 0) 615 return ret; 616 617 struct rte_port *port = &ports[port_id]; 618 619 /* 620 * Starting a bonded port also starts all slaves under the bonded 621 * device. So if this port is bond device, we need to modify the 622 * port status of these slaves. 623 */ 624 if (port->bond_flag == 1) 625 return change_bonding_slave_port_status(port_id, false); 626 } 627 628 return 0; 629 } 630 631 static int 632 eth_dev_stop_mp(uint16_t port_id) 633 { 634 int ret; 635 636 if (is_proc_primary()) { 637 ret = rte_eth_dev_stop(port_id); 638 if (ret != 0) 639 return ret; 640 641 struct rte_port *port = &ports[port_id]; 642 643 /* 644 * Stopping a bonded port also stops all slaves under the bonded 645 * device. So if this port is bond device, we need to modify the 646 * port status of these slaves. 647 */ 648 if (port->bond_flag == 1) 649 return change_bonding_slave_port_status(port_id, true); 650 } 651 652 return 0; 653 } 654 655 static void 656 mempool_free_mp(struct rte_mempool *mp) 657 { 658 if (is_proc_primary()) 659 rte_mempool_free(mp); 660 } 661 662 static int 663 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 664 { 665 if (is_proc_primary()) 666 return rte_eth_dev_set_mtu(port_id, mtu); 667 668 return 0; 669 } 670 671 /* Forward function declarations */ 672 static void setup_attached_port(portid_t pi); 673 static void check_all_ports_link_status(uint32_t port_mask); 674 static int eth_event_callback(portid_t port_id, 675 enum rte_eth_event_type type, 676 void *param, void *ret_param); 677 static void dev_event_callback(const char *device_name, 678 enum rte_dev_event_type type, 679 void *param); 680 static void fill_xstats_display_info(void); 681 682 /* 683 * Check if all the ports are started. 684 * If yes, return positive value. If not, return zero. 685 */ 686 static int all_ports_started(void); 687 688 #ifdef RTE_LIB_GSO 689 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 690 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 691 #endif 692 693 /* Holds the registered mbuf dynamic flags names. */ 694 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 695 696 697 /* 698 * Helper function to check if socket is already discovered. 699 * If yes, return positive value. If not, return zero. 700 */ 701 int 702 new_socket_id(unsigned int socket_id) 703 { 704 unsigned int i; 705 706 for (i = 0; i < num_sockets; i++) { 707 if (socket_ids[i] == socket_id) 708 return 0; 709 } 710 return 1; 711 } 712 713 /* 714 * Setup default configuration. 715 */ 716 static void 717 set_default_fwd_lcores_config(void) 718 { 719 unsigned int i; 720 unsigned int nb_lc; 721 unsigned int sock_num; 722 723 nb_lc = 0; 724 for (i = 0; i < RTE_MAX_LCORE; i++) { 725 if (!rte_lcore_is_enabled(i)) 726 continue; 727 sock_num = rte_lcore_to_socket_id(i); 728 if (new_socket_id(sock_num)) { 729 if (num_sockets >= RTE_MAX_NUMA_NODES) { 730 rte_exit(EXIT_FAILURE, 731 "Total sockets greater than %u\n", 732 RTE_MAX_NUMA_NODES); 733 } 734 socket_ids[num_sockets++] = sock_num; 735 } 736 if (i == rte_get_main_lcore()) 737 continue; 738 fwd_lcores_cpuids[nb_lc++] = i; 739 } 740 nb_lcores = (lcoreid_t) nb_lc; 741 nb_cfg_lcores = nb_lcores; 742 nb_fwd_lcores = 1; 743 } 744 745 static void 746 set_def_peer_eth_addrs(void) 747 { 748 portid_t i; 749 750 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 751 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 752 peer_eth_addrs[i].addr_bytes[5] = i; 753 } 754 } 755 756 static void 757 set_default_fwd_ports_config(void) 758 { 759 portid_t pt_id; 760 int i = 0; 761 762 RTE_ETH_FOREACH_DEV(pt_id) { 763 fwd_ports_ids[i++] = pt_id; 764 765 /* Update sockets info according to the attached device */ 766 int socket_id = rte_eth_dev_socket_id(pt_id); 767 if (socket_id >= 0 && new_socket_id(socket_id)) { 768 if (num_sockets >= RTE_MAX_NUMA_NODES) { 769 rte_exit(EXIT_FAILURE, 770 "Total sockets greater than %u\n", 771 RTE_MAX_NUMA_NODES); 772 } 773 socket_ids[num_sockets++] = socket_id; 774 } 775 } 776 777 nb_cfg_ports = nb_ports; 778 nb_fwd_ports = nb_ports; 779 } 780 781 void 782 set_def_fwd_config(void) 783 { 784 set_default_fwd_lcores_config(); 785 set_def_peer_eth_addrs(); 786 set_default_fwd_ports_config(); 787 } 788 789 #ifndef RTE_EXEC_ENV_WINDOWS 790 /* extremely pessimistic estimation of memory required to create a mempool */ 791 static int 792 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 793 { 794 unsigned int n_pages, mbuf_per_pg, leftover; 795 uint64_t total_mem, mbuf_mem, obj_sz; 796 797 /* there is no good way to predict how much space the mempool will 798 * occupy because it will allocate chunks on the fly, and some of those 799 * will come from default DPDK memory while some will come from our 800 * external memory, so just assume 128MB will be enough for everyone. 801 */ 802 uint64_t hdr_mem = 128 << 20; 803 804 /* account for possible non-contiguousness */ 805 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 806 if (obj_sz > pgsz) { 807 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 808 return -1; 809 } 810 811 mbuf_per_pg = pgsz / obj_sz; 812 leftover = (nb_mbufs % mbuf_per_pg) > 0; 813 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 814 815 mbuf_mem = n_pages * pgsz; 816 817 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 818 819 if (total_mem > SIZE_MAX) { 820 TESTPMD_LOG(ERR, "Memory size too big\n"); 821 return -1; 822 } 823 *out = (size_t)total_mem; 824 825 return 0; 826 } 827 828 static int 829 pagesz_flags(uint64_t page_sz) 830 { 831 /* as per mmap() manpage, all page sizes are log2 of page size 832 * shifted by MAP_HUGE_SHIFT 833 */ 834 int log2 = rte_log2_u64(page_sz); 835 836 return (log2 << HUGE_SHIFT); 837 } 838 839 static void * 840 alloc_mem(size_t memsz, size_t pgsz, bool huge) 841 { 842 void *addr; 843 int flags; 844 845 /* allocate anonymous hugepages */ 846 flags = MAP_ANONYMOUS | MAP_PRIVATE; 847 if (huge) 848 flags |= HUGE_FLAG | pagesz_flags(pgsz); 849 850 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 851 if (addr == MAP_FAILED) 852 return NULL; 853 854 return addr; 855 } 856 857 struct extmem_param { 858 void *addr; 859 size_t len; 860 size_t pgsz; 861 rte_iova_t *iova_table; 862 unsigned int iova_table_len; 863 }; 864 865 static int 866 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 867 bool huge) 868 { 869 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 870 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 871 unsigned int cur_page, n_pages, pgsz_idx; 872 size_t mem_sz, cur_pgsz; 873 rte_iova_t *iovas = NULL; 874 void *addr; 875 int ret; 876 877 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 878 /* skip anything that is too big */ 879 if (pgsizes[pgsz_idx] > SIZE_MAX) 880 continue; 881 882 cur_pgsz = pgsizes[pgsz_idx]; 883 884 /* if we were told not to allocate hugepages, override */ 885 if (!huge) 886 cur_pgsz = sysconf(_SC_PAGESIZE); 887 888 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 889 if (ret < 0) { 890 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 891 return -1; 892 } 893 894 /* allocate our memory */ 895 addr = alloc_mem(mem_sz, cur_pgsz, huge); 896 897 /* if we couldn't allocate memory with a specified page size, 898 * that doesn't mean we can't do it with other page sizes, so 899 * try another one. 900 */ 901 if (addr == NULL) 902 continue; 903 904 /* store IOVA addresses for every page in this memory area */ 905 n_pages = mem_sz / cur_pgsz; 906 907 iovas = malloc(sizeof(*iovas) * n_pages); 908 909 if (iovas == NULL) { 910 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 911 goto fail; 912 } 913 /* lock memory if it's not huge pages */ 914 if (!huge) 915 mlock(addr, mem_sz); 916 917 /* populate IOVA addresses */ 918 for (cur_page = 0; cur_page < n_pages; cur_page++) { 919 rte_iova_t iova; 920 size_t offset; 921 void *cur; 922 923 offset = cur_pgsz * cur_page; 924 cur = RTE_PTR_ADD(addr, offset); 925 926 /* touch the page before getting its IOVA */ 927 *(volatile char *)cur = 0; 928 929 iova = rte_mem_virt2iova(cur); 930 931 iovas[cur_page] = iova; 932 } 933 934 break; 935 } 936 /* if we couldn't allocate anything */ 937 if (iovas == NULL) 938 return -1; 939 940 param->addr = addr; 941 param->len = mem_sz; 942 param->pgsz = cur_pgsz; 943 param->iova_table = iovas; 944 param->iova_table_len = n_pages; 945 946 return 0; 947 fail: 948 free(iovas); 949 if (addr) 950 munmap(addr, mem_sz); 951 952 return -1; 953 } 954 955 static int 956 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 957 { 958 struct extmem_param param; 959 int socket_id, ret; 960 961 memset(¶m, 0, sizeof(param)); 962 963 /* check if our heap exists */ 964 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 965 if (socket_id < 0) { 966 /* create our heap */ 967 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 968 if (ret < 0) { 969 TESTPMD_LOG(ERR, "Cannot create heap\n"); 970 return -1; 971 } 972 } 973 974 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 975 if (ret < 0) { 976 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 977 return -1; 978 } 979 980 /* we now have a valid memory area, so add it to heap */ 981 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 982 param.addr, param.len, param.iova_table, 983 param.iova_table_len, param.pgsz); 984 985 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 986 987 /* not needed any more */ 988 free(param.iova_table); 989 990 if (ret < 0) { 991 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 992 munmap(param.addr, param.len); 993 return -1; 994 } 995 996 /* success */ 997 998 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 999 param.len >> 20); 1000 1001 return 0; 1002 } 1003 static void 1004 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1005 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1006 { 1007 uint16_t pid = 0; 1008 int ret; 1009 1010 RTE_ETH_FOREACH_DEV(pid) { 1011 struct rte_eth_dev_info dev_info; 1012 1013 ret = eth_dev_info_get_print_err(pid, &dev_info); 1014 if (ret != 0) { 1015 TESTPMD_LOG(DEBUG, 1016 "unable to get device info for port %d on addr 0x%p," 1017 "mempool unmapping will not be performed\n", 1018 pid, memhdr->addr); 1019 continue; 1020 } 1021 1022 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1023 if (ret) { 1024 TESTPMD_LOG(DEBUG, 1025 "unable to DMA unmap addr 0x%p " 1026 "for device %s\n", 1027 memhdr->addr, rte_dev_name(dev_info.device)); 1028 } 1029 } 1030 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1031 if (ret) { 1032 TESTPMD_LOG(DEBUG, 1033 "unable to un-register addr 0x%p\n", memhdr->addr); 1034 } 1035 } 1036 1037 static void 1038 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1039 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1040 { 1041 uint16_t pid = 0; 1042 size_t page_size = sysconf(_SC_PAGESIZE); 1043 int ret; 1044 1045 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1046 page_size); 1047 if (ret) { 1048 TESTPMD_LOG(DEBUG, 1049 "unable to register addr 0x%p\n", memhdr->addr); 1050 return; 1051 } 1052 RTE_ETH_FOREACH_DEV(pid) { 1053 struct rte_eth_dev_info dev_info; 1054 1055 ret = eth_dev_info_get_print_err(pid, &dev_info); 1056 if (ret != 0) { 1057 TESTPMD_LOG(DEBUG, 1058 "unable to get device info for port %d on addr 0x%p," 1059 "mempool mapping will not be performed\n", 1060 pid, memhdr->addr); 1061 continue; 1062 } 1063 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1064 if (ret) { 1065 TESTPMD_LOG(DEBUG, 1066 "unable to DMA map addr 0x%p " 1067 "for device %s\n", 1068 memhdr->addr, rte_dev_name(dev_info.device)); 1069 } 1070 } 1071 } 1072 #endif 1073 1074 static unsigned int 1075 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1076 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1077 { 1078 struct rte_pktmbuf_extmem *xmem; 1079 unsigned int ext_num, zone_num, elt_num; 1080 uint16_t elt_size; 1081 1082 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1083 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1084 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1085 1086 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1087 if (xmem == NULL) { 1088 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1089 "external buffer descriptors\n"); 1090 *ext_mem = NULL; 1091 return 0; 1092 } 1093 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1094 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1095 const struct rte_memzone *mz; 1096 char mz_name[RTE_MEMZONE_NAMESIZE]; 1097 int ret; 1098 1099 ret = snprintf(mz_name, sizeof(mz_name), 1100 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1101 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1102 errno = ENAMETOOLONG; 1103 ext_num = 0; 1104 break; 1105 } 1106 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1107 socket_id, 1108 RTE_MEMZONE_IOVA_CONTIG | 1109 RTE_MEMZONE_1GB | 1110 RTE_MEMZONE_SIZE_HINT_ONLY); 1111 if (mz == NULL) { 1112 /* 1113 * The caller exits on external buffer creation 1114 * error, so there is no need to free memzones. 1115 */ 1116 errno = ENOMEM; 1117 ext_num = 0; 1118 break; 1119 } 1120 xseg->buf_ptr = mz->addr; 1121 xseg->buf_iova = mz->iova; 1122 xseg->buf_len = EXTBUF_ZONE_SIZE; 1123 xseg->elt_size = elt_size; 1124 } 1125 if (ext_num == 0 && xmem != NULL) { 1126 free(xmem); 1127 xmem = NULL; 1128 } 1129 *ext_mem = xmem; 1130 return ext_num; 1131 } 1132 1133 /* 1134 * Configuration initialisation done once at init time. 1135 */ 1136 static struct rte_mempool * 1137 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1138 unsigned int socket_id, uint16_t size_idx) 1139 { 1140 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1141 struct rte_mempool *rte_mp = NULL; 1142 #ifndef RTE_EXEC_ENV_WINDOWS 1143 uint32_t mb_size; 1144 1145 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1146 #endif 1147 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1148 if (!is_proc_primary()) { 1149 rte_mp = rte_mempool_lookup(pool_name); 1150 if (rte_mp == NULL) 1151 rte_exit(EXIT_FAILURE, 1152 "Get mbuf pool for socket %u failed: %s\n", 1153 socket_id, rte_strerror(rte_errno)); 1154 return rte_mp; 1155 } 1156 1157 TESTPMD_LOG(INFO, 1158 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1159 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1160 1161 switch (mp_alloc_type) { 1162 case MP_ALLOC_NATIVE: 1163 { 1164 /* wrapper to rte_mempool_create() */ 1165 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1166 rte_mbuf_best_mempool_ops()); 1167 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1168 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1169 break; 1170 } 1171 #ifndef RTE_EXEC_ENV_WINDOWS 1172 case MP_ALLOC_ANON: 1173 { 1174 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1175 mb_size, (unsigned int) mb_mempool_cache, 1176 sizeof(struct rte_pktmbuf_pool_private), 1177 socket_id, mempool_flags); 1178 if (rte_mp == NULL) 1179 goto err; 1180 1181 if (rte_mempool_populate_anon(rte_mp) == 0) { 1182 rte_mempool_free(rte_mp); 1183 rte_mp = NULL; 1184 goto err; 1185 } 1186 rte_pktmbuf_pool_init(rte_mp, NULL); 1187 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1188 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1189 break; 1190 } 1191 case MP_ALLOC_XMEM: 1192 case MP_ALLOC_XMEM_HUGE: 1193 { 1194 int heap_socket; 1195 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1196 1197 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1198 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1199 1200 heap_socket = 1201 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1202 if (heap_socket < 0) 1203 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1204 1205 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1206 rte_mbuf_best_mempool_ops()); 1207 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1208 mb_mempool_cache, 0, mbuf_seg_size, 1209 heap_socket); 1210 break; 1211 } 1212 #endif 1213 case MP_ALLOC_XBUF: 1214 { 1215 struct rte_pktmbuf_extmem *ext_mem; 1216 unsigned int ext_num; 1217 1218 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1219 socket_id, pool_name, &ext_mem); 1220 if (ext_num == 0) 1221 rte_exit(EXIT_FAILURE, 1222 "Can't create pinned data buffers\n"); 1223 1224 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1225 rte_mbuf_best_mempool_ops()); 1226 rte_mp = rte_pktmbuf_pool_create_extbuf 1227 (pool_name, nb_mbuf, mb_mempool_cache, 1228 0, mbuf_seg_size, socket_id, 1229 ext_mem, ext_num); 1230 free(ext_mem); 1231 break; 1232 } 1233 default: 1234 { 1235 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1236 } 1237 } 1238 1239 #ifndef RTE_EXEC_ENV_WINDOWS 1240 err: 1241 #endif 1242 if (rte_mp == NULL) { 1243 rte_exit(EXIT_FAILURE, 1244 "Creation of mbuf pool for socket %u failed: %s\n", 1245 socket_id, rte_strerror(rte_errno)); 1246 } else if (verbose_level > 0) { 1247 rte_mempool_dump(stdout, rte_mp); 1248 } 1249 return rte_mp; 1250 } 1251 1252 /* 1253 * Check given socket id is valid or not with NUMA mode, 1254 * if valid, return 0, else return -1 1255 */ 1256 static int 1257 check_socket_id(const unsigned int socket_id) 1258 { 1259 static int warning_once = 0; 1260 1261 if (new_socket_id(socket_id)) { 1262 if (!warning_once && numa_support) 1263 fprintf(stderr, 1264 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1265 warning_once = 1; 1266 return -1; 1267 } 1268 return 0; 1269 } 1270 1271 /* 1272 * Get the allowed maximum number of RX queues. 1273 * *pid return the port id which has minimal value of 1274 * max_rx_queues in all ports. 1275 */ 1276 queueid_t 1277 get_allowed_max_nb_rxq(portid_t *pid) 1278 { 1279 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1280 bool max_rxq_valid = false; 1281 portid_t pi; 1282 struct rte_eth_dev_info dev_info; 1283 1284 RTE_ETH_FOREACH_DEV(pi) { 1285 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1286 continue; 1287 1288 max_rxq_valid = true; 1289 if (dev_info.max_rx_queues < allowed_max_rxq) { 1290 allowed_max_rxq = dev_info.max_rx_queues; 1291 *pid = pi; 1292 } 1293 } 1294 return max_rxq_valid ? allowed_max_rxq : 0; 1295 } 1296 1297 /* 1298 * Check input rxq is valid or not. 1299 * If input rxq is not greater than any of maximum number 1300 * of RX queues of all ports, it is valid. 1301 * if valid, return 0, else return -1 1302 */ 1303 int 1304 check_nb_rxq(queueid_t rxq) 1305 { 1306 queueid_t allowed_max_rxq; 1307 portid_t pid = 0; 1308 1309 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1310 if (rxq > allowed_max_rxq) { 1311 fprintf(stderr, 1312 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1313 rxq, allowed_max_rxq, pid); 1314 return -1; 1315 } 1316 return 0; 1317 } 1318 1319 /* 1320 * Get the allowed maximum number of TX queues. 1321 * *pid return the port id which has minimal value of 1322 * max_tx_queues in all ports. 1323 */ 1324 queueid_t 1325 get_allowed_max_nb_txq(portid_t *pid) 1326 { 1327 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1328 bool max_txq_valid = false; 1329 portid_t pi; 1330 struct rte_eth_dev_info dev_info; 1331 1332 RTE_ETH_FOREACH_DEV(pi) { 1333 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1334 continue; 1335 1336 max_txq_valid = true; 1337 if (dev_info.max_tx_queues < allowed_max_txq) { 1338 allowed_max_txq = dev_info.max_tx_queues; 1339 *pid = pi; 1340 } 1341 } 1342 return max_txq_valid ? allowed_max_txq : 0; 1343 } 1344 1345 /* 1346 * Check input txq is valid or not. 1347 * If input txq is not greater than any of maximum number 1348 * of TX queues of all ports, it is valid. 1349 * if valid, return 0, else return -1 1350 */ 1351 int 1352 check_nb_txq(queueid_t txq) 1353 { 1354 queueid_t allowed_max_txq; 1355 portid_t pid = 0; 1356 1357 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1358 if (txq > allowed_max_txq) { 1359 fprintf(stderr, 1360 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1361 txq, allowed_max_txq, pid); 1362 return -1; 1363 } 1364 return 0; 1365 } 1366 1367 /* 1368 * Get the allowed maximum number of RXDs of every rx queue. 1369 * *pid return the port id which has minimal value of 1370 * max_rxd in all queues of all ports. 1371 */ 1372 static uint16_t 1373 get_allowed_max_nb_rxd(portid_t *pid) 1374 { 1375 uint16_t allowed_max_rxd = UINT16_MAX; 1376 portid_t pi; 1377 struct rte_eth_dev_info dev_info; 1378 1379 RTE_ETH_FOREACH_DEV(pi) { 1380 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1381 continue; 1382 1383 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1384 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1385 *pid = pi; 1386 } 1387 } 1388 return allowed_max_rxd; 1389 } 1390 1391 /* 1392 * Get the allowed minimal number of RXDs of every rx queue. 1393 * *pid return the port id which has minimal value of 1394 * min_rxd in all queues of all ports. 1395 */ 1396 static uint16_t 1397 get_allowed_min_nb_rxd(portid_t *pid) 1398 { 1399 uint16_t allowed_min_rxd = 0; 1400 portid_t pi; 1401 struct rte_eth_dev_info dev_info; 1402 1403 RTE_ETH_FOREACH_DEV(pi) { 1404 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1405 continue; 1406 1407 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1408 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1409 *pid = pi; 1410 } 1411 } 1412 1413 return allowed_min_rxd; 1414 } 1415 1416 /* 1417 * Check input rxd is valid or not. 1418 * If input rxd is not greater than any of maximum number 1419 * of RXDs of every Rx queues and is not less than any of 1420 * minimal number of RXDs of every Rx queues, it is valid. 1421 * if valid, return 0, else return -1 1422 */ 1423 int 1424 check_nb_rxd(queueid_t rxd) 1425 { 1426 uint16_t allowed_max_rxd; 1427 uint16_t allowed_min_rxd; 1428 portid_t pid = 0; 1429 1430 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1431 if (rxd > allowed_max_rxd) { 1432 fprintf(stderr, 1433 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1434 rxd, allowed_max_rxd, pid); 1435 return -1; 1436 } 1437 1438 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1439 if (rxd < allowed_min_rxd) { 1440 fprintf(stderr, 1441 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1442 rxd, allowed_min_rxd, pid); 1443 return -1; 1444 } 1445 1446 return 0; 1447 } 1448 1449 /* 1450 * Get the allowed maximum number of TXDs of every rx queues. 1451 * *pid return the port id which has minimal value of 1452 * max_txd in every tx queue. 1453 */ 1454 static uint16_t 1455 get_allowed_max_nb_txd(portid_t *pid) 1456 { 1457 uint16_t allowed_max_txd = UINT16_MAX; 1458 portid_t pi; 1459 struct rte_eth_dev_info dev_info; 1460 1461 RTE_ETH_FOREACH_DEV(pi) { 1462 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1463 continue; 1464 1465 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1466 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1467 *pid = pi; 1468 } 1469 } 1470 return allowed_max_txd; 1471 } 1472 1473 /* 1474 * Get the allowed maximum number of TXDs of every tx queues. 1475 * *pid return the port id which has minimal value of 1476 * min_txd in every tx queue. 1477 */ 1478 static uint16_t 1479 get_allowed_min_nb_txd(portid_t *pid) 1480 { 1481 uint16_t allowed_min_txd = 0; 1482 portid_t pi; 1483 struct rte_eth_dev_info dev_info; 1484 1485 RTE_ETH_FOREACH_DEV(pi) { 1486 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1487 continue; 1488 1489 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1490 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1491 *pid = pi; 1492 } 1493 } 1494 1495 return allowed_min_txd; 1496 } 1497 1498 /* 1499 * Check input txd is valid or not. 1500 * If input txd is not greater than any of maximum number 1501 * of TXDs of every Rx queues, it is valid. 1502 * if valid, return 0, else return -1 1503 */ 1504 int 1505 check_nb_txd(queueid_t txd) 1506 { 1507 uint16_t allowed_max_txd; 1508 uint16_t allowed_min_txd; 1509 portid_t pid = 0; 1510 1511 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1512 if (txd > allowed_max_txd) { 1513 fprintf(stderr, 1514 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1515 txd, allowed_max_txd, pid); 1516 return -1; 1517 } 1518 1519 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1520 if (txd < allowed_min_txd) { 1521 fprintf(stderr, 1522 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1523 txd, allowed_min_txd, pid); 1524 return -1; 1525 } 1526 return 0; 1527 } 1528 1529 1530 /* 1531 * Get the allowed maximum number of hairpin queues. 1532 * *pid return the port id which has minimal value of 1533 * max_hairpin_queues in all ports. 1534 */ 1535 queueid_t 1536 get_allowed_max_nb_hairpinq(portid_t *pid) 1537 { 1538 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1539 portid_t pi; 1540 struct rte_eth_hairpin_cap cap; 1541 1542 RTE_ETH_FOREACH_DEV(pi) { 1543 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1544 *pid = pi; 1545 return 0; 1546 } 1547 if (cap.max_nb_queues < allowed_max_hairpinq) { 1548 allowed_max_hairpinq = cap.max_nb_queues; 1549 *pid = pi; 1550 } 1551 } 1552 return allowed_max_hairpinq; 1553 } 1554 1555 /* 1556 * Check input hairpin is valid or not. 1557 * If input hairpin is not greater than any of maximum number 1558 * of hairpin queues of all ports, it is valid. 1559 * if valid, return 0, else return -1 1560 */ 1561 int 1562 check_nb_hairpinq(queueid_t hairpinq) 1563 { 1564 queueid_t allowed_max_hairpinq; 1565 portid_t pid = 0; 1566 1567 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1568 if (hairpinq > allowed_max_hairpinq) { 1569 fprintf(stderr, 1570 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1571 hairpinq, allowed_max_hairpinq, pid); 1572 return -1; 1573 } 1574 return 0; 1575 } 1576 1577 static int 1578 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1579 { 1580 uint32_t eth_overhead; 1581 1582 if (dev_info->max_mtu != UINT16_MAX && 1583 dev_info->max_rx_pktlen > dev_info->max_mtu) 1584 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1585 else 1586 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1587 1588 return eth_overhead; 1589 } 1590 1591 static void 1592 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1593 { 1594 struct rte_port *port = &ports[pid]; 1595 int ret; 1596 int i; 1597 1598 eth_rx_metadata_negotiate_mp(pid); 1599 1600 port->dev_conf.txmode = tx_mode; 1601 port->dev_conf.rxmode = rx_mode; 1602 1603 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1604 if (ret != 0) 1605 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1606 1607 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1608 port->dev_conf.txmode.offloads &= 1609 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1610 1611 /* Apply Rx offloads configuration */ 1612 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1613 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1614 /* Apply Tx offloads configuration */ 1615 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1616 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1617 1618 if (eth_link_speed) 1619 port->dev_conf.link_speeds = eth_link_speed; 1620 1621 if (max_rx_pkt_len) 1622 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1623 get_eth_overhead(&port->dev_info); 1624 1625 /* set flag to initialize port/queue */ 1626 port->need_reconfig = 1; 1627 port->need_reconfig_queues = 1; 1628 port->socket_id = socket_id; 1629 port->tx_metadata = 0; 1630 1631 /* 1632 * Check for maximum number of segments per MTU. 1633 * Accordingly update the mbuf data size. 1634 */ 1635 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1636 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1637 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1638 uint16_t mtu; 1639 1640 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1641 uint16_t data_size = (mtu + eth_overhead) / 1642 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1643 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1644 1645 if (buffer_size > mbuf_data_size[0]) { 1646 mbuf_data_size[0] = buffer_size; 1647 TESTPMD_LOG(WARNING, 1648 "Configured mbuf size of the first segment %hu\n", 1649 mbuf_data_size[0]); 1650 } 1651 } 1652 } 1653 } 1654 1655 static void 1656 init_config(void) 1657 { 1658 portid_t pid; 1659 struct rte_mempool *mbp; 1660 unsigned int nb_mbuf_per_pool; 1661 lcoreid_t lc_id; 1662 #ifdef RTE_LIB_GRO 1663 struct rte_gro_param gro_param; 1664 #endif 1665 #ifdef RTE_LIB_GSO 1666 uint32_t gso_types; 1667 #endif 1668 1669 /* Configuration of logical cores. */ 1670 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1671 sizeof(struct fwd_lcore *) * nb_lcores, 1672 RTE_CACHE_LINE_SIZE); 1673 if (fwd_lcores == NULL) { 1674 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1675 "failed\n", nb_lcores); 1676 } 1677 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1678 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1679 sizeof(struct fwd_lcore), 1680 RTE_CACHE_LINE_SIZE); 1681 if (fwd_lcores[lc_id] == NULL) { 1682 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1683 "failed\n"); 1684 } 1685 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1686 } 1687 1688 RTE_ETH_FOREACH_DEV(pid) { 1689 uint32_t socket_id; 1690 1691 if (numa_support) { 1692 socket_id = port_numa[pid]; 1693 if (port_numa[pid] == NUMA_NO_CONFIG) { 1694 socket_id = rte_eth_dev_socket_id(pid); 1695 1696 /* 1697 * if socket_id is invalid, 1698 * set to the first available socket. 1699 */ 1700 if (check_socket_id(socket_id) < 0) 1701 socket_id = socket_ids[0]; 1702 } 1703 } else { 1704 socket_id = (socket_num == UMA_NO_CONFIG) ? 1705 0 : socket_num; 1706 } 1707 /* Apply default TxRx configuration for all ports */ 1708 init_config_port_offloads(pid, socket_id); 1709 } 1710 /* 1711 * Create pools of mbuf. 1712 * If NUMA support is disabled, create a single pool of mbuf in 1713 * socket 0 memory by default. 1714 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1715 * 1716 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1717 * nb_txd can be configured at run time. 1718 */ 1719 if (param_total_num_mbufs) 1720 nb_mbuf_per_pool = param_total_num_mbufs; 1721 else { 1722 nb_mbuf_per_pool = RX_DESC_MAX + 1723 (nb_lcores * mb_mempool_cache) + 1724 TX_DESC_MAX + MAX_PKT_BURST; 1725 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1726 } 1727 1728 if (numa_support) { 1729 uint8_t i, j; 1730 1731 for (i = 0; i < num_sockets; i++) 1732 for (j = 0; j < mbuf_data_size_n; j++) 1733 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1734 mbuf_pool_create(mbuf_data_size[j], 1735 nb_mbuf_per_pool, 1736 socket_ids[i], j); 1737 } else { 1738 uint8_t i; 1739 1740 for (i = 0; i < mbuf_data_size_n; i++) 1741 mempools[i] = mbuf_pool_create 1742 (mbuf_data_size[i], 1743 nb_mbuf_per_pool, 1744 socket_num == UMA_NO_CONFIG ? 1745 0 : socket_num, i); 1746 } 1747 1748 init_port_config(); 1749 1750 #ifdef RTE_LIB_GSO 1751 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1752 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1753 #endif 1754 /* 1755 * Records which Mbuf pool to use by each logical core, if needed. 1756 */ 1757 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1758 mbp = mbuf_pool_find( 1759 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1760 1761 if (mbp == NULL) 1762 mbp = mbuf_pool_find(0, 0); 1763 fwd_lcores[lc_id]->mbp = mbp; 1764 #ifdef RTE_LIB_GSO 1765 /* initialize GSO context */ 1766 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1767 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1768 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1769 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1770 RTE_ETHER_CRC_LEN; 1771 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1772 #endif 1773 } 1774 1775 fwd_config_setup(); 1776 1777 #ifdef RTE_LIB_GRO 1778 /* create a gro context for each lcore */ 1779 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1780 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1781 gro_param.max_item_per_flow = MAX_PKT_BURST; 1782 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1783 gro_param.socket_id = rte_lcore_to_socket_id( 1784 fwd_lcores_cpuids[lc_id]); 1785 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1786 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1787 rte_exit(EXIT_FAILURE, 1788 "rte_gro_ctx_create() failed\n"); 1789 } 1790 } 1791 #endif 1792 } 1793 1794 1795 void 1796 reconfig(portid_t new_port_id, unsigned socket_id) 1797 { 1798 /* Reconfiguration of Ethernet ports. */ 1799 init_config_port_offloads(new_port_id, socket_id); 1800 init_port_config(); 1801 } 1802 1803 int 1804 init_fwd_streams(void) 1805 { 1806 portid_t pid; 1807 struct rte_port *port; 1808 streamid_t sm_id, nb_fwd_streams_new; 1809 queueid_t q; 1810 1811 /* set socket id according to numa or not */ 1812 RTE_ETH_FOREACH_DEV(pid) { 1813 port = &ports[pid]; 1814 if (nb_rxq > port->dev_info.max_rx_queues) { 1815 fprintf(stderr, 1816 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1817 nb_rxq, port->dev_info.max_rx_queues); 1818 return -1; 1819 } 1820 if (nb_txq > port->dev_info.max_tx_queues) { 1821 fprintf(stderr, 1822 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1823 nb_txq, port->dev_info.max_tx_queues); 1824 return -1; 1825 } 1826 if (numa_support) { 1827 if (port_numa[pid] != NUMA_NO_CONFIG) 1828 port->socket_id = port_numa[pid]; 1829 else { 1830 port->socket_id = rte_eth_dev_socket_id(pid); 1831 1832 /* 1833 * if socket_id is invalid, 1834 * set to the first available socket. 1835 */ 1836 if (check_socket_id(port->socket_id) < 0) 1837 port->socket_id = socket_ids[0]; 1838 } 1839 } 1840 else { 1841 if (socket_num == UMA_NO_CONFIG) 1842 port->socket_id = 0; 1843 else 1844 port->socket_id = socket_num; 1845 } 1846 } 1847 1848 q = RTE_MAX(nb_rxq, nb_txq); 1849 if (q == 0) { 1850 fprintf(stderr, 1851 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1852 return -1; 1853 } 1854 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1855 if (nb_fwd_streams_new == nb_fwd_streams) 1856 return 0; 1857 /* clear the old */ 1858 if (fwd_streams != NULL) { 1859 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1860 if (fwd_streams[sm_id] == NULL) 1861 continue; 1862 rte_free(fwd_streams[sm_id]); 1863 fwd_streams[sm_id] = NULL; 1864 } 1865 rte_free(fwd_streams); 1866 fwd_streams = NULL; 1867 } 1868 1869 /* init new */ 1870 nb_fwd_streams = nb_fwd_streams_new; 1871 if (nb_fwd_streams) { 1872 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1873 sizeof(struct fwd_stream *) * nb_fwd_streams, 1874 RTE_CACHE_LINE_SIZE); 1875 if (fwd_streams == NULL) 1876 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1877 " (struct fwd_stream *)) failed\n", 1878 nb_fwd_streams); 1879 1880 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1881 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1882 " struct fwd_stream", sizeof(struct fwd_stream), 1883 RTE_CACHE_LINE_SIZE); 1884 if (fwd_streams[sm_id] == NULL) 1885 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1886 "(struct fwd_stream) failed\n"); 1887 } 1888 } 1889 1890 return 0; 1891 } 1892 1893 static void 1894 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1895 { 1896 uint64_t total_burst, sburst; 1897 uint64_t nb_burst; 1898 uint64_t burst_stats[4]; 1899 uint16_t pktnb_stats[4]; 1900 uint16_t nb_pkt; 1901 int burst_percent[4], sburstp; 1902 int i; 1903 1904 /* 1905 * First compute the total number of packet bursts and the 1906 * two highest numbers of bursts of the same number of packets. 1907 */ 1908 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1909 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1910 1911 /* Show stats for 0 burst size always */ 1912 total_burst = pbs->pkt_burst_spread[0]; 1913 burst_stats[0] = pbs->pkt_burst_spread[0]; 1914 pktnb_stats[0] = 0; 1915 1916 /* Find the next 2 burst sizes with highest occurrences. */ 1917 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1918 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1919 1920 if (nb_burst == 0) 1921 continue; 1922 1923 total_burst += nb_burst; 1924 1925 if (nb_burst > burst_stats[1]) { 1926 burst_stats[2] = burst_stats[1]; 1927 pktnb_stats[2] = pktnb_stats[1]; 1928 burst_stats[1] = nb_burst; 1929 pktnb_stats[1] = nb_pkt; 1930 } else if (nb_burst > burst_stats[2]) { 1931 burst_stats[2] = nb_burst; 1932 pktnb_stats[2] = nb_pkt; 1933 } 1934 } 1935 if (total_burst == 0) 1936 return; 1937 1938 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1939 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1940 if (i == 3) { 1941 printf("%d%% of other]\n", 100 - sburstp); 1942 return; 1943 } 1944 1945 sburst += burst_stats[i]; 1946 if (sburst == total_burst) { 1947 printf("%d%% of %d pkts]\n", 1948 100 - sburstp, (int) pktnb_stats[i]); 1949 return; 1950 } 1951 1952 burst_percent[i] = 1953 (double)burst_stats[i] / total_burst * 100; 1954 printf("%d%% of %d pkts + ", 1955 burst_percent[i], (int) pktnb_stats[i]); 1956 sburstp += burst_percent[i]; 1957 } 1958 } 1959 1960 static void 1961 fwd_stream_stats_display(streamid_t stream_id) 1962 { 1963 struct fwd_stream *fs; 1964 static const char *fwd_top_stats_border = "-------"; 1965 1966 fs = fwd_streams[stream_id]; 1967 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1968 (fs->fwd_dropped == 0)) 1969 return; 1970 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1971 "TX Port=%2d/Queue=%2d %s\n", 1972 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1973 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1974 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1975 " TX-dropped: %-14"PRIu64, 1976 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1977 1978 /* if checksum mode */ 1979 if (cur_fwd_eng == &csum_fwd_engine) { 1980 printf(" RX- bad IP checksum: %-14"PRIu64 1981 " Rx- bad L4 checksum: %-14"PRIu64 1982 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1983 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1984 fs->rx_bad_outer_l4_csum); 1985 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1986 fs->rx_bad_outer_ip_csum); 1987 } else { 1988 printf("\n"); 1989 } 1990 1991 if (record_burst_stats) { 1992 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1993 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1994 } 1995 } 1996 1997 void 1998 fwd_stats_display(void) 1999 { 2000 static const char *fwd_stats_border = "----------------------"; 2001 static const char *acc_stats_border = "+++++++++++++++"; 2002 struct { 2003 struct fwd_stream *rx_stream; 2004 struct fwd_stream *tx_stream; 2005 uint64_t tx_dropped; 2006 uint64_t rx_bad_ip_csum; 2007 uint64_t rx_bad_l4_csum; 2008 uint64_t rx_bad_outer_l4_csum; 2009 uint64_t rx_bad_outer_ip_csum; 2010 } ports_stats[RTE_MAX_ETHPORTS]; 2011 uint64_t total_rx_dropped = 0; 2012 uint64_t total_tx_dropped = 0; 2013 uint64_t total_rx_nombuf = 0; 2014 struct rte_eth_stats stats; 2015 uint64_t fwd_cycles = 0; 2016 uint64_t total_recv = 0; 2017 uint64_t total_xmit = 0; 2018 struct rte_port *port; 2019 streamid_t sm_id; 2020 portid_t pt_id; 2021 int ret; 2022 int i; 2023 2024 memset(ports_stats, 0, sizeof(ports_stats)); 2025 2026 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2027 struct fwd_stream *fs = fwd_streams[sm_id]; 2028 2029 if (cur_fwd_config.nb_fwd_streams > 2030 cur_fwd_config.nb_fwd_ports) { 2031 fwd_stream_stats_display(sm_id); 2032 } else { 2033 ports_stats[fs->tx_port].tx_stream = fs; 2034 ports_stats[fs->rx_port].rx_stream = fs; 2035 } 2036 2037 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2038 2039 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2040 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2041 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2042 fs->rx_bad_outer_l4_csum; 2043 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2044 fs->rx_bad_outer_ip_csum; 2045 2046 if (record_core_cycles) 2047 fwd_cycles += fs->core_cycles; 2048 } 2049 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2050 pt_id = fwd_ports_ids[i]; 2051 port = &ports[pt_id]; 2052 2053 ret = rte_eth_stats_get(pt_id, &stats); 2054 if (ret != 0) { 2055 fprintf(stderr, 2056 "%s: Error: failed to get stats (port %u): %d", 2057 __func__, pt_id, ret); 2058 continue; 2059 } 2060 stats.ipackets -= port->stats.ipackets; 2061 stats.opackets -= port->stats.opackets; 2062 stats.ibytes -= port->stats.ibytes; 2063 stats.obytes -= port->stats.obytes; 2064 stats.imissed -= port->stats.imissed; 2065 stats.oerrors -= port->stats.oerrors; 2066 stats.rx_nombuf -= port->stats.rx_nombuf; 2067 2068 total_recv += stats.ipackets; 2069 total_xmit += stats.opackets; 2070 total_rx_dropped += stats.imissed; 2071 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2072 total_tx_dropped += stats.oerrors; 2073 total_rx_nombuf += stats.rx_nombuf; 2074 2075 printf("\n %s Forward statistics for port %-2d %s\n", 2076 fwd_stats_border, pt_id, fwd_stats_border); 2077 2078 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2079 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2080 stats.ipackets + stats.imissed); 2081 2082 if (cur_fwd_eng == &csum_fwd_engine) { 2083 printf(" Bad-ipcsum: %-14"PRIu64 2084 " Bad-l4csum: %-14"PRIu64 2085 "Bad-outer-l4csum: %-14"PRIu64"\n", 2086 ports_stats[pt_id].rx_bad_ip_csum, 2087 ports_stats[pt_id].rx_bad_l4_csum, 2088 ports_stats[pt_id].rx_bad_outer_l4_csum); 2089 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2090 ports_stats[pt_id].rx_bad_outer_ip_csum); 2091 } 2092 if (stats.ierrors + stats.rx_nombuf > 0) { 2093 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2094 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2095 } 2096 2097 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2098 "TX-total: %-"PRIu64"\n", 2099 stats.opackets, ports_stats[pt_id].tx_dropped, 2100 stats.opackets + ports_stats[pt_id].tx_dropped); 2101 2102 if (record_burst_stats) { 2103 if (ports_stats[pt_id].rx_stream) 2104 pkt_burst_stats_display("RX", 2105 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2106 if (ports_stats[pt_id].tx_stream) 2107 pkt_burst_stats_display("TX", 2108 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2109 } 2110 2111 printf(" %s--------------------------------%s\n", 2112 fwd_stats_border, fwd_stats_border); 2113 } 2114 2115 printf("\n %s Accumulated forward statistics for all ports" 2116 "%s\n", 2117 acc_stats_border, acc_stats_border); 2118 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2119 "%-"PRIu64"\n" 2120 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2121 "%-"PRIu64"\n", 2122 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2123 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2124 if (total_rx_nombuf > 0) 2125 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2126 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2127 "%s\n", 2128 acc_stats_border, acc_stats_border); 2129 if (record_core_cycles) { 2130 #define CYC_PER_MHZ 1E6 2131 if (total_recv > 0 || total_xmit > 0) { 2132 uint64_t total_pkts = 0; 2133 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2134 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2135 total_pkts = total_xmit; 2136 else 2137 total_pkts = total_recv; 2138 2139 printf("\n CPU cycles/packet=%.2F (total cycles=" 2140 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2141 " MHz Clock\n", 2142 (double) fwd_cycles / total_pkts, 2143 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2144 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2145 } 2146 } 2147 } 2148 2149 void 2150 fwd_stats_reset(void) 2151 { 2152 streamid_t sm_id; 2153 portid_t pt_id; 2154 int ret; 2155 int i; 2156 2157 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2158 pt_id = fwd_ports_ids[i]; 2159 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2160 if (ret != 0) 2161 fprintf(stderr, 2162 "%s: Error: failed to clear stats (port %u):%d", 2163 __func__, pt_id, ret); 2164 } 2165 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2166 struct fwd_stream *fs = fwd_streams[sm_id]; 2167 2168 fs->rx_packets = 0; 2169 fs->tx_packets = 0; 2170 fs->fwd_dropped = 0; 2171 fs->rx_bad_ip_csum = 0; 2172 fs->rx_bad_l4_csum = 0; 2173 fs->rx_bad_outer_l4_csum = 0; 2174 fs->rx_bad_outer_ip_csum = 0; 2175 2176 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2177 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2178 fs->core_cycles = 0; 2179 } 2180 } 2181 2182 static void 2183 flush_fwd_rx_queues(void) 2184 { 2185 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2186 portid_t rxp; 2187 portid_t port_id; 2188 queueid_t rxq; 2189 uint16_t nb_rx; 2190 uint16_t i; 2191 uint8_t j; 2192 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2193 uint64_t timer_period; 2194 2195 if (num_procs > 1) { 2196 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2197 return; 2198 } 2199 2200 /* convert to number of cycles */ 2201 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2202 2203 for (j = 0; j < 2; j++) { 2204 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2205 for (rxq = 0; rxq < nb_rxq; rxq++) { 2206 port_id = fwd_ports_ids[rxp]; 2207 2208 /* Polling stopped queues is prohibited. */ 2209 if (ports[port_id].rxq[rxq].state == 2210 RTE_ETH_QUEUE_STATE_STOPPED) 2211 continue; 2212 2213 /** 2214 * testpmd can stuck in the below do while loop 2215 * if rte_eth_rx_burst() always returns nonzero 2216 * packets. So timer is added to exit this loop 2217 * after 1sec timer expiry. 2218 */ 2219 prev_tsc = rte_rdtsc(); 2220 do { 2221 nb_rx = rte_eth_rx_burst(port_id, rxq, 2222 pkts_burst, MAX_PKT_BURST); 2223 for (i = 0; i < nb_rx; i++) 2224 rte_pktmbuf_free(pkts_burst[i]); 2225 2226 cur_tsc = rte_rdtsc(); 2227 diff_tsc = cur_tsc - prev_tsc; 2228 timer_tsc += diff_tsc; 2229 } while ((nb_rx > 0) && 2230 (timer_tsc < timer_period)); 2231 timer_tsc = 0; 2232 } 2233 } 2234 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2235 } 2236 } 2237 2238 static void 2239 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2240 { 2241 struct fwd_stream **fsm; 2242 streamid_t nb_fs; 2243 streamid_t sm_id; 2244 #ifdef RTE_LIB_BITRATESTATS 2245 uint64_t tics_per_1sec; 2246 uint64_t tics_datum; 2247 uint64_t tics_current; 2248 uint16_t i, cnt_ports; 2249 2250 cnt_ports = nb_ports; 2251 tics_datum = rte_rdtsc(); 2252 tics_per_1sec = rte_get_timer_hz(); 2253 #endif 2254 fsm = &fwd_streams[fc->stream_idx]; 2255 nb_fs = fc->stream_nb; 2256 do { 2257 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2258 if (!fsm[sm_id]->disabled) 2259 (*pkt_fwd)(fsm[sm_id]); 2260 #ifdef RTE_LIB_BITRATESTATS 2261 if (bitrate_enabled != 0 && 2262 bitrate_lcore_id == rte_lcore_id()) { 2263 tics_current = rte_rdtsc(); 2264 if (tics_current - tics_datum >= tics_per_1sec) { 2265 /* Periodic bitrate calculation */ 2266 for (i = 0; i < cnt_ports; i++) 2267 rte_stats_bitrate_calc(bitrate_data, 2268 ports_ids[i]); 2269 tics_datum = tics_current; 2270 } 2271 } 2272 #endif 2273 #ifdef RTE_LIB_LATENCYSTATS 2274 if (latencystats_enabled != 0 && 2275 latencystats_lcore_id == rte_lcore_id()) 2276 rte_latencystats_update(); 2277 #endif 2278 2279 } while (! fc->stopped); 2280 } 2281 2282 static int 2283 start_pkt_forward_on_core(void *fwd_arg) 2284 { 2285 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2286 cur_fwd_config.fwd_eng->packet_fwd); 2287 return 0; 2288 } 2289 2290 /* 2291 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2292 * Used to start communication flows in network loopback test configurations. 2293 */ 2294 static int 2295 run_one_txonly_burst_on_core(void *fwd_arg) 2296 { 2297 struct fwd_lcore *fwd_lc; 2298 struct fwd_lcore tmp_lcore; 2299 2300 fwd_lc = (struct fwd_lcore *) fwd_arg; 2301 tmp_lcore = *fwd_lc; 2302 tmp_lcore.stopped = 1; 2303 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2304 return 0; 2305 } 2306 2307 /* 2308 * Launch packet forwarding: 2309 * - Setup per-port forwarding context. 2310 * - launch logical cores with their forwarding configuration. 2311 */ 2312 static void 2313 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2314 { 2315 unsigned int i; 2316 unsigned int lc_id; 2317 int diag; 2318 2319 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2320 lc_id = fwd_lcores_cpuids[i]; 2321 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2322 fwd_lcores[i]->stopped = 0; 2323 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2324 fwd_lcores[i], lc_id); 2325 if (diag != 0) 2326 fprintf(stderr, 2327 "launch lcore %u failed - diag=%d\n", 2328 lc_id, diag); 2329 } 2330 } 2331 } 2332 2333 /* 2334 * Launch packet forwarding configuration. 2335 */ 2336 void 2337 start_packet_forwarding(int with_tx_first) 2338 { 2339 port_fwd_begin_t port_fwd_begin; 2340 port_fwd_end_t port_fwd_end; 2341 stream_init_t stream_init = cur_fwd_eng->stream_init; 2342 unsigned int i; 2343 2344 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2345 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2346 2347 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2348 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2349 2350 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2351 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2352 (!nb_rxq || !nb_txq)) 2353 rte_exit(EXIT_FAILURE, 2354 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2355 cur_fwd_eng->fwd_mode_name); 2356 2357 if (all_ports_started() == 0) { 2358 fprintf(stderr, "Not all ports were started\n"); 2359 return; 2360 } 2361 if (test_done == 0) { 2362 fprintf(stderr, "Packet forwarding already started\n"); 2363 return; 2364 } 2365 2366 fwd_config_setup(); 2367 2368 pkt_fwd_config_display(&cur_fwd_config); 2369 if (!pkt_fwd_shared_rxq_check()) 2370 return; 2371 2372 if (stream_init != NULL) 2373 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2374 stream_init(fwd_streams[i]); 2375 2376 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2377 if (port_fwd_begin != NULL) { 2378 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2379 if (port_fwd_begin(fwd_ports_ids[i])) { 2380 fprintf(stderr, 2381 "Packet forwarding is not ready\n"); 2382 return; 2383 } 2384 } 2385 } 2386 2387 if (with_tx_first) { 2388 port_fwd_begin = tx_only_engine.port_fwd_begin; 2389 if (port_fwd_begin != NULL) { 2390 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2391 if (port_fwd_begin(fwd_ports_ids[i])) { 2392 fprintf(stderr, 2393 "Packet forwarding is not ready\n"); 2394 return; 2395 } 2396 } 2397 } 2398 } 2399 2400 test_done = 0; 2401 2402 if(!no_flush_rx) 2403 flush_fwd_rx_queues(); 2404 2405 rxtx_config_display(); 2406 2407 fwd_stats_reset(); 2408 if (with_tx_first) { 2409 while (with_tx_first--) { 2410 launch_packet_forwarding( 2411 run_one_txonly_burst_on_core); 2412 rte_eal_mp_wait_lcore(); 2413 } 2414 port_fwd_end = tx_only_engine.port_fwd_end; 2415 if (port_fwd_end != NULL) { 2416 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2417 (*port_fwd_end)(fwd_ports_ids[i]); 2418 } 2419 } 2420 launch_packet_forwarding(start_pkt_forward_on_core); 2421 } 2422 2423 void 2424 stop_packet_forwarding(void) 2425 { 2426 port_fwd_end_t port_fwd_end; 2427 lcoreid_t lc_id; 2428 portid_t pt_id; 2429 int i; 2430 2431 if (test_done) { 2432 fprintf(stderr, "Packet forwarding not started\n"); 2433 return; 2434 } 2435 printf("Telling cores to stop..."); 2436 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2437 fwd_lcores[lc_id]->stopped = 1; 2438 printf("\nWaiting for lcores to finish...\n"); 2439 rte_eal_mp_wait_lcore(); 2440 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2441 if (port_fwd_end != NULL) { 2442 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2443 pt_id = fwd_ports_ids[i]; 2444 (*port_fwd_end)(pt_id); 2445 } 2446 } 2447 2448 fwd_stats_display(); 2449 2450 printf("\nDone.\n"); 2451 test_done = 1; 2452 } 2453 2454 void 2455 dev_set_link_up(portid_t pid) 2456 { 2457 if (rte_eth_dev_set_link_up(pid) < 0) 2458 fprintf(stderr, "\nSet link up fail.\n"); 2459 } 2460 2461 void 2462 dev_set_link_down(portid_t pid) 2463 { 2464 if (rte_eth_dev_set_link_down(pid) < 0) 2465 fprintf(stderr, "\nSet link down fail.\n"); 2466 } 2467 2468 static int 2469 all_ports_started(void) 2470 { 2471 portid_t pi; 2472 struct rte_port *port; 2473 2474 RTE_ETH_FOREACH_DEV(pi) { 2475 port = &ports[pi]; 2476 /* Check if there is a port which is not started */ 2477 if ((port->port_status != RTE_PORT_STARTED) && 2478 (port->slave_flag == 0)) 2479 return 0; 2480 } 2481 2482 /* No port is not started */ 2483 return 1; 2484 } 2485 2486 int 2487 port_is_stopped(portid_t port_id) 2488 { 2489 struct rte_port *port = &ports[port_id]; 2490 2491 if ((port->port_status != RTE_PORT_STOPPED) && 2492 (port->slave_flag == 0)) 2493 return 0; 2494 return 1; 2495 } 2496 2497 int 2498 all_ports_stopped(void) 2499 { 2500 portid_t pi; 2501 2502 RTE_ETH_FOREACH_DEV(pi) { 2503 if (!port_is_stopped(pi)) 2504 return 0; 2505 } 2506 2507 return 1; 2508 } 2509 2510 int 2511 port_is_started(portid_t port_id) 2512 { 2513 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2514 return 0; 2515 2516 if (ports[port_id].port_status != RTE_PORT_STARTED) 2517 return 0; 2518 2519 return 1; 2520 } 2521 2522 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2523 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2524 2525 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2526 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2527 2528 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2529 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2530 2531 2532 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2533 static int 2534 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2535 { 2536 queueid_t qi; 2537 struct rte_eth_hairpin_conf hairpin_conf = { 2538 .peer_count = 1, 2539 }; 2540 int i; 2541 int diag; 2542 struct rte_port *port = &ports[pi]; 2543 uint16_t peer_rx_port = pi; 2544 uint16_t peer_tx_port = pi; 2545 uint32_t manual = 1; 2546 uint32_t tx_exp = hairpin_mode & 0x10; 2547 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2548 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2549 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2550 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2551 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2552 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2553 2554 if (!(hairpin_mode & 0xf)) { 2555 peer_rx_port = pi; 2556 peer_tx_port = pi; 2557 manual = 0; 2558 } else if (hairpin_mode & 0x1) { 2559 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2560 RTE_ETH_DEV_NO_OWNER); 2561 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2562 peer_tx_port = rte_eth_find_next_owned_by(0, 2563 RTE_ETH_DEV_NO_OWNER); 2564 if (p_pi != RTE_MAX_ETHPORTS) { 2565 peer_rx_port = p_pi; 2566 } else { 2567 uint16_t next_pi; 2568 2569 /* Last port will be the peer RX port of the first. */ 2570 RTE_ETH_FOREACH_DEV(next_pi) 2571 peer_rx_port = next_pi; 2572 } 2573 manual = 1; 2574 } else if (hairpin_mode & 0x2) { 2575 if (cnt_pi & 0x1) { 2576 peer_rx_port = p_pi; 2577 } else { 2578 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2579 RTE_ETH_DEV_NO_OWNER); 2580 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2581 peer_rx_port = pi; 2582 } 2583 peer_tx_port = peer_rx_port; 2584 manual = 1; 2585 } 2586 2587 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2588 hairpin_conf.peers[0].port = peer_rx_port; 2589 hairpin_conf.peers[0].queue = i + nb_rxq; 2590 hairpin_conf.manual_bind = !!manual; 2591 hairpin_conf.tx_explicit = !!tx_exp; 2592 hairpin_conf.force_memory = !!tx_force_memory; 2593 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2594 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2595 diag = rte_eth_tx_hairpin_queue_setup 2596 (pi, qi, nb_txd, &hairpin_conf); 2597 i++; 2598 if (diag == 0) 2599 continue; 2600 2601 /* Fail to setup rx queue, return */ 2602 if (port->port_status == RTE_PORT_HANDLING) 2603 port->port_status = RTE_PORT_STOPPED; 2604 else 2605 fprintf(stderr, 2606 "Port %d can not be set back to stopped\n", pi); 2607 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2608 pi); 2609 /* try to reconfigure queues next time */ 2610 port->need_reconfig_queues = 1; 2611 return -1; 2612 } 2613 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2614 hairpin_conf.peers[0].port = peer_tx_port; 2615 hairpin_conf.peers[0].queue = i + nb_txq; 2616 hairpin_conf.manual_bind = !!manual; 2617 hairpin_conf.tx_explicit = !!tx_exp; 2618 hairpin_conf.force_memory = !!rx_force_memory; 2619 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2620 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2621 diag = rte_eth_rx_hairpin_queue_setup 2622 (pi, qi, nb_rxd, &hairpin_conf); 2623 i++; 2624 if (diag == 0) 2625 continue; 2626 2627 /* Fail to setup rx queue, return */ 2628 if (port->port_status == RTE_PORT_HANDLING) 2629 port->port_status = RTE_PORT_STOPPED; 2630 else 2631 fprintf(stderr, 2632 "Port %d can not be set back to stopped\n", pi); 2633 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2634 pi); 2635 /* try to reconfigure queues next time */ 2636 port->need_reconfig_queues = 1; 2637 return -1; 2638 } 2639 return 0; 2640 } 2641 2642 /* Configure the Rx with optional split. */ 2643 int 2644 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2645 uint16_t nb_rx_desc, unsigned int socket_id, 2646 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2647 { 2648 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2649 unsigned int i, mp_n; 2650 int ret; 2651 2652 if (rx_pkt_nb_segs <= 1 || 2653 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2654 rx_conf->rx_seg = NULL; 2655 rx_conf->rx_nseg = 0; 2656 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2657 nb_rx_desc, socket_id, 2658 rx_conf, mp); 2659 goto exit; 2660 } 2661 for (i = 0; i < rx_pkt_nb_segs; i++) { 2662 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2663 struct rte_mempool *mpx; 2664 /* 2665 * Use last valid pool for the segments with number 2666 * exceeding the pool index. 2667 */ 2668 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2669 mpx = mbuf_pool_find(socket_id, mp_n); 2670 /* Handle zero as mbuf data buffer size. */ 2671 rx_seg->length = rx_pkt_seg_lengths[i] ? 2672 rx_pkt_seg_lengths[i] : 2673 mbuf_data_size[mp_n]; 2674 rx_seg->offset = i < rx_pkt_nb_offs ? 2675 rx_pkt_seg_offsets[i] : 0; 2676 rx_seg->mp = mpx ? mpx : mp; 2677 } 2678 rx_conf->rx_nseg = rx_pkt_nb_segs; 2679 rx_conf->rx_seg = rx_useg; 2680 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2681 socket_id, rx_conf, NULL); 2682 rx_conf->rx_seg = NULL; 2683 rx_conf->rx_nseg = 0; 2684 exit: 2685 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2686 RTE_ETH_QUEUE_STATE_STOPPED : 2687 RTE_ETH_QUEUE_STATE_STARTED; 2688 return ret; 2689 } 2690 2691 static int 2692 alloc_xstats_display_info(portid_t pi) 2693 { 2694 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2695 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2696 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2697 2698 if (xstats_display_num == 0) 2699 return 0; 2700 2701 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2702 if (*ids_supp == NULL) 2703 goto fail_ids_supp; 2704 2705 *prev_values = calloc(xstats_display_num, 2706 sizeof(**prev_values)); 2707 if (*prev_values == NULL) 2708 goto fail_prev_values; 2709 2710 *curr_values = calloc(xstats_display_num, 2711 sizeof(**curr_values)); 2712 if (*curr_values == NULL) 2713 goto fail_curr_values; 2714 2715 ports[pi].xstats_info.allocated = true; 2716 2717 return 0; 2718 2719 fail_curr_values: 2720 free(*prev_values); 2721 fail_prev_values: 2722 free(*ids_supp); 2723 fail_ids_supp: 2724 return -ENOMEM; 2725 } 2726 2727 static void 2728 free_xstats_display_info(portid_t pi) 2729 { 2730 if (!ports[pi].xstats_info.allocated) 2731 return; 2732 free(ports[pi].xstats_info.ids_supp); 2733 free(ports[pi].xstats_info.prev_values); 2734 free(ports[pi].xstats_info.curr_values); 2735 ports[pi].xstats_info.allocated = false; 2736 } 2737 2738 /** Fill helper structures for specified port to show extended statistics. */ 2739 static void 2740 fill_xstats_display_info_for_port(portid_t pi) 2741 { 2742 unsigned int stat, stat_supp; 2743 const char *xstat_name; 2744 struct rte_port *port; 2745 uint64_t *ids_supp; 2746 int rc; 2747 2748 if (xstats_display_num == 0) 2749 return; 2750 2751 if (pi == (portid_t)RTE_PORT_ALL) { 2752 fill_xstats_display_info(); 2753 return; 2754 } 2755 2756 port = &ports[pi]; 2757 if (port->port_status != RTE_PORT_STARTED) 2758 return; 2759 2760 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2761 rte_exit(EXIT_FAILURE, 2762 "Failed to allocate xstats display memory\n"); 2763 2764 ids_supp = port->xstats_info.ids_supp; 2765 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2766 xstat_name = xstats_display[stat].name; 2767 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2768 ids_supp + stat_supp); 2769 if (rc != 0) { 2770 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2771 xstat_name, pi, stat); 2772 continue; 2773 } 2774 stat_supp++; 2775 } 2776 2777 port->xstats_info.ids_supp_sz = stat_supp; 2778 } 2779 2780 /** Fill helper structures for all ports to show extended statistics. */ 2781 static void 2782 fill_xstats_display_info(void) 2783 { 2784 portid_t pi; 2785 2786 if (xstats_display_num == 0) 2787 return; 2788 2789 RTE_ETH_FOREACH_DEV(pi) 2790 fill_xstats_display_info_for_port(pi); 2791 } 2792 2793 int 2794 start_port(portid_t pid) 2795 { 2796 int diag, need_check_link_status = -1; 2797 portid_t pi; 2798 portid_t p_pi = RTE_MAX_ETHPORTS; 2799 portid_t pl[RTE_MAX_ETHPORTS]; 2800 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2801 uint16_t cnt_pi = 0; 2802 uint16_t cfg_pi = 0; 2803 int peer_pi; 2804 queueid_t qi; 2805 struct rte_port *port; 2806 struct rte_eth_hairpin_cap cap; 2807 2808 if (port_id_is_invalid(pid, ENABLED_WARN)) 2809 return 0; 2810 2811 RTE_ETH_FOREACH_DEV(pi) { 2812 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2813 continue; 2814 2815 if (port_is_bonding_slave(pi)) { 2816 fprintf(stderr, 2817 "Please remove port %d from bonded device.\n", 2818 pi); 2819 continue; 2820 } 2821 2822 need_check_link_status = 0; 2823 port = &ports[pi]; 2824 if (port->port_status == RTE_PORT_STOPPED) 2825 port->port_status = RTE_PORT_HANDLING; 2826 else { 2827 fprintf(stderr, "Port %d is now not stopped\n", pi); 2828 continue; 2829 } 2830 2831 if (port->need_reconfig > 0) { 2832 struct rte_eth_conf dev_conf; 2833 int k; 2834 2835 port->need_reconfig = 0; 2836 2837 if (flow_isolate_all) { 2838 int ret = port_flow_isolate(pi, 1); 2839 if (ret) { 2840 fprintf(stderr, 2841 "Failed to apply isolated mode on port %d\n", 2842 pi); 2843 return -1; 2844 } 2845 } 2846 configure_rxtx_dump_callbacks(0); 2847 printf("Configuring Port %d (socket %u)\n", pi, 2848 port->socket_id); 2849 if (nb_hairpinq > 0 && 2850 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2851 fprintf(stderr, 2852 "Port %d doesn't support hairpin queues\n", 2853 pi); 2854 return -1; 2855 } 2856 2857 /* configure port */ 2858 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2859 nb_txq + nb_hairpinq, 2860 &(port->dev_conf)); 2861 if (diag != 0) { 2862 if (port->port_status == RTE_PORT_HANDLING) 2863 port->port_status = RTE_PORT_STOPPED; 2864 else 2865 fprintf(stderr, 2866 "Port %d can not be set back to stopped\n", 2867 pi); 2868 fprintf(stderr, "Fail to configure port %d\n", 2869 pi); 2870 /* try to reconfigure port next time */ 2871 port->need_reconfig = 1; 2872 return -1; 2873 } 2874 /* get device configuration*/ 2875 if (0 != 2876 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2877 fprintf(stderr, 2878 "port %d can not get device configuration\n", 2879 pi); 2880 return -1; 2881 } 2882 /* Apply Rx offloads configuration */ 2883 if (dev_conf.rxmode.offloads != 2884 port->dev_conf.rxmode.offloads) { 2885 port->dev_conf.rxmode.offloads |= 2886 dev_conf.rxmode.offloads; 2887 for (k = 0; 2888 k < port->dev_info.max_rx_queues; 2889 k++) 2890 port->rxq[k].conf.offloads |= 2891 dev_conf.rxmode.offloads; 2892 } 2893 /* Apply Tx offloads configuration */ 2894 if (dev_conf.txmode.offloads != 2895 port->dev_conf.txmode.offloads) { 2896 port->dev_conf.txmode.offloads |= 2897 dev_conf.txmode.offloads; 2898 for (k = 0; 2899 k < port->dev_info.max_tx_queues; 2900 k++) 2901 port->txq[k].conf.offloads |= 2902 dev_conf.txmode.offloads; 2903 } 2904 } 2905 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2906 port->need_reconfig_queues = 0; 2907 /* setup tx queues */ 2908 for (qi = 0; qi < nb_txq; qi++) { 2909 struct rte_eth_txconf *conf = 2910 &port->txq[qi].conf; 2911 2912 if ((numa_support) && 2913 (txring_numa[pi] != NUMA_NO_CONFIG)) 2914 diag = rte_eth_tx_queue_setup(pi, qi, 2915 port->nb_tx_desc[qi], 2916 txring_numa[pi], 2917 &(port->txq[qi].conf)); 2918 else 2919 diag = rte_eth_tx_queue_setup(pi, qi, 2920 port->nb_tx_desc[qi], 2921 port->socket_id, 2922 &(port->txq[qi].conf)); 2923 2924 if (diag == 0) { 2925 port->txq[qi].state = 2926 conf->tx_deferred_start ? 2927 RTE_ETH_QUEUE_STATE_STOPPED : 2928 RTE_ETH_QUEUE_STATE_STARTED; 2929 continue; 2930 } 2931 2932 /* Fail to setup tx queue, return */ 2933 if (port->port_status == RTE_PORT_HANDLING) 2934 port->port_status = RTE_PORT_STOPPED; 2935 else 2936 fprintf(stderr, 2937 "Port %d can not be set back to stopped\n", 2938 pi); 2939 fprintf(stderr, 2940 "Fail to configure port %d tx queues\n", 2941 pi); 2942 /* try to reconfigure queues next time */ 2943 port->need_reconfig_queues = 1; 2944 return -1; 2945 } 2946 for (qi = 0; qi < nb_rxq; qi++) { 2947 /* setup rx queues */ 2948 if ((numa_support) && 2949 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2950 struct rte_mempool * mp = 2951 mbuf_pool_find 2952 (rxring_numa[pi], 0); 2953 if (mp == NULL) { 2954 fprintf(stderr, 2955 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2956 rxring_numa[pi]); 2957 return -1; 2958 } 2959 2960 diag = rx_queue_setup(pi, qi, 2961 port->nb_rx_desc[qi], 2962 rxring_numa[pi], 2963 &(port->rxq[qi].conf), 2964 mp); 2965 } else { 2966 struct rte_mempool *mp = 2967 mbuf_pool_find 2968 (port->socket_id, 0); 2969 if (mp == NULL) { 2970 fprintf(stderr, 2971 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2972 port->socket_id); 2973 return -1; 2974 } 2975 diag = rx_queue_setup(pi, qi, 2976 port->nb_rx_desc[qi], 2977 port->socket_id, 2978 &(port->rxq[qi].conf), 2979 mp); 2980 } 2981 if (diag == 0) 2982 continue; 2983 2984 /* Fail to setup rx queue, return */ 2985 if (port->port_status == RTE_PORT_HANDLING) 2986 port->port_status = RTE_PORT_STOPPED; 2987 else 2988 fprintf(stderr, 2989 "Port %d can not be set back to stopped\n", 2990 pi); 2991 fprintf(stderr, 2992 "Fail to configure port %d rx queues\n", 2993 pi); 2994 /* try to reconfigure queues next time */ 2995 port->need_reconfig_queues = 1; 2996 return -1; 2997 } 2998 /* setup hairpin queues */ 2999 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3000 return -1; 3001 } 3002 configure_rxtx_dump_callbacks(verbose_level); 3003 if (clear_ptypes) { 3004 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3005 NULL, 0); 3006 if (diag < 0) 3007 fprintf(stderr, 3008 "Port %d: Failed to disable Ptype parsing\n", 3009 pi); 3010 } 3011 3012 p_pi = pi; 3013 cnt_pi++; 3014 3015 /* start port */ 3016 diag = eth_dev_start_mp(pi); 3017 if (diag < 0) { 3018 fprintf(stderr, "Fail to start port %d: %s\n", 3019 pi, rte_strerror(-diag)); 3020 3021 /* Fail to setup rx queue, return */ 3022 if (port->port_status == RTE_PORT_HANDLING) 3023 port->port_status = RTE_PORT_STOPPED; 3024 else 3025 fprintf(stderr, 3026 "Port %d can not be set back to stopped\n", 3027 pi); 3028 continue; 3029 } 3030 3031 if (port->port_status == RTE_PORT_HANDLING) 3032 port->port_status = RTE_PORT_STARTED; 3033 else 3034 fprintf(stderr, "Port %d can not be set into started\n", 3035 pi); 3036 3037 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3038 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3039 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3040 3041 /* at least one port started, need checking link status */ 3042 need_check_link_status = 1; 3043 3044 pl[cfg_pi++] = pi; 3045 } 3046 3047 if (need_check_link_status == 1 && !no_link_check) 3048 check_all_ports_link_status(RTE_PORT_ALL); 3049 else if (need_check_link_status == 0) 3050 fprintf(stderr, "Please stop the ports first\n"); 3051 3052 if (hairpin_mode & 0xf) { 3053 uint16_t i; 3054 int j; 3055 3056 /* bind all started hairpin ports */ 3057 for (i = 0; i < cfg_pi; i++) { 3058 pi = pl[i]; 3059 /* bind current Tx to all peer Rx */ 3060 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3061 RTE_MAX_ETHPORTS, 1); 3062 if (peer_pi < 0) 3063 return peer_pi; 3064 for (j = 0; j < peer_pi; j++) { 3065 if (!port_is_started(peer_pl[j])) 3066 continue; 3067 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3068 if (diag < 0) { 3069 fprintf(stderr, 3070 "Error during binding hairpin Tx port %u to %u: %s\n", 3071 pi, peer_pl[j], 3072 rte_strerror(-diag)); 3073 return -1; 3074 } 3075 } 3076 /* bind all peer Tx to current Rx */ 3077 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3078 RTE_MAX_ETHPORTS, 0); 3079 if (peer_pi < 0) 3080 return peer_pi; 3081 for (j = 0; j < peer_pi; j++) { 3082 if (!port_is_started(peer_pl[j])) 3083 continue; 3084 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3085 if (diag < 0) { 3086 fprintf(stderr, 3087 "Error during binding hairpin Tx port %u to %u: %s\n", 3088 peer_pl[j], pi, 3089 rte_strerror(-diag)); 3090 return -1; 3091 } 3092 } 3093 } 3094 } 3095 3096 fill_xstats_display_info_for_port(pid); 3097 3098 printf("Done\n"); 3099 return 0; 3100 } 3101 3102 void 3103 stop_port(portid_t pid) 3104 { 3105 portid_t pi; 3106 struct rte_port *port; 3107 int need_check_link_status = 0; 3108 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3109 int peer_pi; 3110 3111 if (port_id_is_invalid(pid, ENABLED_WARN)) 3112 return; 3113 3114 printf("Stopping ports...\n"); 3115 3116 RTE_ETH_FOREACH_DEV(pi) { 3117 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3118 continue; 3119 3120 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3121 fprintf(stderr, 3122 "Please remove port %d from forwarding configuration.\n", 3123 pi); 3124 continue; 3125 } 3126 3127 if (port_is_bonding_slave(pi)) { 3128 fprintf(stderr, 3129 "Please remove port %d from bonded device.\n", 3130 pi); 3131 continue; 3132 } 3133 3134 port = &ports[pi]; 3135 if (port->port_status == RTE_PORT_STARTED) 3136 port->port_status = RTE_PORT_HANDLING; 3137 else 3138 continue; 3139 3140 if (hairpin_mode & 0xf) { 3141 int j; 3142 3143 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3144 /* unbind all peer Tx from current Rx */ 3145 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3146 RTE_MAX_ETHPORTS, 0); 3147 if (peer_pi < 0) 3148 continue; 3149 for (j = 0; j < peer_pi; j++) { 3150 if (!port_is_started(peer_pl[j])) 3151 continue; 3152 rte_eth_hairpin_unbind(peer_pl[j], pi); 3153 } 3154 } 3155 3156 if (port->flow_list) 3157 port_flow_flush(pi); 3158 3159 if (eth_dev_stop_mp(pi) != 0) 3160 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3161 pi); 3162 3163 if (port->port_status == RTE_PORT_HANDLING) 3164 port->port_status = RTE_PORT_STOPPED; 3165 else 3166 fprintf(stderr, "Port %d can not be set into stopped\n", 3167 pi); 3168 need_check_link_status = 1; 3169 } 3170 if (need_check_link_status && !no_link_check) 3171 check_all_ports_link_status(RTE_PORT_ALL); 3172 3173 printf("Done\n"); 3174 } 3175 3176 static void 3177 remove_invalid_ports_in(portid_t *array, portid_t *total) 3178 { 3179 portid_t i; 3180 portid_t new_total = 0; 3181 3182 for (i = 0; i < *total; i++) 3183 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3184 array[new_total] = array[i]; 3185 new_total++; 3186 } 3187 *total = new_total; 3188 } 3189 3190 static void 3191 remove_invalid_ports(void) 3192 { 3193 remove_invalid_ports_in(ports_ids, &nb_ports); 3194 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3195 nb_cfg_ports = nb_fwd_ports; 3196 } 3197 3198 static void 3199 flush_port_owned_resources(portid_t pi) 3200 { 3201 mcast_addr_pool_destroy(pi); 3202 port_flow_flush(pi); 3203 port_flex_item_flush(pi); 3204 port_action_handle_flush(pi); 3205 } 3206 3207 static void 3208 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3209 { 3210 struct rte_port *port; 3211 portid_t slave_pid; 3212 uint16_t i; 3213 3214 for (i = 0; i < num_slaves; i++) { 3215 slave_pid = slave_pids[i]; 3216 if (port_is_started(slave_pid) == 1) { 3217 if (rte_eth_dev_stop(slave_pid) != 0) 3218 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3219 slave_pid); 3220 3221 port = &ports[slave_pid]; 3222 port->port_status = RTE_PORT_STOPPED; 3223 } 3224 3225 clear_port_slave_flag(slave_pid); 3226 3227 /* Close slave device when testpmd quit or is killed. */ 3228 if (cl_quit == 1 || f_quit == 1) 3229 rte_eth_dev_close(slave_pid); 3230 } 3231 } 3232 3233 void 3234 close_port(portid_t pid) 3235 { 3236 portid_t pi; 3237 struct rte_port *port; 3238 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3239 int num_slaves = 0; 3240 3241 if (port_id_is_invalid(pid, ENABLED_WARN)) 3242 return; 3243 3244 printf("Closing ports...\n"); 3245 3246 RTE_ETH_FOREACH_DEV(pi) { 3247 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3248 continue; 3249 3250 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3251 fprintf(stderr, 3252 "Please remove port %d from forwarding configuration.\n", 3253 pi); 3254 continue; 3255 } 3256 3257 if (port_is_bonding_slave(pi)) { 3258 fprintf(stderr, 3259 "Please remove port %d from bonded device.\n", 3260 pi); 3261 continue; 3262 } 3263 3264 port = &ports[pi]; 3265 if (port->port_status == RTE_PORT_CLOSED) { 3266 fprintf(stderr, "Port %d is already closed\n", pi); 3267 continue; 3268 } 3269 3270 if (is_proc_primary()) { 3271 flush_port_owned_resources(pi); 3272 #ifdef RTE_NET_BOND 3273 if (port->bond_flag == 1) 3274 num_slaves = rte_eth_bond_slaves_get(pi, 3275 slave_pids, RTE_MAX_ETHPORTS); 3276 #endif 3277 rte_eth_dev_close(pi); 3278 /* 3279 * If this port is bonded device, all slaves under the 3280 * device need to be removed or closed. 3281 */ 3282 if (port->bond_flag == 1 && num_slaves > 0) 3283 clear_bonding_slave_device(slave_pids, 3284 num_slaves); 3285 } 3286 3287 free_xstats_display_info(pi); 3288 } 3289 3290 remove_invalid_ports(); 3291 printf("Done\n"); 3292 } 3293 3294 void 3295 reset_port(portid_t pid) 3296 { 3297 int diag; 3298 portid_t pi; 3299 struct rte_port *port; 3300 3301 if (port_id_is_invalid(pid, ENABLED_WARN)) 3302 return; 3303 3304 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3305 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3306 fprintf(stderr, 3307 "Can not reset port(s), please stop port(s) first.\n"); 3308 return; 3309 } 3310 3311 printf("Resetting ports...\n"); 3312 3313 RTE_ETH_FOREACH_DEV(pi) { 3314 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3315 continue; 3316 3317 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3318 fprintf(stderr, 3319 "Please remove port %d from forwarding configuration.\n", 3320 pi); 3321 continue; 3322 } 3323 3324 if (port_is_bonding_slave(pi)) { 3325 fprintf(stderr, 3326 "Please remove port %d from bonded device.\n", 3327 pi); 3328 continue; 3329 } 3330 3331 diag = rte_eth_dev_reset(pi); 3332 if (diag == 0) { 3333 port = &ports[pi]; 3334 port->need_reconfig = 1; 3335 port->need_reconfig_queues = 1; 3336 } else { 3337 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3338 pi, diag); 3339 } 3340 } 3341 3342 printf("Done\n"); 3343 } 3344 3345 void 3346 attach_port(char *identifier) 3347 { 3348 portid_t pi; 3349 struct rte_dev_iterator iterator; 3350 3351 printf("Attaching a new port...\n"); 3352 3353 if (identifier == NULL) { 3354 fprintf(stderr, "Invalid parameters are specified\n"); 3355 return; 3356 } 3357 3358 if (rte_dev_probe(identifier) < 0) { 3359 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3360 return; 3361 } 3362 3363 /* first attach mode: event */ 3364 if (setup_on_probe_event) { 3365 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3366 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3367 if (ports[pi].port_status == RTE_PORT_HANDLING && 3368 ports[pi].need_setup != 0) 3369 setup_attached_port(pi); 3370 return; 3371 } 3372 3373 /* second attach mode: iterator */ 3374 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3375 /* setup ports matching the devargs used for probing */ 3376 if (port_is_forwarding(pi)) 3377 continue; /* port was already attached before */ 3378 setup_attached_port(pi); 3379 } 3380 } 3381 3382 static void 3383 setup_attached_port(portid_t pi) 3384 { 3385 unsigned int socket_id; 3386 int ret; 3387 3388 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3389 /* if socket_id is invalid, set to the first available socket. */ 3390 if (check_socket_id(socket_id) < 0) 3391 socket_id = socket_ids[0]; 3392 reconfig(pi, socket_id); 3393 ret = rte_eth_promiscuous_enable(pi); 3394 if (ret != 0) 3395 fprintf(stderr, 3396 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3397 pi, rte_strerror(-ret)); 3398 3399 ports_ids[nb_ports++] = pi; 3400 fwd_ports_ids[nb_fwd_ports++] = pi; 3401 nb_cfg_ports = nb_fwd_ports; 3402 ports[pi].need_setup = 0; 3403 ports[pi].port_status = RTE_PORT_STOPPED; 3404 3405 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3406 printf("Done\n"); 3407 } 3408 3409 static void 3410 detach_device(struct rte_device *dev) 3411 { 3412 portid_t sibling; 3413 3414 if (dev == NULL) { 3415 fprintf(stderr, "Device already removed\n"); 3416 return; 3417 } 3418 3419 printf("Removing a device...\n"); 3420 3421 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3422 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3423 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3424 fprintf(stderr, "Port %u not stopped\n", 3425 sibling); 3426 return; 3427 } 3428 flush_port_owned_resources(sibling); 3429 } 3430 } 3431 3432 if (rte_dev_remove(dev) < 0) { 3433 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3434 return; 3435 } 3436 remove_invalid_ports(); 3437 3438 printf("Device is detached\n"); 3439 printf("Now total ports is %d\n", nb_ports); 3440 printf("Done\n"); 3441 return; 3442 } 3443 3444 void 3445 detach_port_device(portid_t port_id) 3446 { 3447 int ret; 3448 struct rte_eth_dev_info dev_info; 3449 3450 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3451 return; 3452 3453 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3454 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3455 fprintf(stderr, "Port not stopped\n"); 3456 return; 3457 } 3458 fprintf(stderr, "Port was not closed\n"); 3459 } 3460 3461 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3462 if (ret != 0) { 3463 TESTPMD_LOG(ERR, 3464 "Failed to get device info for port %d, not detaching\n", 3465 port_id); 3466 return; 3467 } 3468 detach_device(dev_info.device); 3469 } 3470 3471 void 3472 detach_devargs(char *identifier) 3473 { 3474 struct rte_dev_iterator iterator; 3475 struct rte_devargs da; 3476 portid_t port_id; 3477 3478 printf("Removing a device...\n"); 3479 3480 memset(&da, 0, sizeof(da)); 3481 if (rte_devargs_parsef(&da, "%s", identifier)) { 3482 fprintf(stderr, "cannot parse identifier\n"); 3483 return; 3484 } 3485 3486 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3487 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3488 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3489 fprintf(stderr, "Port %u not stopped\n", 3490 port_id); 3491 rte_eth_iterator_cleanup(&iterator); 3492 rte_devargs_reset(&da); 3493 return; 3494 } 3495 flush_port_owned_resources(port_id); 3496 } 3497 } 3498 3499 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3500 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3501 da.name, rte_bus_name(da.bus)); 3502 rte_devargs_reset(&da); 3503 return; 3504 } 3505 3506 remove_invalid_ports(); 3507 3508 printf("Device %s is detached\n", identifier); 3509 printf("Now total ports is %d\n", nb_ports); 3510 printf("Done\n"); 3511 rte_devargs_reset(&da); 3512 } 3513 3514 void 3515 pmd_test_exit(void) 3516 { 3517 portid_t pt_id; 3518 unsigned int i; 3519 int ret; 3520 3521 if (test_done == 0) 3522 stop_packet_forwarding(); 3523 3524 #ifndef RTE_EXEC_ENV_WINDOWS 3525 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3526 if (mempools[i]) { 3527 if (mp_alloc_type == MP_ALLOC_ANON) 3528 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3529 NULL); 3530 } 3531 } 3532 #endif 3533 if (ports != NULL) { 3534 no_link_check = 1; 3535 RTE_ETH_FOREACH_DEV(pt_id) { 3536 printf("\nStopping port %d...\n", pt_id); 3537 fflush(stdout); 3538 stop_port(pt_id); 3539 } 3540 RTE_ETH_FOREACH_DEV(pt_id) { 3541 printf("\nShutting down port %d...\n", pt_id); 3542 fflush(stdout); 3543 close_port(pt_id); 3544 } 3545 } 3546 3547 if (hot_plug) { 3548 ret = rte_dev_event_monitor_stop(); 3549 if (ret) { 3550 RTE_LOG(ERR, EAL, 3551 "fail to stop device event monitor."); 3552 return; 3553 } 3554 3555 ret = rte_dev_event_callback_unregister(NULL, 3556 dev_event_callback, NULL); 3557 if (ret < 0) { 3558 RTE_LOG(ERR, EAL, 3559 "fail to unregister device event callback.\n"); 3560 return; 3561 } 3562 3563 ret = rte_dev_hotplug_handle_disable(); 3564 if (ret) { 3565 RTE_LOG(ERR, EAL, 3566 "fail to disable hotplug handling.\n"); 3567 return; 3568 } 3569 } 3570 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3571 if (mempools[i]) 3572 mempool_free_mp(mempools[i]); 3573 } 3574 free(xstats_display); 3575 3576 printf("\nBye...\n"); 3577 } 3578 3579 typedef void (*cmd_func_t)(void); 3580 struct pmd_test_command { 3581 const char *cmd_name; 3582 cmd_func_t cmd_func; 3583 }; 3584 3585 /* Check the link status of all ports in up to 9s, and print them finally */ 3586 static void 3587 check_all_ports_link_status(uint32_t port_mask) 3588 { 3589 #define CHECK_INTERVAL 100 /* 100ms */ 3590 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3591 portid_t portid; 3592 uint8_t count, all_ports_up, print_flag = 0; 3593 struct rte_eth_link link; 3594 int ret; 3595 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3596 3597 printf("Checking link statuses...\n"); 3598 fflush(stdout); 3599 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3600 all_ports_up = 1; 3601 RTE_ETH_FOREACH_DEV(portid) { 3602 if ((port_mask & (1 << portid)) == 0) 3603 continue; 3604 memset(&link, 0, sizeof(link)); 3605 ret = rte_eth_link_get_nowait(portid, &link); 3606 if (ret < 0) { 3607 all_ports_up = 0; 3608 if (print_flag == 1) 3609 fprintf(stderr, 3610 "Port %u link get failed: %s\n", 3611 portid, rte_strerror(-ret)); 3612 continue; 3613 } 3614 /* print link status if flag set */ 3615 if (print_flag == 1) { 3616 rte_eth_link_to_str(link_status, 3617 sizeof(link_status), &link); 3618 printf("Port %d %s\n", portid, link_status); 3619 continue; 3620 } 3621 /* clear all_ports_up flag if any link down */ 3622 if (link.link_status == RTE_ETH_LINK_DOWN) { 3623 all_ports_up = 0; 3624 break; 3625 } 3626 } 3627 /* after finally printing all link status, get out */ 3628 if (print_flag == 1) 3629 break; 3630 3631 if (all_ports_up == 0) { 3632 fflush(stdout); 3633 rte_delay_ms(CHECK_INTERVAL); 3634 } 3635 3636 /* set the print_flag if all ports up or timeout */ 3637 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3638 print_flag = 1; 3639 } 3640 3641 if (lsc_interrupt) 3642 break; 3643 } 3644 } 3645 3646 static void 3647 rmv_port_callback(void *arg) 3648 { 3649 int need_to_start = 0; 3650 int org_no_link_check = no_link_check; 3651 portid_t port_id = (intptr_t)arg; 3652 struct rte_eth_dev_info dev_info; 3653 int ret; 3654 3655 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3656 3657 if (!test_done && port_is_forwarding(port_id)) { 3658 need_to_start = 1; 3659 stop_packet_forwarding(); 3660 } 3661 no_link_check = 1; 3662 stop_port(port_id); 3663 no_link_check = org_no_link_check; 3664 3665 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3666 if (ret != 0) 3667 TESTPMD_LOG(ERR, 3668 "Failed to get device info for port %d, not detaching\n", 3669 port_id); 3670 else { 3671 struct rte_device *device = dev_info.device; 3672 close_port(port_id); 3673 detach_device(device); /* might be already removed or have more ports */ 3674 } 3675 if (need_to_start) 3676 start_packet_forwarding(0); 3677 } 3678 3679 /* This function is used by the interrupt thread */ 3680 static int 3681 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3682 void *ret_param) 3683 { 3684 RTE_SET_USED(param); 3685 RTE_SET_USED(ret_param); 3686 3687 if (type >= RTE_ETH_EVENT_MAX) { 3688 fprintf(stderr, 3689 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3690 port_id, __func__, type); 3691 fflush(stderr); 3692 } else if (event_print_mask & (UINT32_C(1) << type)) { 3693 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3694 eth_event_desc[type]); 3695 fflush(stdout); 3696 } 3697 3698 switch (type) { 3699 case RTE_ETH_EVENT_NEW: 3700 ports[port_id].need_setup = 1; 3701 ports[port_id].port_status = RTE_PORT_HANDLING; 3702 break; 3703 case RTE_ETH_EVENT_INTR_RMV: 3704 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3705 break; 3706 if (rte_eal_alarm_set(100000, 3707 rmv_port_callback, (void *)(intptr_t)port_id)) 3708 fprintf(stderr, 3709 "Could not set up deferred device removal\n"); 3710 break; 3711 case RTE_ETH_EVENT_DESTROY: 3712 ports[port_id].port_status = RTE_PORT_CLOSED; 3713 printf("Port %u is closed\n", port_id); 3714 break; 3715 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3716 uint16_t rxq_id; 3717 int ret; 3718 3719 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3720 for (rxq_id = 0; ; rxq_id++) { 3721 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3722 NULL); 3723 if (ret <= 0) 3724 break; 3725 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3726 port_id, rxq_id); 3727 3728 #ifdef RTE_NET_MLX5 3729 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3730 #endif 3731 } 3732 break; 3733 } 3734 default: 3735 break; 3736 } 3737 return 0; 3738 } 3739 3740 static int 3741 register_eth_event_callback(void) 3742 { 3743 int ret; 3744 enum rte_eth_event_type event; 3745 3746 for (event = RTE_ETH_EVENT_UNKNOWN; 3747 event < RTE_ETH_EVENT_MAX; event++) { 3748 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3749 event, 3750 eth_event_callback, 3751 NULL); 3752 if (ret != 0) { 3753 TESTPMD_LOG(ERR, "Failed to register callback for " 3754 "%s event\n", eth_event_desc[event]); 3755 return -1; 3756 } 3757 } 3758 3759 return 0; 3760 } 3761 3762 /* This function is used by the interrupt thread */ 3763 static void 3764 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3765 __rte_unused void *arg) 3766 { 3767 uint16_t port_id; 3768 int ret; 3769 3770 if (type >= RTE_DEV_EVENT_MAX) { 3771 fprintf(stderr, "%s called upon invalid event %d\n", 3772 __func__, type); 3773 fflush(stderr); 3774 } 3775 3776 switch (type) { 3777 case RTE_DEV_EVENT_REMOVE: 3778 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3779 device_name); 3780 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3781 if (ret) { 3782 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3783 device_name); 3784 return; 3785 } 3786 /* 3787 * Because the user's callback is invoked in eal interrupt 3788 * callback, the interrupt callback need to be finished before 3789 * it can be unregistered when detaching device. So finish 3790 * callback soon and use a deferred removal to detach device 3791 * is need. It is a workaround, once the device detaching be 3792 * moved into the eal in the future, the deferred removal could 3793 * be deleted. 3794 */ 3795 if (rte_eal_alarm_set(100000, 3796 rmv_port_callback, (void *)(intptr_t)port_id)) 3797 RTE_LOG(ERR, EAL, 3798 "Could not set up deferred device removal\n"); 3799 break; 3800 case RTE_DEV_EVENT_ADD: 3801 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3802 device_name); 3803 /* TODO: After finish kernel driver binding, 3804 * begin to attach port. 3805 */ 3806 break; 3807 default: 3808 break; 3809 } 3810 } 3811 3812 static void 3813 rxtx_port_config(portid_t pid) 3814 { 3815 uint16_t qid; 3816 uint64_t offloads; 3817 struct rte_port *port = &ports[pid]; 3818 3819 for (qid = 0; qid < nb_rxq; qid++) { 3820 offloads = port->rxq[qid].conf.offloads; 3821 port->rxq[qid].conf = port->dev_info.default_rxconf; 3822 3823 if (rxq_share > 0 && 3824 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3825 /* Non-zero share group to enable RxQ share. */ 3826 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3827 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3828 } 3829 3830 if (offloads != 0) 3831 port->rxq[qid].conf.offloads = offloads; 3832 3833 /* Check if any Rx parameters have been passed */ 3834 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3835 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3836 3837 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3838 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3839 3840 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3841 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3842 3843 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3844 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3845 3846 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3847 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3848 3849 port->nb_rx_desc[qid] = nb_rxd; 3850 } 3851 3852 for (qid = 0; qid < nb_txq; qid++) { 3853 offloads = port->txq[qid].conf.offloads; 3854 port->txq[qid].conf = port->dev_info.default_txconf; 3855 if (offloads != 0) 3856 port->txq[qid].conf.offloads = offloads; 3857 3858 /* Check if any Tx parameters have been passed */ 3859 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3860 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3861 3862 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3863 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3864 3865 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3866 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3867 3868 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3869 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3870 3871 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3872 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3873 3874 port->nb_tx_desc[qid] = nb_txd; 3875 } 3876 } 3877 3878 /* 3879 * Helper function to set MTU from frame size 3880 * 3881 * port->dev_info should be set before calling this function. 3882 * 3883 * return 0 on success, negative on error 3884 */ 3885 int 3886 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3887 { 3888 struct rte_port *port = &ports[portid]; 3889 uint32_t eth_overhead; 3890 uint16_t mtu, new_mtu; 3891 3892 eth_overhead = get_eth_overhead(&port->dev_info); 3893 3894 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3895 printf("Failed to get MTU for port %u\n", portid); 3896 return -1; 3897 } 3898 3899 new_mtu = max_rx_pktlen - eth_overhead; 3900 3901 if (mtu == new_mtu) 3902 return 0; 3903 3904 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3905 fprintf(stderr, 3906 "Failed to set MTU to %u for port %u\n", 3907 new_mtu, portid); 3908 return -1; 3909 } 3910 3911 port->dev_conf.rxmode.mtu = new_mtu; 3912 3913 return 0; 3914 } 3915 3916 void 3917 init_port_config(void) 3918 { 3919 portid_t pid; 3920 struct rte_port *port; 3921 int ret, i; 3922 3923 RTE_ETH_FOREACH_DEV(pid) { 3924 port = &ports[pid]; 3925 3926 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3927 if (ret != 0) 3928 return; 3929 3930 if (nb_rxq > 1) { 3931 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3932 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3933 rss_hf & port->dev_info.flow_type_rss_offloads; 3934 } else { 3935 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3936 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3937 } 3938 3939 if (port->dcb_flag == 0) { 3940 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3941 port->dev_conf.rxmode.mq_mode = 3942 (enum rte_eth_rx_mq_mode) 3943 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3944 } else { 3945 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3946 port->dev_conf.rxmode.offloads &= 3947 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3948 3949 for (i = 0; 3950 i < port->dev_info.nb_rx_queues; 3951 i++) 3952 port->rxq[i].conf.offloads &= 3953 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3954 } 3955 } 3956 3957 rxtx_port_config(pid); 3958 3959 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3960 if (ret != 0) 3961 return; 3962 3963 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3964 port->dev_conf.intr_conf.lsc = 1; 3965 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3966 port->dev_conf.intr_conf.rmv = 1; 3967 } 3968 } 3969 3970 void set_port_slave_flag(portid_t slave_pid) 3971 { 3972 struct rte_port *port; 3973 3974 port = &ports[slave_pid]; 3975 port->slave_flag = 1; 3976 } 3977 3978 void clear_port_slave_flag(portid_t slave_pid) 3979 { 3980 struct rte_port *port; 3981 3982 port = &ports[slave_pid]; 3983 port->slave_flag = 0; 3984 } 3985 3986 uint8_t port_is_bonding_slave(portid_t slave_pid) 3987 { 3988 struct rte_port *port; 3989 struct rte_eth_dev_info dev_info; 3990 int ret; 3991 3992 port = &ports[slave_pid]; 3993 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3994 if (ret != 0) { 3995 TESTPMD_LOG(ERR, 3996 "Failed to get device info for port id %d," 3997 "cannot determine if the port is a bonded slave", 3998 slave_pid); 3999 return 0; 4000 } 4001 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4002 return 1; 4003 return 0; 4004 } 4005 4006 const uint16_t vlan_tags[] = { 4007 0, 1, 2, 3, 4, 5, 6, 7, 4008 8, 9, 10, 11, 12, 13, 14, 15, 4009 16, 17, 18, 19, 20, 21, 22, 23, 4010 24, 25, 26, 27, 28, 29, 30, 31 4011 }; 4012 4013 static int 4014 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4015 enum dcb_mode_enable dcb_mode, 4016 enum rte_eth_nb_tcs num_tcs, 4017 uint8_t pfc_en) 4018 { 4019 uint8_t i; 4020 int32_t rc; 4021 struct rte_eth_rss_conf rss_conf; 4022 4023 /* 4024 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4025 * given above, and the number of traffic classes available for use. 4026 */ 4027 if (dcb_mode == DCB_VT_ENABLED) { 4028 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4029 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4030 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4031 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4032 4033 /* VMDQ+DCB RX and TX configurations */ 4034 vmdq_rx_conf->enable_default_pool = 0; 4035 vmdq_rx_conf->default_pool = 0; 4036 vmdq_rx_conf->nb_queue_pools = 4037 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4038 vmdq_tx_conf->nb_queue_pools = 4039 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4040 4041 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4042 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4043 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4044 vmdq_rx_conf->pool_map[i].pools = 4045 1 << (i % vmdq_rx_conf->nb_queue_pools); 4046 } 4047 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4048 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4049 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4050 } 4051 4052 /* set DCB mode of RX and TX of multiple queues */ 4053 eth_conf->rxmode.mq_mode = 4054 (enum rte_eth_rx_mq_mode) 4055 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4056 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4057 } else { 4058 struct rte_eth_dcb_rx_conf *rx_conf = 4059 ð_conf->rx_adv_conf.dcb_rx_conf; 4060 struct rte_eth_dcb_tx_conf *tx_conf = 4061 ð_conf->tx_adv_conf.dcb_tx_conf; 4062 4063 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4064 4065 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4066 if (rc != 0) 4067 return rc; 4068 4069 rx_conf->nb_tcs = num_tcs; 4070 tx_conf->nb_tcs = num_tcs; 4071 4072 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4073 rx_conf->dcb_tc[i] = i % num_tcs; 4074 tx_conf->dcb_tc[i] = i % num_tcs; 4075 } 4076 4077 eth_conf->rxmode.mq_mode = 4078 (enum rte_eth_rx_mq_mode) 4079 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4080 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4081 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4082 } 4083 4084 if (pfc_en) 4085 eth_conf->dcb_capability_en = 4086 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4087 else 4088 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4089 4090 return 0; 4091 } 4092 4093 int 4094 init_port_dcb_config(portid_t pid, 4095 enum dcb_mode_enable dcb_mode, 4096 enum rte_eth_nb_tcs num_tcs, 4097 uint8_t pfc_en) 4098 { 4099 struct rte_eth_conf port_conf; 4100 struct rte_port *rte_port; 4101 int retval; 4102 uint16_t i; 4103 4104 if (num_procs > 1) { 4105 printf("The multi-process feature doesn't support dcb.\n"); 4106 return -ENOTSUP; 4107 } 4108 rte_port = &ports[pid]; 4109 4110 /* retain the original device configuration. */ 4111 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4112 4113 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4114 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4115 if (retval < 0) 4116 return retval; 4117 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4118 /* remove RSS HASH offload for DCB in vt mode */ 4119 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4120 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4121 for (i = 0; i < nb_rxq; i++) 4122 rte_port->rxq[i].conf.offloads &= 4123 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4124 } 4125 4126 /* re-configure the device . */ 4127 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4128 if (retval < 0) 4129 return retval; 4130 4131 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4132 if (retval != 0) 4133 return retval; 4134 4135 /* If dev_info.vmdq_pool_base is greater than 0, 4136 * the queue id of vmdq pools is started after pf queues. 4137 */ 4138 if (dcb_mode == DCB_VT_ENABLED && 4139 rte_port->dev_info.vmdq_pool_base > 0) { 4140 fprintf(stderr, 4141 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4142 pid); 4143 return -1; 4144 } 4145 4146 /* Assume the ports in testpmd have the same dcb capability 4147 * and has the same number of rxq and txq in dcb mode 4148 */ 4149 if (dcb_mode == DCB_VT_ENABLED) { 4150 if (rte_port->dev_info.max_vfs > 0) { 4151 nb_rxq = rte_port->dev_info.nb_rx_queues; 4152 nb_txq = rte_port->dev_info.nb_tx_queues; 4153 } else { 4154 nb_rxq = rte_port->dev_info.max_rx_queues; 4155 nb_txq = rte_port->dev_info.max_tx_queues; 4156 } 4157 } else { 4158 /*if vt is disabled, use all pf queues */ 4159 if (rte_port->dev_info.vmdq_pool_base == 0) { 4160 nb_rxq = rte_port->dev_info.max_rx_queues; 4161 nb_txq = rte_port->dev_info.max_tx_queues; 4162 } else { 4163 nb_rxq = (queueid_t)num_tcs; 4164 nb_txq = (queueid_t)num_tcs; 4165 4166 } 4167 } 4168 rx_free_thresh = 64; 4169 4170 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4171 4172 rxtx_port_config(pid); 4173 /* VLAN filter */ 4174 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4175 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4176 rx_vft_set(pid, vlan_tags[i], 1); 4177 4178 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4179 if (retval != 0) 4180 return retval; 4181 4182 rte_port->dcb_flag = 1; 4183 4184 /* Enter DCB configuration status */ 4185 dcb_config = 1; 4186 4187 return 0; 4188 } 4189 4190 static void 4191 init_port(void) 4192 { 4193 int i; 4194 4195 /* Configuration of Ethernet ports. */ 4196 ports = rte_zmalloc("testpmd: ports", 4197 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4198 RTE_CACHE_LINE_SIZE); 4199 if (ports == NULL) { 4200 rte_exit(EXIT_FAILURE, 4201 "rte_zmalloc(%d struct rte_port) failed\n", 4202 RTE_MAX_ETHPORTS); 4203 } 4204 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4205 ports[i].xstats_info.allocated = false; 4206 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4207 LIST_INIT(&ports[i].flow_tunnel_list); 4208 /* Initialize ports NUMA structures */ 4209 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4210 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4211 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4212 } 4213 4214 static void 4215 force_quit(void) 4216 { 4217 pmd_test_exit(); 4218 prompt_exit(); 4219 } 4220 4221 static void 4222 print_stats(void) 4223 { 4224 uint8_t i; 4225 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4226 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4227 4228 /* Clear screen and move to top left */ 4229 printf("%s%s", clr, top_left); 4230 4231 printf("\nPort statistics ===================================="); 4232 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4233 nic_stats_display(fwd_ports_ids[i]); 4234 4235 fflush(stdout); 4236 } 4237 4238 static void 4239 signal_handler(int signum) 4240 { 4241 if (signum == SIGINT || signum == SIGTERM) { 4242 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4243 signum); 4244 #ifdef RTE_LIB_PDUMP 4245 /* uninitialize packet capture framework */ 4246 rte_pdump_uninit(); 4247 #endif 4248 #ifdef RTE_LIB_LATENCYSTATS 4249 if (latencystats_enabled != 0) 4250 rte_latencystats_uninit(); 4251 #endif 4252 force_quit(); 4253 /* Set flag to indicate the force termination. */ 4254 f_quit = 1; 4255 /* exit with the expected status */ 4256 #ifndef RTE_EXEC_ENV_WINDOWS 4257 signal(signum, SIG_DFL); 4258 kill(getpid(), signum); 4259 #endif 4260 } 4261 } 4262 4263 int 4264 main(int argc, char** argv) 4265 { 4266 int diag; 4267 portid_t port_id; 4268 uint16_t count; 4269 int ret; 4270 4271 signal(SIGINT, signal_handler); 4272 signal(SIGTERM, signal_handler); 4273 4274 testpmd_logtype = rte_log_register("testpmd"); 4275 if (testpmd_logtype < 0) 4276 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4277 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4278 4279 diag = rte_eal_init(argc, argv); 4280 if (diag < 0) 4281 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4282 rte_strerror(rte_errno)); 4283 4284 ret = register_eth_event_callback(); 4285 if (ret != 0) 4286 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4287 4288 #ifdef RTE_LIB_PDUMP 4289 /* initialize packet capture framework */ 4290 rte_pdump_init(); 4291 #endif 4292 4293 count = 0; 4294 RTE_ETH_FOREACH_DEV(port_id) { 4295 ports_ids[count] = port_id; 4296 count++; 4297 } 4298 nb_ports = (portid_t) count; 4299 if (nb_ports == 0) 4300 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4301 4302 /* allocate port structures, and init them */ 4303 init_port(); 4304 4305 set_def_fwd_config(); 4306 if (nb_lcores == 0) 4307 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4308 "Check the core mask argument\n"); 4309 4310 /* Bitrate/latency stats disabled by default */ 4311 #ifdef RTE_LIB_BITRATESTATS 4312 bitrate_enabled = 0; 4313 #endif 4314 #ifdef RTE_LIB_LATENCYSTATS 4315 latencystats_enabled = 0; 4316 #endif 4317 4318 /* on FreeBSD, mlockall() is disabled by default */ 4319 #ifdef RTE_EXEC_ENV_FREEBSD 4320 do_mlockall = 0; 4321 #else 4322 do_mlockall = 1; 4323 #endif 4324 4325 argc -= diag; 4326 argv += diag; 4327 if (argc > 1) 4328 launch_args_parse(argc, argv); 4329 4330 #ifndef RTE_EXEC_ENV_WINDOWS 4331 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4332 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4333 strerror(errno)); 4334 } 4335 #endif 4336 4337 if (tx_first && interactive) 4338 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4339 "interactive mode.\n"); 4340 4341 if (tx_first && lsc_interrupt) { 4342 fprintf(stderr, 4343 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4344 lsc_interrupt = 0; 4345 } 4346 4347 if (!nb_rxq && !nb_txq) 4348 fprintf(stderr, 4349 "Warning: Either rx or tx queues should be non-zero\n"); 4350 4351 if (nb_rxq > 1 && nb_rxq > nb_txq) 4352 fprintf(stderr, 4353 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4354 nb_rxq, nb_txq); 4355 4356 init_config(); 4357 4358 if (hot_plug) { 4359 ret = rte_dev_hotplug_handle_enable(); 4360 if (ret) { 4361 RTE_LOG(ERR, EAL, 4362 "fail to enable hotplug handling."); 4363 return -1; 4364 } 4365 4366 ret = rte_dev_event_monitor_start(); 4367 if (ret) { 4368 RTE_LOG(ERR, EAL, 4369 "fail to start device event monitoring."); 4370 return -1; 4371 } 4372 4373 ret = rte_dev_event_callback_register(NULL, 4374 dev_event_callback, NULL); 4375 if (ret) { 4376 RTE_LOG(ERR, EAL, 4377 "fail to register device event callback\n"); 4378 return -1; 4379 } 4380 } 4381 4382 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4383 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4384 4385 /* set all ports to promiscuous mode by default */ 4386 RTE_ETH_FOREACH_DEV(port_id) { 4387 ret = rte_eth_promiscuous_enable(port_id); 4388 if (ret != 0) 4389 fprintf(stderr, 4390 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4391 port_id, rte_strerror(-ret)); 4392 } 4393 4394 #ifdef RTE_LIB_METRICS 4395 /* Init metrics library */ 4396 rte_metrics_init(rte_socket_id()); 4397 #endif 4398 4399 #ifdef RTE_LIB_LATENCYSTATS 4400 if (latencystats_enabled != 0) { 4401 int ret = rte_latencystats_init(1, NULL); 4402 if (ret) 4403 fprintf(stderr, 4404 "Warning: latencystats init() returned error %d\n", 4405 ret); 4406 fprintf(stderr, "Latencystats running on lcore %d\n", 4407 latencystats_lcore_id); 4408 } 4409 #endif 4410 4411 /* Setup bitrate stats */ 4412 #ifdef RTE_LIB_BITRATESTATS 4413 if (bitrate_enabled != 0) { 4414 bitrate_data = rte_stats_bitrate_create(); 4415 if (bitrate_data == NULL) 4416 rte_exit(EXIT_FAILURE, 4417 "Could not allocate bitrate data.\n"); 4418 rte_stats_bitrate_reg(bitrate_data); 4419 } 4420 #endif 4421 #ifdef RTE_LIB_CMDLINE 4422 if (init_cmdline() != 0) 4423 rte_exit(EXIT_FAILURE, 4424 "Could not initialise cmdline context.\n"); 4425 4426 if (strlen(cmdline_filename) != 0) 4427 cmdline_read_from_file(cmdline_filename); 4428 4429 if (interactive == 1) { 4430 if (auto_start) { 4431 printf("Start automatic packet forwarding\n"); 4432 start_packet_forwarding(0); 4433 } 4434 prompt(); 4435 pmd_test_exit(); 4436 } else 4437 #endif 4438 { 4439 char c; 4440 int rc; 4441 4442 f_quit = 0; 4443 4444 printf("No commandline core given, start packet forwarding\n"); 4445 start_packet_forwarding(tx_first); 4446 if (stats_period != 0) { 4447 uint64_t prev_time = 0, cur_time, diff_time = 0; 4448 uint64_t timer_period; 4449 4450 /* Convert to number of cycles */ 4451 timer_period = stats_period * rte_get_timer_hz(); 4452 4453 while (f_quit == 0) { 4454 cur_time = rte_get_timer_cycles(); 4455 diff_time += cur_time - prev_time; 4456 4457 if (diff_time >= timer_period) { 4458 print_stats(); 4459 /* Reset the timer */ 4460 diff_time = 0; 4461 } 4462 /* Sleep to avoid unnecessary checks */ 4463 prev_time = cur_time; 4464 rte_delay_us_sleep(US_PER_S); 4465 } 4466 } 4467 4468 printf("Press enter to exit\n"); 4469 rc = read(0, &c, 1); 4470 pmd_test_exit(); 4471 if (rc < 0) 4472 return 1; 4473 } 4474 4475 ret = rte_eal_cleanup(); 4476 if (ret != 0) 4477 rte_exit(EXIT_FAILURE, 4478 "EAL cleanup failed: %s\n", strerror(-ret)); 4479 4480 return EXIT_SUCCESS; 4481 } 4482