1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 251 /* 252 * Configuration of packet segments used by the "txonly" processing engine. 253 */ 254 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 255 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 256 TXONLY_DEF_PACKET_LEN, 257 }; 258 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 259 260 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 261 /**< Split policy for packets to TX. */ 262 263 uint8_t txonly_multi_flow; 264 /**< Whether multiple flows are generated in TXONLY mode. */ 265 266 uint32_t tx_pkt_times_inter; 267 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 268 269 uint32_t tx_pkt_times_intra; 270 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 271 272 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 273 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 274 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 275 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 276 277 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 278 uint8_t dcb_config = 0; 279 280 /* 281 * Configurable number of RX/TX queues. 282 */ 283 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 284 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 285 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 286 287 /* 288 * Configurable number of RX/TX ring descriptors. 289 * Defaults are supplied by drivers via ethdev. 290 */ 291 #define RTE_TEST_RX_DESC_DEFAULT 0 292 #define RTE_TEST_TX_DESC_DEFAULT 0 293 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 294 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 295 296 #define RTE_PMD_PARAM_UNSET -1 297 /* 298 * Configurable values of RX and TX ring threshold registers. 299 */ 300 301 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 302 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 303 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 304 305 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 306 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 307 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 308 309 /* 310 * Configurable value of RX free threshold. 311 */ 312 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 313 314 /* 315 * Configurable value of RX drop enable. 316 */ 317 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 318 319 /* 320 * Configurable value of TX free threshold. 321 */ 322 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 323 324 /* 325 * Configurable value of TX RS bit threshold. 326 */ 327 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 328 329 /* 330 * Configurable value of buffered packets before sending. 331 */ 332 uint16_t noisy_tx_sw_bufsz; 333 334 /* 335 * Configurable value of packet buffer timeout. 336 */ 337 uint16_t noisy_tx_sw_buf_flush_time; 338 339 /* 340 * Configurable value for size of VNF internal memory area 341 * used for simulating noisy neighbour behaviour 342 */ 343 uint64_t noisy_lkup_mem_sz; 344 345 /* 346 * Configurable value of number of random writes done in 347 * VNF simulation memory area. 348 */ 349 uint64_t noisy_lkup_num_writes; 350 351 /* 352 * Configurable value of number of random reads done in 353 * VNF simulation memory area. 354 */ 355 uint64_t noisy_lkup_num_reads; 356 357 /* 358 * Configurable value of number of random reads/writes done in 359 * VNF simulation memory area. 360 */ 361 uint64_t noisy_lkup_num_reads_writes; 362 363 /* 364 * Receive Side Scaling (RSS) configuration. 365 */ 366 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 367 368 /* 369 * Port topology configuration 370 */ 371 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 372 373 /* 374 * Avoids to flush all the RX streams before starts forwarding. 375 */ 376 uint8_t no_flush_rx = 0; /* flush by default */ 377 378 /* 379 * Flow API isolated mode. 380 */ 381 uint8_t flow_isolate_all; 382 383 /* 384 * Avoids to check link status when starting/stopping a port. 385 */ 386 uint8_t no_link_check = 0; /* check by default */ 387 388 /* 389 * Don't automatically start all ports in interactive mode. 390 */ 391 uint8_t no_device_start = 0; 392 393 /* 394 * Enable link status change notification 395 */ 396 uint8_t lsc_interrupt = 1; /* enabled by default */ 397 398 /* 399 * Enable device removal notification. 400 */ 401 uint8_t rmv_interrupt = 1; /* enabled by default */ 402 403 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 404 405 /* After attach, port setup is called on event or by iterator */ 406 bool setup_on_probe_event = true; 407 408 /* Clear ptypes on port initialization. */ 409 uint8_t clear_ptypes = true; 410 411 /* Hairpin ports configuration mode. */ 412 uint16_t hairpin_mode; 413 414 /* Pretty printing of ethdev events */ 415 static const char * const eth_event_desc[] = { 416 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 417 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 418 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 419 [RTE_ETH_EVENT_INTR_RESET] = "reset", 420 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 421 [RTE_ETH_EVENT_IPSEC] = "IPsec", 422 [RTE_ETH_EVENT_MACSEC] = "MACsec", 423 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 424 [RTE_ETH_EVENT_NEW] = "device probed", 425 [RTE_ETH_EVENT_DESTROY] = "device released", 426 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 427 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 428 [RTE_ETH_EVENT_MAX] = NULL, 429 }; 430 431 /* 432 * Display or mask ether events 433 * Default to all events except VF_MBOX 434 */ 435 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 436 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 437 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 438 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 439 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 440 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 441 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 442 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 443 /* 444 * Decide if all memory are locked for performance. 445 */ 446 int do_mlockall = 0; 447 448 #ifdef RTE_LIB_LATENCYSTATS 449 450 /* 451 * Set when latency stats is enabled in the commandline 452 */ 453 uint8_t latencystats_enabled; 454 455 /* 456 * Lcore ID to service latency statistics. 457 */ 458 lcoreid_t latencystats_lcore_id = -1; 459 460 #endif 461 462 /* 463 * Ethernet device configuration. 464 */ 465 struct rte_eth_rxmode rx_mode; 466 467 struct rte_eth_txmode tx_mode = { 468 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 469 }; 470 471 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 472 473 /* 474 * Display zero values by default for xstats 475 */ 476 uint8_t xstats_hide_zero; 477 478 /* 479 * Measure of CPU cycles disabled by default 480 */ 481 uint8_t record_core_cycles; 482 483 /* 484 * Display of RX and TX bursts disabled by default 485 */ 486 uint8_t record_burst_stats; 487 488 /* 489 * Number of ports per shared Rx queue group, 0 disable. 490 */ 491 uint32_t rxq_share; 492 493 unsigned int num_sockets = 0; 494 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 495 496 #ifdef RTE_LIB_BITRATESTATS 497 /* Bitrate statistics */ 498 struct rte_stats_bitrates *bitrate_data; 499 lcoreid_t bitrate_lcore_id; 500 uint8_t bitrate_enabled; 501 #endif 502 503 #ifdef RTE_LIB_GRO 504 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 505 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 506 #endif 507 508 /* 509 * hexadecimal bitmask of RX mq mode can be enabled. 510 */ 511 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 512 513 /* 514 * Used to set forced link speed 515 */ 516 uint32_t eth_link_speed; 517 518 /* 519 * ID of the current process in multi-process, used to 520 * configure the queues to be polled. 521 */ 522 int proc_id; 523 524 /* 525 * Number of processes in multi-process, used to 526 * configure the queues to be polled. 527 */ 528 unsigned int num_procs = 1; 529 530 static void 531 eth_rx_metadata_negotiate_mp(uint16_t port_id) 532 { 533 uint64_t rx_meta_features = 0; 534 int ret; 535 536 if (!is_proc_primary()) 537 return; 538 539 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 540 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 541 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 542 543 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 544 if (ret == 0) { 545 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 546 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 547 port_id); 548 } 549 550 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 551 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 552 port_id); 553 } 554 555 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 556 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 557 port_id); 558 } 559 } else if (ret != -ENOTSUP) { 560 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 561 port_id, rte_strerror(-ret)); 562 } 563 } 564 565 static int 566 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 567 const struct rte_eth_conf *dev_conf) 568 { 569 if (is_proc_primary()) 570 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 571 dev_conf); 572 return 0; 573 } 574 575 static int 576 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 577 { 578 #ifdef RTE_NET_BOND 579 580 portid_t slave_pids[RTE_MAX_ETHPORTS]; 581 struct rte_port *port; 582 int num_slaves; 583 portid_t slave_pid; 584 int i; 585 586 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 587 RTE_MAX_ETHPORTS); 588 if (num_slaves < 0) { 589 fprintf(stderr, "Failed to get slave list for port = %u\n", 590 bond_pid); 591 return num_slaves; 592 } 593 594 for (i = 0; i < num_slaves; i++) { 595 slave_pid = slave_pids[i]; 596 port = &ports[slave_pid]; 597 port->port_status = 598 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 599 } 600 #else 601 RTE_SET_USED(bond_pid); 602 RTE_SET_USED(is_stop); 603 #endif 604 return 0; 605 } 606 607 static int 608 eth_dev_start_mp(uint16_t port_id) 609 { 610 int ret; 611 612 if (is_proc_primary()) { 613 ret = rte_eth_dev_start(port_id); 614 if (ret != 0) 615 return ret; 616 617 struct rte_port *port = &ports[port_id]; 618 619 /* 620 * Starting a bonded port also starts all slaves under the bonded 621 * device. So if this port is bond device, we need to modify the 622 * port status of these slaves. 623 */ 624 if (port->bond_flag == 1) 625 return change_bonding_slave_port_status(port_id, false); 626 } 627 628 return 0; 629 } 630 631 static int 632 eth_dev_stop_mp(uint16_t port_id) 633 { 634 int ret; 635 636 if (is_proc_primary()) { 637 ret = rte_eth_dev_stop(port_id); 638 if (ret != 0) 639 return ret; 640 641 struct rte_port *port = &ports[port_id]; 642 643 /* 644 * Stopping a bonded port also stops all slaves under the bonded 645 * device. So if this port is bond device, we need to modify the 646 * port status of these slaves. 647 */ 648 if (port->bond_flag == 1) 649 return change_bonding_slave_port_status(port_id, true); 650 } 651 652 return 0; 653 } 654 655 static void 656 mempool_free_mp(struct rte_mempool *mp) 657 { 658 if (is_proc_primary()) 659 rte_mempool_free(mp); 660 } 661 662 static int 663 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 664 { 665 if (is_proc_primary()) 666 return rte_eth_dev_set_mtu(port_id, mtu); 667 668 return 0; 669 } 670 671 /* Forward function declarations */ 672 static void setup_attached_port(portid_t pi); 673 static void check_all_ports_link_status(uint32_t port_mask); 674 static int eth_event_callback(portid_t port_id, 675 enum rte_eth_event_type type, 676 void *param, void *ret_param); 677 static void dev_event_callback(const char *device_name, 678 enum rte_dev_event_type type, 679 void *param); 680 static void fill_xstats_display_info(void); 681 682 /* 683 * Check if all the ports are started. 684 * If yes, return positive value. If not, return zero. 685 */ 686 static int all_ports_started(void); 687 688 #ifdef RTE_LIB_GSO 689 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 690 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 691 #endif 692 693 /* Holds the registered mbuf dynamic flags names. */ 694 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 695 696 697 /* 698 * Helper function to check if socket is already discovered. 699 * If yes, return positive value. If not, return zero. 700 */ 701 int 702 new_socket_id(unsigned int socket_id) 703 { 704 unsigned int i; 705 706 for (i = 0; i < num_sockets; i++) { 707 if (socket_ids[i] == socket_id) 708 return 0; 709 } 710 return 1; 711 } 712 713 /* 714 * Setup default configuration. 715 */ 716 static void 717 set_default_fwd_lcores_config(void) 718 { 719 unsigned int i; 720 unsigned int nb_lc; 721 unsigned int sock_num; 722 723 nb_lc = 0; 724 for (i = 0; i < RTE_MAX_LCORE; i++) { 725 if (!rte_lcore_is_enabled(i)) 726 continue; 727 sock_num = rte_lcore_to_socket_id(i); 728 if (new_socket_id(sock_num)) { 729 if (num_sockets >= RTE_MAX_NUMA_NODES) { 730 rte_exit(EXIT_FAILURE, 731 "Total sockets greater than %u\n", 732 RTE_MAX_NUMA_NODES); 733 } 734 socket_ids[num_sockets++] = sock_num; 735 } 736 if (i == rte_get_main_lcore()) 737 continue; 738 fwd_lcores_cpuids[nb_lc++] = i; 739 } 740 nb_lcores = (lcoreid_t) nb_lc; 741 nb_cfg_lcores = nb_lcores; 742 nb_fwd_lcores = 1; 743 } 744 745 static void 746 set_def_peer_eth_addrs(void) 747 { 748 portid_t i; 749 750 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 751 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 752 peer_eth_addrs[i].addr_bytes[5] = i; 753 } 754 } 755 756 static void 757 set_default_fwd_ports_config(void) 758 { 759 portid_t pt_id; 760 int i = 0; 761 762 RTE_ETH_FOREACH_DEV(pt_id) { 763 fwd_ports_ids[i++] = pt_id; 764 765 /* Update sockets info according to the attached device */ 766 int socket_id = rte_eth_dev_socket_id(pt_id); 767 if (socket_id >= 0 && new_socket_id(socket_id)) { 768 if (num_sockets >= RTE_MAX_NUMA_NODES) { 769 rte_exit(EXIT_FAILURE, 770 "Total sockets greater than %u\n", 771 RTE_MAX_NUMA_NODES); 772 } 773 socket_ids[num_sockets++] = socket_id; 774 } 775 } 776 777 nb_cfg_ports = nb_ports; 778 nb_fwd_ports = nb_ports; 779 } 780 781 void 782 set_def_fwd_config(void) 783 { 784 set_default_fwd_lcores_config(); 785 set_def_peer_eth_addrs(); 786 set_default_fwd_ports_config(); 787 } 788 789 #ifndef RTE_EXEC_ENV_WINDOWS 790 /* extremely pessimistic estimation of memory required to create a mempool */ 791 static int 792 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 793 { 794 unsigned int n_pages, mbuf_per_pg, leftover; 795 uint64_t total_mem, mbuf_mem, obj_sz; 796 797 /* there is no good way to predict how much space the mempool will 798 * occupy because it will allocate chunks on the fly, and some of those 799 * will come from default DPDK memory while some will come from our 800 * external memory, so just assume 128MB will be enough for everyone. 801 */ 802 uint64_t hdr_mem = 128 << 20; 803 804 /* account for possible non-contiguousness */ 805 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 806 if (obj_sz > pgsz) { 807 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 808 return -1; 809 } 810 811 mbuf_per_pg = pgsz / obj_sz; 812 leftover = (nb_mbufs % mbuf_per_pg) > 0; 813 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 814 815 mbuf_mem = n_pages * pgsz; 816 817 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 818 819 if (total_mem > SIZE_MAX) { 820 TESTPMD_LOG(ERR, "Memory size too big\n"); 821 return -1; 822 } 823 *out = (size_t)total_mem; 824 825 return 0; 826 } 827 828 static int 829 pagesz_flags(uint64_t page_sz) 830 { 831 /* as per mmap() manpage, all page sizes are log2 of page size 832 * shifted by MAP_HUGE_SHIFT 833 */ 834 int log2 = rte_log2_u64(page_sz); 835 836 return (log2 << HUGE_SHIFT); 837 } 838 839 static void * 840 alloc_mem(size_t memsz, size_t pgsz, bool huge) 841 { 842 void *addr; 843 int flags; 844 845 /* allocate anonymous hugepages */ 846 flags = MAP_ANONYMOUS | MAP_PRIVATE; 847 if (huge) 848 flags |= HUGE_FLAG | pagesz_flags(pgsz); 849 850 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 851 if (addr == MAP_FAILED) 852 return NULL; 853 854 return addr; 855 } 856 857 struct extmem_param { 858 void *addr; 859 size_t len; 860 size_t pgsz; 861 rte_iova_t *iova_table; 862 unsigned int iova_table_len; 863 }; 864 865 static int 866 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 867 bool huge) 868 { 869 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 870 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 871 unsigned int cur_page, n_pages, pgsz_idx; 872 size_t mem_sz, cur_pgsz; 873 rte_iova_t *iovas = NULL; 874 void *addr; 875 int ret; 876 877 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 878 /* skip anything that is too big */ 879 if (pgsizes[pgsz_idx] > SIZE_MAX) 880 continue; 881 882 cur_pgsz = pgsizes[pgsz_idx]; 883 884 /* if we were told not to allocate hugepages, override */ 885 if (!huge) 886 cur_pgsz = sysconf(_SC_PAGESIZE); 887 888 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 889 if (ret < 0) { 890 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 891 return -1; 892 } 893 894 /* allocate our memory */ 895 addr = alloc_mem(mem_sz, cur_pgsz, huge); 896 897 /* if we couldn't allocate memory with a specified page size, 898 * that doesn't mean we can't do it with other page sizes, so 899 * try another one. 900 */ 901 if (addr == NULL) 902 continue; 903 904 /* store IOVA addresses for every page in this memory area */ 905 n_pages = mem_sz / cur_pgsz; 906 907 iovas = malloc(sizeof(*iovas) * n_pages); 908 909 if (iovas == NULL) { 910 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 911 goto fail; 912 } 913 /* lock memory if it's not huge pages */ 914 if (!huge) 915 mlock(addr, mem_sz); 916 917 /* populate IOVA addresses */ 918 for (cur_page = 0; cur_page < n_pages; cur_page++) { 919 rte_iova_t iova; 920 size_t offset; 921 void *cur; 922 923 offset = cur_pgsz * cur_page; 924 cur = RTE_PTR_ADD(addr, offset); 925 926 /* touch the page before getting its IOVA */ 927 *(volatile char *)cur = 0; 928 929 iova = rte_mem_virt2iova(cur); 930 931 iovas[cur_page] = iova; 932 } 933 934 break; 935 } 936 /* if we couldn't allocate anything */ 937 if (iovas == NULL) 938 return -1; 939 940 param->addr = addr; 941 param->len = mem_sz; 942 param->pgsz = cur_pgsz; 943 param->iova_table = iovas; 944 param->iova_table_len = n_pages; 945 946 return 0; 947 fail: 948 free(iovas); 949 if (addr) 950 munmap(addr, mem_sz); 951 952 return -1; 953 } 954 955 static int 956 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 957 { 958 struct extmem_param param; 959 int socket_id, ret; 960 961 memset(¶m, 0, sizeof(param)); 962 963 /* check if our heap exists */ 964 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 965 if (socket_id < 0) { 966 /* create our heap */ 967 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 968 if (ret < 0) { 969 TESTPMD_LOG(ERR, "Cannot create heap\n"); 970 return -1; 971 } 972 } 973 974 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 975 if (ret < 0) { 976 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 977 return -1; 978 } 979 980 /* we now have a valid memory area, so add it to heap */ 981 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 982 param.addr, param.len, param.iova_table, 983 param.iova_table_len, param.pgsz); 984 985 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 986 987 /* not needed any more */ 988 free(param.iova_table); 989 990 if (ret < 0) { 991 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 992 munmap(param.addr, param.len); 993 return -1; 994 } 995 996 /* success */ 997 998 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 999 param.len >> 20); 1000 1001 return 0; 1002 } 1003 static void 1004 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1005 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1006 { 1007 uint16_t pid = 0; 1008 int ret; 1009 1010 RTE_ETH_FOREACH_DEV(pid) { 1011 struct rte_eth_dev_info dev_info; 1012 1013 ret = eth_dev_info_get_print_err(pid, &dev_info); 1014 if (ret != 0) { 1015 TESTPMD_LOG(DEBUG, 1016 "unable to get device info for port %d on addr 0x%p," 1017 "mempool unmapping will not be performed\n", 1018 pid, memhdr->addr); 1019 continue; 1020 } 1021 1022 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1023 if (ret) { 1024 TESTPMD_LOG(DEBUG, 1025 "unable to DMA unmap addr 0x%p " 1026 "for device %s\n", 1027 memhdr->addr, rte_dev_name(dev_info.device)); 1028 } 1029 } 1030 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1031 if (ret) { 1032 TESTPMD_LOG(DEBUG, 1033 "unable to un-register addr 0x%p\n", memhdr->addr); 1034 } 1035 } 1036 1037 static void 1038 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1039 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1040 { 1041 uint16_t pid = 0; 1042 size_t page_size = sysconf(_SC_PAGESIZE); 1043 int ret; 1044 1045 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1046 page_size); 1047 if (ret) { 1048 TESTPMD_LOG(DEBUG, 1049 "unable to register addr 0x%p\n", memhdr->addr); 1050 return; 1051 } 1052 RTE_ETH_FOREACH_DEV(pid) { 1053 struct rte_eth_dev_info dev_info; 1054 1055 ret = eth_dev_info_get_print_err(pid, &dev_info); 1056 if (ret != 0) { 1057 TESTPMD_LOG(DEBUG, 1058 "unable to get device info for port %d on addr 0x%p," 1059 "mempool mapping will not be performed\n", 1060 pid, memhdr->addr); 1061 continue; 1062 } 1063 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1064 if (ret) { 1065 TESTPMD_LOG(DEBUG, 1066 "unable to DMA map addr 0x%p " 1067 "for device %s\n", 1068 memhdr->addr, rte_dev_name(dev_info.device)); 1069 } 1070 } 1071 } 1072 #endif 1073 1074 static unsigned int 1075 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1076 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1077 { 1078 struct rte_pktmbuf_extmem *xmem; 1079 unsigned int ext_num, zone_num, elt_num; 1080 uint16_t elt_size; 1081 1082 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1083 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1084 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1085 1086 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1087 if (xmem == NULL) { 1088 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1089 "external buffer descriptors\n"); 1090 *ext_mem = NULL; 1091 return 0; 1092 } 1093 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1094 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1095 const struct rte_memzone *mz; 1096 char mz_name[RTE_MEMZONE_NAMESIZE]; 1097 int ret; 1098 1099 ret = snprintf(mz_name, sizeof(mz_name), 1100 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1101 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1102 errno = ENAMETOOLONG; 1103 ext_num = 0; 1104 break; 1105 } 1106 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1107 socket_id, 1108 RTE_MEMZONE_IOVA_CONTIG | 1109 RTE_MEMZONE_1GB | 1110 RTE_MEMZONE_SIZE_HINT_ONLY); 1111 if (mz == NULL) { 1112 /* 1113 * The caller exits on external buffer creation 1114 * error, so there is no need to free memzones. 1115 */ 1116 errno = ENOMEM; 1117 ext_num = 0; 1118 break; 1119 } 1120 xseg->buf_ptr = mz->addr; 1121 xseg->buf_iova = mz->iova; 1122 xseg->buf_len = EXTBUF_ZONE_SIZE; 1123 xseg->elt_size = elt_size; 1124 } 1125 if (ext_num == 0 && xmem != NULL) { 1126 free(xmem); 1127 xmem = NULL; 1128 } 1129 *ext_mem = xmem; 1130 return ext_num; 1131 } 1132 1133 /* 1134 * Configuration initialisation done once at init time. 1135 */ 1136 static struct rte_mempool * 1137 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1138 unsigned int socket_id, uint16_t size_idx) 1139 { 1140 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1141 struct rte_mempool *rte_mp = NULL; 1142 #ifndef RTE_EXEC_ENV_WINDOWS 1143 uint32_t mb_size; 1144 1145 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1146 #endif 1147 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1148 if (!is_proc_primary()) { 1149 rte_mp = rte_mempool_lookup(pool_name); 1150 if (rte_mp == NULL) 1151 rte_exit(EXIT_FAILURE, 1152 "Get mbuf pool for socket %u failed: %s\n", 1153 socket_id, rte_strerror(rte_errno)); 1154 return rte_mp; 1155 } 1156 1157 TESTPMD_LOG(INFO, 1158 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1159 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1160 1161 switch (mp_alloc_type) { 1162 case MP_ALLOC_NATIVE: 1163 { 1164 /* wrapper to rte_mempool_create() */ 1165 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1166 rte_mbuf_best_mempool_ops()); 1167 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1168 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1169 break; 1170 } 1171 #ifndef RTE_EXEC_ENV_WINDOWS 1172 case MP_ALLOC_ANON: 1173 { 1174 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1175 mb_size, (unsigned int) mb_mempool_cache, 1176 sizeof(struct rte_pktmbuf_pool_private), 1177 socket_id, mempool_flags); 1178 if (rte_mp == NULL) 1179 goto err; 1180 1181 if (rte_mempool_populate_anon(rte_mp) == 0) { 1182 rte_mempool_free(rte_mp); 1183 rte_mp = NULL; 1184 goto err; 1185 } 1186 rte_pktmbuf_pool_init(rte_mp, NULL); 1187 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1188 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1189 break; 1190 } 1191 case MP_ALLOC_XMEM: 1192 case MP_ALLOC_XMEM_HUGE: 1193 { 1194 int heap_socket; 1195 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1196 1197 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1198 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1199 1200 heap_socket = 1201 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1202 if (heap_socket < 0) 1203 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1204 1205 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1206 rte_mbuf_best_mempool_ops()); 1207 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1208 mb_mempool_cache, 0, mbuf_seg_size, 1209 heap_socket); 1210 break; 1211 } 1212 #endif 1213 case MP_ALLOC_XBUF: 1214 { 1215 struct rte_pktmbuf_extmem *ext_mem; 1216 unsigned int ext_num; 1217 1218 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1219 socket_id, pool_name, &ext_mem); 1220 if (ext_num == 0) 1221 rte_exit(EXIT_FAILURE, 1222 "Can't create pinned data buffers\n"); 1223 1224 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1225 rte_mbuf_best_mempool_ops()); 1226 rte_mp = rte_pktmbuf_pool_create_extbuf 1227 (pool_name, nb_mbuf, mb_mempool_cache, 1228 0, mbuf_seg_size, socket_id, 1229 ext_mem, ext_num); 1230 free(ext_mem); 1231 break; 1232 } 1233 default: 1234 { 1235 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1236 } 1237 } 1238 1239 #ifndef RTE_EXEC_ENV_WINDOWS 1240 err: 1241 #endif 1242 if (rte_mp == NULL) { 1243 rte_exit(EXIT_FAILURE, 1244 "Creation of mbuf pool for socket %u failed: %s\n", 1245 socket_id, rte_strerror(rte_errno)); 1246 } else if (verbose_level > 0) { 1247 rte_mempool_dump(stdout, rte_mp); 1248 } 1249 return rte_mp; 1250 } 1251 1252 /* 1253 * Check given socket id is valid or not with NUMA mode, 1254 * if valid, return 0, else return -1 1255 */ 1256 static int 1257 check_socket_id(const unsigned int socket_id) 1258 { 1259 static int warning_once = 0; 1260 1261 if (new_socket_id(socket_id)) { 1262 if (!warning_once && numa_support) 1263 fprintf(stderr, 1264 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1265 warning_once = 1; 1266 return -1; 1267 } 1268 return 0; 1269 } 1270 1271 /* 1272 * Get the allowed maximum number of RX queues. 1273 * *pid return the port id which has minimal value of 1274 * max_rx_queues in all ports. 1275 */ 1276 queueid_t 1277 get_allowed_max_nb_rxq(portid_t *pid) 1278 { 1279 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1280 bool max_rxq_valid = false; 1281 portid_t pi; 1282 struct rte_eth_dev_info dev_info; 1283 1284 RTE_ETH_FOREACH_DEV(pi) { 1285 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1286 continue; 1287 1288 max_rxq_valid = true; 1289 if (dev_info.max_rx_queues < allowed_max_rxq) { 1290 allowed_max_rxq = dev_info.max_rx_queues; 1291 *pid = pi; 1292 } 1293 } 1294 return max_rxq_valid ? allowed_max_rxq : 0; 1295 } 1296 1297 /* 1298 * Check input rxq is valid or not. 1299 * If input rxq is not greater than any of maximum number 1300 * of RX queues of all ports, it is valid. 1301 * if valid, return 0, else return -1 1302 */ 1303 int 1304 check_nb_rxq(queueid_t rxq) 1305 { 1306 queueid_t allowed_max_rxq; 1307 portid_t pid = 0; 1308 1309 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1310 if (rxq > allowed_max_rxq) { 1311 fprintf(stderr, 1312 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1313 rxq, allowed_max_rxq, pid); 1314 return -1; 1315 } 1316 return 0; 1317 } 1318 1319 /* 1320 * Get the allowed maximum number of TX queues. 1321 * *pid return the port id which has minimal value of 1322 * max_tx_queues in all ports. 1323 */ 1324 queueid_t 1325 get_allowed_max_nb_txq(portid_t *pid) 1326 { 1327 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1328 bool max_txq_valid = false; 1329 portid_t pi; 1330 struct rte_eth_dev_info dev_info; 1331 1332 RTE_ETH_FOREACH_DEV(pi) { 1333 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1334 continue; 1335 1336 max_txq_valid = true; 1337 if (dev_info.max_tx_queues < allowed_max_txq) { 1338 allowed_max_txq = dev_info.max_tx_queues; 1339 *pid = pi; 1340 } 1341 } 1342 return max_txq_valid ? allowed_max_txq : 0; 1343 } 1344 1345 /* 1346 * Check input txq is valid or not. 1347 * If input txq is not greater than any of maximum number 1348 * of TX queues of all ports, it is valid. 1349 * if valid, return 0, else return -1 1350 */ 1351 int 1352 check_nb_txq(queueid_t txq) 1353 { 1354 queueid_t allowed_max_txq; 1355 portid_t pid = 0; 1356 1357 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1358 if (txq > allowed_max_txq) { 1359 fprintf(stderr, 1360 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1361 txq, allowed_max_txq, pid); 1362 return -1; 1363 } 1364 return 0; 1365 } 1366 1367 /* 1368 * Get the allowed maximum number of RXDs of every rx queue. 1369 * *pid return the port id which has minimal value of 1370 * max_rxd in all queues of all ports. 1371 */ 1372 static uint16_t 1373 get_allowed_max_nb_rxd(portid_t *pid) 1374 { 1375 uint16_t allowed_max_rxd = UINT16_MAX; 1376 portid_t pi; 1377 struct rte_eth_dev_info dev_info; 1378 1379 RTE_ETH_FOREACH_DEV(pi) { 1380 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1381 continue; 1382 1383 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1384 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1385 *pid = pi; 1386 } 1387 } 1388 return allowed_max_rxd; 1389 } 1390 1391 /* 1392 * Get the allowed minimal number of RXDs of every rx queue. 1393 * *pid return the port id which has minimal value of 1394 * min_rxd in all queues of all ports. 1395 */ 1396 static uint16_t 1397 get_allowed_min_nb_rxd(portid_t *pid) 1398 { 1399 uint16_t allowed_min_rxd = 0; 1400 portid_t pi; 1401 struct rte_eth_dev_info dev_info; 1402 1403 RTE_ETH_FOREACH_DEV(pi) { 1404 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1405 continue; 1406 1407 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1408 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1409 *pid = pi; 1410 } 1411 } 1412 1413 return allowed_min_rxd; 1414 } 1415 1416 /* 1417 * Check input rxd is valid or not. 1418 * If input rxd is not greater than any of maximum number 1419 * of RXDs of every Rx queues and is not less than any of 1420 * minimal number of RXDs of every Rx queues, it is valid. 1421 * if valid, return 0, else return -1 1422 */ 1423 int 1424 check_nb_rxd(queueid_t rxd) 1425 { 1426 uint16_t allowed_max_rxd; 1427 uint16_t allowed_min_rxd; 1428 portid_t pid = 0; 1429 1430 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1431 if (rxd > allowed_max_rxd) { 1432 fprintf(stderr, 1433 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1434 rxd, allowed_max_rxd, pid); 1435 return -1; 1436 } 1437 1438 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1439 if (rxd < allowed_min_rxd) { 1440 fprintf(stderr, 1441 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1442 rxd, allowed_min_rxd, pid); 1443 return -1; 1444 } 1445 1446 return 0; 1447 } 1448 1449 /* 1450 * Get the allowed maximum number of TXDs of every rx queues. 1451 * *pid return the port id which has minimal value of 1452 * max_txd in every tx queue. 1453 */ 1454 static uint16_t 1455 get_allowed_max_nb_txd(portid_t *pid) 1456 { 1457 uint16_t allowed_max_txd = UINT16_MAX; 1458 portid_t pi; 1459 struct rte_eth_dev_info dev_info; 1460 1461 RTE_ETH_FOREACH_DEV(pi) { 1462 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1463 continue; 1464 1465 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1466 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1467 *pid = pi; 1468 } 1469 } 1470 return allowed_max_txd; 1471 } 1472 1473 /* 1474 * Get the allowed maximum number of TXDs of every tx queues. 1475 * *pid return the port id which has minimal value of 1476 * min_txd in every tx queue. 1477 */ 1478 static uint16_t 1479 get_allowed_min_nb_txd(portid_t *pid) 1480 { 1481 uint16_t allowed_min_txd = 0; 1482 portid_t pi; 1483 struct rte_eth_dev_info dev_info; 1484 1485 RTE_ETH_FOREACH_DEV(pi) { 1486 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1487 continue; 1488 1489 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1490 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1491 *pid = pi; 1492 } 1493 } 1494 1495 return allowed_min_txd; 1496 } 1497 1498 /* 1499 * Check input txd is valid or not. 1500 * If input txd is not greater than any of maximum number 1501 * of TXDs of every Rx queues, it is valid. 1502 * if valid, return 0, else return -1 1503 */ 1504 int 1505 check_nb_txd(queueid_t txd) 1506 { 1507 uint16_t allowed_max_txd; 1508 uint16_t allowed_min_txd; 1509 portid_t pid = 0; 1510 1511 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1512 if (txd > allowed_max_txd) { 1513 fprintf(stderr, 1514 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1515 txd, allowed_max_txd, pid); 1516 return -1; 1517 } 1518 1519 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1520 if (txd < allowed_min_txd) { 1521 fprintf(stderr, 1522 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1523 txd, allowed_min_txd, pid); 1524 return -1; 1525 } 1526 return 0; 1527 } 1528 1529 1530 /* 1531 * Get the allowed maximum number of hairpin queues. 1532 * *pid return the port id which has minimal value of 1533 * max_hairpin_queues in all ports. 1534 */ 1535 queueid_t 1536 get_allowed_max_nb_hairpinq(portid_t *pid) 1537 { 1538 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1539 portid_t pi; 1540 struct rte_eth_hairpin_cap cap; 1541 1542 RTE_ETH_FOREACH_DEV(pi) { 1543 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1544 *pid = pi; 1545 return 0; 1546 } 1547 if (cap.max_nb_queues < allowed_max_hairpinq) { 1548 allowed_max_hairpinq = cap.max_nb_queues; 1549 *pid = pi; 1550 } 1551 } 1552 return allowed_max_hairpinq; 1553 } 1554 1555 /* 1556 * Check input hairpin is valid or not. 1557 * If input hairpin is not greater than any of maximum number 1558 * of hairpin queues of all ports, it is valid. 1559 * if valid, return 0, else return -1 1560 */ 1561 int 1562 check_nb_hairpinq(queueid_t hairpinq) 1563 { 1564 queueid_t allowed_max_hairpinq; 1565 portid_t pid = 0; 1566 1567 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1568 if (hairpinq > allowed_max_hairpinq) { 1569 fprintf(stderr, 1570 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1571 hairpinq, allowed_max_hairpinq, pid); 1572 return -1; 1573 } 1574 return 0; 1575 } 1576 1577 static int 1578 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1579 { 1580 uint32_t eth_overhead; 1581 1582 if (dev_info->max_mtu != UINT16_MAX && 1583 dev_info->max_rx_pktlen > dev_info->max_mtu) 1584 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1585 else 1586 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1587 1588 return eth_overhead; 1589 } 1590 1591 static void 1592 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1593 { 1594 struct rte_port *port = &ports[pid]; 1595 int ret; 1596 int i; 1597 1598 eth_rx_metadata_negotiate_mp(pid); 1599 1600 port->dev_conf.txmode = tx_mode; 1601 port->dev_conf.rxmode = rx_mode; 1602 1603 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1604 if (ret != 0) 1605 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1606 1607 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1608 port->dev_conf.txmode.offloads &= 1609 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1610 1611 /* Apply Rx offloads configuration */ 1612 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1613 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1614 /* Apply Tx offloads configuration */ 1615 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1616 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1617 1618 if (eth_link_speed) 1619 port->dev_conf.link_speeds = eth_link_speed; 1620 1621 if (max_rx_pkt_len) 1622 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1623 get_eth_overhead(&port->dev_info); 1624 1625 /* set flag to initialize port/queue */ 1626 port->need_reconfig = 1; 1627 port->need_reconfig_queues = 1; 1628 port->socket_id = socket_id; 1629 port->tx_metadata = 0; 1630 1631 /* 1632 * Check for maximum number of segments per MTU. 1633 * Accordingly update the mbuf data size. 1634 */ 1635 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1636 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1637 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1638 uint16_t mtu; 1639 1640 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1641 uint16_t data_size = (mtu + eth_overhead) / 1642 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1643 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1644 1645 if (buffer_size > mbuf_data_size[0]) { 1646 mbuf_data_size[0] = buffer_size; 1647 TESTPMD_LOG(WARNING, 1648 "Configured mbuf size of the first segment %hu\n", 1649 mbuf_data_size[0]); 1650 } 1651 } 1652 } 1653 } 1654 1655 static void 1656 init_config(void) 1657 { 1658 portid_t pid; 1659 struct rte_mempool *mbp; 1660 unsigned int nb_mbuf_per_pool; 1661 lcoreid_t lc_id; 1662 #ifdef RTE_LIB_GRO 1663 struct rte_gro_param gro_param; 1664 #endif 1665 #ifdef RTE_LIB_GSO 1666 uint32_t gso_types; 1667 #endif 1668 1669 /* Configuration of logical cores. */ 1670 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1671 sizeof(struct fwd_lcore *) * nb_lcores, 1672 RTE_CACHE_LINE_SIZE); 1673 if (fwd_lcores == NULL) { 1674 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1675 "failed\n", nb_lcores); 1676 } 1677 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1678 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1679 sizeof(struct fwd_lcore), 1680 RTE_CACHE_LINE_SIZE); 1681 if (fwd_lcores[lc_id] == NULL) { 1682 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1683 "failed\n"); 1684 } 1685 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1686 } 1687 1688 RTE_ETH_FOREACH_DEV(pid) { 1689 uint32_t socket_id; 1690 1691 if (numa_support) { 1692 socket_id = port_numa[pid]; 1693 if (port_numa[pid] == NUMA_NO_CONFIG) { 1694 socket_id = rte_eth_dev_socket_id(pid); 1695 1696 /* 1697 * if socket_id is invalid, 1698 * set to the first available socket. 1699 */ 1700 if (check_socket_id(socket_id) < 0) 1701 socket_id = socket_ids[0]; 1702 } 1703 } else { 1704 socket_id = (socket_num == UMA_NO_CONFIG) ? 1705 0 : socket_num; 1706 } 1707 /* Apply default TxRx configuration for all ports */ 1708 init_config_port_offloads(pid, socket_id); 1709 } 1710 /* 1711 * Create pools of mbuf. 1712 * If NUMA support is disabled, create a single pool of mbuf in 1713 * socket 0 memory by default. 1714 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1715 * 1716 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1717 * nb_txd can be configured at run time. 1718 */ 1719 if (param_total_num_mbufs) 1720 nb_mbuf_per_pool = param_total_num_mbufs; 1721 else { 1722 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1723 (nb_lcores * mb_mempool_cache) + 1724 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1725 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1726 } 1727 1728 if (numa_support) { 1729 uint8_t i, j; 1730 1731 for (i = 0; i < num_sockets; i++) 1732 for (j = 0; j < mbuf_data_size_n; j++) 1733 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1734 mbuf_pool_create(mbuf_data_size[j], 1735 nb_mbuf_per_pool, 1736 socket_ids[i], j); 1737 } else { 1738 uint8_t i; 1739 1740 for (i = 0; i < mbuf_data_size_n; i++) 1741 mempools[i] = mbuf_pool_create 1742 (mbuf_data_size[i], 1743 nb_mbuf_per_pool, 1744 socket_num == UMA_NO_CONFIG ? 1745 0 : socket_num, i); 1746 } 1747 1748 init_port_config(); 1749 1750 #ifdef RTE_LIB_GSO 1751 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1752 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1753 #endif 1754 /* 1755 * Records which Mbuf pool to use by each logical core, if needed. 1756 */ 1757 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1758 mbp = mbuf_pool_find( 1759 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1760 1761 if (mbp == NULL) 1762 mbp = mbuf_pool_find(0, 0); 1763 fwd_lcores[lc_id]->mbp = mbp; 1764 #ifdef RTE_LIB_GSO 1765 /* initialize GSO context */ 1766 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1767 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1768 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1769 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1770 RTE_ETHER_CRC_LEN; 1771 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1772 #endif 1773 } 1774 1775 fwd_config_setup(); 1776 1777 #ifdef RTE_LIB_GRO 1778 /* create a gro context for each lcore */ 1779 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1780 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1781 gro_param.max_item_per_flow = MAX_PKT_BURST; 1782 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1783 gro_param.socket_id = rte_lcore_to_socket_id( 1784 fwd_lcores_cpuids[lc_id]); 1785 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1786 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1787 rte_exit(EXIT_FAILURE, 1788 "rte_gro_ctx_create() failed\n"); 1789 } 1790 } 1791 #endif 1792 } 1793 1794 1795 void 1796 reconfig(portid_t new_port_id, unsigned socket_id) 1797 { 1798 /* Reconfiguration of Ethernet ports. */ 1799 init_config_port_offloads(new_port_id, socket_id); 1800 init_port_config(); 1801 } 1802 1803 int 1804 init_fwd_streams(void) 1805 { 1806 portid_t pid; 1807 struct rte_port *port; 1808 streamid_t sm_id, nb_fwd_streams_new; 1809 queueid_t q; 1810 1811 /* set socket id according to numa or not */ 1812 RTE_ETH_FOREACH_DEV(pid) { 1813 port = &ports[pid]; 1814 if (nb_rxq > port->dev_info.max_rx_queues) { 1815 fprintf(stderr, 1816 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1817 nb_rxq, port->dev_info.max_rx_queues); 1818 return -1; 1819 } 1820 if (nb_txq > port->dev_info.max_tx_queues) { 1821 fprintf(stderr, 1822 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1823 nb_txq, port->dev_info.max_tx_queues); 1824 return -1; 1825 } 1826 if (numa_support) { 1827 if (port_numa[pid] != NUMA_NO_CONFIG) 1828 port->socket_id = port_numa[pid]; 1829 else { 1830 port->socket_id = rte_eth_dev_socket_id(pid); 1831 1832 /* 1833 * if socket_id is invalid, 1834 * set to the first available socket. 1835 */ 1836 if (check_socket_id(port->socket_id) < 0) 1837 port->socket_id = socket_ids[0]; 1838 } 1839 } 1840 else { 1841 if (socket_num == UMA_NO_CONFIG) 1842 port->socket_id = 0; 1843 else 1844 port->socket_id = socket_num; 1845 } 1846 } 1847 1848 q = RTE_MAX(nb_rxq, nb_txq); 1849 if (q == 0) { 1850 fprintf(stderr, 1851 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1852 return -1; 1853 } 1854 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1855 if (nb_fwd_streams_new == nb_fwd_streams) 1856 return 0; 1857 /* clear the old */ 1858 if (fwd_streams != NULL) { 1859 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1860 if (fwd_streams[sm_id] == NULL) 1861 continue; 1862 rte_free(fwd_streams[sm_id]); 1863 fwd_streams[sm_id] = NULL; 1864 } 1865 rte_free(fwd_streams); 1866 fwd_streams = NULL; 1867 } 1868 1869 /* init new */ 1870 nb_fwd_streams = nb_fwd_streams_new; 1871 if (nb_fwd_streams) { 1872 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1873 sizeof(struct fwd_stream *) * nb_fwd_streams, 1874 RTE_CACHE_LINE_SIZE); 1875 if (fwd_streams == NULL) 1876 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1877 " (struct fwd_stream *)) failed\n", 1878 nb_fwd_streams); 1879 1880 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1881 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1882 " struct fwd_stream", sizeof(struct fwd_stream), 1883 RTE_CACHE_LINE_SIZE); 1884 if (fwd_streams[sm_id] == NULL) 1885 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1886 "(struct fwd_stream) failed\n"); 1887 } 1888 } 1889 1890 return 0; 1891 } 1892 1893 static void 1894 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1895 { 1896 uint64_t total_burst, sburst; 1897 uint64_t nb_burst; 1898 uint64_t burst_stats[4]; 1899 uint16_t pktnb_stats[4]; 1900 uint16_t nb_pkt; 1901 int burst_percent[4], sburstp; 1902 int i; 1903 1904 /* 1905 * First compute the total number of packet bursts and the 1906 * two highest numbers of bursts of the same number of packets. 1907 */ 1908 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1909 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1910 1911 /* Show stats for 0 burst size always */ 1912 total_burst = pbs->pkt_burst_spread[0]; 1913 burst_stats[0] = pbs->pkt_burst_spread[0]; 1914 pktnb_stats[0] = 0; 1915 1916 /* Find the next 2 burst sizes with highest occurrences. */ 1917 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1918 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1919 1920 if (nb_burst == 0) 1921 continue; 1922 1923 total_burst += nb_burst; 1924 1925 if (nb_burst > burst_stats[1]) { 1926 burst_stats[2] = burst_stats[1]; 1927 pktnb_stats[2] = pktnb_stats[1]; 1928 burst_stats[1] = nb_burst; 1929 pktnb_stats[1] = nb_pkt; 1930 } else if (nb_burst > burst_stats[2]) { 1931 burst_stats[2] = nb_burst; 1932 pktnb_stats[2] = nb_pkt; 1933 } 1934 } 1935 if (total_burst == 0) 1936 return; 1937 1938 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1939 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1940 if (i == 3) { 1941 printf("%d%% of other]\n", 100 - sburstp); 1942 return; 1943 } 1944 1945 sburst += burst_stats[i]; 1946 if (sburst == total_burst) { 1947 printf("%d%% of %d pkts]\n", 1948 100 - sburstp, (int) pktnb_stats[i]); 1949 return; 1950 } 1951 1952 burst_percent[i] = 1953 (double)burst_stats[i] / total_burst * 100; 1954 printf("%d%% of %d pkts + ", 1955 burst_percent[i], (int) pktnb_stats[i]); 1956 sburstp += burst_percent[i]; 1957 } 1958 } 1959 1960 static void 1961 fwd_stream_stats_display(streamid_t stream_id) 1962 { 1963 struct fwd_stream *fs; 1964 static const char *fwd_top_stats_border = "-------"; 1965 1966 fs = fwd_streams[stream_id]; 1967 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1968 (fs->fwd_dropped == 0)) 1969 return; 1970 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1971 "TX Port=%2d/Queue=%2d %s\n", 1972 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1973 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1974 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1975 " TX-dropped: %-14"PRIu64, 1976 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1977 1978 /* if checksum mode */ 1979 if (cur_fwd_eng == &csum_fwd_engine) { 1980 printf(" RX- bad IP checksum: %-14"PRIu64 1981 " Rx- bad L4 checksum: %-14"PRIu64 1982 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1983 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1984 fs->rx_bad_outer_l4_csum); 1985 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1986 fs->rx_bad_outer_ip_csum); 1987 } else { 1988 printf("\n"); 1989 } 1990 1991 if (record_burst_stats) { 1992 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1993 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1994 } 1995 } 1996 1997 void 1998 fwd_stats_display(void) 1999 { 2000 static const char *fwd_stats_border = "----------------------"; 2001 static const char *acc_stats_border = "+++++++++++++++"; 2002 struct { 2003 struct fwd_stream *rx_stream; 2004 struct fwd_stream *tx_stream; 2005 uint64_t tx_dropped; 2006 uint64_t rx_bad_ip_csum; 2007 uint64_t rx_bad_l4_csum; 2008 uint64_t rx_bad_outer_l4_csum; 2009 uint64_t rx_bad_outer_ip_csum; 2010 } ports_stats[RTE_MAX_ETHPORTS]; 2011 uint64_t total_rx_dropped = 0; 2012 uint64_t total_tx_dropped = 0; 2013 uint64_t total_rx_nombuf = 0; 2014 struct rte_eth_stats stats; 2015 uint64_t fwd_cycles = 0; 2016 uint64_t total_recv = 0; 2017 uint64_t total_xmit = 0; 2018 struct rte_port *port; 2019 streamid_t sm_id; 2020 portid_t pt_id; 2021 int ret; 2022 int i; 2023 2024 memset(ports_stats, 0, sizeof(ports_stats)); 2025 2026 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2027 struct fwd_stream *fs = fwd_streams[sm_id]; 2028 2029 if (cur_fwd_config.nb_fwd_streams > 2030 cur_fwd_config.nb_fwd_ports) { 2031 fwd_stream_stats_display(sm_id); 2032 } else { 2033 ports_stats[fs->tx_port].tx_stream = fs; 2034 ports_stats[fs->rx_port].rx_stream = fs; 2035 } 2036 2037 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2038 2039 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2040 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2041 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2042 fs->rx_bad_outer_l4_csum; 2043 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2044 fs->rx_bad_outer_ip_csum; 2045 2046 if (record_core_cycles) 2047 fwd_cycles += fs->core_cycles; 2048 } 2049 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2050 pt_id = fwd_ports_ids[i]; 2051 port = &ports[pt_id]; 2052 2053 ret = rte_eth_stats_get(pt_id, &stats); 2054 if (ret != 0) { 2055 fprintf(stderr, 2056 "%s: Error: failed to get stats (port %u): %d", 2057 __func__, pt_id, ret); 2058 continue; 2059 } 2060 stats.ipackets -= port->stats.ipackets; 2061 stats.opackets -= port->stats.opackets; 2062 stats.ibytes -= port->stats.ibytes; 2063 stats.obytes -= port->stats.obytes; 2064 stats.imissed -= port->stats.imissed; 2065 stats.oerrors -= port->stats.oerrors; 2066 stats.rx_nombuf -= port->stats.rx_nombuf; 2067 2068 total_recv += stats.ipackets; 2069 total_xmit += stats.opackets; 2070 total_rx_dropped += stats.imissed; 2071 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2072 total_tx_dropped += stats.oerrors; 2073 total_rx_nombuf += stats.rx_nombuf; 2074 2075 printf("\n %s Forward statistics for port %-2d %s\n", 2076 fwd_stats_border, pt_id, fwd_stats_border); 2077 2078 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2079 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2080 stats.ipackets + stats.imissed); 2081 2082 if (cur_fwd_eng == &csum_fwd_engine) { 2083 printf(" Bad-ipcsum: %-14"PRIu64 2084 " Bad-l4csum: %-14"PRIu64 2085 "Bad-outer-l4csum: %-14"PRIu64"\n", 2086 ports_stats[pt_id].rx_bad_ip_csum, 2087 ports_stats[pt_id].rx_bad_l4_csum, 2088 ports_stats[pt_id].rx_bad_outer_l4_csum); 2089 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2090 ports_stats[pt_id].rx_bad_outer_ip_csum); 2091 } 2092 if (stats.ierrors + stats.rx_nombuf > 0) { 2093 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2094 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2095 } 2096 2097 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2098 "TX-total: %-"PRIu64"\n", 2099 stats.opackets, ports_stats[pt_id].tx_dropped, 2100 stats.opackets + ports_stats[pt_id].tx_dropped); 2101 2102 if (record_burst_stats) { 2103 if (ports_stats[pt_id].rx_stream) 2104 pkt_burst_stats_display("RX", 2105 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2106 if (ports_stats[pt_id].tx_stream) 2107 pkt_burst_stats_display("TX", 2108 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2109 } 2110 2111 printf(" %s--------------------------------%s\n", 2112 fwd_stats_border, fwd_stats_border); 2113 } 2114 2115 printf("\n %s Accumulated forward statistics for all ports" 2116 "%s\n", 2117 acc_stats_border, acc_stats_border); 2118 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2119 "%-"PRIu64"\n" 2120 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2121 "%-"PRIu64"\n", 2122 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2123 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2124 if (total_rx_nombuf > 0) 2125 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2126 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2127 "%s\n", 2128 acc_stats_border, acc_stats_border); 2129 if (record_core_cycles) { 2130 #define CYC_PER_MHZ 1E6 2131 if (total_recv > 0 || total_xmit > 0) { 2132 uint64_t total_pkts = 0; 2133 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2134 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2135 total_pkts = total_xmit; 2136 else 2137 total_pkts = total_recv; 2138 2139 printf("\n CPU cycles/packet=%.2F (total cycles=" 2140 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2141 " MHz Clock\n", 2142 (double) fwd_cycles / total_pkts, 2143 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2144 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2145 } 2146 } 2147 } 2148 2149 void 2150 fwd_stats_reset(void) 2151 { 2152 streamid_t sm_id; 2153 portid_t pt_id; 2154 int ret; 2155 int i; 2156 2157 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2158 pt_id = fwd_ports_ids[i]; 2159 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2160 if (ret != 0) 2161 fprintf(stderr, 2162 "%s: Error: failed to clear stats (port %u):%d", 2163 __func__, pt_id, ret); 2164 } 2165 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2166 struct fwd_stream *fs = fwd_streams[sm_id]; 2167 2168 fs->rx_packets = 0; 2169 fs->tx_packets = 0; 2170 fs->fwd_dropped = 0; 2171 fs->rx_bad_ip_csum = 0; 2172 fs->rx_bad_l4_csum = 0; 2173 fs->rx_bad_outer_l4_csum = 0; 2174 fs->rx_bad_outer_ip_csum = 0; 2175 2176 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2177 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2178 fs->core_cycles = 0; 2179 } 2180 } 2181 2182 static void 2183 flush_fwd_rx_queues(void) 2184 { 2185 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2186 portid_t rxp; 2187 portid_t port_id; 2188 queueid_t rxq; 2189 uint16_t nb_rx; 2190 uint16_t i; 2191 uint8_t j; 2192 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2193 uint64_t timer_period; 2194 2195 if (num_procs > 1) { 2196 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2197 return; 2198 } 2199 2200 /* convert to number of cycles */ 2201 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2202 2203 for (j = 0; j < 2; j++) { 2204 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2205 for (rxq = 0; rxq < nb_rxq; rxq++) { 2206 port_id = fwd_ports_ids[rxp]; 2207 2208 /* Polling stopped queues is prohibited. */ 2209 if (ports[port_id].rxq[rxq].state == 2210 RTE_ETH_QUEUE_STATE_STOPPED) 2211 continue; 2212 2213 /** 2214 * testpmd can stuck in the below do while loop 2215 * if rte_eth_rx_burst() always returns nonzero 2216 * packets. So timer is added to exit this loop 2217 * after 1sec timer expiry. 2218 */ 2219 prev_tsc = rte_rdtsc(); 2220 do { 2221 nb_rx = rte_eth_rx_burst(port_id, rxq, 2222 pkts_burst, MAX_PKT_BURST); 2223 for (i = 0; i < nb_rx; i++) 2224 rte_pktmbuf_free(pkts_burst[i]); 2225 2226 cur_tsc = rte_rdtsc(); 2227 diff_tsc = cur_tsc - prev_tsc; 2228 timer_tsc += diff_tsc; 2229 } while ((nb_rx > 0) && 2230 (timer_tsc < timer_period)); 2231 timer_tsc = 0; 2232 } 2233 } 2234 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2235 } 2236 } 2237 2238 static void 2239 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2240 { 2241 struct fwd_stream **fsm; 2242 streamid_t nb_fs; 2243 streamid_t sm_id; 2244 #ifdef RTE_LIB_BITRATESTATS 2245 uint64_t tics_per_1sec; 2246 uint64_t tics_datum; 2247 uint64_t tics_current; 2248 uint16_t i, cnt_ports; 2249 2250 cnt_ports = nb_ports; 2251 tics_datum = rte_rdtsc(); 2252 tics_per_1sec = rte_get_timer_hz(); 2253 #endif 2254 fsm = &fwd_streams[fc->stream_idx]; 2255 nb_fs = fc->stream_nb; 2256 do { 2257 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2258 if (!fsm[sm_id]->disabled) 2259 (*pkt_fwd)(fsm[sm_id]); 2260 #ifdef RTE_LIB_BITRATESTATS 2261 if (bitrate_enabled != 0 && 2262 bitrate_lcore_id == rte_lcore_id()) { 2263 tics_current = rte_rdtsc(); 2264 if (tics_current - tics_datum >= tics_per_1sec) { 2265 /* Periodic bitrate calculation */ 2266 for (i = 0; i < cnt_ports; i++) 2267 rte_stats_bitrate_calc(bitrate_data, 2268 ports_ids[i]); 2269 tics_datum = tics_current; 2270 } 2271 } 2272 #endif 2273 #ifdef RTE_LIB_LATENCYSTATS 2274 if (latencystats_enabled != 0 && 2275 latencystats_lcore_id == rte_lcore_id()) 2276 rte_latencystats_update(); 2277 #endif 2278 2279 } while (! fc->stopped); 2280 } 2281 2282 static int 2283 start_pkt_forward_on_core(void *fwd_arg) 2284 { 2285 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2286 cur_fwd_config.fwd_eng->packet_fwd); 2287 return 0; 2288 } 2289 2290 /* 2291 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2292 * Used to start communication flows in network loopback test configurations. 2293 */ 2294 static int 2295 run_one_txonly_burst_on_core(void *fwd_arg) 2296 { 2297 struct fwd_lcore *fwd_lc; 2298 struct fwd_lcore tmp_lcore; 2299 2300 fwd_lc = (struct fwd_lcore *) fwd_arg; 2301 tmp_lcore = *fwd_lc; 2302 tmp_lcore.stopped = 1; 2303 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2304 return 0; 2305 } 2306 2307 /* 2308 * Launch packet forwarding: 2309 * - Setup per-port forwarding context. 2310 * - launch logical cores with their forwarding configuration. 2311 */ 2312 static void 2313 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2314 { 2315 unsigned int i; 2316 unsigned int lc_id; 2317 int diag; 2318 2319 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2320 lc_id = fwd_lcores_cpuids[i]; 2321 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2322 fwd_lcores[i]->stopped = 0; 2323 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2324 fwd_lcores[i], lc_id); 2325 if (diag != 0) 2326 fprintf(stderr, 2327 "launch lcore %u failed - diag=%d\n", 2328 lc_id, diag); 2329 } 2330 } 2331 } 2332 2333 /* 2334 * Launch packet forwarding configuration. 2335 */ 2336 void 2337 start_packet_forwarding(int with_tx_first) 2338 { 2339 port_fwd_begin_t port_fwd_begin; 2340 port_fwd_end_t port_fwd_end; 2341 stream_init_t stream_init = cur_fwd_eng->stream_init; 2342 unsigned int i; 2343 2344 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2345 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2346 2347 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2348 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2349 2350 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2351 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2352 (!nb_rxq || !nb_txq)) 2353 rte_exit(EXIT_FAILURE, 2354 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2355 cur_fwd_eng->fwd_mode_name); 2356 2357 if (all_ports_started() == 0) { 2358 fprintf(stderr, "Not all ports were started\n"); 2359 return; 2360 } 2361 if (test_done == 0) { 2362 fprintf(stderr, "Packet forwarding already started\n"); 2363 return; 2364 } 2365 2366 fwd_config_setup(); 2367 2368 pkt_fwd_config_display(&cur_fwd_config); 2369 if (!pkt_fwd_shared_rxq_check()) 2370 return; 2371 2372 if (stream_init != NULL) 2373 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2374 stream_init(fwd_streams[i]); 2375 2376 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2377 if (port_fwd_begin != NULL) { 2378 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2379 if (port_fwd_begin(fwd_ports_ids[i])) { 2380 fprintf(stderr, 2381 "Packet forwarding is not ready\n"); 2382 return; 2383 } 2384 } 2385 } 2386 2387 if (with_tx_first) { 2388 port_fwd_begin = tx_only_engine.port_fwd_begin; 2389 if (port_fwd_begin != NULL) { 2390 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2391 if (port_fwd_begin(fwd_ports_ids[i])) { 2392 fprintf(stderr, 2393 "Packet forwarding is not ready\n"); 2394 return; 2395 } 2396 } 2397 } 2398 } 2399 2400 test_done = 0; 2401 2402 if(!no_flush_rx) 2403 flush_fwd_rx_queues(); 2404 2405 rxtx_config_display(); 2406 2407 fwd_stats_reset(); 2408 if (with_tx_first) { 2409 while (with_tx_first--) { 2410 launch_packet_forwarding( 2411 run_one_txonly_burst_on_core); 2412 rte_eal_mp_wait_lcore(); 2413 } 2414 port_fwd_end = tx_only_engine.port_fwd_end; 2415 if (port_fwd_end != NULL) { 2416 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2417 (*port_fwd_end)(fwd_ports_ids[i]); 2418 } 2419 } 2420 launch_packet_forwarding(start_pkt_forward_on_core); 2421 } 2422 2423 void 2424 stop_packet_forwarding(void) 2425 { 2426 port_fwd_end_t port_fwd_end; 2427 lcoreid_t lc_id; 2428 portid_t pt_id; 2429 int i; 2430 2431 if (test_done) { 2432 fprintf(stderr, "Packet forwarding not started\n"); 2433 return; 2434 } 2435 printf("Telling cores to stop..."); 2436 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2437 fwd_lcores[lc_id]->stopped = 1; 2438 printf("\nWaiting for lcores to finish...\n"); 2439 rte_eal_mp_wait_lcore(); 2440 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2441 if (port_fwd_end != NULL) { 2442 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2443 pt_id = fwd_ports_ids[i]; 2444 (*port_fwd_end)(pt_id); 2445 } 2446 } 2447 2448 fwd_stats_display(); 2449 2450 printf("\nDone.\n"); 2451 test_done = 1; 2452 } 2453 2454 void 2455 dev_set_link_up(portid_t pid) 2456 { 2457 if (rte_eth_dev_set_link_up(pid) < 0) 2458 fprintf(stderr, "\nSet link up fail.\n"); 2459 } 2460 2461 void 2462 dev_set_link_down(portid_t pid) 2463 { 2464 if (rte_eth_dev_set_link_down(pid) < 0) 2465 fprintf(stderr, "\nSet link down fail.\n"); 2466 } 2467 2468 static int 2469 all_ports_started(void) 2470 { 2471 portid_t pi; 2472 struct rte_port *port; 2473 2474 RTE_ETH_FOREACH_DEV(pi) { 2475 port = &ports[pi]; 2476 /* Check if there is a port which is not started */ 2477 if ((port->port_status != RTE_PORT_STARTED) && 2478 (port->slave_flag == 0)) 2479 return 0; 2480 } 2481 2482 /* No port is not started */ 2483 return 1; 2484 } 2485 2486 int 2487 port_is_stopped(portid_t port_id) 2488 { 2489 struct rte_port *port = &ports[port_id]; 2490 2491 if ((port->port_status != RTE_PORT_STOPPED) && 2492 (port->slave_flag == 0)) 2493 return 0; 2494 return 1; 2495 } 2496 2497 int 2498 all_ports_stopped(void) 2499 { 2500 portid_t pi; 2501 2502 RTE_ETH_FOREACH_DEV(pi) { 2503 if (!port_is_stopped(pi)) 2504 return 0; 2505 } 2506 2507 return 1; 2508 } 2509 2510 int 2511 port_is_started(portid_t port_id) 2512 { 2513 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2514 return 0; 2515 2516 if (ports[port_id].port_status != RTE_PORT_STARTED) 2517 return 0; 2518 2519 return 1; 2520 } 2521 2522 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2523 static int 2524 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2525 { 2526 queueid_t qi; 2527 struct rte_eth_hairpin_conf hairpin_conf = { 2528 .peer_count = 1, 2529 }; 2530 int i; 2531 int diag; 2532 struct rte_port *port = &ports[pi]; 2533 uint16_t peer_rx_port = pi; 2534 uint16_t peer_tx_port = pi; 2535 uint32_t manual = 1; 2536 uint32_t tx_exp = hairpin_mode & 0x10; 2537 2538 if (!(hairpin_mode & 0xf)) { 2539 peer_rx_port = pi; 2540 peer_tx_port = pi; 2541 manual = 0; 2542 } else if (hairpin_mode & 0x1) { 2543 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2544 RTE_ETH_DEV_NO_OWNER); 2545 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2546 peer_tx_port = rte_eth_find_next_owned_by(0, 2547 RTE_ETH_DEV_NO_OWNER); 2548 if (p_pi != RTE_MAX_ETHPORTS) { 2549 peer_rx_port = p_pi; 2550 } else { 2551 uint16_t next_pi; 2552 2553 /* Last port will be the peer RX port of the first. */ 2554 RTE_ETH_FOREACH_DEV(next_pi) 2555 peer_rx_port = next_pi; 2556 } 2557 manual = 1; 2558 } else if (hairpin_mode & 0x2) { 2559 if (cnt_pi & 0x1) { 2560 peer_rx_port = p_pi; 2561 } else { 2562 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2563 RTE_ETH_DEV_NO_OWNER); 2564 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2565 peer_rx_port = pi; 2566 } 2567 peer_tx_port = peer_rx_port; 2568 manual = 1; 2569 } 2570 2571 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2572 hairpin_conf.peers[0].port = peer_rx_port; 2573 hairpin_conf.peers[0].queue = i + nb_rxq; 2574 hairpin_conf.manual_bind = !!manual; 2575 hairpin_conf.tx_explicit = !!tx_exp; 2576 diag = rte_eth_tx_hairpin_queue_setup 2577 (pi, qi, nb_txd, &hairpin_conf); 2578 i++; 2579 if (diag == 0) 2580 continue; 2581 2582 /* Fail to setup rx queue, return */ 2583 if (port->port_status == RTE_PORT_HANDLING) 2584 port->port_status = RTE_PORT_STOPPED; 2585 else 2586 fprintf(stderr, 2587 "Port %d can not be set back to stopped\n", pi); 2588 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2589 pi); 2590 /* try to reconfigure queues next time */ 2591 port->need_reconfig_queues = 1; 2592 return -1; 2593 } 2594 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2595 hairpin_conf.peers[0].port = peer_tx_port; 2596 hairpin_conf.peers[0].queue = i + nb_txq; 2597 hairpin_conf.manual_bind = !!manual; 2598 hairpin_conf.tx_explicit = !!tx_exp; 2599 diag = rte_eth_rx_hairpin_queue_setup 2600 (pi, qi, nb_rxd, &hairpin_conf); 2601 i++; 2602 if (diag == 0) 2603 continue; 2604 2605 /* Fail to setup rx queue, return */ 2606 if (port->port_status == RTE_PORT_HANDLING) 2607 port->port_status = RTE_PORT_STOPPED; 2608 else 2609 fprintf(stderr, 2610 "Port %d can not be set back to stopped\n", pi); 2611 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2612 pi); 2613 /* try to reconfigure queues next time */ 2614 port->need_reconfig_queues = 1; 2615 return -1; 2616 } 2617 return 0; 2618 } 2619 2620 /* Configure the Rx with optional split. */ 2621 int 2622 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2623 uint16_t nb_rx_desc, unsigned int socket_id, 2624 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2625 { 2626 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2627 unsigned int i, mp_n; 2628 int ret; 2629 2630 if (rx_pkt_nb_segs <= 1 || 2631 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2632 rx_conf->rx_seg = NULL; 2633 rx_conf->rx_nseg = 0; 2634 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2635 nb_rx_desc, socket_id, 2636 rx_conf, mp); 2637 goto exit; 2638 } 2639 for (i = 0; i < rx_pkt_nb_segs; i++) { 2640 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2641 struct rte_mempool *mpx; 2642 /* 2643 * Use last valid pool for the segments with number 2644 * exceeding the pool index. 2645 */ 2646 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2647 mpx = mbuf_pool_find(socket_id, mp_n); 2648 /* Handle zero as mbuf data buffer size. */ 2649 rx_seg->length = rx_pkt_seg_lengths[i] ? 2650 rx_pkt_seg_lengths[i] : 2651 mbuf_data_size[mp_n]; 2652 rx_seg->offset = i < rx_pkt_nb_offs ? 2653 rx_pkt_seg_offsets[i] : 0; 2654 rx_seg->mp = mpx ? mpx : mp; 2655 } 2656 rx_conf->rx_nseg = rx_pkt_nb_segs; 2657 rx_conf->rx_seg = rx_useg; 2658 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2659 socket_id, rx_conf, NULL); 2660 rx_conf->rx_seg = NULL; 2661 rx_conf->rx_nseg = 0; 2662 exit: 2663 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2664 RTE_ETH_QUEUE_STATE_STOPPED : 2665 RTE_ETH_QUEUE_STATE_STARTED; 2666 return ret; 2667 } 2668 2669 static int 2670 alloc_xstats_display_info(portid_t pi) 2671 { 2672 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2673 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2674 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2675 2676 if (xstats_display_num == 0) 2677 return 0; 2678 2679 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2680 if (*ids_supp == NULL) 2681 goto fail_ids_supp; 2682 2683 *prev_values = calloc(xstats_display_num, 2684 sizeof(**prev_values)); 2685 if (*prev_values == NULL) 2686 goto fail_prev_values; 2687 2688 *curr_values = calloc(xstats_display_num, 2689 sizeof(**curr_values)); 2690 if (*curr_values == NULL) 2691 goto fail_curr_values; 2692 2693 ports[pi].xstats_info.allocated = true; 2694 2695 return 0; 2696 2697 fail_curr_values: 2698 free(*prev_values); 2699 fail_prev_values: 2700 free(*ids_supp); 2701 fail_ids_supp: 2702 return -ENOMEM; 2703 } 2704 2705 static void 2706 free_xstats_display_info(portid_t pi) 2707 { 2708 if (!ports[pi].xstats_info.allocated) 2709 return; 2710 free(ports[pi].xstats_info.ids_supp); 2711 free(ports[pi].xstats_info.prev_values); 2712 free(ports[pi].xstats_info.curr_values); 2713 ports[pi].xstats_info.allocated = false; 2714 } 2715 2716 /** Fill helper structures for specified port to show extended statistics. */ 2717 static void 2718 fill_xstats_display_info_for_port(portid_t pi) 2719 { 2720 unsigned int stat, stat_supp; 2721 const char *xstat_name; 2722 struct rte_port *port; 2723 uint64_t *ids_supp; 2724 int rc; 2725 2726 if (xstats_display_num == 0) 2727 return; 2728 2729 if (pi == (portid_t)RTE_PORT_ALL) { 2730 fill_xstats_display_info(); 2731 return; 2732 } 2733 2734 port = &ports[pi]; 2735 if (port->port_status != RTE_PORT_STARTED) 2736 return; 2737 2738 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2739 rte_exit(EXIT_FAILURE, 2740 "Failed to allocate xstats display memory\n"); 2741 2742 ids_supp = port->xstats_info.ids_supp; 2743 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2744 xstat_name = xstats_display[stat].name; 2745 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2746 ids_supp + stat_supp); 2747 if (rc != 0) { 2748 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2749 xstat_name, pi, stat); 2750 continue; 2751 } 2752 stat_supp++; 2753 } 2754 2755 port->xstats_info.ids_supp_sz = stat_supp; 2756 } 2757 2758 /** Fill helper structures for all ports to show extended statistics. */ 2759 static void 2760 fill_xstats_display_info(void) 2761 { 2762 portid_t pi; 2763 2764 if (xstats_display_num == 0) 2765 return; 2766 2767 RTE_ETH_FOREACH_DEV(pi) 2768 fill_xstats_display_info_for_port(pi); 2769 } 2770 2771 int 2772 start_port(portid_t pid) 2773 { 2774 int diag, need_check_link_status = -1; 2775 portid_t pi; 2776 portid_t p_pi = RTE_MAX_ETHPORTS; 2777 portid_t pl[RTE_MAX_ETHPORTS]; 2778 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2779 uint16_t cnt_pi = 0; 2780 uint16_t cfg_pi = 0; 2781 int peer_pi; 2782 queueid_t qi; 2783 struct rte_port *port; 2784 struct rte_eth_hairpin_cap cap; 2785 2786 if (port_id_is_invalid(pid, ENABLED_WARN)) 2787 return 0; 2788 2789 RTE_ETH_FOREACH_DEV(pi) { 2790 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2791 continue; 2792 2793 if (port_is_bonding_slave(pi)) { 2794 fprintf(stderr, 2795 "Please remove port %d from bonded device.\n", 2796 pi); 2797 continue; 2798 } 2799 2800 need_check_link_status = 0; 2801 port = &ports[pi]; 2802 if (port->port_status == RTE_PORT_STOPPED) 2803 port->port_status = RTE_PORT_HANDLING; 2804 else { 2805 fprintf(stderr, "Port %d is now not stopped\n", pi); 2806 continue; 2807 } 2808 2809 if (port->need_reconfig > 0) { 2810 struct rte_eth_conf dev_conf; 2811 int k; 2812 2813 port->need_reconfig = 0; 2814 2815 if (flow_isolate_all) { 2816 int ret = port_flow_isolate(pi, 1); 2817 if (ret) { 2818 fprintf(stderr, 2819 "Failed to apply isolated mode on port %d\n", 2820 pi); 2821 return -1; 2822 } 2823 } 2824 configure_rxtx_dump_callbacks(0); 2825 printf("Configuring Port %d (socket %u)\n", pi, 2826 port->socket_id); 2827 if (nb_hairpinq > 0 && 2828 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2829 fprintf(stderr, 2830 "Port %d doesn't support hairpin queues\n", 2831 pi); 2832 return -1; 2833 } 2834 2835 /* configure port */ 2836 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2837 nb_txq + nb_hairpinq, 2838 &(port->dev_conf)); 2839 if (diag != 0) { 2840 if (port->port_status == RTE_PORT_HANDLING) 2841 port->port_status = RTE_PORT_STOPPED; 2842 else 2843 fprintf(stderr, 2844 "Port %d can not be set back to stopped\n", 2845 pi); 2846 fprintf(stderr, "Fail to configure port %d\n", 2847 pi); 2848 /* try to reconfigure port next time */ 2849 port->need_reconfig = 1; 2850 return -1; 2851 } 2852 /* get device configuration*/ 2853 if (0 != 2854 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2855 fprintf(stderr, 2856 "port %d can not get device configuration\n", 2857 pi); 2858 return -1; 2859 } 2860 /* Apply Rx offloads configuration */ 2861 if (dev_conf.rxmode.offloads != 2862 port->dev_conf.rxmode.offloads) { 2863 port->dev_conf.rxmode.offloads |= 2864 dev_conf.rxmode.offloads; 2865 for (k = 0; 2866 k < port->dev_info.max_rx_queues; 2867 k++) 2868 port->rxq[k].conf.offloads |= 2869 dev_conf.rxmode.offloads; 2870 } 2871 /* Apply Tx offloads configuration */ 2872 if (dev_conf.txmode.offloads != 2873 port->dev_conf.txmode.offloads) { 2874 port->dev_conf.txmode.offloads |= 2875 dev_conf.txmode.offloads; 2876 for (k = 0; 2877 k < port->dev_info.max_tx_queues; 2878 k++) 2879 port->txq[k].conf.offloads |= 2880 dev_conf.txmode.offloads; 2881 } 2882 } 2883 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2884 port->need_reconfig_queues = 0; 2885 /* setup tx queues */ 2886 for (qi = 0; qi < nb_txq; qi++) { 2887 struct rte_eth_txconf *conf = 2888 &port->txq[qi].conf; 2889 2890 if ((numa_support) && 2891 (txring_numa[pi] != NUMA_NO_CONFIG)) 2892 diag = rte_eth_tx_queue_setup(pi, qi, 2893 port->nb_tx_desc[qi], 2894 txring_numa[pi], 2895 &(port->txq[qi].conf)); 2896 else 2897 diag = rte_eth_tx_queue_setup(pi, qi, 2898 port->nb_tx_desc[qi], 2899 port->socket_id, 2900 &(port->txq[qi].conf)); 2901 2902 if (diag == 0) { 2903 port->txq[qi].state = 2904 conf->tx_deferred_start ? 2905 RTE_ETH_QUEUE_STATE_STOPPED : 2906 RTE_ETH_QUEUE_STATE_STARTED; 2907 continue; 2908 } 2909 2910 /* Fail to setup tx queue, return */ 2911 if (port->port_status == RTE_PORT_HANDLING) 2912 port->port_status = RTE_PORT_STOPPED; 2913 else 2914 fprintf(stderr, 2915 "Port %d can not be set back to stopped\n", 2916 pi); 2917 fprintf(stderr, 2918 "Fail to configure port %d tx queues\n", 2919 pi); 2920 /* try to reconfigure queues next time */ 2921 port->need_reconfig_queues = 1; 2922 return -1; 2923 } 2924 for (qi = 0; qi < nb_rxq; qi++) { 2925 /* setup rx queues */ 2926 if ((numa_support) && 2927 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2928 struct rte_mempool * mp = 2929 mbuf_pool_find 2930 (rxring_numa[pi], 0); 2931 if (mp == NULL) { 2932 fprintf(stderr, 2933 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2934 rxring_numa[pi]); 2935 return -1; 2936 } 2937 2938 diag = rx_queue_setup(pi, qi, 2939 port->nb_rx_desc[qi], 2940 rxring_numa[pi], 2941 &(port->rxq[qi].conf), 2942 mp); 2943 } else { 2944 struct rte_mempool *mp = 2945 mbuf_pool_find 2946 (port->socket_id, 0); 2947 if (mp == NULL) { 2948 fprintf(stderr, 2949 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2950 port->socket_id); 2951 return -1; 2952 } 2953 diag = rx_queue_setup(pi, qi, 2954 port->nb_rx_desc[qi], 2955 port->socket_id, 2956 &(port->rxq[qi].conf), 2957 mp); 2958 } 2959 if (diag == 0) 2960 continue; 2961 2962 /* Fail to setup rx queue, return */ 2963 if (port->port_status == RTE_PORT_HANDLING) 2964 port->port_status = RTE_PORT_STOPPED; 2965 else 2966 fprintf(stderr, 2967 "Port %d can not be set back to stopped\n", 2968 pi); 2969 fprintf(stderr, 2970 "Fail to configure port %d rx queues\n", 2971 pi); 2972 /* try to reconfigure queues next time */ 2973 port->need_reconfig_queues = 1; 2974 return -1; 2975 } 2976 /* setup hairpin queues */ 2977 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2978 return -1; 2979 } 2980 configure_rxtx_dump_callbacks(verbose_level); 2981 if (clear_ptypes) { 2982 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2983 NULL, 0); 2984 if (diag < 0) 2985 fprintf(stderr, 2986 "Port %d: Failed to disable Ptype parsing\n", 2987 pi); 2988 } 2989 2990 p_pi = pi; 2991 cnt_pi++; 2992 2993 /* start port */ 2994 diag = eth_dev_start_mp(pi); 2995 if (diag < 0) { 2996 fprintf(stderr, "Fail to start port %d: %s\n", 2997 pi, rte_strerror(-diag)); 2998 2999 /* Fail to setup rx queue, return */ 3000 if (port->port_status == RTE_PORT_HANDLING) 3001 port->port_status = RTE_PORT_STOPPED; 3002 else 3003 fprintf(stderr, 3004 "Port %d can not be set back to stopped\n", 3005 pi); 3006 continue; 3007 } 3008 3009 if (port->port_status == RTE_PORT_HANDLING) 3010 port->port_status = RTE_PORT_STARTED; 3011 else 3012 fprintf(stderr, "Port %d can not be set into started\n", 3013 pi); 3014 3015 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3016 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3017 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3018 3019 /* at least one port started, need checking link status */ 3020 need_check_link_status = 1; 3021 3022 pl[cfg_pi++] = pi; 3023 } 3024 3025 if (need_check_link_status == 1 && !no_link_check) 3026 check_all_ports_link_status(RTE_PORT_ALL); 3027 else if (need_check_link_status == 0) 3028 fprintf(stderr, "Please stop the ports first\n"); 3029 3030 if (hairpin_mode & 0xf) { 3031 uint16_t i; 3032 int j; 3033 3034 /* bind all started hairpin ports */ 3035 for (i = 0; i < cfg_pi; i++) { 3036 pi = pl[i]; 3037 /* bind current Tx to all peer Rx */ 3038 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3039 RTE_MAX_ETHPORTS, 1); 3040 if (peer_pi < 0) 3041 return peer_pi; 3042 for (j = 0; j < peer_pi; j++) { 3043 if (!port_is_started(peer_pl[j])) 3044 continue; 3045 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3046 if (diag < 0) { 3047 fprintf(stderr, 3048 "Error during binding hairpin Tx port %u to %u: %s\n", 3049 pi, peer_pl[j], 3050 rte_strerror(-diag)); 3051 return -1; 3052 } 3053 } 3054 /* bind all peer Tx to current Rx */ 3055 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3056 RTE_MAX_ETHPORTS, 0); 3057 if (peer_pi < 0) 3058 return peer_pi; 3059 for (j = 0; j < peer_pi; j++) { 3060 if (!port_is_started(peer_pl[j])) 3061 continue; 3062 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3063 if (diag < 0) { 3064 fprintf(stderr, 3065 "Error during binding hairpin Tx port %u to %u: %s\n", 3066 peer_pl[j], pi, 3067 rte_strerror(-diag)); 3068 return -1; 3069 } 3070 } 3071 } 3072 } 3073 3074 fill_xstats_display_info_for_port(pid); 3075 3076 printf("Done\n"); 3077 return 0; 3078 } 3079 3080 void 3081 stop_port(portid_t pid) 3082 { 3083 portid_t pi; 3084 struct rte_port *port; 3085 int need_check_link_status = 0; 3086 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3087 int peer_pi; 3088 3089 if (port_id_is_invalid(pid, ENABLED_WARN)) 3090 return; 3091 3092 printf("Stopping ports...\n"); 3093 3094 RTE_ETH_FOREACH_DEV(pi) { 3095 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3096 continue; 3097 3098 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3099 fprintf(stderr, 3100 "Please remove port %d from forwarding configuration.\n", 3101 pi); 3102 continue; 3103 } 3104 3105 if (port_is_bonding_slave(pi)) { 3106 fprintf(stderr, 3107 "Please remove port %d from bonded device.\n", 3108 pi); 3109 continue; 3110 } 3111 3112 port = &ports[pi]; 3113 if (port->port_status == RTE_PORT_STARTED) 3114 port->port_status = RTE_PORT_HANDLING; 3115 else 3116 continue; 3117 3118 if (hairpin_mode & 0xf) { 3119 int j; 3120 3121 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3122 /* unbind all peer Tx from current Rx */ 3123 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3124 RTE_MAX_ETHPORTS, 0); 3125 if (peer_pi < 0) 3126 continue; 3127 for (j = 0; j < peer_pi; j++) { 3128 if (!port_is_started(peer_pl[j])) 3129 continue; 3130 rte_eth_hairpin_unbind(peer_pl[j], pi); 3131 } 3132 } 3133 3134 if (port->flow_list) 3135 port_flow_flush(pi); 3136 3137 if (eth_dev_stop_mp(pi) != 0) 3138 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3139 pi); 3140 3141 if (port->port_status == RTE_PORT_HANDLING) 3142 port->port_status = RTE_PORT_STOPPED; 3143 else 3144 fprintf(stderr, "Port %d can not be set into stopped\n", 3145 pi); 3146 need_check_link_status = 1; 3147 } 3148 if (need_check_link_status && !no_link_check) 3149 check_all_ports_link_status(RTE_PORT_ALL); 3150 3151 printf("Done\n"); 3152 } 3153 3154 static void 3155 remove_invalid_ports_in(portid_t *array, portid_t *total) 3156 { 3157 portid_t i; 3158 portid_t new_total = 0; 3159 3160 for (i = 0; i < *total; i++) 3161 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3162 array[new_total] = array[i]; 3163 new_total++; 3164 } 3165 *total = new_total; 3166 } 3167 3168 static void 3169 remove_invalid_ports(void) 3170 { 3171 remove_invalid_ports_in(ports_ids, &nb_ports); 3172 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3173 nb_cfg_ports = nb_fwd_ports; 3174 } 3175 3176 static void 3177 flush_port_owned_resources(portid_t pi) 3178 { 3179 mcast_addr_pool_destroy(pi); 3180 port_flow_flush(pi); 3181 port_flex_item_flush(pi); 3182 port_action_handle_flush(pi); 3183 } 3184 3185 static void 3186 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3187 { 3188 struct rte_port *port; 3189 portid_t slave_pid; 3190 uint16_t i; 3191 3192 for (i = 0; i < num_slaves; i++) { 3193 slave_pid = slave_pids[i]; 3194 if (port_is_started(slave_pid) == 1) { 3195 if (rte_eth_dev_stop(slave_pid) != 0) 3196 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3197 slave_pid); 3198 3199 port = &ports[slave_pid]; 3200 port->port_status = RTE_PORT_STOPPED; 3201 } 3202 3203 clear_port_slave_flag(slave_pid); 3204 3205 /* Close slave device when testpmd quit or is killed. */ 3206 if (cl_quit == 1 || f_quit == 1) 3207 rte_eth_dev_close(slave_pid); 3208 } 3209 } 3210 3211 void 3212 close_port(portid_t pid) 3213 { 3214 portid_t pi; 3215 struct rte_port *port; 3216 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3217 int num_slaves = 0; 3218 3219 if (port_id_is_invalid(pid, ENABLED_WARN)) 3220 return; 3221 3222 printf("Closing ports...\n"); 3223 3224 RTE_ETH_FOREACH_DEV(pi) { 3225 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3226 continue; 3227 3228 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3229 fprintf(stderr, 3230 "Please remove port %d from forwarding configuration.\n", 3231 pi); 3232 continue; 3233 } 3234 3235 if (port_is_bonding_slave(pi)) { 3236 fprintf(stderr, 3237 "Please remove port %d from bonded device.\n", 3238 pi); 3239 continue; 3240 } 3241 3242 port = &ports[pi]; 3243 if (port->port_status == RTE_PORT_CLOSED) { 3244 fprintf(stderr, "Port %d is already closed\n", pi); 3245 continue; 3246 } 3247 3248 if (is_proc_primary()) { 3249 flush_port_owned_resources(pi); 3250 #ifdef RTE_NET_BOND 3251 if (port->bond_flag == 1) 3252 num_slaves = rte_eth_bond_slaves_get(pi, 3253 slave_pids, RTE_MAX_ETHPORTS); 3254 #endif 3255 rte_eth_dev_close(pi); 3256 /* 3257 * If this port is bonded device, all slaves under the 3258 * device need to be removed or closed. 3259 */ 3260 if (port->bond_flag == 1 && num_slaves > 0) 3261 clear_bonding_slave_device(slave_pids, 3262 num_slaves); 3263 } 3264 3265 free_xstats_display_info(pi); 3266 } 3267 3268 remove_invalid_ports(); 3269 printf("Done\n"); 3270 } 3271 3272 void 3273 reset_port(portid_t pid) 3274 { 3275 int diag; 3276 portid_t pi; 3277 struct rte_port *port; 3278 3279 if (port_id_is_invalid(pid, ENABLED_WARN)) 3280 return; 3281 3282 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3283 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3284 fprintf(stderr, 3285 "Can not reset port(s), please stop port(s) first.\n"); 3286 return; 3287 } 3288 3289 printf("Resetting ports...\n"); 3290 3291 RTE_ETH_FOREACH_DEV(pi) { 3292 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3293 continue; 3294 3295 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3296 fprintf(stderr, 3297 "Please remove port %d from forwarding configuration.\n", 3298 pi); 3299 continue; 3300 } 3301 3302 if (port_is_bonding_slave(pi)) { 3303 fprintf(stderr, 3304 "Please remove port %d from bonded device.\n", 3305 pi); 3306 continue; 3307 } 3308 3309 diag = rte_eth_dev_reset(pi); 3310 if (diag == 0) { 3311 port = &ports[pi]; 3312 port->need_reconfig = 1; 3313 port->need_reconfig_queues = 1; 3314 } else { 3315 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3316 pi, diag); 3317 } 3318 } 3319 3320 printf("Done\n"); 3321 } 3322 3323 void 3324 attach_port(char *identifier) 3325 { 3326 portid_t pi; 3327 struct rte_dev_iterator iterator; 3328 3329 printf("Attaching a new port...\n"); 3330 3331 if (identifier == NULL) { 3332 fprintf(stderr, "Invalid parameters are specified\n"); 3333 return; 3334 } 3335 3336 if (rte_dev_probe(identifier) < 0) { 3337 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3338 return; 3339 } 3340 3341 /* first attach mode: event */ 3342 if (setup_on_probe_event) { 3343 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3344 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3345 if (ports[pi].port_status == RTE_PORT_HANDLING && 3346 ports[pi].need_setup != 0) 3347 setup_attached_port(pi); 3348 return; 3349 } 3350 3351 /* second attach mode: iterator */ 3352 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3353 /* setup ports matching the devargs used for probing */ 3354 if (port_is_forwarding(pi)) 3355 continue; /* port was already attached before */ 3356 setup_attached_port(pi); 3357 } 3358 } 3359 3360 static void 3361 setup_attached_port(portid_t pi) 3362 { 3363 unsigned int socket_id; 3364 int ret; 3365 3366 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3367 /* if socket_id is invalid, set to the first available socket. */ 3368 if (check_socket_id(socket_id) < 0) 3369 socket_id = socket_ids[0]; 3370 reconfig(pi, socket_id); 3371 ret = rte_eth_promiscuous_enable(pi); 3372 if (ret != 0) 3373 fprintf(stderr, 3374 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3375 pi, rte_strerror(-ret)); 3376 3377 ports_ids[nb_ports++] = pi; 3378 fwd_ports_ids[nb_fwd_ports++] = pi; 3379 nb_cfg_ports = nb_fwd_ports; 3380 ports[pi].need_setup = 0; 3381 ports[pi].port_status = RTE_PORT_STOPPED; 3382 3383 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3384 printf("Done\n"); 3385 } 3386 3387 static void 3388 detach_device(struct rte_device *dev) 3389 { 3390 portid_t sibling; 3391 3392 if (dev == NULL) { 3393 fprintf(stderr, "Device already removed\n"); 3394 return; 3395 } 3396 3397 printf("Removing a device...\n"); 3398 3399 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3400 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3401 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3402 fprintf(stderr, "Port %u not stopped\n", 3403 sibling); 3404 return; 3405 } 3406 flush_port_owned_resources(sibling); 3407 } 3408 } 3409 3410 if (rte_dev_remove(dev) < 0) { 3411 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3412 return; 3413 } 3414 remove_invalid_ports(); 3415 3416 printf("Device is detached\n"); 3417 printf("Now total ports is %d\n", nb_ports); 3418 printf("Done\n"); 3419 return; 3420 } 3421 3422 void 3423 detach_port_device(portid_t port_id) 3424 { 3425 int ret; 3426 struct rte_eth_dev_info dev_info; 3427 3428 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3429 return; 3430 3431 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3432 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3433 fprintf(stderr, "Port not stopped\n"); 3434 return; 3435 } 3436 fprintf(stderr, "Port was not closed\n"); 3437 } 3438 3439 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3440 if (ret != 0) { 3441 TESTPMD_LOG(ERR, 3442 "Failed to get device info for port %d, not detaching\n", 3443 port_id); 3444 return; 3445 } 3446 detach_device(dev_info.device); 3447 } 3448 3449 void 3450 detach_devargs(char *identifier) 3451 { 3452 struct rte_dev_iterator iterator; 3453 struct rte_devargs da; 3454 portid_t port_id; 3455 3456 printf("Removing a device...\n"); 3457 3458 memset(&da, 0, sizeof(da)); 3459 if (rte_devargs_parsef(&da, "%s", identifier)) { 3460 fprintf(stderr, "cannot parse identifier\n"); 3461 return; 3462 } 3463 3464 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3465 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3466 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3467 fprintf(stderr, "Port %u not stopped\n", 3468 port_id); 3469 rte_eth_iterator_cleanup(&iterator); 3470 rte_devargs_reset(&da); 3471 return; 3472 } 3473 flush_port_owned_resources(port_id); 3474 } 3475 } 3476 3477 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3478 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3479 da.name, rte_bus_name(da.bus)); 3480 rte_devargs_reset(&da); 3481 return; 3482 } 3483 3484 remove_invalid_ports(); 3485 3486 printf("Device %s is detached\n", identifier); 3487 printf("Now total ports is %d\n", nb_ports); 3488 printf("Done\n"); 3489 rte_devargs_reset(&da); 3490 } 3491 3492 void 3493 pmd_test_exit(void) 3494 { 3495 portid_t pt_id; 3496 unsigned int i; 3497 int ret; 3498 3499 if (test_done == 0) 3500 stop_packet_forwarding(); 3501 3502 #ifndef RTE_EXEC_ENV_WINDOWS 3503 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3504 if (mempools[i]) { 3505 if (mp_alloc_type == MP_ALLOC_ANON) 3506 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3507 NULL); 3508 } 3509 } 3510 #endif 3511 if (ports != NULL) { 3512 no_link_check = 1; 3513 RTE_ETH_FOREACH_DEV(pt_id) { 3514 printf("\nStopping port %d...\n", pt_id); 3515 fflush(stdout); 3516 stop_port(pt_id); 3517 } 3518 RTE_ETH_FOREACH_DEV(pt_id) { 3519 printf("\nShutting down port %d...\n", pt_id); 3520 fflush(stdout); 3521 close_port(pt_id); 3522 } 3523 } 3524 3525 if (hot_plug) { 3526 ret = rte_dev_event_monitor_stop(); 3527 if (ret) { 3528 RTE_LOG(ERR, EAL, 3529 "fail to stop device event monitor."); 3530 return; 3531 } 3532 3533 ret = rte_dev_event_callback_unregister(NULL, 3534 dev_event_callback, NULL); 3535 if (ret < 0) { 3536 RTE_LOG(ERR, EAL, 3537 "fail to unregister device event callback.\n"); 3538 return; 3539 } 3540 3541 ret = rte_dev_hotplug_handle_disable(); 3542 if (ret) { 3543 RTE_LOG(ERR, EAL, 3544 "fail to disable hotplug handling.\n"); 3545 return; 3546 } 3547 } 3548 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3549 if (mempools[i]) 3550 mempool_free_mp(mempools[i]); 3551 } 3552 free(xstats_display); 3553 3554 printf("\nBye...\n"); 3555 } 3556 3557 typedef void (*cmd_func_t)(void); 3558 struct pmd_test_command { 3559 const char *cmd_name; 3560 cmd_func_t cmd_func; 3561 }; 3562 3563 /* Check the link status of all ports in up to 9s, and print them finally */ 3564 static void 3565 check_all_ports_link_status(uint32_t port_mask) 3566 { 3567 #define CHECK_INTERVAL 100 /* 100ms */ 3568 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3569 portid_t portid; 3570 uint8_t count, all_ports_up, print_flag = 0; 3571 struct rte_eth_link link; 3572 int ret; 3573 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3574 3575 printf("Checking link statuses...\n"); 3576 fflush(stdout); 3577 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3578 all_ports_up = 1; 3579 RTE_ETH_FOREACH_DEV(portid) { 3580 if ((port_mask & (1 << portid)) == 0) 3581 continue; 3582 memset(&link, 0, sizeof(link)); 3583 ret = rte_eth_link_get_nowait(portid, &link); 3584 if (ret < 0) { 3585 all_ports_up = 0; 3586 if (print_flag == 1) 3587 fprintf(stderr, 3588 "Port %u link get failed: %s\n", 3589 portid, rte_strerror(-ret)); 3590 continue; 3591 } 3592 /* print link status if flag set */ 3593 if (print_flag == 1) { 3594 rte_eth_link_to_str(link_status, 3595 sizeof(link_status), &link); 3596 printf("Port %d %s\n", portid, link_status); 3597 continue; 3598 } 3599 /* clear all_ports_up flag if any link down */ 3600 if (link.link_status == RTE_ETH_LINK_DOWN) { 3601 all_ports_up = 0; 3602 break; 3603 } 3604 } 3605 /* after finally printing all link status, get out */ 3606 if (print_flag == 1) 3607 break; 3608 3609 if (all_ports_up == 0) { 3610 fflush(stdout); 3611 rte_delay_ms(CHECK_INTERVAL); 3612 } 3613 3614 /* set the print_flag if all ports up or timeout */ 3615 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3616 print_flag = 1; 3617 } 3618 3619 if (lsc_interrupt) 3620 break; 3621 } 3622 } 3623 3624 static void 3625 rmv_port_callback(void *arg) 3626 { 3627 int need_to_start = 0; 3628 int org_no_link_check = no_link_check; 3629 portid_t port_id = (intptr_t)arg; 3630 struct rte_eth_dev_info dev_info; 3631 int ret; 3632 3633 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3634 3635 if (!test_done && port_is_forwarding(port_id)) { 3636 need_to_start = 1; 3637 stop_packet_forwarding(); 3638 } 3639 no_link_check = 1; 3640 stop_port(port_id); 3641 no_link_check = org_no_link_check; 3642 3643 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3644 if (ret != 0) 3645 TESTPMD_LOG(ERR, 3646 "Failed to get device info for port %d, not detaching\n", 3647 port_id); 3648 else { 3649 struct rte_device *device = dev_info.device; 3650 close_port(port_id); 3651 detach_device(device); /* might be already removed or have more ports */ 3652 } 3653 if (need_to_start) 3654 start_packet_forwarding(0); 3655 } 3656 3657 /* This function is used by the interrupt thread */ 3658 static int 3659 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3660 void *ret_param) 3661 { 3662 RTE_SET_USED(param); 3663 RTE_SET_USED(ret_param); 3664 3665 if (type >= RTE_ETH_EVENT_MAX) { 3666 fprintf(stderr, 3667 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3668 port_id, __func__, type); 3669 fflush(stderr); 3670 } else if (event_print_mask & (UINT32_C(1) << type)) { 3671 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3672 eth_event_desc[type]); 3673 fflush(stdout); 3674 } 3675 3676 switch (type) { 3677 case RTE_ETH_EVENT_NEW: 3678 ports[port_id].need_setup = 1; 3679 ports[port_id].port_status = RTE_PORT_HANDLING; 3680 break; 3681 case RTE_ETH_EVENT_INTR_RMV: 3682 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3683 break; 3684 if (rte_eal_alarm_set(100000, 3685 rmv_port_callback, (void *)(intptr_t)port_id)) 3686 fprintf(stderr, 3687 "Could not set up deferred device removal\n"); 3688 break; 3689 case RTE_ETH_EVENT_DESTROY: 3690 ports[port_id].port_status = RTE_PORT_CLOSED; 3691 printf("Port %u is closed\n", port_id); 3692 break; 3693 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3694 uint16_t rxq_id; 3695 int ret; 3696 3697 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3698 for (rxq_id = 0; ; rxq_id++) { 3699 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3700 NULL); 3701 if (ret <= 0) 3702 break; 3703 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3704 port_id, rxq_id); 3705 3706 #ifdef RTE_NET_MLX5 3707 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3708 #endif 3709 } 3710 break; 3711 } 3712 default: 3713 break; 3714 } 3715 return 0; 3716 } 3717 3718 static int 3719 register_eth_event_callback(void) 3720 { 3721 int ret; 3722 enum rte_eth_event_type event; 3723 3724 for (event = RTE_ETH_EVENT_UNKNOWN; 3725 event < RTE_ETH_EVENT_MAX; event++) { 3726 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3727 event, 3728 eth_event_callback, 3729 NULL); 3730 if (ret != 0) { 3731 TESTPMD_LOG(ERR, "Failed to register callback for " 3732 "%s event\n", eth_event_desc[event]); 3733 return -1; 3734 } 3735 } 3736 3737 return 0; 3738 } 3739 3740 /* This function is used by the interrupt thread */ 3741 static void 3742 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3743 __rte_unused void *arg) 3744 { 3745 uint16_t port_id; 3746 int ret; 3747 3748 if (type >= RTE_DEV_EVENT_MAX) { 3749 fprintf(stderr, "%s called upon invalid event %d\n", 3750 __func__, type); 3751 fflush(stderr); 3752 } 3753 3754 switch (type) { 3755 case RTE_DEV_EVENT_REMOVE: 3756 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3757 device_name); 3758 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3759 if (ret) { 3760 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3761 device_name); 3762 return; 3763 } 3764 /* 3765 * Because the user's callback is invoked in eal interrupt 3766 * callback, the interrupt callback need to be finished before 3767 * it can be unregistered when detaching device. So finish 3768 * callback soon and use a deferred removal to detach device 3769 * is need. It is a workaround, once the device detaching be 3770 * moved into the eal in the future, the deferred removal could 3771 * be deleted. 3772 */ 3773 if (rte_eal_alarm_set(100000, 3774 rmv_port_callback, (void *)(intptr_t)port_id)) 3775 RTE_LOG(ERR, EAL, 3776 "Could not set up deferred device removal\n"); 3777 break; 3778 case RTE_DEV_EVENT_ADD: 3779 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3780 device_name); 3781 /* TODO: After finish kernel driver binding, 3782 * begin to attach port. 3783 */ 3784 break; 3785 default: 3786 break; 3787 } 3788 } 3789 3790 static void 3791 rxtx_port_config(portid_t pid) 3792 { 3793 uint16_t qid; 3794 uint64_t offloads; 3795 struct rte_port *port = &ports[pid]; 3796 3797 for (qid = 0; qid < nb_rxq; qid++) { 3798 offloads = port->rxq[qid].conf.offloads; 3799 port->rxq[qid].conf = port->dev_info.default_rxconf; 3800 3801 if (rxq_share > 0 && 3802 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3803 /* Non-zero share group to enable RxQ share. */ 3804 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3805 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3806 } 3807 3808 if (offloads != 0) 3809 port->rxq[qid].conf.offloads = offloads; 3810 3811 /* Check if any Rx parameters have been passed */ 3812 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3813 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3814 3815 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3816 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3817 3818 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3819 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3820 3821 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3822 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3823 3824 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3825 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3826 3827 port->nb_rx_desc[qid] = nb_rxd; 3828 } 3829 3830 for (qid = 0; qid < nb_txq; qid++) { 3831 offloads = port->txq[qid].conf.offloads; 3832 port->txq[qid].conf = port->dev_info.default_txconf; 3833 if (offloads != 0) 3834 port->txq[qid].conf.offloads = offloads; 3835 3836 /* Check if any Tx parameters have been passed */ 3837 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3838 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3839 3840 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3841 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3842 3843 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3844 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3845 3846 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3847 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3848 3849 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3850 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3851 3852 port->nb_tx_desc[qid] = nb_txd; 3853 } 3854 } 3855 3856 /* 3857 * Helper function to set MTU from frame size 3858 * 3859 * port->dev_info should be set before calling this function. 3860 * 3861 * return 0 on success, negative on error 3862 */ 3863 int 3864 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3865 { 3866 struct rte_port *port = &ports[portid]; 3867 uint32_t eth_overhead; 3868 uint16_t mtu, new_mtu; 3869 3870 eth_overhead = get_eth_overhead(&port->dev_info); 3871 3872 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3873 printf("Failed to get MTU for port %u\n", portid); 3874 return -1; 3875 } 3876 3877 new_mtu = max_rx_pktlen - eth_overhead; 3878 3879 if (mtu == new_mtu) 3880 return 0; 3881 3882 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3883 fprintf(stderr, 3884 "Failed to set MTU to %u for port %u\n", 3885 new_mtu, portid); 3886 return -1; 3887 } 3888 3889 port->dev_conf.rxmode.mtu = new_mtu; 3890 3891 return 0; 3892 } 3893 3894 void 3895 init_port_config(void) 3896 { 3897 portid_t pid; 3898 struct rte_port *port; 3899 int ret, i; 3900 3901 RTE_ETH_FOREACH_DEV(pid) { 3902 port = &ports[pid]; 3903 3904 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3905 if (ret != 0) 3906 return; 3907 3908 if (nb_rxq > 1) { 3909 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3910 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3911 rss_hf & port->dev_info.flow_type_rss_offloads; 3912 } else { 3913 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3914 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3915 } 3916 3917 if (port->dcb_flag == 0) { 3918 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3919 port->dev_conf.rxmode.mq_mode = 3920 (enum rte_eth_rx_mq_mode) 3921 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3922 } else { 3923 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3924 port->dev_conf.rxmode.offloads &= 3925 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3926 3927 for (i = 0; 3928 i < port->dev_info.nb_rx_queues; 3929 i++) 3930 port->rxq[i].conf.offloads &= 3931 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3932 } 3933 } 3934 3935 rxtx_port_config(pid); 3936 3937 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3938 if (ret != 0) 3939 return; 3940 3941 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3942 port->dev_conf.intr_conf.lsc = 1; 3943 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3944 port->dev_conf.intr_conf.rmv = 1; 3945 } 3946 } 3947 3948 void set_port_slave_flag(portid_t slave_pid) 3949 { 3950 struct rte_port *port; 3951 3952 port = &ports[slave_pid]; 3953 port->slave_flag = 1; 3954 } 3955 3956 void clear_port_slave_flag(portid_t slave_pid) 3957 { 3958 struct rte_port *port; 3959 3960 port = &ports[slave_pid]; 3961 port->slave_flag = 0; 3962 } 3963 3964 uint8_t port_is_bonding_slave(portid_t slave_pid) 3965 { 3966 struct rte_port *port; 3967 struct rte_eth_dev_info dev_info; 3968 int ret; 3969 3970 port = &ports[slave_pid]; 3971 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3972 if (ret != 0) { 3973 TESTPMD_LOG(ERR, 3974 "Failed to get device info for port id %d," 3975 "cannot determine if the port is a bonded slave", 3976 slave_pid); 3977 return 0; 3978 } 3979 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3980 return 1; 3981 return 0; 3982 } 3983 3984 const uint16_t vlan_tags[] = { 3985 0, 1, 2, 3, 4, 5, 6, 7, 3986 8, 9, 10, 11, 12, 13, 14, 15, 3987 16, 17, 18, 19, 20, 21, 22, 23, 3988 24, 25, 26, 27, 28, 29, 30, 31 3989 }; 3990 3991 static int 3992 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3993 enum dcb_mode_enable dcb_mode, 3994 enum rte_eth_nb_tcs num_tcs, 3995 uint8_t pfc_en) 3996 { 3997 uint8_t i; 3998 int32_t rc; 3999 struct rte_eth_rss_conf rss_conf; 4000 4001 /* 4002 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4003 * given above, and the number of traffic classes available for use. 4004 */ 4005 if (dcb_mode == DCB_VT_ENABLED) { 4006 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4007 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4008 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4009 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4010 4011 /* VMDQ+DCB RX and TX configurations */ 4012 vmdq_rx_conf->enable_default_pool = 0; 4013 vmdq_rx_conf->default_pool = 0; 4014 vmdq_rx_conf->nb_queue_pools = 4015 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4016 vmdq_tx_conf->nb_queue_pools = 4017 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4018 4019 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4020 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4021 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4022 vmdq_rx_conf->pool_map[i].pools = 4023 1 << (i % vmdq_rx_conf->nb_queue_pools); 4024 } 4025 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4026 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4027 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4028 } 4029 4030 /* set DCB mode of RX and TX of multiple queues */ 4031 eth_conf->rxmode.mq_mode = 4032 (enum rte_eth_rx_mq_mode) 4033 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4034 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4035 } else { 4036 struct rte_eth_dcb_rx_conf *rx_conf = 4037 ð_conf->rx_adv_conf.dcb_rx_conf; 4038 struct rte_eth_dcb_tx_conf *tx_conf = 4039 ð_conf->tx_adv_conf.dcb_tx_conf; 4040 4041 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4042 4043 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4044 if (rc != 0) 4045 return rc; 4046 4047 rx_conf->nb_tcs = num_tcs; 4048 tx_conf->nb_tcs = num_tcs; 4049 4050 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4051 rx_conf->dcb_tc[i] = i % num_tcs; 4052 tx_conf->dcb_tc[i] = i % num_tcs; 4053 } 4054 4055 eth_conf->rxmode.mq_mode = 4056 (enum rte_eth_rx_mq_mode) 4057 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4058 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4059 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4060 } 4061 4062 if (pfc_en) 4063 eth_conf->dcb_capability_en = 4064 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4065 else 4066 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4067 4068 return 0; 4069 } 4070 4071 int 4072 init_port_dcb_config(portid_t pid, 4073 enum dcb_mode_enable dcb_mode, 4074 enum rte_eth_nb_tcs num_tcs, 4075 uint8_t pfc_en) 4076 { 4077 struct rte_eth_conf port_conf; 4078 struct rte_port *rte_port; 4079 int retval; 4080 uint16_t i; 4081 4082 if (num_procs > 1) { 4083 printf("The multi-process feature doesn't support dcb.\n"); 4084 return -ENOTSUP; 4085 } 4086 rte_port = &ports[pid]; 4087 4088 /* retain the original device configuration. */ 4089 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4090 4091 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4092 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4093 if (retval < 0) 4094 return retval; 4095 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4096 /* remove RSS HASH offload for DCB in vt mode */ 4097 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4098 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4099 for (i = 0; i < nb_rxq; i++) 4100 rte_port->rxq[i].conf.offloads &= 4101 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4102 } 4103 4104 /* re-configure the device . */ 4105 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4106 if (retval < 0) 4107 return retval; 4108 4109 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4110 if (retval != 0) 4111 return retval; 4112 4113 /* If dev_info.vmdq_pool_base is greater than 0, 4114 * the queue id of vmdq pools is started after pf queues. 4115 */ 4116 if (dcb_mode == DCB_VT_ENABLED && 4117 rte_port->dev_info.vmdq_pool_base > 0) { 4118 fprintf(stderr, 4119 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4120 pid); 4121 return -1; 4122 } 4123 4124 /* Assume the ports in testpmd have the same dcb capability 4125 * and has the same number of rxq and txq in dcb mode 4126 */ 4127 if (dcb_mode == DCB_VT_ENABLED) { 4128 if (rte_port->dev_info.max_vfs > 0) { 4129 nb_rxq = rte_port->dev_info.nb_rx_queues; 4130 nb_txq = rte_port->dev_info.nb_tx_queues; 4131 } else { 4132 nb_rxq = rte_port->dev_info.max_rx_queues; 4133 nb_txq = rte_port->dev_info.max_tx_queues; 4134 } 4135 } else { 4136 /*if vt is disabled, use all pf queues */ 4137 if (rte_port->dev_info.vmdq_pool_base == 0) { 4138 nb_rxq = rte_port->dev_info.max_rx_queues; 4139 nb_txq = rte_port->dev_info.max_tx_queues; 4140 } else { 4141 nb_rxq = (queueid_t)num_tcs; 4142 nb_txq = (queueid_t)num_tcs; 4143 4144 } 4145 } 4146 rx_free_thresh = 64; 4147 4148 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4149 4150 rxtx_port_config(pid); 4151 /* VLAN filter */ 4152 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4153 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4154 rx_vft_set(pid, vlan_tags[i], 1); 4155 4156 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4157 if (retval != 0) 4158 return retval; 4159 4160 rte_port->dcb_flag = 1; 4161 4162 /* Enter DCB configuration status */ 4163 dcb_config = 1; 4164 4165 return 0; 4166 } 4167 4168 static void 4169 init_port(void) 4170 { 4171 int i; 4172 4173 /* Configuration of Ethernet ports. */ 4174 ports = rte_zmalloc("testpmd: ports", 4175 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4176 RTE_CACHE_LINE_SIZE); 4177 if (ports == NULL) { 4178 rte_exit(EXIT_FAILURE, 4179 "rte_zmalloc(%d struct rte_port) failed\n", 4180 RTE_MAX_ETHPORTS); 4181 } 4182 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4183 ports[i].xstats_info.allocated = false; 4184 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4185 LIST_INIT(&ports[i].flow_tunnel_list); 4186 /* Initialize ports NUMA structures */ 4187 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4188 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4189 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4190 } 4191 4192 static void 4193 force_quit(void) 4194 { 4195 pmd_test_exit(); 4196 prompt_exit(); 4197 } 4198 4199 static void 4200 print_stats(void) 4201 { 4202 uint8_t i; 4203 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4204 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4205 4206 /* Clear screen and move to top left */ 4207 printf("%s%s", clr, top_left); 4208 4209 printf("\nPort statistics ===================================="); 4210 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4211 nic_stats_display(fwd_ports_ids[i]); 4212 4213 fflush(stdout); 4214 } 4215 4216 static void 4217 signal_handler(int signum) 4218 { 4219 if (signum == SIGINT || signum == SIGTERM) { 4220 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4221 signum); 4222 #ifdef RTE_LIB_PDUMP 4223 /* uninitialize packet capture framework */ 4224 rte_pdump_uninit(); 4225 #endif 4226 #ifdef RTE_LIB_LATENCYSTATS 4227 if (latencystats_enabled != 0) 4228 rte_latencystats_uninit(); 4229 #endif 4230 force_quit(); 4231 /* Set flag to indicate the force termination. */ 4232 f_quit = 1; 4233 /* exit with the expected status */ 4234 #ifndef RTE_EXEC_ENV_WINDOWS 4235 signal(signum, SIG_DFL); 4236 kill(getpid(), signum); 4237 #endif 4238 } 4239 } 4240 4241 int 4242 main(int argc, char** argv) 4243 { 4244 int diag; 4245 portid_t port_id; 4246 uint16_t count; 4247 int ret; 4248 4249 signal(SIGINT, signal_handler); 4250 signal(SIGTERM, signal_handler); 4251 4252 testpmd_logtype = rte_log_register("testpmd"); 4253 if (testpmd_logtype < 0) 4254 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4255 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4256 4257 diag = rte_eal_init(argc, argv); 4258 if (diag < 0) 4259 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4260 rte_strerror(rte_errno)); 4261 4262 ret = register_eth_event_callback(); 4263 if (ret != 0) 4264 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4265 4266 #ifdef RTE_LIB_PDUMP 4267 /* initialize packet capture framework */ 4268 rte_pdump_init(); 4269 #endif 4270 4271 count = 0; 4272 RTE_ETH_FOREACH_DEV(port_id) { 4273 ports_ids[count] = port_id; 4274 count++; 4275 } 4276 nb_ports = (portid_t) count; 4277 if (nb_ports == 0) 4278 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4279 4280 /* allocate port structures, and init them */ 4281 init_port(); 4282 4283 set_def_fwd_config(); 4284 if (nb_lcores == 0) 4285 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4286 "Check the core mask argument\n"); 4287 4288 /* Bitrate/latency stats disabled by default */ 4289 #ifdef RTE_LIB_BITRATESTATS 4290 bitrate_enabled = 0; 4291 #endif 4292 #ifdef RTE_LIB_LATENCYSTATS 4293 latencystats_enabled = 0; 4294 #endif 4295 4296 /* on FreeBSD, mlockall() is disabled by default */ 4297 #ifdef RTE_EXEC_ENV_FREEBSD 4298 do_mlockall = 0; 4299 #else 4300 do_mlockall = 1; 4301 #endif 4302 4303 argc -= diag; 4304 argv += diag; 4305 if (argc > 1) 4306 launch_args_parse(argc, argv); 4307 4308 #ifndef RTE_EXEC_ENV_WINDOWS 4309 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4310 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4311 strerror(errno)); 4312 } 4313 #endif 4314 4315 if (tx_first && interactive) 4316 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4317 "interactive mode.\n"); 4318 4319 if (tx_first && lsc_interrupt) { 4320 fprintf(stderr, 4321 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4322 lsc_interrupt = 0; 4323 } 4324 4325 if (!nb_rxq && !nb_txq) 4326 fprintf(stderr, 4327 "Warning: Either rx or tx queues should be non-zero\n"); 4328 4329 if (nb_rxq > 1 && nb_rxq > nb_txq) 4330 fprintf(stderr, 4331 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4332 nb_rxq, nb_txq); 4333 4334 init_config(); 4335 4336 if (hot_plug) { 4337 ret = rte_dev_hotplug_handle_enable(); 4338 if (ret) { 4339 RTE_LOG(ERR, EAL, 4340 "fail to enable hotplug handling."); 4341 return -1; 4342 } 4343 4344 ret = rte_dev_event_monitor_start(); 4345 if (ret) { 4346 RTE_LOG(ERR, EAL, 4347 "fail to start device event monitoring."); 4348 return -1; 4349 } 4350 4351 ret = rte_dev_event_callback_register(NULL, 4352 dev_event_callback, NULL); 4353 if (ret) { 4354 RTE_LOG(ERR, EAL, 4355 "fail to register device event callback\n"); 4356 return -1; 4357 } 4358 } 4359 4360 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4361 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4362 4363 /* set all ports to promiscuous mode by default */ 4364 RTE_ETH_FOREACH_DEV(port_id) { 4365 ret = rte_eth_promiscuous_enable(port_id); 4366 if (ret != 0) 4367 fprintf(stderr, 4368 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4369 port_id, rte_strerror(-ret)); 4370 } 4371 4372 #ifdef RTE_LIB_METRICS 4373 /* Init metrics library */ 4374 rte_metrics_init(rte_socket_id()); 4375 #endif 4376 4377 #ifdef RTE_LIB_LATENCYSTATS 4378 if (latencystats_enabled != 0) { 4379 int ret = rte_latencystats_init(1, NULL); 4380 if (ret) 4381 fprintf(stderr, 4382 "Warning: latencystats init() returned error %d\n", 4383 ret); 4384 fprintf(stderr, "Latencystats running on lcore %d\n", 4385 latencystats_lcore_id); 4386 } 4387 #endif 4388 4389 /* Setup bitrate stats */ 4390 #ifdef RTE_LIB_BITRATESTATS 4391 if (bitrate_enabled != 0) { 4392 bitrate_data = rte_stats_bitrate_create(); 4393 if (bitrate_data == NULL) 4394 rte_exit(EXIT_FAILURE, 4395 "Could not allocate bitrate data.\n"); 4396 rte_stats_bitrate_reg(bitrate_data); 4397 } 4398 #endif 4399 #ifdef RTE_LIB_CMDLINE 4400 if (init_cmdline() != 0) 4401 rte_exit(EXIT_FAILURE, 4402 "Could not initialise cmdline context.\n"); 4403 4404 if (strlen(cmdline_filename) != 0) 4405 cmdline_read_from_file(cmdline_filename); 4406 4407 if (interactive == 1) { 4408 if (auto_start) { 4409 printf("Start automatic packet forwarding\n"); 4410 start_packet_forwarding(0); 4411 } 4412 prompt(); 4413 pmd_test_exit(); 4414 } else 4415 #endif 4416 { 4417 char c; 4418 int rc; 4419 4420 f_quit = 0; 4421 4422 printf("No commandline core given, start packet forwarding\n"); 4423 start_packet_forwarding(tx_first); 4424 if (stats_period != 0) { 4425 uint64_t prev_time = 0, cur_time, diff_time = 0; 4426 uint64_t timer_period; 4427 4428 /* Convert to number of cycles */ 4429 timer_period = stats_period * rte_get_timer_hz(); 4430 4431 while (f_quit == 0) { 4432 cur_time = rte_get_timer_cycles(); 4433 diff_time += cur_time - prev_time; 4434 4435 if (diff_time >= timer_period) { 4436 print_stats(); 4437 /* Reset the timer */ 4438 diff_time = 0; 4439 } 4440 /* Sleep to avoid unnecessary checks */ 4441 prev_time = cur_time; 4442 rte_delay_us_sleep(US_PER_S); 4443 } 4444 } 4445 4446 printf("Press enter to exit\n"); 4447 rc = read(0, &c, 1); 4448 pmd_test_exit(); 4449 if (rc < 0) 4450 return 1; 4451 } 4452 4453 ret = rte_eal_cleanup(); 4454 if (ret != 0) 4455 rte_exit(EXIT_FAILURE, 4456 "EAL cleanup failed: %s\n", strerror(-ret)); 4457 4458 return EXIT_SUCCESS; 4459 } 4460