1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 73 #include "testpmd.h" 74 75 #ifndef MAP_HUGETLB 76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 77 #define HUGE_FLAG (0x40000) 78 #else 79 #define HUGE_FLAG MAP_HUGETLB 80 #endif 81 82 #ifndef MAP_HUGE_SHIFT 83 /* older kernels (or FreeBSD) will not have this define */ 84 #define HUGE_SHIFT (26) 85 #else 86 #define HUGE_SHIFT MAP_HUGE_SHIFT 87 #endif 88 89 #define EXTMEM_HEAP_NAME "extmem" 90 /* 91 * Zone size with the malloc overhead (max of debug and release variants) 92 * must fit into the smallest supported hugepage size (2M), 93 * so that an IOVA-contiguous zone of this size can always be allocated 94 * if there are free 2M hugepages. 95 */ 96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 97 98 uint16_t verbose_level = 0; /**< Silent by default. */ 99 int testpmd_logtype; /**< Log type for testpmd logs */ 100 101 /* use main core for command line ? */ 102 uint8_t interactive = 0; 103 uint8_t auto_start = 0; 104 uint8_t tx_first; 105 char cmdline_filename[PATH_MAX] = {0}; 106 107 /* 108 * NUMA support configuration. 109 * When set, the NUMA support attempts to dispatch the allocation of the 110 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 111 * probed ports among the CPU sockets 0 and 1. 112 * Otherwise, all memory is allocated from CPU socket 0. 113 */ 114 uint8_t numa_support = 1; /**< numa enabled by default */ 115 116 /* 117 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 118 * not configured. 119 */ 120 uint8_t socket_num = UMA_NO_CONFIG; 121 122 /* 123 * Select mempool allocation type: 124 * - native: use regular DPDK memory 125 * - anon: use regular DPDK memory to create mempool, but populate using 126 * anonymous memory (may not be IOVA-contiguous) 127 * - xmem: use externally allocated hugepage memory 128 */ 129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 130 131 /* 132 * Store specified sockets on which memory pool to be used by ports 133 * is allocated. 134 */ 135 uint8_t port_numa[RTE_MAX_ETHPORTS]; 136 137 /* 138 * Store specified sockets on which RX ring to be used by ports 139 * is allocated. 140 */ 141 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 142 143 /* 144 * Store specified sockets on which TX ring to be used by ports 145 * is allocated. 146 */ 147 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 148 149 /* 150 * Record the Ethernet address of peer target ports to which packets are 151 * forwarded. 152 * Must be instantiated with the ethernet addresses of peer traffic generator 153 * ports. 154 */ 155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 156 portid_t nb_peer_eth_addrs = 0; 157 158 /* 159 * Probed Target Environment. 160 */ 161 struct rte_port *ports; /**< For all probed ethernet ports. */ 162 portid_t nb_ports; /**< Number of probed ethernet ports. */ 163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 164 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 165 166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 167 168 /* 169 * Test Forwarding Configuration. 170 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 171 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 172 */ 173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 175 portid_t nb_cfg_ports; /**< Number of configured ports. */ 176 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 177 178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 180 181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 182 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 183 184 /* 185 * Forwarding engines. 186 */ 187 struct fwd_engine * fwd_engines[] = { 188 &io_fwd_engine, 189 &mac_fwd_engine, 190 &mac_swap_engine, 191 &flow_gen_engine, 192 &rx_only_engine, 193 &tx_only_engine, 194 &csum_fwd_engine, 195 &icmp_echo_engine, 196 &noisy_vnf_engine, 197 &five_tuple_swap_fwd_engine, 198 #ifdef RTE_LIBRTE_IEEE1588 199 &ieee1588_fwd_engine, 200 #endif 201 &shared_rxq_engine, 202 NULL, 203 }; 204 205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 206 uint16_t mempool_flags; 207 208 struct fwd_config cur_fwd_config; 209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 210 uint32_t retry_enabled; 211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 213 214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 216 DEFAULT_MBUF_DATA_SIZE 217 }; /**< Mbuf data space size. */ 218 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 219 * specified on command-line. */ 220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 221 222 /** Extended statistics to show. */ 223 struct rte_eth_xstat_name *xstats_display; 224 225 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 226 227 /* 228 * In container, it cannot terminate the process which running with 'stats-period' 229 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 230 */ 231 uint8_t f_quit; 232 233 /* 234 * Max Rx frame size, set by '--max-pkt-len' parameter. 235 */ 236 uint32_t max_rx_pkt_len; 237 238 /* 239 * Configuration of packet segments used to scatter received packets 240 * if some of split features is configured. 241 */ 242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 243 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 245 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 246 247 /* 248 * Configuration of packet segments used by the "txonly" processing engine. 249 */ 250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 252 TXONLY_DEF_PACKET_LEN, 253 }; 254 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 255 256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 257 /**< Split policy for packets to TX. */ 258 259 uint8_t txonly_multi_flow; 260 /**< Whether multiple flows are generated in TXONLY mode. */ 261 262 uint32_t tx_pkt_times_inter; 263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 264 265 uint32_t tx_pkt_times_intra; 266 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 267 268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 272 273 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 274 uint8_t dcb_config = 0; 275 276 /* 277 * Configurable number of RX/TX queues. 278 */ 279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 282 283 /* 284 * Configurable number of RX/TX ring descriptors. 285 * Defaults are supplied by drivers via ethdev. 286 */ 287 #define RTE_TEST_RX_DESC_DEFAULT 0 288 #define RTE_TEST_TX_DESC_DEFAULT 0 289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 291 292 #define RTE_PMD_PARAM_UNSET -1 293 /* 294 * Configurable values of RX and TX ring threshold registers. 295 */ 296 297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 300 301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 304 305 /* 306 * Configurable value of RX free threshold. 307 */ 308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 309 310 /* 311 * Configurable value of RX drop enable. 312 */ 313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 314 315 /* 316 * Configurable value of TX free threshold. 317 */ 318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 319 320 /* 321 * Configurable value of TX RS bit threshold. 322 */ 323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 324 325 /* 326 * Configurable value of buffered packets before sending. 327 */ 328 uint16_t noisy_tx_sw_bufsz; 329 330 /* 331 * Configurable value of packet buffer timeout. 332 */ 333 uint16_t noisy_tx_sw_buf_flush_time; 334 335 /* 336 * Configurable value for size of VNF internal memory area 337 * used for simulating noisy neighbour behaviour 338 */ 339 uint64_t noisy_lkup_mem_sz; 340 341 /* 342 * Configurable value of number of random writes done in 343 * VNF simulation memory area. 344 */ 345 uint64_t noisy_lkup_num_writes; 346 347 /* 348 * Configurable value of number of random reads done in 349 * VNF simulation memory area. 350 */ 351 uint64_t noisy_lkup_num_reads; 352 353 /* 354 * Configurable value of number of random reads/writes done in 355 * VNF simulation memory area. 356 */ 357 uint64_t noisy_lkup_num_reads_writes; 358 359 /* 360 * Receive Side Scaling (RSS) configuration. 361 */ 362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 363 364 /* 365 * Port topology configuration 366 */ 367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 368 369 /* 370 * Avoids to flush all the RX streams before starts forwarding. 371 */ 372 uint8_t no_flush_rx = 0; /* flush by default */ 373 374 /* 375 * Flow API isolated mode. 376 */ 377 uint8_t flow_isolate_all; 378 379 /* 380 * Avoids to check link status when starting/stopping a port. 381 */ 382 uint8_t no_link_check = 0; /* check by default */ 383 384 /* 385 * Don't automatically start all ports in interactive mode. 386 */ 387 uint8_t no_device_start = 0; 388 389 /* 390 * Enable link status change notification 391 */ 392 uint8_t lsc_interrupt = 1; /* enabled by default */ 393 394 /* 395 * Enable device removal notification. 396 */ 397 uint8_t rmv_interrupt = 1; /* enabled by default */ 398 399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 400 401 /* After attach, port setup is called on event or by iterator */ 402 bool setup_on_probe_event = true; 403 404 /* Clear ptypes on port initialization. */ 405 uint8_t clear_ptypes = true; 406 407 /* Hairpin ports configuration mode. */ 408 uint16_t hairpin_mode; 409 410 /* Pretty printing of ethdev events */ 411 static const char * const eth_event_desc[] = { 412 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 413 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 414 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 415 [RTE_ETH_EVENT_INTR_RESET] = "reset", 416 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 417 [RTE_ETH_EVENT_IPSEC] = "IPsec", 418 [RTE_ETH_EVENT_MACSEC] = "MACsec", 419 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 420 [RTE_ETH_EVENT_NEW] = "device probed", 421 [RTE_ETH_EVENT_DESTROY] = "device released", 422 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 423 [RTE_ETH_EVENT_MAX] = NULL, 424 }; 425 426 /* 427 * Display or mask ether events 428 * Default to all events except VF_MBOX 429 */ 430 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 431 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 432 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 433 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 434 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 435 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 436 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 437 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 438 /* 439 * Decide if all memory are locked for performance. 440 */ 441 int do_mlockall = 0; 442 443 /* 444 * NIC bypass mode configuration options. 445 */ 446 447 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 448 /* The NIC bypass watchdog timeout. */ 449 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 450 #endif 451 452 453 #ifdef RTE_LIB_LATENCYSTATS 454 455 /* 456 * Set when latency stats is enabled in the commandline 457 */ 458 uint8_t latencystats_enabled; 459 460 /* 461 * Lcore ID to service latency statistics. 462 */ 463 lcoreid_t latencystats_lcore_id = -1; 464 465 #endif 466 467 /* 468 * Ethernet device configuration. 469 */ 470 struct rte_eth_rxmode rx_mode; 471 472 struct rte_eth_txmode tx_mode = { 473 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 474 }; 475 476 struct rte_eth_fdir_conf fdir_conf = { 477 .mode = RTE_FDIR_MODE_NONE, 478 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 479 .status = RTE_FDIR_REPORT_STATUS, 480 .mask = { 481 .vlan_tci_mask = 0xFFEF, 482 .ipv4_mask = { 483 .src_ip = 0xFFFFFFFF, 484 .dst_ip = 0xFFFFFFFF, 485 }, 486 .ipv6_mask = { 487 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 488 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 489 }, 490 .src_port_mask = 0xFFFF, 491 .dst_port_mask = 0xFFFF, 492 .mac_addr_byte_mask = 0xFF, 493 .tunnel_type_mask = 1, 494 .tunnel_id_mask = 0xFFFFFFFF, 495 }, 496 .drop_queue = 127, 497 }; 498 499 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 500 501 /* 502 * Display zero values by default for xstats 503 */ 504 uint8_t xstats_hide_zero; 505 506 /* 507 * Measure of CPU cycles disabled by default 508 */ 509 uint8_t record_core_cycles; 510 511 /* 512 * Display of RX and TX bursts disabled by default 513 */ 514 uint8_t record_burst_stats; 515 516 /* 517 * Number of ports per shared Rx queue group, 0 disable. 518 */ 519 uint32_t rxq_share; 520 521 unsigned int num_sockets = 0; 522 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 523 524 #ifdef RTE_LIB_BITRATESTATS 525 /* Bitrate statistics */ 526 struct rte_stats_bitrates *bitrate_data; 527 lcoreid_t bitrate_lcore_id; 528 uint8_t bitrate_enabled; 529 #endif 530 531 #ifdef RTE_LIB_GRO 532 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 533 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 534 #endif 535 536 /* 537 * hexadecimal bitmask of RX mq mode can be enabled. 538 */ 539 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 540 541 /* 542 * Used to set forced link speed 543 */ 544 uint32_t eth_link_speed; 545 546 /* 547 * ID of the current process in multi-process, used to 548 * configure the queues to be polled. 549 */ 550 int proc_id; 551 552 /* 553 * Number of processes in multi-process, used to 554 * configure the queues to be polled. 555 */ 556 unsigned int num_procs = 1; 557 558 static void 559 eth_rx_metadata_negotiate_mp(uint16_t port_id) 560 { 561 uint64_t rx_meta_features = 0; 562 int ret; 563 564 if (!is_proc_primary()) 565 return; 566 567 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 568 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 569 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 570 571 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 572 if (ret == 0) { 573 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 574 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 575 port_id); 576 } 577 578 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 579 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 580 port_id); 581 } 582 583 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 584 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 585 port_id); 586 } 587 } else if (ret != -ENOTSUP) { 588 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 589 port_id, rte_strerror(-ret)); 590 } 591 } 592 593 static int 594 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 595 const struct rte_eth_conf *dev_conf) 596 { 597 if (is_proc_primary()) 598 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 599 dev_conf); 600 return 0; 601 } 602 603 static int 604 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 605 { 606 #ifdef RTE_NET_BOND 607 608 portid_t slave_pids[RTE_MAX_ETHPORTS]; 609 struct rte_port *port; 610 int num_slaves; 611 portid_t slave_pid; 612 int i; 613 614 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 615 RTE_MAX_ETHPORTS); 616 if (num_slaves < 0) { 617 fprintf(stderr, "Failed to get slave list for port = %u\n", 618 bond_pid); 619 return num_slaves; 620 } 621 622 for (i = 0; i < num_slaves; i++) { 623 slave_pid = slave_pids[i]; 624 port = &ports[slave_pid]; 625 port->port_status = 626 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 627 } 628 #else 629 RTE_SET_USED(bond_pid); 630 RTE_SET_USED(is_stop); 631 #endif 632 return 0; 633 } 634 635 static int 636 eth_dev_start_mp(uint16_t port_id) 637 { 638 int ret; 639 640 if (is_proc_primary()) { 641 ret = rte_eth_dev_start(port_id); 642 if (ret != 0) 643 return ret; 644 645 struct rte_port *port = &ports[port_id]; 646 647 /* 648 * Starting a bonded port also starts all slaves under the bonded 649 * device. So if this port is bond device, we need to modify the 650 * port status of these slaves. 651 */ 652 if (port->bond_flag == 1) 653 return change_bonding_slave_port_status(port_id, false); 654 } 655 656 return 0; 657 } 658 659 static int 660 eth_dev_stop_mp(uint16_t port_id) 661 { 662 int ret; 663 664 if (is_proc_primary()) { 665 ret = rte_eth_dev_stop(port_id); 666 if (ret != 0) 667 return ret; 668 669 struct rte_port *port = &ports[port_id]; 670 671 /* 672 * Stopping a bonded port also stops all slaves under the bonded 673 * device. So if this port is bond device, we need to modify the 674 * port status of these slaves. 675 */ 676 if (port->bond_flag == 1) 677 return change_bonding_slave_port_status(port_id, true); 678 } 679 680 return 0; 681 } 682 683 static void 684 mempool_free_mp(struct rte_mempool *mp) 685 { 686 if (is_proc_primary()) 687 rte_mempool_free(mp); 688 } 689 690 static int 691 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 692 { 693 if (is_proc_primary()) 694 return rte_eth_dev_set_mtu(port_id, mtu); 695 696 return 0; 697 } 698 699 /* Forward function declarations */ 700 static void setup_attached_port(portid_t pi); 701 static void check_all_ports_link_status(uint32_t port_mask); 702 static int eth_event_callback(portid_t port_id, 703 enum rte_eth_event_type type, 704 void *param, void *ret_param); 705 static void dev_event_callback(const char *device_name, 706 enum rte_dev_event_type type, 707 void *param); 708 static void fill_xstats_display_info(void); 709 710 /* 711 * Check if all the ports are started. 712 * If yes, return positive value. If not, return zero. 713 */ 714 static int all_ports_started(void); 715 716 #ifdef RTE_LIB_GSO 717 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 718 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 719 #endif 720 721 /* Holds the registered mbuf dynamic flags names. */ 722 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 723 724 725 /* 726 * Helper function to check if socket is already discovered. 727 * If yes, return positive value. If not, return zero. 728 */ 729 int 730 new_socket_id(unsigned int socket_id) 731 { 732 unsigned int i; 733 734 for (i = 0; i < num_sockets; i++) { 735 if (socket_ids[i] == socket_id) 736 return 0; 737 } 738 return 1; 739 } 740 741 /* 742 * Setup default configuration. 743 */ 744 static void 745 set_default_fwd_lcores_config(void) 746 { 747 unsigned int i; 748 unsigned int nb_lc; 749 unsigned int sock_num; 750 751 nb_lc = 0; 752 for (i = 0; i < RTE_MAX_LCORE; i++) { 753 if (!rte_lcore_is_enabled(i)) 754 continue; 755 sock_num = rte_lcore_to_socket_id(i); 756 if (new_socket_id(sock_num)) { 757 if (num_sockets >= RTE_MAX_NUMA_NODES) { 758 rte_exit(EXIT_FAILURE, 759 "Total sockets greater than %u\n", 760 RTE_MAX_NUMA_NODES); 761 } 762 socket_ids[num_sockets++] = sock_num; 763 } 764 if (i == rte_get_main_lcore()) 765 continue; 766 fwd_lcores_cpuids[nb_lc++] = i; 767 } 768 nb_lcores = (lcoreid_t) nb_lc; 769 nb_cfg_lcores = nb_lcores; 770 nb_fwd_lcores = 1; 771 } 772 773 static void 774 set_def_peer_eth_addrs(void) 775 { 776 portid_t i; 777 778 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 779 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 780 peer_eth_addrs[i].addr_bytes[5] = i; 781 } 782 } 783 784 static void 785 set_default_fwd_ports_config(void) 786 { 787 portid_t pt_id; 788 int i = 0; 789 790 RTE_ETH_FOREACH_DEV(pt_id) { 791 fwd_ports_ids[i++] = pt_id; 792 793 /* Update sockets info according to the attached device */ 794 int socket_id = rte_eth_dev_socket_id(pt_id); 795 if (socket_id >= 0 && new_socket_id(socket_id)) { 796 if (num_sockets >= RTE_MAX_NUMA_NODES) { 797 rte_exit(EXIT_FAILURE, 798 "Total sockets greater than %u\n", 799 RTE_MAX_NUMA_NODES); 800 } 801 socket_ids[num_sockets++] = socket_id; 802 } 803 } 804 805 nb_cfg_ports = nb_ports; 806 nb_fwd_ports = nb_ports; 807 } 808 809 void 810 set_def_fwd_config(void) 811 { 812 set_default_fwd_lcores_config(); 813 set_def_peer_eth_addrs(); 814 set_default_fwd_ports_config(); 815 } 816 817 #ifndef RTE_EXEC_ENV_WINDOWS 818 /* extremely pessimistic estimation of memory required to create a mempool */ 819 static int 820 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 821 { 822 unsigned int n_pages, mbuf_per_pg, leftover; 823 uint64_t total_mem, mbuf_mem, obj_sz; 824 825 /* there is no good way to predict how much space the mempool will 826 * occupy because it will allocate chunks on the fly, and some of those 827 * will come from default DPDK memory while some will come from our 828 * external memory, so just assume 128MB will be enough for everyone. 829 */ 830 uint64_t hdr_mem = 128 << 20; 831 832 /* account for possible non-contiguousness */ 833 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 834 if (obj_sz > pgsz) { 835 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 836 return -1; 837 } 838 839 mbuf_per_pg = pgsz / obj_sz; 840 leftover = (nb_mbufs % mbuf_per_pg) > 0; 841 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 842 843 mbuf_mem = n_pages * pgsz; 844 845 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 846 847 if (total_mem > SIZE_MAX) { 848 TESTPMD_LOG(ERR, "Memory size too big\n"); 849 return -1; 850 } 851 *out = (size_t)total_mem; 852 853 return 0; 854 } 855 856 static int 857 pagesz_flags(uint64_t page_sz) 858 { 859 /* as per mmap() manpage, all page sizes are log2 of page size 860 * shifted by MAP_HUGE_SHIFT 861 */ 862 int log2 = rte_log2_u64(page_sz); 863 864 return (log2 << HUGE_SHIFT); 865 } 866 867 static void * 868 alloc_mem(size_t memsz, size_t pgsz, bool huge) 869 { 870 void *addr; 871 int flags; 872 873 /* allocate anonymous hugepages */ 874 flags = MAP_ANONYMOUS | MAP_PRIVATE; 875 if (huge) 876 flags |= HUGE_FLAG | pagesz_flags(pgsz); 877 878 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 879 if (addr == MAP_FAILED) 880 return NULL; 881 882 return addr; 883 } 884 885 struct extmem_param { 886 void *addr; 887 size_t len; 888 size_t pgsz; 889 rte_iova_t *iova_table; 890 unsigned int iova_table_len; 891 }; 892 893 static int 894 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 895 bool huge) 896 { 897 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 898 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 899 unsigned int cur_page, n_pages, pgsz_idx; 900 size_t mem_sz, cur_pgsz; 901 rte_iova_t *iovas = NULL; 902 void *addr; 903 int ret; 904 905 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 906 /* skip anything that is too big */ 907 if (pgsizes[pgsz_idx] > SIZE_MAX) 908 continue; 909 910 cur_pgsz = pgsizes[pgsz_idx]; 911 912 /* if we were told not to allocate hugepages, override */ 913 if (!huge) 914 cur_pgsz = sysconf(_SC_PAGESIZE); 915 916 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 917 if (ret < 0) { 918 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 919 return -1; 920 } 921 922 /* allocate our memory */ 923 addr = alloc_mem(mem_sz, cur_pgsz, huge); 924 925 /* if we couldn't allocate memory with a specified page size, 926 * that doesn't mean we can't do it with other page sizes, so 927 * try another one. 928 */ 929 if (addr == NULL) 930 continue; 931 932 /* store IOVA addresses for every page in this memory area */ 933 n_pages = mem_sz / cur_pgsz; 934 935 iovas = malloc(sizeof(*iovas) * n_pages); 936 937 if (iovas == NULL) { 938 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 939 goto fail; 940 } 941 /* lock memory if it's not huge pages */ 942 if (!huge) 943 mlock(addr, mem_sz); 944 945 /* populate IOVA addresses */ 946 for (cur_page = 0; cur_page < n_pages; cur_page++) { 947 rte_iova_t iova; 948 size_t offset; 949 void *cur; 950 951 offset = cur_pgsz * cur_page; 952 cur = RTE_PTR_ADD(addr, offset); 953 954 /* touch the page before getting its IOVA */ 955 *(volatile char *)cur = 0; 956 957 iova = rte_mem_virt2iova(cur); 958 959 iovas[cur_page] = iova; 960 } 961 962 break; 963 } 964 /* if we couldn't allocate anything */ 965 if (iovas == NULL) 966 return -1; 967 968 param->addr = addr; 969 param->len = mem_sz; 970 param->pgsz = cur_pgsz; 971 param->iova_table = iovas; 972 param->iova_table_len = n_pages; 973 974 return 0; 975 fail: 976 free(iovas); 977 if (addr) 978 munmap(addr, mem_sz); 979 980 return -1; 981 } 982 983 static int 984 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 985 { 986 struct extmem_param param; 987 int socket_id, ret; 988 989 memset(¶m, 0, sizeof(param)); 990 991 /* check if our heap exists */ 992 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 993 if (socket_id < 0) { 994 /* create our heap */ 995 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 996 if (ret < 0) { 997 TESTPMD_LOG(ERR, "Cannot create heap\n"); 998 return -1; 999 } 1000 } 1001 1002 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 1003 if (ret < 0) { 1004 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 1005 return -1; 1006 } 1007 1008 /* we now have a valid memory area, so add it to heap */ 1009 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 1010 param.addr, param.len, param.iova_table, 1011 param.iova_table_len, param.pgsz); 1012 1013 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1014 1015 /* not needed any more */ 1016 free(param.iova_table); 1017 1018 if (ret < 0) { 1019 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1020 munmap(param.addr, param.len); 1021 return -1; 1022 } 1023 1024 /* success */ 1025 1026 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1027 param.len >> 20); 1028 1029 return 0; 1030 } 1031 static void 1032 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1033 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1034 { 1035 uint16_t pid = 0; 1036 int ret; 1037 1038 RTE_ETH_FOREACH_DEV(pid) { 1039 struct rte_eth_dev_info dev_info; 1040 1041 ret = eth_dev_info_get_print_err(pid, &dev_info); 1042 if (ret != 0) { 1043 TESTPMD_LOG(DEBUG, 1044 "unable to get device info for port %d on addr 0x%p," 1045 "mempool unmapping will not be performed\n", 1046 pid, memhdr->addr); 1047 continue; 1048 } 1049 1050 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1051 if (ret) { 1052 TESTPMD_LOG(DEBUG, 1053 "unable to DMA unmap addr 0x%p " 1054 "for device %s\n", 1055 memhdr->addr, dev_info.device->name); 1056 } 1057 } 1058 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1059 if (ret) { 1060 TESTPMD_LOG(DEBUG, 1061 "unable to un-register addr 0x%p\n", memhdr->addr); 1062 } 1063 } 1064 1065 static void 1066 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1067 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1068 { 1069 uint16_t pid = 0; 1070 size_t page_size = sysconf(_SC_PAGESIZE); 1071 int ret; 1072 1073 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1074 page_size); 1075 if (ret) { 1076 TESTPMD_LOG(DEBUG, 1077 "unable to register addr 0x%p\n", memhdr->addr); 1078 return; 1079 } 1080 RTE_ETH_FOREACH_DEV(pid) { 1081 struct rte_eth_dev_info dev_info; 1082 1083 ret = eth_dev_info_get_print_err(pid, &dev_info); 1084 if (ret != 0) { 1085 TESTPMD_LOG(DEBUG, 1086 "unable to get device info for port %d on addr 0x%p," 1087 "mempool mapping will not be performed\n", 1088 pid, memhdr->addr); 1089 continue; 1090 } 1091 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1092 if (ret) { 1093 TESTPMD_LOG(DEBUG, 1094 "unable to DMA map addr 0x%p " 1095 "for device %s\n", 1096 memhdr->addr, dev_info.device->name); 1097 } 1098 } 1099 } 1100 #endif 1101 1102 static unsigned int 1103 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1104 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1105 { 1106 struct rte_pktmbuf_extmem *xmem; 1107 unsigned int ext_num, zone_num, elt_num; 1108 uint16_t elt_size; 1109 1110 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1111 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1112 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1113 1114 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1115 if (xmem == NULL) { 1116 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1117 "external buffer descriptors\n"); 1118 *ext_mem = NULL; 1119 return 0; 1120 } 1121 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1122 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1123 const struct rte_memzone *mz; 1124 char mz_name[RTE_MEMZONE_NAMESIZE]; 1125 int ret; 1126 1127 ret = snprintf(mz_name, sizeof(mz_name), 1128 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1129 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1130 errno = ENAMETOOLONG; 1131 ext_num = 0; 1132 break; 1133 } 1134 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1135 socket_id, 1136 RTE_MEMZONE_IOVA_CONTIG | 1137 RTE_MEMZONE_1GB | 1138 RTE_MEMZONE_SIZE_HINT_ONLY); 1139 if (mz == NULL) { 1140 /* 1141 * The caller exits on external buffer creation 1142 * error, so there is no need to free memzones. 1143 */ 1144 errno = ENOMEM; 1145 ext_num = 0; 1146 break; 1147 } 1148 xseg->buf_ptr = mz->addr; 1149 xseg->buf_iova = mz->iova; 1150 xseg->buf_len = EXTBUF_ZONE_SIZE; 1151 xseg->elt_size = elt_size; 1152 } 1153 if (ext_num == 0 && xmem != NULL) { 1154 free(xmem); 1155 xmem = NULL; 1156 } 1157 *ext_mem = xmem; 1158 return ext_num; 1159 } 1160 1161 /* 1162 * Configuration initialisation done once at init time. 1163 */ 1164 static struct rte_mempool * 1165 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1166 unsigned int socket_id, uint16_t size_idx) 1167 { 1168 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1169 struct rte_mempool *rte_mp = NULL; 1170 #ifndef RTE_EXEC_ENV_WINDOWS 1171 uint32_t mb_size; 1172 1173 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1174 #endif 1175 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1176 if (!is_proc_primary()) { 1177 rte_mp = rte_mempool_lookup(pool_name); 1178 if (rte_mp == NULL) 1179 rte_exit(EXIT_FAILURE, 1180 "Get mbuf pool for socket %u failed: %s\n", 1181 socket_id, rte_strerror(rte_errno)); 1182 return rte_mp; 1183 } 1184 1185 TESTPMD_LOG(INFO, 1186 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1187 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1188 1189 switch (mp_alloc_type) { 1190 case MP_ALLOC_NATIVE: 1191 { 1192 /* wrapper to rte_mempool_create() */ 1193 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1194 rte_mbuf_best_mempool_ops()); 1195 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1196 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1197 break; 1198 } 1199 #ifndef RTE_EXEC_ENV_WINDOWS 1200 case MP_ALLOC_ANON: 1201 { 1202 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1203 mb_size, (unsigned int) mb_mempool_cache, 1204 sizeof(struct rte_pktmbuf_pool_private), 1205 socket_id, mempool_flags); 1206 if (rte_mp == NULL) 1207 goto err; 1208 1209 if (rte_mempool_populate_anon(rte_mp) == 0) { 1210 rte_mempool_free(rte_mp); 1211 rte_mp = NULL; 1212 goto err; 1213 } 1214 rte_pktmbuf_pool_init(rte_mp, NULL); 1215 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1216 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1217 break; 1218 } 1219 case MP_ALLOC_XMEM: 1220 case MP_ALLOC_XMEM_HUGE: 1221 { 1222 int heap_socket; 1223 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1224 1225 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1226 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1227 1228 heap_socket = 1229 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1230 if (heap_socket < 0) 1231 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1232 1233 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1234 rte_mbuf_best_mempool_ops()); 1235 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1236 mb_mempool_cache, 0, mbuf_seg_size, 1237 heap_socket); 1238 break; 1239 } 1240 #endif 1241 case MP_ALLOC_XBUF: 1242 { 1243 struct rte_pktmbuf_extmem *ext_mem; 1244 unsigned int ext_num; 1245 1246 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1247 socket_id, pool_name, &ext_mem); 1248 if (ext_num == 0) 1249 rte_exit(EXIT_FAILURE, 1250 "Can't create pinned data buffers\n"); 1251 1252 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1253 rte_mbuf_best_mempool_ops()); 1254 rte_mp = rte_pktmbuf_pool_create_extbuf 1255 (pool_name, nb_mbuf, mb_mempool_cache, 1256 0, mbuf_seg_size, socket_id, 1257 ext_mem, ext_num); 1258 free(ext_mem); 1259 break; 1260 } 1261 default: 1262 { 1263 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1264 } 1265 } 1266 1267 #ifndef RTE_EXEC_ENV_WINDOWS 1268 err: 1269 #endif 1270 if (rte_mp == NULL) { 1271 rte_exit(EXIT_FAILURE, 1272 "Creation of mbuf pool for socket %u failed: %s\n", 1273 socket_id, rte_strerror(rte_errno)); 1274 } else if (verbose_level > 0) { 1275 rte_mempool_dump(stdout, rte_mp); 1276 } 1277 return rte_mp; 1278 } 1279 1280 /* 1281 * Check given socket id is valid or not with NUMA mode, 1282 * if valid, return 0, else return -1 1283 */ 1284 static int 1285 check_socket_id(const unsigned int socket_id) 1286 { 1287 static int warning_once = 0; 1288 1289 if (new_socket_id(socket_id)) { 1290 if (!warning_once && numa_support) 1291 fprintf(stderr, 1292 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1293 warning_once = 1; 1294 return -1; 1295 } 1296 return 0; 1297 } 1298 1299 /* 1300 * Get the allowed maximum number of RX queues. 1301 * *pid return the port id which has minimal value of 1302 * max_rx_queues in all ports. 1303 */ 1304 queueid_t 1305 get_allowed_max_nb_rxq(portid_t *pid) 1306 { 1307 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1308 bool max_rxq_valid = false; 1309 portid_t pi; 1310 struct rte_eth_dev_info dev_info; 1311 1312 RTE_ETH_FOREACH_DEV(pi) { 1313 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1314 continue; 1315 1316 max_rxq_valid = true; 1317 if (dev_info.max_rx_queues < allowed_max_rxq) { 1318 allowed_max_rxq = dev_info.max_rx_queues; 1319 *pid = pi; 1320 } 1321 } 1322 return max_rxq_valid ? allowed_max_rxq : 0; 1323 } 1324 1325 /* 1326 * Check input rxq is valid or not. 1327 * If input rxq is not greater than any of maximum number 1328 * of RX queues of all ports, it is valid. 1329 * if valid, return 0, else return -1 1330 */ 1331 int 1332 check_nb_rxq(queueid_t rxq) 1333 { 1334 queueid_t allowed_max_rxq; 1335 portid_t pid = 0; 1336 1337 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1338 if (rxq > allowed_max_rxq) { 1339 fprintf(stderr, 1340 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1341 rxq, allowed_max_rxq, pid); 1342 return -1; 1343 } 1344 return 0; 1345 } 1346 1347 /* 1348 * Get the allowed maximum number of TX queues. 1349 * *pid return the port id which has minimal value of 1350 * max_tx_queues in all ports. 1351 */ 1352 queueid_t 1353 get_allowed_max_nb_txq(portid_t *pid) 1354 { 1355 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1356 bool max_txq_valid = false; 1357 portid_t pi; 1358 struct rte_eth_dev_info dev_info; 1359 1360 RTE_ETH_FOREACH_DEV(pi) { 1361 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1362 continue; 1363 1364 max_txq_valid = true; 1365 if (dev_info.max_tx_queues < allowed_max_txq) { 1366 allowed_max_txq = dev_info.max_tx_queues; 1367 *pid = pi; 1368 } 1369 } 1370 return max_txq_valid ? allowed_max_txq : 0; 1371 } 1372 1373 /* 1374 * Check input txq is valid or not. 1375 * If input txq is not greater than any of maximum number 1376 * of TX queues of all ports, it is valid. 1377 * if valid, return 0, else return -1 1378 */ 1379 int 1380 check_nb_txq(queueid_t txq) 1381 { 1382 queueid_t allowed_max_txq; 1383 portid_t pid = 0; 1384 1385 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1386 if (txq > allowed_max_txq) { 1387 fprintf(stderr, 1388 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1389 txq, allowed_max_txq, pid); 1390 return -1; 1391 } 1392 return 0; 1393 } 1394 1395 /* 1396 * Get the allowed maximum number of RXDs of every rx queue. 1397 * *pid return the port id which has minimal value of 1398 * max_rxd in all queues of all ports. 1399 */ 1400 static uint16_t 1401 get_allowed_max_nb_rxd(portid_t *pid) 1402 { 1403 uint16_t allowed_max_rxd = UINT16_MAX; 1404 portid_t pi; 1405 struct rte_eth_dev_info dev_info; 1406 1407 RTE_ETH_FOREACH_DEV(pi) { 1408 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1409 continue; 1410 1411 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1412 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1413 *pid = pi; 1414 } 1415 } 1416 return allowed_max_rxd; 1417 } 1418 1419 /* 1420 * Get the allowed minimal number of RXDs of every rx queue. 1421 * *pid return the port id which has minimal value of 1422 * min_rxd in all queues of all ports. 1423 */ 1424 static uint16_t 1425 get_allowed_min_nb_rxd(portid_t *pid) 1426 { 1427 uint16_t allowed_min_rxd = 0; 1428 portid_t pi; 1429 struct rte_eth_dev_info dev_info; 1430 1431 RTE_ETH_FOREACH_DEV(pi) { 1432 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1433 continue; 1434 1435 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1436 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1437 *pid = pi; 1438 } 1439 } 1440 1441 return allowed_min_rxd; 1442 } 1443 1444 /* 1445 * Check input rxd is valid or not. 1446 * If input rxd is not greater than any of maximum number 1447 * of RXDs of every Rx queues and is not less than any of 1448 * minimal number of RXDs of every Rx queues, it is valid. 1449 * if valid, return 0, else return -1 1450 */ 1451 int 1452 check_nb_rxd(queueid_t rxd) 1453 { 1454 uint16_t allowed_max_rxd; 1455 uint16_t allowed_min_rxd; 1456 portid_t pid = 0; 1457 1458 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1459 if (rxd > allowed_max_rxd) { 1460 fprintf(stderr, 1461 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1462 rxd, allowed_max_rxd, pid); 1463 return -1; 1464 } 1465 1466 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1467 if (rxd < allowed_min_rxd) { 1468 fprintf(stderr, 1469 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1470 rxd, allowed_min_rxd, pid); 1471 return -1; 1472 } 1473 1474 return 0; 1475 } 1476 1477 /* 1478 * Get the allowed maximum number of TXDs of every rx queues. 1479 * *pid return the port id which has minimal value of 1480 * max_txd in every tx queue. 1481 */ 1482 static uint16_t 1483 get_allowed_max_nb_txd(portid_t *pid) 1484 { 1485 uint16_t allowed_max_txd = UINT16_MAX; 1486 portid_t pi; 1487 struct rte_eth_dev_info dev_info; 1488 1489 RTE_ETH_FOREACH_DEV(pi) { 1490 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1491 continue; 1492 1493 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1494 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1495 *pid = pi; 1496 } 1497 } 1498 return allowed_max_txd; 1499 } 1500 1501 /* 1502 * Get the allowed maximum number of TXDs of every tx queues. 1503 * *pid return the port id which has minimal value of 1504 * min_txd in every tx queue. 1505 */ 1506 static uint16_t 1507 get_allowed_min_nb_txd(portid_t *pid) 1508 { 1509 uint16_t allowed_min_txd = 0; 1510 portid_t pi; 1511 struct rte_eth_dev_info dev_info; 1512 1513 RTE_ETH_FOREACH_DEV(pi) { 1514 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1515 continue; 1516 1517 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1518 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1519 *pid = pi; 1520 } 1521 } 1522 1523 return allowed_min_txd; 1524 } 1525 1526 /* 1527 * Check input txd is valid or not. 1528 * If input txd is not greater than any of maximum number 1529 * of TXDs of every Rx queues, it is valid. 1530 * if valid, return 0, else return -1 1531 */ 1532 int 1533 check_nb_txd(queueid_t txd) 1534 { 1535 uint16_t allowed_max_txd; 1536 uint16_t allowed_min_txd; 1537 portid_t pid = 0; 1538 1539 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1540 if (txd > allowed_max_txd) { 1541 fprintf(stderr, 1542 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1543 txd, allowed_max_txd, pid); 1544 return -1; 1545 } 1546 1547 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1548 if (txd < allowed_min_txd) { 1549 fprintf(stderr, 1550 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1551 txd, allowed_min_txd, pid); 1552 return -1; 1553 } 1554 return 0; 1555 } 1556 1557 1558 /* 1559 * Get the allowed maximum number of hairpin queues. 1560 * *pid return the port id which has minimal value of 1561 * max_hairpin_queues in all ports. 1562 */ 1563 queueid_t 1564 get_allowed_max_nb_hairpinq(portid_t *pid) 1565 { 1566 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1567 portid_t pi; 1568 struct rte_eth_hairpin_cap cap; 1569 1570 RTE_ETH_FOREACH_DEV(pi) { 1571 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1572 *pid = pi; 1573 return 0; 1574 } 1575 if (cap.max_nb_queues < allowed_max_hairpinq) { 1576 allowed_max_hairpinq = cap.max_nb_queues; 1577 *pid = pi; 1578 } 1579 } 1580 return allowed_max_hairpinq; 1581 } 1582 1583 /* 1584 * Check input hairpin is valid or not. 1585 * If input hairpin is not greater than any of maximum number 1586 * of hairpin queues of all ports, it is valid. 1587 * if valid, return 0, else return -1 1588 */ 1589 int 1590 check_nb_hairpinq(queueid_t hairpinq) 1591 { 1592 queueid_t allowed_max_hairpinq; 1593 portid_t pid = 0; 1594 1595 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1596 if (hairpinq > allowed_max_hairpinq) { 1597 fprintf(stderr, 1598 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1599 hairpinq, allowed_max_hairpinq, pid); 1600 return -1; 1601 } 1602 return 0; 1603 } 1604 1605 static int 1606 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1607 { 1608 uint32_t eth_overhead; 1609 1610 if (dev_info->max_mtu != UINT16_MAX && 1611 dev_info->max_rx_pktlen > dev_info->max_mtu) 1612 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1613 else 1614 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1615 1616 return eth_overhead; 1617 } 1618 1619 static void 1620 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1621 { 1622 struct rte_port *port = &ports[pid]; 1623 int ret; 1624 int i; 1625 1626 eth_rx_metadata_negotiate_mp(pid); 1627 1628 port->dev_conf.txmode = tx_mode; 1629 port->dev_conf.rxmode = rx_mode; 1630 1631 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1632 if (ret != 0) 1633 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1634 1635 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1636 port->dev_conf.txmode.offloads &= 1637 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1638 1639 /* Apply Rx offloads configuration */ 1640 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1641 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads; 1642 /* Apply Tx offloads configuration */ 1643 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1644 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads; 1645 1646 if (eth_link_speed) 1647 port->dev_conf.link_speeds = eth_link_speed; 1648 1649 if (max_rx_pkt_len) 1650 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1651 get_eth_overhead(&port->dev_info); 1652 1653 /* set flag to initialize port/queue */ 1654 port->need_reconfig = 1; 1655 port->need_reconfig_queues = 1; 1656 port->socket_id = socket_id; 1657 port->tx_metadata = 0; 1658 1659 /* 1660 * Check for maximum number of segments per MTU. 1661 * Accordingly update the mbuf data size. 1662 */ 1663 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1664 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1665 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1666 uint16_t mtu; 1667 1668 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1669 uint16_t data_size = (mtu + eth_overhead) / 1670 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1671 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1672 1673 if (buffer_size > mbuf_data_size[0]) { 1674 mbuf_data_size[0] = buffer_size; 1675 TESTPMD_LOG(WARNING, 1676 "Configured mbuf size of the first segment %hu\n", 1677 mbuf_data_size[0]); 1678 } 1679 } 1680 } 1681 } 1682 1683 static void 1684 init_config(void) 1685 { 1686 portid_t pid; 1687 struct rte_mempool *mbp; 1688 unsigned int nb_mbuf_per_pool; 1689 lcoreid_t lc_id; 1690 #ifdef RTE_LIB_GRO 1691 struct rte_gro_param gro_param; 1692 #endif 1693 #ifdef RTE_LIB_GSO 1694 uint32_t gso_types; 1695 #endif 1696 1697 /* Configuration of logical cores. */ 1698 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1699 sizeof(struct fwd_lcore *) * nb_lcores, 1700 RTE_CACHE_LINE_SIZE); 1701 if (fwd_lcores == NULL) { 1702 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1703 "failed\n", nb_lcores); 1704 } 1705 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1706 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1707 sizeof(struct fwd_lcore), 1708 RTE_CACHE_LINE_SIZE); 1709 if (fwd_lcores[lc_id] == NULL) { 1710 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1711 "failed\n"); 1712 } 1713 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1714 } 1715 1716 RTE_ETH_FOREACH_DEV(pid) { 1717 uint32_t socket_id; 1718 1719 if (numa_support) { 1720 socket_id = port_numa[pid]; 1721 if (port_numa[pid] == NUMA_NO_CONFIG) { 1722 socket_id = rte_eth_dev_socket_id(pid); 1723 1724 /* 1725 * if socket_id is invalid, 1726 * set to the first available socket. 1727 */ 1728 if (check_socket_id(socket_id) < 0) 1729 socket_id = socket_ids[0]; 1730 } 1731 } else { 1732 socket_id = (socket_num == UMA_NO_CONFIG) ? 1733 0 : socket_num; 1734 } 1735 /* Apply default TxRx configuration for all ports */ 1736 init_config_port_offloads(pid, socket_id); 1737 } 1738 /* 1739 * Create pools of mbuf. 1740 * If NUMA support is disabled, create a single pool of mbuf in 1741 * socket 0 memory by default. 1742 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1743 * 1744 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1745 * nb_txd can be configured at run time. 1746 */ 1747 if (param_total_num_mbufs) 1748 nb_mbuf_per_pool = param_total_num_mbufs; 1749 else { 1750 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1751 (nb_lcores * mb_mempool_cache) + 1752 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1753 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1754 } 1755 1756 if (numa_support) { 1757 uint8_t i, j; 1758 1759 for (i = 0; i < num_sockets; i++) 1760 for (j = 0; j < mbuf_data_size_n; j++) 1761 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1762 mbuf_pool_create(mbuf_data_size[j], 1763 nb_mbuf_per_pool, 1764 socket_ids[i], j); 1765 } else { 1766 uint8_t i; 1767 1768 for (i = 0; i < mbuf_data_size_n; i++) 1769 mempools[i] = mbuf_pool_create 1770 (mbuf_data_size[i], 1771 nb_mbuf_per_pool, 1772 socket_num == UMA_NO_CONFIG ? 1773 0 : socket_num, i); 1774 } 1775 1776 init_port_config(); 1777 1778 #ifdef RTE_LIB_GSO 1779 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1780 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1781 #endif 1782 /* 1783 * Records which Mbuf pool to use by each logical core, if needed. 1784 */ 1785 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1786 mbp = mbuf_pool_find( 1787 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1788 1789 if (mbp == NULL) 1790 mbp = mbuf_pool_find(0, 0); 1791 fwd_lcores[lc_id]->mbp = mbp; 1792 #ifdef RTE_LIB_GSO 1793 /* initialize GSO context */ 1794 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1795 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1796 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1797 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1798 RTE_ETHER_CRC_LEN; 1799 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1800 #endif 1801 } 1802 1803 fwd_config_setup(); 1804 1805 #ifdef RTE_LIB_GRO 1806 /* create a gro context for each lcore */ 1807 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1808 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1809 gro_param.max_item_per_flow = MAX_PKT_BURST; 1810 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1811 gro_param.socket_id = rte_lcore_to_socket_id( 1812 fwd_lcores_cpuids[lc_id]); 1813 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1814 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1815 rte_exit(EXIT_FAILURE, 1816 "rte_gro_ctx_create() failed\n"); 1817 } 1818 } 1819 #endif 1820 } 1821 1822 1823 void 1824 reconfig(portid_t new_port_id, unsigned socket_id) 1825 { 1826 /* Reconfiguration of Ethernet ports. */ 1827 init_config_port_offloads(new_port_id, socket_id); 1828 init_port_config(); 1829 } 1830 1831 1832 int 1833 init_fwd_streams(void) 1834 { 1835 portid_t pid; 1836 struct rte_port *port; 1837 streamid_t sm_id, nb_fwd_streams_new; 1838 queueid_t q; 1839 1840 /* set socket id according to numa or not */ 1841 RTE_ETH_FOREACH_DEV(pid) { 1842 port = &ports[pid]; 1843 if (nb_rxq > port->dev_info.max_rx_queues) { 1844 fprintf(stderr, 1845 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1846 nb_rxq, port->dev_info.max_rx_queues); 1847 return -1; 1848 } 1849 if (nb_txq > port->dev_info.max_tx_queues) { 1850 fprintf(stderr, 1851 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1852 nb_txq, port->dev_info.max_tx_queues); 1853 return -1; 1854 } 1855 if (numa_support) { 1856 if (port_numa[pid] != NUMA_NO_CONFIG) 1857 port->socket_id = port_numa[pid]; 1858 else { 1859 port->socket_id = rte_eth_dev_socket_id(pid); 1860 1861 /* 1862 * if socket_id is invalid, 1863 * set to the first available socket. 1864 */ 1865 if (check_socket_id(port->socket_id) < 0) 1866 port->socket_id = socket_ids[0]; 1867 } 1868 } 1869 else { 1870 if (socket_num == UMA_NO_CONFIG) 1871 port->socket_id = 0; 1872 else 1873 port->socket_id = socket_num; 1874 } 1875 } 1876 1877 q = RTE_MAX(nb_rxq, nb_txq); 1878 if (q == 0) { 1879 fprintf(stderr, 1880 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1881 return -1; 1882 } 1883 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1884 if (nb_fwd_streams_new == nb_fwd_streams) 1885 return 0; 1886 /* clear the old */ 1887 if (fwd_streams != NULL) { 1888 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1889 if (fwd_streams[sm_id] == NULL) 1890 continue; 1891 rte_free(fwd_streams[sm_id]); 1892 fwd_streams[sm_id] = NULL; 1893 } 1894 rte_free(fwd_streams); 1895 fwd_streams = NULL; 1896 } 1897 1898 /* init new */ 1899 nb_fwd_streams = nb_fwd_streams_new; 1900 if (nb_fwd_streams) { 1901 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1902 sizeof(struct fwd_stream *) * nb_fwd_streams, 1903 RTE_CACHE_LINE_SIZE); 1904 if (fwd_streams == NULL) 1905 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1906 " (struct fwd_stream *)) failed\n", 1907 nb_fwd_streams); 1908 1909 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1910 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1911 " struct fwd_stream", sizeof(struct fwd_stream), 1912 RTE_CACHE_LINE_SIZE); 1913 if (fwd_streams[sm_id] == NULL) 1914 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1915 "(struct fwd_stream) failed\n"); 1916 } 1917 } 1918 1919 return 0; 1920 } 1921 1922 static void 1923 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1924 { 1925 uint64_t total_burst, sburst; 1926 uint64_t nb_burst; 1927 uint64_t burst_stats[4]; 1928 uint16_t pktnb_stats[4]; 1929 uint16_t nb_pkt; 1930 int burst_percent[4], sburstp; 1931 int i; 1932 1933 /* 1934 * First compute the total number of packet bursts and the 1935 * two highest numbers of bursts of the same number of packets. 1936 */ 1937 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1938 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1939 1940 /* Show stats for 0 burst size always */ 1941 total_burst = pbs->pkt_burst_spread[0]; 1942 burst_stats[0] = pbs->pkt_burst_spread[0]; 1943 pktnb_stats[0] = 0; 1944 1945 /* Find the next 2 burst sizes with highest occurrences. */ 1946 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1947 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1948 1949 if (nb_burst == 0) 1950 continue; 1951 1952 total_burst += nb_burst; 1953 1954 if (nb_burst > burst_stats[1]) { 1955 burst_stats[2] = burst_stats[1]; 1956 pktnb_stats[2] = pktnb_stats[1]; 1957 burst_stats[1] = nb_burst; 1958 pktnb_stats[1] = nb_pkt; 1959 } else if (nb_burst > burst_stats[2]) { 1960 burst_stats[2] = nb_burst; 1961 pktnb_stats[2] = nb_pkt; 1962 } 1963 } 1964 if (total_burst == 0) 1965 return; 1966 1967 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1968 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1969 if (i == 3) { 1970 printf("%d%% of other]\n", 100 - sburstp); 1971 return; 1972 } 1973 1974 sburst += burst_stats[i]; 1975 if (sburst == total_burst) { 1976 printf("%d%% of %d pkts]\n", 1977 100 - sburstp, (int) pktnb_stats[i]); 1978 return; 1979 } 1980 1981 burst_percent[i] = 1982 (double)burst_stats[i] / total_burst * 100; 1983 printf("%d%% of %d pkts + ", 1984 burst_percent[i], (int) pktnb_stats[i]); 1985 sburstp += burst_percent[i]; 1986 } 1987 } 1988 1989 static void 1990 fwd_stream_stats_display(streamid_t stream_id) 1991 { 1992 struct fwd_stream *fs; 1993 static const char *fwd_top_stats_border = "-------"; 1994 1995 fs = fwd_streams[stream_id]; 1996 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1997 (fs->fwd_dropped == 0)) 1998 return; 1999 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 2000 "TX Port=%2d/Queue=%2d %s\n", 2001 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 2002 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 2003 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 2004 " TX-dropped: %-14"PRIu64, 2005 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 2006 2007 /* if checksum mode */ 2008 if (cur_fwd_eng == &csum_fwd_engine) { 2009 printf(" RX- bad IP checksum: %-14"PRIu64 2010 " Rx- bad L4 checksum: %-14"PRIu64 2011 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 2012 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 2013 fs->rx_bad_outer_l4_csum); 2014 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2015 fs->rx_bad_outer_ip_csum); 2016 } else { 2017 printf("\n"); 2018 } 2019 2020 if (record_burst_stats) { 2021 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2022 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2023 } 2024 } 2025 2026 void 2027 fwd_stats_display(void) 2028 { 2029 static const char *fwd_stats_border = "----------------------"; 2030 static const char *acc_stats_border = "+++++++++++++++"; 2031 struct { 2032 struct fwd_stream *rx_stream; 2033 struct fwd_stream *tx_stream; 2034 uint64_t tx_dropped; 2035 uint64_t rx_bad_ip_csum; 2036 uint64_t rx_bad_l4_csum; 2037 uint64_t rx_bad_outer_l4_csum; 2038 uint64_t rx_bad_outer_ip_csum; 2039 } ports_stats[RTE_MAX_ETHPORTS]; 2040 uint64_t total_rx_dropped = 0; 2041 uint64_t total_tx_dropped = 0; 2042 uint64_t total_rx_nombuf = 0; 2043 struct rte_eth_stats stats; 2044 uint64_t fwd_cycles = 0; 2045 uint64_t total_recv = 0; 2046 uint64_t total_xmit = 0; 2047 struct rte_port *port; 2048 streamid_t sm_id; 2049 portid_t pt_id; 2050 int ret; 2051 int i; 2052 2053 memset(ports_stats, 0, sizeof(ports_stats)); 2054 2055 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2056 struct fwd_stream *fs = fwd_streams[sm_id]; 2057 2058 if (cur_fwd_config.nb_fwd_streams > 2059 cur_fwd_config.nb_fwd_ports) { 2060 fwd_stream_stats_display(sm_id); 2061 } else { 2062 ports_stats[fs->tx_port].tx_stream = fs; 2063 ports_stats[fs->rx_port].rx_stream = fs; 2064 } 2065 2066 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2067 2068 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2069 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2070 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2071 fs->rx_bad_outer_l4_csum; 2072 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2073 fs->rx_bad_outer_ip_csum; 2074 2075 if (record_core_cycles) 2076 fwd_cycles += fs->core_cycles; 2077 } 2078 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2079 pt_id = fwd_ports_ids[i]; 2080 port = &ports[pt_id]; 2081 2082 ret = rte_eth_stats_get(pt_id, &stats); 2083 if (ret != 0) { 2084 fprintf(stderr, 2085 "%s: Error: failed to get stats (port %u): %d", 2086 __func__, pt_id, ret); 2087 continue; 2088 } 2089 stats.ipackets -= port->stats.ipackets; 2090 stats.opackets -= port->stats.opackets; 2091 stats.ibytes -= port->stats.ibytes; 2092 stats.obytes -= port->stats.obytes; 2093 stats.imissed -= port->stats.imissed; 2094 stats.oerrors -= port->stats.oerrors; 2095 stats.rx_nombuf -= port->stats.rx_nombuf; 2096 2097 total_recv += stats.ipackets; 2098 total_xmit += stats.opackets; 2099 total_rx_dropped += stats.imissed; 2100 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2101 total_tx_dropped += stats.oerrors; 2102 total_rx_nombuf += stats.rx_nombuf; 2103 2104 printf("\n %s Forward statistics for port %-2d %s\n", 2105 fwd_stats_border, pt_id, fwd_stats_border); 2106 2107 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2108 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2109 stats.ipackets + stats.imissed); 2110 2111 if (cur_fwd_eng == &csum_fwd_engine) { 2112 printf(" Bad-ipcsum: %-14"PRIu64 2113 " Bad-l4csum: %-14"PRIu64 2114 "Bad-outer-l4csum: %-14"PRIu64"\n", 2115 ports_stats[pt_id].rx_bad_ip_csum, 2116 ports_stats[pt_id].rx_bad_l4_csum, 2117 ports_stats[pt_id].rx_bad_outer_l4_csum); 2118 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2119 ports_stats[pt_id].rx_bad_outer_ip_csum); 2120 } 2121 if (stats.ierrors + stats.rx_nombuf > 0) { 2122 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2123 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2124 } 2125 2126 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2127 "TX-total: %-"PRIu64"\n", 2128 stats.opackets, ports_stats[pt_id].tx_dropped, 2129 stats.opackets + ports_stats[pt_id].tx_dropped); 2130 2131 if (record_burst_stats) { 2132 if (ports_stats[pt_id].rx_stream) 2133 pkt_burst_stats_display("RX", 2134 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2135 if (ports_stats[pt_id].tx_stream) 2136 pkt_burst_stats_display("TX", 2137 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2138 } 2139 2140 printf(" %s--------------------------------%s\n", 2141 fwd_stats_border, fwd_stats_border); 2142 } 2143 2144 printf("\n %s Accumulated forward statistics for all ports" 2145 "%s\n", 2146 acc_stats_border, acc_stats_border); 2147 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2148 "%-"PRIu64"\n" 2149 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2150 "%-"PRIu64"\n", 2151 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2152 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2153 if (total_rx_nombuf > 0) 2154 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2155 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2156 "%s\n", 2157 acc_stats_border, acc_stats_border); 2158 if (record_core_cycles) { 2159 #define CYC_PER_MHZ 1E6 2160 if (total_recv > 0 || total_xmit > 0) { 2161 uint64_t total_pkts = 0; 2162 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2163 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2164 total_pkts = total_xmit; 2165 else 2166 total_pkts = total_recv; 2167 2168 printf("\n CPU cycles/packet=%.2F (total cycles=" 2169 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2170 " MHz Clock\n", 2171 (double) fwd_cycles / total_pkts, 2172 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2173 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2174 } 2175 } 2176 } 2177 2178 void 2179 fwd_stats_reset(void) 2180 { 2181 streamid_t sm_id; 2182 portid_t pt_id; 2183 int ret; 2184 int i; 2185 2186 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2187 pt_id = fwd_ports_ids[i]; 2188 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2189 if (ret != 0) 2190 fprintf(stderr, 2191 "%s: Error: failed to clear stats (port %u):%d", 2192 __func__, pt_id, ret); 2193 } 2194 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2195 struct fwd_stream *fs = fwd_streams[sm_id]; 2196 2197 fs->rx_packets = 0; 2198 fs->tx_packets = 0; 2199 fs->fwd_dropped = 0; 2200 fs->rx_bad_ip_csum = 0; 2201 fs->rx_bad_l4_csum = 0; 2202 fs->rx_bad_outer_l4_csum = 0; 2203 fs->rx_bad_outer_ip_csum = 0; 2204 2205 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2206 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2207 fs->core_cycles = 0; 2208 } 2209 } 2210 2211 static void 2212 flush_fwd_rx_queues(void) 2213 { 2214 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2215 portid_t rxp; 2216 portid_t port_id; 2217 queueid_t rxq; 2218 uint16_t nb_rx; 2219 uint16_t i; 2220 uint8_t j; 2221 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2222 uint64_t timer_period; 2223 2224 if (num_procs > 1) { 2225 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2226 return; 2227 } 2228 2229 /* convert to number of cycles */ 2230 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2231 2232 for (j = 0; j < 2; j++) { 2233 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2234 for (rxq = 0; rxq < nb_rxq; rxq++) { 2235 port_id = fwd_ports_ids[rxp]; 2236 /** 2237 * testpmd can stuck in the below do while loop 2238 * if rte_eth_rx_burst() always returns nonzero 2239 * packets. So timer is added to exit this loop 2240 * after 1sec timer expiry. 2241 */ 2242 prev_tsc = rte_rdtsc(); 2243 do { 2244 nb_rx = rte_eth_rx_burst(port_id, rxq, 2245 pkts_burst, MAX_PKT_BURST); 2246 for (i = 0; i < nb_rx; i++) 2247 rte_pktmbuf_free(pkts_burst[i]); 2248 2249 cur_tsc = rte_rdtsc(); 2250 diff_tsc = cur_tsc - prev_tsc; 2251 timer_tsc += diff_tsc; 2252 } while ((nb_rx > 0) && 2253 (timer_tsc < timer_period)); 2254 timer_tsc = 0; 2255 } 2256 } 2257 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2258 } 2259 } 2260 2261 static void 2262 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2263 { 2264 struct fwd_stream **fsm; 2265 streamid_t nb_fs; 2266 streamid_t sm_id; 2267 #ifdef RTE_LIB_BITRATESTATS 2268 uint64_t tics_per_1sec; 2269 uint64_t tics_datum; 2270 uint64_t tics_current; 2271 uint16_t i, cnt_ports; 2272 2273 cnt_ports = nb_ports; 2274 tics_datum = rte_rdtsc(); 2275 tics_per_1sec = rte_get_timer_hz(); 2276 #endif 2277 fsm = &fwd_streams[fc->stream_idx]; 2278 nb_fs = fc->stream_nb; 2279 do { 2280 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2281 (*pkt_fwd)(fsm[sm_id]); 2282 #ifdef RTE_LIB_BITRATESTATS 2283 if (bitrate_enabled != 0 && 2284 bitrate_lcore_id == rte_lcore_id()) { 2285 tics_current = rte_rdtsc(); 2286 if (tics_current - tics_datum >= tics_per_1sec) { 2287 /* Periodic bitrate calculation */ 2288 for (i = 0; i < cnt_ports; i++) 2289 rte_stats_bitrate_calc(bitrate_data, 2290 ports_ids[i]); 2291 tics_datum = tics_current; 2292 } 2293 } 2294 #endif 2295 #ifdef RTE_LIB_LATENCYSTATS 2296 if (latencystats_enabled != 0 && 2297 latencystats_lcore_id == rte_lcore_id()) 2298 rte_latencystats_update(); 2299 #endif 2300 2301 } while (! fc->stopped); 2302 } 2303 2304 static int 2305 start_pkt_forward_on_core(void *fwd_arg) 2306 { 2307 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2308 cur_fwd_config.fwd_eng->packet_fwd); 2309 return 0; 2310 } 2311 2312 /* 2313 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2314 * Used to start communication flows in network loopback test configurations. 2315 */ 2316 static int 2317 run_one_txonly_burst_on_core(void *fwd_arg) 2318 { 2319 struct fwd_lcore *fwd_lc; 2320 struct fwd_lcore tmp_lcore; 2321 2322 fwd_lc = (struct fwd_lcore *) fwd_arg; 2323 tmp_lcore = *fwd_lc; 2324 tmp_lcore.stopped = 1; 2325 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2326 return 0; 2327 } 2328 2329 /* 2330 * Launch packet forwarding: 2331 * - Setup per-port forwarding context. 2332 * - launch logical cores with their forwarding configuration. 2333 */ 2334 static void 2335 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2336 { 2337 unsigned int i; 2338 unsigned int lc_id; 2339 int diag; 2340 2341 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2342 lc_id = fwd_lcores_cpuids[i]; 2343 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2344 fwd_lcores[i]->stopped = 0; 2345 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2346 fwd_lcores[i], lc_id); 2347 if (diag != 0) 2348 fprintf(stderr, 2349 "launch lcore %u failed - diag=%d\n", 2350 lc_id, diag); 2351 } 2352 } 2353 } 2354 2355 /* 2356 * Launch packet forwarding configuration. 2357 */ 2358 void 2359 start_packet_forwarding(int with_tx_first) 2360 { 2361 port_fwd_begin_t port_fwd_begin; 2362 port_fwd_end_t port_fwd_end; 2363 unsigned int i; 2364 2365 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2366 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2367 2368 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2369 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2370 2371 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2372 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2373 (!nb_rxq || !nb_txq)) 2374 rte_exit(EXIT_FAILURE, 2375 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2376 cur_fwd_eng->fwd_mode_name); 2377 2378 if (all_ports_started() == 0) { 2379 fprintf(stderr, "Not all ports were started\n"); 2380 return; 2381 } 2382 if (test_done == 0) { 2383 fprintf(stderr, "Packet forwarding already started\n"); 2384 return; 2385 } 2386 2387 fwd_config_setup(); 2388 2389 pkt_fwd_config_display(&cur_fwd_config); 2390 if (!pkt_fwd_shared_rxq_check()) 2391 return; 2392 2393 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2394 if (port_fwd_begin != NULL) { 2395 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2396 if (port_fwd_begin(fwd_ports_ids[i])) { 2397 fprintf(stderr, 2398 "Packet forwarding is not ready\n"); 2399 return; 2400 } 2401 } 2402 } 2403 2404 if (with_tx_first) { 2405 port_fwd_begin = tx_only_engine.port_fwd_begin; 2406 if (port_fwd_begin != NULL) { 2407 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2408 if (port_fwd_begin(fwd_ports_ids[i])) { 2409 fprintf(stderr, 2410 "Packet forwarding is not ready\n"); 2411 return; 2412 } 2413 } 2414 } 2415 } 2416 2417 test_done = 0; 2418 2419 if(!no_flush_rx) 2420 flush_fwd_rx_queues(); 2421 2422 rxtx_config_display(); 2423 2424 fwd_stats_reset(); 2425 if (with_tx_first) { 2426 while (with_tx_first--) { 2427 launch_packet_forwarding( 2428 run_one_txonly_burst_on_core); 2429 rte_eal_mp_wait_lcore(); 2430 } 2431 port_fwd_end = tx_only_engine.port_fwd_end; 2432 if (port_fwd_end != NULL) { 2433 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2434 (*port_fwd_end)(fwd_ports_ids[i]); 2435 } 2436 } 2437 launch_packet_forwarding(start_pkt_forward_on_core); 2438 } 2439 2440 void 2441 stop_packet_forwarding(void) 2442 { 2443 port_fwd_end_t port_fwd_end; 2444 lcoreid_t lc_id; 2445 portid_t pt_id; 2446 int i; 2447 2448 if (test_done) { 2449 fprintf(stderr, "Packet forwarding not started\n"); 2450 return; 2451 } 2452 printf("Telling cores to stop..."); 2453 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2454 fwd_lcores[lc_id]->stopped = 1; 2455 printf("\nWaiting for lcores to finish...\n"); 2456 rte_eal_mp_wait_lcore(); 2457 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2458 if (port_fwd_end != NULL) { 2459 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2460 pt_id = fwd_ports_ids[i]; 2461 (*port_fwd_end)(pt_id); 2462 } 2463 } 2464 2465 fwd_stats_display(); 2466 2467 printf("\nDone.\n"); 2468 test_done = 1; 2469 } 2470 2471 void 2472 dev_set_link_up(portid_t pid) 2473 { 2474 if (rte_eth_dev_set_link_up(pid) < 0) 2475 fprintf(stderr, "\nSet link up fail.\n"); 2476 } 2477 2478 void 2479 dev_set_link_down(portid_t pid) 2480 { 2481 if (rte_eth_dev_set_link_down(pid) < 0) 2482 fprintf(stderr, "\nSet link down fail.\n"); 2483 } 2484 2485 static int 2486 all_ports_started(void) 2487 { 2488 portid_t pi; 2489 struct rte_port *port; 2490 2491 RTE_ETH_FOREACH_DEV(pi) { 2492 port = &ports[pi]; 2493 /* Check if there is a port which is not started */ 2494 if ((port->port_status != RTE_PORT_STARTED) && 2495 (port->slave_flag == 0)) 2496 return 0; 2497 } 2498 2499 /* No port is not started */ 2500 return 1; 2501 } 2502 2503 int 2504 port_is_stopped(portid_t port_id) 2505 { 2506 struct rte_port *port = &ports[port_id]; 2507 2508 if ((port->port_status != RTE_PORT_STOPPED) && 2509 (port->slave_flag == 0)) 2510 return 0; 2511 return 1; 2512 } 2513 2514 int 2515 all_ports_stopped(void) 2516 { 2517 portid_t pi; 2518 2519 RTE_ETH_FOREACH_DEV(pi) { 2520 if (!port_is_stopped(pi)) 2521 return 0; 2522 } 2523 2524 return 1; 2525 } 2526 2527 int 2528 port_is_started(portid_t port_id) 2529 { 2530 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2531 return 0; 2532 2533 if (ports[port_id].port_status != RTE_PORT_STARTED) 2534 return 0; 2535 2536 return 1; 2537 } 2538 2539 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2540 static int 2541 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2542 { 2543 queueid_t qi; 2544 struct rte_eth_hairpin_conf hairpin_conf = { 2545 .peer_count = 1, 2546 }; 2547 int i; 2548 int diag; 2549 struct rte_port *port = &ports[pi]; 2550 uint16_t peer_rx_port = pi; 2551 uint16_t peer_tx_port = pi; 2552 uint32_t manual = 1; 2553 uint32_t tx_exp = hairpin_mode & 0x10; 2554 2555 if (!(hairpin_mode & 0xf)) { 2556 peer_rx_port = pi; 2557 peer_tx_port = pi; 2558 manual = 0; 2559 } else if (hairpin_mode & 0x1) { 2560 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2561 RTE_ETH_DEV_NO_OWNER); 2562 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2563 peer_tx_port = rte_eth_find_next_owned_by(0, 2564 RTE_ETH_DEV_NO_OWNER); 2565 if (p_pi != RTE_MAX_ETHPORTS) { 2566 peer_rx_port = p_pi; 2567 } else { 2568 uint16_t next_pi; 2569 2570 /* Last port will be the peer RX port of the first. */ 2571 RTE_ETH_FOREACH_DEV(next_pi) 2572 peer_rx_port = next_pi; 2573 } 2574 manual = 1; 2575 } else if (hairpin_mode & 0x2) { 2576 if (cnt_pi & 0x1) { 2577 peer_rx_port = p_pi; 2578 } else { 2579 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2580 RTE_ETH_DEV_NO_OWNER); 2581 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2582 peer_rx_port = pi; 2583 } 2584 peer_tx_port = peer_rx_port; 2585 manual = 1; 2586 } 2587 2588 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2589 hairpin_conf.peers[0].port = peer_rx_port; 2590 hairpin_conf.peers[0].queue = i + nb_rxq; 2591 hairpin_conf.manual_bind = !!manual; 2592 hairpin_conf.tx_explicit = !!tx_exp; 2593 diag = rte_eth_tx_hairpin_queue_setup 2594 (pi, qi, nb_txd, &hairpin_conf); 2595 i++; 2596 if (diag == 0) 2597 continue; 2598 2599 /* Fail to setup rx queue, return */ 2600 if (port->port_status == RTE_PORT_HANDLING) 2601 port->port_status = RTE_PORT_STOPPED; 2602 else 2603 fprintf(stderr, 2604 "Port %d can not be set back to stopped\n", pi); 2605 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2606 pi); 2607 /* try to reconfigure queues next time */ 2608 port->need_reconfig_queues = 1; 2609 return -1; 2610 } 2611 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2612 hairpin_conf.peers[0].port = peer_tx_port; 2613 hairpin_conf.peers[0].queue = i + nb_txq; 2614 hairpin_conf.manual_bind = !!manual; 2615 hairpin_conf.tx_explicit = !!tx_exp; 2616 diag = rte_eth_rx_hairpin_queue_setup 2617 (pi, qi, nb_rxd, &hairpin_conf); 2618 i++; 2619 if (diag == 0) 2620 continue; 2621 2622 /* Fail to setup rx queue, return */ 2623 if (port->port_status == RTE_PORT_HANDLING) 2624 port->port_status = RTE_PORT_STOPPED; 2625 else 2626 fprintf(stderr, 2627 "Port %d can not be set back to stopped\n", pi); 2628 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2629 pi); 2630 /* try to reconfigure queues next time */ 2631 port->need_reconfig_queues = 1; 2632 return -1; 2633 } 2634 return 0; 2635 } 2636 2637 /* Configure the Rx with optional split. */ 2638 int 2639 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2640 uint16_t nb_rx_desc, unsigned int socket_id, 2641 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2642 { 2643 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2644 unsigned int i, mp_n; 2645 int ret; 2646 2647 if (rx_pkt_nb_segs <= 1 || 2648 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2649 rx_conf->rx_seg = NULL; 2650 rx_conf->rx_nseg = 0; 2651 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2652 nb_rx_desc, socket_id, 2653 rx_conf, mp); 2654 return ret; 2655 } 2656 for (i = 0; i < rx_pkt_nb_segs; i++) { 2657 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2658 struct rte_mempool *mpx; 2659 /* 2660 * Use last valid pool for the segments with number 2661 * exceeding the pool index. 2662 */ 2663 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2664 mpx = mbuf_pool_find(socket_id, mp_n); 2665 /* Handle zero as mbuf data buffer size. */ 2666 rx_seg->length = rx_pkt_seg_lengths[i] ? 2667 rx_pkt_seg_lengths[i] : 2668 mbuf_data_size[mp_n]; 2669 rx_seg->offset = i < rx_pkt_nb_offs ? 2670 rx_pkt_seg_offsets[i] : 0; 2671 rx_seg->mp = mpx ? mpx : mp; 2672 } 2673 rx_conf->rx_nseg = rx_pkt_nb_segs; 2674 rx_conf->rx_seg = rx_useg; 2675 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2676 socket_id, rx_conf, NULL); 2677 rx_conf->rx_seg = NULL; 2678 rx_conf->rx_nseg = 0; 2679 return ret; 2680 } 2681 2682 static int 2683 alloc_xstats_display_info(portid_t pi) 2684 { 2685 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2686 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2687 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2688 2689 if (xstats_display_num == 0) 2690 return 0; 2691 2692 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2693 if (*ids_supp == NULL) 2694 goto fail_ids_supp; 2695 2696 *prev_values = calloc(xstats_display_num, 2697 sizeof(**prev_values)); 2698 if (*prev_values == NULL) 2699 goto fail_prev_values; 2700 2701 *curr_values = calloc(xstats_display_num, 2702 sizeof(**curr_values)); 2703 if (*curr_values == NULL) 2704 goto fail_curr_values; 2705 2706 ports[pi].xstats_info.allocated = true; 2707 2708 return 0; 2709 2710 fail_curr_values: 2711 free(*prev_values); 2712 fail_prev_values: 2713 free(*ids_supp); 2714 fail_ids_supp: 2715 return -ENOMEM; 2716 } 2717 2718 static void 2719 free_xstats_display_info(portid_t pi) 2720 { 2721 if (!ports[pi].xstats_info.allocated) 2722 return; 2723 free(ports[pi].xstats_info.ids_supp); 2724 free(ports[pi].xstats_info.prev_values); 2725 free(ports[pi].xstats_info.curr_values); 2726 ports[pi].xstats_info.allocated = false; 2727 } 2728 2729 /** Fill helper structures for specified port to show extended statistics. */ 2730 static void 2731 fill_xstats_display_info_for_port(portid_t pi) 2732 { 2733 unsigned int stat, stat_supp; 2734 const char *xstat_name; 2735 struct rte_port *port; 2736 uint64_t *ids_supp; 2737 int rc; 2738 2739 if (xstats_display_num == 0) 2740 return; 2741 2742 if (pi == (portid_t)RTE_PORT_ALL) { 2743 fill_xstats_display_info(); 2744 return; 2745 } 2746 2747 port = &ports[pi]; 2748 if (port->port_status != RTE_PORT_STARTED) 2749 return; 2750 2751 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2752 rte_exit(EXIT_FAILURE, 2753 "Failed to allocate xstats display memory\n"); 2754 2755 ids_supp = port->xstats_info.ids_supp; 2756 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2757 xstat_name = xstats_display[stat].name; 2758 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2759 ids_supp + stat_supp); 2760 if (rc != 0) { 2761 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2762 xstat_name, pi, stat); 2763 continue; 2764 } 2765 stat_supp++; 2766 } 2767 2768 port->xstats_info.ids_supp_sz = stat_supp; 2769 } 2770 2771 /** Fill helper structures for all ports to show extended statistics. */ 2772 static void 2773 fill_xstats_display_info(void) 2774 { 2775 portid_t pi; 2776 2777 if (xstats_display_num == 0) 2778 return; 2779 2780 RTE_ETH_FOREACH_DEV(pi) 2781 fill_xstats_display_info_for_port(pi); 2782 } 2783 2784 int 2785 start_port(portid_t pid) 2786 { 2787 int diag, need_check_link_status = -1; 2788 portid_t pi; 2789 portid_t p_pi = RTE_MAX_ETHPORTS; 2790 portid_t pl[RTE_MAX_ETHPORTS]; 2791 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2792 uint16_t cnt_pi = 0; 2793 uint16_t cfg_pi = 0; 2794 int peer_pi; 2795 queueid_t qi; 2796 struct rte_port *port; 2797 struct rte_eth_hairpin_cap cap; 2798 2799 if (port_id_is_invalid(pid, ENABLED_WARN)) 2800 return 0; 2801 2802 RTE_ETH_FOREACH_DEV(pi) { 2803 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2804 continue; 2805 2806 if (port_is_bonding_slave(pi)) { 2807 fprintf(stderr, 2808 "Please remove port %d from bonded device.\n", 2809 pi); 2810 continue; 2811 } 2812 2813 need_check_link_status = 0; 2814 port = &ports[pi]; 2815 if (port->port_status == RTE_PORT_STOPPED) 2816 port->port_status = RTE_PORT_HANDLING; 2817 else { 2818 fprintf(stderr, "Port %d is now not stopped\n", pi); 2819 continue; 2820 } 2821 2822 if (port->need_reconfig > 0) { 2823 struct rte_eth_conf dev_conf; 2824 int k; 2825 2826 port->need_reconfig = 0; 2827 2828 if (flow_isolate_all) { 2829 int ret = port_flow_isolate(pi, 1); 2830 if (ret) { 2831 fprintf(stderr, 2832 "Failed to apply isolated mode on port %d\n", 2833 pi); 2834 return -1; 2835 } 2836 } 2837 configure_rxtx_dump_callbacks(0); 2838 printf("Configuring Port %d (socket %u)\n", pi, 2839 port->socket_id); 2840 if (nb_hairpinq > 0 && 2841 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2842 fprintf(stderr, 2843 "Port %d doesn't support hairpin queues\n", 2844 pi); 2845 return -1; 2846 } 2847 2848 /* configure port */ 2849 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2850 nb_txq + nb_hairpinq, 2851 &(port->dev_conf)); 2852 if (diag != 0) { 2853 if (port->port_status == RTE_PORT_HANDLING) 2854 port->port_status = RTE_PORT_STOPPED; 2855 else 2856 fprintf(stderr, 2857 "Port %d can not be set back to stopped\n", 2858 pi); 2859 fprintf(stderr, "Fail to configure port %d\n", 2860 pi); 2861 /* try to reconfigure port next time */ 2862 port->need_reconfig = 1; 2863 return -1; 2864 } 2865 /* get device configuration*/ 2866 if (0 != 2867 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2868 fprintf(stderr, 2869 "port %d can not get device configuration\n", 2870 pi); 2871 return -1; 2872 } 2873 /* Apply Rx offloads configuration */ 2874 if (dev_conf.rxmode.offloads != 2875 port->dev_conf.rxmode.offloads) { 2876 port->dev_conf.rxmode.offloads |= 2877 dev_conf.rxmode.offloads; 2878 for (k = 0; 2879 k < port->dev_info.max_rx_queues; 2880 k++) 2881 port->rx_conf[k].offloads |= 2882 dev_conf.rxmode.offloads; 2883 } 2884 /* Apply Tx offloads configuration */ 2885 if (dev_conf.txmode.offloads != 2886 port->dev_conf.txmode.offloads) { 2887 port->dev_conf.txmode.offloads |= 2888 dev_conf.txmode.offloads; 2889 for (k = 0; 2890 k < port->dev_info.max_tx_queues; 2891 k++) 2892 port->tx_conf[k].offloads |= 2893 dev_conf.txmode.offloads; 2894 } 2895 } 2896 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2897 port->need_reconfig_queues = 0; 2898 /* setup tx queues */ 2899 for (qi = 0; qi < nb_txq; qi++) { 2900 if ((numa_support) && 2901 (txring_numa[pi] != NUMA_NO_CONFIG)) 2902 diag = rte_eth_tx_queue_setup(pi, qi, 2903 port->nb_tx_desc[qi], 2904 txring_numa[pi], 2905 &(port->tx_conf[qi])); 2906 else 2907 diag = rte_eth_tx_queue_setup(pi, qi, 2908 port->nb_tx_desc[qi], 2909 port->socket_id, 2910 &(port->tx_conf[qi])); 2911 2912 if (diag == 0) 2913 continue; 2914 2915 /* Fail to setup tx queue, return */ 2916 if (port->port_status == RTE_PORT_HANDLING) 2917 port->port_status = RTE_PORT_STOPPED; 2918 else 2919 fprintf(stderr, 2920 "Port %d can not be set back to stopped\n", 2921 pi); 2922 fprintf(stderr, 2923 "Fail to configure port %d tx queues\n", 2924 pi); 2925 /* try to reconfigure queues next time */ 2926 port->need_reconfig_queues = 1; 2927 return -1; 2928 } 2929 for (qi = 0; qi < nb_rxq; qi++) { 2930 /* setup rx queues */ 2931 if ((numa_support) && 2932 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2933 struct rte_mempool * mp = 2934 mbuf_pool_find 2935 (rxring_numa[pi], 0); 2936 if (mp == NULL) { 2937 fprintf(stderr, 2938 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2939 rxring_numa[pi]); 2940 return -1; 2941 } 2942 2943 diag = rx_queue_setup(pi, qi, 2944 port->nb_rx_desc[qi], 2945 rxring_numa[pi], 2946 &(port->rx_conf[qi]), 2947 mp); 2948 } else { 2949 struct rte_mempool *mp = 2950 mbuf_pool_find 2951 (port->socket_id, 0); 2952 if (mp == NULL) { 2953 fprintf(stderr, 2954 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2955 port->socket_id); 2956 return -1; 2957 } 2958 diag = rx_queue_setup(pi, qi, 2959 port->nb_rx_desc[qi], 2960 port->socket_id, 2961 &(port->rx_conf[qi]), 2962 mp); 2963 } 2964 if (diag == 0) 2965 continue; 2966 2967 /* Fail to setup rx queue, return */ 2968 if (port->port_status == RTE_PORT_HANDLING) 2969 port->port_status = RTE_PORT_STOPPED; 2970 else 2971 fprintf(stderr, 2972 "Port %d can not be set back to stopped\n", 2973 pi); 2974 fprintf(stderr, 2975 "Fail to configure port %d rx queues\n", 2976 pi); 2977 /* try to reconfigure queues next time */ 2978 port->need_reconfig_queues = 1; 2979 return -1; 2980 } 2981 /* setup hairpin queues */ 2982 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2983 return -1; 2984 } 2985 configure_rxtx_dump_callbacks(verbose_level); 2986 if (clear_ptypes) { 2987 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2988 NULL, 0); 2989 if (diag < 0) 2990 fprintf(stderr, 2991 "Port %d: Failed to disable Ptype parsing\n", 2992 pi); 2993 } 2994 2995 p_pi = pi; 2996 cnt_pi++; 2997 2998 /* start port */ 2999 diag = eth_dev_start_mp(pi); 3000 if (diag < 0) { 3001 fprintf(stderr, "Fail to start port %d: %s\n", 3002 pi, rte_strerror(-diag)); 3003 3004 /* Fail to setup rx queue, return */ 3005 if (port->port_status == RTE_PORT_HANDLING) 3006 port->port_status = RTE_PORT_STOPPED; 3007 else 3008 fprintf(stderr, 3009 "Port %d can not be set back to stopped\n", 3010 pi); 3011 continue; 3012 } 3013 3014 if (port->port_status == RTE_PORT_HANDLING) 3015 port->port_status = RTE_PORT_STARTED; 3016 else 3017 fprintf(stderr, "Port %d can not be set into started\n", 3018 pi); 3019 3020 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3021 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3022 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3023 3024 /* at least one port started, need checking link status */ 3025 need_check_link_status = 1; 3026 3027 pl[cfg_pi++] = pi; 3028 } 3029 3030 if (need_check_link_status == 1 && !no_link_check) 3031 check_all_ports_link_status(RTE_PORT_ALL); 3032 else if (need_check_link_status == 0) 3033 fprintf(stderr, "Please stop the ports first\n"); 3034 3035 if (hairpin_mode & 0xf) { 3036 uint16_t i; 3037 int j; 3038 3039 /* bind all started hairpin ports */ 3040 for (i = 0; i < cfg_pi; i++) { 3041 pi = pl[i]; 3042 /* bind current Tx to all peer Rx */ 3043 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3044 RTE_MAX_ETHPORTS, 1); 3045 if (peer_pi < 0) 3046 return peer_pi; 3047 for (j = 0; j < peer_pi; j++) { 3048 if (!port_is_started(peer_pl[j])) 3049 continue; 3050 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3051 if (diag < 0) { 3052 fprintf(stderr, 3053 "Error during binding hairpin Tx port %u to %u: %s\n", 3054 pi, peer_pl[j], 3055 rte_strerror(-diag)); 3056 return -1; 3057 } 3058 } 3059 /* bind all peer Tx to current Rx */ 3060 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3061 RTE_MAX_ETHPORTS, 0); 3062 if (peer_pi < 0) 3063 return peer_pi; 3064 for (j = 0; j < peer_pi; j++) { 3065 if (!port_is_started(peer_pl[j])) 3066 continue; 3067 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3068 if (diag < 0) { 3069 fprintf(stderr, 3070 "Error during binding hairpin Tx port %u to %u: %s\n", 3071 peer_pl[j], pi, 3072 rte_strerror(-diag)); 3073 return -1; 3074 } 3075 } 3076 } 3077 } 3078 3079 fill_xstats_display_info_for_port(pid); 3080 3081 printf("Done\n"); 3082 return 0; 3083 } 3084 3085 void 3086 stop_port(portid_t pid) 3087 { 3088 portid_t pi; 3089 struct rte_port *port; 3090 int need_check_link_status = 0; 3091 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3092 int peer_pi; 3093 3094 if (port_id_is_invalid(pid, ENABLED_WARN)) 3095 return; 3096 3097 printf("Stopping ports...\n"); 3098 3099 RTE_ETH_FOREACH_DEV(pi) { 3100 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3101 continue; 3102 3103 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3104 fprintf(stderr, 3105 "Please remove port %d from forwarding configuration.\n", 3106 pi); 3107 continue; 3108 } 3109 3110 if (port_is_bonding_slave(pi)) { 3111 fprintf(stderr, 3112 "Please remove port %d from bonded device.\n", 3113 pi); 3114 continue; 3115 } 3116 3117 port = &ports[pi]; 3118 if (port->port_status == RTE_PORT_STARTED) 3119 port->port_status = RTE_PORT_HANDLING; 3120 else 3121 continue; 3122 3123 if (hairpin_mode & 0xf) { 3124 int j; 3125 3126 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3127 /* unbind all peer Tx from current Rx */ 3128 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3129 RTE_MAX_ETHPORTS, 0); 3130 if (peer_pi < 0) 3131 continue; 3132 for (j = 0; j < peer_pi; j++) { 3133 if (!port_is_started(peer_pl[j])) 3134 continue; 3135 rte_eth_hairpin_unbind(peer_pl[j], pi); 3136 } 3137 } 3138 3139 if (port->flow_list) 3140 port_flow_flush(pi); 3141 3142 if (eth_dev_stop_mp(pi) != 0) 3143 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3144 pi); 3145 3146 if (port->port_status == RTE_PORT_HANDLING) 3147 port->port_status = RTE_PORT_STOPPED; 3148 else 3149 fprintf(stderr, "Port %d can not be set into stopped\n", 3150 pi); 3151 need_check_link_status = 1; 3152 } 3153 if (need_check_link_status && !no_link_check) 3154 check_all_ports_link_status(RTE_PORT_ALL); 3155 3156 printf("Done\n"); 3157 } 3158 3159 static void 3160 remove_invalid_ports_in(portid_t *array, portid_t *total) 3161 { 3162 portid_t i; 3163 portid_t new_total = 0; 3164 3165 for (i = 0; i < *total; i++) 3166 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3167 array[new_total] = array[i]; 3168 new_total++; 3169 } 3170 *total = new_total; 3171 } 3172 3173 static void 3174 remove_invalid_ports(void) 3175 { 3176 remove_invalid_ports_in(ports_ids, &nb_ports); 3177 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3178 nb_cfg_ports = nb_fwd_ports; 3179 } 3180 3181 void 3182 close_port(portid_t pid) 3183 { 3184 portid_t pi; 3185 struct rte_port *port; 3186 3187 if (port_id_is_invalid(pid, ENABLED_WARN)) 3188 return; 3189 3190 printf("Closing ports...\n"); 3191 3192 RTE_ETH_FOREACH_DEV(pi) { 3193 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3194 continue; 3195 3196 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3197 fprintf(stderr, 3198 "Please remove port %d from forwarding configuration.\n", 3199 pi); 3200 continue; 3201 } 3202 3203 if (port_is_bonding_slave(pi)) { 3204 fprintf(stderr, 3205 "Please remove port %d from bonded device.\n", 3206 pi); 3207 continue; 3208 } 3209 3210 port = &ports[pi]; 3211 if (port->port_status == RTE_PORT_CLOSED) { 3212 fprintf(stderr, "Port %d is already closed\n", pi); 3213 continue; 3214 } 3215 3216 if (is_proc_primary()) { 3217 port_flow_flush(pi); 3218 port_flex_item_flush(pi); 3219 port_action_handle_flush(pi); 3220 rte_eth_dev_close(pi); 3221 } 3222 3223 free_xstats_display_info(pi); 3224 } 3225 3226 remove_invalid_ports(); 3227 printf("Done\n"); 3228 } 3229 3230 void 3231 reset_port(portid_t pid) 3232 { 3233 int diag; 3234 portid_t pi; 3235 struct rte_port *port; 3236 3237 if (port_id_is_invalid(pid, ENABLED_WARN)) 3238 return; 3239 3240 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3241 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3242 fprintf(stderr, 3243 "Can not reset port(s), please stop port(s) first.\n"); 3244 return; 3245 } 3246 3247 printf("Resetting ports...\n"); 3248 3249 RTE_ETH_FOREACH_DEV(pi) { 3250 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3251 continue; 3252 3253 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3254 fprintf(stderr, 3255 "Please remove port %d from forwarding configuration.\n", 3256 pi); 3257 continue; 3258 } 3259 3260 if (port_is_bonding_slave(pi)) { 3261 fprintf(stderr, 3262 "Please remove port %d from bonded device.\n", 3263 pi); 3264 continue; 3265 } 3266 3267 diag = rte_eth_dev_reset(pi); 3268 if (diag == 0) { 3269 port = &ports[pi]; 3270 port->need_reconfig = 1; 3271 port->need_reconfig_queues = 1; 3272 } else { 3273 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3274 pi, diag); 3275 } 3276 } 3277 3278 printf("Done\n"); 3279 } 3280 3281 void 3282 attach_port(char *identifier) 3283 { 3284 portid_t pi; 3285 struct rte_dev_iterator iterator; 3286 3287 printf("Attaching a new port...\n"); 3288 3289 if (identifier == NULL) { 3290 fprintf(stderr, "Invalid parameters are specified\n"); 3291 return; 3292 } 3293 3294 if (rte_dev_probe(identifier) < 0) { 3295 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3296 return; 3297 } 3298 3299 /* first attach mode: event */ 3300 if (setup_on_probe_event) { 3301 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3302 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3303 if (ports[pi].port_status == RTE_PORT_HANDLING && 3304 ports[pi].need_setup != 0) 3305 setup_attached_port(pi); 3306 return; 3307 } 3308 3309 /* second attach mode: iterator */ 3310 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3311 /* setup ports matching the devargs used for probing */ 3312 if (port_is_forwarding(pi)) 3313 continue; /* port was already attached before */ 3314 setup_attached_port(pi); 3315 } 3316 } 3317 3318 static void 3319 setup_attached_port(portid_t pi) 3320 { 3321 unsigned int socket_id; 3322 int ret; 3323 3324 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3325 /* if socket_id is invalid, set to the first available socket. */ 3326 if (check_socket_id(socket_id) < 0) 3327 socket_id = socket_ids[0]; 3328 reconfig(pi, socket_id); 3329 ret = rte_eth_promiscuous_enable(pi); 3330 if (ret != 0) 3331 fprintf(stderr, 3332 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3333 pi, rte_strerror(-ret)); 3334 3335 ports_ids[nb_ports++] = pi; 3336 fwd_ports_ids[nb_fwd_ports++] = pi; 3337 nb_cfg_ports = nb_fwd_ports; 3338 ports[pi].need_setup = 0; 3339 ports[pi].port_status = RTE_PORT_STOPPED; 3340 3341 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3342 printf("Done\n"); 3343 } 3344 3345 static void 3346 detach_device(struct rte_device *dev) 3347 { 3348 portid_t sibling; 3349 3350 if (dev == NULL) { 3351 fprintf(stderr, "Device already removed\n"); 3352 return; 3353 } 3354 3355 printf("Removing a device...\n"); 3356 3357 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3358 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3359 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3360 fprintf(stderr, "Port %u not stopped\n", 3361 sibling); 3362 return; 3363 } 3364 port_flow_flush(sibling); 3365 } 3366 } 3367 3368 if (rte_dev_remove(dev) < 0) { 3369 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3370 return; 3371 } 3372 remove_invalid_ports(); 3373 3374 printf("Device is detached\n"); 3375 printf("Now total ports is %d\n", nb_ports); 3376 printf("Done\n"); 3377 return; 3378 } 3379 3380 void 3381 detach_port_device(portid_t port_id) 3382 { 3383 int ret; 3384 struct rte_eth_dev_info dev_info; 3385 3386 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3387 return; 3388 3389 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3390 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3391 fprintf(stderr, "Port not stopped\n"); 3392 return; 3393 } 3394 fprintf(stderr, "Port was not closed\n"); 3395 } 3396 3397 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3398 if (ret != 0) { 3399 TESTPMD_LOG(ERR, 3400 "Failed to get device info for port %d, not detaching\n", 3401 port_id); 3402 return; 3403 } 3404 detach_device(dev_info.device); 3405 } 3406 3407 void 3408 detach_devargs(char *identifier) 3409 { 3410 struct rte_dev_iterator iterator; 3411 struct rte_devargs da; 3412 portid_t port_id; 3413 3414 printf("Removing a device...\n"); 3415 3416 memset(&da, 0, sizeof(da)); 3417 if (rte_devargs_parsef(&da, "%s", identifier)) { 3418 fprintf(stderr, "cannot parse identifier\n"); 3419 return; 3420 } 3421 3422 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3423 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3424 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3425 fprintf(stderr, "Port %u not stopped\n", 3426 port_id); 3427 rte_eth_iterator_cleanup(&iterator); 3428 rte_devargs_reset(&da); 3429 return; 3430 } 3431 port_flow_flush(port_id); 3432 } 3433 } 3434 3435 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3436 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3437 da.name, da.bus->name); 3438 rte_devargs_reset(&da); 3439 return; 3440 } 3441 3442 remove_invalid_ports(); 3443 3444 printf("Device %s is detached\n", identifier); 3445 printf("Now total ports is %d\n", nb_ports); 3446 printf("Done\n"); 3447 rte_devargs_reset(&da); 3448 } 3449 3450 void 3451 pmd_test_exit(void) 3452 { 3453 portid_t pt_id; 3454 unsigned int i; 3455 int ret; 3456 3457 if (test_done == 0) 3458 stop_packet_forwarding(); 3459 3460 #ifndef RTE_EXEC_ENV_WINDOWS 3461 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3462 if (mempools[i]) { 3463 if (mp_alloc_type == MP_ALLOC_ANON) 3464 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3465 NULL); 3466 } 3467 } 3468 #endif 3469 if (ports != NULL) { 3470 no_link_check = 1; 3471 RTE_ETH_FOREACH_DEV(pt_id) { 3472 printf("\nStopping port %d...\n", pt_id); 3473 fflush(stdout); 3474 stop_port(pt_id); 3475 } 3476 RTE_ETH_FOREACH_DEV(pt_id) { 3477 printf("\nShutting down port %d...\n", pt_id); 3478 fflush(stdout); 3479 close_port(pt_id); 3480 } 3481 } 3482 3483 if (hot_plug) { 3484 ret = rte_dev_event_monitor_stop(); 3485 if (ret) { 3486 RTE_LOG(ERR, EAL, 3487 "fail to stop device event monitor."); 3488 return; 3489 } 3490 3491 ret = rte_dev_event_callback_unregister(NULL, 3492 dev_event_callback, NULL); 3493 if (ret < 0) { 3494 RTE_LOG(ERR, EAL, 3495 "fail to unregister device event callback.\n"); 3496 return; 3497 } 3498 3499 ret = rte_dev_hotplug_handle_disable(); 3500 if (ret) { 3501 RTE_LOG(ERR, EAL, 3502 "fail to disable hotplug handling.\n"); 3503 return; 3504 } 3505 } 3506 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3507 if (mempools[i]) 3508 mempool_free_mp(mempools[i]); 3509 } 3510 free(xstats_display); 3511 3512 printf("\nBye...\n"); 3513 } 3514 3515 typedef void (*cmd_func_t)(void); 3516 struct pmd_test_command { 3517 const char *cmd_name; 3518 cmd_func_t cmd_func; 3519 }; 3520 3521 /* Check the link status of all ports in up to 9s, and print them finally */ 3522 static void 3523 check_all_ports_link_status(uint32_t port_mask) 3524 { 3525 #define CHECK_INTERVAL 100 /* 100ms */ 3526 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3527 portid_t portid; 3528 uint8_t count, all_ports_up, print_flag = 0; 3529 struct rte_eth_link link; 3530 int ret; 3531 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3532 3533 printf("Checking link statuses...\n"); 3534 fflush(stdout); 3535 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3536 all_ports_up = 1; 3537 RTE_ETH_FOREACH_DEV(portid) { 3538 if ((port_mask & (1 << portid)) == 0) 3539 continue; 3540 memset(&link, 0, sizeof(link)); 3541 ret = rte_eth_link_get_nowait(portid, &link); 3542 if (ret < 0) { 3543 all_ports_up = 0; 3544 if (print_flag == 1) 3545 fprintf(stderr, 3546 "Port %u link get failed: %s\n", 3547 portid, rte_strerror(-ret)); 3548 continue; 3549 } 3550 /* print link status if flag set */ 3551 if (print_flag == 1) { 3552 rte_eth_link_to_str(link_status, 3553 sizeof(link_status), &link); 3554 printf("Port %d %s\n", portid, link_status); 3555 continue; 3556 } 3557 /* clear all_ports_up flag if any link down */ 3558 if (link.link_status == RTE_ETH_LINK_DOWN) { 3559 all_ports_up = 0; 3560 break; 3561 } 3562 } 3563 /* after finally printing all link status, get out */ 3564 if (print_flag == 1) 3565 break; 3566 3567 if (all_ports_up == 0) { 3568 fflush(stdout); 3569 rte_delay_ms(CHECK_INTERVAL); 3570 } 3571 3572 /* set the print_flag if all ports up or timeout */ 3573 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3574 print_flag = 1; 3575 } 3576 3577 if (lsc_interrupt) 3578 break; 3579 } 3580 } 3581 3582 static void 3583 rmv_port_callback(void *arg) 3584 { 3585 int need_to_start = 0; 3586 int org_no_link_check = no_link_check; 3587 portid_t port_id = (intptr_t)arg; 3588 struct rte_eth_dev_info dev_info; 3589 int ret; 3590 3591 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3592 3593 if (!test_done && port_is_forwarding(port_id)) { 3594 need_to_start = 1; 3595 stop_packet_forwarding(); 3596 } 3597 no_link_check = 1; 3598 stop_port(port_id); 3599 no_link_check = org_no_link_check; 3600 3601 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3602 if (ret != 0) 3603 TESTPMD_LOG(ERR, 3604 "Failed to get device info for port %d, not detaching\n", 3605 port_id); 3606 else { 3607 struct rte_device *device = dev_info.device; 3608 close_port(port_id); 3609 detach_device(device); /* might be already removed or have more ports */ 3610 } 3611 if (need_to_start) 3612 start_packet_forwarding(0); 3613 } 3614 3615 /* This function is used by the interrupt thread */ 3616 static int 3617 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3618 void *ret_param) 3619 { 3620 RTE_SET_USED(param); 3621 RTE_SET_USED(ret_param); 3622 3623 if (type >= RTE_ETH_EVENT_MAX) { 3624 fprintf(stderr, 3625 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3626 port_id, __func__, type); 3627 fflush(stderr); 3628 } else if (event_print_mask & (UINT32_C(1) << type)) { 3629 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3630 eth_event_desc[type]); 3631 fflush(stdout); 3632 } 3633 3634 switch (type) { 3635 case RTE_ETH_EVENT_NEW: 3636 ports[port_id].need_setup = 1; 3637 ports[port_id].port_status = RTE_PORT_HANDLING; 3638 break; 3639 case RTE_ETH_EVENT_INTR_RMV: 3640 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3641 break; 3642 if (rte_eal_alarm_set(100000, 3643 rmv_port_callback, (void *)(intptr_t)port_id)) 3644 fprintf(stderr, 3645 "Could not set up deferred device removal\n"); 3646 break; 3647 case RTE_ETH_EVENT_DESTROY: 3648 ports[port_id].port_status = RTE_PORT_CLOSED; 3649 printf("Port %u is closed\n", port_id); 3650 break; 3651 default: 3652 break; 3653 } 3654 return 0; 3655 } 3656 3657 static int 3658 register_eth_event_callback(void) 3659 { 3660 int ret; 3661 enum rte_eth_event_type event; 3662 3663 for (event = RTE_ETH_EVENT_UNKNOWN; 3664 event < RTE_ETH_EVENT_MAX; event++) { 3665 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3666 event, 3667 eth_event_callback, 3668 NULL); 3669 if (ret != 0) { 3670 TESTPMD_LOG(ERR, "Failed to register callback for " 3671 "%s event\n", eth_event_desc[event]); 3672 return -1; 3673 } 3674 } 3675 3676 return 0; 3677 } 3678 3679 /* This function is used by the interrupt thread */ 3680 static void 3681 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3682 __rte_unused void *arg) 3683 { 3684 uint16_t port_id; 3685 int ret; 3686 3687 if (type >= RTE_DEV_EVENT_MAX) { 3688 fprintf(stderr, "%s called upon invalid event %d\n", 3689 __func__, type); 3690 fflush(stderr); 3691 } 3692 3693 switch (type) { 3694 case RTE_DEV_EVENT_REMOVE: 3695 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3696 device_name); 3697 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3698 if (ret) { 3699 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3700 device_name); 3701 return; 3702 } 3703 /* 3704 * Because the user's callback is invoked in eal interrupt 3705 * callback, the interrupt callback need to be finished before 3706 * it can be unregistered when detaching device. So finish 3707 * callback soon and use a deferred removal to detach device 3708 * is need. It is a workaround, once the device detaching be 3709 * moved into the eal in the future, the deferred removal could 3710 * be deleted. 3711 */ 3712 if (rte_eal_alarm_set(100000, 3713 rmv_port_callback, (void *)(intptr_t)port_id)) 3714 RTE_LOG(ERR, EAL, 3715 "Could not set up deferred device removal\n"); 3716 break; 3717 case RTE_DEV_EVENT_ADD: 3718 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3719 device_name); 3720 /* TODO: After finish kernel driver binding, 3721 * begin to attach port. 3722 */ 3723 break; 3724 default: 3725 break; 3726 } 3727 } 3728 3729 static void 3730 rxtx_port_config(portid_t pid) 3731 { 3732 uint16_t qid; 3733 uint64_t offloads; 3734 struct rte_port *port = &ports[pid]; 3735 3736 for (qid = 0; qid < nb_rxq; qid++) { 3737 offloads = port->rx_conf[qid].offloads; 3738 port->rx_conf[qid] = port->dev_info.default_rxconf; 3739 3740 if (rxq_share > 0 && 3741 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3742 /* Non-zero share group to enable RxQ share. */ 3743 port->rx_conf[qid].share_group = pid / rxq_share + 1; 3744 port->rx_conf[qid].share_qid = qid; /* Equal mapping. */ 3745 } 3746 3747 if (offloads != 0) 3748 port->rx_conf[qid].offloads = offloads; 3749 3750 /* Check if any Rx parameters have been passed */ 3751 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3752 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3753 3754 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3755 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3756 3757 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3758 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3759 3760 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3761 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3762 3763 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3764 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3765 3766 port->nb_rx_desc[qid] = nb_rxd; 3767 } 3768 3769 for (qid = 0; qid < nb_txq; qid++) { 3770 offloads = port->tx_conf[qid].offloads; 3771 port->tx_conf[qid] = port->dev_info.default_txconf; 3772 if (offloads != 0) 3773 port->tx_conf[qid].offloads = offloads; 3774 3775 /* Check if any Tx parameters have been passed */ 3776 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3777 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3778 3779 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3780 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3781 3782 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3783 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3784 3785 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3786 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3787 3788 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3789 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3790 3791 port->nb_tx_desc[qid] = nb_txd; 3792 } 3793 } 3794 3795 /* 3796 * Helper function to set MTU from frame size 3797 * 3798 * port->dev_info should be set before calling this function. 3799 * 3800 * return 0 on success, negative on error 3801 */ 3802 int 3803 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3804 { 3805 struct rte_port *port = &ports[portid]; 3806 uint32_t eth_overhead; 3807 uint16_t mtu, new_mtu; 3808 3809 eth_overhead = get_eth_overhead(&port->dev_info); 3810 3811 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3812 printf("Failed to get MTU for port %u\n", portid); 3813 return -1; 3814 } 3815 3816 new_mtu = max_rx_pktlen - eth_overhead; 3817 3818 if (mtu == new_mtu) 3819 return 0; 3820 3821 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3822 fprintf(stderr, 3823 "Failed to set MTU to %u for port %u\n", 3824 new_mtu, portid); 3825 return -1; 3826 } 3827 3828 port->dev_conf.rxmode.mtu = new_mtu; 3829 3830 return 0; 3831 } 3832 3833 void 3834 init_port_config(void) 3835 { 3836 portid_t pid; 3837 struct rte_port *port; 3838 int ret, i; 3839 3840 RTE_ETH_FOREACH_DEV(pid) { 3841 port = &ports[pid]; 3842 port->dev_conf.fdir_conf = fdir_conf; 3843 3844 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3845 if (ret != 0) 3846 return; 3847 3848 if (nb_rxq > 1) { 3849 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3850 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3851 rss_hf & port->dev_info.flow_type_rss_offloads; 3852 } else { 3853 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3854 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3855 } 3856 3857 if (port->dcb_flag == 0) { 3858 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3859 port->dev_conf.rxmode.mq_mode = 3860 (enum rte_eth_rx_mq_mode) 3861 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3862 } else { 3863 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3864 port->dev_conf.rxmode.offloads &= 3865 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3866 3867 for (i = 0; 3868 i < port->dev_info.nb_rx_queues; 3869 i++) 3870 port->rx_conf[i].offloads &= 3871 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3872 } 3873 } 3874 3875 rxtx_port_config(pid); 3876 3877 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3878 if (ret != 0) 3879 return; 3880 3881 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3882 rte_pmd_ixgbe_bypass_init(pid); 3883 #endif 3884 3885 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3886 port->dev_conf.intr_conf.lsc = 1; 3887 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3888 port->dev_conf.intr_conf.rmv = 1; 3889 } 3890 } 3891 3892 void set_port_slave_flag(portid_t slave_pid) 3893 { 3894 struct rte_port *port; 3895 3896 port = &ports[slave_pid]; 3897 port->slave_flag = 1; 3898 } 3899 3900 void clear_port_slave_flag(portid_t slave_pid) 3901 { 3902 struct rte_port *port; 3903 3904 port = &ports[slave_pid]; 3905 port->slave_flag = 0; 3906 } 3907 3908 uint8_t port_is_bonding_slave(portid_t slave_pid) 3909 { 3910 struct rte_port *port; 3911 struct rte_eth_dev_info dev_info; 3912 int ret; 3913 3914 port = &ports[slave_pid]; 3915 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3916 if (ret != 0) { 3917 TESTPMD_LOG(ERR, 3918 "Failed to get device info for port id %d," 3919 "cannot determine if the port is a bonded slave", 3920 slave_pid); 3921 return 0; 3922 } 3923 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3924 return 1; 3925 return 0; 3926 } 3927 3928 const uint16_t vlan_tags[] = { 3929 0, 1, 2, 3, 4, 5, 6, 7, 3930 8, 9, 10, 11, 12, 13, 14, 15, 3931 16, 17, 18, 19, 20, 21, 22, 23, 3932 24, 25, 26, 27, 28, 29, 30, 31 3933 }; 3934 3935 static int 3936 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3937 enum dcb_mode_enable dcb_mode, 3938 enum rte_eth_nb_tcs num_tcs, 3939 uint8_t pfc_en) 3940 { 3941 uint8_t i; 3942 int32_t rc; 3943 struct rte_eth_rss_conf rss_conf; 3944 3945 /* 3946 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3947 * given above, and the number of traffic classes available for use. 3948 */ 3949 if (dcb_mode == DCB_VT_ENABLED) { 3950 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3951 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3952 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3953 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3954 3955 /* VMDQ+DCB RX and TX configurations */ 3956 vmdq_rx_conf->enable_default_pool = 0; 3957 vmdq_rx_conf->default_pool = 0; 3958 vmdq_rx_conf->nb_queue_pools = 3959 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3960 vmdq_tx_conf->nb_queue_pools = 3961 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3962 3963 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3964 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3965 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3966 vmdq_rx_conf->pool_map[i].pools = 3967 1 << (i % vmdq_rx_conf->nb_queue_pools); 3968 } 3969 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3970 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3971 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3972 } 3973 3974 /* set DCB mode of RX and TX of multiple queues */ 3975 eth_conf->rxmode.mq_mode = 3976 (enum rte_eth_rx_mq_mode) 3977 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 3978 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 3979 } else { 3980 struct rte_eth_dcb_rx_conf *rx_conf = 3981 ð_conf->rx_adv_conf.dcb_rx_conf; 3982 struct rte_eth_dcb_tx_conf *tx_conf = 3983 ð_conf->tx_adv_conf.dcb_tx_conf; 3984 3985 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3986 3987 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3988 if (rc != 0) 3989 return rc; 3990 3991 rx_conf->nb_tcs = num_tcs; 3992 tx_conf->nb_tcs = num_tcs; 3993 3994 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3995 rx_conf->dcb_tc[i] = i % num_tcs; 3996 tx_conf->dcb_tc[i] = i % num_tcs; 3997 } 3998 3999 eth_conf->rxmode.mq_mode = 4000 (enum rte_eth_rx_mq_mode) 4001 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4002 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4003 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4004 } 4005 4006 if (pfc_en) 4007 eth_conf->dcb_capability_en = 4008 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4009 else 4010 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4011 4012 return 0; 4013 } 4014 4015 int 4016 init_port_dcb_config(portid_t pid, 4017 enum dcb_mode_enable dcb_mode, 4018 enum rte_eth_nb_tcs num_tcs, 4019 uint8_t pfc_en) 4020 { 4021 struct rte_eth_conf port_conf; 4022 struct rte_port *rte_port; 4023 int retval; 4024 uint16_t i; 4025 4026 if (num_procs > 1) { 4027 printf("The multi-process feature doesn't support dcb.\n"); 4028 return -ENOTSUP; 4029 } 4030 rte_port = &ports[pid]; 4031 4032 /* retain the original device configuration. */ 4033 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4034 4035 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4036 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4037 if (retval < 0) 4038 return retval; 4039 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4040 /* remove RSS HASH offload for DCB in vt mode */ 4041 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4042 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4043 for (i = 0; i < nb_rxq; i++) 4044 rte_port->rx_conf[i].offloads &= 4045 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4046 } 4047 4048 /* re-configure the device . */ 4049 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4050 if (retval < 0) 4051 return retval; 4052 4053 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4054 if (retval != 0) 4055 return retval; 4056 4057 /* If dev_info.vmdq_pool_base is greater than 0, 4058 * the queue id of vmdq pools is started after pf queues. 4059 */ 4060 if (dcb_mode == DCB_VT_ENABLED && 4061 rte_port->dev_info.vmdq_pool_base > 0) { 4062 fprintf(stderr, 4063 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4064 pid); 4065 return -1; 4066 } 4067 4068 /* Assume the ports in testpmd have the same dcb capability 4069 * and has the same number of rxq and txq in dcb mode 4070 */ 4071 if (dcb_mode == DCB_VT_ENABLED) { 4072 if (rte_port->dev_info.max_vfs > 0) { 4073 nb_rxq = rte_port->dev_info.nb_rx_queues; 4074 nb_txq = rte_port->dev_info.nb_tx_queues; 4075 } else { 4076 nb_rxq = rte_port->dev_info.max_rx_queues; 4077 nb_txq = rte_port->dev_info.max_tx_queues; 4078 } 4079 } else { 4080 /*if vt is disabled, use all pf queues */ 4081 if (rte_port->dev_info.vmdq_pool_base == 0) { 4082 nb_rxq = rte_port->dev_info.max_rx_queues; 4083 nb_txq = rte_port->dev_info.max_tx_queues; 4084 } else { 4085 nb_rxq = (queueid_t)num_tcs; 4086 nb_txq = (queueid_t)num_tcs; 4087 4088 } 4089 } 4090 rx_free_thresh = 64; 4091 4092 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4093 4094 rxtx_port_config(pid); 4095 /* VLAN filter */ 4096 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4097 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4098 rx_vft_set(pid, vlan_tags[i], 1); 4099 4100 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4101 if (retval != 0) 4102 return retval; 4103 4104 rte_port->dcb_flag = 1; 4105 4106 /* Enter DCB configuration status */ 4107 dcb_config = 1; 4108 4109 return 0; 4110 } 4111 4112 static void 4113 init_port(void) 4114 { 4115 int i; 4116 4117 /* Configuration of Ethernet ports. */ 4118 ports = rte_zmalloc("testpmd: ports", 4119 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4120 RTE_CACHE_LINE_SIZE); 4121 if (ports == NULL) { 4122 rte_exit(EXIT_FAILURE, 4123 "rte_zmalloc(%d struct rte_port) failed\n", 4124 RTE_MAX_ETHPORTS); 4125 } 4126 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4127 ports[i].xstats_info.allocated = false; 4128 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4129 LIST_INIT(&ports[i].flow_tunnel_list); 4130 /* Initialize ports NUMA structures */ 4131 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4132 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4133 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4134 } 4135 4136 static void 4137 force_quit(void) 4138 { 4139 pmd_test_exit(); 4140 prompt_exit(); 4141 } 4142 4143 static void 4144 print_stats(void) 4145 { 4146 uint8_t i; 4147 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4148 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4149 4150 /* Clear screen and move to top left */ 4151 printf("%s%s", clr, top_left); 4152 4153 printf("\nPort statistics ===================================="); 4154 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4155 nic_stats_display(fwd_ports_ids[i]); 4156 4157 fflush(stdout); 4158 } 4159 4160 static void 4161 signal_handler(int signum) 4162 { 4163 if (signum == SIGINT || signum == SIGTERM) { 4164 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4165 signum); 4166 #ifdef RTE_LIB_PDUMP 4167 /* uninitialize packet capture framework */ 4168 rte_pdump_uninit(); 4169 #endif 4170 #ifdef RTE_LIB_LATENCYSTATS 4171 if (latencystats_enabled != 0) 4172 rte_latencystats_uninit(); 4173 #endif 4174 force_quit(); 4175 /* Set flag to indicate the force termination. */ 4176 f_quit = 1; 4177 /* exit with the expected status */ 4178 #ifndef RTE_EXEC_ENV_WINDOWS 4179 signal(signum, SIG_DFL); 4180 kill(getpid(), signum); 4181 #endif 4182 } 4183 } 4184 4185 int 4186 main(int argc, char** argv) 4187 { 4188 int diag; 4189 portid_t port_id; 4190 uint16_t count; 4191 int ret; 4192 4193 signal(SIGINT, signal_handler); 4194 signal(SIGTERM, signal_handler); 4195 4196 testpmd_logtype = rte_log_register("testpmd"); 4197 if (testpmd_logtype < 0) 4198 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4199 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4200 4201 diag = rte_eal_init(argc, argv); 4202 if (diag < 0) 4203 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4204 rte_strerror(rte_errno)); 4205 4206 ret = register_eth_event_callback(); 4207 if (ret != 0) 4208 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4209 4210 #ifdef RTE_LIB_PDUMP 4211 /* initialize packet capture framework */ 4212 rte_pdump_init(); 4213 #endif 4214 4215 count = 0; 4216 RTE_ETH_FOREACH_DEV(port_id) { 4217 ports_ids[count] = port_id; 4218 count++; 4219 } 4220 nb_ports = (portid_t) count; 4221 if (nb_ports == 0) 4222 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4223 4224 /* allocate port structures, and init them */ 4225 init_port(); 4226 4227 set_def_fwd_config(); 4228 if (nb_lcores == 0) 4229 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4230 "Check the core mask argument\n"); 4231 4232 /* Bitrate/latency stats disabled by default */ 4233 #ifdef RTE_LIB_BITRATESTATS 4234 bitrate_enabled = 0; 4235 #endif 4236 #ifdef RTE_LIB_LATENCYSTATS 4237 latencystats_enabled = 0; 4238 #endif 4239 4240 /* on FreeBSD, mlockall() is disabled by default */ 4241 #ifdef RTE_EXEC_ENV_FREEBSD 4242 do_mlockall = 0; 4243 #else 4244 do_mlockall = 1; 4245 #endif 4246 4247 argc -= diag; 4248 argv += diag; 4249 if (argc > 1) 4250 launch_args_parse(argc, argv); 4251 4252 #ifndef RTE_EXEC_ENV_WINDOWS 4253 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4254 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4255 strerror(errno)); 4256 } 4257 #endif 4258 4259 if (tx_first && interactive) 4260 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4261 "interactive mode.\n"); 4262 4263 if (tx_first && lsc_interrupt) { 4264 fprintf(stderr, 4265 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4266 lsc_interrupt = 0; 4267 } 4268 4269 if (!nb_rxq && !nb_txq) 4270 fprintf(stderr, 4271 "Warning: Either rx or tx queues should be non-zero\n"); 4272 4273 if (nb_rxq > 1 && nb_rxq > nb_txq) 4274 fprintf(stderr, 4275 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4276 nb_rxq, nb_txq); 4277 4278 init_config(); 4279 4280 if (hot_plug) { 4281 ret = rte_dev_hotplug_handle_enable(); 4282 if (ret) { 4283 RTE_LOG(ERR, EAL, 4284 "fail to enable hotplug handling."); 4285 return -1; 4286 } 4287 4288 ret = rte_dev_event_monitor_start(); 4289 if (ret) { 4290 RTE_LOG(ERR, EAL, 4291 "fail to start device event monitoring."); 4292 return -1; 4293 } 4294 4295 ret = rte_dev_event_callback_register(NULL, 4296 dev_event_callback, NULL); 4297 if (ret) { 4298 RTE_LOG(ERR, EAL, 4299 "fail to register device event callback\n"); 4300 return -1; 4301 } 4302 } 4303 4304 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4305 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4306 4307 /* set all ports to promiscuous mode by default */ 4308 RTE_ETH_FOREACH_DEV(port_id) { 4309 ret = rte_eth_promiscuous_enable(port_id); 4310 if (ret != 0) 4311 fprintf(stderr, 4312 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4313 port_id, rte_strerror(-ret)); 4314 } 4315 4316 #ifdef RTE_LIB_METRICS 4317 /* Init metrics library */ 4318 rte_metrics_init(rte_socket_id()); 4319 #endif 4320 4321 #ifdef RTE_LIB_LATENCYSTATS 4322 if (latencystats_enabled != 0) { 4323 int ret = rte_latencystats_init(1, NULL); 4324 if (ret) 4325 fprintf(stderr, 4326 "Warning: latencystats init() returned error %d\n", 4327 ret); 4328 fprintf(stderr, "Latencystats running on lcore %d\n", 4329 latencystats_lcore_id); 4330 } 4331 #endif 4332 4333 /* Setup bitrate stats */ 4334 #ifdef RTE_LIB_BITRATESTATS 4335 if (bitrate_enabled != 0) { 4336 bitrate_data = rte_stats_bitrate_create(); 4337 if (bitrate_data == NULL) 4338 rte_exit(EXIT_FAILURE, 4339 "Could not allocate bitrate data.\n"); 4340 rte_stats_bitrate_reg(bitrate_data); 4341 } 4342 #endif 4343 #ifdef RTE_LIB_CMDLINE 4344 if (strlen(cmdline_filename) != 0) 4345 cmdline_read_from_file(cmdline_filename); 4346 4347 if (interactive == 1) { 4348 if (auto_start) { 4349 printf("Start automatic packet forwarding\n"); 4350 start_packet_forwarding(0); 4351 } 4352 prompt(); 4353 pmd_test_exit(); 4354 } else 4355 #endif 4356 { 4357 char c; 4358 int rc; 4359 4360 f_quit = 0; 4361 4362 printf("No commandline core given, start packet forwarding\n"); 4363 start_packet_forwarding(tx_first); 4364 if (stats_period != 0) { 4365 uint64_t prev_time = 0, cur_time, diff_time = 0; 4366 uint64_t timer_period; 4367 4368 /* Convert to number of cycles */ 4369 timer_period = stats_period * rte_get_timer_hz(); 4370 4371 while (f_quit == 0) { 4372 cur_time = rte_get_timer_cycles(); 4373 diff_time += cur_time - prev_time; 4374 4375 if (diff_time >= timer_period) { 4376 print_stats(); 4377 /* Reset the timer */ 4378 diff_time = 0; 4379 } 4380 /* Sleep to avoid unnecessary checks */ 4381 prev_time = cur_time; 4382 rte_delay_us_sleep(US_PER_S); 4383 } 4384 } 4385 4386 printf("Press enter to exit\n"); 4387 rc = read(0, &c, 1); 4388 pmd_test_exit(); 4389 if (rc < 0) 4390 return 1; 4391 } 4392 4393 ret = rte_eal_cleanup(); 4394 if (ret != 0) 4395 rte_exit(EXIT_FAILURE, 4396 "EAL cleanup failed: %s\n", strerror(-ret)); 4397 4398 return EXIT_SUCCESS; 4399 } 4400