1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 73 #include "testpmd.h" 74 75 #ifndef MAP_HUGETLB 76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 77 #define HUGE_FLAG (0x40000) 78 #else 79 #define HUGE_FLAG MAP_HUGETLB 80 #endif 81 82 #ifndef MAP_HUGE_SHIFT 83 /* older kernels (or FreeBSD) will not have this define */ 84 #define HUGE_SHIFT (26) 85 #else 86 #define HUGE_SHIFT MAP_HUGE_SHIFT 87 #endif 88 89 #define EXTMEM_HEAP_NAME "extmem" 90 /* 91 * Zone size with the malloc overhead (max of debug and release variants) 92 * must fit into the smallest supported hugepage size (2M), 93 * so that an IOVA-contiguous zone of this size can always be allocated 94 * if there are free 2M hugepages. 95 */ 96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 97 98 uint16_t verbose_level = 0; /**< Silent by default. */ 99 int testpmd_logtype; /**< Log type for testpmd logs */ 100 101 /* use main core for command line ? */ 102 uint8_t interactive = 0; 103 uint8_t auto_start = 0; 104 uint8_t tx_first; 105 char cmdline_filename[PATH_MAX] = {0}; 106 107 /* 108 * NUMA support configuration. 109 * When set, the NUMA support attempts to dispatch the allocation of the 110 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 111 * probed ports among the CPU sockets 0 and 1. 112 * Otherwise, all memory is allocated from CPU socket 0. 113 */ 114 uint8_t numa_support = 1; /**< numa enabled by default */ 115 116 /* 117 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 118 * not configured. 119 */ 120 uint8_t socket_num = UMA_NO_CONFIG; 121 122 /* 123 * Select mempool allocation type: 124 * - native: use regular DPDK memory 125 * - anon: use regular DPDK memory to create mempool, but populate using 126 * anonymous memory (may not be IOVA-contiguous) 127 * - xmem: use externally allocated hugepage memory 128 */ 129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 130 131 /* 132 * Store specified sockets on which memory pool to be used by ports 133 * is allocated. 134 */ 135 uint8_t port_numa[RTE_MAX_ETHPORTS]; 136 137 /* 138 * Store specified sockets on which RX ring to be used by ports 139 * is allocated. 140 */ 141 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 142 143 /* 144 * Store specified sockets on which TX ring to be used by ports 145 * is allocated. 146 */ 147 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 148 149 /* 150 * Record the Ethernet address of peer target ports to which packets are 151 * forwarded. 152 * Must be instantiated with the ethernet addresses of peer traffic generator 153 * ports. 154 */ 155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 156 portid_t nb_peer_eth_addrs = 0; 157 158 /* 159 * Probed Target Environment. 160 */ 161 struct rte_port *ports; /**< For all probed ethernet ports. */ 162 portid_t nb_ports; /**< Number of probed ethernet ports. */ 163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 164 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 165 166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 167 168 /* 169 * Test Forwarding Configuration. 170 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 171 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 172 */ 173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 175 portid_t nb_cfg_ports; /**< Number of configured ports. */ 176 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 177 178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 180 181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 182 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 183 184 /* 185 * Forwarding engines. 186 */ 187 struct fwd_engine * fwd_engines[] = { 188 &io_fwd_engine, 189 &mac_fwd_engine, 190 &mac_swap_engine, 191 &flow_gen_engine, 192 &rx_only_engine, 193 &tx_only_engine, 194 &csum_fwd_engine, 195 &icmp_echo_engine, 196 &noisy_vnf_engine, 197 &five_tuple_swap_fwd_engine, 198 #ifdef RTE_LIBRTE_IEEE1588 199 &ieee1588_fwd_engine, 200 #endif 201 &shared_rxq_engine, 202 NULL, 203 }; 204 205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 206 uint16_t mempool_flags; 207 208 struct fwd_config cur_fwd_config; 209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 210 uint32_t retry_enabled; 211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 213 214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 216 DEFAULT_MBUF_DATA_SIZE 217 }; /**< Mbuf data space size. */ 218 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 219 * specified on command-line. */ 220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 221 222 /** Extended statistics to show. */ 223 struct rte_eth_xstat_name *xstats_display; 224 225 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 226 227 /* 228 * In container, it cannot terminate the process which running with 'stats-period' 229 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 230 */ 231 uint8_t f_quit; 232 233 /* 234 * Max Rx frame size, set by '--max-pkt-len' parameter. 235 */ 236 uint32_t max_rx_pkt_len; 237 238 /* 239 * Configuration of packet segments used to scatter received packets 240 * if some of split features is configured. 241 */ 242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 243 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 245 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 246 247 /* 248 * Configuration of packet segments used by the "txonly" processing engine. 249 */ 250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 252 TXONLY_DEF_PACKET_LEN, 253 }; 254 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 255 256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 257 /**< Split policy for packets to TX. */ 258 259 uint8_t txonly_multi_flow; 260 /**< Whether multiple flows are generated in TXONLY mode. */ 261 262 uint32_t tx_pkt_times_inter; 263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 264 265 uint32_t tx_pkt_times_intra; 266 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 267 268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 272 273 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 274 uint8_t dcb_config = 0; 275 276 /* 277 * Configurable number of RX/TX queues. 278 */ 279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 282 283 /* 284 * Configurable number of RX/TX ring descriptors. 285 * Defaults are supplied by drivers via ethdev. 286 */ 287 #define RTE_TEST_RX_DESC_DEFAULT 0 288 #define RTE_TEST_TX_DESC_DEFAULT 0 289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 291 292 #define RTE_PMD_PARAM_UNSET -1 293 /* 294 * Configurable values of RX and TX ring threshold registers. 295 */ 296 297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 300 301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 304 305 /* 306 * Configurable value of RX free threshold. 307 */ 308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 309 310 /* 311 * Configurable value of RX drop enable. 312 */ 313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 314 315 /* 316 * Configurable value of TX free threshold. 317 */ 318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 319 320 /* 321 * Configurable value of TX RS bit threshold. 322 */ 323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 324 325 /* 326 * Configurable value of buffered packets before sending. 327 */ 328 uint16_t noisy_tx_sw_bufsz; 329 330 /* 331 * Configurable value of packet buffer timeout. 332 */ 333 uint16_t noisy_tx_sw_buf_flush_time; 334 335 /* 336 * Configurable value for size of VNF internal memory area 337 * used for simulating noisy neighbour behaviour 338 */ 339 uint64_t noisy_lkup_mem_sz; 340 341 /* 342 * Configurable value of number of random writes done in 343 * VNF simulation memory area. 344 */ 345 uint64_t noisy_lkup_num_writes; 346 347 /* 348 * Configurable value of number of random reads done in 349 * VNF simulation memory area. 350 */ 351 uint64_t noisy_lkup_num_reads; 352 353 /* 354 * Configurable value of number of random reads/writes done in 355 * VNF simulation memory area. 356 */ 357 uint64_t noisy_lkup_num_reads_writes; 358 359 /* 360 * Receive Side Scaling (RSS) configuration. 361 */ 362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 363 364 /* 365 * Port topology configuration 366 */ 367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 368 369 /* 370 * Avoids to flush all the RX streams before starts forwarding. 371 */ 372 uint8_t no_flush_rx = 0; /* flush by default */ 373 374 /* 375 * Flow API isolated mode. 376 */ 377 uint8_t flow_isolate_all; 378 379 /* 380 * Avoids to check link status when starting/stopping a port. 381 */ 382 uint8_t no_link_check = 0; /* check by default */ 383 384 /* 385 * Don't automatically start all ports in interactive mode. 386 */ 387 uint8_t no_device_start = 0; 388 389 /* 390 * Enable link status change notification 391 */ 392 uint8_t lsc_interrupt = 1; /* enabled by default */ 393 394 /* 395 * Enable device removal notification. 396 */ 397 uint8_t rmv_interrupt = 1; /* enabled by default */ 398 399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 400 401 /* After attach, port setup is called on event or by iterator */ 402 bool setup_on_probe_event = true; 403 404 /* Clear ptypes on port initialization. */ 405 uint8_t clear_ptypes = true; 406 407 /* Hairpin ports configuration mode. */ 408 uint16_t hairpin_mode; 409 410 /* Pretty printing of ethdev events */ 411 static const char * const eth_event_desc[] = { 412 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 413 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 414 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 415 [RTE_ETH_EVENT_INTR_RESET] = "reset", 416 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 417 [RTE_ETH_EVENT_IPSEC] = "IPsec", 418 [RTE_ETH_EVENT_MACSEC] = "MACsec", 419 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 420 [RTE_ETH_EVENT_NEW] = "device probed", 421 [RTE_ETH_EVENT_DESTROY] = "device released", 422 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 423 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 424 [RTE_ETH_EVENT_MAX] = NULL, 425 }; 426 427 /* 428 * Display or mask ether events 429 * Default to all events except VF_MBOX 430 */ 431 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 432 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 433 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 434 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 435 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 436 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 437 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 438 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 439 /* 440 * Decide if all memory are locked for performance. 441 */ 442 int do_mlockall = 0; 443 444 /* 445 * NIC bypass mode configuration options. 446 */ 447 448 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 449 /* The NIC bypass watchdog timeout. */ 450 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 451 #endif 452 453 454 #ifdef RTE_LIB_LATENCYSTATS 455 456 /* 457 * Set when latency stats is enabled in the commandline 458 */ 459 uint8_t latencystats_enabled; 460 461 /* 462 * Lcore ID to service latency statistics. 463 */ 464 lcoreid_t latencystats_lcore_id = -1; 465 466 #endif 467 468 /* 469 * Ethernet device configuration. 470 */ 471 struct rte_eth_rxmode rx_mode; 472 473 struct rte_eth_txmode tx_mode = { 474 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 475 }; 476 477 struct rte_eth_fdir_conf fdir_conf = { 478 .mode = RTE_FDIR_MODE_NONE, 479 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 480 .status = RTE_FDIR_REPORT_STATUS, 481 .mask = { 482 .vlan_tci_mask = 0xFFEF, 483 .ipv4_mask = { 484 .src_ip = 0xFFFFFFFF, 485 .dst_ip = 0xFFFFFFFF, 486 }, 487 .ipv6_mask = { 488 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 489 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 490 }, 491 .src_port_mask = 0xFFFF, 492 .dst_port_mask = 0xFFFF, 493 .mac_addr_byte_mask = 0xFF, 494 .tunnel_type_mask = 1, 495 .tunnel_id_mask = 0xFFFFFFFF, 496 }, 497 .drop_queue = 127, 498 }; 499 500 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 501 502 /* 503 * Display zero values by default for xstats 504 */ 505 uint8_t xstats_hide_zero; 506 507 /* 508 * Measure of CPU cycles disabled by default 509 */ 510 uint8_t record_core_cycles; 511 512 /* 513 * Display of RX and TX bursts disabled by default 514 */ 515 uint8_t record_burst_stats; 516 517 /* 518 * Number of ports per shared Rx queue group, 0 disable. 519 */ 520 uint32_t rxq_share; 521 522 unsigned int num_sockets = 0; 523 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 524 525 #ifdef RTE_LIB_BITRATESTATS 526 /* Bitrate statistics */ 527 struct rte_stats_bitrates *bitrate_data; 528 lcoreid_t bitrate_lcore_id; 529 uint8_t bitrate_enabled; 530 #endif 531 532 #ifdef RTE_LIB_GRO 533 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 534 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 535 #endif 536 537 /* 538 * hexadecimal bitmask of RX mq mode can be enabled. 539 */ 540 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 541 542 /* 543 * Used to set forced link speed 544 */ 545 uint32_t eth_link_speed; 546 547 /* 548 * ID of the current process in multi-process, used to 549 * configure the queues to be polled. 550 */ 551 int proc_id; 552 553 /* 554 * Number of processes in multi-process, used to 555 * configure the queues to be polled. 556 */ 557 unsigned int num_procs = 1; 558 559 static void 560 eth_rx_metadata_negotiate_mp(uint16_t port_id) 561 { 562 uint64_t rx_meta_features = 0; 563 int ret; 564 565 if (!is_proc_primary()) 566 return; 567 568 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 569 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 570 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 571 572 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 573 if (ret == 0) { 574 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 575 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 576 port_id); 577 } 578 579 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 580 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 581 port_id); 582 } 583 584 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 585 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 586 port_id); 587 } 588 } else if (ret != -ENOTSUP) { 589 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 590 port_id, rte_strerror(-ret)); 591 } 592 } 593 594 static int 595 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 596 const struct rte_eth_conf *dev_conf) 597 { 598 if (is_proc_primary()) 599 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 600 dev_conf); 601 return 0; 602 } 603 604 static int 605 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 606 { 607 #ifdef RTE_NET_BOND 608 609 portid_t slave_pids[RTE_MAX_ETHPORTS]; 610 struct rte_port *port; 611 int num_slaves; 612 portid_t slave_pid; 613 int i; 614 615 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 616 RTE_MAX_ETHPORTS); 617 if (num_slaves < 0) { 618 fprintf(stderr, "Failed to get slave list for port = %u\n", 619 bond_pid); 620 return num_slaves; 621 } 622 623 for (i = 0; i < num_slaves; i++) { 624 slave_pid = slave_pids[i]; 625 port = &ports[slave_pid]; 626 port->port_status = 627 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 628 } 629 #else 630 RTE_SET_USED(bond_pid); 631 RTE_SET_USED(is_stop); 632 #endif 633 return 0; 634 } 635 636 static int 637 eth_dev_start_mp(uint16_t port_id) 638 { 639 int ret; 640 641 if (is_proc_primary()) { 642 ret = rte_eth_dev_start(port_id); 643 if (ret != 0) 644 return ret; 645 646 struct rte_port *port = &ports[port_id]; 647 648 /* 649 * Starting a bonded port also starts all slaves under the bonded 650 * device. So if this port is bond device, we need to modify the 651 * port status of these slaves. 652 */ 653 if (port->bond_flag == 1) 654 return change_bonding_slave_port_status(port_id, false); 655 } 656 657 return 0; 658 } 659 660 static int 661 eth_dev_stop_mp(uint16_t port_id) 662 { 663 int ret; 664 665 if (is_proc_primary()) { 666 ret = rte_eth_dev_stop(port_id); 667 if (ret != 0) 668 return ret; 669 670 struct rte_port *port = &ports[port_id]; 671 672 /* 673 * Stopping a bonded port also stops all slaves under the bonded 674 * device. So if this port is bond device, we need to modify the 675 * port status of these slaves. 676 */ 677 if (port->bond_flag == 1) 678 return change_bonding_slave_port_status(port_id, true); 679 } 680 681 return 0; 682 } 683 684 static void 685 mempool_free_mp(struct rte_mempool *mp) 686 { 687 if (is_proc_primary()) 688 rte_mempool_free(mp); 689 } 690 691 static int 692 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 693 { 694 if (is_proc_primary()) 695 return rte_eth_dev_set_mtu(port_id, mtu); 696 697 return 0; 698 } 699 700 /* Forward function declarations */ 701 static void setup_attached_port(portid_t pi); 702 static void check_all_ports_link_status(uint32_t port_mask); 703 static int eth_event_callback(portid_t port_id, 704 enum rte_eth_event_type type, 705 void *param, void *ret_param); 706 static void dev_event_callback(const char *device_name, 707 enum rte_dev_event_type type, 708 void *param); 709 static void fill_xstats_display_info(void); 710 711 /* 712 * Check if all the ports are started. 713 * If yes, return positive value. If not, return zero. 714 */ 715 static int all_ports_started(void); 716 717 #ifdef RTE_LIB_GSO 718 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 719 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 720 #endif 721 722 /* Holds the registered mbuf dynamic flags names. */ 723 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 724 725 726 /* 727 * Helper function to check if socket is already discovered. 728 * If yes, return positive value. If not, return zero. 729 */ 730 int 731 new_socket_id(unsigned int socket_id) 732 { 733 unsigned int i; 734 735 for (i = 0; i < num_sockets; i++) { 736 if (socket_ids[i] == socket_id) 737 return 0; 738 } 739 return 1; 740 } 741 742 /* 743 * Setup default configuration. 744 */ 745 static void 746 set_default_fwd_lcores_config(void) 747 { 748 unsigned int i; 749 unsigned int nb_lc; 750 unsigned int sock_num; 751 752 nb_lc = 0; 753 for (i = 0; i < RTE_MAX_LCORE; i++) { 754 if (!rte_lcore_is_enabled(i)) 755 continue; 756 sock_num = rte_lcore_to_socket_id(i); 757 if (new_socket_id(sock_num)) { 758 if (num_sockets >= RTE_MAX_NUMA_NODES) { 759 rte_exit(EXIT_FAILURE, 760 "Total sockets greater than %u\n", 761 RTE_MAX_NUMA_NODES); 762 } 763 socket_ids[num_sockets++] = sock_num; 764 } 765 if (i == rte_get_main_lcore()) 766 continue; 767 fwd_lcores_cpuids[nb_lc++] = i; 768 } 769 nb_lcores = (lcoreid_t) nb_lc; 770 nb_cfg_lcores = nb_lcores; 771 nb_fwd_lcores = 1; 772 } 773 774 static void 775 set_def_peer_eth_addrs(void) 776 { 777 portid_t i; 778 779 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 780 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 781 peer_eth_addrs[i].addr_bytes[5] = i; 782 } 783 } 784 785 static void 786 set_default_fwd_ports_config(void) 787 { 788 portid_t pt_id; 789 int i = 0; 790 791 RTE_ETH_FOREACH_DEV(pt_id) { 792 fwd_ports_ids[i++] = pt_id; 793 794 /* Update sockets info according to the attached device */ 795 int socket_id = rte_eth_dev_socket_id(pt_id); 796 if (socket_id >= 0 && new_socket_id(socket_id)) { 797 if (num_sockets >= RTE_MAX_NUMA_NODES) { 798 rte_exit(EXIT_FAILURE, 799 "Total sockets greater than %u\n", 800 RTE_MAX_NUMA_NODES); 801 } 802 socket_ids[num_sockets++] = socket_id; 803 } 804 } 805 806 nb_cfg_ports = nb_ports; 807 nb_fwd_ports = nb_ports; 808 } 809 810 void 811 set_def_fwd_config(void) 812 { 813 set_default_fwd_lcores_config(); 814 set_def_peer_eth_addrs(); 815 set_default_fwd_ports_config(); 816 } 817 818 #ifndef RTE_EXEC_ENV_WINDOWS 819 /* extremely pessimistic estimation of memory required to create a mempool */ 820 static int 821 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 822 { 823 unsigned int n_pages, mbuf_per_pg, leftover; 824 uint64_t total_mem, mbuf_mem, obj_sz; 825 826 /* there is no good way to predict how much space the mempool will 827 * occupy because it will allocate chunks on the fly, and some of those 828 * will come from default DPDK memory while some will come from our 829 * external memory, so just assume 128MB will be enough for everyone. 830 */ 831 uint64_t hdr_mem = 128 << 20; 832 833 /* account for possible non-contiguousness */ 834 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 835 if (obj_sz > pgsz) { 836 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 837 return -1; 838 } 839 840 mbuf_per_pg = pgsz / obj_sz; 841 leftover = (nb_mbufs % mbuf_per_pg) > 0; 842 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 843 844 mbuf_mem = n_pages * pgsz; 845 846 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 847 848 if (total_mem > SIZE_MAX) { 849 TESTPMD_LOG(ERR, "Memory size too big\n"); 850 return -1; 851 } 852 *out = (size_t)total_mem; 853 854 return 0; 855 } 856 857 static int 858 pagesz_flags(uint64_t page_sz) 859 { 860 /* as per mmap() manpage, all page sizes are log2 of page size 861 * shifted by MAP_HUGE_SHIFT 862 */ 863 int log2 = rte_log2_u64(page_sz); 864 865 return (log2 << HUGE_SHIFT); 866 } 867 868 static void * 869 alloc_mem(size_t memsz, size_t pgsz, bool huge) 870 { 871 void *addr; 872 int flags; 873 874 /* allocate anonymous hugepages */ 875 flags = MAP_ANONYMOUS | MAP_PRIVATE; 876 if (huge) 877 flags |= HUGE_FLAG | pagesz_flags(pgsz); 878 879 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 880 if (addr == MAP_FAILED) 881 return NULL; 882 883 return addr; 884 } 885 886 struct extmem_param { 887 void *addr; 888 size_t len; 889 size_t pgsz; 890 rte_iova_t *iova_table; 891 unsigned int iova_table_len; 892 }; 893 894 static int 895 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 896 bool huge) 897 { 898 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 899 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 900 unsigned int cur_page, n_pages, pgsz_idx; 901 size_t mem_sz, cur_pgsz; 902 rte_iova_t *iovas = NULL; 903 void *addr; 904 int ret; 905 906 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 907 /* skip anything that is too big */ 908 if (pgsizes[pgsz_idx] > SIZE_MAX) 909 continue; 910 911 cur_pgsz = pgsizes[pgsz_idx]; 912 913 /* if we were told not to allocate hugepages, override */ 914 if (!huge) 915 cur_pgsz = sysconf(_SC_PAGESIZE); 916 917 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 918 if (ret < 0) { 919 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 920 return -1; 921 } 922 923 /* allocate our memory */ 924 addr = alloc_mem(mem_sz, cur_pgsz, huge); 925 926 /* if we couldn't allocate memory with a specified page size, 927 * that doesn't mean we can't do it with other page sizes, so 928 * try another one. 929 */ 930 if (addr == NULL) 931 continue; 932 933 /* store IOVA addresses for every page in this memory area */ 934 n_pages = mem_sz / cur_pgsz; 935 936 iovas = malloc(sizeof(*iovas) * n_pages); 937 938 if (iovas == NULL) { 939 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 940 goto fail; 941 } 942 /* lock memory if it's not huge pages */ 943 if (!huge) 944 mlock(addr, mem_sz); 945 946 /* populate IOVA addresses */ 947 for (cur_page = 0; cur_page < n_pages; cur_page++) { 948 rte_iova_t iova; 949 size_t offset; 950 void *cur; 951 952 offset = cur_pgsz * cur_page; 953 cur = RTE_PTR_ADD(addr, offset); 954 955 /* touch the page before getting its IOVA */ 956 *(volatile char *)cur = 0; 957 958 iova = rte_mem_virt2iova(cur); 959 960 iovas[cur_page] = iova; 961 } 962 963 break; 964 } 965 /* if we couldn't allocate anything */ 966 if (iovas == NULL) 967 return -1; 968 969 param->addr = addr; 970 param->len = mem_sz; 971 param->pgsz = cur_pgsz; 972 param->iova_table = iovas; 973 param->iova_table_len = n_pages; 974 975 return 0; 976 fail: 977 free(iovas); 978 if (addr) 979 munmap(addr, mem_sz); 980 981 return -1; 982 } 983 984 static int 985 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 986 { 987 struct extmem_param param; 988 int socket_id, ret; 989 990 memset(¶m, 0, sizeof(param)); 991 992 /* check if our heap exists */ 993 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 994 if (socket_id < 0) { 995 /* create our heap */ 996 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 997 if (ret < 0) { 998 TESTPMD_LOG(ERR, "Cannot create heap\n"); 999 return -1; 1000 } 1001 } 1002 1003 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 1004 if (ret < 0) { 1005 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 1006 return -1; 1007 } 1008 1009 /* we now have a valid memory area, so add it to heap */ 1010 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 1011 param.addr, param.len, param.iova_table, 1012 param.iova_table_len, param.pgsz); 1013 1014 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1015 1016 /* not needed any more */ 1017 free(param.iova_table); 1018 1019 if (ret < 0) { 1020 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1021 munmap(param.addr, param.len); 1022 return -1; 1023 } 1024 1025 /* success */ 1026 1027 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1028 param.len >> 20); 1029 1030 return 0; 1031 } 1032 static void 1033 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1034 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1035 { 1036 uint16_t pid = 0; 1037 int ret; 1038 1039 RTE_ETH_FOREACH_DEV(pid) { 1040 struct rte_eth_dev_info dev_info; 1041 1042 ret = eth_dev_info_get_print_err(pid, &dev_info); 1043 if (ret != 0) { 1044 TESTPMD_LOG(DEBUG, 1045 "unable to get device info for port %d on addr 0x%p," 1046 "mempool unmapping will not be performed\n", 1047 pid, memhdr->addr); 1048 continue; 1049 } 1050 1051 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1052 if (ret) { 1053 TESTPMD_LOG(DEBUG, 1054 "unable to DMA unmap addr 0x%p " 1055 "for device %s\n", 1056 memhdr->addr, dev_info.device->name); 1057 } 1058 } 1059 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1060 if (ret) { 1061 TESTPMD_LOG(DEBUG, 1062 "unable to un-register addr 0x%p\n", memhdr->addr); 1063 } 1064 } 1065 1066 static void 1067 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1068 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1069 { 1070 uint16_t pid = 0; 1071 size_t page_size = sysconf(_SC_PAGESIZE); 1072 int ret; 1073 1074 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1075 page_size); 1076 if (ret) { 1077 TESTPMD_LOG(DEBUG, 1078 "unable to register addr 0x%p\n", memhdr->addr); 1079 return; 1080 } 1081 RTE_ETH_FOREACH_DEV(pid) { 1082 struct rte_eth_dev_info dev_info; 1083 1084 ret = eth_dev_info_get_print_err(pid, &dev_info); 1085 if (ret != 0) { 1086 TESTPMD_LOG(DEBUG, 1087 "unable to get device info for port %d on addr 0x%p," 1088 "mempool mapping will not be performed\n", 1089 pid, memhdr->addr); 1090 continue; 1091 } 1092 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1093 if (ret) { 1094 TESTPMD_LOG(DEBUG, 1095 "unable to DMA map addr 0x%p " 1096 "for device %s\n", 1097 memhdr->addr, dev_info.device->name); 1098 } 1099 } 1100 } 1101 #endif 1102 1103 static unsigned int 1104 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1105 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1106 { 1107 struct rte_pktmbuf_extmem *xmem; 1108 unsigned int ext_num, zone_num, elt_num; 1109 uint16_t elt_size; 1110 1111 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1112 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1113 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1114 1115 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1116 if (xmem == NULL) { 1117 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1118 "external buffer descriptors\n"); 1119 *ext_mem = NULL; 1120 return 0; 1121 } 1122 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1123 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1124 const struct rte_memzone *mz; 1125 char mz_name[RTE_MEMZONE_NAMESIZE]; 1126 int ret; 1127 1128 ret = snprintf(mz_name, sizeof(mz_name), 1129 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1130 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1131 errno = ENAMETOOLONG; 1132 ext_num = 0; 1133 break; 1134 } 1135 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1136 socket_id, 1137 RTE_MEMZONE_IOVA_CONTIG | 1138 RTE_MEMZONE_1GB | 1139 RTE_MEMZONE_SIZE_HINT_ONLY); 1140 if (mz == NULL) { 1141 /* 1142 * The caller exits on external buffer creation 1143 * error, so there is no need to free memzones. 1144 */ 1145 errno = ENOMEM; 1146 ext_num = 0; 1147 break; 1148 } 1149 xseg->buf_ptr = mz->addr; 1150 xseg->buf_iova = mz->iova; 1151 xseg->buf_len = EXTBUF_ZONE_SIZE; 1152 xseg->elt_size = elt_size; 1153 } 1154 if (ext_num == 0 && xmem != NULL) { 1155 free(xmem); 1156 xmem = NULL; 1157 } 1158 *ext_mem = xmem; 1159 return ext_num; 1160 } 1161 1162 /* 1163 * Configuration initialisation done once at init time. 1164 */ 1165 static struct rte_mempool * 1166 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1167 unsigned int socket_id, uint16_t size_idx) 1168 { 1169 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1170 struct rte_mempool *rte_mp = NULL; 1171 #ifndef RTE_EXEC_ENV_WINDOWS 1172 uint32_t mb_size; 1173 1174 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1175 #endif 1176 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1177 if (!is_proc_primary()) { 1178 rte_mp = rte_mempool_lookup(pool_name); 1179 if (rte_mp == NULL) 1180 rte_exit(EXIT_FAILURE, 1181 "Get mbuf pool for socket %u failed: %s\n", 1182 socket_id, rte_strerror(rte_errno)); 1183 return rte_mp; 1184 } 1185 1186 TESTPMD_LOG(INFO, 1187 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1188 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1189 1190 switch (mp_alloc_type) { 1191 case MP_ALLOC_NATIVE: 1192 { 1193 /* wrapper to rte_mempool_create() */ 1194 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1195 rte_mbuf_best_mempool_ops()); 1196 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1197 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1198 break; 1199 } 1200 #ifndef RTE_EXEC_ENV_WINDOWS 1201 case MP_ALLOC_ANON: 1202 { 1203 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1204 mb_size, (unsigned int) mb_mempool_cache, 1205 sizeof(struct rte_pktmbuf_pool_private), 1206 socket_id, mempool_flags); 1207 if (rte_mp == NULL) 1208 goto err; 1209 1210 if (rte_mempool_populate_anon(rte_mp) == 0) { 1211 rte_mempool_free(rte_mp); 1212 rte_mp = NULL; 1213 goto err; 1214 } 1215 rte_pktmbuf_pool_init(rte_mp, NULL); 1216 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1217 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1218 break; 1219 } 1220 case MP_ALLOC_XMEM: 1221 case MP_ALLOC_XMEM_HUGE: 1222 { 1223 int heap_socket; 1224 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1225 1226 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1227 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1228 1229 heap_socket = 1230 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1231 if (heap_socket < 0) 1232 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1233 1234 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1235 rte_mbuf_best_mempool_ops()); 1236 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1237 mb_mempool_cache, 0, mbuf_seg_size, 1238 heap_socket); 1239 break; 1240 } 1241 #endif 1242 case MP_ALLOC_XBUF: 1243 { 1244 struct rte_pktmbuf_extmem *ext_mem; 1245 unsigned int ext_num; 1246 1247 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1248 socket_id, pool_name, &ext_mem); 1249 if (ext_num == 0) 1250 rte_exit(EXIT_FAILURE, 1251 "Can't create pinned data buffers\n"); 1252 1253 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1254 rte_mbuf_best_mempool_ops()); 1255 rte_mp = rte_pktmbuf_pool_create_extbuf 1256 (pool_name, nb_mbuf, mb_mempool_cache, 1257 0, mbuf_seg_size, socket_id, 1258 ext_mem, ext_num); 1259 free(ext_mem); 1260 break; 1261 } 1262 default: 1263 { 1264 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1265 } 1266 } 1267 1268 #ifndef RTE_EXEC_ENV_WINDOWS 1269 err: 1270 #endif 1271 if (rte_mp == NULL) { 1272 rte_exit(EXIT_FAILURE, 1273 "Creation of mbuf pool for socket %u failed: %s\n", 1274 socket_id, rte_strerror(rte_errno)); 1275 } else if (verbose_level > 0) { 1276 rte_mempool_dump(stdout, rte_mp); 1277 } 1278 return rte_mp; 1279 } 1280 1281 /* 1282 * Check given socket id is valid or not with NUMA mode, 1283 * if valid, return 0, else return -1 1284 */ 1285 static int 1286 check_socket_id(const unsigned int socket_id) 1287 { 1288 static int warning_once = 0; 1289 1290 if (new_socket_id(socket_id)) { 1291 if (!warning_once && numa_support) 1292 fprintf(stderr, 1293 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1294 warning_once = 1; 1295 return -1; 1296 } 1297 return 0; 1298 } 1299 1300 /* 1301 * Get the allowed maximum number of RX queues. 1302 * *pid return the port id which has minimal value of 1303 * max_rx_queues in all ports. 1304 */ 1305 queueid_t 1306 get_allowed_max_nb_rxq(portid_t *pid) 1307 { 1308 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1309 bool max_rxq_valid = false; 1310 portid_t pi; 1311 struct rte_eth_dev_info dev_info; 1312 1313 RTE_ETH_FOREACH_DEV(pi) { 1314 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1315 continue; 1316 1317 max_rxq_valid = true; 1318 if (dev_info.max_rx_queues < allowed_max_rxq) { 1319 allowed_max_rxq = dev_info.max_rx_queues; 1320 *pid = pi; 1321 } 1322 } 1323 return max_rxq_valid ? allowed_max_rxq : 0; 1324 } 1325 1326 /* 1327 * Check input rxq is valid or not. 1328 * If input rxq is not greater than any of maximum number 1329 * of RX queues of all ports, it is valid. 1330 * if valid, return 0, else return -1 1331 */ 1332 int 1333 check_nb_rxq(queueid_t rxq) 1334 { 1335 queueid_t allowed_max_rxq; 1336 portid_t pid = 0; 1337 1338 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1339 if (rxq > allowed_max_rxq) { 1340 fprintf(stderr, 1341 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1342 rxq, allowed_max_rxq, pid); 1343 return -1; 1344 } 1345 return 0; 1346 } 1347 1348 /* 1349 * Get the allowed maximum number of TX queues. 1350 * *pid return the port id which has minimal value of 1351 * max_tx_queues in all ports. 1352 */ 1353 queueid_t 1354 get_allowed_max_nb_txq(portid_t *pid) 1355 { 1356 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1357 bool max_txq_valid = false; 1358 portid_t pi; 1359 struct rte_eth_dev_info dev_info; 1360 1361 RTE_ETH_FOREACH_DEV(pi) { 1362 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1363 continue; 1364 1365 max_txq_valid = true; 1366 if (dev_info.max_tx_queues < allowed_max_txq) { 1367 allowed_max_txq = dev_info.max_tx_queues; 1368 *pid = pi; 1369 } 1370 } 1371 return max_txq_valid ? allowed_max_txq : 0; 1372 } 1373 1374 /* 1375 * Check input txq is valid or not. 1376 * If input txq is not greater than any of maximum number 1377 * of TX queues of all ports, it is valid. 1378 * if valid, return 0, else return -1 1379 */ 1380 int 1381 check_nb_txq(queueid_t txq) 1382 { 1383 queueid_t allowed_max_txq; 1384 portid_t pid = 0; 1385 1386 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1387 if (txq > allowed_max_txq) { 1388 fprintf(stderr, 1389 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1390 txq, allowed_max_txq, pid); 1391 return -1; 1392 } 1393 return 0; 1394 } 1395 1396 /* 1397 * Get the allowed maximum number of RXDs of every rx queue. 1398 * *pid return the port id which has minimal value of 1399 * max_rxd in all queues of all ports. 1400 */ 1401 static uint16_t 1402 get_allowed_max_nb_rxd(portid_t *pid) 1403 { 1404 uint16_t allowed_max_rxd = UINT16_MAX; 1405 portid_t pi; 1406 struct rte_eth_dev_info dev_info; 1407 1408 RTE_ETH_FOREACH_DEV(pi) { 1409 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1410 continue; 1411 1412 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1413 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1414 *pid = pi; 1415 } 1416 } 1417 return allowed_max_rxd; 1418 } 1419 1420 /* 1421 * Get the allowed minimal number of RXDs of every rx queue. 1422 * *pid return the port id which has minimal value of 1423 * min_rxd in all queues of all ports. 1424 */ 1425 static uint16_t 1426 get_allowed_min_nb_rxd(portid_t *pid) 1427 { 1428 uint16_t allowed_min_rxd = 0; 1429 portid_t pi; 1430 struct rte_eth_dev_info dev_info; 1431 1432 RTE_ETH_FOREACH_DEV(pi) { 1433 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1434 continue; 1435 1436 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1437 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1438 *pid = pi; 1439 } 1440 } 1441 1442 return allowed_min_rxd; 1443 } 1444 1445 /* 1446 * Check input rxd is valid or not. 1447 * If input rxd is not greater than any of maximum number 1448 * of RXDs of every Rx queues and is not less than any of 1449 * minimal number of RXDs of every Rx queues, it is valid. 1450 * if valid, return 0, else return -1 1451 */ 1452 int 1453 check_nb_rxd(queueid_t rxd) 1454 { 1455 uint16_t allowed_max_rxd; 1456 uint16_t allowed_min_rxd; 1457 portid_t pid = 0; 1458 1459 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1460 if (rxd > allowed_max_rxd) { 1461 fprintf(stderr, 1462 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1463 rxd, allowed_max_rxd, pid); 1464 return -1; 1465 } 1466 1467 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1468 if (rxd < allowed_min_rxd) { 1469 fprintf(stderr, 1470 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1471 rxd, allowed_min_rxd, pid); 1472 return -1; 1473 } 1474 1475 return 0; 1476 } 1477 1478 /* 1479 * Get the allowed maximum number of TXDs of every rx queues. 1480 * *pid return the port id which has minimal value of 1481 * max_txd in every tx queue. 1482 */ 1483 static uint16_t 1484 get_allowed_max_nb_txd(portid_t *pid) 1485 { 1486 uint16_t allowed_max_txd = UINT16_MAX; 1487 portid_t pi; 1488 struct rte_eth_dev_info dev_info; 1489 1490 RTE_ETH_FOREACH_DEV(pi) { 1491 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1492 continue; 1493 1494 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1495 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1496 *pid = pi; 1497 } 1498 } 1499 return allowed_max_txd; 1500 } 1501 1502 /* 1503 * Get the allowed maximum number of TXDs of every tx queues. 1504 * *pid return the port id which has minimal value of 1505 * min_txd in every tx queue. 1506 */ 1507 static uint16_t 1508 get_allowed_min_nb_txd(portid_t *pid) 1509 { 1510 uint16_t allowed_min_txd = 0; 1511 portid_t pi; 1512 struct rte_eth_dev_info dev_info; 1513 1514 RTE_ETH_FOREACH_DEV(pi) { 1515 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1516 continue; 1517 1518 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1519 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1520 *pid = pi; 1521 } 1522 } 1523 1524 return allowed_min_txd; 1525 } 1526 1527 /* 1528 * Check input txd is valid or not. 1529 * If input txd is not greater than any of maximum number 1530 * of TXDs of every Rx queues, it is valid. 1531 * if valid, return 0, else return -1 1532 */ 1533 int 1534 check_nb_txd(queueid_t txd) 1535 { 1536 uint16_t allowed_max_txd; 1537 uint16_t allowed_min_txd; 1538 portid_t pid = 0; 1539 1540 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1541 if (txd > allowed_max_txd) { 1542 fprintf(stderr, 1543 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1544 txd, allowed_max_txd, pid); 1545 return -1; 1546 } 1547 1548 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1549 if (txd < allowed_min_txd) { 1550 fprintf(stderr, 1551 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1552 txd, allowed_min_txd, pid); 1553 return -1; 1554 } 1555 return 0; 1556 } 1557 1558 1559 /* 1560 * Get the allowed maximum number of hairpin queues. 1561 * *pid return the port id which has minimal value of 1562 * max_hairpin_queues in all ports. 1563 */ 1564 queueid_t 1565 get_allowed_max_nb_hairpinq(portid_t *pid) 1566 { 1567 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1568 portid_t pi; 1569 struct rte_eth_hairpin_cap cap; 1570 1571 RTE_ETH_FOREACH_DEV(pi) { 1572 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1573 *pid = pi; 1574 return 0; 1575 } 1576 if (cap.max_nb_queues < allowed_max_hairpinq) { 1577 allowed_max_hairpinq = cap.max_nb_queues; 1578 *pid = pi; 1579 } 1580 } 1581 return allowed_max_hairpinq; 1582 } 1583 1584 /* 1585 * Check input hairpin is valid or not. 1586 * If input hairpin is not greater than any of maximum number 1587 * of hairpin queues of all ports, it is valid. 1588 * if valid, return 0, else return -1 1589 */ 1590 int 1591 check_nb_hairpinq(queueid_t hairpinq) 1592 { 1593 queueid_t allowed_max_hairpinq; 1594 portid_t pid = 0; 1595 1596 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1597 if (hairpinq > allowed_max_hairpinq) { 1598 fprintf(stderr, 1599 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1600 hairpinq, allowed_max_hairpinq, pid); 1601 return -1; 1602 } 1603 return 0; 1604 } 1605 1606 static int 1607 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1608 { 1609 uint32_t eth_overhead; 1610 1611 if (dev_info->max_mtu != UINT16_MAX && 1612 dev_info->max_rx_pktlen > dev_info->max_mtu) 1613 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1614 else 1615 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1616 1617 return eth_overhead; 1618 } 1619 1620 static void 1621 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1622 { 1623 struct rte_port *port = &ports[pid]; 1624 int ret; 1625 int i; 1626 1627 eth_rx_metadata_negotiate_mp(pid); 1628 1629 port->dev_conf.txmode = tx_mode; 1630 port->dev_conf.rxmode = rx_mode; 1631 1632 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1633 if (ret != 0) 1634 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1635 1636 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1637 port->dev_conf.txmode.offloads &= 1638 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1639 1640 /* Apply Rx offloads configuration */ 1641 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1642 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1643 /* Apply Tx offloads configuration */ 1644 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1645 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1646 1647 if (eth_link_speed) 1648 port->dev_conf.link_speeds = eth_link_speed; 1649 1650 if (max_rx_pkt_len) 1651 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1652 get_eth_overhead(&port->dev_info); 1653 1654 /* set flag to initialize port/queue */ 1655 port->need_reconfig = 1; 1656 port->need_reconfig_queues = 1; 1657 port->socket_id = socket_id; 1658 port->tx_metadata = 0; 1659 1660 /* 1661 * Check for maximum number of segments per MTU. 1662 * Accordingly update the mbuf data size. 1663 */ 1664 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1665 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1666 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1667 uint16_t mtu; 1668 1669 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1670 uint16_t data_size = (mtu + eth_overhead) / 1671 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1672 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1673 1674 if (buffer_size > mbuf_data_size[0]) { 1675 mbuf_data_size[0] = buffer_size; 1676 TESTPMD_LOG(WARNING, 1677 "Configured mbuf size of the first segment %hu\n", 1678 mbuf_data_size[0]); 1679 } 1680 } 1681 } 1682 } 1683 1684 static void 1685 init_config(void) 1686 { 1687 portid_t pid; 1688 struct rte_mempool *mbp; 1689 unsigned int nb_mbuf_per_pool; 1690 lcoreid_t lc_id; 1691 #ifdef RTE_LIB_GRO 1692 struct rte_gro_param gro_param; 1693 #endif 1694 #ifdef RTE_LIB_GSO 1695 uint32_t gso_types; 1696 #endif 1697 1698 /* Configuration of logical cores. */ 1699 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1700 sizeof(struct fwd_lcore *) * nb_lcores, 1701 RTE_CACHE_LINE_SIZE); 1702 if (fwd_lcores == NULL) { 1703 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1704 "failed\n", nb_lcores); 1705 } 1706 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1707 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1708 sizeof(struct fwd_lcore), 1709 RTE_CACHE_LINE_SIZE); 1710 if (fwd_lcores[lc_id] == NULL) { 1711 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1712 "failed\n"); 1713 } 1714 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1715 } 1716 1717 RTE_ETH_FOREACH_DEV(pid) { 1718 uint32_t socket_id; 1719 1720 if (numa_support) { 1721 socket_id = port_numa[pid]; 1722 if (port_numa[pid] == NUMA_NO_CONFIG) { 1723 socket_id = rte_eth_dev_socket_id(pid); 1724 1725 /* 1726 * if socket_id is invalid, 1727 * set to the first available socket. 1728 */ 1729 if (check_socket_id(socket_id) < 0) 1730 socket_id = socket_ids[0]; 1731 } 1732 } else { 1733 socket_id = (socket_num == UMA_NO_CONFIG) ? 1734 0 : socket_num; 1735 } 1736 /* Apply default TxRx configuration for all ports */ 1737 init_config_port_offloads(pid, socket_id); 1738 } 1739 /* 1740 * Create pools of mbuf. 1741 * If NUMA support is disabled, create a single pool of mbuf in 1742 * socket 0 memory by default. 1743 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1744 * 1745 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1746 * nb_txd can be configured at run time. 1747 */ 1748 if (param_total_num_mbufs) 1749 nb_mbuf_per_pool = param_total_num_mbufs; 1750 else { 1751 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1752 (nb_lcores * mb_mempool_cache) + 1753 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1754 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1755 } 1756 1757 if (numa_support) { 1758 uint8_t i, j; 1759 1760 for (i = 0; i < num_sockets; i++) 1761 for (j = 0; j < mbuf_data_size_n; j++) 1762 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1763 mbuf_pool_create(mbuf_data_size[j], 1764 nb_mbuf_per_pool, 1765 socket_ids[i], j); 1766 } else { 1767 uint8_t i; 1768 1769 for (i = 0; i < mbuf_data_size_n; i++) 1770 mempools[i] = mbuf_pool_create 1771 (mbuf_data_size[i], 1772 nb_mbuf_per_pool, 1773 socket_num == UMA_NO_CONFIG ? 1774 0 : socket_num, i); 1775 } 1776 1777 init_port_config(); 1778 1779 #ifdef RTE_LIB_GSO 1780 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1781 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1782 #endif 1783 /* 1784 * Records which Mbuf pool to use by each logical core, if needed. 1785 */ 1786 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1787 mbp = mbuf_pool_find( 1788 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1789 1790 if (mbp == NULL) 1791 mbp = mbuf_pool_find(0, 0); 1792 fwd_lcores[lc_id]->mbp = mbp; 1793 #ifdef RTE_LIB_GSO 1794 /* initialize GSO context */ 1795 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1796 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1797 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1798 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1799 RTE_ETHER_CRC_LEN; 1800 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1801 #endif 1802 } 1803 1804 fwd_config_setup(); 1805 1806 #ifdef RTE_LIB_GRO 1807 /* create a gro context for each lcore */ 1808 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1809 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1810 gro_param.max_item_per_flow = MAX_PKT_BURST; 1811 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1812 gro_param.socket_id = rte_lcore_to_socket_id( 1813 fwd_lcores_cpuids[lc_id]); 1814 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1815 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1816 rte_exit(EXIT_FAILURE, 1817 "rte_gro_ctx_create() failed\n"); 1818 } 1819 } 1820 #endif 1821 } 1822 1823 1824 void 1825 reconfig(portid_t new_port_id, unsigned socket_id) 1826 { 1827 /* Reconfiguration of Ethernet ports. */ 1828 init_config_port_offloads(new_port_id, socket_id); 1829 init_port_config(); 1830 } 1831 1832 int 1833 init_fwd_streams(void) 1834 { 1835 portid_t pid; 1836 struct rte_port *port; 1837 streamid_t sm_id, nb_fwd_streams_new; 1838 queueid_t q; 1839 1840 /* set socket id according to numa or not */ 1841 RTE_ETH_FOREACH_DEV(pid) { 1842 port = &ports[pid]; 1843 if (nb_rxq > port->dev_info.max_rx_queues) { 1844 fprintf(stderr, 1845 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1846 nb_rxq, port->dev_info.max_rx_queues); 1847 return -1; 1848 } 1849 if (nb_txq > port->dev_info.max_tx_queues) { 1850 fprintf(stderr, 1851 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1852 nb_txq, port->dev_info.max_tx_queues); 1853 return -1; 1854 } 1855 if (numa_support) { 1856 if (port_numa[pid] != NUMA_NO_CONFIG) 1857 port->socket_id = port_numa[pid]; 1858 else { 1859 port->socket_id = rte_eth_dev_socket_id(pid); 1860 1861 /* 1862 * if socket_id is invalid, 1863 * set to the first available socket. 1864 */ 1865 if (check_socket_id(port->socket_id) < 0) 1866 port->socket_id = socket_ids[0]; 1867 } 1868 } 1869 else { 1870 if (socket_num == UMA_NO_CONFIG) 1871 port->socket_id = 0; 1872 else 1873 port->socket_id = socket_num; 1874 } 1875 } 1876 1877 q = RTE_MAX(nb_rxq, nb_txq); 1878 if (q == 0) { 1879 fprintf(stderr, 1880 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1881 return -1; 1882 } 1883 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1884 if (nb_fwd_streams_new == nb_fwd_streams) 1885 return 0; 1886 /* clear the old */ 1887 if (fwd_streams != NULL) { 1888 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1889 if (fwd_streams[sm_id] == NULL) 1890 continue; 1891 rte_free(fwd_streams[sm_id]); 1892 fwd_streams[sm_id] = NULL; 1893 } 1894 rte_free(fwd_streams); 1895 fwd_streams = NULL; 1896 } 1897 1898 /* init new */ 1899 nb_fwd_streams = nb_fwd_streams_new; 1900 if (nb_fwd_streams) { 1901 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1902 sizeof(struct fwd_stream *) * nb_fwd_streams, 1903 RTE_CACHE_LINE_SIZE); 1904 if (fwd_streams == NULL) 1905 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1906 " (struct fwd_stream *)) failed\n", 1907 nb_fwd_streams); 1908 1909 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1910 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1911 " struct fwd_stream", sizeof(struct fwd_stream), 1912 RTE_CACHE_LINE_SIZE); 1913 if (fwd_streams[sm_id] == NULL) 1914 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1915 "(struct fwd_stream) failed\n"); 1916 } 1917 } 1918 1919 return 0; 1920 } 1921 1922 static void 1923 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1924 { 1925 uint64_t total_burst, sburst; 1926 uint64_t nb_burst; 1927 uint64_t burst_stats[4]; 1928 uint16_t pktnb_stats[4]; 1929 uint16_t nb_pkt; 1930 int burst_percent[4], sburstp; 1931 int i; 1932 1933 /* 1934 * First compute the total number of packet bursts and the 1935 * two highest numbers of bursts of the same number of packets. 1936 */ 1937 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1938 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1939 1940 /* Show stats for 0 burst size always */ 1941 total_burst = pbs->pkt_burst_spread[0]; 1942 burst_stats[0] = pbs->pkt_burst_spread[0]; 1943 pktnb_stats[0] = 0; 1944 1945 /* Find the next 2 burst sizes with highest occurrences. */ 1946 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1947 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1948 1949 if (nb_burst == 0) 1950 continue; 1951 1952 total_burst += nb_burst; 1953 1954 if (nb_burst > burst_stats[1]) { 1955 burst_stats[2] = burst_stats[1]; 1956 pktnb_stats[2] = pktnb_stats[1]; 1957 burst_stats[1] = nb_burst; 1958 pktnb_stats[1] = nb_pkt; 1959 } else if (nb_burst > burst_stats[2]) { 1960 burst_stats[2] = nb_burst; 1961 pktnb_stats[2] = nb_pkt; 1962 } 1963 } 1964 if (total_burst == 0) 1965 return; 1966 1967 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1968 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1969 if (i == 3) { 1970 printf("%d%% of other]\n", 100 - sburstp); 1971 return; 1972 } 1973 1974 sburst += burst_stats[i]; 1975 if (sburst == total_burst) { 1976 printf("%d%% of %d pkts]\n", 1977 100 - sburstp, (int) pktnb_stats[i]); 1978 return; 1979 } 1980 1981 burst_percent[i] = 1982 (double)burst_stats[i] / total_burst * 100; 1983 printf("%d%% of %d pkts + ", 1984 burst_percent[i], (int) pktnb_stats[i]); 1985 sburstp += burst_percent[i]; 1986 } 1987 } 1988 1989 static void 1990 fwd_stream_stats_display(streamid_t stream_id) 1991 { 1992 struct fwd_stream *fs; 1993 static const char *fwd_top_stats_border = "-------"; 1994 1995 fs = fwd_streams[stream_id]; 1996 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1997 (fs->fwd_dropped == 0)) 1998 return; 1999 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 2000 "TX Port=%2d/Queue=%2d %s\n", 2001 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 2002 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 2003 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 2004 " TX-dropped: %-14"PRIu64, 2005 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 2006 2007 /* if checksum mode */ 2008 if (cur_fwd_eng == &csum_fwd_engine) { 2009 printf(" RX- bad IP checksum: %-14"PRIu64 2010 " Rx- bad L4 checksum: %-14"PRIu64 2011 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 2012 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 2013 fs->rx_bad_outer_l4_csum); 2014 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2015 fs->rx_bad_outer_ip_csum); 2016 } else { 2017 printf("\n"); 2018 } 2019 2020 if (record_burst_stats) { 2021 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2022 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2023 } 2024 } 2025 2026 void 2027 fwd_stats_display(void) 2028 { 2029 static const char *fwd_stats_border = "----------------------"; 2030 static const char *acc_stats_border = "+++++++++++++++"; 2031 struct { 2032 struct fwd_stream *rx_stream; 2033 struct fwd_stream *tx_stream; 2034 uint64_t tx_dropped; 2035 uint64_t rx_bad_ip_csum; 2036 uint64_t rx_bad_l4_csum; 2037 uint64_t rx_bad_outer_l4_csum; 2038 uint64_t rx_bad_outer_ip_csum; 2039 } ports_stats[RTE_MAX_ETHPORTS]; 2040 uint64_t total_rx_dropped = 0; 2041 uint64_t total_tx_dropped = 0; 2042 uint64_t total_rx_nombuf = 0; 2043 struct rte_eth_stats stats; 2044 uint64_t fwd_cycles = 0; 2045 uint64_t total_recv = 0; 2046 uint64_t total_xmit = 0; 2047 struct rte_port *port; 2048 streamid_t sm_id; 2049 portid_t pt_id; 2050 int ret; 2051 int i; 2052 2053 memset(ports_stats, 0, sizeof(ports_stats)); 2054 2055 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2056 struct fwd_stream *fs = fwd_streams[sm_id]; 2057 2058 if (cur_fwd_config.nb_fwd_streams > 2059 cur_fwd_config.nb_fwd_ports) { 2060 fwd_stream_stats_display(sm_id); 2061 } else { 2062 ports_stats[fs->tx_port].tx_stream = fs; 2063 ports_stats[fs->rx_port].rx_stream = fs; 2064 } 2065 2066 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2067 2068 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2069 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2070 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2071 fs->rx_bad_outer_l4_csum; 2072 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2073 fs->rx_bad_outer_ip_csum; 2074 2075 if (record_core_cycles) 2076 fwd_cycles += fs->core_cycles; 2077 } 2078 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2079 pt_id = fwd_ports_ids[i]; 2080 port = &ports[pt_id]; 2081 2082 ret = rte_eth_stats_get(pt_id, &stats); 2083 if (ret != 0) { 2084 fprintf(stderr, 2085 "%s: Error: failed to get stats (port %u): %d", 2086 __func__, pt_id, ret); 2087 continue; 2088 } 2089 stats.ipackets -= port->stats.ipackets; 2090 stats.opackets -= port->stats.opackets; 2091 stats.ibytes -= port->stats.ibytes; 2092 stats.obytes -= port->stats.obytes; 2093 stats.imissed -= port->stats.imissed; 2094 stats.oerrors -= port->stats.oerrors; 2095 stats.rx_nombuf -= port->stats.rx_nombuf; 2096 2097 total_recv += stats.ipackets; 2098 total_xmit += stats.opackets; 2099 total_rx_dropped += stats.imissed; 2100 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2101 total_tx_dropped += stats.oerrors; 2102 total_rx_nombuf += stats.rx_nombuf; 2103 2104 printf("\n %s Forward statistics for port %-2d %s\n", 2105 fwd_stats_border, pt_id, fwd_stats_border); 2106 2107 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2108 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2109 stats.ipackets + stats.imissed); 2110 2111 if (cur_fwd_eng == &csum_fwd_engine) { 2112 printf(" Bad-ipcsum: %-14"PRIu64 2113 " Bad-l4csum: %-14"PRIu64 2114 "Bad-outer-l4csum: %-14"PRIu64"\n", 2115 ports_stats[pt_id].rx_bad_ip_csum, 2116 ports_stats[pt_id].rx_bad_l4_csum, 2117 ports_stats[pt_id].rx_bad_outer_l4_csum); 2118 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2119 ports_stats[pt_id].rx_bad_outer_ip_csum); 2120 } 2121 if (stats.ierrors + stats.rx_nombuf > 0) { 2122 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2123 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2124 } 2125 2126 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2127 "TX-total: %-"PRIu64"\n", 2128 stats.opackets, ports_stats[pt_id].tx_dropped, 2129 stats.opackets + ports_stats[pt_id].tx_dropped); 2130 2131 if (record_burst_stats) { 2132 if (ports_stats[pt_id].rx_stream) 2133 pkt_burst_stats_display("RX", 2134 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2135 if (ports_stats[pt_id].tx_stream) 2136 pkt_burst_stats_display("TX", 2137 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2138 } 2139 2140 printf(" %s--------------------------------%s\n", 2141 fwd_stats_border, fwd_stats_border); 2142 } 2143 2144 printf("\n %s Accumulated forward statistics for all ports" 2145 "%s\n", 2146 acc_stats_border, acc_stats_border); 2147 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2148 "%-"PRIu64"\n" 2149 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2150 "%-"PRIu64"\n", 2151 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2152 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2153 if (total_rx_nombuf > 0) 2154 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2155 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2156 "%s\n", 2157 acc_stats_border, acc_stats_border); 2158 if (record_core_cycles) { 2159 #define CYC_PER_MHZ 1E6 2160 if (total_recv > 0 || total_xmit > 0) { 2161 uint64_t total_pkts = 0; 2162 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2163 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2164 total_pkts = total_xmit; 2165 else 2166 total_pkts = total_recv; 2167 2168 printf("\n CPU cycles/packet=%.2F (total cycles=" 2169 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2170 " MHz Clock\n", 2171 (double) fwd_cycles / total_pkts, 2172 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2173 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2174 } 2175 } 2176 } 2177 2178 void 2179 fwd_stats_reset(void) 2180 { 2181 streamid_t sm_id; 2182 portid_t pt_id; 2183 int ret; 2184 int i; 2185 2186 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2187 pt_id = fwd_ports_ids[i]; 2188 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2189 if (ret != 0) 2190 fprintf(stderr, 2191 "%s: Error: failed to clear stats (port %u):%d", 2192 __func__, pt_id, ret); 2193 } 2194 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2195 struct fwd_stream *fs = fwd_streams[sm_id]; 2196 2197 fs->rx_packets = 0; 2198 fs->tx_packets = 0; 2199 fs->fwd_dropped = 0; 2200 fs->rx_bad_ip_csum = 0; 2201 fs->rx_bad_l4_csum = 0; 2202 fs->rx_bad_outer_l4_csum = 0; 2203 fs->rx_bad_outer_ip_csum = 0; 2204 2205 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2206 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2207 fs->core_cycles = 0; 2208 } 2209 } 2210 2211 static void 2212 flush_fwd_rx_queues(void) 2213 { 2214 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2215 portid_t rxp; 2216 portid_t port_id; 2217 queueid_t rxq; 2218 uint16_t nb_rx; 2219 uint16_t i; 2220 uint8_t j; 2221 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2222 uint64_t timer_period; 2223 2224 if (num_procs > 1) { 2225 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2226 return; 2227 } 2228 2229 /* convert to number of cycles */ 2230 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2231 2232 for (j = 0; j < 2; j++) { 2233 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2234 for (rxq = 0; rxq < nb_rxq; rxq++) { 2235 port_id = fwd_ports_ids[rxp]; 2236 2237 /* Polling stopped queues is prohibited. */ 2238 if (ports[port_id].rxq[rxq].state == 2239 RTE_ETH_QUEUE_STATE_STOPPED) 2240 continue; 2241 2242 /** 2243 * testpmd can stuck in the below do while loop 2244 * if rte_eth_rx_burst() always returns nonzero 2245 * packets. So timer is added to exit this loop 2246 * after 1sec timer expiry. 2247 */ 2248 prev_tsc = rte_rdtsc(); 2249 do { 2250 nb_rx = rte_eth_rx_burst(port_id, rxq, 2251 pkts_burst, MAX_PKT_BURST); 2252 for (i = 0; i < nb_rx; i++) 2253 rte_pktmbuf_free(pkts_burst[i]); 2254 2255 cur_tsc = rte_rdtsc(); 2256 diff_tsc = cur_tsc - prev_tsc; 2257 timer_tsc += diff_tsc; 2258 } while ((nb_rx > 0) && 2259 (timer_tsc < timer_period)); 2260 timer_tsc = 0; 2261 } 2262 } 2263 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2264 } 2265 } 2266 2267 static void 2268 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2269 { 2270 struct fwd_stream **fsm; 2271 streamid_t nb_fs; 2272 streamid_t sm_id; 2273 #ifdef RTE_LIB_BITRATESTATS 2274 uint64_t tics_per_1sec; 2275 uint64_t tics_datum; 2276 uint64_t tics_current; 2277 uint16_t i, cnt_ports; 2278 2279 cnt_ports = nb_ports; 2280 tics_datum = rte_rdtsc(); 2281 tics_per_1sec = rte_get_timer_hz(); 2282 #endif 2283 fsm = &fwd_streams[fc->stream_idx]; 2284 nb_fs = fc->stream_nb; 2285 do { 2286 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2287 if (!fsm[sm_id]->disabled) 2288 (*pkt_fwd)(fsm[sm_id]); 2289 #ifdef RTE_LIB_BITRATESTATS 2290 if (bitrate_enabled != 0 && 2291 bitrate_lcore_id == rte_lcore_id()) { 2292 tics_current = rte_rdtsc(); 2293 if (tics_current - tics_datum >= tics_per_1sec) { 2294 /* Periodic bitrate calculation */ 2295 for (i = 0; i < cnt_ports; i++) 2296 rte_stats_bitrate_calc(bitrate_data, 2297 ports_ids[i]); 2298 tics_datum = tics_current; 2299 } 2300 } 2301 #endif 2302 #ifdef RTE_LIB_LATENCYSTATS 2303 if (latencystats_enabled != 0 && 2304 latencystats_lcore_id == rte_lcore_id()) 2305 rte_latencystats_update(); 2306 #endif 2307 2308 } while (! fc->stopped); 2309 } 2310 2311 static int 2312 start_pkt_forward_on_core(void *fwd_arg) 2313 { 2314 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2315 cur_fwd_config.fwd_eng->packet_fwd); 2316 return 0; 2317 } 2318 2319 /* 2320 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2321 * Used to start communication flows in network loopback test configurations. 2322 */ 2323 static int 2324 run_one_txonly_burst_on_core(void *fwd_arg) 2325 { 2326 struct fwd_lcore *fwd_lc; 2327 struct fwd_lcore tmp_lcore; 2328 2329 fwd_lc = (struct fwd_lcore *) fwd_arg; 2330 tmp_lcore = *fwd_lc; 2331 tmp_lcore.stopped = 1; 2332 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2333 return 0; 2334 } 2335 2336 /* 2337 * Launch packet forwarding: 2338 * - Setup per-port forwarding context. 2339 * - launch logical cores with their forwarding configuration. 2340 */ 2341 static void 2342 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2343 { 2344 unsigned int i; 2345 unsigned int lc_id; 2346 int diag; 2347 2348 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2349 lc_id = fwd_lcores_cpuids[i]; 2350 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2351 fwd_lcores[i]->stopped = 0; 2352 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2353 fwd_lcores[i], lc_id); 2354 if (diag != 0) 2355 fprintf(stderr, 2356 "launch lcore %u failed - diag=%d\n", 2357 lc_id, diag); 2358 } 2359 } 2360 } 2361 2362 /* 2363 * Launch packet forwarding configuration. 2364 */ 2365 void 2366 start_packet_forwarding(int with_tx_first) 2367 { 2368 port_fwd_begin_t port_fwd_begin; 2369 port_fwd_end_t port_fwd_end; 2370 stream_init_t stream_init = cur_fwd_eng->stream_init; 2371 unsigned int i; 2372 2373 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2374 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2375 2376 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2377 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2378 2379 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2380 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2381 (!nb_rxq || !nb_txq)) 2382 rte_exit(EXIT_FAILURE, 2383 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2384 cur_fwd_eng->fwd_mode_name); 2385 2386 if (all_ports_started() == 0) { 2387 fprintf(stderr, "Not all ports were started\n"); 2388 return; 2389 } 2390 if (test_done == 0) { 2391 fprintf(stderr, "Packet forwarding already started\n"); 2392 return; 2393 } 2394 2395 fwd_config_setup(); 2396 2397 pkt_fwd_config_display(&cur_fwd_config); 2398 if (!pkt_fwd_shared_rxq_check()) 2399 return; 2400 2401 if (stream_init != NULL) 2402 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2403 stream_init(fwd_streams[i]); 2404 2405 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2406 if (port_fwd_begin != NULL) { 2407 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2408 if (port_fwd_begin(fwd_ports_ids[i])) { 2409 fprintf(stderr, 2410 "Packet forwarding is not ready\n"); 2411 return; 2412 } 2413 } 2414 } 2415 2416 if (with_tx_first) { 2417 port_fwd_begin = tx_only_engine.port_fwd_begin; 2418 if (port_fwd_begin != NULL) { 2419 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2420 if (port_fwd_begin(fwd_ports_ids[i])) { 2421 fprintf(stderr, 2422 "Packet forwarding is not ready\n"); 2423 return; 2424 } 2425 } 2426 } 2427 } 2428 2429 test_done = 0; 2430 2431 if(!no_flush_rx) 2432 flush_fwd_rx_queues(); 2433 2434 rxtx_config_display(); 2435 2436 fwd_stats_reset(); 2437 if (with_tx_first) { 2438 while (with_tx_first--) { 2439 launch_packet_forwarding( 2440 run_one_txonly_burst_on_core); 2441 rte_eal_mp_wait_lcore(); 2442 } 2443 port_fwd_end = tx_only_engine.port_fwd_end; 2444 if (port_fwd_end != NULL) { 2445 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2446 (*port_fwd_end)(fwd_ports_ids[i]); 2447 } 2448 } 2449 launch_packet_forwarding(start_pkt_forward_on_core); 2450 } 2451 2452 void 2453 stop_packet_forwarding(void) 2454 { 2455 port_fwd_end_t port_fwd_end; 2456 lcoreid_t lc_id; 2457 portid_t pt_id; 2458 int i; 2459 2460 if (test_done) { 2461 fprintf(stderr, "Packet forwarding not started\n"); 2462 return; 2463 } 2464 printf("Telling cores to stop..."); 2465 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2466 fwd_lcores[lc_id]->stopped = 1; 2467 printf("\nWaiting for lcores to finish...\n"); 2468 rte_eal_mp_wait_lcore(); 2469 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2470 if (port_fwd_end != NULL) { 2471 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2472 pt_id = fwd_ports_ids[i]; 2473 (*port_fwd_end)(pt_id); 2474 } 2475 } 2476 2477 fwd_stats_display(); 2478 2479 printf("\nDone.\n"); 2480 test_done = 1; 2481 } 2482 2483 void 2484 dev_set_link_up(portid_t pid) 2485 { 2486 if (rte_eth_dev_set_link_up(pid) < 0) 2487 fprintf(stderr, "\nSet link up fail.\n"); 2488 } 2489 2490 void 2491 dev_set_link_down(portid_t pid) 2492 { 2493 if (rte_eth_dev_set_link_down(pid) < 0) 2494 fprintf(stderr, "\nSet link down fail.\n"); 2495 } 2496 2497 static int 2498 all_ports_started(void) 2499 { 2500 portid_t pi; 2501 struct rte_port *port; 2502 2503 RTE_ETH_FOREACH_DEV(pi) { 2504 port = &ports[pi]; 2505 /* Check if there is a port which is not started */ 2506 if ((port->port_status != RTE_PORT_STARTED) && 2507 (port->slave_flag == 0)) 2508 return 0; 2509 } 2510 2511 /* No port is not started */ 2512 return 1; 2513 } 2514 2515 int 2516 port_is_stopped(portid_t port_id) 2517 { 2518 struct rte_port *port = &ports[port_id]; 2519 2520 if ((port->port_status != RTE_PORT_STOPPED) && 2521 (port->slave_flag == 0)) 2522 return 0; 2523 return 1; 2524 } 2525 2526 int 2527 all_ports_stopped(void) 2528 { 2529 portid_t pi; 2530 2531 RTE_ETH_FOREACH_DEV(pi) { 2532 if (!port_is_stopped(pi)) 2533 return 0; 2534 } 2535 2536 return 1; 2537 } 2538 2539 int 2540 port_is_started(portid_t port_id) 2541 { 2542 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2543 return 0; 2544 2545 if (ports[port_id].port_status != RTE_PORT_STARTED) 2546 return 0; 2547 2548 return 1; 2549 } 2550 2551 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2552 static int 2553 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2554 { 2555 queueid_t qi; 2556 struct rte_eth_hairpin_conf hairpin_conf = { 2557 .peer_count = 1, 2558 }; 2559 int i; 2560 int diag; 2561 struct rte_port *port = &ports[pi]; 2562 uint16_t peer_rx_port = pi; 2563 uint16_t peer_tx_port = pi; 2564 uint32_t manual = 1; 2565 uint32_t tx_exp = hairpin_mode & 0x10; 2566 2567 if (!(hairpin_mode & 0xf)) { 2568 peer_rx_port = pi; 2569 peer_tx_port = pi; 2570 manual = 0; 2571 } else if (hairpin_mode & 0x1) { 2572 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2573 RTE_ETH_DEV_NO_OWNER); 2574 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2575 peer_tx_port = rte_eth_find_next_owned_by(0, 2576 RTE_ETH_DEV_NO_OWNER); 2577 if (p_pi != RTE_MAX_ETHPORTS) { 2578 peer_rx_port = p_pi; 2579 } else { 2580 uint16_t next_pi; 2581 2582 /* Last port will be the peer RX port of the first. */ 2583 RTE_ETH_FOREACH_DEV(next_pi) 2584 peer_rx_port = next_pi; 2585 } 2586 manual = 1; 2587 } else if (hairpin_mode & 0x2) { 2588 if (cnt_pi & 0x1) { 2589 peer_rx_port = p_pi; 2590 } else { 2591 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2592 RTE_ETH_DEV_NO_OWNER); 2593 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2594 peer_rx_port = pi; 2595 } 2596 peer_tx_port = peer_rx_port; 2597 manual = 1; 2598 } 2599 2600 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2601 hairpin_conf.peers[0].port = peer_rx_port; 2602 hairpin_conf.peers[0].queue = i + nb_rxq; 2603 hairpin_conf.manual_bind = !!manual; 2604 hairpin_conf.tx_explicit = !!tx_exp; 2605 diag = rte_eth_tx_hairpin_queue_setup 2606 (pi, qi, nb_txd, &hairpin_conf); 2607 i++; 2608 if (diag == 0) 2609 continue; 2610 2611 /* Fail to setup rx queue, return */ 2612 if (port->port_status == RTE_PORT_HANDLING) 2613 port->port_status = RTE_PORT_STOPPED; 2614 else 2615 fprintf(stderr, 2616 "Port %d can not be set back to stopped\n", pi); 2617 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2618 pi); 2619 /* try to reconfigure queues next time */ 2620 port->need_reconfig_queues = 1; 2621 return -1; 2622 } 2623 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2624 hairpin_conf.peers[0].port = peer_tx_port; 2625 hairpin_conf.peers[0].queue = i + nb_txq; 2626 hairpin_conf.manual_bind = !!manual; 2627 hairpin_conf.tx_explicit = !!tx_exp; 2628 diag = rte_eth_rx_hairpin_queue_setup 2629 (pi, qi, nb_rxd, &hairpin_conf); 2630 i++; 2631 if (diag == 0) 2632 continue; 2633 2634 /* Fail to setup rx queue, return */ 2635 if (port->port_status == RTE_PORT_HANDLING) 2636 port->port_status = RTE_PORT_STOPPED; 2637 else 2638 fprintf(stderr, 2639 "Port %d can not be set back to stopped\n", pi); 2640 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2641 pi); 2642 /* try to reconfigure queues next time */ 2643 port->need_reconfig_queues = 1; 2644 return -1; 2645 } 2646 return 0; 2647 } 2648 2649 /* Configure the Rx with optional split. */ 2650 int 2651 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2652 uint16_t nb_rx_desc, unsigned int socket_id, 2653 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2654 { 2655 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2656 unsigned int i, mp_n; 2657 int ret; 2658 2659 if (rx_pkt_nb_segs <= 1 || 2660 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2661 rx_conf->rx_seg = NULL; 2662 rx_conf->rx_nseg = 0; 2663 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2664 nb_rx_desc, socket_id, 2665 rx_conf, mp); 2666 goto exit; 2667 } 2668 for (i = 0; i < rx_pkt_nb_segs; i++) { 2669 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2670 struct rte_mempool *mpx; 2671 /* 2672 * Use last valid pool for the segments with number 2673 * exceeding the pool index. 2674 */ 2675 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2676 mpx = mbuf_pool_find(socket_id, mp_n); 2677 /* Handle zero as mbuf data buffer size. */ 2678 rx_seg->length = rx_pkt_seg_lengths[i] ? 2679 rx_pkt_seg_lengths[i] : 2680 mbuf_data_size[mp_n]; 2681 rx_seg->offset = i < rx_pkt_nb_offs ? 2682 rx_pkt_seg_offsets[i] : 0; 2683 rx_seg->mp = mpx ? mpx : mp; 2684 } 2685 rx_conf->rx_nseg = rx_pkt_nb_segs; 2686 rx_conf->rx_seg = rx_useg; 2687 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2688 socket_id, rx_conf, NULL); 2689 rx_conf->rx_seg = NULL; 2690 rx_conf->rx_nseg = 0; 2691 exit: 2692 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2693 RTE_ETH_QUEUE_STATE_STOPPED : 2694 RTE_ETH_QUEUE_STATE_STARTED; 2695 return ret; 2696 } 2697 2698 static int 2699 alloc_xstats_display_info(portid_t pi) 2700 { 2701 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2702 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2703 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2704 2705 if (xstats_display_num == 0) 2706 return 0; 2707 2708 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2709 if (*ids_supp == NULL) 2710 goto fail_ids_supp; 2711 2712 *prev_values = calloc(xstats_display_num, 2713 sizeof(**prev_values)); 2714 if (*prev_values == NULL) 2715 goto fail_prev_values; 2716 2717 *curr_values = calloc(xstats_display_num, 2718 sizeof(**curr_values)); 2719 if (*curr_values == NULL) 2720 goto fail_curr_values; 2721 2722 ports[pi].xstats_info.allocated = true; 2723 2724 return 0; 2725 2726 fail_curr_values: 2727 free(*prev_values); 2728 fail_prev_values: 2729 free(*ids_supp); 2730 fail_ids_supp: 2731 return -ENOMEM; 2732 } 2733 2734 static void 2735 free_xstats_display_info(portid_t pi) 2736 { 2737 if (!ports[pi].xstats_info.allocated) 2738 return; 2739 free(ports[pi].xstats_info.ids_supp); 2740 free(ports[pi].xstats_info.prev_values); 2741 free(ports[pi].xstats_info.curr_values); 2742 ports[pi].xstats_info.allocated = false; 2743 } 2744 2745 /** Fill helper structures for specified port to show extended statistics. */ 2746 static void 2747 fill_xstats_display_info_for_port(portid_t pi) 2748 { 2749 unsigned int stat, stat_supp; 2750 const char *xstat_name; 2751 struct rte_port *port; 2752 uint64_t *ids_supp; 2753 int rc; 2754 2755 if (xstats_display_num == 0) 2756 return; 2757 2758 if (pi == (portid_t)RTE_PORT_ALL) { 2759 fill_xstats_display_info(); 2760 return; 2761 } 2762 2763 port = &ports[pi]; 2764 if (port->port_status != RTE_PORT_STARTED) 2765 return; 2766 2767 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2768 rte_exit(EXIT_FAILURE, 2769 "Failed to allocate xstats display memory\n"); 2770 2771 ids_supp = port->xstats_info.ids_supp; 2772 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2773 xstat_name = xstats_display[stat].name; 2774 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2775 ids_supp + stat_supp); 2776 if (rc != 0) { 2777 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2778 xstat_name, pi, stat); 2779 continue; 2780 } 2781 stat_supp++; 2782 } 2783 2784 port->xstats_info.ids_supp_sz = stat_supp; 2785 } 2786 2787 /** Fill helper structures for all ports to show extended statistics. */ 2788 static void 2789 fill_xstats_display_info(void) 2790 { 2791 portid_t pi; 2792 2793 if (xstats_display_num == 0) 2794 return; 2795 2796 RTE_ETH_FOREACH_DEV(pi) 2797 fill_xstats_display_info_for_port(pi); 2798 } 2799 2800 int 2801 start_port(portid_t pid) 2802 { 2803 int diag, need_check_link_status = -1; 2804 portid_t pi; 2805 portid_t p_pi = RTE_MAX_ETHPORTS; 2806 portid_t pl[RTE_MAX_ETHPORTS]; 2807 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2808 uint16_t cnt_pi = 0; 2809 uint16_t cfg_pi = 0; 2810 int peer_pi; 2811 queueid_t qi; 2812 struct rte_port *port; 2813 struct rte_eth_hairpin_cap cap; 2814 2815 if (port_id_is_invalid(pid, ENABLED_WARN)) 2816 return 0; 2817 2818 RTE_ETH_FOREACH_DEV(pi) { 2819 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2820 continue; 2821 2822 if (port_is_bonding_slave(pi)) { 2823 fprintf(stderr, 2824 "Please remove port %d from bonded device.\n", 2825 pi); 2826 continue; 2827 } 2828 2829 need_check_link_status = 0; 2830 port = &ports[pi]; 2831 if (port->port_status == RTE_PORT_STOPPED) 2832 port->port_status = RTE_PORT_HANDLING; 2833 else { 2834 fprintf(stderr, "Port %d is now not stopped\n", pi); 2835 continue; 2836 } 2837 2838 if (port->need_reconfig > 0) { 2839 struct rte_eth_conf dev_conf; 2840 int k; 2841 2842 port->need_reconfig = 0; 2843 2844 if (flow_isolate_all) { 2845 int ret = port_flow_isolate(pi, 1); 2846 if (ret) { 2847 fprintf(stderr, 2848 "Failed to apply isolated mode on port %d\n", 2849 pi); 2850 return -1; 2851 } 2852 } 2853 configure_rxtx_dump_callbacks(0); 2854 printf("Configuring Port %d (socket %u)\n", pi, 2855 port->socket_id); 2856 if (nb_hairpinq > 0 && 2857 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2858 fprintf(stderr, 2859 "Port %d doesn't support hairpin queues\n", 2860 pi); 2861 return -1; 2862 } 2863 2864 /* configure port */ 2865 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2866 nb_txq + nb_hairpinq, 2867 &(port->dev_conf)); 2868 if (diag != 0) { 2869 if (port->port_status == RTE_PORT_HANDLING) 2870 port->port_status = RTE_PORT_STOPPED; 2871 else 2872 fprintf(stderr, 2873 "Port %d can not be set back to stopped\n", 2874 pi); 2875 fprintf(stderr, "Fail to configure port %d\n", 2876 pi); 2877 /* try to reconfigure port next time */ 2878 port->need_reconfig = 1; 2879 return -1; 2880 } 2881 /* get device configuration*/ 2882 if (0 != 2883 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2884 fprintf(stderr, 2885 "port %d can not get device configuration\n", 2886 pi); 2887 return -1; 2888 } 2889 /* Apply Rx offloads configuration */ 2890 if (dev_conf.rxmode.offloads != 2891 port->dev_conf.rxmode.offloads) { 2892 port->dev_conf.rxmode.offloads |= 2893 dev_conf.rxmode.offloads; 2894 for (k = 0; 2895 k < port->dev_info.max_rx_queues; 2896 k++) 2897 port->rxq[k].conf.offloads |= 2898 dev_conf.rxmode.offloads; 2899 } 2900 /* Apply Tx offloads configuration */ 2901 if (dev_conf.txmode.offloads != 2902 port->dev_conf.txmode.offloads) { 2903 port->dev_conf.txmode.offloads |= 2904 dev_conf.txmode.offloads; 2905 for (k = 0; 2906 k < port->dev_info.max_tx_queues; 2907 k++) 2908 port->txq[k].conf.offloads |= 2909 dev_conf.txmode.offloads; 2910 } 2911 } 2912 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2913 port->need_reconfig_queues = 0; 2914 /* setup tx queues */ 2915 for (qi = 0; qi < nb_txq; qi++) { 2916 struct rte_eth_txconf *conf = 2917 &port->txq[qi].conf; 2918 2919 if ((numa_support) && 2920 (txring_numa[pi] != NUMA_NO_CONFIG)) 2921 diag = rte_eth_tx_queue_setup(pi, qi, 2922 port->nb_tx_desc[qi], 2923 txring_numa[pi], 2924 &(port->txq[qi].conf)); 2925 else 2926 diag = rte_eth_tx_queue_setup(pi, qi, 2927 port->nb_tx_desc[qi], 2928 port->socket_id, 2929 &(port->txq[qi].conf)); 2930 2931 if (diag == 0) { 2932 port->txq[qi].state = 2933 conf->tx_deferred_start ? 2934 RTE_ETH_QUEUE_STATE_STOPPED : 2935 RTE_ETH_QUEUE_STATE_STARTED; 2936 continue; 2937 } 2938 2939 /* Fail to setup tx queue, return */ 2940 if (port->port_status == RTE_PORT_HANDLING) 2941 port->port_status = RTE_PORT_STOPPED; 2942 else 2943 fprintf(stderr, 2944 "Port %d can not be set back to stopped\n", 2945 pi); 2946 fprintf(stderr, 2947 "Fail to configure port %d tx queues\n", 2948 pi); 2949 /* try to reconfigure queues next time */ 2950 port->need_reconfig_queues = 1; 2951 return -1; 2952 } 2953 for (qi = 0; qi < nb_rxq; qi++) { 2954 /* setup rx queues */ 2955 if ((numa_support) && 2956 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2957 struct rte_mempool * mp = 2958 mbuf_pool_find 2959 (rxring_numa[pi], 0); 2960 if (mp == NULL) { 2961 fprintf(stderr, 2962 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2963 rxring_numa[pi]); 2964 return -1; 2965 } 2966 2967 diag = rx_queue_setup(pi, qi, 2968 port->nb_rx_desc[qi], 2969 rxring_numa[pi], 2970 &(port->rxq[qi].conf), 2971 mp); 2972 } else { 2973 struct rte_mempool *mp = 2974 mbuf_pool_find 2975 (port->socket_id, 0); 2976 if (mp == NULL) { 2977 fprintf(stderr, 2978 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2979 port->socket_id); 2980 return -1; 2981 } 2982 diag = rx_queue_setup(pi, qi, 2983 port->nb_rx_desc[qi], 2984 port->socket_id, 2985 &(port->rxq[qi].conf), 2986 mp); 2987 } 2988 if (diag == 0) 2989 continue; 2990 2991 /* Fail to setup rx queue, return */ 2992 if (port->port_status == RTE_PORT_HANDLING) 2993 port->port_status = RTE_PORT_STOPPED; 2994 else 2995 fprintf(stderr, 2996 "Port %d can not be set back to stopped\n", 2997 pi); 2998 fprintf(stderr, 2999 "Fail to configure port %d rx queues\n", 3000 pi); 3001 /* try to reconfigure queues next time */ 3002 port->need_reconfig_queues = 1; 3003 return -1; 3004 } 3005 /* setup hairpin queues */ 3006 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3007 return -1; 3008 } 3009 configure_rxtx_dump_callbacks(verbose_level); 3010 if (clear_ptypes) { 3011 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3012 NULL, 0); 3013 if (diag < 0) 3014 fprintf(stderr, 3015 "Port %d: Failed to disable Ptype parsing\n", 3016 pi); 3017 } 3018 3019 p_pi = pi; 3020 cnt_pi++; 3021 3022 /* start port */ 3023 diag = eth_dev_start_mp(pi); 3024 if (diag < 0) { 3025 fprintf(stderr, "Fail to start port %d: %s\n", 3026 pi, rte_strerror(-diag)); 3027 3028 /* Fail to setup rx queue, return */ 3029 if (port->port_status == RTE_PORT_HANDLING) 3030 port->port_status = RTE_PORT_STOPPED; 3031 else 3032 fprintf(stderr, 3033 "Port %d can not be set back to stopped\n", 3034 pi); 3035 continue; 3036 } 3037 3038 if (port->port_status == RTE_PORT_HANDLING) 3039 port->port_status = RTE_PORT_STARTED; 3040 else 3041 fprintf(stderr, "Port %d can not be set into started\n", 3042 pi); 3043 3044 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3045 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3046 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3047 3048 /* at least one port started, need checking link status */ 3049 need_check_link_status = 1; 3050 3051 pl[cfg_pi++] = pi; 3052 } 3053 3054 if (need_check_link_status == 1 && !no_link_check) 3055 check_all_ports_link_status(RTE_PORT_ALL); 3056 else if (need_check_link_status == 0) 3057 fprintf(stderr, "Please stop the ports first\n"); 3058 3059 if (hairpin_mode & 0xf) { 3060 uint16_t i; 3061 int j; 3062 3063 /* bind all started hairpin ports */ 3064 for (i = 0; i < cfg_pi; i++) { 3065 pi = pl[i]; 3066 /* bind current Tx to all peer Rx */ 3067 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3068 RTE_MAX_ETHPORTS, 1); 3069 if (peer_pi < 0) 3070 return peer_pi; 3071 for (j = 0; j < peer_pi; j++) { 3072 if (!port_is_started(peer_pl[j])) 3073 continue; 3074 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3075 if (diag < 0) { 3076 fprintf(stderr, 3077 "Error during binding hairpin Tx port %u to %u: %s\n", 3078 pi, peer_pl[j], 3079 rte_strerror(-diag)); 3080 return -1; 3081 } 3082 } 3083 /* bind all peer Tx to current Rx */ 3084 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3085 RTE_MAX_ETHPORTS, 0); 3086 if (peer_pi < 0) 3087 return peer_pi; 3088 for (j = 0; j < peer_pi; j++) { 3089 if (!port_is_started(peer_pl[j])) 3090 continue; 3091 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3092 if (diag < 0) { 3093 fprintf(stderr, 3094 "Error during binding hairpin Tx port %u to %u: %s\n", 3095 peer_pl[j], pi, 3096 rte_strerror(-diag)); 3097 return -1; 3098 } 3099 } 3100 } 3101 } 3102 3103 fill_xstats_display_info_for_port(pid); 3104 3105 printf("Done\n"); 3106 return 0; 3107 } 3108 3109 void 3110 stop_port(portid_t pid) 3111 { 3112 portid_t pi; 3113 struct rte_port *port; 3114 int need_check_link_status = 0; 3115 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3116 int peer_pi; 3117 3118 if (port_id_is_invalid(pid, ENABLED_WARN)) 3119 return; 3120 3121 printf("Stopping ports...\n"); 3122 3123 RTE_ETH_FOREACH_DEV(pi) { 3124 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3125 continue; 3126 3127 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3128 fprintf(stderr, 3129 "Please remove port %d from forwarding configuration.\n", 3130 pi); 3131 continue; 3132 } 3133 3134 if (port_is_bonding_slave(pi)) { 3135 fprintf(stderr, 3136 "Please remove port %d from bonded device.\n", 3137 pi); 3138 continue; 3139 } 3140 3141 port = &ports[pi]; 3142 if (port->port_status == RTE_PORT_STARTED) 3143 port->port_status = RTE_PORT_HANDLING; 3144 else 3145 continue; 3146 3147 if (hairpin_mode & 0xf) { 3148 int j; 3149 3150 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3151 /* unbind all peer Tx from current Rx */ 3152 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3153 RTE_MAX_ETHPORTS, 0); 3154 if (peer_pi < 0) 3155 continue; 3156 for (j = 0; j < peer_pi; j++) { 3157 if (!port_is_started(peer_pl[j])) 3158 continue; 3159 rte_eth_hairpin_unbind(peer_pl[j], pi); 3160 } 3161 } 3162 3163 if (port->flow_list) 3164 port_flow_flush(pi); 3165 3166 if (eth_dev_stop_mp(pi) != 0) 3167 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3168 pi); 3169 3170 if (port->port_status == RTE_PORT_HANDLING) 3171 port->port_status = RTE_PORT_STOPPED; 3172 else 3173 fprintf(stderr, "Port %d can not be set into stopped\n", 3174 pi); 3175 need_check_link_status = 1; 3176 } 3177 if (need_check_link_status && !no_link_check) 3178 check_all_ports_link_status(RTE_PORT_ALL); 3179 3180 printf("Done\n"); 3181 } 3182 3183 static void 3184 remove_invalid_ports_in(portid_t *array, portid_t *total) 3185 { 3186 portid_t i; 3187 portid_t new_total = 0; 3188 3189 for (i = 0; i < *total; i++) 3190 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3191 array[new_total] = array[i]; 3192 new_total++; 3193 } 3194 *total = new_total; 3195 } 3196 3197 static void 3198 remove_invalid_ports(void) 3199 { 3200 remove_invalid_ports_in(ports_ids, &nb_ports); 3201 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3202 nb_cfg_ports = nb_fwd_ports; 3203 } 3204 3205 void 3206 close_port(portid_t pid) 3207 { 3208 portid_t pi; 3209 struct rte_port *port; 3210 3211 if (port_id_is_invalid(pid, ENABLED_WARN)) 3212 return; 3213 3214 printf("Closing ports...\n"); 3215 3216 RTE_ETH_FOREACH_DEV(pi) { 3217 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3218 continue; 3219 3220 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3221 fprintf(stderr, 3222 "Please remove port %d from forwarding configuration.\n", 3223 pi); 3224 continue; 3225 } 3226 3227 if (port_is_bonding_slave(pi)) { 3228 fprintf(stderr, 3229 "Please remove port %d from bonded device.\n", 3230 pi); 3231 continue; 3232 } 3233 3234 port = &ports[pi]; 3235 if (port->port_status == RTE_PORT_CLOSED) { 3236 fprintf(stderr, "Port %d is already closed\n", pi); 3237 continue; 3238 } 3239 3240 if (is_proc_primary()) { 3241 mcast_addr_pool_destroy(pi); 3242 port_flow_flush(pi); 3243 port_flex_item_flush(pi); 3244 port_action_handle_flush(pi); 3245 rte_eth_dev_close(pi); 3246 } 3247 3248 free_xstats_display_info(pi); 3249 } 3250 3251 remove_invalid_ports(); 3252 printf("Done\n"); 3253 } 3254 3255 void 3256 reset_port(portid_t pid) 3257 { 3258 int diag; 3259 portid_t pi; 3260 struct rte_port *port; 3261 3262 if (port_id_is_invalid(pid, ENABLED_WARN)) 3263 return; 3264 3265 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3266 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3267 fprintf(stderr, 3268 "Can not reset port(s), please stop port(s) first.\n"); 3269 return; 3270 } 3271 3272 printf("Resetting ports...\n"); 3273 3274 RTE_ETH_FOREACH_DEV(pi) { 3275 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3276 continue; 3277 3278 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3279 fprintf(stderr, 3280 "Please remove port %d from forwarding configuration.\n", 3281 pi); 3282 continue; 3283 } 3284 3285 if (port_is_bonding_slave(pi)) { 3286 fprintf(stderr, 3287 "Please remove port %d from bonded device.\n", 3288 pi); 3289 continue; 3290 } 3291 3292 diag = rte_eth_dev_reset(pi); 3293 if (diag == 0) { 3294 port = &ports[pi]; 3295 port->need_reconfig = 1; 3296 port->need_reconfig_queues = 1; 3297 } else { 3298 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3299 pi, diag); 3300 } 3301 } 3302 3303 printf("Done\n"); 3304 } 3305 3306 void 3307 attach_port(char *identifier) 3308 { 3309 portid_t pi; 3310 struct rte_dev_iterator iterator; 3311 3312 printf("Attaching a new port...\n"); 3313 3314 if (identifier == NULL) { 3315 fprintf(stderr, "Invalid parameters are specified\n"); 3316 return; 3317 } 3318 3319 if (rte_dev_probe(identifier) < 0) { 3320 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3321 return; 3322 } 3323 3324 /* first attach mode: event */ 3325 if (setup_on_probe_event) { 3326 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3327 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3328 if (ports[pi].port_status == RTE_PORT_HANDLING && 3329 ports[pi].need_setup != 0) 3330 setup_attached_port(pi); 3331 return; 3332 } 3333 3334 /* second attach mode: iterator */ 3335 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3336 /* setup ports matching the devargs used for probing */ 3337 if (port_is_forwarding(pi)) 3338 continue; /* port was already attached before */ 3339 setup_attached_port(pi); 3340 } 3341 } 3342 3343 static void 3344 setup_attached_port(portid_t pi) 3345 { 3346 unsigned int socket_id; 3347 int ret; 3348 3349 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3350 /* if socket_id is invalid, set to the first available socket. */ 3351 if (check_socket_id(socket_id) < 0) 3352 socket_id = socket_ids[0]; 3353 reconfig(pi, socket_id); 3354 ret = rte_eth_promiscuous_enable(pi); 3355 if (ret != 0) 3356 fprintf(stderr, 3357 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3358 pi, rte_strerror(-ret)); 3359 3360 ports_ids[nb_ports++] = pi; 3361 fwd_ports_ids[nb_fwd_ports++] = pi; 3362 nb_cfg_ports = nb_fwd_ports; 3363 ports[pi].need_setup = 0; 3364 ports[pi].port_status = RTE_PORT_STOPPED; 3365 3366 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3367 printf("Done\n"); 3368 } 3369 3370 static void 3371 detach_device(struct rte_device *dev) 3372 { 3373 portid_t sibling; 3374 3375 if (dev == NULL) { 3376 fprintf(stderr, "Device already removed\n"); 3377 return; 3378 } 3379 3380 printf("Removing a device...\n"); 3381 3382 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3383 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3384 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3385 fprintf(stderr, "Port %u not stopped\n", 3386 sibling); 3387 return; 3388 } 3389 port_flow_flush(sibling); 3390 } 3391 } 3392 3393 if (rte_dev_remove(dev) < 0) { 3394 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3395 return; 3396 } 3397 remove_invalid_ports(); 3398 3399 printf("Device is detached\n"); 3400 printf("Now total ports is %d\n", nb_ports); 3401 printf("Done\n"); 3402 return; 3403 } 3404 3405 void 3406 detach_port_device(portid_t port_id) 3407 { 3408 int ret; 3409 struct rte_eth_dev_info dev_info; 3410 3411 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3412 return; 3413 3414 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3415 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3416 fprintf(stderr, "Port not stopped\n"); 3417 return; 3418 } 3419 fprintf(stderr, "Port was not closed\n"); 3420 } 3421 3422 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3423 if (ret != 0) { 3424 TESTPMD_LOG(ERR, 3425 "Failed to get device info for port %d, not detaching\n", 3426 port_id); 3427 return; 3428 } 3429 detach_device(dev_info.device); 3430 } 3431 3432 void 3433 detach_devargs(char *identifier) 3434 { 3435 struct rte_dev_iterator iterator; 3436 struct rte_devargs da; 3437 portid_t port_id; 3438 3439 printf("Removing a device...\n"); 3440 3441 memset(&da, 0, sizeof(da)); 3442 if (rte_devargs_parsef(&da, "%s", identifier)) { 3443 fprintf(stderr, "cannot parse identifier\n"); 3444 return; 3445 } 3446 3447 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3448 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3449 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3450 fprintf(stderr, "Port %u not stopped\n", 3451 port_id); 3452 rte_eth_iterator_cleanup(&iterator); 3453 rte_devargs_reset(&da); 3454 return; 3455 } 3456 port_flow_flush(port_id); 3457 } 3458 } 3459 3460 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3461 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3462 da.name, da.bus->name); 3463 rte_devargs_reset(&da); 3464 return; 3465 } 3466 3467 remove_invalid_ports(); 3468 3469 printf("Device %s is detached\n", identifier); 3470 printf("Now total ports is %d\n", nb_ports); 3471 printf("Done\n"); 3472 rte_devargs_reset(&da); 3473 } 3474 3475 void 3476 pmd_test_exit(void) 3477 { 3478 portid_t pt_id; 3479 unsigned int i; 3480 int ret; 3481 3482 if (test_done == 0) 3483 stop_packet_forwarding(); 3484 3485 #ifndef RTE_EXEC_ENV_WINDOWS 3486 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3487 if (mempools[i]) { 3488 if (mp_alloc_type == MP_ALLOC_ANON) 3489 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3490 NULL); 3491 } 3492 } 3493 #endif 3494 if (ports != NULL) { 3495 no_link_check = 1; 3496 RTE_ETH_FOREACH_DEV(pt_id) { 3497 printf("\nStopping port %d...\n", pt_id); 3498 fflush(stdout); 3499 stop_port(pt_id); 3500 } 3501 RTE_ETH_FOREACH_DEV(pt_id) { 3502 printf("\nShutting down port %d...\n", pt_id); 3503 fflush(stdout); 3504 close_port(pt_id); 3505 } 3506 } 3507 3508 if (hot_plug) { 3509 ret = rte_dev_event_monitor_stop(); 3510 if (ret) { 3511 RTE_LOG(ERR, EAL, 3512 "fail to stop device event monitor."); 3513 return; 3514 } 3515 3516 ret = rte_dev_event_callback_unregister(NULL, 3517 dev_event_callback, NULL); 3518 if (ret < 0) { 3519 RTE_LOG(ERR, EAL, 3520 "fail to unregister device event callback.\n"); 3521 return; 3522 } 3523 3524 ret = rte_dev_hotplug_handle_disable(); 3525 if (ret) { 3526 RTE_LOG(ERR, EAL, 3527 "fail to disable hotplug handling.\n"); 3528 return; 3529 } 3530 } 3531 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3532 if (mempools[i]) 3533 mempool_free_mp(mempools[i]); 3534 } 3535 free(xstats_display); 3536 3537 printf("\nBye...\n"); 3538 } 3539 3540 typedef void (*cmd_func_t)(void); 3541 struct pmd_test_command { 3542 const char *cmd_name; 3543 cmd_func_t cmd_func; 3544 }; 3545 3546 /* Check the link status of all ports in up to 9s, and print them finally */ 3547 static void 3548 check_all_ports_link_status(uint32_t port_mask) 3549 { 3550 #define CHECK_INTERVAL 100 /* 100ms */ 3551 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3552 portid_t portid; 3553 uint8_t count, all_ports_up, print_flag = 0; 3554 struct rte_eth_link link; 3555 int ret; 3556 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3557 3558 printf("Checking link statuses...\n"); 3559 fflush(stdout); 3560 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3561 all_ports_up = 1; 3562 RTE_ETH_FOREACH_DEV(portid) { 3563 if ((port_mask & (1 << portid)) == 0) 3564 continue; 3565 memset(&link, 0, sizeof(link)); 3566 ret = rte_eth_link_get_nowait(portid, &link); 3567 if (ret < 0) { 3568 all_ports_up = 0; 3569 if (print_flag == 1) 3570 fprintf(stderr, 3571 "Port %u link get failed: %s\n", 3572 portid, rte_strerror(-ret)); 3573 continue; 3574 } 3575 /* print link status if flag set */ 3576 if (print_flag == 1) { 3577 rte_eth_link_to_str(link_status, 3578 sizeof(link_status), &link); 3579 printf("Port %d %s\n", portid, link_status); 3580 continue; 3581 } 3582 /* clear all_ports_up flag if any link down */ 3583 if (link.link_status == RTE_ETH_LINK_DOWN) { 3584 all_ports_up = 0; 3585 break; 3586 } 3587 } 3588 /* after finally printing all link status, get out */ 3589 if (print_flag == 1) 3590 break; 3591 3592 if (all_ports_up == 0) { 3593 fflush(stdout); 3594 rte_delay_ms(CHECK_INTERVAL); 3595 } 3596 3597 /* set the print_flag if all ports up or timeout */ 3598 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3599 print_flag = 1; 3600 } 3601 3602 if (lsc_interrupt) 3603 break; 3604 } 3605 } 3606 3607 static void 3608 rmv_port_callback(void *arg) 3609 { 3610 int need_to_start = 0; 3611 int org_no_link_check = no_link_check; 3612 portid_t port_id = (intptr_t)arg; 3613 struct rte_eth_dev_info dev_info; 3614 int ret; 3615 3616 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3617 3618 if (!test_done && port_is_forwarding(port_id)) { 3619 need_to_start = 1; 3620 stop_packet_forwarding(); 3621 } 3622 no_link_check = 1; 3623 stop_port(port_id); 3624 no_link_check = org_no_link_check; 3625 3626 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3627 if (ret != 0) 3628 TESTPMD_LOG(ERR, 3629 "Failed to get device info for port %d, not detaching\n", 3630 port_id); 3631 else { 3632 struct rte_device *device = dev_info.device; 3633 close_port(port_id); 3634 detach_device(device); /* might be already removed or have more ports */ 3635 } 3636 if (need_to_start) 3637 start_packet_forwarding(0); 3638 } 3639 3640 /* This function is used by the interrupt thread */ 3641 static int 3642 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3643 void *ret_param) 3644 { 3645 RTE_SET_USED(param); 3646 RTE_SET_USED(ret_param); 3647 3648 if (type >= RTE_ETH_EVENT_MAX) { 3649 fprintf(stderr, 3650 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3651 port_id, __func__, type); 3652 fflush(stderr); 3653 } else if (event_print_mask & (UINT32_C(1) << type)) { 3654 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3655 eth_event_desc[type]); 3656 fflush(stdout); 3657 } 3658 3659 switch (type) { 3660 case RTE_ETH_EVENT_NEW: 3661 ports[port_id].need_setup = 1; 3662 ports[port_id].port_status = RTE_PORT_HANDLING; 3663 break; 3664 case RTE_ETH_EVENT_INTR_RMV: 3665 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3666 break; 3667 if (rte_eal_alarm_set(100000, 3668 rmv_port_callback, (void *)(intptr_t)port_id)) 3669 fprintf(stderr, 3670 "Could not set up deferred device removal\n"); 3671 break; 3672 case RTE_ETH_EVENT_DESTROY: 3673 ports[port_id].port_status = RTE_PORT_CLOSED; 3674 printf("Port %u is closed\n", port_id); 3675 break; 3676 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3677 uint16_t rxq_id; 3678 int ret; 3679 3680 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3681 for (rxq_id = 0; ; rxq_id++) { 3682 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3683 NULL); 3684 if (ret <= 0) 3685 break; 3686 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3687 port_id, rxq_id); 3688 } 3689 break; 3690 } 3691 default: 3692 break; 3693 } 3694 return 0; 3695 } 3696 3697 static int 3698 register_eth_event_callback(void) 3699 { 3700 int ret; 3701 enum rte_eth_event_type event; 3702 3703 for (event = RTE_ETH_EVENT_UNKNOWN; 3704 event < RTE_ETH_EVENT_MAX; event++) { 3705 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3706 event, 3707 eth_event_callback, 3708 NULL); 3709 if (ret != 0) { 3710 TESTPMD_LOG(ERR, "Failed to register callback for " 3711 "%s event\n", eth_event_desc[event]); 3712 return -1; 3713 } 3714 } 3715 3716 return 0; 3717 } 3718 3719 /* This function is used by the interrupt thread */ 3720 static void 3721 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3722 __rte_unused void *arg) 3723 { 3724 uint16_t port_id; 3725 int ret; 3726 3727 if (type >= RTE_DEV_EVENT_MAX) { 3728 fprintf(stderr, "%s called upon invalid event %d\n", 3729 __func__, type); 3730 fflush(stderr); 3731 } 3732 3733 switch (type) { 3734 case RTE_DEV_EVENT_REMOVE: 3735 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3736 device_name); 3737 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3738 if (ret) { 3739 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3740 device_name); 3741 return; 3742 } 3743 /* 3744 * Because the user's callback is invoked in eal interrupt 3745 * callback, the interrupt callback need to be finished before 3746 * it can be unregistered when detaching device. So finish 3747 * callback soon and use a deferred removal to detach device 3748 * is need. It is a workaround, once the device detaching be 3749 * moved into the eal in the future, the deferred removal could 3750 * be deleted. 3751 */ 3752 if (rte_eal_alarm_set(100000, 3753 rmv_port_callback, (void *)(intptr_t)port_id)) 3754 RTE_LOG(ERR, EAL, 3755 "Could not set up deferred device removal\n"); 3756 break; 3757 case RTE_DEV_EVENT_ADD: 3758 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3759 device_name); 3760 /* TODO: After finish kernel driver binding, 3761 * begin to attach port. 3762 */ 3763 break; 3764 default: 3765 break; 3766 } 3767 } 3768 3769 static void 3770 rxtx_port_config(portid_t pid) 3771 { 3772 uint16_t qid; 3773 uint64_t offloads; 3774 struct rte_port *port = &ports[pid]; 3775 3776 for (qid = 0; qid < nb_rxq; qid++) { 3777 offloads = port->rxq[qid].conf.offloads; 3778 port->rxq[qid].conf = port->dev_info.default_rxconf; 3779 3780 if (rxq_share > 0 && 3781 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3782 /* Non-zero share group to enable RxQ share. */ 3783 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3784 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3785 } 3786 3787 if (offloads != 0) 3788 port->rxq[qid].conf.offloads = offloads; 3789 3790 /* Check if any Rx parameters have been passed */ 3791 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3792 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3793 3794 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3795 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3796 3797 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3798 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3799 3800 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3801 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3802 3803 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3804 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3805 3806 port->nb_rx_desc[qid] = nb_rxd; 3807 } 3808 3809 for (qid = 0; qid < nb_txq; qid++) { 3810 offloads = port->txq[qid].conf.offloads; 3811 port->txq[qid].conf = port->dev_info.default_txconf; 3812 if (offloads != 0) 3813 port->txq[qid].conf.offloads = offloads; 3814 3815 /* Check if any Tx parameters have been passed */ 3816 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3817 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3818 3819 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3820 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3821 3822 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3823 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3824 3825 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3826 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3827 3828 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3829 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3830 3831 port->nb_tx_desc[qid] = nb_txd; 3832 } 3833 } 3834 3835 /* 3836 * Helper function to set MTU from frame size 3837 * 3838 * port->dev_info should be set before calling this function. 3839 * 3840 * return 0 on success, negative on error 3841 */ 3842 int 3843 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3844 { 3845 struct rte_port *port = &ports[portid]; 3846 uint32_t eth_overhead; 3847 uint16_t mtu, new_mtu; 3848 3849 eth_overhead = get_eth_overhead(&port->dev_info); 3850 3851 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3852 printf("Failed to get MTU for port %u\n", portid); 3853 return -1; 3854 } 3855 3856 new_mtu = max_rx_pktlen - eth_overhead; 3857 3858 if (mtu == new_mtu) 3859 return 0; 3860 3861 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3862 fprintf(stderr, 3863 "Failed to set MTU to %u for port %u\n", 3864 new_mtu, portid); 3865 return -1; 3866 } 3867 3868 port->dev_conf.rxmode.mtu = new_mtu; 3869 3870 return 0; 3871 } 3872 3873 void 3874 init_port_config(void) 3875 { 3876 portid_t pid; 3877 struct rte_port *port; 3878 int ret, i; 3879 3880 RTE_ETH_FOREACH_DEV(pid) { 3881 port = &ports[pid]; 3882 port->dev_conf.fdir_conf = fdir_conf; 3883 3884 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3885 if (ret != 0) 3886 return; 3887 3888 if (nb_rxq > 1) { 3889 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3890 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3891 rss_hf & port->dev_info.flow_type_rss_offloads; 3892 } else { 3893 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3894 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3895 } 3896 3897 if (port->dcb_flag == 0) { 3898 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3899 port->dev_conf.rxmode.mq_mode = 3900 (enum rte_eth_rx_mq_mode) 3901 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3902 } else { 3903 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3904 port->dev_conf.rxmode.offloads &= 3905 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3906 3907 for (i = 0; 3908 i < port->dev_info.nb_rx_queues; 3909 i++) 3910 port->rxq[i].conf.offloads &= 3911 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3912 } 3913 } 3914 3915 rxtx_port_config(pid); 3916 3917 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3918 if (ret != 0) 3919 return; 3920 3921 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3922 rte_pmd_ixgbe_bypass_init(pid); 3923 #endif 3924 3925 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3926 port->dev_conf.intr_conf.lsc = 1; 3927 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3928 port->dev_conf.intr_conf.rmv = 1; 3929 } 3930 } 3931 3932 void set_port_slave_flag(portid_t slave_pid) 3933 { 3934 struct rte_port *port; 3935 3936 port = &ports[slave_pid]; 3937 port->slave_flag = 1; 3938 } 3939 3940 void clear_port_slave_flag(portid_t slave_pid) 3941 { 3942 struct rte_port *port; 3943 3944 port = &ports[slave_pid]; 3945 port->slave_flag = 0; 3946 } 3947 3948 uint8_t port_is_bonding_slave(portid_t slave_pid) 3949 { 3950 struct rte_port *port; 3951 struct rte_eth_dev_info dev_info; 3952 int ret; 3953 3954 port = &ports[slave_pid]; 3955 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3956 if (ret != 0) { 3957 TESTPMD_LOG(ERR, 3958 "Failed to get device info for port id %d," 3959 "cannot determine if the port is a bonded slave", 3960 slave_pid); 3961 return 0; 3962 } 3963 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3964 return 1; 3965 return 0; 3966 } 3967 3968 const uint16_t vlan_tags[] = { 3969 0, 1, 2, 3, 4, 5, 6, 7, 3970 8, 9, 10, 11, 12, 13, 14, 15, 3971 16, 17, 18, 19, 20, 21, 22, 23, 3972 24, 25, 26, 27, 28, 29, 30, 31 3973 }; 3974 3975 static int 3976 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3977 enum dcb_mode_enable dcb_mode, 3978 enum rte_eth_nb_tcs num_tcs, 3979 uint8_t pfc_en) 3980 { 3981 uint8_t i; 3982 int32_t rc; 3983 struct rte_eth_rss_conf rss_conf; 3984 3985 /* 3986 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3987 * given above, and the number of traffic classes available for use. 3988 */ 3989 if (dcb_mode == DCB_VT_ENABLED) { 3990 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3991 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3992 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3993 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3994 3995 /* VMDQ+DCB RX and TX configurations */ 3996 vmdq_rx_conf->enable_default_pool = 0; 3997 vmdq_rx_conf->default_pool = 0; 3998 vmdq_rx_conf->nb_queue_pools = 3999 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4000 vmdq_tx_conf->nb_queue_pools = 4001 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4002 4003 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4004 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4005 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4006 vmdq_rx_conf->pool_map[i].pools = 4007 1 << (i % vmdq_rx_conf->nb_queue_pools); 4008 } 4009 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4010 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4011 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4012 } 4013 4014 /* set DCB mode of RX and TX of multiple queues */ 4015 eth_conf->rxmode.mq_mode = 4016 (enum rte_eth_rx_mq_mode) 4017 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4018 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4019 } else { 4020 struct rte_eth_dcb_rx_conf *rx_conf = 4021 ð_conf->rx_adv_conf.dcb_rx_conf; 4022 struct rte_eth_dcb_tx_conf *tx_conf = 4023 ð_conf->tx_adv_conf.dcb_tx_conf; 4024 4025 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4026 4027 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4028 if (rc != 0) 4029 return rc; 4030 4031 rx_conf->nb_tcs = num_tcs; 4032 tx_conf->nb_tcs = num_tcs; 4033 4034 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4035 rx_conf->dcb_tc[i] = i % num_tcs; 4036 tx_conf->dcb_tc[i] = i % num_tcs; 4037 } 4038 4039 eth_conf->rxmode.mq_mode = 4040 (enum rte_eth_rx_mq_mode) 4041 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4042 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4043 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4044 } 4045 4046 if (pfc_en) 4047 eth_conf->dcb_capability_en = 4048 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4049 else 4050 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4051 4052 return 0; 4053 } 4054 4055 int 4056 init_port_dcb_config(portid_t pid, 4057 enum dcb_mode_enable dcb_mode, 4058 enum rte_eth_nb_tcs num_tcs, 4059 uint8_t pfc_en) 4060 { 4061 struct rte_eth_conf port_conf; 4062 struct rte_port *rte_port; 4063 int retval; 4064 uint16_t i; 4065 4066 if (num_procs > 1) { 4067 printf("The multi-process feature doesn't support dcb.\n"); 4068 return -ENOTSUP; 4069 } 4070 rte_port = &ports[pid]; 4071 4072 /* retain the original device configuration. */ 4073 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4074 4075 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4076 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4077 if (retval < 0) 4078 return retval; 4079 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4080 /* remove RSS HASH offload for DCB in vt mode */ 4081 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4082 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4083 for (i = 0; i < nb_rxq; i++) 4084 rte_port->rxq[i].conf.offloads &= 4085 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4086 } 4087 4088 /* re-configure the device . */ 4089 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4090 if (retval < 0) 4091 return retval; 4092 4093 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4094 if (retval != 0) 4095 return retval; 4096 4097 /* If dev_info.vmdq_pool_base is greater than 0, 4098 * the queue id of vmdq pools is started after pf queues. 4099 */ 4100 if (dcb_mode == DCB_VT_ENABLED && 4101 rte_port->dev_info.vmdq_pool_base > 0) { 4102 fprintf(stderr, 4103 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4104 pid); 4105 return -1; 4106 } 4107 4108 /* Assume the ports in testpmd have the same dcb capability 4109 * and has the same number of rxq and txq in dcb mode 4110 */ 4111 if (dcb_mode == DCB_VT_ENABLED) { 4112 if (rte_port->dev_info.max_vfs > 0) { 4113 nb_rxq = rte_port->dev_info.nb_rx_queues; 4114 nb_txq = rte_port->dev_info.nb_tx_queues; 4115 } else { 4116 nb_rxq = rte_port->dev_info.max_rx_queues; 4117 nb_txq = rte_port->dev_info.max_tx_queues; 4118 } 4119 } else { 4120 /*if vt is disabled, use all pf queues */ 4121 if (rte_port->dev_info.vmdq_pool_base == 0) { 4122 nb_rxq = rte_port->dev_info.max_rx_queues; 4123 nb_txq = rte_port->dev_info.max_tx_queues; 4124 } else { 4125 nb_rxq = (queueid_t)num_tcs; 4126 nb_txq = (queueid_t)num_tcs; 4127 4128 } 4129 } 4130 rx_free_thresh = 64; 4131 4132 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4133 4134 rxtx_port_config(pid); 4135 /* VLAN filter */ 4136 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4137 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4138 rx_vft_set(pid, vlan_tags[i], 1); 4139 4140 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4141 if (retval != 0) 4142 return retval; 4143 4144 rte_port->dcb_flag = 1; 4145 4146 /* Enter DCB configuration status */ 4147 dcb_config = 1; 4148 4149 return 0; 4150 } 4151 4152 static void 4153 init_port(void) 4154 { 4155 int i; 4156 4157 /* Configuration of Ethernet ports. */ 4158 ports = rte_zmalloc("testpmd: ports", 4159 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4160 RTE_CACHE_LINE_SIZE); 4161 if (ports == NULL) { 4162 rte_exit(EXIT_FAILURE, 4163 "rte_zmalloc(%d struct rte_port) failed\n", 4164 RTE_MAX_ETHPORTS); 4165 } 4166 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4167 ports[i].xstats_info.allocated = false; 4168 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4169 LIST_INIT(&ports[i].flow_tunnel_list); 4170 /* Initialize ports NUMA structures */ 4171 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4172 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4173 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4174 } 4175 4176 static void 4177 force_quit(void) 4178 { 4179 pmd_test_exit(); 4180 prompt_exit(); 4181 } 4182 4183 static void 4184 print_stats(void) 4185 { 4186 uint8_t i; 4187 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4188 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4189 4190 /* Clear screen and move to top left */ 4191 printf("%s%s", clr, top_left); 4192 4193 printf("\nPort statistics ===================================="); 4194 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4195 nic_stats_display(fwd_ports_ids[i]); 4196 4197 fflush(stdout); 4198 } 4199 4200 static void 4201 signal_handler(int signum) 4202 { 4203 if (signum == SIGINT || signum == SIGTERM) { 4204 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4205 signum); 4206 #ifdef RTE_LIB_PDUMP 4207 /* uninitialize packet capture framework */ 4208 rte_pdump_uninit(); 4209 #endif 4210 #ifdef RTE_LIB_LATENCYSTATS 4211 if (latencystats_enabled != 0) 4212 rte_latencystats_uninit(); 4213 #endif 4214 force_quit(); 4215 /* Set flag to indicate the force termination. */ 4216 f_quit = 1; 4217 /* exit with the expected status */ 4218 #ifndef RTE_EXEC_ENV_WINDOWS 4219 signal(signum, SIG_DFL); 4220 kill(getpid(), signum); 4221 #endif 4222 } 4223 } 4224 4225 int 4226 main(int argc, char** argv) 4227 { 4228 int diag; 4229 portid_t port_id; 4230 uint16_t count; 4231 int ret; 4232 4233 signal(SIGINT, signal_handler); 4234 signal(SIGTERM, signal_handler); 4235 4236 testpmd_logtype = rte_log_register("testpmd"); 4237 if (testpmd_logtype < 0) 4238 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4239 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4240 4241 diag = rte_eal_init(argc, argv); 4242 if (diag < 0) 4243 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4244 rte_strerror(rte_errno)); 4245 4246 ret = register_eth_event_callback(); 4247 if (ret != 0) 4248 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4249 4250 #ifdef RTE_LIB_PDUMP 4251 /* initialize packet capture framework */ 4252 rte_pdump_init(); 4253 #endif 4254 4255 count = 0; 4256 RTE_ETH_FOREACH_DEV(port_id) { 4257 ports_ids[count] = port_id; 4258 count++; 4259 } 4260 nb_ports = (portid_t) count; 4261 if (nb_ports == 0) 4262 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4263 4264 /* allocate port structures, and init them */ 4265 init_port(); 4266 4267 set_def_fwd_config(); 4268 if (nb_lcores == 0) 4269 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4270 "Check the core mask argument\n"); 4271 4272 /* Bitrate/latency stats disabled by default */ 4273 #ifdef RTE_LIB_BITRATESTATS 4274 bitrate_enabled = 0; 4275 #endif 4276 #ifdef RTE_LIB_LATENCYSTATS 4277 latencystats_enabled = 0; 4278 #endif 4279 4280 /* on FreeBSD, mlockall() is disabled by default */ 4281 #ifdef RTE_EXEC_ENV_FREEBSD 4282 do_mlockall = 0; 4283 #else 4284 do_mlockall = 1; 4285 #endif 4286 4287 argc -= diag; 4288 argv += diag; 4289 if (argc > 1) 4290 launch_args_parse(argc, argv); 4291 4292 #ifndef RTE_EXEC_ENV_WINDOWS 4293 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4294 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4295 strerror(errno)); 4296 } 4297 #endif 4298 4299 if (tx_first && interactive) 4300 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4301 "interactive mode.\n"); 4302 4303 if (tx_first && lsc_interrupt) { 4304 fprintf(stderr, 4305 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4306 lsc_interrupt = 0; 4307 } 4308 4309 if (!nb_rxq && !nb_txq) 4310 fprintf(stderr, 4311 "Warning: Either rx or tx queues should be non-zero\n"); 4312 4313 if (nb_rxq > 1 && nb_rxq > nb_txq) 4314 fprintf(stderr, 4315 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4316 nb_rxq, nb_txq); 4317 4318 init_config(); 4319 4320 if (hot_plug) { 4321 ret = rte_dev_hotplug_handle_enable(); 4322 if (ret) { 4323 RTE_LOG(ERR, EAL, 4324 "fail to enable hotplug handling."); 4325 return -1; 4326 } 4327 4328 ret = rte_dev_event_monitor_start(); 4329 if (ret) { 4330 RTE_LOG(ERR, EAL, 4331 "fail to start device event monitoring."); 4332 return -1; 4333 } 4334 4335 ret = rte_dev_event_callback_register(NULL, 4336 dev_event_callback, NULL); 4337 if (ret) { 4338 RTE_LOG(ERR, EAL, 4339 "fail to register device event callback\n"); 4340 return -1; 4341 } 4342 } 4343 4344 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4345 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4346 4347 /* set all ports to promiscuous mode by default */ 4348 RTE_ETH_FOREACH_DEV(port_id) { 4349 ret = rte_eth_promiscuous_enable(port_id); 4350 if (ret != 0) 4351 fprintf(stderr, 4352 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4353 port_id, rte_strerror(-ret)); 4354 } 4355 4356 #ifdef RTE_LIB_METRICS 4357 /* Init metrics library */ 4358 rte_metrics_init(rte_socket_id()); 4359 #endif 4360 4361 #ifdef RTE_LIB_LATENCYSTATS 4362 if (latencystats_enabled != 0) { 4363 int ret = rte_latencystats_init(1, NULL); 4364 if (ret) 4365 fprintf(stderr, 4366 "Warning: latencystats init() returned error %d\n", 4367 ret); 4368 fprintf(stderr, "Latencystats running on lcore %d\n", 4369 latencystats_lcore_id); 4370 } 4371 #endif 4372 4373 /* Setup bitrate stats */ 4374 #ifdef RTE_LIB_BITRATESTATS 4375 if (bitrate_enabled != 0) { 4376 bitrate_data = rte_stats_bitrate_create(); 4377 if (bitrate_data == NULL) 4378 rte_exit(EXIT_FAILURE, 4379 "Could not allocate bitrate data.\n"); 4380 rte_stats_bitrate_reg(bitrate_data); 4381 } 4382 #endif 4383 #ifdef RTE_LIB_CMDLINE 4384 if (init_cmdline() != 0) 4385 rte_exit(EXIT_FAILURE, 4386 "Could not initialise cmdline context.\n"); 4387 4388 if (strlen(cmdline_filename) != 0) 4389 cmdline_read_from_file(cmdline_filename); 4390 4391 if (interactive == 1) { 4392 if (auto_start) { 4393 printf("Start automatic packet forwarding\n"); 4394 start_packet_forwarding(0); 4395 } 4396 prompt(); 4397 pmd_test_exit(); 4398 } else 4399 #endif 4400 { 4401 char c; 4402 int rc; 4403 4404 f_quit = 0; 4405 4406 printf("No commandline core given, start packet forwarding\n"); 4407 start_packet_forwarding(tx_first); 4408 if (stats_period != 0) { 4409 uint64_t prev_time = 0, cur_time, diff_time = 0; 4410 uint64_t timer_period; 4411 4412 /* Convert to number of cycles */ 4413 timer_period = stats_period * rte_get_timer_hz(); 4414 4415 while (f_quit == 0) { 4416 cur_time = rte_get_timer_cycles(); 4417 diff_time += cur_time - prev_time; 4418 4419 if (diff_time >= timer_period) { 4420 print_stats(); 4421 /* Reset the timer */ 4422 diff_time = 0; 4423 } 4424 /* Sleep to avoid unnecessary checks */ 4425 prev_time = cur_time; 4426 rte_delay_us_sleep(US_PER_S); 4427 } 4428 } 4429 4430 printf("Press enter to exit\n"); 4431 rc = read(0, &c, 1); 4432 pmd_test_exit(); 4433 if (rc < 0) 4434 return 1; 4435 } 4436 4437 ret = rte_eal_cleanup(); 4438 if (ret != 0) 4439 rte_exit(EXIT_FAILURE, 4440 "EAL cleanup failed: %s\n", strerror(-ret)); 4441 4442 return EXIT_SUCCESS; 4443 } 4444