1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 73 #include "testpmd.h" 74 75 #ifndef MAP_HUGETLB 76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 77 #define HUGE_FLAG (0x40000) 78 #else 79 #define HUGE_FLAG MAP_HUGETLB 80 #endif 81 82 #ifndef MAP_HUGE_SHIFT 83 /* older kernels (or FreeBSD) will not have this define */ 84 #define HUGE_SHIFT (26) 85 #else 86 #define HUGE_SHIFT MAP_HUGE_SHIFT 87 #endif 88 89 #define EXTMEM_HEAP_NAME "extmem" 90 /* 91 * Zone size with the malloc overhead (max of debug and release variants) 92 * must fit into the smallest supported hugepage size (2M), 93 * so that an IOVA-contiguous zone of this size can always be allocated 94 * if there are free 2M hugepages. 95 */ 96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 97 98 uint16_t verbose_level = 0; /**< Silent by default. */ 99 int testpmd_logtype; /**< Log type for testpmd logs */ 100 101 /* use main core for command line ? */ 102 uint8_t interactive = 0; 103 uint8_t auto_start = 0; 104 uint8_t tx_first; 105 char cmdline_filename[PATH_MAX] = {0}; 106 107 /* 108 * NUMA support configuration. 109 * When set, the NUMA support attempts to dispatch the allocation of the 110 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 111 * probed ports among the CPU sockets 0 and 1. 112 * Otherwise, all memory is allocated from CPU socket 0. 113 */ 114 uint8_t numa_support = 1; /**< numa enabled by default */ 115 116 /* 117 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 118 * not configured. 119 */ 120 uint8_t socket_num = UMA_NO_CONFIG; 121 122 /* 123 * Select mempool allocation type: 124 * - native: use regular DPDK memory 125 * - anon: use regular DPDK memory to create mempool, but populate using 126 * anonymous memory (may not be IOVA-contiguous) 127 * - xmem: use externally allocated hugepage memory 128 */ 129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 130 131 /* 132 * Store specified sockets on which memory pool to be used by ports 133 * is allocated. 134 */ 135 uint8_t port_numa[RTE_MAX_ETHPORTS]; 136 137 /* 138 * Store specified sockets on which RX ring to be used by ports 139 * is allocated. 140 */ 141 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 142 143 /* 144 * Store specified sockets on which TX ring to be used by ports 145 * is allocated. 146 */ 147 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 148 149 /* 150 * Record the Ethernet address of peer target ports to which packets are 151 * forwarded. 152 * Must be instantiated with the ethernet addresses of peer traffic generator 153 * ports. 154 */ 155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 156 portid_t nb_peer_eth_addrs = 0; 157 158 /* 159 * Probed Target Environment. 160 */ 161 struct rte_port *ports; /**< For all probed ethernet ports. */ 162 portid_t nb_ports; /**< Number of probed ethernet ports. */ 163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 164 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 165 166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 167 168 /* 169 * Test Forwarding Configuration. 170 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 171 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 172 */ 173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 175 portid_t nb_cfg_ports; /**< Number of configured ports. */ 176 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 177 178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 180 181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 182 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 183 184 /* 185 * Forwarding engines. 186 */ 187 struct fwd_engine * fwd_engines[] = { 188 &io_fwd_engine, 189 &mac_fwd_engine, 190 &mac_swap_engine, 191 &flow_gen_engine, 192 &rx_only_engine, 193 &tx_only_engine, 194 &csum_fwd_engine, 195 &icmp_echo_engine, 196 &noisy_vnf_engine, 197 &five_tuple_swap_fwd_engine, 198 #ifdef RTE_LIBRTE_IEEE1588 199 &ieee1588_fwd_engine, 200 #endif 201 &shared_rxq_engine, 202 NULL, 203 }; 204 205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 206 uint16_t mempool_flags; 207 208 struct fwd_config cur_fwd_config; 209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 210 uint32_t retry_enabled; 211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 213 214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 216 DEFAULT_MBUF_DATA_SIZE 217 }; /**< Mbuf data space size. */ 218 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 219 * specified on command-line. */ 220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 221 222 /** Extended statistics to show. */ 223 struct rte_eth_xstat_name *xstats_display; 224 225 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 226 227 /* 228 * In container, it cannot terminate the process which running with 'stats-period' 229 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 230 */ 231 uint8_t f_quit; 232 233 /* 234 * Max Rx frame size, set by '--max-pkt-len' parameter. 235 */ 236 uint32_t max_rx_pkt_len; 237 238 /* 239 * Configuration of packet segments used to scatter received packets 240 * if some of split features is configured. 241 */ 242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 243 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 245 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 246 247 /* 248 * Configuration of packet segments used by the "txonly" processing engine. 249 */ 250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 252 TXONLY_DEF_PACKET_LEN, 253 }; 254 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 255 256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 257 /**< Split policy for packets to TX. */ 258 259 uint8_t txonly_multi_flow; 260 /**< Whether multiple flows are generated in TXONLY mode. */ 261 262 uint32_t tx_pkt_times_inter; 263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 264 265 uint32_t tx_pkt_times_intra; 266 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 267 268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 272 273 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 274 uint8_t dcb_config = 0; 275 276 /* 277 * Configurable number of RX/TX queues. 278 */ 279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 282 283 /* 284 * Configurable number of RX/TX ring descriptors. 285 * Defaults are supplied by drivers via ethdev. 286 */ 287 #define RTE_TEST_RX_DESC_DEFAULT 0 288 #define RTE_TEST_TX_DESC_DEFAULT 0 289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 291 292 #define RTE_PMD_PARAM_UNSET -1 293 /* 294 * Configurable values of RX and TX ring threshold registers. 295 */ 296 297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 300 301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 304 305 /* 306 * Configurable value of RX free threshold. 307 */ 308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 309 310 /* 311 * Configurable value of RX drop enable. 312 */ 313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 314 315 /* 316 * Configurable value of TX free threshold. 317 */ 318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 319 320 /* 321 * Configurable value of TX RS bit threshold. 322 */ 323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 324 325 /* 326 * Configurable value of buffered packets before sending. 327 */ 328 uint16_t noisy_tx_sw_bufsz; 329 330 /* 331 * Configurable value of packet buffer timeout. 332 */ 333 uint16_t noisy_tx_sw_buf_flush_time; 334 335 /* 336 * Configurable value for size of VNF internal memory area 337 * used for simulating noisy neighbour behaviour 338 */ 339 uint64_t noisy_lkup_mem_sz; 340 341 /* 342 * Configurable value of number of random writes done in 343 * VNF simulation memory area. 344 */ 345 uint64_t noisy_lkup_num_writes; 346 347 /* 348 * Configurable value of number of random reads done in 349 * VNF simulation memory area. 350 */ 351 uint64_t noisy_lkup_num_reads; 352 353 /* 354 * Configurable value of number of random reads/writes done in 355 * VNF simulation memory area. 356 */ 357 uint64_t noisy_lkup_num_reads_writes; 358 359 /* 360 * Receive Side Scaling (RSS) configuration. 361 */ 362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 363 364 /* 365 * Port topology configuration 366 */ 367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 368 369 /* 370 * Avoids to flush all the RX streams before starts forwarding. 371 */ 372 uint8_t no_flush_rx = 0; /* flush by default */ 373 374 /* 375 * Flow API isolated mode. 376 */ 377 uint8_t flow_isolate_all; 378 379 /* 380 * Avoids to check link status when starting/stopping a port. 381 */ 382 uint8_t no_link_check = 0; /* check by default */ 383 384 /* 385 * Don't automatically start all ports in interactive mode. 386 */ 387 uint8_t no_device_start = 0; 388 389 /* 390 * Enable link status change notification 391 */ 392 uint8_t lsc_interrupt = 1; /* enabled by default */ 393 394 /* 395 * Enable device removal notification. 396 */ 397 uint8_t rmv_interrupt = 1; /* enabled by default */ 398 399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 400 401 /* After attach, port setup is called on event or by iterator */ 402 bool setup_on_probe_event = true; 403 404 /* Clear ptypes on port initialization. */ 405 uint8_t clear_ptypes = true; 406 407 /* Hairpin ports configuration mode. */ 408 uint16_t hairpin_mode; 409 410 /* Pretty printing of ethdev events */ 411 static const char * const eth_event_desc[] = { 412 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 413 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 414 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 415 [RTE_ETH_EVENT_INTR_RESET] = "reset", 416 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 417 [RTE_ETH_EVENT_IPSEC] = "IPsec", 418 [RTE_ETH_EVENT_MACSEC] = "MACsec", 419 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 420 [RTE_ETH_EVENT_NEW] = "device probed", 421 [RTE_ETH_EVENT_DESTROY] = "device released", 422 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 423 [RTE_ETH_EVENT_MAX] = NULL, 424 }; 425 426 /* 427 * Display or mask ether events 428 * Default to all events except VF_MBOX 429 */ 430 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 431 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 432 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 433 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 434 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 435 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 436 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 437 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 438 /* 439 * Decide if all memory are locked for performance. 440 */ 441 int do_mlockall = 0; 442 443 /* 444 * NIC bypass mode configuration options. 445 */ 446 447 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 448 /* The NIC bypass watchdog timeout. */ 449 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 450 #endif 451 452 453 #ifdef RTE_LIB_LATENCYSTATS 454 455 /* 456 * Set when latency stats is enabled in the commandline 457 */ 458 uint8_t latencystats_enabled; 459 460 /* 461 * Lcore ID to service latency statistics. 462 */ 463 lcoreid_t latencystats_lcore_id = -1; 464 465 #endif 466 467 /* 468 * Ethernet device configuration. 469 */ 470 struct rte_eth_rxmode rx_mode; 471 472 struct rte_eth_txmode tx_mode = { 473 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 474 }; 475 476 struct rte_eth_fdir_conf fdir_conf = { 477 .mode = RTE_FDIR_MODE_NONE, 478 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 479 .status = RTE_FDIR_REPORT_STATUS, 480 .mask = { 481 .vlan_tci_mask = 0xFFEF, 482 .ipv4_mask = { 483 .src_ip = 0xFFFFFFFF, 484 .dst_ip = 0xFFFFFFFF, 485 }, 486 .ipv6_mask = { 487 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 488 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 489 }, 490 .src_port_mask = 0xFFFF, 491 .dst_port_mask = 0xFFFF, 492 .mac_addr_byte_mask = 0xFF, 493 .tunnel_type_mask = 1, 494 .tunnel_id_mask = 0xFFFFFFFF, 495 }, 496 .drop_queue = 127, 497 }; 498 499 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 500 501 /* 502 * Display zero values by default for xstats 503 */ 504 uint8_t xstats_hide_zero; 505 506 /* 507 * Measure of CPU cycles disabled by default 508 */ 509 uint8_t record_core_cycles; 510 511 /* 512 * Display of RX and TX bursts disabled by default 513 */ 514 uint8_t record_burst_stats; 515 516 /* 517 * Number of ports per shared Rx queue group, 0 disable. 518 */ 519 uint32_t rxq_share; 520 521 unsigned int num_sockets = 0; 522 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 523 524 #ifdef RTE_LIB_BITRATESTATS 525 /* Bitrate statistics */ 526 struct rte_stats_bitrates *bitrate_data; 527 lcoreid_t bitrate_lcore_id; 528 uint8_t bitrate_enabled; 529 #endif 530 531 #ifdef RTE_LIB_GRO 532 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 533 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 534 #endif 535 536 /* 537 * hexadecimal bitmask of RX mq mode can be enabled. 538 */ 539 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 540 541 /* 542 * Used to set forced link speed 543 */ 544 uint32_t eth_link_speed; 545 546 /* 547 * ID of the current process in multi-process, used to 548 * configure the queues to be polled. 549 */ 550 int proc_id; 551 552 /* 553 * Number of processes in multi-process, used to 554 * configure the queues to be polled. 555 */ 556 unsigned int num_procs = 1; 557 558 static void 559 eth_rx_metadata_negotiate_mp(uint16_t port_id) 560 { 561 uint64_t rx_meta_features = 0; 562 int ret; 563 564 if (!is_proc_primary()) 565 return; 566 567 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 568 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 569 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 570 571 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 572 if (ret == 0) { 573 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 574 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 575 port_id); 576 } 577 578 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 579 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 580 port_id); 581 } 582 583 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 584 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 585 port_id); 586 } 587 } else if (ret != -ENOTSUP) { 588 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 589 port_id, rte_strerror(-ret)); 590 } 591 } 592 593 static int 594 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 595 const struct rte_eth_conf *dev_conf) 596 { 597 if (is_proc_primary()) 598 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 599 dev_conf); 600 return 0; 601 } 602 603 static int 604 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 605 { 606 #ifdef RTE_NET_BOND 607 608 portid_t slave_pids[RTE_MAX_ETHPORTS]; 609 struct rte_port *port; 610 int num_slaves; 611 portid_t slave_pid; 612 int i; 613 614 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 615 RTE_MAX_ETHPORTS); 616 if (num_slaves < 0) { 617 fprintf(stderr, "Failed to get slave list for port = %u\n", 618 bond_pid); 619 return num_slaves; 620 } 621 622 for (i = 0; i < num_slaves; i++) { 623 slave_pid = slave_pids[i]; 624 port = &ports[slave_pid]; 625 port->port_status = 626 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 627 } 628 #else 629 RTE_SET_USED(bond_pid); 630 RTE_SET_USED(is_stop); 631 #endif 632 return 0; 633 } 634 635 static int 636 eth_dev_start_mp(uint16_t port_id) 637 { 638 int ret; 639 640 if (is_proc_primary()) { 641 ret = rte_eth_dev_start(port_id); 642 if (ret != 0) 643 return ret; 644 645 struct rte_port *port = &ports[port_id]; 646 647 /* 648 * Starting a bonded port also starts all slaves under the bonded 649 * device. So if this port is bond device, we need to modify the 650 * port status of these slaves. 651 */ 652 if (port->bond_flag == 1) 653 return change_bonding_slave_port_status(port_id, false); 654 } 655 656 return 0; 657 } 658 659 static int 660 eth_dev_stop_mp(uint16_t port_id) 661 { 662 int ret; 663 664 if (is_proc_primary()) { 665 ret = rte_eth_dev_stop(port_id); 666 if (ret != 0) 667 return ret; 668 669 struct rte_port *port = &ports[port_id]; 670 671 /* 672 * Stopping a bonded port also stops all slaves under the bonded 673 * device. So if this port is bond device, we need to modify the 674 * port status of these slaves. 675 */ 676 if (port->bond_flag == 1) 677 return change_bonding_slave_port_status(port_id, true); 678 } 679 680 return 0; 681 } 682 683 static void 684 mempool_free_mp(struct rte_mempool *mp) 685 { 686 if (is_proc_primary()) 687 rte_mempool_free(mp); 688 } 689 690 static int 691 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 692 { 693 if (is_proc_primary()) 694 return rte_eth_dev_set_mtu(port_id, mtu); 695 696 return 0; 697 } 698 699 /* Forward function declarations */ 700 static void setup_attached_port(portid_t pi); 701 static void check_all_ports_link_status(uint32_t port_mask); 702 static int eth_event_callback(portid_t port_id, 703 enum rte_eth_event_type type, 704 void *param, void *ret_param); 705 static void dev_event_callback(const char *device_name, 706 enum rte_dev_event_type type, 707 void *param); 708 static void fill_xstats_display_info(void); 709 710 /* 711 * Check if all the ports are started. 712 * If yes, return positive value. If not, return zero. 713 */ 714 static int all_ports_started(void); 715 716 #ifdef RTE_LIB_GSO 717 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 718 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 719 #endif 720 721 /* Holds the registered mbuf dynamic flags names. */ 722 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 723 724 725 /* 726 * Helper function to check if socket is already discovered. 727 * If yes, return positive value. If not, return zero. 728 */ 729 int 730 new_socket_id(unsigned int socket_id) 731 { 732 unsigned int i; 733 734 for (i = 0; i < num_sockets; i++) { 735 if (socket_ids[i] == socket_id) 736 return 0; 737 } 738 return 1; 739 } 740 741 /* 742 * Setup default configuration. 743 */ 744 static void 745 set_default_fwd_lcores_config(void) 746 { 747 unsigned int i; 748 unsigned int nb_lc; 749 unsigned int sock_num; 750 751 nb_lc = 0; 752 for (i = 0; i < RTE_MAX_LCORE; i++) { 753 if (!rte_lcore_is_enabled(i)) 754 continue; 755 sock_num = rte_lcore_to_socket_id(i); 756 if (new_socket_id(sock_num)) { 757 if (num_sockets >= RTE_MAX_NUMA_NODES) { 758 rte_exit(EXIT_FAILURE, 759 "Total sockets greater than %u\n", 760 RTE_MAX_NUMA_NODES); 761 } 762 socket_ids[num_sockets++] = sock_num; 763 } 764 if (i == rte_get_main_lcore()) 765 continue; 766 fwd_lcores_cpuids[nb_lc++] = i; 767 } 768 nb_lcores = (lcoreid_t) nb_lc; 769 nb_cfg_lcores = nb_lcores; 770 nb_fwd_lcores = 1; 771 } 772 773 static void 774 set_def_peer_eth_addrs(void) 775 { 776 portid_t i; 777 778 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 779 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 780 peer_eth_addrs[i].addr_bytes[5] = i; 781 } 782 } 783 784 static void 785 set_default_fwd_ports_config(void) 786 { 787 portid_t pt_id; 788 int i = 0; 789 790 RTE_ETH_FOREACH_DEV(pt_id) { 791 fwd_ports_ids[i++] = pt_id; 792 793 /* Update sockets info according to the attached device */ 794 int socket_id = rte_eth_dev_socket_id(pt_id); 795 if (socket_id >= 0 && new_socket_id(socket_id)) { 796 if (num_sockets >= RTE_MAX_NUMA_NODES) { 797 rte_exit(EXIT_FAILURE, 798 "Total sockets greater than %u\n", 799 RTE_MAX_NUMA_NODES); 800 } 801 socket_ids[num_sockets++] = socket_id; 802 } 803 } 804 805 nb_cfg_ports = nb_ports; 806 nb_fwd_ports = nb_ports; 807 } 808 809 void 810 set_def_fwd_config(void) 811 { 812 set_default_fwd_lcores_config(); 813 set_def_peer_eth_addrs(); 814 set_default_fwd_ports_config(); 815 } 816 817 #ifndef RTE_EXEC_ENV_WINDOWS 818 /* extremely pessimistic estimation of memory required to create a mempool */ 819 static int 820 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 821 { 822 unsigned int n_pages, mbuf_per_pg, leftover; 823 uint64_t total_mem, mbuf_mem, obj_sz; 824 825 /* there is no good way to predict how much space the mempool will 826 * occupy because it will allocate chunks on the fly, and some of those 827 * will come from default DPDK memory while some will come from our 828 * external memory, so just assume 128MB will be enough for everyone. 829 */ 830 uint64_t hdr_mem = 128 << 20; 831 832 /* account for possible non-contiguousness */ 833 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 834 if (obj_sz > pgsz) { 835 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 836 return -1; 837 } 838 839 mbuf_per_pg = pgsz / obj_sz; 840 leftover = (nb_mbufs % mbuf_per_pg) > 0; 841 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 842 843 mbuf_mem = n_pages * pgsz; 844 845 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 846 847 if (total_mem > SIZE_MAX) { 848 TESTPMD_LOG(ERR, "Memory size too big\n"); 849 return -1; 850 } 851 *out = (size_t)total_mem; 852 853 return 0; 854 } 855 856 static int 857 pagesz_flags(uint64_t page_sz) 858 { 859 /* as per mmap() manpage, all page sizes are log2 of page size 860 * shifted by MAP_HUGE_SHIFT 861 */ 862 int log2 = rte_log2_u64(page_sz); 863 864 return (log2 << HUGE_SHIFT); 865 } 866 867 static void * 868 alloc_mem(size_t memsz, size_t pgsz, bool huge) 869 { 870 void *addr; 871 int flags; 872 873 /* allocate anonymous hugepages */ 874 flags = MAP_ANONYMOUS | MAP_PRIVATE; 875 if (huge) 876 flags |= HUGE_FLAG | pagesz_flags(pgsz); 877 878 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 879 if (addr == MAP_FAILED) 880 return NULL; 881 882 return addr; 883 } 884 885 struct extmem_param { 886 void *addr; 887 size_t len; 888 size_t pgsz; 889 rte_iova_t *iova_table; 890 unsigned int iova_table_len; 891 }; 892 893 static int 894 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 895 bool huge) 896 { 897 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 898 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 899 unsigned int cur_page, n_pages, pgsz_idx; 900 size_t mem_sz, cur_pgsz; 901 rte_iova_t *iovas = NULL; 902 void *addr; 903 int ret; 904 905 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 906 /* skip anything that is too big */ 907 if (pgsizes[pgsz_idx] > SIZE_MAX) 908 continue; 909 910 cur_pgsz = pgsizes[pgsz_idx]; 911 912 /* if we were told not to allocate hugepages, override */ 913 if (!huge) 914 cur_pgsz = sysconf(_SC_PAGESIZE); 915 916 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 917 if (ret < 0) { 918 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 919 return -1; 920 } 921 922 /* allocate our memory */ 923 addr = alloc_mem(mem_sz, cur_pgsz, huge); 924 925 /* if we couldn't allocate memory with a specified page size, 926 * that doesn't mean we can't do it with other page sizes, so 927 * try another one. 928 */ 929 if (addr == NULL) 930 continue; 931 932 /* store IOVA addresses for every page in this memory area */ 933 n_pages = mem_sz / cur_pgsz; 934 935 iovas = malloc(sizeof(*iovas) * n_pages); 936 937 if (iovas == NULL) { 938 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 939 goto fail; 940 } 941 /* lock memory if it's not huge pages */ 942 if (!huge) 943 mlock(addr, mem_sz); 944 945 /* populate IOVA addresses */ 946 for (cur_page = 0; cur_page < n_pages; cur_page++) { 947 rte_iova_t iova; 948 size_t offset; 949 void *cur; 950 951 offset = cur_pgsz * cur_page; 952 cur = RTE_PTR_ADD(addr, offset); 953 954 /* touch the page before getting its IOVA */ 955 *(volatile char *)cur = 0; 956 957 iova = rte_mem_virt2iova(cur); 958 959 iovas[cur_page] = iova; 960 } 961 962 break; 963 } 964 /* if we couldn't allocate anything */ 965 if (iovas == NULL) 966 return -1; 967 968 param->addr = addr; 969 param->len = mem_sz; 970 param->pgsz = cur_pgsz; 971 param->iova_table = iovas; 972 param->iova_table_len = n_pages; 973 974 return 0; 975 fail: 976 free(iovas); 977 if (addr) 978 munmap(addr, mem_sz); 979 980 return -1; 981 } 982 983 static int 984 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 985 { 986 struct extmem_param param; 987 int socket_id, ret; 988 989 memset(¶m, 0, sizeof(param)); 990 991 /* check if our heap exists */ 992 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 993 if (socket_id < 0) { 994 /* create our heap */ 995 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 996 if (ret < 0) { 997 TESTPMD_LOG(ERR, "Cannot create heap\n"); 998 return -1; 999 } 1000 } 1001 1002 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 1003 if (ret < 0) { 1004 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 1005 return -1; 1006 } 1007 1008 /* we now have a valid memory area, so add it to heap */ 1009 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 1010 param.addr, param.len, param.iova_table, 1011 param.iova_table_len, param.pgsz); 1012 1013 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1014 1015 /* not needed any more */ 1016 free(param.iova_table); 1017 1018 if (ret < 0) { 1019 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1020 munmap(param.addr, param.len); 1021 return -1; 1022 } 1023 1024 /* success */ 1025 1026 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1027 param.len >> 20); 1028 1029 return 0; 1030 } 1031 static void 1032 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1033 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1034 { 1035 uint16_t pid = 0; 1036 int ret; 1037 1038 RTE_ETH_FOREACH_DEV(pid) { 1039 struct rte_eth_dev_info dev_info; 1040 1041 ret = eth_dev_info_get_print_err(pid, &dev_info); 1042 if (ret != 0) { 1043 TESTPMD_LOG(DEBUG, 1044 "unable to get device info for port %d on addr 0x%p," 1045 "mempool unmapping will not be performed\n", 1046 pid, memhdr->addr); 1047 continue; 1048 } 1049 1050 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1051 if (ret) { 1052 TESTPMD_LOG(DEBUG, 1053 "unable to DMA unmap addr 0x%p " 1054 "for device %s\n", 1055 memhdr->addr, dev_info.device->name); 1056 } 1057 } 1058 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1059 if (ret) { 1060 TESTPMD_LOG(DEBUG, 1061 "unable to un-register addr 0x%p\n", memhdr->addr); 1062 } 1063 } 1064 1065 static void 1066 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1067 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1068 { 1069 uint16_t pid = 0; 1070 size_t page_size = sysconf(_SC_PAGESIZE); 1071 int ret; 1072 1073 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1074 page_size); 1075 if (ret) { 1076 TESTPMD_LOG(DEBUG, 1077 "unable to register addr 0x%p\n", memhdr->addr); 1078 return; 1079 } 1080 RTE_ETH_FOREACH_DEV(pid) { 1081 struct rte_eth_dev_info dev_info; 1082 1083 ret = eth_dev_info_get_print_err(pid, &dev_info); 1084 if (ret != 0) { 1085 TESTPMD_LOG(DEBUG, 1086 "unable to get device info for port %d on addr 0x%p," 1087 "mempool mapping will not be performed\n", 1088 pid, memhdr->addr); 1089 continue; 1090 } 1091 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1092 if (ret) { 1093 TESTPMD_LOG(DEBUG, 1094 "unable to DMA map addr 0x%p " 1095 "for device %s\n", 1096 memhdr->addr, dev_info.device->name); 1097 } 1098 } 1099 } 1100 #endif 1101 1102 static unsigned int 1103 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1104 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1105 { 1106 struct rte_pktmbuf_extmem *xmem; 1107 unsigned int ext_num, zone_num, elt_num; 1108 uint16_t elt_size; 1109 1110 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1111 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1112 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1113 1114 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1115 if (xmem == NULL) { 1116 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1117 "external buffer descriptors\n"); 1118 *ext_mem = NULL; 1119 return 0; 1120 } 1121 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1122 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1123 const struct rte_memzone *mz; 1124 char mz_name[RTE_MEMZONE_NAMESIZE]; 1125 int ret; 1126 1127 ret = snprintf(mz_name, sizeof(mz_name), 1128 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1129 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1130 errno = ENAMETOOLONG; 1131 ext_num = 0; 1132 break; 1133 } 1134 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1135 socket_id, 1136 RTE_MEMZONE_IOVA_CONTIG | 1137 RTE_MEMZONE_1GB | 1138 RTE_MEMZONE_SIZE_HINT_ONLY); 1139 if (mz == NULL) { 1140 /* 1141 * The caller exits on external buffer creation 1142 * error, so there is no need to free memzones. 1143 */ 1144 errno = ENOMEM; 1145 ext_num = 0; 1146 break; 1147 } 1148 xseg->buf_ptr = mz->addr; 1149 xseg->buf_iova = mz->iova; 1150 xseg->buf_len = EXTBUF_ZONE_SIZE; 1151 xseg->elt_size = elt_size; 1152 } 1153 if (ext_num == 0 && xmem != NULL) { 1154 free(xmem); 1155 xmem = NULL; 1156 } 1157 *ext_mem = xmem; 1158 return ext_num; 1159 } 1160 1161 /* 1162 * Configuration initialisation done once at init time. 1163 */ 1164 static struct rte_mempool * 1165 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1166 unsigned int socket_id, uint16_t size_idx) 1167 { 1168 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1169 struct rte_mempool *rte_mp = NULL; 1170 #ifndef RTE_EXEC_ENV_WINDOWS 1171 uint32_t mb_size; 1172 1173 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1174 #endif 1175 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1176 if (!is_proc_primary()) { 1177 rte_mp = rte_mempool_lookup(pool_name); 1178 if (rte_mp == NULL) 1179 rte_exit(EXIT_FAILURE, 1180 "Get mbuf pool for socket %u failed: %s\n", 1181 socket_id, rte_strerror(rte_errno)); 1182 return rte_mp; 1183 } 1184 1185 TESTPMD_LOG(INFO, 1186 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1187 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1188 1189 switch (mp_alloc_type) { 1190 case MP_ALLOC_NATIVE: 1191 { 1192 /* wrapper to rte_mempool_create() */ 1193 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1194 rte_mbuf_best_mempool_ops()); 1195 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1196 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1197 break; 1198 } 1199 #ifndef RTE_EXEC_ENV_WINDOWS 1200 case MP_ALLOC_ANON: 1201 { 1202 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1203 mb_size, (unsigned int) mb_mempool_cache, 1204 sizeof(struct rte_pktmbuf_pool_private), 1205 socket_id, mempool_flags); 1206 if (rte_mp == NULL) 1207 goto err; 1208 1209 if (rte_mempool_populate_anon(rte_mp) == 0) { 1210 rte_mempool_free(rte_mp); 1211 rte_mp = NULL; 1212 goto err; 1213 } 1214 rte_pktmbuf_pool_init(rte_mp, NULL); 1215 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1216 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1217 break; 1218 } 1219 case MP_ALLOC_XMEM: 1220 case MP_ALLOC_XMEM_HUGE: 1221 { 1222 int heap_socket; 1223 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1224 1225 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1226 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1227 1228 heap_socket = 1229 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1230 if (heap_socket < 0) 1231 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1232 1233 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1234 rte_mbuf_best_mempool_ops()); 1235 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1236 mb_mempool_cache, 0, mbuf_seg_size, 1237 heap_socket); 1238 break; 1239 } 1240 #endif 1241 case MP_ALLOC_XBUF: 1242 { 1243 struct rte_pktmbuf_extmem *ext_mem; 1244 unsigned int ext_num; 1245 1246 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1247 socket_id, pool_name, &ext_mem); 1248 if (ext_num == 0) 1249 rte_exit(EXIT_FAILURE, 1250 "Can't create pinned data buffers\n"); 1251 1252 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1253 rte_mbuf_best_mempool_ops()); 1254 rte_mp = rte_pktmbuf_pool_create_extbuf 1255 (pool_name, nb_mbuf, mb_mempool_cache, 1256 0, mbuf_seg_size, socket_id, 1257 ext_mem, ext_num); 1258 free(ext_mem); 1259 break; 1260 } 1261 default: 1262 { 1263 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1264 } 1265 } 1266 1267 #ifndef RTE_EXEC_ENV_WINDOWS 1268 err: 1269 #endif 1270 if (rte_mp == NULL) { 1271 rte_exit(EXIT_FAILURE, 1272 "Creation of mbuf pool for socket %u failed: %s\n", 1273 socket_id, rte_strerror(rte_errno)); 1274 } else if (verbose_level > 0) { 1275 rte_mempool_dump(stdout, rte_mp); 1276 } 1277 return rte_mp; 1278 } 1279 1280 /* 1281 * Check given socket id is valid or not with NUMA mode, 1282 * if valid, return 0, else return -1 1283 */ 1284 static int 1285 check_socket_id(const unsigned int socket_id) 1286 { 1287 static int warning_once = 0; 1288 1289 if (new_socket_id(socket_id)) { 1290 if (!warning_once && numa_support) 1291 fprintf(stderr, 1292 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1293 warning_once = 1; 1294 return -1; 1295 } 1296 return 0; 1297 } 1298 1299 /* 1300 * Get the allowed maximum number of RX queues. 1301 * *pid return the port id which has minimal value of 1302 * max_rx_queues in all ports. 1303 */ 1304 queueid_t 1305 get_allowed_max_nb_rxq(portid_t *pid) 1306 { 1307 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1308 bool max_rxq_valid = false; 1309 portid_t pi; 1310 struct rte_eth_dev_info dev_info; 1311 1312 RTE_ETH_FOREACH_DEV(pi) { 1313 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1314 continue; 1315 1316 max_rxq_valid = true; 1317 if (dev_info.max_rx_queues < allowed_max_rxq) { 1318 allowed_max_rxq = dev_info.max_rx_queues; 1319 *pid = pi; 1320 } 1321 } 1322 return max_rxq_valid ? allowed_max_rxq : 0; 1323 } 1324 1325 /* 1326 * Check input rxq is valid or not. 1327 * If input rxq is not greater than any of maximum number 1328 * of RX queues of all ports, it is valid. 1329 * if valid, return 0, else return -1 1330 */ 1331 int 1332 check_nb_rxq(queueid_t rxq) 1333 { 1334 queueid_t allowed_max_rxq; 1335 portid_t pid = 0; 1336 1337 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1338 if (rxq > allowed_max_rxq) { 1339 fprintf(stderr, 1340 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1341 rxq, allowed_max_rxq, pid); 1342 return -1; 1343 } 1344 return 0; 1345 } 1346 1347 /* 1348 * Get the allowed maximum number of TX queues. 1349 * *pid return the port id which has minimal value of 1350 * max_tx_queues in all ports. 1351 */ 1352 queueid_t 1353 get_allowed_max_nb_txq(portid_t *pid) 1354 { 1355 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1356 bool max_txq_valid = false; 1357 portid_t pi; 1358 struct rte_eth_dev_info dev_info; 1359 1360 RTE_ETH_FOREACH_DEV(pi) { 1361 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1362 continue; 1363 1364 max_txq_valid = true; 1365 if (dev_info.max_tx_queues < allowed_max_txq) { 1366 allowed_max_txq = dev_info.max_tx_queues; 1367 *pid = pi; 1368 } 1369 } 1370 return max_txq_valid ? allowed_max_txq : 0; 1371 } 1372 1373 /* 1374 * Check input txq is valid or not. 1375 * If input txq is not greater than any of maximum number 1376 * of TX queues of all ports, it is valid. 1377 * if valid, return 0, else return -1 1378 */ 1379 int 1380 check_nb_txq(queueid_t txq) 1381 { 1382 queueid_t allowed_max_txq; 1383 portid_t pid = 0; 1384 1385 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1386 if (txq > allowed_max_txq) { 1387 fprintf(stderr, 1388 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1389 txq, allowed_max_txq, pid); 1390 return -1; 1391 } 1392 return 0; 1393 } 1394 1395 /* 1396 * Get the allowed maximum number of RXDs of every rx queue. 1397 * *pid return the port id which has minimal value of 1398 * max_rxd in all queues of all ports. 1399 */ 1400 static uint16_t 1401 get_allowed_max_nb_rxd(portid_t *pid) 1402 { 1403 uint16_t allowed_max_rxd = UINT16_MAX; 1404 portid_t pi; 1405 struct rte_eth_dev_info dev_info; 1406 1407 RTE_ETH_FOREACH_DEV(pi) { 1408 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1409 continue; 1410 1411 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1412 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1413 *pid = pi; 1414 } 1415 } 1416 return allowed_max_rxd; 1417 } 1418 1419 /* 1420 * Get the allowed minimal number of RXDs of every rx queue. 1421 * *pid return the port id which has minimal value of 1422 * min_rxd in all queues of all ports. 1423 */ 1424 static uint16_t 1425 get_allowed_min_nb_rxd(portid_t *pid) 1426 { 1427 uint16_t allowed_min_rxd = 0; 1428 portid_t pi; 1429 struct rte_eth_dev_info dev_info; 1430 1431 RTE_ETH_FOREACH_DEV(pi) { 1432 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1433 continue; 1434 1435 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1436 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1437 *pid = pi; 1438 } 1439 } 1440 1441 return allowed_min_rxd; 1442 } 1443 1444 /* 1445 * Check input rxd is valid or not. 1446 * If input rxd is not greater than any of maximum number 1447 * of RXDs of every Rx queues and is not less than any of 1448 * minimal number of RXDs of every Rx queues, it is valid. 1449 * if valid, return 0, else return -1 1450 */ 1451 int 1452 check_nb_rxd(queueid_t rxd) 1453 { 1454 uint16_t allowed_max_rxd; 1455 uint16_t allowed_min_rxd; 1456 portid_t pid = 0; 1457 1458 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1459 if (rxd > allowed_max_rxd) { 1460 fprintf(stderr, 1461 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1462 rxd, allowed_max_rxd, pid); 1463 return -1; 1464 } 1465 1466 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1467 if (rxd < allowed_min_rxd) { 1468 fprintf(stderr, 1469 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1470 rxd, allowed_min_rxd, pid); 1471 return -1; 1472 } 1473 1474 return 0; 1475 } 1476 1477 /* 1478 * Get the allowed maximum number of TXDs of every rx queues. 1479 * *pid return the port id which has minimal value of 1480 * max_txd in every tx queue. 1481 */ 1482 static uint16_t 1483 get_allowed_max_nb_txd(portid_t *pid) 1484 { 1485 uint16_t allowed_max_txd = UINT16_MAX; 1486 portid_t pi; 1487 struct rte_eth_dev_info dev_info; 1488 1489 RTE_ETH_FOREACH_DEV(pi) { 1490 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1491 continue; 1492 1493 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1494 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1495 *pid = pi; 1496 } 1497 } 1498 return allowed_max_txd; 1499 } 1500 1501 /* 1502 * Get the allowed maximum number of TXDs of every tx queues. 1503 * *pid return the port id which has minimal value of 1504 * min_txd in every tx queue. 1505 */ 1506 static uint16_t 1507 get_allowed_min_nb_txd(portid_t *pid) 1508 { 1509 uint16_t allowed_min_txd = 0; 1510 portid_t pi; 1511 struct rte_eth_dev_info dev_info; 1512 1513 RTE_ETH_FOREACH_DEV(pi) { 1514 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1515 continue; 1516 1517 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1518 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1519 *pid = pi; 1520 } 1521 } 1522 1523 return allowed_min_txd; 1524 } 1525 1526 /* 1527 * Check input txd is valid or not. 1528 * If input txd is not greater than any of maximum number 1529 * of TXDs of every Rx queues, it is valid. 1530 * if valid, return 0, else return -1 1531 */ 1532 int 1533 check_nb_txd(queueid_t txd) 1534 { 1535 uint16_t allowed_max_txd; 1536 uint16_t allowed_min_txd; 1537 portid_t pid = 0; 1538 1539 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1540 if (txd > allowed_max_txd) { 1541 fprintf(stderr, 1542 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1543 txd, allowed_max_txd, pid); 1544 return -1; 1545 } 1546 1547 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1548 if (txd < allowed_min_txd) { 1549 fprintf(stderr, 1550 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1551 txd, allowed_min_txd, pid); 1552 return -1; 1553 } 1554 return 0; 1555 } 1556 1557 1558 /* 1559 * Get the allowed maximum number of hairpin queues. 1560 * *pid return the port id which has minimal value of 1561 * max_hairpin_queues in all ports. 1562 */ 1563 queueid_t 1564 get_allowed_max_nb_hairpinq(portid_t *pid) 1565 { 1566 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1567 portid_t pi; 1568 struct rte_eth_hairpin_cap cap; 1569 1570 RTE_ETH_FOREACH_DEV(pi) { 1571 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1572 *pid = pi; 1573 return 0; 1574 } 1575 if (cap.max_nb_queues < allowed_max_hairpinq) { 1576 allowed_max_hairpinq = cap.max_nb_queues; 1577 *pid = pi; 1578 } 1579 } 1580 return allowed_max_hairpinq; 1581 } 1582 1583 /* 1584 * Check input hairpin is valid or not. 1585 * If input hairpin is not greater than any of maximum number 1586 * of hairpin queues of all ports, it is valid. 1587 * if valid, return 0, else return -1 1588 */ 1589 int 1590 check_nb_hairpinq(queueid_t hairpinq) 1591 { 1592 queueid_t allowed_max_hairpinq; 1593 portid_t pid = 0; 1594 1595 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1596 if (hairpinq > allowed_max_hairpinq) { 1597 fprintf(stderr, 1598 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1599 hairpinq, allowed_max_hairpinq, pid); 1600 return -1; 1601 } 1602 return 0; 1603 } 1604 1605 static int 1606 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1607 { 1608 uint32_t eth_overhead; 1609 1610 if (dev_info->max_mtu != UINT16_MAX && 1611 dev_info->max_rx_pktlen > dev_info->max_mtu) 1612 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1613 else 1614 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1615 1616 return eth_overhead; 1617 } 1618 1619 static void 1620 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1621 { 1622 struct rte_port *port = &ports[pid]; 1623 int ret; 1624 int i; 1625 1626 eth_rx_metadata_negotiate_mp(pid); 1627 1628 port->dev_conf.txmode = tx_mode; 1629 port->dev_conf.rxmode = rx_mode; 1630 1631 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1632 if (ret != 0) 1633 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1634 1635 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1636 port->dev_conf.txmode.offloads &= 1637 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1638 1639 /* Apply Rx offloads configuration */ 1640 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1641 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1642 /* Apply Tx offloads configuration */ 1643 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1644 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1645 1646 if (eth_link_speed) 1647 port->dev_conf.link_speeds = eth_link_speed; 1648 1649 if (max_rx_pkt_len) 1650 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1651 get_eth_overhead(&port->dev_info); 1652 1653 /* set flag to initialize port/queue */ 1654 port->need_reconfig = 1; 1655 port->need_reconfig_queues = 1; 1656 port->socket_id = socket_id; 1657 port->tx_metadata = 0; 1658 1659 /* 1660 * Check for maximum number of segments per MTU. 1661 * Accordingly update the mbuf data size. 1662 */ 1663 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1664 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1665 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1666 uint16_t mtu; 1667 1668 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1669 uint16_t data_size = (mtu + eth_overhead) / 1670 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1671 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1672 1673 if (buffer_size > mbuf_data_size[0]) { 1674 mbuf_data_size[0] = buffer_size; 1675 TESTPMD_LOG(WARNING, 1676 "Configured mbuf size of the first segment %hu\n", 1677 mbuf_data_size[0]); 1678 } 1679 } 1680 } 1681 } 1682 1683 static void 1684 init_config(void) 1685 { 1686 portid_t pid; 1687 struct rte_mempool *mbp; 1688 unsigned int nb_mbuf_per_pool; 1689 lcoreid_t lc_id; 1690 #ifdef RTE_LIB_GRO 1691 struct rte_gro_param gro_param; 1692 #endif 1693 #ifdef RTE_LIB_GSO 1694 uint32_t gso_types; 1695 #endif 1696 1697 /* Configuration of logical cores. */ 1698 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1699 sizeof(struct fwd_lcore *) * nb_lcores, 1700 RTE_CACHE_LINE_SIZE); 1701 if (fwd_lcores == NULL) { 1702 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1703 "failed\n", nb_lcores); 1704 } 1705 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1706 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1707 sizeof(struct fwd_lcore), 1708 RTE_CACHE_LINE_SIZE); 1709 if (fwd_lcores[lc_id] == NULL) { 1710 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1711 "failed\n"); 1712 } 1713 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1714 } 1715 1716 RTE_ETH_FOREACH_DEV(pid) { 1717 uint32_t socket_id; 1718 1719 if (numa_support) { 1720 socket_id = port_numa[pid]; 1721 if (port_numa[pid] == NUMA_NO_CONFIG) { 1722 socket_id = rte_eth_dev_socket_id(pid); 1723 1724 /* 1725 * if socket_id is invalid, 1726 * set to the first available socket. 1727 */ 1728 if (check_socket_id(socket_id) < 0) 1729 socket_id = socket_ids[0]; 1730 } 1731 } else { 1732 socket_id = (socket_num == UMA_NO_CONFIG) ? 1733 0 : socket_num; 1734 } 1735 /* Apply default TxRx configuration for all ports */ 1736 init_config_port_offloads(pid, socket_id); 1737 } 1738 /* 1739 * Create pools of mbuf. 1740 * If NUMA support is disabled, create a single pool of mbuf in 1741 * socket 0 memory by default. 1742 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1743 * 1744 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1745 * nb_txd can be configured at run time. 1746 */ 1747 if (param_total_num_mbufs) 1748 nb_mbuf_per_pool = param_total_num_mbufs; 1749 else { 1750 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1751 (nb_lcores * mb_mempool_cache) + 1752 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1753 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1754 } 1755 1756 if (numa_support) { 1757 uint8_t i, j; 1758 1759 for (i = 0; i < num_sockets; i++) 1760 for (j = 0; j < mbuf_data_size_n; j++) 1761 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1762 mbuf_pool_create(mbuf_data_size[j], 1763 nb_mbuf_per_pool, 1764 socket_ids[i], j); 1765 } else { 1766 uint8_t i; 1767 1768 for (i = 0; i < mbuf_data_size_n; i++) 1769 mempools[i] = mbuf_pool_create 1770 (mbuf_data_size[i], 1771 nb_mbuf_per_pool, 1772 socket_num == UMA_NO_CONFIG ? 1773 0 : socket_num, i); 1774 } 1775 1776 init_port_config(); 1777 1778 #ifdef RTE_LIB_GSO 1779 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1780 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1781 #endif 1782 /* 1783 * Records which Mbuf pool to use by each logical core, if needed. 1784 */ 1785 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1786 mbp = mbuf_pool_find( 1787 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1788 1789 if (mbp == NULL) 1790 mbp = mbuf_pool_find(0, 0); 1791 fwd_lcores[lc_id]->mbp = mbp; 1792 #ifdef RTE_LIB_GSO 1793 /* initialize GSO context */ 1794 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1795 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1796 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1797 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1798 RTE_ETHER_CRC_LEN; 1799 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1800 #endif 1801 } 1802 1803 fwd_config_setup(); 1804 1805 #ifdef RTE_LIB_GRO 1806 /* create a gro context for each lcore */ 1807 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1808 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1809 gro_param.max_item_per_flow = MAX_PKT_BURST; 1810 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1811 gro_param.socket_id = rte_lcore_to_socket_id( 1812 fwd_lcores_cpuids[lc_id]); 1813 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1814 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1815 rte_exit(EXIT_FAILURE, 1816 "rte_gro_ctx_create() failed\n"); 1817 } 1818 } 1819 #endif 1820 } 1821 1822 1823 void 1824 reconfig(portid_t new_port_id, unsigned socket_id) 1825 { 1826 /* Reconfiguration of Ethernet ports. */ 1827 init_config_port_offloads(new_port_id, socket_id); 1828 init_port_config(); 1829 } 1830 1831 int 1832 init_fwd_streams(void) 1833 { 1834 portid_t pid; 1835 struct rte_port *port; 1836 streamid_t sm_id, nb_fwd_streams_new; 1837 queueid_t q; 1838 1839 /* set socket id according to numa or not */ 1840 RTE_ETH_FOREACH_DEV(pid) { 1841 port = &ports[pid]; 1842 if (nb_rxq > port->dev_info.max_rx_queues) { 1843 fprintf(stderr, 1844 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1845 nb_rxq, port->dev_info.max_rx_queues); 1846 return -1; 1847 } 1848 if (nb_txq > port->dev_info.max_tx_queues) { 1849 fprintf(stderr, 1850 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1851 nb_txq, port->dev_info.max_tx_queues); 1852 return -1; 1853 } 1854 if (numa_support) { 1855 if (port_numa[pid] != NUMA_NO_CONFIG) 1856 port->socket_id = port_numa[pid]; 1857 else { 1858 port->socket_id = rte_eth_dev_socket_id(pid); 1859 1860 /* 1861 * if socket_id is invalid, 1862 * set to the first available socket. 1863 */ 1864 if (check_socket_id(port->socket_id) < 0) 1865 port->socket_id = socket_ids[0]; 1866 } 1867 } 1868 else { 1869 if (socket_num == UMA_NO_CONFIG) 1870 port->socket_id = 0; 1871 else 1872 port->socket_id = socket_num; 1873 } 1874 } 1875 1876 q = RTE_MAX(nb_rxq, nb_txq); 1877 if (q == 0) { 1878 fprintf(stderr, 1879 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1880 return -1; 1881 } 1882 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1883 if (nb_fwd_streams_new == nb_fwd_streams) 1884 return 0; 1885 /* clear the old */ 1886 if (fwd_streams != NULL) { 1887 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1888 if (fwd_streams[sm_id] == NULL) 1889 continue; 1890 rte_free(fwd_streams[sm_id]); 1891 fwd_streams[sm_id] = NULL; 1892 } 1893 rte_free(fwd_streams); 1894 fwd_streams = NULL; 1895 } 1896 1897 /* init new */ 1898 nb_fwd_streams = nb_fwd_streams_new; 1899 if (nb_fwd_streams) { 1900 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1901 sizeof(struct fwd_stream *) * nb_fwd_streams, 1902 RTE_CACHE_LINE_SIZE); 1903 if (fwd_streams == NULL) 1904 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1905 " (struct fwd_stream *)) failed\n", 1906 nb_fwd_streams); 1907 1908 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1909 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1910 " struct fwd_stream", sizeof(struct fwd_stream), 1911 RTE_CACHE_LINE_SIZE); 1912 if (fwd_streams[sm_id] == NULL) 1913 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1914 "(struct fwd_stream) failed\n"); 1915 } 1916 } 1917 1918 return 0; 1919 } 1920 1921 static void 1922 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1923 { 1924 uint64_t total_burst, sburst; 1925 uint64_t nb_burst; 1926 uint64_t burst_stats[4]; 1927 uint16_t pktnb_stats[4]; 1928 uint16_t nb_pkt; 1929 int burst_percent[4], sburstp; 1930 int i; 1931 1932 /* 1933 * First compute the total number of packet bursts and the 1934 * two highest numbers of bursts of the same number of packets. 1935 */ 1936 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1937 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1938 1939 /* Show stats for 0 burst size always */ 1940 total_burst = pbs->pkt_burst_spread[0]; 1941 burst_stats[0] = pbs->pkt_burst_spread[0]; 1942 pktnb_stats[0] = 0; 1943 1944 /* Find the next 2 burst sizes with highest occurrences. */ 1945 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1946 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1947 1948 if (nb_burst == 0) 1949 continue; 1950 1951 total_burst += nb_burst; 1952 1953 if (nb_burst > burst_stats[1]) { 1954 burst_stats[2] = burst_stats[1]; 1955 pktnb_stats[2] = pktnb_stats[1]; 1956 burst_stats[1] = nb_burst; 1957 pktnb_stats[1] = nb_pkt; 1958 } else if (nb_burst > burst_stats[2]) { 1959 burst_stats[2] = nb_burst; 1960 pktnb_stats[2] = nb_pkt; 1961 } 1962 } 1963 if (total_burst == 0) 1964 return; 1965 1966 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1967 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1968 if (i == 3) { 1969 printf("%d%% of other]\n", 100 - sburstp); 1970 return; 1971 } 1972 1973 sburst += burst_stats[i]; 1974 if (sburst == total_burst) { 1975 printf("%d%% of %d pkts]\n", 1976 100 - sburstp, (int) pktnb_stats[i]); 1977 return; 1978 } 1979 1980 burst_percent[i] = 1981 (double)burst_stats[i] / total_burst * 100; 1982 printf("%d%% of %d pkts + ", 1983 burst_percent[i], (int) pktnb_stats[i]); 1984 sburstp += burst_percent[i]; 1985 } 1986 } 1987 1988 static void 1989 fwd_stream_stats_display(streamid_t stream_id) 1990 { 1991 struct fwd_stream *fs; 1992 static const char *fwd_top_stats_border = "-------"; 1993 1994 fs = fwd_streams[stream_id]; 1995 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1996 (fs->fwd_dropped == 0)) 1997 return; 1998 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1999 "TX Port=%2d/Queue=%2d %s\n", 2000 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 2001 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 2002 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 2003 " TX-dropped: %-14"PRIu64, 2004 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 2005 2006 /* if checksum mode */ 2007 if (cur_fwd_eng == &csum_fwd_engine) { 2008 printf(" RX- bad IP checksum: %-14"PRIu64 2009 " Rx- bad L4 checksum: %-14"PRIu64 2010 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 2011 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 2012 fs->rx_bad_outer_l4_csum); 2013 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2014 fs->rx_bad_outer_ip_csum); 2015 } else { 2016 printf("\n"); 2017 } 2018 2019 if (record_burst_stats) { 2020 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2021 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2022 } 2023 } 2024 2025 void 2026 fwd_stats_display(void) 2027 { 2028 static const char *fwd_stats_border = "----------------------"; 2029 static const char *acc_stats_border = "+++++++++++++++"; 2030 struct { 2031 struct fwd_stream *rx_stream; 2032 struct fwd_stream *tx_stream; 2033 uint64_t tx_dropped; 2034 uint64_t rx_bad_ip_csum; 2035 uint64_t rx_bad_l4_csum; 2036 uint64_t rx_bad_outer_l4_csum; 2037 uint64_t rx_bad_outer_ip_csum; 2038 } ports_stats[RTE_MAX_ETHPORTS]; 2039 uint64_t total_rx_dropped = 0; 2040 uint64_t total_tx_dropped = 0; 2041 uint64_t total_rx_nombuf = 0; 2042 struct rte_eth_stats stats; 2043 uint64_t fwd_cycles = 0; 2044 uint64_t total_recv = 0; 2045 uint64_t total_xmit = 0; 2046 struct rte_port *port; 2047 streamid_t sm_id; 2048 portid_t pt_id; 2049 int ret; 2050 int i; 2051 2052 memset(ports_stats, 0, sizeof(ports_stats)); 2053 2054 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2055 struct fwd_stream *fs = fwd_streams[sm_id]; 2056 2057 if (cur_fwd_config.nb_fwd_streams > 2058 cur_fwd_config.nb_fwd_ports) { 2059 fwd_stream_stats_display(sm_id); 2060 } else { 2061 ports_stats[fs->tx_port].tx_stream = fs; 2062 ports_stats[fs->rx_port].rx_stream = fs; 2063 } 2064 2065 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2066 2067 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2068 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2069 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2070 fs->rx_bad_outer_l4_csum; 2071 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2072 fs->rx_bad_outer_ip_csum; 2073 2074 if (record_core_cycles) 2075 fwd_cycles += fs->core_cycles; 2076 } 2077 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2078 pt_id = fwd_ports_ids[i]; 2079 port = &ports[pt_id]; 2080 2081 ret = rte_eth_stats_get(pt_id, &stats); 2082 if (ret != 0) { 2083 fprintf(stderr, 2084 "%s: Error: failed to get stats (port %u): %d", 2085 __func__, pt_id, ret); 2086 continue; 2087 } 2088 stats.ipackets -= port->stats.ipackets; 2089 stats.opackets -= port->stats.opackets; 2090 stats.ibytes -= port->stats.ibytes; 2091 stats.obytes -= port->stats.obytes; 2092 stats.imissed -= port->stats.imissed; 2093 stats.oerrors -= port->stats.oerrors; 2094 stats.rx_nombuf -= port->stats.rx_nombuf; 2095 2096 total_recv += stats.ipackets; 2097 total_xmit += stats.opackets; 2098 total_rx_dropped += stats.imissed; 2099 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2100 total_tx_dropped += stats.oerrors; 2101 total_rx_nombuf += stats.rx_nombuf; 2102 2103 printf("\n %s Forward statistics for port %-2d %s\n", 2104 fwd_stats_border, pt_id, fwd_stats_border); 2105 2106 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2107 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2108 stats.ipackets + stats.imissed); 2109 2110 if (cur_fwd_eng == &csum_fwd_engine) { 2111 printf(" Bad-ipcsum: %-14"PRIu64 2112 " Bad-l4csum: %-14"PRIu64 2113 "Bad-outer-l4csum: %-14"PRIu64"\n", 2114 ports_stats[pt_id].rx_bad_ip_csum, 2115 ports_stats[pt_id].rx_bad_l4_csum, 2116 ports_stats[pt_id].rx_bad_outer_l4_csum); 2117 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2118 ports_stats[pt_id].rx_bad_outer_ip_csum); 2119 } 2120 if (stats.ierrors + stats.rx_nombuf > 0) { 2121 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2122 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2123 } 2124 2125 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2126 "TX-total: %-"PRIu64"\n", 2127 stats.opackets, ports_stats[pt_id].tx_dropped, 2128 stats.opackets + ports_stats[pt_id].tx_dropped); 2129 2130 if (record_burst_stats) { 2131 if (ports_stats[pt_id].rx_stream) 2132 pkt_burst_stats_display("RX", 2133 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2134 if (ports_stats[pt_id].tx_stream) 2135 pkt_burst_stats_display("TX", 2136 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2137 } 2138 2139 printf(" %s--------------------------------%s\n", 2140 fwd_stats_border, fwd_stats_border); 2141 } 2142 2143 printf("\n %s Accumulated forward statistics for all ports" 2144 "%s\n", 2145 acc_stats_border, acc_stats_border); 2146 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2147 "%-"PRIu64"\n" 2148 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2149 "%-"PRIu64"\n", 2150 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2151 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2152 if (total_rx_nombuf > 0) 2153 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2154 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2155 "%s\n", 2156 acc_stats_border, acc_stats_border); 2157 if (record_core_cycles) { 2158 #define CYC_PER_MHZ 1E6 2159 if (total_recv > 0 || total_xmit > 0) { 2160 uint64_t total_pkts = 0; 2161 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2162 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2163 total_pkts = total_xmit; 2164 else 2165 total_pkts = total_recv; 2166 2167 printf("\n CPU cycles/packet=%.2F (total cycles=" 2168 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2169 " MHz Clock\n", 2170 (double) fwd_cycles / total_pkts, 2171 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2172 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2173 } 2174 } 2175 } 2176 2177 void 2178 fwd_stats_reset(void) 2179 { 2180 streamid_t sm_id; 2181 portid_t pt_id; 2182 int ret; 2183 int i; 2184 2185 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2186 pt_id = fwd_ports_ids[i]; 2187 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2188 if (ret != 0) 2189 fprintf(stderr, 2190 "%s: Error: failed to clear stats (port %u):%d", 2191 __func__, pt_id, ret); 2192 } 2193 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2194 struct fwd_stream *fs = fwd_streams[sm_id]; 2195 2196 fs->rx_packets = 0; 2197 fs->tx_packets = 0; 2198 fs->fwd_dropped = 0; 2199 fs->rx_bad_ip_csum = 0; 2200 fs->rx_bad_l4_csum = 0; 2201 fs->rx_bad_outer_l4_csum = 0; 2202 fs->rx_bad_outer_ip_csum = 0; 2203 2204 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2205 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2206 fs->core_cycles = 0; 2207 } 2208 } 2209 2210 static void 2211 flush_fwd_rx_queues(void) 2212 { 2213 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2214 portid_t rxp; 2215 portid_t port_id; 2216 queueid_t rxq; 2217 uint16_t nb_rx; 2218 uint16_t i; 2219 uint8_t j; 2220 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2221 uint64_t timer_period; 2222 2223 if (num_procs > 1) { 2224 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2225 return; 2226 } 2227 2228 /* convert to number of cycles */ 2229 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2230 2231 for (j = 0; j < 2; j++) { 2232 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2233 for (rxq = 0; rxq < nb_rxq; rxq++) { 2234 port_id = fwd_ports_ids[rxp]; 2235 2236 /* Polling stopped queues is prohibited. */ 2237 if (ports[port_id].rxq[rxq].state == 2238 RTE_ETH_QUEUE_STATE_STOPPED) 2239 continue; 2240 2241 /** 2242 * testpmd can stuck in the below do while loop 2243 * if rte_eth_rx_burst() always returns nonzero 2244 * packets. So timer is added to exit this loop 2245 * after 1sec timer expiry. 2246 */ 2247 prev_tsc = rte_rdtsc(); 2248 do { 2249 nb_rx = rte_eth_rx_burst(port_id, rxq, 2250 pkts_burst, MAX_PKT_BURST); 2251 for (i = 0; i < nb_rx; i++) 2252 rte_pktmbuf_free(pkts_burst[i]); 2253 2254 cur_tsc = rte_rdtsc(); 2255 diff_tsc = cur_tsc - prev_tsc; 2256 timer_tsc += diff_tsc; 2257 } while ((nb_rx > 0) && 2258 (timer_tsc < timer_period)); 2259 timer_tsc = 0; 2260 } 2261 } 2262 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2263 } 2264 } 2265 2266 static void 2267 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2268 { 2269 struct fwd_stream **fsm; 2270 streamid_t nb_fs; 2271 streamid_t sm_id; 2272 #ifdef RTE_LIB_BITRATESTATS 2273 uint64_t tics_per_1sec; 2274 uint64_t tics_datum; 2275 uint64_t tics_current; 2276 uint16_t i, cnt_ports; 2277 2278 cnt_ports = nb_ports; 2279 tics_datum = rte_rdtsc(); 2280 tics_per_1sec = rte_get_timer_hz(); 2281 #endif 2282 fsm = &fwd_streams[fc->stream_idx]; 2283 nb_fs = fc->stream_nb; 2284 do { 2285 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2286 if (!fsm[sm_id]->disabled) 2287 (*pkt_fwd)(fsm[sm_id]); 2288 #ifdef RTE_LIB_BITRATESTATS 2289 if (bitrate_enabled != 0 && 2290 bitrate_lcore_id == rte_lcore_id()) { 2291 tics_current = rte_rdtsc(); 2292 if (tics_current - tics_datum >= tics_per_1sec) { 2293 /* Periodic bitrate calculation */ 2294 for (i = 0; i < cnt_ports; i++) 2295 rte_stats_bitrate_calc(bitrate_data, 2296 ports_ids[i]); 2297 tics_datum = tics_current; 2298 } 2299 } 2300 #endif 2301 #ifdef RTE_LIB_LATENCYSTATS 2302 if (latencystats_enabled != 0 && 2303 latencystats_lcore_id == rte_lcore_id()) 2304 rte_latencystats_update(); 2305 #endif 2306 2307 } while (! fc->stopped); 2308 } 2309 2310 static int 2311 start_pkt_forward_on_core(void *fwd_arg) 2312 { 2313 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2314 cur_fwd_config.fwd_eng->packet_fwd); 2315 return 0; 2316 } 2317 2318 /* 2319 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2320 * Used to start communication flows in network loopback test configurations. 2321 */ 2322 static int 2323 run_one_txonly_burst_on_core(void *fwd_arg) 2324 { 2325 struct fwd_lcore *fwd_lc; 2326 struct fwd_lcore tmp_lcore; 2327 2328 fwd_lc = (struct fwd_lcore *) fwd_arg; 2329 tmp_lcore = *fwd_lc; 2330 tmp_lcore.stopped = 1; 2331 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2332 return 0; 2333 } 2334 2335 /* 2336 * Launch packet forwarding: 2337 * - Setup per-port forwarding context. 2338 * - launch logical cores with their forwarding configuration. 2339 */ 2340 static void 2341 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2342 { 2343 unsigned int i; 2344 unsigned int lc_id; 2345 int diag; 2346 2347 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2348 lc_id = fwd_lcores_cpuids[i]; 2349 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2350 fwd_lcores[i]->stopped = 0; 2351 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2352 fwd_lcores[i], lc_id); 2353 if (diag != 0) 2354 fprintf(stderr, 2355 "launch lcore %u failed - diag=%d\n", 2356 lc_id, diag); 2357 } 2358 } 2359 } 2360 2361 /* 2362 * Launch packet forwarding configuration. 2363 */ 2364 void 2365 start_packet_forwarding(int with_tx_first) 2366 { 2367 port_fwd_begin_t port_fwd_begin; 2368 port_fwd_end_t port_fwd_end; 2369 stream_init_t stream_init = cur_fwd_eng->stream_init; 2370 unsigned int i; 2371 2372 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2373 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2374 2375 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2376 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2377 2378 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2379 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2380 (!nb_rxq || !nb_txq)) 2381 rte_exit(EXIT_FAILURE, 2382 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2383 cur_fwd_eng->fwd_mode_name); 2384 2385 if (all_ports_started() == 0) { 2386 fprintf(stderr, "Not all ports were started\n"); 2387 return; 2388 } 2389 if (test_done == 0) { 2390 fprintf(stderr, "Packet forwarding already started\n"); 2391 return; 2392 } 2393 2394 fwd_config_setup(); 2395 2396 pkt_fwd_config_display(&cur_fwd_config); 2397 if (!pkt_fwd_shared_rxq_check()) 2398 return; 2399 2400 if (stream_init != NULL) 2401 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2402 stream_init(fwd_streams[i]); 2403 2404 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2405 if (port_fwd_begin != NULL) { 2406 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2407 if (port_fwd_begin(fwd_ports_ids[i])) { 2408 fprintf(stderr, 2409 "Packet forwarding is not ready\n"); 2410 return; 2411 } 2412 } 2413 } 2414 2415 if (with_tx_first) { 2416 port_fwd_begin = tx_only_engine.port_fwd_begin; 2417 if (port_fwd_begin != NULL) { 2418 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2419 if (port_fwd_begin(fwd_ports_ids[i])) { 2420 fprintf(stderr, 2421 "Packet forwarding is not ready\n"); 2422 return; 2423 } 2424 } 2425 } 2426 } 2427 2428 test_done = 0; 2429 2430 if(!no_flush_rx) 2431 flush_fwd_rx_queues(); 2432 2433 rxtx_config_display(); 2434 2435 fwd_stats_reset(); 2436 if (with_tx_first) { 2437 while (with_tx_first--) { 2438 launch_packet_forwarding( 2439 run_one_txonly_burst_on_core); 2440 rte_eal_mp_wait_lcore(); 2441 } 2442 port_fwd_end = tx_only_engine.port_fwd_end; 2443 if (port_fwd_end != NULL) { 2444 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2445 (*port_fwd_end)(fwd_ports_ids[i]); 2446 } 2447 } 2448 launch_packet_forwarding(start_pkt_forward_on_core); 2449 } 2450 2451 void 2452 stop_packet_forwarding(void) 2453 { 2454 port_fwd_end_t port_fwd_end; 2455 lcoreid_t lc_id; 2456 portid_t pt_id; 2457 int i; 2458 2459 if (test_done) { 2460 fprintf(stderr, "Packet forwarding not started\n"); 2461 return; 2462 } 2463 printf("Telling cores to stop..."); 2464 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2465 fwd_lcores[lc_id]->stopped = 1; 2466 printf("\nWaiting for lcores to finish...\n"); 2467 rte_eal_mp_wait_lcore(); 2468 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2469 if (port_fwd_end != NULL) { 2470 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2471 pt_id = fwd_ports_ids[i]; 2472 (*port_fwd_end)(pt_id); 2473 } 2474 } 2475 2476 fwd_stats_display(); 2477 2478 printf("\nDone.\n"); 2479 test_done = 1; 2480 } 2481 2482 void 2483 dev_set_link_up(portid_t pid) 2484 { 2485 if (rte_eth_dev_set_link_up(pid) < 0) 2486 fprintf(stderr, "\nSet link up fail.\n"); 2487 } 2488 2489 void 2490 dev_set_link_down(portid_t pid) 2491 { 2492 if (rte_eth_dev_set_link_down(pid) < 0) 2493 fprintf(stderr, "\nSet link down fail.\n"); 2494 } 2495 2496 static int 2497 all_ports_started(void) 2498 { 2499 portid_t pi; 2500 struct rte_port *port; 2501 2502 RTE_ETH_FOREACH_DEV(pi) { 2503 port = &ports[pi]; 2504 /* Check if there is a port which is not started */ 2505 if ((port->port_status != RTE_PORT_STARTED) && 2506 (port->slave_flag == 0)) 2507 return 0; 2508 } 2509 2510 /* No port is not started */ 2511 return 1; 2512 } 2513 2514 int 2515 port_is_stopped(portid_t port_id) 2516 { 2517 struct rte_port *port = &ports[port_id]; 2518 2519 if ((port->port_status != RTE_PORT_STOPPED) && 2520 (port->slave_flag == 0)) 2521 return 0; 2522 return 1; 2523 } 2524 2525 int 2526 all_ports_stopped(void) 2527 { 2528 portid_t pi; 2529 2530 RTE_ETH_FOREACH_DEV(pi) { 2531 if (!port_is_stopped(pi)) 2532 return 0; 2533 } 2534 2535 return 1; 2536 } 2537 2538 int 2539 port_is_started(portid_t port_id) 2540 { 2541 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2542 return 0; 2543 2544 if (ports[port_id].port_status != RTE_PORT_STARTED) 2545 return 0; 2546 2547 return 1; 2548 } 2549 2550 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2551 static int 2552 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2553 { 2554 queueid_t qi; 2555 struct rte_eth_hairpin_conf hairpin_conf = { 2556 .peer_count = 1, 2557 }; 2558 int i; 2559 int diag; 2560 struct rte_port *port = &ports[pi]; 2561 uint16_t peer_rx_port = pi; 2562 uint16_t peer_tx_port = pi; 2563 uint32_t manual = 1; 2564 uint32_t tx_exp = hairpin_mode & 0x10; 2565 2566 if (!(hairpin_mode & 0xf)) { 2567 peer_rx_port = pi; 2568 peer_tx_port = pi; 2569 manual = 0; 2570 } else if (hairpin_mode & 0x1) { 2571 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2572 RTE_ETH_DEV_NO_OWNER); 2573 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2574 peer_tx_port = rte_eth_find_next_owned_by(0, 2575 RTE_ETH_DEV_NO_OWNER); 2576 if (p_pi != RTE_MAX_ETHPORTS) { 2577 peer_rx_port = p_pi; 2578 } else { 2579 uint16_t next_pi; 2580 2581 /* Last port will be the peer RX port of the first. */ 2582 RTE_ETH_FOREACH_DEV(next_pi) 2583 peer_rx_port = next_pi; 2584 } 2585 manual = 1; 2586 } else if (hairpin_mode & 0x2) { 2587 if (cnt_pi & 0x1) { 2588 peer_rx_port = p_pi; 2589 } else { 2590 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2591 RTE_ETH_DEV_NO_OWNER); 2592 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2593 peer_rx_port = pi; 2594 } 2595 peer_tx_port = peer_rx_port; 2596 manual = 1; 2597 } 2598 2599 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2600 hairpin_conf.peers[0].port = peer_rx_port; 2601 hairpin_conf.peers[0].queue = i + nb_rxq; 2602 hairpin_conf.manual_bind = !!manual; 2603 hairpin_conf.tx_explicit = !!tx_exp; 2604 diag = rte_eth_tx_hairpin_queue_setup 2605 (pi, qi, nb_txd, &hairpin_conf); 2606 i++; 2607 if (diag == 0) 2608 continue; 2609 2610 /* Fail to setup rx queue, return */ 2611 if (port->port_status == RTE_PORT_HANDLING) 2612 port->port_status = RTE_PORT_STOPPED; 2613 else 2614 fprintf(stderr, 2615 "Port %d can not be set back to stopped\n", pi); 2616 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2617 pi); 2618 /* try to reconfigure queues next time */ 2619 port->need_reconfig_queues = 1; 2620 return -1; 2621 } 2622 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2623 hairpin_conf.peers[0].port = peer_tx_port; 2624 hairpin_conf.peers[0].queue = i + nb_txq; 2625 hairpin_conf.manual_bind = !!manual; 2626 hairpin_conf.tx_explicit = !!tx_exp; 2627 diag = rte_eth_rx_hairpin_queue_setup 2628 (pi, qi, nb_rxd, &hairpin_conf); 2629 i++; 2630 if (diag == 0) 2631 continue; 2632 2633 /* Fail to setup rx queue, return */ 2634 if (port->port_status == RTE_PORT_HANDLING) 2635 port->port_status = RTE_PORT_STOPPED; 2636 else 2637 fprintf(stderr, 2638 "Port %d can not be set back to stopped\n", pi); 2639 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2640 pi); 2641 /* try to reconfigure queues next time */ 2642 port->need_reconfig_queues = 1; 2643 return -1; 2644 } 2645 return 0; 2646 } 2647 2648 /* Configure the Rx with optional split. */ 2649 int 2650 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2651 uint16_t nb_rx_desc, unsigned int socket_id, 2652 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2653 { 2654 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2655 unsigned int i, mp_n; 2656 int ret; 2657 2658 if (rx_pkt_nb_segs <= 1 || 2659 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2660 rx_conf->rx_seg = NULL; 2661 rx_conf->rx_nseg = 0; 2662 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2663 nb_rx_desc, socket_id, 2664 rx_conf, mp); 2665 goto exit; 2666 } 2667 for (i = 0; i < rx_pkt_nb_segs; i++) { 2668 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2669 struct rte_mempool *mpx; 2670 /* 2671 * Use last valid pool for the segments with number 2672 * exceeding the pool index. 2673 */ 2674 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2675 mpx = mbuf_pool_find(socket_id, mp_n); 2676 /* Handle zero as mbuf data buffer size. */ 2677 rx_seg->length = rx_pkt_seg_lengths[i] ? 2678 rx_pkt_seg_lengths[i] : 2679 mbuf_data_size[mp_n]; 2680 rx_seg->offset = i < rx_pkt_nb_offs ? 2681 rx_pkt_seg_offsets[i] : 0; 2682 rx_seg->mp = mpx ? mpx : mp; 2683 } 2684 rx_conf->rx_nseg = rx_pkt_nb_segs; 2685 rx_conf->rx_seg = rx_useg; 2686 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2687 socket_id, rx_conf, NULL); 2688 rx_conf->rx_seg = NULL; 2689 rx_conf->rx_nseg = 0; 2690 exit: 2691 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2692 RTE_ETH_QUEUE_STATE_STOPPED : 2693 RTE_ETH_QUEUE_STATE_STARTED; 2694 return ret; 2695 } 2696 2697 static int 2698 alloc_xstats_display_info(portid_t pi) 2699 { 2700 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2701 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2702 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2703 2704 if (xstats_display_num == 0) 2705 return 0; 2706 2707 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2708 if (*ids_supp == NULL) 2709 goto fail_ids_supp; 2710 2711 *prev_values = calloc(xstats_display_num, 2712 sizeof(**prev_values)); 2713 if (*prev_values == NULL) 2714 goto fail_prev_values; 2715 2716 *curr_values = calloc(xstats_display_num, 2717 sizeof(**curr_values)); 2718 if (*curr_values == NULL) 2719 goto fail_curr_values; 2720 2721 ports[pi].xstats_info.allocated = true; 2722 2723 return 0; 2724 2725 fail_curr_values: 2726 free(*prev_values); 2727 fail_prev_values: 2728 free(*ids_supp); 2729 fail_ids_supp: 2730 return -ENOMEM; 2731 } 2732 2733 static void 2734 free_xstats_display_info(portid_t pi) 2735 { 2736 if (!ports[pi].xstats_info.allocated) 2737 return; 2738 free(ports[pi].xstats_info.ids_supp); 2739 free(ports[pi].xstats_info.prev_values); 2740 free(ports[pi].xstats_info.curr_values); 2741 ports[pi].xstats_info.allocated = false; 2742 } 2743 2744 /** Fill helper structures for specified port to show extended statistics. */ 2745 static void 2746 fill_xstats_display_info_for_port(portid_t pi) 2747 { 2748 unsigned int stat, stat_supp; 2749 const char *xstat_name; 2750 struct rte_port *port; 2751 uint64_t *ids_supp; 2752 int rc; 2753 2754 if (xstats_display_num == 0) 2755 return; 2756 2757 if (pi == (portid_t)RTE_PORT_ALL) { 2758 fill_xstats_display_info(); 2759 return; 2760 } 2761 2762 port = &ports[pi]; 2763 if (port->port_status != RTE_PORT_STARTED) 2764 return; 2765 2766 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2767 rte_exit(EXIT_FAILURE, 2768 "Failed to allocate xstats display memory\n"); 2769 2770 ids_supp = port->xstats_info.ids_supp; 2771 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2772 xstat_name = xstats_display[stat].name; 2773 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2774 ids_supp + stat_supp); 2775 if (rc != 0) { 2776 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2777 xstat_name, pi, stat); 2778 continue; 2779 } 2780 stat_supp++; 2781 } 2782 2783 port->xstats_info.ids_supp_sz = stat_supp; 2784 } 2785 2786 /** Fill helper structures for all ports to show extended statistics. */ 2787 static void 2788 fill_xstats_display_info(void) 2789 { 2790 portid_t pi; 2791 2792 if (xstats_display_num == 0) 2793 return; 2794 2795 RTE_ETH_FOREACH_DEV(pi) 2796 fill_xstats_display_info_for_port(pi); 2797 } 2798 2799 int 2800 start_port(portid_t pid) 2801 { 2802 int diag, need_check_link_status = -1; 2803 portid_t pi; 2804 portid_t p_pi = RTE_MAX_ETHPORTS; 2805 portid_t pl[RTE_MAX_ETHPORTS]; 2806 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2807 uint16_t cnt_pi = 0; 2808 uint16_t cfg_pi = 0; 2809 int peer_pi; 2810 queueid_t qi; 2811 struct rte_port *port; 2812 struct rte_eth_hairpin_cap cap; 2813 2814 if (port_id_is_invalid(pid, ENABLED_WARN)) 2815 return 0; 2816 2817 RTE_ETH_FOREACH_DEV(pi) { 2818 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2819 continue; 2820 2821 if (port_is_bonding_slave(pi)) { 2822 fprintf(stderr, 2823 "Please remove port %d from bonded device.\n", 2824 pi); 2825 continue; 2826 } 2827 2828 need_check_link_status = 0; 2829 port = &ports[pi]; 2830 if (port->port_status == RTE_PORT_STOPPED) 2831 port->port_status = RTE_PORT_HANDLING; 2832 else { 2833 fprintf(stderr, "Port %d is now not stopped\n", pi); 2834 continue; 2835 } 2836 2837 if (port->need_reconfig > 0) { 2838 struct rte_eth_conf dev_conf; 2839 int k; 2840 2841 port->need_reconfig = 0; 2842 2843 if (flow_isolate_all) { 2844 int ret = port_flow_isolate(pi, 1); 2845 if (ret) { 2846 fprintf(stderr, 2847 "Failed to apply isolated mode on port %d\n", 2848 pi); 2849 return -1; 2850 } 2851 } 2852 configure_rxtx_dump_callbacks(0); 2853 printf("Configuring Port %d (socket %u)\n", pi, 2854 port->socket_id); 2855 if (nb_hairpinq > 0 && 2856 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2857 fprintf(stderr, 2858 "Port %d doesn't support hairpin queues\n", 2859 pi); 2860 return -1; 2861 } 2862 2863 /* configure port */ 2864 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2865 nb_txq + nb_hairpinq, 2866 &(port->dev_conf)); 2867 if (diag != 0) { 2868 if (port->port_status == RTE_PORT_HANDLING) 2869 port->port_status = RTE_PORT_STOPPED; 2870 else 2871 fprintf(stderr, 2872 "Port %d can not be set back to stopped\n", 2873 pi); 2874 fprintf(stderr, "Fail to configure port %d\n", 2875 pi); 2876 /* try to reconfigure port next time */ 2877 port->need_reconfig = 1; 2878 return -1; 2879 } 2880 /* get device configuration*/ 2881 if (0 != 2882 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2883 fprintf(stderr, 2884 "port %d can not get device configuration\n", 2885 pi); 2886 return -1; 2887 } 2888 /* Apply Rx offloads configuration */ 2889 if (dev_conf.rxmode.offloads != 2890 port->dev_conf.rxmode.offloads) { 2891 port->dev_conf.rxmode.offloads |= 2892 dev_conf.rxmode.offloads; 2893 for (k = 0; 2894 k < port->dev_info.max_rx_queues; 2895 k++) 2896 port->rxq[k].conf.offloads |= 2897 dev_conf.rxmode.offloads; 2898 } 2899 /* Apply Tx offloads configuration */ 2900 if (dev_conf.txmode.offloads != 2901 port->dev_conf.txmode.offloads) { 2902 port->dev_conf.txmode.offloads |= 2903 dev_conf.txmode.offloads; 2904 for (k = 0; 2905 k < port->dev_info.max_tx_queues; 2906 k++) 2907 port->txq[k].conf.offloads |= 2908 dev_conf.txmode.offloads; 2909 } 2910 } 2911 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2912 port->need_reconfig_queues = 0; 2913 /* setup tx queues */ 2914 for (qi = 0; qi < nb_txq; qi++) { 2915 struct rte_eth_txconf *conf = 2916 &port->txq[qi].conf; 2917 2918 if ((numa_support) && 2919 (txring_numa[pi] != NUMA_NO_CONFIG)) 2920 diag = rte_eth_tx_queue_setup(pi, qi, 2921 port->nb_tx_desc[qi], 2922 txring_numa[pi], 2923 &(port->txq[qi].conf)); 2924 else 2925 diag = rte_eth_tx_queue_setup(pi, qi, 2926 port->nb_tx_desc[qi], 2927 port->socket_id, 2928 &(port->txq[qi].conf)); 2929 2930 if (diag == 0) { 2931 port->txq[qi].state = 2932 conf->tx_deferred_start ? 2933 RTE_ETH_QUEUE_STATE_STOPPED : 2934 RTE_ETH_QUEUE_STATE_STARTED; 2935 continue; 2936 } 2937 2938 /* Fail to setup tx queue, return */ 2939 if (port->port_status == RTE_PORT_HANDLING) 2940 port->port_status = RTE_PORT_STOPPED; 2941 else 2942 fprintf(stderr, 2943 "Port %d can not be set back to stopped\n", 2944 pi); 2945 fprintf(stderr, 2946 "Fail to configure port %d tx queues\n", 2947 pi); 2948 /* try to reconfigure queues next time */ 2949 port->need_reconfig_queues = 1; 2950 return -1; 2951 } 2952 for (qi = 0; qi < nb_rxq; qi++) { 2953 /* setup rx queues */ 2954 if ((numa_support) && 2955 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2956 struct rte_mempool * mp = 2957 mbuf_pool_find 2958 (rxring_numa[pi], 0); 2959 if (mp == NULL) { 2960 fprintf(stderr, 2961 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2962 rxring_numa[pi]); 2963 return -1; 2964 } 2965 2966 diag = rx_queue_setup(pi, qi, 2967 port->nb_rx_desc[qi], 2968 rxring_numa[pi], 2969 &(port->rxq[qi].conf), 2970 mp); 2971 } else { 2972 struct rte_mempool *mp = 2973 mbuf_pool_find 2974 (port->socket_id, 0); 2975 if (mp == NULL) { 2976 fprintf(stderr, 2977 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2978 port->socket_id); 2979 return -1; 2980 } 2981 diag = rx_queue_setup(pi, qi, 2982 port->nb_rx_desc[qi], 2983 port->socket_id, 2984 &(port->rxq[qi].conf), 2985 mp); 2986 } 2987 if (diag == 0) 2988 continue; 2989 2990 /* Fail to setup rx queue, return */ 2991 if (port->port_status == RTE_PORT_HANDLING) 2992 port->port_status = RTE_PORT_STOPPED; 2993 else 2994 fprintf(stderr, 2995 "Port %d can not be set back to stopped\n", 2996 pi); 2997 fprintf(stderr, 2998 "Fail to configure port %d rx queues\n", 2999 pi); 3000 /* try to reconfigure queues next time */ 3001 port->need_reconfig_queues = 1; 3002 return -1; 3003 } 3004 /* setup hairpin queues */ 3005 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3006 return -1; 3007 } 3008 configure_rxtx_dump_callbacks(verbose_level); 3009 if (clear_ptypes) { 3010 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3011 NULL, 0); 3012 if (diag < 0) 3013 fprintf(stderr, 3014 "Port %d: Failed to disable Ptype parsing\n", 3015 pi); 3016 } 3017 3018 p_pi = pi; 3019 cnt_pi++; 3020 3021 /* start port */ 3022 diag = eth_dev_start_mp(pi); 3023 if (diag < 0) { 3024 fprintf(stderr, "Fail to start port %d: %s\n", 3025 pi, rte_strerror(-diag)); 3026 3027 /* Fail to setup rx queue, return */ 3028 if (port->port_status == RTE_PORT_HANDLING) 3029 port->port_status = RTE_PORT_STOPPED; 3030 else 3031 fprintf(stderr, 3032 "Port %d can not be set back to stopped\n", 3033 pi); 3034 continue; 3035 } 3036 3037 if (port->port_status == RTE_PORT_HANDLING) 3038 port->port_status = RTE_PORT_STARTED; 3039 else 3040 fprintf(stderr, "Port %d can not be set into started\n", 3041 pi); 3042 3043 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3044 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3045 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3046 3047 /* at least one port started, need checking link status */ 3048 need_check_link_status = 1; 3049 3050 pl[cfg_pi++] = pi; 3051 } 3052 3053 if (need_check_link_status == 1 && !no_link_check) 3054 check_all_ports_link_status(RTE_PORT_ALL); 3055 else if (need_check_link_status == 0) 3056 fprintf(stderr, "Please stop the ports first\n"); 3057 3058 if (hairpin_mode & 0xf) { 3059 uint16_t i; 3060 int j; 3061 3062 /* bind all started hairpin ports */ 3063 for (i = 0; i < cfg_pi; i++) { 3064 pi = pl[i]; 3065 /* bind current Tx to all peer Rx */ 3066 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3067 RTE_MAX_ETHPORTS, 1); 3068 if (peer_pi < 0) 3069 return peer_pi; 3070 for (j = 0; j < peer_pi; j++) { 3071 if (!port_is_started(peer_pl[j])) 3072 continue; 3073 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3074 if (diag < 0) { 3075 fprintf(stderr, 3076 "Error during binding hairpin Tx port %u to %u: %s\n", 3077 pi, peer_pl[j], 3078 rte_strerror(-diag)); 3079 return -1; 3080 } 3081 } 3082 /* bind all peer Tx to current Rx */ 3083 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3084 RTE_MAX_ETHPORTS, 0); 3085 if (peer_pi < 0) 3086 return peer_pi; 3087 for (j = 0; j < peer_pi; j++) { 3088 if (!port_is_started(peer_pl[j])) 3089 continue; 3090 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3091 if (diag < 0) { 3092 fprintf(stderr, 3093 "Error during binding hairpin Tx port %u to %u: %s\n", 3094 peer_pl[j], pi, 3095 rte_strerror(-diag)); 3096 return -1; 3097 } 3098 } 3099 } 3100 } 3101 3102 fill_xstats_display_info_for_port(pid); 3103 3104 printf("Done\n"); 3105 return 0; 3106 } 3107 3108 void 3109 stop_port(portid_t pid) 3110 { 3111 portid_t pi; 3112 struct rte_port *port; 3113 int need_check_link_status = 0; 3114 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3115 int peer_pi; 3116 3117 if (port_id_is_invalid(pid, ENABLED_WARN)) 3118 return; 3119 3120 printf("Stopping ports...\n"); 3121 3122 RTE_ETH_FOREACH_DEV(pi) { 3123 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3124 continue; 3125 3126 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3127 fprintf(stderr, 3128 "Please remove port %d from forwarding configuration.\n", 3129 pi); 3130 continue; 3131 } 3132 3133 if (port_is_bonding_slave(pi)) { 3134 fprintf(stderr, 3135 "Please remove port %d from bonded device.\n", 3136 pi); 3137 continue; 3138 } 3139 3140 port = &ports[pi]; 3141 if (port->port_status == RTE_PORT_STARTED) 3142 port->port_status = RTE_PORT_HANDLING; 3143 else 3144 continue; 3145 3146 if (hairpin_mode & 0xf) { 3147 int j; 3148 3149 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3150 /* unbind all peer Tx from current Rx */ 3151 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3152 RTE_MAX_ETHPORTS, 0); 3153 if (peer_pi < 0) 3154 continue; 3155 for (j = 0; j < peer_pi; j++) { 3156 if (!port_is_started(peer_pl[j])) 3157 continue; 3158 rte_eth_hairpin_unbind(peer_pl[j], pi); 3159 } 3160 } 3161 3162 if (port->flow_list) 3163 port_flow_flush(pi); 3164 3165 if (eth_dev_stop_mp(pi) != 0) 3166 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3167 pi); 3168 3169 if (port->port_status == RTE_PORT_HANDLING) 3170 port->port_status = RTE_PORT_STOPPED; 3171 else 3172 fprintf(stderr, "Port %d can not be set into stopped\n", 3173 pi); 3174 need_check_link_status = 1; 3175 } 3176 if (need_check_link_status && !no_link_check) 3177 check_all_ports_link_status(RTE_PORT_ALL); 3178 3179 printf("Done\n"); 3180 } 3181 3182 static void 3183 remove_invalid_ports_in(portid_t *array, portid_t *total) 3184 { 3185 portid_t i; 3186 portid_t new_total = 0; 3187 3188 for (i = 0; i < *total; i++) 3189 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3190 array[new_total] = array[i]; 3191 new_total++; 3192 } 3193 *total = new_total; 3194 } 3195 3196 static void 3197 remove_invalid_ports(void) 3198 { 3199 remove_invalid_ports_in(ports_ids, &nb_ports); 3200 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3201 nb_cfg_ports = nb_fwd_ports; 3202 } 3203 3204 void 3205 close_port(portid_t pid) 3206 { 3207 portid_t pi; 3208 struct rte_port *port; 3209 3210 if (port_id_is_invalid(pid, ENABLED_WARN)) 3211 return; 3212 3213 printf("Closing ports...\n"); 3214 3215 RTE_ETH_FOREACH_DEV(pi) { 3216 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3217 continue; 3218 3219 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3220 fprintf(stderr, 3221 "Please remove port %d from forwarding configuration.\n", 3222 pi); 3223 continue; 3224 } 3225 3226 if (port_is_bonding_slave(pi)) { 3227 fprintf(stderr, 3228 "Please remove port %d from bonded device.\n", 3229 pi); 3230 continue; 3231 } 3232 3233 port = &ports[pi]; 3234 if (port->port_status == RTE_PORT_CLOSED) { 3235 fprintf(stderr, "Port %d is already closed\n", pi); 3236 continue; 3237 } 3238 3239 if (is_proc_primary()) { 3240 mcast_addr_pool_destroy(pi); 3241 port_flow_flush(pi); 3242 port_flex_item_flush(pi); 3243 port_action_handle_flush(pi); 3244 rte_eth_dev_close(pi); 3245 } 3246 3247 free_xstats_display_info(pi); 3248 } 3249 3250 remove_invalid_ports(); 3251 printf("Done\n"); 3252 } 3253 3254 void 3255 reset_port(portid_t pid) 3256 { 3257 int diag; 3258 portid_t pi; 3259 struct rte_port *port; 3260 3261 if (port_id_is_invalid(pid, ENABLED_WARN)) 3262 return; 3263 3264 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3265 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3266 fprintf(stderr, 3267 "Can not reset port(s), please stop port(s) first.\n"); 3268 return; 3269 } 3270 3271 printf("Resetting ports...\n"); 3272 3273 RTE_ETH_FOREACH_DEV(pi) { 3274 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3275 continue; 3276 3277 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3278 fprintf(stderr, 3279 "Please remove port %d from forwarding configuration.\n", 3280 pi); 3281 continue; 3282 } 3283 3284 if (port_is_bonding_slave(pi)) { 3285 fprintf(stderr, 3286 "Please remove port %d from bonded device.\n", 3287 pi); 3288 continue; 3289 } 3290 3291 diag = rte_eth_dev_reset(pi); 3292 if (diag == 0) { 3293 port = &ports[pi]; 3294 port->need_reconfig = 1; 3295 port->need_reconfig_queues = 1; 3296 } else { 3297 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3298 pi, diag); 3299 } 3300 } 3301 3302 printf("Done\n"); 3303 } 3304 3305 void 3306 attach_port(char *identifier) 3307 { 3308 portid_t pi; 3309 struct rte_dev_iterator iterator; 3310 3311 printf("Attaching a new port...\n"); 3312 3313 if (identifier == NULL) { 3314 fprintf(stderr, "Invalid parameters are specified\n"); 3315 return; 3316 } 3317 3318 if (rte_dev_probe(identifier) < 0) { 3319 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3320 return; 3321 } 3322 3323 /* first attach mode: event */ 3324 if (setup_on_probe_event) { 3325 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3326 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3327 if (ports[pi].port_status == RTE_PORT_HANDLING && 3328 ports[pi].need_setup != 0) 3329 setup_attached_port(pi); 3330 return; 3331 } 3332 3333 /* second attach mode: iterator */ 3334 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3335 /* setup ports matching the devargs used for probing */ 3336 if (port_is_forwarding(pi)) 3337 continue; /* port was already attached before */ 3338 setup_attached_port(pi); 3339 } 3340 } 3341 3342 static void 3343 setup_attached_port(portid_t pi) 3344 { 3345 unsigned int socket_id; 3346 int ret; 3347 3348 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3349 /* if socket_id is invalid, set to the first available socket. */ 3350 if (check_socket_id(socket_id) < 0) 3351 socket_id = socket_ids[0]; 3352 reconfig(pi, socket_id); 3353 ret = rte_eth_promiscuous_enable(pi); 3354 if (ret != 0) 3355 fprintf(stderr, 3356 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3357 pi, rte_strerror(-ret)); 3358 3359 ports_ids[nb_ports++] = pi; 3360 fwd_ports_ids[nb_fwd_ports++] = pi; 3361 nb_cfg_ports = nb_fwd_ports; 3362 ports[pi].need_setup = 0; 3363 ports[pi].port_status = RTE_PORT_STOPPED; 3364 3365 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3366 printf("Done\n"); 3367 } 3368 3369 static void 3370 detach_device(struct rte_device *dev) 3371 { 3372 portid_t sibling; 3373 3374 if (dev == NULL) { 3375 fprintf(stderr, "Device already removed\n"); 3376 return; 3377 } 3378 3379 printf("Removing a device...\n"); 3380 3381 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3382 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3383 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3384 fprintf(stderr, "Port %u not stopped\n", 3385 sibling); 3386 return; 3387 } 3388 port_flow_flush(sibling); 3389 } 3390 } 3391 3392 if (rte_dev_remove(dev) < 0) { 3393 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3394 return; 3395 } 3396 remove_invalid_ports(); 3397 3398 printf("Device is detached\n"); 3399 printf("Now total ports is %d\n", nb_ports); 3400 printf("Done\n"); 3401 return; 3402 } 3403 3404 void 3405 detach_port_device(portid_t port_id) 3406 { 3407 int ret; 3408 struct rte_eth_dev_info dev_info; 3409 3410 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3411 return; 3412 3413 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3414 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3415 fprintf(stderr, "Port not stopped\n"); 3416 return; 3417 } 3418 fprintf(stderr, "Port was not closed\n"); 3419 } 3420 3421 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3422 if (ret != 0) { 3423 TESTPMD_LOG(ERR, 3424 "Failed to get device info for port %d, not detaching\n", 3425 port_id); 3426 return; 3427 } 3428 detach_device(dev_info.device); 3429 } 3430 3431 void 3432 detach_devargs(char *identifier) 3433 { 3434 struct rte_dev_iterator iterator; 3435 struct rte_devargs da; 3436 portid_t port_id; 3437 3438 printf("Removing a device...\n"); 3439 3440 memset(&da, 0, sizeof(da)); 3441 if (rte_devargs_parsef(&da, "%s", identifier)) { 3442 fprintf(stderr, "cannot parse identifier\n"); 3443 return; 3444 } 3445 3446 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3447 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3448 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3449 fprintf(stderr, "Port %u not stopped\n", 3450 port_id); 3451 rte_eth_iterator_cleanup(&iterator); 3452 rte_devargs_reset(&da); 3453 return; 3454 } 3455 port_flow_flush(port_id); 3456 } 3457 } 3458 3459 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3460 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3461 da.name, da.bus->name); 3462 rte_devargs_reset(&da); 3463 return; 3464 } 3465 3466 remove_invalid_ports(); 3467 3468 printf("Device %s is detached\n", identifier); 3469 printf("Now total ports is %d\n", nb_ports); 3470 printf("Done\n"); 3471 rte_devargs_reset(&da); 3472 } 3473 3474 void 3475 pmd_test_exit(void) 3476 { 3477 portid_t pt_id; 3478 unsigned int i; 3479 int ret; 3480 3481 if (test_done == 0) 3482 stop_packet_forwarding(); 3483 3484 #ifndef RTE_EXEC_ENV_WINDOWS 3485 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3486 if (mempools[i]) { 3487 if (mp_alloc_type == MP_ALLOC_ANON) 3488 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3489 NULL); 3490 } 3491 } 3492 #endif 3493 if (ports != NULL) { 3494 no_link_check = 1; 3495 RTE_ETH_FOREACH_DEV(pt_id) { 3496 printf("\nStopping port %d...\n", pt_id); 3497 fflush(stdout); 3498 stop_port(pt_id); 3499 } 3500 RTE_ETH_FOREACH_DEV(pt_id) { 3501 printf("\nShutting down port %d...\n", pt_id); 3502 fflush(stdout); 3503 close_port(pt_id); 3504 } 3505 } 3506 3507 if (hot_plug) { 3508 ret = rte_dev_event_monitor_stop(); 3509 if (ret) { 3510 RTE_LOG(ERR, EAL, 3511 "fail to stop device event monitor."); 3512 return; 3513 } 3514 3515 ret = rte_dev_event_callback_unregister(NULL, 3516 dev_event_callback, NULL); 3517 if (ret < 0) { 3518 RTE_LOG(ERR, EAL, 3519 "fail to unregister device event callback.\n"); 3520 return; 3521 } 3522 3523 ret = rte_dev_hotplug_handle_disable(); 3524 if (ret) { 3525 RTE_LOG(ERR, EAL, 3526 "fail to disable hotplug handling.\n"); 3527 return; 3528 } 3529 } 3530 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3531 if (mempools[i]) 3532 mempool_free_mp(mempools[i]); 3533 } 3534 free(xstats_display); 3535 3536 printf("\nBye...\n"); 3537 } 3538 3539 typedef void (*cmd_func_t)(void); 3540 struct pmd_test_command { 3541 const char *cmd_name; 3542 cmd_func_t cmd_func; 3543 }; 3544 3545 /* Check the link status of all ports in up to 9s, and print them finally */ 3546 static void 3547 check_all_ports_link_status(uint32_t port_mask) 3548 { 3549 #define CHECK_INTERVAL 100 /* 100ms */ 3550 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3551 portid_t portid; 3552 uint8_t count, all_ports_up, print_flag = 0; 3553 struct rte_eth_link link; 3554 int ret; 3555 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3556 3557 printf("Checking link statuses...\n"); 3558 fflush(stdout); 3559 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3560 all_ports_up = 1; 3561 RTE_ETH_FOREACH_DEV(portid) { 3562 if ((port_mask & (1 << portid)) == 0) 3563 continue; 3564 memset(&link, 0, sizeof(link)); 3565 ret = rte_eth_link_get_nowait(portid, &link); 3566 if (ret < 0) { 3567 all_ports_up = 0; 3568 if (print_flag == 1) 3569 fprintf(stderr, 3570 "Port %u link get failed: %s\n", 3571 portid, rte_strerror(-ret)); 3572 continue; 3573 } 3574 /* print link status if flag set */ 3575 if (print_flag == 1) { 3576 rte_eth_link_to_str(link_status, 3577 sizeof(link_status), &link); 3578 printf("Port %d %s\n", portid, link_status); 3579 continue; 3580 } 3581 /* clear all_ports_up flag if any link down */ 3582 if (link.link_status == RTE_ETH_LINK_DOWN) { 3583 all_ports_up = 0; 3584 break; 3585 } 3586 } 3587 /* after finally printing all link status, get out */ 3588 if (print_flag == 1) 3589 break; 3590 3591 if (all_ports_up == 0) { 3592 fflush(stdout); 3593 rte_delay_ms(CHECK_INTERVAL); 3594 } 3595 3596 /* set the print_flag if all ports up or timeout */ 3597 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3598 print_flag = 1; 3599 } 3600 3601 if (lsc_interrupt) 3602 break; 3603 } 3604 } 3605 3606 static void 3607 rmv_port_callback(void *arg) 3608 { 3609 int need_to_start = 0; 3610 int org_no_link_check = no_link_check; 3611 portid_t port_id = (intptr_t)arg; 3612 struct rte_eth_dev_info dev_info; 3613 int ret; 3614 3615 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3616 3617 if (!test_done && port_is_forwarding(port_id)) { 3618 need_to_start = 1; 3619 stop_packet_forwarding(); 3620 } 3621 no_link_check = 1; 3622 stop_port(port_id); 3623 no_link_check = org_no_link_check; 3624 3625 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3626 if (ret != 0) 3627 TESTPMD_LOG(ERR, 3628 "Failed to get device info for port %d, not detaching\n", 3629 port_id); 3630 else { 3631 struct rte_device *device = dev_info.device; 3632 close_port(port_id); 3633 detach_device(device); /* might be already removed or have more ports */ 3634 } 3635 if (need_to_start) 3636 start_packet_forwarding(0); 3637 } 3638 3639 /* This function is used by the interrupt thread */ 3640 static int 3641 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3642 void *ret_param) 3643 { 3644 RTE_SET_USED(param); 3645 RTE_SET_USED(ret_param); 3646 3647 if (type >= RTE_ETH_EVENT_MAX) { 3648 fprintf(stderr, 3649 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3650 port_id, __func__, type); 3651 fflush(stderr); 3652 } else if (event_print_mask & (UINT32_C(1) << type)) { 3653 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3654 eth_event_desc[type]); 3655 fflush(stdout); 3656 } 3657 3658 switch (type) { 3659 case RTE_ETH_EVENT_NEW: 3660 ports[port_id].need_setup = 1; 3661 ports[port_id].port_status = RTE_PORT_HANDLING; 3662 break; 3663 case RTE_ETH_EVENT_INTR_RMV: 3664 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3665 break; 3666 if (rte_eal_alarm_set(100000, 3667 rmv_port_callback, (void *)(intptr_t)port_id)) 3668 fprintf(stderr, 3669 "Could not set up deferred device removal\n"); 3670 break; 3671 case RTE_ETH_EVENT_DESTROY: 3672 ports[port_id].port_status = RTE_PORT_CLOSED; 3673 printf("Port %u is closed\n", port_id); 3674 break; 3675 default: 3676 break; 3677 } 3678 return 0; 3679 } 3680 3681 static int 3682 register_eth_event_callback(void) 3683 { 3684 int ret; 3685 enum rte_eth_event_type event; 3686 3687 for (event = RTE_ETH_EVENT_UNKNOWN; 3688 event < RTE_ETH_EVENT_MAX; event++) { 3689 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3690 event, 3691 eth_event_callback, 3692 NULL); 3693 if (ret != 0) { 3694 TESTPMD_LOG(ERR, "Failed to register callback for " 3695 "%s event\n", eth_event_desc[event]); 3696 return -1; 3697 } 3698 } 3699 3700 return 0; 3701 } 3702 3703 /* This function is used by the interrupt thread */ 3704 static void 3705 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3706 __rte_unused void *arg) 3707 { 3708 uint16_t port_id; 3709 int ret; 3710 3711 if (type >= RTE_DEV_EVENT_MAX) { 3712 fprintf(stderr, "%s called upon invalid event %d\n", 3713 __func__, type); 3714 fflush(stderr); 3715 } 3716 3717 switch (type) { 3718 case RTE_DEV_EVENT_REMOVE: 3719 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3720 device_name); 3721 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3722 if (ret) { 3723 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3724 device_name); 3725 return; 3726 } 3727 /* 3728 * Because the user's callback is invoked in eal interrupt 3729 * callback, the interrupt callback need to be finished before 3730 * it can be unregistered when detaching device. So finish 3731 * callback soon and use a deferred removal to detach device 3732 * is need. It is a workaround, once the device detaching be 3733 * moved into the eal in the future, the deferred removal could 3734 * be deleted. 3735 */ 3736 if (rte_eal_alarm_set(100000, 3737 rmv_port_callback, (void *)(intptr_t)port_id)) 3738 RTE_LOG(ERR, EAL, 3739 "Could not set up deferred device removal\n"); 3740 break; 3741 case RTE_DEV_EVENT_ADD: 3742 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3743 device_name); 3744 /* TODO: After finish kernel driver binding, 3745 * begin to attach port. 3746 */ 3747 break; 3748 default: 3749 break; 3750 } 3751 } 3752 3753 static void 3754 rxtx_port_config(portid_t pid) 3755 { 3756 uint16_t qid; 3757 uint64_t offloads; 3758 struct rte_port *port = &ports[pid]; 3759 3760 for (qid = 0; qid < nb_rxq; qid++) { 3761 offloads = port->rxq[qid].conf.offloads; 3762 port->rxq[qid].conf = port->dev_info.default_rxconf; 3763 3764 if (rxq_share > 0 && 3765 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3766 /* Non-zero share group to enable RxQ share. */ 3767 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3768 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3769 } 3770 3771 if (offloads != 0) 3772 port->rxq[qid].conf.offloads = offloads; 3773 3774 /* Check if any Rx parameters have been passed */ 3775 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3776 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3777 3778 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3779 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3780 3781 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3782 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3783 3784 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3785 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3786 3787 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3788 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3789 3790 port->nb_rx_desc[qid] = nb_rxd; 3791 } 3792 3793 for (qid = 0; qid < nb_txq; qid++) { 3794 offloads = port->txq[qid].conf.offloads; 3795 port->txq[qid].conf = port->dev_info.default_txconf; 3796 if (offloads != 0) 3797 port->txq[qid].conf.offloads = offloads; 3798 3799 /* Check if any Tx parameters have been passed */ 3800 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3801 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3802 3803 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3804 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3805 3806 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3807 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3808 3809 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3810 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3811 3812 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3813 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3814 3815 port->nb_tx_desc[qid] = nb_txd; 3816 } 3817 } 3818 3819 /* 3820 * Helper function to set MTU from frame size 3821 * 3822 * port->dev_info should be set before calling this function. 3823 * 3824 * return 0 on success, negative on error 3825 */ 3826 int 3827 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3828 { 3829 struct rte_port *port = &ports[portid]; 3830 uint32_t eth_overhead; 3831 uint16_t mtu, new_mtu; 3832 3833 eth_overhead = get_eth_overhead(&port->dev_info); 3834 3835 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3836 printf("Failed to get MTU for port %u\n", portid); 3837 return -1; 3838 } 3839 3840 new_mtu = max_rx_pktlen - eth_overhead; 3841 3842 if (mtu == new_mtu) 3843 return 0; 3844 3845 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3846 fprintf(stderr, 3847 "Failed to set MTU to %u for port %u\n", 3848 new_mtu, portid); 3849 return -1; 3850 } 3851 3852 port->dev_conf.rxmode.mtu = new_mtu; 3853 3854 return 0; 3855 } 3856 3857 void 3858 init_port_config(void) 3859 { 3860 portid_t pid; 3861 struct rte_port *port; 3862 int ret, i; 3863 3864 RTE_ETH_FOREACH_DEV(pid) { 3865 port = &ports[pid]; 3866 port->dev_conf.fdir_conf = fdir_conf; 3867 3868 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3869 if (ret != 0) 3870 return; 3871 3872 if (nb_rxq > 1) { 3873 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3874 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3875 rss_hf & port->dev_info.flow_type_rss_offloads; 3876 } else { 3877 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3878 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3879 } 3880 3881 if (port->dcb_flag == 0) { 3882 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3883 port->dev_conf.rxmode.mq_mode = 3884 (enum rte_eth_rx_mq_mode) 3885 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3886 } else { 3887 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3888 port->dev_conf.rxmode.offloads &= 3889 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3890 3891 for (i = 0; 3892 i < port->dev_info.nb_rx_queues; 3893 i++) 3894 port->rxq[i].conf.offloads &= 3895 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3896 } 3897 } 3898 3899 rxtx_port_config(pid); 3900 3901 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3902 if (ret != 0) 3903 return; 3904 3905 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3906 rte_pmd_ixgbe_bypass_init(pid); 3907 #endif 3908 3909 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3910 port->dev_conf.intr_conf.lsc = 1; 3911 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3912 port->dev_conf.intr_conf.rmv = 1; 3913 } 3914 } 3915 3916 void set_port_slave_flag(portid_t slave_pid) 3917 { 3918 struct rte_port *port; 3919 3920 port = &ports[slave_pid]; 3921 port->slave_flag = 1; 3922 } 3923 3924 void clear_port_slave_flag(portid_t slave_pid) 3925 { 3926 struct rte_port *port; 3927 3928 port = &ports[slave_pid]; 3929 port->slave_flag = 0; 3930 } 3931 3932 uint8_t port_is_bonding_slave(portid_t slave_pid) 3933 { 3934 struct rte_port *port; 3935 struct rte_eth_dev_info dev_info; 3936 int ret; 3937 3938 port = &ports[slave_pid]; 3939 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3940 if (ret != 0) { 3941 TESTPMD_LOG(ERR, 3942 "Failed to get device info for port id %d," 3943 "cannot determine if the port is a bonded slave", 3944 slave_pid); 3945 return 0; 3946 } 3947 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3948 return 1; 3949 return 0; 3950 } 3951 3952 const uint16_t vlan_tags[] = { 3953 0, 1, 2, 3, 4, 5, 6, 7, 3954 8, 9, 10, 11, 12, 13, 14, 15, 3955 16, 17, 18, 19, 20, 21, 22, 23, 3956 24, 25, 26, 27, 28, 29, 30, 31 3957 }; 3958 3959 static int 3960 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3961 enum dcb_mode_enable dcb_mode, 3962 enum rte_eth_nb_tcs num_tcs, 3963 uint8_t pfc_en) 3964 { 3965 uint8_t i; 3966 int32_t rc; 3967 struct rte_eth_rss_conf rss_conf; 3968 3969 /* 3970 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3971 * given above, and the number of traffic classes available for use. 3972 */ 3973 if (dcb_mode == DCB_VT_ENABLED) { 3974 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3975 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3976 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3977 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3978 3979 /* VMDQ+DCB RX and TX configurations */ 3980 vmdq_rx_conf->enable_default_pool = 0; 3981 vmdq_rx_conf->default_pool = 0; 3982 vmdq_rx_conf->nb_queue_pools = 3983 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3984 vmdq_tx_conf->nb_queue_pools = 3985 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3986 3987 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3988 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3989 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3990 vmdq_rx_conf->pool_map[i].pools = 3991 1 << (i % vmdq_rx_conf->nb_queue_pools); 3992 } 3993 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3994 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3995 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3996 } 3997 3998 /* set DCB mode of RX and TX of multiple queues */ 3999 eth_conf->rxmode.mq_mode = 4000 (enum rte_eth_rx_mq_mode) 4001 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4002 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4003 } else { 4004 struct rte_eth_dcb_rx_conf *rx_conf = 4005 ð_conf->rx_adv_conf.dcb_rx_conf; 4006 struct rte_eth_dcb_tx_conf *tx_conf = 4007 ð_conf->tx_adv_conf.dcb_tx_conf; 4008 4009 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4010 4011 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4012 if (rc != 0) 4013 return rc; 4014 4015 rx_conf->nb_tcs = num_tcs; 4016 tx_conf->nb_tcs = num_tcs; 4017 4018 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4019 rx_conf->dcb_tc[i] = i % num_tcs; 4020 tx_conf->dcb_tc[i] = i % num_tcs; 4021 } 4022 4023 eth_conf->rxmode.mq_mode = 4024 (enum rte_eth_rx_mq_mode) 4025 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4026 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4027 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4028 } 4029 4030 if (pfc_en) 4031 eth_conf->dcb_capability_en = 4032 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4033 else 4034 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4035 4036 return 0; 4037 } 4038 4039 int 4040 init_port_dcb_config(portid_t pid, 4041 enum dcb_mode_enable dcb_mode, 4042 enum rte_eth_nb_tcs num_tcs, 4043 uint8_t pfc_en) 4044 { 4045 struct rte_eth_conf port_conf; 4046 struct rte_port *rte_port; 4047 int retval; 4048 uint16_t i; 4049 4050 if (num_procs > 1) { 4051 printf("The multi-process feature doesn't support dcb.\n"); 4052 return -ENOTSUP; 4053 } 4054 rte_port = &ports[pid]; 4055 4056 /* retain the original device configuration. */ 4057 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4058 4059 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4060 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4061 if (retval < 0) 4062 return retval; 4063 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4064 /* remove RSS HASH offload for DCB in vt mode */ 4065 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4066 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4067 for (i = 0; i < nb_rxq; i++) 4068 rte_port->rxq[i].conf.offloads &= 4069 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4070 } 4071 4072 /* re-configure the device . */ 4073 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4074 if (retval < 0) 4075 return retval; 4076 4077 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4078 if (retval != 0) 4079 return retval; 4080 4081 /* If dev_info.vmdq_pool_base is greater than 0, 4082 * the queue id of vmdq pools is started after pf queues. 4083 */ 4084 if (dcb_mode == DCB_VT_ENABLED && 4085 rte_port->dev_info.vmdq_pool_base > 0) { 4086 fprintf(stderr, 4087 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4088 pid); 4089 return -1; 4090 } 4091 4092 /* Assume the ports in testpmd have the same dcb capability 4093 * and has the same number of rxq and txq in dcb mode 4094 */ 4095 if (dcb_mode == DCB_VT_ENABLED) { 4096 if (rte_port->dev_info.max_vfs > 0) { 4097 nb_rxq = rte_port->dev_info.nb_rx_queues; 4098 nb_txq = rte_port->dev_info.nb_tx_queues; 4099 } else { 4100 nb_rxq = rte_port->dev_info.max_rx_queues; 4101 nb_txq = rte_port->dev_info.max_tx_queues; 4102 } 4103 } else { 4104 /*if vt is disabled, use all pf queues */ 4105 if (rte_port->dev_info.vmdq_pool_base == 0) { 4106 nb_rxq = rte_port->dev_info.max_rx_queues; 4107 nb_txq = rte_port->dev_info.max_tx_queues; 4108 } else { 4109 nb_rxq = (queueid_t)num_tcs; 4110 nb_txq = (queueid_t)num_tcs; 4111 4112 } 4113 } 4114 rx_free_thresh = 64; 4115 4116 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4117 4118 rxtx_port_config(pid); 4119 /* VLAN filter */ 4120 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4121 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4122 rx_vft_set(pid, vlan_tags[i], 1); 4123 4124 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4125 if (retval != 0) 4126 return retval; 4127 4128 rte_port->dcb_flag = 1; 4129 4130 /* Enter DCB configuration status */ 4131 dcb_config = 1; 4132 4133 return 0; 4134 } 4135 4136 static void 4137 init_port(void) 4138 { 4139 int i; 4140 4141 /* Configuration of Ethernet ports. */ 4142 ports = rte_zmalloc("testpmd: ports", 4143 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4144 RTE_CACHE_LINE_SIZE); 4145 if (ports == NULL) { 4146 rte_exit(EXIT_FAILURE, 4147 "rte_zmalloc(%d struct rte_port) failed\n", 4148 RTE_MAX_ETHPORTS); 4149 } 4150 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4151 ports[i].xstats_info.allocated = false; 4152 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4153 LIST_INIT(&ports[i].flow_tunnel_list); 4154 /* Initialize ports NUMA structures */ 4155 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4156 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4157 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4158 } 4159 4160 static void 4161 force_quit(void) 4162 { 4163 pmd_test_exit(); 4164 prompt_exit(); 4165 } 4166 4167 static void 4168 print_stats(void) 4169 { 4170 uint8_t i; 4171 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4172 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4173 4174 /* Clear screen and move to top left */ 4175 printf("%s%s", clr, top_left); 4176 4177 printf("\nPort statistics ===================================="); 4178 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4179 nic_stats_display(fwd_ports_ids[i]); 4180 4181 fflush(stdout); 4182 } 4183 4184 static void 4185 signal_handler(int signum) 4186 { 4187 if (signum == SIGINT || signum == SIGTERM) { 4188 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4189 signum); 4190 #ifdef RTE_LIB_PDUMP 4191 /* uninitialize packet capture framework */ 4192 rte_pdump_uninit(); 4193 #endif 4194 #ifdef RTE_LIB_LATENCYSTATS 4195 if (latencystats_enabled != 0) 4196 rte_latencystats_uninit(); 4197 #endif 4198 force_quit(); 4199 /* Set flag to indicate the force termination. */ 4200 f_quit = 1; 4201 /* exit with the expected status */ 4202 #ifndef RTE_EXEC_ENV_WINDOWS 4203 signal(signum, SIG_DFL); 4204 kill(getpid(), signum); 4205 #endif 4206 } 4207 } 4208 4209 int 4210 main(int argc, char** argv) 4211 { 4212 int diag; 4213 portid_t port_id; 4214 uint16_t count; 4215 int ret; 4216 4217 signal(SIGINT, signal_handler); 4218 signal(SIGTERM, signal_handler); 4219 4220 testpmd_logtype = rte_log_register("testpmd"); 4221 if (testpmd_logtype < 0) 4222 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4223 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4224 4225 diag = rte_eal_init(argc, argv); 4226 if (diag < 0) 4227 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4228 rte_strerror(rte_errno)); 4229 4230 ret = register_eth_event_callback(); 4231 if (ret != 0) 4232 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4233 4234 #ifdef RTE_LIB_PDUMP 4235 /* initialize packet capture framework */ 4236 rte_pdump_init(); 4237 #endif 4238 4239 count = 0; 4240 RTE_ETH_FOREACH_DEV(port_id) { 4241 ports_ids[count] = port_id; 4242 count++; 4243 } 4244 nb_ports = (portid_t) count; 4245 if (nb_ports == 0) 4246 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4247 4248 /* allocate port structures, and init them */ 4249 init_port(); 4250 4251 set_def_fwd_config(); 4252 if (nb_lcores == 0) 4253 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4254 "Check the core mask argument\n"); 4255 4256 /* Bitrate/latency stats disabled by default */ 4257 #ifdef RTE_LIB_BITRATESTATS 4258 bitrate_enabled = 0; 4259 #endif 4260 #ifdef RTE_LIB_LATENCYSTATS 4261 latencystats_enabled = 0; 4262 #endif 4263 4264 /* on FreeBSD, mlockall() is disabled by default */ 4265 #ifdef RTE_EXEC_ENV_FREEBSD 4266 do_mlockall = 0; 4267 #else 4268 do_mlockall = 1; 4269 #endif 4270 4271 argc -= diag; 4272 argv += diag; 4273 if (argc > 1) 4274 launch_args_parse(argc, argv); 4275 4276 #ifndef RTE_EXEC_ENV_WINDOWS 4277 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4278 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4279 strerror(errno)); 4280 } 4281 #endif 4282 4283 if (tx_first && interactive) 4284 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4285 "interactive mode.\n"); 4286 4287 if (tx_first && lsc_interrupt) { 4288 fprintf(stderr, 4289 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4290 lsc_interrupt = 0; 4291 } 4292 4293 if (!nb_rxq && !nb_txq) 4294 fprintf(stderr, 4295 "Warning: Either rx or tx queues should be non-zero\n"); 4296 4297 if (nb_rxq > 1 && nb_rxq > nb_txq) 4298 fprintf(stderr, 4299 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4300 nb_rxq, nb_txq); 4301 4302 init_config(); 4303 4304 if (hot_plug) { 4305 ret = rte_dev_hotplug_handle_enable(); 4306 if (ret) { 4307 RTE_LOG(ERR, EAL, 4308 "fail to enable hotplug handling."); 4309 return -1; 4310 } 4311 4312 ret = rte_dev_event_monitor_start(); 4313 if (ret) { 4314 RTE_LOG(ERR, EAL, 4315 "fail to start device event monitoring."); 4316 return -1; 4317 } 4318 4319 ret = rte_dev_event_callback_register(NULL, 4320 dev_event_callback, NULL); 4321 if (ret) { 4322 RTE_LOG(ERR, EAL, 4323 "fail to register device event callback\n"); 4324 return -1; 4325 } 4326 } 4327 4328 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4329 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4330 4331 /* set all ports to promiscuous mode by default */ 4332 RTE_ETH_FOREACH_DEV(port_id) { 4333 ret = rte_eth_promiscuous_enable(port_id); 4334 if (ret != 0) 4335 fprintf(stderr, 4336 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4337 port_id, rte_strerror(-ret)); 4338 } 4339 4340 #ifdef RTE_LIB_METRICS 4341 /* Init metrics library */ 4342 rte_metrics_init(rte_socket_id()); 4343 #endif 4344 4345 #ifdef RTE_LIB_LATENCYSTATS 4346 if (latencystats_enabled != 0) { 4347 int ret = rte_latencystats_init(1, NULL); 4348 if (ret) 4349 fprintf(stderr, 4350 "Warning: latencystats init() returned error %d\n", 4351 ret); 4352 fprintf(stderr, "Latencystats running on lcore %d\n", 4353 latencystats_lcore_id); 4354 } 4355 #endif 4356 4357 /* Setup bitrate stats */ 4358 #ifdef RTE_LIB_BITRATESTATS 4359 if (bitrate_enabled != 0) { 4360 bitrate_data = rte_stats_bitrate_create(); 4361 if (bitrate_data == NULL) 4362 rte_exit(EXIT_FAILURE, 4363 "Could not allocate bitrate data.\n"); 4364 rte_stats_bitrate_reg(bitrate_data); 4365 } 4366 #endif 4367 #ifdef RTE_LIB_CMDLINE 4368 if (init_cmdline() != 0) 4369 rte_exit(EXIT_FAILURE, 4370 "Could not initialise cmdline context.\n"); 4371 4372 if (strlen(cmdline_filename) != 0) 4373 cmdline_read_from_file(cmdline_filename); 4374 4375 if (interactive == 1) { 4376 if (auto_start) { 4377 printf("Start automatic packet forwarding\n"); 4378 start_packet_forwarding(0); 4379 } 4380 prompt(); 4381 pmd_test_exit(); 4382 } else 4383 #endif 4384 { 4385 char c; 4386 int rc; 4387 4388 f_quit = 0; 4389 4390 printf("No commandline core given, start packet forwarding\n"); 4391 start_packet_forwarding(tx_first); 4392 if (stats_period != 0) { 4393 uint64_t prev_time = 0, cur_time, diff_time = 0; 4394 uint64_t timer_period; 4395 4396 /* Convert to number of cycles */ 4397 timer_period = stats_period * rte_get_timer_hz(); 4398 4399 while (f_quit == 0) { 4400 cur_time = rte_get_timer_cycles(); 4401 diff_time += cur_time - prev_time; 4402 4403 if (diff_time >= timer_period) { 4404 print_stats(); 4405 /* Reset the timer */ 4406 diff_time = 0; 4407 } 4408 /* Sleep to avoid unnecessary checks */ 4409 prev_time = cur_time; 4410 rte_delay_us_sleep(US_PER_S); 4411 } 4412 } 4413 4414 printf("Press enter to exit\n"); 4415 rc = read(0, &c, 1); 4416 pmd_test_exit(); 4417 if (rc < 0) 4418 return 1; 4419 } 4420 4421 ret = rte_eal_cleanup(); 4422 if (ret != 0) 4423 rte_exit(EXIT_FAILURE, 4424 "EAL cleanup failed: %s\n", strerror(-ret)); 4425 4426 return EXIT_SUCCESS; 4427 } 4428