1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 251 /* 252 * Configuration of packet segments used by the "txonly" processing engine. 253 */ 254 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 255 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 256 TXONLY_DEF_PACKET_LEN, 257 }; 258 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 259 260 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 261 /**< Split policy for packets to TX. */ 262 263 uint8_t txonly_multi_flow; 264 /**< Whether multiple flows are generated in TXONLY mode. */ 265 266 uint32_t tx_pkt_times_inter; 267 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 268 269 uint32_t tx_pkt_times_intra; 270 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 271 272 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 273 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 274 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 275 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 276 277 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 278 uint8_t dcb_config = 0; 279 280 /* 281 * Configurable number of RX/TX queues. 282 */ 283 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 284 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 285 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 286 287 /* 288 * Configurable number of RX/TX ring descriptors. 289 * Defaults are supplied by drivers via ethdev. 290 */ 291 #define RTE_TEST_RX_DESC_DEFAULT 0 292 #define RTE_TEST_TX_DESC_DEFAULT 0 293 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 294 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 295 296 #define RTE_PMD_PARAM_UNSET -1 297 /* 298 * Configurable values of RX and TX ring threshold registers. 299 */ 300 301 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 302 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 303 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 304 305 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 306 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 307 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 308 309 /* 310 * Configurable value of RX free threshold. 311 */ 312 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 313 314 /* 315 * Configurable value of RX drop enable. 316 */ 317 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 318 319 /* 320 * Configurable value of TX free threshold. 321 */ 322 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 323 324 /* 325 * Configurable value of TX RS bit threshold. 326 */ 327 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 328 329 /* 330 * Configurable value of buffered packets before sending. 331 */ 332 uint16_t noisy_tx_sw_bufsz; 333 334 /* 335 * Configurable value of packet buffer timeout. 336 */ 337 uint16_t noisy_tx_sw_buf_flush_time; 338 339 /* 340 * Configurable value for size of VNF internal memory area 341 * used for simulating noisy neighbour behaviour 342 */ 343 uint64_t noisy_lkup_mem_sz; 344 345 /* 346 * Configurable value of number of random writes done in 347 * VNF simulation memory area. 348 */ 349 uint64_t noisy_lkup_num_writes; 350 351 /* 352 * Configurable value of number of random reads done in 353 * VNF simulation memory area. 354 */ 355 uint64_t noisy_lkup_num_reads; 356 357 /* 358 * Configurable value of number of random reads/writes done in 359 * VNF simulation memory area. 360 */ 361 uint64_t noisy_lkup_num_reads_writes; 362 363 /* 364 * Receive Side Scaling (RSS) configuration. 365 */ 366 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 367 368 /* 369 * Port topology configuration 370 */ 371 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 372 373 /* 374 * Avoids to flush all the RX streams before starts forwarding. 375 */ 376 uint8_t no_flush_rx = 0; /* flush by default */ 377 378 /* 379 * Flow API isolated mode. 380 */ 381 uint8_t flow_isolate_all; 382 383 /* 384 * Avoids to check link status when starting/stopping a port. 385 */ 386 uint8_t no_link_check = 0; /* check by default */ 387 388 /* 389 * Don't automatically start all ports in interactive mode. 390 */ 391 uint8_t no_device_start = 0; 392 393 /* 394 * Enable link status change notification 395 */ 396 uint8_t lsc_interrupt = 1; /* enabled by default */ 397 398 /* 399 * Enable device removal notification. 400 */ 401 uint8_t rmv_interrupt = 1; /* enabled by default */ 402 403 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 404 405 /* After attach, port setup is called on event or by iterator */ 406 bool setup_on_probe_event = true; 407 408 /* Clear ptypes on port initialization. */ 409 uint8_t clear_ptypes = true; 410 411 /* Hairpin ports configuration mode. */ 412 uint16_t hairpin_mode; 413 414 /* Pretty printing of ethdev events */ 415 static const char * const eth_event_desc[] = { 416 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 417 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 418 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 419 [RTE_ETH_EVENT_INTR_RESET] = "reset", 420 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 421 [RTE_ETH_EVENT_IPSEC] = "IPsec", 422 [RTE_ETH_EVENT_MACSEC] = "MACsec", 423 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 424 [RTE_ETH_EVENT_NEW] = "device probed", 425 [RTE_ETH_EVENT_DESTROY] = "device released", 426 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 427 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 428 [RTE_ETH_EVENT_MAX] = NULL, 429 }; 430 431 /* 432 * Display or mask ether events 433 * Default to all events except VF_MBOX 434 */ 435 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 436 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 437 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 438 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 439 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 440 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 441 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 442 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 443 /* 444 * Decide if all memory are locked for performance. 445 */ 446 int do_mlockall = 0; 447 448 #ifdef RTE_LIB_LATENCYSTATS 449 450 /* 451 * Set when latency stats is enabled in the commandline 452 */ 453 uint8_t latencystats_enabled; 454 455 /* 456 * Lcore ID to service latency statistics. 457 */ 458 lcoreid_t latencystats_lcore_id = -1; 459 460 #endif 461 462 /* 463 * Ethernet device configuration. 464 */ 465 struct rte_eth_rxmode rx_mode; 466 467 struct rte_eth_txmode tx_mode = { 468 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 469 }; 470 471 struct rte_eth_fdir_conf fdir_conf = { 472 .mode = RTE_FDIR_MODE_NONE, 473 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 474 .status = RTE_FDIR_REPORT_STATUS, 475 .mask = { 476 .vlan_tci_mask = 0xFFEF, 477 .ipv4_mask = { 478 .src_ip = 0xFFFFFFFF, 479 .dst_ip = 0xFFFFFFFF, 480 }, 481 .ipv6_mask = { 482 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 483 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 484 }, 485 .src_port_mask = 0xFFFF, 486 .dst_port_mask = 0xFFFF, 487 .mac_addr_byte_mask = 0xFF, 488 .tunnel_type_mask = 1, 489 .tunnel_id_mask = 0xFFFFFFFF, 490 }, 491 .drop_queue = 127, 492 }; 493 494 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 495 496 /* 497 * Display zero values by default for xstats 498 */ 499 uint8_t xstats_hide_zero; 500 501 /* 502 * Measure of CPU cycles disabled by default 503 */ 504 uint8_t record_core_cycles; 505 506 /* 507 * Display of RX and TX bursts disabled by default 508 */ 509 uint8_t record_burst_stats; 510 511 /* 512 * Number of ports per shared Rx queue group, 0 disable. 513 */ 514 uint32_t rxq_share; 515 516 unsigned int num_sockets = 0; 517 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 518 519 #ifdef RTE_LIB_BITRATESTATS 520 /* Bitrate statistics */ 521 struct rte_stats_bitrates *bitrate_data; 522 lcoreid_t bitrate_lcore_id; 523 uint8_t bitrate_enabled; 524 #endif 525 526 #ifdef RTE_LIB_GRO 527 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 528 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 529 #endif 530 531 /* 532 * hexadecimal bitmask of RX mq mode can be enabled. 533 */ 534 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 535 536 /* 537 * Used to set forced link speed 538 */ 539 uint32_t eth_link_speed; 540 541 /* 542 * ID of the current process in multi-process, used to 543 * configure the queues to be polled. 544 */ 545 int proc_id; 546 547 /* 548 * Number of processes in multi-process, used to 549 * configure the queues to be polled. 550 */ 551 unsigned int num_procs = 1; 552 553 static void 554 eth_rx_metadata_negotiate_mp(uint16_t port_id) 555 { 556 uint64_t rx_meta_features = 0; 557 int ret; 558 559 if (!is_proc_primary()) 560 return; 561 562 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 563 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 564 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 565 566 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 567 if (ret == 0) { 568 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 569 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 570 port_id); 571 } 572 573 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 574 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 575 port_id); 576 } 577 578 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 579 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 580 port_id); 581 } 582 } else if (ret != -ENOTSUP) { 583 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 584 port_id, rte_strerror(-ret)); 585 } 586 } 587 588 static int 589 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 590 const struct rte_eth_conf *dev_conf) 591 { 592 if (is_proc_primary()) 593 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 594 dev_conf); 595 return 0; 596 } 597 598 static int 599 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 600 { 601 #ifdef RTE_NET_BOND 602 603 portid_t slave_pids[RTE_MAX_ETHPORTS]; 604 struct rte_port *port; 605 int num_slaves; 606 portid_t slave_pid; 607 int i; 608 609 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 610 RTE_MAX_ETHPORTS); 611 if (num_slaves < 0) { 612 fprintf(stderr, "Failed to get slave list for port = %u\n", 613 bond_pid); 614 return num_slaves; 615 } 616 617 for (i = 0; i < num_slaves; i++) { 618 slave_pid = slave_pids[i]; 619 port = &ports[slave_pid]; 620 port->port_status = 621 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 622 } 623 #else 624 RTE_SET_USED(bond_pid); 625 RTE_SET_USED(is_stop); 626 #endif 627 return 0; 628 } 629 630 static int 631 eth_dev_start_mp(uint16_t port_id) 632 { 633 int ret; 634 635 if (is_proc_primary()) { 636 ret = rte_eth_dev_start(port_id); 637 if (ret != 0) 638 return ret; 639 640 struct rte_port *port = &ports[port_id]; 641 642 /* 643 * Starting a bonded port also starts all slaves under the bonded 644 * device. So if this port is bond device, we need to modify the 645 * port status of these slaves. 646 */ 647 if (port->bond_flag == 1) 648 return change_bonding_slave_port_status(port_id, false); 649 } 650 651 return 0; 652 } 653 654 static int 655 eth_dev_stop_mp(uint16_t port_id) 656 { 657 int ret; 658 659 if (is_proc_primary()) { 660 ret = rte_eth_dev_stop(port_id); 661 if (ret != 0) 662 return ret; 663 664 struct rte_port *port = &ports[port_id]; 665 666 /* 667 * Stopping a bonded port also stops all slaves under the bonded 668 * device. So if this port is bond device, we need to modify the 669 * port status of these slaves. 670 */ 671 if (port->bond_flag == 1) 672 return change_bonding_slave_port_status(port_id, true); 673 } 674 675 return 0; 676 } 677 678 static void 679 mempool_free_mp(struct rte_mempool *mp) 680 { 681 if (is_proc_primary()) 682 rte_mempool_free(mp); 683 } 684 685 static int 686 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 687 { 688 if (is_proc_primary()) 689 return rte_eth_dev_set_mtu(port_id, mtu); 690 691 return 0; 692 } 693 694 /* Forward function declarations */ 695 static void setup_attached_port(portid_t pi); 696 static void check_all_ports_link_status(uint32_t port_mask); 697 static int eth_event_callback(portid_t port_id, 698 enum rte_eth_event_type type, 699 void *param, void *ret_param); 700 static void dev_event_callback(const char *device_name, 701 enum rte_dev_event_type type, 702 void *param); 703 static void fill_xstats_display_info(void); 704 705 /* 706 * Check if all the ports are started. 707 * If yes, return positive value. If not, return zero. 708 */ 709 static int all_ports_started(void); 710 711 #ifdef RTE_LIB_GSO 712 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 713 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 714 #endif 715 716 /* Holds the registered mbuf dynamic flags names. */ 717 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 718 719 720 /* 721 * Helper function to check if socket is already discovered. 722 * If yes, return positive value. If not, return zero. 723 */ 724 int 725 new_socket_id(unsigned int socket_id) 726 { 727 unsigned int i; 728 729 for (i = 0; i < num_sockets; i++) { 730 if (socket_ids[i] == socket_id) 731 return 0; 732 } 733 return 1; 734 } 735 736 /* 737 * Setup default configuration. 738 */ 739 static void 740 set_default_fwd_lcores_config(void) 741 { 742 unsigned int i; 743 unsigned int nb_lc; 744 unsigned int sock_num; 745 746 nb_lc = 0; 747 for (i = 0; i < RTE_MAX_LCORE; i++) { 748 if (!rte_lcore_is_enabled(i)) 749 continue; 750 sock_num = rte_lcore_to_socket_id(i); 751 if (new_socket_id(sock_num)) { 752 if (num_sockets >= RTE_MAX_NUMA_NODES) { 753 rte_exit(EXIT_FAILURE, 754 "Total sockets greater than %u\n", 755 RTE_MAX_NUMA_NODES); 756 } 757 socket_ids[num_sockets++] = sock_num; 758 } 759 if (i == rte_get_main_lcore()) 760 continue; 761 fwd_lcores_cpuids[nb_lc++] = i; 762 } 763 nb_lcores = (lcoreid_t) nb_lc; 764 nb_cfg_lcores = nb_lcores; 765 nb_fwd_lcores = 1; 766 } 767 768 static void 769 set_def_peer_eth_addrs(void) 770 { 771 portid_t i; 772 773 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 774 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 775 peer_eth_addrs[i].addr_bytes[5] = i; 776 } 777 } 778 779 static void 780 set_default_fwd_ports_config(void) 781 { 782 portid_t pt_id; 783 int i = 0; 784 785 RTE_ETH_FOREACH_DEV(pt_id) { 786 fwd_ports_ids[i++] = pt_id; 787 788 /* Update sockets info according to the attached device */ 789 int socket_id = rte_eth_dev_socket_id(pt_id); 790 if (socket_id >= 0 && new_socket_id(socket_id)) { 791 if (num_sockets >= RTE_MAX_NUMA_NODES) { 792 rte_exit(EXIT_FAILURE, 793 "Total sockets greater than %u\n", 794 RTE_MAX_NUMA_NODES); 795 } 796 socket_ids[num_sockets++] = socket_id; 797 } 798 } 799 800 nb_cfg_ports = nb_ports; 801 nb_fwd_ports = nb_ports; 802 } 803 804 void 805 set_def_fwd_config(void) 806 { 807 set_default_fwd_lcores_config(); 808 set_def_peer_eth_addrs(); 809 set_default_fwd_ports_config(); 810 } 811 812 #ifndef RTE_EXEC_ENV_WINDOWS 813 /* extremely pessimistic estimation of memory required to create a mempool */ 814 static int 815 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 816 { 817 unsigned int n_pages, mbuf_per_pg, leftover; 818 uint64_t total_mem, mbuf_mem, obj_sz; 819 820 /* there is no good way to predict how much space the mempool will 821 * occupy because it will allocate chunks on the fly, and some of those 822 * will come from default DPDK memory while some will come from our 823 * external memory, so just assume 128MB will be enough for everyone. 824 */ 825 uint64_t hdr_mem = 128 << 20; 826 827 /* account for possible non-contiguousness */ 828 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 829 if (obj_sz > pgsz) { 830 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 831 return -1; 832 } 833 834 mbuf_per_pg = pgsz / obj_sz; 835 leftover = (nb_mbufs % mbuf_per_pg) > 0; 836 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 837 838 mbuf_mem = n_pages * pgsz; 839 840 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 841 842 if (total_mem > SIZE_MAX) { 843 TESTPMD_LOG(ERR, "Memory size too big\n"); 844 return -1; 845 } 846 *out = (size_t)total_mem; 847 848 return 0; 849 } 850 851 static int 852 pagesz_flags(uint64_t page_sz) 853 { 854 /* as per mmap() manpage, all page sizes are log2 of page size 855 * shifted by MAP_HUGE_SHIFT 856 */ 857 int log2 = rte_log2_u64(page_sz); 858 859 return (log2 << HUGE_SHIFT); 860 } 861 862 static void * 863 alloc_mem(size_t memsz, size_t pgsz, bool huge) 864 { 865 void *addr; 866 int flags; 867 868 /* allocate anonymous hugepages */ 869 flags = MAP_ANONYMOUS | MAP_PRIVATE; 870 if (huge) 871 flags |= HUGE_FLAG | pagesz_flags(pgsz); 872 873 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 874 if (addr == MAP_FAILED) 875 return NULL; 876 877 return addr; 878 } 879 880 struct extmem_param { 881 void *addr; 882 size_t len; 883 size_t pgsz; 884 rte_iova_t *iova_table; 885 unsigned int iova_table_len; 886 }; 887 888 static int 889 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 890 bool huge) 891 { 892 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 893 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 894 unsigned int cur_page, n_pages, pgsz_idx; 895 size_t mem_sz, cur_pgsz; 896 rte_iova_t *iovas = NULL; 897 void *addr; 898 int ret; 899 900 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 901 /* skip anything that is too big */ 902 if (pgsizes[pgsz_idx] > SIZE_MAX) 903 continue; 904 905 cur_pgsz = pgsizes[pgsz_idx]; 906 907 /* if we were told not to allocate hugepages, override */ 908 if (!huge) 909 cur_pgsz = sysconf(_SC_PAGESIZE); 910 911 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 912 if (ret < 0) { 913 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 914 return -1; 915 } 916 917 /* allocate our memory */ 918 addr = alloc_mem(mem_sz, cur_pgsz, huge); 919 920 /* if we couldn't allocate memory with a specified page size, 921 * that doesn't mean we can't do it with other page sizes, so 922 * try another one. 923 */ 924 if (addr == NULL) 925 continue; 926 927 /* store IOVA addresses for every page in this memory area */ 928 n_pages = mem_sz / cur_pgsz; 929 930 iovas = malloc(sizeof(*iovas) * n_pages); 931 932 if (iovas == NULL) { 933 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 934 goto fail; 935 } 936 /* lock memory if it's not huge pages */ 937 if (!huge) 938 mlock(addr, mem_sz); 939 940 /* populate IOVA addresses */ 941 for (cur_page = 0; cur_page < n_pages; cur_page++) { 942 rte_iova_t iova; 943 size_t offset; 944 void *cur; 945 946 offset = cur_pgsz * cur_page; 947 cur = RTE_PTR_ADD(addr, offset); 948 949 /* touch the page before getting its IOVA */ 950 *(volatile char *)cur = 0; 951 952 iova = rte_mem_virt2iova(cur); 953 954 iovas[cur_page] = iova; 955 } 956 957 break; 958 } 959 /* if we couldn't allocate anything */ 960 if (iovas == NULL) 961 return -1; 962 963 param->addr = addr; 964 param->len = mem_sz; 965 param->pgsz = cur_pgsz; 966 param->iova_table = iovas; 967 param->iova_table_len = n_pages; 968 969 return 0; 970 fail: 971 free(iovas); 972 if (addr) 973 munmap(addr, mem_sz); 974 975 return -1; 976 } 977 978 static int 979 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 980 { 981 struct extmem_param param; 982 int socket_id, ret; 983 984 memset(¶m, 0, sizeof(param)); 985 986 /* check if our heap exists */ 987 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 988 if (socket_id < 0) { 989 /* create our heap */ 990 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 991 if (ret < 0) { 992 TESTPMD_LOG(ERR, "Cannot create heap\n"); 993 return -1; 994 } 995 } 996 997 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 998 if (ret < 0) { 999 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 1000 return -1; 1001 } 1002 1003 /* we now have a valid memory area, so add it to heap */ 1004 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 1005 param.addr, param.len, param.iova_table, 1006 param.iova_table_len, param.pgsz); 1007 1008 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 1009 1010 /* not needed any more */ 1011 free(param.iova_table); 1012 1013 if (ret < 0) { 1014 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 1015 munmap(param.addr, param.len); 1016 return -1; 1017 } 1018 1019 /* success */ 1020 1021 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1022 param.len >> 20); 1023 1024 return 0; 1025 } 1026 static void 1027 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1028 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1029 { 1030 uint16_t pid = 0; 1031 int ret; 1032 1033 RTE_ETH_FOREACH_DEV(pid) { 1034 struct rte_eth_dev_info dev_info; 1035 1036 ret = eth_dev_info_get_print_err(pid, &dev_info); 1037 if (ret != 0) { 1038 TESTPMD_LOG(DEBUG, 1039 "unable to get device info for port %d on addr 0x%p," 1040 "mempool unmapping will not be performed\n", 1041 pid, memhdr->addr); 1042 continue; 1043 } 1044 1045 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1046 if (ret) { 1047 TESTPMD_LOG(DEBUG, 1048 "unable to DMA unmap addr 0x%p " 1049 "for device %s\n", 1050 memhdr->addr, rte_dev_name(dev_info.device)); 1051 } 1052 } 1053 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1054 if (ret) { 1055 TESTPMD_LOG(DEBUG, 1056 "unable to un-register addr 0x%p\n", memhdr->addr); 1057 } 1058 } 1059 1060 static void 1061 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1062 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1063 { 1064 uint16_t pid = 0; 1065 size_t page_size = sysconf(_SC_PAGESIZE); 1066 int ret; 1067 1068 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1069 page_size); 1070 if (ret) { 1071 TESTPMD_LOG(DEBUG, 1072 "unable to register addr 0x%p\n", memhdr->addr); 1073 return; 1074 } 1075 RTE_ETH_FOREACH_DEV(pid) { 1076 struct rte_eth_dev_info dev_info; 1077 1078 ret = eth_dev_info_get_print_err(pid, &dev_info); 1079 if (ret != 0) { 1080 TESTPMD_LOG(DEBUG, 1081 "unable to get device info for port %d on addr 0x%p," 1082 "mempool mapping will not be performed\n", 1083 pid, memhdr->addr); 1084 continue; 1085 } 1086 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1087 if (ret) { 1088 TESTPMD_LOG(DEBUG, 1089 "unable to DMA map addr 0x%p " 1090 "for device %s\n", 1091 memhdr->addr, rte_dev_name(dev_info.device)); 1092 } 1093 } 1094 } 1095 #endif 1096 1097 static unsigned int 1098 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1099 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1100 { 1101 struct rte_pktmbuf_extmem *xmem; 1102 unsigned int ext_num, zone_num, elt_num; 1103 uint16_t elt_size; 1104 1105 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1106 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1107 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1108 1109 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1110 if (xmem == NULL) { 1111 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1112 "external buffer descriptors\n"); 1113 *ext_mem = NULL; 1114 return 0; 1115 } 1116 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1117 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1118 const struct rte_memzone *mz; 1119 char mz_name[RTE_MEMZONE_NAMESIZE]; 1120 int ret; 1121 1122 ret = snprintf(mz_name, sizeof(mz_name), 1123 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1124 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1125 errno = ENAMETOOLONG; 1126 ext_num = 0; 1127 break; 1128 } 1129 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1130 socket_id, 1131 RTE_MEMZONE_IOVA_CONTIG | 1132 RTE_MEMZONE_1GB | 1133 RTE_MEMZONE_SIZE_HINT_ONLY); 1134 if (mz == NULL) { 1135 /* 1136 * The caller exits on external buffer creation 1137 * error, so there is no need to free memzones. 1138 */ 1139 errno = ENOMEM; 1140 ext_num = 0; 1141 break; 1142 } 1143 xseg->buf_ptr = mz->addr; 1144 xseg->buf_iova = mz->iova; 1145 xseg->buf_len = EXTBUF_ZONE_SIZE; 1146 xseg->elt_size = elt_size; 1147 } 1148 if (ext_num == 0 && xmem != NULL) { 1149 free(xmem); 1150 xmem = NULL; 1151 } 1152 *ext_mem = xmem; 1153 return ext_num; 1154 } 1155 1156 /* 1157 * Configuration initialisation done once at init time. 1158 */ 1159 static struct rte_mempool * 1160 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1161 unsigned int socket_id, uint16_t size_idx) 1162 { 1163 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1164 struct rte_mempool *rte_mp = NULL; 1165 #ifndef RTE_EXEC_ENV_WINDOWS 1166 uint32_t mb_size; 1167 1168 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1169 #endif 1170 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1171 if (!is_proc_primary()) { 1172 rte_mp = rte_mempool_lookup(pool_name); 1173 if (rte_mp == NULL) 1174 rte_exit(EXIT_FAILURE, 1175 "Get mbuf pool for socket %u failed: %s\n", 1176 socket_id, rte_strerror(rte_errno)); 1177 return rte_mp; 1178 } 1179 1180 TESTPMD_LOG(INFO, 1181 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1182 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1183 1184 switch (mp_alloc_type) { 1185 case MP_ALLOC_NATIVE: 1186 { 1187 /* wrapper to rte_mempool_create() */ 1188 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1189 rte_mbuf_best_mempool_ops()); 1190 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1191 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1192 break; 1193 } 1194 #ifndef RTE_EXEC_ENV_WINDOWS 1195 case MP_ALLOC_ANON: 1196 { 1197 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1198 mb_size, (unsigned int) mb_mempool_cache, 1199 sizeof(struct rte_pktmbuf_pool_private), 1200 socket_id, mempool_flags); 1201 if (rte_mp == NULL) 1202 goto err; 1203 1204 if (rte_mempool_populate_anon(rte_mp) == 0) { 1205 rte_mempool_free(rte_mp); 1206 rte_mp = NULL; 1207 goto err; 1208 } 1209 rte_pktmbuf_pool_init(rte_mp, NULL); 1210 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1211 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1212 break; 1213 } 1214 case MP_ALLOC_XMEM: 1215 case MP_ALLOC_XMEM_HUGE: 1216 { 1217 int heap_socket; 1218 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1219 1220 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1221 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1222 1223 heap_socket = 1224 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1225 if (heap_socket < 0) 1226 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1227 1228 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1229 rte_mbuf_best_mempool_ops()); 1230 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1231 mb_mempool_cache, 0, mbuf_seg_size, 1232 heap_socket); 1233 break; 1234 } 1235 #endif 1236 case MP_ALLOC_XBUF: 1237 { 1238 struct rte_pktmbuf_extmem *ext_mem; 1239 unsigned int ext_num; 1240 1241 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1242 socket_id, pool_name, &ext_mem); 1243 if (ext_num == 0) 1244 rte_exit(EXIT_FAILURE, 1245 "Can't create pinned data buffers\n"); 1246 1247 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1248 rte_mbuf_best_mempool_ops()); 1249 rte_mp = rte_pktmbuf_pool_create_extbuf 1250 (pool_name, nb_mbuf, mb_mempool_cache, 1251 0, mbuf_seg_size, socket_id, 1252 ext_mem, ext_num); 1253 free(ext_mem); 1254 break; 1255 } 1256 default: 1257 { 1258 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1259 } 1260 } 1261 1262 #ifndef RTE_EXEC_ENV_WINDOWS 1263 err: 1264 #endif 1265 if (rte_mp == NULL) { 1266 rte_exit(EXIT_FAILURE, 1267 "Creation of mbuf pool for socket %u failed: %s\n", 1268 socket_id, rte_strerror(rte_errno)); 1269 } else if (verbose_level > 0) { 1270 rte_mempool_dump(stdout, rte_mp); 1271 } 1272 return rte_mp; 1273 } 1274 1275 /* 1276 * Check given socket id is valid or not with NUMA mode, 1277 * if valid, return 0, else return -1 1278 */ 1279 static int 1280 check_socket_id(const unsigned int socket_id) 1281 { 1282 static int warning_once = 0; 1283 1284 if (new_socket_id(socket_id)) { 1285 if (!warning_once && numa_support) 1286 fprintf(stderr, 1287 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1288 warning_once = 1; 1289 return -1; 1290 } 1291 return 0; 1292 } 1293 1294 /* 1295 * Get the allowed maximum number of RX queues. 1296 * *pid return the port id which has minimal value of 1297 * max_rx_queues in all ports. 1298 */ 1299 queueid_t 1300 get_allowed_max_nb_rxq(portid_t *pid) 1301 { 1302 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1303 bool max_rxq_valid = false; 1304 portid_t pi; 1305 struct rte_eth_dev_info dev_info; 1306 1307 RTE_ETH_FOREACH_DEV(pi) { 1308 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1309 continue; 1310 1311 max_rxq_valid = true; 1312 if (dev_info.max_rx_queues < allowed_max_rxq) { 1313 allowed_max_rxq = dev_info.max_rx_queues; 1314 *pid = pi; 1315 } 1316 } 1317 return max_rxq_valid ? allowed_max_rxq : 0; 1318 } 1319 1320 /* 1321 * Check input rxq is valid or not. 1322 * If input rxq is not greater than any of maximum number 1323 * of RX queues of all ports, it is valid. 1324 * if valid, return 0, else return -1 1325 */ 1326 int 1327 check_nb_rxq(queueid_t rxq) 1328 { 1329 queueid_t allowed_max_rxq; 1330 portid_t pid = 0; 1331 1332 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1333 if (rxq > allowed_max_rxq) { 1334 fprintf(stderr, 1335 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1336 rxq, allowed_max_rxq, pid); 1337 return -1; 1338 } 1339 return 0; 1340 } 1341 1342 /* 1343 * Get the allowed maximum number of TX queues. 1344 * *pid return the port id which has minimal value of 1345 * max_tx_queues in all ports. 1346 */ 1347 queueid_t 1348 get_allowed_max_nb_txq(portid_t *pid) 1349 { 1350 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1351 bool max_txq_valid = false; 1352 portid_t pi; 1353 struct rte_eth_dev_info dev_info; 1354 1355 RTE_ETH_FOREACH_DEV(pi) { 1356 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1357 continue; 1358 1359 max_txq_valid = true; 1360 if (dev_info.max_tx_queues < allowed_max_txq) { 1361 allowed_max_txq = dev_info.max_tx_queues; 1362 *pid = pi; 1363 } 1364 } 1365 return max_txq_valid ? allowed_max_txq : 0; 1366 } 1367 1368 /* 1369 * Check input txq is valid or not. 1370 * If input txq is not greater than any of maximum number 1371 * of TX queues of all ports, it is valid. 1372 * if valid, return 0, else return -1 1373 */ 1374 int 1375 check_nb_txq(queueid_t txq) 1376 { 1377 queueid_t allowed_max_txq; 1378 portid_t pid = 0; 1379 1380 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1381 if (txq > allowed_max_txq) { 1382 fprintf(stderr, 1383 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1384 txq, allowed_max_txq, pid); 1385 return -1; 1386 } 1387 return 0; 1388 } 1389 1390 /* 1391 * Get the allowed maximum number of RXDs of every rx queue. 1392 * *pid return the port id which has minimal value of 1393 * max_rxd in all queues of all ports. 1394 */ 1395 static uint16_t 1396 get_allowed_max_nb_rxd(portid_t *pid) 1397 { 1398 uint16_t allowed_max_rxd = UINT16_MAX; 1399 portid_t pi; 1400 struct rte_eth_dev_info dev_info; 1401 1402 RTE_ETH_FOREACH_DEV(pi) { 1403 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1404 continue; 1405 1406 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1407 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1408 *pid = pi; 1409 } 1410 } 1411 return allowed_max_rxd; 1412 } 1413 1414 /* 1415 * Get the allowed minimal number of RXDs of every rx queue. 1416 * *pid return the port id which has minimal value of 1417 * min_rxd in all queues of all ports. 1418 */ 1419 static uint16_t 1420 get_allowed_min_nb_rxd(portid_t *pid) 1421 { 1422 uint16_t allowed_min_rxd = 0; 1423 portid_t pi; 1424 struct rte_eth_dev_info dev_info; 1425 1426 RTE_ETH_FOREACH_DEV(pi) { 1427 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1428 continue; 1429 1430 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1431 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1432 *pid = pi; 1433 } 1434 } 1435 1436 return allowed_min_rxd; 1437 } 1438 1439 /* 1440 * Check input rxd is valid or not. 1441 * If input rxd is not greater than any of maximum number 1442 * of RXDs of every Rx queues and is not less than any of 1443 * minimal number of RXDs of every Rx queues, it is valid. 1444 * if valid, return 0, else return -1 1445 */ 1446 int 1447 check_nb_rxd(queueid_t rxd) 1448 { 1449 uint16_t allowed_max_rxd; 1450 uint16_t allowed_min_rxd; 1451 portid_t pid = 0; 1452 1453 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1454 if (rxd > allowed_max_rxd) { 1455 fprintf(stderr, 1456 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1457 rxd, allowed_max_rxd, pid); 1458 return -1; 1459 } 1460 1461 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1462 if (rxd < allowed_min_rxd) { 1463 fprintf(stderr, 1464 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1465 rxd, allowed_min_rxd, pid); 1466 return -1; 1467 } 1468 1469 return 0; 1470 } 1471 1472 /* 1473 * Get the allowed maximum number of TXDs of every rx queues. 1474 * *pid return the port id which has minimal value of 1475 * max_txd in every tx queue. 1476 */ 1477 static uint16_t 1478 get_allowed_max_nb_txd(portid_t *pid) 1479 { 1480 uint16_t allowed_max_txd = UINT16_MAX; 1481 portid_t pi; 1482 struct rte_eth_dev_info dev_info; 1483 1484 RTE_ETH_FOREACH_DEV(pi) { 1485 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1486 continue; 1487 1488 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1489 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1490 *pid = pi; 1491 } 1492 } 1493 return allowed_max_txd; 1494 } 1495 1496 /* 1497 * Get the allowed maximum number of TXDs of every tx queues. 1498 * *pid return the port id which has minimal value of 1499 * min_txd in every tx queue. 1500 */ 1501 static uint16_t 1502 get_allowed_min_nb_txd(portid_t *pid) 1503 { 1504 uint16_t allowed_min_txd = 0; 1505 portid_t pi; 1506 struct rte_eth_dev_info dev_info; 1507 1508 RTE_ETH_FOREACH_DEV(pi) { 1509 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1510 continue; 1511 1512 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1513 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1514 *pid = pi; 1515 } 1516 } 1517 1518 return allowed_min_txd; 1519 } 1520 1521 /* 1522 * Check input txd is valid or not. 1523 * If input txd is not greater than any of maximum number 1524 * of TXDs of every Rx queues, it is valid. 1525 * if valid, return 0, else return -1 1526 */ 1527 int 1528 check_nb_txd(queueid_t txd) 1529 { 1530 uint16_t allowed_max_txd; 1531 uint16_t allowed_min_txd; 1532 portid_t pid = 0; 1533 1534 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1535 if (txd > allowed_max_txd) { 1536 fprintf(stderr, 1537 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1538 txd, allowed_max_txd, pid); 1539 return -1; 1540 } 1541 1542 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1543 if (txd < allowed_min_txd) { 1544 fprintf(stderr, 1545 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1546 txd, allowed_min_txd, pid); 1547 return -1; 1548 } 1549 return 0; 1550 } 1551 1552 1553 /* 1554 * Get the allowed maximum number of hairpin queues. 1555 * *pid return the port id which has minimal value of 1556 * max_hairpin_queues in all ports. 1557 */ 1558 queueid_t 1559 get_allowed_max_nb_hairpinq(portid_t *pid) 1560 { 1561 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1562 portid_t pi; 1563 struct rte_eth_hairpin_cap cap; 1564 1565 RTE_ETH_FOREACH_DEV(pi) { 1566 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1567 *pid = pi; 1568 return 0; 1569 } 1570 if (cap.max_nb_queues < allowed_max_hairpinq) { 1571 allowed_max_hairpinq = cap.max_nb_queues; 1572 *pid = pi; 1573 } 1574 } 1575 return allowed_max_hairpinq; 1576 } 1577 1578 /* 1579 * Check input hairpin is valid or not. 1580 * If input hairpin is not greater than any of maximum number 1581 * of hairpin queues of all ports, it is valid. 1582 * if valid, return 0, else return -1 1583 */ 1584 int 1585 check_nb_hairpinq(queueid_t hairpinq) 1586 { 1587 queueid_t allowed_max_hairpinq; 1588 portid_t pid = 0; 1589 1590 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1591 if (hairpinq > allowed_max_hairpinq) { 1592 fprintf(stderr, 1593 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1594 hairpinq, allowed_max_hairpinq, pid); 1595 return -1; 1596 } 1597 return 0; 1598 } 1599 1600 static int 1601 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1602 { 1603 uint32_t eth_overhead; 1604 1605 if (dev_info->max_mtu != UINT16_MAX && 1606 dev_info->max_rx_pktlen > dev_info->max_mtu) 1607 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1608 else 1609 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1610 1611 return eth_overhead; 1612 } 1613 1614 static void 1615 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1616 { 1617 struct rte_port *port = &ports[pid]; 1618 int ret; 1619 int i; 1620 1621 eth_rx_metadata_negotiate_mp(pid); 1622 1623 port->dev_conf.txmode = tx_mode; 1624 port->dev_conf.rxmode = rx_mode; 1625 1626 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1627 if (ret != 0) 1628 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1629 1630 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1631 port->dev_conf.txmode.offloads &= 1632 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1633 1634 /* Apply Rx offloads configuration */ 1635 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1636 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1637 /* Apply Tx offloads configuration */ 1638 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1639 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1640 1641 if (eth_link_speed) 1642 port->dev_conf.link_speeds = eth_link_speed; 1643 1644 if (max_rx_pkt_len) 1645 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1646 get_eth_overhead(&port->dev_info); 1647 1648 /* set flag to initialize port/queue */ 1649 port->need_reconfig = 1; 1650 port->need_reconfig_queues = 1; 1651 port->socket_id = socket_id; 1652 port->tx_metadata = 0; 1653 1654 /* 1655 * Check for maximum number of segments per MTU. 1656 * Accordingly update the mbuf data size. 1657 */ 1658 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1659 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1660 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1661 uint16_t mtu; 1662 1663 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1664 uint16_t data_size = (mtu + eth_overhead) / 1665 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1666 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1667 1668 if (buffer_size > mbuf_data_size[0]) { 1669 mbuf_data_size[0] = buffer_size; 1670 TESTPMD_LOG(WARNING, 1671 "Configured mbuf size of the first segment %hu\n", 1672 mbuf_data_size[0]); 1673 } 1674 } 1675 } 1676 } 1677 1678 static void 1679 init_config(void) 1680 { 1681 portid_t pid; 1682 struct rte_mempool *mbp; 1683 unsigned int nb_mbuf_per_pool; 1684 lcoreid_t lc_id; 1685 #ifdef RTE_LIB_GRO 1686 struct rte_gro_param gro_param; 1687 #endif 1688 #ifdef RTE_LIB_GSO 1689 uint32_t gso_types; 1690 #endif 1691 1692 /* Configuration of logical cores. */ 1693 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1694 sizeof(struct fwd_lcore *) * nb_lcores, 1695 RTE_CACHE_LINE_SIZE); 1696 if (fwd_lcores == NULL) { 1697 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1698 "failed\n", nb_lcores); 1699 } 1700 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1701 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1702 sizeof(struct fwd_lcore), 1703 RTE_CACHE_LINE_SIZE); 1704 if (fwd_lcores[lc_id] == NULL) { 1705 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1706 "failed\n"); 1707 } 1708 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1709 } 1710 1711 RTE_ETH_FOREACH_DEV(pid) { 1712 uint32_t socket_id; 1713 1714 if (numa_support) { 1715 socket_id = port_numa[pid]; 1716 if (port_numa[pid] == NUMA_NO_CONFIG) { 1717 socket_id = rte_eth_dev_socket_id(pid); 1718 1719 /* 1720 * if socket_id is invalid, 1721 * set to the first available socket. 1722 */ 1723 if (check_socket_id(socket_id) < 0) 1724 socket_id = socket_ids[0]; 1725 } 1726 } else { 1727 socket_id = (socket_num == UMA_NO_CONFIG) ? 1728 0 : socket_num; 1729 } 1730 /* Apply default TxRx configuration for all ports */ 1731 init_config_port_offloads(pid, socket_id); 1732 } 1733 /* 1734 * Create pools of mbuf. 1735 * If NUMA support is disabled, create a single pool of mbuf in 1736 * socket 0 memory by default. 1737 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1738 * 1739 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1740 * nb_txd can be configured at run time. 1741 */ 1742 if (param_total_num_mbufs) 1743 nb_mbuf_per_pool = param_total_num_mbufs; 1744 else { 1745 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1746 (nb_lcores * mb_mempool_cache) + 1747 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1748 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1749 } 1750 1751 if (numa_support) { 1752 uint8_t i, j; 1753 1754 for (i = 0; i < num_sockets; i++) 1755 for (j = 0; j < mbuf_data_size_n; j++) 1756 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1757 mbuf_pool_create(mbuf_data_size[j], 1758 nb_mbuf_per_pool, 1759 socket_ids[i], j); 1760 } else { 1761 uint8_t i; 1762 1763 for (i = 0; i < mbuf_data_size_n; i++) 1764 mempools[i] = mbuf_pool_create 1765 (mbuf_data_size[i], 1766 nb_mbuf_per_pool, 1767 socket_num == UMA_NO_CONFIG ? 1768 0 : socket_num, i); 1769 } 1770 1771 init_port_config(); 1772 1773 #ifdef RTE_LIB_GSO 1774 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1775 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1776 #endif 1777 /* 1778 * Records which Mbuf pool to use by each logical core, if needed. 1779 */ 1780 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1781 mbp = mbuf_pool_find( 1782 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1783 1784 if (mbp == NULL) 1785 mbp = mbuf_pool_find(0, 0); 1786 fwd_lcores[lc_id]->mbp = mbp; 1787 #ifdef RTE_LIB_GSO 1788 /* initialize GSO context */ 1789 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1790 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1791 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1792 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1793 RTE_ETHER_CRC_LEN; 1794 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1795 #endif 1796 } 1797 1798 fwd_config_setup(); 1799 1800 #ifdef RTE_LIB_GRO 1801 /* create a gro context for each lcore */ 1802 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1803 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1804 gro_param.max_item_per_flow = MAX_PKT_BURST; 1805 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1806 gro_param.socket_id = rte_lcore_to_socket_id( 1807 fwd_lcores_cpuids[lc_id]); 1808 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1809 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1810 rte_exit(EXIT_FAILURE, 1811 "rte_gro_ctx_create() failed\n"); 1812 } 1813 } 1814 #endif 1815 } 1816 1817 1818 void 1819 reconfig(portid_t new_port_id, unsigned socket_id) 1820 { 1821 /* Reconfiguration of Ethernet ports. */ 1822 init_config_port_offloads(new_port_id, socket_id); 1823 init_port_config(); 1824 } 1825 1826 int 1827 init_fwd_streams(void) 1828 { 1829 portid_t pid; 1830 struct rte_port *port; 1831 streamid_t sm_id, nb_fwd_streams_new; 1832 queueid_t q; 1833 1834 /* set socket id according to numa or not */ 1835 RTE_ETH_FOREACH_DEV(pid) { 1836 port = &ports[pid]; 1837 if (nb_rxq > port->dev_info.max_rx_queues) { 1838 fprintf(stderr, 1839 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1840 nb_rxq, port->dev_info.max_rx_queues); 1841 return -1; 1842 } 1843 if (nb_txq > port->dev_info.max_tx_queues) { 1844 fprintf(stderr, 1845 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1846 nb_txq, port->dev_info.max_tx_queues); 1847 return -1; 1848 } 1849 if (numa_support) { 1850 if (port_numa[pid] != NUMA_NO_CONFIG) 1851 port->socket_id = port_numa[pid]; 1852 else { 1853 port->socket_id = rte_eth_dev_socket_id(pid); 1854 1855 /* 1856 * if socket_id is invalid, 1857 * set to the first available socket. 1858 */ 1859 if (check_socket_id(port->socket_id) < 0) 1860 port->socket_id = socket_ids[0]; 1861 } 1862 } 1863 else { 1864 if (socket_num == UMA_NO_CONFIG) 1865 port->socket_id = 0; 1866 else 1867 port->socket_id = socket_num; 1868 } 1869 } 1870 1871 q = RTE_MAX(nb_rxq, nb_txq); 1872 if (q == 0) { 1873 fprintf(stderr, 1874 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1875 return -1; 1876 } 1877 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1878 if (nb_fwd_streams_new == nb_fwd_streams) 1879 return 0; 1880 /* clear the old */ 1881 if (fwd_streams != NULL) { 1882 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1883 if (fwd_streams[sm_id] == NULL) 1884 continue; 1885 rte_free(fwd_streams[sm_id]); 1886 fwd_streams[sm_id] = NULL; 1887 } 1888 rte_free(fwd_streams); 1889 fwd_streams = NULL; 1890 } 1891 1892 /* init new */ 1893 nb_fwd_streams = nb_fwd_streams_new; 1894 if (nb_fwd_streams) { 1895 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1896 sizeof(struct fwd_stream *) * nb_fwd_streams, 1897 RTE_CACHE_LINE_SIZE); 1898 if (fwd_streams == NULL) 1899 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1900 " (struct fwd_stream *)) failed\n", 1901 nb_fwd_streams); 1902 1903 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1904 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1905 " struct fwd_stream", sizeof(struct fwd_stream), 1906 RTE_CACHE_LINE_SIZE); 1907 if (fwd_streams[sm_id] == NULL) 1908 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1909 "(struct fwd_stream) failed\n"); 1910 } 1911 } 1912 1913 return 0; 1914 } 1915 1916 static void 1917 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1918 { 1919 uint64_t total_burst, sburst; 1920 uint64_t nb_burst; 1921 uint64_t burst_stats[4]; 1922 uint16_t pktnb_stats[4]; 1923 uint16_t nb_pkt; 1924 int burst_percent[4], sburstp; 1925 int i; 1926 1927 /* 1928 * First compute the total number of packet bursts and the 1929 * two highest numbers of bursts of the same number of packets. 1930 */ 1931 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1932 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1933 1934 /* Show stats for 0 burst size always */ 1935 total_burst = pbs->pkt_burst_spread[0]; 1936 burst_stats[0] = pbs->pkt_burst_spread[0]; 1937 pktnb_stats[0] = 0; 1938 1939 /* Find the next 2 burst sizes with highest occurrences. */ 1940 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1941 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1942 1943 if (nb_burst == 0) 1944 continue; 1945 1946 total_burst += nb_burst; 1947 1948 if (nb_burst > burst_stats[1]) { 1949 burst_stats[2] = burst_stats[1]; 1950 pktnb_stats[2] = pktnb_stats[1]; 1951 burst_stats[1] = nb_burst; 1952 pktnb_stats[1] = nb_pkt; 1953 } else if (nb_burst > burst_stats[2]) { 1954 burst_stats[2] = nb_burst; 1955 pktnb_stats[2] = nb_pkt; 1956 } 1957 } 1958 if (total_burst == 0) 1959 return; 1960 1961 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1962 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1963 if (i == 3) { 1964 printf("%d%% of other]\n", 100 - sburstp); 1965 return; 1966 } 1967 1968 sburst += burst_stats[i]; 1969 if (sburst == total_burst) { 1970 printf("%d%% of %d pkts]\n", 1971 100 - sburstp, (int) pktnb_stats[i]); 1972 return; 1973 } 1974 1975 burst_percent[i] = 1976 (double)burst_stats[i] / total_burst * 100; 1977 printf("%d%% of %d pkts + ", 1978 burst_percent[i], (int) pktnb_stats[i]); 1979 sburstp += burst_percent[i]; 1980 } 1981 } 1982 1983 static void 1984 fwd_stream_stats_display(streamid_t stream_id) 1985 { 1986 struct fwd_stream *fs; 1987 static const char *fwd_top_stats_border = "-------"; 1988 1989 fs = fwd_streams[stream_id]; 1990 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1991 (fs->fwd_dropped == 0)) 1992 return; 1993 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1994 "TX Port=%2d/Queue=%2d %s\n", 1995 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1996 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1997 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1998 " TX-dropped: %-14"PRIu64, 1999 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 2000 2001 /* if checksum mode */ 2002 if (cur_fwd_eng == &csum_fwd_engine) { 2003 printf(" RX- bad IP checksum: %-14"PRIu64 2004 " Rx- bad L4 checksum: %-14"PRIu64 2005 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 2006 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 2007 fs->rx_bad_outer_l4_csum); 2008 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 2009 fs->rx_bad_outer_ip_csum); 2010 } else { 2011 printf("\n"); 2012 } 2013 2014 if (record_burst_stats) { 2015 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2016 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2017 } 2018 } 2019 2020 void 2021 fwd_stats_display(void) 2022 { 2023 static const char *fwd_stats_border = "----------------------"; 2024 static const char *acc_stats_border = "+++++++++++++++"; 2025 struct { 2026 struct fwd_stream *rx_stream; 2027 struct fwd_stream *tx_stream; 2028 uint64_t tx_dropped; 2029 uint64_t rx_bad_ip_csum; 2030 uint64_t rx_bad_l4_csum; 2031 uint64_t rx_bad_outer_l4_csum; 2032 uint64_t rx_bad_outer_ip_csum; 2033 } ports_stats[RTE_MAX_ETHPORTS]; 2034 uint64_t total_rx_dropped = 0; 2035 uint64_t total_tx_dropped = 0; 2036 uint64_t total_rx_nombuf = 0; 2037 struct rte_eth_stats stats; 2038 uint64_t fwd_cycles = 0; 2039 uint64_t total_recv = 0; 2040 uint64_t total_xmit = 0; 2041 struct rte_port *port; 2042 streamid_t sm_id; 2043 portid_t pt_id; 2044 int ret; 2045 int i; 2046 2047 memset(ports_stats, 0, sizeof(ports_stats)); 2048 2049 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2050 struct fwd_stream *fs = fwd_streams[sm_id]; 2051 2052 if (cur_fwd_config.nb_fwd_streams > 2053 cur_fwd_config.nb_fwd_ports) { 2054 fwd_stream_stats_display(sm_id); 2055 } else { 2056 ports_stats[fs->tx_port].tx_stream = fs; 2057 ports_stats[fs->rx_port].rx_stream = fs; 2058 } 2059 2060 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2061 2062 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2063 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2064 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2065 fs->rx_bad_outer_l4_csum; 2066 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2067 fs->rx_bad_outer_ip_csum; 2068 2069 if (record_core_cycles) 2070 fwd_cycles += fs->core_cycles; 2071 } 2072 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2073 pt_id = fwd_ports_ids[i]; 2074 port = &ports[pt_id]; 2075 2076 ret = rte_eth_stats_get(pt_id, &stats); 2077 if (ret != 0) { 2078 fprintf(stderr, 2079 "%s: Error: failed to get stats (port %u): %d", 2080 __func__, pt_id, ret); 2081 continue; 2082 } 2083 stats.ipackets -= port->stats.ipackets; 2084 stats.opackets -= port->stats.opackets; 2085 stats.ibytes -= port->stats.ibytes; 2086 stats.obytes -= port->stats.obytes; 2087 stats.imissed -= port->stats.imissed; 2088 stats.oerrors -= port->stats.oerrors; 2089 stats.rx_nombuf -= port->stats.rx_nombuf; 2090 2091 total_recv += stats.ipackets; 2092 total_xmit += stats.opackets; 2093 total_rx_dropped += stats.imissed; 2094 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2095 total_tx_dropped += stats.oerrors; 2096 total_rx_nombuf += stats.rx_nombuf; 2097 2098 printf("\n %s Forward statistics for port %-2d %s\n", 2099 fwd_stats_border, pt_id, fwd_stats_border); 2100 2101 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2102 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2103 stats.ipackets + stats.imissed); 2104 2105 if (cur_fwd_eng == &csum_fwd_engine) { 2106 printf(" Bad-ipcsum: %-14"PRIu64 2107 " Bad-l4csum: %-14"PRIu64 2108 "Bad-outer-l4csum: %-14"PRIu64"\n", 2109 ports_stats[pt_id].rx_bad_ip_csum, 2110 ports_stats[pt_id].rx_bad_l4_csum, 2111 ports_stats[pt_id].rx_bad_outer_l4_csum); 2112 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2113 ports_stats[pt_id].rx_bad_outer_ip_csum); 2114 } 2115 if (stats.ierrors + stats.rx_nombuf > 0) { 2116 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2117 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2118 } 2119 2120 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2121 "TX-total: %-"PRIu64"\n", 2122 stats.opackets, ports_stats[pt_id].tx_dropped, 2123 stats.opackets + ports_stats[pt_id].tx_dropped); 2124 2125 if (record_burst_stats) { 2126 if (ports_stats[pt_id].rx_stream) 2127 pkt_burst_stats_display("RX", 2128 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2129 if (ports_stats[pt_id].tx_stream) 2130 pkt_burst_stats_display("TX", 2131 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2132 } 2133 2134 printf(" %s--------------------------------%s\n", 2135 fwd_stats_border, fwd_stats_border); 2136 } 2137 2138 printf("\n %s Accumulated forward statistics for all ports" 2139 "%s\n", 2140 acc_stats_border, acc_stats_border); 2141 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2142 "%-"PRIu64"\n" 2143 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2144 "%-"PRIu64"\n", 2145 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2146 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2147 if (total_rx_nombuf > 0) 2148 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2149 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2150 "%s\n", 2151 acc_stats_border, acc_stats_border); 2152 if (record_core_cycles) { 2153 #define CYC_PER_MHZ 1E6 2154 if (total_recv > 0 || total_xmit > 0) { 2155 uint64_t total_pkts = 0; 2156 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2157 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2158 total_pkts = total_xmit; 2159 else 2160 total_pkts = total_recv; 2161 2162 printf("\n CPU cycles/packet=%.2F (total cycles=" 2163 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2164 " MHz Clock\n", 2165 (double) fwd_cycles / total_pkts, 2166 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2167 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2168 } 2169 } 2170 } 2171 2172 void 2173 fwd_stats_reset(void) 2174 { 2175 streamid_t sm_id; 2176 portid_t pt_id; 2177 int ret; 2178 int i; 2179 2180 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2181 pt_id = fwd_ports_ids[i]; 2182 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2183 if (ret != 0) 2184 fprintf(stderr, 2185 "%s: Error: failed to clear stats (port %u):%d", 2186 __func__, pt_id, ret); 2187 } 2188 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2189 struct fwd_stream *fs = fwd_streams[sm_id]; 2190 2191 fs->rx_packets = 0; 2192 fs->tx_packets = 0; 2193 fs->fwd_dropped = 0; 2194 fs->rx_bad_ip_csum = 0; 2195 fs->rx_bad_l4_csum = 0; 2196 fs->rx_bad_outer_l4_csum = 0; 2197 fs->rx_bad_outer_ip_csum = 0; 2198 2199 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2200 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2201 fs->core_cycles = 0; 2202 } 2203 } 2204 2205 static void 2206 flush_fwd_rx_queues(void) 2207 { 2208 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2209 portid_t rxp; 2210 portid_t port_id; 2211 queueid_t rxq; 2212 uint16_t nb_rx; 2213 uint16_t i; 2214 uint8_t j; 2215 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2216 uint64_t timer_period; 2217 2218 if (num_procs > 1) { 2219 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2220 return; 2221 } 2222 2223 /* convert to number of cycles */ 2224 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2225 2226 for (j = 0; j < 2; j++) { 2227 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2228 for (rxq = 0; rxq < nb_rxq; rxq++) { 2229 port_id = fwd_ports_ids[rxp]; 2230 2231 /* Polling stopped queues is prohibited. */ 2232 if (ports[port_id].rxq[rxq].state == 2233 RTE_ETH_QUEUE_STATE_STOPPED) 2234 continue; 2235 2236 /** 2237 * testpmd can stuck in the below do while loop 2238 * if rte_eth_rx_burst() always returns nonzero 2239 * packets. So timer is added to exit this loop 2240 * after 1sec timer expiry. 2241 */ 2242 prev_tsc = rte_rdtsc(); 2243 do { 2244 nb_rx = rte_eth_rx_burst(port_id, rxq, 2245 pkts_burst, MAX_PKT_BURST); 2246 for (i = 0; i < nb_rx; i++) 2247 rte_pktmbuf_free(pkts_burst[i]); 2248 2249 cur_tsc = rte_rdtsc(); 2250 diff_tsc = cur_tsc - prev_tsc; 2251 timer_tsc += diff_tsc; 2252 } while ((nb_rx > 0) && 2253 (timer_tsc < timer_period)); 2254 timer_tsc = 0; 2255 } 2256 } 2257 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2258 } 2259 } 2260 2261 static void 2262 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2263 { 2264 struct fwd_stream **fsm; 2265 streamid_t nb_fs; 2266 streamid_t sm_id; 2267 #ifdef RTE_LIB_BITRATESTATS 2268 uint64_t tics_per_1sec; 2269 uint64_t tics_datum; 2270 uint64_t tics_current; 2271 uint16_t i, cnt_ports; 2272 2273 cnt_ports = nb_ports; 2274 tics_datum = rte_rdtsc(); 2275 tics_per_1sec = rte_get_timer_hz(); 2276 #endif 2277 fsm = &fwd_streams[fc->stream_idx]; 2278 nb_fs = fc->stream_nb; 2279 do { 2280 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2281 if (!fsm[sm_id]->disabled) 2282 (*pkt_fwd)(fsm[sm_id]); 2283 #ifdef RTE_LIB_BITRATESTATS 2284 if (bitrate_enabled != 0 && 2285 bitrate_lcore_id == rte_lcore_id()) { 2286 tics_current = rte_rdtsc(); 2287 if (tics_current - tics_datum >= tics_per_1sec) { 2288 /* Periodic bitrate calculation */ 2289 for (i = 0; i < cnt_ports; i++) 2290 rte_stats_bitrate_calc(bitrate_data, 2291 ports_ids[i]); 2292 tics_datum = tics_current; 2293 } 2294 } 2295 #endif 2296 #ifdef RTE_LIB_LATENCYSTATS 2297 if (latencystats_enabled != 0 && 2298 latencystats_lcore_id == rte_lcore_id()) 2299 rte_latencystats_update(); 2300 #endif 2301 2302 } while (! fc->stopped); 2303 } 2304 2305 static int 2306 start_pkt_forward_on_core(void *fwd_arg) 2307 { 2308 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2309 cur_fwd_config.fwd_eng->packet_fwd); 2310 return 0; 2311 } 2312 2313 /* 2314 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2315 * Used to start communication flows in network loopback test configurations. 2316 */ 2317 static int 2318 run_one_txonly_burst_on_core(void *fwd_arg) 2319 { 2320 struct fwd_lcore *fwd_lc; 2321 struct fwd_lcore tmp_lcore; 2322 2323 fwd_lc = (struct fwd_lcore *) fwd_arg; 2324 tmp_lcore = *fwd_lc; 2325 tmp_lcore.stopped = 1; 2326 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2327 return 0; 2328 } 2329 2330 /* 2331 * Launch packet forwarding: 2332 * - Setup per-port forwarding context. 2333 * - launch logical cores with their forwarding configuration. 2334 */ 2335 static void 2336 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2337 { 2338 unsigned int i; 2339 unsigned int lc_id; 2340 int diag; 2341 2342 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2343 lc_id = fwd_lcores_cpuids[i]; 2344 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2345 fwd_lcores[i]->stopped = 0; 2346 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2347 fwd_lcores[i], lc_id); 2348 if (diag != 0) 2349 fprintf(stderr, 2350 "launch lcore %u failed - diag=%d\n", 2351 lc_id, diag); 2352 } 2353 } 2354 } 2355 2356 /* 2357 * Launch packet forwarding configuration. 2358 */ 2359 void 2360 start_packet_forwarding(int with_tx_first) 2361 { 2362 port_fwd_begin_t port_fwd_begin; 2363 port_fwd_end_t port_fwd_end; 2364 stream_init_t stream_init = cur_fwd_eng->stream_init; 2365 unsigned int i; 2366 2367 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2368 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2369 2370 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2371 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2372 2373 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2374 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2375 (!nb_rxq || !nb_txq)) 2376 rte_exit(EXIT_FAILURE, 2377 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2378 cur_fwd_eng->fwd_mode_name); 2379 2380 if (all_ports_started() == 0) { 2381 fprintf(stderr, "Not all ports were started\n"); 2382 return; 2383 } 2384 if (test_done == 0) { 2385 fprintf(stderr, "Packet forwarding already started\n"); 2386 return; 2387 } 2388 2389 fwd_config_setup(); 2390 2391 pkt_fwd_config_display(&cur_fwd_config); 2392 if (!pkt_fwd_shared_rxq_check()) 2393 return; 2394 2395 if (stream_init != NULL) 2396 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2397 stream_init(fwd_streams[i]); 2398 2399 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2400 if (port_fwd_begin != NULL) { 2401 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2402 if (port_fwd_begin(fwd_ports_ids[i])) { 2403 fprintf(stderr, 2404 "Packet forwarding is not ready\n"); 2405 return; 2406 } 2407 } 2408 } 2409 2410 if (with_tx_first) { 2411 port_fwd_begin = tx_only_engine.port_fwd_begin; 2412 if (port_fwd_begin != NULL) { 2413 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2414 if (port_fwd_begin(fwd_ports_ids[i])) { 2415 fprintf(stderr, 2416 "Packet forwarding is not ready\n"); 2417 return; 2418 } 2419 } 2420 } 2421 } 2422 2423 test_done = 0; 2424 2425 if(!no_flush_rx) 2426 flush_fwd_rx_queues(); 2427 2428 rxtx_config_display(); 2429 2430 fwd_stats_reset(); 2431 if (with_tx_first) { 2432 while (with_tx_first--) { 2433 launch_packet_forwarding( 2434 run_one_txonly_burst_on_core); 2435 rte_eal_mp_wait_lcore(); 2436 } 2437 port_fwd_end = tx_only_engine.port_fwd_end; 2438 if (port_fwd_end != NULL) { 2439 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2440 (*port_fwd_end)(fwd_ports_ids[i]); 2441 } 2442 } 2443 launch_packet_forwarding(start_pkt_forward_on_core); 2444 } 2445 2446 void 2447 stop_packet_forwarding(void) 2448 { 2449 port_fwd_end_t port_fwd_end; 2450 lcoreid_t lc_id; 2451 portid_t pt_id; 2452 int i; 2453 2454 if (test_done) { 2455 fprintf(stderr, "Packet forwarding not started\n"); 2456 return; 2457 } 2458 printf("Telling cores to stop..."); 2459 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2460 fwd_lcores[lc_id]->stopped = 1; 2461 printf("\nWaiting for lcores to finish...\n"); 2462 rte_eal_mp_wait_lcore(); 2463 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2464 if (port_fwd_end != NULL) { 2465 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2466 pt_id = fwd_ports_ids[i]; 2467 (*port_fwd_end)(pt_id); 2468 } 2469 } 2470 2471 fwd_stats_display(); 2472 2473 printf("\nDone.\n"); 2474 test_done = 1; 2475 } 2476 2477 void 2478 dev_set_link_up(portid_t pid) 2479 { 2480 if (rte_eth_dev_set_link_up(pid) < 0) 2481 fprintf(stderr, "\nSet link up fail.\n"); 2482 } 2483 2484 void 2485 dev_set_link_down(portid_t pid) 2486 { 2487 if (rte_eth_dev_set_link_down(pid) < 0) 2488 fprintf(stderr, "\nSet link down fail.\n"); 2489 } 2490 2491 static int 2492 all_ports_started(void) 2493 { 2494 portid_t pi; 2495 struct rte_port *port; 2496 2497 RTE_ETH_FOREACH_DEV(pi) { 2498 port = &ports[pi]; 2499 /* Check if there is a port which is not started */ 2500 if ((port->port_status != RTE_PORT_STARTED) && 2501 (port->slave_flag == 0)) 2502 return 0; 2503 } 2504 2505 /* No port is not started */ 2506 return 1; 2507 } 2508 2509 int 2510 port_is_stopped(portid_t port_id) 2511 { 2512 struct rte_port *port = &ports[port_id]; 2513 2514 if ((port->port_status != RTE_PORT_STOPPED) && 2515 (port->slave_flag == 0)) 2516 return 0; 2517 return 1; 2518 } 2519 2520 int 2521 all_ports_stopped(void) 2522 { 2523 portid_t pi; 2524 2525 RTE_ETH_FOREACH_DEV(pi) { 2526 if (!port_is_stopped(pi)) 2527 return 0; 2528 } 2529 2530 return 1; 2531 } 2532 2533 int 2534 port_is_started(portid_t port_id) 2535 { 2536 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2537 return 0; 2538 2539 if (ports[port_id].port_status != RTE_PORT_STARTED) 2540 return 0; 2541 2542 return 1; 2543 } 2544 2545 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2546 static int 2547 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2548 { 2549 queueid_t qi; 2550 struct rte_eth_hairpin_conf hairpin_conf = { 2551 .peer_count = 1, 2552 }; 2553 int i; 2554 int diag; 2555 struct rte_port *port = &ports[pi]; 2556 uint16_t peer_rx_port = pi; 2557 uint16_t peer_tx_port = pi; 2558 uint32_t manual = 1; 2559 uint32_t tx_exp = hairpin_mode & 0x10; 2560 2561 if (!(hairpin_mode & 0xf)) { 2562 peer_rx_port = pi; 2563 peer_tx_port = pi; 2564 manual = 0; 2565 } else if (hairpin_mode & 0x1) { 2566 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2567 RTE_ETH_DEV_NO_OWNER); 2568 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2569 peer_tx_port = rte_eth_find_next_owned_by(0, 2570 RTE_ETH_DEV_NO_OWNER); 2571 if (p_pi != RTE_MAX_ETHPORTS) { 2572 peer_rx_port = p_pi; 2573 } else { 2574 uint16_t next_pi; 2575 2576 /* Last port will be the peer RX port of the first. */ 2577 RTE_ETH_FOREACH_DEV(next_pi) 2578 peer_rx_port = next_pi; 2579 } 2580 manual = 1; 2581 } else if (hairpin_mode & 0x2) { 2582 if (cnt_pi & 0x1) { 2583 peer_rx_port = p_pi; 2584 } else { 2585 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2586 RTE_ETH_DEV_NO_OWNER); 2587 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2588 peer_rx_port = pi; 2589 } 2590 peer_tx_port = peer_rx_port; 2591 manual = 1; 2592 } 2593 2594 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2595 hairpin_conf.peers[0].port = peer_rx_port; 2596 hairpin_conf.peers[0].queue = i + nb_rxq; 2597 hairpin_conf.manual_bind = !!manual; 2598 hairpin_conf.tx_explicit = !!tx_exp; 2599 diag = rte_eth_tx_hairpin_queue_setup 2600 (pi, qi, nb_txd, &hairpin_conf); 2601 i++; 2602 if (diag == 0) 2603 continue; 2604 2605 /* Fail to setup rx queue, return */ 2606 if (port->port_status == RTE_PORT_HANDLING) 2607 port->port_status = RTE_PORT_STOPPED; 2608 else 2609 fprintf(stderr, 2610 "Port %d can not be set back to stopped\n", pi); 2611 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2612 pi); 2613 /* try to reconfigure queues next time */ 2614 port->need_reconfig_queues = 1; 2615 return -1; 2616 } 2617 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2618 hairpin_conf.peers[0].port = peer_tx_port; 2619 hairpin_conf.peers[0].queue = i + nb_txq; 2620 hairpin_conf.manual_bind = !!manual; 2621 hairpin_conf.tx_explicit = !!tx_exp; 2622 diag = rte_eth_rx_hairpin_queue_setup 2623 (pi, qi, nb_rxd, &hairpin_conf); 2624 i++; 2625 if (diag == 0) 2626 continue; 2627 2628 /* Fail to setup rx queue, return */ 2629 if (port->port_status == RTE_PORT_HANDLING) 2630 port->port_status = RTE_PORT_STOPPED; 2631 else 2632 fprintf(stderr, 2633 "Port %d can not be set back to stopped\n", pi); 2634 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2635 pi); 2636 /* try to reconfigure queues next time */ 2637 port->need_reconfig_queues = 1; 2638 return -1; 2639 } 2640 return 0; 2641 } 2642 2643 /* Configure the Rx with optional split. */ 2644 int 2645 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2646 uint16_t nb_rx_desc, unsigned int socket_id, 2647 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2648 { 2649 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2650 unsigned int i, mp_n; 2651 int ret; 2652 2653 if (rx_pkt_nb_segs <= 1 || 2654 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2655 rx_conf->rx_seg = NULL; 2656 rx_conf->rx_nseg = 0; 2657 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2658 nb_rx_desc, socket_id, 2659 rx_conf, mp); 2660 goto exit; 2661 } 2662 for (i = 0; i < rx_pkt_nb_segs; i++) { 2663 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2664 struct rte_mempool *mpx; 2665 /* 2666 * Use last valid pool for the segments with number 2667 * exceeding the pool index. 2668 */ 2669 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2670 mpx = mbuf_pool_find(socket_id, mp_n); 2671 /* Handle zero as mbuf data buffer size. */ 2672 rx_seg->length = rx_pkt_seg_lengths[i] ? 2673 rx_pkt_seg_lengths[i] : 2674 mbuf_data_size[mp_n]; 2675 rx_seg->offset = i < rx_pkt_nb_offs ? 2676 rx_pkt_seg_offsets[i] : 0; 2677 rx_seg->mp = mpx ? mpx : mp; 2678 } 2679 rx_conf->rx_nseg = rx_pkt_nb_segs; 2680 rx_conf->rx_seg = rx_useg; 2681 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2682 socket_id, rx_conf, NULL); 2683 rx_conf->rx_seg = NULL; 2684 rx_conf->rx_nseg = 0; 2685 exit: 2686 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2687 RTE_ETH_QUEUE_STATE_STOPPED : 2688 RTE_ETH_QUEUE_STATE_STARTED; 2689 return ret; 2690 } 2691 2692 static int 2693 alloc_xstats_display_info(portid_t pi) 2694 { 2695 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2696 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2697 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2698 2699 if (xstats_display_num == 0) 2700 return 0; 2701 2702 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2703 if (*ids_supp == NULL) 2704 goto fail_ids_supp; 2705 2706 *prev_values = calloc(xstats_display_num, 2707 sizeof(**prev_values)); 2708 if (*prev_values == NULL) 2709 goto fail_prev_values; 2710 2711 *curr_values = calloc(xstats_display_num, 2712 sizeof(**curr_values)); 2713 if (*curr_values == NULL) 2714 goto fail_curr_values; 2715 2716 ports[pi].xstats_info.allocated = true; 2717 2718 return 0; 2719 2720 fail_curr_values: 2721 free(*prev_values); 2722 fail_prev_values: 2723 free(*ids_supp); 2724 fail_ids_supp: 2725 return -ENOMEM; 2726 } 2727 2728 static void 2729 free_xstats_display_info(portid_t pi) 2730 { 2731 if (!ports[pi].xstats_info.allocated) 2732 return; 2733 free(ports[pi].xstats_info.ids_supp); 2734 free(ports[pi].xstats_info.prev_values); 2735 free(ports[pi].xstats_info.curr_values); 2736 ports[pi].xstats_info.allocated = false; 2737 } 2738 2739 /** Fill helper structures for specified port to show extended statistics. */ 2740 static void 2741 fill_xstats_display_info_for_port(portid_t pi) 2742 { 2743 unsigned int stat, stat_supp; 2744 const char *xstat_name; 2745 struct rte_port *port; 2746 uint64_t *ids_supp; 2747 int rc; 2748 2749 if (xstats_display_num == 0) 2750 return; 2751 2752 if (pi == (portid_t)RTE_PORT_ALL) { 2753 fill_xstats_display_info(); 2754 return; 2755 } 2756 2757 port = &ports[pi]; 2758 if (port->port_status != RTE_PORT_STARTED) 2759 return; 2760 2761 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2762 rte_exit(EXIT_FAILURE, 2763 "Failed to allocate xstats display memory\n"); 2764 2765 ids_supp = port->xstats_info.ids_supp; 2766 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2767 xstat_name = xstats_display[stat].name; 2768 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2769 ids_supp + stat_supp); 2770 if (rc != 0) { 2771 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2772 xstat_name, pi, stat); 2773 continue; 2774 } 2775 stat_supp++; 2776 } 2777 2778 port->xstats_info.ids_supp_sz = stat_supp; 2779 } 2780 2781 /** Fill helper structures for all ports to show extended statistics. */ 2782 static void 2783 fill_xstats_display_info(void) 2784 { 2785 portid_t pi; 2786 2787 if (xstats_display_num == 0) 2788 return; 2789 2790 RTE_ETH_FOREACH_DEV(pi) 2791 fill_xstats_display_info_for_port(pi); 2792 } 2793 2794 int 2795 start_port(portid_t pid) 2796 { 2797 int diag, need_check_link_status = -1; 2798 portid_t pi; 2799 portid_t p_pi = RTE_MAX_ETHPORTS; 2800 portid_t pl[RTE_MAX_ETHPORTS]; 2801 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2802 uint16_t cnt_pi = 0; 2803 uint16_t cfg_pi = 0; 2804 int peer_pi; 2805 queueid_t qi; 2806 struct rte_port *port; 2807 struct rte_eth_hairpin_cap cap; 2808 2809 if (port_id_is_invalid(pid, ENABLED_WARN)) 2810 return 0; 2811 2812 RTE_ETH_FOREACH_DEV(pi) { 2813 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2814 continue; 2815 2816 if (port_is_bonding_slave(pi)) { 2817 fprintf(stderr, 2818 "Please remove port %d from bonded device.\n", 2819 pi); 2820 continue; 2821 } 2822 2823 need_check_link_status = 0; 2824 port = &ports[pi]; 2825 if (port->port_status == RTE_PORT_STOPPED) 2826 port->port_status = RTE_PORT_HANDLING; 2827 else { 2828 fprintf(stderr, "Port %d is now not stopped\n", pi); 2829 continue; 2830 } 2831 2832 if (port->need_reconfig > 0) { 2833 struct rte_eth_conf dev_conf; 2834 int k; 2835 2836 port->need_reconfig = 0; 2837 2838 if (flow_isolate_all) { 2839 int ret = port_flow_isolate(pi, 1); 2840 if (ret) { 2841 fprintf(stderr, 2842 "Failed to apply isolated mode on port %d\n", 2843 pi); 2844 return -1; 2845 } 2846 } 2847 configure_rxtx_dump_callbacks(0); 2848 printf("Configuring Port %d (socket %u)\n", pi, 2849 port->socket_id); 2850 if (nb_hairpinq > 0 && 2851 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2852 fprintf(stderr, 2853 "Port %d doesn't support hairpin queues\n", 2854 pi); 2855 return -1; 2856 } 2857 2858 /* configure port */ 2859 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2860 nb_txq + nb_hairpinq, 2861 &(port->dev_conf)); 2862 if (diag != 0) { 2863 if (port->port_status == RTE_PORT_HANDLING) 2864 port->port_status = RTE_PORT_STOPPED; 2865 else 2866 fprintf(stderr, 2867 "Port %d can not be set back to stopped\n", 2868 pi); 2869 fprintf(stderr, "Fail to configure port %d\n", 2870 pi); 2871 /* try to reconfigure port next time */ 2872 port->need_reconfig = 1; 2873 return -1; 2874 } 2875 /* get device configuration*/ 2876 if (0 != 2877 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2878 fprintf(stderr, 2879 "port %d can not get device configuration\n", 2880 pi); 2881 return -1; 2882 } 2883 /* Apply Rx offloads configuration */ 2884 if (dev_conf.rxmode.offloads != 2885 port->dev_conf.rxmode.offloads) { 2886 port->dev_conf.rxmode.offloads |= 2887 dev_conf.rxmode.offloads; 2888 for (k = 0; 2889 k < port->dev_info.max_rx_queues; 2890 k++) 2891 port->rxq[k].conf.offloads |= 2892 dev_conf.rxmode.offloads; 2893 } 2894 /* Apply Tx offloads configuration */ 2895 if (dev_conf.txmode.offloads != 2896 port->dev_conf.txmode.offloads) { 2897 port->dev_conf.txmode.offloads |= 2898 dev_conf.txmode.offloads; 2899 for (k = 0; 2900 k < port->dev_info.max_tx_queues; 2901 k++) 2902 port->txq[k].conf.offloads |= 2903 dev_conf.txmode.offloads; 2904 } 2905 } 2906 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2907 port->need_reconfig_queues = 0; 2908 /* setup tx queues */ 2909 for (qi = 0; qi < nb_txq; qi++) { 2910 struct rte_eth_txconf *conf = 2911 &port->txq[qi].conf; 2912 2913 if ((numa_support) && 2914 (txring_numa[pi] != NUMA_NO_CONFIG)) 2915 diag = rte_eth_tx_queue_setup(pi, qi, 2916 port->nb_tx_desc[qi], 2917 txring_numa[pi], 2918 &(port->txq[qi].conf)); 2919 else 2920 diag = rte_eth_tx_queue_setup(pi, qi, 2921 port->nb_tx_desc[qi], 2922 port->socket_id, 2923 &(port->txq[qi].conf)); 2924 2925 if (diag == 0) { 2926 port->txq[qi].state = 2927 conf->tx_deferred_start ? 2928 RTE_ETH_QUEUE_STATE_STOPPED : 2929 RTE_ETH_QUEUE_STATE_STARTED; 2930 continue; 2931 } 2932 2933 /* Fail to setup tx queue, return */ 2934 if (port->port_status == RTE_PORT_HANDLING) 2935 port->port_status = RTE_PORT_STOPPED; 2936 else 2937 fprintf(stderr, 2938 "Port %d can not be set back to stopped\n", 2939 pi); 2940 fprintf(stderr, 2941 "Fail to configure port %d tx queues\n", 2942 pi); 2943 /* try to reconfigure queues next time */ 2944 port->need_reconfig_queues = 1; 2945 return -1; 2946 } 2947 for (qi = 0; qi < nb_rxq; qi++) { 2948 /* setup rx queues */ 2949 if ((numa_support) && 2950 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2951 struct rte_mempool * mp = 2952 mbuf_pool_find 2953 (rxring_numa[pi], 0); 2954 if (mp == NULL) { 2955 fprintf(stderr, 2956 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2957 rxring_numa[pi]); 2958 return -1; 2959 } 2960 2961 diag = rx_queue_setup(pi, qi, 2962 port->nb_rx_desc[qi], 2963 rxring_numa[pi], 2964 &(port->rxq[qi].conf), 2965 mp); 2966 } else { 2967 struct rte_mempool *mp = 2968 mbuf_pool_find 2969 (port->socket_id, 0); 2970 if (mp == NULL) { 2971 fprintf(stderr, 2972 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2973 port->socket_id); 2974 return -1; 2975 } 2976 diag = rx_queue_setup(pi, qi, 2977 port->nb_rx_desc[qi], 2978 port->socket_id, 2979 &(port->rxq[qi].conf), 2980 mp); 2981 } 2982 if (diag == 0) 2983 continue; 2984 2985 /* Fail to setup rx queue, return */ 2986 if (port->port_status == RTE_PORT_HANDLING) 2987 port->port_status = RTE_PORT_STOPPED; 2988 else 2989 fprintf(stderr, 2990 "Port %d can not be set back to stopped\n", 2991 pi); 2992 fprintf(stderr, 2993 "Fail to configure port %d rx queues\n", 2994 pi); 2995 /* try to reconfigure queues next time */ 2996 port->need_reconfig_queues = 1; 2997 return -1; 2998 } 2999 /* setup hairpin queues */ 3000 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3001 return -1; 3002 } 3003 configure_rxtx_dump_callbacks(verbose_level); 3004 if (clear_ptypes) { 3005 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3006 NULL, 0); 3007 if (diag < 0) 3008 fprintf(stderr, 3009 "Port %d: Failed to disable Ptype parsing\n", 3010 pi); 3011 } 3012 3013 p_pi = pi; 3014 cnt_pi++; 3015 3016 /* start port */ 3017 diag = eth_dev_start_mp(pi); 3018 if (diag < 0) { 3019 fprintf(stderr, "Fail to start port %d: %s\n", 3020 pi, rte_strerror(-diag)); 3021 3022 /* Fail to setup rx queue, return */ 3023 if (port->port_status == RTE_PORT_HANDLING) 3024 port->port_status = RTE_PORT_STOPPED; 3025 else 3026 fprintf(stderr, 3027 "Port %d can not be set back to stopped\n", 3028 pi); 3029 continue; 3030 } 3031 3032 if (port->port_status == RTE_PORT_HANDLING) 3033 port->port_status = RTE_PORT_STARTED; 3034 else 3035 fprintf(stderr, "Port %d can not be set into started\n", 3036 pi); 3037 3038 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3039 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3040 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3041 3042 /* at least one port started, need checking link status */ 3043 need_check_link_status = 1; 3044 3045 pl[cfg_pi++] = pi; 3046 } 3047 3048 if (need_check_link_status == 1 && !no_link_check) 3049 check_all_ports_link_status(RTE_PORT_ALL); 3050 else if (need_check_link_status == 0) 3051 fprintf(stderr, "Please stop the ports first\n"); 3052 3053 if (hairpin_mode & 0xf) { 3054 uint16_t i; 3055 int j; 3056 3057 /* bind all started hairpin ports */ 3058 for (i = 0; i < cfg_pi; i++) { 3059 pi = pl[i]; 3060 /* bind current Tx to all peer Rx */ 3061 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3062 RTE_MAX_ETHPORTS, 1); 3063 if (peer_pi < 0) 3064 return peer_pi; 3065 for (j = 0; j < peer_pi; j++) { 3066 if (!port_is_started(peer_pl[j])) 3067 continue; 3068 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3069 if (diag < 0) { 3070 fprintf(stderr, 3071 "Error during binding hairpin Tx port %u to %u: %s\n", 3072 pi, peer_pl[j], 3073 rte_strerror(-diag)); 3074 return -1; 3075 } 3076 } 3077 /* bind all peer Tx to current Rx */ 3078 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3079 RTE_MAX_ETHPORTS, 0); 3080 if (peer_pi < 0) 3081 return peer_pi; 3082 for (j = 0; j < peer_pi; j++) { 3083 if (!port_is_started(peer_pl[j])) 3084 continue; 3085 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3086 if (diag < 0) { 3087 fprintf(stderr, 3088 "Error during binding hairpin Tx port %u to %u: %s\n", 3089 peer_pl[j], pi, 3090 rte_strerror(-diag)); 3091 return -1; 3092 } 3093 } 3094 } 3095 } 3096 3097 fill_xstats_display_info_for_port(pid); 3098 3099 printf("Done\n"); 3100 return 0; 3101 } 3102 3103 void 3104 stop_port(portid_t pid) 3105 { 3106 portid_t pi; 3107 struct rte_port *port; 3108 int need_check_link_status = 0; 3109 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3110 int peer_pi; 3111 3112 if (port_id_is_invalid(pid, ENABLED_WARN)) 3113 return; 3114 3115 printf("Stopping ports...\n"); 3116 3117 RTE_ETH_FOREACH_DEV(pi) { 3118 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3119 continue; 3120 3121 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3122 fprintf(stderr, 3123 "Please remove port %d from forwarding configuration.\n", 3124 pi); 3125 continue; 3126 } 3127 3128 if (port_is_bonding_slave(pi)) { 3129 fprintf(stderr, 3130 "Please remove port %d from bonded device.\n", 3131 pi); 3132 continue; 3133 } 3134 3135 port = &ports[pi]; 3136 if (port->port_status == RTE_PORT_STARTED) 3137 port->port_status = RTE_PORT_HANDLING; 3138 else 3139 continue; 3140 3141 if (hairpin_mode & 0xf) { 3142 int j; 3143 3144 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3145 /* unbind all peer Tx from current Rx */ 3146 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3147 RTE_MAX_ETHPORTS, 0); 3148 if (peer_pi < 0) 3149 continue; 3150 for (j = 0; j < peer_pi; j++) { 3151 if (!port_is_started(peer_pl[j])) 3152 continue; 3153 rte_eth_hairpin_unbind(peer_pl[j], pi); 3154 } 3155 } 3156 3157 if (port->flow_list) 3158 port_flow_flush(pi); 3159 3160 if (eth_dev_stop_mp(pi) != 0) 3161 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3162 pi); 3163 3164 if (port->port_status == RTE_PORT_HANDLING) 3165 port->port_status = RTE_PORT_STOPPED; 3166 else 3167 fprintf(stderr, "Port %d can not be set into stopped\n", 3168 pi); 3169 need_check_link_status = 1; 3170 } 3171 if (need_check_link_status && !no_link_check) 3172 check_all_ports_link_status(RTE_PORT_ALL); 3173 3174 printf("Done\n"); 3175 } 3176 3177 static void 3178 remove_invalid_ports_in(portid_t *array, portid_t *total) 3179 { 3180 portid_t i; 3181 portid_t new_total = 0; 3182 3183 for (i = 0; i < *total; i++) 3184 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3185 array[new_total] = array[i]; 3186 new_total++; 3187 } 3188 *total = new_total; 3189 } 3190 3191 static void 3192 remove_invalid_ports(void) 3193 { 3194 remove_invalid_ports_in(ports_ids, &nb_ports); 3195 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3196 nb_cfg_ports = nb_fwd_ports; 3197 } 3198 3199 static void 3200 flush_port_owned_resources(portid_t pi) 3201 { 3202 mcast_addr_pool_destroy(pi); 3203 port_flow_flush(pi); 3204 port_flex_item_flush(pi); 3205 port_action_handle_flush(pi); 3206 } 3207 3208 static void 3209 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3210 { 3211 struct rte_port *port; 3212 portid_t slave_pid; 3213 uint16_t i; 3214 3215 for (i = 0; i < num_slaves; i++) { 3216 slave_pid = slave_pids[i]; 3217 if (port_is_started(slave_pid) == 1) { 3218 if (rte_eth_dev_stop(slave_pid) != 0) 3219 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3220 slave_pid); 3221 3222 port = &ports[slave_pid]; 3223 port->port_status = RTE_PORT_STOPPED; 3224 } 3225 3226 clear_port_slave_flag(slave_pid); 3227 3228 /* Close slave device when testpmd quit or is killed. */ 3229 if (cl_quit == 1 || f_quit == 1) 3230 rte_eth_dev_close(slave_pid); 3231 } 3232 } 3233 3234 void 3235 close_port(portid_t pid) 3236 { 3237 portid_t pi; 3238 struct rte_port *port; 3239 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3240 int num_slaves = 0; 3241 3242 if (port_id_is_invalid(pid, ENABLED_WARN)) 3243 return; 3244 3245 printf("Closing ports...\n"); 3246 3247 RTE_ETH_FOREACH_DEV(pi) { 3248 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3249 continue; 3250 3251 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3252 fprintf(stderr, 3253 "Please remove port %d from forwarding configuration.\n", 3254 pi); 3255 continue; 3256 } 3257 3258 if (port_is_bonding_slave(pi)) { 3259 fprintf(stderr, 3260 "Please remove port %d from bonded device.\n", 3261 pi); 3262 continue; 3263 } 3264 3265 port = &ports[pi]; 3266 if (port->port_status == RTE_PORT_CLOSED) { 3267 fprintf(stderr, "Port %d is already closed\n", pi); 3268 continue; 3269 } 3270 3271 if (is_proc_primary()) { 3272 flush_port_owned_resources(pi); 3273 #ifdef RTE_NET_BOND 3274 if (port->bond_flag == 1) 3275 num_slaves = rte_eth_bond_slaves_get(pi, 3276 slave_pids, RTE_MAX_ETHPORTS); 3277 #endif 3278 rte_eth_dev_close(pi); 3279 /* 3280 * If this port is bonded device, all slaves under the 3281 * device need to be removed or closed. 3282 */ 3283 if (port->bond_flag == 1 && num_slaves > 0) 3284 clear_bonding_slave_device(slave_pids, 3285 num_slaves); 3286 } 3287 3288 free_xstats_display_info(pi); 3289 } 3290 3291 remove_invalid_ports(); 3292 printf("Done\n"); 3293 } 3294 3295 void 3296 reset_port(portid_t pid) 3297 { 3298 int diag; 3299 portid_t pi; 3300 struct rte_port *port; 3301 3302 if (port_id_is_invalid(pid, ENABLED_WARN)) 3303 return; 3304 3305 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3306 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3307 fprintf(stderr, 3308 "Can not reset port(s), please stop port(s) first.\n"); 3309 return; 3310 } 3311 3312 printf("Resetting ports...\n"); 3313 3314 RTE_ETH_FOREACH_DEV(pi) { 3315 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3316 continue; 3317 3318 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3319 fprintf(stderr, 3320 "Please remove port %d from forwarding configuration.\n", 3321 pi); 3322 continue; 3323 } 3324 3325 if (port_is_bonding_slave(pi)) { 3326 fprintf(stderr, 3327 "Please remove port %d from bonded device.\n", 3328 pi); 3329 continue; 3330 } 3331 3332 diag = rte_eth_dev_reset(pi); 3333 if (diag == 0) { 3334 port = &ports[pi]; 3335 port->need_reconfig = 1; 3336 port->need_reconfig_queues = 1; 3337 } else { 3338 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3339 pi, diag); 3340 } 3341 } 3342 3343 printf("Done\n"); 3344 } 3345 3346 void 3347 attach_port(char *identifier) 3348 { 3349 portid_t pi; 3350 struct rte_dev_iterator iterator; 3351 3352 printf("Attaching a new port...\n"); 3353 3354 if (identifier == NULL) { 3355 fprintf(stderr, "Invalid parameters are specified\n"); 3356 return; 3357 } 3358 3359 if (rte_dev_probe(identifier) < 0) { 3360 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3361 return; 3362 } 3363 3364 /* first attach mode: event */ 3365 if (setup_on_probe_event) { 3366 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3367 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3368 if (ports[pi].port_status == RTE_PORT_HANDLING && 3369 ports[pi].need_setup != 0) 3370 setup_attached_port(pi); 3371 return; 3372 } 3373 3374 /* second attach mode: iterator */ 3375 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3376 /* setup ports matching the devargs used for probing */ 3377 if (port_is_forwarding(pi)) 3378 continue; /* port was already attached before */ 3379 setup_attached_port(pi); 3380 } 3381 } 3382 3383 static void 3384 setup_attached_port(portid_t pi) 3385 { 3386 unsigned int socket_id; 3387 int ret; 3388 3389 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3390 /* if socket_id is invalid, set to the first available socket. */ 3391 if (check_socket_id(socket_id) < 0) 3392 socket_id = socket_ids[0]; 3393 reconfig(pi, socket_id); 3394 ret = rte_eth_promiscuous_enable(pi); 3395 if (ret != 0) 3396 fprintf(stderr, 3397 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3398 pi, rte_strerror(-ret)); 3399 3400 ports_ids[nb_ports++] = pi; 3401 fwd_ports_ids[nb_fwd_ports++] = pi; 3402 nb_cfg_ports = nb_fwd_ports; 3403 ports[pi].need_setup = 0; 3404 ports[pi].port_status = RTE_PORT_STOPPED; 3405 3406 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3407 printf("Done\n"); 3408 } 3409 3410 static void 3411 detach_device(struct rte_device *dev) 3412 { 3413 portid_t sibling; 3414 3415 if (dev == NULL) { 3416 fprintf(stderr, "Device already removed\n"); 3417 return; 3418 } 3419 3420 printf("Removing a device...\n"); 3421 3422 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3423 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3424 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3425 fprintf(stderr, "Port %u not stopped\n", 3426 sibling); 3427 return; 3428 } 3429 flush_port_owned_resources(sibling); 3430 } 3431 } 3432 3433 if (rte_dev_remove(dev) < 0) { 3434 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3435 return; 3436 } 3437 remove_invalid_ports(); 3438 3439 printf("Device is detached\n"); 3440 printf("Now total ports is %d\n", nb_ports); 3441 printf("Done\n"); 3442 return; 3443 } 3444 3445 void 3446 detach_port_device(portid_t port_id) 3447 { 3448 int ret; 3449 struct rte_eth_dev_info dev_info; 3450 3451 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3452 return; 3453 3454 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3455 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3456 fprintf(stderr, "Port not stopped\n"); 3457 return; 3458 } 3459 fprintf(stderr, "Port was not closed\n"); 3460 } 3461 3462 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3463 if (ret != 0) { 3464 TESTPMD_LOG(ERR, 3465 "Failed to get device info for port %d, not detaching\n", 3466 port_id); 3467 return; 3468 } 3469 detach_device(dev_info.device); 3470 } 3471 3472 void 3473 detach_devargs(char *identifier) 3474 { 3475 struct rte_dev_iterator iterator; 3476 struct rte_devargs da; 3477 portid_t port_id; 3478 3479 printf("Removing a device...\n"); 3480 3481 memset(&da, 0, sizeof(da)); 3482 if (rte_devargs_parsef(&da, "%s", identifier)) { 3483 fprintf(stderr, "cannot parse identifier\n"); 3484 return; 3485 } 3486 3487 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3488 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3489 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3490 fprintf(stderr, "Port %u not stopped\n", 3491 port_id); 3492 rte_eth_iterator_cleanup(&iterator); 3493 rte_devargs_reset(&da); 3494 return; 3495 } 3496 flush_port_owned_resources(port_id); 3497 } 3498 } 3499 3500 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3501 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3502 da.name, rte_bus_name(da.bus)); 3503 rte_devargs_reset(&da); 3504 return; 3505 } 3506 3507 remove_invalid_ports(); 3508 3509 printf("Device %s is detached\n", identifier); 3510 printf("Now total ports is %d\n", nb_ports); 3511 printf("Done\n"); 3512 rte_devargs_reset(&da); 3513 } 3514 3515 void 3516 pmd_test_exit(void) 3517 { 3518 portid_t pt_id; 3519 unsigned int i; 3520 int ret; 3521 3522 if (test_done == 0) 3523 stop_packet_forwarding(); 3524 3525 #ifndef RTE_EXEC_ENV_WINDOWS 3526 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3527 if (mempools[i]) { 3528 if (mp_alloc_type == MP_ALLOC_ANON) 3529 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3530 NULL); 3531 } 3532 } 3533 #endif 3534 if (ports != NULL) { 3535 no_link_check = 1; 3536 RTE_ETH_FOREACH_DEV(pt_id) { 3537 printf("\nStopping port %d...\n", pt_id); 3538 fflush(stdout); 3539 stop_port(pt_id); 3540 } 3541 RTE_ETH_FOREACH_DEV(pt_id) { 3542 printf("\nShutting down port %d...\n", pt_id); 3543 fflush(stdout); 3544 close_port(pt_id); 3545 } 3546 } 3547 3548 if (hot_plug) { 3549 ret = rte_dev_event_monitor_stop(); 3550 if (ret) { 3551 RTE_LOG(ERR, EAL, 3552 "fail to stop device event monitor."); 3553 return; 3554 } 3555 3556 ret = rte_dev_event_callback_unregister(NULL, 3557 dev_event_callback, NULL); 3558 if (ret < 0) { 3559 RTE_LOG(ERR, EAL, 3560 "fail to unregister device event callback.\n"); 3561 return; 3562 } 3563 3564 ret = rte_dev_hotplug_handle_disable(); 3565 if (ret) { 3566 RTE_LOG(ERR, EAL, 3567 "fail to disable hotplug handling.\n"); 3568 return; 3569 } 3570 } 3571 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3572 if (mempools[i]) 3573 mempool_free_mp(mempools[i]); 3574 } 3575 free(xstats_display); 3576 3577 printf("\nBye...\n"); 3578 } 3579 3580 typedef void (*cmd_func_t)(void); 3581 struct pmd_test_command { 3582 const char *cmd_name; 3583 cmd_func_t cmd_func; 3584 }; 3585 3586 /* Check the link status of all ports in up to 9s, and print them finally */ 3587 static void 3588 check_all_ports_link_status(uint32_t port_mask) 3589 { 3590 #define CHECK_INTERVAL 100 /* 100ms */ 3591 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3592 portid_t portid; 3593 uint8_t count, all_ports_up, print_flag = 0; 3594 struct rte_eth_link link; 3595 int ret; 3596 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3597 3598 printf("Checking link statuses...\n"); 3599 fflush(stdout); 3600 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3601 all_ports_up = 1; 3602 RTE_ETH_FOREACH_DEV(portid) { 3603 if ((port_mask & (1 << portid)) == 0) 3604 continue; 3605 memset(&link, 0, sizeof(link)); 3606 ret = rte_eth_link_get_nowait(portid, &link); 3607 if (ret < 0) { 3608 all_ports_up = 0; 3609 if (print_flag == 1) 3610 fprintf(stderr, 3611 "Port %u link get failed: %s\n", 3612 portid, rte_strerror(-ret)); 3613 continue; 3614 } 3615 /* print link status if flag set */ 3616 if (print_flag == 1) { 3617 rte_eth_link_to_str(link_status, 3618 sizeof(link_status), &link); 3619 printf("Port %d %s\n", portid, link_status); 3620 continue; 3621 } 3622 /* clear all_ports_up flag if any link down */ 3623 if (link.link_status == RTE_ETH_LINK_DOWN) { 3624 all_ports_up = 0; 3625 break; 3626 } 3627 } 3628 /* after finally printing all link status, get out */ 3629 if (print_flag == 1) 3630 break; 3631 3632 if (all_ports_up == 0) { 3633 fflush(stdout); 3634 rte_delay_ms(CHECK_INTERVAL); 3635 } 3636 3637 /* set the print_flag if all ports up or timeout */ 3638 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3639 print_flag = 1; 3640 } 3641 3642 if (lsc_interrupt) 3643 break; 3644 } 3645 } 3646 3647 static void 3648 rmv_port_callback(void *arg) 3649 { 3650 int need_to_start = 0; 3651 int org_no_link_check = no_link_check; 3652 portid_t port_id = (intptr_t)arg; 3653 struct rte_eth_dev_info dev_info; 3654 int ret; 3655 3656 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3657 3658 if (!test_done && port_is_forwarding(port_id)) { 3659 need_to_start = 1; 3660 stop_packet_forwarding(); 3661 } 3662 no_link_check = 1; 3663 stop_port(port_id); 3664 no_link_check = org_no_link_check; 3665 3666 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3667 if (ret != 0) 3668 TESTPMD_LOG(ERR, 3669 "Failed to get device info for port %d, not detaching\n", 3670 port_id); 3671 else { 3672 struct rte_device *device = dev_info.device; 3673 close_port(port_id); 3674 detach_device(device); /* might be already removed or have more ports */ 3675 } 3676 if (need_to_start) 3677 start_packet_forwarding(0); 3678 } 3679 3680 /* This function is used by the interrupt thread */ 3681 static int 3682 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3683 void *ret_param) 3684 { 3685 RTE_SET_USED(param); 3686 RTE_SET_USED(ret_param); 3687 3688 if (type >= RTE_ETH_EVENT_MAX) { 3689 fprintf(stderr, 3690 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3691 port_id, __func__, type); 3692 fflush(stderr); 3693 } else if (event_print_mask & (UINT32_C(1) << type)) { 3694 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3695 eth_event_desc[type]); 3696 fflush(stdout); 3697 } 3698 3699 switch (type) { 3700 case RTE_ETH_EVENT_NEW: 3701 ports[port_id].need_setup = 1; 3702 ports[port_id].port_status = RTE_PORT_HANDLING; 3703 break; 3704 case RTE_ETH_EVENT_INTR_RMV: 3705 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3706 break; 3707 if (rte_eal_alarm_set(100000, 3708 rmv_port_callback, (void *)(intptr_t)port_id)) 3709 fprintf(stderr, 3710 "Could not set up deferred device removal\n"); 3711 break; 3712 case RTE_ETH_EVENT_DESTROY: 3713 ports[port_id].port_status = RTE_PORT_CLOSED; 3714 printf("Port %u is closed\n", port_id); 3715 break; 3716 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3717 uint16_t rxq_id; 3718 int ret; 3719 3720 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3721 for (rxq_id = 0; ; rxq_id++) { 3722 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3723 NULL); 3724 if (ret <= 0) 3725 break; 3726 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3727 port_id, rxq_id); 3728 3729 #ifdef RTE_NET_MLX5 3730 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3731 #endif 3732 } 3733 break; 3734 } 3735 default: 3736 break; 3737 } 3738 return 0; 3739 } 3740 3741 static int 3742 register_eth_event_callback(void) 3743 { 3744 int ret; 3745 enum rte_eth_event_type event; 3746 3747 for (event = RTE_ETH_EVENT_UNKNOWN; 3748 event < RTE_ETH_EVENT_MAX; event++) { 3749 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3750 event, 3751 eth_event_callback, 3752 NULL); 3753 if (ret != 0) { 3754 TESTPMD_LOG(ERR, "Failed to register callback for " 3755 "%s event\n", eth_event_desc[event]); 3756 return -1; 3757 } 3758 } 3759 3760 return 0; 3761 } 3762 3763 /* This function is used by the interrupt thread */ 3764 static void 3765 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3766 __rte_unused void *arg) 3767 { 3768 uint16_t port_id; 3769 int ret; 3770 3771 if (type >= RTE_DEV_EVENT_MAX) { 3772 fprintf(stderr, "%s called upon invalid event %d\n", 3773 __func__, type); 3774 fflush(stderr); 3775 } 3776 3777 switch (type) { 3778 case RTE_DEV_EVENT_REMOVE: 3779 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3780 device_name); 3781 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3782 if (ret) { 3783 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3784 device_name); 3785 return; 3786 } 3787 /* 3788 * Because the user's callback is invoked in eal interrupt 3789 * callback, the interrupt callback need to be finished before 3790 * it can be unregistered when detaching device. So finish 3791 * callback soon and use a deferred removal to detach device 3792 * is need. It is a workaround, once the device detaching be 3793 * moved into the eal in the future, the deferred removal could 3794 * be deleted. 3795 */ 3796 if (rte_eal_alarm_set(100000, 3797 rmv_port_callback, (void *)(intptr_t)port_id)) 3798 RTE_LOG(ERR, EAL, 3799 "Could not set up deferred device removal\n"); 3800 break; 3801 case RTE_DEV_EVENT_ADD: 3802 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3803 device_name); 3804 /* TODO: After finish kernel driver binding, 3805 * begin to attach port. 3806 */ 3807 break; 3808 default: 3809 break; 3810 } 3811 } 3812 3813 static void 3814 rxtx_port_config(portid_t pid) 3815 { 3816 uint16_t qid; 3817 uint64_t offloads; 3818 struct rte_port *port = &ports[pid]; 3819 3820 for (qid = 0; qid < nb_rxq; qid++) { 3821 offloads = port->rxq[qid].conf.offloads; 3822 port->rxq[qid].conf = port->dev_info.default_rxconf; 3823 3824 if (rxq_share > 0 && 3825 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3826 /* Non-zero share group to enable RxQ share. */ 3827 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3828 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3829 } 3830 3831 if (offloads != 0) 3832 port->rxq[qid].conf.offloads = offloads; 3833 3834 /* Check if any Rx parameters have been passed */ 3835 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3836 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3837 3838 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3839 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3840 3841 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3842 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3843 3844 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3845 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3846 3847 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3848 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3849 3850 port->nb_rx_desc[qid] = nb_rxd; 3851 } 3852 3853 for (qid = 0; qid < nb_txq; qid++) { 3854 offloads = port->txq[qid].conf.offloads; 3855 port->txq[qid].conf = port->dev_info.default_txconf; 3856 if (offloads != 0) 3857 port->txq[qid].conf.offloads = offloads; 3858 3859 /* Check if any Tx parameters have been passed */ 3860 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3861 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3862 3863 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3864 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3865 3866 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3867 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3868 3869 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3870 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3871 3872 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3873 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3874 3875 port->nb_tx_desc[qid] = nb_txd; 3876 } 3877 } 3878 3879 /* 3880 * Helper function to set MTU from frame size 3881 * 3882 * port->dev_info should be set before calling this function. 3883 * 3884 * return 0 on success, negative on error 3885 */ 3886 int 3887 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3888 { 3889 struct rte_port *port = &ports[portid]; 3890 uint32_t eth_overhead; 3891 uint16_t mtu, new_mtu; 3892 3893 eth_overhead = get_eth_overhead(&port->dev_info); 3894 3895 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3896 printf("Failed to get MTU for port %u\n", portid); 3897 return -1; 3898 } 3899 3900 new_mtu = max_rx_pktlen - eth_overhead; 3901 3902 if (mtu == new_mtu) 3903 return 0; 3904 3905 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3906 fprintf(stderr, 3907 "Failed to set MTU to %u for port %u\n", 3908 new_mtu, portid); 3909 return -1; 3910 } 3911 3912 port->dev_conf.rxmode.mtu = new_mtu; 3913 3914 return 0; 3915 } 3916 3917 void 3918 init_port_config(void) 3919 { 3920 portid_t pid; 3921 struct rte_port *port; 3922 int ret, i; 3923 3924 RTE_ETH_FOREACH_DEV(pid) { 3925 port = &ports[pid]; 3926 port->dev_conf.fdir_conf = fdir_conf; 3927 3928 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3929 if (ret != 0) 3930 return; 3931 3932 if (nb_rxq > 1) { 3933 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3934 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3935 rss_hf & port->dev_info.flow_type_rss_offloads; 3936 } else { 3937 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3938 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3939 } 3940 3941 if (port->dcb_flag == 0) { 3942 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3943 port->dev_conf.rxmode.mq_mode = 3944 (enum rte_eth_rx_mq_mode) 3945 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3946 } else { 3947 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3948 port->dev_conf.rxmode.offloads &= 3949 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3950 3951 for (i = 0; 3952 i < port->dev_info.nb_rx_queues; 3953 i++) 3954 port->rxq[i].conf.offloads &= 3955 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3956 } 3957 } 3958 3959 rxtx_port_config(pid); 3960 3961 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3962 if (ret != 0) 3963 return; 3964 3965 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3966 rte_pmd_ixgbe_bypass_init(pid); 3967 #endif 3968 3969 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3970 port->dev_conf.intr_conf.lsc = 1; 3971 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3972 port->dev_conf.intr_conf.rmv = 1; 3973 } 3974 } 3975 3976 void set_port_slave_flag(portid_t slave_pid) 3977 { 3978 struct rte_port *port; 3979 3980 port = &ports[slave_pid]; 3981 port->slave_flag = 1; 3982 } 3983 3984 void clear_port_slave_flag(portid_t slave_pid) 3985 { 3986 struct rte_port *port; 3987 3988 port = &ports[slave_pid]; 3989 port->slave_flag = 0; 3990 } 3991 3992 uint8_t port_is_bonding_slave(portid_t slave_pid) 3993 { 3994 struct rte_port *port; 3995 struct rte_eth_dev_info dev_info; 3996 int ret; 3997 3998 port = &ports[slave_pid]; 3999 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4000 if (ret != 0) { 4001 TESTPMD_LOG(ERR, 4002 "Failed to get device info for port id %d," 4003 "cannot determine if the port is a bonded slave", 4004 slave_pid); 4005 return 0; 4006 } 4007 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4008 return 1; 4009 return 0; 4010 } 4011 4012 const uint16_t vlan_tags[] = { 4013 0, 1, 2, 3, 4, 5, 6, 7, 4014 8, 9, 10, 11, 12, 13, 14, 15, 4015 16, 17, 18, 19, 20, 21, 22, 23, 4016 24, 25, 26, 27, 28, 29, 30, 31 4017 }; 4018 4019 static int 4020 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4021 enum dcb_mode_enable dcb_mode, 4022 enum rte_eth_nb_tcs num_tcs, 4023 uint8_t pfc_en) 4024 { 4025 uint8_t i; 4026 int32_t rc; 4027 struct rte_eth_rss_conf rss_conf; 4028 4029 /* 4030 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4031 * given above, and the number of traffic classes available for use. 4032 */ 4033 if (dcb_mode == DCB_VT_ENABLED) { 4034 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4035 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4036 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4037 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4038 4039 /* VMDQ+DCB RX and TX configurations */ 4040 vmdq_rx_conf->enable_default_pool = 0; 4041 vmdq_rx_conf->default_pool = 0; 4042 vmdq_rx_conf->nb_queue_pools = 4043 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4044 vmdq_tx_conf->nb_queue_pools = 4045 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4046 4047 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4048 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4049 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4050 vmdq_rx_conf->pool_map[i].pools = 4051 1 << (i % vmdq_rx_conf->nb_queue_pools); 4052 } 4053 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4054 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4055 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4056 } 4057 4058 /* set DCB mode of RX and TX of multiple queues */ 4059 eth_conf->rxmode.mq_mode = 4060 (enum rte_eth_rx_mq_mode) 4061 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4062 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4063 } else { 4064 struct rte_eth_dcb_rx_conf *rx_conf = 4065 ð_conf->rx_adv_conf.dcb_rx_conf; 4066 struct rte_eth_dcb_tx_conf *tx_conf = 4067 ð_conf->tx_adv_conf.dcb_tx_conf; 4068 4069 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4070 4071 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4072 if (rc != 0) 4073 return rc; 4074 4075 rx_conf->nb_tcs = num_tcs; 4076 tx_conf->nb_tcs = num_tcs; 4077 4078 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4079 rx_conf->dcb_tc[i] = i % num_tcs; 4080 tx_conf->dcb_tc[i] = i % num_tcs; 4081 } 4082 4083 eth_conf->rxmode.mq_mode = 4084 (enum rte_eth_rx_mq_mode) 4085 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4086 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4087 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4088 } 4089 4090 if (pfc_en) 4091 eth_conf->dcb_capability_en = 4092 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4093 else 4094 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4095 4096 return 0; 4097 } 4098 4099 int 4100 init_port_dcb_config(portid_t pid, 4101 enum dcb_mode_enable dcb_mode, 4102 enum rte_eth_nb_tcs num_tcs, 4103 uint8_t pfc_en) 4104 { 4105 struct rte_eth_conf port_conf; 4106 struct rte_port *rte_port; 4107 int retval; 4108 uint16_t i; 4109 4110 if (num_procs > 1) { 4111 printf("The multi-process feature doesn't support dcb.\n"); 4112 return -ENOTSUP; 4113 } 4114 rte_port = &ports[pid]; 4115 4116 /* retain the original device configuration. */ 4117 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4118 4119 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4120 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4121 if (retval < 0) 4122 return retval; 4123 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4124 /* remove RSS HASH offload for DCB in vt mode */ 4125 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4126 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4127 for (i = 0; i < nb_rxq; i++) 4128 rte_port->rxq[i].conf.offloads &= 4129 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4130 } 4131 4132 /* re-configure the device . */ 4133 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4134 if (retval < 0) 4135 return retval; 4136 4137 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4138 if (retval != 0) 4139 return retval; 4140 4141 /* If dev_info.vmdq_pool_base is greater than 0, 4142 * the queue id of vmdq pools is started after pf queues. 4143 */ 4144 if (dcb_mode == DCB_VT_ENABLED && 4145 rte_port->dev_info.vmdq_pool_base > 0) { 4146 fprintf(stderr, 4147 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4148 pid); 4149 return -1; 4150 } 4151 4152 /* Assume the ports in testpmd have the same dcb capability 4153 * and has the same number of rxq and txq in dcb mode 4154 */ 4155 if (dcb_mode == DCB_VT_ENABLED) { 4156 if (rte_port->dev_info.max_vfs > 0) { 4157 nb_rxq = rte_port->dev_info.nb_rx_queues; 4158 nb_txq = rte_port->dev_info.nb_tx_queues; 4159 } else { 4160 nb_rxq = rte_port->dev_info.max_rx_queues; 4161 nb_txq = rte_port->dev_info.max_tx_queues; 4162 } 4163 } else { 4164 /*if vt is disabled, use all pf queues */ 4165 if (rte_port->dev_info.vmdq_pool_base == 0) { 4166 nb_rxq = rte_port->dev_info.max_rx_queues; 4167 nb_txq = rte_port->dev_info.max_tx_queues; 4168 } else { 4169 nb_rxq = (queueid_t)num_tcs; 4170 nb_txq = (queueid_t)num_tcs; 4171 4172 } 4173 } 4174 rx_free_thresh = 64; 4175 4176 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4177 4178 rxtx_port_config(pid); 4179 /* VLAN filter */ 4180 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4181 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4182 rx_vft_set(pid, vlan_tags[i], 1); 4183 4184 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4185 if (retval != 0) 4186 return retval; 4187 4188 rte_port->dcb_flag = 1; 4189 4190 /* Enter DCB configuration status */ 4191 dcb_config = 1; 4192 4193 return 0; 4194 } 4195 4196 static void 4197 init_port(void) 4198 { 4199 int i; 4200 4201 /* Configuration of Ethernet ports. */ 4202 ports = rte_zmalloc("testpmd: ports", 4203 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4204 RTE_CACHE_LINE_SIZE); 4205 if (ports == NULL) { 4206 rte_exit(EXIT_FAILURE, 4207 "rte_zmalloc(%d struct rte_port) failed\n", 4208 RTE_MAX_ETHPORTS); 4209 } 4210 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4211 ports[i].xstats_info.allocated = false; 4212 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4213 LIST_INIT(&ports[i].flow_tunnel_list); 4214 /* Initialize ports NUMA structures */ 4215 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4216 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4217 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4218 } 4219 4220 static void 4221 force_quit(void) 4222 { 4223 pmd_test_exit(); 4224 prompt_exit(); 4225 } 4226 4227 static void 4228 print_stats(void) 4229 { 4230 uint8_t i; 4231 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4232 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4233 4234 /* Clear screen and move to top left */ 4235 printf("%s%s", clr, top_left); 4236 4237 printf("\nPort statistics ===================================="); 4238 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4239 nic_stats_display(fwd_ports_ids[i]); 4240 4241 fflush(stdout); 4242 } 4243 4244 static void 4245 signal_handler(int signum) 4246 { 4247 if (signum == SIGINT || signum == SIGTERM) { 4248 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4249 signum); 4250 #ifdef RTE_LIB_PDUMP 4251 /* uninitialize packet capture framework */ 4252 rte_pdump_uninit(); 4253 #endif 4254 #ifdef RTE_LIB_LATENCYSTATS 4255 if (latencystats_enabled != 0) 4256 rte_latencystats_uninit(); 4257 #endif 4258 force_quit(); 4259 /* Set flag to indicate the force termination. */ 4260 f_quit = 1; 4261 /* exit with the expected status */ 4262 #ifndef RTE_EXEC_ENV_WINDOWS 4263 signal(signum, SIG_DFL); 4264 kill(getpid(), signum); 4265 #endif 4266 } 4267 } 4268 4269 int 4270 main(int argc, char** argv) 4271 { 4272 int diag; 4273 portid_t port_id; 4274 uint16_t count; 4275 int ret; 4276 4277 signal(SIGINT, signal_handler); 4278 signal(SIGTERM, signal_handler); 4279 4280 testpmd_logtype = rte_log_register("testpmd"); 4281 if (testpmd_logtype < 0) 4282 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4283 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4284 4285 diag = rte_eal_init(argc, argv); 4286 if (diag < 0) 4287 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4288 rte_strerror(rte_errno)); 4289 4290 ret = register_eth_event_callback(); 4291 if (ret != 0) 4292 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4293 4294 #ifdef RTE_LIB_PDUMP 4295 /* initialize packet capture framework */ 4296 rte_pdump_init(); 4297 #endif 4298 4299 count = 0; 4300 RTE_ETH_FOREACH_DEV(port_id) { 4301 ports_ids[count] = port_id; 4302 count++; 4303 } 4304 nb_ports = (portid_t) count; 4305 if (nb_ports == 0) 4306 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4307 4308 /* allocate port structures, and init them */ 4309 init_port(); 4310 4311 set_def_fwd_config(); 4312 if (nb_lcores == 0) 4313 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4314 "Check the core mask argument\n"); 4315 4316 /* Bitrate/latency stats disabled by default */ 4317 #ifdef RTE_LIB_BITRATESTATS 4318 bitrate_enabled = 0; 4319 #endif 4320 #ifdef RTE_LIB_LATENCYSTATS 4321 latencystats_enabled = 0; 4322 #endif 4323 4324 /* on FreeBSD, mlockall() is disabled by default */ 4325 #ifdef RTE_EXEC_ENV_FREEBSD 4326 do_mlockall = 0; 4327 #else 4328 do_mlockall = 1; 4329 #endif 4330 4331 argc -= diag; 4332 argv += diag; 4333 if (argc > 1) 4334 launch_args_parse(argc, argv); 4335 4336 #ifndef RTE_EXEC_ENV_WINDOWS 4337 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4338 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4339 strerror(errno)); 4340 } 4341 #endif 4342 4343 if (tx_first && interactive) 4344 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4345 "interactive mode.\n"); 4346 4347 if (tx_first && lsc_interrupt) { 4348 fprintf(stderr, 4349 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4350 lsc_interrupt = 0; 4351 } 4352 4353 if (!nb_rxq && !nb_txq) 4354 fprintf(stderr, 4355 "Warning: Either rx or tx queues should be non-zero\n"); 4356 4357 if (nb_rxq > 1 && nb_rxq > nb_txq) 4358 fprintf(stderr, 4359 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4360 nb_rxq, nb_txq); 4361 4362 init_config(); 4363 4364 if (hot_plug) { 4365 ret = rte_dev_hotplug_handle_enable(); 4366 if (ret) { 4367 RTE_LOG(ERR, EAL, 4368 "fail to enable hotplug handling."); 4369 return -1; 4370 } 4371 4372 ret = rte_dev_event_monitor_start(); 4373 if (ret) { 4374 RTE_LOG(ERR, EAL, 4375 "fail to start device event monitoring."); 4376 return -1; 4377 } 4378 4379 ret = rte_dev_event_callback_register(NULL, 4380 dev_event_callback, NULL); 4381 if (ret) { 4382 RTE_LOG(ERR, EAL, 4383 "fail to register device event callback\n"); 4384 return -1; 4385 } 4386 } 4387 4388 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4389 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4390 4391 /* set all ports to promiscuous mode by default */ 4392 RTE_ETH_FOREACH_DEV(port_id) { 4393 ret = rte_eth_promiscuous_enable(port_id); 4394 if (ret != 0) 4395 fprintf(stderr, 4396 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4397 port_id, rte_strerror(-ret)); 4398 } 4399 4400 #ifdef RTE_LIB_METRICS 4401 /* Init metrics library */ 4402 rte_metrics_init(rte_socket_id()); 4403 #endif 4404 4405 #ifdef RTE_LIB_LATENCYSTATS 4406 if (latencystats_enabled != 0) { 4407 int ret = rte_latencystats_init(1, NULL); 4408 if (ret) 4409 fprintf(stderr, 4410 "Warning: latencystats init() returned error %d\n", 4411 ret); 4412 fprintf(stderr, "Latencystats running on lcore %d\n", 4413 latencystats_lcore_id); 4414 } 4415 #endif 4416 4417 /* Setup bitrate stats */ 4418 #ifdef RTE_LIB_BITRATESTATS 4419 if (bitrate_enabled != 0) { 4420 bitrate_data = rte_stats_bitrate_create(); 4421 if (bitrate_data == NULL) 4422 rte_exit(EXIT_FAILURE, 4423 "Could not allocate bitrate data.\n"); 4424 rte_stats_bitrate_reg(bitrate_data); 4425 } 4426 #endif 4427 #ifdef RTE_LIB_CMDLINE 4428 if (init_cmdline() != 0) 4429 rte_exit(EXIT_FAILURE, 4430 "Could not initialise cmdline context.\n"); 4431 4432 if (strlen(cmdline_filename) != 0) 4433 cmdline_read_from_file(cmdline_filename); 4434 4435 if (interactive == 1) { 4436 if (auto_start) { 4437 printf("Start automatic packet forwarding\n"); 4438 start_packet_forwarding(0); 4439 } 4440 prompt(); 4441 pmd_test_exit(); 4442 } else 4443 #endif 4444 { 4445 char c; 4446 int rc; 4447 4448 f_quit = 0; 4449 4450 printf("No commandline core given, start packet forwarding\n"); 4451 start_packet_forwarding(tx_first); 4452 if (stats_period != 0) { 4453 uint64_t prev_time = 0, cur_time, diff_time = 0; 4454 uint64_t timer_period; 4455 4456 /* Convert to number of cycles */ 4457 timer_period = stats_period * rte_get_timer_hz(); 4458 4459 while (f_quit == 0) { 4460 cur_time = rte_get_timer_cycles(); 4461 diff_time += cur_time - prev_time; 4462 4463 if (diff_time >= timer_period) { 4464 print_stats(); 4465 /* Reset the timer */ 4466 diff_time = 0; 4467 } 4468 /* Sleep to avoid unnecessary checks */ 4469 prev_time = cur_time; 4470 rte_delay_us_sleep(US_PER_S); 4471 } 4472 } 4473 4474 printf("Press enter to exit\n"); 4475 rc = read(0, &c, 1); 4476 pmd_test_exit(); 4477 if (rc < 0) 4478 return 1; 4479 } 4480 4481 ret = rte_eal_cleanup(); 4482 if (ret != 0) 4483 rte_exit(EXIT_FAILURE, 4484 "EAL cleanup failed: %s\n", strerror(-ret)); 4485 4486 return EXIT_SUCCESS; 4487 } 4488