1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 /* 253 * Configuration of packet segments used by the "txonly" processing engine. 254 */ 255 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 256 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 257 TXONLY_DEF_PACKET_LEN, 258 }; 259 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 260 261 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 262 /**< Split policy for packets to TX. */ 263 264 uint8_t txonly_multi_flow; 265 /**< Whether multiple flows are generated in TXONLY mode. */ 266 267 uint32_t tx_pkt_times_inter; 268 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 269 270 uint32_t tx_pkt_times_intra; 271 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 272 273 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 274 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 275 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 276 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 277 278 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 279 uint8_t dcb_config = 0; 280 281 /* 282 * Configurable number of RX/TX queues. 283 */ 284 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 285 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 286 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 287 288 /* 289 * Configurable number of RX/TX ring descriptors. 290 * Defaults are supplied by drivers via ethdev. 291 */ 292 #define RX_DESC_DEFAULT 0 293 #define TX_DESC_DEFAULT 0 294 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 295 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 296 297 #define RTE_PMD_PARAM_UNSET -1 298 /* 299 * Configurable values of RX and TX ring threshold registers. 300 */ 301 302 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 303 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 304 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 305 306 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 307 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 308 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 309 310 /* 311 * Configurable value of RX free threshold. 312 */ 313 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 314 315 /* 316 * Configurable value of RX drop enable. 317 */ 318 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 319 320 /* 321 * Configurable value of TX free threshold. 322 */ 323 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 324 325 /* 326 * Configurable value of TX RS bit threshold. 327 */ 328 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 329 330 /* 331 * Configurable value of buffered packets before sending. 332 */ 333 uint16_t noisy_tx_sw_bufsz; 334 335 /* 336 * Configurable value of packet buffer timeout. 337 */ 338 uint16_t noisy_tx_sw_buf_flush_time; 339 340 /* 341 * Configurable value for size of VNF internal memory area 342 * used for simulating noisy neighbour behaviour 343 */ 344 uint64_t noisy_lkup_mem_sz; 345 346 /* 347 * Configurable value of number of random writes done in 348 * VNF simulation memory area. 349 */ 350 uint64_t noisy_lkup_num_writes; 351 352 /* 353 * Configurable value of number of random reads done in 354 * VNF simulation memory area. 355 */ 356 uint64_t noisy_lkup_num_reads; 357 358 /* 359 * Configurable value of number of random reads/writes done in 360 * VNF simulation memory area. 361 */ 362 uint64_t noisy_lkup_num_reads_writes; 363 364 /* 365 * Receive Side Scaling (RSS) configuration. 366 */ 367 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 368 369 /* 370 * Port topology configuration 371 */ 372 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 373 374 /* 375 * Avoids to flush all the RX streams before starts forwarding. 376 */ 377 uint8_t no_flush_rx = 0; /* flush by default */ 378 379 /* 380 * Flow API isolated mode. 381 */ 382 uint8_t flow_isolate_all; 383 384 /* 385 * Avoids to check link status when starting/stopping a port. 386 */ 387 uint8_t no_link_check = 0; /* check by default */ 388 389 /* 390 * Don't automatically start all ports in interactive mode. 391 */ 392 uint8_t no_device_start = 0; 393 394 /* 395 * Enable link status change notification 396 */ 397 uint8_t lsc_interrupt = 1; /* enabled by default */ 398 399 /* 400 * Enable device removal notification. 401 */ 402 uint8_t rmv_interrupt = 1; /* enabled by default */ 403 404 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 405 406 /* After attach, port setup is called on event or by iterator */ 407 bool setup_on_probe_event = true; 408 409 /* Clear ptypes on port initialization. */ 410 uint8_t clear_ptypes = true; 411 412 /* Hairpin ports configuration mode. */ 413 uint32_t hairpin_mode; 414 415 /* Pretty printing of ethdev events */ 416 static const char * const eth_event_desc[] = { 417 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 418 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 419 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 420 [RTE_ETH_EVENT_INTR_RESET] = "reset", 421 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 422 [RTE_ETH_EVENT_IPSEC] = "IPsec", 423 [RTE_ETH_EVENT_MACSEC] = "MACsec", 424 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 425 [RTE_ETH_EVENT_NEW] = "device probed", 426 [RTE_ETH_EVENT_DESTROY] = "device released", 427 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 428 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 429 [RTE_ETH_EVENT_ERR_RECOVERING] = "error recovering", 430 [RTE_ETH_EVENT_RECOVERY_SUCCESS] = "error recovery successful", 431 [RTE_ETH_EVENT_RECOVERY_FAILED] = "error recovery failed", 432 [RTE_ETH_EVENT_MAX] = NULL, 433 }; 434 435 /* 436 * Display or mask ether events 437 * Default to all events except VF_MBOX 438 */ 439 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 440 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 441 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 442 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 443 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 444 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 445 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 446 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED) | 447 (UINT32_C(1) << RTE_ETH_EVENT_ERR_RECOVERING) | 448 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_SUCCESS) | 449 (UINT32_C(1) << RTE_ETH_EVENT_RECOVERY_FAILED); 450 /* 451 * Decide if all memory are locked for performance. 452 */ 453 int do_mlockall = 0; 454 455 #ifdef RTE_LIB_LATENCYSTATS 456 457 /* 458 * Set when latency stats is enabled in the commandline 459 */ 460 uint8_t latencystats_enabled; 461 462 /* 463 * Lcore ID to service latency statistics. 464 */ 465 lcoreid_t latencystats_lcore_id = -1; 466 467 #endif 468 469 /* 470 * Ethernet device configuration. 471 */ 472 struct rte_eth_rxmode rx_mode; 473 474 struct rte_eth_txmode tx_mode = { 475 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 476 }; 477 478 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 479 480 /* 481 * Display zero values by default for xstats 482 */ 483 uint8_t xstats_hide_zero; 484 485 /* 486 * Measure of CPU cycles disabled by default 487 */ 488 uint8_t record_core_cycles; 489 490 /* 491 * Display of RX and TX bursts disabled by default 492 */ 493 uint8_t record_burst_stats; 494 495 /* 496 * Number of ports per shared Rx queue group, 0 disable. 497 */ 498 uint32_t rxq_share; 499 500 unsigned int num_sockets = 0; 501 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 502 503 #ifdef RTE_LIB_BITRATESTATS 504 /* Bitrate statistics */ 505 struct rte_stats_bitrates *bitrate_data; 506 lcoreid_t bitrate_lcore_id; 507 uint8_t bitrate_enabled; 508 #endif 509 510 #ifdef RTE_LIB_GRO 511 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 512 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 513 #endif 514 515 /* 516 * hexadecimal bitmask of RX mq mode can be enabled. 517 */ 518 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 519 520 /* 521 * Used to set forced link speed 522 */ 523 uint32_t eth_link_speed; 524 525 /* 526 * ID of the current process in multi-process, used to 527 * configure the queues to be polled. 528 */ 529 int proc_id; 530 531 /* 532 * Number of processes in multi-process, used to 533 * configure the queues to be polled. 534 */ 535 unsigned int num_procs = 1; 536 537 static void 538 eth_rx_metadata_negotiate_mp(uint16_t port_id) 539 { 540 uint64_t rx_meta_features = 0; 541 int ret; 542 543 if (!is_proc_primary()) 544 return; 545 546 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 547 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 548 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 549 550 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 551 if (ret == 0) { 552 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 553 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 554 port_id); 555 } 556 557 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 558 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 559 port_id); 560 } 561 562 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 563 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 564 port_id); 565 } 566 } else if (ret != -ENOTSUP) { 567 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 568 port_id, rte_strerror(-ret)); 569 } 570 } 571 572 static int 573 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 574 const struct rte_eth_conf *dev_conf) 575 { 576 if (is_proc_primary()) 577 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 578 dev_conf); 579 return 0; 580 } 581 582 static int 583 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 584 { 585 #ifdef RTE_NET_BOND 586 587 portid_t slave_pids[RTE_MAX_ETHPORTS]; 588 struct rte_port *port; 589 int num_slaves; 590 portid_t slave_pid; 591 int i; 592 593 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 594 RTE_MAX_ETHPORTS); 595 if (num_slaves < 0) { 596 fprintf(stderr, "Failed to get slave list for port = %u\n", 597 bond_pid); 598 return num_slaves; 599 } 600 601 for (i = 0; i < num_slaves; i++) { 602 slave_pid = slave_pids[i]; 603 port = &ports[slave_pid]; 604 port->port_status = 605 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 606 } 607 #else 608 RTE_SET_USED(bond_pid); 609 RTE_SET_USED(is_stop); 610 #endif 611 return 0; 612 } 613 614 static int 615 eth_dev_start_mp(uint16_t port_id) 616 { 617 int ret; 618 619 if (is_proc_primary()) { 620 ret = rte_eth_dev_start(port_id); 621 if (ret != 0) 622 return ret; 623 624 struct rte_port *port = &ports[port_id]; 625 626 /* 627 * Starting a bonded port also starts all slaves under the bonded 628 * device. So if this port is bond device, we need to modify the 629 * port status of these slaves. 630 */ 631 if (port->bond_flag == 1) 632 return change_bonding_slave_port_status(port_id, false); 633 } 634 635 return 0; 636 } 637 638 static int 639 eth_dev_stop_mp(uint16_t port_id) 640 { 641 int ret; 642 643 if (is_proc_primary()) { 644 ret = rte_eth_dev_stop(port_id); 645 if (ret != 0) 646 return ret; 647 648 struct rte_port *port = &ports[port_id]; 649 650 /* 651 * Stopping a bonded port also stops all slaves under the bonded 652 * device. So if this port is bond device, we need to modify the 653 * port status of these slaves. 654 */ 655 if (port->bond_flag == 1) 656 return change_bonding_slave_port_status(port_id, true); 657 } 658 659 return 0; 660 } 661 662 static void 663 mempool_free_mp(struct rte_mempool *mp) 664 { 665 if (is_proc_primary()) 666 rte_mempool_free(mp); 667 } 668 669 static int 670 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 671 { 672 if (is_proc_primary()) 673 return rte_eth_dev_set_mtu(port_id, mtu); 674 675 return 0; 676 } 677 678 /* Forward function declarations */ 679 static void setup_attached_port(portid_t pi); 680 static void check_all_ports_link_status(uint32_t port_mask); 681 static int eth_event_callback(portid_t port_id, 682 enum rte_eth_event_type type, 683 void *param, void *ret_param); 684 static void dev_event_callback(const char *device_name, 685 enum rte_dev_event_type type, 686 void *param); 687 static void fill_xstats_display_info(void); 688 689 /* 690 * Check if all the ports are started. 691 * If yes, return positive value. If not, return zero. 692 */ 693 static int all_ports_started(void); 694 695 #ifdef RTE_LIB_GSO 696 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 697 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 698 #endif 699 700 /* Holds the registered mbuf dynamic flags names. */ 701 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 702 703 704 /* 705 * Helper function to check if socket is already discovered. 706 * If yes, return positive value. If not, return zero. 707 */ 708 int 709 new_socket_id(unsigned int socket_id) 710 { 711 unsigned int i; 712 713 for (i = 0; i < num_sockets; i++) { 714 if (socket_ids[i] == socket_id) 715 return 0; 716 } 717 return 1; 718 } 719 720 /* 721 * Setup default configuration. 722 */ 723 static void 724 set_default_fwd_lcores_config(void) 725 { 726 unsigned int i; 727 unsigned int nb_lc; 728 unsigned int sock_num; 729 730 nb_lc = 0; 731 for (i = 0; i < RTE_MAX_LCORE; i++) { 732 if (!rte_lcore_is_enabled(i)) 733 continue; 734 sock_num = rte_lcore_to_socket_id(i); 735 if (new_socket_id(sock_num)) { 736 if (num_sockets >= RTE_MAX_NUMA_NODES) { 737 rte_exit(EXIT_FAILURE, 738 "Total sockets greater than %u\n", 739 RTE_MAX_NUMA_NODES); 740 } 741 socket_ids[num_sockets++] = sock_num; 742 } 743 if (i == rte_get_main_lcore()) 744 continue; 745 fwd_lcores_cpuids[nb_lc++] = i; 746 } 747 nb_lcores = (lcoreid_t) nb_lc; 748 nb_cfg_lcores = nb_lcores; 749 nb_fwd_lcores = 1; 750 } 751 752 static void 753 set_def_peer_eth_addrs(void) 754 { 755 portid_t i; 756 757 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 758 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 759 peer_eth_addrs[i].addr_bytes[5] = i; 760 } 761 } 762 763 static void 764 set_default_fwd_ports_config(void) 765 { 766 portid_t pt_id; 767 int i = 0; 768 769 RTE_ETH_FOREACH_DEV(pt_id) { 770 fwd_ports_ids[i++] = pt_id; 771 772 /* Update sockets info according to the attached device */ 773 int socket_id = rte_eth_dev_socket_id(pt_id); 774 if (socket_id >= 0 && new_socket_id(socket_id)) { 775 if (num_sockets >= RTE_MAX_NUMA_NODES) { 776 rte_exit(EXIT_FAILURE, 777 "Total sockets greater than %u\n", 778 RTE_MAX_NUMA_NODES); 779 } 780 socket_ids[num_sockets++] = socket_id; 781 } 782 } 783 784 nb_cfg_ports = nb_ports; 785 nb_fwd_ports = nb_ports; 786 } 787 788 void 789 set_def_fwd_config(void) 790 { 791 set_default_fwd_lcores_config(); 792 set_def_peer_eth_addrs(); 793 set_default_fwd_ports_config(); 794 } 795 796 #ifndef RTE_EXEC_ENV_WINDOWS 797 /* extremely pessimistic estimation of memory required to create a mempool */ 798 static int 799 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 800 { 801 unsigned int n_pages, mbuf_per_pg, leftover; 802 uint64_t total_mem, mbuf_mem, obj_sz; 803 804 /* there is no good way to predict how much space the mempool will 805 * occupy because it will allocate chunks on the fly, and some of those 806 * will come from default DPDK memory while some will come from our 807 * external memory, so just assume 128MB will be enough for everyone. 808 */ 809 uint64_t hdr_mem = 128 << 20; 810 811 /* account for possible non-contiguousness */ 812 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 813 if (obj_sz > pgsz) { 814 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 815 return -1; 816 } 817 818 mbuf_per_pg = pgsz / obj_sz; 819 leftover = (nb_mbufs % mbuf_per_pg) > 0; 820 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 821 822 mbuf_mem = n_pages * pgsz; 823 824 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 825 826 if (total_mem > SIZE_MAX) { 827 TESTPMD_LOG(ERR, "Memory size too big\n"); 828 return -1; 829 } 830 *out = (size_t)total_mem; 831 832 return 0; 833 } 834 835 static int 836 pagesz_flags(uint64_t page_sz) 837 { 838 /* as per mmap() manpage, all page sizes are log2 of page size 839 * shifted by MAP_HUGE_SHIFT 840 */ 841 int log2 = rte_log2_u64(page_sz); 842 843 return (log2 << HUGE_SHIFT); 844 } 845 846 static void * 847 alloc_mem(size_t memsz, size_t pgsz, bool huge) 848 { 849 void *addr; 850 int flags; 851 852 /* allocate anonymous hugepages */ 853 flags = MAP_ANONYMOUS | MAP_PRIVATE; 854 if (huge) 855 flags |= HUGE_FLAG | pagesz_flags(pgsz); 856 857 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 858 if (addr == MAP_FAILED) 859 return NULL; 860 861 return addr; 862 } 863 864 struct extmem_param { 865 void *addr; 866 size_t len; 867 size_t pgsz; 868 rte_iova_t *iova_table; 869 unsigned int iova_table_len; 870 }; 871 872 static int 873 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 874 bool huge) 875 { 876 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 877 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 878 unsigned int cur_page, n_pages, pgsz_idx; 879 size_t mem_sz, cur_pgsz; 880 rte_iova_t *iovas = NULL; 881 void *addr; 882 int ret; 883 884 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 885 /* skip anything that is too big */ 886 if (pgsizes[pgsz_idx] > SIZE_MAX) 887 continue; 888 889 cur_pgsz = pgsizes[pgsz_idx]; 890 891 /* if we were told not to allocate hugepages, override */ 892 if (!huge) 893 cur_pgsz = sysconf(_SC_PAGESIZE); 894 895 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 896 if (ret < 0) { 897 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 898 return -1; 899 } 900 901 /* allocate our memory */ 902 addr = alloc_mem(mem_sz, cur_pgsz, huge); 903 904 /* if we couldn't allocate memory with a specified page size, 905 * that doesn't mean we can't do it with other page sizes, so 906 * try another one. 907 */ 908 if (addr == NULL) 909 continue; 910 911 /* store IOVA addresses for every page in this memory area */ 912 n_pages = mem_sz / cur_pgsz; 913 914 iovas = malloc(sizeof(*iovas) * n_pages); 915 916 if (iovas == NULL) { 917 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 918 goto fail; 919 } 920 /* lock memory if it's not huge pages */ 921 if (!huge) 922 mlock(addr, mem_sz); 923 924 /* populate IOVA addresses */ 925 for (cur_page = 0; cur_page < n_pages; cur_page++) { 926 rte_iova_t iova; 927 size_t offset; 928 void *cur; 929 930 offset = cur_pgsz * cur_page; 931 cur = RTE_PTR_ADD(addr, offset); 932 933 /* touch the page before getting its IOVA */ 934 *(volatile char *)cur = 0; 935 936 iova = rte_mem_virt2iova(cur); 937 938 iovas[cur_page] = iova; 939 } 940 941 break; 942 } 943 /* if we couldn't allocate anything */ 944 if (iovas == NULL) 945 return -1; 946 947 param->addr = addr; 948 param->len = mem_sz; 949 param->pgsz = cur_pgsz; 950 param->iova_table = iovas; 951 param->iova_table_len = n_pages; 952 953 return 0; 954 fail: 955 free(iovas); 956 if (addr) 957 munmap(addr, mem_sz); 958 959 return -1; 960 } 961 962 static int 963 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 964 { 965 struct extmem_param param; 966 int socket_id, ret; 967 968 memset(¶m, 0, sizeof(param)); 969 970 /* check if our heap exists */ 971 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 972 if (socket_id < 0) { 973 /* create our heap */ 974 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 975 if (ret < 0) { 976 TESTPMD_LOG(ERR, "Cannot create heap\n"); 977 return -1; 978 } 979 } 980 981 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 982 if (ret < 0) { 983 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 984 return -1; 985 } 986 987 /* we now have a valid memory area, so add it to heap */ 988 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 989 param.addr, param.len, param.iova_table, 990 param.iova_table_len, param.pgsz); 991 992 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 993 994 /* not needed any more */ 995 free(param.iova_table); 996 997 if (ret < 0) { 998 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 999 munmap(param.addr, param.len); 1000 return -1; 1001 } 1002 1003 /* success */ 1004 1005 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1006 param.len >> 20); 1007 1008 return 0; 1009 } 1010 static void 1011 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1012 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1013 { 1014 uint16_t pid = 0; 1015 int ret; 1016 1017 RTE_ETH_FOREACH_DEV(pid) { 1018 struct rte_eth_dev_info dev_info; 1019 1020 ret = eth_dev_info_get_print_err(pid, &dev_info); 1021 if (ret != 0) { 1022 TESTPMD_LOG(DEBUG, 1023 "unable to get device info for port %d on addr 0x%p," 1024 "mempool unmapping will not be performed\n", 1025 pid, memhdr->addr); 1026 continue; 1027 } 1028 1029 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1030 if (ret) { 1031 TESTPMD_LOG(DEBUG, 1032 "unable to DMA unmap addr 0x%p " 1033 "for device %s\n", 1034 memhdr->addr, rte_dev_name(dev_info.device)); 1035 } 1036 } 1037 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1038 if (ret) { 1039 TESTPMD_LOG(DEBUG, 1040 "unable to un-register addr 0x%p\n", memhdr->addr); 1041 } 1042 } 1043 1044 static void 1045 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1046 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1047 { 1048 uint16_t pid = 0; 1049 size_t page_size = sysconf(_SC_PAGESIZE); 1050 int ret; 1051 1052 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1053 page_size); 1054 if (ret) { 1055 TESTPMD_LOG(DEBUG, 1056 "unable to register addr 0x%p\n", memhdr->addr); 1057 return; 1058 } 1059 RTE_ETH_FOREACH_DEV(pid) { 1060 struct rte_eth_dev_info dev_info; 1061 1062 ret = eth_dev_info_get_print_err(pid, &dev_info); 1063 if (ret != 0) { 1064 TESTPMD_LOG(DEBUG, 1065 "unable to get device info for port %d on addr 0x%p," 1066 "mempool mapping will not be performed\n", 1067 pid, memhdr->addr); 1068 continue; 1069 } 1070 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1071 if (ret) { 1072 TESTPMD_LOG(DEBUG, 1073 "unable to DMA map addr 0x%p " 1074 "for device %s\n", 1075 memhdr->addr, rte_dev_name(dev_info.device)); 1076 } 1077 } 1078 } 1079 #endif 1080 1081 static unsigned int 1082 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1083 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1084 { 1085 struct rte_pktmbuf_extmem *xmem; 1086 unsigned int ext_num, zone_num, elt_num; 1087 uint16_t elt_size; 1088 1089 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1090 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1091 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1092 1093 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1094 if (xmem == NULL) { 1095 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1096 "external buffer descriptors\n"); 1097 *ext_mem = NULL; 1098 return 0; 1099 } 1100 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1101 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1102 const struct rte_memzone *mz; 1103 char mz_name[RTE_MEMZONE_NAMESIZE]; 1104 int ret; 1105 1106 ret = snprintf(mz_name, sizeof(mz_name), 1107 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1108 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1109 errno = ENAMETOOLONG; 1110 ext_num = 0; 1111 break; 1112 } 1113 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1114 socket_id, 1115 RTE_MEMZONE_IOVA_CONTIG | 1116 RTE_MEMZONE_1GB | 1117 RTE_MEMZONE_SIZE_HINT_ONLY); 1118 if (mz == NULL) { 1119 /* 1120 * The caller exits on external buffer creation 1121 * error, so there is no need to free memzones. 1122 */ 1123 errno = ENOMEM; 1124 ext_num = 0; 1125 break; 1126 } 1127 xseg->buf_ptr = mz->addr; 1128 xseg->buf_iova = mz->iova; 1129 xseg->buf_len = EXTBUF_ZONE_SIZE; 1130 xseg->elt_size = elt_size; 1131 } 1132 if (ext_num == 0 && xmem != NULL) { 1133 free(xmem); 1134 xmem = NULL; 1135 } 1136 *ext_mem = xmem; 1137 return ext_num; 1138 } 1139 1140 /* 1141 * Configuration initialisation done once at init time. 1142 */ 1143 static struct rte_mempool * 1144 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1145 unsigned int socket_id, uint16_t size_idx) 1146 { 1147 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1148 struct rte_mempool *rte_mp = NULL; 1149 #ifndef RTE_EXEC_ENV_WINDOWS 1150 uint32_t mb_size; 1151 1152 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1153 #endif 1154 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1155 if (!is_proc_primary()) { 1156 rte_mp = rte_mempool_lookup(pool_name); 1157 if (rte_mp == NULL) 1158 rte_exit(EXIT_FAILURE, 1159 "Get mbuf pool for socket %u failed: %s\n", 1160 socket_id, rte_strerror(rte_errno)); 1161 return rte_mp; 1162 } 1163 1164 TESTPMD_LOG(INFO, 1165 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1166 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1167 1168 switch (mp_alloc_type) { 1169 case MP_ALLOC_NATIVE: 1170 { 1171 /* wrapper to rte_mempool_create() */ 1172 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1173 rte_mbuf_best_mempool_ops()); 1174 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1175 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1176 break; 1177 } 1178 #ifndef RTE_EXEC_ENV_WINDOWS 1179 case MP_ALLOC_ANON: 1180 { 1181 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1182 mb_size, (unsigned int) mb_mempool_cache, 1183 sizeof(struct rte_pktmbuf_pool_private), 1184 socket_id, mempool_flags); 1185 if (rte_mp == NULL) 1186 goto err; 1187 1188 if (rte_mempool_populate_anon(rte_mp) == 0) { 1189 rte_mempool_free(rte_mp); 1190 rte_mp = NULL; 1191 goto err; 1192 } 1193 rte_pktmbuf_pool_init(rte_mp, NULL); 1194 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1195 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1196 break; 1197 } 1198 case MP_ALLOC_XMEM: 1199 case MP_ALLOC_XMEM_HUGE: 1200 { 1201 int heap_socket; 1202 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1203 1204 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1205 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1206 1207 heap_socket = 1208 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1209 if (heap_socket < 0) 1210 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1211 1212 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1213 rte_mbuf_best_mempool_ops()); 1214 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1215 mb_mempool_cache, 0, mbuf_seg_size, 1216 heap_socket); 1217 break; 1218 } 1219 #endif 1220 case MP_ALLOC_XBUF: 1221 { 1222 struct rte_pktmbuf_extmem *ext_mem; 1223 unsigned int ext_num; 1224 1225 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1226 socket_id, pool_name, &ext_mem); 1227 if (ext_num == 0) 1228 rte_exit(EXIT_FAILURE, 1229 "Can't create pinned data buffers\n"); 1230 1231 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1232 rte_mbuf_best_mempool_ops()); 1233 rte_mp = rte_pktmbuf_pool_create_extbuf 1234 (pool_name, nb_mbuf, mb_mempool_cache, 1235 0, mbuf_seg_size, socket_id, 1236 ext_mem, ext_num); 1237 free(ext_mem); 1238 break; 1239 } 1240 default: 1241 { 1242 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1243 } 1244 } 1245 1246 #ifndef RTE_EXEC_ENV_WINDOWS 1247 err: 1248 #endif 1249 if (rte_mp == NULL) { 1250 rte_exit(EXIT_FAILURE, 1251 "Creation of mbuf pool for socket %u failed: %s\n", 1252 socket_id, rte_strerror(rte_errno)); 1253 } else if (verbose_level > 0) { 1254 rte_mempool_dump(stdout, rte_mp); 1255 } 1256 return rte_mp; 1257 } 1258 1259 /* 1260 * Check given socket id is valid or not with NUMA mode, 1261 * if valid, return 0, else return -1 1262 */ 1263 static int 1264 check_socket_id(const unsigned int socket_id) 1265 { 1266 static int warning_once = 0; 1267 1268 if (new_socket_id(socket_id)) { 1269 if (!warning_once && numa_support) 1270 fprintf(stderr, 1271 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1272 warning_once = 1; 1273 return -1; 1274 } 1275 return 0; 1276 } 1277 1278 /* 1279 * Get the allowed maximum number of RX queues. 1280 * *pid return the port id which has minimal value of 1281 * max_rx_queues in all ports. 1282 */ 1283 queueid_t 1284 get_allowed_max_nb_rxq(portid_t *pid) 1285 { 1286 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1287 bool max_rxq_valid = false; 1288 portid_t pi; 1289 struct rte_eth_dev_info dev_info; 1290 1291 RTE_ETH_FOREACH_DEV(pi) { 1292 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1293 continue; 1294 1295 max_rxq_valid = true; 1296 if (dev_info.max_rx_queues < allowed_max_rxq) { 1297 allowed_max_rxq = dev_info.max_rx_queues; 1298 *pid = pi; 1299 } 1300 } 1301 return max_rxq_valid ? allowed_max_rxq : 0; 1302 } 1303 1304 /* 1305 * Check input rxq is valid or not. 1306 * If input rxq is not greater than any of maximum number 1307 * of RX queues of all ports, it is valid. 1308 * if valid, return 0, else return -1 1309 */ 1310 int 1311 check_nb_rxq(queueid_t rxq) 1312 { 1313 queueid_t allowed_max_rxq; 1314 portid_t pid = 0; 1315 1316 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1317 if (rxq > allowed_max_rxq) { 1318 fprintf(stderr, 1319 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1320 rxq, allowed_max_rxq, pid); 1321 return -1; 1322 } 1323 return 0; 1324 } 1325 1326 /* 1327 * Get the allowed maximum number of TX queues. 1328 * *pid return the port id which has minimal value of 1329 * max_tx_queues in all ports. 1330 */ 1331 queueid_t 1332 get_allowed_max_nb_txq(portid_t *pid) 1333 { 1334 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1335 bool max_txq_valid = false; 1336 portid_t pi; 1337 struct rte_eth_dev_info dev_info; 1338 1339 RTE_ETH_FOREACH_DEV(pi) { 1340 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1341 continue; 1342 1343 max_txq_valid = true; 1344 if (dev_info.max_tx_queues < allowed_max_txq) { 1345 allowed_max_txq = dev_info.max_tx_queues; 1346 *pid = pi; 1347 } 1348 } 1349 return max_txq_valid ? allowed_max_txq : 0; 1350 } 1351 1352 /* 1353 * Check input txq is valid or not. 1354 * If input txq is not greater than any of maximum number 1355 * of TX queues of all ports, it is valid. 1356 * if valid, return 0, else return -1 1357 */ 1358 int 1359 check_nb_txq(queueid_t txq) 1360 { 1361 queueid_t allowed_max_txq; 1362 portid_t pid = 0; 1363 1364 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1365 if (txq > allowed_max_txq) { 1366 fprintf(stderr, 1367 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1368 txq, allowed_max_txq, pid); 1369 return -1; 1370 } 1371 return 0; 1372 } 1373 1374 /* 1375 * Get the allowed maximum number of RXDs of every rx queue. 1376 * *pid return the port id which has minimal value of 1377 * max_rxd in all queues of all ports. 1378 */ 1379 static uint16_t 1380 get_allowed_max_nb_rxd(portid_t *pid) 1381 { 1382 uint16_t allowed_max_rxd = UINT16_MAX; 1383 portid_t pi; 1384 struct rte_eth_dev_info dev_info; 1385 1386 RTE_ETH_FOREACH_DEV(pi) { 1387 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1388 continue; 1389 1390 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1391 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1392 *pid = pi; 1393 } 1394 } 1395 return allowed_max_rxd; 1396 } 1397 1398 /* 1399 * Get the allowed minimal number of RXDs of every rx queue. 1400 * *pid return the port id which has minimal value of 1401 * min_rxd in all queues of all ports. 1402 */ 1403 static uint16_t 1404 get_allowed_min_nb_rxd(portid_t *pid) 1405 { 1406 uint16_t allowed_min_rxd = 0; 1407 portid_t pi; 1408 struct rte_eth_dev_info dev_info; 1409 1410 RTE_ETH_FOREACH_DEV(pi) { 1411 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1412 continue; 1413 1414 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1415 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1416 *pid = pi; 1417 } 1418 } 1419 1420 return allowed_min_rxd; 1421 } 1422 1423 /* 1424 * Check input rxd is valid or not. 1425 * If input rxd is not greater than any of maximum number 1426 * of RXDs of every Rx queues and is not less than any of 1427 * minimal number of RXDs of every Rx queues, it is valid. 1428 * if valid, return 0, else return -1 1429 */ 1430 int 1431 check_nb_rxd(queueid_t rxd) 1432 { 1433 uint16_t allowed_max_rxd; 1434 uint16_t allowed_min_rxd; 1435 portid_t pid = 0; 1436 1437 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1438 if (rxd > allowed_max_rxd) { 1439 fprintf(stderr, 1440 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1441 rxd, allowed_max_rxd, pid); 1442 return -1; 1443 } 1444 1445 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1446 if (rxd < allowed_min_rxd) { 1447 fprintf(stderr, 1448 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1449 rxd, allowed_min_rxd, pid); 1450 return -1; 1451 } 1452 1453 return 0; 1454 } 1455 1456 /* 1457 * Get the allowed maximum number of TXDs of every rx queues. 1458 * *pid return the port id which has minimal value of 1459 * max_txd in every tx queue. 1460 */ 1461 static uint16_t 1462 get_allowed_max_nb_txd(portid_t *pid) 1463 { 1464 uint16_t allowed_max_txd = UINT16_MAX; 1465 portid_t pi; 1466 struct rte_eth_dev_info dev_info; 1467 1468 RTE_ETH_FOREACH_DEV(pi) { 1469 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1470 continue; 1471 1472 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1473 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1474 *pid = pi; 1475 } 1476 } 1477 return allowed_max_txd; 1478 } 1479 1480 /* 1481 * Get the allowed maximum number of TXDs of every tx queues. 1482 * *pid return the port id which has minimal value of 1483 * min_txd in every tx queue. 1484 */ 1485 static uint16_t 1486 get_allowed_min_nb_txd(portid_t *pid) 1487 { 1488 uint16_t allowed_min_txd = 0; 1489 portid_t pi; 1490 struct rte_eth_dev_info dev_info; 1491 1492 RTE_ETH_FOREACH_DEV(pi) { 1493 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1494 continue; 1495 1496 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1497 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1498 *pid = pi; 1499 } 1500 } 1501 1502 return allowed_min_txd; 1503 } 1504 1505 /* 1506 * Check input txd is valid or not. 1507 * If input txd is not greater than any of maximum number 1508 * of TXDs of every Rx queues, it is valid. 1509 * if valid, return 0, else return -1 1510 */ 1511 int 1512 check_nb_txd(queueid_t txd) 1513 { 1514 uint16_t allowed_max_txd; 1515 uint16_t allowed_min_txd; 1516 portid_t pid = 0; 1517 1518 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1519 if (txd > allowed_max_txd) { 1520 fprintf(stderr, 1521 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1522 txd, allowed_max_txd, pid); 1523 return -1; 1524 } 1525 1526 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1527 if (txd < allowed_min_txd) { 1528 fprintf(stderr, 1529 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1530 txd, allowed_min_txd, pid); 1531 return -1; 1532 } 1533 return 0; 1534 } 1535 1536 1537 /* 1538 * Get the allowed maximum number of hairpin queues. 1539 * *pid return the port id which has minimal value of 1540 * max_hairpin_queues in all ports. 1541 */ 1542 queueid_t 1543 get_allowed_max_nb_hairpinq(portid_t *pid) 1544 { 1545 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1546 portid_t pi; 1547 struct rte_eth_hairpin_cap cap; 1548 1549 RTE_ETH_FOREACH_DEV(pi) { 1550 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1551 *pid = pi; 1552 return 0; 1553 } 1554 if (cap.max_nb_queues < allowed_max_hairpinq) { 1555 allowed_max_hairpinq = cap.max_nb_queues; 1556 *pid = pi; 1557 } 1558 } 1559 return allowed_max_hairpinq; 1560 } 1561 1562 /* 1563 * Check input hairpin is valid or not. 1564 * If input hairpin is not greater than any of maximum number 1565 * of hairpin queues of all ports, it is valid. 1566 * if valid, return 0, else return -1 1567 */ 1568 int 1569 check_nb_hairpinq(queueid_t hairpinq) 1570 { 1571 queueid_t allowed_max_hairpinq; 1572 portid_t pid = 0; 1573 1574 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1575 if (hairpinq > allowed_max_hairpinq) { 1576 fprintf(stderr, 1577 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1578 hairpinq, allowed_max_hairpinq, pid); 1579 return -1; 1580 } 1581 return 0; 1582 } 1583 1584 static int 1585 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1586 { 1587 uint32_t eth_overhead; 1588 1589 if (dev_info->max_mtu != UINT16_MAX && 1590 dev_info->max_rx_pktlen > dev_info->max_mtu) 1591 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1592 else 1593 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1594 1595 return eth_overhead; 1596 } 1597 1598 static void 1599 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1600 { 1601 struct rte_port *port = &ports[pid]; 1602 int ret; 1603 int i; 1604 1605 eth_rx_metadata_negotiate_mp(pid); 1606 1607 port->dev_conf.txmode = tx_mode; 1608 port->dev_conf.rxmode = rx_mode; 1609 1610 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1611 if (ret != 0) 1612 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1613 1614 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1615 port->dev_conf.txmode.offloads &= 1616 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1617 1618 /* Apply Rx offloads configuration */ 1619 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1620 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1621 /* Apply Tx offloads configuration */ 1622 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1623 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1624 1625 if (eth_link_speed) 1626 port->dev_conf.link_speeds = eth_link_speed; 1627 1628 if (max_rx_pkt_len) 1629 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1630 get_eth_overhead(&port->dev_info); 1631 1632 /* set flag to initialize port/queue */ 1633 port->need_reconfig = 1; 1634 port->need_reconfig_queues = 1; 1635 port->socket_id = socket_id; 1636 port->tx_metadata = 0; 1637 1638 /* 1639 * Check for maximum number of segments per MTU. 1640 * Accordingly update the mbuf data size. 1641 */ 1642 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1643 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1644 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1645 uint16_t mtu; 1646 1647 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1648 uint16_t data_size = (mtu + eth_overhead) / 1649 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1650 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1651 1652 if (buffer_size > mbuf_data_size[0]) { 1653 mbuf_data_size[0] = buffer_size; 1654 TESTPMD_LOG(WARNING, 1655 "Configured mbuf size of the first segment %hu\n", 1656 mbuf_data_size[0]); 1657 } 1658 } 1659 } 1660 } 1661 1662 static void 1663 init_config(void) 1664 { 1665 portid_t pid; 1666 struct rte_mempool *mbp; 1667 unsigned int nb_mbuf_per_pool; 1668 lcoreid_t lc_id; 1669 #ifdef RTE_LIB_GRO 1670 struct rte_gro_param gro_param; 1671 #endif 1672 #ifdef RTE_LIB_GSO 1673 uint32_t gso_types; 1674 #endif 1675 1676 /* Configuration of logical cores. */ 1677 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1678 sizeof(struct fwd_lcore *) * nb_lcores, 1679 RTE_CACHE_LINE_SIZE); 1680 if (fwd_lcores == NULL) { 1681 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1682 "failed\n", nb_lcores); 1683 } 1684 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1685 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1686 sizeof(struct fwd_lcore), 1687 RTE_CACHE_LINE_SIZE); 1688 if (fwd_lcores[lc_id] == NULL) { 1689 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1690 "failed\n"); 1691 } 1692 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1693 } 1694 1695 RTE_ETH_FOREACH_DEV(pid) { 1696 uint32_t socket_id; 1697 1698 if (numa_support) { 1699 socket_id = port_numa[pid]; 1700 if (port_numa[pid] == NUMA_NO_CONFIG) { 1701 socket_id = rte_eth_dev_socket_id(pid); 1702 1703 /* 1704 * if socket_id is invalid, 1705 * set to the first available socket. 1706 */ 1707 if (check_socket_id(socket_id) < 0) 1708 socket_id = socket_ids[0]; 1709 } 1710 } else { 1711 socket_id = (socket_num == UMA_NO_CONFIG) ? 1712 0 : socket_num; 1713 } 1714 /* Apply default TxRx configuration for all ports */ 1715 init_config_port_offloads(pid, socket_id); 1716 } 1717 /* 1718 * Create pools of mbuf. 1719 * If NUMA support is disabled, create a single pool of mbuf in 1720 * socket 0 memory by default. 1721 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1722 * 1723 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1724 * nb_txd can be configured at run time. 1725 */ 1726 if (param_total_num_mbufs) 1727 nb_mbuf_per_pool = param_total_num_mbufs; 1728 else { 1729 nb_mbuf_per_pool = RX_DESC_MAX + 1730 (nb_lcores * mb_mempool_cache) + 1731 TX_DESC_MAX + MAX_PKT_BURST; 1732 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1733 } 1734 1735 if (numa_support) { 1736 uint8_t i, j; 1737 1738 for (i = 0; i < num_sockets; i++) 1739 for (j = 0; j < mbuf_data_size_n; j++) 1740 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1741 mbuf_pool_create(mbuf_data_size[j], 1742 nb_mbuf_per_pool, 1743 socket_ids[i], j); 1744 } else { 1745 uint8_t i; 1746 1747 for (i = 0; i < mbuf_data_size_n; i++) 1748 mempools[i] = mbuf_pool_create 1749 (mbuf_data_size[i], 1750 nb_mbuf_per_pool, 1751 socket_num == UMA_NO_CONFIG ? 1752 0 : socket_num, i); 1753 } 1754 1755 init_port_config(); 1756 1757 #ifdef RTE_LIB_GSO 1758 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1759 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1760 #endif 1761 /* 1762 * Records which Mbuf pool to use by each logical core, if needed. 1763 */ 1764 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1765 mbp = mbuf_pool_find( 1766 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1767 1768 if (mbp == NULL) 1769 mbp = mbuf_pool_find(0, 0); 1770 fwd_lcores[lc_id]->mbp = mbp; 1771 #ifdef RTE_LIB_GSO 1772 /* initialize GSO context */ 1773 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1774 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1775 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1776 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1777 RTE_ETHER_CRC_LEN; 1778 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1779 #endif 1780 } 1781 1782 fwd_config_setup(); 1783 1784 #ifdef RTE_LIB_GRO 1785 /* create a gro context for each lcore */ 1786 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1787 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1788 gro_param.max_item_per_flow = MAX_PKT_BURST; 1789 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1790 gro_param.socket_id = rte_lcore_to_socket_id( 1791 fwd_lcores_cpuids[lc_id]); 1792 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1793 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1794 rte_exit(EXIT_FAILURE, 1795 "rte_gro_ctx_create() failed\n"); 1796 } 1797 } 1798 #endif 1799 } 1800 1801 1802 void 1803 reconfig(portid_t new_port_id, unsigned socket_id) 1804 { 1805 /* Reconfiguration of Ethernet ports. */ 1806 init_config_port_offloads(new_port_id, socket_id); 1807 init_port_config(); 1808 } 1809 1810 int 1811 init_fwd_streams(void) 1812 { 1813 portid_t pid; 1814 struct rte_port *port; 1815 streamid_t sm_id, nb_fwd_streams_new; 1816 queueid_t q; 1817 1818 /* set socket id according to numa or not */ 1819 RTE_ETH_FOREACH_DEV(pid) { 1820 port = &ports[pid]; 1821 if (nb_rxq > port->dev_info.max_rx_queues) { 1822 fprintf(stderr, 1823 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1824 nb_rxq, port->dev_info.max_rx_queues); 1825 return -1; 1826 } 1827 if (nb_txq > port->dev_info.max_tx_queues) { 1828 fprintf(stderr, 1829 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1830 nb_txq, port->dev_info.max_tx_queues); 1831 return -1; 1832 } 1833 if (numa_support) { 1834 if (port_numa[pid] != NUMA_NO_CONFIG) 1835 port->socket_id = port_numa[pid]; 1836 else { 1837 port->socket_id = rte_eth_dev_socket_id(pid); 1838 1839 /* 1840 * if socket_id is invalid, 1841 * set to the first available socket. 1842 */ 1843 if (check_socket_id(port->socket_id) < 0) 1844 port->socket_id = socket_ids[0]; 1845 } 1846 } 1847 else { 1848 if (socket_num == UMA_NO_CONFIG) 1849 port->socket_id = 0; 1850 else 1851 port->socket_id = socket_num; 1852 } 1853 } 1854 1855 q = RTE_MAX(nb_rxq, nb_txq); 1856 if (q == 0) { 1857 fprintf(stderr, 1858 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1859 return -1; 1860 } 1861 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1862 if (nb_fwd_streams_new == nb_fwd_streams) 1863 return 0; 1864 /* clear the old */ 1865 if (fwd_streams != NULL) { 1866 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1867 if (fwd_streams[sm_id] == NULL) 1868 continue; 1869 rte_free(fwd_streams[sm_id]); 1870 fwd_streams[sm_id] = NULL; 1871 } 1872 rte_free(fwd_streams); 1873 fwd_streams = NULL; 1874 } 1875 1876 /* init new */ 1877 nb_fwd_streams = nb_fwd_streams_new; 1878 if (nb_fwd_streams) { 1879 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1880 sizeof(struct fwd_stream *) * nb_fwd_streams, 1881 RTE_CACHE_LINE_SIZE); 1882 if (fwd_streams == NULL) 1883 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1884 " (struct fwd_stream *)) failed\n", 1885 nb_fwd_streams); 1886 1887 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1888 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1889 " struct fwd_stream", sizeof(struct fwd_stream), 1890 RTE_CACHE_LINE_SIZE); 1891 if (fwd_streams[sm_id] == NULL) 1892 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1893 "(struct fwd_stream) failed\n"); 1894 } 1895 } 1896 1897 return 0; 1898 } 1899 1900 static void 1901 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1902 { 1903 uint64_t total_burst, sburst; 1904 uint64_t nb_burst; 1905 uint64_t burst_stats[4]; 1906 uint16_t pktnb_stats[4]; 1907 uint16_t nb_pkt; 1908 int burst_percent[4], sburstp; 1909 int i; 1910 1911 /* 1912 * First compute the total number of packet bursts and the 1913 * two highest numbers of bursts of the same number of packets. 1914 */ 1915 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1916 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1917 1918 /* Show stats for 0 burst size always */ 1919 total_burst = pbs->pkt_burst_spread[0]; 1920 burst_stats[0] = pbs->pkt_burst_spread[0]; 1921 pktnb_stats[0] = 0; 1922 1923 /* Find the next 2 burst sizes with highest occurrences. */ 1924 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1925 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1926 1927 if (nb_burst == 0) 1928 continue; 1929 1930 total_burst += nb_burst; 1931 1932 if (nb_burst > burst_stats[1]) { 1933 burst_stats[2] = burst_stats[1]; 1934 pktnb_stats[2] = pktnb_stats[1]; 1935 burst_stats[1] = nb_burst; 1936 pktnb_stats[1] = nb_pkt; 1937 } else if (nb_burst > burst_stats[2]) { 1938 burst_stats[2] = nb_burst; 1939 pktnb_stats[2] = nb_pkt; 1940 } 1941 } 1942 if (total_burst == 0) 1943 return; 1944 1945 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1946 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1947 if (i == 3) { 1948 printf("%d%% of other]\n", 100 - sburstp); 1949 return; 1950 } 1951 1952 sburst += burst_stats[i]; 1953 if (sburst == total_burst) { 1954 printf("%d%% of %d pkts]\n", 1955 100 - sburstp, (int) pktnb_stats[i]); 1956 return; 1957 } 1958 1959 burst_percent[i] = 1960 (double)burst_stats[i] / total_burst * 100; 1961 printf("%d%% of %d pkts + ", 1962 burst_percent[i], (int) pktnb_stats[i]); 1963 sburstp += burst_percent[i]; 1964 } 1965 } 1966 1967 static void 1968 fwd_stream_stats_display(streamid_t stream_id) 1969 { 1970 struct fwd_stream *fs; 1971 static const char *fwd_top_stats_border = "-------"; 1972 1973 fs = fwd_streams[stream_id]; 1974 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1975 (fs->fwd_dropped == 0)) 1976 return; 1977 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1978 "TX Port=%2d/Queue=%2d %s\n", 1979 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1980 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1981 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1982 " TX-dropped: %-14"PRIu64, 1983 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1984 1985 /* if checksum mode */ 1986 if (cur_fwd_eng == &csum_fwd_engine) { 1987 printf(" RX- bad IP checksum: %-14"PRIu64 1988 " Rx- bad L4 checksum: %-14"PRIu64 1989 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1990 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1991 fs->rx_bad_outer_l4_csum); 1992 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1993 fs->rx_bad_outer_ip_csum); 1994 } else { 1995 printf("\n"); 1996 } 1997 1998 if (record_burst_stats) { 1999 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 2000 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 2001 } 2002 } 2003 2004 void 2005 fwd_stats_display(void) 2006 { 2007 static const char *fwd_stats_border = "----------------------"; 2008 static const char *acc_stats_border = "+++++++++++++++"; 2009 struct { 2010 struct fwd_stream *rx_stream; 2011 struct fwd_stream *tx_stream; 2012 uint64_t tx_dropped; 2013 uint64_t rx_bad_ip_csum; 2014 uint64_t rx_bad_l4_csum; 2015 uint64_t rx_bad_outer_l4_csum; 2016 uint64_t rx_bad_outer_ip_csum; 2017 } ports_stats[RTE_MAX_ETHPORTS]; 2018 uint64_t total_rx_dropped = 0; 2019 uint64_t total_tx_dropped = 0; 2020 uint64_t total_rx_nombuf = 0; 2021 struct rte_eth_stats stats; 2022 uint64_t fwd_cycles = 0; 2023 uint64_t total_recv = 0; 2024 uint64_t total_xmit = 0; 2025 struct rte_port *port; 2026 streamid_t sm_id; 2027 portid_t pt_id; 2028 int ret; 2029 int i; 2030 2031 memset(ports_stats, 0, sizeof(ports_stats)); 2032 2033 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2034 struct fwd_stream *fs = fwd_streams[sm_id]; 2035 2036 if (cur_fwd_config.nb_fwd_streams > 2037 cur_fwd_config.nb_fwd_ports) { 2038 fwd_stream_stats_display(sm_id); 2039 } else { 2040 ports_stats[fs->tx_port].tx_stream = fs; 2041 ports_stats[fs->rx_port].rx_stream = fs; 2042 } 2043 2044 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2045 2046 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2047 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2048 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2049 fs->rx_bad_outer_l4_csum; 2050 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2051 fs->rx_bad_outer_ip_csum; 2052 2053 if (record_core_cycles) 2054 fwd_cycles += fs->core_cycles; 2055 } 2056 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2057 pt_id = fwd_ports_ids[i]; 2058 port = &ports[pt_id]; 2059 2060 ret = rte_eth_stats_get(pt_id, &stats); 2061 if (ret != 0) { 2062 fprintf(stderr, 2063 "%s: Error: failed to get stats (port %u): %d", 2064 __func__, pt_id, ret); 2065 continue; 2066 } 2067 stats.ipackets -= port->stats.ipackets; 2068 stats.opackets -= port->stats.opackets; 2069 stats.ibytes -= port->stats.ibytes; 2070 stats.obytes -= port->stats.obytes; 2071 stats.imissed -= port->stats.imissed; 2072 stats.oerrors -= port->stats.oerrors; 2073 stats.rx_nombuf -= port->stats.rx_nombuf; 2074 2075 total_recv += stats.ipackets; 2076 total_xmit += stats.opackets; 2077 total_rx_dropped += stats.imissed; 2078 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2079 total_tx_dropped += stats.oerrors; 2080 total_rx_nombuf += stats.rx_nombuf; 2081 2082 printf("\n %s Forward statistics for port %-2d %s\n", 2083 fwd_stats_border, pt_id, fwd_stats_border); 2084 2085 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2086 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2087 stats.ipackets + stats.imissed); 2088 2089 if (cur_fwd_eng == &csum_fwd_engine) { 2090 printf(" Bad-ipcsum: %-14"PRIu64 2091 " Bad-l4csum: %-14"PRIu64 2092 "Bad-outer-l4csum: %-14"PRIu64"\n", 2093 ports_stats[pt_id].rx_bad_ip_csum, 2094 ports_stats[pt_id].rx_bad_l4_csum, 2095 ports_stats[pt_id].rx_bad_outer_l4_csum); 2096 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2097 ports_stats[pt_id].rx_bad_outer_ip_csum); 2098 } 2099 if (stats.ierrors + stats.rx_nombuf > 0) { 2100 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2101 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2102 } 2103 2104 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2105 "TX-total: %-"PRIu64"\n", 2106 stats.opackets, ports_stats[pt_id].tx_dropped, 2107 stats.opackets + ports_stats[pt_id].tx_dropped); 2108 2109 if (record_burst_stats) { 2110 if (ports_stats[pt_id].rx_stream) 2111 pkt_burst_stats_display("RX", 2112 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2113 if (ports_stats[pt_id].tx_stream) 2114 pkt_burst_stats_display("TX", 2115 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2116 } 2117 2118 printf(" %s--------------------------------%s\n", 2119 fwd_stats_border, fwd_stats_border); 2120 } 2121 2122 printf("\n %s Accumulated forward statistics for all ports" 2123 "%s\n", 2124 acc_stats_border, acc_stats_border); 2125 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2126 "%-"PRIu64"\n" 2127 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2128 "%-"PRIu64"\n", 2129 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2130 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2131 if (total_rx_nombuf > 0) 2132 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2133 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2134 "%s\n", 2135 acc_stats_border, acc_stats_border); 2136 if (record_core_cycles) { 2137 #define CYC_PER_MHZ 1E6 2138 if (total_recv > 0 || total_xmit > 0) { 2139 uint64_t total_pkts = 0; 2140 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2141 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2142 total_pkts = total_xmit; 2143 else 2144 total_pkts = total_recv; 2145 2146 printf("\n CPU cycles/packet=%.2F (total cycles=" 2147 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2148 " MHz Clock\n", 2149 (double) fwd_cycles / total_pkts, 2150 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2151 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2152 } 2153 } 2154 } 2155 2156 void 2157 fwd_stats_reset(void) 2158 { 2159 streamid_t sm_id; 2160 portid_t pt_id; 2161 int ret; 2162 int i; 2163 2164 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2165 pt_id = fwd_ports_ids[i]; 2166 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2167 if (ret != 0) 2168 fprintf(stderr, 2169 "%s: Error: failed to clear stats (port %u):%d", 2170 __func__, pt_id, ret); 2171 } 2172 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2173 struct fwd_stream *fs = fwd_streams[sm_id]; 2174 2175 fs->rx_packets = 0; 2176 fs->tx_packets = 0; 2177 fs->fwd_dropped = 0; 2178 fs->rx_bad_ip_csum = 0; 2179 fs->rx_bad_l4_csum = 0; 2180 fs->rx_bad_outer_l4_csum = 0; 2181 fs->rx_bad_outer_ip_csum = 0; 2182 2183 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2184 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2185 fs->core_cycles = 0; 2186 } 2187 } 2188 2189 static void 2190 flush_fwd_rx_queues(void) 2191 { 2192 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2193 portid_t rxp; 2194 portid_t port_id; 2195 queueid_t rxq; 2196 uint16_t nb_rx; 2197 uint16_t i; 2198 uint8_t j; 2199 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2200 uint64_t timer_period; 2201 2202 if (num_procs > 1) { 2203 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2204 return; 2205 } 2206 2207 /* convert to number of cycles */ 2208 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2209 2210 for (j = 0; j < 2; j++) { 2211 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2212 for (rxq = 0; rxq < nb_rxq; rxq++) { 2213 port_id = fwd_ports_ids[rxp]; 2214 2215 /* Polling stopped queues is prohibited. */ 2216 if (ports[port_id].rxq[rxq].state == 2217 RTE_ETH_QUEUE_STATE_STOPPED) 2218 continue; 2219 2220 /** 2221 * testpmd can stuck in the below do while loop 2222 * if rte_eth_rx_burst() always returns nonzero 2223 * packets. So timer is added to exit this loop 2224 * after 1sec timer expiry. 2225 */ 2226 prev_tsc = rte_rdtsc(); 2227 do { 2228 nb_rx = rte_eth_rx_burst(port_id, rxq, 2229 pkts_burst, MAX_PKT_BURST); 2230 for (i = 0; i < nb_rx; i++) 2231 rte_pktmbuf_free(pkts_burst[i]); 2232 2233 cur_tsc = rte_rdtsc(); 2234 diff_tsc = cur_tsc - prev_tsc; 2235 timer_tsc += diff_tsc; 2236 } while ((nb_rx > 0) && 2237 (timer_tsc < timer_period)); 2238 timer_tsc = 0; 2239 } 2240 } 2241 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2242 } 2243 } 2244 2245 static void 2246 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2247 { 2248 struct fwd_stream **fsm; 2249 streamid_t nb_fs; 2250 streamid_t sm_id; 2251 #ifdef RTE_LIB_BITRATESTATS 2252 uint64_t tics_per_1sec; 2253 uint64_t tics_datum; 2254 uint64_t tics_current; 2255 uint16_t i, cnt_ports; 2256 2257 cnt_ports = nb_ports; 2258 tics_datum = rte_rdtsc(); 2259 tics_per_1sec = rte_get_timer_hz(); 2260 #endif 2261 fsm = &fwd_streams[fc->stream_idx]; 2262 nb_fs = fc->stream_nb; 2263 do { 2264 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2265 if (!fsm[sm_id]->disabled) 2266 (*pkt_fwd)(fsm[sm_id]); 2267 #ifdef RTE_LIB_BITRATESTATS 2268 if (bitrate_enabled != 0 && 2269 bitrate_lcore_id == rte_lcore_id()) { 2270 tics_current = rte_rdtsc(); 2271 if (tics_current - tics_datum >= tics_per_1sec) { 2272 /* Periodic bitrate calculation */ 2273 for (i = 0; i < cnt_ports; i++) 2274 rte_stats_bitrate_calc(bitrate_data, 2275 ports_ids[i]); 2276 tics_datum = tics_current; 2277 } 2278 } 2279 #endif 2280 #ifdef RTE_LIB_LATENCYSTATS 2281 if (latencystats_enabled != 0 && 2282 latencystats_lcore_id == rte_lcore_id()) 2283 rte_latencystats_update(); 2284 #endif 2285 2286 } while (! fc->stopped); 2287 } 2288 2289 static int 2290 start_pkt_forward_on_core(void *fwd_arg) 2291 { 2292 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2293 cur_fwd_config.fwd_eng->packet_fwd); 2294 return 0; 2295 } 2296 2297 /* 2298 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2299 * Used to start communication flows in network loopback test configurations. 2300 */ 2301 static int 2302 run_one_txonly_burst_on_core(void *fwd_arg) 2303 { 2304 struct fwd_lcore *fwd_lc; 2305 struct fwd_lcore tmp_lcore; 2306 2307 fwd_lc = (struct fwd_lcore *) fwd_arg; 2308 tmp_lcore = *fwd_lc; 2309 tmp_lcore.stopped = 1; 2310 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2311 return 0; 2312 } 2313 2314 /* 2315 * Launch packet forwarding: 2316 * - Setup per-port forwarding context. 2317 * - launch logical cores with their forwarding configuration. 2318 */ 2319 static void 2320 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2321 { 2322 unsigned int i; 2323 unsigned int lc_id; 2324 int diag; 2325 2326 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2327 lc_id = fwd_lcores_cpuids[i]; 2328 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2329 fwd_lcores[i]->stopped = 0; 2330 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2331 fwd_lcores[i], lc_id); 2332 if (diag != 0) 2333 fprintf(stderr, 2334 "launch lcore %u failed - diag=%d\n", 2335 lc_id, diag); 2336 } 2337 } 2338 } 2339 2340 /* 2341 * Launch packet forwarding configuration. 2342 */ 2343 void 2344 start_packet_forwarding(int with_tx_first) 2345 { 2346 port_fwd_begin_t port_fwd_begin; 2347 port_fwd_end_t port_fwd_end; 2348 stream_init_t stream_init = cur_fwd_eng->stream_init; 2349 unsigned int i; 2350 2351 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2352 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2353 2354 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2355 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2356 2357 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2358 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2359 (!nb_rxq || !nb_txq)) 2360 rte_exit(EXIT_FAILURE, 2361 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2362 cur_fwd_eng->fwd_mode_name); 2363 2364 if (all_ports_started() == 0) { 2365 fprintf(stderr, "Not all ports were started\n"); 2366 return; 2367 } 2368 if (test_done == 0) { 2369 fprintf(stderr, "Packet forwarding already started\n"); 2370 return; 2371 } 2372 2373 fwd_config_setup(); 2374 2375 pkt_fwd_config_display(&cur_fwd_config); 2376 if (!pkt_fwd_shared_rxq_check()) 2377 return; 2378 2379 if (stream_init != NULL) 2380 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2381 stream_init(fwd_streams[i]); 2382 2383 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2384 if (port_fwd_begin != NULL) { 2385 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2386 if (port_fwd_begin(fwd_ports_ids[i])) { 2387 fprintf(stderr, 2388 "Packet forwarding is not ready\n"); 2389 return; 2390 } 2391 } 2392 } 2393 2394 if (with_tx_first) { 2395 port_fwd_begin = tx_only_engine.port_fwd_begin; 2396 if (port_fwd_begin != NULL) { 2397 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2398 if (port_fwd_begin(fwd_ports_ids[i])) { 2399 fprintf(stderr, 2400 "Packet forwarding is not ready\n"); 2401 return; 2402 } 2403 } 2404 } 2405 } 2406 2407 test_done = 0; 2408 2409 if(!no_flush_rx) 2410 flush_fwd_rx_queues(); 2411 2412 rxtx_config_display(); 2413 2414 fwd_stats_reset(); 2415 if (with_tx_first) { 2416 while (with_tx_first--) { 2417 launch_packet_forwarding( 2418 run_one_txonly_burst_on_core); 2419 rte_eal_mp_wait_lcore(); 2420 } 2421 port_fwd_end = tx_only_engine.port_fwd_end; 2422 if (port_fwd_end != NULL) { 2423 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2424 (*port_fwd_end)(fwd_ports_ids[i]); 2425 } 2426 } 2427 launch_packet_forwarding(start_pkt_forward_on_core); 2428 } 2429 2430 void 2431 stop_packet_forwarding(void) 2432 { 2433 port_fwd_end_t port_fwd_end; 2434 lcoreid_t lc_id; 2435 portid_t pt_id; 2436 int i; 2437 2438 if (test_done) { 2439 fprintf(stderr, "Packet forwarding not started\n"); 2440 return; 2441 } 2442 printf("Telling cores to stop..."); 2443 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2444 fwd_lcores[lc_id]->stopped = 1; 2445 printf("\nWaiting for lcores to finish...\n"); 2446 rte_eal_mp_wait_lcore(); 2447 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2448 if (port_fwd_end != NULL) { 2449 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2450 pt_id = fwd_ports_ids[i]; 2451 (*port_fwd_end)(pt_id); 2452 } 2453 } 2454 2455 fwd_stats_display(); 2456 2457 printf("\nDone.\n"); 2458 test_done = 1; 2459 } 2460 2461 void 2462 dev_set_link_up(portid_t pid) 2463 { 2464 if (rte_eth_dev_set_link_up(pid) < 0) 2465 fprintf(stderr, "\nSet link up fail.\n"); 2466 } 2467 2468 void 2469 dev_set_link_down(portid_t pid) 2470 { 2471 if (rte_eth_dev_set_link_down(pid) < 0) 2472 fprintf(stderr, "\nSet link down fail.\n"); 2473 } 2474 2475 static int 2476 all_ports_started(void) 2477 { 2478 portid_t pi; 2479 struct rte_port *port; 2480 2481 RTE_ETH_FOREACH_DEV(pi) { 2482 port = &ports[pi]; 2483 /* Check if there is a port which is not started */ 2484 if ((port->port_status != RTE_PORT_STARTED) && 2485 (port->slave_flag == 0)) 2486 return 0; 2487 } 2488 2489 /* No port is not started */ 2490 return 1; 2491 } 2492 2493 int 2494 port_is_stopped(portid_t port_id) 2495 { 2496 struct rte_port *port = &ports[port_id]; 2497 2498 if ((port->port_status != RTE_PORT_STOPPED) && 2499 (port->slave_flag == 0)) 2500 return 0; 2501 return 1; 2502 } 2503 2504 int 2505 all_ports_stopped(void) 2506 { 2507 portid_t pi; 2508 2509 RTE_ETH_FOREACH_DEV(pi) { 2510 if (!port_is_stopped(pi)) 2511 return 0; 2512 } 2513 2514 return 1; 2515 } 2516 2517 int 2518 port_is_started(portid_t port_id) 2519 { 2520 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2521 return 0; 2522 2523 if (ports[port_id].port_status != RTE_PORT_STARTED) 2524 return 0; 2525 2526 return 1; 2527 } 2528 2529 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2530 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2531 2532 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2533 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2534 2535 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2536 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2537 2538 2539 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2540 static int 2541 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2542 { 2543 queueid_t qi; 2544 struct rte_eth_hairpin_conf hairpin_conf = { 2545 .peer_count = 1, 2546 }; 2547 int i; 2548 int diag; 2549 struct rte_port *port = &ports[pi]; 2550 uint16_t peer_rx_port = pi; 2551 uint16_t peer_tx_port = pi; 2552 uint32_t manual = 1; 2553 uint32_t tx_exp = hairpin_mode & 0x10; 2554 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2555 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2556 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2557 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2558 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2559 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2560 2561 if (!(hairpin_mode & 0xf)) { 2562 peer_rx_port = pi; 2563 peer_tx_port = pi; 2564 manual = 0; 2565 } else if (hairpin_mode & 0x1) { 2566 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2567 RTE_ETH_DEV_NO_OWNER); 2568 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2569 peer_tx_port = rte_eth_find_next_owned_by(0, 2570 RTE_ETH_DEV_NO_OWNER); 2571 if (p_pi != RTE_MAX_ETHPORTS) { 2572 peer_rx_port = p_pi; 2573 } else { 2574 uint16_t next_pi; 2575 2576 /* Last port will be the peer RX port of the first. */ 2577 RTE_ETH_FOREACH_DEV(next_pi) 2578 peer_rx_port = next_pi; 2579 } 2580 manual = 1; 2581 } else if (hairpin_mode & 0x2) { 2582 if (cnt_pi & 0x1) { 2583 peer_rx_port = p_pi; 2584 } else { 2585 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2586 RTE_ETH_DEV_NO_OWNER); 2587 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2588 peer_rx_port = pi; 2589 } 2590 peer_tx_port = peer_rx_port; 2591 manual = 1; 2592 } 2593 2594 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2595 hairpin_conf.peers[0].port = peer_rx_port; 2596 hairpin_conf.peers[0].queue = i + nb_rxq; 2597 hairpin_conf.manual_bind = !!manual; 2598 hairpin_conf.tx_explicit = !!tx_exp; 2599 hairpin_conf.force_memory = !!tx_force_memory; 2600 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2601 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2602 diag = rte_eth_tx_hairpin_queue_setup 2603 (pi, qi, nb_txd, &hairpin_conf); 2604 i++; 2605 if (diag == 0) 2606 continue; 2607 2608 /* Fail to setup rx queue, return */ 2609 if (port->port_status == RTE_PORT_HANDLING) 2610 port->port_status = RTE_PORT_STOPPED; 2611 else 2612 fprintf(stderr, 2613 "Port %d can not be set back to stopped\n", pi); 2614 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2615 pi); 2616 /* try to reconfigure queues next time */ 2617 port->need_reconfig_queues = 1; 2618 return -1; 2619 } 2620 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2621 hairpin_conf.peers[0].port = peer_tx_port; 2622 hairpin_conf.peers[0].queue = i + nb_txq; 2623 hairpin_conf.manual_bind = !!manual; 2624 hairpin_conf.tx_explicit = !!tx_exp; 2625 hairpin_conf.force_memory = !!rx_force_memory; 2626 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2627 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2628 diag = rte_eth_rx_hairpin_queue_setup 2629 (pi, qi, nb_rxd, &hairpin_conf); 2630 i++; 2631 if (diag == 0) 2632 continue; 2633 2634 /* Fail to setup rx queue, return */ 2635 if (port->port_status == RTE_PORT_HANDLING) 2636 port->port_status = RTE_PORT_STOPPED; 2637 else 2638 fprintf(stderr, 2639 "Port %d can not be set back to stopped\n", pi); 2640 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2641 pi); 2642 /* try to reconfigure queues next time */ 2643 port->need_reconfig_queues = 1; 2644 return -1; 2645 } 2646 return 0; 2647 } 2648 2649 /* Configure the Rx with optional split. */ 2650 int 2651 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2652 uint16_t nb_rx_desc, unsigned int socket_id, 2653 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2654 { 2655 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2656 unsigned int i, mp_n; 2657 int ret; 2658 2659 if (rx_pkt_nb_segs <= 1 || 2660 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2661 rx_conf->rx_seg = NULL; 2662 rx_conf->rx_nseg = 0; 2663 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2664 nb_rx_desc, socket_id, 2665 rx_conf, mp); 2666 goto exit; 2667 } 2668 for (i = 0; i < rx_pkt_nb_segs; i++) { 2669 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2670 struct rte_mempool *mpx; 2671 /* 2672 * Use last valid pool for the segments with number 2673 * exceeding the pool index. 2674 */ 2675 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2676 mpx = mbuf_pool_find(socket_id, mp_n); 2677 /* Handle zero as mbuf data buffer size. */ 2678 rx_seg->offset = i < rx_pkt_nb_offs ? 2679 rx_pkt_seg_offsets[i] : 0; 2680 rx_seg->mp = mpx ? mpx : mp; 2681 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2682 rx_seg->proto_hdr = rx_pkt_hdr_protos[i]; 2683 } else { 2684 rx_seg->length = rx_pkt_seg_lengths[i] ? 2685 rx_pkt_seg_lengths[i] : 2686 mbuf_data_size[mp_n]; 2687 } 2688 } 2689 rx_conf->rx_nseg = rx_pkt_nb_segs; 2690 rx_conf->rx_seg = rx_useg; 2691 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2692 socket_id, rx_conf, NULL); 2693 rx_conf->rx_seg = NULL; 2694 rx_conf->rx_nseg = 0; 2695 exit: 2696 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2697 RTE_ETH_QUEUE_STATE_STOPPED : 2698 RTE_ETH_QUEUE_STATE_STARTED; 2699 return ret; 2700 } 2701 2702 static int 2703 alloc_xstats_display_info(portid_t pi) 2704 { 2705 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2706 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2707 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2708 2709 if (xstats_display_num == 0) 2710 return 0; 2711 2712 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2713 if (*ids_supp == NULL) 2714 goto fail_ids_supp; 2715 2716 *prev_values = calloc(xstats_display_num, 2717 sizeof(**prev_values)); 2718 if (*prev_values == NULL) 2719 goto fail_prev_values; 2720 2721 *curr_values = calloc(xstats_display_num, 2722 sizeof(**curr_values)); 2723 if (*curr_values == NULL) 2724 goto fail_curr_values; 2725 2726 ports[pi].xstats_info.allocated = true; 2727 2728 return 0; 2729 2730 fail_curr_values: 2731 free(*prev_values); 2732 fail_prev_values: 2733 free(*ids_supp); 2734 fail_ids_supp: 2735 return -ENOMEM; 2736 } 2737 2738 static void 2739 free_xstats_display_info(portid_t pi) 2740 { 2741 if (!ports[pi].xstats_info.allocated) 2742 return; 2743 free(ports[pi].xstats_info.ids_supp); 2744 free(ports[pi].xstats_info.prev_values); 2745 free(ports[pi].xstats_info.curr_values); 2746 ports[pi].xstats_info.allocated = false; 2747 } 2748 2749 /** Fill helper structures for specified port to show extended statistics. */ 2750 static void 2751 fill_xstats_display_info_for_port(portid_t pi) 2752 { 2753 unsigned int stat, stat_supp; 2754 const char *xstat_name; 2755 struct rte_port *port; 2756 uint64_t *ids_supp; 2757 int rc; 2758 2759 if (xstats_display_num == 0) 2760 return; 2761 2762 if (pi == (portid_t)RTE_PORT_ALL) { 2763 fill_xstats_display_info(); 2764 return; 2765 } 2766 2767 port = &ports[pi]; 2768 if (port->port_status != RTE_PORT_STARTED) 2769 return; 2770 2771 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2772 rte_exit(EXIT_FAILURE, 2773 "Failed to allocate xstats display memory\n"); 2774 2775 ids_supp = port->xstats_info.ids_supp; 2776 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2777 xstat_name = xstats_display[stat].name; 2778 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2779 ids_supp + stat_supp); 2780 if (rc != 0) { 2781 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2782 xstat_name, pi, stat); 2783 continue; 2784 } 2785 stat_supp++; 2786 } 2787 2788 port->xstats_info.ids_supp_sz = stat_supp; 2789 } 2790 2791 /** Fill helper structures for all ports to show extended statistics. */ 2792 static void 2793 fill_xstats_display_info(void) 2794 { 2795 portid_t pi; 2796 2797 if (xstats_display_num == 0) 2798 return; 2799 2800 RTE_ETH_FOREACH_DEV(pi) 2801 fill_xstats_display_info_for_port(pi); 2802 } 2803 2804 int 2805 start_port(portid_t pid) 2806 { 2807 int diag, need_check_link_status = -1; 2808 portid_t pi; 2809 portid_t p_pi = RTE_MAX_ETHPORTS; 2810 portid_t pl[RTE_MAX_ETHPORTS]; 2811 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2812 uint16_t cnt_pi = 0; 2813 uint16_t cfg_pi = 0; 2814 int peer_pi; 2815 queueid_t qi; 2816 struct rte_port *port; 2817 struct rte_eth_hairpin_cap cap; 2818 2819 if (port_id_is_invalid(pid, ENABLED_WARN)) 2820 return 0; 2821 2822 RTE_ETH_FOREACH_DEV(pi) { 2823 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2824 continue; 2825 2826 if (port_is_bonding_slave(pi)) { 2827 fprintf(stderr, 2828 "Please remove port %d from bonded device.\n", 2829 pi); 2830 continue; 2831 } 2832 2833 need_check_link_status = 0; 2834 port = &ports[pi]; 2835 if (port->port_status == RTE_PORT_STOPPED) 2836 port->port_status = RTE_PORT_HANDLING; 2837 else { 2838 fprintf(stderr, "Port %d is now not stopped\n", pi); 2839 continue; 2840 } 2841 2842 if (port->need_reconfig > 0) { 2843 struct rte_eth_conf dev_conf; 2844 int k; 2845 2846 port->need_reconfig = 0; 2847 2848 if (flow_isolate_all) { 2849 int ret = port_flow_isolate(pi, 1); 2850 if (ret) { 2851 fprintf(stderr, 2852 "Failed to apply isolated mode on port %d\n", 2853 pi); 2854 return -1; 2855 } 2856 } 2857 configure_rxtx_dump_callbacks(0); 2858 printf("Configuring Port %d (socket %u)\n", pi, 2859 port->socket_id); 2860 if (nb_hairpinq > 0 && 2861 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2862 fprintf(stderr, 2863 "Port %d doesn't support hairpin queues\n", 2864 pi); 2865 return -1; 2866 } 2867 2868 /* configure port */ 2869 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2870 nb_txq + nb_hairpinq, 2871 &(port->dev_conf)); 2872 if (diag != 0) { 2873 if (port->port_status == RTE_PORT_HANDLING) 2874 port->port_status = RTE_PORT_STOPPED; 2875 else 2876 fprintf(stderr, 2877 "Port %d can not be set back to stopped\n", 2878 pi); 2879 fprintf(stderr, "Fail to configure port %d\n", 2880 pi); 2881 /* try to reconfigure port next time */ 2882 port->need_reconfig = 1; 2883 return -1; 2884 } 2885 /* get device configuration*/ 2886 if (0 != 2887 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2888 fprintf(stderr, 2889 "port %d can not get device configuration\n", 2890 pi); 2891 return -1; 2892 } 2893 /* Apply Rx offloads configuration */ 2894 if (dev_conf.rxmode.offloads != 2895 port->dev_conf.rxmode.offloads) { 2896 port->dev_conf.rxmode.offloads |= 2897 dev_conf.rxmode.offloads; 2898 for (k = 0; 2899 k < port->dev_info.max_rx_queues; 2900 k++) 2901 port->rxq[k].conf.offloads |= 2902 dev_conf.rxmode.offloads; 2903 } 2904 /* Apply Tx offloads configuration */ 2905 if (dev_conf.txmode.offloads != 2906 port->dev_conf.txmode.offloads) { 2907 port->dev_conf.txmode.offloads |= 2908 dev_conf.txmode.offloads; 2909 for (k = 0; 2910 k < port->dev_info.max_tx_queues; 2911 k++) 2912 port->txq[k].conf.offloads |= 2913 dev_conf.txmode.offloads; 2914 } 2915 } 2916 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2917 port->need_reconfig_queues = 0; 2918 /* setup tx queues */ 2919 for (qi = 0; qi < nb_txq; qi++) { 2920 struct rte_eth_txconf *conf = 2921 &port->txq[qi].conf; 2922 2923 if ((numa_support) && 2924 (txring_numa[pi] != NUMA_NO_CONFIG)) 2925 diag = rte_eth_tx_queue_setup(pi, qi, 2926 port->nb_tx_desc[qi], 2927 txring_numa[pi], 2928 &(port->txq[qi].conf)); 2929 else 2930 diag = rte_eth_tx_queue_setup(pi, qi, 2931 port->nb_tx_desc[qi], 2932 port->socket_id, 2933 &(port->txq[qi].conf)); 2934 2935 if (diag == 0) { 2936 port->txq[qi].state = 2937 conf->tx_deferred_start ? 2938 RTE_ETH_QUEUE_STATE_STOPPED : 2939 RTE_ETH_QUEUE_STATE_STARTED; 2940 continue; 2941 } 2942 2943 /* Fail to setup tx queue, return */ 2944 if (port->port_status == RTE_PORT_HANDLING) 2945 port->port_status = RTE_PORT_STOPPED; 2946 else 2947 fprintf(stderr, 2948 "Port %d can not be set back to stopped\n", 2949 pi); 2950 fprintf(stderr, 2951 "Fail to configure port %d tx queues\n", 2952 pi); 2953 /* try to reconfigure queues next time */ 2954 port->need_reconfig_queues = 1; 2955 return -1; 2956 } 2957 for (qi = 0; qi < nb_rxq; qi++) { 2958 /* setup rx queues */ 2959 if ((numa_support) && 2960 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2961 struct rte_mempool * mp = 2962 mbuf_pool_find 2963 (rxring_numa[pi], 0); 2964 if (mp == NULL) { 2965 fprintf(stderr, 2966 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2967 rxring_numa[pi]); 2968 return -1; 2969 } 2970 2971 diag = rx_queue_setup(pi, qi, 2972 port->nb_rx_desc[qi], 2973 rxring_numa[pi], 2974 &(port->rxq[qi].conf), 2975 mp); 2976 } else { 2977 struct rte_mempool *mp = 2978 mbuf_pool_find 2979 (port->socket_id, 0); 2980 if (mp == NULL) { 2981 fprintf(stderr, 2982 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2983 port->socket_id); 2984 return -1; 2985 } 2986 diag = rx_queue_setup(pi, qi, 2987 port->nb_rx_desc[qi], 2988 port->socket_id, 2989 &(port->rxq[qi].conf), 2990 mp); 2991 } 2992 if (diag == 0) 2993 continue; 2994 2995 /* Fail to setup rx queue, return */ 2996 if (port->port_status == RTE_PORT_HANDLING) 2997 port->port_status = RTE_PORT_STOPPED; 2998 else 2999 fprintf(stderr, 3000 "Port %d can not be set back to stopped\n", 3001 pi); 3002 fprintf(stderr, 3003 "Fail to configure port %d rx queues\n", 3004 pi); 3005 /* try to reconfigure queues next time */ 3006 port->need_reconfig_queues = 1; 3007 return -1; 3008 } 3009 /* setup hairpin queues */ 3010 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3011 return -1; 3012 } 3013 configure_rxtx_dump_callbacks(verbose_level); 3014 if (clear_ptypes) { 3015 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3016 NULL, 0); 3017 if (diag < 0) 3018 fprintf(stderr, 3019 "Port %d: Failed to disable Ptype parsing\n", 3020 pi); 3021 } 3022 3023 p_pi = pi; 3024 cnt_pi++; 3025 3026 /* start port */ 3027 diag = eth_dev_start_mp(pi); 3028 if (diag < 0) { 3029 fprintf(stderr, "Fail to start port %d: %s\n", 3030 pi, rte_strerror(-diag)); 3031 3032 /* Fail to setup rx queue, return */ 3033 if (port->port_status == RTE_PORT_HANDLING) 3034 port->port_status = RTE_PORT_STOPPED; 3035 else 3036 fprintf(stderr, 3037 "Port %d can not be set back to stopped\n", 3038 pi); 3039 continue; 3040 } 3041 3042 if (port->port_status == RTE_PORT_HANDLING) 3043 port->port_status = RTE_PORT_STARTED; 3044 else 3045 fprintf(stderr, "Port %d can not be set into started\n", 3046 pi); 3047 3048 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3049 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3050 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3051 3052 /* at least one port started, need checking link status */ 3053 need_check_link_status = 1; 3054 3055 pl[cfg_pi++] = pi; 3056 } 3057 3058 if (need_check_link_status == 1 && !no_link_check) 3059 check_all_ports_link_status(RTE_PORT_ALL); 3060 else if (need_check_link_status == 0) 3061 fprintf(stderr, "Please stop the ports first\n"); 3062 3063 if (hairpin_mode & 0xf) { 3064 uint16_t i; 3065 int j; 3066 3067 /* bind all started hairpin ports */ 3068 for (i = 0; i < cfg_pi; i++) { 3069 pi = pl[i]; 3070 /* bind current Tx to all peer Rx */ 3071 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3072 RTE_MAX_ETHPORTS, 1); 3073 if (peer_pi < 0) 3074 return peer_pi; 3075 for (j = 0; j < peer_pi; j++) { 3076 if (!port_is_started(peer_pl[j])) 3077 continue; 3078 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3079 if (diag < 0) { 3080 fprintf(stderr, 3081 "Error during binding hairpin Tx port %u to %u: %s\n", 3082 pi, peer_pl[j], 3083 rte_strerror(-diag)); 3084 return -1; 3085 } 3086 } 3087 /* bind all peer Tx to current Rx */ 3088 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3089 RTE_MAX_ETHPORTS, 0); 3090 if (peer_pi < 0) 3091 return peer_pi; 3092 for (j = 0; j < peer_pi; j++) { 3093 if (!port_is_started(peer_pl[j])) 3094 continue; 3095 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3096 if (diag < 0) { 3097 fprintf(stderr, 3098 "Error during binding hairpin Tx port %u to %u: %s\n", 3099 peer_pl[j], pi, 3100 rte_strerror(-diag)); 3101 return -1; 3102 } 3103 } 3104 } 3105 } 3106 3107 fill_xstats_display_info_for_port(pid); 3108 3109 printf("Done\n"); 3110 return 0; 3111 } 3112 3113 void 3114 stop_port(portid_t pid) 3115 { 3116 portid_t pi; 3117 struct rte_port *port; 3118 int need_check_link_status = 0; 3119 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3120 int peer_pi; 3121 3122 if (port_id_is_invalid(pid, ENABLED_WARN)) 3123 return; 3124 3125 printf("Stopping ports...\n"); 3126 3127 RTE_ETH_FOREACH_DEV(pi) { 3128 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3129 continue; 3130 3131 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3132 fprintf(stderr, 3133 "Please remove port %d from forwarding configuration.\n", 3134 pi); 3135 continue; 3136 } 3137 3138 if (port_is_bonding_slave(pi)) { 3139 fprintf(stderr, 3140 "Please remove port %d from bonded device.\n", 3141 pi); 3142 continue; 3143 } 3144 3145 port = &ports[pi]; 3146 if (port->port_status == RTE_PORT_STARTED) 3147 port->port_status = RTE_PORT_HANDLING; 3148 else 3149 continue; 3150 3151 if (hairpin_mode & 0xf) { 3152 int j; 3153 3154 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3155 /* unbind all peer Tx from current Rx */ 3156 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3157 RTE_MAX_ETHPORTS, 0); 3158 if (peer_pi < 0) 3159 continue; 3160 for (j = 0; j < peer_pi; j++) { 3161 if (!port_is_started(peer_pl[j])) 3162 continue; 3163 rte_eth_hairpin_unbind(peer_pl[j], pi); 3164 } 3165 } 3166 3167 if (port->flow_list) 3168 port_flow_flush(pi); 3169 3170 if (eth_dev_stop_mp(pi) != 0) 3171 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3172 pi); 3173 3174 if (port->port_status == RTE_PORT_HANDLING) 3175 port->port_status = RTE_PORT_STOPPED; 3176 else 3177 fprintf(stderr, "Port %d can not be set into stopped\n", 3178 pi); 3179 need_check_link_status = 1; 3180 } 3181 if (need_check_link_status && !no_link_check) 3182 check_all_ports_link_status(RTE_PORT_ALL); 3183 3184 printf("Done\n"); 3185 } 3186 3187 static void 3188 remove_invalid_ports_in(portid_t *array, portid_t *total) 3189 { 3190 portid_t i; 3191 portid_t new_total = 0; 3192 3193 for (i = 0; i < *total; i++) 3194 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3195 array[new_total] = array[i]; 3196 new_total++; 3197 } 3198 *total = new_total; 3199 } 3200 3201 static void 3202 remove_invalid_ports(void) 3203 { 3204 remove_invalid_ports_in(ports_ids, &nb_ports); 3205 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3206 nb_cfg_ports = nb_fwd_ports; 3207 } 3208 3209 static void 3210 flush_port_owned_resources(portid_t pi) 3211 { 3212 mcast_addr_pool_destroy(pi); 3213 port_flow_flush(pi); 3214 port_flex_item_flush(pi); 3215 port_action_handle_flush(pi); 3216 } 3217 3218 static void 3219 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3220 { 3221 struct rte_port *port; 3222 portid_t slave_pid; 3223 uint16_t i; 3224 3225 for (i = 0; i < num_slaves; i++) { 3226 slave_pid = slave_pids[i]; 3227 if (port_is_started(slave_pid) == 1) { 3228 if (rte_eth_dev_stop(slave_pid) != 0) 3229 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3230 slave_pid); 3231 3232 port = &ports[slave_pid]; 3233 port->port_status = RTE_PORT_STOPPED; 3234 } 3235 3236 clear_port_slave_flag(slave_pid); 3237 3238 /* Close slave device when testpmd quit or is killed. */ 3239 if (cl_quit == 1 || f_quit == 1) 3240 rte_eth_dev_close(slave_pid); 3241 } 3242 } 3243 3244 void 3245 close_port(portid_t pid) 3246 { 3247 portid_t pi; 3248 struct rte_port *port; 3249 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3250 int num_slaves = 0; 3251 3252 if (port_id_is_invalid(pid, ENABLED_WARN)) 3253 return; 3254 3255 printf("Closing ports...\n"); 3256 3257 RTE_ETH_FOREACH_DEV(pi) { 3258 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3259 continue; 3260 3261 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3262 fprintf(stderr, 3263 "Please remove port %d from forwarding configuration.\n", 3264 pi); 3265 continue; 3266 } 3267 3268 if (port_is_bonding_slave(pi)) { 3269 fprintf(stderr, 3270 "Please remove port %d from bonded device.\n", 3271 pi); 3272 continue; 3273 } 3274 3275 port = &ports[pi]; 3276 if (port->port_status == RTE_PORT_CLOSED) { 3277 fprintf(stderr, "Port %d is already closed\n", pi); 3278 continue; 3279 } 3280 3281 if (is_proc_primary()) { 3282 flush_port_owned_resources(pi); 3283 #ifdef RTE_NET_BOND 3284 if (port->bond_flag == 1) 3285 num_slaves = rte_eth_bond_slaves_get(pi, 3286 slave_pids, RTE_MAX_ETHPORTS); 3287 #endif 3288 rte_eth_dev_close(pi); 3289 /* 3290 * If this port is bonded device, all slaves under the 3291 * device need to be removed or closed. 3292 */ 3293 if (port->bond_flag == 1 && num_slaves > 0) 3294 clear_bonding_slave_device(slave_pids, 3295 num_slaves); 3296 } 3297 3298 free_xstats_display_info(pi); 3299 } 3300 3301 remove_invalid_ports(); 3302 printf("Done\n"); 3303 } 3304 3305 void 3306 reset_port(portid_t pid) 3307 { 3308 int diag; 3309 portid_t pi; 3310 struct rte_port *port; 3311 3312 if (port_id_is_invalid(pid, ENABLED_WARN)) 3313 return; 3314 3315 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3316 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3317 fprintf(stderr, 3318 "Can not reset port(s), please stop port(s) first.\n"); 3319 return; 3320 } 3321 3322 printf("Resetting ports...\n"); 3323 3324 RTE_ETH_FOREACH_DEV(pi) { 3325 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3326 continue; 3327 3328 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3329 fprintf(stderr, 3330 "Please remove port %d from forwarding configuration.\n", 3331 pi); 3332 continue; 3333 } 3334 3335 if (port_is_bonding_slave(pi)) { 3336 fprintf(stderr, 3337 "Please remove port %d from bonded device.\n", 3338 pi); 3339 continue; 3340 } 3341 3342 if (is_proc_primary()) { 3343 diag = rte_eth_dev_reset(pi); 3344 if (diag == 0) { 3345 port = &ports[pi]; 3346 port->need_reconfig = 1; 3347 port->need_reconfig_queues = 1; 3348 } else { 3349 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3350 pi, diag); 3351 } 3352 } 3353 } 3354 3355 printf("Done\n"); 3356 } 3357 3358 void 3359 attach_port(char *identifier) 3360 { 3361 portid_t pi; 3362 struct rte_dev_iterator iterator; 3363 3364 printf("Attaching a new port...\n"); 3365 3366 if (identifier == NULL) { 3367 fprintf(stderr, "Invalid parameters are specified\n"); 3368 return; 3369 } 3370 3371 if (rte_dev_probe(identifier) < 0) { 3372 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3373 return; 3374 } 3375 3376 /* first attach mode: event */ 3377 if (setup_on_probe_event) { 3378 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3379 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3380 if (ports[pi].port_status == RTE_PORT_HANDLING && 3381 ports[pi].need_setup != 0) 3382 setup_attached_port(pi); 3383 return; 3384 } 3385 3386 /* second attach mode: iterator */ 3387 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3388 /* setup ports matching the devargs used for probing */ 3389 if (port_is_forwarding(pi)) 3390 continue; /* port was already attached before */ 3391 setup_attached_port(pi); 3392 } 3393 } 3394 3395 static void 3396 setup_attached_port(portid_t pi) 3397 { 3398 unsigned int socket_id; 3399 int ret; 3400 3401 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3402 /* if socket_id is invalid, set to the first available socket. */ 3403 if (check_socket_id(socket_id) < 0) 3404 socket_id = socket_ids[0]; 3405 reconfig(pi, socket_id); 3406 ret = rte_eth_promiscuous_enable(pi); 3407 if (ret != 0) 3408 fprintf(stderr, 3409 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3410 pi, rte_strerror(-ret)); 3411 3412 ports_ids[nb_ports++] = pi; 3413 fwd_ports_ids[nb_fwd_ports++] = pi; 3414 nb_cfg_ports = nb_fwd_ports; 3415 ports[pi].need_setup = 0; 3416 ports[pi].port_status = RTE_PORT_STOPPED; 3417 3418 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3419 printf("Done\n"); 3420 } 3421 3422 static void 3423 detach_device(struct rte_device *dev) 3424 { 3425 portid_t sibling; 3426 3427 if (dev == NULL) { 3428 fprintf(stderr, "Device already removed\n"); 3429 return; 3430 } 3431 3432 printf("Removing a device...\n"); 3433 3434 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3435 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3436 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3437 fprintf(stderr, "Port %u not stopped\n", 3438 sibling); 3439 return; 3440 } 3441 flush_port_owned_resources(sibling); 3442 } 3443 } 3444 3445 if (rte_dev_remove(dev) < 0) { 3446 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3447 return; 3448 } 3449 remove_invalid_ports(); 3450 3451 printf("Device is detached\n"); 3452 printf("Now total ports is %d\n", nb_ports); 3453 printf("Done\n"); 3454 return; 3455 } 3456 3457 void 3458 detach_port_device(portid_t port_id) 3459 { 3460 int ret; 3461 struct rte_eth_dev_info dev_info; 3462 3463 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3464 return; 3465 3466 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3467 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3468 fprintf(stderr, "Port not stopped\n"); 3469 return; 3470 } 3471 fprintf(stderr, "Port was not closed\n"); 3472 } 3473 3474 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3475 if (ret != 0) { 3476 TESTPMD_LOG(ERR, 3477 "Failed to get device info for port %d, not detaching\n", 3478 port_id); 3479 return; 3480 } 3481 detach_device(dev_info.device); 3482 } 3483 3484 void 3485 detach_devargs(char *identifier) 3486 { 3487 struct rte_dev_iterator iterator; 3488 struct rte_devargs da; 3489 portid_t port_id; 3490 3491 printf("Removing a device...\n"); 3492 3493 memset(&da, 0, sizeof(da)); 3494 if (rte_devargs_parsef(&da, "%s", identifier)) { 3495 fprintf(stderr, "cannot parse identifier\n"); 3496 return; 3497 } 3498 3499 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3500 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3501 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3502 fprintf(stderr, "Port %u not stopped\n", 3503 port_id); 3504 rte_eth_iterator_cleanup(&iterator); 3505 rte_devargs_reset(&da); 3506 return; 3507 } 3508 flush_port_owned_resources(port_id); 3509 } 3510 } 3511 3512 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3513 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3514 da.name, rte_bus_name(da.bus)); 3515 rte_devargs_reset(&da); 3516 return; 3517 } 3518 3519 remove_invalid_ports(); 3520 3521 printf("Device %s is detached\n", identifier); 3522 printf("Now total ports is %d\n", nb_ports); 3523 printf("Done\n"); 3524 rte_devargs_reset(&da); 3525 } 3526 3527 void 3528 pmd_test_exit(void) 3529 { 3530 portid_t pt_id; 3531 unsigned int i; 3532 int ret; 3533 3534 if (test_done == 0) 3535 stop_packet_forwarding(); 3536 3537 #ifndef RTE_EXEC_ENV_WINDOWS 3538 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3539 if (mempools[i]) { 3540 if (mp_alloc_type == MP_ALLOC_ANON) 3541 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3542 NULL); 3543 } 3544 } 3545 #endif 3546 if (ports != NULL) { 3547 no_link_check = 1; 3548 RTE_ETH_FOREACH_DEV(pt_id) { 3549 printf("\nStopping port %d...\n", pt_id); 3550 fflush(stdout); 3551 stop_port(pt_id); 3552 } 3553 RTE_ETH_FOREACH_DEV(pt_id) { 3554 printf("\nShutting down port %d...\n", pt_id); 3555 fflush(stdout); 3556 close_port(pt_id); 3557 } 3558 } 3559 3560 if (hot_plug) { 3561 ret = rte_dev_event_monitor_stop(); 3562 if (ret) { 3563 RTE_LOG(ERR, EAL, 3564 "fail to stop device event monitor."); 3565 return; 3566 } 3567 3568 ret = rte_dev_event_callback_unregister(NULL, 3569 dev_event_callback, NULL); 3570 if (ret < 0) { 3571 RTE_LOG(ERR, EAL, 3572 "fail to unregister device event callback.\n"); 3573 return; 3574 } 3575 3576 ret = rte_dev_hotplug_handle_disable(); 3577 if (ret) { 3578 RTE_LOG(ERR, EAL, 3579 "fail to disable hotplug handling.\n"); 3580 return; 3581 } 3582 } 3583 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3584 if (mempools[i]) 3585 mempool_free_mp(mempools[i]); 3586 } 3587 free(xstats_display); 3588 3589 printf("\nBye...\n"); 3590 } 3591 3592 typedef void (*cmd_func_t)(void); 3593 struct pmd_test_command { 3594 const char *cmd_name; 3595 cmd_func_t cmd_func; 3596 }; 3597 3598 /* Check the link status of all ports in up to 9s, and print them finally */ 3599 static void 3600 check_all_ports_link_status(uint32_t port_mask) 3601 { 3602 #define CHECK_INTERVAL 100 /* 100ms */ 3603 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3604 portid_t portid; 3605 uint8_t count, all_ports_up, print_flag = 0; 3606 struct rte_eth_link link; 3607 int ret; 3608 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3609 3610 printf("Checking link statuses...\n"); 3611 fflush(stdout); 3612 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3613 all_ports_up = 1; 3614 RTE_ETH_FOREACH_DEV(portid) { 3615 if ((port_mask & (1 << portid)) == 0) 3616 continue; 3617 memset(&link, 0, sizeof(link)); 3618 ret = rte_eth_link_get_nowait(portid, &link); 3619 if (ret < 0) { 3620 all_ports_up = 0; 3621 if (print_flag == 1) 3622 fprintf(stderr, 3623 "Port %u link get failed: %s\n", 3624 portid, rte_strerror(-ret)); 3625 continue; 3626 } 3627 /* print link status if flag set */ 3628 if (print_flag == 1) { 3629 rte_eth_link_to_str(link_status, 3630 sizeof(link_status), &link); 3631 printf("Port %d %s\n", portid, link_status); 3632 continue; 3633 } 3634 /* clear all_ports_up flag if any link down */ 3635 if (link.link_status == RTE_ETH_LINK_DOWN) { 3636 all_ports_up = 0; 3637 break; 3638 } 3639 } 3640 /* after finally printing all link status, get out */ 3641 if (print_flag == 1) 3642 break; 3643 3644 if (all_ports_up == 0) { 3645 fflush(stdout); 3646 rte_delay_ms(CHECK_INTERVAL); 3647 } 3648 3649 /* set the print_flag if all ports up or timeout */ 3650 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3651 print_flag = 1; 3652 } 3653 3654 if (lsc_interrupt) 3655 break; 3656 } 3657 } 3658 3659 static void 3660 rmv_port_callback(void *arg) 3661 { 3662 int need_to_start = 0; 3663 int org_no_link_check = no_link_check; 3664 portid_t port_id = (intptr_t)arg; 3665 struct rte_eth_dev_info dev_info; 3666 int ret; 3667 3668 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3669 3670 if (!test_done && port_is_forwarding(port_id)) { 3671 need_to_start = 1; 3672 stop_packet_forwarding(); 3673 } 3674 no_link_check = 1; 3675 stop_port(port_id); 3676 no_link_check = org_no_link_check; 3677 3678 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3679 if (ret != 0) 3680 TESTPMD_LOG(ERR, 3681 "Failed to get device info for port %d, not detaching\n", 3682 port_id); 3683 else { 3684 struct rte_device *device = dev_info.device; 3685 close_port(port_id); 3686 detach_device(device); /* might be already removed or have more ports */ 3687 } 3688 if (need_to_start) 3689 start_packet_forwarding(0); 3690 } 3691 3692 /* This function is used by the interrupt thread */ 3693 static int 3694 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3695 void *ret_param) 3696 { 3697 RTE_SET_USED(param); 3698 RTE_SET_USED(ret_param); 3699 3700 if (type >= RTE_ETH_EVENT_MAX) { 3701 fprintf(stderr, 3702 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3703 port_id, __func__, type); 3704 fflush(stderr); 3705 } else if (event_print_mask & (UINT32_C(1) << type)) { 3706 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3707 eth_event_desc[type]); 3708 fflush(stdout); 3709 } 3710 3711 switch (type) { 3712 case RTE_ETH_EVENT_NEW: 3713 ports[port_id].need_setup = 1; 3714 ports[port_id].port_status = RTE_PORT_HANDLING; 3715 break; 3716 case RTE_ETH_EVENT_INTR_RMV: 3717 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3718 break; 3719 if (rte_eal_alarm_set(100000, 3720 rmv_port_callback, (void *)(intptr_t)port_id)) 3721 fprintf(stderr, 3722 "Could not set up deferred device removal\n"); 3723 break; 3724 case RTE_ETH_EVENT_DESTROY: 3725 ports[port_id].port_status = RTE_PORT_CLOSED; 3726 printf("Port %u is closed\n", port_id); 3727 break; 3728 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3729 uint16_t rxq_id; 3730 int ret; 3731 3732 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3733 for (rxq_id = 0; ; rxq_id++) { 3734 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3735 NULL); 3736 if (ret <= 0) 3737 break; 3738 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3739 port_id, rxq_id); 3740 3741 #ifdef RTE_NET_MLX5 3742 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3743 #endif 3744 } 3745 break; 3746 } 3747 default: 3748 break; 3749 } 3750 return 0; 3751 } 3752 3753 static int 3754 register_eth_event_callback(void) 3755 { 3756 int ret; 3757 enum rte_eth_event_type event; 3758 3759 for (event = RTE_ETH_EVENT_UNKNOWN; 3760 event < RTE_ETH_EVENT_MAX; event++) { 3761 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3762 event, 3763 eth_event_callback, 3764 NULL); 3765 if (ret != 0) { 3766 TESTPMD_LOG(ERR, "Failed to register callback for " 3767 "%s event\n", eth_event_desc[event]); 3768 return -1; 3769 } 3770 } 3771 3772 return 0; 3773 } 3774 3775 /* This function is used by the interrupt thread */ 3776 static void 3777 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3778 __rte_unused void *arg) 3779 { 3780 uint16_t port_id; 3781 int ret; 3782 3783 if (type >= RTE_DEV_EVENT_MAX) { 3784 fprintf(stderr, "%s called upon invalid event %d\n", 3785 __func__, type); 3786 fflush(stderr); 3787 } 3788 3789 switch (type) { 3790 case RTE_DEV_EVENT_REMOVE: 3791 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3792 device_name); 3793 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3794 if (ret) { 3795 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3796 device_name); 3797 return; 3798 } 3799 /* 3800 * Because the user's callback is invoked in eal interrupt 3801 * callback, the interrupt callback need to be finished before 3802 * it can be unregistered when detaching device. So finish 3803 * callback soon and use a deferred removal to detach device 3804 * is need. It is a workaround, once the device detaching be 3805 * moved into the eal in the future, the deferred removal could 3806 * be deleted. 3807 */ 3808 if (rte_eal_alarm_set(100000, 3809 rmv_port_callback, (void *)(intptr_t)port_id)) 3810 RTE_LOG(ERR, EAL, 3811 "Could not set up deferred device removal\n"); 3812 break; 3813 case RTE_DEV_EVENT_ADD: 3814 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3815 device_name); 3816 /* TODO: After finish kernel driver binding, 3817 * begin to attach port. 3818 */ 3819 break; 3820 default: 3821 break; 3822 } 3823 } 3824 3825 static void 3826 rxtx_port_config(portid_t pid) 3827 { 3828 uint16_t qid; 3829 uint64_t offloads; 3830 struct rte_port *port = &ports[pid]; 3831 3832 for (qid = 0; qid < nb_rxq; qid++) { 3833 offloads = port->rxq[qid].conf.offloads; 3834 port->rxq[qid].conf = port->dev_info.default_rxconf; 3835 3836 if (rxq_share > 0 && 3837 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3838 /* Non-zero share group to enable RxQ share. */ 3839 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3840 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3841 } 3842 3843 if (offloads != 0) 3844 port->rxq[qid].conf.offloads = offloads; 3845 3846 /* Check if any Rx parameters have been passed */ 3847 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3848 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3849 3850 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3851 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3852 3853 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3854 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3855 3856 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3857 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3858 3859 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3860 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3861 3862 port->nb_rx_desc[qid] = nb_rxd; 3863 } 3864 3865 for (qid = 0; qid < nb_txq; qid++) { 3866 offloads = port->txq[qid].conf.offloads; 3867 port->txq[qid].conf = port->dev_info.default_txconf; 3868 if (offloads != 0) 3869 port->txq[qid].conf.offloads = offloads; 3870 3871 /* Check if any Tx parameters have been passed */ 3872 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3873 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3874 3875 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3876 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3877 3878 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3879 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3880 3881 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3882 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3883 3884 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3885 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3886 3887 port->nb_tx_desc[qid] = nb_txd; 3888 } 3889 } 3890 3891 /* 3892 * Helper function to set MTU from frame size 3893 * 3894 * port->dev_info should be set before calling this function. 3895 * 3896 * return 0 on success, negative on error 3897 */ 3898 int 3899 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3900 { 3901 struct rte_port *port = &ports[portid]; 3902 uint32_t eth_overhead; 3903 uint16_t mtu, new_mtu; 3904 3905 eth_overhead = get_eth_overhead(&port->dev_info); 3906 3907 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3908 printf("Failed to get MTU for port %u\n", portid); 3909 return -1; 3910 } 3911 3912 new_mtu = max_rx_pktlen - eth_overhead; 3913 3914 if (mtu == new_mtu) 3915 return 0; 3916 3917 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3918 fprintf(stderr, 3919 "Failed to set MTU to %u for port %u\n", 3920 new_mtu, portid); 3921 return -1; 3922 } 3923 3924 port->dev_conf.rxmode.mtu = new_mtu; 3925 3926 return 0; 3927 } 3928 3929 void 3930 init_port_config(void) 3931 { 3932 portid_t pid; 3933 struct rte_port *port; 3934 int ret, i; 3935 3936 RTE_ETH_FOREACH_DEV(pid) { 3937 port = &ports[pid]; 3938 3939 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3940 if (ret != 0) 3941 return; 3942 3943 if (nb_rxq > 1) { 3944 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3945 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3946 rss_hf & port->dev_info.flow_type_rss_offloads; 3947 } else { 3948 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3949 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3950 } 3951 3952 if (port->dcb_flag == 0) { 3953 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3954 port->dev_conf.rxmode.mq_mode = 3955 (enum rte_eth_rx_mq_mode) 3956 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3957 } else { 3958 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3959 port->dev_conf.rxmode.offloads &= 3960 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3961 3962 for (i = 0; 3963 i < port->dev_info.nb_rx_queues; 3964 i++) 3965 port->rxq[i].conf.offloads &= 3966 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3967 } 3968 } 3969 3970 rxtx_port_config(pid); 3971 3972 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3973 if (ret != 0) 3974 return; 3975 3976 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3977 port->dev_conf.intr_conf.lsc = 1; 3978 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3979 port->dev_conf.intr_conf.rmv = 1; 3980 } 3981 } 3982 3983 void set_port_slave_flag(portid_t slave_pid) 3984 { 3985 struct rte_port *port; 3986 3987 port = &ports[slave_pid]; 3988 port->slave_flag = 1; 3989 } 3990 3991 void clear_port_slave_flag(portid_t slave_pid) 3992 { 3993 struct rte_port *port; 3994 3995 port = &ports[slave_pid]; 3996 port->slave_flag = 0; 3997 } 3998 3999 uint8_t port_is_bonding_slave(portid_t slave_pid) 4000 { 4001 struct rte_port *port; 4002 struct rte_eth_dev_info dev_info; 4003 int ret; 4004 4005 port = &ports[slave_pid]; 4006 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 4007 if (ret != 0) { 4008 TESTPMD_LOG(ERR, 4009 "Failed to get device info for port id %d," 4010 "cannot determine if the port is a bonded slave", 4011 slave_pid); 4012 return 0; 4013 } 4014 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4015 return 1; 4016 return 0; 4017 } 4018 4019 const uint16_t vlan_tags[] = { 4020 0, 1, 2, 3, 4, 5, 6, 7, 4021 8, 9, 10, 11, 12, 13, 14, 15, 4022 16, 17, 18, 19, 20, 21, 22, 23, 4023 24, 25, 26, 27, 28, 29, 30, 31 4024 }; 4025 4026 static int 4027 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4028 enum dcb_mode_enable dcb_mode, 4029 enum rte_eth_nb_tcs num_tcs, 4030 uint8_t pfc_en) 4031 { 4032 uint8_t i; 4033 int32_t rc; 4034 struct rte_eth_rss_conf rss_conf; 4035 4036 /* 4037 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4038 * given above, and the number of traffic classes available for use. 4039 */ 4040 if (dcb_mode == DCB_VT_ENABLED) { 4041 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4042 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4043 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4044 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4045 4046 /* VMDQ+DCB RX and TX configurations */ 4047 vmdq_rx_conf->enable_default_pool = 0; 4048 vmdq_rx_conf->default_pool = 0; 4049 vmdq_rx_conf->nb_queue_pools = 4050 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4051 vmdq_tx_conf->nb_queue_pools = 4052 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4053 4054 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4055 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4056 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4057 vmdq_rx_conf->pool_map[i].pools = 4058 1 << (i % vmdq_rx_conf->nb_queue_pools); 4059 } 4060 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4061 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4062 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4063 } 4064 4065 /* set DCB mode of RX and TX of multiple queues */ 4066 eth_conf->rxmode.mq_mode = 4067 (enum rte_eth_rx_mq_mode) 4068 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4069 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4070 } else { 4071 struct rte_eth_dcb_rx_conf *rx_conf = 4072 ð_conf->rx_adv_conf.dcb_rx_conf; 4073 struct rte_eth_dcb_tx_conf *tx_conf = 4074 ð_conf->tx_adv_conf.dcb_tx_conf; 4075 4076 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4077 4078 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4079 if (rc != 0) 4080 return rc; 4081 4082 rx_conf->nb_tcs = num_tcs; 4083 tx_conf->nb_tcs = num_tcs; 4084 4085 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4086 rx_conf->dcb_tc[i] = i % num_tcs; 4087 tx_conf->dcb_tc[i] = i % num_tcs; 4088 } 4089 4090 eth_conf->rxmode.mq_mode = 4091 (enum rte_eth_rx_mq_mode) 4092 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4093 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4094 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4095 } 4096 4097 if (pfc_en) 4098 eth_conf->dcb_capability_en = 4099 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4100 else 4101 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4102 4103 return 0; 4104 } 4105 4106 int 4107 init_port_dcb_config(portid_t pid, 4108 enum dcb_mode_enable dcb_mode, 4109 enum rte_eth_nb_tcs num_tcs, 4110 uint8_t pfc_en) 4111 { 4112 struct rte_eth_conf port_conf; 4113 struct rte_port *rte_port; 4114 int retval; 4115 uint16_t i; 4116 4117 if (num_procs > 1) { 4118 printf("The multi-process feature doesn't support dcb.\n"); 4119 return -ENOTSUP; 4120 } 4121 rte_port = &ports[pid]; 4122 4123 /* retain the original device configuration. */ 4124 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4125 4126 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4127 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4128 if (retval < 0) 4129 return retval; 4130 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4131 /* remove RSS HASH offload for DCB in vt mode */ 4132 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4133 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4134 for (i = 0; i < nb_rxq; i++) 4135 rte_port->rxq[i].conf.offloads &= 4136 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4137 } 4138 4139 /* re-configure the device . */ 4140 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4141 if (retval < 0) 4142 return retval; 4143 4144 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4145 if (retval != 0) 4146 return retval; 4147 4148 /* If dev_info.vmdq_pool_base is greater than 0, 4149 * the queue id of vmdq pools is started after pf queues. 4150 */ 4151 if (dcb_mode == DCB_VT_ENABLED && 4152 rte_port->dev_info.vmdq_pool_base > 0) { 4153 fprintf(stderr, 4154 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4155 pid); 4156 return -1; 4157 } 4158 4159 /* Assume the ports in testpmd have the same dcb capability 4160 * and has the same number of rxq and txq in dcb mode 4161 */ 4162 if (dcb_mode == DCB_VT_ENABLED) { 4163 if (rte_port->dev_info.max_vfs > 0) { 4164 nb_rxq = rte_port->dev_info.nb_rx_queues; 4165 nb_txq = rte_port->dev_info.nb_tx_queues; 4166 } else { 4167 nb_rxq = rte_port->dev_info.max_rx_queues; 4168 nb_txq = rte_port->dev_info.max_tx_queues; 4169 } 4170 } else { 4171 /*if vt is disabled, use all pf queues */ 4172 if (rte_port->dev_info.vmdq_pool_base == 0) { 4173 nb_rxq = rte_port->dev_info.max_rx_queues; 4174 nb_txq = rte_port->dev_info.max_tx_queues; 4175 } else { 4176 nb_rxq = (queueid_t)num_tcs; 4177 nb_txq = (queueid_t)num_tcs; 4178 4179 } 4180 } 4181 rx_free_thresh = 64; 4182 4183 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4184 4185 rxtx_port_config(pid); 4186 /* VLAN filter */ 4187 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4188 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4189 rx_vft_set(pid, vlan_tags[i], 1); 4190 4191 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4192 if (retval != 0) 4193 return retval; 4194 4195 rte_port->dcb_flag = 1; 4196 4197 /* Enter DCB configuration status */ 4198 dcb_config = 1; 4199 4200 return 0; 4201 } 4202 4203 static void 4204 init_port(void) 4205 { 4206 int i; 4207 4208 /* Configuration of Ethernet ports. */ 4209 ports = rte_zmalloc("testpmd: ports", 4210 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4211 RTE_CACHE_LINE_SIZE); 4212 if (ports == NULL) { 4213 rte_exit(EXIT_FAILURE, 4214 "rte_zmalloc(%d struct rte_port) failed\n", 4215 RTE_MAX_ETHPORTS); 4216 } 4217 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 4218 ports[i].fwd_mac_swap = 1; 4219 ports[i].xstats_info.allocated = false; 4220 LIST_INIT(&ports[i].flow_tunnel_list); 4221 } 4222 /* Initialize ports NUMA structures */ 4223 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4224 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4225 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4226 } 4227 4228 static void 4229 force_quit(void) 4230 { 4231 pmd_test_exit(); 4232 prompt_exit(); 4233 } 4234 4235 static void 4236 print_stats(void) 4237 { 4238 uint8_t i; 4239 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4240 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4241 4242 /* Clear screen and move to top left */ 4243 printf("%s%s", clr, top_left); 4244 4245 printf("\nPort statistics ===================================="); 4246 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4247 nic_stats_display(fwd_ports_ids[i]); 4248 4249 fflush(stdout); 4250 } 4251 4252 static void 4253 signal_handler(int signum) 4254 { 4255 if (signum == SIGINT || signum == SIGTERM) { 4256 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4257 signum); 4258 #ifdef RTE_LIB_PDUMP 4259 /* uninitialize packet capture framework */ 4260 rte_pdump_uninit(); 4261 #endif 4262 #ifdef RTE_LIB_LATENCYSTATS 4263 if (latencystats_enabled != 0) 4264 rte_latencystats_uninit(); 4265 #endif 4266 force_quit(); 4267 /* Set flag to indicate the force termination. */ 4268 f_quit = 1; 4269 /* exit with the expected status */ 4270 #ifndef RTE_EXEC_ENV_WINDOWS 4271 signal(signum, SIG_DFL); 4272 kill(getpid(), signum); 4273 #endif 4274 } 4275 } 4276 4277 int 4278 main(int argc, char** argv) 4279 { 4280 int diag; 4281 portid_t port_id; 4282 uint16_t count; 4283 int ret; 4284 4285 signal(SIGINT, signal_handler); 4286 signal(SIGTERM, signal_handler); 4287 4288 testpmd_logtype = rte_log_register("testpmd"); 4289 if (testpmd_logtype < 0) 4290 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4291 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4292 4293 diag = rte_eal_init(argc, argv); 4294 if (diag < 0) 4295 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4296 rte_strerror(rte_errno)); 4297 4298 ret = register_eth_event_callback(); 4299 if (ret != 0) 4300 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4301 4302 #ifdef RTE_LIB_PDUMP 4303 /* initialize packet capture framework */ 4304 rte_pdump_init(); 4305 #endif 4306 4307 count = 0; 4308 RTE_ETH_FOREACH_DEV(port_id) { 4309 ports_ids[count] = port_id; 4310 count++; 4311 } 4312 nb_ports = (portid_t) count; 4313 if (nb_ports == 0) 4314 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4315 4316 /* allocate port structures, and init them */ 4317 init_port(); 4318 4319 set_def_fwd_config(); 4320 if (nb_lcores == 0) 4321 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4322 "Check the core mask argument\n"); 4323 4324 /* Bitrate/latency stats disabled by default */ 4325 #ifdef RTE_LIB_BITRATESTATS 4326 bitrate_enabled = 0; 4327 #endif 4328 #ifdef RTE_LIB_LATENCYSTATS 4329 latencystats_enabled = 0; 4330 #endif 4331 4332 /* on FreeBSD, mlockall() is disabled by default */ 4333 #ifdef RTE_EXEC_ENV_FREEBSD 4334 do_mlockall = 0; 4335 #else 4336 do_mlockall = 1; 4337 #endif 4338 4339 argc -= diag; 4340 argv += diag; 4341 if (argc > 1) 4342 launch_args_parse(argc, argv); 4343 4344 #ifndef RTE_EXEC_ENV_WINDOWS 4345 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4346 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4347 strerror(errno)); 4348 } 4349 #endif 4350 4351 if (tx_first && interactive) 4352 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4353 "interactive mode.\n"); 4354 4355 if (tx_first && lsc_interrupt) { 4356 fprintf(stderr, 4357 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4358 lsc_interrupt = 0; 4359 } 4360 4361 if (!nb_rxq && !nb_txq) 4362 fprintf(stderr, 4363 "Warning: Either rx or tx queues should be non-zero\n"); 4364 4365 if (nb_rxq > 1 && nb_rxq > nb_txq) 4366 fprintf(stderr, 4367 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4368 nb_rxq, nb_txq); 4369 4370 init_config(); 4371 4372 if (hot_plug) { 4373 ret = rte_dev_hotplug_handle_enable(); 4374 if (ret) { 4375 RTE_LOG(ERR, EAL, 4376 "fail to enable hotplug handling."); 4377 return -1; 4378 } 4379 4380 ret = rte_dev_event_monitor_start(); 4381 if (ret) { 4382 RTE_LOG(ERR, EAL, 4383 "fail to start device event monitoring."); 4384 return -1; 4385 } 4386 4387 ret = rte_dev_event_callback_register(NULL, 4388 dev_event_callback, NULL); 4389 if (ret) { 4390 RTE_LOG(ERR, EAL, 4391 "fail to register device event callback\n"); 4392 return -1; 4393 } 4394 } 4395 4396 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4397 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4398 4399 /* set all ports to promiscuous mode by default */ 4400 RTE_ETH_FOREACH_DEV(port_id) { 4401 ret = rte_eth_promiscuous_enable(port_id); 4402 if (ret != 0) 4403 fprintf(stderr, 4404 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4405 port_id, rte_strerror(-ret)); 4406 } 4407 4408 #ifdef RTE_LIB_METRICS 4409 /* Init metrics library */ 4410 rte_metrics_init(rte_socket_id()); 4411 #endif 4412 4413 #ifdef RTE_LIB_LATENCYSTATS 4414 if (latencystats_enabled != 0) { 4415 int ret = rte_latencystats_init(1, NULL); 4416 if (ret) 4417 fprintf(stderr, 4418 "Warning: latencystats init() returned error %d\n", 4419 ret); 4420 fprintf(stderr, "Latencystats running on lcore %d\n", 4421 latencystats_lcore_id); 4422 } 4423 #endif 4424 4425 /* Setup bitrate stats */ 4426 #ifdef RTE_LIB_BITRATESTATS 4427 if (bitrate_enabled != 0) { 4428 bitrate_data = rte_stats_bitrate_create(); 4429 if (bitrate_data == NULL) 4430 rte_exit(EXIT_FAILURE, 4431 "Could not allocate bitrate data.\n"); 4432 rte_stats_bitrate_reg(bitrate_data); 4433 } 4434 #endif 4435 #ifdef RTE_LIB_CMDLINE 4436 if (init_cmdline() != 0) 4437 rte_exit(EXIT_FAILURE, 4438 "Could not initialise cmdline context.\n"); 4439 4440 if (strlen(cmdline_filename) != 0) 4441 cmdline_read_from_file(cmdline_filename); 4442 4443 if (interactive == 1) { 4444 if (auto_start) { 4445 printf("Start automatic packet forwarding\n"); 4446 start_packet_forwarding(0); 4447 } 4448 prompt(); 4449 pmd_test_exit(); 4450 } else 4451 #endif 4452 { 4453 char c; 4454 int rc; 4455 4456 f_quit = 0; 4457 4458 printf("No commandline core given, start packet forwarding\n"); 4459 start_packet_forwarding(tx_first); 4460 if (stats_period != 0) { 4461 uint64_t prev_time = 0, cur_time, diff_time = 0; 4462 uint64_t timer_period; 4463 4464 /* Convert to number of cycles */ 4465 timer_period = stats_period * rte_get_timer_hz(); 4466 4467 while (f_quit == 0) { 4468 cur_time = rte_get_timer_cycles(); 4469 diff_time += cur_time - prev_time; 4470 4471 if (diff_time >= timer_period) { 4472 print_stats(); 4473 /* Reset the timer */ 4474 diff_time = 0; 4475 } 4476 /* Sleep to avoid unnecessary checks */ 4477 prev_time = cur_time; 4478 rte_delay_us_sleep(US_PER_S); 4479 } 4480 } 4481 4482 printf("Press enter to exit\n"); 4483 rc = read(0, &c, 1); 4484 pmd_test_exit(); 4485 if (rc < 0) 4486 return 1; 4487 } 4488 4489 ret = rte_eal_cleanup(); 4490 if (ret != 0) 4491 rte_exit(EXIT_FAILURE, 4492 "EAL cleanup failed: %s\n", strerror(-ret)); 4493 4494 return EXIT_SUCCESS; 4495 } 4496