1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_bus.h> 36 #include <rte_eal.h> 37 #include <rte_alarm.h> 38 #include <rte_per_lcore.h> 39 #include <rte_lcore.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 #ifdef RTE_NET_BOND 70 #include <rte_eth_bond.h> 71 #endif 72 #ifdef RTE_NET_MLX5 73 #include "mlx5_testpmd.h" 74 #endif 75 76 #include "testpmd.h" 77 78 #ifndef MAP_HUGETLB 79 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 80 #define HUGE_FLAG (0x40000) 81 #else 82 #define HUGE_FLAG MAP_HUGETLB 83 #endif 84 85 #ifndef MAP_HUGE_SHIFT 86 /* older kernels (or FreeBSD) will not have this define */ 87 #define HUGE_SHIFT (26) 88 #else 89 #define HUGE_SHIFT MAP_HUGE_SHIFT 90 #endif 91 92 #define EXTMEM_HEAP_NAME "extmem" 93 /* 94 * Zone size with the malloc overhead (max of debug and release variants) 95 * must fit into the smallest supported hugepage size (2M), 96 * so that an IOVA-contiguous zone of this size can always be allocated 97 * if there are free 2M hugepages. 98 */ 99 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 100 101 uint16_t verbose_level = 0; /**< Silent by default. */ 102 int testpmd_logtype; /**< Log type for testpmd logs */ 103 104 /* use main core for command line ? */ 105 uint8_t interactive = 0; 106 uint8_t auto_start = 0; 107 uint8_t tx_first; 108 char cmdline_filename[PATH_MAX] = {0}; 109 110 /* 111 * NUMA support configuration. 112 * When set, the NUMA support attempts to dispatch the allocation of the 113 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 114 * probed ports among the CPU sockets 0 and 1. 115 * Otherwise, all memory is allocated from CPU socket 0. 116 */ 117 uint8_t numa_support = 1; /**< numa enabled by default */ 118 119 /* 120 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 121 * not configured. 122 */ 123 uint8_t socket_num = UMA_NO_CONFIG; 124 125 /* 126 * Select mempool allocation type: 127 * - native: use regular DPDK memory 128 * - anon: use regular DPDK memory to create mempool, but populate using 129 * anonymous memory (may not be IOVA-contiguous) 130 * - xmem: use externally allocated hugepage memory 131 */ 132 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 133 134 /* 135 * Store specified sockets on which memory pool to be used by ports 136 * is allocated. 137 */ 138 uint8_t port_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which RX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Store specified sockets on which TX ring to be used by ports 148 * is allocated. 149 */ 150 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 151 152 /* 153 * Record the Ethernet address of peer target ports to which packets are 154 * forwarded. 155 * Must be instantiated with the ethernet addresses of peer traffic generator 156 * ports. 157 */ 158 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 159 portid_t nb_peer_eth_addrs = 0; 160 161 /* 162 * Probed Target Environment. 163 */ 164 struct rte_port *ports; /**< For all probed ethernet ports. */ 165 portid_t nb_ports; /**< Number of probed ethernet ports. */ 166 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 167 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 168 169 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 170 171 /* 172 * Test Forwarding Configuration. 173 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 174 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 175 */ 176 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 177 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 178 portid_t nb_cfg_ports; /**< Number of configured ports. */ 179 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 180 181 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 182 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 183 184 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 185 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 186 187 /* 188 * Forwarding engines. 189 */ 190 struct fwd_engine * fwd_engines[] = { 191 &io_fwd_engine, 192 &mac_fwd_engine, 193 &mac_swap_engine, 194 &flow_gen_engine, 195 &rx_only_engine, 196 &tx_only_engine, 197 &csum_fwd_engine, 198 &icmp_echo_engine, 199 &noisy_vnf_engine, 200 &five_tuple_swap_fwd_engine, 201 #ifdef RTE_LIBRTE_IEEE1588 202 &ieee1588_fwd_engine, 203 #endif 204 &shared_rxq_engine, 205 NULL, 206 }; 207 208 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 209 uint16_t mempool_flags; 210 211 struct fwd_config cur_fwd_config; 212 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 213 uint32_t retry_enabled; 214 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 215 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 216 217 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 218 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 219 DEFAULT_MBUF_DATA_SIZE 220 }; /**< Mbuf data space size. */ 221 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 222 * specified on command-line. */ 223 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 224 225 /** Extended statistics to show. */ 226 struct rte_eth_xstat_name *xstats_display; 227 228 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 229 230 /* 231 * In container, it cannot terminate the process which running with 'stats-period' 232 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 233 */ 234 uint8_t f_quit; 235 uint8_t cl_quit; /* Quit testpmd from cmdline. */ 236 237 /* 238 * Max Rx frame size, set by '--max-pkt-len' parameter. 239 */ 240 uint32_t max_rx_pkt_len; 241 242 /* 243 * Configuration of packet segments used to scatter received packets 244 * if some of split features is configured. 245 */ 246 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 247 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 248 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 249 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 250 uint32_t rx_pkt_hdr_protos[MAX_SEGS_BUFFER_SPLIT]; 251 252 /* 253 * Configuration of packet segments used by the "txonly" processing engine. 254 */ 255 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 256 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 257 TXONLY_DEF_PACKET_LEN, 258 }; 259 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 260 261 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 262 /**< Split policy for packets to TX. */ 263 264 uint8_t txonly_multi_flow; 265 /**< Whether multiple flows are generated in TXONLY mode. */ 266 267 uint32_t tx_pkt_times_inter; 268 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 269 270 uint32_t tx_pkt_times_intra; 271 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 272 273 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 274 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 275 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 276 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 277 278 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 279 uint8_t dcb_config = 0; 280 281 /* 282 * Configurable number of RX/TX queues. 283 */ 284 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 285 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 286 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 287 288 /* 289 * Configurable number of RX/TX ring descriptors. 290 * Defaults are supplied by drivers via ethdev. 291 */ 292 #define RX_DESC_DEFAULT 0 293 #define TX_DESC_DEFAULT 0 294 uint16_t nb_rxd = RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 295 uint16_t nb_txd = TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 296 297 #define RTE_PMD_PARAM_UNSET -1 298 /* 299 * Configurable values of RX and TX ring threshold registers. 300 */ 301 302 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 303 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 304 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 305 306 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 307 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 308 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 309 310 /* 311 * Configurable value of RX free threshold. 312 */ 313 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 314 315 /* 316 * Configurable value of RX drop enable. 317 */ 318 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 319 320 /* 321 * Configurable value of TX free threshold. 322 */ 323 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 324 325 /* 326 * Configurable value of TX RS bit threshold. 327 */ 328 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 329 330 /* 331 * Configurable value of buffered packets before sending. 332 */ 333 uint16_t noisy_tx_sw_bufsz; 334 335 /* 336 * Configurable value of packet buffer timeout. 337 */ 338 uint16_t noisy_tx_sw_buf_flush_time; 339 340 /* 341 * Configurable value for size of VNF internal memory area 342 * used for simulating noisy neighbour behaviour 343 */ 344 uint64_t noisy_lkup_mem_sz; 345 346 /* 347 * Configurable value of number of random writes done in 348 * VNF simulation memory area. 349 */ 350 uint64_t noisy_lkup_num_writes; 351 352 /* 353 * Configurable value of number of random reads done in 354 * VNF simulation memory area. 355 */ 356 uint64_t noisy_lkup_num_reads; 357 358 /* 359 * Configurable value of number of random reads/writes done in 360 * VNF simulation memory area. 361 */ 362 uint64_t noisy_lkup_num_reads_writes; 363 364 /* 365 * Receive Side Scaling (RSS) configuration. 366 */ 367 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 368 369 /* 370 * Port topology configuration 371 */ 372 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 373 374 /* 375 * Avoids to flush all the RX streams before starts forwarding. 376 */ 377 uint8_t no_flush_rx = 0; /* flush by default */ 378 379 /* 380 * Flow API isolated mode. 381 */ 382 uint8_t flow_isolate_all; 383 384 /* 385 * Avoids to check link status when starting/stopping a port. 386 */ 387 uint8_t no_link_check = 0; /* check by default */ 388 389 /* 390 * Don't automatically start all ports in interactive mode. 391 */ 392 uint8_t no_device_start = 0; 393 394 /* 395 * Enable link status change notification 396 */ 397 uint8_t lsc_interrupt = 1; /* enabled by default */ 398 399 /* 400 * Enable device removal notification. 401 */ 402 uint8_t rmv_interrupt = 1; /* enabled by default */ 403 404 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 405 406 /* After attach, port setup is called on event or by iterator */ 407 bool setup_on_probe_event = true; 408 409 /* Clear ptypes on port initialization. */ 410 uint8_t clear_ptypes = true; 411 412 /* Hairpin ports configuration mode. */ 413 uint32_t hairpin_mode; 414 415 /* Pretty printing of ethdev events */ 416 static const char * const eth_event_desc[] = { 417 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 418 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 419 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 420 [RTE_ETH_EVENT_INTR_RESET] = "reset", 421 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 422 [RTE_ETH_EVENT_IPSEC] = "IPsec", 423 [RTE_ETH_EVENT_MACSEC] = "MACsec", 424 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 425 [RTE_ETH_EVENT_NEW] = "device probed", 426 [RTE_ETH_EVENT_DESTROY] = "device released", 427 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 428 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached", 429 [RTE_ETH_EVENT_MAX] = NULL, 430 }; 431 432 /* 433 * Display or mask ether events 434 * Default to all events except VF_MBOX 435 */ 436 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 437 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 438 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 439 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 440 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 441 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 442 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 443 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 444 /* 445 * Decide if all memory are locked for performance. 446 */ 447 int do_mlockall = 0; 448 449 #ifdef RTE_LIB_LATENCYSTATS 450 451 /* 452 * Set when latency stats is enabled in the commandline 453 */ 454 uint8_t latencystats_enabled; 455 456 /* 457 * Lcore ID to service latency statistics. 458 */ 459 lcoreid_t latencystats_lcore_id = -1; 460 461 #endif 462 463 /* 464 * Ethernet device configuration. 465 */ 466 struct rte_eth_rxmode rx_mode; 467 468 struct rte_eth_txmode tx_mode = { 469 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 470 }; 471 472 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 473 474 /* 475 * Display zero values by default for xstats 476 */ 477 uint8_t xstats_hide_zero; 478 479 /* 480 * Measure of CPU cycles disabled by default 481 */ 482 uint8_t record_core_cycles; 483 484 /* 485 * Display of RX and TX bursts disabled by default 486 */ 487 uint8_t record_burst_stats; 488 489 /* 490 * Number of ports per shared Rx queue group, 0 disable. 491 */ 492 uint32_t rxq_share; 493 494 unsigned int num_sockets = 0; 495 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 496 497 #ifdef RTE_LIB_BITRATESTATS 498 /* Bitrate statistics */ 499 struct rte_stats_bitrates *bitrate_data; 500 lcoreid_t bitrate_lcore_id; 501 uint8_t bitrate_enabled; 502 #endif 503 504 #ifdef RTE_LIB_GRO 505 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 506 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 507 #endif 508 509 /* 510 * hexadecimal bitmask of RX mq mode can be enabled. 511 */ 512 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 513 514 /* 515 * Used to set forced link speed 516 */ 517 uint32_t eth_link_speed; 518 519 /* 520 * ID of the current process in multi-process, used to 521 * configure the queues to be polled. 522 */ 523 int proc_id; 524 525 /* 526 * Number of processes in multi-process, used to 527 * configure the queues to be polled. 528 */ 529 unsigned int num_procs = 1; 530 531 static void 532 eth_rx_metadata_negotiate_mp(uint16_t port_id) 533 { 534 uint64_t rx_meta_features = 0; 535 int ret; 536 537 if (!is_proc_primary()) 538 return; 539 540 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 541 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 542 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 543 544 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 545 if (ret == 0) { 546 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 547 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 548 port_id); 549 } 550 551 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 552 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 553 port_id); 554 } 555 556 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 557 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 558 port_id); 559 } 560 } else if (ret != -ENOTSUP) { 561 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 562 port_id, rte_strerror(-ret)); 563 } 564 } 565 566 static int 567 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 568 const struct rte_eth_conf *dev_conf) 569 { 570 if (is_proc_primary()) 571 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 572 dev_conf); 573 return 0; 574 } 575 576 static int 577 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop) 578 { 579 #ifdef RTE_NET_BOND 580 581 portid_t slave_pids[RTE_MAX_ETHPORTS]; 582 struct rte_port *port; 583 int num_slaves; 584 portid_t slave_pid; 585 int i; 586 587 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids, 588 RTE_MAX_ETHPORTS); 589 if (num_slaves < 0) { 590 fprintf(stderr, "Failed to get slave list for port = %u\n", 591 bond_pid); 592 return num_slaves; 593 } 594 595 for (i = 0; i < num_slaves; i++) { 596 slave_pid = slave_pids[i]; 597 port = &ports[slave_pid]; 598 port->port_status = 599 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED; 600 } 601 #else 602 RTE_SET_USED(bond_pid); 603 RTE_SET_USED(is_stop); 604 #endif 605 return 0; 606 } 607 608 static int 609 eth_dev_start_mp(uint16_t port_id) 610 { 611 int ret; 612 613 if (is_proc_primary()) { 614 ret = rte_eth_dev_start(port_id); 615 if (ret != 0) 616 return ret; 617 618 struct rte_port *port = &ports[port_id]; 619 620 /* 621 * Starting a bonded port also starts all slaves under the bonded 622 * device. So if this port is bond device, we need to modify the 623 * port status of these slaves. 624 */ 625 if (port->bond_flag == 1) 626 return change_bonding_slave_port_status(port_id, false); 627 } 628 629 return 0; 630 } 631 632 static int 633 eth_dev_stop_mp(uint16_t port_id) 634 { 635 int ret; 636 637 if (is_proc_primary()) { 638 ret = rte_eth_dev_stop(port_id); 639 if (ret != 0) 640 return ret; 641 642 struct rte_port *port = &ports[port_id]; 643 644 /* 645 * Stopping a bonded port also stops all slaves under the bonded 646 * device. So if this port is bond device, we need to modify the 647 * port status of these slaves. 648 */ 649 if (port->bond_flag == 1) 650 return change_bonding_slave_port_status(port_id, true); 651 } 652 653 return 0; 654 } 655 656 static void 657 mempool_free_mp(struct rte_mempool *mp) 658 { 659 if (is_proc_primary()) 660 rte_mempool_free(mp); 661 } 662 663 static int 664 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 665 { 666 if (is_proc_primary()) 667 return rte_eth_dev_set_mtu(port_id, mtu); 668 669 return 0; 670 } 671 672 /* Forward function declarations */ 673 static void setup_attached_port(portid_t pi); 674 static void check_all_ports_link_status(uint32_t port_mask); 675 static int eth_event_callback(portid_t port_id, 676 enum rte_eth_event_type type, 677 void *param, void *ret_param); 678 static void dev_event_callback(const char *device_name, 679 enum rte_dev_event_type type, 680 void *param); 681 static void fill_xstats_display_info(void); 682 683 /* 684 * Check if all the ports are started. 685 * If yes, return positive value. If not, return zero. 686 */ 687 static int all_ports_started(void); 688 689 #ifdef RTE_LIB_GSO 690 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 691 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 692 #endif 693 694 /* Holds the registered mbuf dynamic flags names. */ 695 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 696 697 698 /* 699 * Helper function to check if socket is already discovered. 700 * If yes, return positive value. If not, return zero. 701 */ 702 int 703 new_socket_id(unsigned int socket_id) 704 { 705 unsigned int i; 706 707 for (i = 0; i < num_sockets; i++) { 708 if (socket_ids[i] == socket_id) 709 return 0; 710 } 711 return 1; 712 } 713 714 /* 715 * Setup default configuration. 716 */ 717 static void 718 set_default_fwd_lcores_config(void) 719 { 720 unsigned int i; 721 unsigned int nb_lc; 722 unsigned int sock_num; 723 724 nb_lc = 0; 725 for (i = 0; i < RTE_MAX_LCORE; i++) { 726 if (!rte_lcore_is_enabled(i)) 727 continue; 728 sock_num = rte_lcore_to_socket_id(i); 729 if (new_socket_id(sock_num)) { 730 if (num_sockets >= RTE_MAX_NUMA_NODES) { 731 rte_exit(EXIT_FAILURE, 732 "Total sockets greater than %u\n", 733 RTE_MAX_NUMA_NODES); 734 } 735 socket_ids[num_sockets++] = sock_num; 736 } 737 if (i == rte_get_main_lcore()) 738 continue; 739 fwd_lcores_cpuids[nb_lc++] = i; 740 } 741 nb_lcores = (lcoreid_t) nb_lc; 742 nb_cfg_lcores = nb_lcores; 743 nb_fwd_lcores = 1; 744 } 745 746 static void 747 set_def_peer_eth_addrs(void) 748 { 749 portid_t i; 750 751 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 752 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 753 peer_eth_addrs[i].addr_bytes[5] = i; 754 } 755 } 756 757 static void 758 set_default_fwd_ports_config(void) 759 { 760 portid_t pt_id; 761 int i = 0; 762 763 RTE_ETH_FOREACH_DEV(pt_id) { 764 fwd_ports_ids[i++] = pt_id; 765 766 /* Update sockets info according to the attached device */ 767 int socket_id = rte_eth_dev_socket_id(pt_id); 768 if (socket_id >= 0 && new_socket_id(socket_id)) { 769 if (num_sockets >= RTE_MAX_NUMA_NODES) { 770 rte_exit(EXIT_FAILURE, 771 "Total sockets greater than %u\n", 772 RTE_MAX_NUMA_NODES); 773 } 774 socket_ids[num_sockets++] = socket_id; 775 } 776 } 777 778 nb_cfg_ports = nb_ports; 779 nb_fwd_ports = nb_ports; 780 } 781 782 void 783 set_def_fwd_config(void) 784 { 785 set_default_fwd_lcores_config(); 786 set_def_peer_eth_addrs(); 787 set_default_fwd_ports_config(); 788 } 789 790 #ifndef RTE_EXEC_ENV_WINDOWS 791 /* extremely pessimistic estimation of memory required to create a mempool */ 792 static int 793 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 794 { 795 unsigned int n_pages, mbuf_per_pg, leftover; 796 uint64_t total_mem, mbuf_mem, obj_sz; 797 798 /* there is no good way to predict how much space the mempool will 799 * occupy because it will allocate chunks on the fly, and some of those 800 * will come from default DPDK memory while some will come from our 801 * external memory, so just assume 128MB will be enough for everyone. 802 */ 803 uint64_t hdr_mem = 128 << 20; 804 805 /* account for possible non-contiguousness */ 806 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 807 if (obj_sz > pgsz) { 808 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 809 return -1; 810 } 811 812 mbuf_per_pg = pgsz / obj_sz; 813 leftover = (nb_mbufs % mbuf_per_pg) > 0; 814 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 815 816 mbuf_mem = n_pages * pgsz; 817 818 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 819 820 if (total_mem > SIZE_MAX) { 821 TESTPMD_LOG(ERR, "Memory size too big\n"); 822 return -1; 823 } 824 *out = (size_t)total_mem; 825 826 return 0; 827 } 828 829 static int 830 pagesz_flags(uint64_t page_sz) 831 { 832 /* as per mmap() manpage, all page sizes are log2 of page size 833 * shifted by MAP_HUGE_SHIFT 834 */ 835 int log2 = rte_log2_u64(page_sz); 836 837 return (log2 << HUGE_SHIFT); 838 } 839 840 static void * 841 alloc_mem(size_t memsz, size_t pgsz, bool huge) 842 { 843 void *addr; 844 int flags; 845 846 /* allocate anonymous hugepages */ 847 flags = MAP_ANONYMOUS | MAP_PRIVATE; 848 if (huge) 849 flags |= HUGE_FLAG | pagesz_flags(pgsz); 850 851 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 852 if (addr == MAP_FAILED) 853 return NULL; 854 855 return addr; 856 } 857 858 struct extmem_param { 859 void *addr; 860 size_t len; 861 size_t pgsz; 862 rte_iova_t *iova_table; 863 unsigned int iova_table_len; 864 }; 865 866 static int 867 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 868 bool huge) 869 { 870 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 871 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 872 unsigned int cur_page, n_pages, pgsz_idx; 873 size_t mem_sz, cur_pgsz; 874 rte_iova_t *iovas = NULL; 875 void *addr; 876 int ret; 877 878 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 879 /* skip anything that is too big */ 880 if (pgsizes[pgsz_idx] > SIZE_MAX) 881 continue; 882 883 cur_pgsz = pgsizes[pgsz_idx]; 884 885 /* if we were told not to allocate hugepages, override */ 886 if (!huge) 887 cur_pgsz = sysconf(_SC_PAGESIZE); 888 889 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 890 if (ret < 0) { 891 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 892 return -1; 893 } 894 895 /* allocate our memory */ 896 addr = alloc_mem(mem_sz, cur_pgsz, huge); 897 898 /* if we couldn't allocate memory with a specified page size, 899 * that doesn't mean we can't do it with other page sizes, so 900 * try another one. 901 */ 902 if (addr == NULL) 903 continue; 904 905 /* store IOVA addresses for every page in this memory area */ 906 n_pages = mem_sz / cur_pgsz; 907 908 iovas = malloc(sizeof(*iovas) * n_pages); 909 910 if (iovas == NULL) { 911 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 912 goto fail; 913 } 914 /* lock memory if it's not huge pages */ 915 if (!huge) 916 mlock(addr, mem_sz); 917 918 /* populate IOVA addresses */ 919 for (cur_page = 0; cur_page < n_pages; cur_page++) { 920 rte_iova_t iova; 921 size_t offset; 922 void *cur; 923 924 offset = cur_pgsz * cur_page; 925 cur = RTE_PTR_ADD(addr, offset); 926 927 /* touch the page before getting its IOVA */ 928 *(volatile char *)cur = 0; 929 930 iova = rte_mem_virt2iova(cur); 931 932 iovas[cur_page] = iova; 933 } 934 935 break; 936 } 937 /* if we couldn't allocate anything */ 938 if (iovas == NULL) 939 return -1; 940 941 param->addr = addr; 942 param->len = mem_sz; 943 param->pgsz = cur_pgsz; 944 param->iova_table = iovas; 945 param->iova_table_len = n_pages; 946 947 return 0; 948 fail: 949 free(iovas); 950 if (addr) 951 munmap(addr, mem_sz); 952 953 return -1; 954 } 955 956 static int 957 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 958 { 959 struct extmem_param param; 960 int socket_id, ret; 961 962 memset(¶m, 0, sizeof(param)); 963 964 /* check if our heap exists */ 965 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 966 if (socket_id < 0) { 967 /* create our heap */ 968 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 969 if (ret < 0) { 970 TESTPMD_LOG(ERR, "Cannot create heap\n"); 971 return -1; 972 } 973 } 974 975 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 976 if (ret < 0) { 977 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 978 return -1; 979 } 980 981 /* we now have a valid memory area, so add it to heap */ 982 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 983 param.addr, param.len, param.iova_table, 984 param.iova_table_len, param.pgsz); 985 986 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 987 988 /* not needed any more */ 989 free(param.iova_table); 990 991 if (ret < 0) { 992 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 993 munmap(param.addr, param.len); 994 return -1; 995 } 996 997 /* success */ 998 999 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 1000 param.len >> 20); 1001 1002 return 0; 1003 } 1004 static void 1005 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1006 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1007 { 1008 uint16_t pid = 0; 1009 int ret; 1010 1011 RTE_ETH_FOREACH_DEV(pid) { 1012 struct rte_eth_dev_info dev_info; 1013 1014 ret = eth_dev_info_get_print_err(pid, &dev_info); 1015 if (ret != 0) { 1016 TESTPMD_LOG(DEBUG, 1017 "unable to get device info for port %d on addr 0x%p," 1018 "mempool unmapping will not be performed\n", 1019 pid, memhdr->addr); 1020 continue; 1021 } 1022 1023 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1024 if (ret) { 1025 TESTPMD_LOG(DEBUG, 1026 "unable to DMA unmap addr 0x%p " 1027 "for device %s\n", 1028 memhdr->addr, rte_dev_name(dev_info.device)); 1029 } 1030 } 1031 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1032 if (ret) { 1033 TESTPMD_LOG(DEBUG, 1034 "unable to un-register addr 0x%p\n", memhdr->addr); 1035 } 1036 } 1037 1038 static void 1039 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1040 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1041 { 1042 uint16_t pid = 0; 1043 size_t page_size = sysconf(_SC_PAGESIZE); 1044 int ret; 1045 1046 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1047 page_size); 1048 if (ret) { 1049 TESTPMD_LOG(DEBUG, 1050 "unable to register addr 0x%p\n", memhdr->addr); 1051 return; 1052 } 1053 RTE_ETH_FOREACH_DEV(pid) { 1054 struct rte_eth_dev_info dev_info; 1055 1056 ret = eth_dev_info_get_print_err(pid, &dev_info); 1057 if (ret != 0) { 1058 TESTPMD_LOG(DEBUG, 1059 "unable to get device info for port %d on addr 0x%p," 1060 "mempool mapping will not be performed\n", 1061 pid, memhdr->addr); 1062 continue; 1063 } 1064 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1065 if (ret) { 1066 TESTPMD_LOG(DEBUG, 1067 "unable to DMA map addr 0x%p " 1068 "for device %s\n", 1069 memhdr->addr, rte_dev_name(dev_info.device)); 1070 } 1071 } 1072 } 1073 #endif 1074 1075 static unsigned int 1076 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1077 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1078 { 1079 struct rte_pktmbuf_extmem *xmem; 1080 unsigned int ext_num, zone_num, elt_num; 1081 uint16_t elt_size; 1082 1083 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1084 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1085 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1086 1087 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1088 if (xmem == NULL) { 1089 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1090 "external buffer descriptors\n"); 1091 *ext_mem = NULL; 1092 return 0; 1093 } 1094 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1095 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1096 const struct rte_memzone *mz; 1097 char mz_name[RTE_MEMZONE_NAMESIZE]; 1098 int ret; 1099 1100 ret = snprintf(mz_name, sizeof(mz_name), 1101 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1102 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1103 errno = ENAMETOOLONG; 1104 ext_num = 0; 1105 break; 1106 } 1107 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1108 socket_id, 1109 RTE_MEMZONE_IOVA_CONTIG | 1110 RTE_MEMZONE_1GB | 1111 RTE_MEMZONE_SIZE_HINT_ONLY); 1112 if (mz == NULL) { 1113 /* 1114 * The caller exits on external buffer creation 1115 * error, so there is no need to free memzones. 1116 */ 1117 errno = ENOMEM; 1118 ext_num = 0; 1119 break; 1120 } 1121 xseg->buf_ptr = mz->addr; 1122 xseg->buf_iova = mz->iova; 1123 xseg->buf_len = EXTBUF_ZONE_SIZE; 1124 xseg->elt_size = elt_size; 1125 } 1126 if (ext_num == 0 && xmem != NULL) { 1127 free(xmem); 1128 xmem = NULL; 1129 } 1130 *ext_mem = xmem; 1131 return ext_num; 1132 } 1133 1134 /* 1135 * Configuration initialisation done once at init time. 1136 */ 1137 static struct rte_mempool * 1138 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1139 unsigned int socket_id, uint16_t size_idx) 1140 { 1141 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1142 struct rte_mempool *rte_mp = NULL; 1143 #ifndef RTE_EXEC_ENV_WINDOWS 1144 uint32_t mb_size; 1145 1146 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1147 #endif 1148 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1149 if (!is_proc_primary()) { 1150 rte_mp = rte_mempool_lookup(pool_name); 1151 if (rte_mp == NULL) 1152 rte_exit(EXIT_FAILURE, 1153 "Get mbuf pool for socket %u failed: %s\n", 1154 socket_id, rte_strerror(rte_errno)); 1155 return rte_mp; 1156 } 1157 1158 TESTPMD_LOG(INFO, 1159 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1160 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1161 1162 switch (mp_alloc_type) { 1163 case MP_ALLOC_NATIVE: 1164 { 1165 /* wrapper to rte_mempool_create() */ 1166 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1167 rte_mbuf_best_mempool_ops()); 1168 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1169 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1170 break; 1171 } 1172 #ifndef RTE_EXEC_ENV_WINDOWS 1173 case MP_ALLOC_ANON: 1174 { 1175 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1176 mb_size, (unsigned int) mb_mempool_cache, 1177 sizeof(struct rte_pktmbuf_pool_private), 1178 socket_id, mempool_flags); 1179 if (rte_mp == NULL) 1180 goto err; 1181 1182 if (rte_mempool_populate_anon(rte_mp) == 0) { 1183 rte_mempool_free(rte_mp); 1184 rte_mp = NULL; 1185 goto err; 1186 } 1187 rte_pktmbuf_pool_init(rte_mp, NULL); 1188 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1189 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1190 break; 1191 } 1192 case MP_ALLOC_XMEM: 1193 case MP_ALLOC_XMEM_HUGE: 1194 { 1195 int heap_socket; 1196 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1197 1198 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1199 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1200 1201 heap_socket = 1202 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1203 if (heap_socket < 0) 1204 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1205 1206 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1207 rte_mbuf_best_mempool_ops()); 1208 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1209 mb_mempool_cache, 0, mbuf_seg_size, 1210 heap_socket); 1211 break; 1212 } 1213 #endif 1214 case MP_ALLOC_XBUF: 1215 { 1216 struct rte_pktmbuf_extmem *ext_mem; 1217 unsigned int ext_num; 1218 1219 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1220 socket_id, pool_name, &ext_mem); 1221 if (ext_num == 0) 1222 rte_exit(EXIT_FAILURE, 1223 "Can't create pinned data buffers\n"); 1224 1225 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1226 rte_mbuf_best_mempool_ops()); 1227 rte_mp = rte_pktmbuf_pool_create_extbuf 1228 (pool_name, nb_mbuf, mb_mempool_cache, 1229 0, mbuf_seg_size, socket_id, 1230 ext_mem, ext_num); 1231 free(ext_mem); 1232 break; 1233 } 1234 default: 1235 { 1236 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1237 } 1238 } 1239 1240 #ifndef RTE_EXEC_ENV_WINDOWS 1241 err: 1242 #endif 1243 if (rte_mp == NULL) { 1244 rte_exit(EXIT_FAILURE, 1245 "Creation of mbuf pool for socket %u failed: %s\n", 1246 socket_id, rte_strerror(rte_errno)); 1247 } else if (verbose_level > 0) { 1248 rte_mempool_dump(stdout, rte_mp); 1249 } 1250 return rte_mp; 1251 } 1252 1253 /* 1254 * Check given socket id is valid or not with NUMA mode, 1255 * if valid, return 0, else return -1 1256 */ 1257 static int 1258 check_socket_id(const unsigned int socket_id) 1259 { 1260 static int warning_once = 0; 1261 1262 if (new_socket_id(socket_id)) { 1263 if (!warning_once && numa_support) 1264 fprintf(stderr, 1265 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1266 warning_once = 1; 1267 return -1; 1268 } 1269 return 0; 1270 } 1271 1272 /* 1273 * Get the allowed maximum number of RX queues. 1274 * *pid return the port id which has minimal value of 1275 * max_rx_queues in all ports. 1276 */ 1277 queueid_t 1278 get_allowed_max_nb_rxq(portid_t *pid) 1279 { 1280 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1281 bool max_rxq_valid = false; 1282 portid_t pi; 1283 struct rte_eth_dev_info dev_info; 1284 1285 RTE_ETH_FOREACH_DEV(pi) { 1286 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1287 continue; 1288 1289 max_rxq_valid = true; 1290 if (dev_info.max_rx_queues < allowed_max_rxq) { 1291 allowed_max_rxq = dev_info.max_rx_queues; 1292 *pid = pi; 1293 } 1294 } 1295 return max_rxq_valid ? allowed_max_rxq : 0; 1296 } 1297 1298 /* 1299 * Check input rxq is valid or not. 1300 * If input rxq is not greater than any of maximum number 1301 * of RX queues of all ports, it is valid. 1302 * if valid, return 0, else return -1 1303 */ 1304 int 1305 check_nb_rxq(queueid_t rxq) 1306 { 1307 queueid_t allowed_max_rxq; 1308 portid_t pid = 0; 1309 1310 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1311 if (rxq > allowed_max_rxq) { 1312 fprintf(stderr, 1313 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1314 rxq, allowed_max_rxq, pid); 1315 return -1; 1316 } 1317 return 0; 1318 } 1319 1320 /* 1321 * Get the allowed maximum number of TX queues. 1322 * *pid return the port id which has minimal value of 1323 * max_tx_queues in all ports. 1324 */ 1325 queueid_t 1326 get_allowed_max_nb_txq(portid_t *pid) 1327 { 1328 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1329 bool max_txq_valid = false; 1330 portid_t pi; 1331 struct rte_eth_dev_info dev_info; 1332 1333 RTE_ETH_FOREACH_DEV(pi) { 1334 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1335 continue; 1336 1337 max_txq_valid = true; 1338 if (dev_info.max_tx_queues < allowed_max_txq) { 1339 allowed_max_txq = dev_info.max_tx_queues; 1340 *pid = pi; 1341 } 1342 } 1343 return max_txq_valid ? allowed_max_txq : 0; 1344 } 1345 1346 /* 1347 * Check input txq is valid or not. 1348 * If input txq is not greater than any of maximum number 1349 * of TX queues of all ports, it is valid. 1350 * if valid, return 0, else return -1 1351 */ 1352 int 1353 check_nb_txq(queueid_t txq) 1354 { 1355 queueid_t allowed_max_txq; 1356 portid_t pid = 0; 1357 1358 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1359 if (txq > allowed_max_txq) { 1360 fprintf(stderr, 1361 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1362 txq, allowed_max_txq, pid); 1363 return -1; 1364 } 1365 return 0; 1366 } 1367 1368 /* 1369 * Get the allowed maximum number of RXDs of every rx queue. 1370 * *pid return the port id which has minimal value of 1371 * max_rxd in all queues of all ports. 1372 */ 1373 static uint16_t 1374 get_allowed_max_nb_rxd(portid_t *pid) 1375 { 1376 uint16_t allowed_max_rxd = UINT16_MAX; 1377 portid_t pi; 1378 struct rte_eth_dev_info dev_info; 1379 1380 RTE_ETH_FOREACH_DEV(pi) { 1381 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1382 continue; 1383 1384 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1385 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1386 *pid = pi; 1387 } 1388 } 1389 return allowed_max_rxd; 1390 } 1391 1392 /* 1393 * Get the allowed minimal number of RXDs of every rx queue. 1394 * *pid return the port id which has minimal value of 1395 * min_rxd in all queues of all ports. 1396 */ 1397 static uint16_t 1398 get_allowed_min_nb_rxd(portid_t *pid) 1399 { 1400 uint16_t allowed_min_rxd = 0; 1401 portid_t pi; 1402 struct rte_eth_dev_info dev_info; 1403 1404 RTE_ETH_FOREACH_DEV(pi) { 1405 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1406 continue; 1407 1408 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1409 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1410 *pid = pi; 1411 } 1412 } 1413 1414 return allowed_min_rxd; 1415 } 1416 1417 /* 1418 * Check input rxd is valid or not. 1419 * If input rxd is not greater than any of maximum number 1420 * of RXDs of every Rx queues and is not less than any of 1421 * minimal number of RXDs of every Rx queues, it is valid. 1422 * if valid, return 0, else return -1 1423 */ 1424 int 1425 check_nb_rxd(queueid_t rxd) 1426 { 1427 uint16_t allowed_max_rxd; 1428 uint16_t allowed_min_rxd; 1429 portid_t pid = 0; 1430 1431 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1432 if (rxd > allowed_max_rxd) { 1433 fprintf(stderr, 1434 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1435 rxd, allowed_max_rxd, pid); 1436 return -1; 1437 } 1438 1439 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1440 if (rxd < allowed_min_rxd) { 1441 fprintf(stderr, 1442 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1443 rxd, allowed_min_rxd, pid); 1444 return -1; 1445 } 1446 1447 return 0; 1448 } 1449 1450 /* 1451 * Get the allowed maximum number of TXDs of every rx queues. 1452 * *pid return the port id which has minimal value of 1453 * max_txd in every tx queue. 1454 */ 1455 static uint16_t 1456 get_allowed_max_nb_txd(portid_t *pid) 1457 { 1458 uint16_t allowed_max_txd = UINT16_MAX; 1459 portid_t pi; 1460 struct rte_eth_dev_info dev_info; 1461 1462 RTE_ETH_FOREACH_DEV(pi) { 1463 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1464 continue; 1465 1466 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1467 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1468 *pid = pi; 1469 } 1470 } 1471 return allowed_max_txd; 1472 } 1473 1474 /* 1475 * Get the allowed maximum number of TXDs of every tx queues. 1476 * *pid return the port id which has minimal value of 1477 * min_txd in every tx queue. 1478 */ 1479 static uint16_t 1480 get_allowed_min_nb_txd(portid_t *pid) 1481 { 1482 uint16_t allowed_min_txd = 0; 1483 portid_t pi; 1484 struct rte_eth_dev_info dev_info; 1485 1486 RTE_ETH_FOREACH_DEV(pi) { 1487 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1488 continue; 1489 1490 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1491 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1492 *pid = pi; 1493 } 1494 } 1495 1496 return allowed_min_txd; 1497 } 1498 1499 /* 1500 * Check input txd is valid or not. 1501 * If input txd is not greater than any of maximum number 1502 * of TXDs of every Rx queues, it is valid. 1503 * if valid, return 0, else return -1 1504 */ 1505 int 1506 check_nb_txd(queueid_t txd) 1507 { 1508 uint16_t allowed_max_txd; 1509 uint16_t allowed_min_txd; 1510 portid_t pid = 0; 1511 1512 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1513 if (txd > allowed_max_txd) { 1514 fprintf(stderr, 1515 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1516 txd, allowed_max_txd, pid); 1517 return -1; 1518 } 1519 1520 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1521 if (txd < allowed_min_txd) { 1522 fprintf(stderr, 1523 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1524 txd, allowed_min_txd, pid); 1525 return -1; 1526 } 1527 return 0; 1528 } 1529 1530 1531 /* 1532 * Get the allowed maximum number of hairpin queues. 1533 * *pid return the port id which has minimal value of 1534 * max_hairpin_queues in all ports. 1535 */ 1536 queueid_t 1537 get_allowed_max_nb_hairpinq(portid_t *pid) 1538 { 1539 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1540 portid_t pi; 1541 struct rte_eth_hairpin_cap cap; 1542 1543 RTE_ETH_FOREACH_DEV(pi) { 1544 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1545 *pid = pi; 1546 return 0; 1547 } 1548 if (cap.max_nb_queues < allowed_max_hairpinq) { 1549 allowed_max_hairpinq = cap.max_nb_queues; 1550 *pid = pi; 1551 } 1552 } 1553 return allowed_max_hairpinq; 1554 } 1555 1556 /* 1557 * Check input hairpin is valid or not. 1558 * If input hairpin is not greater than any of maximum number 1559 * of hairpin queues of all ports, it is valid. 1560 * if valid, return 0, else return -1 1561 */ 1562 int 1563 check_nb_hairpinq(queueid_t hairpinq) 1564 { 1565 queueid_t allowed_max_hairpinq; 1566 portid_t pid = 0; 1567 1568 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1569 if (hairpinq > allowed_max_hairpinq) { 1570 fprintf(stderr, 1571 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1572 hairpinq, allowed_max_hairpinq, pid); 1573 return -1; 1574 } 1575 return 0; 1576 } 1577 1578 static int 1579 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1580 { 1581 uint32_t eth_overhead; 1582 1583 if (dev_info->max_mtu != UINT16_MAX && 1584 dev_info->max_rx_pktlen > dev_info->max_mtu) 1585 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1586 else 1587 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1588 1589 return eth_overhead; 1590 } 1591 1592 static void 1593 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1594 { 1595 struct rte_port *port = &ports[pid]; 1596 int ret; 1597 int i; 1598 1599 eth_rx_metadata_negotiate_mp(pid); 1600 1601 port->dev_conf.txmode = tx_mode; 1602 port->dev_conf.rxmode = rx_mode; 1603 1604 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1605 if (ret != 0) 1606 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1607 1608 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1609 port->dev_conf.txmode.offloads &= 1610 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1611 1612 /* Apply Rx offloads configuration */ 1613 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1614 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads; 1615 /* Apply Tx offloads configuration */ 1616 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1617 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads; 1618 1619 if (eth_link_speed) 1620 port->dev_conf.link_speeds = eth_link_speed; 1621 1622 if (max_rx_pkt_len) 1623 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1624 get_eth_overhead(&port->dev_info); 1625 1626 /* set flag to initialize port/queue */ 1627 port->need_reconfig = 1; 1628 port->need_reconfig_queues = 1; 1629 port->socket_id = socket_id; 1630 port->tx_metadata = 0; 1631 1632 /* 1633 * Check for maximum number of segments per MTU. 1634 * Accordingly update the mbuf data size. 1635 */ 1636 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1637 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1638 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1639 uint16_t mtu; 1640 1641 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1642 uint16_t data_size = (mtu + eth_overhead) / 1643 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1644 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1645 1646 if (buffer_size > mbuf_data_size[0]) { 1647 mbuf_data_size[0] = buffer_size; 1648 TESTPMD_LOG(WARNING, 1649 "Configured mbuf size of the first segment %hu\n", 1650 mbuf_data_size[0]); 1651 } 1652 } 1653 } 1654 } 1655 1656 static void 1657 init_config(void) 1658 { 1659 portid_t pid; 1660 struct rte_mempool *mbp; 1661 unsigned int nb_mbuf_per_pool; 1662 lcoreid_t lc_id; 1663 #ifdef RTE_LIB_GRO 1664 struct rte_gro_param gro_param; 1665 #endif 1666 #ifdef RTE_LIB_GSO 1667 uint32_t gso_types; 1668 #endif 1669 1670 /* Configuration of logical cores. */ 1671 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1672 sizeof(struct fwd_lcore *) * nb_lcores, 1673 RTE_CACHE_LINE_SIZE); 1674 if (fwd_lcores == NULL) { 1675 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1676 "failed\n", nb_lcores); 1677 } 1678 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1679 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1680 sizeof(struct fwd_lcore), 1681 RTE_CACHE_LINE_SIZE); 1682 if (fwd_lcores[lc_id] == NULL) { 1683 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1684 "failed\n"); 1685 } 1686 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1687 } 1688 1689 RTE_ETH_FOREACH_DEV(pid) { 1690 uint32_t socket_id; 1691 1692 if (numa_support) { 1693 socket_id = port_numa[pid]; 1694 if (port_numa[pid] == NUMA_NO_CONFIG) { 1695 socket_id = rte_eth_dev_socket_id(pid); 1696 1697 /* 1698 * if socket_id is invalid, 1699 * set to the first available socket. 1700 */ 1701 if (check_socket_id(socket_id) < 0) 1702 socket_id = socket_ids[0]; 1703 } 1704 } else { 1705 socket_id = (socket_num == UMA_NO_CONFIG) ? 1706 0 : socket_num; 1707 } 1708 /* Apply default TxRx configuration for all ports */ 1709 init_config_port_offloads(pid, socket_id); 1710 } 1711 /* 1712 * Create pools of mbuf. 1713 * If NUMA support is disabled, create a single pool of mbuf in 1714 * socket 0 memory by default. 1715 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1716 * 1717 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1718 * nb_txd can be configured at run time. 1719 */ 1720 if (param_total_num_mbufs) 1721 nb_mbuf_per_pool = param_total_num_mbufs; 1722 else { 1723 nb_mbuf_per_pool = RX_DESC_MAX + 1724 (nb_lcores * mb_mempool_cache) + 1725 TX_DESC_MAX + MAX_PKT_BURST; 1726 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1727 } 1728 1729 if (numa_support) { 1730 uint8_t i, j; 1731 1732 for (i = 0; i < num_sockets; i++) 1733 for (j = 0; j < mbuf_data_size_n; j++) 1734 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1735 mbuf_pool_create(mbuf_data_size[j], 1736 nb_mbuf_per_pool, 1737 socket_ids[i], j); 1738 } else { 1739 uint8_t i; 1740 1741 for (i = 0; i < mbuf_data_size_n; i++) 1742 mempools[i] = mbuf_pool_create 1743 (mbuf_data_size[i], 1744 nb_mbuf_per_pool, 1745 socket_num == UMA_NO_CONFIG ? 1746 0 : socket_num, i); 1747 } 1748 1749 init_port_config(); 1750 1751 #ifdef RTE_LIB_GSO 1752 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1753 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1754 #endif 1755 /* 1756 * Records which Mbuf pool to use by each logical core, if needed. 1757 */ 1758 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1759 mbp = mbuf_pool_find( 1760 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1761 1762 if (mbp == NULL) 1763 mbp = mbuf_pool_find(0, 0); 1764 fwd_lcores[lc_id]->mbp = mbp; 1765 #ifdef RTE_LIB_GSO 1766 /* initialize GSO context */ 1767 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1768 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1769 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1770 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1771 RTE_ETHER_CRC_LEN; 1772 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1773 #endif 1774 } 1775 1776 fwd_config_setup(); 1777 1778 #ifdef RTE_LIB_GRO 1779 /* create a gro context for each lcore */ 1780 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1781 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1782 gro_param.max_item_per_flow = MAX_PKT_BURST; 1783 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1784 gro_param.socket_id = rte_lcore_to_socket_id( 1785 fwd_lcores_cpuids[lc_id]); 1786 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1787 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1788 rte_exit(EXIT_FAILURE, 1789 "rte_gro_ctx_create() failed\n"); 1790 } 1791 } 1792 #endif 1793 } 1794 1795 1796 void 1797 reconfig(portid_t new_port_id, unsigned socket_id) 1798 { 1799 /* Reconfiguration of Ethernet ports. */ 1800 init_config_port_offloads(new_port_id, socket_id); 1801 init_port_config(); 1802 } 1803 1804 int 1805 init_fwd_streams(void) 1806 { 1807 portid_t pid; 1808 struct rte_port *port; 1809 streamid_t sm_id, nb_fwd_streams_new; 1810 queueid_t q; 1811 1812 /* set socket id according to numa or not */ 1813 RTE_ETH_FOREACH_DEV(pid) { 1814 port = &ports[pid]; 1815 if (nb_rxq > port->dev_info.max_rx_queues) { 1816 fprintf(stderr, 1817 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1818 nb_rxq, port->dev_info.max_rx_queues); 1819 return -1; 1820 } 1821 if (nb_txq > port->dev_info.max_tx_queues) { 1822 fprintf(stderr, 1823 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1824 nb_txq, port->dev_info.max_tx_queues); 1825 return -1; 1826 } 1827 if (numa_support) { 1828 if (port_numa[pid] != NUMA_NO_CONFIG) 1829 port->socket_id = port_numa[pid]; 1830 else { 1831 port->socket_id = rte_eth_dev_socket_id(pid); 1832 1833 /* 1834 * if socket_id is invalid, 1835 * set to the first available socket. 1836 */ 1837 if (check_socket_id(port->socket_id) < 0) 1838 port->socket_id = socket_ids[0]; 1839 } 1840 } 1841 else { 1842 if (socket_num == UMA_NO_CONFIG) 1843 port->socket_id = 0; 1844 else 1845 port->socket_id = socket_num; 1846 } 1847 } 1848 1849 q = RTE_MAX(nb_rxq, nb_txq); 1850 if (q == 0) { 1851 fprintf(stderr, 1852 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1853 return -1; 1854 } 1855 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1856 if (nb_fwd_streams_new == nb_fwd_streams) 1857 return 0; 1858 /* clear the old */ 1859 if (fwd_streams != NULL) { 1860 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1861 if (fwd_streams[sm_id] == NULL) 1862 continue; 1863 rte_free(fwd_streams[sm_id]); 1864 fwd_streams[sm_id] = NULL; 1865 } 1866 rte_free(fwd_streams); 1867 fwd_streams = NULL; 1868 } 1869 1870 /* init new */ 1871 nb_fwd_streams = nb_fwd_streams_new; 1872 if (nb_fwd_streams) { 1873 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1874 sizeof(struct fwd_stream *) * nb_fwd_streams, 1875 RTE_CACHE_LINE_SIZE); 1876 if (fwd_streams == NULL) 1877 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1878 " (struct fwd_stream *)) failed\n", 1879 nb_fwd_streams); 1880 1881 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1882 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1883 " struct fwd_stream", sizeof(struct fwd_stream), 1884 RTE_CACHE_LINE_SIZE); 1885 if (fwd_streams[sm_id] == NULL) 1886 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1887 "(struct fwd_stream) failed\n"); 1888 } 1889 } 1890 1891 return 0; 1892 } 1893 1894 static void 1895 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1896 { 1897 uint64_t total_burst, sburst; 1898 uint64_t nb_burst; 1899 uint64_t burst_stats[4]; 1900 uint16_t pktnb_stats[4]; 1901 uint16_t nb_pkt; 1902 int burst_percent[4], sburstp; 1903 int i; 1904 1905 /* 1906 * First compute the total number of packet bursts and the 1907 * two highest numbers of bursts of the same number of packets. 1908 */ 1909 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1910 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1911 1912 /* Show stats for 0 burst size always */ 1913 total_burst = pbs->pkt_burst_spread[0]; 1914 burst_stats[0] = pbs->pkt_burst_spread[0]; 1915 pktnb_stats[0] = 0; 1916 1917 /* Find the next 2 burst sizes with highest occurrences. */ 1918 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1919 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1920 1921 if (nb_burst == 0) 1922 continue; 1923 1924 total_burst += nb_burst; 1925 1926 if (nb_burst > burst_stats[1]) { 1927 burst_stats[2] = burst_stats[1]; 1928 pktnb_stats[2] = pktnb_stats[1]; 1929 burst_stats[1] = nb_burst; 1930 pktnb_stats[1] = nb_pkt; 1931 } else if (nb_burst > burst_stats[2]) { 1932 burst_stats[2] = nb_burst; 1933 pktnb_stats[2] = nb_pkt; 1934 } 1935 } 1936 if (total_burst == 0) 1937 return; 1938 1939 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1940 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1941 if (i == 3) { 1942 printf("%d%% of other]\n", 100 - sburstp); 1943 return; 1944 } 1945 1946 sburst += burst_stats[i]; 1947 if (sburst == total_burst) { 1948 printf("%d%% of %d pkts]\n", 1949 100 - sburstp, (int) pktnb_stats[i]); 1950 return; 1951 } 1952 1953 burst_percent[i] = 1954 (double)burst_stats[i] / total_burst * 100; 1955 printf("%d%% of %d pkts + ", 1956 burst_percent[i], (int) pktnb_stats[i]); 1957 sburstp += burst_percent[i]; 1958 } 1959 } 1960 1961 static void 1962 fwd_stream_stats_display(streamid_t stream_id) 1963 { 1964 struct fwd_stream *fs; 1965 static const char *fwd_top_stats_border = "-------"; 1966 1967 fs = fwd_streams[stream_id]; 1968 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1969 (fs->fwd_dropped == 0)) 1970 return; 1971 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1972 "TX Port=%2d/Queue=%2d %s\n", 1973 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1974 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1975 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1976 " TX-dropped: %-14"PRIu64, 1977 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1978 1979 /* if checksum mode */ 1980 if (cur_fwd_eng == &csum_fwd_engine) { 1981 printf(" RX- bad IP checksum: %-14"PRIu64 1982 " Rx- bad L4 checksum: %-14"PRIu64 1983 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1984 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1985 fs->rx_bad_outer_l4_csum); 1986 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1987 fs->rx_bad_outer_ip_csum); 1988 } else { 1989 printf("\n"); 1990 } 1991 1992 if (record_burst_stats) { 1993 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1994 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1995 } 1996 } 1997 1998 void 1999 fwd_stats_display(void) 2000 { 2001 static const char *fwd_stats_border = "----------------------"; 2002 static const char *acc_stats_border = "+++++++++++++++"; 2003 struct { 2004 struct fwd_stream *rx_stream; 2005 struct fwd_stream *tx_stream; 2006 uint64_t tx_dropped; 2007 uint64_t rx_bad_ip_csum; 2008 uint64_t rx_bad_l4_csum; 2009 uint64_t rx_bad_outer_l4_csum; 2010 uint64_t rx_bad_outer_ip_csum; 2011 } ports_stats[RTE_MAX_ETHPORTS]; 2012 uint64_t total_rx_dropped = 0; 2013 uint64_t total_tx_dropped = 0; 2014 uint64_t total_rx_nombuf = 0; 2015 struct rte_eth_stats stats; 2016 uint64_t fwd_cycles = 0; 2017 uint64_t total_recv = 0; 2018 uint64_t total_xmit = 0; 2019 struct rte_port *port; 2020 streamid_t sm_id; 2021 portid_t pt_id; 2022 int ret; 2023 int i; 2024 2025 memset(ports_stats, 0, sizeof(ports_stats)); 2026 2027 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2028 struct fwd_stream *fs = fwd_streams[sm_id]; 2029 2030 if (cur_fwd_config.nb_fwd_streams > 2031 cur_fwd_config.nb_fwd_ports) { 2032 fwd_stream_stats_display(sm_id); 2033 } else { 2034 ports_stats[fs->tx_port].tx_stream = fs; 2035 ports_stats[fs->rx_port].rx_stream = fs; 2036 } 2037 2038 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2039 2040 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2041 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2042 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2043 fs->rx_bad_outer_l4_csum; 2044 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2045 fs->rx_bad_outer_ip_csum; 2046 2047 if (record_core_cycles) 2048 fwd_cycles += fs->core_cycles; 2049 } 2050 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2051 pt_id = fwd_ports_ids[i]; 2052 port = &ports[pt_id]; 2053 2054 ret = rte_eth_stats_get(pt_id, &stats); 2055 if (ret != 0) { 2056 fprintf(stderr, 2057 "%s: Error: failed to get stats (port %u): %d", 2058 __func__, pt_id, ret); 2059 continue; 2060 } 2061 stats.ipackets -= port->stats.ipackets; 2062 stats.opackets -= port->stats.opackets; 2063 stats.ibytes -= port->stats.ibytes; 2064 stats.obytes -= port->stats.obytes; 2065 stats.imissed -= port->stats.imissed; 2066 stats.oerrors -= port->stats.oerrors; 2067 stats.rx_nombuf -= port->stats.rx_nombuf; 2068 2069 total_recv += stats.ipackets; 2070 total_xmit += stats.opackets; 2071 total_rx_dropped += stats.imissed; 2072 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2073 total_tx_dropped += stats.oerrors; 2074 total_rx_nombuf += stats.rx_nombuf; 2075 2076 printf("\n %s Forward statistics for port %-2d %s\n", 2077 fwd_stats_border, pt_id, fwd_stats_border); 2078 2079 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2080 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2081 stats.ipackets + stats.imissed); 2082 2083 if (cur_fwd_eng == &csum_fwd_engine) { 2084 printf(" Bad-ipcsum: %-14"PRIu64 2085 " Bad-l4csum: %-14"PRIu64 2086 "Bad-outer-l4csum: %-14"PRIu64"\n", 2087 ports_stats[pt_id].rx_bad_ip_csum, 2088 ports_stats[pt_id].rx_bad_l4_csum, 2089 ports_stats[pt_id].rx_bad_outer_l4_csum); 2090 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2091 ports_stats[pt_id].rx_bad_outer_ip_csum); 2092 } 2093 if (stats.ierrors + stats.rx_nombuf > 0) { 2094 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2095 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2096 } 2097 2098 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2099 "TX-total: %-"PRIu64"\n", 2100 stats.opackets, ports_stats[pt_id].tx_dropped, 2101 stats.opackets + ports_stats[pt_id].tx_dropped); 2102 2103 if (record_burst_stats) { 2104 if (ports_stats[pt_id].rx_stream) 2105 pkt_burst_stats_display("RX", 2106 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2107 if (ports_stats[pt_id].tx_stream) 2108 pkt_burst_stats_display("TX", 2109 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2110 } 2111 2112 printf(" %s--------------------------------%s\n", 2113 fwd_stats_border, fwd_stats_border); 2114 } 2115 2116 printf("\n %s Accumulated forward statistics for all ports" 2117 "%s\n", 2118 acc_stats_border, acc_stats_border); 2119 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2120 "%-"PRIu64"\n" 2121 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2122 "%-"PRIu64"\n", 2123 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2124 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2125 if (total_rx_nombuf > 0) 2126 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2127 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2128 "%s\n", 2129 acc_stats_border, acc_stats_border); 2130 if (record_core_cycles) { 2131 #define CYC_PER_MHZ 1E6 2132 if (total_recv > 0 || total_xmit > 0) { 2133 uint64_t total_pkts = 0; 2134 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2135 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2136 total_pkts = total_xmit; 2137 else 2138 total_pkts = total_recv; 2139 2140 printf("\n CPU cycles/packet=%.2F (total cycles=" 2141 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2142 " MHz Clock\n", 2143 (double) fwd_cycles / total_pkts, 2144 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2145 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2146 } 2147 } 2148 } 2149 2150 void 2151 fwd_stats_reset(void) 2152 { 2153 streamid_t sm_id; 2154 portid_t pt_id; 2155 int ret; 2156 int i; 2157 2158 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2159 pt_id = fwd_ports_ids[i]; 2160 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2161 if (ret != 0) 2162 fprintf(stderr, 2163 "%s: Error: failed to clear stats (port %u):%d", 2164 __func__, pt_id, ret); 2165 } 2166 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2167 struct fwd_stream *fs = fwd_streams[sm_id]; 2168 2169 fs->rx_packets = 0; 2170 fs->tx_packets = 0; 2171 fs->fwd_dropped = 0; 2172 fs->rx_bad_ip_csum = 0; 2173 fs->rx_bad_l4_csum = 0; 2174 fs->rx_bad_outer_l4_csum = 0; 2175 fs->rx_bad_outer_ip_csum = 0; 2176 2177 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2178 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2179 fs->core_cycles = 0; 2180 } 2181 } 2182 2183 static void 2184 flush_fwd_rx_queues(void) 2185 { 2186 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2187 portid_t rxp; 2188 portid_t port_id; 2189 queueid_t rxq; 2190 uint16_t nb_rx; 2191 uint16_t i; 2192 uint8_t j; 2193 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2194 uint64_t timer_period; 2195 2196 if (num_procs > 1) { 2197 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2198 return; 2199 } 2200 2201 /* convert to number of cycles */ 2202 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2203 2204 for (j = 0; j < 2; j++) { 2205 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2206 for (rxq = 0; rxq < nb_rxq; rxq++) { 2207 port_id = fwd_ports_ids[rxp]; 2208 2209 /* Polling stopped queues is prohibited. */ 2210 if (ports[port_id].rxq[rxq].state == 2211 RTE_ETH_QUEUE_STATE_STOPPED) 2212 continue; 2213 2214 /** 2215 * testpmd can stuck in the below do while loop 2216 * if rte_eth_rx_burst() always returns nonzero 2217 * packets. So timer is added to exit this loop 2218 * after 1sec timer expiry. 2219 */ 2220 prev_tsc = rte_rdtsc(); 2221 do { 2222 nb_rx = rte_eth_rx_burst(port_id, rxq, 2223 pkts_burst, MAX_PKT_BURST); 2224 for (i = 0; i < nb_rx; i++) 2225 rte_pktmbuf_free(pkts_burst[i]); 2226 2227 cur_tsc = rte_rdtsc(); 2228 diff_tsc = cur_tsc - prev_tsc; 2229 timer_tsc += diff_tsc; 2230 } while ((nb_rx > 0) && 2231 (timer_tsc < timer_period)); 2232 timer_tsc = 0; 2233 } 2234 } 2235 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2236 } 2237 } 2238 2239 static void 2240 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2241 { 2242 struct fwd_stream **fsm; 2243 streamid_t nb_fs; 2244 streamid_t sm_id; 2245 #ifdef RTE_LIB_BITRATESTATS 2246 uint64_t tics_per_1sec; 2247 uint64_t tics_datum; 2248 uint64_t tics_current; 2249 uint16_t i, cnt_ports; 2250 2251 cnt_ports = nb_ports; 2252 tics_datum = rte_rdtsc(); 2253 tics_per_1sec = rte_get_timer_hz(); 2254 #endif 2255 fsm = &fwd_streams[fc->stream_idx]; 2256 nb_fs = fc->stream_nb; 2257 do { 2258 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2259 if (!fsm[sm_id]->disabled) 2260 (*pkt_fwd)(fsm[sm_id]); 2261 #ifdef RTE_LIB_BITRATESTATS 2262 if (bitrate_enabled != 0 && 2263 bitrate_lcore_id == rte_lcore_id()) { 2264 tics_current = rte_rdtsc(); 2265 if (tics_current - tics_datum >= tics_per_1sec) { 2266 /* Periodic bitrate calculation */ 2267 for (i = 0; i < cnt_ports; i++) 2268 rte_stats_bitrate_calc(bitrate_data, 2269 ports_ids[i]); 2270 tics_datum = tics_current; 2271 } 2272 } 2273 #endif 2274 #ifdef RTE_LIB_LATENCYSTATS 2275 if (latencystats_enabled != 0 && 2276 latencystats_lcore_id == rte_lcore_id()) 2277 rte_latencystats_update(); 2278 #endif 2279 2280 } while (! fc->stopped); 2281 } 2282 2283 static int 2284 start_pkt_forward_on_core(void *fwd_arg) 2285 { 2286 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2287 cur_fwd_config.fwd_eng->packet_fwd); 2288 return 0; 2289 } 2290 2291 /* 2292 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2293 * Used to start communication flows in network loopback test configurations. 2294 */ 2295 static int 2296 run_one_txonly_burst_on_core(void *fwd_arg) 2297 { 2298 struct fwd_lcore *fwd_lc; 2299 struct fwd_lcore tmp_lcore; 2300 2301 fwd_lc = (struct fwd_lcore *) fwd_arg; 2302 tmp_lcore = *fwd_lc; 2303 tmp_lcore.stopped = 1; 2304 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2305 return 0; 2306 } 2307 2308 /* 2309 * Launch packet forwarding: 2310 * - Setup per-port forwarding context. 2311 * - launch logical cores with their forwarding configuration. 2312 */ 2313 static void 2314 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2315 { 2316 unsigned int i; 2317 unsigned int lc_id; 2318 int diag; 2319 2320 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2321 lc_id = fwd_lcores_cpuids[i]; 2322 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2323 fwd_lcores[i]->stopped = 0; 2324 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2325 fwd_lcores[i], lc_id); 2326 if (diag != 0) 2327 fprintf(stderr, 2328 "launch lcore %u failed - diag=%d\n", 2329 lc_id, diag); 2330 } 2331 } 2332 } 2333 2334 /* 2335 * Launch packet forwarding configuration. 2336 */ 2337 void 2338 start_packet_forwarding(int with_tx_first) 2339 { 2340 port_fwd_begin_t port_fwd_begin; 2341 port_fwd_end_t port_fwd_end; 2342 stream_init_t stream_init = cur_fwd_eng->stream_init; 2343 unsigned int i; 2344 2345 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2346 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2347 2348 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2349 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2350 2351 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2352 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2353 (!nb_rxq || !nb_txq)) 2354 rte_exit(EXIT_FAILURE, 2355 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2356 cur_fwd_eng->fwd_mode_name); 2357 2358 if (all_ports_started() == 0) { 2359 fprintf(stderr, "Not all ports were started\n"); 2360 return; 2361 } 2362 if (test_done == 0) { 2363 fprintf(stderr, "Packet forwarding already started\n"); 2364 return; 2365 } 2366 2367 fwd_config_setup(); 2368 2369 pkt_fwd_config_display(&cur_fwd_config); 2370 if (!pkt_fwd_shared_rxq_check()) 2371 return; 2372 2373 if (stream_init != NULL) 2374 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) 2375 stream_init(fwd_streams[i]); 2376 2377 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2378 if (port_fwd_begin != NULL) { 2379 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2380 if (port_fwd_begin(fwd_ports_ids[i])) { 2381 fprintf(stderr, 2382 "Packet forwarding is not ready\n"); 2383 return; 2384 } 2385 } 2386 } 2387 2388 if (with_tx_first) { 2389 port_fwd_begin = tx_only_engine.port_fwd_begin; 2390 if (port_fwd_begin != NULL) { 2391 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2392 if (port_fwd_begin(fwd_ports_ids[i])) { 2393 fprintf(stderr, 2394 "Packet forwarding is not ready\n"); 2395 return; 2396 } 2397 } 2398 } 2399 } 2400 2401 test_done = 0; 2402 2403 if(!no_flush_rx) 2404 flush_fwd_rx_queues(); 2405 2406 rxtx_config_display(); 2407 2408 fwd_stats_reset(); 2409 if (with_tx_first) { 2410 while (with_tx_first--) { 2411 launch_packet_forwarding( 2412 run_one_txonly_burst_on_core); 2413 rte_eal_mp_wait_lcore(); 2414 } 2415 port_fwd_end = tx_only_engine.port_fwd_end; 2416 if (port_fwd_end != NULL) { 2417 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2418 (*port_fwd_end)(fwd_ports_ids[i]); 2419 } 2420 } 2421 launch_packet_forwarding(start_pkt_forward_on_core); 2422 } 2423 2424 void 2425 stop_packet_forwarding(void) 2426 { 2427 port_fwd_end_t port_fwd_end; 2428 lcoreid_t lc_id; 2429 portid_t pt_id; 2430 int i; 2431 2432 if (test_done) { 2433 fprintf(stderr, "Packet forwarding not started\n"); 2434 return; 2435 } 2436 printf("Telling cores to stop..."); 2437 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2438 fwd_lcores[lc_id]->stopped = 1; 2439 printf("\nWaiting for lcores to finish...\n"); 2440 rte_eal_mp_wait_lcore(); 2441 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2442 if (port_fwd_end != NULL) { 2443 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2444 pt_id = fwd_ports_ids[i]; 2445 (*port_fwd_end)(pt_id); 2446 } 2447 } 2448 2449 fwd_stats_display(); 2450 2451 printf("\nDone.\n"); 2452 test_done = 1; 2453 } 2454 2455 void 2456 dev_set_link_up(portid_t pid) 2457 { 2458 if (rte_eth_dev_set_link_up(pid) < 0) 2459 fprintf(stderr, "\nSet link up fail.\n"); 2460 } 2461 2462 void 2463 dev_set_link_down(portid_t pid) 2464 { 2465 if (rte_eth_dev_set_link_down(pid) < 0) 2466 fprintf(stderr, "\nSet link down fail.\n"); 2467 } 2468 2469 static int 2470 all_ports_started(void) 2471 { 2472 portid_t pi; 2473 struct rte_port *port; 2474 2475 RTE_ETH_FOREACH_DEV(pi) { 2476 port = &ports[pi]; 2477 /* Check if there is a port which is not started */ 2478 if ((port->port_status != RTE_PORT_STARTED) && 2479 (port->slave_flag == 0)) 2480 return 0; 2481 } 2482 2483 /* No port is not started */ 2484 return 1; 2485 } 2486 2487 int 2488 port_is_stopped(portid_t port_id) 2489 { 2490 struct rte_port *port = &ports[port_id]; 2491 2492 if ((port->port_status != RTE_PORT_STOPPED) && 2493 (port->slave_flag == 0)) 2494 return 0; 2495 return 1; 2496 } 2497 2498 int 2499 all_ports_stopped(void) 2500 { 2501 portid_t pi; 2502 2503 RTE_ETH_FOREACH_DEV(pi) { 2504 if (!port_is_stopped(pi)) 2505 return 0; 2506 } 2507 2508 return 1; 2509 } 2510 2511 int 2512 port_is_started(portid_t port_id) 2513 { 2514 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2515 return 0; 2516 2517 if (ports[port_id].port_status != RTE_PORT_STARTED) 2518 return 0; 2519 2520 return 1; 2521 } 2522 2523 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 2524 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 2525 2526 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 2527 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 2528 2529 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 2530 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 2531 2532 2533 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2534 static int 2535 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2536 { 2537 queueid_t qi; 2538 struct rte_eth_hairpin_conf hairpin_conf = { 2539 .peer_count = 1, 2540 }; 2541 int i; 2542 int diag; 2543 struct rte_port *port = &ports[pi]; 2544 uint16_t peer_rx_port = pi; 2545 uint16_t peer_tx_port = pi; 2546 uint32_t manual = 1; 2547 uint32_t tx_exp = hairpin_mode & 0x10; 2548 uint32_t rx_force_memory = hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY; 2549 uint32_t rx_locked_memory = hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY; 2550 uint32_t rx_rte_memory = hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY; 2551 uint32_t tx_force_memory = hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY; 2552 uint32_t tx_locked_memory = hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY; 2553 uint32_t tx_rte_memory = hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY; 2554 2555 if (!(hairpin_mode & 0xf)) { 2556 peer_rx_port = pi; 2557 peer_tx_port = pi; 2558 manual = 0; 2559 } else if (hairpin_mode & 0x1) { 2560 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2561 RTE_ETH_DEV_NO_OWNER); 2562 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2563 peer_tx_port = rte_eth_find_next_owned_by(0, 2564 RTE_ETH_DEV_NO_OWNER); 2565 if (p_pi != RTE_MAX_ETHPORTS) { 2566 peer_rx_port = p_pi; 2567 } else { 2568 uint16_t next_pi; 2569 2570 /* Last port will be the peer RX port of the first. */ 2571 RTE_ETH_FOREACH_DEV(next_pi) 2572 peer_rx_port = next_pi; 2573 } 2574 manual = 1; 2575 } else if (hairpin_mode & 0x2) { 2576 if (cnt_pi & 0x1) { 2577 peer_rx_port = p_pi; 2578 } else { 2579 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2580 RTE_ETH_DEV_NO_OWNER); 2581 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2582 peer_rx_port = pi; 2583 } 2584 peer_tx_port = peer_rx_port; 2585 manual = 1; 2586 } 2587 2588 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2589 hairpin_conf.peers[0].port = peer_rx_port; 2590 hairpin_conf.peers[0].queue = i + nb_rxq; 2591 hairpin_conf.manual_bind = !!manual; 2592 hairpin_conf.tx_explicit = !!tx_exp; 2593 hairpin_conf.force_memory = !!tx_force_memory; 2594 hairpin_conf.use_locked_device_memory = !!tx_locked_memory; 2595 hairpin_conf.use_rte_memory = !!tx_rte_memory; 2596 diag = rte_eth_tx_hairpin_queue_setup 2597 (pi, qi, nb_txd, &hairpin_conf); 2598 i++; 2599 if (diag == 0) 2600 continue; 2601 2602 /* Fail to setup rx queue, return */ 2603 if (port->port_status == RTE_PORT_HANDLING) 2604 port->port_status = RTE_PORT_STOPPED; 2605 else 2606 fprintf(stderr, 2607 "Port %d can not be set back to stopped\n", pi); 2608 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2609 pi); 2610 /* try to reconfigure queues next time */ 2611 port->need_reconfig_queues = 1; 2612 return -1; 2613 } 2614 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2615 hairpin_conf.peers[0].port = peer_tx_port; 2616 hairpin_conf.peers[0].queue = i + nb_txq; 2617 hairpin_conf.manual_bind = !!manual; 2618 hairpin_conf.tx_explicit = !!tx_exp; 2619 hairpin_conf.force_memory = !!rx_force_memory; 2620 hairpin_conf.use_locked_device_memory = !!rx_locked_memory; 2621 hairpin_conf.use_rte_memory = !!rx_rte_memory; 2622 diag = rte_eth_rx_hairpin_queue_setup 2623 (pi, qi, nb_rxd, &hairpin_conf); 2624 i++; 2625 if (diag == 0) 2626 continue; 2627 2628 /* Fail to setup rx queue, return */ 2629 if (port->port_status == RTE_PORT_HANDLING) 2630 port->port_status = RTE_PORT_STOPPED; 2631 else 2632 fprintf(stderr, 2633 "Port %d can not be set back to stopped\n", pi); 2634 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2635 pi); 2636 /* try to reconfigure queues next time */ 2637 port->need_reconfig_queues = 1; 2638 return -1; 2639 } 2640 return 0; 2641 } 2642 2643 /* Configure the Rx with optional split. */ 2644 int 2645 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2646 uint16_t nb_rx_desc, unsigned int socket_id, 2647 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2648 { 2649 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2650 unsigned int i, mp_n; 2651 int ret; 2652 2653 if (rx_pkt_nb_segs <= 1 || 2654 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2655 rx_conf->rx_seg = NULL; 2656 rx_conf->rx_nseg = 0; 2657 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2658 nb_rx_desc, socket_id, 2659 rx_conf, mp); 2660 goto exit; 2661 } 2662 for (i = 0; i < rx_pkt_nb_segs; i++) { 2663 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2664 struct rte_mempool *mpx; 2665 /* 2666 * Use last valid pool for the segments with number 2667 * exceeding the pool index. 2668 */ 2669 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2670 mpx = mbuf_pool_find(socket_id, mp_n); 2671 /* Handle zero as mbuf data buffer size. */ 2672 rx_seg->offset = i < rx_pkt_nb_offs ? 2673 rx_pkt_seg_offsets[i] : 0; 2674 rx_seg->mp = mpx ? mpx : mp; 2675 if (rx_pkt_hdr_protos[i] != 0 && rx_pkt_seg_lengths[i] == 0) { 2676 rx_seg->proto_hdr = rx_pkt_hdr_protos[i]; 2677 } else { 2678 rx_seg->length = rx_pkt_seg_lengths[i] ? 2679 rx_pkt_seg_lengths[i] : 2680 mbuf_data_size[mp_n]; 2681 } 2682 } 2683 rx_conf->rx_nseg = rx_pkt_nb_segs; 2684 rx_conf->rx_seg = rx_useg; 2685 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2686 socket_id, rx_conf, NULL); 2687 rx_conf->rx_seg = NULL; 2688 rx_conf->rx_nseg = 0; 2689 exit: 2690 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ? 2691 RTE_ETH_QUEUE_STATE_STOPPED : 2692 RTE_ETH_QUEUE_STATE_STARTED; 2693 return ret; 2694 } 2695 2696 static int 2697 alloc_xstats_display_info(portid_t pi) 2698 { 2699 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2700 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2701 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2702 2703 if (xstats_display_num == 0) 2704 return 0; 2705 2706 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2707 if (*ids_supp == NULL) 2708 goto fail_ids_supp; 2709 2710 *prev_values = calloc(xstats_display_num, 2711 sizeof(**prev_values)); 2712 if (*prev_values == NULL) 2713 goto fail_prev_values; 2714 2715 *curr_values = calloc(xstats_display_num, 2716 sizeof(**curr_values)); 2717 if (*curr_values == NULL) 2718 goto fail_curr_values; 2719 2720 ports[pi].xstats_info.allocated = true; 2721 2722 return 0; 2723 2724 fail_curr_values: 2725 free(*prev_values); 2726 fail_prev_values: 2727 free(*ids_supp); 2728 fail_ids_supp: 2729 return -ENOMEM; 2730 } 2731 2732 static void 2733 free_xstats_display_info(portid_t pi) 2734 { 2735 if (!ports[pi].xstats_info.allocated) 2736 return; 2737 free(ports[pi].xstats_info.ids_supp); 2738 free(ports[pi].xstats_info.prev_values); 2739 free(ports[pi].xstats_info.curr_values); 2740 ports[pi].xstats_info.allocated = false; 2741 } 2742 2743 /** Fill helper structures for specified port to show extended statistics. */ 2744 static void 2745 fill_xstats_display_info_for_port(portid_t pi) 2746 { 2747 unsigned int stat, stat_supp; 2748 const char *xstat_name; 2749 struct rte_port *port; 2750 uint64_t *ids_supp; 2751 int rc; 2752 2753 if (xstats_display_num == 0) 2754 return; 2755 2756 if (pi == (portid_t)RTE_PORT_ALL) { 2757 fill_xstats_display_info(); 2758 return; 2759 } 2760 2761 port = &ports[pi]; 2762 if (port->port_status != RTE_PORT_STARTED) 2763 return; 2764 2765 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2766 rte_exit(EXIT_FAILURE, 2767 "Failed to allocate xstats display memory\n"); 2768 2769 ids_supp = port->xstats_info.ids_supp; 2770 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2771 xstat_name = xstats_display[stat].name; 2772 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2773 ids_supp + stat_supp); 2774 if (rc != 0) { 2775 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2776 xstat_name, pi, stat); 2777 continue; 2778 } 2779 stat_supp++; 2780 } 2781 2782 port->xstats_info.ids_supp_sz = stat_supp; 2783 } 2784 2785 /** Fill helper structures for all ports to show extended statistics. */ 2786 static void 2787 fill_xstats_display_info(void) 2788 { 2789 portid_t pi; 2790 2791 if (xstats_display_num == 0) 2792 return; 2793 2794 RTE_ETH_FOREACH_DEV(pi) 2795 fill_xstats_display_info_for_port(pi); 2796 } 2797 2798 int 2799 start_port(portid_t pid) 2800 { 2801 int diag, need_check_link_status = -1; 2802 portid_t pi; 2803 portid_t p_pi = RTE_MAX_ETHPORTS; 2804 portid_t pl[RTE_MAX_ETHPORTS]; 2805 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2806 uint16_t cnt_pi = 0; 2807 uint16_t cfg_pi = 0; 2808 int peer_pi; 2809 queueid_t qi; 2810 struct rte_port *port; 2811 struct rte_eth_hairpin_cap cap; 2812 2813 if (port_id_is_invalid(pid, ENABLED_WARN)) 2814 return 0; 2815 2816 RTE_ETH_FOREACH_DEV(pi) { 2817 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2818 continue; 2819 2820 if (port_is_bonding_slave(pi)) { 2821 fprintf(stderr, 2822 "Please remove port %d from bonded device.\n", 2823 pi); 2824 continue; 2825 } 2826 2827 need_check_link_status = 0; 2828 port = &ports[pi]; 2829 if (port->port_status == RTE_PORT_STOPPED) 2830 port->port_status = RTE_PORT_HANDLING; 2831 else { 2832 fprintf(stderr, "Port %d is now not stopped\n", pi); 2833 continue; 2834 } 2835 2836 if (port->need_reconfig > 0) { 2837 struct rte_eth_conf dev_conf; 2838 int k; 2839 2840 port->need_reconfig = 0; 2841 2842 if (flow_isolate_all) { 2843 int ret = port_flow_isolate(pi, 1); 2844 if (ret) { 2845 fprintf(stderr, 2846 "Failed to apply isolated mode on port %d\n", 2847 pi); 2848 return -1; 2849 } 2850 } 2851 configure_rxtx_dump_callbacks(0); 2852 printf("Configuring Port %d (socket %u)\n", pi, 2853 port->socket_id); 2854 if (nb_hairpinq > 0 && 2855 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2856 fprintf(stderr, 2857 "Port %d doesn't support hairpin queues\n", 2858 pi); 2859 return -1; 2860 } 2861 2862 /* configure port */ 2863 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2864 nb_txq + nb_hairpinq, 2865 &(port->dev_conf)); 2866 if (diag != 0) { 2867 if (port->port_status == RTE_PORT_HANDLING) 2868 port->port_status = RTE_PORT_STOPPED; 2869 else 2870 fprintf(stderr, 2871 "Port %d can not be set back to stopped\n", 2872 pi); 2873 fprintf(stderr, "Fail to configure port %d\n", 2874 pi); 2875 /* try to reconfigure port next time */ 2876 port->need_reconfig = 1; 2877 return -1; 2878 } 2879 /* get device configuration*/ 2880 if (0 != 2881 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2882 fprintf(stderr, 2883 "port %d can not get device configuration\n", 2884 pi); 2885 return -1; 2886 } 2887 /* Apply Rx offloads configuration */ 2888 if (dev_conf.rxmode.offloads != 2889 port->dev_conf.rxmode.offloads) { 2890 port->dev_conf.rxmode.offloads |= 2891 dev_conf.rxmode.offloads; 2892 for (k = 0; 2893 k < port->dev_info.max_rx_queues; 2894 k++) 2895 port->rxq[k].conf.offloads |= 2896 dev_conf.rxmode.offloads; 2897 } 2898 /* Apply Tx offloads configuration */ 2899 if (dev_conf.txmode.offloads != 2900 port->dev_conf.txmode.offloads) { 2901 port->dev_conf.txmode.offloads |= 2902 dev_conf.txmode.offloads; 2903 for (k = 0; 2904 k < port->dev_info.max_tx_queues; 2905 k++) 2906 port->txq[k].conf.offloads |= 2907 dev_conf.txmode.offloads; 2908 } 2909 } 2910 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2911 port->need_reconfig_queues = 0; 2912 /* setup tx queues */ 2913 for (qi = 0; qi < nb_txq; qi++) { 2914 struct rte_eth_txconf *conf = 2915 &port->txq[qi].conf; 2916 2917 if ((numa_support) && 2918 (txring_numa[pi] != NUMA_NO_CONFIG)) 2919 diag = rte_eth_tx_queue_setup(pi, qi, 2920 port->nb_tx_desc[qi], 2921 txring_numa[pi], 2922 &(port->txq[qi].conf)); 2923 else 2924 diag = rte_eth_tx_queue_setup(pi, qi, 2925 port->nb_tx_desc[qi], 2926 port->socket_id, 2927 &(port->txq[qi].conf)); 2928 2929 if (diag == 0) { 2930 port->txq[qi].state = 2931 conf->tx_deferred_start ? 2932 RTE_ETH_QUEUE_STATE_STOPPED : 2933 RTE_ETH_QUEUE_STATE_STARTED; 2934 continue; 2935 } 2936 2937 /* Fail to setup tx queue, return */ 2938 if (port->port_status == RTE_PORT_HANDLING) 2939 port->port_status = RTE_PORT_STOPPED; 2940 else 2941 fprintf(stderr, 2942 "Port %d can not be set back to stopped\n", 2943 pi); 2944 fprintf(stderr, 2945 "Fail to configure port %d tx queues\n", 2946 pi); 2947 /* try to reconfigure queues next time */ 2948 port->need_reconfig_queues = 1; 2949 return -1; 2950 } 2951 for (qi = 0; qi < nb_rxq; qi++) { 2952 /* setup rx queues */ 2953 if ((numa_support) && 2954 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2955 struct rte_mempool * mp = 2956 mbuf_pool_find 2957 (rxring_numa[pi], 0); 2958 if (mp == NULL) { 2959 fprintf(stderr, 2960 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2961 rxring_numa[pi]); 2962 return -1; 2963 } 2964 2965 diag = rx_queue_setup(pi, qi, 2966 port->nb_rx_desc[qi], 2967 rxring_numa[pi], 2968 &(port->rxq[qi].conf), 2969 mp); 2970 } else { 2971 struct rte_mempool *mp = 2972 mbuf_pool_find 2973 (port->socket_id, 0); 2974 if (mp == NULL) { 2975 fprintf(stderr, 2976 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2977 port->socket_id); 2978 return -1; 2979 } 2980 diag = rx_queue_setup(pi, qi, 2981 port->nb_rx_desc[qi], 2982 port->socket_id, 2983 &(port->rxq[qi].conf), 2984 mp); 2985 } 2986 if (diag == 0) 2987 continue; 2988 2989 /* Fail to setup rx queue, return */ 2990 if (port->port_status == RTE_PORT_HANDLING) 2991 port->port_status = RTE_PORT_STOPPED; 2992 else 2993 fprintf(stderr, 2994 "Port %d can not be set back to stopped\n", 2995 pi); 2996 fprintf(stderr, 2997 "Fail to configure port %d rx queues\n", 2998 pi); 2999 /* try to reconfigure queues next time */ 3000 port->need_reconfig_queues = 1; 3001 return -1; 3002 } 3003 /* setup hairpin queues */ 3004 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 3005 return -1; 3006 } 3007 configure_rxtx_dump_callbacks(verbose_level); 3008 if (clear_ptypes) { 3009 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 3010 NULL, 0); 3011 if (diag < 0) 3012 fprintf(stderr, 3013 "Port %d: Failed to disable Ptype parsing\n", 3014 pi); 3015 } 3016 3017 p_pi = pi; 3018 cnt_pi++; 3019 3020 /* start port */ 3021 diag = eth_dev_start_mp(pi); 3022 if (diag < 0) { 3023 fprintf(stderr, "Fail to start port %d: %s\n", 3024 pi, rte_strerror(-diag)); 3025 3026 /* Fail to setup rx queue, return */ 3027 if (port->port_status == RTE_PORT_HANDLING) 3028 port->port_status = RTE_PORT_STOPPED; 3029 else 3030 fprintf(stderr, 3031 "Port %d can not be set back to stopped\n", 3032 pi); 3033 continue; 3034 } 3035 3036 if (port->port_status == RTE_PORT_HANDLING) 3037 port->port_status = RTE_PORT_STARTED; 3038 else 3039 fprintf(stderr, "Port %d can not be set into started\n", 3040 pi); 3041 3042 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 3043 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 3044 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 3045 3046 /* at least one port started, need checking link status */ 3047 need_check_link_status = 1; 3048 3049 pl[cfg_pi++] = pi; 3050 } 3051 3052 if (need_check_link_status == 1 && !no_link_check) 3053 check_all_ports_link_status(RTE_PORT_ALL); 3054 else if (need_check_link_status == 0) 3055 fprintf(stderr, "Please stop the ports first\n"); 3056 3057 if (hairpin_mode & 0xf) { 3058 uint16_t i; 3059 int j; 3060 3061 /* bind all started hairpin ports */ 3062 for (i = 0; i < cfg_pi; i++) { 3063 pi = pl[i]; 3064 /* bind current Tx to all peer Rx */ 3065 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3066 RTE_MAX_ETHPORTS, 1); 3067 if (peer_pi < 0) 3068 return peer_pi; 3069 for (j = 0; j < peer_pi; j++) { 3070 if (!port_is_started(peer_pl[j])) 3071 continue; 3072 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 3073 if (diag < 0) { 3074 fprintf(stderr, 3075 "Error during binding hairpin Tx port %u to %u: %s\n", 3076 pi, peer_pl[j], 3077 rte_strerror(-diag)); 3078 return -1; 3079 } 3080 } 3081 /* bind all peer Tx to current Rx */ 3082 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3083 RTE_MAX_ETHPORTS, 0); 3084 if (peer_pi < 0) 3085 return peer_pi; 3086 for (j = 0; j < peer_pi; j++) { 3087 if (!port_is_started(peer_pl[j])) 3088 continue; 3089 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3090 if (diag < 0) { 3091 fprintf(stderr, 3092 "Error during binding hairpin Tx port %u to %u: %s\n", 3093 peer_pl[j], pi, 3094 rte_strerror(-diag)); 3095 return -1; 3096 } 3097 } 3098 } 3099 } 3100 3101 fill_xstats_display_info_for_port(pid); 3102 3103 printf("Done\n"); 3104 return 0; 3105 } 3106 3107 void 3108 stop_port(portid_t pid) 3109 { 3110 portid_t pi; 3111 struct rte_port *port; 3112 int need_check_link_status = 0; 3113 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3114 int peer_pi; 3115 3116 if (port_id_is_invalid(pid, ENABLED_WARN)) 3117 return; 3118 3119 printf("Stopping ports...\n"); 3120 3121 RTE_ETH_FOREACH_DEV(pi) { 3122 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3123 continue; 3124 3125 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3126 fprintf(stderr, 3127 "Please remove port %d from forwarding configuration.\n", 3128 pi); 3129 continue; 3130 } 3131 3132 if (port_is_bonding_slave(pi)) { 3133 fprintf(stderr, 3134 "Please remove port %d from bonded device.\n", 3135 pi); 3136 continue; 3137 } 3138 3139 port = &ports[pi]; 3140 if (port->port_status == RTE_PORT_STARTED) 3141 port->port_status = RTE_PORT_HANDLING; 3142 else 3143 continue; 3144 3145 if (hairpin_mode & 0xf) { 3146 int j; 3147 3148 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3149 /* unbind all peer Tx from current Rx */ 3150 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3151 RTE_MAX_ETHPORTS, 0); 3152 if (peer_pi < 0) 3153 continue; 3154 for (j = 0; j < peer_pi; j++) { 3155 if (!port_is_started(peer_pl[j])) 3156 continue; 3157 rte_eth_hairpin_unbind(peer_pl[j], pi); 3158 } 3159 } 3160 3161 if (port->flow_list) 3162 port_flow_flush(pi); 3163 3164 if (eth_dev_stop_mp(pi) != 0) 3165 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3166 pi); 3167 3168 if (port->port_status == RTE_PORT_HANDLING) 3169 port->port_status = RTE_PORT_STOPPED; 3170 else 3171 fprintf(stderr, "Port %d can not be set into stopped\n", 3172 pi); 3173 need_check_link_status = 1; 3174 } 3175 if (need_check_link_status && !no_link_check) 3176 check_all_ports_link_status(RTE_PORT_ALL); 3177 3178 printf("Done\n"); 3179 } 3180 3181 static void 3182 remove_invalid_ports_in(portid_t *array, portid_t *total) 3183 { 3184 portid_t i; 3185 portid_t new_total = 0; 3186 3187 for (i = 0; i < *total; i++) 3188 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3189 array[new_total] = array[i]; 3190 new_total++; 3191 } 3192 *total = new_total; 3193 } 3194 3195 static void 3196 remove_invalid_ports(void) 3197 { 3198 remove_invalid_ports_in(ports_ids, &nb_ports); 3199 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3200 nb_cfg_ports = nb_fwd_ports; 3201 } 3202 3203 static void 3204 flush_port_owned_resources(portid_t pi) 3205 { 3206 mcast_addr_pool_destroy(pi); 3207 port_flow_flush(pi); 3208 port_flex_item_flush(pi); 3209 port_action_handle_flush(pi); 3210 } 3211 3212 static void 3213 clear_bonding_slave_device(portid_t *slave_pids, uint16_t num_slaves) 3214 { 3215 struct rte_port *port; 3216 portid_t slave_pid; 3217 uint16_t i; 3218 3219 for (i = 0; i < num_slaves; i++) { 3220 slave_pid = slave_pids[i]; 3221 if (port_is_started(slave_pid) == 1) { 3222 if (rte_eth_dev_stop(slave_pid) != 0) 3223 fprintf(stderr, "rte_eth_dev_stop failed for port %u\n", 3224 slave_pid); 3225 3226 port = &ports[slave_pid]; 3227 port->port_status = RTE_PORT_STOPPED; 3228 } 3229 3230 clear_port_slave_flag(slave_pid); 3231 3232 /* Close slave device when testpmd quit or is killed. */ 3233 if (cl_quit == 1 || f_quit == 1) 3234 rte_eth_dev_close(slave_pid); 3235 } 3236 } 3237 3238 void 3239 close_port(portid_t pid) 3240 { 3241 portid_t pi; 3242 struct rte_port *port; 3243 portid_t slave_pids[RTE_MAX_ETHPORTS]; 3244 int num_slaves = 0; 3245 3246 if (port_id_is_invalid(pid, ENABLED_WARN)) 3247 return; 3248 3249 printf("Closing ports...\n"); 3250 3251 RTE_ETH_FOREACH_DEV(pi) { 3252 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3253 continue; 3254 3255 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3256 fprintf(stderr, 3257 "Please remove port %d from forwarding configuration.\n", 3258 pi); 3259 continue; 3260 } 3261 3262 if (port_is_bonding_slave(pi)) { 3263 fprintf(stderr, 3264 "Please remove port %d from bonded device.\n", 3265 pi); 3266 continue; 3267 } 3268 3269 port = &ports[pi]; 3270 if (port->port_status == RTE_PORT_CLOSED) { 3271 fprintf(stderr, "Port %d is already closed\n", pi); 3272 continue; 3273 } 3274 3275 if (is_proc_primary()) { 3276 flush_port_owned_resources(pi); 3277 #ifdef RTE_NET_BOND 3278 if (port->bond_flag == 1) 3279 num_slaves = rte_eth_bond_slaves_get(pi, 3280 slave_pids, RTE_MAX_ETHPORTS); 3281 #endif 3282 rte_eth_dev_close(pi); 3283 /* 3284 * If this port is bonded device, all slaves under the 3285 * device need to be removed or closed. 3286 */ 3287 if (port->bond_flag == 1 && num_slaves > 0) 3288 clear_bonding_slave_device(slave_pids, 3289 num_slaves); 3290 } 3291 3292 free_xstats_display_info(pi); 3293 } 3294 3295 remove_invalid_ports(); 3296 printf("Done\n"); 3297 } 3298 3299 void 3300 reset_port(portid_t pid) 3301 { 3302 int diag; 3303 portid_t pi; 3304 struct rte_port *port; 3305 3306 if (port_id_is_invalid(pid, ENABLED_WARN)) 3307 return; 3308 3309 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3310 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3311 fprintf(stderr, 3312 "Can not reset port(s), please stop port(s) first.\n"); 3313 return; 3314 } 3315 3316 printf("Resetting ports...\n"); 3317 3318 RTE_ETH_FOREACH_DEV(pi) { 3319 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3320 continue; 3321 3322 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3323 fprintf(stderr, 3324 "Please remove port %d from forwarding configuration.\n", 3325 pi); 3326 continue; 3327 } 3328 3329 if (port_is_bonding_slave(pi)) { 3330 fprintf(stderr, 3331 "Please remove port %d from bonded device.\n", 3332 pi); 3333 continue; 3334 } 3335 3336 diag = rte_eth_dev_reset(pi); 3337 if (diag == 0) { 3338 port = &ports[pi]; 3339 port->need_reconfig = 1; 3340 port->need_reconfig_queues = 1; 3341 } else { 3342 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3343 pi, diag); 3344 } 3345 } 3346 3347 printf("Done\n"); 3348 } 3349 3350 void 3351 attach_port(char *identifier) 3352 { 3353 portid_t pi; 3354 struct rte_dev_iterator iterator; 3355 3356 printf("Attaching a new port...\n"); 3357 3358 if (identifier == NULL) { 3359 fprintf(stderr, "Invalid parameters are specified\n"); 3360 return; 3361 } 3362 3363 if (rte_dev_probe(identifier) < 0) { 3364 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3365 return; 3366 } 3367 3368 /* first attach mode: event */ 3369 if (setup_on_probe_event) { 3370 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3371 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3372 if (ports[pi].port_status == RTE_PORT_HANDLING && 3373 ports[pi].need_setup != 0) 3374 setup_attached_port(pi); 3375 return; 3376 } 3377 3378 /* second attach mode: iterator */ 3379 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3380 /* setup ports matching the devargs used for probing */ 3381 if (port_is_forwarding(pi)) 3382 continue; /* port was already attached before */ 3383 setup_attached_port(pi); 3384 } 3385 } 3386 3387 static void 3388 setup_attached_port(portid_t pi) 3389 { 3390 unsigned int socket_id; 3391 int ret; 3392 3393 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3394 /* if socket_id is invalid, set to the first available socket. */ 3395 if (check_socket_id(socket_id) < 0) 3396 socket_id = socket_ids[0]; 3397 reconfig(pi, socket_id); 3398 ret = rte_eth_promiscuous_enable(pi); 3399 if (ret != 0) 3400 fprintf(stderr, 3401 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3402 pi, rte_strerror(-ret)); 3403 3404 ports_ids[nb_ports++] = pi; 3405 fwd_ports_ids[nb_fwd_ports++] = pi; 3406 nb_cfg_ports = nb_fwd_ports; 3407 ports[pi].need_setup = 0; 3408 ports[pi].port_status = RTE_PORT_STOPPED; 3409 3410 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3411 printf("Done\n"); 3412 } 3413 3414 static void 3415 detach_device(struct rte_device *dev) 3416 { 3417 portid_t sibling; 3418 3419 if (dev == NULL) { 3420 fprintf(stderr, "Device already removed\n"); 3421 return; 3422 } 3423 3424 printf("Removing a device...\n"); 3425 3426 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3427 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3428 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3429 fprintf(stderr, "Port %u not stopped\n", 3430 sibling); 3431 return; 3432 } 3433 flush_port_owned_resources(sibling); 3434 } 3435 } 3436 3437 if (rte_dev_remove(dev) < 0) { 3438 TESTPMD_LOG(ERR, "Failed to detach device %s\n", rte_dev_name(dev)); 3439 return; 3440 } 3441 remove_invalid_ports(); 3442 3443 printf("Device is detached\n"); 3444 printf("Now total ports is %d\n", nb_ports); 3445 printf("Done\n"); 3446 return; 3447 } 3448 3449 void 3450 detach_port_device(portid_t port_id) 3451 { 3452 int ret; 3453 struct rte_eth_dev_info dev_info; 3454 3455 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3456 return; 3457 3458 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3459 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3460 fprintf(stderr, "Port not stopped\n"); 3461 return; 3462 } 3463 fprintf(stderr, "Port was not closed\n"); 3464 } 3465 3466 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3467 if (ret != 0) { 3468 TESTPMD_LOG(ERR, 3469 "Failed to get device info for port %d, not detaching\n", 3470 port_id); 3471 return; 3472 } 3473 detach_device(dev_info.device); 3474 } 3475 3476 void 3477 detach_devargs(char *identifier) 3478 { 3479 struct rte_dev_iterator iterator; 3480 struct rte_devargs da; 3481 portid_t port_id; 3482 3483 printf("Removing a device...\n"); 3484 3485 memset(&da, 0, sizeof(da)); 3486 if (rte_devargs_parsef(&da, "%s", identifier)) { 3487 fprintf(stderr, "cannot parse identifier\n"); 3488 return; 3489 } 3490 3491 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3492 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3493 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3494 fprintf(stderr, "Port %u not stopped\n", 3495 port_id); 3496 rte_eth_iterator_cleanup(&iterator); 3497 rte_devargs_reset(&da); 3498 return; 3499 } 3500 flush_port_owned_resources(port_id); 3501 } 3502 } 3503 3504 if (rte_eal_hotplug_remove(rte_bus_name(da.bus), da.name) != 0) { 3505 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3506 da.name, rte_bus_name(da.bus)); 3507 rte_devargs_reset(&da); 3508 return; 3509 } 3510 3511 remove_invalid_ports(); 3512 3513 printf("Device %s is detached\n", identifier); 3514 printf("Now total ports is %d\n", nb_ports); 3515 printf("Done\n"); 3516 rte_devargs_reset(&da); 3517 } 3518 3519 void 3520 pmd_test_exit(void) 3521 { 3522 portid_t pt_id; 3523 unsigned int i; 3524 int ret; 3525 3526 if (test_done == 0) 3527 stop_packet_forwarding(); 3528 3529 #ifndef RTE_EXEC_ENV_WINDOWS 3530 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3531 if (mempools[i]) { 3532 if (mp_alloc_type == MP_ALLOC_ANON) 3533 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3534 NULL); 3535 } 3536 } 3537 #endif 3538 if (ports != NULL) { 3539 no_link_check = 1; 3540 RTE_ETH_FOREACH_DEV(pt_id) { 3541 printf("\nStopping port %d...\n", pt_id); 3542 fflush(stdout); 3543 stop_port(pt_id); 3544 } 3545 RTE_ETH_FOREACH_DEV(pt_id) { 3546 printf("\nShutting down port %d...\n", pt_id); 3547 fflush(stdout); 3548 close_port(pt_id); 3549 } 3550 } 3551 3552 if (hot_plug) { 3553 ret = rte_dev_event_monitor_stop(); 3554 if (ret) { 3555 RTE_LOG(ERR, EAL, 3556 "fail to stop device event monitor."); 3557 return; 3558 } 3559 3560 ret = rte_dev_event_callback_unregister(NULL, 3561 dev_event_callback, NULL); 3562 if (ret < 0) { 3563 RTE_LOG(ERR, EAL, 3564 "fail to unregister device event callback.\n"); 3565 return; 3566 } 3567 3568 ret = rte_dev_hotplug_handle_disable(); 3569 if (ret) { 3570 RTE_LOG(ERR, EAL, 3571 "fail to disable hotplug handling.\n"); 3572 return; 3573 } 3574 } 3575 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3576 if (mempools[i]) 3577 mempool_free_mp(mempools[i]); 3578 } 3579 free(xstats_display); 3580 3581 printf("\nBye...\n"); 3582 } 3583 3584 typedef void (*cmd_func_t)(void); 3585 struct pmd_test_command { 3586 const char *cmd_name; 3587 cmd_func_t cmd_func; 3588 }; 3589 3590 /* Check the link status of all ports in up to 9s, and print them finally */ 3591 static void 3592 check_all_ports_link_status(uint32_t port_mask) 3593 { 3594 #define CHECK_INTERVAL 100 /* 100ms */ 3595 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3596 portid_t portid; 3597 uint8_t count, all_ports_up, print_flag = 0; 3598 struct rte_eth_link link; 3599 int ret; 3600 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3601 3602 printf("Checking link statuses...\n"); 3603 fflush(stdout); 3604 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3605 all_ports_up = 1; 3606 RTE_ETH_FOREACH_DEV(portid) { 3607 if ((port_mask & (1 << portid)) == 0) 3608 continue; 3609 memset(&link, 0, sizeof(link)); 3610 ret = rte_eth_link_get_nowait(portid, &link); 3611 if (ret < 0) { 3612 all_ports_up = 0; 3613 if (print_flag == 1) 3614 fprintf(stderr, 3615 "Port %u link get failed: %s\n", 3616 portid, rte_strerror(-ret)); 3617 continue; 3618 } 3619 /* print link status if flag set */ 3620 if (print_flag == 1) { 3621 rte_eth_link_to_str(link_status, 3622 sizeof(link_status), &link); 3623 printf("Port %d %s\n", portid, link_status); 3624 continue; 3625 } 3626 /* clear all_ports_up flag if any link down */ 3627 if (link.link_status == RTE_ETH_LINK_DOWN) { 3628 all_ports_up = 0; 3629 break; 3630 } 3631 } 3632 /* after finally printing all link status, get out */ 3633 if (print_flag == 1) 3634 break; 3635 3636 if (all_ports_up == 0) { 3637 fflush(stdout); 3638 rte_delay_ms(CHECK_INTERVAL); 3639 } 3640 3641 /* set the print_flag if all ports up or timeout */ 3642 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3643 print_flag = 1; 3644 } 3645 3646 if (lsc_interrupt) 3647 break; 3648 } 3649 } 3650 3651 static void 3652 rmv_port_callback(void *arg) 3653 { 3654 int need_to_start = 0; 3655 int org_no_link_check = no_link_check; 3656 portid_t port_id = (intptr_t)arg; 3657 struct rte_eth_dev_info dev_info; 3658 int ret; 3659 3660 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3661 3662 if (!test_done && port_is_forwarding(port_id)) { 3663 need_to_start = 1; 3664 stop_packet_forwarding(); 3665 } 3666 no_link_check = 1; 3667 stop_port(port_id); 3668 no_link_check = org_no_link_check; 3669 3670 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3671 if (ret != 0) 3672 TESTPMD_LOG(ERR, 3673 "Failed to get device info for port %d, not detaching\n", 3674 port_id); 3675 else { 3676 struct rte_device *device = dev_info.device; 3677 close_port(port_id); 3678 detach_device(device); /* might be already removed or have more ports */ 3679 } 3680 if (need_to_start) 3681 start_packet_forwarding(0); 3682 } 3683 3684 /* This function is used by the interrupt thread */ 3685 static int 3686 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3687 void *ret_param) 3688 { 3689 RTE_SET_USED(param); 3690 RTE_SET_USED(ret_param); 3691 3692 if (type >= RTE_ETH_EVENT_MAX) { 3693 fprintf(stderr, 3694 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3695 port_id, __func__, type); 3696 fflush(stderr); 3697 } else if (event_print_mask & (UINT32_C(1) << type)) { 3698 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3699 eth_event_desc[type]); 3700 fflush(stdout); 3701 } 3702 3703 switch (type) { 3704 case RTE_ETH_EVENT_NEW: 3705 ports[port_id].need_setup = 1; 3706 ports[port_id].port_status = RTE_PORT_HANDLING; 3707 break; 3708 case RTE_ETH_EVENT_INTR_RMV: 3709 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3710 break; 3711 if (rte_eal_alarm_set(100000, 3712 rmv_port_callback, (void *)(intptr_t)port_id)) 3713 fprintf(stderr, 3714 "Could not set up deferred device removal\n"); 3715 break; 3716 case RTE_ETH_EVENT_DESTROY: 3717 ports[port_id].port_status = RTE_PORT_CLOSED; 3718 printf("Port %u is closed\n", port_id); 3719 break; 3720 case RTE_ETH_EVENT_RX_AVAIL_THRESH: { 3721 uint16_t rxq_id; 3722 int ret; 3723 3724 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */ 3725 for (rxq_id = 0; ; rxq_id++) { 3726 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id, 3727 NULL); 3728 if (ret <= 0) 3729 break; 3730 printf("Received avail_thresh event, port: %u, rxq_id: %u\n", 3731 port_id, rxq_id); 3732 3733 #ifdef RTE_NET_MLX5 3734 mlx5_test_avail_thresh_event_handler(port_id, rxq_id); 3735 #endif 3736 } 3737 break; 3738 } 3739 default: 3740 break; 3741 } 3742 return 0; 3743 } 3744 3745 static int 3746 register_eth_event_callback(void) 3747 { 3748 int ret; 3749 enum rte_eth_event_type event; 3750 3751 for (event = RTE_ETH_EVENT_UNKNOWN; 3752 event < RTE_ETH_EVENT_MAX; event++) { 3753 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3754 event, 3755 eth_event_callback, 3756 NULL); 3757 if (ret != 0) { 3758 TESTPMD_LOG(ERR, "Failed to register callback for " 3759 "%s event\n", eth_event_desc[event]); 3760 return -1; 3761 } 3762 } 3763 3764 return 0; 3765 } 3766 3767 /* This function is used by the interrupt thread */ 3768 static void 3769 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3770 __rte_unused void *arg) 3771 { 3772 uint16_t port_id; 3773 int ret; 3774 3775 if (type >= RTE_DEV_EVENT_MAX) { 3776 fprintf(stderr, "%s called upon invalid event %d\n", 3777 __func__, type); 3778 fflush(stderr); 3779 } 3780 3781 switch (type) { 3782 case RTE_DEV_EVENT_REMOVE: 3783 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3784 device_name); 3785 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3786 if (ret) { 3787 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3788 device_name); 3789 return; 3790 } 3791 /* 3792 * Because the user's callback is invoked in eal interrupt 3793 * callback, the interrupt callback need to be finished before 3794 * it can be unregistered when detaching device. So finish 3795 * callback soon and use a deferred removal to detach device 3796 * is need. It is a workaround, once the device detaching be 3797 * moved into the eal in the future, the deferred removal could 3798 * be deleted. 3799 */ 3800 if (rte_eal_alarm_set(100000, 3801 rmv_port_callback, (void *)(intptr_t)port_id)) 3802 RTE_LOG(ERR, EAL, 3803 "Could not set up deferred device removal\n"); 3804 break; 3805 case RTE_DEV_EVENT_ADD: 3806 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3807 device_name); 3808 /* TODO: After finish kernel driver binding, 3809 * begin to attach port. 3810 */ 3811 break; 3812 default: 3813 break; 3814 } 3815 } 3816 3817 static void 3818 rxtx_port_config(portid_t pid) 3819 { 3820 uint16_t qid; 3821 uint64_t offloads; 3822 struct rte_port *port = &ports[pid]; 3823 3824 for (qid = 0; qid < nb_rxq; qid++) { 3825 offloads = port->rxq[qid].conf.offloads; 3826 port->rxq[qid].conf = port->dev_info.default_rxconf; 3827 3828 if (rxq_share > 0 && 3829 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3830 /* Non-zero share group to enable RxQ share. */ 3831 port->rxq[qid].conf.share_group = pid / rxq_share + 1; 3832 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */ 3833 } 3834 3835 if (offloads != 0) 3836 port->rxq[qid].conf.offloads = offloads; 3837 3838 /* Check if any Rx parameters have been passed */ 3839 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3840 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh; 3841 3842 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3843 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh; 3844 3845 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3846 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh; 3847 3848 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3849 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh; 3850 3851 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3852 port->rxq[qid].conf.rx_drop_en = rx_drop_en; 3853 3854 port->nb_rx_desc[qid] = nb_rxd; 3855 } 3856 3857 for (qid = 0; qid < nb_txq; qid++) { 3858 offloads = port->txq[qid].conf.offloads; 3859 port->txq[qid].conf = port->dev_info.default_txconf; 3860 if (offloads != 0) 3861 port->txq[qid].conf.offloads = offloads; 3862 3863 /* Check if any Tx parameters have been passed */ 3864 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3865 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh; 3866 3867 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3868 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh; 3869 3870 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3871 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh; 3872 3873 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3874 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh; 3875 3876 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3877 port->txq[qid].conf.tx_free_thresh = tx_free_thresh; 3878 3879 port->nb_tx_desc[qid] = nb_txd; 3880 } 3881 } 3882 3883 /* 3884 * Helper function to set MTU from frame size 3885 * 3886 * port->dev_info should be set before calling this function. 3887 * 3888 * return 0 on success, negative on error 3889 */ 3890 int 3891 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3892 { 3893 struct rte_port *port = &ports[portid]; 3894 uint32_t eth_overhead; 3895 uint16_t mtu, new_mtu; 3896 3897 eth_overhead = get_eth_overhead(&port->dev_info); 3898 3899 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3900 printf("Failed to get MTU for port %u\n", portid); 3901 return -1; 3902 } 3903 3904 new_mtu = max_rx_pktlen - eth_overhead; 3905 3906 if (mtu == new_mtu) 3907 return 0; 3908 3909 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3910 fprintf(stderr, 3911 "Failed to set MTU to %u for port %u\n", 3912 new_mtu, portid); 3913 return -1; 3914 } 3915 3916 port->dev_conf.rxmode.mtu = new_mtu; 3917 3918 return 0; 3919 } 3920 3921 void 3922 init_port_config(void) 3923 { 3924 portid_t pid; 3925 struct rte_port *port; 3926 int ret, i; 3927 3928 RTE_ETH_FOREACH_DEV(pid) { 3929 port = &ports[pid]; 3930 3931 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3932 if (ret != 0) 3933 return; 3934 3935 if (nb_rxq > 1) { 3936 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3937 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3938 rss_hf & port->dev_info.flow_type_rss_offloads; 3939 } else { 3940 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3941 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3942 } 3943 3944 if (port->dcb_flag == 0) { 3945 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3946 port->dev_conf.rxmode.mq_mode = 3947 (enum rte_eth_rx_mq_mode) 3948 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3949 } else { 3950 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3951 port->dev_conf.rxmode.offloads &= 3952 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3953 3954 for (i = 0; 3955 i < port->dev_info.nb_rx_queues; 3956 i++) 3957 port->rxq[i].conf.offloads &= 3958 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3959 } 3960 } 3961 3962 rxtx_port_config(pid); 3963 3964 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3965 if (ret != 0) 3966 return; 3967 3968 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3969 port->dev_conf.intr_conf.lsc = 1; 3970 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3971 port->dev_conf.intr_conf.rmv = 1; 3972 } 3973 } 3974 3975 void set_port_slave_flag(portid_t slave_pid) 3976 { 3977 struct rte_port *port; 3978 3979 port = &ports[slave_pid]; 3980 port->slave_flag = 1; 3981 } 3982 3983 void clear_port_slave_flag(portid_t slave_pid) 3984 { 3985 struct rte_port *port; 3986 3987 port = &ports[slave_pid]; 3988 port->slave_flag = 0; 3989 } 3990 3991 uint8_t port_is_bonding_slave(portid_t slave_pid) 3992 { 3993 struct rte_port *port; 3994 struct rte_eth_dev_info dev_info; 3995 int ret; 3996 3997 port = &ports[slave_pid]; 3998 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3999 if (ret != 0) { 4000 TESTPMD_LOG(ERR, 4001 "Failed to get device info for port id %d," 4002 "cannot determine if the port is a bonded slave", 4003 slave_pid); 4004 return 0; 4005 } 4006 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 4007 return 1; 4008 return 0; 4009 } 4010 4011 const uint16_t vlan_tags[] = { 4012 0, 1, 2, 3, 4, 5, 6, 7, 4013 8, 9, 10, 11, 12, 13, 14, 15, 4014 16, 17, 18, 19, 20, 21, 22, 23, 4015 24, 25, 26, 27, 28, 29, 30, 31 4016 }; 4017 4018 static int 4019 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 4020 enum dcb_mode_enable dcb_mode, 4021 enum rte_eth_nb_tcs num_tcs, 4022 uint8_t pfc_en) 4023 { 4024 uint8_t i; 4025 int32_t rc; 4026 struct rte_eth_rss_conf rss_conf; 4027 4028 /* 4029 * Builds up the correct configuration for dcb+vt based on the vlan tags array 4030 * given above, and the number of traffic classes available for use. 4031 */ 4032 if (dcb_mode == DCB_VT_ENABLED) { 4033 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 4034 ð_conf->rx_adv_conf.vmdq_dcb_conf; 4035 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 4036 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 4037 4038 /* VMDQ+DCB RX and TX configurations */ 4039 vmdq_rx_conf->enable_default_pool = 0; 4040 vmdq_rx_conf->default_pool = 0; 4041 vmdq_rx_conf->nb_queue_pools = 4042 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4043 vmdq_tx_conf->nb_queue_pools = 4044 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 4045 4046 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 4047 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 4048 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 4049 vmdq_rx_conf->pool_map[i].pools = 4050 1 << (i % vmdq_rx_conf->nb_queue_pools); 4051 } 4052 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4053 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 4054 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 4055 } 4056 4057 /* set DCB mode of RX and TX of multiple queues */ 4058 eth_conf->rxmode.mq_mode = 4059 (enum rte_eth_rx_mq_mode) 4060 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 4061 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 4062 } else { 4063 struct rte_eth_dcb_rx_conf *rx_conf = 4064 ð_conf->rx_adv_conf.dcb_rx_conf; 4065 struct rte_eth_dcb_tx_conf *tx_conf = 4066 ð_conf->tx_adv_conf.dcb_tx_conf; 4067 4068 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 4069 4070 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 4071 if (rc != 0) 4072 return rc; 4073 4074 rx_conf->nb_tcs = num_tcs; 4075 tx_conf->nb_tcs = num_tcs; 4076 4077 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4078 rx_conf->dcb_tc[i] = i % num_tcs; 4079 tx_conf->dcb_tc[i] = i % num_tcs; 4080 } 4081 4082 eth_conf->rxmode.mq_mode = 4083 (enum rte_eth_rx_mq_mode) 4084 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 4085 eth_conf->rx_adv_conf.rss_conf = rss_conf; 4086 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 4087 } 4088 4089 if (pfc_en) 4090 eth_conf->dcb_capability_en = 4091 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 4092 else 4093 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 4094 4095 return 0; 4096 } 4097 4098 int 4099 init_port_dcb_config(portid_t pid, 4100 enum dcb_mode_enable dcb_mode, 4101 enum rte_eth_nb_tcs num_tcs, 4102 uint8_t pfc_en) 4103 { 4104 struct rte_eth_conf port_conf; 4105 struct rte_port *rte_port; 4106 int retval; 4107 uint16_t i; 4108 4109 if (num_procs > 1) { 4110 printf("The multi-process feature doesn't support dcb.\n"); 4111 return -ENOTSUP; 4112 } 4113 rte_port = &ports[pid]; 4114 4115 /* retain the original device configuration. */ 4116 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 4117 4118 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 4119 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 4120 if (retval < 0) 4121 return retval; 4122 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4123 /* remove RSS HASH offload for DCB in vt mode */ 4124 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 4125 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4126 for (i = 0; i < nb_rxq; i++) 4127 rte_port->rxq[i].conf.offloads &= 4128 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 4129 } 4130 4131 /* re-configure the device . */ 4132 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 4133 if (retval < 0) 4134 return retval; 4135 4136 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 4137 if (retval != 0) 4138 return retval; 4139 4140 /* If dev_info.vmdq_pool_base is greater than 0, 4141 * the queue id of vmdq pools is started after pf queues. 4142 */ 4143 if (dcb_mode == DCB_VT_ENABLED && 4144 rte_port->dev_info.vmdq_pool_base > 0) { 4145 fprintf(stderr, 4146 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 4147 pid); 4148 return -1; 4149 } 4150 4151 /* Assume the ports in testpmd have the same dcb capability 4152 * and has the same number of rxq and txq in dcb mode 4153 */ 4154 if (dcb_mode == DCB_VT_ENABLED) { 4155 if (rte_port->dev_info.max_vfs > 0) { 4156 nb_rxq = rte_port->dev_info.nb_rx_queues; 4157 nb_txq = rte_port->dev_info.nb_tx_queues; 4158 } else { 4159 nb_rxq = rte_port->dev_info.max_rx_queues; 4160 nb_txq = rte_port->dev_info.max_tx_queues; 4161 } 4162 } else { 4163 /*if vt is disabled, use all pf queues */ 4164 if (rte_port->dev_info.vmdq_pool_base == 0) { 4165 nb_rxq = rte_port->dev_info.max_rx_queues; 4166 nb_txq = rte_port->dev_info.max_tx_queues; 4167 } else { 4168 nb_rxq = (queueid_t)num_tcs; 4169 nb_txq = (queueid_t)num_tcs; 4170 4171 } 4172 } 4173 rx_free_thresh = 64; 4174 4175 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4176 4177 rxtx_port_config(pid); 4178 /* VLAN filter */ 4179 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4180 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4181 rx_vft_set(pid, vlan_tags[i], 1); 4182 4183 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4184 if (retval != 0) 4185 return retval; 4186 4187 rte_port->dcb_flag = 1; 4188 4189 /* Enter DCB configuration status */ 4190 dcb_config = 1; 4191 4192 return 0; 4193 } 4194 4195 static void 4196 init_port(void) 4197 { 4198 int i; 4199 4200 /* Configuration of Ethernet ports. */ 4201 ports = rte_zmalloc("testpmd: ports", 4202 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4203 RTE_CACHE_LINE_SIZE); 4204 if (ports == NULL) { 4205 rte_exit(EXIT_FAILURE, 4206 "rte_zmalloc(%d struct rte_port) failed\n", 4207 RTE_MAX_ETHPORTS); 4208 } 4209 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4210 ports[i].xstats_info.allocated = false; 4211 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4212 LIST_INIT(&ports[i].flow_tunnel_list); 4213 /* Initialize ports NUMA structures */ 4214 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4215 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4216 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4217 } 4218 4219 static void 4220 force_quit(void) 4221 { 4222 pmd_test_exit(); 4223 prompt_exit(); 4224 } 4225 4226 static void 4227 print_stats(void) 4228 { 4229 uint8_t i; 4230 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4231 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4232 4233 /* Clear screen and move to top left */ 4234 printf("%s%s", clr, top_left); 4235 4236 printf("\nPort statistics ===================================="); 4237 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4238 nic_stats_display(fwd_ports_ids[i]); 4239 4240 fflush(stdout); 4241 } 4242 4243 static void 4244 signal_handler(int signum) 4245 { 4246 if (signum == SIGINT || signum == SIGTERM) { 4247 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4248 signum); 4249 #ifdef RTE_LIB_PDUMP 4250 /* uninitialize packet capture framework */ 4251 rte_pdump_uninit(); 4252 #endif 4253 #ifdef RTE_LIB_LATENCYSTATS 4254 if (latencystats_enabled != 0) 4255 rte_latencystats_uninit(); 4256 #endif 4257 force_quit(); 4258 /* Set flag to indicate the force termination. */ 4259 f_quit = 1; 4260 /* exit with the expected status */ 4261 #ifndef RTE_EXEC_ENV_WINDOWS 4262 signal(signum, SIG_DFL); 4263 kill(getpid(), signum); 4264 #endif 4265 } 4266 } 4267 4268 int 4269 main(int argc, char** argv) 4270 { 4271 int diag; 4272 portid_t port_id; 4273 uint16_t count; 4274 int ret; 4275 4276 signal(SIGINT, signal_handler); 4277 signal(SIGTERM, signal_handler); 4278 4279 testpmd_logtype = rte_log_register("testpmd"); 4280 if (testpmd_logtype < 0) 4281 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4282 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4283 4284 diag = rte_eal_init(argc, argv); 4285 if (diag < 0) 4286 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4287 rte_strerror(rte_errno)); 4288 4289 ret = register_eth_event_callback(); 4290 if (ret != 0) 4291 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4292 4293 #ifdef RTE_LIB_PDUMP 4294 /* initialize packet capture framework */ 4295 rte_pdump_init(); 4296 #endif 4297 4298 count = 0; 4299 RTE_ETH_FOREACH_DEV(port_id) { 4300 ports_ids[count] = port_id; 4301 count++; 4302 } 4303 nb_ports = (portid_t) count; 4304 if (nb_ports == 0) 4305 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4306 4307 /* allocate port structures, and init them */ 4308 init_port(); 4309 4310 set_def_fwd_config(); 4311 if (nb_lcores == 0) 4312 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4313 "Check the core mask argument\n"); 4314 4315 /* Bitrate/latency stats disabled by default */ 4316 #ifdef RTE_LIB_BITRATESTATS 4317 bitrate_enabled = 0; 4318 #endif 4319 #ifdef RTE_LIB_LATENCYSTATS 4320 latencystats_enabled = 0; 4321 #endif 4322 4323 /* on FreeBSD, mlockall() is disabled by default */ 4324 #ifdef RTE_EXEC_ENV_FREEBSD 4325 do_mlockall = 0; 4326 #else 4327 do_mlockall = 1; 4328 #endif 4329 4330 argc -= diag; 4331 argv += diag; 4332 if (argc > 1) 4333 launch_args_parse(argc, argv); 4334 4335 #ifndef RTE_EXEC_ENV_WINDOWS 4336 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4337 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4338 strerror(errno)); 4339 } 4340 #endif 4341 4342 if (tx_first && interactive) 4343 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4344 "interactive mode.\n"); 4345 4346 if (tx_first && lsc_interrupt) { 4347 fprintf(stderr, 4348 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4349 lsc_interrupt = 0; 4350 } 4351 4352 if (!nb_rxq && !nb_txq) 4353 fprintf(stderr, 4354 "Warning: Either rx or tx queues should be non-zero\n"); 4355 4356 if (nb_rxq > 1 && nb_rxq > nb_txq) 4357 fprintf(stderr, 4358 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4359 nb_rxq, nb_txq); 4360 4361 init_config(); 4362 4363 if (hot_plug) { 4364 ret = rte_dev_hotplug_handle_enable(); 4365 if (ret) { 4366 RTE_LOG(ERR, EAL, 4367 "fail to enable hotplug handling."); 4368 return -1; 4369 } 4370 4371 ret = rte_dev_event_monitor_start(); 4372 if (ret) { 4373 RTE_LOG(ERR, EAL, 4374 "fail to start device event monitoring."); 4375 return -1; 4376 } 4377 4378 ret = rte_dev_event_callback_register(NULL, 4379 dev_event_callback, NULL); 4380 if (ret) { 4381 RTE_LOG(ERR, EAL, 4382 "fail to register device event callback\n"); 4383 return -1; 4384 } 4385 } 4386 4387 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4388 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4389 4390 /* set all ports to promiscuous mode by default */ 4391 RTE_ETH_FOREACH_DEV(port_id) { 4392 ret = rte_eth_promiscuous_enable(port_id); 4393 if (ret != 0) 4394 fprintf(stderr, 4395 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4396 port_id, rte_strerror(-ret)); 4397 } 4398 4399 #ifdef RTE_LIB_METRICS 4400 /* Init metrics library */ 4401 rte_metrics_init(rte_socket_id()); 4402 #endif 4403 4404 #ifdef RTE_LIB_LATENCYSTATS 4405 if (latencystats_enabled != 0) { 4406 int ret = rte_latencystats_init(1, NULL); 4407 if (ret) 4408 fprintf(stderr, 4409 "Warning: latencystats init() returned error %d\n", 4410 ret); 4411 fprintf(stderr, "Latencystats running on lcore %d\n", 4412 latencystats_lcore_id); 4413 } 4414 #endif 4415 4416 /* Setup bitrate stats */ 4417 #ifdef RTE_LIB_BITRATESTATS 4418 if (bitrate_enabled != 0) { 4419 bitrate_data = rte_stats_bitrate_create(); 4420 if (bitrate_data == NULL) 4421 rte_exit(EXIT_FAILURE, 4422 "Could not allocate bitrate data.\n"); 4423 rte_stats_bitrate_reg(bitrate_data); 4424 } 4425 #endif 4426 #ifdef RTE_LIB_CMDLINE 4427 if (init_cmdline() != 0) 4428 rte_exit(EXIT_FAILURE, 4429 "Could not initialise cmdline context.\n"); 4430 4431 if (strlen(cmdline_filename) != 0) 4432 cmdline_read_from_file(cmdline_filename); 4433 4434 if (interactive == 1) { 4435 if (auto_start) { 4436 printf("Start automatic packet forwarding\n"); 4437 start_packet_forwarding(0); 4438 } 4439 prompt(); 4440 pmd_test_exit(); 4441 } else 4442 #endif 4443 { 4444 char c; 4445 int rc; 4446 4447 f_quit = 0; 4448 4449 printf("No commandline core given, start packet forwarding\n"); 4450 start_packet_forwarding(tx_first); 4451 if (stats_period != 0) { 4452 uint64_t prev_time = 0, cur_time, diff_time = 0; 4453 uint64_t timer_period; 4454 4455 /* Convert to number of cycles */ 4456 timer_period = stats_period * rte_get_timer_hz(); 4457 4458 while (f_quit == 0) { 4459 cur_time = rte_get_timer_cycles(); 4460 diff_time += cur_time - prev_time; 4461 4462 if (diff_time >= timer_period) { 4463 print_stats(); 4464 /* Reset the timer */ 4465 diff_time = 0; 4466 } 4467 /* Sleep to avoid unnecessary checks */ 4468 prev_time = cur_time; 4469 rte_delay_us_sleep(US_PER_S); 4470 } 4471 } 4472 4473 printf("Press enter to exit\n"); 4474 rc = read(0, &c, 1); 4475 pmd_test_exit(); 4476 if (rc < 0) 4477 return 1; 4478 } 4479 4480 ret = rte_eal_cleanup(); 4481 if (ret != 0) 4482 rte_exit(EXIT_FAILURE, 4483 "EAL cleanup failed: %s\n", strerror(-ret)); 4484 4485 return EXIT_SUCCESS; 4486 } 4487