1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 70 #include "testpmd.h" 71 72 #ifndef MAP_HUGETLB 73 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 74 #define HUGE_FLAG (0x40000) 75 #else 76 #define HUGE_FLAG MAP_HUGETLB 77 #endif 78 79 #ifndef MAP_HUGE_SHIFT 80 /* older kernels (or FreeBSD) will not have this define */ 81 #define HUGE_SHIFT (26) 82 #else 83 #define HUGE_SHIFT MAP_HUGE_SHIFT 84 #endif 85 86 #define EXTMEM_HEAP_NAME "extmem" 87 /* 88 * Zone size with the malloc overhead (max of debug and release variants) 89 * must fit into the smallest supported hugepage size (2M), 90 * so that an IOVA-contiguous zone of this size can always be allocated 91 * if there are free 2M hugepages. 92 */ 93 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 94 95 uint16_t verbose_level = 0; /**< Silent by default. */ 96 int testpmd_logtype; /**< Log type for testpmd logs */ 97 98 /* use main core for command line ? */ 99 uint8_t interactive = 0; 100 uint8_t auto_start = 0; 101 uint8_t tx_first; 102 char cmdline_filename[PATH_MAX] = {0}; 103 104 /* 105 * NUMA support configuration. 106 * When set, the NUMA support attempts to dispatch the allocation of the 107 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 108 * probed ports among the CPU sockets 0 and 1. 109 * Otherwise, all memory is allocated from CPU socket 0. 110 */ 111 uint8_t numa_support = 1; /**< numa enabled by default */ 112 113 /* 114 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 115 * not configured. 116 */ 117 uint8_t socket_num = UMA_NO_CONFIG; 118 119 /* 120 * Select mempool allocation type: 121 * - native: use regular DPDK memory 122 * - anon: use regular DPDK memory to create mempool, but populate using 123 * anonymous memory (may not be IOVA-contiguous) 124 * - xmem: use externally allocated hugepage memory 125 */ 126 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 127 128 /* 129 * Store specified sockets on which memory pool to be used by ports 130 * is allocated. 131 */ 132 uint8_t port_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Store specified sockets on which RX ring to be used by ports 136 * is allocated. 137 */ 138 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which TX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Record the Ethernet address of peer target ports to which packets are 148 * forwarded. 149 * Must be instantiated with the ethernet addresses of peer traffic generator 150 * ports. 151 */ 152 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 153 portid_t nb_peer_eth_addrs = 0; 154 155 /* 156 * Probed Target Environment. 157 */ 158 struct rte_port *ports; /**< For all probed ethernet ports. */ 159 portid_t nb_ports; /**< Number of probed ethernet ports. */ 160 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 161 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 162 163 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 164 165 /* 166 * Test Forwarding Configuration. 167 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 168 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 169 */ 170 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 171 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 172 portid_t nb_cfg_ports; /**< Number of configured ports. */ 173 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 174 175 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 176 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 177 178 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 179 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 180 181 /* 182 * Forwarding engines. 183 */ 184 struct fwd_engine * fwd_engines[] = { 185 &io_fwd_engine, 186 &mac_fwd_engine, 187 &mac_swap_engine, 188 &flow_gen_engine, 189 &rx_only_engine, 190 &tx_only_engine, 191 &csum_fwd_engine, 192 &icmp_echo_engine, 193 &noisy_vnf_engine, 194 &five_tuple_swap_fwd_engine, 195 #ifdef RTE_LIBRTE_IEEE1588 196 &ieee1588_fwd_engine, 197 #endif 198 &shared_rxq_engine, 199 NULL, 200 }; 201 202 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 203 uint16_t mempool_flags; 204 205 struct fwd_config cur_fwd_config; 206 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 207 uint32_t retry_enabled; 208 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 209 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 210 211 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 212 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 213 DEFAULT_MBUF_DATA_SIZE 214 }; /**< Mbuf data space size. */ 215 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 216 * specified on command-line. */ 217 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 218 219 /** Extended statistics to show. */ 220 struct rte_eth_xstat_name *xstats_display; 221 222 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 223 224 /* 225 * In container, it cannot terminate the process which running with 'stats-period' 226 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 227 */ 228 uint8_t f_quit; 229 230 /* 231 * Max Rx frame size, set by '--max-pkt-len' parameter. 232 */ 233 uint32_t max_rx_pkt_len; 234 235 /* 236 * Configuration of packet segments used to scatter received packets 237 * if some of split features is configured. 238 */ 239 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 240 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 241 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 242 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 243 244 /* 245 * Configuration of packet segments used by the "txonly" processing engine. 246 */ 247 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 248 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 249 TXONLY_DEF_PACKET_LEN, 250 }; 251 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 252 253 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 254 /**< Split policy for packets to TX. */ 255 256 uint8_t txonly_multi_flow; 257 /**< Whether multiple flows are generated in TXONLY mode. */ 258 259 uint32_t tx_pkt_times_inter; 260 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 261 262 uint32_t tx_pkt_times_intra; 263 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 264 265 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 266 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 267 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 268 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 269 270 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 271 uint8_t dcb_config = 0; 272 273 /* 274 * Configurable number of RX/TX queues. 275 */ 276 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 277 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 278 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 279 280 /* 281 * Configurable number of RX/TX ring descriptors. 282 * Defaults are supplied by drivers via ethdev. 283 */ 284 #define RTE_TEST_RX_DESC_DEFAULT 0 285 #define RTE_TEST_TX_DESC_DEFAULT 0 286 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 287 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 288 289 #define RTE_PMD_PARAM_UNSET -1 290 /* 291 * Configurable values of RX and TX ring threshold registers. 292 */ 293 294 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 295 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 296 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 297 298 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 299 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 300 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 301 302 /* 303 * Configurable value of RX free threshold. 304 */ 305 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 306 307 /* 308 * Configurable value of RX drop enable. 309 */ 310 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 311 312 /* 313 * Configurable value of TX free threshold. 314 */ 315 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 316 317 /* 318 * Configurable value of TX RS bit threshold. 319 */ 320 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 321 322 /* 323 * Configurable value of buffered packets before sending. 324 */ 325 uint16_t noisy_tx_sw_bufsz; 326 327 /* 328 * Configurable value of packet buffer timeout. 329 */ 330 uint16_t noisy_tx_sw_buf_flush_time; 331 332 /* 333 * Configurable value for size of VNF internal memory area 334 * used for simulating noisy neighbour behaviour 335 */ 336 uint64_t noisy_lkup_mem_sz; 337 338 /* 339 * Configurable value of number of random writes done in 340 * VNF simulation memory area. 341 */ 342 uint64_t noisy_lkup_num_writes; 343 344 /* 345 * Configurable value of number of random reads done in 346 * VNF simulation memory area. 347 */ 348 uint64_t noisy_lkup_num_reads; 349 350 /* 351 * Configurable value of number of random reads/writes done in 352 * VNF simulation memory area. 353 */ 354 uint64_t noisy_lkup_num_reads_writes; 355 356 /* 357 * Receive Side Scaling (RSS) configuration. 358 */ 359 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 360 361 /* 362 * Port topology configuration 363 */ 364 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 365 366 /* 367 * Avoids to flush all the RX streams before starts forwarding. 368 */ 369 uint8_t no_flush_rx = 0; /* flush by default */ 370 371 /* 372 * Flow API isolated mode. 373 */ 374 uint8_t flow_isolate_all; 375 376 /* 377 * Avoids to check link status when starting/stopping a port. 378 */ 379 uint8_t no_link_check = 0; /* check by default */ 380 381 /* 382 * Don't automatically start all ports in interactive mode. 383 */ 384 uint8_t no_device_start = 0; 385 386 /* 387 * Enable link status change notification 388 */ 389 uint8_t lsc_interrupt = 1; /* enabled by default */ 390 391 /* 392 * Enable device removal notification. 393 */ 394 uint8_t rmv_interrupt = 1; /* enabled by default */ 395 396 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 397 398 /* After attach, port setup is called on event or by iterator */ 399 bool setup_on_probe_event = true; 400 401 /* Clear ptypes on port initialization. */ 402 uint8_t clear_ptypes = true; 403 404 /* Hairpin ports configuration mode. */ 405 uint16_t hairpin_mode; 406 407 /* Pretty printing of ethdev events */ 408 static const char * const eth_event_desc[] = { 409 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 410 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 411 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 412 [RTE_ETH_EVENT_INTR_RESET] = "reset", 413 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 414 [RTE_ETH_EVENT_IPSEC] = "IPsec", 415 [RTE_ETH_EVENT_MACSEC] = "MACsec", 416 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 417 [RTE_ETH_EVENT_NEW] = "device probed", 418 [RTE_ETH_EVENT_DESTROY] = "device released", 419 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 420 [RTE_ETH_EVENT_MAX] = NULL, 421 }; 422 423 /* 424 * Display or mask ether events 425 * Default to all events except VF_MBOX 426 */ 427 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 428 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 429 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 430 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 431 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 432 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 433 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 434 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 435 /* 436 * Decide if all memory are locked for performance. 437 */ 438 int do_mlockall = 0; 439 440 /* 441 * NIC bypass mode configuration options. 442 */ 443 444 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 445 /* The NIC bypass watchdog timeout. */ 446 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 447 #endif 448 449 450 #ifdef RTE_LIB_LATENCYSTATS 451 452 /* 453 * Set when latency stats is enabled in the commandline 454 */ 455 uint8_t latencystats_enabled; 456 457 /* 458 * Lcore ID to service latency statistics. 459 */ 460 lcoreid_t latencystats_lcore_id = -1; 461 462 #endif 463 464 /* 465 * Ethernet device configuration. 466 */ 467 struct rte_eth_rxmode rx_mode; 468 469 struct rte_eth_txmode tx_mode = { 470 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 471 }; 472 473 struct rte_eth_fdir_conf fdir_conf = { 474 .mode = RTE_FDIR_MODE_NONE, 475 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 476 .status = RTE_FDIR_REPORT_STATUS, 477 .mask = { 478 .vlan_tci_mask = 0xFFEF, 479 .ipv4_mask = { 480 .src_ip = 0xFFFFFFFF, 481 .dst_ip = 0xFFFFFFFF, 482 }, 483 .ipv6_mask = { 484 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 485 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 486 }, 487 .src_port_mask = 0xFFFF, 488 .dst_port_mask = 0xFFFF, 489 .mac_addr_byte_mask = 0xFF, 490 .tunnel_type_mask = 1, 491 .tunnel_id_mask = 0xFFFFFFFF, 492 }, 493 .drop_queue = 127, 494 }; 495 496 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 497 498 /* 499 * Display zero values by default for xstats 500 */ 501 uint8_t xstats_hide_zero; 502 503 /* 504 * Measure of CPU cycles disabled by default 505 */ 506 uint8_t record_core_cycles; 507 508 /* 509 * Display of RX and TX bursts disabled by default 510 */ 511 uint8_t record_burst_stats; 512 513 /* 514 * Number of ports per shared Rx queue group, 0 disable. 515 */ 516 uint32_t rxq_share; 517 518 unsigned int num_sockets = 0; 519 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 520 521 #ifdef RTE_LIB_BITRATESTATS 522 /* Bitrate statistics */ 523 struct rte_stats_bitrates *bitrate_data; 524 lcoreid_t bitrate_lcore_id; 525 uint8_t bitrate_enabled; 526 #endif 527 528 #ifdef RTE_LIB_GRO 529 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 530 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 531 #endif 532 533 /* 534 * hexadecimal bitmask of RX mq mode can be enabled. 535 */ 536 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 537 538 /* 539 * Used to set forced link speed 540 */ 541 uint32_t eth_link_speed; 542 543 /* 544 * ID of the current process in multi-process, used to 545 * configure the queues to be polled. 546 */ 547 int proc_id; 548 549 /* 550 * Number of processes in multi-process, used to 551 * configure the queues to be polled. 552 */ 553 unsigned int num_procs = 1; 554 555 static void 556 eth_rx_metadata_negotiate_mp(uint16_t port_id) 557 { 558 uint64_t rx_meta_features = 0; 559 int ret; 560 561 if (!is_proc_primary()) 562 return; 563 564 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 565 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 566 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 567 568 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 569 if (ret == 0) { 570 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 571 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 572 port_id); 573 } 574 575 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 576 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 577 port_id); 578 } 579 580 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 581 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 582 port_id); 583 } 584 } else if (ret != -ENOTSUP) { 585 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 586 port_id, rte_strerror(-ret)); 587 } 588 } 589 590 static int 591 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 592 const struct rte_eth_conf *dev_conf) 593 { 594 if (is_proc_primary()) 595 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 596 dev_conf); 597 return 0; 598 } 599 600 static int 601 eth_dev_start_mp(uint16_t port_id) 602 { 603 if (is_proc_primary()) 604 return rte_eth_dev_start(port_id); 605 606 return 0; 607 } 608 609 static int 610 eth_dev_stop_mp(uint16_t port_id) 611 { 612 if (is_proc_primary()) 613 return rte_eth_dev_stop(port_id); 614 615 return 0; 616 } 617 618 static void 619 mempool_free_mp(struct rte_mempool *mp) 620 { 621 if (is_proc_primary()) 622 rte_mempool_free(mp); 623 } 624 625 static int 626 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 627 { 628 if (is_proc_primary()) 629 return rte_eth_dev_set_mtu(port_id, mtu); 630 631 return 0; 632 } 633 634 /* Forward function declarations */ 635 static void setup_attached_port(portid_t pi); 636 static void check_all_ports_link_status(uint32_t port_mask); 637 static int eth_event_callback(portid_t port_id, 638 enum rte_eth_event_type type, 639 void *param, void *ret_param); 640 static void dev_event_callback(const char *device_name, 641 enum rte_dev_event_type type, 642 void *param); 643 static void fill_xstats_display_info(void); 644 645 /* 646 * Check if all the ports are started. 647 * If yes, return positive value. If not, return zero. 648 */ 649 static int all_ports_started(void); 650 651 #ifdef RTE_LIB_GSO 652 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 653 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 654 #endif 655 656 /* Holds the registered mbuf dynamic flags names. */ 657 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 658 659 660 /* 661 * Helper function to check if socket is already discovered. 662 * If yes, return positive value. If not, return zero. 663 */ 664 int 665 new_socket_id(unsigned int socket_id) 666 { 667 unsigned int i; 668 669 for (i = 0; i < num_sockets; i++) { 670 if (socket_ids[i] == socket_id) 671 return 0; 672 } 673 return 1; 674 } 675 676 /* 677 * Setup default configuration. 678 */ 679 static void 680 set_default_fwd_lcores_config(void) 681 { 682 unsigned int i; 683 unsigned int nb_lc; 684 unsigned int sock_num; 685 686 nb_lc = 0; 687 for (i = 0; i < RTE_MAX_LCORE; i++) { 688 if (!rte_lcore_is_enabled(i)) 689 continue; 690 sock_num = rte_lcore_to_socket_id(i); 691 if (new_socket_id(sock_num)) { 692 if (num_sockets >= RTE_MAX_NUMA_NODES) { 693 rte_exit(EXIT_FAILURE, 694 "Total sockets greater than %u\n", 695 RTE_MAX_NUMA_NODES); 696 } 697 socket_ids[num_sockets++] = sock_num; 698 } 699 if (i == rte_get_main_lcore()) 700 continue; 701 fwd_lcores_cpuids[nb_lc++] = i; 702 } 703 nb_lcores = (lcoreid_t) nb_lc; 704 nb_cfg_lcores = nb_lcores; 705 nb_fwd_lcores = 1; 706 } 707 708 static void 709 set_def_peer_eth_addrs(void) 710 { 711 portid_t i; 712 713 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 714 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 715 peer_eth_addrs[i].addr_bytes[5] = i; 716 } 717 } 718 719 static void 720 set_default_fwd_ports_config(void) 721 { 722 portid_t pt_id; 723 int i = 0; 724 725 RTE_ETH_FOREACH_DEV(pt_id) { 726 fwd_ports_ids[i++] = pt_id; 727 728 /* Update sockets info according to the attached device */ 729 int socket_id = rte_eth_dev_socket_id(pt_id); 730 if (socket_id >= 0 && new_socket_id(socket_id)) { 731 if (num_sockets >= RTE_MAX_NUMA_NODES) { 732 rte_exit(EXIT_FAILURE, 733 "Total sockets greater than %u\n", 734 RTE_MAX_NUMA_NODES); 735 } 736 socket_ids[num_sockets++] = socket_id; 737 } 738 } 739 740 nb_cfg_ports = nb_ports; 741 nb_fwd_ports = nb_ports; 742 } 743 744 void 745 set_def_fwd_config(void) 746 { 747 set_default_fwd_lcores_config(); 748 set_def_peer_eth_addrs(); 749 set_default_fwd_ports_config(); 750 } 751 752 #ifndef RTE_EXEC_ENV_WINDOWS 753 /* extremely pessimistic estimation of memory required to create a mempool */ 754 static int 755 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 756 { 757 unsigned int n_pages, mbuf_per_pg, leftover; 758 uint64_t total_mem, mbuf_mem, obj_sz; 759 760 /* there is no good way to predict how much space the mempool will 761 * occupy because it will allocate chunks on the fly, and some of those 762 * will come from default DPDK memory while some will come from our 763 * external memory, so just assume 128MB will be enough for everyone. 764 */ 765 uint64_t hdr_mem = 128 << 20; 766 767 /* account for possible non-contiguousness */ 768 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 769 if (obj_sz > pgsz) { 770 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 771 return -1; 772 } 773 774 mbuf_per_pg = pgsz / obj_sz; 775 leftover = (nb_mbufs % mbuf_per_pg) > 0; 776 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 777 778 mbuf_mem = n_pages * pgsz; 779 780 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 781 782 if (total_mem > SIZE_MAX) { 783 TESTPMD_LOG(ERR, "Memory size too big\n"); 784 return -1; 785 } 786 *out = (size_t)total_mem; 787 788 return 0; 789 } 790 791 static int 792 pagesz_flags(uint64_t page_sz) 793 { 794 /* as per mmap() manpage, all page sizes are log2 of page size 795 * shifted by MAP_HUGE_SHIFT 796 */ 797 int log2 = rte_log2_u64(page_sz); 798 799 return (log2 << HUGE_SHIFT); 800 } 801 802 static void * 803 alloc_mem(size_t memsz, size_t pgsz, bool huge) 804 { 805 void *addr; 806 int flags; 807 808 /* allocate anonymous hugepages */ 809 flags = MAP_ANONYMOUS | MAP_PRIVATE; 810 if (huge) 811 flags |= HUGE_FLAG | pagesz_flags(pgsz); 812 813 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 814 if (addr == MAP_FAILED) 815 return NULL; 816 817 return addr; 818 } 819 820 struct extmem_param { 821 void *addr; 822 size_t len; 823 size_t pgsz; 824 rte_iova_t *iova_table; 825 unsigned int iova_table_len; 826 }; 827 828 static int 829 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 830 bool huge) 831 { 832 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 833 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 834 unsigned int cur_page, n_pages, pgsz_idx; 835 size_t mem_sz, cur_pgsz; 836 rte_iova_t *iovas = NULL; 837 void *addr; 838 int ret; 839 840 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 841 /* skip anything that is too big */ 842 if (pgsizes[pgsz_idx] > SIZE_MAX) 843 continue; 844 845 cur_pgsz = pgsizes[pgsz_idx]; 846 847 /* if we were told not to allocate hugepages, override */ 848 if (!huge) 849 cur_pgsz = sysconf(_SC_PAGESIZE); 850 851 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 852 if (ret < 0) { 853 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 854 return -1; 855 } 856 857 /* allocate our memory */ 858 addr = alloc_mem(mem_sz, cur_pgsz, huge); 859 860 /* if we couldn't allocate memory with a specified page size, 861 * that doesn't mean we can't do it with other page sizes, so 862 * try another one. 863 */ 864 if (addr == NULL) 865 continue; 866 867 /* store IOVA addresses for every page in this memory area */ 868 n_pages = mem_sz / cur_pgsz; 869 870 iovas = malloc(sizeof(*iovas) * n_pages); 871 872 if (iovas == NULL) { 873 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 874 goto fail; 875 } 876 /* lock memory if it's not huge pages */ 877 if (!huge) 878 mlock(addr, mem_sz); 879 880 /* populate IOVA addresses */ 881 for (cur_page = 0; cur_page < n_pages; cur_page++) { 882 rte_iova_t iova; 883 size_t offset; 884 void *cur; 885 886 offset = cur_pgsz * cur_page; 887 cur = RTE_PTR_ADD(addr, offset); 888 889 /* touch the page before getting its IOVA */ 890 *(volatile char *)cur = 0; 891 892 iova = rte_mem_virt2iova(cur); 893 894 iovas[cur_page] = iova; 895 } 896 897 break; 898 } 899 /* if we couldn't allocate anything */ 900 if (iovas == NULL) 901 return -1; 902 903 param->addr = addr; 904 param->len = mem_sz; 905 param->pgsz = cur_pgsz; 906 param->iova_table = iovas; 907 param->iova_table_len = n_pages; 908 909 return 0; 910 fail: 911 free(iovas); 912 if (addr) 913 munmap(addr, mem_sz); 914 915 return -1; 916 } 917 918 static int 919 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 920 { 921 struct extmem_param param; 922 int socket_id, ret; 923 924 memset(¶m, 0, sizeof(param)); 925 926 /* check if our heap exists */ 927 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 928 if (socket_id < 0) { 929 /* create our heap */ 930 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 931 if (ret < 0) { 932 TESTPMD_LOG(ERR, "Cannot create heap\n"); 933 return -1; 934 } 935 } 936 937 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 938 if (ret < 0) { 939 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 940 return -1; 941 } 942 943 /* we now have a valid memory area, so add it to heap */ 944 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 945 param.addr, param.len, param.iova_table, 946 param.iova_table_len, param.pgsz); 947 948 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 949 950 /* not needed any more */ 951 free(param.iova_table); 952 953 if (ret < 0) { 954 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 955 munmap(param.addr, param.len); 956 return -1; 957 } 958 959 /* success */ 960 961 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 962 param.len >> 20); 963 964 return 0; 965 } 966 static void 967 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 968 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 969 { 970 uint16_t pid = 0; 971 int ret; 972 973 RTE_ETH_FOREACH_DEV(pid) { 974 struct rte_eth_dev_info dev_info; 975 976 ret = eth_dev_info_get_print_err(pid, &dev_info); 977 if (ret != 0) { 978 TESTPMD_LOG(DEBUG, 979 "unable to get device info for port %d on addr 0x%p," 980 "mempool unmapping will not be performed\n", 981 pid, memhdr->addr); 982 continue; 983 } 984 985 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 986 if (ret) { 987 TESTPMD_LOG(DEBUG, 988 "unable to DMA unmap addr 0x%p " 989 "for device %s\n", 990 memhdr->addr, dev_info.device->name); 991 } 992 } 993 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 994 if (ret) { 995 TESTPMD_LOG(DEBUG, 996 "unable to un-register addr 0x%p\n", memhdr->addr); 997 } 998 } 999 1000 static void 1001 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1002 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1003 { 1004 uint16_t pid = 0; 1005 size_t page_size = sysconf(_SC_PAGESIZE); 1006 int ret; 1007 1008 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1009 page_size); 1010 if (ret) { 1011 TESTPMD_LOG(DEBUG, 1012 "unable to register addr 0x%p\n", memhdr->addr); 1013 return; 1014 } 1015 RTE_ETH_FOREACH_DEV(pid) { 1016 struct rte_eth_dev_info dev_info; 1017 1018 ret = eth_dev_info_get_print_err(pid, &dev_info); 1019 if (ret != 0) { 1020 TESTPMD_LOG(DEBUG, 1021 "unable to get device info for port %d on addr 0x%p," 1022 "mempool mapping will not be performed\n", 1023 pid, memhdr->addr); 1024 continue; 1025 } 1026 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1027 if (ret) { 1028 TESTPMD_LOG(DEBUG, 1029 "unable to DMA map addr 0x%p " 1030 "for device %s\n", 1031 memhdr->addr, dev_info.device->name); 1032 } 1033 } 1034 } 1035 #endif 1036 1037 static unsigned int 1038 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1039 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1040 { 1041 struct rte_pktmbuf_extmem *xmem; 1042 unsigned int ext_num, zone_num, elt_num; 1043 uint16_t elt_size; 1044 1045 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1046 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1047 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1048 1049 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1050 if (xmem == NULL) { 1051 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1052 "external buffer descriptors\n"); 1053 *ext_mem = NULL; 1054 return 0; 1055 } 1056 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1057 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1058 const struct rte_memzone *mz; 1059 char mz_name[RTE_MEMZONE_NAMESIZE]; 1060 int ret; 1061 1062 ret = snprintf(mz_name, sizeof(mz_name), 1063 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1064 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1065 errno = ENAMETOOLONG; 1066 ext_num = 0; 1067 break; 1068 } 1069 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1070 socket_id, 1071 RTE_MEMZONE_IOVA_CONTIG | 1072 RTE_MEMZONE_1GB | 1073 RTE_MEMZONE_SIZE_HINT_ONLY); 1074 if (mz == NULL) { 1075 /* 1076 * The caller exits on external buffer creation 1077 * error, so there is no need to free memzones. 1078 */ 1079 errno = ENOMEM; 1080 ext_num = 0; 1081 break; 1082 } 1083 xseg->buf_ptr = mz->addr; 1084 xseg->buf_iova = mz->iova; 1085 xseg->buf_len = EXTBUF_ZONE_SIZE; 1086 xseg->elt_size = elt_size; 1087 } 1088 if (ext_num == 0 && xmem != NULL) { 1089 free(xmem); 1090 xmem = NULL; 1091 } 1092 *ext_mem = xmem; 1093 return ext_num; 1094 } 1095 1096 /* 1097 * Configuration initialisation done once at init time. 1098 */ 1099 static struct rte_mempool * 1100 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1101 unsigned int socket_id, uint16_t size_idx) 1102 { 1103 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1104 struct rte_mempool *rte_mp = NULL; 1105 #ifndef RTE_EXEC_ENV_WINDOWS 1106 uint32_t mb_size; 1107 1108 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1109 #endif 1110 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1111 if (!is_proc_primary()) { 1112 rte_mp = rte_mempool_lookup(pool_name); 1113 if (rte_mp == NULL) 1114 rte_exit(EXIT_FAILURE, 1115 "Get mbuf pool for socket %u failed: %s\n", 1116 socket_id, rte_strerror(rte_errno)); 1117 return rte_mp; 1118 } 1119 1120 TESTPMD_LOG(INFO, 1121 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1122 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1123 1124 switch (mp_alloc_type) { 1125 case MP_ALLOC_NATIVE: 1126 { 1127 /* wrapper to rte_mempool_create() */ 1128 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1129 rte_mbuf_best_mempool_ops()); 1130 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1131 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1132 break; 1133 } 1134 #ifndef RTE_EXEC_ENV_WINDOWS 1135 case MP_ALLOC_ANON: 1136 { 1137 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1138 mb_size, (unsigned int) mb_mempool_cache, 1139 sizeof(struct rte_pktmbuf_pool_private), 1140 socket_id, mempool_flags); 1141 if (rte_mp == NULL) 1142 goto err; 1143 1144 if (rte_mempool_populate_anon(rte_mp) == 0) { 1145 rte_mempool_free(rte_mp); 1146 rte_mp = NULL; 1147 goto err; 1148 } 1149 rte_pktmbuf_pool_init(rte_mp, NULL); 1150 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1151 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1152 break; 1153 } 1154 case MP_ALLOC_XMEM: 1155 case MP_ALLOC_XMEM_HUGE: 1156 { 1157 int heap_socket; 1158 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1159 1160 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1161 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1162 1163 heap_socket = 1164 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1165 if (heap_socket < 0) 1166 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1167 1168 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1169 rte_mbuf_best_mempool_ops()); 1170 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1171 mb_mempool_cache, 0, mbuf_seg_size, 1172 heap_socket); 1173 break; 1174 } 1175 #endif 1176 case MP_ALLOC_XBUF: 1177 { 1178 struct rte_pktmbuf_extmem *ext_mem; 1179 unsigned int ext_num; 1180 1181 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1182 socket_id, pool_name, &ext_mem); 1183 if (ext_num == 0) 1184 rte_exit(EXIT_FAILURE, 1185 "Can't create pinned data buffers\n"); 1186 1187 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1188 rte_mbuf_best_mempool_ops()); 1189 rte_mp = rte_pktmbuf_pool_create_extbuf 1190 (pool_name, nb_mbuf, mb_mempool_cache, 1191 0, mbuf_seg_size, socket_id, 1192 ext_mem, ext_num); 1193 free(ext_mem); 1194 break; 1195 } 1196 default: 1197 { 1198 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1199 } 1200 } 1201 1202 #ifndef RTE_EXEC_ENV_WINDOWS 1203 err: 1204 #endif 1205 if (rte_mp == NULL) { 1206 rte_exit(EXIT_FAILURE, 1207 "Creation of mbuf pool for socket %u failed: %s\n", 1208 socket_id, rte_strerror(rte_errno)); 1209 } else if (verbose_level > 0) { 1210 rte_mempool_dump(stdout, rte_mp); 1211 } 1212 return rte_mp; 1213 } 1214 1215 /* 1216 * Check given socket id is valid or not with NUMA mode, 1217 * if valid, return 0, else return -1 1218 */ 1219 static int 1220 check_socket_id(const unsigned int socket_id) 1221 { 1222 static int warning_once = 0; 1223 1224 if (new_socket_id(socket_id)) { 1225 if (!warning_once && numa_support) 1226 fprintf(stderr, 1227 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1228 warning_once = 1; 1229 return -1; 1230 } 1231 return 0; 1232 } 1233 1234 /* 1235 * Get the allowed maximum number of RX queues. 1236 * *pid return the port id which has minimal value of 1237 * max_rx_queues in all ports. 1238 */ 1239 queueid_t 1240 get_allowed_max_nb_rxq(portid_t *pid) 1241 { 1242 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1243 bool max_rxq_valid = false; 1244 portid_t pi; 1245 struct rte_eth_dev_info dev_info; 1246 1247 RTE_ETH_FOREACH_DEV(pi) { 1248 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1249 continue; 1250 1251 max_rxq_valid = true; 1252 if (dev_info.max_rx_queues < allowed_max_rxq) { 1253 allowed_max_rxq = dev_info.max_rx_queues; 1254 *pid = pi; 1255 } 1256 } 1257 return max_rxq_valid ? allowed_max_rxq : 0; 1258 } 1259 1260 /* 1261 * Check input rxq is valid or not. 1262 * If input rxq is not greater than any of maximum number 1263 * of RX queues of all ports, it is valid. 1264 * if valid, return 0, else return -1 1265 */ 1266 int 1267 check_nb_rxq(queueid_t rxq) 1268 { 1269 queueid_t allowed_max_rxq; 1270 portid_t pid = 0; 1271 1272 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1273 if (rxq > allowed_max_rxq) { 1274 fprintf(stderr, 1275 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1276 rxq, allowed_max_rxq, pid); 1277 return -1; 1278 } 1279 return 0; 1280 } 1281 1282 /* 1283 * Get the allowed maximum number of TX queues. 1284 * *pid return the port id which has minimal value of 1285 * max_tx_queues in all ports. 1286 */ 1287 queueid_t 1288 get_allowed_max_nb_txq(portid_t *pid) 1289 { 1290 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1291 bool max_txq_valid = false; 1292 portid_t pi; 1293 struct rte_eth_dev_info dev_info; 1294 1295 RTE_ETH_FOREACH_DEV(pi) { 1296 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1297 continue; 1298 1299 max_txq_valid = true; 1300 if (dev_info.max_tx_queues < allowed_max_txq) { 1301 allowed_max_txq = dev_info.max_tx_queues; 1302 *pid = pi; 1303 } 1304 } 1305 return max_txq_valid ? allowed_max_txq : 0; 1306 } 1307 1308 /* 1309 * Check input txq is valid or not. 1310 * If input txq is not greater than any of maximum number 1311 * of TX queues of all ports, it is valid. 1312 * if valid, return 0, else return -1 1313 */ 1314 int 1315 check_nb_txq(queueid_t txq) 1316 { 1317 queueid_t allowed_max_txq; 1318 portid_t pid = 0; 1319 1320 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1321 if (txq > allowed_max_txq) { 1322 fprintf(stderr, 1323 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1324 txq, allowed_max_txq, pid); 1325 return -1; 1326 } 1327 return 0; 1328 } 1329 1330 /* 1331 * Get the allowed maximum number of RXDs of every rx queue. 1332 * *pid return the port id which has minimal value of 1333 * max_rxd in all queues of all ports. 1334 */ 1335 static uint16_t 1336 get_allowed_max_nb_rxd(portid_t *pid) 1337 { 1338 uint16_t allowed_max_rxd = UINT16_MAX; 1339 portid_t pi; 1340 struct rte_eth_dev_info dev_info; 1341 1342 RTE_ETH_FOREACH_DEV(pi) { 1343 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1344 continue; 1345 1346 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1347 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1348 *pid = pi; 1349 } 1350 } 1351 return allowed_max_rxd; 1352 } 1353 1354 /* 1355 * Get the allowed minimal number of RXDs of every rx queue. 1356 * *pid return the port id which has minimal value of 1357 * min_rxd in all queues of all ports. 1358 */ 1359 static uint16_t 1360 get_allowed_min_nb_rxd(portid_t *pid) 1361 { 1362 uint16_t allowed_min_rxd = 0; 1363 portid_t pi; 1364 struct rte_eth_dev_info dev_info; 1365 1366 RTE_ETH_FOREACH_DEV(pi) { 1367 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1368 continue; 1369 1370 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1371 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1372 *pid = pi; 1373 } 1374 } 1375 1376 return allowed_min_rxd; 1377 } 1378 1379 /* 1380 * Check input rxd is valid or not. 1381 * If input rxd is not greater than any of maximum number 1382 * of RXDs of every Rx queues and is not less than any of 1383 * minimal number of RXDs of every Rx queues, it is valid. 1384 * if valid, return 0, else return -1 1385 */ 1386 int 1387 check_nb_rxd(queueid_t rxd) 1388 { 1389 uint16_t allowed_max_rxd; 1390 uint16_t allowed_min_rxd; 1391 portid_t pid = 0; 1392 1393 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1394 if (rxd > allowed_max_rxd) { 1395 fprintf(stderr, 1396 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1397 rxd, allowed_max_rxd, pid); 1398 return -1; 1399 } 1400 1401 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1402 if (rxd < allowed_min_rxd) { 1403 fprintf(stderr, 1404 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1405 rxd, allowed_min_rxd, pid); 1406 return -1; 1407 } 1408 1409 return 0; 1410 } 1411 1412 /* 1413 * Get the allowed maximum number of TXDs of every rx queues. 1414 * *pid return the port id which has minimal value of 1415 * max_txd in every tx queue. 1416 */ 1417 static uint16_t 1418 get_allowed_max_nb_txd(portid_t *pid) 1419 { 1420 uint16_t allowed_max_txd = UINT16_MAX; 1421 portid_t pi; 1422 struct rte_eth_dev_info dev_info; 1423 1424 RTE_ETH_FOREACH_DEV(pi) { 1425 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1426 continue; 1427 1428 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1429 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1430 *pid = pi; 1431 } 1432 } 1433 return allowed_max_txd; 1434 } 1435 1436 /* 1437 * Get the allowed maximum number of TXDs of every tx queues. 1438 * *pid return the port id which has minimal value of 1439 * min_txd in every tx queue. 1440 */ 1441 static uint16_t 1442 get_allowed_min_nb_txd(portid_t *pid) 1443 { 1444 uint16_t allowed_min_txd = 0; 1445 portid_t pi; 1446 struct rte_eth_dev_info dev_info; 1447 1448 RTE_ETH_FOREACH_DEV(pi) { 1449 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1450 continue; 1451 1452 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1453 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1454 *pid = pi; 1455 } 1456 } 1457 1458 return allowed_min_txd; 1459 } 1460 1461 /* 1462 * Check input txd is valid or not. 1463 * If input txd is not greater than any of maximum number 1464 * of TXDs of every Rx queues, it is valid. 1465 * if valid, return 0, else return -1 1466 */ 1467 int 1468 check_nb_txd(queueid_t txd) 1469 { 1470 uint16_t allowed_max_txd; 1471 uint16_t allowed_min_txd; 1472 portid_t pid = 0; 1473 1474 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1475 if (txd > allowed_max_txd) { 1476 fprintf(stderr, 1477 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1478 txd, allowed_max_txd, pid); 1479 return -1; 1480 } 1481 1482 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1483 if (txd < allowed_min_txd) { 1484 fprintf(stderr, 1485 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1486 txd, allowed_min_txd, pid); 1487 return -1; 1488 } 1489 return 0; 1490 } 1491 1492 1493 /* 1494 * Get the allowed maximum number of hairpin queues. 1495 * *pid return the port id which has minimal value of 1496 * max_hairpin_queues in all ports. 1497 */ 1498 queueid_t 1499 get_allowed_max_nb_hairpinq(portid_t *pid) 1500 { 1501 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1502 portid_t pi; 1503 struct rte_eth_hairpin_cap cap; 1504 1505 RTE_ETH_FOREACH_DEV(pi) { 1506 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1507 *pid = pi; 1508 return 0; 1509 } 1510 if (cap.max_nb_queues < allowed_max_hairpinq) { 1511 allowed_max_hairpinq = cap.max_nb_queues; 1512 *pid = pi; 1513 } 1514 } 1515 return allowed_max_hairpinq; 1516 } 1517 1518 /* 1519 * Check input hairpin is valid or not. 1520 * If input hairpin is not greater than any of maximum number 1521 * of hairpin queues of all ports, it is valid. 1522 * if valid, return 0, else return -1 1523 */ 1524 int 1525 check_nb_hairpinq(queueid_t hairpinq) 1526 { 1527 queueid_t allowed_max_hairpinq; 1528 portid_t pid = 0; 1529 1530 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1531 if (hairpinq > allowed_max_hairpinq) { 1532 fprintf(stderr, 1533 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1534 hairpinq, allowed_max_hairpinq, pid); 1535 return -1; 1536 } 1537 return 0; 1538 } 1539 1540 static int 1541 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1542 { 1543 uint32_t eth_overhead; 1544 1545 if (dev_info->max_mtu != UINT16_MAX && 1546 dev_info->max_rx_pktlen > dev_info->max_mtu) 1547 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1548 else 1549 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1550 1551 return eth_overhead; 1552 } 1553 1554 static void 1555 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1556 { 1557 struct rte_port *port = &ports[pid]; 1558 int ret; 1559 int i; 1560 1561 eth_rx_metadata_negotiate_mp(pid); 1562 1563 port->dev_conf.txmode = tx_mode; 1564 port->dev_conf.rxmode = rx_mode; 1565 1566 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1567 if (ret != 0) 1568 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1569 1570 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1571 port->dev_conf.txmode.offloads &= 1572 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1573 1574 /* Apply Rx offloads configuration */ 1575 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1576 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads; 1577 /* Apply Tx offloads configuration */ 1578 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1579 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads; 1580 1581 if (eth_link_speed) 1582 port->dev_conf.link_speeds = eth_link_speed; 1583 1584 if (max_rx_pkt_len) 1585 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1586 get_eth_overhead(&port->dev_info); 1587 1588 /* set flag to initialize port/queue */ 1589 port->need_reconfig = 1; 1590 port->need_reconfig_queues = 1; 1591 port->socket_id = socket_id; 1592 port->tx_metadata = 0; 1593 1594 /* 1595 * Check for maximum number of segments per MTU. 1596 * Accordingly update the mbuf data size. 1597 */ 1598 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1599 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1600 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1601 uint16_t mtu; 1602 1603 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1604 uint16_t data_size = (mtu + eth_overhead) / 1605 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1606 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1607 1608 if (buffer_size > mbuf_data_size[0]) { 1609 mbuf_data_size[0] = buffer_size; 1610 TESTPMD_LOG(WARNING, 1611 "Configured mbuf size of the first segment %hu\n", 1612 mbuf_data_size[0]); 1613 } 1614 } 1615 } 1616 } 1617 1618 static void 1619 init_config(void) 1620 { 1621 portid_t pid; 1622 struct rte_mempool *mbp; 1623 unsigned int nb_mbuf_per_pool; 1624 lcoreid_t lc_id; 1625 #ifdef RTE_LIB_GRO 1626 struct rte_gro_param gro_param; 1627 #endif 1628 #ifdef RTE_LIB_GSO 1629 uint32_t gso_types; 1630 #endif 1631 1632 /* Configuration of logical cores. */ 1633 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1634 sizeof(struct fwd_lcore *) * nb_lcores, 1635 RTE_CACHE_LINE_SIZE); 1636 if (fwd_lcores == NULL) { 1637 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1638 "failed\n", nb_lcores); 1639 } 1640 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1641 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1642 sizeof(struct fwd_lcore), 1643 RTE_CACHE_LINE_SIZE); 1644 if (fwd_lcores[lc_id] == NULL) { 1645 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1646 "failed\n"); 1647 } 1648 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1649 } 1650 1651 RTE_ETH_FOREACH_DEV(pid) { 1652 uint32_t socket_id; 1653 1654 if (numa_support) { 1655 socket_id = port_numa[pid]; 1656 if (port_numa[pid] == NUMA_NO_CONFIG) { 1657 socket_id = rte_eth_dev_socket_id(pid); 1658 1659 /* 1660 * if socket_id is invalid, 1661 * set to the first available socket. 1662 */ 1663 if (check_socket_id(socket_id) < 0) 1664 socket_id = socket_ids[0]; 1665 } 1666 } else { 1667 socket_id = (socket_num == UMA_NO_CONFIG) ? 1668 0 : socket_num; 1669 } 1670 /* Apply default TxRx configuration for all ports */ 1671 init_config_port_offloads(pid, socket_id); 1672 } 1673 /* 1674 * Create pools of mbuf. 1675 * If NUMA support is disabled, create a single pool of mbuf in 1676 * socket 0 memory by default. 1677 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1678 * 1679 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1680 * nb_txd can be configured at run time. 1681 */ 1682 if (param_total_num_mbufs) 1683 nb_mbuf_per_pool = param_total_num_mbufs; 1684 else { 1685 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1686 (nb_lcores * mb_mempool_cache) + 1687 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1688 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1689 } 1690 1691 if (numa_support) { 1692 uint8_t i, j; 1693 1694 for (i = 0; i < num_sockets; i++) 1695 for (j = 0; j < mbuf_data_size_n; j++) 1696 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1697 mbuf_pool_create(mbuf_data_size[j], 1698 nb_mbuf_per_pool, 1699 socket_ids[i], j); 1700 } else { 1701 uint8_t i; 1702 1703 for (i = 0; i < mbuf_data_size_n; i++) 1704 mempools[i] = mbuf_pool_create 1705 (mbuf_data_size[i], 1706 nb_mbuf_per_pool, 1707 socket_num == UMA_NO_CONFIG ? 1708 0 : socket_num, i); 1709 } 1710 1711 init_port_config(); 1712 1713 #ifdef RTE_LIB_GSO 1714 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1715 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1716 #endif 1717 /* 1718 * Records which Mbuf pool to use by each logical core, if needed. 1719 */ 1720 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1721 mbp = mbuf_pool_find( 1722 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1723 1724 if (mbp == NULL) 1725 mbp = mbuf_pool_find(0, 0); 1726 fwd_lcores[lc_id]->mbp = mbp; 1727 #ifdef RTE_LIB_GSO 1728 /* initialize GSO context */ 1729 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1730 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1731 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1732 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1733 RTE_ETHER_CRC_LEN; 1734 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1735 #endif 1736 } 1737 1738 fwd_config_setup(); 1739 1740 #ifdef RTE_LIB_GRO 1741 /* create a gro context for each lcore */ 1742 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1743 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1744 gro_param.max_item_per_flow = MAX_PKT_BURST; 1745 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1746 gro_param.socket_id = rte_lcore_to_socket_id( 1747 fwd_lcores_cpuids[lc_id]); 1748 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1749 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1750 rte_exit(EXIT_FAILURE, 1751 "rte_gro_ctx_create() failed\n"); 1752 } 1753 } 1754 #endif 1755 } 1756 1757 1758 void 1759 reconfig(portid_t new_port_id, unsigned socket_id) 1760 { 1761 /* Reconfiguration of Ethernet ports. */ 1762 init_config_port_offloads(new_port_id, socket_id); 1763 init_port_config(); 1764 } 1765 1766 1767 int 1768 init_fwd_streams(void) 1769 { 1770 portid_t pid; 1771 struct rte_port *port; 1772 streamid_t sm_id, nb_fwd_streams_new; 1773 queueid_t q; 1774 1775 /* set socket id according to numa or not */ 1776 RTE_ETH_FOREACH_DEV(pid) { 1777 port = &ports[pid]; 1778 if (nb_rxq > port->dev_info.max_rx_queues) { 1779 fprintf(stderr, 1780 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1781 nb_rxq, port->dev_info.max_rx_queues); 1782 return -1; 1783 } 1784 if (nb_txq > port->dev_info.max_tx_queues) { 1785 fprintf(stderr, 1786 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1787 nb_txq, port->dev_info.max_tx_queues); 1788 return -1; 1789 } 1790 if (numa_support) { 1791 if (port_numa[pid] != NUMA_NO_CONFIG) 1792 port->socket_id = port_numa[pid]; 1793 else { 1794 port->socket_id = rte_eth_dev_socket_id(pid); 1795 1796 /* 1797 * if socket_id is invalid, 1798 * set to the first available socket. 1799 */ 1800 if (check_socket_id(port->socket_id) < 0) 1801 port->socket_id = socket_ids[0]; 1802 } 1803 } 1804 else { 1805 if (socket_num == UMA_NO_CONFIG) 1806 port->socket_id = 0; 1807 else 1808 port->socket_id = socket_num; 1809 } 1810 } 1811 1812 q = RTE_MAX(nb_rxq, nb_txq); 1813 if (q == 0) { 1814 fprintf(stderr, 1815 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1816 return -1; 1817 } 1818 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1819 if (nb_fwd_streams_new == nb_fwd_streams) 1820 return 0; 1821 /* clear the old */ 1822 if (fwd_streams != NULL) { 1823 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1824 if (fwd_streams[sm_id] == NULL) 1825 continue; 1826 rte_free(fwd_streams[sm_id]); 1827 fwd_streams[sm_id] = NULL; 1828 } 1829 rte_free(fwd_streams); 1830 fwd_streams = NULL; 1831 } 1832 1833 /* init new */ 1834 nb_fwd_streams = nb_fwd_streams_new; 1835 if (nb_fwd_streams) { 1836 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1837 sizeof(struct fwd_stream *) * nb_fwd_streams, 1838 RTE_CACHE_LINE_SIZE); 1839 if (fwd_streams == NULL) 1840 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1841 " (struct fwd_stream *)) failed\n", 1842 nb_fwd_streams); 1843 1844 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1845 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1846 " struct fwd_stream", sizeof(struct fwd_stream), 1847 RTE_CACHE_LINE_SIZE); 1848 if (fwd_streams[sm_id] == NULL) 1849 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1850 "(struct fwd_stream) failed\n"); 1851 } 1852 } 1853 1854 return 0; 1855 } 1856 1857 static void 1858 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1859 { 1860 uint64_t total_burst, sburst; 1861 uint64_t nb_burst; 1862 uint64_t burst_stats[4]; 1863 uint16_t pktnb_stats[4]; 1864 uint16_t nb_pkt; 1865 int burst_percent[4], sburstp; 1866 int i; 1867 1868 /* 1869 * First compute the total number of packet bursts and the 1870 * two highest numbers of bursts of the same number of packets. 1871 */ 1872 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1873 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1874 1875 /* Show stats for 0 burst size always */ 1876 total_burst = pbs->pkt_burst_spread[0]; 1877 burst_stats[0] = pbs->pkt_burst_spread[0]; 1878 pktnb_stats[0] = 0; 1879 1880 /* Find the next 2 burst sizes with highest occurrences. */ 1881 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1882 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1883 1884 if (nb_burst == 0) 1885 continue; 1886 1887 total_burst += nb_burst; 1888 1889 if (nb_burst > burst_stats[1]) { 1890 burst_stats[2] = burst_stats[1]; 1891 pktnb_stats[2] = pktnb_stats[1]; 1892 burst_stats[1] = nb_burst; 1893 pktnb_stats[1] = nb_pkt; 1894 } else if (nb_burst > burst_stats[2]) { 1895 burst_stats[2] = nb_burst; 1896 pktnb_stats[2] = nb_pkt; 1897 } 1898 } 1899 if (total_burst == 0) 1900 return; 1901 1902 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1903 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1904 if (i == 3) { 1905 printf("%d%% of other]\n", 100 - sburstp); 1906 return; 1907 } 1908 1909 sburst += burst_stats[i]; 1910 if (sburst == total_burst) { 1911 printf("%d%% of %d pkts]\n", 1912 100 - sburstp, (int) pktnb_stats[i]); 1913 return; 1914 } 1915 1916 burst_percent[i] = 1917 (double)burst_stats[i] / total_burst * 100; 1918 printf("%d%% of %d pkts + ", 1919 burst_percent[i], (int) pktnb_stats[i]); 1920 sburstp += burst_percent[i]; 1921 } 1922 } 1923 1924 static void 1925 fwd_stream_stats_display(streamid_t stream_id) 1926 { 1927 struct fwd_stream *fs; 1928 static const char *fwd_top_stats_border = "-------"; 1929 1930 fs = fwd_streams[stream_id]; 1931 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1932 (fs->fwd_dropped == 0)) 1933 return; 1934 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1935 "TX Port=%2d/Queue=%2d %s\n", 1936 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1937 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1938 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1939 " TX-dropped: %-14"PRIu64, 1940 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1941 1942 /* if checksum mode */ 1943 if (cur_fwd_eng == &csum_fwd_engine) { 1944 printf(" RX- bad IP checksum: %-14"PRIu64 1945 " Rx- bad L4 checksum: %-14"PRIu64 1946 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1947 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1948 fs->rx_bad_outer_l4_csum); 1949 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1950 fs->rx_bad_outer_ip_csum); 1951 } else { 1952 printf("\n"); 1953 } 1954 1955 if (record_burst_stats) { 1956 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1957 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1958 } 1959 } 1960 1961 void 1962 fwd_stats_display(void) 1963 { 1964 static const char *fwd_stats_border = "----------------------"; 1965 static const char *acc_stats_border = "+++++++++++++++"; 1966 struct { 1967 struct fwd_stream *rx_stream; 1968 struct fwd_stream *tx_stream; 1969 uint64_t tx_dropped; 1970 uint64_t rx_bad_ip_csum; 1971 uint64_t rx_bad_l4_csum; 1972 uint64_t rx_bad_outer_l4_csum; 1973 uint64_t rx_bad_outer_ip_csum; 1974 } ports_stats[RTE_MAX_ETHPORTS]; 1975 uint64_t total_rx_dropped = 0; 1976 uint64_t total_tx_dropped = 0; 1977 uint64_t total_rx_nombuf = 0; 1978 struct rte_eth_stats stats; 1979 uint64_t fwd_cycles = 0; 1980 uint64_t total_recv = 0; 1981 uint64_t total_xmit = 0; 1982 struct rte_port *port; 1983 streamid_t sm_id; 1984 portid_t pt_id; 1985 int ret; 1986 int i; 1987 1988 memset(ports_stats, 0, sizeof(ports_stats)); 1989 1990 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1991 struct fwd_stream *fs = fwd_streams[sm_id]; 1992 1993 if (cur_fwd_config.nb_fwd_streams > 1994 cur_fwd_config.nb_fwd_ports) { 1995 fwd_stream_stats_display(sm_id); 1996 } else { 1997 ports_stats[fs->tx_port].tx_stream = fs; 1998 ports_stats[fs->rx_port].rx_stream = fs; 1999 } 2000 2001 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2002 2003 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2004 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2005 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2006 fs->rx_bad_outer_l4_csum; 2007 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2008 fs->rx_bad_outer_ip_csum; 2009 2010 if (record_core_cycles) 2011 fwd_cycles += fs->core_cycles; 2012 } 2013 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2014 pt_id = fwd_ports_ids[i]; 2015 port = &ports[pt_id]; 2016 2017 ret = rte_eth_stats_get(pt_id, &stats); 2018 if (ret != 0) { 2019 fprintf(stderr, 2020 "%s: Error: failed to get stats (port %u): %d", 2021 __func__, pt_id, ret); 2022 continue; 2023 } 2024 stats.ipackets -= port->stats.ipackets; 2025 stats.opackets -= port->stats.opackets; 2026 stats.ibytes -= port->stats.ibytes; 2027 stats.obytes -= port->stats.obytes; 2028 stats.imissed -= port->stats.imissed; 2029 stats.oerrors -= port->stats.oerrors; 2030 stats.rx_nombuf -= port->stats.rx_nombuf; 2031 2032 total_recv += stats.ipackets; 2033 total_xmit += stats.opackets; 2034 total_rx_dropped += stats.imissed; 2035 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2036 total_tx_dropped += stats.oerrors; 2037 total_rx_nombuf += stats.rx_nombuf; 2038 2039 printf("\n %s Forward statistics for port %-2d %s\n", 2040 fwd_stats_border, pt_id, fwd_stats_border); 2041 2042 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2043 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2044 stats.ipackets + stats.imissed); 2045 2046 if (cur_fwd_eng == &csum_fwd_engine) { 2047 printf(" Bad-ipcsum: %-14"PRIu64 2048 " Bad-l4csum: %-14"PRIu64 2049 "Bad-outer-l4csum: %-14"PRIu64"\n", 2050 ports_stats[pt_id].rx_bad_ip_csum, 2051 ports_stats[pt_id].rx_bad_l4_csum, 2052 ports_stats[pt_id].rx_bad_outer_l4_csum); 2053 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2054 ports_stats[pt_id].rx_bad_outer_ip_csum); 2055 } 2056 if (stats.ierrors + stats.rx_nombuf > 0) { 2057 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2058 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2059 } 2060 2061 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2062 "TX-total: %-"PRIu64"\n", 2063 stats.opackets, ports_stats[pt_id].tx_dropped, 2064 stats.opackets + ports_stats[pt_id].tx_dropped); 2065 2066 if (record_burst_stats) { 2067 if (ports_stats[pt_id].rx_stream) 2068 pkt_burst_stats_display("RX", 2069 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2070 if (ports_stats[pt_id].tx_stream) 2071 pkt_burst_stats_display("TX", 2072 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2073 } 2074 2075 printf(" %s--------------------------------%s\n", 2076 fwd_stats_border, fwd_stats_border); 2077 } 2078 2079 printf("\n %s Accumulated forward statistics for all ports" 2080 "%s\n", 2081 acc_stats_border, acc_stats_border); 2082 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2083 "%-"PRIu64"\n" 2084 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2085 "%-"PRIu64"\n", 2086 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2087 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2088 if (total_rx_nombuf > 0) 2089 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2090 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2091 "%s\n", 2092 acc_stats_border, acc_stats_border); 2093 if (record_core_cycles) { 2094 #define CYC_PER_MHZ 1E6 2095 if (total_recv > 0 || total_xmit > 0) { 2096 uint64_t total_pkts = 0; 2097 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2098 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2099 total_pkts = total_xmit; 2100 else 2101 total_pkts = total_recv; 2102 2103 printf("\n CPU cycles/packet=%.2F (total cycles=" 2104 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2105 " MHz Clock\n", 2106 (double) fwd_cycles / total_pkts, 2107 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2108 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2109 } 2110 } 2111 } 2112 2113 void 2114 fwd_stats_reset(void) 2115 { 2116 streamid_t sm_id; 2117 portid_t pt_id; 2118 int ret; 2119 int i; 2120 2121 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2122 pt_id = fwd_ports_ids[i]; 2123 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2124 if (ret != 0) 2125 fprintf(stderr, 2126 "%s: Error: failed to clear stats (port %u):%d", 2127 __func__, pt_id, ret); 2128 } 2129 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2130 struct fwd_stream *fs = fwd_streams[sm_id]; 2131 2132 fs->rx_packets = 0; 2133 fs->tx_packets = 0; 2134 fs->fwd_dropped = 0; 2135 fs->rx_bad_ip_csum = 0; 2136 fs->rx_bad_l4_csum = 0; 2137 fs->rx_bad_outer_l4_csum = 0; 2138 fs->rx_bad_outer_ip_csum = 0; 2139 2140 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2141 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2142 fs->core_cycles = 0; 2143 } 2144 } 2145 2146 static void 2147 flush_fwd_rx_queues(void) 2148 { 2149 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2150 portid_t rxp; 2151 portid_t port_id; 2152 queueid_t rxq; 2153 uint16_t nb_rx; 2154 uint16_t i; 2155 uint8_t j; 2156 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2157 uint64_t timer_period; 2158 2159 if (num_procs > 1) { 2160 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2161 return; 2162 } 2163 2164 /* convert to number of cycles */ 2165 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2166 2167 for (j = 0; j < 2; j++) { 2168 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2169 for (rxq = 0; rxq < nb_rxq; rxq++) { 2170 port_id = fwd_ports_ids[rxp]; 2171 /** 2172 * testpmd can stuck in the below do while loop 2173 * if rte_eth_rx_burst() always returns nonzero 2174 * packets. So timer is added to exit this loop 2175 * after 1sec timer expiry. 2176 */ 2177 prev_tsc = rte_rdtsc(); 2178 do { 2179 nb_rx = rte_eth_rx_burst(port_id, rxq, 2180 pkts_burst, MAX_PKT_BURST); 2181 for (i = 0; i < nb_rx; i++) 2182 rte_pktmbuf_free(pkts_burst[i]); 2183 2184 cur_tsc = rte_rdtsc(); 2185 diff_tsc = cur_tsc - prev_tsc; 2186 timer_tsc += diff_tsc; 2187 } while ((nb_rx > 0) && 2188 (timer_tsc < timer_period)); 2189 timer_tsc = 0; 2190 } 2191 } 2192 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2193 } 2194 } 2195 2196 static void 2197 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2198 { 2199 struct fwd_stream **fsm; 2200 streamid_t nb_fs; 2201 streamid_t sm_id; 2202 #ifdef RTE_LIB_BITRATESTATS 2203 uint64_t tics_per_1sec; 2204 uint64_t tics_datum; 2205 uint64_t tics_current; 2206 uint16_t i, cnt_ports; 2207 2208 cnt_ports = nb_ports; 2209 tics_datum = rte_rdtsc(); 2210 tics_per_1sec = rte_get_timer_hz(); 2211 #endif 2212 fsm = &fwd_streams[fc->stream_idx]; 2213 nb_fs = fc->stream_nb; 2214 do { 2215 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2216 (*pkt_fwd)(fsm[sm_id]); 2217 #ifdef RTE_LIB_BITRATESTATS 2218 if (bitrate_enabled != 0 && 2219 bitrate_lcore_id == rte_lcore_id()) { 2220 tics_current = rte_rdtsc(); 2221 if (tics_current - tics_datum >= tics_per_1sec) { 2222 /* Periodic bitrate calculation */ 2223 for (i = 0; i < cnt_ports; i++) 2224 rte_stats_bitrate_calc(bitrate_data, 2225 ports_ids[i]); 2226 tics_datum = tics_current; 2227 } 2228 } 2229 #endif 2230 #ifdef RTE_LIB_LATENCYSTATS 2231 if (latencystats_enabled != 0 && 2232 latencystats_lcore_id == rte_lcore_id()) 2233 rte_latencystats_update(); 2234 #endif 2235 2236 } while (! fc->stopped); 2237 } 2238 2239 static int 2240 start_pkt_forward_on_core(void *fwd_arg) 2241 { 2242 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2243 cur_fwd_config.fwd_eng->packet_fwd); 2244 return 0; 2245 } 2246 2247 /* 2248 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2249 * Used to start communication flows in network loopback test configurations. 2250 */ 2251 static int 2252 run_one_txonly_burst_on_core(void *fwd_arg) 2253 { 2254 struct fwd_lcore *fwd_lc; 2255 struct fwd_lcore tmp_lcore; 2256 2257 fwd_lc = (struct fwd_lcore *) fwd_arg; 2258 tmp_lcore = *fwd_lc; 2259 tmp_lcore.stopped = 1; 2260 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2261 return 0; 2262 } 2263 2264 /* 2265 * Launch packet forwarding: 2266 * - Setup per-port forwarding context. 2267 * - launch logical cores with their forwarding configuration. 2268 */ 2269 static void 2270 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2271 { 2272 unsigned int i; 2273 unsigned int lc_id; 2274 int diag; 2275 2276 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2277 lc_id = fwd_lcores_cpuids[i]; 2278 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2279 fwd_lcores[i]->stopped = 0; 2280 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2281 fwd_lcores[i], lc_id); 2282 if (diag != 0) 2283 fprintf(stderr, 2284 "launch lcore %u failed - diag=%d\n", 2285 lc_id, diag); 2286 } 2287 } 2288 } 2289 2290 /* 2291 * Launch packet forwarding configuration. 2292 */ 2293 void 2294 start_packet_forwarding(int with_tx_first) 2295 { 2296 port_fwd_begin_t port_fwd_begin; 2297 port_fwd_end_t port_fwd_end; 2298 unsigned int i; 2299 2300 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2301 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2302 2303 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2304 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2305 2306 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2307 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2308 (!nb_rxq || !nb_txq)) 2309 rte_exit(EXIT_FAILURE, 2310 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2311 cur_fwd_eng->fwd_mode_name); 2312 2313 if (all_ports_started() == 0) { 2314 fprintf(stderr, "Not all ports were started\n"); 2315 return; 2316 } 2317 if (test_done == 0) { 2318 fprintf(stderr, "Packet forwarding already started\n"); 2319 return; 2320 } 2321 2322 fwd_config_setup(); 2323 2324 pkt_fwd_config_display(&cur_fwd_config); 2325 if (!pkt_fwd_shared_rxq_check()) 2326 return; 2327 2328 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2329 if (port_fwd_begin != NULL) { 2330 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2331 if (port_fwd_begin(fwd_ports_ids[i])) { 2332 fprintf(stderr, 2333 "Packet forwarding is not ready\n"); 2334 return; 2335 } 2336 } 2337 } 2338 2339 if (with_tx_first) { 2340 port_fwd_begin = tx_only_engine.port_fwd_begin; 2341 if (port_fwd_begin != NULL) { 2342 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2343 if (port_fwd_begin(fwd_ports_ids[i])) { 2344 fprintf(stderr, 2345 "Packet forwarding is not ready\n"); 2346 return; 2347 } 2348 } 2349 } 2350 } 2351 2352 test_done = 0; 2353 2354 if(!no_flush_rx) 2355 flush_fwd_rx_queues(); 2356 2357 rxtx_config_display(); 2358 2359 fwd_stats_reset(); 2360 if (with_tx_first) { 2361 while (with_tx_first--) { 2362 launch_packet_forwarding( 2363 run_one_txonly_burst_on_core); 2364 rte_eal_mp_wait_lcore(); 2365 } 2366 port_fwd_end = tx_only_engine.port_fwd_end; 2367 if (port_fwd_end != NULL) { 2368 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2369 (*port_fwd_end)(fwd_ports_ids[i]); 2370 } 2371 } 2372 launch_packet_forwarding(start_pkt_forward_on_core); 2373 } 2374 2375 void 2376 stop_packet_forwarding(void) 2377 { 2378 port_fwd_end_t port_fwd_end; 2379 lcoreid_t lc_id; 2380 portid_t pt_id; 2381 int i; 2382 2383 if (test_done) { 2384 fprintf(stderr, "Packet forwarding not started\n"); 2385 return; 2386 } 2387 printf("Telling cores to stop..."); 2388 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2389 fwd_lcores[lc_id]->stopped = 1; 2390 printf("\nWaiting for lcores to finish...\n"); 2391 rte_eal_mp_wait_lcore(); 2392 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2393 if (port_fwd_end != NULL) { 2394 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2395 pt_id = fwd_ports_ids[i]; 2396 (*port_fwd_end)(pt_id); 2397 } 2398 } 2399 2400 fwd_stats_display(); 2401 2402 printf("\nDone.\n"); 2403 test_done = 1; 2404 } 2405 2406 void 2407 dev_set_link_up(portid_t pid) 2408 { 2409 if (rte_eth_dev_set_link_up(pid) < 0) 2410 fprintf(stderr, "\nSet link up fail.\n"); 2411 } 2412 2413 void 2414 dev_set_link_down(portid_t pid) 2415 { 2416 if (rte_eth_dev_set_link_down(pid) < 0) 2417 fprintf(stderr, "\nSet link down fail.\n"); 2418 } 2419 2420 static int 2421 all_ports_started(void) 2422 { 2423 portid_t pi; 2424 struct rte_port *port; 2425 2426 RTE_ETH_FOREACH_DEV(pi) { 2427 port = &ports[pi]; 2428 /* Check if there is a port which is not started */ 2429 if ((port->port_status != RTE_PORT_STARTED) && 2430 (port->slave_flag == 0)) 2431 return 0; 2432 } 2433 2434 /* No port is not started */ 2435 return 1; 2436 } 2437 2438 int 2439 port_is_stopped(portid_t port_id) 2440 { 2441 struct rte_port *port = &ports[port_id]; 2442 2443 if ((port->port_status != RTE_PORT_STOPPED) && 2444 (port->slave_flag == 0)) 2445 return 0; 2446 return 1; 2447 } 2448 2449 int 2450 all_ports_stopped(void) 2451 { 2452 portid_t pi; 2453 2454 RTE_ETH_FOREACH_DEV(pi) { 2455 if (!port_is_stopped(pi)) 2456 return 0; 2457 } 2458 2459 return 1; 2460 } 2461 2462 int 2463 port_is_started(portid_t port_id) 2464 { 2465 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2466 return 0; 2467 2468 if (ports[port_id].port_status != RTE_PORT_STARTED) 2469 return 0; 2470 2471 return 1; 2472 } 2473 2474 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2475 static int 2476 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2477 { 2478 queueid_t qi; 2479 struct rte_eth_hairpin_conf hairpin_conf = { 2480 .peer_count = 1, 2481 }; 2482 int i; 2483 int diag; 2484 struct rte_port *port = &ports[pi]; 2485 uint16_t peer_rx_port = pi; 2486 uint16_t peer_tx_port = pi; 2487 uint32_t manual = 1; 2488 uint32_t tx_exp = hairpin_mode & 0x10; 2489 2490 if (!(hairpin_mode & 0xf)) { 2491 peer_rx_port = pi; 2492 peer_tx_port = pi; 2493 manual = 0; 2494 } else if (hairpin_mode & 0x1) { 2495 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2496 RTE_ETH_DEV_NO_OWNER); 2497 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2498 peer_tx_port = rte_eth_find_next_owned_by(0, 2499 RTE_ETH_DEV_NO_OWNER); 2500 if (p_pi != RTE_MAX_ETHPORTS) { 2501 peer_rx_port = p_pi; 2502 } else { 2503 uint16_t next_pi; 2504 2505 /* Last port will be the peer RX port of the first. */ 2506 RTE_ETH_FOREACH_DEV(next_pi) 2507 peer_rx_port = next_pi; 2508 } 2509 manual = 1; 2510 } else if (hairpin_mode & 0x2) { 2511 if (cnt_pi & 0x1) { 2512 peer_rx_port = p_pi; 2513 } else { 2514 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2515 RTE_ETH_DEV_NO_OWNER); 2516 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2517 peer_rx_port = pi; 2518 } 2519 peer_tx_port = peer_rx_port; 2520 manual = 1; 2521 } 2522 2523 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2524 hairpin_conf.peers[0].port = peer_rx_port; 2525 hairpin_conf.peers[0].queue = i + nb_rxq; 2526 hairpin_conf.manual_bind = !!manual; 2527 hairpin_conf.tx_explicit = !!tx_exp; 2528 diag = rte_eth_tx_hairpin_queue_setup 2529 (pi, qi, nb_txd, &hairpin_conf); 2530 i++; 2531 if (diag == 0) 2532 continue; 2533 2534 /* Fail to setup rx queue, return */ 2535 if (port->port_status == RTE_PORT_HANDLING) 2536 port->port_status = RTE_PORT_STOPPED; 2537 else 2538 fprintf(stderr, 2539 "Port %d can not be set back to stopped\n", pi); 2540 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2541 pi); 2542 /* try to reconfigure queues next time */ 2543 port->need_reconfig_queues = 1; 2544 return -1; 2545 } 2546 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2547 hairpin_conf.peers[0].port = peer_tx_port; 2548 hairpin_conf.peers[0].queue = i + nb_txq; 2549 hairpin_conf.manual_bind = !!manual; 2550 hairpin_conf.tx_explicit = !!tx_exp; 2551 diag = rte_eth_rx_hairpin_queue_setup 2552 (pi, qi, nb_rxd, &hairpin_conf); 2553 i++; 2554 if (diag == 0) 2555 continue; 2556 2557 /* Fail to setup rx queue, return */ 2558 if (port->port_status == RTE_PORT_HANDLING) 2559 port->port_status = RTE_PORT_STOPPED; 2560 else 2561 fprintf(stderr, 2562 "Port %d can not be set back to stopped\n", pi); 2563 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2564 pi); 2565 /* try to reconfigure queues next time */ 2566 port->need_reconfig_queues = 1; 2567 return -1; 2568 } 2569 return 0; 2570 } 2571 2572 /* Configure the Rx with optional split. */ 2573 int 2574 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2575 uint16_t nb_rx_desc, unsigned int socket_id, 2576 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2577 { 2578 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2579 unsigned int i, mp_n; 2580 int ret; 2581 2582 if (rx_pkt_nb_segs <= 1 || 2583 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2584 rx_conf->rx_seg = NULL; 2585 rx_conf->rx_nseg = 0; 2586 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2587 nb_rx_desc, socket_id, 2588 rx_conf, mp); 2589 return ret; 2590 } 2591 for (i = 0; i < rx_pkt_nb_segs; i++) { 2592 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2593 struct rte_mempool *mpx; 2594 /* 2595 * Use last valid pool for the segments with number 2596 * exceeding the pool index. 2597 */ 2598 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2599 mpx = mbuf_pool_find(socket_id, mp_n); 2600 /* Handle zero as mbuf data buffer size. */ 2601 rx_seg->length = rx_pkt_seg_lengths[i] ? 2602 rx_pkt_seg_lengths[i] : 2603 mbuf_data_size[mp_n]; 2604 rx_seg->offset = i < rx_pkt_nb_offs ? 2605 rx_pkt_seg_offsets[i] : 0; 2606 rx_seg->mp = mpx ? mpx : mp; 2607 } 2608 rx_conf->rx_nseg = rx_pkt_nb_segs; 2609 rx_conf->rx_seg = rx_useg; 2610 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2611 socket_id, rx_conf, NULL); 2612 rx_conf->rx_seg = NULL; 2613 rx_conf->rx_nseg = 0; 2614 return ret; 2615 } 2616 2617 static int 2618 alloc_xstats_display_info(portid_t pi) 2619 { 2620 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2621 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2622 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2623 2624 if (xstats_display_num == 0) 2625 return 0; 2626 2627 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2628 if (*ids_supp == NULL) 2629 goto fail_ids_supp; 2630 2631 *prev_values = calloc(xstats_display_num, 2632 sizeof(**prev_values)); 2633 if (*prev_values == NULL) 2634 goto fail_prev_values; 2635 2636 *curr_values = calloc(xstats_display_num, 2637 sizeof(**curr_values)); 2638 if (*curr_values == NULL) 2639 goto fail_curr_values; 2640 2641 ports[pi].xstats_info.allocated = true; 2642 2643 return 0; 2644 2645 fail_curr_values: 2646 free(*prev_values); 2647 fail_prev_values: 2648 free(*ids_supp); 2649 fail_ids_supp: 2650 return -ENOMEM; 2651 } 2652 2653 static void 2654 free_xstats_display_info(portid_t pi) 2655 { 2656 if (!ports[pi].xstats_info.allocated) 2657 return; 2658 free(ports[pi].xstats_info.ids_supp); 2659 free(ports[pi].xstats_info.prev_values); 2660 free(ports[pi].xstats_info.curr_values); 2661 ports[pi].xstats_info.allocated = false; 2662 } 2663 2664 /** Fill helper structures for specified port to show extended statistics. */ 2665 static void 2666 fill_xstats_display_info_for_port(portid_t pi) 2667 { 2668 unsigned int stat, stat_supp; 2669 const char *xstat_name; 2670 struct rte_port *port; 2671 uint64_t *ids_supp; 2672 int rc; 2673 2674 if (xstats_display_num == 0) 2675 return; 2676 2677 if (pi == (portid_t)RTE_PORT_ALL) { 2678 fill_xstats_display_info(); 2679 return; 2680 } 2681 2682 port = &ports[pi]; 2683 if (port->port_status != RTE_PORT_STARTED) 2684 return; 2685 2686 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2687 rte_exit(EXIT_FAILURE, 2688 "Failed to allocate xstats display memory\n"); 2689 2690 ids_supp = port->xstats_info.ids_supp; 2691 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2692 xstat_name = xstats_display[stat].name; 2693 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2694 ids_supp + stat_supp); 2695 if (rc != 0) { 2696 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2697 xstat_name, pi, stat); 2698 continue; 2699 } 2700 stat_supp++; 2701 } 2702 2703 port->xstats_info.ids_supp_sz = stat_supp; 2704 } 2705 2706 /** Fill helper structures for all ports to show extended statistics. */ 2707 static void 2708 fill_xstats_display_info(void) 2709 { 2710 portid_t pi; 2711 2712 if (xstats_display_num == 0) 2713 return; 2714 2715 RTE_ETH_FOREACH_DEV(pi) 2716 fill_xstats_display_info_for_port(pi); 2717 } 2718 2719 int 2720 start_port(portid_t pid) 2721 { 2722 int diag, need_check_link_status = -1; 2723 portid_t pi; 2724 portid_t p_pi = RTE_MAX_ETHPORTS; 2725 portid_t pl[RTE_MAX_ETHPORTS]; 2726 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2727 uint16_t cnt_pi = 0; 2728 uint16_t cfg_pi = 0; 2729 int peer_pi; 2730 queueid_t qi; 2731 struct rte_port *port; 2732 struct rte_eth_hairpin_cap cap; 2733 2734 if (port_id_is_invalid(pid, ENABLED_WARN)) 2735 return 0; 2736 2737 RTE_ETH_FOREACH_DEV(pi) { 2738 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2739 continue; 2740 2741 if (port_is_bonding_slave(pi)) { 2742 fprintf(stderr, 2743 "Please remove port %d from bonded device.\n", 2744 pi); 2745 continue; 2746 } 2747 2748 need_check_link_status = 0; 2749 port = &ports[pi]; 2750 if (port->port_status == RTE_PORT_STOPPED) 2751 port->port_status = RTE_PORT_HANDLING; 2752 else { 2753 fprintf(stderr, "Port %d is now not stopped\n", pi); 2754 continue; 2755 } 2756 2757 if (port->need_reconfig > 0) { 2758 struct rte_eth_conf dev_conf; 2759 int k; 2760 2761 port->need_reconfig = 0; 2762 2763 if (flow_isolate_all) { 2764 int ret = port_flow_isolate(pi, 1); 2765 if (ret) { 2766 fprintf(stderr, 2767 "Failed to apply isolated mode on port %d\n", 2768 pi); 2769 return -1; 2770 } 2771 } 2772 configure_rxtx_dump_callbacks(0); 2773 printf("Configuring Port %d (socket %u)\n", pi, 2774 port->socket_id); 2775 if (nb_hairpinq > 0 && 2776 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2777 fprintf(stderr, 2778 "Port %d doesn't support hairpin queues\n", 2779 pi); 2780 return -1; 2781 } 2782 2783 /* configure port */ 2784 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2785 nb_txq + nb_hairpinq, 2786 &(port->dev_conf)); 2787 if (diag != 0) { 2788 if (port->port_status == RTE_PORT_HANDLING) 2789 port->port_status = RTE_PORT_STOPPED; 2790 else 2791 fprintf(stderr, 2792 "Port %d can not be set back to stopped\n", 2793 pi); 2794 fprintf(stderr, "Fail to configure port %d\n", 2795 pi); 2796 /* try to reconfigure port next time */ 2797 port->need_reconfig = 1; 2798 return -1; 2799 } 2800 /* get device configuration*/ 2801 if (0 != 2802 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2803 fprintf(stderr, 2804 "port %d can not get device configuration\n", 2805 pi); 2806 return -1; 2807 } 2808 /* Apply Rx offloads configuration */ 2809 if (dev_conf.rxmode.offloads != 2810 port->dev_conf.rxmode.offloads) { 2811 port->dev_conf.rxmode.offloads |= 2812 dev_conf.rxmode.offloads; 2813 for (k = 0; 2814 k < port->dev_info.max_rx_queues; 2815 k++) 2816 port->rx_conf[k].offloads |= 2817 dev_conf.rxmode.offloads; 2818 } 2819 /* Apply Tx offloads configuration */ 2820 if (dev_conf.txmode.offloads != 2821 port->dev_conf.txmode.offloads) { 2822 port->dev_conf.txmode.offloads |= 2823 dev_conf.txmode.offloads; 2824 for (k = 0; 2825 k < port->dev_info.max_tx_queues; 2826 k++) 2827 port->tx_conf[k].offloads |= 2828 dev_conf.txmode.offloads; 2829 } 2830 } 2831 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2832 port->need_reconfig_queues = 0; 2833 /* setup tx queues */ 2834 for (qi = 0; qi < nb_txq; qi++) { 2835 if ((numa_support) && 2836 (txring_numa[pi] != NUMA_NO_CONFIG)) 2837 diag = rte_eth_tx_queue_setup(pi, qi, 2838 port->nb_tx_desc[qi], 2839 txring_numa[pi], 2840 &(port->tx_conf[qi])); 2841 else 2842 diag = rte_eth_tx_queue_setup(pi, qi, 2843 port->nb_tx_desc[qi], 2844 port->socket_id, 2845 &(port->tx_conf[qi])); 2846 2847 if (diag == 0) 2848 continue; 2849 2850 /* Fail to setup tx queue, return */ 2851 if (port->port_status == RTE_PORT_HANDLING) 2852 port->port_status = RTE_PORT_STOPPED; 2853 else 2854 fprintf(stderr, 2855 "Port %d can not be set back to stopped\n", 2856 pi); 2857 fprintf(stderr, 2858 "Fail to configure port %d tx queues\n", 2859 pi); 2860 /* try to reconfigure queues next time */ 2861 port->need_reconfig_queues = 1; 2862 return -1; 2863 } 2864 for (qi = 0; qi < nb_rxq; qi++) { 2865 /* setup rx queues */ 2866 if ((numa_support) && 2867 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2868 struct rte_mempool * mp = 2869 mbuf_pool_find 2870 (rxring_numa[pi], 0); 2871 if (mp == NULL) { 2872 fprintf(stderr, 2873 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2874 rxring_numa[pi]); 2875 return -1; 2876 } 2877 2878 diag = rx_queue_setup(pi, qi, 2879 port->nb_rx_desc[qi], 2880 rxring_numa[pi], 2881 &(port->rx_conf[qi]), 2882 mp); 2883 } else { 2884 struct rte_mempool *mp = 2885 mbuf_pool_find 2886 (port->socket_id, 0); 2887 if (mp == NULL) { 2888 fprintf(stderr, 2889 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2890 port->socket_id); 2891 return -1; 2892 } 2893 diag = rx_queue_setup(pi, qi, 2894 port->nb_rx_desc[qi], 2895 port->socket_id, 2896 &(port->rx_conf[qi]), 2897 mp); 2898 } 2899 if (diag == 0) 2900 continue; 2901 2902 /* Fail to setup rx queue, return */ 2903 if (port->port_status == RTE_PORT_HANDLING) 2904 port->port_status = RTE_PORT_STOPPED; 2905 else 2906 fprintf(stderr, 2907 "Port %d can not be set back to stopped\n", 2908 pi); 2909 fprintf(stderr, 2910 "Fail to configure port %d rx queues\n", 2911 pi); 2912 /* try to reconfigure queues next time */ 2913 port->need_reconfig_queues = 1; 2914 return -1; 2915 } 2916 /* setup hairpin queues */ 2917 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2918 return -1; 2919 } 2920 configure_rxtx_dump_callbacks(verbose_level); 2921 if (clear_ptypes) { 2922 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2923 NULL, 0); 2924 if (diag < 0) 2925 fprintf(stderr, 2926 "Port %d: Failed to disable Ptype parsing\n", 2927 pi); 2928 } 2929 2930 p_pi = pi; 2931 cnt_pi++; 2932 2933 /* start port */ 2934 diag = eth_dev_start_mp(pi); 2935 if (diag < 0) { 2936 fprintf(stderr, "Fail to start port %d: %s\n", 2937 pi, rte_strerror(-diag)); 2938 2939 /* Fail to setup rx queue, return */ 2940 if (port->port_status == RTE_PORT_HANDLING) 2941 port->port_status = RTE_PORT_STOPPED; 2942 else 2943 fprintf(stderr, 2944 "Port %d can not be set back to stopped\n", 2945 pi); 2946 continue; 2947 } 2948 2949 if (port->port_status == RTE_PORT_HANDLING) 2950 port->port_status = RTE_PORT_STARTED; 2951 else 2952 fprintf(stderr, "Port %d can not be set into started\n", 2953 pi); 2954 2955 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 2956 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 2957 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 2958 2959 /* at least one port started, need checking link status */ 2960 need_check_link_status = 1; 2961 2962 pl[cfg_pi++] = pi; 2963 } 2964 2965 if (need_check_link_status == 1 && !no_link_check) 2966 check_all_ports_link_status(RTE_PORT_ALL); 2967 else if (need_check_link_status == 0) 2968 fprintf(stderr, "Please stop the ports first\n"); 2969 2970 if (hairpin_mode & 0xf) { 2971 uint16_t i; 2972 int j; 2973 2974 /* bind all started hairpin ports */ 2975 for (i = 0; i < cfg_pi; i++) { 2976 pi = pl[i]; 2977 /* bind current Tx to all peer Rx */ 2978 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2979 RTE_MAX_ETHPORTS, 1); 2980 if (peer_pi < 0) 2981 return peer_pi; 2982 for (j = 0; j < peer_pi; j++) { 2983 if (!port_is_started(peer_pl[j])) 2984 continue; 2985 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2986 if (diag < 0) { 2987 fprintf(stderr, 2988 "Error during binding hairpin Tx port %u to %u: %s\n", 2989 pi, peer_pl[j], 2990 rte_strerror(-diag)); 2991 return -1; 2992 } 2993 } 2994 /* bind all peer Tx to current Rx */ 2995 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2996 RTE_MAX_ETHPORTS, 0); 2997 if (peer_pi < 0) 2998 return peer_pi; 2999 for (j = 0; j < peer_pi; j++) { 3000 if (!port_is_started(peer_pl[j])) 3001 continue; 3002 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3003 if (diag < 0) { 3004 fprintf(stderr, 3005 "Error during binding hairpin Tx port %u to %u: %s\n", 3006 peer_pl[j], pi, 3007 rte_strerror(-diag)); 3008 return -1; 3009 } 3010 } 3011 } 3012 } 3013 3014 fill_xstats_display_info_for_port(pid); 3015 3016 printf("Done\n"); 3017 return 0; 3018 } 3019 3020 void 3021 stop_port(portid_t pid) 3022 { 3023 portid_t pi; 3024 struct rte_port *port; 3025 int need_check_link_status = 0; 3026 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3027 int peer_pi; 3028 3029 if (port_id_is_invalid(pid, ENABLED_WARN)) 3030 return; 3031 3032 printf("Stopping ports...\n"); 3033 3034 RTE_ETH_FOREACH_DEV(pi) { 3035 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3036 continue; 3037 3038 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3039 fprintf(stderr, 3040 "Please remove port %d from forwarding configuration.\n", 3041 pi); 3042 continue; 3043 } 3044 3045 if (port_is_bonding_slave(pi)) { 3046 fprintf(stderr, 3047 "Please remove port %d from bonded device.\n", 3048 pi); 3049 continue; 3050 } 3051 3052 port = &ports[pi]; 3053 if (port->port_status == RTE_PORT_STARTED) 3054 port->port_status = RTE_PORT_HANDLING; 3055 else 3056 continue; 3057 3058 if (hairpin_mode & 0xf) { 3059 int j; 3060 3061 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3062 /* unbind all peer Tx from current Rx */ 3063 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3064 RTE_MAX_ETHPORTS, 0); 3065 if (peer_pi < 0) 3066 continue; 3067 for (j = 0; j < peer_pi; j++) { 3068 if (!port_is_started(peer_pl[j])) 3069 continue; 3070 rte_eth_hairpin_unbind(peer_pl[j], pi); 3071 } 3072 } 3073 3074 if (port->flow_list) 3075 port_flow_flush(pi); 3076 3077 if (eth_dev_stop_mp(pi) != 0) 3078 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3079 pi); 3080 3081 if (port->port_status == RTE_PORT_HANDLING) 3082 port->port_status = RTE_PORT_STOPPED; 3083 else 3084 fprintf(stderr, "Port %d can not be set into stopped\n", 3085 pi); 3086 need_check_link_status = 1; 3087 } 3088 if (need_check_link_status && !no_link_check) 3089 check_all_ports_link_status(RTE_PORT_ALL); 3090 3091 printf("Done\n"); 3092 } 3093 3094 static void 3095 remove_invalid_ports_in(portid_t *array, portid_t *total) 3096 { 3097 portid_t i; 3098 portid_t new_total = 0; 3099 3100 for (i = 0; i < *total; i++) 3101 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3102 array[new_total] = array[i]; 3103 new_total++; 3104 } 3105 *total = new_total; 3106 } 3107 3108 static void 3109 remove_invalid_ports(void) 3110 { 3111 remove_invalid_ports_in(ports_ids, &nb_ports); 3112 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3113 nb_cfg_ports = nb_fwd_ports; 3114 } 3115 3116 void 3117 close_port(portid_t pid) 3118 { 3119 portid_t pi; 3120 struct rte_port *port; 3121 3122 if (port_id_is_invalid(pid, ENABLED_WARN)) 3123 return; 3124 3125 printf("Closing ports...\n"); 3126 3127 RTE_ETH_FOREACH_DEV(pi) { 3128 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3129 continue; 3130 3131 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3132 fprintf(stderr, 3133 "Please remove port %d from forwarding configuration.\n", 3134 pi); 3135 continue; 3136 } 3137 3138 if (port_is_bonding_slave(pi)) { 3139 fprintf(stderr, 3140 "Please remove port %d from bonded device.\n", 3141 pi); 3142 continue; 3143 } 3144 3145 port = &ports[pi]; 3146 if (port->port_status == RTE_PORT_CLOSED) { 3147 fprintf(stderr, "Port %d is already closed\n", pi); 3148 continue; 3149 } 3150 3151 if (is_proc_primary()) { 3152 port_flow_flush(pi); 3153 port_flex_item_flush(pi); 3154 rte_eth_dev_close(pi); 3155 } 3156 3157 free_xstats_display_info(pi); 3158 } 3159 3160 remove_invalid_ports(); 3161 printf("Done\n"); 3162 } 3163 3164 void 3165 reset_port(portid_t pid) 3166 { 3167 int diag; 3168 portid_t pi; 3169 struct rte_port *port; 3170 3171 if (port_id_is_invalid(pid, ENABLED_WARN)) 3172 return; 3173 3174 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3175 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3176 fprintf(stderr, 3177 "Can not reset port(s), please stop port(s) first.\n"); 3178 return; 3179 } 3180 3181 printf("Resetting ports...\n"); 3182 3183 RTE_ETH_FOREACH_DEV(pi) { 3184 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3185 continue; 3186 3187 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3188 fprintf(stderr, 3189 "Please remove port %d from forwarding configuration.\n", 3190 pi); 3191 continue; 3192 } 3193 3194 if (port_is_bonding_slave(pi)) { 3195 fprintf(stderr, 3196 "Please remove port %d from bonded device.\n", 3197 pi); 3198 continue; 3199 } 3200 3201 diag = rte_eth_dev_reset(pi); 3202 if (diag == 0) { 3203 port = &ports[pi]; 3204 port->need_reconfig = 1; 3205 port->need_reconfig_queues = 1; 3206 } else { 3207 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3208 pi, diag); 3209 } 3210 } 3211 3212 printf("Done\n"); 3213 } 3214 3215 void 3216 attach_port(char *identifier) 3217 { 3218 portid_t pi; 3219 struct rte_dev_iterator iterator; 3220 3221 printf("Attaching a new port...\n"); 3222 3223 if (identifier == NULL) { 3224 fprintf(stderr, "Invalid parameters are specified\n"); 3225 return; 3226 } 3227 3228 if (rte_dev_probe(identifier) < 0) { 3229 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3230 return; 3231 } 3232 3233 /* first attach mode: event */ 3234 if (setup_on_probe_event) { 3235 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3236 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3237 if (ports[pi].port_status == RTE_PORT_HANDLING && 3238 ports[pi].need_setup != 0) 3239 setup_attached_port(pi); 3240 return; 3241 } 3242 3243 /* second attach mode: iterator */ 3244 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3245 /* setup ports matching the devargs used for probing */ 3246 if (port_is_forwarding(pi)) 3247 continue; /* port was already attached before */ 3248 setup_attached_port(pi); 3249 } 3250 } 3251 3252 static void 3253 setup_attached_port(portid_t pi) 3254 { 3255 unsigned int socket_id; 3256 int ret; 3257 3258 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3259 /* if socket_id is invalid, set to the first available socket. */ 3260 if (check_socket_id(socket_id) < 0) 3261 socket_id = socket_ids[0]; 3262 reconfig(pi, socket_id); 3263 ret = rte_eth_promiscuous_enable(pi); 3264 if (ret != 0) 3265 fprintf(stderr, 3266 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3267 pi, rte_strerror(-ret)); 3268 3269 ports_ids[nb_ports++] = pi; 3270 fwd_ports_ids[nb_fwd_ports++] = pi; 3271 nb_cfg_ports = nb_fwd_ports; 3272 ports[pi].need_setup = 0; 3273 ports[pi].port_status = RTE_PORT_STOPPED; 3274 3275 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3276 printf("Done\n"); 3277 } 3278 3279 static void 3280 detach_device(struct rte_device *dev) 3281 { 3282 portid_t sibling; 3283 3284 if (dev == NULL) { 3285 fprintf(stderr, "Device already removed\n"); 3286 return; 3287 } 3288 3289 printf("Removing a device...\n"); 3290 3291 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3292 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3293 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3294 fprintf(stderr, "Port %u not stopped\n", 3295 sibling); 3296 return; 3297 } 3298 port_flow_flush(sibling); 3299 } 3300 } 3301 3302 if (rte_dev_remove(dev) < 0) { 3303 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3304 return; 3305 } 3306 remove_invalid_ports(); 3307 3308 printf("Device is detached\n"); 3309 printf("Now total ports is %d\n", nb_ports); 3310 printf("Done\n"); 3311 return; 3312 } 3313 3314 void 3315 detach_port_device(portid_t port_id) 3316 { 3317 int ret; 3318 struct rte_eth_dev_info dev_info; 3319 3320 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3321 return; 3322 3323 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3324 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3325 fprintf(stderr, "Port not stopped\n"); 3326 return; 3327 } 3328 fprintf(stderr, "Port was not closed\n"); 3329 } 3330 3331 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3332 if (ret != 0) { 3333 TESTPMD_LOG(ERR, 3334 "Failed to get device info for port %d, not detaching\n", 3335 port_id); 3336 return; 3337 } 3338 detach_device(dev_info.device); 3339 } 3340 3341 void 3342 detach_devargs(char *identifier) 3343 { 3344 struct rte_dev_iterator iterator; 3345 struct rte_devargs da; 3346 portid_t port_id; 3347 3348 printf("Removing a device...\n"); 3349 3350 memset(&da, 0, sizeof(da)); 3351 if (rte_devargs_parsef(&da, "%s", identifier)) { 3352 fprintf(stderr, "cannot parse identifier\n"); 3353 return; 3354 } 3355 3356 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3357 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3358 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3359 fprintf(stderr, "Port %u not stopped\n", 3360 port_id); 3361 rte_eth_iterator_cleanup(&iterator); 3362 rte_devargs_reset(&da); 3363 return; 3364 } 3365 port_flow_flush(port_id); 3366 } 3367 } 3368 3369 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3370 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3371 da.name, da.bus->name); 3372 rte_devargs_reset(&da); 3373 return; 3374 } 3375 3376 remove_invalid_ports(); 3377 3378 printf("Device %s is detached\n", identifier); 3379 printf("Now total ports is %d\n", nb_ports); 3380 printf("Done\n"); 3381 rte_devargs_reset(&da); 3382 } 3383 3384 void 3385 pmd_test_exit(void) 3386 { 3387 portid_t pt_id; 3388 unsigned int i; 3389 int ret; 3390 3391 if (test_done == 0) 3392 stop_packet_forwarding(); 3393 3394 #ifndef RTE_EXEC_ENV_WINDOWS 3395 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3396 if (mempools[i]) { 3397 if (mp_alloc_type == MP_ALLOC_ANON) 3398 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3399 NULL); 3400 } 3401 } 3402 #endif 3403 if (ports != NULL) { 3404 no_link_check = 1; 3405 RTE_ETH_FOREACH_DEV(pt_id) { 3406 printf("\nStopping port %d...\n", pt_id); 3407 fflush(stdout); 3408 stop_port(pt_id); 3409 } 3410 RTE_ETH_FOREACH_DEV(pt_id) { 3411 printf("\nShutting down port %d...\n", pt_id); 3412 fflush(stdout); 3413 close_port(pt_id); 3414 } 3415 } 3416 3417 if (hot_plug) { 3418 ret = rte_dev_event_monitor_stop(); 3419 if (ret) { 3420 RTE_LOG(ERR, EAL, 3421 "fail to stop device event monitor."); 3422 return; 3423 } 3424 3425 ret = rte_dev_event_callback_unregister(NULL, 3426 dev_event_callback, NULL); 3427 if (ret < 0) { 3428 RTE_LOG(ERR, EAL, 3429 "fail to unregister device event callback.\n"); 3430 return; 3431 } 3432 3433 ret = rte_dev_hotplug_handle_disable(); 3434 if (ret) { 3435 RTE_LOG(ERR, EAL, 3436 "fail to disable hotplug handling.\n"); 3437 return; 3438 } 3439 } 3440 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3441 if (mempools[i]) 3442 mempool_free_mp(mempools[i]); 3443 } 3444 free(xstats_display); 3445 3446 printf("\nBye...\n"); 3447 } 3448 3449 typedef void (*cmd_func_t)(void); 3450 struct pmd_test_command { 3451 const char *cmd_name; 3452 cmd_func_t cmd_func; 3453 }; 3454 3455 /* Check the link status of all ports in up to 9s, and print them finally */ 3456 static void 3457 check_all_ports_link_status(uint32_t port_mask) 3458 { 3459 #define CHECK_INTERVAL 100 /* 100ms */ 3460 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3461 portid_t portid; 3462 uint8_t count, all_ports_up, print_flag = 0; 3463 struct rte_eth_link link; 3464 int ret; 3465 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3466 3467 printf("Checking link statuses...\n"); 3468 fflush(stdout); 3469 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3470 all_ports_up = 1; 3471 RTE_ETH_FOREACH_DEV(portid) { 3472 if ((port_mask & (1 << portid)) == 0) 3473 continue; 3474 memset(&link, 0, sizeof(link)); 3475 ret = rte_eth_link_get_nowait(portid, &link); 3476 if (ret < 0) { 3477 all_ports_up = 0; 3478 if (print_flag == 1) 3479 fprintf(stderr, 3480 "Port %u link get failed: %s\n", 3481 portid, rte_strerror(-ret)); 3482 continue; 3483 } 3484 /* print link status if flag set */ 3485 if (print_flag == 1) { 3486 rte_eth_link_to_str(link_status, 3487 sizeof(link_status), &link); 3488 printf("Port %d %s\n", portid, link_status); 3489 continue; 3490 } 3491 /* clear all_ports_up flag if any link down */ 3492 if (link.link_status == RTE_ETH_LINK_DOWN) { 3493 all_ports_up = 0; 3494 break; 3495 } 3496 } 3497 /* after finally printing all link status, get out */ 3498 if (print_flag == 1) 3499 break; 3500 3501 if (all_ports_up == 0) { 3502 fflush(stdout); 3503 rte_delay_ms(CHECK_INTERVAL); 3504 } 3505 3506 /* set the print_flag if all ports up or timeout */ 3507 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3508 print_flag = 1; 3509 } 3510 3511 if (lsc_interrupt) 3512 break; 3513 } 3514 } 3515 3516 static void 3517 rmv_port_callback(void *arg) 3518 { 3519 int need_to_start = 0; 3520 int org_no_link_check = no_link_check; 3521 portid_t port_id = (intptr_t)arg; 3522 struct rte_eth_dev_info dev_info; 3523 int ret; 3524 3525 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3526 3527 if (!test_done && port_is_forwarding(port_id)) { 3528 need_to_start = 1; 3529 stop_packet_forwarding(); 3530 } 3531 no_link_check = 1; 3532 stop_port(port_id); 3533 no_link_check = org_no_link_check; 3534 3535 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3536 if (ret != 0) 3537 TESTPMD_LOG(ERR, 3538 "Failed to get device info for port %d, not detaching\n", 3539 port_id); 3540 else { 3541 struct rte_device *device = dev_info.device; 3542 close_port(port_id); 3543 detach_device(device); /* might be already removed or have more ports */ 3544 } 3545 if (need_to_start) 3546 start_packet_forwarding(0); 3547 } 3548 3549 /* This function is used by the interrupt thread */ 3550 static int 3551 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3552 void *ret_param) 3553 { 3554 RTE_SET_USED(param); 3555 RTE_SET_USED(ret_param); 3556 3557 if (type >= RTE_ETH_EVENT_MAX) { 3558 fprintf(stderr, 3559 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3560 port_id, __func__, type); 3561 fflush(stderr); 3562 } else if (event_print_mask & (UINT32_C(1) << type)) { 3563 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3564 eth_event_desc[type]); 3565 fflush(stdout); 3566 } 3567 3568 switch (type) { 3569 case RTE_ETH_EVENT_NEW: 3570 ports[port_id].need_setup = 1; 3571 ports[port_id].port_status = RTE_PORT_HANDLING; 3572 break; 3573 case RTE_ETH_EVENT_INTR_RMV: 3574 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3575 break; 3576 if (rte_eal_alarm_set(100000, 3577 rmv_port_callback, (void *)(intptr_t)port_id)) 3578 fprintf(stderr, 3579 "Could not set up deferred device removal\n"); 3580 break; 3581 case RTE_ETH_EVENT_DESTROY: 3582 ports[port_id].port_status = RTE_PORT_CLOSED; 3583 printf("Port %u is closed\n", port_id); 3584 break; 3585 default: 3586 break; 3587 } 3588 return 0; 3589 } 3590 3591 static int 3592 register_eth_event_callback(void) 3593 { 3594 int ret; 3595 enum rte_eth_event_type event; 3596 3597 for (event = RTE_ETH_EVENT_UNKNOWN; 3598 event < RTE_ETH_EVENT_MAX; event++) { 3599 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3600 event, 3601 eth_event_callback, 3602 NULL); 3603 if (ret != 0) { 3604 TESTPMD_LOG(ERR, "Failed to register callback for " 3605 "%s event\n", eth_event_desc[event]); 3606 return -1; 3607 } 3608 } 3609 3610 return 0; 3611 } 3612 3613 /* This function is used by the interrupt thread */ 3614 static void 3615 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3616 __rte_unused void *arg) 3617 { 3618 uint16_t port_id; 3619 int ret; 3620 3621 if (type >= RTE_DEV_EVENT_MAX) { 3622 fprintf(stderr, "%s called upon invalid event %d\n", 3623 __func__, type); 3624 fflush(stderr); 3625 } 3626 3627 switch (type) { 3628 case RTE_DEV_EVENT_REMOVE: 3629 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3630 device_name); 3631 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3632 if (ret) { 3633 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3634 device_name); 3635 return; 3636 } 3637 /* 3638 * Because the user's callback is invoked in eal interrupt 3639 * callback, the interrupt callback need to be finished before 3640 * it can be unregistered when detaching device. So finish 3641 * callback soon and use a deferred removal to detach device 3642 * is need. It is a workaround, once the device detaching be 3643 * moved into the eal in the future, the deferred removal could 3644 * be deleted. 3645 */ 3646 if (rte_eal_alarm_set(100000, 3647 rmv_port_callback, (void *)(intptr_t)port_id)) 3648 RTE_LOG(ERR, EAL, 3649 "Could not set up deferred device removal\n"); 3650 break; 3651 case RTE_DEV_EVENT_ADD: 3652 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3653 device_name); 3654 /* TODO: After finish kernel driver binding, 3655 * begin to attach port. 3656 */ 3657 break; 3658 default: 3659 break; 3660 } 3661 } 3662 3663 static void 3664 rxtx_port_config(portid_t pid) 3665 { 3666 uint16_t qid; 3667 uint64_t offloads; 3668 struct rte_port *port = &ports[pid]; 3669 3670 for (qid = 0; qid < nb_rxq; qid++) { 3671 offloads = port->rx_conf[qid].offloads; 3672 port->rx_conf[qid] = port->dev_info.default_rxconf; 3673 3674 if (rxq_share > 0 && 3675 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3676 /* Non-zero share group to enable RxQ share. */ 3677 port->rx_conf[qid].share_group = pid / rxq_share + 1; 3678 port->rx_conf[qid].share_qid = qid; /* Equal mapping. */ 3679 } 3680 3681 if (offloads != 0) 3682 port->rx_conf[qid].offloads = offloads; 3683 3684 /* Check if any Rx parameters have been passed */ 3685 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3686 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3687 3688 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3689 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3690 3691 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3692 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3693 3694 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3695 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3696 3697 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3698 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3699 3700 port->nb_rx_desc[qid] = nb_rxd; 3701 } 3702 3703 for (qid = 0; qid < nb_txq; qid++) { 3704 offloads = port->tx_conf[qid].offloads; 3705 port->tx_conf[qid] = port->dev_info.default_txconf; 3706 if (offloads != 0) 3707 port->tx_conf[qid].offloads = offloads; 3708 3709 /* Check if any Tx parameters have been passed */ 3710 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3711 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3712 3713 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3714 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3715 3716 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3717 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3718 3719 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3720 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3721 3722 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3723 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3724 3725 port->nb_tx_desc[qid] = nb_txd; 3726 } 3727 } 3728 3729 /* 3730 * Helper function to set MTU from frame size 3731 * 3732 * port->dev_info should be set before calling this function. 3733 * 3734 * return 0 on success, negative on error 3735 */ 3736 int 3737 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3738 { 3739 struct rte_port *port = &ports[portid]; 3740 uint32_t eth_overhead; 3741 uint16_t mtu, new_mtu; 3742 3743 eth_overhead = get_eth_overhead(&port->dev_info); 3744 3745 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3746 printf("Failed to get MTU for port %u\n", portid); 3747 return -1; 3748 } 3749 3750 new_mtu = max_rx_pktlen - eth_overhead; 3751 3752 if (mtu == new_mtu) 3753 return 0; 3754 3755 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3756 fprintf(stderr, 3757 "Failed to set MTU to %u for port %u\n", 3758 new_mtu, portid); 3759 return -1; 3760 } 3761 3762 port->dev_conf.rxmode.mtu = new_mtu; 3763 3764 return 0; 3765 } 3766 3767 void 3768 init_port_config(void) 3769 { 3770 portid_t pid; 3771 struct rte_port *port; 3772 int ret, i; 3773 3774 RTE_ETH_FOREACH_DEV(pid) { 3775 port = &ports[pid]; 3776 port->dev_conf.fdir_conf = fdir_conf; 3777 3778 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3779 if (ret != 0) 3780 return; 3781 3782 if (nb_rxq > 1) { 3783 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3784 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3785 rss_hf & port->dev_info.flow_type_rss_offloads; 3786 } else { 3787 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3788 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3789 } 3790 3791 if (port->dcb_flag == 0) { 3792 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3793 port->dev_conf.rxmode.mq_mode = 3794 (enum rte_eth_rx_mq_mode) 3795 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3796 } else { 3797 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3798 port->dev_conf.rxmode.offloads &= 3799 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3800 3801 for (i = 0; 3802 i < port->dev_info.nb_rx_queues; 3803 i++) 3804 port->rx_conf[i].offloads &= 3805 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3806 } 3807 } 3808 3809 rxtx_port_config(pid); 3810 3811 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3812 if (ret != 0) 3813 return; 3814 3815 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3816 rte_pmd_ixgbe_bypass_init(pid); 3817 #endif 3818 3819 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3820 port->dev_conf.intr_conf.lsc = 1; 3821 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3822 port->dev_conf.intr_conf.rmv = 1; 3823 } 3824 } 3825 3826 void set_port_slave_flag(portid_t slave_pid) 3827 { 3828 struct rte_port *port; 3829 3830 port = &ports[slave_pid]; 3831 port->slave_flag = 1; 3832 } 3833 3834 void clear_port_slave_flag(portid_t slave_pid) 3835 { 3836 struct rte_port *port; 3837 3838 port = &ports[slave_pid]; 3839 port->slave_flag = 0; 3840 } 3841 3842 uint8_t port_is_bonding_slave(portid_t slave_pid) 3843 { 3844 struct rte_port *port; 3845 struct rte_eth_dev_info dev_info; 3846 int ret; 3847 3848 port = &ports[slave_pid]; 3849 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3850 if (ret != 0) { 3851 TESTPMD_LOG(ERR, 3852 "Failed to get device info for port id %d," 3853 "cannot determine if the port is a bonded slave", 3854 slave_pid); 3855 return 0; 3856 } 3857 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3858 return 1; 3859 return 0; 3860 } 3861 3862 const uint16_t vlan_tags[] = { 3863 0, 1, 2, 3, 4, 5, 6, 7, 3864 8, 9, 10, 11, 12, 13, 14, 15, 3865 16, 17, 18, 19, 20, 21, 22, 23, 3866 24, 25, 26, 27, 28, 29, 30, 31 3867 }; 3868 3869 static int 3870 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3871 enum dcb_mode_enable dcb_mode, 3872 enum rte_eth_nb_tcs num_tcs, 3873 uint8_t pfc_en) 3874 { 3875 uint8_t i; 3876 int32_t rc; 3877 struct rte_eth_rss_conf rss_conf; 3878 3879 /* 3880 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3881 * given above, and the number of traffic classes available for use. 3882 */ 3883 if (dcb_mode == DCB_VT_ENABLED) { 3884 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3885 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3886 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3887 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3888 3889 /* VMDQ+DCB RX and TX configurations */ 3890 vmdq_rx_conf->enable_default_pool = 0; 3891 vmdq_rx_conf->default_pool = 0; 3892 vmdq_rx_conf->nb_queue_pools = 3893 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3894 vmdq_tx_conf->nb_queue_pools = 3895 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3896 3897 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3898 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3899 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3900 vmdq_rx_conf->pool_map[i].pools = 3901 1 << (i % vmdq_rx_conf->nb_queue_pools); 3902 } 3903 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3904 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3905 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3906 } 3907 3908 /* set DCB mode of RX and TX of multiple queues */ 3909 eth_conf->rxmode.mq_mode = 3910 (enum rte_eth_rx_mq_mode) 3911 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 3912 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 3913 } else { 3914 struct rte_eth_dcb_rx_conf *rx_conf = 3915 ð_conf->rx_adv_conf.dcb_rx_conf; 3916 struct rte_eth_dcb_tx_conf *tx_conf = 3917 ð_conf->tx_adv_conf.dcb_tx_conf; 3918 3919 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3920 3921 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3922 if (rc != 0) 3923 return rc; 3924 3925 rx_conf->nb_tcs = num_tcs; 3926 tx_conf->nb_tcs = num_tcs; 3927 3928 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3929 rx_conf->dcb_tc[i] = i % num_tcs; 3930 tx_conf->dcb_tc[i] = i % num_tcs; 3931 } 3932 3933 eth_conf->rxmode.mq_mode = 3934 (enum rte_eth_rx_mq_mode) 3935 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 3936 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3937 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 3938 } 3939 3940 if (pfc_en) 3941 eth_conf->dcb_capability_en = 3942 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 3943 else 3944 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 3945 3946 return 0; 3947 } 3948 3949 int 3950 init_port_dcb_config(portid_t pid, 3951 enum dcb_mode_enable dcb_mode, 3952 enum rte_eth_nb_tcs num_tcs, 3953 uint8_t pfc_en) 3954 { 3955 struct rte_eth_conf port_conf; 3956 struct rte_port *rte_port; 3957 int retval; 3958 uint16_t i; 3959 3960 if (num_procs > 1) { 3961 printf("The multi-process feature doesn't support dcb.\n"); 3962 return -ENOTSUP; 3963 } 3964 rte_port = &ports[pid]; 3965 3966 /* retain the original device configuration. */ 3967 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 3968 3969 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3970 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3971 if (retval < 0) 3972 return retval; 3973 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 3974 /* remove RSS HASH offload for DCB in vt mode */ 3975 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 3976 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3977 for (i = 0; i < nb_rxq; i++) 3978 rte_port->rx_conf[i].offloads &= 3979 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3980 } 3981 3982 /* re-configure the device . */ 3983 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3984 if (retval < 0) 3985 return retval; 3986 3987 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3988 if (retval != 0) 3989 return retval; 3990 3991 /* If dev_info.vmdq_pool_base is greater than 0, 3992 * the queue id of vmdq pools is started after pf queues. 3993 */ 3994 if (dcb_mode == DCB_VT_ENABLED && 3995 rte_port->dev_info.vmdq_pool_base > 0) { 3996 fprintf(stderr, 3997 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 3998 pid); 3999 return -1; 4000 } 4001 4002 /* Assume the ports in testpmd have the same dcb capability 4003 * and has the same number of rxq and txq in dcb mode 4004 */ 4005 if (dcb_mode == DCB_VT_ENABLED) { 4006 if (rte_port->dev_info.max_vfs > 0) { 4007 nb_rxq = rte_port->dev_info.nb_rx_queues; 4008 nb_txq = rte_port->dev_info.nb_tx_queues; 4009 } else { 4010 nb_rxq = rte_port->dev_info.max_rx_queues; 4011 nb_txq = rte_port->dev_info.max_tx_queues; 4012 } 4013 } else { 4014 /*if vt is disabled, use all pf queues */ 4015 if (rte_port->dev_info.vmdq_pool_base == 0) { 4016 nb_rxq = rte_port->dev_info.max_rx_queues; 4017 nb_txq = rte_port->dev_info.max_tx_queues; 4018 } else { 4019 nb_rxq = (queueid_t)num_tcs; 4020 nb_txq = (queueid_t)num_tcs; 4021 4022 } 4023 } 4024 rx_free_thresh = 64; 4025 4026 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4027 4028 rxtx_port_config(pid); 4029 /* VLAN filter */ 4030 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4031 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4032 rx_vft_set(pid, vlan_tags[i], 1); 4033 4034 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4035 if (retval != 0) 4036 return retval; 4037 4038 rte_port->dcb_flag = 1; 4039 4040 /* Enter DCB configuration status */ 4041 dcb_config = 1; 4042 4043 return 0; 4044 } 4045 4046 static void 4047 init_port(void) 4048 { 4049 int i; 4050 4051 /* Configuration of Ethernet ports. */ 4052 ports = rte_zmalloc("testpmd: ports", 4053 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4054 RTE_CACHE_LINE_SIZE); 4055 if (ports == NULL) { 4056 rte_exit(EXIT_FAILURE, 4057 "rte_zmalloc(%d struct rte_port) failed\n", 4058 RTE_MAX_ETHPORTS); 4059 } 4060 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4061 ports[i].xstats_info.allocated = false; 4062 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4063 LIST_INIT(&ports[i].flow_tunnel_list); 4064 /* Initialize ports NUMA structures */ 4065 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4066 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4067 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4068 } 4069 4070 static void 4071 force_quit(void) 4072 { 4073 pmd_test_exit(); 4074 prompt_exit(); 4075 } 4076 4077 static void 4078 print_stats(void) 4079 { 4080 uint8_t i; 4081 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4082 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4083 4084 /* Clear screen and move to top left */ 4085 printf("%s%s", clr, top_left); 4086 4087 printf("\nPort statistics ===================================="); 4088 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4089 nic_stats_display(fwd_ports_ids[i]); 4090 4091 fflush(stdout); 4092 } 4093 4094 static void 4095 signal_handler(int signum) 4096 { 4097 if (signum == SIGINT || signum == SIGTERM) { 4098 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4099 signum); 4100 #ifdef RTE_LIB_PDUMP 4101 /* uninitialize packet capture framework */ 4102 rte_pdump_uninit(); 4103 #endif 4104 #ifdef RTE_LIB_LATENCYSTATS 4105 if (latencystats_enabled != 0) 4106 rte_latencystats_uninit(); 4107 #endif 4108 force_quit(); 4109 /* Set flag to indicate the force termination. */ 4110 f_quit = 1; 4111 /* exit with the expected status */ 4112 #ifndef RTE_EXEC_ENV_WINDOWS 4113 signal(signum, SIG_DFL); 4114 kill(getpid(), signum); 4115 #endif 4116 } 4117 } 4118 4119 int 4120 main(int argc, char** argv) 4121 { 4122 int diag; 4123 portid_t port_id; 4124 uint16_t count; 4125 int ret; 4126 4127 signal(SIGINT, signal_handler); 4128 signal(SIGTERM, signal_handler); 4129 4130 testpmd_logtype = rte_log_register("testpmd"); 4131 if (testpmd_logtype < 0) 4132 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4133 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4134 4135 diag = rte_eal_init(argc, argv); 4136 if (diag < 0) 4137 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4138 rte_strerror(rte_errno)); 4139 4140 ret = register_eth_event_callback(); 4141 if (ret != 0) 4142 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4143 4144 #ifdef RTE_LIB_PDUMP 4145 /* initialize packet capture framework */ 4146 rte_pdump_init(); 4147 #endif 4148 4149 count = 0; 4150 RTE_ETH_FOREACH_DEV(port_id) { 4151 ports_ids[count] = port_id; 4152 count++; 4153 } 4154 nb_ports = (portid_t) count; 4155 if (nb_ports == 0) 4156 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4157 4158 /* allocate port structures, and init them */ 4159 init_port(); 4160 4161 set_def_fwd_config(); 4162 if (nb_lcores == 0) 4163 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4164 "Check the core mask argument\n"); 4165 4166 /* Bitrate/latency stats disabled by default */ 4167 #ifdef RTE_LIB_BITRATESTATS 4168 bitrate_enabled = 0; 4169 #endif 4170 #ifdef RTE_LIB_LATENCYSTATS 4171 latencystats_enabled = 0; 4172 #endif 4173 4174 /* on FreeBSD, mlockall() is disabled by default */ 4175 #ifdef RTE_EXEC_ENV_FREEBSD 4176 do_mlockall = 0; 4177 #else 4178 do_mlockall = 1; 4179 #endif 4180 4181 argc -= diag; 4182 argv += diag; 4183 if (argc > 1) 4184 launch_args_parse(argc, argv); 4185 4186 #ifndef RTE_EXEC_ENV_WINDOWS 4187 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4188 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4189 strerror(errno)); 4190 } 4191 #endif 4192 4193 if (tx_first && interactive) 4194 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4195 "interactive mode.\n"); 4196 4197 if (tx_first && lsc_interrupt) { 4198 fprintf(stderr, 4199 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4200 lsc_interrupt = 0; 4201 } 4202 4203 if (!nb_rxq && !nb_txq) 4204 fprintf(stderr, 4205 "Warning: Either rx or tx queues should be non-zero\n"); 4206 4207 if (nb_rxq > 1 && nb_rxq > nb_txq) 4208 fprintf(stderr, 4209 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4210 nb_rxq, nb_txq); 4211 4212 init_config(); 4213 4214 if (hot_plug) { 4215 ret = rte_dev_hotplug_handle_enable(); 4216 if (ret) { 4217 RTE_LOG(ERR, EAL, 4218 "fail to enable hotplug handling."); 4219 return -1; 4220 } 4221 4222 ret = rte_dev_event_monitor_start(); 4223 if (ret) { 4224 RTE_LOG(ERR, EAL, 4225 "fail to start device event monitoring."); 4226 return -1; 4227 } 4228 4229 ret = rte_dev_event_callback_register(NULL, 4230 dev_event_callback, NULL); 4231 if (ret) { 4232 RTE_LOG(ERR, EAL, 4233 "fail to register device event callback\n"); 4234 return -1; 4235 } 4236 } 4237 4238 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4239 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4240 4241 /* set all ports to promiscuous mode by default */ 4242 RTE_ETH_FOREACH_DEV(port_id) { 4243 ret = rte_eth_promiscuous_enable(port_id); 4244 if (ret != 0) 4245 fprintf(stderr, 4246 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4247 port_id, rte_strerror(-ret)); 4248 } 4249 4250 #ifdef RTE_LIB_METRICS 4251 /* Init metrics library */ 4252 rte_metrics_init(rte_socket_id()); 4253 #endif 4254 4255 #ifdef RTE_LIB_LATENCYSTATS 4256 if (latencystats_enabled != 0) { 4257 int ret = rte_latencystats_init(1, NULL); 4258 if (ret) 4259 fprintf(stderr, 4260 "Warning: latencystats init() returned error %d\n", 4261 ret); 4262 fprintf(stderr, "Latencystats running on lcore %d\n", 4263 latencystats_lcore_id); 4264 } 4265 #endif 4266 4267 /* Setup bitrate stats */ 4268 #ifdef RTE_LIB_BITRATESTATS 4269 if (bitrate_enabled != 0) { 4270 bitrate_data = rte_stats_bitrate_create(); 4271 if (bitrate_data == NULL) 4272 rte_exit(EXIT_FAILURE, 4273 "Could not allocate bitrate data.\n"); 4274 rte_stats_bitrate_reg(bitrate_data); 4275 } 4276 #endif 4277 #ifdef RTE_LIB_CMDLINE 4278 if (strlen(cmdline_filename) != 0) 4279 cmdline_read_from_file(cmdline_filename); 4280 4281 if (interactive == 1) { 4282 if (auto_start) { 4283 printf("Start automatic packet forwarding\n"); 4284 start_packet_forwarding(0); 4285 } 4286 prompt(); 4287 pmd_test_exit(); 4288 } else 4289 #endif 4290 { 4291 char c; 4292 int rc; 4293 4294 f_quit = 0; 4295 4296 printf("No commandline core given, start packet forwarding\n"); 4297 start_packet_forwarding(tx_first); 4298 if (stats_period != 0) { 4299 uint64_t prev_time = 0, cur_time, diff_time = 0; 4300 uint64_t timer_period; 4301 4302 /* Convert to number of cycles */ 4303 timer_period = stats_period * rte_get_timer_hz(); 4304 4305 while (f_quit == 0) { 4306 cur_time = rte_get_timer_cycles(); 4307 diff_time += cur_time - prev_time; 4308 4309 if (diff_time >= timer_period) { 4310 print_stats(); 4311 /* Reset the timer */ 4312 diff_time = 0; 4313 } 4314 /* Sleep to avoid unnecessary checks */ 4315 prev_time = cur_time; 4316 rte_delay_us_sleep(US_PER_S); 4317 } 4318 } 4319 4320 printf("Press enter to exit\n"); 4321 rc = read(0, &c, 1); 4322 pmd_test_exit(); 4323 if (rc < 0) 4324 return 1; 4325 } 4326 4327 ret = rte_eal_cleanup(); 4328 if (ret != 0) 4329 rte_exit(EXIT_FAILURE, 4330 "EAL cleanup failed: %s\n", strerror(-ret)); 4331 4332 return EXIT_SUCCESS; 4333 } 4334