1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 70 #include "testpmd.h" 71 72 #ifndef MAP_HUGETLB 73 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 74 #define HUGE_FLAG (0x40000) 75 #else 76 #define HUGE_FLAG MAP_HUGETLB 77 #endif 78 79 #ifndef MAP_HUGE_SHIFT 80 /* older kernels (or FreeBSD) will not have this define */ 81 #define HUGE_SHIFT (26) 82 #else 83 #define HUGE_SHIFT MAP_HUGE_SHIFT 84 #endif 85 86 #define EXTMEM_HEAP_NAME "extmem" 87 /* 88 * Zone size with the malloc overhead (max of debug and release variants) 89 * must fit into the smallest supported hugepage size (2M), 90 * so that an IOVA-contiguous zone of this size can always be allocated 91 * if there are free 2M hugepages. 92 */ 93 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE) 94 95 uint16_t verbose_level = 0; /**< Silent by default. */ 96 int testpmd_logtype; /**< Log type for testpmd logs */ 97 98 /* use main core for command line ? */ 99 uint8_t interactive = 0; 100 uint8_t auto_start = 0; 101 uint8_t tx_first; 102 char cmdline_filename[PATH_MAX] = {0}; 103 104 /* 105 * NUMA support configuration. 106 * When set, the NUMA support attempts to dispatch the allocation of the 107 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 108 * probed ports among the CPU sockets 0 and 1. 109 * Otherwise, all memory is allocated from CPU socket 0. 110 */ 111 uint8_t numa_support = 1; /**< numa enabled by default */ 112 113 /* 114 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 115 * not configured. 116 */ 117 uint8_t socket_num = UMA_NO_CONFIG; 118 119 /* 120 * Select mempool allocation type: 121 * - native: use regular DPDK memory 122 * - anon: use regular DPDK memory to create mempool, but populate using 123 * anonymous memory (may not be IOVA-contiguous) 124 * - xmem: use externally allocated hugepage memory 125 */ 126 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 127 128 /* 129 * Store specified sockets on which memory pool to be used by ports 130 * is allocated. 131 */ 132 uint8_t port_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Store specified sockets on which RX ring to be used by ports 136 * is allocated. 137 */ 138 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Store specified sockets on which TX ring to be used by ports 142 * is allocated. 143 */ 144 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 145 146 /* 147 * Record the Ethernet address of peer target ports to which packets are 148 * forwarded. 149 * Must be instantiated with the ethernet addresses of peer traffic generator 150 * ports. 151 */ 152 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 153 portid_t nb_peer_eth_addrs = 0; 154 155 /* 156 * Probed Target Environment. 157 */ 158 struct rte_port *ports; /**< For all probed ethernet ports. */ 159 portid_t nb_ports; /**< Number of probed ethernet ports. */ 160 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 161 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 162 163 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 164 165 /* 166 * Test Forwarding Configuration. 167 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 168 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 169 */ 170 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 171 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 172 portid_t nb_cfg_ports; /**< Number of configured ports. */ 173 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 174 175 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 176 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 177 178 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 179 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 180 181 /* 182 * Forwarding engines. 183 */ 184 struct fwd_engine * fwd_engines[] = { 185 &io_fwd_engine, 186 &mac_fwd_engine, 187 &mac_swap_engine, 188 &flow_gen_engine, 189 &rx_only_engine, 190 &tx_only_engine, 191 &csum_fwd_engine, 192 &icmp_echo_engine, 193 &noisy_vnf_engine, 194 &five_tuple_swap_fwd_engine, 195 #ifdef RTE_LIBRTE_IEEE1588 196 &ieee1588_fwd_engine, 197 #endif 198 &shared_rxq_engine, 199 NULL, 200 }; 201 202 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 203 uint16_t mempool_flags; 204 205 struct fwd_config cur_fwd_config; 206 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 207 uint32_t retry_enabled; 208 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 209 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 210 211 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 212 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 213 DEFAULT_MBUF_DATA_SIZE 214 }; /**< Mbuf data space size. */ 215 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 216 * specified on command-line. */ 217 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 218 219 /** Extended statistics to show. */ 220 struct rte_eth_xstat_name *xstats_display; 221 222 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 223 224 /* 225 * In container, it cannot terminate the process which running with 'stats-period' 226 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 227 */ 228 uint8_t f_quit; 229 230 /* 231 * Max Rx frame size, set by '--max-pkt-len' parameter. 232 */ 233 uint32_t max_rx_pkt_len; 234 235 /* 236 * Configuration of packet segments used to scatter received packets 237 * if some of split features is configured. 238 */ 239 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 240 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 241 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 242 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 243 244 /* 245 * Configuration of packet segments used by the "txonly" processing engine. 246 */ 247 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 248 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 249 TXONLY_DEF_PACKET_LEN, 250 }; 251 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 252 253 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 254 /**< Split policy for packets to TX. */ 255 256 uint8_t txonly_multi_flow; 257 /**< Whether multiple flows are generated in TXONLY mode. */ 258 259 uint32_t tx_pkt_times_inter; 260 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 261 262 uint32_t tx_pkt_times_intra; 263 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 264 265 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 266 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 267 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 268 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 269 270 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 271 uint8_t dcb_config = 0; 272 273 /* 274 * Configurable number of RX/TX queues. 275 */ 276 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 277 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 278 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 279 280 /* 281 * Configurable number of RX/TX ring descriptors. 282 * Defaults are supplied by drivers via ethdev. 283 */ 284 #define RTE_TEST_RX_DESC_DEFAULT 0 285 #define RTE_TEST_TX_DESC_DEFAULT 0 286 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 287 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 288 289 #define RTE_PMD_PARAM_UNSET -1 290 /* 291 * Configurable values of RX and TX ring threshold registers. 292 */ 293 294 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 295 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 296 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 297 298 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 299 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 300 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 301 302 /* 303 * Configurable value of RX free threshold. 304 */ 305 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 306 307 /* 308 * Configurable value of RX drop enable. 309 */ 310 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 311 312 /* 313 * Configurable value of TX free threshold. 314 */ 315 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 316 317 /* 318 * Configurable value of TX RS bit threshold. 319 */ 320 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 321 322 /* 323 * Configurable value of buffered packets before sending. 324 */ 325 uint16_t noisy_tx_sw_bufsz; 326 327 /* 328 * Configurable value of packet buffer timeout. 329 */ 330 uint16_t noisy_tx_sw_buf_flush_time; 331 332 /* 333 * Configurable value for size of VNF internal memory area 334 * used for simulating noisy neighbour behaviour 335 */ 336 uint64_t noisy_lkup_mem_sz; 337 338 /* 339 * Configurable value of number of random writes done in 340 * VNF simulation memory area. 341 */ 342 uint64_t noisy_lkup_num_writes; 343 344 /* 345 * Configurable value of number of random reads done in 346 * VNF simulation memory area. 347 */ 348 uint64_t noisy_lkup_num_reads; 349 350 /* 351 * Configurable value of number of random reads/writes done in 352 * VNF simulation memory area. 353 */ 354 uint64_t noisy_lkup_num_reads_writes; 355 356 /* 357 * Receive Side Scaling (RSS) configuration. 358 */ 359 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 360 361 /* 362 * Port topology configuration 363 */ 364 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 365 366 /* 367 * Avoids to flush all the RX streams before starts forwarding. 368 */ 369 uint8_t no_flush_rx = 0; /* flush by default */ 370 371 /* 372 * Flow API isolated mode. 373 */ 374 uint8_t flow_isolate_all; 375 376 /* 377 * Avoids to check link status when starting/stopping a port. 378 */ 379 uint8_t no_link_check = 0; /* check by default */ 380 381 /* 382 * Don't automatically start all ports in interactive mode. 383 */ 384 uint8_t no_device_start = 0; 385 386 /* 387 * Enable link status change notification 388 */ 389 uint8_t lsc_interrupt = 1; /* enabled by default */ 390 391 /* 392 * Enable device removal notification. 393 */ 394 uint8_t rmv_interrupt = 1; /* enabled by default */ 395 396 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 397 398 /* After attach, port setup is called on event or by iterator */ 399 bool setup_on_probe_event = true; 400 401 /* Clear ptypes on port initialization. */ 402 uint8_t clear_ptypes = true; 403 404 /* Hairpin ports configuration mode. */ 405 uint16_t hairpin_mode; 406 407 /* Pretty printing of ethdev events */ 408 static const char * const eth_event_desc[] = { 409 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 410 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 411 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 412 [RTE_ETH_EVENT_INTR_RESET] = "reset", 413 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 414 [RTE_ETH_EVENT_IPSEC] = "IPsec", 415 [RTE_ETH_EVENT_MACSEC] = "MACsec", 416 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 417 [RTE_ETH_EVENT_NEW] = "device probed", 418 [RTE_ETH_EVENT_DESTROY] = "device released", 419 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 420 [RTE_ETH_EVENT_MAX] = NULL, 421 }; 422 423 /* 424 * Display or mask ether events 425 * Default to all events except VF_MBOX 426 */ 427 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 428 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 429 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 430 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 431 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 432 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 433 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 434 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 435 /* 436 * Decide if all memory are locked for performance. 437 */ 438 int do_mlockall = 0; 439 440 /* 441 * NIC bypass mode configuration options. 442 */ 443 444 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 445 /* The NIC bypass watchdog timeout. */ 446 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 447 #endif 448 449 450 #ifdef RTE_LIB_LATENCYSTATS 451 452 /* 453 * Set when latency stats is enabled in the commandline 454 */ 455 uint8_t latencystats_enabled; 456 457 /* 458 * Lcore ID to service latency statistics. 459 */ 460 lcoreid_t latencystats_lcore_id = -1; 461 462 #endif 463 464 /* 465 * Ethernet device configuration. 466 */ 467 struct rte_eth_rxmode rx_mode; 468 469 struct rte_eth_txmode tx_mode = { 470 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 471 }; 472 473 struct rte_eth_fdir_conf fdir_conf = { 474 .mode = RTE_FDIR_MODE_NONE, 475 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 476 .status = RTE_FDIR_REPORT_STATUS, 477 .mask = { 478 .vlan_tci_mask = 0xFFEF, 479 .ipv4_mask = { 480 .src_ip = 0xFFFFFFFF, 481 .dst_ip = 0xFFFFFFFF, 482 }, 483 .ipv6_mask = { 484 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 485 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 486 }, 487 .src_port_mask = 0xFFFF, 488 .dst_port_mask = 0xFFFF, 489 .mac_addr_byte_mask = 0xFF, 490 .tunnel_type_mask = 1, 491 .tunnel_id_mask = 0xFFFFFFFF, 492 }, 493 .drop_queue = 127, 494 }; 495 496 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 497 498 /* 499 * Display zero values by default for xstats 500 */ 501 uint8_t xstats_hide_zero; 502 503 /* 504 * Measure of CPU cycles disabled by default 505 */ 506 uint8_t record_core_cycles; 507 508 /* 509 * Display of RX and TX bursts disabled by default 510 */ 511 uint8_t record_burst_stats; 512 513 /* 514 * Number of ports per shared Rx queue group, 0 disable. 515 */ 516 uint32_t rxq_share; 517 518 unsigned int num_sockets = 0; 519 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 520 521 #ifdef RTE_LIB_BITRATESTATS 522 /* Bitrate statistics */ 523 struct rte_stats_bitrates *bitrate_data; 524 lcoreid_t bitrate_lcore_id; 525 uint8_t bitrate_enabled; 526 #endif 527 528 #ifdef RTE_LIB_GRO 529 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 530 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 531 #endif 532 533 /* 534 * hexadecimal bitmask of RX mq mode can be enabled. 535 */ 536 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 537 538 /* 539 * Used to set forced link speed 540 */ 541 uint32_t eth_link_speed; 542 543 /* 544 * ID of the current process in multi-process, used to 545 * configure the queues to be polled. 546 */ 547 int proc_id; 548 549 /* 550 * Number of processes in multi-process, used to 551 * configure the queues to be polled. 552 */ 553 unsigned int num_procs = 1; 554 555 static void 556 eth_rx_metadata_negotiate_mp(uint16_t port_id) 557 { 558 uint64_t rx_meta_features = 0; 559 int ret; 560 561 if (!is_proc_primary()) 562 return; 563 564 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 565 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 566 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 567 568 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 569 if (ret == 0) { 570 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 571 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 572 port_id); 573 } 574 575 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 576 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 577 port_id); 578 } 579 580 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 581 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 582 port_id); 583 } 584 } else if (ret != -ENOTSUP) { 585 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 586 port_id, rte_strerror(-ret)); 587 } 588 } 589 590 static int 591 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 592 const struct rte_eth_conf *dev_conf) 593 { 594 if (is_proc_primary()) 595 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 596 dev_conf); 597 return 0; 598 } 599 600 static int 601 eth_dev_start_mp(uint16_t port_id) 602 { 603 if (is_proc_primary()) 604 return rte_eth_dev_start(port_id); 605 606 return 0; 607 } 608 609 static int 610 eth_dev_stop_mp(uint16_t port_id) 611 { 612 if (is_proc_primary()) 613 return rte_eth_dev_stop(port_id); 614 615 return 0; 616 } 617 618 static void 619 mempool_free_mp(struct rte_mempool *mp) 620 { 621 if (is_proc_primary()) 622 rte_mempool_free(mp); 623 } 624 625 static int 626 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 627 { 628 if (is_proc_primary()) 629 return rte_eth_dev_set_mtu(port_id, mtu); 630 631 return 0; 632 } 633 634 /* Forward function declarations */ 635 static void setup_attached_port(portid_t pi); 636 static void check_all_ports_link_status(uint32_t port_mask); 637 static int eth_event_callback(portid_t port_id, 638 enum rte_eth_event_type type, 639 void *param, void *ret_param); 640 static void dev_event_callback(const char *device_name, 641 enum rte_dev_event_type type, 642 void *param); 643 static void fill_xstats_display_info(void); 644 645 /* 646 * Check if all the ports are started. 647 * If yes, return positive value. If not, return zero. 648 */ 649 static int all_ports_started(void); 650 651 #ifdef RTE_LIB_GSO 652 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 653 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 654 #endif 655 656 /* Holds the registered mbuf dynamic flags names. */ 657 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 658 659 660 /* 661 * Helper function to check if socket is already discovered. 662 * If yes, return positive value. If not, return zero. 663 */ 664 int 665 new_socket_id(unsigned int socket_id) 666 { 667 unsigned int i; 668 669 for (i = 0; i < num_sockets; i++) { 670 if (socket_ids[i] == socket_id) 671 return 0; 672 } 673 return 1; 674 } 675 676 /* 677 * Setup default configuration. 678 */ 679 static void 680 set_default_fwd_lcores_config(void) 681 { 682 unsigned int i; 683 unsigned int nb_lc; 684 unsigned int sock_num; 685 686 nb_lc = 0; 687 for (i = 0; i < RTE_MAX_LCORE; i++) { 688 if (!rte_lcore_is_enabled(i)) 689 continue; 690 sock_num = rte_lcore_to_socket_id(i); 691 if (new_socket_id(sock_num)) { 692 if (num_sockets >= RTE_MAX_NUMA_NODES) { 693 rte_exit(EXIT_FAILURE, 694 "Total sockets greater than %u\n", 695 RTE_MAX_NUMA_NODES); 696 } 697 socket_ids[num_sockets++] = sock_num; 698 } 699 if (i == rte_get_main_lcore()) 700 continue; 701 fwd_lcores_cpuids[nb_lc++] = i; 702 } 703 nb_lcores = (lcoreid_t) nb_lc; 704 nb_cfg_lcores = nb_lcores; 705 nb_fwd_lcores = 1; 706 } 707 708 static void 709 set_def_peer_eth_addrs(void) 710 { 711 portid_t i; 712 713 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 714 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 715 peer_eth_addrs[i].addr_bytes[5] = i; 716 } 717 } 718 719 static void 720 set_default_fwd_ports_config(void) 721 { 722 portid_t pt_id; 723 int i = 0; 724 725 RTE_ETH_FOREACH_DEV(pt_id) { 726 fwd_ports_ids[i++] = pt_id; 727 728 /* Update sockets info according to the attached device */ 729 int socket_id = rte_eth_dev_socket_id(pt_id); 730 if (socket_id >= 0 && new_socket_id(socket_id)) { 731 if (num_sockets >= RTE_MAX_NUMA_NODES) { 732 rte_exit(EXIT_FAILURE, 733 "Total sockets greater than %u\n", 734 RTE_MAX_NUMA_NODES); 735 } 736 socket_ids[num_sockets++] = socket_id; 737 } 738 } 739 740 nb_cfg_ports = nb_ports; 741 nb_fwd_ports = nb_ports; 742 } 743 744 void 745 set_def_fwd_config(void) 746 { 747 set_default_fwd_lcores_config(); 748 set_def_peer_eth_addrs(); 749 set_default_fwd_ports_config(); 750 } 751 752 #ifndef RTE_EXEC_ENV_WINDOWS 753 /* extremely pessimistic estimation of memory required to create a mempool */ 754 static int 755 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 756 { 757 unsigned int n_pages, mbuf_per_pg, leftover; 758 uint64_t total_mem, mbuf_mem, obj_sz; 759 760 /* there is no good way to predict how much space the mempool will 761 * occupy because it will allocate chunks on the fly, and some of those 762 * will come from default DPDK memory while some will come from our 763 * external memory, so just assume 128MB will be enough for everyone. 764 */ 765 uint64_t hdr_mem = 128 << 20; 766 767 /* account for possible non-contiguousness */ 768 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 769 if (obj_sz > pgsz) { 770 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 771 return -1; 772 } 773 774 mbuf_per_pg = pgsz / obj_sz; 775 leftover = (nb_mbufs % mbuf_per_pg) > 0; 776 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 777 778 mbuf_mem = n_pages * pgsz; 779 780 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 781 782 if (total_mem > SIZE_MAX) { 783 TESTPMD_LOG(ERR, "Memory size too big\n"); 784 return -1; 785 } 786 *out = (size_t)total_mem; 787 788 return 0; 789 } 790 791 static int 792 pagesz_flags(uint64_t page_sz) 793 { 794 /* as per mmap() manpage, all page sizes are log2 of page size 795 * shifted by MAP_HUGE_SHIFT 796 */ 797 int log2 = rte_log2_u64(page_sz); 798 799 return (log2 << HUGE_SHIFT); 800 } 801 802 static void * 803 alloc_mem(size_t memsz, size_t pgsz, bool huge) 804 { 805 void *addr; 806 int flags; 807 808 /* allocate anonymous hugepages */ 809 flags = MAP_ANONYMOUS | MAP_PRIVATE; 810 if (huge) 811 flags |= HUGE_FLAG | pagesz_flags(pgsz); 812 813 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 814 if (addr == MAP_FAILED) 815 return NULL; 816 817 return addr; 818 } 819 820 struct extmem_param { 821 void *addr; 822 size_t len; 823 size_t pgsz; 824 rte_iova_t *iova_table; 825 unsigned int iova_table_len; 826 }; 827 828 static int 829 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 830 bool huge) 831 { 832 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 833 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 834 unsigned int cur_page, n_pages, pgsz_idx; 835 size_t mem_sz, cur_pgsz; 836 rte_iova_t *iovas = NULL; 837 void *addr; 838 int ret; 839 840 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 841 /* skip anything that is too big */ 842 if (pgsizes[pgsz_idx] > SIZE_MAX) 843 continue; 844 845 cur_pgsz = pgsizes[pgsz_idx]; 846 847 /* if we were told not to allocate hugepages, override */ 848 if (!huge) 849 cur_pgsz = sysconf(_SC_PAGESIZE); 850 851 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 852 if (ret < 0) { 853 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 854 return -1; 855 } 856 857 /* allocate our memory */ 858 addr = alloc_mem(mem_sz, cur_pgsz, huge); 859 860 /* if we couldn't allocate memory with a specified page size, 861 * that doesn't mean we can't do it with other page sizes, so 862 * try another one. 863 */ 864 if (addr == NULL) 865 continue; 866 867 /* store IOVA addresses for every page in this memory area */ 868 n_pages = mem_sz / cur_pgsz; 869 870 iovas = malloc(sizeof(*iovas) * n_pages); 871 872 if (iovas == NULL) { 873 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 874 goto fail; 875 } 876 /* lock memory if it's not huge pages */ 877 if (!huge) 878 mlock(addr, mem_sz); 879 880 /* populate IOVA addresses */ 881 for (cur_page = 0; cur_page < n_pages; cur_page++) { 882 rte_iova_t iova; 883 size_t offset; 884 void *cur; 885 886 offset = cur_pgsz * cur_page; 887 cur = RTE_PTR_ADD(addr, offset); 888 889 /* touch the page before getting its IOVA */ 890 *(volatile char *)cur = 0; 891 892 iova = rte_mem_virt2iova(cur); 893 894 iovas[cur_page] = iova; 895 } 896 897 break; 898 } 899 /* if we couldn't allocate anything */ 900 if (iovas == NULL) 901 return -1; 902 903 param->addr = addr; 904 param->len = mem_sz; 905 param->pgsz = cur_pgsz; 906 param->iova_table = iovas; 907 param->iova_table_len = n_pages; 908 909 return 0; 910 fail: 911 free(iovas); 912 if (addr) 913 munmap(addr, mem_sz); 914 915 return -1; 916 } 917 918 static int 919 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 920 { 921 struct extmem_param param; 922 int socket_id, ret; 923 924 memset(¶m, 0, sizeof(param)); 925 926 /* check if our heap exists */ 927 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 928 if (socket_id < 0) { 929 /* create our heap */ 930 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 931 if (ret < 0) { 932 TESTPMD_LOG(ERR, "Cannot create heap\n"); 933 return -1; 934 } 935 } 936 937 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 938 if (ret < 0) { 939 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 940 return -1; 941 } 942 943 /* we now have a valid memory area, so add it to heap */ 944 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 945 param.addr, param.len, param.iova_table, 946 param.iova_table_len, param.pgsz); 947 948 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 949 950 /* not needed any more */ 951 free(param.iova_table); 952 953 if (ret < 0) { 954 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 955 munmap(param.addr, param.len); 956 return -1; 957 } 958 959 /* success */ 960 961 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 962 param.len >> 20); 963 964 return 0; 965 } 966 static void 967 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 968 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 969 { 970 uint16_t pid = 0; 971 int ret; 972 973 RTE_ETH_FOREACH_DEV(pid) { 974 struct rte_eth_dev_info dev_info; 975 976 ret = eth_dev_info_get_print_err(pid, &dev_info); 977 if (ret != 0) { 978 TESTPMD_LOG(DEBUG, 979 "unable to get device info for port %d on addr 0x%p," 980 "mempool unmapping will not be performed\n", 981 pid, memhdr->addr); 982 continue; 983 } 984 985 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 986 if (ret) { 987 TESTPMD_LOG(DEBUG, 988 "unable to DMA unmap addr 0x%p " 989 "for device %s\n", 990 memhdr->addr, dev_info.device->name); 991 } 992 } 993 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 994 if (ret) { 995 TESTPMD_LOG(DEBUG, 996 "unable to un-register addr 0x%p\n", memhdr->addr); 997 } 998 } 999 1000 static void 1001 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1002 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1003 { 1004 uint16_t pid = 0; 1005 size_t page_size = sysconf(_SC_PAGESIZE); 1006 int ret; 1007 1008 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1009 page_size); 1010 if (ret) { 1011 TESTPMD_LOG(DEBUG, 1012 "unable to register addr 0x%p\n", memhdr->addr); 1013 return; 1014 } 1015 RTE_ETH_FOREACH_DEV(pid) { 1016 struct rte_eth_dev_info dev_info; 1017 1018 ret = eth_dev_info_get_print_err(pid, &dev_info); 1019 if (ret != 0) { 1020 TESTPMD_LOG(DEBUG, 1021 "unable to get device info for port %d on addr 0x%p," 1022 "mempool mapping will not be performed\n", 1023 pid, memhdr->addr); 1024 continue; 1025 } 1026 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1027 if (ret) { 1028 TESTPMD_LOG(DEBUG, 1029 "unable to DMA map addr 0x%p " 1030 "for device %s\n", 1031 memhdr->addr, dev_info.device->name); 1032 } 1033 } 1034 } 1035 #endif 1036 1037 static unsigned int 1038 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1039 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1040 { 1041 struct rte_pktmbuf_extmem *xmem; 1042 unsigned int ext_num, zone_num, elt_num; 1043 uint16_t elt_size; 1044 1045 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1046 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1047 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1048 1049 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1050 if (xmem == NULL) { 1051 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1052 "external buffer descriptors\n"); 1053 *ext_mem = NULL; 1054 return 0; 1055 } 1056 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1057 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1058 const struct rte_memzone *mz; 1059 char mz_name[RTE_MEMZONE_NAMESIZE]; 1060 int ret; 1061 1062 ret = snprintf(mz_name, sizeof(mz_name), 1063 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1064 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1065 errno = ENAMETOOLONG; 1066 ext_num = 0; 1067 break; 1068 } 1069 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE, 1070 socket_id, 1071 RTE_MEMZONE_IOVA_CONTIG | 1072 RTE_MEMZONE_1GB | 1073 RTE_MEMZONE_SIZE_HINT_ONLY); 1074 if (mz == NULL) { 1075 /* 1076 * The caller exits on external buffer creation 1077 * error, so there is no need to free memzones. 1078 */ 1079 errno = ENOMEM; 1080 ext_num = 0; 1081 break; 1082 } 1083 xseg->buf_ptr = mz->addr; 1084 xseg->buf_iova = mz->iova; 1085 xseg->buf_len = EXTBUF_ZONE_SIZE; 1086 xseg->elt_size = elt_size; 1087 } 1088 if (ext_num == 0 && xmem != NULL) { 1089 free(xmem); 1090 xmem = NULL; 1091 } 1092 *ext_mem = xmem; 1093 return ext_num; 1094 } 1095 1096 /* 1097 * Configuration initialisation done once at init time. 1098 */ 1099 static struct rte_mempool * 1100 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1101 unsigned int socket_id, uint16_t size_idx) 1102 { 1103 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1104 struct rte_mempool *rte_mp = NULL; 1105 #ifndef RTE_EXEC_ENV_WINDOWS 1106 uint32_t mb_size; 1107 1108 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1109 #endif 1110 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1111 if (!is_proc_primary()) { 1112 rte_mp = rte_mempool_lookup(pool_name); 1113 if (rte_mp == NULL) 1114 rte_exit(EXIT_FAILURE, 1115 "Get mbuf pool for socket %u failed: %s\n", 1116 socket_id, rte_strerror(rte_errno)); 1117 return rte_mp; 1118 } 1119 1120 TESTPMD_LOG(INFO, 1121 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1122 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1123 1124 switch (mp_alloc_type) { 1125 case MP_ALLOC_NATIVE: 1126 { 1127 /* wrapper to rte_mempool_create() */ 1128 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1129 rte_mbuf_best_mempool_ops()); 1130 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1131 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1132 break; 1133 } 1134 #ifndef RTE_EXEC_ENV_WINDOWS 1135 case MP_ALLOC_ANON: 1136 { 1137 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1138 mb_size, (unsigned int) mb_mempool_cache, 1139 sizeof(struct rte_pktmbuf_pool_private), 1140 socket_id, mempool_flags); 1141 if (rte_mp == NULL) 1142 goto err; 1143 1144 if (rte_mempool_populate_anon(rte_mp) == 0) { 1145 rte_mempool_free(rte_mp); 1146 rte_mp = NULL; 1147 goto err; 1148 } 1149 rte_pktmbuf_pool_init(rte_mp, NULL); 1150 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1151 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1152 break; 1153 } 1154 case MP_ALLOC_XMEM: 1155 case MP_ALLOC_XMEM_HUGE: 1156 { 1157 int heap_socket; 1158 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1159 1160 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1161 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1162 1163 heap_socket = 1164 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1165 if (heap_socket < 0) 1166 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1167 1168 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1169 rte_mbuf_best_mempool_ops()); 1170 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1171 mb_mempool_cache, 0, mbuf_seg_size, 1172 heap_socket); 1173 break; 1174 } 1175 #endif 1176 case MP_ALLOC_XBUF: 1177 { 1178 struct rte_pktmbuf_extmem *ext_mem; 1179 unsigned int ext_num; 1180 1181 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1182 socket_id, pool_name, &ext_mem); 1183 if (ext_num == 0) 1184 rte_exit(EXIT_FAILURE, 1185 "Can't create pinned data buffers\n"); 1186 1187 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1188 rte_mbuf_best_mempool_ops()); 1189 rte_mp = rte_pktmbuf_pool_create_extbuf 1190 (pool_name, nb_mbuf, mb_mempool_cache, 1191 0, mbuf_seg_size, socket_id, 1192 ext_mem, ext_num); 1193 free(ext_mem); 1194 break; 1195 } 1196 default: 1197 { 1198 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1199 } 1200 } 1201 1202 #ifndef RTE_EXEC_ENV_WINDOWS 1203 err: 1204 #endif 1205 if (rte_mp == NULL) { 1206 rte_exit(EXIT_FAILURE, 1207 "Creation of mbuf pool for socket %u failed: %s\n", 1208 socket_id, rte_strerror(rte_errno)); 1209 } else if (verbose_level > 0) { 1210 rte_mempool_dump(stdout, rte_mp); 1211 } 1212 return rte_mp; 1213 } 1214 1215 /* 1216 * Check given socket id is valid or not with NUMA mode, 1217 * if valid, return 0, else return -1 1218 */ 1219 static int 1220 check_socket_id(const unsigned int socket_id) 1221 { 1222 static int warning_once = 0; 1223 1224 if (new_socket_id(socket_id)) { 1225 if (!warning_once && numa_support) 1226 fprintf(stderr, 1227 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1228 warning_once = 1; 1229 return -1; 1230 } 1231 return 0; 1232 } 1233 1234 /* 1235 * Get the allowed maximum number of RX queues. 1236 * *pid return the port id which has minimal value of 1237 * max_rx_queues in all ports. 1238 */ 1239 queueid_t 1240 get_allowed_max_nb_rxq(portid_t *pid) 1241 { 1242 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1243 bool max_rxq_valid = false; 1244 portid_t pi; 1245 struct rte_eth_dev_info dev_info; 1246 1247 RTE_ETH_FOREACH_DEV(pi) { 1248 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1249 continue; 1250 1251 max_rxq_valid = true; 1252 if (dev_info.max_rx_queues < allowed_max_rxq) { 1253 allowed_max_rxq = dev_info.max_rx_queues; 1254 *pid = pi; 1255 } 1256 } 1257 return max_rxq_valid ? allowed_max_rxq : 0; 1258 } 1259 1260 /* 1261 * Check input rxq is valid or not. 1262 * If input rxq is not greater than any of maximum number 1263 * of RX queues of all ports, it is valid. 1264 * if valid, return 0, else return -1 1265 */ 1266 int 1267 check_nb_rxq(queueid_t rxq) 1268 { 1269 queueid_t allowed_max_rxq; 1270 portid_t pid = 0; 1271 1272 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1273 if (rxq > allowed_max_rxq) { 1274 fprintf(stderr, 1275 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1276 rxq, allowed_max_rxq, pid); 1277 return -1; 1278 } 1279 return 0; 1280 } 1281 1282 /* 1283 * Get the allowed maximum number of TX queues. 1284 * *pid return the port id which has minimal value of 1285 * max_tx_queues in all ports. 1286 */ 1287 queueid_t 1288 get_allowed_max_nb_txq(portid_t *pid) 1289 { 1290 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1291 bool max_txq_valid = false; 1292 portid_t pi; 1293 struct rte_eth_dev_info dev_info; 1294 1295 RTE_ETH_FOREACH_DEV(pi) { 1296 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1297 continue; 1298 1299 max_txq_valid = true; 1300 if (dev_info.max_tx_queues < allowed_max_txq) { 1301 allowed_max_txq = dev_info.max_tx_queues; 1302 *pid = pi; 1303 } 1304 } 1305 return max_txq_valid ? allowed_max_txq : 0; 1306 } 1307 1308 /* 1309 * Check input txq is valid or not. 1310 * If input txq is not greater than any of maximum number 1311 * of TX queues of all ports, it is valid. 1312 * if valid, return 0, else return -1 1313 */ 1314 int 1315 check_nb_txq(queueid_t txq) 1316 { 1317 queueid_t allowed_max_txq; 1318 portid_t pid = 0; 1319 1320 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1321 if (txq > allowed_max_txq) { 1322 fprintf(stderr, 1323 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1324 txq, allowed_max_txq, pid); 1325 return -1; 1326 } 1327 return 0; 1328 } 1329 1330 /* 1331 * Get the allowed maximum number of RXDs of every rx queue. 1332 * *pid return the port id which has minimal value of 1333 * max_rxd in all queues of all ports. 1334 */ 1335 static uint16_t 1336 get_allowed_max_nb_rxd(portid_t *pid) 1337 { 1338 uint16_t allowed_max_rxd = UINT16_MAX; 1339 portid_t pi; 1340 struct rte_eth_dev_info dev_info; 1341 1342 RTE_ETH_FOREACH_DEV(pi) { 1343 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1344 continue; 1345 1346 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1347 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1348 *pid = pi; 1349 } 1350 } 1351 return allowed_max_rxd; 1352 } 1353 1354 /* 1355 * Get the allowed minimal number of RXDs of every rx queue. 1356 * *pid return the port id which has minimal value of 1357 * min_rxd in all queues of all ports. 1358 */ 1359 static uint16_t 1360 get_allowed_min_nb_rxd(portid_t *pid) 1361 { 1362 uint16_t allowed_min_rxd = 0; 1363 portid_t pi; 1364 struct rte_eth_dev_info dev_info; 1365 1366 RTE_ETH_FOREACH_DEV(pi) { 1367 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1368 continue; 1369 1370 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1371 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1372 *pid = pi; 1373 } 1374 } 1375 1376 return allowed_min_rxd; 1377 } 1378 1379 /* 1380 * Check input rxd is valid or not. 1381 * If input rxd is not greater than any of maximum number 1382 * of RXDs of every Rx queues and is not less than any of 1383 * minimal number of RXDs of every Rx queues, it is valid. 1384 * if valid, return 0, else return -1 1385 */ 1386 int 1387 check_nb_rxd(queueid_t rxd) 1388 { 1389 uint16_t allowed_max_rxd; 1390 uint16_t allowed_min_rxd; 1391 portid_t pid = 0; 1392 1393 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1394 if (rxd > allowed_max_rxd) { 1395 fprintf(stderr, 1396 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1397 rxd, allowed_max_rxd, pid); 1398 return -1; 1399 } 1400 1401 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1402 if (rxd < allowed_min_rxd) { 1403 fprintf(stderr, 1404 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1405 rxd, allowed_min_rxd, pid); 1406 return -1; 1407 } 1408 1409 return 0; 1410 } 1411 1412 /* 1413 * Get the allowed maximum number of TXDs of every rx queues. 1414 * *pid return the port id which has minimal value of 1415 * max_txd in every tx queue. 1416 */ 1417 static uint16_t 1418 get_allowed_max_nb_txd(portid_t *pid) 1419 { 1420 uint16_t allowed_max_txd = UINT16_MAX; 1421 portid_t pi; 1422 struct rte_eth_dev_info dev_info; 1423 1424 RTE_ETH_FOREACH_DEV(pi) { 1425 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1426 continue; 1427 1428 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1429 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1430 *pid = pi; 1431 } 1432 } 1433 return allowed_max_txd; 1434 } 1435 1436 /* 1437 * Get the allowed maximum number of TXDs of every tx queues. 1438 * *pid return the port id which has minimal value of 1439 * min_txd in every tx queue. 1440 */ 1441 static uint16_t 1442 get_allowed_min_nb_txd(portid_t *pid) 1443 { 1444 uint16_t allowed_min_txd = 0; 1445 portid_t pi; 1446 struct rte_eth_dev_info dev_info; 1447 1448 RTE_ETH_FOREACH_DEV(pi) { 1449 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1450 continue; 1451 1452 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1453 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1454 *pid = pi; 1455 } 1456 } 1457 1458 return allowed_min_txd; 1459 } 1460 1461 /* 1462 * Check input txd is valid or not. 1463 * If input txd is not greater than any of maximum number 1464 * of TXDs of every Rx queues, it is valid. 1465 * if valid, return 0, else return -1 1466 */ 1467 int 1468 check_nb_txd(queueid_t txd) 1469 { 1470 uint16_t allowed_max_txd; 1471 uint16_t allowed_min_txd; 1472 portid_t pid = 0; 1473 1474 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1475 if (txd > allowed_max_txd) { 1476 fprintf(stderr, 1477 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1478 txd, allowed_max_txd, pid); 1479 return -1; 1480 } 1481 1482 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1483 if (txd < allowed_min_txd) { 1484 fprintf(stderr, 1485 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1486 txd, allowed_min_txd, pid); 1487 return -1; 1488 } 1489 return 0; 1490 } 1491 1492 1493 /* 1494 * Get the allowed maximum number of hairpin queues. 1495 * *pid return the port id which has minimal value of 1496 * max_hairpin_queues in all ports. 1497 */ 1498 queueid_t 1499 get_allowed_max_nb_hairpinq(portid_t *pid) 1500 { 1501 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1502 portid_t pi; 1503 struct rte_eth_hairpin_cap cap; 1504 1505 RTE_ETH_FOREACH_DEV(pi) { 1506 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1507 *pid = pi; 1508 return 0; 1509 } 1510 if (cap.max_nb_queues < allowed_max_hairpinq) { 1511 allowed_max_hairpinq = cap.max_nb_queues; 1512 *pid = pi; 1513 } 1514 } 1515 return allowed_max_hairpinq; 1516 } 1517 1518 /* 1519 * Check input hairpin is valid or not. 1520 * If input hairpin is not greater than any of maximum number 1521 * of hairpin queues of all ports, it is valid. 1522 * if valid, return 0, else return -1 1523 */ 1524 int 1525 check_nb_hairpinq(queueid_t hairpinq) 1526 { 1527 queueid_t allowed_max_hairpinq; 1528 portid_t pid = 0; 1529 1530 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1531 if (hairpinq > allowed_max_hairpinq) { 1532 fprintf(stderr, 1533 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1534 hairpinq, allowed_max_hairpinq, pid); 1535 return -1; 1536 } 1537 return 0; 1538 } 1539 1540 static int 1541 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1542 { 1543 uint32_t eth_overhead; 1544 1545 if (dev_info->max_mtu != UINT16_MAX && 1546 dev_info->max_rx_pktlen > dev_info->max_mtu) 1547 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1548 else 1549 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1550 1551 return eth_overhead; 1552 } 1553 1554 static void 1555 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1556 { 1557 struct rte_port *port = &ports[pid]; 1558 int ret; 1559 int i; 1560 1561 eth_rx_metadata_negotiate_mp(pid); 1562 1563 port->dev_conf.txmode = tx_mode; 1564 port->dev_conf.rxmode = rx_mode; 1565 1566 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1567 if (ret != 0) 1568 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1569 1570 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1571 port->dev_conf.txmode.offloads &= 1572 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1573 1574 /* Apply Rx offloads configuration */ 1575 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1576 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads; 1577 /* Apply Tx offloads configuration */ 1578 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1579 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads; 1580 1581 if (eth_link_speed) 1582 port->dev_conf.link_speeds = eth_link_speed; 1583 1584 if (max_rx_pkt_len) 1585 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1586 get_eth_overhead(&port->dev_info); 1587 1588 /* set flag to initialize port/queue */ 1589 port->need_reconfig = 1; 1590 port->need_reconfig_queues = 1; 1591 port->socket_id = socket_id; 1592 port->tx_metadata = 0; 1593 1594 /* 1595 * Check for maximum number of segments per MTU. 1596 * Accordingly update the mbuf data size. 1597 */ 1598 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1599 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1600 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1601 uint16_t mtu; 1602 1603 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1604 uint16_t data_size = (mtu + eth_overhead) / 1605 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1606 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1607 1608 if (buffer_size > mbuf_data_size[0]) { 1609 mbuf_data_size[0] = buffer_size; 1610 TESTPMD_LOG(WARNING, 1611 "Configured mbuf size of the first segment %hu\n", 1612 mbuf_data_size[0]); 1613 } 1614 } 1615 } 1616 } 1617 1618 static void 1619 init_config(void) 1620 { 1621 portid_t pid; 1622 struct rte_mempool *mbp; 1623 unsigned int nb_mbuf_per_pool; 1624 lcoreid_t lc_id; 1625 #ifdef RTE_LIB_GRO 1626 struct rte_gro_param gro_param; 1627 #endif 1628 #ifdef RTE_LIB_GSO 1629 uint32_t gso_types; 1630 #endif 1631 1632 /* Configuration of logical cores. */ 1633 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1634 sizeof(struct fwd_lcore *) * nb_lcores, 1635 RTE_CACHE_LINE_SIZE); 1636 if (fwd_lcores == NULL) { 1637 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1638 "failed\n", nb_lcores); 1639 } 1640 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1641 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1642 sizeof(struct fwd_lcore), 1643 RTE_CACHE_LINE_SIZE); 1644 if (fwd_lcores[lc_id] == NULL) { 1645 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1646 "failed\n"); 1647 } 1648 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1649 } 1650 1651 RTE_ETH_FOREACH_DEV(pid) { 1652 uint32_t socket_id; 1653 1654 if (numa_support) { 1655 socket_id = port_numa[pid]; 1656 if (port_numa[pid] == NUMA_NO_CONFIG) { 1657 socket_id = rte_eth_dev_socket_id(pid); 1658 1659 /* 1660 * if socket_id is invalid, 1661 * set to the first available socket. 1662 */ 1663 if (check_socket_id(socket_id) < 0) 1664 socket_id = socket_ids[0]; 1665 } 1666 } else { 1667 socket_id = (socket_num == UMA_NO_CONFIG) ? 1668 0 : socket_num; 1669 } 1670 /* Apply default TxRx configuration for all ports */ 1671 init_config_port_offloads(pid, socket_id); 1672 } 1673 /* 1674 * Create pools of mbuf. 1675 * If NUMA support is disabled, create a single pool of mbuf in 1676 * socket 0 memory by default. 1677 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1678 * 1679 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1680 * nb_txd can be configured at run time. 1681 */ 1682 if (param_total_num_mbufs) 1683 nb_mbuf_per_pool = param_total_num_mbufs; 1684 else { 1685 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1686 (nb_lcores * mb_mempool_cache) + 1687 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1688 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1689 } 1690 1691 if (numa_support) { 1692 uint8_t i, j; 1693 1694 for (i = 0; i < num_sockets; i++) 1695 for (j = 0; j < mbuf_data_size_n; j++) 1696 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1697 mbuf_pool_create(mbuf_data_size[j], 1698 nb_mbuf_per_pool, 1699 socket_ids[i], j); 1700 } else { 1701 uint8_t i; 1702 1703 for (i = 0; i < mbuf_data_size_n; i++) 1704 mempools[i] = mbuf_pool_create 1705 (mbuf_data_size[i], 1706 nb_mbuf_per_pool, 1707 socket_num == UMA_NO_CONFIG ? 1708 0 : socket_num, i); 1709 } 1710 1711 init_port_config(); 1712 1713 #ifdef RTE_LIB_GSO 1714 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1715 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1716 #endif 1717 /* 1718 * Records which Mbuf pool to use by each logical core, if needed. 1719 */ 1720 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1721 mbp = mbuf_pool_find( 1722 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1723 1724 if (mbp == NULL) 1725 mbp = mbuf_pool_find(0, 0); 1726 fwd_lcores[lc_id]->mbp = mbp; 1727 #ifdef RTE_LIB_GSO 1728 /* initialize GSO context */ 1729 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1730 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1731 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1732 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1733 RTE_ETHER_CRC_LEN; 1734 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1735 #endif 1736 } 1737 1738 fwd_config_setup(); 1739 1740 #ifdef RTE_LIB_GRO 1741 /* create a gro context for each lcore */ 1742 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1743 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1744 gro_param.max_item_per_flow = MAX_PKT_BURST; 1745 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1746 gro_param.socket_id = rte_lcore_to_socket_id( 1747 fwd_lcores_cpuids[lc_id]); 1748 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1749 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1750 rte_exit(EXIT_FAILURE, 1751 "rte_gro_ctx_create() failed\n"); 1752 } 1753 } 1754 #endif 1755 } 1756 1757 1758 void 1759 reconfig(portid_t new_port_id, unsigned socket_id) 1760 { 1761 /* Reconfiguration of Ethernet ports. */ 1762 init_config_port_offloads(new_port_id, socket_id); 1763 init_port_config(); 1764 } 1765 1766 1767 int 1768 init_fwd_streams(void) 1769 { 1770 portid_t pid; 1771 struct rte_port *port; 1772 streamid_t sm_id, nb_fwd_streams_new; 1773 queueid_t q; 1774 1775 /* set socket id according to numa or not */ 1776 RTE_ETH_FOREACH_DEV(pid) { 1777 port = &ports[pid]; 1778 if (nb_rxq > port->dev_info.max_rx_queues) { 1779 fprintf(stderr, 1780 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1781 nb_rxq, port->dev_info.max_rx_queues); 1782 return -1; 1783 } 1784 if (nb_txq > port->dev_info.max_tx_queues) { 1785 fprintf(stderr, 1786 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1787 nb_txq, port->dev_info.max_tx_queues); 1788 return -1; 1789 } 1790 if (numa_support) { 1791 if (port_numa[pid] != NUMA_NO_CONFIG) 1792 port->socket_id = port_numa[pid]; 1793 else { 1794 port->socket_id = rte_eth_dev_socket_id(pid); 1795 1796 /* 1797 * if socket_id is invalid, 1798 * set to the first available socket. 1799 */ 1800 if (check_socket_id(port->socket_id) < 0) 1801 port->socket_id = socket_ids[0]; 1802 } 1803 } 1804 else { 1805 if (socket_num == UMA_NO_CONFIG) 1806 port->socket_id = 0; 1807 else 1808 port->socket_id = socket_num; 1809 } 1810 } 1811 1812 q = RTE_MAX(nb_rxq, nb_txq); 1813 if (q == 0) { 1814 fprintf(stderr, 1815 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1816 return -1; 1817 } 1818 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1819 if (nb_fwd_streams_new == nb_fwd_streams) 1820 return 0; 1821 /* clear the old */ 1822 if (fwd_streams != NULL) { 1823 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1824 if (fwd_streams[sm_id] == NULL) 1825 continue; 1826 rte_free(fwd_streams[sm_id]); 1827 fwd_streams[sm_id] = NULL; 1828 } 1829 rte_free(fwd_streams); 1830 fwd_streams = NULL; 1831 } 1832 1833 /* init new */ 1834 nb_fwd_streams = nb_fwd_streams_new; 1835 if (nb_fwd_streams) { 1836 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1837 sizeof(struct fwd_stream *) * nb_fwd_streams, 1838 RTE_CACHE_LINE_SIZE); 1839 if (fwd_streams == NULL) 1840 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1841 " (struct fwd_stream *)) failed\n", 1842 nb_fwd_streams); 1843 1844 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1845 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1846 " struct fwd_stream", sizeof(struct fwd_stream), 1847 RTE_CACHE_LINE_SIZE); 1848 if (fwd_streams[sm_id] == NULL) 1849 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1850 "(struct fwd_stream) failed\n"); 1851 } 1852 } 1853 1854 return 0; 1855 } 1856 1857 static void 1858 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1859 { 1860 uint64_t total_burst, sburst; 1861 uint64_t nb_burst; 1862 uint64_t burst_stats[4]; 1863 uint16_t pktnb_stats[4]; 1864 uint16_t nb_pkt; 1865 int burst_percent[4], sburstp; 1866 int i; 1867 1868 /* 1869 * First compute the total number of packet bursts and the 1870 * two highest numbers of bursts of the same number of packets. 1871 */ 1872 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1873 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1874 1875 /* Show stats for 0 burst size always */ 1876 total_burst = pbs->pkt_burst_spread[0]; 1877 burst_stats[0] = pbs->pkt_burst_spread[0]; 1878 pktnb_stats[0] = 0; 1879 1880 /* Find the next 2 burst sizes with highest occurrences. */ 1881 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1882 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1883 1884 if (nb_burst == 0) 1885 continue; 1886 1887 total_burst += nb_burst; 1888 1889 if (nb_burst > burst_stats[1]) { 1890 burst_stats[2] = burst_stats[1]; 1891 pktnb_stats[2] = pktnb_stats[1]; 1892 burst_stats[1] = nb_burst; 1893 pktnb_stats[1] = nb_pkt; 1894 } else if (nb_burst > burst_stats[2]) { 1895 burst_stats[2] = nb_burst; 1896 pktnb_stats[2] = nb_pkt; 1897 } 1898 } 1899 if (total_burst == 0) 1900 return; 1901 1902 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1903 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1904 if (i == 3) { 1905 printf("%d%% of other]\n", 100 - sburstp); 1906 return; 1907 } 1908 1909 sburst += burst_stats[i]; 1910 if (sburst == total_burst) { 1911 printf("%d%% of %d pkts]\n", 1912 100 - sburstp, (int) pktnb_stats[i]); 1913 return; 1914 } 1915 1916 burst_percent[i] = 1917 (double)burst_stats[i] / total_burst * 100; 1918 printf("%d%% of %d pkts + ", 1919 burst_percent[i], (int) pktnb_stats[i]); 1920 sburstp += burst_percent[i]; 1921 } 1922 } 1923 1924 static void 1925 fwd_stream_stats_display(streamid_t stream_id) 1926 { 1927 struct fwd_stream *fs; 1928 static const char *fwd_top_stats_border = "-------"; 1929 1930 fs = fwd_streams[stream_id]; 1931 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1932 (fs->fwd_dropped == 0)) 1933 return; 1934 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1935 "TX Port=%2d/Queue=%2d %s\n", 1936 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1937 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1938 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1939 " TX-dropped: %-14"PRIu64, 1940 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1941 1942 /* if checksum mode */ 1943 if (cur_fwd_eng == &csum_fwd_engine) { 1944 printf(" RX- bad IP checksum: %-14"PRIu64 1945 " Rx- bad L4 checksum: %-14"PRIu64 1946 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1947 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1948 fs->rx_bad_outer_l4_csum); 1949 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1950 fs->rx_bad_outer_ip_csum); 1951 } else { 1952 printf("\n"); 1953 } 1954 1955 if (record_burst_stats) { 1956 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1957 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1958 } 1959 } 1960 1961 void 1962 fwd_stats_display(void) 1963 { 1964 static const char *fwd_stats_border = "----------------------"; 1965 static const char *acc_stats_border = "+++++++++++++++"; 1966 struct { 1967 struct fwd_stream *rx_stream; 1968 struct fwd_stream *tx_stream; 1969 uint64_t tx_dropped; 1970 uint64_t rx_bad_ip_csum; 1971 uint64_t rx_bad_l4_csum; 1972 uint64_t rx_bad_outer_l4_csum; 1973 uint64_t rx_bad_outer_ip_csum; 1974 } ports_stats[RTE_MAX_ETHPORTS]; 1975 uint64_t total_rx_dropped = 0; 1976 uint64_t total_tx_dropped = 0; 1977 uint64_t total_rx_nombuf = 0; 1978 struct rte_eth_stats stats; 1979 uint64_t fwd_cycles = 0; 1980 uint64_t total_recv = 0; 1981 uint64_t total_xmit = 0; 1982 struct rte_port *port; 1983 streamid_t sm_id; 1984 portid_t pt_id; 1985 int i; 1986 1987 memset(ports_stats, 0, sizeof(ports_stats)); 1988 1989 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1990 struct fwd_stream *fs = fwd_streams[sm_id]; 1991 1992 if (cur_fwd_config.nb_fwd_streams > 1993 cur_fwd_config.nb_fwd_ports) { 1994 fwd_stream_stats_display(sm_id); 1995 } else { 1996 ports_stats[fs->tx_port].tx_stream = fs; 1997 ports_stats[fs->rx_port].rx_stream = fs; 1998 } 1999 2000 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2001 2002 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2003 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2004 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2005 fs->rx_bad_outer_l4_csum; 2006 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2007 fs->rx_bad_outer_ip_csum; 2008 2009 if (record_core_cycles) 2010 fwd_cycles += fs->core_cycles; 2011 } 2012 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2013 pt_id = fwd_ports_ids[i]; 2014 port = &ports[pt_id]; 2015 2016 rte_eth_stats_get(pt_id, &stats); 2017 stats.ipackets -= port->stats.ipackets; 2018 stats.opackets -= port->stats.opackets; 2019 stats.ibytes -= port->stats.ibytes; 2020 stats.obytes -= port->stats.obytes; 2021 stats.imissed -= port->stats.imissed; 2022 stats.oerrors -= port->stats.oerrors; 2023 stats.rx_nombuf -= port->stats.rx_nombuf; 2024 2025 total_recv += stats.ipackets; 2026 total_xmit += stats.opackets; 2027 total_rx_dropped += stats.imissed; 2028 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2029 total_tx_dropped += stats.oerrors; 2030 total_rx_nombuf += stats.rx_nombuf; 2031 2032 printf("\n %s Forward statistics for port %-2d %s\n", 2033 fwd_stats_border, pt_id, fwd_stats_border); 2034 2035 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2036 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2037 stats.ipackets + stats.imissed); 2038 2039 if (cur_fwd_eng == &csum_fwd_engine) { 2040 printf(" Bad-ipcsum: %-14"PRIu64 2041 " Bad-l4csum: %-14"PRIu64 2042 "Bad-outer-l4csum: %-14"PRIu64"\n", 2043 ports_stats[pt_id].rx_bad_ip_csum, 2044 ports_stats[pt_id].rx_bad_l4_csum, 2045 ports_stats[pt_id].rx_bad_outer_l4_csum); 2046 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2047 ports_stats[pt_id].rx_bad_outer_ip_csum); 2048 } 2049 if (stats.ierrors + stats.rx_nombuf > 0) { 2050 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2051 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2052 } 2053 2054 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2055 "TX-total: %-"PRIu64"\n", 2056 stats.opackets, ports_stats[pt_id].tx_dropped, 2057 stats.opackets + ports_stats[pt_id].tx_dropped); 2058 2059 if (record_burst_stats) { 2060 if (ports_stats[pt_id].rx_stream) 2061 pkt_burst_stats_display("RX", 2062 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2063 if (ports_stats[pt_id].tx_stream) 2064 pkt_burst_stats_display("TX", 2065 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2066 } 2067 2068 printf(" %s--------------------------------%s\n", 2069 fwd_stats_border, fwd_stats_border); 2070 } 2071 2072 printf("\n %s Accumulated forward statistics for all ports" 2073 "%s\n", 2074 acc_stats_border, acc_stats_border); 2075 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2076 "%-"PRIu64"\n" 2077 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2078 "%-"PRIu64"\n", 2079 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2080 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2081 if (total_rx_nombuf > 0) 2082 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2083 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2084 "%s\n", 2085 acc_stats_border, acc_stats_border); 2086 if (record_core_cycles) { 2087 #define CYC_PER_MHZ 1E6 2088 if (total_recv > 0 || total_xmit > 0) { 2089 uint64_t total_pkts = 0; 2090 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2091 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2092 total_pkts = total_xmit; 2093 else 2094 total_pkts = total_recv; 2095 2096 printf("\n CPU cycles/packet=%.2F (total cycles=" 2097 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2098 " MHz Clock\n", 2099 (double) fwd_cycles / total_pkts, 2100 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2101 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2102 } 2103 } 2104 } 2105 2106 void 2107 fwd_stats_reset(void) 2108 { 2109 streamid_t sm_id; 2110 portid_t pt_id; 2111 int i; 2112 2113 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2114 pt_id = fwd_ports_ids[i]; 2115 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2116 } 2117 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2118 struct fwd_stream *fs = fwd_streams[sm_id]; 2119 2120 fs->rx_packets = 0; 2121 fs->tx_packets = 0; 2122 fs->fwd_dropped = 0; 2123 fs->rx_bad_ip_csum = 0; 2124 fs->rx_bad_l4_csum = 0; 2125 fs->rx_bad_outer_l4_csum = 0; 2126 fs->rx_bad_outer_ip_csum = 0; 2127 2128 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2129 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2130 fs->core_cycles = 0; 2131 } 2132 } 2133 2134 static void 2135 flush_fwd_rx_queues(void) 2136 { 2137 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2138 portid_t rxp; 2139 portid_t port_id; 2140 queueid_t rxq; 2141 uint16_t nb_rx; 2142 uint16_t i; 2143 uint8_t j; 2144 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2145 uint64_t timer_period; 2146 2147 if (num_procs > 1) { 2148 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2149 return; 2150 } 2151 2152 /* convert to number of cycles */ 2153 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2154 2155 for (j = 0; j < 2; j++) { 2156 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2157 for (rxq = 0; rxq < nb_rxq; rxq++) { 2158 port_id = fwd_ports_ids[rxp]; 2159 /** 2160 * testpmd can stuck in the below do while loop 2161 * if rte_eth_rx_burst() always returns nonzero 2162 * packets. So timer is added to exit this loop 2163 * after 1sec timer expiry. 2164 */ 2165 prev_tsc = rte_rdtsc(); 2166 do { 2167 nb_rx = rte_eth_rx_burst(port_id, rxq, 2168 pkts_burst, MAX_PKT_BURST); 2169 for (i = 0; i < nb_rx; i++) 2170 rte_pktmbuf_free(pkts_burst[i]); 2171 2172 cur_tsc = rte_rdtsc(); 2173 diff_tsc = cur_tsc - prev_tsc; 2174 timer_tsc += diff_tsc; 2175 } while ((nb_rx > 0) && 2176 (timer_tsc < timer_period)); 2177 timer_tsc = 0; 2178 } 2179 } 2180 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2181 } 2182 } 2183 2184 static void 2185 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2186 { 2187 struct fwd_stream **fsm; 2188 streamid_t nb_fs; 2189 streamid_t sm_id; 2190 #ifdef RTE_LIB_BITRATESTATS 2191 uint64_t tics_per_1sec; 2192 uint64_t tics_datum; 2193 uint64_t tics_current; 2194 uint16_t i, cnt_ports; 2195 2196 cnt_ports = nb_ports; 2197 tics_datum = rte_rdtsc(); 2198 tics_per_1sec = rte_get_timer_hz(); 2199 #endif 2200 fsm = &fwd_streams[fc->stream_idx]; 2201 nb_fs = fc->stream_nb; 2202 do { 2203 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2204 (*pkt_fwd)(fsm[sm_id]); 2205 #ifdef RTE_LIB_BITRATESTATS 2206 if (bitrate_enabled != 0 && 2207 bitrate_lcore_id == rte_lcore_id()) { 2208 tics_current = rte_rdtsc(); 2209 if (tics_current - tics_datum >= tics_per_1sec) { 2210 /* Periodic bitrate calculation */ 2211 for (i = 0; i < cnt_ports; i++) 2212 rte_stats_bitrate_calc(bitrate_data, 2213 ports_ids[i]); 2214 tics_datum = tics_current; 2215 } 2216 } 2217 #endif 2218 #ifdef RTE_LIB_LATENCYSTATS 2219 if (latencystats_enabled != 0 && 2220 latencystats_lcore_id == rte_lcore_id()) 2221 rte_latencystats_update(); 2222 #endif 2223 2224 } while (! fc->stopped); 2225 } 2226 2227 static int 2228 start_pkt_forward_on_core(void *fwd_arg) 2229 { 2230 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2231 cur_fwd_config.fwd_eng->packet_fwd); 2232 return 0; 2233 } 2234 2235 /* 2236 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2237 * Used to start communication flows in network loopback test configurations. 2238 */ 2239 static int 2240 run_one_txonly_burst_on_core(void *fwd_arg) 2241 { 2242 struct fwd_lcore *fwd_lc; 2243 struct fwd_lcore tmp_lcore; 2244 2245 fwd_lc = (struct fwd_lcore *) fwd_arg; 2246 tmp_lcore = *fwd_lc; 2247 tmp_lcore.stopped = 1; 2248 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2249 return 0; 2250 } 2251 2252 /* 2253 * Launch packet forwarding: 2254 * - Setup per-port forwarding context. 2255 * - launch logical cores with their forwarding configuration. 2256 */ 2257 static void 2258 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2259 { 2260 unsigned int i; 2261 unsigned int lc_id; 2262 int diag; 2263 2264 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2265 lc_id = fwd_lcores_cpuids[i]; 2266 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2267 fwd_lcores[i]->stopped = 0; 2268 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2269 fwd_lcores[i], lc_id); 2270 if (diag != 0) 2271 fprintf(stderr, 2272 "launch lcore %u failed - diag=%d\n", 2273 lc_id, diag); 2274 } 2275 } 2276 } 2277 2278 /* 2279 * Launch packet forwarding configuration. 2280 */ 2281 void 2282 start_packet_forwarding(int with_tx_first) 2283 { 2284 port_fwd_begin_t port_fwd_begin; 2285 port_fwd_end_t port_fwd_end; 2286 unsigned int i; 2287 2288 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2289 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2290 2291 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2292 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2293 2294 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2295 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2296 (!nb_rxq || !nb_txq)) 2297 rte_exit(EXIT_FAILURE, 2298 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2299 cur_fwd_eng->fwd_mode_name); 2300 2301 if (all_ports_started() == 0) { 2302 fprintf(stderr, "Not all ports were started\n"); 2303 return; 2304 } 2305 if (test_done == 0) { 2306 fprintf(stderr, "Packet forwarding already started\n"); 2307 return; 2308 } 2309 2310 fwd_config_setup(); 2311 2312 pkt_fwd_config_display(&cur_fwd_config); 2313 if (!pkt_fwd_shared_rxq_check()) 2314 return; 2315 2316 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2317 if (port_fwd_begin != NULL) { 2318 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2319 if (port_fwd_begin(fwd_ports_ids[i])) { 2320 fprintf(stderr, 2321 "Packet forwarding is not ready\n"); 2322 return; 2323 } 2324 } 2325 } 2326 2327 if (with_tx_first) { 2328 port_fwd_begin = tx_only_engine.port_fwd_begin; 2329 if (port_fwd_begin != NULL) { 2330 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2331 if (port_fwd_begin(fwd_ports_ids[i])) { 2332 fprintf(stderr, 2333 "Packet forwarding is not ready\n"); 2334 return; 2335 } 2336 } 2337 } 2338 } 2339 2340 test_done = 0; 2341 2342 if(!no_flush_rx) 2343 flush_fwd_rx_queues(); 2344 2345 rxtx_config_display(); 2346 2347 fwd_stats_reset(); 2348 if (with_tx_first) { 2349 while (with_tx_first--) { 2350 launch_packet_forwarding( 2351 run_one_txonly_burst_on_core); 2352 rte_eal_mp_wait_lcore(); 2353 } 2354 port_fwd_end = tx_only_engine.port_fwd_end; 2355 if (port_fwd_end != NULL) { 2356 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2357 (*port_fwd_end)(fwd_ports_ids[i]); 2358 } 2359 } 2360 launch_packet_forwarding(start_pkt_forward_on_core); 2361 } 2362 2363 void 2364 stop_packet_forwarding(void) 2365 { 2366 port_fwd_end_t port_fwd_end; 2367 lcoreid_t lc_id; 2368 portid_t pt_id; 2369 int i; 2370 2371 if (test_done) { 2372 fprintf(stderr, "Packet forwarding not started\n"); 2373 return; 2374 } 2375 printf("Telling cores to stop..."); 2376 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2377 fwd_lcores[lc_id]->stopped = 1; 2378 printf("\nWaiting for lcores to finish...\n"); 2379 rte_eal_mp_wait_lcore(); 2380 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2381 if (port_fwd_end != NULL) { 2382 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2383 pt_id = fwd_ports_ids[i]; 2384 (*port_fwd_end)(pt_id); 2385 } 2386 } 2387 2388 fwd_stats_display(); 2389 2390 printf("\nDone.\n"); 2391 test_done = 1; 2392 } 2393 2394 void 2395 dev_set_link_up(portid_t pid) 2396 { 2397 if (rte_eth_dev_set_link_up(pid) < 0) 2398 fprintf(stderr, "\nSet link up fail.\n"); 2399 } 2400 2401 void 2402 dev_set_link_down(portid_t pid) 2403 { 2404 if (rte_eth_dev_set_link_down(pid) < 0) 2405 fprintf(stderr, "\nSet link down fail.\n"); 2406 } 2407 2408 static int 2409 all_ports_started(void) 2410 { 2411 portid_t pi; 2412 struct rte_port *port; 2413 2414 RTE_ETH_FOREACH_DEV(pi) { 2415 port = &ports[pi]; 2416 /* Check if there is a port which is not started */ 2417 if ((port->port_status != RTE_PORT_STARTED) && 2418 (port->slave_flag == 0)) 2419 return 0; 2420 } 2421 2422 /* No port is not started */ 2423 return 1; 2424 } 2425 2426 int 2427 port_is_stopped(portid_t port_id) 2428 { 2429 struct rte_port *port = &ports[port_id]; 2430 2431 if ((port->port_status != RTE_PORT_STOPPED) && 2432 (port->slave_flag == 0)) 2433 return 0; 2434 return 1; 2435 } 2436 2437 int 2438 all_ports_stopped(void) 2439 { 2440 portid_t pi; 2441 2442 RTE_ETH_FOREACH_DEV(pi) { 2443 if (!port_is_stopped(pi)) 2444 return 0; 2445 } 2446 2447 return 1; 2448 } 2449 2450 int 2451 port_is_started(portid_t port_id) 2452 { 2453 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2454 return 0; 2455 2456 if (ports[port_id].port_status != RTE_PORT_STARTED) 2457 return 0; 2458 2459 return 1; 2460 } 2461 2462 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2463 static int 2464 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2465 { 2466 queueid_t qi; 2467 struct rte_eth_hairpin_conf hairpin_conf = { 2468 .peer_count = 1, 2469 }; 2470 int i; 2471 int diag; 2472 struct rte_port *port = &ports[pi]; 2473 uint16_t peer_rx_port = pi; 2474 uint16_t peer_tx_port = pi; 2475 uint32_t manual = 1; 2476 uint32_t tx_exp = hairpin_mode & 0x10; 2477 2478 if (!(hairpin_mode & 0xf)) { 2479 peer_rx_port = pi; 2480 peer_tx_port = pi; 2481 manual = 0; 2482 } else if (hairpin_mode & 0x1) { 2483 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2484 RTE_ETH_DEV_NO_OWNER); 2485 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2486 peer_tx_port = rte_eth_find_next_owned_by(0, 2487 RTE_ETH_DEV_NO_OWNER); 2488 if (p_pi != RTE_MAX_ETHPORTS) { 2489 peer_rx_port = p_pi; 2490 } else { 2491 uint16_t next_pi; 2492 2493 /* Last port will be the peer RX port of the first. */ 2494 RTE_ETH_FOREACH_DEV(next_pi) 2495 peer_rx_port = next_pi; 2496 } 2497 manual = 1; 2498 } else if (hairpin_mode & 0x2) { 2499 if (cnt_pi & 0x1) { 2500 peer_rx_port = p_pi; 2501 } else { 2502 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2503 RTE_ETH_DEV_NO_OWNER); 2504 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2505 peer_rx_port = pi; 2506 } 2507 peer_tx_port = peer_rx_port; 2508 manual = 1; 2509 } 2510 2511 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2512 hairpin_conf.peers[0].port = peer_rx_port; 2513 hairpin_conf.peers[0].queue = i + nb_rxq; 2514 hairpin_conf.manual_bind = !!manual; 2515 hairpin_conf.tx_explicit = !!tx_exp; 2516 diag = rte_eth_tx_hairpin_queue_setup 2517 (pi, qi, nb_txd, &hairpin_conf); 2518 i++; 2519 if (diag == 0) 2520 continue; 2521 2522 /* Fail to setup rx queue, return */ 2523 if (port->port_status == RTE_PORT_HANDLING) 2524 port->port_status = RTE_PORT_STOPPED; 2525 else 2526 fprintf(stderr, 2527 "Port %d can not be set back to stopped\n", pi); 2528 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2529 pi); 2530 /* try to reconfigure queues next time */ 2531 port->need_reconfig_queues = 1; 2532 return -1; 2533 } 2534 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2535 hairpin_conf.peers[0].port = peer_tx_port; 2536 hairpin_conf.peers[0].queue = i + nb_txq; 2537 hairpin_conf.manual_bind = !!manual; 2538 hairpin_conf.tx_explicit = !!tx_exp; 2539 diag = rte_eth_rx_hairpin_queue_setup 2540 (pi, qi, nb_rxd, &hairpin_conf); 2541 i++; 2542 if (diag == 0) 2543 continue; 2544 2545 /* Fail to setup rx queue, return */ 2546 if (port->port_status == RTE_PORT_HANDLING) 2547 port->port_status = RTE_PORT_STOPPED; 2548 else 2549 fprintf(stderr, 2550 "Port %d can not be set back to stopped\n", pi); 2551 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2552 pi); 2553 /* try to reconfigure queues next time */ 2554 port->need_reconfig_queues = 1; 2555 return -1; 2556 } 2557 return 0; 2558 } 2559 2560 /* Configure the Rx with optional split. */ 2561 int 2562 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2563 uint16_t nb_rx_desc, unsigned int socket_id, 2564 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2565 { 2566 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2567 unsigned int i, mp_n; 2568 int ret; 2569 2570 if (rx_pkt_nb_segs <= 1 || 2571 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2572 rx_conf->rx_seg = NULL; 2573 rx_conf->rx_nseg = 0; 2574 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2575 nb_rx_desc, socket_id, 2576 rx_conf, mp); 2577 return ret; 2578 } 2579 for (i = 0; i < rx_pkt_nb_segs; i++) { 2580 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2581 struct rte_mempool *mpx; 2582 /* 2583 * Use last valid pool for the segments with number 2584 * exceeding the pool index. 2585 */ 2586 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2587 mpx = mbuf_pool_find(socket_id, mp_n); 2588 /* Handle zero as mbuf data buffer size. */ 2589 rx_seg->length = rx_pkt_seg_lengths[i] ? 2590 rx_pkt_seg_lengths[i] : 2591 mbuf_data_size[mp_n]; 2592 rx_seg->offset = i < rx_pkt_nb_offs ? 2593 rx_pkt_seg_offsets[i] : 0; 2594 rx_seg->mp = mpx ? mpx : mp; 2595 } 2596 rx_conf->rx_nseg = rx_pkt_nb_segs; 2597 rx_conf->rx_seg = rx_useg; 2598 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2599 socket_id, rx_conf, NULL); 2600 rx_conf->rx_seg = NULL; 2601 rx_conf->rx_nseg = 0; 2602 return ret; 2603 } 2604 2605 static int 2606 alloc_xstats_display_info(portid_t pi) 2607 { 2608 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2609 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2610 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2611 2612 if (xstats_display_num == 0) 2613 return 0; 2614 2615 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2616 if (*ids_supp == NULL) 2617 goto fail_ids_supp; 2618 2619 *prev_values = calloc(xstats_display_num, 2620 sizeof(**prev_values)); 2621 if (*prev_values == NULL) 2622 goto fail_prev_values; 2623 2624 *curr_values = calloc(xstats_display_num, 2625 sizeof(**curr_values)); 2626 if (*curr_values == NULL) 2627 goto fail_curr_values; 2628 2629 ports[pi].xstats_info.allocated = true; 2630 2631 return 0; 2632 2633 fail_curr_values: 2634 free(*prev_values); 2635 fail_prev_values: 2636 free(*ids_supp); 2637 fail_ids_supp: 2638 return -ENOMEM; 2639 } 2640 2641 static void 2642 free_xstats_display_info(portid_t pi) 2643 { 2644 if (!ports[pi].xstats_info.allocated) 2645 return; 2646 free(ports[pi].xstats_info.ids_supp); 2647 free(ports[pi].xstats_info.prev_values); 2648 free(ports[pi].xstats_info.curr_values); 2649 ports[pi].xstats_info.allocated = false; 2650 } 2651 2652 /** Fill helper structures for specified port to show extended statistics. */ 2653 static void 2654 fill_xstats_display_info_for_port(portid_t pi) 2655 { 2656 unsigned int stat, stat_supp; 2657 const char *xstat_name; 2658 struct rte_port *port; 2659 uint64_t *ids_supp; 2660 int rc; 2661 2662 if (xstats_display_num == 0) 2663 return; 2664 2665 if (pi == (portid_t)RTE_PORT_ALL) { 2666 fill_xstats_display_info(); 2667 return; 2668 } 2669 2670 port = &ports[pi]; 2671 if (port->port_status != RTE_PORT_STARTED) 2672 return; 2673 2674 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2675 rte_exit(EXIT_FAILURE, 2676 "Failed to allocate xstats display memory\n"); 2677 2678 ids_supp = port->xstats_info.ids_supp; 2679 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2680 xstat_name = xstats_display[stat].name; 2681 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2682 ids_supp + stat_supp); 2683 if (rc != 0) { 2684 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2685 xstat_name, pi, stat); 2686 continue; 2687 } 2688 stat_supp++; 2689 } 2690 2691 port->xstats_info.ids_supp_sz = stat_supp; 2692 } 2693 2694 /** Fill helper structures for all ports to show extended statistics. */ 2695 static void 2696 fill_xstats_display_info(void) 2697 { 2698 portid_t pi; 2699 2700 if (xstats_display_num == 0) 2701 return; 2702 2703 RTE_ETH_FOREACH_DEV(pi) 2704 fill_xstats_display_info_for_port(pi); 2705 } 2706 2707 int 2708 start_port(portid_t pid) 2709 { 2710 int diag, need_check_link_status = -1; 2711 portid_t pi; 2712 portid_t p_pi = RTE_MAX_ETHPORTS; 2713 portid_t pl[RTE_MAX_ETHPORTS]; 2714 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2715 uint16_t cnt_pi = 0; 2716 uint16_t cfg_pi = 0; 2717 int peer_pi; 2718 queueid_t qi; 2719 struct rte_port *port; 2720 struct rte_eth_hairpin_cap cap; 2721 2722 if (port_id_is_invalid(pid, ENABLED_WARN)) 2723 return 0; 2724 2725 RTE_ETH_FOREACH_DEV(pi) { 2726 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2727 continue; 2728 2729 if (port_is_bonding_slave(pi)) { 2730 fprintf(stderr, 2731 "Please remove port %d from bonded device.\n", 2732 pi); 2733 continue; 2734 } 2735 2736 need_check_link_status = 0; 2737 port = &ports[pi]; 2738 if (port->port_status == RTE_PORT_STOPPED) 2739 port->port_status = RTE_PORT_HANDLING; 2740 else { 2741 fprintf(stderr, "Port %d is now not stopped\n", pi); 2742 continue; 2743 } 2744 2745 if (port->need_reconfig > 0) { 2746 struct rte_eth_conf dev_conf; 2747 int k; 2748 2749 port->need_reconfig = 0; 2750 2751 if (flow_isolate_all) { 2752 int ret = port_flow_isolate(pi, 1); 2753 if (ret) { 2754 fprintf(stderr, 2755 "Failed to apply isolated mode on port %d\n", 2756 pi); 2757 return -1; 2758 } 2759 } 2760 configure_rxtx_dump_callbacks(0); 2761 printf("Configuring Port %d (socket %u)\n", pi, 2762 port->socket_id); 2763 if (nb_hairpinq > 0 && 2764 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2765 fprintf(stderr, 2766 "Port %d doesn't support hairpin queues\n", 2767 pi); 2768 return -1; 2769 } 2770 2771 /* configure port */ 2772 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2773 nb_txq + nb_hairpinq, 2774 &(port->dev_conf)); 2775 if (diag != 0) { 2776 if (port->port_status == RTE_PORT_HANDLING) 2777 port->port_status = RTE_PORT_STOPPED; 2778 else 2779 fprintf(stderr, 2780 "Port %d can not be set back to stopped\n", 2781 pi); 2782 fprintf(stderr, "Fail to configure port %d\n", 2783 pi); 2784 /* try to reconfigure port next time */ 2785 port->need_reconfig = 1; 2786 return -1; 2787 } 2788 /* get device configuration*/ 2789 if (0 != 2790 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2791 fprintf(stderr, 2792 "port %d can not get device configuration\n", 2793 pi); 2794 return -1; 2795 } 2796 /* Apply Rx offloads configuration */ 2797 if (dev_conf.rxmode.offloads != 2798 port->dev_conf.rxmode.offloads) { 2799 port->dev_conf.rxmode.offloads |= 2800 dev_conf.rxmode.offloads; 2801 for (k = 0; 2802 k < port->dev_info.max_rx_queues; 2803 k++) 2804 port->rx_conf[k].offloads |= 2805 dev_conf.rxmode.offloads; 2806 } 2807 /* Apply Tx offloads configuration */ 2808 if (dev_conf.txmode.offloads != 2809 port->dev_conf.txmode.offloads) { 2810 port->dev_conf.txmode.offloads |= 2811 dev_conf.txmode.offloads; 2812 for (k = 0; 2813 k < port->dev_info.max_tx_queues; 2814 k++) 2815 port->tx_conf[k].offloads |= 2816 dev_conf.txmode.offloads; 2817 } 2818 } 2819 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2820 port->need_reconfig_queues = 0; 2821 /* setup tx queues */ 2822 for (qi = 0; qi < nb_txq; qi++) { 2823 if ((numa_support) && 2824 (txring_numa[pi] != NUMA_NO_CONFIG)) 2825 diag = rte_eth_tx_queue_setup(pi, qi, 2826 port->nb_tx_desc[qi], 2827 txring_numa[pi], 2828 &(port->tx_conf[qi])); 2829 else 2830 diag = rte_eth_tx_queue_setup(pi, qi, 2831 port->nb_tx_desc[qi], 2832 port->socket_id, 2833 &(port->tx_conf[qi])); 2834 2835 if (diag == 0) 2836 continue; 2837 2838 /* Fail to setup tx queue, return */ 2839 if (port->port_status == RTE_PORT_HANDLING) 2840 port->port_status = RTE_PORT_STOPPED; 2841 else 2842 fprintf(stderr, 2843 "Port %d can not be set back to stopped\n", 2844 pi); 2845 fprintf(stderr, 2846 "Fail to configure port %d tx queues\n", 2847 pi); 2848 /* try to reconfigure queues next time */ 2849 port->need_reconfig_queues = 1; 2850 return -1; 2851 } 2852 for (qi = 0; qi < nb_rxq; qi++) { 2853 /* setup rx queues */ 2854 if ((numa_support) && 2855 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2856 struct rte_mempool * mp = 2857 mbuf_pool_find 2858 (rxring_numa[pi], 0); 2859 if (mp == NULL) { 2860 fprintf(stderr, 2861 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2862 rxring_numa[pi]); 2863 return -1; 2864 } 2865 2866 diag = rx_queue_setup(pi, qi, 2867 port->nb_rx_desc[qi], 2868 rxring_numa[pi], 2869 &(port->rx_conf[qi]), 2870 mp); 2871 } else { 2872 struct rte_mempool *mp = 2873 mbuf_pool_find 2874 (port->socket_id, 0); 2875 if (mp == NULL) { 2876 fprintf(stderr, 2877 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2878 port->socket_id); 2879 return -1; 2880 } 2881 diag = rx_queue_setup(pi, qi, 2882 port->nb_rx_desc[qi], 2883 port->socket_id, 2884 &(port->rx_conf[qi]), 2885 mp); 2886 } 2887 if (diag == 0) 2888 continue; 2889 2890 /* Fail to setup rx queue, return */ 2891 if (port->port_status == RTE_PORT_HANDLING) 2892 port->port_status = RTE_PORT_STOPPED; 2893 else 2894 fprintf(stderr, 2895 "Port %d can not be set back to stopped\n", 2896 pi); 2897 fprintf(stderr, 2898 "Fail to configure port %d rx queues\n", 2899 pi); 2900 /* try to reconfigure queues next time */ 2901 port->need_reconfig_queues = 1; 2902 return -1; 2903 } 2904 /* setup hairpin queues */ 2905 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2906 return -1; 2907 } 2908 configure_rxtx_dump_callbacks(verbose_level); 2909 if (clear_ptypes) { 2910 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2911 NULL, 0); 2912 if (diag < 0) 2913 fprintf(stderr, 2914 "Port %d: Failed to disable Ptype parsing\n", 2915 pi); 2916 } 2917 2918 p_pi = pi; 2919 cnt_pi++; 2920 2921 /* start port */ 2922 diag = eth_dev_start_mp(pi); 2923 if (diag < 0) { 2924 fprintf(stderr, "Fail to start port %d: %s\n", 2925 pi, rte_strerror(-diag)); 2926 2927 /* Fail to setup rx queue, return */ 2928 if (port->port_status == RTE_PORT_HANDLING) 2929 port->port_status = RTE_PORT_STOPPED; 2930 else 2931 fprintf(stderr, 2932 "Port %d can not be set back to stopped\n", 2933 pi); 2934 continue; 2935 } 2936 2937 if (port->port_status == RTE_PORT_HANDLING) 2938 port->port_status = RTE_PORT_STARTED; 2939 else 2940 fprintf(stderr, "Port %d can not be set into started\n", 2941 pi); 2942 2943 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 2944 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 2945 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 2946 2947 /* at least one port started, need checking link status */ 2948 need_check_link_status = 1; 2949 2950 pl[cfg_pi++] = pi; 2951 } 2952 2953 if (need_check_link_status == 1 && !no_link_check) 2954 check_all_ports_link_status(RTE_PORT_ALL); 2955 else if (need_check_link_status == 0) 2956 fprintf(stderr, "Please stop the ports first\n"); 2957 2958 if (hairpin_mode & 0xf) { 2959 uint16_t i; 2960 int j; 2961 2962 /* bind all started hairpin ports */ 2963 for (i = 0; i < cfg_pi; i++) { 2964 pi = pl[i]; 2965 /* bind current Tx to all peer Rx */ 2966 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2967 RTE_MAX_ETHPORTS, 1); 2968 if (peer_pi < 0) 2969 return peer_pi; 2970 for (j = 0; j < peer_pi; j++) { 2971 if (!port_is_started(peer_pl[j])) 2972 continue; 2973 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2974 if (diag < 0) { 2975 fprintf(stderr, 2976 "Error during binding hairpin Tx port %u to %u: %s\n", 2977 pi, peer_pl[j], 2978 rte_strerror(-diag)); 2979 return -1; 2980 } 2981 } 2982 /* bind all peer Tx to current Rx */ 2983 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2984 RTE_MAX_ETHPORTS, 0); 2985 if (peer_pi < 0) 2986 return peer_pi; 2987 for (j = 0; j < peer_pi; j++) { 2988 if (!port_is_started(peer_pl[j])) 2989 continue; 2990 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2991 if (diag < 0) { 2992 fprintf(stderr, 2993 "Error during binding hairpin Tx port %u to %u: %s\n", 2994 peer_pl[j], pi, 2995 rte_strerror(-diag)); 2996 return -1; 2997 } 2998 } 2999 } 3000 } 3001 3002 fill_xstats_display_info_for_port(pid); 3003 3004 printf("Done\n"); 3005 return 0; 3006 } 3007 3008 void 3009 stop_port(portid_t pid) 3010 { 3011 portid_t pi; 3012 struct rte_port *port; 3013 int need_check_link_status = 0; 3014 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3015 int peer_pi; 3016 3017 if (port_id_is_invalid(pid, ENABLED_WARN)) 3018 return; 3019 3020 printf("Stopping ports...\n"); 3021 3022 RTE_ETH_FOREACH_DEV(pi) { 3023 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3024 continue; 3025 3026 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3027 fprintf(stderr, 3028 "Please remove port %d from forwarding configuration.\n", 3029 pi); 3030 continue; 3031 } 3032 3033 if (port_is_bonding_slave(pi)) { 3034 fprintf(stderr, 3035 "Please remove port %d from bonded device.\n", 3036 pi); 3037 continue; 3038 } 3039 3040 port = &ports[pi]; 3041 if (port->port_status == RTE_PORT_STARTED) 3042 port->port_status = RTE_PORT_HANDLING; 3043 else 3044 continue; 3045 3046 if (hairpin_mode & 0xf) { 3047 int j; 3048 3049 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3050 /* unbind all peer Tx from current Rx */ 3051 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3052 RTE_MAX_ETHPORTS, 0); 3053 if (peer_pi < 0) 3054 continue; 3055 for (j = 0; j < peer_pi; j++) { 3056 if (!port_is_started(peer_pl[j])) 3057 continue; 3058 rte_eth_hairpin_unbind(peer_pl[j], pi); 3059 } 3060 } 3061 3062 if (port->flow_list) 3063 port_flow_flush(pi); 3064 3065 if (eth_dev_stop_mp(pi) != 0) 3066 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3067 pi); 3068 3069 if (port->port_status == RTE_PORT_HANDLING) 3070 port->port_status = RTE_PORT_STOPPED; 3071 else 3072 fprintf(stderr, "Port %d can not be set into stopped\n", 3073 pi); 3074 need_check_link_status = 1; 3075 } 3076 if (need_check_link_status && !no_link_check) 3077 check_all_ports_link_status(RTE_PORT_ALL); 3078 3079 printf("Done\n"); 3080 } 3081 3082 static void 3083 remove_invalid_ports_in(portid_t *array, portid_t *total) 3084 { 3085 portid_t i; 3086 portid_t new_total = 0; 3087 3088 for (i = 0; i < *total; i++) 3089 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3090 array[new_total] = array[i]; 3091 new_total++; 3092 } 3093 *total = new_total; 3094 } 3095 3096 static void 3097 remove_invalid_ports(void) 3098 { 3099 remove_invalid_ports_in(ports_ids, &nb_ports); 3100 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3101 nb_cfg_ports = nb_fwd_ports; 3102 } 3103 3104 void 3105 close_port(portid_t pid) 3106 { 3107 portid_t pi; 3108 struct rte_port *port; 3109 3110 if (port_id_is_invalid(pid, ENABLED_WARN)) 3111 return; 3112 3113 printf("Closing ports...\n"); 3114 3115 RTE_ETH_FOREACH_DEV(pi) { 3116 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3117 continue; 3118 3119 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3120 fprintf(stderr, 3121 "Please remove port %d from forwarding configuration.\n", 3122 pi); 3123 continue; 3124 } 3125 3126 if (port_is_bonding_slave(pi)) { 3127 fprintf(stderr, 3128 "Please remove port %d from bonded device.\n", 3129 pi); 3130 continue; 3131 } 3132 3133 port = &ports[pi]; 3134 if (port->port_status == RTE_PORT_CLOSED) { 3135 fprintf(stderr, "Port %d is already closed\n", pi); 3136 continue; 3137 } 3138 3139 if (is_proc_primary()) { 3140 port_flow_flush(pi); 3141 port_flex_item_flush(pi); 3142 rte_eth_dev_close(pi); 3143 } 3144 3145 free_xstats_display_info(pi); 3146 } 3147 3148 remove_invalid_ports(); 3149 printf("Done\n"); 3150 } 3151 3152 void 3153 reset_port(portid_t pid) 3154 { 3155 int diag; 3156 portid_t pi; 3157 struct rte_port *port; 3158 3159 if (port_id_is_invalid(pid, ENABLED_WARN)) 3160 return; 3161 3162 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3163 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3164 fprintf(stderr, 3165 "Can not reset port(s), please stop port(s) first.\n"); 3166 return; 3167 } 3168 3169 printf("Resetting ports...\n"); 3170 3171 RTE_ETH_FOREACH_DEV(pi) { 3172 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3173 continue; 3174 3175 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3176 fprintf(stderr, 3177 "Please remove port %d from forwarding configuration.\n", 3178 pi); 3179 continue; 3180 } 3181 3182 if (port_is_bonding_slave(pi)) { 3183 fprintf(stderr, 3184 "Please remove port %d from bonded device.\n", 3185 pi); 3186 continue; 3187 } 3188 3189 diag = rte_eth_dev_reset(pi); 3190 if (diag == 0) { 3191 port = &ports[pi]; 3192 port->need_reconfig = 1; 3193 port->need_reconfig_queues = 1; 3194 } else { 3195 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3196 pi, diag); 3197 } 3198 } 3199 3200 printf("Done\n"); 3201 } 3202 3203 void 3204 attach_port(char *identifier) 3205 { 3206 portid_t pi; 3207 struct rte_dev_iterator iterator; 3208 3209 printf("Attaching a new port...\n"); 3210 3211 if (identifier == NULL) { 3212 fprintf(stderr, "Invalid parameters are specified\n"); 3213 return; 3214 } 3215 3216 if (rte_dev_probe(identifier) < 0) { 3217 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3218 return; 3219 } 3220 3221 /* first attach mode: event */ 3222 if (setup_on_probe_event) { 3223 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3224 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3225 if (ports[pi].port_status == RTE_PORT_HANDLING && 3226 ports[pi].need_setup != 0) 3227 setup_attached_port(pi); 3228 return; 3229 } 3230 3231 /* second attach mode: iterator */ 3232 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3233 /* setup ports matching the devargs used for probing */ 3234 if (port_is_forwarding(pi)) 3235 continue; /* port was already attached before */ 3236 setup_attached_port(pi); 3237 } 3238 } 3239 3240 static void 3241 setup_attached_port(portid_t pi) 3242 { 3243 unsigned int socket_id; 3244 int ret; 3245 3246 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3247 /* if socket_id is invalid, set to the first available socket. */ 3248 if (check_socket_id(socket_id) < 0) 3249 socket_id = socket_ids[0]; 3250 reconfig(pi, socket_id); 3251 ret = rte_eth_promiscuous_enable(pi); 3252 if (ret != 0) 3253 fprintf(stderr, 3254 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3255 pi, rte_strerror(-ret)); 3256 3257 ports_ids[nb_ports++] = pi; 3258 fwd_ports_ids[nb_fwd_ports++] = pi; 3259 nb_cfg_ports = nb_fwd_ports; 3260 ports[pi].need_setup = 0; 3261 ports[pi].port_status = RTE_PORT_STOPPED; 3262 3263 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3264 printf("Done\n"); 3265 } 3266 3267 static void 3268 detach_device(struct rte_device *dev) 3269 { 3270 portid_t sibling; 3271 3272 if (dev == NULL) { 3273 fprintf(stderr, "Device already removed\n"); 3274 return; 3275 } 3276 3277 printf("Removing a device...\n"); 3278 3279 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3280 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3281 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3282 fprintf(stderr, "Port %u not stopped\n", 3283 sibling); 3284 return; 3285 } 3286 port_flow_flush(sibling); 3287 } 3288 } 3289 3290 if (rte_dev_remove(dev) < 0) { 3291 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3292 return; 3293 } 3294 remove_invalid_ports(); 3295 3296 printf("Device is detached\n"); 3297 printf("Now total ports is %d\n", nb_ports); 3298 printf("Done\n"); 3299 return; 3300 } 3301 3302 void 3303 detach_port_device(portid_t port_id) 3304 { 3305 int ret; 3306 struct rte_eth_dev_info dev_info; 3307 3308 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3309 return; 3310 3311 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3312 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3313 fprintf(stderr, "Port not stopped\n"); 3314 return; 3315 } 3316 fprintf(stderr, "Port was not closed\n"); 3317 } 3318 3319 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3320 if (ret != 0) { 3321 TESTPMD_LOG(ERR, 3322 "Failed to get device info for port %d, not detaching\n", 3323 port_id); 3324 return; 3325 } 3326 detach_device(dev_info.device); 3327 } 3328 3329 void 3330 detach_devargs(char *identifier) 3331 { 3332 struct rte_dev_iterator iterator; 3333 struct rte_devargs da; 3334 portid_t port_id; 3335 3336 printf("Removing a device...\n"); 3337 3338 memset(&da, 0, sizeof(da)); 3339 if (rte_devargs_parsef(&da, "%s", identifier)) { 3340 fprintf(stderr, "cannot parse identifier\n"); 3341 return; 3342 } 3343 3344 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3345 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3346 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3347 fprintf(stderr, "Port %u not stopped\n", 3348 port_id); 3349 rte_eth_iterator_cleanup(&iterator); 3350 rte_devargs_reset(&da); 3351 return; 3352 } 3353 port_flow_flush(port_id); 3354 } 3355 } 3356 3357 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3358 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3359 da.name, da.bus->name); 3360 rte_devargs_reset(&da); 3361 return; 3362 } 3363 3364 remove_invalid_ports(); 3365 3366 printf("Device %s is detached\n", identifier); 3367 printf("Now total ports is %d\n", nb_ports); 3368 printf("Done\n"); 3369 rte_devargs_reset(&da); 3370 } 3371 3372 void 3373 pmd_test_exit(void) 3374 { 3375 portid_t pt_id; 3376 unsigned int i; 3377 int ret; 3378 3379 if (test_done == 0) 3380 stop_packet_forwarding(); 3381 3382 #ifndef RTE_EXEC_ENV_WINDOWS 3383 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3384 if (mempools[i]) { 3385 if (mp_alloc_type == MP_ALLOC_ANON) 3386 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3387 NULL); 3388 } 3389 } 3390 #endif 3391 if (ports != NULL) { 3392 no_link_check = 1; 3393 RTE_ETH_FOREACH_DEV(pt_id) { 3394 printf("\nStopping port %d...\n", pt_id); 3395 fflush(stdout); 3396 stop_port(pt_id); 3397 } 3398 RTE_ETH_FOREACH_DEV(pt_id) { 3399 printf("\nShutting down port %d...\n", pt_id); 3400 fflush(stdout); 3401 close_port(pt_id); 3402 } 3403 } 3404 3405 if (hot_plug) { 3406 ret = rte_dev_event_monitor_stop(); 3407 if (ret) { 3408 RTE_LOG(ERR, EAL, 3409 "fail to stop device event monitor."); 3410 return; 3411 } 3412 3413 ret = rte_dev_event_callback_unregister(NULL, 3414 dev_event_callback, NULL); 3415 if (ret < 0) { 3416 RTE_LOG(ERR, EAL, 3417 "fail to unregister device event callback.\n"); 3418 return; 3419 } 3420 3421 ret = rte_dev_hotplug_handle_disable(); 3422 if (ret) { 3423 RTE_LOG(ERR, EAL, 3424 "fail to disable hotplug handling.\n"); 3425 return; 3426 } 3427 } 3428 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3429 if (mempools[i]) 3430 mempool_free_mp(mempools[i]); 3431 } 3432 free(xstats_display); 3433 3434 printf("\nBye...\n"); 3435 } 3436 3437 typedef void (*cmd_func_t)(void); 3438 struct pmd_test_command { 3439 const char *cmd_name; 3440 cmd_func_t cmd_func; 3441 }; 3442 3443 /* Check the link status of all ports in up to 9s, and print them finally */ 3444 static void 3445 check_all_ports_link_status(uint32_t port_mask) 3446 { 3447 #define CHECK_INTERVAL 100 /* 100ms */ 3448 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3449 portid_t portid; 3450 uint8_t count, all_ports_up, print_flag = 0; 3451 struct rte_eth_link link; 3452 int ret; 3453 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3454 3455 printf("Checking link statuses...\n"); 3456 fflush(stdout); 3457 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3458 all_ports_up = 1; 3459 RTE_ETH_FOREACH_DEV(portid) { 3460 if ((port_mask & (1 << portid)) == 0) 3461 continue; 3462 memset(&link, 0, sizeof(link)); 3463 ret = rte_eth_link_get_nowait(portid, &link); 3464 if (ret < 0) { 3465 all_ports_up = 0; 3466 if (print_flag == 1) 3467 fprintf(stderr, 3468 "Port %u link get failed: %s\n", 3469 portid, rte_strerror(-ret)); 3470 continue; 3471 } 3472 /* print link status if flag set */ 3473 if (print_flag == 1) { 3474 rte_eth_link_to_str(link_status, 3475 sizeof(link_status), &link); 3476 printf("Port %d %s\n", portid, link_status); 3477 continue; 3478 } 3479 /* clear all_ports_up flag if any link down */ 3480 if (link.link_status == RTE_ETH_LINK_DOWN) { 3481 all_ports_up = 0; 3482 break; 3483 } 3484 } 3485 /* after finally printing all link status, get out */ 3486 if (print_flag == 1) 3487 break; 3488 3489 if (all_ports_up == 0) { 3490 fflush(stdout); 3491 rte_delay_ms(CHECK_INTERVAL); 3492 } 3493 3494 /* set the print_flag if all ports up or timeout */ 3495 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3496 print_flag = 1; 3497 } 3498 3499 if (lsc_interrupt) 3500 break; 3501 } 3502 } 3503 3504 static void 3505 rmv_port_callback(void *arg) 3506 { 3507 int need_to_start = 0; 3508 int org_no_link_check = no_link_check; 3509 portid_t port_id = (intptr_t)arg; 3510 struct rte_eth_dev_info dev_info; 3511 int ret; 3512 3513 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3514 3515 if (!test_done && port_is_forwarding(port_id)) { 3516 need_to_start = 1; 3517 stop_packet_forwarding(); 3518 } 3519 no_link_check = 1; 3520 stop_port(port_id); 3521 no_link_check = org_no_link_check; 3522 3523 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3524 if (ret != 0) 3525 TESTPMD_LOG(ERR, 3526 "Failed to get device info for port %d, not detaching\n", 3527 port_id); 3528 else { 3529 struct rte_device *device = dev_info.device; 3530 close_port(port_id); 3531 detach_device(device); /* might be already removed or have more ports */ 3532 } 3533 if (need_to_start) 3534 start_packet_forwarding(0); 3535 } 3536 3537 /* This function is used by the interrupt thread */ 3538 static int 3539 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3540 void *ret_param) 3541 { 3542 RTE_SET_USED(param); 3543 RTE_SET_USED(ret_param); 3544 3545 if (type >= RTE_ETH_EVENT_MAX) { 3546 fprintf(stderr, 3547 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3548 port_id, __func__, type); 3549 fflush(stderr); 3550 } else if (event_print_mask & (UINT32_C(1) << type)) { 3551 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3552 eth_event_desc[type]); 3553 fflush(stdout); 3554 } 3555 3556 switch (type) { 3557 case RTE_ETH_EVENT_NEW: 3558 ports[port_id].need_setup = 1; 3559 ports[port_id].port_status = RTE_PORT_HANDLING; 3560 break; 3561 case RTE_ETH_EVENT_INTR_RMV: 3562 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3563 break; 3564 if (rte_eal_alarm_set(100000, 3565 rmv_port_callback, (void *)(intptr_t)port_id)) 3566 fprintf(stderr, 3567 "Could not set up deferred device removal\n"); 3568 break; 3569 case RTE_ETH_EVENT_DESTROY: 3570 ports[port_id].port_status = RTE_PORT_CLOSED; 3571 printf("Port %u is closed\n", port_id); 3572 break; 3573 default: 3574 break; 3575 } 3576 return 0; 3577 } 3578 3579 static int 3580 register_eth_event_callback(void) 3581 { 3582 int ret; 3583 enum rte_eth_event_type event; 3584 3585 for (event = RTE_ETH_EVENT_UNKNOWN; 3586 event < RTE_ETH_EVENT_MAX; event++) { 3587 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3588 event, 3589 eth_event_callback, 3590 NULL); 3591 if (ret != 0) { 3592 TESTPMD_LOG(ERR, "Failed to register callback for " 3593 "%s event\n", eth_event_desc[event]); 3594 return -1; 3595 } 3596 } 3597 3598 return 0; 3599 } 3600 3601 /* This function is used by the interrupt thread */ 3602 static void 3603 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3604 __rte_unused void *arg) 3605 { 3606 uint16_t port_id; 3607 int ret; 3608 3609 if (type >= RTE_DEV_EVENT_MAX) { 3610 fprintf(stderr, "%s called upon invalid event %d\n", 3611 __func__, type); 3612 fflush(stderr); 3613 } 3614 3615 switch (type) { 3616 case RTE_DEV_EVENT_REMOVE: 3617 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3618 device_name); 3619 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3620 if (ret) { 3621 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3622 device_name); 3623 return; 3624 } 3625 /* 3626 * Because the user's callback is invoked in eal interrupt 3627 * callback, the interrupt callback need to be finished before 3628 * it can be unregistered when detaching device. So finish 3629 * callback soon and use a deferred removal to detach device 3630 * is need. It is a workaround, once the device detaching be 3631 * moved into the eal in the future, the deferred removal could 3632 * be deleted. 3633 */ 3634 if (rte_eal_alarm_set(100000, 3635 rmv_port_callback, (void *)(intptr_t)port_id)) 3636 RTE_LOG(ERR, EAL, 3637 "Could not set up deferred device removal\n"); 3638 break; 3639 case RTE_DEV_EVENT_ADD: 3640 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3641 device_name); 3642 /* TODO: After finish kernel driver binding, 3643 * begin to attach port. 3644 */ 3645 break; 3646 default: 3647 break; 3648 } 3649 } 3650 3651 static void 3652 rxtx_port_config(portid_t pid) 3653 { 3654 uint16_t qid; 3655 uint64_t offloads; 3656 struct rte_port *port = &ports[pid]; 3657 3658 for (qid = 0; qid < nb_rxq; qid++) { 3659 offloads = port->rx_conf[qid].offloads; 3660 port->rx_conf[qid] = port->dev_info.default_rxconf; 3661 3662 if (rxq_share > 0 && 3663 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3664 /* Non-zero share group to enable RxQ share. */ 3665 port->rx_conf[qid].share_group = pid / rxq_share + 1; 3666 port->rx_conf[qid].share_qid = qid; /* Equal mapping. */ 3667 } 3668 3669 if (offloads != 0) 3670 port->rx_conf[qid].offloads = offloads; 3671 3672 /* Check if any Rx parameters have been passed */ 3673 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3674 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3675 3676 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3677 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3678 3679 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3680 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3681 3682 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3683 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3684 3685 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3686 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3687 3688 port->nb_rx_desc[qid] = nb_rxd; 3689 } 3690 3691 for (qid = 0; qid < nb_txq; qid++) { 3692 offloads = port->tx_conf[qid].offloads; 3693 port->tx_conf[qid] = port->dev_info.default_txconf; 3694 if (offloads != 0) 3695 port->tx_conf[qid].offloads = offloads; 3696 3697 /* Check if any Tx parameters have been passed */ 3698 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3699 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3700 3701 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3702 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3703 3704 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3705 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3706 3707 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3708 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3709 3710 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3711 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3712 3713 port->nb_tx_desc[qid] = nb_txd; 3714 } 3715 } 3716 3717 /* 3718 * Helper function to set MTU from frame size 3719 * 3720 * port->dev_info should be set before calling this function. 3721 * 3722 * return 0 on success, negative on error 3723 */ 3724 int 3725 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3726 { 3727 struct rte_port *port = &ports[portid]; 3728 uint32_t eth_overhead; 3729 uint16_t mtu, new_mtu; 3730 3731 eth_overhead = get_eth_overhead(&port->dev_info); 3732 3733 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3734 printf("Failed to get MTU for port %u\n", portid); 3735 return -1; 3736 } 3737 3738 new_mtu = max_rx_pktlen - eth_overhead; 3739 3740 if (mtu == new_mtu) 3741 return 0; 3742 3743 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3744 fprintf(stderr, 3745 "Failed to set MTU to %u for port %u\n", 3746 new_mtu, portid); 3747 return -1; 3748 } 3749 3750 port->dev_conf.rxmode.mtu = new_mtu; 3751 3752 return 0; 3753 } 3754 3755 void 3756 init_port_config(void) 3757 { 3758 portid_t pid; 3759 struct rte_port *port; 3760 int ret, i; 3761 3762 RTE_ETH_FOREACH_DEV(pid) { 3763 port = &ports[pid]; 3764 port->dev_conf.fdir_conf = fdir_conf; 3765 3766 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3767 if (ret != 0) 3768 return; 3769 3770 if (nb_rxq > 1) { 3771 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3772 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3773 rss_hf & port->dev_info.flow_type_rss_offloads; 3774 } else { 3775 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3776 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3777 } 3778 3779 if (port->dcb_flag == 0) { 3780 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3781 port->dev_conf.rxmode.mq_mode = 3782 (enum rte_eth_rx_mq_mode) 3783 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3784 } else { 3785 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3786 port->dev_conf.rxmode.offloads &= 3787 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3788 3789 for (i = 0; 3790 i < port->dev_info.nb_rx_queues; 3791 i++) 3792 port->rx_conf[i].offloads &= 3793 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3794 } 3795 } 3796 3797 rxtx_port_config(pid); 3798 3799 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3800 if (ret != 0) 3801 return; 3802 3803 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3804 rte_pmd_ixgbe_bypass_init(pid); 3805 #endif 3806 3807 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3808 port->dev_conf.intr_conf.lsc = 1; 3809 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3810 port->dev_conf.intr_conf.rmv = 1; 3811 } 3812 } 3813 3814 void set_port_slave_flag(portid_t slave_pid) 3815 { 3816 struct rte_port *port; 3817 3818 port = &ports[slave_pid]; 3819 port->slave_flag = 1; 3820 } 3821 3822 void clear_port_slave_flag(portid_t slave_pid) 3823 { 3824 struct rte_port *port; 3825 3826 port = &ports[slave_pid]; 3827 port->slave_flag = 0; 3828 } 3829 3830 uint8_t port_is_bonding_slave(portid_t slave_pid) 3831 { 3832 struct rte_port *port; 3833 struct rte_eth_dev_info dev_info; 3834 int ret; 3835 3836 port = &ports[slave_pid]; 3837 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3838 if (ret != 0) { 3839 TESTPMD_LOG(ERR, 3840 "Failed to get device info for port id %d," 3841 "cannot determine if the port is a bonded slave", 3842 slave_pid); 3843 return 0; 3844 } 3845 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3846 return 1; 3847 return 0; 3848 } 3849 3850 const uint16_t vlan_tags[] = { 3851 0, 1, 2, 3, 4, 5, 6, 7, 3852 8, 9, 10, 11, 12, 13, 14, 15, 3853 16, 17, 18, 19, 20, 21, 22, 23, 3854 24, 25, 26, 27, 28, 29, 30, 31 3855 }; 3856 3857 static int 3858 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3859 enum dcb_mode_enable dcb_mode, 3860 enum rte_eth_nb_tcs num_tcs, 3861 uint8_t pfc_en) 3862 { 3863 uint8_t i; 3864 int32_t rc; 3865 struct rte_eth_rss_conf rss_conf; 3866 3867 /* 3868 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3869 * given above, and the number of traffic classes available for use. 3870 */ 3871 if (dcb_mode == DCB_VT_ENABLED) { 3872 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3873 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3874 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3875 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3876 3877 /* VMDQ+DCB RX and TX configurations */ 3878 vmdq_rx_conf->enable_default_pool = 0; 3879 vmdq_rx_conf->default_pool = 0; 3880 vmdq_rx_conf->nb_queue_pools = 3881 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3882 vmdq_tx_conf->nb_queue_pools = 3883 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3884 3885 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3886 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3887 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3888 vmdq_rx_conf->pool_map[i].pools = 3889 1 << (i % vmdq_rx_conf->nb_queue_pools); 3890 } 3891 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3892 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3893 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3894 } 3895 3896 /* set DCB mode of RX and TX of multiple queues */ 3897 eth_conf->rxmode.mq_mode = 3898 (enum rte_eth_rx_mq_mode) 3899 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 3900 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 3901 } else { 3902 struct rte_eth_dcb_rx_conf *rx_conf = 3903 ð_conf->rx_adv_conf.dcb_rx_conf; 3904 struct rte_eth_dcb_tx_conf *tx_conf = 3905 ð_conf->tx_adv_conf.dcb_tx_conf; 3906 3907 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3908 3909 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3910 if (rc != 0) 3911 return rc; 3912 3913 rx_conf->nb_tcs = num_tcs; 3914 tx_conf->nb_tcs = num_tcs; 3915 3916 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3917 rx_conf->dcb_tc[i] = i % num_tcs; 3918 tx_conf->dcb_tc[i] = i % num_tcs; 3919 } 3920 3921 eth_conf->rxmode.mq_mode = 3922 (enum rte_eth_rx_mq_mode) 3923 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 3924 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3925 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 3926 } 3927 3928 if (pfc_en) 3929 eth_conf->dcb_capability_en = 3930 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 3931 else 3932 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 3933 3934 return 0; 3935 } 3936 3937 int 3938 init_port_dcb_config(portid_t pid, 3939 enum dcb_mode_enable dcb_mode, 3940 enum rte_eth_nb_tcs num_tcs, 3941 uint8_t pfc_en) 3942 { 3943 struct rte_eth_conf port_conf; 3944 struct rte_port *rte_port; 3945 int retval; 3946 uint16_t i; 3947 3948 if (num_procs > 1) { 3949 printf("The multi-process feature doesn't support dcb.\n"); 3950 return -ENOTSUP; 3951 } 3952 rte_port = &ports[pid]; 3953 3954 /* retain the original device configuration. */ 3955 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 3956 3957 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3958 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3959 if (retval < 0) 3960 return retval; 3961 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 3962 /* remove RSS HASH offload for DCB in vt mode */ 3963 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 3964 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3965 for (i = 0; i < nb_rxq; i++) 3966 rte_port->rx_conf[i].offloads &= 3967 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3968 } 3969 3970 /* re-configure the device . */ 3971 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3972 if (retval < 0) 3973 return retval; 3974 3975 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3976 if (retval != 0) 3977 return retval; 3978 3979 /* If dev_info.vmdq_pool_base is greater than 0, 3980 * the queue id of vmdq pools is started after pf queues. 3981 */ 3982 if (dcb_mode == DCB_VT_ENABLED && 3983 rte_port->dev_info.vmdq_pool_base > 0) { 3984 fprintf(stderr, 3985 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 3986 pid); 3987 return -1; 3988 } 3989 3990 /* Assume the ports in testpmd have the same dcb capability 3991 * and has the same number of rxq and txq in dcb mode 3992 */ 3993 if (dcb_mode == DCB_VT_ENABLED) { 3994 if (rte_port->dev_info.max_vfs > 0) { 3995 nb_rxq = rte_port->dev_info.nb_rx_queues; 3996 nb_txq = rte_port->dev_info.nb_tx_queues; 3997 } else { 3998 nb_rxq = rte_port->dev_info.max_rx_queues; 3999 nb_txq = rte_port->dev_info.max_tx_queues; 4000 } 4001 } else { 4002 /*if vt is disabled, use all pf queues */ 4003 if (rte_port->dev_info.vmdq_pool_base == 0) { 4004 nb_rxq = rte_port->dev_info.max_rx_queues; 4005 nb_txq = rte_port->dev_info.max_tx_queues; 4006 } else { 4007 nb_rxq = (queueid_t)num_tcs; 4008 nb_txq = (queueid_t)num_tcs; 4009 4010 } 4011 } 4012 rx_free_thresh = 64; 4013 4014 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4015 4016 rxtx_port_config(pid); 4017 /* VLAN filter */ 4018 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4019 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4020 rx_vft_set(pid, vlan_tags[i], 1); 4021 4022 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4023 if (retval != 0) 4024 return retval; 4025 4026 rte_port->dcb_flag = 1; 4027 4028 /* Enter DCB configuration status */ 4029 dcb_config = 1; 4030 4031 return 0; 4032 } 4033 4034 static void 4035 init_port(void) 4036 { 4037 int i; 4038 4039 /* Configuration of Ethernet ports. */ 4040 ports = rte_zmalloc("testpmd: ports", 4041 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4042 RTE_CACHE_LINE_SIZE); 4043 if (ports == NULL) { 4044 rte_exit(EXIT_FAILURE, 4045 "rte_zmalloc(%d struct rte_port) failed\n", 4046 RTE_MAX_ETHPORTS); 4047 } 4048 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4049 ports[i].xstats_info.allocated = false; 4050 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4051 LIST_INIT(&ports[i].flow_tunnel_list); 4052 /* Initialize ports NUMA structures */ 4053 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4054 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4055 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4056 } 4057 4058 static void 4059 force_quit(void) 4060 { 4061 pmd_test_exit(); 4062 prompt_exit(); 4063 } 4064 4065 static void 4066 print_stats(void) 4067 { 4068 uint8_t i; 4069 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4070 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4071 4072 /* Clear screen and move to top left */ 4073 printf("%s%s", clr, top_left); 4074 4075 printf("\nPort statistics ===================================="); 4076 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4077 nic_stats_display(fwd_ports_ids[i]); 4078 4079 fflush(stdout); 4080 } 4081 4082 static void 4083 signal_handler(int signum) 4084 { 4085 if (signum == SIGINT || signum == SIGTERM) { 4086 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4087 signum); 4088 #ifdef RTE_LIB_PDUMP 4089 /* uninitialize packet capture framework */ 4090 rte_pdump_uninit(); 4091 #endif 4092 #ifdef RTE_LIB_LATENCYSTATS 4093 if (latencystats_enabled != 0) 4094 rte_latencystats_uninit(); 4095 #endif 4096 force_quit(); 4097 /* Set flag to indicate the force termination. */ 4098 f_quit = 1; 4099 /* exit with the expected status */ 4100 #ifndef RTE_EXEC_ENV_WINDOWS 4101 signal(signum, SIG_DFL); 4102 kill(getpid(), signum); 4103 #endif 4104 } 4105 } 4106 4107 int 4108 main(int argc, char** argv) 4109 { 4110 int diag; 4111 portid_t port_id; 4112 uint16_t count; 4113 int ret; 4114 4115 signal(SIGINT, signal_handler); 4116 signal(SIGTERM, signal_handler); 4117 4118 testpmd_logtype = rte_log_register("testpmd"); 4119 if (testpmd_logtype < 0) 4120 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4121 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4122 4123 diag = rte_eal_init(argc, argv); 4124 if (diag < 0) 4125 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4126 rte_strerror(rte_errno)); 4127 4128 ret = register_eth_event_callback(); 4129 if (ret != 0) 4130 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4131 4132 #ifdef RTE_LIB_PDUMP 4133 /* initialize packet capture framework */ 4134 rte_pdump_init(); 4135 #endif 4136 4137 count = 0; 4138 RTE_ETH_FOREACH_DEV(port_id) { 4139 ports_ids[count] = port_id; 4140 count++; 4141 } 4142 nb_ports = (portid_t) count; 4143 if (nb_ports == 0) 4144 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4145 4146 /* allocate port structures, and init them */ 4147 init_port(); 4148 4149 set_def_fwd_config(); 4150 if (nb_lcores == 0) 4151 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4152 "Check the core mask argument\n"); 4153 4154 /* Bitrate/latency stats disabled by default */ 4155 #ifdef RTE_LIB_BITRATESTATS 4156 bitrate_enabled = 0; 4157 #endif 4158 #ifdef RTE_LIB_LATENCYSTATS 4159 latencystats_enabled = 0; 4160 #endif 4161 4162 /* on FreeBSD, mlockall() is disabled by default */ 4163 #ifdef RTE_EXEC_ENV_FREEBSD 4164 do_mlockall = 0; 4165 #else 4166 do_mlockall = 1; 4167 #endif 4168 4169 argc -= diag; 4170 argv += diag; 4171 if (argc > 1) 4172 launch_args_parse(argc, argv); 4173 4174 #ifndef RTE_EXEC_ENV_WINDOWS 4175 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4176 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4177 strerror(errno)); 4178 } 4179 #endif 4180 4181 if (tx_first && interactive) 4182 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4183 "interactive mode.\n"); 4184 4185 if (tx_first && lsc_interrupt) { 4186 fprintf(stderr, 4187 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4188 lsc_interrupt = 0; 4189 } 4190 4191 if (!nb_rxq && !nb_txq) 4192 fprintf(stderr, 4193 "Warning: Either rx or tx queues should be non-zero\n"); 4194 4195 if (nb_rxq > 1 && nb_rxq > nb_txq) 4196 fprintf(stderr, 4197 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4198 nb_rxq, nb_txq); 4199 4200 init_config(); 4201 4202 if (hot_plug) { 4203 ret = rte_dev_hotplug_handle_enable(); 4204 if (ret) { 4205 RTE_LOG(ERR, EAL, 4206 "fail to enable hotplug handling."); 4207 return -1; 4208 } 4209 4210 ret = rte_dev_event_monitor_start(); 4211 if (ret) { 4212 RTE_LOG(ERR, EAL, 4213 "fail to start device event monitoring."); 4214 return -1; 4215 } 4216 4217 ret = rte_dev_event_callback_register(NULL, 4218 dev_event_callback, NULL); 4219 if (ret) { 4220 RTE_LOG(ERR, EAL, 4221 "fail to register device event callback\n"); 4222 return -1; 4223 } 4224 } 4225 4226 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4227 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4228 4229 /* set all ports to promiscuous mode by default */ 4230 RTE_ETH_FOREACH_DEV(port_id) { 4231 ret = rte_eth_promiscuous_enable(port_id); 4232 if (ret != 0) 4233 fprintf(stderr, 4234 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4235 port_id, rte_strerror(-ret)); 4236 } 4237 4238 #ifdef RTE_LIB_METRICS 4239 /* Init metrics library */ 4240 rte_metrics_init(rte_socket_id()); 4241 #endif 4242 4243 #ifdef RTE_LIB_LATENCYSTATS 4244 if (latencystats_enabled != 0) { 4245 int ret = rte_latencystats_init(1, NULL); 4246 if (ret) 4247 fprintf(stderr, 4248 "Warning: latencystats init() returned error %d\n", 4249 ret); 4250 fprintf(stderr, "Latencystats running on lcore %d\n", 4251 latencystats_lcore_id); 4252 } 4253 #endif 4254 4255 /* Setup bitrate stats */ 4256 #ifdef RTE_LIB_BITRATESTATS 4257 if (bitrate_enabled != 0) { 4258 bitrate_data = rte_stats_bitrate_create(); 4259 if (bitrate_data == NULL) 4260 rte_exit(EXIT_FAILURE, 4261 "Could not allocate bitrate data.\n"); 4262 rte_stats_bitrate_reg(bitrate_data); 4263 } 4264 #endif 4265 #ifdef RTE_LIB_CMDLINE 4266 if (strlen(cmdline_filename) != 0) 4267 cmdline_read_from_file(cmdline_filename); 4268 4269 if (interactive == 1) { 4270 if (auto_start) { 4271 printf("Start automatic packet forwarding\n"); 4272 start_packet_forwarding(0); 4273 } 4274 prompt(); 4275 pmd_test_exit(); 4276 } else 4277 #endif 4278 { 4279 char c; 4280 int rc; 4281 4282 f_quit = 0; 4283 4284 printf("No commandline core given, start packet forwarding\n"); 4285 start_packet_forwarding(tx_first); 4286 if (stats_period != 0) { 4287 uint64_t prev_time = 0, cur_time, diff_time = 0; 4288 uint64_t timer_period; 4289 4290 /* Convert to number of cycles */ 4291 timer_period = stats_period * rte_get_timer_hz(); 4292 4293 while (f_quit == 0) { 4294 cur_time = rte_get_timer_cycles(); 4295 diff_time += cur_time - prev_time; 4296 4297 if (diff_time >= timer_period) { 4298 print_stats(); 4299 /* Reset the timer */ 4300 diff_time = 0; 4301 } 4302 /* Sleep to avoid unnecessary checks */ 4303 prev_time = cur_time; 4304 rte_delay_us_sleep(US_PER_S); 4305 } 4306 } 4307 4308 printf("Press enter to exit\n"); 4309 rc = read(0, &c, 1); 4310 pmd_test_exit(); 4311 if (rc < 0) 4312 return 1; 4313 } 4314 4315 ret = rte_eal_cleanup(); 4316 if (ret != 0) 4317 rte_exit(EXIT_FAILURE, 4318 "EAL cleanup failed: %s\n", strerror(-ret)); 4319 4320 return EXIT_SUCCESS; 4321 } 4322