1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIB_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIB_LATENCYSTATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use main core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 220 221 /* 222 * Configuration of packet segments used by the "txonly" processing engine. 223 */ 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 226 TXONLY_DEF_PACKET_LEN, 227 }; 228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 229 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 231 /**< Split policy for packets to TX. */ 232 233 uint8_t txonly_multi_flow; 234 /**< Whether multiple flows are generated in TXONLY mode. */ 235 236 uint32_t tx_pkt_times_inter; 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 238 239 uint32_t tx_pkt_times_intra; 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 241 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 245 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 247 uint8_t dcb_config = 0; 248 249 /* Whether the dcb is in testing status */ 250 uint8_t dcb_test = 0; 251 252 /* 253 * Configurable number of RX/TX queues. 254 */ 255 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 256 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 257 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 258 259 /* 260 * Configurable number of RX/TX ring descriptors. 261 * Defaults are supplied by drivers via ethdev. 262 */ 263 #define RTE_TEST_RX_DESC_DEFAULT 0 264 #define RTE_TEST_TX_DESC_DEFAULT 0 265 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 266 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 267 268 #define RTE_PMD_PARAM_UNSET -1 269 /* 270 * Configurable values of RX and TX ring threshold registers. 271 */ 272 273 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 274 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 275 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 276 277 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 278 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 279 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 280 281 /* 282 * Configurable value of RX free threshold. 283 */ 284 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 285 286 /* 287 * Configurable value of RX drop enable. 288 */ 289 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 290 291 /* 292 * Configurable value of TX free threshold. 293 */ 294 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 295 296 /* 297 * Configurable value of TX RS bit threshold. 298 */ 299 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 300 301 /* 302 * Configurable value of buffered packets before sending. 303 */ 304 uint16_t noisy_tx_sw_bufsz; 305 306 /* 307 * Configurable value of packet buffer timeout. 308 */ 309 uint16_t noisy_tx_sw_buf_flush_time; 310 311 /* 312 * Configurable value for size of VNF internal memory area 313 * used for simulating noisy neighbour behaviour 314 */ 315 uint64_t noisy_lkup_mem_sz; 316 317 /* 318 * Configurable value of number of random writes done in 319 * VNF simulation memory area. 320 */ 321 uint64_t noisy_lkup_num_writes; 322 323 /* 324 * Configurable value of number of random reads done in 325 * VNF simulation memory area. 326 */ 327 uint64_t noisy_lkup_num_reads; 328 329 /* 330 * Configurable value of number of random reads/writes done in 331 * VNF simulation memory area. 332 */ 333 uint64_t noisy_lkup_num_reads_writes; 334 335 /* 336 * Receive Side Scaling (RSS) configuration. 337 */ 338 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 339 340 /* 341 * Port topology configuration 342 */ 343 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 344 345 /* 346 * Avoids to flush all the RX streams before starts forwarding. 347 */ 348 uint8_t no_flush_rx = 0; /* flush by default */ 349 350 /* 351 * Flow API isolated mode. 352 */ 353 uint8_t flow_isolate_all; 354 355 /* 356 * Avoids to check link status when starting/stopping a port. 357 */ 358 uint8_t no_link_check = 0; /* check by default */ 359 360 /* 361 * Don't automatically start all ports in interactive mode. 362 */ 363 uint8_t no_device_start = 0; 364 365 /* 366 * Enable link status change notification 367 */ 368 uint8_t lsc_interrupt = 1; /* enabled by default */ 369 370 /* 371 * Enable device removal notification. 372 */ 373 uint8_t rmv_interrupt = 1; /* enabled by default */ 374 375 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 376 377 /* After attach, port setup is called on event or by iterator */ 378 bool setup_on_probe_event = true; 379 380 /* Clear ptypes on port initialization. */ 381 uint8_t clear_ptypes = true; 382 383 /* Hairpin ports configuration mode. */ 384 uint16_t hairpin_mode; 385 386 /* Pretty printing of ethdev events */ 387 static const char * const eth_event_desc[] = { 388 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 389 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 390 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 391 [RTE_ETH_EVENT_INTR_RESET] = "reset", 392 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 393 [RTE_ETH_EVENT_IPSEC] = "IPsec", 394 [RTE_ETH_EVENT_MACSEC] = "MACsec", 395 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 396 [RTE_ETH_EVENT_NEW] = "device probed", 397 [RTE_ETH_EVENT_DESTROY] = "device released", 398 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 399 [RTE_ETH_EVENT_MAX] = NULL, 400 }; 401 402 /* 403 * Display or mask ether events 404 * Default to all events except VF_MBOX 405 */ 406 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 407 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 408 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 410 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 411 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 412 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 413 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 414 /* 415 * Decide if all memory are locked for performance. 416 */ 417 int do_mlockall = 0; 418 419 /* 420 * NIC bypass mode configuration options. 421 */ 422 423 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 424 /* The NIC bypass watchdog timeout. */ 425 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 426 #endif 427 428 429 #ifdef RTE_LIB_LATENCYSTATS 430 431 /* 432 * Set when latency stats is enabled in the commandline 433 */ 434 uint8_t latencystats_enabled; 435 436 /* 437 * Lcore ID to serive latency statistics. 438 */ 439 lcoreid_t latencystats_lcore_id = -1; 440 441 #endif 442 443 /* 444 * Ethernet device configuration. 445 */ 446 struct rte_eth_rxmode rx_mode = { 447 /* Default maximum frame length. 448 * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead" 449 * in init_config(). 450 */ 451 .max_rx_pkt_len = 0, 452 }; 453 454 struct rte_eth_txmode tx_mode = { 455 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 456 }; 457 458 struct rte_fdir_conf fdir_conf = { 459 .mode = RTE_FDIR_MODE_NONE, 460 .pballoc = RTE_FDIR_PBALLOC_64K, 461 .status = RTE_FDIR_REPORT_STATUS, 462 .mask = { 463 .vlan_tci_mask = 0xFFEF, 464 .ipv4_mask = { 465 .src_ip = 0xFFFFFFFF, 466 .dst_ip = 0xFFFFFFFF, 467 }, 468 .ipv6_mask = { 469 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 470 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 471 }, 472 .src_port_mask = 0xFFFF, 473 .dst_port_mask = 0xFFFF, 474 .mac_addr_byte_mask = 0xFF, 475 .tunnel_type_mask = 1, 476 .tunnel_id_mask = 0xFFFFFFFF, 477 }, 478 .drop_queue = 127, 479 }; 480 481 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 482 483 /* 484 * Display zero values by default for xstats 485 */ 486 uint8_t xstats_hide_zero; 487 488 /* 489 * Measure of CPU cycles disabled by default 490 */ 491 uint8_t record_core_cycles; 492 493 /* 494 * Display of RX and TX bursts disabled by default 495 */ 496 uint8_t record_burst_stats; 497 498 unsigned int num_sockets = 0; 499 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 500 501 #ifdef RTE_LIB_BITRATESTATS 502 /* Bitrate statistics */ 503 struct rte_stats_bitrates *bitrate_data; 504 lcoreid_t bitrate_lcore_id; 505 uint8_t bitrate_enabled; 506 #endif 507 508 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 509 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 510 511 /* 512 * hexadecimal bitmask of RX mq mode can be enabled. 513 */ 514 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 515 516 /* 517 * Used to set forced link speed 518 */ 519 uint32_t eth_link_speed; 520 521 /* Forward function declarations */ 522 static void setup_attached_port(portid_t pi); 523 static void check_all_ports_link_status(uint32_t port_mask); 524 static int eth_event_callback(portid_t port_id, 525 enum rte_eth_event_type type, 526 void *param, void *ret_param); 527 static void dev_event_callback(const char *device_name, 528 enum rte_dev_event_type type, 529 void *param); 530 531 /* 532 * Check if all the ports are started. 533 * If yes, return positive value. If not, return zero. 534 */ 535 static int all_ports_started(void); 536 537 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 538 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 539 540 /* Holds the registered mbuf dynamic flags names. */ 541 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 542 543 /* 544 * Helper function to check if socket is already discovered. 545 * If yes, return positive value. If not, return zero. 546 */ 547 int 548 new_socket_id(unsigned int socket_id) 549 { 550 unsigned int i; 551 552 for (i = 0; i < num_sockets; i++) { 553 if (socket_ids[i] == socket_id) 554 return 0; 555 } 556 return 1; 557 } 558 559 /* 560 * Setup default configuration. 561 */ 562 static void 563 set_default_fwd_lcores_config(void) 564 { 565 unsigned int i; 566 unsigned int nb_lc; 567 unsigned int sock_num; 568 569 nb_lc = 0; 570 for (i = 0; i < RTE_MAX_LCORE; i++) { 571 if (!rte_lcore_is_enabled(i)) 572 continue; 573 sock_num = rte_lcore_to_socket_id(i); 574 if (new_socket_id(sock_num)) { 575 if (num_sockets >= RTE_MAX_NUMA_NODES) { 576 rte_exit(EXIT_FAILURE, 577 "Total sockets greater than %u\n", 578 RTE_MAX_NUMA_NODES); 579 } 580 socket_ids[num_sockets++] = sock_num; 581 } 582 if (i == rte_get_main_lcore()) 583 continue; 584 fwd_lcores_cpuids[nb_lc++] = i; 585 } 586 nb_lcores = (lcoreid_t) nb_lc; 587 nb_cfg_lcores = nb_lcores; 588 nb_fwd_lcores = 1; 589 } 590 591 static void 592 set_def_peer_eth_addrs(void) 593 { 594 portid_t i; 595 596 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 597 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 598 peer_eth_addrs[i].addr_bytes[5] = i; 599 } 600 } 601 602 static void 603 set_default_fwd_ports_config(void) 604 { 605 portid_t pt_id; 606 int i = 0; 607 608 RTE_ETH_FOREACH_DEV(pt_id) { 609 fwd_ports_ids[i++] = pt_id; 610 611 /* Update sockets info according to the attached device */ 612 int socket_id = rte_eth_dev_socket_id(pt_id); 613 if (socket_id >= 0 && new_socket_id(socket_id)) { 614 if (num_sockets >= RTE_MAX_NUMA_NODES) { 615 rte_exit(EXIT_FAILURE, 616 "Total sockets greater than %u\n", 617 RTE_MAX_NUMA_NODES); 618 } 619 socket_ids[num_sockets++] = socket_id; 620 } 621 } 622 623 nb_cfg_ports = nb_ports; 624 nb_fwd_ports = nb_ports; 625 } 626 627 void 628 set_def_fwd_config(void) 629 { 630 set_default_fwd_lcores_config(); 631 set_def_peer_eth_addrs(); 632 set_default_fwd_ports_config(); 633 } 634 635 /* extremely pessimistic estimation of memory required to create a mempool */ 636 static int 637 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 638 { 639 unsigned int n_pages, mbuf_per_pg, leftover; 640 uint64_t total_mem, mbuf_mem, obj_sz; 641 642 /* there is no good way to predict how much space the mempool will 643 * occupy because it will allocate chunks on the fly, and some of those 644 * will come from default DPDK memory while some will come from our 645 * external memory, so just assume 128MB will be enough for everyone. 646 */ 647 uint64_t hdr_mem = 128 << 20; 648 649 /* account for possible non-contiguousness */ 650 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 651 if (obj_sz > pgsz) { 652 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 653 return -1; 654 } 655 656 mbuf_per_pg = pgsz / obj_sz; 657 leftover = (nb_mbufs % mbuf_per_pg) > 0; 658 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 659 660 mbuf_mem = n_pages * pgsz; 661 662 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 663 664 if (total_mem > SIZE_MAX) { 665 TESTPMD_LOG(ERR, "Memory size too big\n"); 666 return -1; 667 } 668 *out = (size_t)total_mem; 669 670 return 0; 671 } 672 673 static int 674 pagesz_flags(uint64_t page_sz) 675 { 676 /* as per mmap() manpage, all page sizes are log2 of page size 677 * shifted by MAP_HUGE_SHIFT 678 */ 679 int log2 = rte_log2_u64(page_sz); 680 681 return (log2 << HUGE_SHIFT); 682 } 683 684 static void * 685 alloc_mem(size_t memsz, size_t pgsz, bool huge) 686 { 687 void *addr; 688 int flags; 689 690 /* allocate anonymous hugepages */ 691 flags = MAP_ANONYMOUS | MAP_PRIVATE; 692 if (huge) 693 flags |= HUGE_FLAG | pagesz_flags(pgsz); 694 695 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 696 if (addr == MAP_FAILED) 697 return NULL; 698 699 return addr; 700 } 701 702 struct extmem_param { 703 void *addr; 704 size_t len; 705 size_t pgsz; 706 rte_iova_t *iova_table; 707 unsigned int iova_table_len; 708 }; 709 710 static int 711 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 712 bool huge) 713 { 714 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 715 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 716 unsigned int cur_page, n_pages, pgsz_idx; 717 size_t mem_sz, cur_pgsz; 718 rte_iova_t *iovas = NULL; 719 void *addr; 720 int ret; 721 722 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 723 /* skip anything that is too big */ 724 if (pgsizes[pgsz_idx] > SIZE_MAX) 725 continue; 726 727 cur_pgsz = pgsizes[pgsz_idx]; 728 729 /* if we were told not to allocate hugepages, override */ 730 if (!huge) 731 cur_pgsz = sysconf(_SC_PAGESIZE); 732 733 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 734 if (ret < 0) { 735 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 736 return -1; 737 } 738 739 /* allocate our memory */ 740 addr = alloc_mem(mem_sz, cur_pgsz, huge); 741 742 /* if we couldn't allocate memory with a specified page size, 743 * that doesn't mean we can't do it with other page sizes, so 744 * try another one. 745 */ 746 if (addr == NULL) 747 continue; 748 749 /* store IOVA addresses for every page in this memory area */ 750 n_pages = mem_sz / cur_pgsz; 751 752 iovas = malloc(sizeof(*iovas) * n_pages); 753 754 if (iovas == NULL) { 755 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 756 goto fail; 757 } 758 /* lock memory if it's not huge pages */ 759 if (!huge) 760 mlock(addr, mem_sz); 761 762 /* populate IOVA addresses */ 763 for (cur_page = 0; cur_page < n_pages; cur_page++) { 764 rte_iova_t iova; 765 size_t offset; 766 void *cur; 767 768 offset = cur_pgsz * cur_page; 769 cur = RTE_PTR_ADD(addr, offset); 770 771 /* touch the page before getting its IOVA */ 772 *(volatile char *)cur = 0; 773 774 iova = rte_mem_virt2iova(cur); 775 776 iovas[cur_page] = iova; 777 } 778 779 break; 780 } 781 /* if we couldn't allocate anything */ 782 if (iovas == NULL) 783 return -1; 784 785 param->addr = addr; 786 param->len = mem_sz; 787 param->pgsz = cur_pgsz; 788 param->iova_table = iovas; 789 param->iova_table_len = n_pages; 790 791 return 0; 792 fail: 793 if (iovas) 794 free(iovas); 795 if (addr) 796 munmap(addr, mem_sz); 797 798 return -1; 799 } 800 801 static int 802 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 803 { 804 struct extmem_param param; 805 int socket_id, ret; 806 807 memset(¶m, 0, sizeof(param)); 808 809 /* check if our heap exists */ 810 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 811 if (socket_id < 0) { 812 /* create our heap */ 813 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 814 if (ret < 0) { 815 TESTPMD_LOG(ERR, "Cannot create heap\n"); 816 return -1; 817 } 818 } 819 820 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 821 if (ret < 0) { 822 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 823 return -1; 824 } 825 826 /* we now have a valid memory area, so add it to heap */ 827 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 828 param.addr, param.len, param.iova_table, 829 param.iova_table_len, param.pgsz); 830 831 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 832 833 /* not needed any more */ 834 free(param.iova_table); 835 836 if (ret < 0) { 837 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 838 munmap(param.addr, param.len); 839 return -1; 840 } 841 842 /* success */ 843 844 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 845 param.len >> 20); 846 847 return 0; 848 } 849 static void 850 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 851 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 852 { 853 uint16_t pid = 0; 854 int ret; 855 856 RTE_ETH_FOREACH_DEV(pid) { 857 struct rte_eth_dev *dev = 858 &rte_eth_devices[pid]; 859 860 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 861 memhdr->len); 862 if (ret) { 863 TESTPMD_LOG(DEBUG, 864 "unable to DMA unmap addr 0x%p " 865 "for device %s\n", 866 memhdr->addr, dev->data->name); 867 } 868 } 869 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 870 if (ret) { 871 TESTPMD_LOG(DEBUG, 872 "unable to un-register addr 0x%p\n", memhdr->addr); 873 } 874 } 875 876 static void 877 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 878 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 879 { 880 uint16_t pid = 0; 881 size_t page_size = sysconf(_SC_PAGESIZE); 882 int ret; 883 884 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 885 page_size); 886 if (ret) { 887 TESTPMD_LOG(DEBUG, 888 "unable to register addr 0x%p\n", memhdr->addr); 889 return; 890 } 891 RTE_ETH_FOREACH_DEV(pid) { 892 struct rte_eth_dev *dev = 893 &rte_eth_devices[pid]; 894 895 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 896 memhdr->len); 897 if (ret) { 898 TESTPMD_LOG(DEBUG, 899 "unable to DMA map addr 0x%p " 900 "for device %s\n", 901 memhdr->addr, dev->data->name); 902 } 903 } 904 } 905 906 static unsigned int 907 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 908 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 909 { 910 struct rte_pktmbuf_extmem *xmem; 911 unsigned int ext_num, zone_num, elt_num; 912 uint16_t elt_size; 913 914 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 915 elt_num = EXTBUF_ZONE_SIZE / elt_size; 916 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 917 918 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 919 if (xmem == NULL) { 920 TESTPMD_LOG(ERR, "Cannot allocate memory for " 921 "external buffer descriptors\n"); 922 *ext_mem = NULL; 923 return 0; 924 } 925 for (ext_num = 0; ext_num < zone_num; ext_num++) { 926 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 927 const struct rte_memzone *mz; 928 char mz_name[RTE_MEMZONE_NAMESIZE]; 929 int ret; 930 931 ret = snprintf(mz_name, sizeof(mz_name), 932 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 933 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 934 errno = ENAMETOOLONG; 935 ext_num = 0; 936 break; 937 } 938 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 939 socket_id, 940 RTE_MEMZONE_IOVA_CONTIG | 941 RTE_MEMZONE_1GB | 942 RTE_MEMZONE_SIZE_HINT_ONLY, 943 EXTBUF_ZONE_SIZE); 944 if (mz == NULL) { 945 /* 946 * The caller exits on external buffer creation 947 * error, so there is no need to free memzones. 948 */ 949 errno = ENOMEM; 950 ext_num = 0; 951 break; 952 } 953 xseg->buf_ptr = mz->addr; 954 xseg->buf_iova = mz->iova; 955 xseg->buf_len = EXTBUF_ZONE_SIZE; 956 xseg->elt_size = elt_size; 957 } 958 if (ext_num == 0 && xmem != NULL) { 959 free(xmem); 960 xmem = NULL; 961 } 962 *ext_mem = xmem; 963 return ext_num; 964 } 965 966 /* 967 * Configuration initialisation done once at init time. 968 */ 969 static struct rte_mempool * 970 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 971 unsigned int socket_id, uint16_t size_idx) 972 { 973 char pool_name[RTE_MEMPOOL_NAMESIZE]; 974 struct rte_mempool *rte_mp = NULL; 975 uint32_t mb_size; 976 977 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 978 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 979 980 TESTPMD_LOG(INFO, 981 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 982 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 983 984 switch (mp_alloc_type) { 985 case MP_ALLOC_NATIVE: 986 { 987 /* wrapper to rte_mempool_create() */ 988 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 989 rte_mbuf_best_mempool_ops()); 990 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 991 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 992 break; 993 } 994 case MP_ALLOC_ANON: 995 { 996 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 997 mb_size, (unsigned int) mb_mempool_cache, 998 sizeof(struct rte_pktmbuf_pool_private), 999 socket_id, mempool_flags); 1000 if (rte_mp == NULL) 1001 goto err; 1002 1003 if (rte_mempool_populate_anon(rte_mp) == 0) { 1004 rte_mempool_free(rte_mp); 1005 rte_mp = NULL; 1006 goto err; 1007 } 1008 rte_pktmbuf_pool_init(rte_mp, NULL); 1009 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1010 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1011 break; 1012 } 1013 case MP_ALLOC_XMEM: 1014 case MP_ALLOC_XMEM_HUGE: 1015 { 1016 int heap_socket; 1017 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1018 1019 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1020 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1021 1022 heap_socket = 1023 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1024 if (heap_socket < 0) 1025 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1026 1027 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1028 rte_mbuf_best_mempool_ops()); 1029 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1030 mb_mempool_cache, 0, mbuf_seg_size, 1031 heap_socket); 1032 break; 1033 } 1034 case MP_ALLOC_XBUF: 1035 { 1036 struct rte_pktmbuf_extmem *ext_mem; 1037 unsigned int ext_num; 1038 1039 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1040 socket_id, pool_name, &ext_mem); 1041 if (ext_num == 0) 1042 rte_exit(EXIT_FAILURE, 1043 "Can't create pinned data buffers\n"); 1044 1045 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1046 rte_mbuf_best_mempool_ops()); 1047 rte_mp = rte_pktmbuf_pool_create_extbuf 1048 (pool_name, nb_mbuf, mb_mempool_cache, 1049 0, mbuf_seg_size, socket_id, 1050 ext_mem, ext_num); 1051 free(ext_mem); 1052 break; 1053 } 1054 default: 1055 { 1056 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1057 } 1058 } 1059 1060 err: 1061 if (rte_mp == NULL) { 1062 rte_exit(EXIT_FAILURE, 1063 "Creation of mbuf pool for socket %u failed: %s\n", 1064 socket_id, rte_strerror(rte_errno)); 1065 } else if (verbose_level > 0) { 1066 rte_mempool_dump(stdout, rte_mp); 1067 } 1068 return rte_mp; 1069 } 1070 1071 /* 1072 * Check given socket id is valid or not with NUMA mode, 1073 * if valid, return 0, else return -1 1074 */ 1075 static int 1076 check_socket_id(const unsigned int socket_id) 1077 { 1078 static int warning_once = 0; 1079 1080 if (new_socket_id(socket_id)) { 1081 if (!warning_once && numa_support) 1082 printf("Warning: NUMA should be configured manually by" 1083 " using --port-numa-config and" 1084 " --ring-numa-config parameters along with" 1085 " --numa.\n"); 1086 warning_once = 1; 1087 return -1; 1088 } 1089 return 0; 1090 } 1091 1092 /* 1093 * Get the allowed maximum number of RX queues. 1094 * *pid return the port id which has minimal value of 1095 * max_rx_queues in all ports. 1096 */ 1097 queueid_t 1098 get_allowed_max_nb_rxq(portid_t *pid) 1099 { 1100 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1101 bool max_rxq_valid = false; 1102 portid_t pi; 1103 struct rte_eth_dev_info dev_info; 1104 1105 RTE_ETH_FOREACH_DEV(pi) { 1106 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1107 continue; 1108 1109 max_rxq_valid = true; 1110 if (dev_info.max_rx_queues < allowed_max_rxq) { 1111 allowed_max_rxq = dev_info.max_rx_queues; 1112 *pid = pi; 1113 } 1114 } 1115 return max_rxq_valid ? allowed_max_rxq : 0; 1116 } 1117 1118 /* 1119 * Check input rxq is valid or not. 1120 * If input rxq is not greater than any of maximum number 1121 * of RX queues of all ports, it is valid. 1122 * if valid, return 0, else return -1 1123 */ 1124 int 1125 check_nb_rxq(queueid_t rxq) 1126 { 1127 queueid_t allowed_max_rxq; 1128 portid_t pid = 0; 1129 1130 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1131 if (rxq > allowed_max_rxq) { 1132 printf("Fail: input rxq (%u) can't be greater " 1133 "than max_rx_queues (%u) of port %u\n", 1134 rxq, 1135 allowed_max_rxq, 1136 pid); 1137 return -1; 1138 } 1139 return 0; 1140 } 1141 1142 /* 1143 * Get the allowed maximum number of TX queues. 1144 * *pid return the port id which has minimal value of 1145 * max_tx_queues in all ports. 1146 */ 1147 queueid_t 1148 get_allowed_max_nb_txq(portid_t *pid) 1149 { 1150 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1151 bool max_txq_valid = false; 1152 portid_t pi; 1153 struct rte_eth_dev_info dev_info; 1154 1155 RTE_ETH_FOREACH_DEV(pi) { 1156 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1157 continue; 1158 1159 max_txq_valid = true; 1160 if (dev_info.max_tx_queues < allowed_max_txq) { 1161 allowed_max_txq = dev_info.max_tx_queues; 1162 *pid = pi; 1163 } 1164 } 1165 return max_txq_valid ? allowed_max_txq : 0; 1166 } 1167 1168 /* 1169 * Check input txq is valid or not. 1170 * If input txq is not greater than any of maximum number 1171 * of TX queues of all ports, it is valid. 1172 * if valid, return 0, else return -1 1173 */ 1174 int 1175 check_nb_txq(queueid_t txq) 1176 { 1177 queueid_t allowed_max_txq; 1178 portid_t pid = 0; 1179 1180 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1181 if (txq > allowed_max_txq) { 1182 printf("Fail: input txq (%u) can't be greater " 1183 "than max_tx_queues (%u) of port %u\n", 1184 txq, 1185 allowed_max_txq, 1186 pid); 1187 return -1; 1188 } 1189 return 0; 1190 } 1191 1192 /* 1193 * Get the allowed maximum number of RXDs of every rx queue. 1194 * *pid return the port id which has minimal value of 1195 * max_rxd in all queues of all ports. 1196 */ 1197 static uint16_t 1198 get_allowed_max_nb_rxd(portid_t *pid) 1199 { 1200 uint16_t allowed_max_rxd = UINT16_MAX; 1201 portid_t pi; 1202 struct rte_eth_dev_info dev_info; 1203 1204 RTE_ETH_FOREACH_DEV(pi) { 1205 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1206 continue; 1207 1208 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1209 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1210 *pid = pi; 1211 } 1212 } 1213 return allowed_max_rxd; 1214 } 1215 1216 /* 1217 * Get the allowed minimal number of RXDs of every rx queue. 1218 * *pid return the port id which has minimal value of 1219 * min_rxd in all queues of all ports. 1220 */ 1221 static uint16_t 1222 get_allowed_min_nb_rxd(portid_t *pid) 1223 { 1224 uint16_t allowed_min_rxd = 0; 1225 portid_t pi; 1226 struct rte_eth_dev_info dev_info; 1227 1228 RTE_ETH_FOREACH_DEV(pi) { 1229 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1230 continue; 1231 1232 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1233 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1234 *pid = pi; 1235 } 1236 } 1237 1238 return allowed_min_rxd; 1239 } 1240 1241 /* 1242 * Check input rxd is valid or not. 1243 * If input rxd is not greater than any of maximum number 1244 * of RXDs of every Rx queues and is not less than any of 1245 * minimal number of RXDs of every Rx queues, it is valid. 1246 * if valid, return 0, else return -1 1247 */ 1248 int 1249 check_nb_rxd(queueid_t rxd) 1250 { 1251 uint16_t allowed_max_rxd; 1252 uint16_t allowed_min_rxd; 1253 portid_t pid = 0; 1254 1255 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1256 if (rxd > allowed_max_rxd) { 1257 printf("Fail: input rxd (%u) can't be greater " 1258 "than max_rxds (%u) of port %u\n", 1259 rxd, 1260 allowed_max_rxd, 1261 pid); 1262 return -1; 1263 } 1264 1265 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1266 if (rxd < allowed_min_rxd) { 1267 printf("Fail: input rxd (%u) can't be less " 1268 "than min_rxds (%u) of port %u\n", 1269 rxd, 1270 allowed_min_rxd, 1271 pid); 1272 return -1; 1273 } 1274 1275 return 0; 1276 } 1277 1278 /* 1279 * Get the allowed maximum number of TXDs of every rx queues. 1280 * *pid return the port id which has minimal value of 1281 * max_txd in every tx queue. 1282 */ 1283 static uint16_t 1284 get_allowed_max_nb_txd(portid_t *pid) 1285 { 1286 uint16_t allowed_max_txd = UINT16_MAX; 1287 portid_t pi; 1288 struct rte_eth_dev_info dev_info; 1289 1290 RTE_ETH_FOREACH_DEV(pi) { 1291 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1292 continue; 1293 1294 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1295 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1296 *pid = pi; 1297 } 1298 } 1299 return allowed_max_txd; 1300 } 1301 1302 /* 1303 * Get the allowed maximum number of TXDs of every tx queues. 1304 * *pid return the port id which has minimal value of 1305 * min_txd in every tx queue. 1306 */ 1307 static uint16_t 1308 get_allowed_min_nb_txd(portid_t *pid) 1309 { 1310 uint16_t allowed_min_txd = 0; 1311 portid_t pi; 1312 struct rte_eth_dev_info dev_info; 1313 1314 RTE_ETH_FOREACH_DEV(pi) { 1315 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1316 continue; 1317 1318 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1319 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1320 *pid = pi; 1321 } 1322 } 1323 1324 return allowed_min_txd; 1325 } 1326 1327 /* 1328 * Check input txd is valid or not. 1329 * If input txd is not greater than any of maximum number 1330 * of TXDs of every Rx queues, it is valid. 1331 * if valid, return 0, else return -1 1332 */ 1333 int 1334 check_nb_txd(queueid_t txd) 1335 { 1336 uint16_t allowed_max_txd; 1337 uint16_t allowed_min_txd; 1338 portid_t pid = 0; 1339 1340 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1341 if (txd > allowed_max_txd) { 1342 printf("Fail: input txd (%u) can't be greater " 1343 "than max_txds (%u) of port %u\n", 1344 txd, 1345 allowed_max_txd, 1346 pid); 1347 return -1; 1348 } 1349 1350 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1351 if (txd < allowed_min_txd) { 1352 printf("Fail: input txd (%u) can't be less " 1353 "than min_txds (%u) of port %u\n", 1354 txd, 1355 allowed_min_txd, 1356 pid); 1357 return -1; 1358 } 1359 return 0; 1360 } 1361 1362 1363 /* 1364 * Get the allowed maximum number of hairpin queues. 1365 * *pid return the port id which has minimal value of 1366 * max_hairpin_queues in all ports. 1367 */ 1368 queueid_t 1369 get_allowed_max_nb_hairpinq(portid_t *pid) 1370 { 1371 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1372 portid_t pi; 1373 struct rte_eth_hairpin_cap cap; 1374 1375 RTE_ETH_FOREACH_DEV(pi) { 1376 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1377 *pid = pi; 1378 return 0; 1379 } 1380 if (cap.max_nb_queues < allowed_max_hairpinq) { 1381 allowed_max_hairpinq = cap.max_nb_queues; 1382 *pid = pi; 1383 } 1384 } 1385 return allowed_max_hairpinq; 1386 } 1387 1388 /* 1389 * Check input hairpin is valid or not. 1390 * If input hairpin is not greater than any of maximum number 1391 * of hairpin queues of all ports, it is valid. 1392 * if valid, return 0, else return -1 1393 */ 1394 int 1395 check_nb_hairpinq(queueid_t hairpinq) 1396 { 1397 queueid_t allowed_max_hairpinq; 1398 portid_t pid = 0; 1399 1400 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1401 if (hairpinq > allowed_max_hairpinq) { 1402 printf("Fail: input hairpin (%u) can't be greater " 1403 "than max_hairpin_queues (%u) of port %u\n", 1404 hairpinq, allowed_max_hairpinq, pid); 1405 return -1; 1406 } 1407 return 0; 1408 } 1409 1410 static void 1411 init_config(void) 1412 { 1413 portid_t pid; 1414 struct rte_port *port; 1415 struct rte_mempool *mbp; 1416 unsigned int nb_mbuf_per_pool; 1417 lcoreid_t lc_id; 1418 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1419 struct rte_gro_param gro_param; 1420 uint32_t gso_types; 1421 uint16_t data_size; 1422 bool warning = 0; 1423 int k; 1424 int ret; 1425 1426 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1427 1428 /* Configuration of logical cores. */ 1429 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1430 sizeof(struct fwd_lcore *) * nb_lcores, 1431 RTE_CACHE_LINE_SIZE); 1432 if (fwd_lcores == NULL) { 1433 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1434 "failed\n", nb_lcores); 1435 } 1436 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1437 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1438 sizeof(struct fwd_lcore), 1439 RTE_CACHE_LINE_SIZE); 1440 if (fwd_lcores[lc_id] == NULL) { 1441 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1442 "failed\n"); 1443 } 1444 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1445 } 1446 1447 RTE_ETH_FOREACH_DEV(pid) { 1448 port = &ports[pid]; 1449 /* Apply default TxRx configuration for all ports */ 1450 port->dev_conf.txmode = tx_mode; 1451 port->dev_conf.rxmode = rx_mode; 1452 1453 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1454 if (ret != 0) 1455 rte_exit(EXIT_FAILURE, 1456 "rte_eth_dev_info_get() failed\n"); 1457 1458 ret = update_jumbo_frame_offload(pid); 1459 if (ret != 0) 1460 printf("Updating jumbo frame offload failed for port %u\n", 1461 pid); 1462 1463 if (!(port->dev_info.tx_offload_capa & 1464 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1465 port->dev_conf.txmode.offloads &= 1466 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1467 if (numa_support) { 1468 if (port_numa[pid] != NUMA_NO_CONFIG) 1469 port_per_socket[port_numa[pid]]++; 1470 else { 1471 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1472 1473 /* 1474 * if socket_id is invalid, 1475 * set to the first available socket. 1476 */ 1477 if (check_socket_id(socket_id) < 0) 1478 socket_id = socket_ids[0]; 1479 port_per_socket[socket_id]++; 1480 } 1481 } 1482 1483 /* Apply Rx offloads configuration */ 1484 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1485 port->rx_conf[k].offloads = 1486 port->dev_conf.rxmode.offloads; 1487 /* Apply Tx offloads configuration */ 1488 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1489 port->tx_conf[k].offloads = 1490 port->dev_conf.txmode.offloads; 1491 1492 if (eth_link_speed) 1493 port->dev_conf.link_speeds = eth_link_speed; 1494 1495 /* set flag to initialize port/queue */ 1496 port->need_reconfig = 1; 1497 port->need_reconfig_queues = 1; 1498 port->tx_metadata = 0; 1499 1500 /* Check for maximum number of segments per MTU. Accordingly 1501 * update the mbuf data size. 1502 */ 1503 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1504 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1505 data_size = rx_mode.max_rx_pkt_len / 1506 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1507 1508 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1509 mbuf_data_size[0]) { 1510 mbuf_data_size[0] = data_size + 1511 RTE_PKTMBUF_HEADROOM; 1512 warning = 1; 1513 } 1514 } 1515 } 1516 1517 if (warning) 1518 TESTPMD_LOG(WARNING, 1519 "Configured mbuf size of the first segment %hu\n", 1520 mbuf_data_size[0]); 1521 /* 1522 * Create pools of mbuf. 1523 * If NUMA support is disabled, create a single pool of mbuf in 1524 * socket 0 memory by default. 1525 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1526 * 1527 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1528 * nb_txd can be configured at run time. 1529 */ 1530 if (param_total_num_mbufs) 1531 nb_mbuf_per_pool = param_total_num_mbufs; 1532 else { 1533 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1534 (nb_lcores * mb_mempool_cache) + 1535 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1536 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1537 } 1538 1539 if (numa_support) { 1540 uint8_t i, j; 1541 1542 for (i = 0; i < num_sockets; i++) 1543 for (j = 0; j < mbuf_data_size_n; j++) 1544 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1545 mbuf_pool_create(mbuf_data_size[j], 1546 nb_mbuf_per_pool, 1547 socket_ids[i], j); 1548 } else { 1549 uint8_t i; 1550 1551 for (i = 0; i < mbuf_data_size_n; i++) 1552 mempools[i] = mbuf_pool_create 1553 (mbuf_data_size[i], 1554 nb_mbuf_per_pool, 1555 socket_num == UMA_NO_CONFIG ? 1556 0 : socket_num, i); 1557 } 1558 1559 init_port_config(); 1560 1561 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1562 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1563 /* 1564 * Records which Mbuf pool to use by each logical core, if needed. 1565 */ 1566 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1567 mbp = mbuf_pool_find( 1568 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1569 1570 if (mbp == NULL) 1571 mbp = mbuf_pool_find(0, 0); 1572 fwd_lcores[lc_id]->mbp = mbp; 1573 /* initialize GSO context */ 1574 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1575 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1576 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1577 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1578 RTE_ETHER_CRC_LEN; 1579 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1580 } 1581 1582 /* Configuration of packet forwarding streams. */ 1583 if (init_fwd_streams() < 0) 1584 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1585 1586 fwd_config_setup(); 1587 1588 /* create a gro context for each lcore */ 1589 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1590 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1591 gro_param.max_item_per_flow = MAX_PKT_BURST; 1592 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1593 gro_param.socket_id = rte_lcore_to_socket_id( 1594 fwd_lcores_cpuids[lc_id]); 1595 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1596 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1597 rte_exit(EXIT_FAILURE, 1598 "rte_gro_ctx_create() failed\n"); 1599 } 1600 } 1601 } 1602 1603 1604 void 1605 reconfig(portid_t new_port_id, unsigned socket_id) 1606 { 1607 struct rte_port *port; 1608 int ret; 1609 1610 /* Reconfiguration of Ethernet ports. */ 1611 port = &ports[new_port_id]; 1612 1613 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1614 if (ret != 0) 1615 return; 1616 1617 /* set flag to initialize port/queue */ 1618 port->need_reconfig = 1; 1619 port->need_reconfig_queues = 1; 1620 port->socket_id = socket_id; 1621 1622 init_port_config(); 1623 } 1624 1625 1626 int 1627 init_fwd_streams(void) 1628 { 1629 portid_t pid; 1630 struct rte_port *port; 1631 streamid_t sm_id, nb_fwd_streams_new; 1632 queueid_t q; 1633 1634 /* set socket id according to numa or not */ 1635 RTE_ETH_FOREACH_DEV(pid) { 1636 port = &ports[pid]; 1637 if (nb_rxq > port->dev_info.max_rx_queues) { 1638 printf("Fail: nb_rxq(%d) is greater than " 1639 "max_rx_queues(%d)\n", nb_rxq, 1640 port->dev_info.max_rx_queues); 1641 return -1; 1642 } 1643 if (nb_txq > port->dev_info.max_tx_queues) { 1644 printf("Fail: nb_txq(%d) is greater than " 1645 "max_tx_queues(%d)\n", nb_txq, 1646 port->dev_info.max_tx_queues); 1647 return -1; 1648 } 1649 if (numa_support) { 1650 if (port_numa[pid] != NUMA_NO_CONFIG) 1651 port->socket_id = port_numa[pid]; 1652 else { 1653 port->socket_id = rte_eth_dev_socket_id(pid); 1654 1655 /* 1656 * if socket_id is invalid, 1657 * set to the first available socket. 1658 */ 1659 if (check_socket_id(port->socket_id) < 0) 1660 port->socket_id = socket_ids[0]; 1661 } 1662 } 1663 else { 1664 if (socket_num == UMA_NO_CONFIG) 1665 port->socket_id = 0; 1666 else 1667 port->socket_id = socket_num; 1668 } 1669 } 1670 1671 q = RTE_MAX(nb_rxq, nb_txq); 1672 if (q == 0) { 1673 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1674 return -1; 1675 } 1676 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1677 if (nb_fwd_streams_new == nb_fwd_streams) 1678 return 0; 1679 /* clear the old */ 1680 if (fwd_streams != NULL) { 1681 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1682 if (fwd_streams[sm_id] == NULL) 1683 continue; 1684 rte_free(fwd_streams[sm_id]); 1685 fwd_streams[sm_id] = NULL; 1686 } 1687 rte_free(fwd_streams); 1688 fwd_streams = NULL; 1689 } 1690 1691 /* init new */ 1692 nb_fwd_streams = nb_fwd_streams_new; 1693 if (nb_fwd_streams) { 1694 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1695 sizeof(struct fwd_stream *) * nb_fwd_streams, 1696 RTE_CACHE_LINE_SIZE); 1697 if (fwd_streams == NULL) 1698 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1699 " (struct fwd_stream *)) failed\n", 1700 nb_fwd_streams); 1701 1702 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1703 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1704 " struct fwd_stream", sizeof(struct fwd_stream), 1705 RTE_CACHE_LINE_SIZE); 1706 if (fwd_streams[sm_id] == NULL) 1707 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1708 "(struct fwd_stream) failed\n"); 1709 } 1710 } 1711 1712 return 0; 1713 } 1714 1715 static void 1716 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1717 { 1718 uint64_t total_burst, sburst; 1719 uint64_t nb_burst; 1720 uint64_t burst_stats[4]; 1721 uint16_t pktnb_stats[4]; 1722 uint16_t nb_pkt; 1723 int burst_percent[4], sburstp; 1724 int i; 1725 1726 /* 1727 * First compute the total number of packet bursts and the 1728 * two highest numbers of bursts of the same number of packets. 1729 */ 1730 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1731 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1732 1733 /* Show stats for 0 burst size always */ 1734 total_burst = pbs->pkt_burst_spread[0]; 1735 burst_stats[0] = pbs->pkt_burst_spread[0]; 1736 pktnb_stats[0] = 0; 1737 1738 /* Find the next 2 burst sizes with highest occurrences. */ 1739 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1740 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1741 1742 if (nb_burst == 0) 1743 continue; 1744 1745 total_burst += nb_burst; 1746 1747 if (nb_burst > burst_stats[1]) { 1748 burst_stats[2] = burst_stats[1]; 1749 pktnb_stats[2] = pktnb_stats[1]; 1750 burst_stats[1] = nb_burst; 1751 pktnb_stats[1] = nb_pkt; 1752 } else if (nb_burst > burst_stats[2]) { 1753 burst_stats[2] = nb_burst; 1754 pktnb_stats[2] = nb_pkt; 1755 } 1756 } 1757 if (total_burst == 0) 1758 return; 1759 1760 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1761 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1762 if (i == 3) { 1763 printf("%d%% of other]\n", 100 - sburstp); 1764 return; 1765 } 1766 1767 sburst += burst_stats[i]; 1768 if (sburst == total_burst) { 1769 printf("%d%% of %d pkts]\n", 1770 100 - sburstp, (int) pktnb_stats[i]); 1771 return; 1772 } 1773 1774 burst_percent[i] = 1775 (double)burst_stats[i] / total_burst * 100; 1776 printf("%d%% of %d pkts + ", 1777 burst_percent[i], (int) pktnb_stats[i]); 1778 sburstp += burst_percent[i]; 1779 } 1780 } 1781 1782 static void 1783 fwd_stream_stats_display(streamid_t stream_id) 1784 { 1785 struct fwd_stream *fs; 1786 static const char *fwd_top_stats_border = "-------"; 1787 1788 fs = fwd_streams[stream_id]; 1789 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1790 (fs->fwd_dropped == 0)) 1791 return; 1792 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1793 "TX Port=%2d/Queue=%2d %s\n", 1794 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1795 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1796 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1797 " TX-dropped: %-14"PRIu64, 1798 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1799 1800 /* if checksum mode */ 1801 if (cur_fwd_eng == &csum_fwd_engine) { 1802 printf(" RX- bad IP checksum: %-14"PRIu64 1803 " Rx- bad L4 checksum: %-14"PRIu64 1804 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1805 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1806 fs->rx_bad_outer_l4_csum); 1807 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1808 fs->rx_bad_outer_ip_csum); 1809 } else { 1810 printf("\n"); 1811 } 1812 1813 if (record_burst_stats) { 1814 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1815 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1816 } 1817 } 1818 1819 void 1820 fwd_stats_display(void) 1821 { 1822 static const char *fwd_stats_border = "----------------------"; 1823 static const char *acc_stats_border = "+++++++++++++++"; 1824 struct { 1825 struct fwd_stream *rx_stream; 1826 struct fwd_stream *tx_stream; 1827 uint64_t tx_dropped; 1828 uint64_t rx_bad_ip_csum; 1829 uint64_t rx_bad_l4_csum; 1830 uint64_t rx_bad_outer_l4_csum; 1831 uint64_t rx_bad_outer_ip_csum; 1832 } ports_stats[RTE_MAX_ETHPORTS]; 1833 uint64_t total_rx_dropped = 0; 1834 uint64_t total_tx_dropped = 0; 1835 uint64_t total_rx_nombuf = 0; 1836 struct rte_eth_stats stats; 1837 uint64_t fwd_cycles = 0; 1838 uint64_t total_recv = 0; 1839 uint64_t total_xmit = 0; 1840 struct rte_port *port; 1841 streamid_t sm_id; 1842 portid_t pt_id; 1843 int i; 1844 1845 memset(ports_stats, 0, sizeof(ports_stats)); 1846 1847 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1848 struct fwd_stream *fs = fwd_streams[sm_id]; 1849 1850 if (cur_fwd_config.nb_fwd_streams > 1851 cur_fwd_config.nb_fwd_ports) { 1852 fwd_stream_stats_display(sm_id); 1853 } else { 1854 ports_stats[fs->tx_port].tx_stream = fs; 1855 ports_stats[fs->rx_port].rx_stream = fs; 1856 } 1857 1858 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1859 1860 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1861 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1862 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1863 fs->rx_bad_outer_l4_csum; 1864 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 1865 fs->rx_bad_outer_ip_csum; 1866 1867 if (record_core_cycles) 1868 fwd_cycles += fs->core_cycles; 1869 } 1870 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1871 pt_id = fwd_ports_ids[i]; 1872 port = &ports[pt_id]; 1873 1874 rte_eth_stats_get(pt_id, &stats); 1875 stats.ipackets -= port->stats.ipackets; 1876 stats.opackets -= port->stats.opackets; 1877 stats.ibytes -= port->stats.ibytes; 1878 stats.obytes -= port->stats.obytes; 1879 stats.imissed -= port->stats.imissed; 1880 stats.oerrors -= port->stats.oerrors; 1881 stats.rx_nombuf -= port->stats.rx_nombuf; 1882 1883 total_recv += stats.ipackets; 1884 total_xmit += stats.opackets; 1885 total_rx_dropped += stats.imissed; 1886 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1887 total_tx_dropped += stats.oerrors; 1888 total_rx_nombuf += stats.rx_nombuf; 1889 1890 printf("\n %s Forward statistics for port %-2d %s\n", 1891 fwd_stats_border, pt_id, fwd_stats_border); 1892 1893 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 1894 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 1895 stats.ipackets + stats.imissed); 1896 1897 if (cur_fwd_eng == &csum_fwd_engine) { 1898 printf(" Bad-ipcsum: %-14"PRIu64 1899 " Bad-l4csum: %-14"PRIu64 1900 "Bad-outer-l4csum: %-14"PRIu64"\n", 1901 ports_stats[pt_id].rx_bad_ip_csum, 1902 ports_stats[pt_id].rx_bad_l4_csum, 1903 ports_stats[pt_id].rx_bad_outer_l4_csum); 1904 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 1905 ports_stats[pt_id].rx_bad_outer_ip_csum); 1906 } 1907 if (stats.ierrors + stats.rx_nombuf > 0) { 1908 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 1909 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 1910 } 1911 1912 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 1913 "TX-total: %-"PRIu64"\n", 1914 stats.opackets, ports_stats[pt_id].tx_dropped, 1915 stats.opackets + ports_stats[pt_id].tx_dropped); 1916 1917 if (record_burst_stats) { 1918 if (ports_stats[pt_id].rx_stream) 1919 pkt_burst_stats_display("RX", 1920 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1921 if (ports_stats[pt_id].tx_stream) 1922 pkt_burst_stats_display("TX", 1923 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1924 } 1925 1926 printf(" %s--------------------------------%s\n", 1927 fwd_stats_border, fwd_stats_border); 1928 } 1929 1930 printf("\n %s Accumulated forward statistics for all ports" 1931 "%s\n", 1932 acc_stats_border, acc_stats_border); 1933 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1934 "%-"PRIu64"\n" 1935 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1936 "%-"PRIu64"\n", 1937 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1938 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1939 if (total_rx_nombuf > 0) 1940 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1941 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1942 "%s\n", 1943 acc_stats_border, acc_stats_border); 1944 if (record_core_cycles) { 1945 #define CYC_PER_MHZ 1E6 1946 if (total_recv > 0 || total_xmit > 0) { 1947 uint64_t total_pkts = 0; 1948 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1949 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1950 total_pkts = total_xmit; 1951 else 1952 total_pkts = total_recv; 1953 1954 printf("\n CPU cycles/packet=%.2F (total cycles=" 1955 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1956 " MHz Clock\n", 1957 (double) fwd_cycles / total_pkts, 1958 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1959 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1960 } 1961 } 1962 } 1963 1964 void 1965 fwd_stats_reset(void) 1966 { 1967 streamid_t sm_id; 1968 portid_t pt_id; 1969 int i; 1970 1971 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1972 pt_id = fwd_ports_ids[i]; 1973 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1974 } 1975 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1976 struct fwd_stream *fs = fwd_streams[sm_id]; 1977 1978 fs->rx_packets = 0; 1979 fs->tx_packets = 0; 1980 fs->fwd_dropped = 0; 1981 fs->rx_bad_ip_csum = 0; 1982 fs->rx_bad_l4_csum = 0; 1983 fs->rx_bad_outer_l4_csum = 0; 1984 fs->rx_bad_outer_ip_csum = 0; 1985 1986 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1987 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1988 fs->core_cycles = 0; 1989 } 1990 } 1991 1992 static void 1993 flush_fwd_rx_queues(void) 1994 { 1995 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1996 portid_t rxp; 1997 portid_t port_id; 1998 queueid_t rxq; 1999 uint16_t nb_rx; 2000 uint16_t i; 2001 uint8_t j; 2002 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2003 uint64_t timer_period; 2004 2005 /* convert to number of cycles */ 2006 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2007 2008 for (j = 0; j < 2; j++) { 2009 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2010 for (rxq = 0; rxq < nb_rxq; rxq++) { 2011 port_id = fwd_ports_ids[rxp]; 2012 /** 2013 * testpmd can stuck in the below do while loop 2014 * if rte_eth_rx_burst() always returns nonzero 2015 * packets. So timer is added to exit this loop 2016 * after 1sec timer expiry. 2017 */ 2018 prev_tsc = rte_rdtsc(); 2019 do { 2020 nb_rx = rte_eth_rx_burst(port_id, rxq, 2021 pkts_burst, MAX_PKT_BURST); 2022 for (i = 0; i < nb_rx; i++) 2023 rte_pktmbuf_free(pkts_burst[i]); 2024 2025 cur_tsc = rte_rdtsc(); 2026 diff_tsc = cur_tsc - prev_tsc; 2027 timer_tsc += diff_tsc; 2028 } while ((nb_rx > 0) && 2029 (timer_tsc < timer_period)); 2030 timer_tsc = 0; 2031 } 2032 } 2033 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2034 } 2035 } 2036 2037 static void 2038 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2039 { 2040 struct fwd_stream **fsm; 2041 streamid_t nb_fs; 2042 streamid_t sm_id; 2043 #ifdef RTE_LIB_BITRATESTATS 2044 uint64_t tics_per_1sec; 2045 uint64_t tics_datum; 2046 uint64_t tics_current; 2047 uint16_t i, cnt_ports; 2048 2049 cnt_ports = nb_ports; 2050 tics_datum = rte_rdtsc(); 2051 tics_per_1sec = rte_get_timer_hz(); 2052 #endif 2053 fsm = &fwd_streams[fc->stream_idx]; 2054 nb_fs = fc->stream_nb; 2055 do { 2056 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2057 (*pkt_fwd)(fsm[sm_id]); 2058 #ifdef RTE_LIB_BITRATESTATS 2059 if (bitrate_enabled != 0 && 2060 bitrate_lcore_id == rte_lcore_id()) { 2061 tics_current = rte_rdtsc(); 2062 if (tics_current - tics_datum >= tics_per_1sec) { 2063 /* Periodic bitrate calculation */ 2064 for (i = 0; i < cnt_ports; i++) 2065 rte_stats_bitrate_calc(bitrate_data, 2066 ports_ids[i]); 2067 tics_datum = tics_current; 2068 } 2069 } 2070 #endif 2071 #ifdef RTE_LIB_LATENCYSTATS 2072 if (latencystats_enabled != 0 && 2073 latencystats_lcore_id == rte_lcore_id()) 2074 rte_latencystats_update(); 2075 #endif 2076 2077 } while (! fc->stopped); 2078 } 2079 2080 static int 2081 start_pkt_forward_on_core(void *fwd_arg) 2082 { 2083 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2084 cur_fwd_config.fwd_eng->packet_fwd); 2085 return 0; 2086 } 2087 2088 /* 2089 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2090 * Used to start communication flows in network loopback test configurations. 2091 */ 2092 static int 2093 run_one_txonly_burst_on_core(void *fwd_arg) 2094 { 2095 struct fwd_lcore *fwd_lc; 2096 struct fwd_lcore tmp_lcore; 2097 2098 fwd_lc = (struct fwd_lcore *) fwd_arg; 2099 tmp_lcore = *fwd_lc; 2100 tmp_lcore.stopped = 1; 2101 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2102 return 0; 2103 } 2104 2105 /* 2106 * Launch packet forwarding: 2107 * - Setup per-port forwarding context. 2108 * - launch logical cores with their forwarding configuration. 2109 */ 2110 static void 2111 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2112 { 2113 port_fwd_begin_t port_fwd_begin; 2114 unsigned int i; 2115 unsigned int lc_id; 2116 int diag; 2117 2118 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2119 if (port_fwd_begin != NULL) { 2120 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2121 (*port_fwd_begin)(fwd_ports_ids[i]); 2122 } 2123 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2124 lc_id = fwd_lcores_cpuids[i]; 2125 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2126 fwd_lcores[i]->stopped = 0; 2127 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2128 fwd_lcores[i], lc_id); 2129 if (diag != 0) 2130 printf("launch lcore %u failed - diag=%d\n", 2131 lc_id, diag); 2132 } 2133 } 2134 } 2135 2136 /* 2137 * Launch packet forwarding configuration. 2138 */ 2139 void 2140 start_packet_forwarding(int with_tx_first) 2141 { 2142 port_fwd_begin_t port_fwd_begin; 2143 port_fwd_end_t port_fwd_end; 2144 struct rte_port *port; 2145 unsigned int i; 2146 portid_t pt_id; 2147 2148 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2149 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2150 2151 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2152 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2153 2154 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2155 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2156 (!nb_rxq || !nb_txq)) 2157 rte_exit(EXIT_FAILURE, 2158 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2159 cur_fwd_eng->fwd_mode_name); 2160 2161 if (all_ports_started() == 0) { 2162 printf("Not all ports were started\n"); 2163 return; 2164 } 2165 if (test_done == 0) { 2166 printf("Packet forwarding already started\n"); 2167 return; 2168 } 2169 2170 2171 if(dcb_test) { 2172 for (i = 0; i < nb_fwd_ports; i++) { 2173 pt_id = fwd_ports_ids[i]; 2174 port = &ports[pt_id]; 2175 if (!port->dcb_flag) { 2176 printf("In DCB mode, all forwarding ports must " 2177 "be configured in this mode.\n"); 2178 return; 2179 } 2180 } 2181 if (nb_fwd_lcores == 1) { 2182 printf("In DCB mode,the nb forwarding cores " 2183 "should be larger than 1.\n"); 2184 return; 2185 } 2186 } 2187 test_done = 0; 2188 2189 fwd_config_setup(); 2190 2191 if(!no_flush_rx) 2192 flush_fwd_rx_queues(); 2193 2194 pkt_fwd_config_display(&cur_fwd_config); 2195 rxtx_config_display(); 2196 2197 fwd_stats_reset(); 2198 if (with_tx_first) { 2199 port_fwd_begin = tx_only_engine.port_fwd_begin; 2200 if (port_fwd_begin != NULL) { 2201 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2202 (*port_fwd_begin)(fwd_ports_ids[i]); 2203 } 2204 while (with_tx_first--) { 2205 launch_packet_forwarding( 2206 run_one_txonly_burst_on_core); 2207 rte_eal_mp_wait_lcore(); 2208 } 2209 port_fwd_end = tx_only_engine.port_fwd_end; 2210 if (port_fwd_end != NULL) { 2211 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2212 (*port_fwd_end)(fwd_ports_ids[i]); 2213 } 2214 } 2215 launch_packet_forwarding(start_pkt_forward_on_core); 2216 } 2217 2218 void 2219 stop_packet_forwarding(void) 2220 { 2221 port_fwd_end_t port_fwd_end; 2222 lcoreid_t lc_id; 2223 portid_t pt_id; 2224 int i; 2225 2226 if (test_done) { 2227 printf("Packet forwarding not started\n"); 2228 return; 2229 } 2230 printf("Telling cores to stop..."); 2231 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2232 fwd_lcores[lc_id]->stopped = 1; 2233 printf("\nWaiting for lcores to finish...\n"); 2234 rte_eal_mp_wait_lcore(); 2235 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2236 if (port_fwd_end != NULL) { 2237 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2238 pt_id = fwd_ports_ids[i]; 2239 (*port_fwd_end)(pt_id); 2240 } 2241 } 2242 2243 fwd_stats_display(); 2244 2245 printf("\nDone.\n"); 2246 test_done = 1; 2247 } 2248 2249 void 2250 dev_set_link_up(portid_t pid) 2251 { 2252 if (rte_eth_dev_set_link_up(pid) < 0) 2253 printf("\nSet link up fail.\n"); 2254 } 2255 2256 void 2257 dev_set_link_down(portid_t pid) 2258 { 2259 if (rte_eth_dev_set_link_down(pid) < 0) 2260 printf("\nSet link down fail.\n"); 2261 } 2262 2263 static int 2264 all_ports_started(void) 2265 { 2266 portid_t pi; 2267 struct rte_port *port; 2268 2269 RTE_ETH_FOREACH_DEV(pi) { 2270 port = &ports[pi]; 2271 /* Check if there is a port which is not started */ 2272 if ((port->port_status != RTE_PORT_STARTED) && 2273 (port->slave_flag == 0)) 2274 return 0; 2275 } 2276 2277 /* No port is not started */ 2278 return 1; 2279 } 2280 2281 int 2282 port_is_stopped(portid_t port_id) 2283 { 2284 struct rte_port *port = &ports[port_id]; 2285 2286 if ((port->port_status != RTE_PORT_STOPPED) && 2287 (port->slave_flag == 0)) 2288 return 0; 2289 return 1; 2290 } 2291 2292 int 2293 all_ports_stopped(void) 2294 { 2295 portid_t pi; 2296 2297 RTE_ETH_FOREACH_DEV(pi) { 2298 if (!port_is_stopped(pi)) 2299 return 0; 2300 } 2301 2302 return 1; 2303 } 2304 2305 int 2306 port_is_started(portid_t port_id) 2307 { 2308 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2309 return 0; 2310 2311 if (ports[port_id].port_status != RTE_PORT_STARTED) 2312 return 0; 2313 2314 return 1; 2315 } 2316 2317 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2318 static int 2319 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2320 { 2321 queueid_t qi; 2322 struct rte_eth_hairpin_conf hairpin_conf = { 2323 .peer_count = 1, 2324 }; 2325 int i; 2326 int diag; 2327 struct rte_port *port = &ports[pi]; 2328 uint16_t peer_rx_port = pi; 2329 uint16_t peer_tx_port = pi; 2330 uint32_t manual = 1; 2331 uint32_t tx_exp = hairpin_mode & 0x10; 2332 2333 if (!(hairpin_mode & 0xf)) { 2334 peer_rx_port = pi; 2335 peer_tx_port = pi; 2336 manual = 0; 2337 } else if (hairpin_mode & 0x1) { 2338 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2339 RTE_ETH_DEV_NO_OWNER); 2340 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2341 peer_tx_port = rte_eth_find_next_owned_by(0, 2342 RTE_ETH_DEV_NO_OWNER); 2343 if (p_pi != RTE_MAX_ETHPORTS) { 2344 peer_rx_port = p_pi; 2345 } else { 2346 uint16_t next_pi; 2347 2348 /* Last port will be the peer RX port of the first. */ 2349 RTE_ETH_FOREACH_DEV(next_pi) 2350 peer_rx_port = next_pi; 2351 } 2352 manual = 1; 2353 } else if (hairpin_mode & 0x2) { 2354 if (cnt_pi & 0x1) { 2355 peer_rx_port = p_pi; 2356 } else { 2357 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2358 RTE_ETH_DEV_NO_OWNER); 2359 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2360 peer_rx_port = pi; 2361 } 2362 peer_tx_port = peer_rx_port; 2363 manual = 1; 2364 } 2365 2366 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2367 hairpin_conf.peers[0].port = peer_rx_port; 2368 hairpin_conf.peers[0].queue = i + nb_rxq; 2369 hairpin_conf.manual_bind = !!manual; 2370 hairpin_conf.tx_explicit = !!tx_exp; 2371 diag = rte_eth_tx_hairpin_queue_setup 2372 (pi, qi, nb_txd, &hairpin_conf); 2373 i++; 2374 if (diag == 0) 2375 continue; 2376 2377 /* Fail to setup rx queue, return */ 2378 if (rte_atomic16_cmpset(&(port->port_status), 2379 RTE_PORT_HANDLING, 2380 RTE_PORT_STOPPED) == 0) 2381 printf("Port %d can not be set back " 2382 "to stopped\n", pi); 2383 printf("Fail to configure port %d hairpin " 2384 "queues\n", pi); 2385 /* try to reconfigure queues next time */ 2386 port->need_reconfig_queues = 1; 2387 return -1; 2388 } 2389 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2390 hairpin_conf.peers[0].port = peer_tx_port; 2391 hairpin_conf.peers[0].queue = i + nb_txq; 2392 hairpin_conf.manual_bind = !!manual; 2393 hairpin_conf.tx_explicit = !!tx_exp; 2394 diag = rte_eth_rx_hairpin_queue_setup 2395 (pi, qi, nb_rxd, &hairpin_conf); 2396 i++; 2397 if (diag == 0) 2398 continue; 2399 2400 /* Fail to setup rx queue, return */ 2401 if (rte_atomic16_cmpset(&(port->port_status), 2402 RTE_PORT_HANDLING, 2403 RTE_PORT_STOPPED) == 0) 2404 printf("Port %d can not be set back " 2405 "to stopped\n", pi); 2406 printf("Fail to configure port %d hairpin " 2407 "queues\n", pi); 2408 /* try to reconfigure queues next time */ 2409 port->need_reconfig_queues = 1; 2410 return -1; 2411 } 2412 return 0; 2413 } 2414 2415 /* Configure the Rx with optional split. */ 2416 int 2417 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2418 uint16_t nb_rx_desc, unsigned int socket_id, 2419 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2420 { 2421 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2422 unsigned int i, mp_n; 2423 int ret; 2424 2425 if (rx_pkt_nb_segs <= 1 || 2426 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2427 rx_conf->rx_seg = NULL; 2428 rx_conf->rx_nseg = 0; 2429 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2430 nb_rx_desc, socket_id, 2431 rx_conf, mp); 2432 return ret; 2433 } 2434 for (i = 0; i < rx_pkt_nb_segs; i++) { 2435 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2436 struct rte_mempool *mpx; 2437 /* 2438 * Use last valid pool for the segments with number 2439 * exceeding the pool index. 2440 */ 2441 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2442 mpx = mbuf_pool_find(socket_id, mp_n); 2443 /* Handle zero as mbuf data buffer size. */ 2444 rx_seg->length = rx_pkt_seg_lengths[i] ? 2445 rx_pkt_seg_lengths[i] : 2446 mbuf_data_size[mp_n]; 2447 rx_seg->offset = i < rx_pkt_nb_offs ? 2448 rx_pkt_seg_offsets[i] : 0; 2449 rx_seg->mp = mpx ? mpx : mp; 2450 } 2451 rx_conf->rx_nseg = rx_pkt_nb_segs; 2452 rx_conf->rx_seg = rx_useg; 2453 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2454 socket_id, rx_conf, NULL); 2455 rx_conf->rx_seg = NULL; 2456 rx_conf->rx_nseg = 0; 2457 return ret; 2458 } 2459 2460 int 2461 start_port(portid_t pid) 2462 { 2463 int diag, need_check_link_status = -1; 2464 portid_t pi; 2465 portid_t p_pi = RTE_MAX_ETHPORTS; 2466 portid_t pl[RTE_MAX_ETHPORTS]; 2467 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2468 uint16_t cnt_pi = 0; 2469 uint16_t cfg_pi = 0; 2470 int peer_pi; 2471 queueid_t qi; 2472 struct rte_port *port; 2473 struct rte_ether_addr mac_addr; 2474 struct rte_eth_hairpin_cap cap; 2475 2476 if (port_id_is_invalid(pid, ENABLED_WARN)) 2477 return 0; 2478 2479 if(dcb_config) 2480 dcb_test = 1; 2481 RTE_ETH_FOREACH_DEV(pi) { 2482 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2483 continue; 2484 2485 need_check_link_status = 0; 2486 port = &ports[pi]; 2487 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2488 RTE_PORT_HANDLING) == 0) { 2489 printf("Port %d is now not stopped\n", pi); 2490 continue; 2491 } 2492 2493 if (port->need_reconfig > 0) { 2494 port->need_reconfig = 0; 2495 2496 if (flow_isolate_all) { 2497 int ret = port_flow_isolate(pi, 1); 2498 if (ret) { 2499 printf("Failed to apply isolated" 2500 " mode on port %d\n", pi); 2501 return -1; 2502 } 2503 } 2504 configure_rxtx_dump_callbacks(0); 2505 printf("Configuring Port %d (socket %u)\n", pi, 2506 port->socket_id); 2507 if (nb_hairpinq > 0 && 2508 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2509 printf("Port %d doesn't support hairpin " 2510 "queues\n", pi); 2511 return -1; 2512 } 2513 /* configure port */ 2514 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2515 nb_txq + nb_hairpinq, 2516 &(port->dev_conf)); 2517 if (diag != 0) { 2518 if (rte_atomic16_cmpset(&(port->port_status), 2519 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2520 printf("Port %d can not be set back " 2521 "to stopped\n", pi); 2522 printf("Fail to configure port %d\n", pi); 2523 /* try to reconfigure port next time */ 2524 port->need_reconfig = 1; 2525 return -1; 2526 } 2527 } 2528 if (port->need_reconfig_queues > 0) { 2529 port->need_reconfig_queues = 0; 2530 /* setup tx queues */ 2531 for (qi = 0; qi < nb_txq; qi++) { 2532 if ((numa_support) && 2533 (txring_numa[pi] != NUMA_NO_CONFIG)) 2534 diag = rte_eth_tx_queue_setup(pi, qi, 2535 port->nb_tx_desc[qi], 2536 txring_numa[pi], 2537 &(port->tx_conf[qi])); 2538 else 2539 diag = rte_eth_tx_queue_setup(pi, qi, 2540 port->nb_tx_desc[qi], 2541 port->socket_id, 2542 &(port->tx_conf[qi])); 2543 2544 if (diag == 0) 2545 continue; 2546 2547 /* Fail to setup tx queue, return */ 2548 if (rte_atomic16_cmpset(&(port->port_status), 2549 RTE_PORT_HANDLING, 2550 RTE_PORT_STOPPED) == 0) 2551 printf("Port %d can not be set back " 2552 "to stopped\n", pi); 2553 printf("Fail to configure port %d tx queues\n", 2554 pi); 2555 /* try to reconfigure queues next time */ 2556 port->need_reconfig_queues = 1; 2557 return -1; 2558 } 2559 for (qi = 0; qi < nb_rxq; qi++) { 2560 /* setup rx queues */ 2561 if ((numa_support) && 2562 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2563 struct rte_mempool * mp = 2564 mbuf_pool_find 2565 (rxring_numa[pi], 0); 2566 if (mp == NULL) { 2567 printf("Failed to setup RX queue:" 2568 "No mempool allocation" 2569 " on the socket %d\n", 2570 rxring_numa[pi]); 2571 return -1; 2572 } 2573 2574 diag = rx_queue_setup(pi, qi, 2575 port->nb_rx_desc[qi], 2576 rxring_numa[pi], 2577 &(port->rx_conf[qi]), 2578 mp); 2579 } else { 2580 struct rte_mempool *mp = 2581 mbuf_pool_find 2582 (port->socket_id, 0); 2583 if (mp == NULL) { 2584 printf("Failed to setup RX queue:" 2585 "No mempool allocation" 2586 " on the socket %d\n", 2587 port->socket_id); 2588 return -1; 2589 } 2590 diag = rx_queue_setup(pi, qi, 2591 port->nb_rx_desc[qi], 2592 port->socket_id, 2593 &(port->rx_conf[qi]), 2594 mp); 2595 } 2596 if (diag == 0) 2597 continue; 2598 2599 /* Fail to setup rx queue, return */ 2600 if (rte_atomic16_cmpset(&(port->port_status), 2601 RTE_PORT_HANDLING, 2602 RTE_PORT_STOPPED) == 0) 2603 printf("Port %d can not be set back " 2604 "to stopped\n", pi); 2605 printf("Fail to configure port %d rx queues\n", 2606 pi); 2607 /* try to reconfigure queues next time */ 2608 port->need_reconfig_queues = 1; 2609 return -1; 2610 } 2611 /* setup hairpin queues */ 2612 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2613 return -1; 2614 } 2615 configure_rxtx_dump_callbacks(verbose_level); 2616 if (clear_ptypes) { 2617 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2618 NULL, 0); 2619 if (diag < 0) 2620 printf( 2621 "Port %d: Failed to disable Ptype parsing\n", 2622 pi); 2623 } 2624 2625 p_pi = pi; 2626 cnt_pi++; 2627 2628 /* start port */ 2629 if (rte_eth_dev_start(pi) < 0) { 2630 printf("Fail to start port %d\n", pi); 2631 2632 /* Fail to setup rx queue, return */ 2633 if (rte_atomic16_cmpset(&(port->port_status), 2634 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2635 printf("Port %d can not be set back to " 2636 "stopped\n", pi); 2637 continue; 2638 } 2639 2640 if (rte_atomic16_cmpset(&(port->port_status), 2641 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2642 printf("Port %d can not be set into started\n", pi); 2643 2644 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2645 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2646 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2647 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2648 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2649 2650 /* at least one port started, need checking link status */ 2651 need_check_link_status = 1; 2652 2653 pl[cfg_pi++] = pi; 2654 } 2655 2656 if (need_check_link_status == 1 && !no_link_check) 2657 check_all_ports_link_status(RTE_PORT_ALL); 2658 else if (need_check_link_status == 0) 2659 printf("Please stop the ports first\n"); 2660 2661 if (hairpin_mode & 0xf) { 2662 uint16_t i; 2663 int j; 2664 2665 /* bind all started hairpin ports */ 2666 for (i = 0; i < cfg_pi; i++) { 2667 pi = pl[i]; 2668 /* bind current Tx to all peer Rx */ 2669 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2670 RTE_MAX_ETHPORTS, 1); 2671 if (peer_pi < 0) 2672 return peer_pi; 2673 for (j = 0; j < peer_pi; j++) { 2674 if (!port_is_started(peer_pl[j])) 2675 continue; 2676 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2677 if (diag < 0) { 2678 printf("Error during binding hairpin" 2679 " Tx port %u to %u: %s\n", 2680 pi, peer_pl[j], 2681 rte_strerror(-diag)); 2682 return -1; 2683 } 2684 } 2685 /* bind all peer Tx to current Rx */ 2686 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2687 RTE_MAX_ETHPORTS, 0); 2688 if (peer_pi < 0) 2689 return peer_pi; 2690 for (j = 0; j < peer_pi; j++) { 2691 if (!port_is_started(peer_pl[j])) 2692 continue; 2693 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2694 if (diag < 0) { 2695 printf("Error during binding hairpin" 2696 " Tx port %u to %u: %s\n", 2697 peer_pl[j], pi, 2698 rte_strerror(-diag)); 2699 return -1; 2700 } 2701 } 2702 } 2703 } 2704 2705 printf("Done\n"); 2706 return 0; 2707 } 2708 2709 void 2710 stop_port(portid_t pid) 2711 { 2712 portid_t pi; 2713 struct rte_port *port; 2714 int need_check_link_status = 0; 2715 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2716 int peer_pi; 2717 2718 if (dcb_test) { 2719 dcb_test = 0; 2720 dcb_config = 0; 2721 } 2722 2723 if (port_id_is_invalid(pid, ENABLED_WARN)) 2724 return; 2725 2726 printf("Stopping ports...\n"); 2727 2728 RTE_ETH_FOREACH_DEV(pi) { 2729 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2730 continue; 2731 2732 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2733 printf("Please remove port %d from forwarding configuration.\n", pi); 2734 continue; 2735 } 2736 2737 if (port_is_bonding_slave(pi)) { 2738 printf("Please remove port %d from bonded device.\n", pi); 2739 continue; 2740 } 2741 2742 port = &ports[pi]; 2743 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2744 RTE_PORT_HANDLING) == 0) 2745 continue; 2746 2747 if (hairpin_mode & 0xf) { 2748 int j; 2749 2750 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2751 /* unbind all peer Tx from current Rx */ 2752 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2753 RTE_MAX_ETHPORTS, 0); 2754 if (peer_pi < 0) 2755 continue; 2756 for (j = 0; j < peer_pi; j++) { 2757 if (!port_is_started(peer_pl[j])) 2758 continue; 2759 rte_eth_hairpin_unbind(peer_pl[j], pi); 2760 } 2761 } 2762 2763 if (port->flow_list) 2764 port_flow_flush(pi); 2765 2766 if (rte_eth_dev_stop(pi) != 0) 2767 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2768 pi); 2769 2770 if (rte_atomic16_cmpset(&(port->port_status), 2771 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2772 printf("Port %d can not be set into stopped\n", pi); 2773 need_check_link_status = 1; 2774 } 2775 if (need_check_link_status && !no_link_check) 2776 check_all_ports_link_status(RTE_PORT_ALL); 2777 2778 printf("Done\n"); 2779 } 2780 2781 static void 2782 remove_invalid_ports_in(portid_t *array, portid_t *total) 2783 { 2784 portid_t i; 2785 portid_t new_total = 0; 2786 2787 for (i = 0; i < *total; i++) 2788 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2789 array[new_total] = array[i]; 2790 new_total++; 2791 } 2792 *total = new_total; 2793 } 2794 2795 static void 2796 remove_invalid_ports(void) 2797 { 2798 remove_invalid_ports_in(ports_ids, &nb_ports); 2799 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2800 nb_cfg_ports = nb_fwd_ports; 2801 } 2802 2803 void 2804 close_port(portid_t pid) 2805 { 2806 portid_t pi; 2807 struct rte_port *port; 2808 2809 if (port_id_is_invalid(pid, ENABLED_WARN)) 2810 return; 2811 2812 printf("Closing ports...\n"); 2813 2814 RTE_ETH_FOREACH_DEV(pi) { 2815 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2816 continue; 2817 2818 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2819 printf("Please remove port %d from forwarding configuration.\n", pi); 2820 continue; 2821 } 2822 2823 if (port_is_bonding_slave(pi)) { 2824 printf("Please remove port %d from bonded device.\n", pi); 2825 continue; 2826 } 2827 2828 port = &ports[pi]; 2829 if (rte_atomic16_cmpset(&(port->port_status), 2830 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2831 printf("Port %d is already closed\n", pi); 2832 continue; 2833 } 2834 2835 port_flow_flush(pi); 2836 rte_eth_dev_close(pi); 2837 } 2838 2839 remove_invalid_ports(); 2840 printf("Done\n"); 2841 } 2842 2843 void 2844 reset_port(portid_t pid) 2845 { 2846 int diag; 2847 portid_t pi; 2848 struct rte_port *port; 2849 2850 if (port_id_is_invalid(pid, ENABLED_WARN)) 2851 return; 2852 2853 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2854 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2855 printf("Can not reset port(s), please stop port(s) first.\n"); 2856 return; 2857 } 2858 2859 printf("Resetting ports...\n"); 2860 2861 RTE_ETH_FOREACH_DEV(pi) { 2862 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2863 continue; 2864 2865 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2866 printf("Please remove port %d from forwarding " 2867 "configuration.\n", pi); 2868 continue; 2869 } 2870 2871 if (port_is_bonding_slave(pi)) { 2872 printf("Please remove port %d from bonded device.\n", 2873 pi); 2874 continue; 2875 } 2876 2877 diag = rte_eth_dev_reset(pi); 2878 if (diag == 0) { 2879 port = &ports[pi]; 2880 port->need_reconfig = 1; 2881 port->need_reconfig_queues = 1; 2882 } else { 2883 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2884 } 2885 } 2886 2887 printf("Done\n"); 2888 } 2889 2890 void 2891 attach_port(char *identifier) 2892 { 2893 portid_t pi; 2894 struct rte_dev_iterator iterator; 2895 2896 printf("Attaching a new port...\n"); 2897 2898 if (identifier == NULL) { 2899 printf("Invalid parameters are specified\n"); 2900 return; 2901 } 2902 2903 if (rte_dev_probe(identifier) < 0) { 2904 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2905 return; 2906 } 2907 2908 /* first attach mode: event */ 2909 if (setup_on_probe_event) { 2910 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2911 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2912 if (ports[pi].port_status == RTE_PORT_HANDLING && 2913 ports[pi].need_setup != 0) 2914 setup_attached_port(pi); 2915 return; 2916 } 2917 2918 /* second attach mode: iterator */ 2919 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2920 /* setup ports matching the devargs used for probing */ 2921 if (port_is_forwarding(pi)) 2922 continue; /* port was already attached before */ 2923 setup_attached_port(pi); 2924 } 2925 } 2926 2927 static void 2928 setup_attached_port(portid_t pi) 2929 { 2930 unsigned int socket_id; 2931 int ret; 2932 2933 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2934 /* if socket_id is invalid, set to the first available socket. */ 2935 if (check_socket_id(socket_id) < 0) 2936 socket_id = socket_ids[0]; 2937 reconfig(pi, socket_id); 2938 ret = rte_eth_promiscuous_enable(pi); 2939 if (ret != 0) 2940 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2941 pi, rte_strerror(-ret)); 2942 2943 ports_ids[nb_ports++] = pi; 2944 fwd_ports_ids[nb_fwd_ports++] = pi; 2945 nb_cfg_ports = nb_fwd_ports; 2946 ports[pi].need_setup = 0; 2947 ports[pi].port_status = RTE_PORT_STOPPED; 2948 2949 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2950 printf("Done\n"); 2951 } 2952 2953 static void 2954 detach_device(struct rte_device *dev) 2955 { 2956 portid_t sibling; 2957 2958 if (dev == NULL) { 2959 printf("Device already removed\n"); 2960 return; 2961 } 2962 2963 printf("Removing a device...\n"); 2964 2965 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2966 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2967 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2968 printf("Port %u not stopped\n", sibling); 2969 return; 2970 } 2971 port_flow_flush(sibling); 2972 } 2973 } 2974 2975 if (rte_dev_remove(dev) < 0) { 2976 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2977 return; 2978 } 2979 remove_invalid_ports(); 2980 2981 printf("Device is detached\n"); 2982 printf("Now total ports is %d\n", nb_ports); 2983 printf("Done\n"); 2984 return; 2985 } 2986 2987 void 2988 detach_port_device(portid_t port_id) 2989 { 2990 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2991 return; 2992 2993 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2994 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2995 printf("Port not stopped\n"); 2996 return; 2997 } 2998 printf("Port was not closed\n"); 2999 } 3000 3001 detach_device(rte_eth_devices[port_id].device); 3002 } 3003 3004 void 3005 detach_devargs(char *identifier) 3006 { 3007 struct rte_dev_iterator iterator; 3008 struct rte_devargs da; 3009 portid_t port_id; 3010 3011 printf("Removing a device...\n"); 3012 3013 memset(&da, 0, sizeof(da)); 3014 if (rte_devargs_parsef(&da, "%s", identifier)) { 3015 printf("cannot parse identifier\n"); 3016 if (da.args) 3017 free(da.args); 3018 return; 3019 } 3020 3021 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3022 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3023 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3024 printf("Port %u not stopped\n", port_id); 3025 rte_eth_iterator_cleanup(&iterator); 3026 return; 3027 } 3028 port_flow_flush(port_id); 3029 } 3030 } 3031 3032 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3033 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3034 da.name, da.bus->name); 3035 return; 3036 } 3037 3038 remove_invalid_ports(); 3039 3040 printf("Device %s is detached\n", identifier); 3041 printf("Now total ports is %d\n", nb_ports); 3042 printf("Done\n"); 3043 } 3044 3045 void 3046 pmd_test_exit(void) 3047 { 3048 portid_t pt_id; 3049 unsigned int i; 3050 int ret; 3051 3052 if (test_done == 0) 3053 stop_packet_forwarding(); 3054 3055 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3056 if (mempools[i]) { 3057 if (mp_alloc_type == MP_ALLOC_ANON) 3058 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3059 NULL); 3060 } 3061 } 3062 if (ports != NULL) { 3063 no_link_check = 1; 3064 RTE_ETH_FOREACH_DEV(pt_id) { 3065 printf("\nStopping port %d...\n", pt_id); 3066 fflush(stdout); 3067 stop_port(pt_id); 3068 } 3069 RTE_ETH_FOREACH_DEV(pt_id) { 3070 printf("\nShutting down port %d...\n", pt_id); 3071 fflush(stdout); 3072 close_port(pt_id); 3073 } 3074 } 3075 3076 if (hot_plug) { 3077 ret = rte_dev_event_monitor_stop(); 3078 if (ret) { 3079 RTE_LOG(ERR, EAL, 3080 "fail to stop device event monitor."); 3081 return; 3082 } 3083 3084 ret = rte_dev_event_callback_unregister(NULL, 3085 dev_event_callback, NULL); 3086 if (ret < 0) { 3087 RTE_LOG(ERR, EAL, 3088 "fail to unregister device event callback.\n"); 3089 return; 3090 } 3091 3092 ret = rte_dev_hotplug_handle_disable(); 3093 if (ret) { 3094 RTE_LOG(ERR, EAL, 3095 "fail to disable hotplug handling.\n"); 3096 return; 3097 } 3098 } 3099 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3100 if (mempools[i]) 3101 rte_mempool_free(mempools[i]); 3102 } 3103 3104 printf("\nBye...\n"); 3105 } 3106 3107 typedef void (*cmd_func_t)(void); 3108 struct pmd_test_command { 3109 const char *cmd_name; 3110 cmd_func_t cmd_func; 3111 }; 3112 3113 /* Check the link status of all ports in up to 9s, and print them finally */ 3114 static void 3115 check_all_ports_link_status(uint32_t port_mask) 3116 { 3117 #define CHECK_INTERVAL 100 /* 100ms */ 3118 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3119 portid_t portid; 3120 uint8_t count, all_ports_up, print_flag = 0; 3121 struct rte_eth_link link; 3122 int ret; 3123 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3124 3125 printf("Checking link statuses...\n"); 3126 fflush(stdout); 3127 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3128 all_ports_up = 1; 3129 RTE_ETH_FOREACH_DEV(portid) { 3130 if ((port_mask & (1 << portid)) == 0) 3131 continue; 3132 memset(&link, 0, sizeof(link)); 3133 ret = rte_eth_link_get_nowait(portid, &link); 3134 if (ret < 0) { 3135 all_ports_up = 0; 3136 if (print_flag == 1) 3137 printf("Port %u link get failed: %s\n", 3138 portid, rte_strerror(-ret)); 3139 continue; 3140 } 3141 /* print link status if flag set */ 3142 if (print_flag == 1) { 3143 rte_eth_link_to_str(link_status, 3144 sizeof(link_status), &link); 3145 printf("Port %d %s\n", portid, link_status); 3146 continue; 3147 } 3148 /* clear all_ports_up flag if any link down */ 3149 if (link.link_status == ETH_LINK_DOWN) { 3150 all_ports_up = 0; 3151 break; 3152 } 3153 } 3154 /* after finally printing all link status, get out */ 3155 if (print_flag == 1) 3156 break; 3157 3158 if (all_ports_up == 0) { 3159 fflush(stdout); 3160 rte_delay_ms(CHECK_INTERVAL); 3161 } 3162 3163 /* set the print_flag if all ports up or timeout */ 3164 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3165 print_flag = 1; 3166 } 3167 3168 if (lsc_interrupt) 3169 break; 3170 } 3171 } 3172 3173 static void 3174 rmv_port_callback(void *arg) 3175 { 3176 int need_to_start = 0; 3177 int org_no_link_check = no_link_check; 3178 portid_t port_id = (intptr_t)arg; 3179 struct rte_device *dev; 3180 3181 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3182 3183 if (!test_done && port_is_forwarding(port_id)) { 3184 need_to_start = 1; 3185 stop_packet_forwarding(); 3186 } 3187 no_link_check = 1; 3188 stop_port(port_id); 3189 no_link_check = org_no_link_check; 3190 3191 /* Save rte_device pointer before closing ethdev port */ 3192 dev = rte_eth_devices[port_id].device; 3193 close_port(port_id); 3194 detach_device(dev); /* might be already removed or have more ports */ 3195 3196 if (need_to_start) 3197 start_packet_forwarding(0); 3198 } 3199 3200 /* This function is used by the interrupt thread */ 3201 static int 3202 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3203 void *ret_param) 3204 { 3205 RTE_SET_USED(param); 3206 RTE_SET_USED(ret_param); 3207 3208 if (type >= RTE_ETH_EVENT_MAX) { 3209 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3210 port_id, __func__, type); 3211 fflush(stderr); 3212 } else if (event_print_mask & (UINT32_C(1) << type)) { 3213 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3214 eth_event_desc[type]); 3215 fflush(stdout); 3216 } 3217 3218 switch (type) { 3219 case RTE_ETH_EVENT_NEW: 3220 ports[port_id].need_setup = 1; 3221 ports[port_id].port_status = RTE_PORT_HANDLING; 3222 break; 3223 case RTE_ETH_EVENT_INTR_RMV: 3224 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3225 break; 3226 if (rte_eal_alarm_set(100000, 3227 rmv_port_callback, (void *)(intptr_t)port_id)) 3228 fprintf(stderr, "Could not set up deferred device removal\n"); 3229 break; 3230 case RTE_ETH_EVENT_DESTROY: 3231 ports[port_id].port_status = RTE_PORT_CLOSED; 3232 printf("Port %u is closed\n", port_id); 3233 break; 3234 default: 3235 break; 3236 } 3237 return 0; 3238 } 3239 3240 static int 3241 register_eth_event_callback(void) 3242 { 3243 int ret; 3244 enum rte_eth_event_type event; 3245 3246 for (event = RTE_ETH_EVENT_UNKNOWN; 3247 event < RTE_ETH_EVENT_MAX; event++) { 3248 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3249 event, 3250 eth_event_callback, 3251 NULL); 3252 if (ret != 0) { 3253 TESTPMD_LOG(ERR, "Failed to register callback for " 3254 "%s event\n", eth_event_desc[event]); 3255 return -1; 3256 } 3257 } 3258 3259 return 0; 3260 } 3261 3262 /* This function is used by the interrupt thread */ 3263 static void 3264 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3265 __rte_unused void *arg) 3266 { 3267 uint16_t port_id; 3268 int ret; 3269 3270 if (type >= RTE_DEV_EVENT_MAX) { 3271 fprintf(stderr, "%s called upon invalid event %d\n", 3272 __func__, type); 3273 fflush(stderr); 3274 } 3275 3276 switch (type) { 3277 case RTE_DEV_EVENT_REMOVE: 3278 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3279 device_name); 3280 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3281 if (ret) { 3282 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3283 device_name); 3284 return; 3285 } 3286 /* 3287 * Because the user's callback is invoked in eal interrupt 3288 * callback, the interrupt callback need to be finished before 3289 * it can be unregistered when detaching device. So finish 3290 * callback soon and use a deferred removal to detach device 3291 * is need. It is a workaround, once the device detaching be 3292 * moved into the eal in the future, the deferred removal could 3293 * be deleted. 3294 */ 3295 if (rte_eal_alarm_set(100000, 3296 rmv_port_callback, (void *)(intptr_t)port_id)) 3297 RTE_LOG(ERR, EAL, 3298 "Could not set up deferred device removal\n"); 3299 break; 3300 case RTE_DEV_EVENT_ADD: 3301 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3302 device_name); 3303 /* TODO: After finish kernel driver binding, 3304 * begin to attach port. 3305 */ 3306 break; 3307 default: 3308 break; 3309 } 3310 } 3311 3312 static void 3313 rxtx_port_config(struct rte_port *port) 3314 { 3315 uint16_t qid; 3316 uint64_t offloads; 3317 3318 for (qid = 0; qid < nb_rxq; qid++) { 3319 offloads = port->rx_conf[qid].offloads; 3320 port->rx_conf[qid] = port->dev_info.default_rxconf; 3321 if (offloads != 0) 3322 port->rx_conf[qid].offloads = offloads; 3323 3324 /* Check if any Rx parameters have been passed */ 3325 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3326 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3327 3328 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3329 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3330 3331 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3332 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3333 3334 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3335 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3336 3337 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3338 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3339 3340 port->nb_rx_desc[qid] = nb_rxd; 3341 } 3342 3343 for (qid = 0; qid < nb_txq; qid++) { 3344 offloads = port->tx_conf[qid].offloads; 3345 port->tx_conf[qid] = port->dev_info.default_txconf; 3346 if (offloads != 0) 3347 port->tx_conf[qid].offloads = offloads; 3348 3349 /* Check if any Tx parameters have been passed */ 3350 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3351 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3352 3353 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3354 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3355 3356 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3357 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3358 3359 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3360 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3361 3362 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3363 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3364 3365 port->nb_tx_desc[qid] = nb_txd; 3366 } 3367 } 3368 3369 /* 3370 * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload, 3371 * MTU is also aligned if JUMBO_FRAME offload is not set. 3372 * 3373 * port->dev_info should be set before calling this function. 3374 * 3375 * return 0 on success, negative on error 3376 */ 3377 int 3378 update_jumbo_frame_offload(portid_t portid) 3379 { 3380 struct rte_port *port = &ports[portid]; 3381 uint32_t eth_overhead; 3382 uint64_t rx_offloads; 3383 int ret; 3384 bool on; 3385 3386 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */ 3387 if (port->dev_info.max_mtu != UINT16_MAX && 3388 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu) 3389 eth_overhead = port->dev_info.max_rx_pktlen - 3390 port->dev_info.max_mtu; 3391 else 3392 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 3393 3394 rx_offloads = port->dev_conf.rxmode.offloads; 3395 3396 /* Default config value is 0 to use PMD specific overhead */ 3397 if (port->dev_conf.rxmode.max_rx_pkt_len == 0) 3398 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead; 3399 3400 if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) { 3401 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3402 on = false; 3403 } else { 3404 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3405 printf("Frame size (%u) is not supported by port %u\n", 3406 port->dev_conf.rxmode.max_rx_pkt_len, 3407 portid); 3408 return -1; 3409 } 3410 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3411 on = true; 3412 } 3413 3414 if (rx_offloads != port->dev_conf.rxmode.offloads) { 3415 uint16_t qid; 3416 3417 port->dev_conf.rxmode.offloads = rx_offloads; 3418 3419 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */ 3420 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) { 3421 if (on) 3422 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3423 else 3424 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3425 } 3426 } 3427 3428 /* If JUMBO_FRAME is set MTU conversion done by ethdev layer, 3429 * if unset do it here 3430 */ 3431 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3432 ret = rte_eth_dev_set_mtu(portid, 3433 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead); 3434 if (ret) 3435 printf("Failed to set MTU to %u for port %u\n", 3436 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead, 3437 portid); 3438 } 3439 3440 return 0; 3441 } 3442 3443 void 3444 init_port_config(void) 3445 { 3446 portid_t pid; 3447 struct rte_port *port; 3448 int ret; 3449 3450 RTE_ETH_FOREACH_DEV(pid) { 3451 port = &ports[pid]; 3452 port->dev_conf.fdir_conf = fdir_conf; 3453 3454 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3455 if (ret != 0) 3456 return; 3457 3458 if (nb_rxq > 1) { 3459 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3460 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3461 rss_hf & port->dev_info.flow_type_rss_offloads; 3462 } else { 3463 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3464 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3465 } 3466 3467 if (port->dcb_flag == 0) { 3468 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3469 port->dev_conf.rxmode.mq_mode = 3470 (enum rte_eth_rx_mq_mode) 3471 (rx_mq_mode & ETH_MQ_RX_RSS); 3472 else 3473 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3474 } 3475 3476 rxtx_port_config(port); 3477 3478 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3479 if (ret != 0) 3480 return; 3481 3482 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3483 rte_pmd_ixgbe_bypass_init(pid); 3484 #endif 3485 3486 if (lsc_interrupt && 3487 (rte_eth_devices[pid].data->dev_flags & 3488 RTE_ETH_DEV_INTR_LSC)) 3489 port->dev_conf.intr_conf.lsc = 1; 3490 if (rmv_interrupt && 3491 (rte_eth_devices[pid].data->dev_flags & 3492 RTE_ETH_DEV_INTR_RMV)) 3493 port->dev_conf.intr_conf.rmv = 1; 3494 } 3495 } 3496 3497 void set_port_slave_flag(portid_t slave_pid) 3498 { 3499 struct rte_port *port; 3500 3501 port = &ports[slave_pid]; 3502 port->slave_flag = 1; 3503 } 3504 3505 void clear_port_slave_flag(portid_t slave_pid) 3506 { 3507 struct rte_port *port; 3508 3509 port = &ports[slave_pid]; 3510 port->slave_flag = 0; 3511 } 3512 3513 uint8_t port_is_bonding_slave(portid_t slave_pid) 3514 { 3515 struct rte_port *port; 3516 3517 port = &ports[slave_pid]; 3518 if ((rte_eth_devices[slave_pid].data->dev_flags & 3519 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3520 return 1; 3521 return 0; 3522 } 3523 3524 const uint16_t vlan_tags[] = { 3525 0, 1, 2, 3, 4, 5, 6, 7, 3526 8, 9, 10, 11, 12, 13, 14, 15, 3527 16, 17, 18, 19, 20, 21, 22, 23, 3528 24, 25, 26, 27, 28, 29, 30, 31 3529 }; 3530 3531 static int 3532 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3533 enum dcb_mode_enable dcb_mode, 3534 enum rte_eth_nb_tcs num_tcs, 3535 uint8_t pfc_en) 3536 { 3537 uint8_t i; 3538 int32_t rc; 3539 struct rte_eth_rss_conf rss_conf; 3540 3541 /* 3542 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3543 * given above, and the number of traffic classes available for use. 3544 */ 3545 if (dcb_mode == DCB_VT_ENABLED) { 3546 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3547 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3548 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3549 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3550 3551 /* VMDQ+DCB RX and TX configurations */ 3552 vmdq_rx_conf->enable_default_pool = 0; 3553 vmdq_rx_conf->default_pool = 0; 3554 vmdq_rx_conf->nb_queue_pools = 3555 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3556 vmdq_tx_conf->nb_queue_pools = 3557 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3558 3559 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3560 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3561 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3562 vmdq_rx_conf->pool_map[i].pools = 3563 1 << (i % vmdq_rx_conf->nb_queue_pools); 3564 } 3565 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3566 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3567 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3568 } 3569 3570 /* set DCB mode of RX and TX of multiple queues */ 3571 eth_conf->rxmode.mq_mode = 3572 (enum rte_eth_rx_mq_mode) 3573 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3574 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3575 } else { 3576 struct rte_eth_dcb_rx_conf *rx_conf = 3577 ð_conf->rx_adv_conf.dcb_rx_conf; 3578 struct rte_eth_dcb_tx_conf *tx_conf = 3579 ð_conf->tx_adv_conf.dcb_tx_conf; 3580 3581 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3582 3583 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3584 if (rc != 0) 3585 return rc; 3586 3587 rx_conf->nb_tcs = num_tcs; 3588 tx_conf->nb_tcs = num_tcs; 3589 3590 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3591 rx_conf->dcb_tc[i] = i % num_tcs; 3592 tx_conf->dcb_tc[i] = i % num_tcs; 3593 } 3594 3595 eth_conf->rxmode.mq_mode = 3596 (enum rte_eth_rx_mq_mode) 3597 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3598 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3599 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3600 } 3601 3602 if (pfc_en) 3603 eth_conf->dcb_capability_en = 3604 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3605 else 3606 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3607 3608 return 0; 3609 } 3610 3611 int 3612 init_port_dcb_config(portid_t pid, 3613 enum dcb_mode_enable dcb_mode, 3614 enum rte_eth_nb_tcs num_tcs, 3615 uint8_t pfc_en) 3616 { 3617 struct rte_eth_conf port_conf; 3618 struct rte_port *rte_port; 3619 int retval; 3620 uint16_t i; 3621 3622 rte_port = &ports[pid]; 3623 3624 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3625 /* Enter DCB configuration status */ 3626 dcb_config = 1; 3627 3628 port_conf.rxmode = rte_port->dev_conf.rxmode; 3629 port_conf.txmode = rte_port->dev_conf.txmode; 3630 3631 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3632 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3633 if (retval < 0) 3634 return retval; 3635 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3636 3637 /* re-configure the device . */ 3638 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3639 if (retval < 0) 3640 return retval; 3641 3642 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3643 if (retval != 0) 3644 return retval; 3645 3646 /* If dev_info.vmdq_pool_base is greater than 0, 3647 * the queue id of vmdq pools is started after pf queues. 3648 */ 3649 if (dcb_mode == DCB_VT_ENABLED && 3650 rte_port->dev_info.vmdq_pool_base > 0) { 3651 printf("VMDQ_DCB multi-queue mode is nonsensical" 3652 " for port %d.", pid); 3653 return -1; 3654 } 3655 3656 /* Assume the ports in testpmd have the same dcb capability 3657 * and has the same number of rxq and txq in dcb mode 3658 */ 3659 if (dcb_mode == DCB_VT_ENABLED) { 3660 if (rte_port->dev_info.max_vfs > 0) { 3661 nb_rxq = rte_port->dev_info.nb_rx_queues; 3662 nb_txq = rte_port->dev_info.nb_tx_queues; 3663 } else { 3664 nb_rxq = rte_port->dev_info.max_rx_queues; 3665 nb_txq = rte_port->dev_info.max_tx_queues; 3666 } 3667 } else { 3668 /*if vt is disabled, use all pf queues */ 3669 if (rte_port->dev_info.vmdq_pool_base == 0) { 3670 nb_rxq = rte_port->dev_info.max_rx_queues; 3671 nb_txq = rte_port->dev_info.max_tx_queues; 3672 } else { 3673 nb_rxq = (queueid_t)num_tcs; 3674 nb_txq = (queueid_t)num_tcs; 3675 3676 } 3677 } 3678 rx_free_thresh = 64; 3679 3680 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3681 3682 rxtx_port_config(rte_port); 3683 /* VLAN filter */ 3684 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3685 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3686 rx_vft_set(pid, vlan_tags[i], 1); 3687 3688 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3689 if (retval != 0) 3690 return retval; 3691 3692 rte_port->dcb_flag = 1; 3693 3694 return 0; 3695 } 3696 3697 static void 3698 init_port(void) 3699 { 3700 int i; 3701 3702 /* Configuration of Ethernet ports. */ 3703 ports = rte_zmalloc("testpmd: ports", 3704 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3705 RTE_CACHE_LINE_SIZE); 3706 if (ports == NULL) { 3707 rte_exit(EXIT_FAILURE, 3708 "rte_zmalloc(%d struct rte_port) failed\n", 3709 RTE_MAX_ETHPORTS); 3710 } 3711 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3712 LIST_INIT(&ports[i].flow_tunnel_list); 3713 /* Initialize ports NUMA structures */ 3714 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3715 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3716 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3717 } 3718 3719 static void 3720 force_quit(void) 3721 { 3722 pmd_test_exit(); 3723 prompt_exit(); 3724 } 3725 3726 static void 3727 print_stats(void) 3728 { 3729 uint8_t i; 3730 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3731 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3732 3733 /* Clear screen and move to top left */ 3734 printf("%s%s", clr, top_left); 3735 3736 printf("\nPort statistics ===================================="); 3737 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3738 nic_stats_display(fwd_ports_ids[i]); 3739 3740 fflush(stdout); 3741 } 3742 3743 static void 3744 signal_handler(int signum) 3745 { 3746 if (signum == SIGINT || signum == SIGTERM) { 3747 printf("\nSignal %d received, preparing to exit...\n", 3748 signum); 3749 #ifdef RTE_LIB_PDUMP 3750 /* uninitialize packet capture framework */ 3751 rte_pdump_uninit(); 3752 #endif 3753 #ifdef RTE_LIB_LATENCYSTATS 3754 if (latencystats_enabled != 0) 3755 rte_latencystats_uninit(); 3756 #endif 3757 force_quit(); 3758 /* Set flag to indicate the force termination. */ 3759 f_quit = 1; 3760 /* exit with the expected status */ 3761 signal(signum, SIG_DFL); 3762 kill(getpid(), signum); 3763 } 3764 } 3765 3766 int 3767 main(int argc, char** argv) 3768 { 3769 int diag; 3770 portid_t port_id; 3771 uint16_t count; 3772 int ret; 3773 3774 signal(SIGINT, signal_handler); 3775 signal(SIGTERM, signal_handler); 3776 3777 testpmd_logtype = rte_log_register("testpmd"); 3778 if (testpmd_logtype < 0) 3779 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3780 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3781 3782 diag = rte_eal_init(argc, argv); 3783 if (diag < 0) 3784 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3785 rte_strerror(rte_errno)); 3786 3787 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3788 rte_exit(EXIT_FAILURE, 3789 "Secondary process type not supported.\n"); 3790 3791 ret = register_eth_event_callback(); 3792 if (ret != 0) 3793 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3794 3795 #ifdef RTE_LIB_PDUMP 3796 /* initialize packet capture framework */ 3797 rte_pdump_init(); 3798 #endif 3799 3800 count = 0; 3801 RTE_ETH_FOREACH_DEV(port_id) { 3802 ports_ids[count] = port_id; 3803 count++; 3804 } 3805 nb_ports = (portid_t) count; 3806 if (nb_ports == 0) 3807 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3808 3809 /* allocate port structures, and init them */ 3810 init_port(); 3811 3812 set_def_fwd_config(); 3813 if (nb_lcores == 0) 3814 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3815 "Check the core mask argument\n"); 3816 3817 /* Bitrate/latency stats disabled by default */ 3818 #ifdef RTE_LIB_BITRATESTATS 3819 bitrate_enabled = 0; 3820 #endif 3821 #ifdef RTE_LIB_LATENCYSTATS 3822 latencystats_enabled = 0; 3823 #endif 3824 3825 /* on FreeBSD, mlockall() is disabled by default */ 3826 #ifdef RTE_EXEC_ENV_FREEBSD 3827 do_mlockall = 0; 3828 #else 3829 do_mlockall = 1; 3830 #endif 3831 3832 argc -= diag; 3833 argv += diag; 3834 if (argc > 1) 3835 launch_args_parse(argc, argv); 3836 3837 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3838 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3839 strerror(errno)); 3840 } 3841 3842 if (tx_first && interactive) 3843 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3844 "interactive mode.\n"); 3845 3846 if (tx_first && lsc_interrupt) { 3847 printf("Warning: lsc_interrupt needs to be off when " 3848 " using tx_first. Disabling.\n"); 3849 lsc_interrupt = 0; 3850 } 3851 3852 if (!nb_rxq && !nb_txq) 3853 printf("Warning: Either rx or tx queues should be non-zero\n"); 3854 3855 if (nb_rxq > 1 && nb_rxq > nb_txq) 3856 printf("Warning: nb_rxq=%d enables RSS configuration, " 3857 "but nb_txq=%d will prevent to fully test it.\n", 3858 nb_rxq, nb_txq); 3859 3860 init_config(); 3861 3862 if (hot_plug) { 3863 ret = rte_dev_hotplug_handle_enable(); 3864 if (ret) { 3865 RTE_LOG(ERR, EAL, 3866 "fail to enable hotplug handling."); 3867 return -1; 3868 } 3869 3870 ret = rte_dev_event_monitor_start(); 3871 if (ret) { 3872 RTE_LOG(ERR, EAL, 3873 "fail to start device event monitoring."); 3874 return -1; 3875 } 3876 3877 ret = rte_dev_event_callback_register(NULL, 3878 dev_event_callback, NULL); 3879 if (ret) { 3880 RTE_LOG(ERR, EAL, 3881 "fail to register device event callback\n"); 3882 return -1; 3883 } 3884 } 3885 3886 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3887 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3888 3889 /* set all ports to promiscuous mode by default */ 3890 RTE_ETH_FOREACH_DEV(port_id) { 3891 ret = rte_eth_promiscuous_enable(port_id); 3892 if (ret != 0) 3893 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3894 port_id, rte_strerror(-ret)); 3895 } 3896 3897 /* Init metrics library */ 3898 rte_metrics_init(rte_socket_id()); 3899 3900 #ifdef RTE_LIB_LATENCYSTATS 3901 if (latencystats_enabled != 0) { 3902 int ret = rte_latencystats_init(1, NULL); 3903 if (ret) 3904 printf("Warning: latencystats init()" 3905 " returned error %d\n", ret); 3906 printf("Latencystats running on lcore %d\n", 3907 latencystats_lcore_id); 3908 } 3909 #endif 3910 3911 /* Setup bitrate stats */ 3912 #ifdef RTE_LIB_BITRATESTATS 3913 if (bitrate_enabled != 0) { 3914 bitrate_data = rte_stats_bitrate_create(); 3915 if (bitrate_data == NULL) 3916 rte_exit(EXIT_FAILURE, 3917 "Could not allocate bitrate data.\n"); 3918 rte_stats_bitrate_reg(bitrate_data); 3919 } 3920 #endif 3921 3922 #ifdef RTE_LIB_CMDLINE 3923 if (strlen(cmdline_filename) != 0) 3924 cmdline_read_from_file(cmdline_filename); 3925 3926 if (interactive == 1) { 3927 if (auto_start) { 3928 printf("Start automatic packet forwarding\n"); 3929 start_packet_forwarding(0); 3930 } 3931 prompt(); 3932 pmd_test_exit(); 3933 } else 3934 #endif 3935 { 3936 char c; 3937 int rc; 3938 3939 f_quit = 0; 3940 3941 printf("No commandline core given, start packet forwarding\n"); 3942 start_packet_forwarding(tx_first); 3943 if (stats_period != 0) { 3944 uint64_t prev_time = 0, cur_time, diff_time = 0; 3945 uint64_t timer_period; 3946 3947 /* Convert to number of cycles */ 3948 timer_period = stats_period * rte_get_timer_hz(); 3949 3950 while (f_quit == 0) { 3951 cur_time = rte_get_timer_cycles(); 3952 diff_time += cur_time - prev_time; 3953 3954 if (diff_time >= timer_period) { 3955 print_stats(); 3956 /* Reset the timer */ 3957 diff_time = 0; 3958 } 3959 /* Sleep to avoid unnecessary checks */ 3960 prev_time = cur_time; 3961 sleep(1); 3962 } 3963 } 3964 3965 printf("Press enter to exit\n"); 3966 rc = read(0, &c, 1); 3967 pmd_test_exit(); 3968 if (rc < 0) 3969 return 1; 3970 } 3971 3972 ret = rte_eal_cleanup(); 3973 if (ret != 0) 3974 rte_exit(EXIT_FAILURE, 3975 "EAL cleanup failed: %s\n", strerror(-ret)); 3976 3977 return EXIT_SUCCESS; 3978 } 3979