1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIB_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIB_LATENCYSTATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use main core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 220 221 /* 222 * Configuration of packet segments used by the "txonly" processing engine. 223 */ 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 226 TXONLY_DEF_PACKET_LEN, 227 }; 228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 229 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 231 /**< Split policy for packets to TX. */ 232 233 uint8_t txonly_multi_flow; 234 /**< Whether multiple flows are generated in TXONLY mode. */ 235 236 uint32_t tx_pkt_times_inter; 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 238 239 uint32_t tx_pkt_times_intra; 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 241 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 245 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 247 uint8_t dcb_config = 0; 248 249 /* Whether the dcb is in testing status */ 250 uint8_t dcb_test = 0; 251 252 /* 253 * Configurable number of RX/TX queues. 254 */ 255 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 256 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 257 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 258 259 /* 260 * Configurable number of RX/TX ring descriptors. 261 * Defaults are supplied by drivers via ethdev. 262 */ 263 #define RTE_TEST_RX_DESC_DEFAULT 0 264 #define RTE_TEST_TX_DESC_DEFAULT 0 265 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 266 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 267 268 #define RTE_PMD_PARAM_UNSET -1 269 /* 270 * Configurable values of RX and TX ring threshold registers. 271 */ 272 273 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 274 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 275 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 276 277 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 278 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 279 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 280 281 /* 282 * Configurable value of RX free threshold. 283 */ 284 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 285 286 /* 287 * Configurable value of RX drop enable. 288 */ 289 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 290 291 /* 292 * Configurable value of TX free threshold. 293 */ 294 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 295 296 /* 297 * Configurable value of TX RS bit threshold. 298 */ 299 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 300 301 /* 302 * Configurable value of buffered packets before sending. 303 */ 304 uint16_t noisy_tx_sw_bufsz; 305 306 /* 307 * Configurable value of packet buffer timeout. 308 */ 309 uint16_t noisy_tx_sw_buf_flush_time; 310 311 /* 312 * Configurable value for size of VNF internal memory area 313 * used for simulating noisy neighbour behaviour 314 */ 315 uint64_t noisy_lkup_mem_sz; 316 317 /* 318 * Configurable value of number of random writes done in 319 * VNF simulation memory area. 320 */ 321 uint64_t noisy_lkup_num_writes; 322 323 /* 324 * Configurable value of number of random reads done in 325 * VNF simulation memory area. 326 */ 327 uint64_t noisy_lkup_num_reads; 328 329 /* 330 * Configurable value of number of random reads/writes done in 331 * VNF simulation memory area. 332 */ 333 uint64_t noisy_lkup_num_reads_writes; 334 335 /* 336 * Receive Side Scaling (RSS) configuration. 337 */ 338 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 339 340 /* 341 * Port topology configuration 342 */ 343 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 344 345 /* 346 * Avoids to flush all the RX streams before starts forwarding. 347 */ 348 uint8_t no_flush_rx = 0; /* flush by default */ 349 350 /* 351 * Flow API isolated mode. 352 */ 353 uint8_t flow_isolate_all; 354 355 /* 356 * Avoids to check link status when starting/stopping a port. 357 */ 358 uint8_t no_link_check = 0; /* check by default */ 359 360 /* 361 * Don't automatically start all ports in interactive mode. 362 */ 363 uint8_t no_device_start = 0; 364 365 /* 366 * Enable link status change notification 367 */ 368 uint8_t lsc_interrupt = 1; /* enabled by default */ 369 370 /* 371 * Enable device removal notification. 372 */ 373 uint8_t rmv_interrupt = 1; /* enabled by default */ 374 375 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 376 377 /* After attach, port setup is called on event or by iterator */ 378 bool setup_on_probe_event = true; 379 380 /* Clear ptypes on port initialization. */ 381 uint8_t clear_ptypes = true; 382 383 /* Hairpin ports configuration mode. */ 384 uint16_t hairpin_mode; 385 386 /* Pretty printing of ethdev events */ 387 static const char * const eth_event_desc[] = { 388 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 389 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 390 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 391 [RTE_ETH_EVENT_INTR_RESET] = "reset", 392 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 393 [RTE_ETH_EVENT_IPSEC] = "IPsec", 394 [RTE_ETH_EVENT_MACSEC] = "MACsec", 395 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 396 [RTE_ETH_EVENT_NEW] = "device probed", 397 [RTE_ETH_EVENT_DESTROY] = "device released", 398 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 399 [RTE_ETH_EVENT_MAX] = NULL, 400 }; 401 402 /* 403 * Display or mask ether events 404 * Default to all events except VF_MBOX 405 */ 406 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 407 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 408 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 410 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 411 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 412 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 413 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 414 /* 415 * Decide if all memory are locked for performance. 416 */ 417 int do_mlockall = 0; 418 419 /* 420 * NIC bypass mode configuration options. 421 */ 422 423 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 424 /* The NIC bypass watchdog timeout. */ 425 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 426 #endif 427 428 429 #ifdef RTE_LIB_LATENCYSTATS 430 431 /* 432 * Set when latency stats is enabled in the commandline 433 */ 434 uint8_t latencystats_enabled; 435 436 /* 437 * Lcore ID to serive latency statistics. 438 */ 439 lcoreid_t latencystats_lcore_id = -1; 440 441 #endif 442 443 /* 444 * Ethernet device configuration. 445 */ 446 struct rte_eth_rxmode rx_mode = { 447 /* Default maximum frame length. 448 * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead" 449 * in init_config(). 450 */ 451 .max_rx_pkt_len = 0, 452 }; 453 454 struct rte_eth_txmode tx_mode = { 455 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 456 }; 457 458 struct rte_fdir_conf fdir_conf = { 459 .mode = RTE_FDIR_MODE_NONE, 460 .pballoc = RTE_FDIR_PBALLOC_64K, 461 .status = RTE_FDIR_REPORT_STATUS, 462 .mask = { 463 .vlan_tci_mask = 0xFFEF, 464 .ipv4_mask = { 465 .src_ip = 0xFFFFFFFF, 466 .dst_ip = 0xFFFFFFFF, 467 }, 468 .ipv6_mask = { 469 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 470 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 471 }, 472 .src_port_mask = 0xFFFF, 473 .dst_port_mask = 0xFFFF, 474 .mac_addr_byte_mask = 0xFF, 475 .tunnel_type_mask = 1, 476 .tunnel_id_mask = 0xFFFFFFFF, 477 }, 478 .drop_queue = 127, 479 }; 480 481 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 482 483 /* 484 * Display zero values by default for xstats 485 */ 486 uint8_t xstats_hide_zero; 487 488 /* 489 * Measure of CPU cycles disabled by default 490 */ 491 uint8_t record_core_cycles; 492 493 /* 494 * Display of RX and TX bursts disabled by default 495 */ 496 uint8_t record_burst_stats; 497 498 unsigned int num_sockets = 0; 499 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 500 501 #ifdef RTE_LIB_BITRATESTATS 502 /* Bitrate statistics */ 503 struct rte_stats_bitrates *bitrate_data; 504 lcoreid_t bitrate_lcore_id; 505 uint8_t bitrate_enabled; 506 #endif 507 508 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 509 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 510 511 /* 512 * hexadecimal bitmask of RX mq mode can be enabled. 513 */ 514 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 515 516 /* 517 * Used to set forced link speed 518 */ 519 uint32_t eth_link_speed; 520 521 /* Forward function declarations */ 522 static void setup_attached_port(portid_t pi); 523 static void check_all_ports_link_status(uint32_t port_mask); 524 static int eth_event_callback(portid_t port_id, 525 enum rte_eth_event_type type, 526 void *param, void *ret_param); 527 static void dev_event_callback(const char *device_name, 528 enum rte_dev_event_type type, 529 void *param); 530 531 /* 532 * Check if all the ports are started. 533 * If yes, return positive value. If not, return zero. 534 */ 535 static int all_ports_started(void); 536 537 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 538 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 539 540 /* Holds the registered mbuf dynamic flags names. */ 541 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 542 543 /* 544 * Helper function to check if socket is already discovered. 545 * If yes, return positive value. If not, return zero. 546 */ 547 int 548 new_socket_id(unsigned int socket_id) 549 { 550 unsigned int i; 551 552 for (i = 0; i < num_sockets; i++) { 553 if (socket_ids[i] == socket_id) 554 return 0; 555 } 556 return 1; 557 } 558 559 /* 560 * Setup default configuration. 561 */ 562 static void 563 set_default_fwd_lcores_config(void) 564 { 565 unsigned int i; 566 unsigned int nb_lc; 567 unsigned int sock_num; 568 569 nb_lc = 0; 570 for (i = 0; i < RTE_MAX_LCORE; i++) { 571 if (!rte_lcore_is_enabled(i)) 572 continue; 573 sock_num = rte_lcore_to_socket_id(i); 574 if (new_socket_id(sock_num)) { 575 if (num_sockets >= RTE_MAX_NUMA_NODES) { 576 rte_exit(EXIT_FAILURE, 577 "Total sockets greater than %u\n", 578 RTE_MAX_NUMA_NODES); 579 } 580 socket_ids[num_sockets++] = sock_num; 581 } 582 if (i == rte_get_main_lcore()) 583 continue; 584 fwd_lcores_cpuids[nb_lc++] = i; 585 } 586 nb_lcores = (lcoreid_t) nb_lc; 587 nb_cfg_lcores = nb_lcores; 588 nb_fwd_lcores = 1; 589 } 590 591 static void 592 set_def_peer_eth_addrs(void) 593 { 594 portid_t i; 595 596 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 597 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 598 peer_eth_addrs[i].addr_bytes[5] = i; 599 } 600 } 601 602 static void 603 set_default_fwd_ports_config(void) 604 { 605 portid_t pt_id; 606 int i = 0; 607 608 RTE_ETH_FOREACH_DEV(pt_id) { 609 fwd_ports_ids[i++] = pt_id; 610 611 /* Update sockets info according to the attached device */ 612 int socket_id = rte_eth_dev_socket_id(pt_id); 613 if (socket_id >= 0 && new_socket_id(socket_id)) { 614 if (num_sockets >= RTE_MAX_NUMA_NODES) { 615 rte_exit(EXIT_FAILURE, 616 "Total sockets greater than %u\n", 617 RTE_MAX_NUMA_NODES); 618 } 619 socket_ids[num_sockets++] = socket_id; 620 } 621 } 622 623 nb_cfg_ports = nb_ports; 624 nb_fwd_ports = nb_ports; 625 } 626 627 void 628 set_def_fwd_config(void) 629 { 630 set_default_fwd_lcores_config(); 631 set_def_peer_eth_addrs(); 632 set_default_fwd_ports_config(); 633 } 634 635 /* extremely pessimistic estimation of memory required to create a mempool */ 636 static int 637 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 638 { 639 unsigned int n_pages, mbuf_per_pg, leftover; 640 uint64_t total_mem, mbuf_mem, obj_sz; 641 642 /* there is no good way to predict how much space the mempool will 643 * occupy because it will allocate chunks on the fly, and some of those 644 * will come from default DPDK memory while some will come from our 645 * external memory, so just assume 128MB will be enough for everyone. 646 */ 647 uint64_t hdr_mem = 128 << 20; 648 649 /* account for possible non-contiguousness */ 650 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 651 if (obj_sz > pgsz) { 652 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 653 return -1; 654 } 655 656 mbuf_per_pg = pgsz / obj_sz; 657 leftover = (nb_mbufs % mbuf_per_pg) > 0; 658 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 659 660 mbuf_mem = n_pages * pgsz; 661 662 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 663 664 if (total_mem > SIZE_MAX) { 665 TESTPMD_LOG(ERR, "Memory size too big\n"); 666 return -1; 667 } 668 *out = (size_t)total_mem; 669 670 return 0; 671 } 672 673 static int 674 pagesz_flags(uint64_t page_sz) 675 { 676 /* as per mmap() manpage, all page sizes are log2 of page size 677 * shifted by MAP_HUGE_SHIFT 678 */ 679 int log2 = rte_log2_u64(page_sz); 680 681 return (log2 << HUGE_SHIFT); 682 } 683 684 static void * 685 alloc_mem(size_t memsz, size_t pgsz, bool huge) 686 { 687 void *addr; 688 int flags; 689 690 /* allocate anonymous hugepages */ 691 flags = MAP_ANONYMOUS | MAP_PRIVATE; 692 if (huge) 693 flags |= HUGE_FLAG | pagesz_flags(pgsz); 694 695 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 696 if (addr == MAP_FAILED) 697 return NULL; 698 699 return addr; 700 } 701 702 struct extmem_param { 703 void *addr; 704 size_t len; 705 size_t pgsz; 706 rte_iova_t *iova_table; 707 unsigned int iova_table_len; 708 }; 709 710 static int 711 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 712 bool huge) 713 { 714 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 715 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 716 unsigned int cur_page, n_pages, pgsz_idx; 717 size_t mem_sz, cur_pgsz; 718 rte_iova_t *iovas = NULL; 719 void *addr; 720 int ret; 721 722 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 723 /* skip anything that is too big */ 724 if (pgsizes[pgsz_idx] > SIZE_MAX) 725 continue; 726 727 cur_pgsz = pgsizes[pgsz_idx]; 728 729 /* if we were told not to allocate hugepages, override */ 730 if (!huge) 731 cur_pgsz = sysconf(_SC_PAGESIZE); 732 733 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 734 if (ret < 0) { 735 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 736 return -1; 737 } 738 739 /* allocate our memory */ 740 addr = alloc_mem(mem_sz, cur_pgsz, huge); 741 742 /* if we couldn't allocate memory with a specified page size, 743 * that doesn't mean we can't do it with other page sizes, so 744 * try another one. 745 */ 746 if (addr == NULL) 747 continue; 748 749 /* store IOVA addresses for every page in this memory area */ 750 n_pages = mem_sz / cur_pgsz; 751 752 iovas = malloc(sizeof(*iovas) * n_pages); 753 754 if (iovas == NULL) { 755 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 756 goto fail; 757 } 758 /* lock memory if it's not huge pages */ 759 if (!huge) 760 mlock(addr, mem_sz); 761 762 /* populate IOVA addresses */ 763 for (cur_page = 0; cur_page < n_pages; cur_page++) { 764 rte_iova_t iova; 765 size_t offset; 766 void *cur; 767 768 offset = cur_pgsz * cur_page; 769 cur = RTE_PTR_ADD(addr, offset); 770 771 /* touch the page before getting its IOVA */ 772 *(volatile char *)cur = 0; 773 774 iova = rte_mem_virt2iova(cur); 775 776 iovas[cur_page] = iova; 777 } 778 779 break; 780 } 781 /* if we couldn't allocate anything */ 782 if (iovas == NULL) 783 return -1; 784 785 param->addr = addr; 786 param->len = mem_sz; 787 param->pgsz = cur_pgsz; 788 param->iova_table = iovas; 789 param->iova_table_len = n_pages; 790 791 return 0; 792 fail: 793 if (iovas) 794 free(iovas); 795 if (addr) 796 munmap(addr, mem_sz); 797 798 return -1; 799 } 800 801 static int 802 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 803 { 804 struct extmem_param param; 805 int socket_id, ret; 806 807 memset(¶m, 0, sizeof(param)); 808 809 /* check if our heap exists */ 810 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 811 if (socket_id < 0) { 812 /* create our heap */ 813 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 814 if (ret < 0) { 815 TESTPMD_LOG(ERR, "Cannot create heap\n"); 816 return -1; 817 } 818 } 819 820 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 821 if (ret < 0) { 822 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 823 return -1; 824 } 825 826 /* we now have a valid memory area, so add it to heap */ 827 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 828 param.addr, param.len, param.iova_table, 829 param.iova_table_len, param.pgsz); 830 831 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 832 833 /* not needed any more */ 834 free(param.iova_table); 835 836 if (ret < 0) { 837 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 838 munmap(param.addr, param.len); 839 return -1; 840 } 841 842 /* success */ 843 844 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 845 param.len >> 20); 846 847 return 0; 848 } 849 static void 850 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 851 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 852 { 853 uint16_t pid = 0; 854 int ret; 855 856 RTE_ETH_FOREACH_DEV(pid) { 857 struct rte_eth_dev *dev = 858 &rte_eth_devices[pid]; 859 860 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 861 memhdr->len); 862 if (ret) { 863 TESTPMD_LOG(DEBUG, 864 "unable to DMA unmap addr 0x%p " 865 "for device %s\n", 866 memhdr->addr, dev->data->name); 867 } 868 } 869 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 870 if (ret) { 871 TESTPMD_LOG(DEBUG, 872 "unable to un-register addr 0x%p\n", memhdr->addr); 873 } 874 } 875 876 static void 877 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 878 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 879 { 880 uint16_t pid = 0; 881 size_t page_size = sysconf(_SC_PAGESIZE); 882 int ret; 883 884 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 885 page_size); 886 if (ret) { 887 TESTPMD_LOG(DEBUG, 888 "unable to register addr 0x%p\n", memhdr->addr); 889 return; 890 } 891 RTE_ETH_FOREACH_DEV(pid) { 892 struct rte_eth_dev *dev = 893 &rte_eth_devices[pid]; 894 895 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 896 memhdr->len); 897 if (ret) { 898 TESTPMD_LOG(DEBUG, 899 "unable to DMA map addr 0x%p " 900 "for device %s\n", 901 memhdr->addr, dev->data->name); 902 } 903 } 904 } 905 906 static unsigned int 907 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 908 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 909 { 910 struct rte_pktmbuf_extmem *xmem; 911 unsigned int ext_num, zone_num, elt_num; 912 uint16_t elt_size; 913 914 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 915 elt_num = EXTBUF_ZONE_SIZE / elt_size; 916 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 917 918 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 919 if (xmem == NULL) { 920 TESTPMD_LOG(ERR, "Cannot allocate memory for " 921 "external buffer descriptors\n"); 922 *ext_mem = NULL; 923 return 0; 924 } 925 for (ext_num = 0; ext_num < zone_num; ext_num++) { 926 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 927 const struct rte_memzone *mz; 928 char mz_name[RTE_MEMZONE_NAMESIZE]; 929 int ret; 930 931 ret = snprintf(mz_name, sizeof(mz_name), 932 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 933 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 934 errno = ENAMETOOLONG; 935 ext_num = 0; 936 break; 937 } 938 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 939 socket_id, 940 RTE_MEMZONE_IOVA_CONTIG | 941 RTE_MEMZONE_1GB | 942 RTE_MEMZONE_SIZE_HINT_ONLY, 943 EXTBUF_ZONE_SIZE); 944 if (mz == NULL) { 945 /* 946 * The caller exits on external buffer creation 947 * error, so there is no need to free memzones. 948 */ 949 errno = ENOMEM; 950 ext_num = 0; 951 break; 952 } 953 xseg->buf_ptr = mz->addr; 954 xseg->buf_iova = mz->iova; 955 xseg->buf_len = EXTBUF_ZONE_SIZE; 956 xseg->elt_size = elt_size; 957 } 958 if (ext_num == 0 && xmem != NULL) { 959 free(xmem); 960 xmem = NULL; 961 } 962 *ext_mem = xmem; 963 return ext_num; 964 } 965 966 /* 967 * Configuration initialisation done once at init time. 968 */ 969 static struct rte_mempool * 970 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 971 unsigned int socket_id, uint16_t size_idx) 972 { 973 char pool_name[RTE_MEMPOOL_NAMESIZE]; 974 struct rte_mempool *rte_mp = NULL; 975 uint32_t mb_size; 976 977 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 978 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 979 980 TESTPMD_LOG(INFO, 981 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 982 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 983 984 switch (mp_alloc_type) { 985 case MP_ALLOC_NATIVE: 986 { 987 /* wrapper to rte_mempool_create() */ 988 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 989 rte_mbuf_best_mempool_ops()); 990 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 991 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 992 break; 993 } 994 case MP_ALLOC_ANON: 995 { 996 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 997 mb_size, (unsigned int) mb_mempool_cache, 998 sizeof(struct rte_pktmbuf_pool_private), 999 socket_id, mempool_flags); 1000 if (rte_mp == NULL) 1001 goto err; 1002 1003 if (rte_mempool_populate_anon(rte_mp) == 0) { 1004 rte_mempool_free(rte_mp); 1005 rte_mp = NULL; 1006 goto err; 1007 } 1008 rte_pktmbuf_pool_init(rte_mp, NULL); 1009 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1010 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1011 break; 1012 } 1013 case MP_ALLOC_XMEM: 1014 case MP_ALLOC_XMEM_HUGE: 1015 { 1016 int heap_socket; 1017 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1018 1019 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1020 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1021 1022 heap_socket = 1023 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1024 if (heap_socket < 0) 1025 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1026 1027 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1028 rte_mbuf_best_mempool_ops()); 1029 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1030 mb_mempool_cache, 0, mbuf_seg_size, 1031 heap_socket); 1032 break; 1033 } 1034 case MP_ALLOC_XBUF: 1035 { 1036 struct rte_pktmbuf_extmem *ext_mem; 1037 unsigned int ext_num; 1038 1039 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1040 socket_id, pool_name, &ext_mem); 1041 if (ext_num == 0) 1042 rte_exit(EXIT_FAILURE, 1043 "Can't create pinned data buffers\n"); 1044 1045 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1046 rte_mbuf_best_mempool_ops()); 1047 rte_mp = rte_pktmbuf_pool_create_extbuf 1048 (pool_name, nb_mbuf, mb_mempool_cache, 1049 0, mbuf_seg_size, socket_id, 1050 ext_mem, ext_num); 1051 free(ext_mem); 1052 break; 1053 } 1054 default: 1055 { 1056 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1057 } 1058 } 1059 1060 err: 1061 if (rte_mp == NULL) { 1062 rte_exit(EXIT_FAILURE, 1063 "Creation of mbuf pool for socket %u failed: %s\n", 1064 socket_id, rte_strerror(rte_errno)); 1065 } else if (verbose_level > 0) { 1066 rte_mempool_dump(stdout, rte_mp); 1067 } 1068 return rte_mp; 1069 } 1070 1071 /* 1072 * Check given socket id is valid or not with NUMA mode, 1073 * if valid, return 0, else return -1 1074 */ 1075 static int 1076 check_socket_id(const unsigned int socket_id) 1077 { 1078 static int warning_once = 0; 1079 1080 if (new_socket_id(socket_id)) { 1081 if (!warning_once && numa_support) 1082 printf("Warning: NUMA should be configured manually by" 1083 " using --port-numa-config and" 1084 " --ring-numa-config parameters along with" 1085 " --numa.\n"); 1086 warning_once = 1; 1087 return -1; 1088 } 1089 return 0; 1090 } 1091 1092 /* 1093 * Get the allowed maximum number of RX queues. 1094 * *pid return the port id which has minimal value of 1095 * max_rx_queues in all ports. 1096 */ 1097 queueid_t 1098 get_allowed_max_nb_rxq(portid_t *pid) 1099 { 1100 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1101 bool max_rxq_valid = false; 1102 portid_t pi; 1103 struct rte_eth_dev_info dev_info; 1104 1105 RTE_ETH_FOREACH_DEV(pi) { 1106 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1107 continue; 1108 1109 max_rxq_valid = true; 1110 if (dev_info.max_rx_queues < allowed_max_rxq) { 1111 allowed_max_rxq = dev_info.max_rx_queues; 1112 *pid = pi; 1113 } 1114 } 1115 return max_rxq_valid ? allowed_max_rxq : 0; 1116 } 1117 1118 /* 1119 * Check input rxq is valid or not. 1120 * If input rxq is not greater than any of maximum number 1121 * of RX queues of all ports, it is valid. 1122 * if valid, return 0, else return -1 1123 */ 1124 int 1125 check_nb_rxq(queueid_t rxq) 1126 { 1127 queueid_t allowed_max_rxq; 1128 portid_t pid = 0; 1129 1130 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1131 if (rxq > allowed_max_rxq) { 1132 printf("Fail: input rxq (%u) can't be greater " 1133 "than max_rx_queues (%u) of port %u\n", 1134 rxq, 1135 allowed_max_rxq, 1136 pid); 1137 return -1; 1138 } 1139 return 0; 1140 } 1141 1142 /* 1143 * Get the allowed maximum number of TX queues. 1144 * *pid return the port id which has minimal value of 1145 * max_tx_queues in all ports. 1146 */ 1147 queueid_t 1148 get_allowed_max_nb_txq(portid_t *pid) 1149 { 1150 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1151 bool max_txq_valid = false; 1152 portid_t pi; 1153 struct rte_eth_dev_info dev_info; 1154 1155 RTE_ETH_FOREACH_DEV(pi) { 1156 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1157 continue; 1158 1159 max_txq_valid = true; 1160 if (dev_info.max_tx_queues < allowed_max_txq) { 1161 allowed_max_txq = dev_info.max_tx_queues; 1162 *pid = pi; 1163 } 1164 } 1165 return max_txq_valid ? allowed_max_txq : 0; 1166 } 1167 1168 /* 1169 * Check input txq is valid or not. 1170 * If input txq is not greater than any of maximum number 1171 * of TX queues of all ports, it is valid. 1172 * if valid, return 0, else return -1 1173 */ 1174 int 1175 check_nb_txq(queueid_t txq) 1176 { 1177 queueid_t allowed_max_txq; 1178 portid_t pid = 0; 1179 1180 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1181 if (txq > allowed_max_txq) { 1182 printf("Fail: input txq (%u) can't be greater " 1183 "than max_tx_queues (%u) of port %u\n", 1184 txq, 1185 allowed_max_txq, 1186 pid); 1187 return -1; 1188 } 1189 return 0; 1190 } 1191 1192 /* 1193 * Get the allowed maximum number of RXDs of every rx queue. 1194 * *pid return the port id which has minimal value of 1195 * max_rxd in all queues of all ports. 1196 */ 1197 static uint16_t 1198 get_allowed_max_nb_rxd(portid_t *pid) 1199 { 1200 uint16_t allowed_max_rxd = UINT16_MAX; 1201 portid_t pi; 1202 struct rte_eth_dev_info dev_info; 1203 1204 RTE_ETH_FOREACH_DEV(pi) { 1205 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1206 continue; 1207 1208 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1209 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1210 *pid = pi; 1211 } 1212 } 1213 return allowed_max_rxd; 1214 } 1215 1216 /* 1217 * Get the allowed minimal number of RXDs of every rx queue. 1218 * *pid return the port id which has minimal value of 1219 * min_rxd in all queues of all ports. 1220 */ 1221 static uint16_t 1222 get_allowed_min_nb_rxd(portid_t *pid) 1223 { 1224 uint16_t allowed_min_rxd = 0; 1225 portid_t pi; 1226 struct rte_eth_dev_info dev_info; 1227 1228 RTE_ETH_FOREACH_DEV(pi) { 1229 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1230 continue; 1231 1232 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1233 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1234 *pid = pi; 1235 } 1236 } 1237 1238 return allowed_min_rxd; 1239 } 1240 1241 /* 1242 * Check input rxd is valid or not. 1243 * If input rxd is not greater than any of maximum number 1244 * of RXDs of every Rx queues and is not less than any of 1245 * minimal number of RXDs of every Rx queues, it is valid. 1246 * if valid, return 0, else return -1 1247 */ 1248 int 1249 check_nb_rxd(queueid_t rxd) 1250 { 1251 uint16_t allowed_max_rxd; 1252 uint16_t allowed_min_rxd; 1253 portid_t pid = 0; 1254 1255 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1256 if (rxd > allowed_max_rxd) { 1257 printf("Fail: input rxd (%u) can't be greater " 1258 "than max_rxds (%u) of port %u\n", 1259 rxd, 1260 allowed_max_rxd, 1261 pid); 1262 return -1; 1263 } 1264 1265 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1266 if (rxd < allowed_min_rxd) { 1267 printf("Fail: input rxd (%u) can't be less " 1268 "than min_rxds (%u) of port %u\n", 1269 rxd, 1270 allowed_min_rxd, 1271 pid); 1272 return -1; 1273 } 1274 1275 return 0; 1276 } 1277 1278 /* 1279 * Get the allowed maximum number of TXDs of every rx queues. 1280 * *pid return the port id which has minimal value of 1281 * max_txd in every tx queue. 1282 */ 1283 static uint16_t 1284 get_allowed_max_nb_txd(portid_t *pid) 1285 { 1286 uint16_t allowed_max_txd = UINT16_MAX; 1287 portid_t pi; 1288 struct rte_eth_dev_info dev_info; 1289 1290 RTE_ETH_FOREACH_DEV(pi) { 1291 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1292 continue; 1293 1294 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1295 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1296 *pid = pi; 1297 } 1298 } 1299 return allowed_max_txd; 1300 } 1301 1302 /* 1303 * Get the allowed maximum number of TXDs of every tx queues. 1304 * *pid return the port id which has minimal value of 1305 * min_txd in every tx queue. 1306 */ 1307 static uint16_t 1308 get_allowed_min_nb_txd(portid_t *pid) 1309 { 1310 uint16_t allowed_min_txd = 0; 1311 portid_t pi; 1312 struct rte_eth_dev_info dev_info; 1313 1314 RTE_ETH_FOREACH_DEV(pi) { 1315 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1316 continue; 1317 1318 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1319 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1320 *pid = pi; 1321 } 1322 } 1323 1324 return allowed_min_txd; 1325 } 1326 1327 /* 1328 * Check input txd is valid or not. 1329 * If input txd is not greater than any of maximum number 1330 * of TXDs of every Rx queues, it is valid. 1331 * if valid, return 0, else return -1 1332 */ 1333 int 1334 check_nb_txd(queueid_t txd) 1335 { 1336 uint16_t allowed_max_txd; 1337 uint16_t allowed_min_txd; 1338 portid_t pid = 0; 1339 1340 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1341 if (txd > allowed_max_txd) { 1342 printf("Fail: input txd (%u) can't be greater " 1343 "than max_txds (%u) of port %u\n", 1344 txd, 1345 allowed_max_txd, 1346 pid); 1347 return -1; 1348 } 1349 1350 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1351 if (txd < allowed_min_txd) { 1352 printf("Fail: input txd (%u) can't be less " 1353 "than min_txds (%u) of port %u\n", 1354 txd, 1355 allowed_min_txd, 1356 pid); 1357 return -1; 1358 } 1359 return 0; 1360 } 1361 1362 1363 /* 1364 * Get the allowed maximum number of hairpin queues. 1365 * *pid return the port id which has minimal value of 1366 * max_hairpin_queues in all ports. 1367 */ 1368 queueid_t 1369 get_allowed_max_nb_hairpinq(portid_t *pid) 1370 { 1371 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1372 portid_t pi; 1373 struct rte_eth_hairpin_cap cap; 1374 1375 RTE_ETH_FOREACH_DEV(pi) { 1376 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1377 *pid = pi; 1378 return 0; 1379 } 1380 if (cap.max_nb_queues < allowed_max_hairpinq) { 1381 allowed_max_hairpinq = cap.max_nb_queues; 1382 *pid = pi; 1383 } 1384 } 1385 return allowed_max_hairpinq; 1386 } 1387 1388 /* 1389 * Check input hairpin is valid or not. 1390 * If input hairpin is not greater than any of maximum number 1391 * of hairpin queues of all ports, it is valid. 1392 * if valid, return 0, else return -1 1393 */ 1394 int 1395 check_nb_hairpinq(queueid_t hairpinq) 1396 { 1397 queueid_t allowed_max_hairpinq; 1398 portid_t pid = 0; 1399 1400 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1401 if (hairpinq > allowed_max_hairpinq) { 1402 printf("Fail: input hairpin (%u) can't be greater " 1403 "than max_hairpin_queues (%u) of port %u\n", 1404 hairpinq, allowed_max_hairpinq, pid); 1405 return -1; 1406 } 1407 return 0; 1408 } 1409 1410 static void 1411 init_config(void) 1412 { 1413 portid_t pid; 1414 struct rte_port *port; 1415 struct rte_mempool *mbp; 1416 unsigned int nb_mbuf_per_pool; 1417 lcoreid_t lc_id; 1418 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1419 struct rte_gro_param gro_param; 1420 uint32_t gso_types; 1421 uint16_t data_size; 1422 bool warning = 0; 1423 int k; 1424 int ret; 1425 1426 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1427 1428 /* Configuration of logical cores. */ 1429 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1430 sizeof(struct fwd_lcore *) * nb_lcores, 1431 RTE_CACHE_LINE_SIZE); 1432 if (fwd_lcores == NULL) { 1433 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1434 "failed\n", nb_lcores); 1435 } 1436 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1437 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1438 sizeof(struct fwd_lcore), 1439 RTE_CACHE_LINE_SIZE); 1440 if (fwd_lcores[lc_id] == NULL) { 1441 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1442 "failed\n"); 1443 } 1444 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1445 } 1446 1447 RTE_ETH_FOREACH_DEV(pid) { 1448 port = &ports[pid]; 1449 /* Apply default TxRx configuration for all ports */ 1450 port->dev_conf.txmode = tx_mode; 1451 port->dev_conf.rxmode = rx_mode; 1452 1453 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1454 if (ret != 0) 1455 rte_exit(EXIT_FAILURE, 1456 "rte_eth_dev_info_get() failed\n"); 1457 1458 ret = update_jumbo_frame_offload(pid); 1459 if (ret != 0) 1460 printf("Updating jumbo frame offload failed for port %u\n", 1461 pid); 1462 1463 if (!(port->dev_info.tx_offload_capa & 1464 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1465 port->dev_conf.txmode.offloads &= 1466 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1467 if (numa_support) { 1468 if (port_numa[pid] != NUMA_NO_CONFIG) 1469 port_per_socket[port_numa[pid]]++; 1470 else { 1471 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1472 1473 /* 1474 * if socket_id is invalid, 1475 * set to the first available socket. 1476 */ 1477 if (check_socket_id(socket_id) < 0) 1478 socket_id = socket_ids[0]; 1479 port_per_socket[socket_id]++; 1480 } 1481 } 1482 1483 /* Apply Rx offloads configuration */ 1484 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1485 port->rx_conf[k].offloads = 1486 port->dev_conf.rxmode.offloads; 1487 /* Apply Tx offloads configuration */ 1488 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1489 port->tx_conf[k].offloads = 1490 port->dev_conf.txmode.offloads; 1491 1492 if (eth_link_speed) 1493 port->dev_conf.link_speeds = eth_link_speed; 1494 1495 /* set flag to initialize port/queue */ 1496 port->need_reconfig = 1; 1497 port->need_reconfig_queues = 1; 1498 port->tx_metadata = 0; 1499 1500 /* Check for maximum number of segments per MTU. Accordingly 1501 * update the mbuf data size. 1502 */ 1503 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1504 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1505 data_size = rx_mode.max_rx_pkt_len / 1506 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1507 1508 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1509 mbuf_data_size[0]) { 1510 mbuf_data_size[0] = data_size + 1511 RTE_PKTMBUF_HEADROOM; 1512 warning = 1; 1513 } 1514 } 1515 } 1516 1517 if (warning) 1518 TESTPMD_LOG(WARNING, 1519 "Configured mbuf size of the first segment %hu\n", 1520 mbuf_data_size[0]); 1521 /* 1522 * Create pools of mbuf. 1523 * If NUMA support is disabled, create a single pool of mbuf in 1524 * socket 0 memory by default. 1525 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1526 * 1527 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1528 * nb_txd can be configured at run time. 1529 */ 1530 if (param_total_num_mbufs) 1531 nb_mbuf_per_pool = param_total_num_mbufs; 1532 else { 1533 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1534 (nb_lcores * mb_mempool_cache) + 1535 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1536 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1537 } 1538 1539 if (numa_support) { 1540 uint8_t i, j; 1541 1542 for (i = 0; i < num_sockets; i++) 1543 for (j = 0; j < mbuf_data_size_n; j++) 1544 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1545 mbuf_pool_create(mbuf_data_size[j], 1546 nb_mbuf_per_pool, 1547 socket_ids[i], j); 1548 } else { 1549 uint8_t i; 1550 1551 for (i = 0; i < mbuf_data_size_n; i++) 1552 mempools[i] = mbuf_pool_create 1553 (mbuf_data_size[i], 1554 nb_mbuf_per_pool, 1555 socket_num == UMA_NO_CONFIG ? 1556 0 : socket_num, i); 1557 } 1558 1559 init_port_config(); 1560 1561 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1562 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1563 /* 1564 * Records which Mbuf pool to use by each logical core, if needed. 1565 */ 1566 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1567 mbp = mbuf_pool_find( 1568 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1569 1570 if (mbp == NULL) 1571 mbp = mbuf_pool_find(0, 0); 1572 fwd_lcores[lc_id]->mbp = mbp; 1573 /* initialize GSO context */ 1574 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1575 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1576 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1577 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1578 RTE_ETHER_CRC_LEN; 1579 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1580 } 1581 1582 /* Configuration of packet forwarding streams. */ 1583 if (init_fwd_streams() < 0) 1584 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1585 1586 fwd_config_setup(); 1587 1588 /* create a gro context for each lcore */ 1589 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1590 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1591 gro_param.max_item_per_flow = MAX_PKT_BURST; 1592 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1593 gro_param.socket_id = rte_lcore_to_socket_id( 1594 fwd_lcores_cpuids[lc_id]); 1595 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1596 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1597 rte_exit(EXIT_FAILURE, 1598 "rte_gro_ctx_create() failed\n"); 1599 } 1600 } 1601 } 1602 1603 1604 void 1605 reconfig(portid_t new_port_id, unsigned socket_id) 1606 { 1607 struct rte_port *port; 1608 int ret; 1609 1610 /* Reconfiguration of Ethernet ports. */ 1611 port = &ports[new_port_id]; 1612 1613 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1614 if (ret != 0) 1615 return; 1616 1617 /* set flag to initialize port/queue */ 1618 port->need_reconfig = 1; 1619 port->need_reconfig_queues = 1; 1620 port->socket_id = socket_id; 1621 1622 init_port_config(); 1623 } 1624 1625 1626 int 1627 init_fwd_streams(void) 1628 { 1629 portid_t pid; 1630 struct rte_port *port; 1631 streamid_t sm_id, nb_fwd_streams_new; 1632 queueid_t q; 1633 1634 /* set socket id according to numa or not */ 1635 RTE_ETH_FOREACH_DEV(pid) { 1636 port = &ports[pid]; 1637 if (nb_rxq > port->dev_info.max_rx_queues) { 1638 printf("Fail: nb_rxq(%d) is greater than " 1639 "max_rx_queues(%d)\n", nb_rxq, 1640 port->dev_info.max_rx_queues); 1641 return -1; 1642 } 1643 if (nb_txq > port->dev_info.max_tx_queues) { 1644 printf("Fail: nb_txq(%d) is greater than " 1645 "max_tx_queues(%d)\n", nb_txq, 1646 port->dev_info.max_tx_queues); 1647 return -1; 1648 } 1649 if (numa_support) { 1650 if (port_numa[pid] != NUMA_NO_CONFIG) 1651 port->socket_id = port_numa[pid]; 1652 else { 1653 port->socket_id = rte_eth_dev_socket_id(pid); 1654 1655 /* 1656 * if socket_id is invalid, 1657 * set to the first available socket. 1658 */ 1659 if (check_socket_id(port->socket_id) < 0) 1660 port->socket_id = socket_ids[0]; 1661 } 1662 } 1663 else { 1664 if (socket_num == UMA_NO_CONFIG) 1665 port->socket_id = 0; 1666 else 1667 port->socket_id = socket_num; 1668 } 1669 } 1670 1671 q = RTE_MAX(nb_rxq, nb_txq); 1672 if (q == 0) { 1673 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1674 return -1; 1675 } 1676 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1677 if (nb_fwd_streams_new == nb_fwd_streams) 1678 return 0; 1679 /* clear the old */ 1680 if (fwd_streams != NULL) { 1681 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1682 if (fwd_streams[sm_id] == NULL) 1683 continue; 1684 rte_free(fwd_streams[sm_id]); 1685 fwd_streams[sm_id] = NULL; 1686 } 1687 rte_free(fwd_streams); 1688 fwd_streams = NULL; 1689 } 1690 1691 /* init new */ 1692 nb_fwd_streams = nb_fwd_streams_new; 1693 if (nb_fwd_streams) { 1694 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1695 sizeof(struct fwd_stream *) * nb_fwd_streams, 1696 RTE_CACHE_LINE_SIZE); 1697 if (fwd_streams == NULL) 1698 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1699 " (struct fwd_stream *)) failed\n", 1700 nb_fwd_streams); 1701 1702 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1703 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1704 " struct fwd_stream", sizeof(struct fwd_stream), 1705 RTE_CACHE_LINE_SIZE); 1706 if (fwd_streams[sm_id] == NULL) 1707 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1708 "(struct fwd_stream) failed\n"); 1709 } 1710 } 1711 1712 return 0; 1713 } 1714 1715 static void 1716 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1717 { 1718 uint64_t total_burst, sburst; 1719 uint64_t nb_burst; 1720 uint64_t burst_stats[4]; 1721 uint16_t pktnb_stats[4]; 1722 uint16_t nb_pkt; 1723 int burst_percent[4], sburstp; 1724 int i; 1725 1726 /* 1727 * First compute the total number of packet bursts and the 1728 * two highest numbers of bursts of the same number of packets. 1729 */ 1730 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1731 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1732 1733 /* Show stats for 0 burst size always */ 1734 total_burst = pbs->pkt_burst_spread[0]; 1735 burst_stats[0] = pbs->pkt_burst_spread[0]; 1736 pktnb_stats[0] = 0; 1737 1738 /* Find the next 2 burst sizes with highest occurrences. */ 1739 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1740 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1741 1742 if (nb_burst == 0) 1743 continue; 1744 1745 total_burst += nb_burst; 1746 1747 if (nb_burst > burst_stats[1]) { 1748 burst_stats[2] = burst_stats[1]; 1749 pktnb_stats[2] = pktnb_stats[1]; 1750 burst_stats[1] = nb_burst; 1751 pktnb_stats[1] = nb_pkt; 1752 } else if (nb_burst > burst_stats[2]) { 1753 burst_stats[2] = nb_burst; 1754 pktnb_stats[2] = nb_pkt; 1755 } 1756 } 1757 if (total_burst == 0) 1758 return; 1759 1760 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1761 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1762 if (i == 3) { 1763 printf("%d%% of other]\n", 100 - sburstp); 1764 return; 1765 } 1766 1767 sburst += burst_stats[i]; 1768 if (sburst == total_burst) { 1769 printf("%d%% of %d pkts]\n", 1770 100 - sburstp, (int) pktnb_stats[i]); 1771 return; 1772 } 1773 1774 burst_percent[i] = 1775 (double)burst_stats[i] / total_burst * 100; 1776 printf("%d%% of %d pkts + ", 1777 burst_percent[i], (int) pktnb_stats[i]); 1778 sburstp += burst_percent[i]; 1779 } 1780 } 1781 1782 static void 1783 fwd_stream_stats_display(streamid_t stream_id) 1784 { 1785 struct fwd_stream *fs; 1786 static const char *fwd_top_stats_border = "-------"; 1787 1788 fs = fwd_streams[stream_id]; 1789 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1790 (fs->fwd_dropped == 0)) 1791 return; 1792 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1793 "TX Port=%2d/Queue=%2d %s\n", 1794 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1795 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1796 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1797 " TX-dropped: %-14"PRIu64, 1798 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1799 1800 /* if checksum mode */ 1801 if (cur_fwd_eng == &csum_fwd_engine) { 1802 printf(" RX- bad IP checksum: %-14"PRIu64 1803 " Rx- bad L4 checksum: %-14"PRIu64 1804 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1805 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1806 fs->rx_bad_outer_l4_csum); 1807 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1808 fs->rx_bad_outer_ip_csum); 1809 } else { 1810 printf("\n"); 1811 } 1812 1813 if (record_burst_stats) { 1814 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1815 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1816 } 1817 } 1818 1819 void 1820 fwd_stats_display(void) 1821 { 1822 static const char *fwd_stats_border = "----------------------"; 1823 static const char *acc_stats_border = "+++++++++++++++"; 1824 struct { 1825 struct fwd_stream *rx_stream; 1826 struct fwd_stream *tx_stream; 1827 uint64_t tx_dropped; 1828 uint64_t rx_bad_ip_csum; 1829 uint64_t rx_bad_l4_csum; 1830 uint64_t rx_bad_outer_l4_csum; 1831 uint64_t rx_bad_outer_ip_csum; 1832 } ports_stats[RTE_MAX_ETHPORTS]; 1833 uint64_t total_rx_dropped = 0; 1834 uint64_t total_tx_dropped = 0; 1835 uint64_t total_rx_nombuf = 0; 1836 struct rte_eth_stats stats; 1837 uint64_t fwd_cycles = 0; 1838 uint64_t total_recv = 0; 1839 uint64_t total_xmit = 0; 1840 struct rte_port *port; 1841 streamid_t sm_id; 1842 portid_t pt_id; 1843 int i; 1844 1845 memset(ports_stats, 0, sizeof(ports_stats)); 1846 1847 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1848 struct fwd_stream *fs = fwd_streams[sm_id]; 1849 1850 if (cur_fwd_config.nb_fwd_streams > 1851 cur_fwd_config.nb_fwd_ports) { 1852 fwd_stream_stats_display(sm_id); 1853 } else { 1854 ports_stats[fs->tx_port].tx_stream = fs; 1855 ports_stats[fs->rx_port].rx_stream = fs; 1856 } 1857 1858 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1859 1860 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1861 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1862 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1863 fs->rx_bad_outer_l4_csum; 1864 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 1865 fs->rx_bad_outer_ip_csum; 1866 1867 if (record_core_cycles) 1868 fwd_cycles += fs->core_cycles; 1869 } 1870 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1871 pt_id = fwd_ports_ids[i]; 1872 port = &ports[pt_id]; 1873 1874 rte_eth_stats_get(pt_id, &stats); 1875 stats.ipackets -= port->stats.ipackets; 1876 stats.opackets -= port->stats.opackets; 1877 stats.ibytes -= port->stats.ibytes; 1878 stats.obytes -= port->stats.obytes; 1879 stats.imissed -= port->stats.imissed; 1880 stats.oerrors -= port->stats.oerrors; 1881 stats.rx_nombuf -= port->stats.rx_nombuf; 1882 1883 total_recv += stats.ipackets; 1884 total_xmit += stats.opackets; 1885 total_rx_dropped += stats.imissed; 1886 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1887 total_tx_dropped += stats.oerrors; 1888 total_rx_nombuf += stats.rx_nombuf; 1889 1890 printf("\n %s Forward statistics for port %-2d %s\n", 1891 fwd_stats_border, pt_id, fwd_stats_border); 1892 1893 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 1894 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 1895 stats.ipackets + stats.imissed); 1896 1897 if (cur_fwd_eng == &csum_fwd_engine) { 1898 printf(" Bad-ipcsum: %-14"PRIu64 1899 " Bad-l4csum: %-14"PRIu64 1900 "Bad-outer-l4csum: %-14"PRIu64"\n", 1901 ports_stats[pt_id].rx_bad_ip_csum, 1902 ports_stats[pt_id].rx_bad_l4_csum, 1903 ports_stats[pt_id].rx_bad_outer_l4_csum); 1904 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 1905 ports_stats[pt_id].rx_bad_outer_ip_csum); 1906 } 1907 if (stats.ierrors + stats.rx_nombuf > 0) { 1908 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 1909 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 1910 } 1911 1912 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 1913 "TX-total: %-"PRIu64"\n", 1914 stats.opackets, ports_stats[pt_id].tx_dropped, 1915 stats.opackets + ports_stats[pt_id].tx_dropped); 1916 1917 if (record_burst_stats) { 1918 if (ports_stats[pt_id].rx_stream) 1919 pkt_burst_stats_display("RX", 1920 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1921 if (ports_stats[pt_id].tx_stream) 1922 pkt_burst_stats_display("TX", 1923 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1924 } 1925 1926 printf(" %s--------------------------------%s\n", 1927 fwd_stats_border, fwd_stats_border); 1928 } 1929 1930 printf("\n %s Accumulated forward statistics for all ports" 1931 "%s\n", 1932 acc_stats_border, acc_stats_border); 1933 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1934 "%-"PRIu64"\n" 1935 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1936 "%-"PRIu64"\n", 1937 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1938 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1939 if (total_rx_nombuf > 0) 1940 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1941 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1942 "%s\n", 1943 acc_stats_border, acc_stats_border); 1944 if (record_core_cycles) { 1945 #define CYC_PER_MHZ 1E6 1946 if (total_recv > 0 || total_xmit > 0) { 1947 uint64_t total_pkts = 0; 1948 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1949 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1950 total_pkts = total_xmit; 1951 else 1952 total_pkts = total_recv; 1953 1954 printf("\n CPU cycles/packet=%.2F (total cycles=" 1955 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1956 " MHz Clock\n", 1957 (double) fwd_cycles / total_pkts, 1958 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1959 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1960 } 1961 } 1962 } 1963 1964 void 1965 fwd_stats_reset(void) 1966 { 1967 streamid_t sm_id; 1968 portid_t pt_id; 1969 int i; 1970 1971 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1972 pt_id = fwd_ports_ids[i]; 1973 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1974 } 1975 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1976 struct fwd_stream *fs = fwd_streams[sm_id]; 1977 1978 fs->rx_packets = 0; 1979 fs->tx_packets = 0; 1980 fs->fwd_dropped = 0; 1981 fs->rx_bad_ip_csum = 0; 1982 fs->rx_bad_l4_csum = 0; 1983 fs->rx_bad_outer_l4_csum = 0; 1984 fs->rx_bad_outer_ip_csum = 0; 1985 1986 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1987 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1988 fs->core_cycles = 0; 1989 } 1990 } 1991 1992 static void 1993 flush_fwd_rx_queues(void) 1994 { 1995 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1996 portid_t rxp; 1997 portid_t port_id; 1998 queueid_t rxq; 1999 uint16_t nb_rx; 2000 uint16_t i; 2001 uint8_t j; 2002 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2003 uint64_t timer_period; 2004 2005 /* convert to number of cycles */ 2006 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2007 2008 for (j = 0; j < 2; j++) { 2009 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2010 for (rxq = 0; rxq < nb_rxq; rxq++) { 2011 port_id = fwd_ports_ids[rxp]; 2012 /** 2013 * testpmd can stuck in the below do while loop 2014 * if rte_eth_rx_burst() always returns nonzero 2015 * packets. So timer is added to exit this loop 2016 * after 1sec timer expiry. 2017 */ 2018 prev_tsc = rte_rdtsc(); 2019 do { 2020 nb_rx = rte_eth_rx_burst(port_id, rxq, 2021 pkts_burst, MAX_PKT_BURST); 2022 for (i = 0; i < nb_rx; i++) 2023 rte_pktmbuf_free(pkts_burst[i]); 2024 2025 cur_tsc = rte_rdtsc(); 2026 diff_tsc = cur_tsc - prev_tsc; 2027 timer_tsc += diff_tsc; 2028 } while ((nb_rx > 0) && 2029 (timer_tsc < timer_period)); 2030 timer_tsc = 0; 2031 } 2032 } 2033 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2034 } 2035 } 2036 2037 static void 2038 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2039 { 2040 struct fwd_stream **fsm; 2041 streamid_t nb_fs; 2042 streamid_t sm_id; 2043 #ifdef RTE_LIB_BITRATESTATS 2044 uint64_t tics_per_1sec; 2045 uint64_t tics_datum; 2046 uint64_t tics_current; 2047 uint16_t i, cnt_ports; 2048 2049 cnt_ports = nb_ports; 2050 tics_datum = rte_rdtsc(); 2051 tics_per_1sec = rte_get_timer_hz(); 2052 #endif 2053 fsm = &fwd_streams[fc->stream_idx]; 2054 nb_fs = fc->stream_nb; 2055 do { 2056 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2057 (*pkt_fwd)(fsm[sm_id]); 2058 #ifdef RTE_LIB_BITRATESTATS 2059 if (bitrate_enabled != 0 && 2060 bitrate_lcore_id == rte_lcore_id()) { 2061 tics_current = rte_rdtsc(); 2062 if (tics_current - tics_datum >= tics_per_1sec) { 2063 /* Periodic bitrate calculation */ 2064 for (i = 0; i < cnt_ports; i++) 2065 rte_stats_bitrate_calc(bitrate_data, 2066 ports_ids[i]); 2067 tics_datum = tics_current; 2068 } 2069 } 2070 #endif 2071 #ifdef RTE_LIB_LATENCYSTATS 2072 if (latencystats_enabled != 0 && 2073 latencystats_lcore_id == rte_lcore_id()) 2074 rte_latencystats_update(); 2075 #endif 2076 2077 } while (! fc->stopped); 2078 } 2079 2080 static int 2081 start_pkt_forward_on_core(void *fwd_arg) 2082 { 2083 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2084 cur_fwd_config.fwd_eng->packet_fwd); 2085 return 0; 2086 } 2087 2088 /* 2089 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2090 * Used to start communication flows in network loopback test configurations. 2091 */ 2092 static int 2093 run_one_txonly_burst_on_core(void *fwd_arg) 2094 { 2095 struct fwd_lcore *fwd_lc; 2096 struct fwd_lcore tmp_lcore; 2097 2098 fwd_lc = (struct fwd_lcore *) fwd_arg; 2099 tmp_lcore = *fwd_lc; 2100 tmp_lcore.stopped = 1; 2101 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2102 return 0; 2103 } 2104 2105 /* 2106 * Launch packet forwarding: 2107 * - Setup per-port forwarding context. 2108 * - launch logical cores with their forwarding configuration. 2109 */ 2110 static void 2111 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2112 { 2113 port_fwd_begin_t port_fwd_begin; 2114 unsigned int i; 2115 unsigned int lc_id; 2116 int diag; 2117 2118 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2119 if (port_fwd_begin != NULL) { 2120 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2121 (*port_fwd_begin)(fwd_ports_ids[i]); 2122 } 2123 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2124 lc_id = fwd_lcores_cpuids[i]; 2125 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2126 fwd_lcores[i]->stopped = 0; 2127 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2128 fwd_lcores[i], lc_id); 2129 if (diag != 0) 2130 printf("launch lcore %u failed - diag=%d\n", 2131 lc_id, diag); 2132 } 2133 } 2134 } 2135 2136 /* 2137 * Launch packet forwarding configuration. 2138 */ 2139 void 2140 start_packet_forwarding(int with_tx_first) 2141 { 2142 port_fwd_begin_t port_fwd_begin; 2143 port_fwd_end_t port_fwd_end; 2144 struct rte_port *port; 2145 unsigned int i; 2146 portid_t pt_id; 2147 2148 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2149 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2150 2151 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2152 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2153 2154 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2155 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2156 (!nb_rxq || !nb_txq)) 2157 rte_exit(EXIT_FAILURE, 2158 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2159 cur_fwd_eng->fwd_mode_name); 2160 2161 if (all_ports_started() == 0) { 2162 printf("Not all ports were started\n"); 2163 return; 2164 } 2165 if (test_done == 0) { 2166 printf("Packet forwarding already started\n"); 2167 return; 2168 } 2169 2170 2171 if(dcb_test) { 2172 for (i = 0; i < nb_fwd_ports; i++) { 2173 pt_id = fwd_ports_ids[i]; 2174 port = &ports[pt_id]; 2175 if (!port->dcb_flag) { 2176 printf("In DCB mode, all forwarding ports must " 2177 "be configured in this mode.\n"); 2178 return; 2179 } 2180 } 2181 if (nb_fwd_lcores == 1) { 2182 printf("In DCB mode,the nb forwarding cores " 2183 "should be larger than 1.\n"); 2184 return; 2185 } 2186 } 2187 test_done = 0; 2188 2189 fwd_config_setup(); 2190 2191 if(!no_flush_rx) 2192 flush_fwd_rx_queues(); 2193 2194 pkt_fwd_config_display(&cur_fwd_config); 2195 rxtx_config_display(); 2196 2197 fwd_stats_reset(); 2198 if (with_tx_first) { 2199 port_fwd_begin = tx_only_engine.port_fwd_begin; 2200 if (port_fwd_begin != NULL) { 2201 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2202 (*port_fwd_begin)(fwd_ports_ids[i]); 2203 } 2204 while (with_tx_first--) { 2205 launch_packet_forwarding( 2206 run_one_txonly_burst_on_core); 2207 rte_eal_mp_wait_lcore(); 2208 } 2209 port_fwd_end = tx_only_engine.port_fwd_end; 2210 if (port_fwd_end != NULL) { 2211 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2212 (*port_fwd_end)(fwd_ports_ids[i]); 2213 } 2214 } 2215 launch_packet_forwarding(start_pkt_forward_on_core); 2216 } 2217 2218 void 2219 stop_packet_forwarding(void) 2220 { 2221 port_fwd_end_t port_fwd_end; 2222 lcoreid_t lc_id; 2223 portid_t pt_id; 2224 int i; 2225 2226 if (test_done) { 2227 printf("Packet forwarding not started\n"); 2228 return; 2229 } 2230 printf("Telling cores to stop..."); 2231 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2232 fwd_lcores[lc_id]->stopped = 1; 2233 printf("\nWaiting for lcores to finish...\n"); 2234 rte_eal_mp_wait_lcore(); 2235 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2236 if (port_fwd_end != NULL) { 2237 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2238 pt_id = fwd_ports_ids[i]; 2239 (*port_fwd_end)(pt_id); 2240 } 2241 } 2242 2243 fwd_stats_display(); 2244 2245 printf("\nDone.\n"); 2246 test_done = 1; 2247 } 2248 2249 void 2250 dev_set_link_up(portid_t pid) 2251 { 2252 if (rte_eth_dev_set_link_up(pid) < 0) 2253 printf("\nSet link up fail.\n"); 2254 } 2255 2256 void 2257 dev_set_link_down(portid_t pid) 2258 { 2259 if (rte_eth_dev_set_link_down(pid) < 0) 2260 printf("\nSet link down fail.\n"); 2261 } 2262 2263 static int 2264 all_ports_started(void) 2265 { 2266 portid_t pi; 2267 struct rte_port *port; 2268 2269 RTE_ETH_FOREACH_DEV(pi) { 2270 port = &ports[pi]; 2271 /* Check if there is a port which is not started */ 2272 if ((port->port_status != RTE_PORT_STARTED) && 2273 (port->slave_flag == 0)) 2274 return 0; 2275 } 2276 2277 /* No port is not started */ 2278 return 1; 2279 } 2280 2281 int 2282 port_is_stopped(portid_t port_id) 2283 { 2284 struct rte_port *port = &ports[port_id]; 2285 2286 if ((port->port_status != RTE_PORT_STOPPED) && 2287 (port->slave_flag == 0)) 2288 return 0; 2289 return 1; 2290 } 2291 2292 int 2293 all_ports_stopped(void) 2294 { 2295 portid_t pi; 2296 2297 RTE_ETH_FOREACH_DEV(pi) { 2298 if (!port_is_stopped(pi)) 2299 return 0; 2300 } 2301 2302 return 1; 2303 } 2304 2305 int 2306 port_is_started(portid_t port_id) 2307 { 2308 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2309 return 0; 2310 2311 if (ports[port_id].port_status != RTE_PORT_STARTED) 2312 return 0; 2313 2314 return 1; 2315 } 2316 2317 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2318 static int 2319 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2320 { 2321 queueid_t qi; 2322 struct rte_eth_hairpin_conf hairpin_conf = { 2323 .peer_count = 1, 2324 }; 2325 int i; 2326 int diag; 2327 struct rte_port *port = &ports[pi]; 2328 uint16_t peer_rx_port = pi; 2329 uint16_t peer_tx_port = pi; 2330 uint32_t manual = 1; 2331 uint32_t tx_exp = hairpin_mode & 0x10; 2332 2333 if (!(hairpin_mode & 0xf)) { 2334 peer_rx_port = pi; 2335 peer_tx_port = pi; 2336 manual = 0; 2337 } else if (hairpin_mode & 0x1) { 2338 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2339 RTE_ETH_DEV_NO_OWNER); 2340 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2341 peer_tx_port = rte_eth_find_next_owned_by(0, 2342 RTE_ETH_DEV_NO_OWNER); 2343 if (p_pi != RTE_MAX_ETHPORTS) { 2344 peer_rx_port = p_pi; 2345 } else { 2346 uint16_t next_pi; 2347 2348 /* Last port will be the peer RX port of the first. */ 2349 RTE_ETH_FOREACH_DEV(next_pi) 2350 peer_rx_port = next_pi; 2351 } 2352 manual = 1; 2353 } else if (hairpin_mode & 0x2) { 2354 if (cnt_pi & 0x1) { 2355 peer_rx_port = p_pi; 2356 } else { 2357 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2358 RTE_ETH_DEV_NO_OWNER); 2359 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2360 peer_rx_port = pi; 2361 } 2362 peer_tx_port = peer_rx_port; 2363 manual = 1; 2364 } 2365 2366 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2367 hairpin_conf.peers[0].port = peer_rx_port; 2368 hairpin_conf.peers[0].queue = i + nb_rxq; 2369 hairpin_conf.manual_bind = !!manual; 2370 hairpin_conf.tx_explicit = !!tx_exp; 2371 diag = rte_eth_tx_hairpin_queue_setup 2372 (pi, qi, nb_txd, &hairpin_conf); 2373 i++; 2374 if (diag == 0) 2375 continue; 2376 2377 /* Fail to setup rx queue, return */ 2378 if (rte_atomic16_cmpset(&(port->port_status), 2379 RTE_PORT_HANDLING, 2380 RTE_PORT_STOPPED) == 0) 2381 printf("Port %d can not be set back " 2382 "to stopped\n", pi); 2383 printf("Fail to configure port %d hairpin " 2384 "queues\n", pi); 2385 /* try to reconfigure queues next time */ 2386 port->need_reconfig_queues = 1; 2387 return -1; 2388 } 2389 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2390 hairpin_conf.peers[0].port = peer_tx_port; 2391 hairpin_conf.peers[0].queue = i + nb_txq; 2392 hairpin_conf.manual_bind = !!manual; 2393 hairpin_conf.tx_explicit = !!tx_exp; 2394 diag = rte_eth_rx_hairpin_queue_setup 2395 (pi, qi, nb_rxd, &hairpin_conf); 2396 i++; 2397 if (diag == 0) 2398 continue; 2399 2400 /* Fail to setup rx queue, return */ 2401 if (rte_atomic16_cmpset(&(port->port_status), 2402 RTE_PORT_HANDLING, 2403 RTE_PORT_STOPPED) == 0) 2404 printf("Port %d can not be set back " 2405 "to stopped\n", pi); 2406 printf("Fail to configure port %d hairpin " 2407 "queues\n", pi); 2408 /* try to reconfigure queues next time */ 2409 port->need_reconfig_queues = 1; 2410 return -1; 2411 } 2412 return 0; 2413 } 2414 2415 /* Configure the Rx with optional split. */ 2416 int 2417 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2418 uint16_t nb_rx_desc, unsigned int socket_id, 2419 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2420 { 2421 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2422 unsigned int i, mp_n; 2423 int ret; 2424 2425 if (rx_pkt_nb_segs <= 1 || 2426 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2427 rx_conf->rx_seg = NULL; 2428 rx_conf->rx_nseg = 0; 2429 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2430 nb_rx_desc, socket_id, 2431 rx_conf, mp); 2432 return ret; 2433 } 2434 for (i = 0; i < rx_pkt_nb_segs; i++) { 2435 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2436 struct rte_mempool *mpx; 2437 /* 2438 * Use last valid pool for the segments with number 2439 * exceeding the pool index. 2440 */ 2441 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2442 mpx = mbuf_pool_find(socket_id, mp_n); 2443 /* Handle zero as mbuf data buffer size. */ 2444 rx_seg->length = rx_pkt_seg_lengths[i] ? 2445 rx_pkt_seg_lengths[i] : 2446 mbuf_data_size[mp_n]; 2447 rx_seg->offset = i < rx_pkt_nb_offs ? 2448 rx_pkt_seg_offsets[i] : 0; 2449 rx_seg->mp = mpx ? mpx : mp; 2450 } 2451 rx_conf->rx_nseg = rx_pkt_nb_segs; 2452 rx_conf->rx_seg = rx_useg; 2453 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2454 socket_id, rx_conf, NULL); 2455 rx_conf->rx_seg = NULL; 2456 rx_conf->rx_nseg = 0; 2457 return ret; 2458 } 2459 2460 int 2461 start_port(portid_t pid) 2462 { 2463 int diag, need_check_link_status = -1; 2464 portid_t pi; 2465 portid_t p_pi = RTE_MAX_ETHPORTS; 2466 portid_t pl[RTE_MAX_ETHPORTS]; 2467 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2468 uint16_t cnt_pi = 0; 2469 uint16_t cfg_pi = 0; 2470 int peer_pi; 2471 queueid_t qi; 2472 struct rte_port *port; 2473 struct rte_ether_addr mac_addr; 2474 struct rte_eth_hairpin_cap cap; 2475 2476 if (port_id_is_invalid(pid, ENABLED_WARN)) 2477 return 0; 2478 2479 if(dcb_config) 2480 dcb_test = 1; 2481 RTE_ETH_FOREACH_DEV(pi) { 2482 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2483 continue; 2484 2485 need_check_link_status = 0; 2486 port = &ports[pi]; 2487 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2488 RTE_PORT_HANDLING) == 0) { 2489 printf("Port %d is now not stopped\n", pi); 2490 continue; 2491 } 2492 2493 if (port->need_reconfig > 0) { 2494 port->need_reconfig = 0; 2495 2496 if (flow_isolate_all) { 2497 int ret = port_flow_isolate(pi, 1); 2498 if (ret) { 2499 printf("Failed to apply isolated" 2500 " mode on port %d\n", pi); 2501 return -1; 2502 } 2503 } 2504 configure_rxtx_dump_callbacks(0); 2505 printf("Configuring Port %d (socket %u)\n", pi, 2506 port->socket_id); 2507 if (nb_hairpinq > 0 && 2508 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2509 printf("Port %d doesn't support hairpin " 2510 "queues\n", pi); 2511 return -1; 2512 } 2513 /* configure port */ 2514 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2515 nb_txq + nb_hairpinq, 2516 &(port->dev_conf)); 2517 if (diag != 0) { 2518 if (rte_atomic16_cmpset(&(port->port_status), 2519 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2520 printf("Port %d can not be set back " 2521 "to stopped\n", pi); 2522 printf("Fail to configure port %d\n", pi); 2523 /* try to reconfigure port next time */ 2524 port->need_reconfig = 1; 2525 return -1; 2526 } 2527 } 2528 if (port->need_reconfig_queues > 0) { 2529 port->need_reconfig_queues = 0; 2530 /* setup tx queues */ 2531 for (qi = 0; qi < nb_txq; qi++) { 2532 if ((numa_support) && 2533 (txring_numa[pi] != NUMA_NO_CONFIG)) 2534 diag = rte_eth_tx_queue_setup(pi, qi, 2535 port->nb_tx_desc[qi], 2536 txring_numa[pi], 2537 &(port->tx_conf[qi])); 2538 else 2539 diag = rte_eth_tx_queue_setup(pi, qi, 2540 port->nb_tx_desc[qi], 2541 port->socket_id, 2542 &(port->tx_conf[qi])); 2543 2544 if (diag == 0) 2545 continue; 2546 2547 /* Fail to setup tx queue, return */ 2548 if (rte_atomic16_cmpset(&(port->port_status), 2549 RTE_PORT_HANDLING, 2550 RTE_PORT_STOPPED) == 0) 2551 printf("Port %d can not be set back " 2552 "to stopped\n", pi); 2553 printf("Fail to configure port %d tx queues\n", 2554 pi); 2555 /* try to reconfigure queues next time */ 2556 port->need_reconfig_queues = 1; 2557 return -1; 2558 } 2559 for (qi = 0; qi < nb_rxq; qi++) { 2560 /* setup rx queues */ 2561 if ((numa_support) && 2562 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2563 struct rte_mempool * mp = 2564 mbuf_pool_find 2565 (rxring_numa[pi], 0); 2566 if (mp == NULL) { 2567 printf("Failed to setup RX queue:" 2568 "No mempool allocation" 2569 " on the socket %d\n", 2570 rxring_numa[pi]); 2571 return -1; 2572 } 2573 2574 diag = rx_queue_setup(pi, qi, 2575 port->nb_rx_desc[qi], 2576 rxring_numa[pi], 2577 &(port->rx_conf[qi]), 2578 mp); 2579 } else { 2580 struct rte_mempool *mp = 2581 mbuf_pool_find 2582 (port->socket_id, 0); 2583 if (mp == NULL) { 2584 printf("Failed to setup RX queue:" 2585 "No mempool allocation" 2586 " on the socket %d\n", 2587 port->socket_id); 2588 return -1; 2589 } 2590 diag = rx_queue_setup(pi, qi, 2591 port->nb_rx_desc[qi], 2592 port->socket_id, 2593 &(port->rx_conf[qi]), 2594 mp); 2595 } 2596 if (diag == 0) 2597 continue; 2598 2599 /* Fail to setup rx queue, return */ 2600 if (rte_atomic16_cmpset(&(port->port_status), 2601 RTE_PORT_HANDLING, 2602 RTE_PORT_STOPPED) == 0) 2603 printf("Port %d can not be set back " 2604 "to stopped\n", pi); 2605 printf("Fail to configure port %d rx queues\n", 2606 pi); 2607 /* try to reconfigure queues next time */ 2608 port->need_reconfig_queues = 1; 2609 return -1; 2610 } 2611 /* setup hairpin queues */ 2612 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2613 return -1; 2614 } 2615 configure_rxtx_dump_callbacks(verbose_level); 2616 if (clear_ptypes) { 2617 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2618 NULL, 0); 2619 if (diag < 0) 2620 printf( 2621 "Port %d: Failed to disable Ptype parsing\n", 2622 pi); 2623 } 2624 2625 p_pi = pi; 2626 cnt_pi++; 2627 2628 /* start port */ 2629 diag = rte_eth_dev_start(pi); 2630 if (diag < 0) { 2631 printf("Fail to start port %d: %s\n", pi, 2632 rte_strerror(-diag)); 2633 2634 /* Fail to setup rx queue, return */ 2635 if (rte_atomic16_cmpset(&(port->port_status), 2636 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2637 printf("Port %d can not be set back to " 2638 "stopped\n", pi); 2639 continue; 2640 } 2641 2642 if (rte_atomic16_cmpset(&(port->port_status), 2643 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2644 printf("Port %d can not be set into started\n", pi); 2645 2646 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2647 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2648 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2649 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2650 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2651 2652 /* at least one port started, need checking link status */ 2653 need_check_link_status = 1; 2654 2655 pl[cfg_pi++] = pi; 2656 } 2657 2658 if (need_check_link_status == 1 && !no_link_check) 2659 check_all_ports_link_status(RTE_PORT_ALL); 2660 else if (need_check_link_status == 0) 2661 printf("Please stop the ports first\n"); 2662 2663 if (hairpin_mode & 0xf) { 2664 uint16_t i; 2665 int j; 2666 2667 /* bind all started hairpin ports */ 2668 for (i = 0; i < cfg_pi; i++) { 2669 pi = pl[i]; 2670 /* bind current Tx to all peer Rx */ 2671 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2672 RTE_MAX_ETHPORTS, 1); 2673 if (peer_pi < 0) 2674 return peer_pi; 2675 for (j = 0; j < peer_pi; j++) { 2676 if (!port_is_started(peer_pl[j])) 2677 continue; 2678 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2679 if (diag < 0) { 2680 printf("Error during binding hairpin" 2681 " Tx port %u to %u: %s\n", 2682 pi, peer_pl[j], 2683 rte_strerror(-diag)); 2684 return -1; 2685 } 2686 } 2687 /* bind all peer Tx to current Rx */ 2688 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2689 RTE_MAX_ETHPORTS, 0); 2690 if (peer_pi < 0) 2691 return peer_pi; 2692 for (j = 0; j < peer_pi; j++) { 2693 if (!port_is_started(peer_pl[j])) 2694 continue; 2695 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2696 if (diag < 0) { 2697 printf("Error during binding hairpin" 2698 " Tx port %u to %u: %s\n", 2699 peer_pl[j], pi, 2700 rte_strerror(-diag)); 2701 return -1; 2702 } 2703 } 2704 } 2705 } 2706 2707 printf("Done\n"); 2708 return 0; 2709 } 2710 2711 void 2712 stop_port(portid_t pid) 2713 { 2714 portid_t pi; 2715 struct rte_port *port; 2716 int need_check_link_status = 0; 2717 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2718 int peer_pi; 2719 2720 if (dcb_test) { 2721 dcb_test = 0; 2722 dcb_config = 0; 2723 } 2724 2725 if (port_id_is_invalid(pid, ENABLED_WARN)) 2726 return; 2727 2728 printf("Stopping ports...\n"); 2729 2730 RTE_ETH_FOREACH_DEV(pi) { 2731 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2732 continue; 2733 2734 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2735 printf("Please remove port %d from forwarding configuration.\n", pi); 2736 continue; 2737 } 2738 2739 if (port_is_bonding_slave(pi)) { 2740 printf("Please remove port %d from bonded device.\n", pi); 2741 continue; 2742 } 2743 2744 port = &ports[pi]; 2745 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2746 RTE_PORT_HANDLING) == 0) 2747 continue; 2748 2749 if (hairpin_mode & 0xf) { 2750 int j; 2751 2752 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2753 /* unbind all peer Tx from current Rx */ 2754 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2755 RTE_MAX_ETHPORTS, 0); 2756 if (peer_pi < 0) 2757 continue; 2758 for (j = 0; j < peer_pi; j++) { 2759 if (!port_is_started(peer_pl[j])) 2760 continue; 2761 rte_eth_hairpin_unbind(peer_pl[j], pi); 2762 } 2763 } 2764 2765 if (port->flow_list) 2766 port_flow_flush(pi); 2767 2768 if (rte_eth_dev_stop(pi) != 0) 2769 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2770 pi); 2771 2772 if (rte_atomic16_cmpset(&(port->port_status), 2773 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2774 printf("Port %d can not be set into stopped\n", pi); 2775 need_check_link_status = 1; 2776 } 2777 if (need_check_link_status && !no_link_check) 2778 check_all_ports_link_status(RTE_PORT_ALL); 2779 2780 printf("Done\n"); 2781 } 2782 2783 static void 2784 remove_invalid_ports_in(portid_t *array, portid_t *total) 2785 { 2786 portid_t i; 2787 portid_t new_total = 0; 2788 2789 for (i = 0; i < *total; i++) 2790 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2791 array[new_total] = array[i]; 2792 new_total++; 2793 } 2794 *total = new_total; 2795 } 2796 2797 static void 2798 remove_invalid_ports(void) 2799 { 2800 remove_invalid_ports_in(ports_ids, &nb_ports); 2801 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2802 nb_cfg_ports = nb_fwd_ports; 2803 } 2804 2805 void 2806 close_port(portid_t pid) 2807 { 2808 portid_t pi; 2809 struct rte_port *port; 2810 2811 if (port_id_is_invalid(pid, ENABLED_WARN)) 2812 return; 2813 2814 printf("Closing ports...\n"); 2815 2816 RTE_ETH_FOREACH_DEV(pi) { 2817 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2818 continue; 2819 2820 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2821 printf("Please remove port %d from forwarding configuration.\n", pi); 2822 continue; 2823 } 2824 2825 if (port_is_bonding_slave(pi)) { 2826 printf("Please remove port %d from bonded device.\n", pi); 2827 continue; 2828 } 2829 2830 port = &ports[pi]; 2831 if (rte_atomic16_cmpset(&(port->port_status), 2832 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2833 printf("Port %d is already closed\n", pi); 2834 continue; 2835 } 2836 2837 port_flow_flush(pi); 2838 rte_eth_dev_close(pi); 2839 } 2840 2841 remove_invalid_ports(); 2842 printf("Done\n"); 2843 } 2844 2845 void 2846 reset_port(portid_t pid) 2847 { 2848 int diag; 2849 portid_t pi; 2850 struct rte_port *port; 2851 2852 if (port_id_is_invalid(pid, ENABLED_WARN)) 2853 return; 2854 2855 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2856 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2857 printf("Can not reset port(s), please stop port(s) first.\n"); 2858 return; 2859 } 2860 2861 printf("Resetting ports...\n"); 2862 2863 RTE_ETH_FOREACH_DEV(pi) { 2864 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2865 continue; 2866 2867 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2868 printf("Please remove port %d from forwarding " 2869 "configuration.\n", pi); 2870 continue; 2871 } 2872 2873 if (port_is_bonding_slave(pi)) { 2874 printf("Please remove port %d from bonded device.\n", 2875 pi); 2876 continue; 2877 } 2878 2879 diag = rte_eth_dev_reset(pi); 2880 if (diag == 0) { 2881 port = &ports[pi]; 2882 port->need_reconfig = 1; 2883 port->need_reconfig_queues = 1; 2884 } else { 2885 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2886 } 2887 } 2888 2889 printf("Done\n"); 2890 } 2891 2892 void 2893 attach_port(char *identifier) 2894 { 2895 portid_t pi; 2896 struct rte_dev_iterator iterator; 2897 2898 printf("Attaching a new port...\n"); 2899 2900 if (identifier == NULL) { 2901 printf("Invalid parameters are specified\n"); 2902 return; 2903 } 2904 2905 if (rte_dev_probe(identifier) < 0) { 2906 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2907 return; 2908 } 2909 2910 /* first attach mode: event */ 2911 if (setup_on_probe_event) { 2912 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2913 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2914 if (ports[pi].port_status == RTE_PORT_HANDLING && 2915 ports[pi].need_setup != 0) 2916 setup_attached_port(pi); 2917 return; 2918 } 2919 2920 /* second attach mode: iterator */ 2921 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2922 /* setup ports matching the devargs used for probing */ 2923 if (port_is_forwarding(pi)) 2924 continue; /* port was already attached before */ 2925 setup_attached_port(pi); 2926 } 2927 } 2928 2929 static void 2930 setup_attached_port(portid_t pi) 2931 { 2932 unsigned int socket_id; 2933 int ret; 2934 2935 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2936 /* if socket_id is invalid, set to the first available socket. */ 2937 if (check_socket_id(socket_id) < 0) 2938 socket_id = socket_ids[0]; 2939 reconfig(pi, socket_id); 2940 ret = rte_eth_promiscuous_enable(pi); 2941 if (ret != 0) 2942 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2943 pi, rte_strerror(-ret)); 2944 2945 ports_ids[nb_ports++] = pi; 2946 fwd_ports_ids[nb_fwd_ports++] = pi; 2947 nb_cfg_ports = nb_fwd_ports; 2948 ports[pi].need_setup = 0; 2949 ports[pi].port_status = RTE_PORT_STOPPED; 2950 2951 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2952 printf("Done\n"); 2953 } 2954 2955 static void 2956 detach_device(struct rte_device *dev) 2957 { 2958 portid_t sibling; 2959 2960 if (dev == NULL) { 2961 printf("Device already removed\n"); 2962 return; 2963 } 2964 2965 printf("Removing a device...\n"); 2966 2967 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2968 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2969 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2970 printf("Port %u not stopped\n", sibling); 2971 return; 2972 } 2973 port_flow_flush(sibling); 2974 } 2975 } 2976 2977 if (rte_dev_remove(dev) < 0) { 2978 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2979 return; 2980 } 2981 remove_invalid_ports(); 2982 2983 printf("Device is detached\n"); 2984 printf("Now total ports is %d\n", nb_ports); 2985 printf("Done\n"); 2986 return; 2987 } 2988 2989 void 2990 detach_port_device(portid_t port_id) 2991 { 2992 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2993 return; 2994 2995 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2996 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2997 printf("Port not stopped\n"); 2998 return; 2999 } 3000 printf("Port was not closed\n"); 3001 } 3002 3003 detach_device(rte_eth_devices[port_id].device); 3004 } 3005 3006 void 3007 detach_devargs(char *identifier) 3008 { 3009 struct rte_dev_iterator iterator; 3010 struct rte_devargs da; 3011 portid_t port_id; 3012 3013 printf("Removing a device...\n"); 3014 3015 memset(&da, 0, sizeof(da)); 3016 if (rte_devargs_parsef(&da, "%s", identifier)) { 3017 printf("cannot parse identifier\n"); 3018 return; 3019 } 3020 3021 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3022 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3023 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3024 printf("Port %u not stopped\n", port_id); 3025 rte_eth_iterator_cleanup(&iterator); 3026 rte_devargs_reset(&da); 3027 return; 3028 } 3029 port_flow_flush(port_id); 3030 } 3031 } 3032 3033 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3034 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3035 da.name, da.bus->name); 3036 rte_devargs_reset(&da); 3037 return; 3038 } 3039 3040 remove_invalid_ports(); 3041 3042 printf("Device %s is detached\n", identifier); 3043 printf("Now total ports is %d\n", nb_ports); 3044 printf("Done\n"); 3045 rte_devargs_reset(&da); 3046 } 3047 3048 void 3049 pmd_test_exit(void) 3050 { 3051 portid_t pt_id; 3052 unsigned int i; 3053 int ret; 3054 3055 if (test_done == 0) 3056 stop_packet_forwarding(); 3057 3058 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3059 if (mempools[i]) { 3060 if (mp_alloc_type == MP_ALLOC_ANON) 3061 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3062 NULL); 3063 } 3064 } 3065 if (ports != NULL) { 3066 no_link_check = 1; 3067 RTE_ETH_FOREACH_DEV(pt_id) { 3068 printf("\nStopping port %d...\n", pt_id); 3069 fflush(stdout); 3070 stop_port(pt_id); 3071 } 3072 RTE_ETH_FOREACH_DEV(pt_id) { 3073 printf("\nShutting down port %d...\n", pt_id); 3074 fflush(stdout); 3075 close_port(pt_id); 3076 } 3077 } 3078 3079 if (hot_plug) { 3080 ret = rte_dev_event_monitor_stop(); 3081 if (ret) { 3082 RTE_LOG(ERR, EAL, 3083 "fail to stop device event monitor."); 3084 return; 3085 } 3086 3087 ret = rte_dev_event_callback_unregister(NULL, 3088 dev_event_callback, NULL); 3089 if (ret < 0) { 3090 RTE_LOG(ERR, EAL, 3091 "fail to unregister device event callback.\n"); 3092 return; 3093 } 3094 3095 ret = rte_dev_hotplug_handle_disable(); 3096 if (ret) { 3097 RTE_LOG(ERR, EAL, 3098 "fail to disable hotplug handling.\n"); 3099 return; 3100 } 3101 } 3102 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3103 if (mempools[i]) 3104 rte_mempool_free(mempools[i]); 3105 } 3106 3107 printf("\nBye...\n"); 3108 } 3109 3110 typedef void (*cmd_func_t)(void); 3111 struct pmd_test_command { 3112 const char *cmd_name; 3113 cmd_func_t cmd_func; 3114 }; 3115 3116 /* Check the link status of all ports in up to 9s, and print them finally */ 3117 static void 3118 check_all_ports_link_status(uint32_t port_mask) 3119 { 3120 #define CHECK_INTERVAL 100 /* 100ms */ 3121 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3122 portid_t portid; 3123 uint8_t count, all_ports_up, print_flag = 0; 3124 struct rte_eth_link link; 3125 int ret; 3126 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3127 3128 printf("Checking link statuses...\n"); 3129 fflush(stdout); 3130 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3131 all_ports_up = 1; 3132 RTE_ETH_FOREACH_DEV(portid) { 3133 if ((port_mask & (1 << portid)) == 0) 3134 continue; 3135 memset(&link, 0, sizeof(link)); 3136 ret = rte_eth_link_get_nowait(portid, &link); 3137 if (ret < 0) { 3138 all_ports_up = 0; 3139 if (print_flag == 1) 3140 printf("Port %u link get failed: %s\n", 3141 portid, rte_strerror(-ret)); 3142 continue; 3143 } 3144 /* print link status if flag set */ 3145 if (print_flag == 1) { 3146 rte_eth_link_to_str(link_status, 3147 sizeof(link_status), &link); 3148 printf("Port %d %s\n", portid, link_status); 3149 continue; 3150 } 3151 /* clear all_ports_up flag if any link down */ 3152 if (link.link_status == ETH_LINK_DOWN) { 3153 all_ports_up = 0; 3154 break; 3155 } 3156 } 3157 /* after finally printing all link status, get out */ 3158 if (print_flag == 1) 3159 break; 3160 3161 if (all_ports_up == 0) { 3162 fflush(stdout); 3163 rte_delay_ms(CHECK_INTERVAL); 3164 } 3165 3166 /* set the print_flag if all ports up or timeout */ 3167 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3168 print_flag = 1; 3169 } 3170 3171 if (lsc_interrupt) 3172 break; 3173 } 3174 } 3175 3176 static void 3177 rmv_port_callback(void *arg) 3178 { 3179 int need_to_start = 0; 3180 int org_no_link_check = no_link_check; 3181 portid_t port_id = (intptr_t)arg; 3182 struct rte_device *dev; 3183 3184 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3185 3186 if (!test_done && port_is_forwarding(port_id)) { 3187 need_to_start = 1; 3188 stop_packet_forwarding(); 3189 } 3190 no_link_check = 1; 3191 stop_port(port_id); 3192 no_link_check = org_no_link_check; 3193 3194 /* Save rte_device pointer before closing ethdev port */ 3195 dev = rte_eth_devices[port_id].device; 3196 close_port(port_id); 3197 detach_device(dev); /* might be already removed or have more ports */ 3198 3199 if (need_to_start) 3200 start_packet_forwarding(0); 3201 } 3202 3203 /* This function is used by the interrupt thread */ 3204 static int 3205 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3206 void *ret_param) 3207 { 3208 RTE_SET_USED(param); 3209 RTE_SET_USED(ret_param); 3210 3211 if (type >= RTE_ETH_EVENT_MAX) { 3212 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3213 port_id, __func__, type); 3214 fflush(stderr); 3215 } else if (event_print_mask & (UINT32_C(1) << type)) { 3216 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3217 eth_event_desc[type]); 3218 fflush(stdout); 3219 } 3220 3221 switch (type) { 3222 case RTE_ETH_EVENT_NEW: 3223 ports[port_id].need_setup = 1; 3224 ports[port_id].port_status = RTE_PORT_HANDLING; 3225 break; 3226 case RTE_ETH_EVENT_INTR_RMV: 3227 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3228 break; 3229 if (rte_eal_alarm_set(100000, 3230 rmv_port_callback, (void *)(intptr_t)port_id)) 3231 fprintf(stderr, "Could not set up deferred device removal\n"); 3232 break; 3233 case RTE_ETH_EVENT_DESTROY: 3234 ports[port_id].port_status = RTE_PORT_CLOSED; 3235 printf("Port %u is closed\n", port_id); 3236 break; 3237 default: 3238 break; 3239 } 3240 return 0; 3241 } 3242 3243 static int 3244 register_eth_event_callback(void) 3245 { 3246 int ret; 3247 enum rte_eth_event_type event; 3248 3249 for (event = RTE_ETH_EVENT_UNKNOWN; 3250 event < RTE_ETH_EVENT_MAX; event++) { 3251 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3252 event, 3253 eth_event_callback, 3254 NULL); 3255 if (ret != 0) { 3256 TESTPMD_LOG(ERR, "Failed to register callback for " 3257 "%s event\n", eth_event_desc[event]); 3258 return -1; 3259 } 3260 } 3261 3262 return 0; 3263 } 3264 3265 /* This function is used by the interrupt thread */ 3266 static void 3267 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3268 __rte_unused void *arg) 3269 { 3270 uint16_t port_id; 3271 int ret; 3272 3273 if (type >= RTE_DEV_EVENT_MAX) { 3274 fprintf(stderr, "%s called upon invalid event %d\n", 3275 __func__, type); 3276 fflush(stderr); 3277 } 3278 3279 switch (type) { 3280 case RTE_DEV_EVENT_REMOVE: 3281 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3282 device_name); 3283 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3284 if (ret) { 3285 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3286 device_name); 3287 return; 3288 } 3289 /* 3290 * Because the user's callback is invoked in eal interrupt 3291 * callback, the interrupt callback need to be finished before 3292 * it can be unregistered when detaching device. So finish 3293 * callback soon and use a deferred removal to detach device 3294 * is need. It is a workaround, once the device detaching be 3295 * moved into the eal in the future, the deferred removal could 3296 * be deleted. 3297 */ 3298 if (rte_eal_alarm_set(100000, 3299 rmv_port_callback, (void *)(intptr_t)port_id)) 3300 RTE_LOG(ERR, EAL, 3301 "Could not set up deferred device removal\n"); 3302 break; 3303 case RTE_DEV_EVENT_ADD: 3304 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3305 device_name); 3306 /* TODO: After finish kernel driver binding, 3307 * begin to attach port. 3308 */ 3309 break; 3310 default: 3311 break; 3312 } 3313 } 3314 3315 static void 3316 rxtx_port_config(struct rte_port *port) 3317 { 3318 uint16_t qid; 3319 uint64_t offloads; 3320 3321 for (qid = 0; qid < nb_rxq; qid++) { 3322 offloads = port->rx_conf[qid].offloads; 3323 port->rx_conf[qid] = port->dev_info.default_rxconf; 3324 if (offloads != 0) 3325 port->rx_conf[qid].offloads = offloads; 3326 3327 /* Check if any Rx parameters have been passed */ 3328 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3329 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3330 3331 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3332 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3333 3334 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3335 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3336 3337 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3338 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3339 3340 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3341 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3342 3343 port->nb_rx_desc[qid] = nb_rxd; 3344 } 3345 3346 for (qid = 0; qid < nb_txq; qid++) { 3347 offloads = port->tx_conf[qid].offloads; 3348 port->tx_conf[qid] = port->dev_info.default_txconf; 3349 if (offloads != 0) 3350 port->tx_conf[qid].offloads = offloads; 3351 3352 /* Check if any Tx parameters have been passed */ 3353 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3354 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3355 3356 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3357 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3358 3359 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3360 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3361 3362 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3363 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3364 3365 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3366 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3367 3368 port->nb_tx_desc[qid] = nb_txd; 3369 } 3370 } 3371 3372 /* 3373 * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload, 3374 * MTU is also aligned if JUMBO_FRAME offload is not set. 3375 * 3376 * port->dev_info should be set before calling this function. 3377 * 3378 * return 0 on success, negative on error 3379 */ 3380 int 3381 update_jumbo_frame_offload(portid_t portid) 3382 { 3383 struct rte_port *port = &ports[portid]; 3384 uint32_t eth_overhead; 3385 uint64_t rx_offloads; 3386 int ret; 3387 bool on; 3388 3389 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */ 3390 if (port->dev_info.max_mtu != UINT16_MAX && 3391 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu) 3392 eth_overhead = port->dev_info.max_rx_pktlen - 3393 port->dev_info.max_mtu; 3394 else 3395 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 3396 3397 rx_offloads = port->dev_conf.rxmode.offloads; 3398 3399 /* Default config value is 0 to use PMD specific overhead */ 3400 if (port->dev_conf.rxmode.max_rx_pkt_len == 0) 3401 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead; 3402 3403 if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) { 3404 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3405 on = false; 3406 } else { 3407 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3408 printf("Frame size (%u) is not supported by port %u\n", 3409 port->dev_conf.rxmode.max_rx_pkt_len, 3410 portid); 3411 return -1; 3412 } 3413 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3414 on = true; 3415 } 3416 3417 if (rx_offloads != port->dev_conf.rxmode.offloads) { 3418 uint16_t qid; 3419 3420 port->dev_conf.rxmode.offloads = rx_offloads; 3421 3422 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */ 3423 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) { 3424 if (on) 3425 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3426 else 3427 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3428 } 3429 } 3430 3431 /* If JUMBO_FRAME is set MTU conversion done by ethdev layer, 3432 * if unset do it here 3433 */ 3434 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3435 ret = rte_eth_dev_set_mtu(portid, 3436 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead); 3437 if (ret) 3438 printf("Failed to set MTU to %u for port %u\n", 3439 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead, 3440 portid); 3441 } 3442 3443 return 0; 3444 } 3445 3446 void 3447 init_port_config(void) 3448 { 3449 portid_t pid; 3450 struct rte_port *port; 3451 int ret; 3452 3453 RTE_ETH_FOREACH_DEV(pid) { 3454 port = &ports[pid]; 3455 port->dev_conf.fdir_conf = fdir_conf; 3456 3457 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3458 if (ret != 0) 3459 return; 3460 3461 if (nb_rxq > 1) { 3462 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3463 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3464 rss_hf & port->dev_info.flow_type_rss_offloads; 3465 } else { 3466 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3467 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3468 } 3469 3470 if (port->dcb_flag == 0) { 3471 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3472 port->dev_conf.rxmode.mq_mode = 3473 (enum rte_eth_rx_mq_mode) 3474 (rx_mq_mode & ETH_MQ_RX_RSS); 3475 else 3476 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3477 } 3478 3479 rxtx_port_config(port); 3480 3481 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3482 if (ret != 0) 3483 return; 3484 3485 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3486 rte_pmd_ixgbe_bypass_init(pid); 3487 #endif 3488 3489 if (lsc_interrupt && 3490 (rte_eth_devices[pid].data->dev_flags & 3491 RTE_ETH_DEV_INTR_LSC)) 3492 port->dev_conf.intr_conf.lsc = 1; 3493 if (rmv_interrupt && 3494 (rte_eth_devices[pid].data->dev_flags & 3495 RTE_ETH_DEV_INTR_RMV)) 3496 port->dev_conf.intr_conf.rmv = 1; 3497 } 3498 } 3499 3500 void set_port_slave_flag(portid_t slave_pid) 3501 { 3502 struct rte_port *port; 3503 3504 port = &ports[slave_pid]; 3505 port->slave_flag = 1; 3506 } 3507 3508 void clear_port_slave_flag(portid_t slave_pid) 3509 { 3510 struct rte_port *port; 3511 3512 port = &ports[slave_pid]; 3513 port->slave_flag = 0; 3514 } 3515 3516 uint8_t port_is_bonding_slave(portid_t slave_pid) 3517 { 3518 struct rte_port *port; 3519 3520 port = &ports[slave_pid]; 3521 if ((rte_eth_devices[slave_pid].data->dev_flags & 3522 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3523 return 1; 3524 return 0; 3525 } 3526 3527 const uint16_t vlan_tags[] = { 3528 0, 1, 2, 3, 4, 5, 6, 7, 3529 8, 9, 10, 11, 12, 13, 14, 15, 3530 16, 17, 18, 19, 20, 21, 22, 23, 3531 24, 25, 26, 27, 28, 29, 30, 31 3532 }; 3533 3534 static int 3535 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3536 enum dcb_mode_enable dcb_mode, 3537 enum rte_eth_nb_tcs num_tcs, 3538 uint8_t pfc_en) 3539 { 3540 uint8_t i; 3541 int32_t rc; 3542 struct rte_eth_rss_conf rss_conf; 3543 3544 /* 3545 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3546 * given above, and the number of traffic classes available for use. 3547 */ 3548 if (dcb_mode == DCB_VT_ENABLED) { 3549 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3550 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3551 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3552 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3553 3554 /* VMDQ+DCB RX and TX configurations */ 3555 vmdq_rx_conf->enable_default_pool = 0; 3556 vmdq_rx_conf->default_pool = 0; 3557 vmdq_rx_conf->nb_queue_pools = 3558 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3559 vmdq_tx_conf->nb_queue_pools = 3560 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3561 3562 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3563 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3564 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3565 vmdq_rx_conf->pool_map[i].pools = 3566 1 << (i % vmdq_rx_conf->nb_queue_pools); 3567 } 3568 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3569 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3570 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3571 } 3572 3573 /* set DCB mode of RX and TX of multiple queues */ 3574 eth_conf->rxmode.mq_mode = 3575 (enum rte_eth_rx_mq_mode) 3576 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3577 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3578 } else { 3579 struct rte_eth_dcb_rx_conf *rx_conf = 3580 ð_conf->rx_adv_conf.dcb_rx_conf; 3581 struct rte_eth_dcb_tx_conf *tx_conf = 3582 ð_conf->tx_adv_conf.dcb_tx_conf; 3583 3584 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3585 3586 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3587 if (rc != 0) 3588 return rc; 3589 3590 rx_conf->nb_tcs = num_tcs; 3591 tx_conf->nb_tcs = num_tcs; 3592 3593 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3594 rx_conf->dcb_tc[i] = i % num_tcs; 3595 tx_conf->dcb_tc[i] = i % num_tcs; 3596 } 3597 3598 eth_conf->rxmode.mq_mode = 3599 (enum rte_eth_rx_mq_mode) 3600 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3601 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3602 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3603 } 3604 3605 if (pfc_en) 3606 eth_conf->dcb_capability_en = 3607 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3608 else 3609 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3610 3611 return 0; 3612 } 3613 3614 int 3615 init_port_dcb_config(portid_t pid, 3616 enum dcb_mode_enable dcb_mode, 3617 enum rte_eth_nb_tcs num_tcs, 3618 uint8_t pfc_en) 3619 { 3620 struct rte_eth_conf port_conf; 3621 struct rte_port *rte_port; 3622 int retval; 3623 uint16_t i; 3624 3625 rte_port = &ports[pid]; 3626 3627 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3628 /* Enter DCB configuration status */ 3629 dcb_config = 1; 3630 3631 port_conf.rxmode = rte_port->dev_conf.rxmode; 3632 port_conf.txmode = rte_port->dev_conf.txmode; 3633 3634 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3635 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3636 if (retval < 0) 3637 return retval; 3638 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3639 3640 /* re-configure the device . */ 3641 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3642 if (retval < 0) 3643 return retval; 3644 3645 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3646 if (retval != 0) 3647 return retval; 3648 3649 /* If dev_info.vmdq_pool_base is greater than 0, 3650 * the queue id of vmdq pools is started after pf queues. 3651 */ 3652 if (dcb_mode == DCB_VT_ENABLED && 3653 rte_port->dev_info.vmdq_pool_base > 0) { 3654 printf("VMDQ_DCB multi-queue mode is nonsensical" 3655 " for port %d.", pid); 3656 return -1; 3657 } 3658 3659 /* Assume the ports in testpmd have the same dcb capability 3660 * and has the same number of rxq and txq in dcb mode 3661 */ 3662 if (dcb_mode == DCB_VT_ENABLED) { 3663 if (rte_port->dev_info.max_vfs > 0) { 3664 nb_rxq = rte_port->dev_info.nb_rx_queues; 3665 nb_txq = rte_port->dev_info.nb_tx_queues; 3666 } else { 3667 nb_rxq = rte_port->dev_info.max_rx_queues; 3668 nb_txq = rte_port->dev_info.max_tx_queues; 3669 } 3670 } else { 3671 /*if vt is disabled, use all pf queues */ 3672 if (rte_port->dev_info.vmdq_pool_base == 0) { 3673 nb_rxq = rte_port->dev_info.max_rx_queues; 3674 nb_txq = rte_port->dev_info.max_tx_queues; 3675 } else { 3676 nb_rxq = (queueid_t)num_tcs; 3677 nb_txq = (queueid_t)num_tcs; 3678 3679 } 3680 } 3681 rx_free_thresh = 64; 3682 3683 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3684 3685 rxtx_port_config(rte_port); 3686 /* VLAN filter */ 3687 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3688 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3689 rx_vft_set(pid, vlan_tags[i], 1); 3690 3691 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3692 if (retval != 0) 3693 return retval; 3694 3695 rte_port->dcb_flag = 1; 3696 3697 return 0; 3698 } 3699 3700 static void 3701 init_port(void) 3702 { 3703 int i; 3704 3705 /* Configuration of Ethernet ports. */ 3706 ports = rte_zmalloc("testpmd: ports", 3707 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3708 RTE_CACHE_LINE_SIZE); 3709 if (ports == NULL) { 3710 rte_exit(EXIT_FAILURE, 3711 "rte_zmalloc(%d struct rte_port) failed\n", 3712 RTE_MAX_ETHPORTS); 3713 } 3714 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3715 LIST_INIT(&ports[i].flow_tunnel_list); 3716 /* Initialize ports NUMA structures */ 3717 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3718 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3719 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3720 } 3721 3722 static void 3723 force_quit(void) 3724 { 3725 pmd_test_exit(); 3726 prompt_exit(); 3727 } 3728 3729 static void 3730 print_stats(void) 3731 { 3732 uint8_t i; 3733 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3734 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3735 3736 /* Clear screen and move to top left */ 3737 printf("%s%s", clr, top_left); 3738 3739 printf("\nPort statistics ===================================="); 3740 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3741 nic_stats_display(fwd_ports_ids[i]); 3742 3743 fflush(stdout); 3744 } 3745 3746 static void 3747 signal_handler(int signum) 3748 { 3749 if (signum == SIGINT || signum == SIGTERM) { 3750 printf("\nSignal %d received, preparing to exit...\n", 3751 signum); 3752 #ifdef RTE_LIB_PDUMP 3753 /* uninitialize packet capture framework */ 3754 rte_pdump_uninit(); 3755 #endif 3756 #ifdef RTE_LIB_LATENCYSTATS 3757 if (latencystats_enabled != 0) 3758 rte_latencystats_uninit(); 3759 #endif 3760 force_quit(); 3761 /* Set flag to indicate the force termination. */ 3762 f_quit = 1; 3763 /* exit with the expected status */ 3764 signal(signum, SIG_DFL); 3765 kill(getpid(), signum); 3766 } 3767 } 3768 3769 int 3770 main(int argc, char** argv) 3771 { 3772 int diag; 3773 portid_t port_id; 3774 uint16_t count; 3775 int ret; 3776 3777 signal(SIGINT, signal_handler); 3778 signal(SIGTERM, signal_handler); 3779 3780 testpmd_logtype = rte_log_register("testpmd"); 3781 if (testpmd_logtype < 0) 3782 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3783 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3784 3785 diag = rte_eal_init(argc, argv); 3786 if (diag < 0) 3787 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3788 rte_strerror(rte_errno)); 3789 3790 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3791 rte_exit(EXIT_FAILURE, 3792 "Secondary process type not supported.\n"); 3793 3794 ret = register_eth_event_callback(); 3795 if (ret != 0) 3796 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3797 3798 #ifdef RTE_LIB_PDUMP 3799 /* initialize packet capture framework */ 3800 rte_pdump_init(); 3801 #endif 3802 3803 count = 0; 3804 RTE_ETH_FOREACH_DEV(port_id) { 3805 ports_ids[count] = port_id; 3806 count++; 3807 } 3808 nb_ports = (portid_t) count; 3809 if (nb_ports == 0) 3810 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3811 3812 /* allocate port structures, and init them */ 3813 init_port(); 3814 3815 set_def_fwd_config(); 3816 if (nb_lcores == 0) 3817 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3818 "Check the core mask argument\n"); 3819 3820 /* Bitrate/latency stats disabled by default */ 3821 #ifdef RTE_LIB_BITRATESTATS 3822 bitrate_enabled = 0; 3823 #endif 3824 #ifdef RTE_LIB_LATENCYSTATS 3825 latencystats_enabled = 0; 3826 #endif 3827 3828 /* on FreeBSD, mlockall() is disabled by default */ 3829 #ifdef RTE_EXEC_ENV_FREEBSD 3830 do_mlockall = 0; 3831 #else 3832 do_mlockall = 1; 3833 #endif 3834 3835 argc -= diag; 3836 argv += diag; 3837 if (argc > 1) 3838 launch_args_parse(argc, argv); 3839 3840 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3841 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3842 strerror(errno)); 3843 } 3844 3845 if (tx_first && interactive) 3846 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3847 "interactive mode.\n"); 3848 3849 if (tx_first && lsc_interrupt) { 3850 printf("Warning: lsc_interrupt needs to be off when " 3851 " using tx_first. Disabling.\n"); 3852 lsc_interrupt = 0; 3853 } 3854 3855 if (!nb_rxq && !nb_txq) 3856 printf("Warning: Either rx or tx queues should be non-zero\n"); 3857 3858 if (nb_rxq > 1 && nb_rxq > nb_txq) 3859 printf("Warning: nb_rxq=%d enables RSS configuration, " 3860 "but nb_txq=%d will prevent to fully test it.\n", 3861 nb_rxq, nb_txq); 3862 3863 init_config(); 3864 3865 if (hot_plug) { 3866 ret = rte_dev_hotplug_handle_enable(); 3867 if (ret) { 3868 RTE_LOG(ERR, EAL, 3869 "fail to enable hotplug handling."); 3870 return -1; 3871 } 3872 3873 ret = rte_dev_event_monitor_start(); 3874 if (ret) { 3875 RTE_LOG(ERR, EAL, 3876 "fail to start device event monitoring."); 3877 return -1; 3878 } 3879 3880 ret = rte_dev_event_callback_register(NULL, 3881 dev_event_callback, NULL); 3882 if (ret) { 3883 RTE_LOG(ERR, EAL, 3884 "fail to register device event callback\n"); 3885 return -1; 3886 } 3887 } 3888 3889 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3890 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3891 3892 /* set all ports to promiscuous mode by default */ 3893 RTE_ETH_FOREACH_DEV(port_id) { 3894 ret = rte_eth_promiscuous_enable(port_id); 3895 if (ret != 0) 3896 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3897 port_id, rte_strerror(-ret)); 3898 } 3899 3900 /* Init metrics library */ 3901 rte_metrics_init(rte_socket_id()); 3902 3903 #ifdef RTE_LIB_LATENCYSTATS 3904 if (latencystats_enabled != 0) { 3905 int ret = rte_latencystats_init(1, NULL); 3906 if (ret) 3907 printf("Warning: latencystats init()" 3908 " returned error %d\n", ret); 3909 printf("Latencystats running on lcore %d\n", 3910 latencystats_lcore_id); 3911 } 3912 #endif 3913 3914 /* Setup bitrate stats */ 3915 #ifdef RTE_LIB_BITRATESTATS 3916 if (bitrate_enabled != 0) { 3917 bitrate_data = rte_stats_bitrate_create(); 3918 if (bitrate_data == NULL) 3919 rte_exit(EXIT_FAILURE, 3920 "Could not allocate bitrate data.\n"); 3921 rte_stats_bitrate_reg(bitrate_data); 3922 } 3923 #endif 3924 3925 #ifdef RTE_LIB_CMDLINE 3926 if (strlen(cmdline_filename) != 0) 3927 cmdline_read_from_file(cmdline_filename); 3928 3929 if (interactive == 1) { 3930 if (auto_start) { 3931 printf("Start automatic packet forwarding\n"); 3932 start_packet_forwarding(0); 3933 } 3934 prompt(); 3935 pmd_test_exit(); 3936 } else 3937 #endif 3938 { 3939 char c; 3940 int rc; 3941 3942 f_quit = 0; 3943 3944 printf("No commandline core given, start packet forwarding\n"); 3945 start_packet_forwarding(tx_first); 3946 if (stats_period != 0) { 3947 uint64_t prev_time = 0, cur_time, diff_time = 0; 3948 uint64_t timer_period; 3949 3950 /* Convert to number of cycles */ 3951 timer_period = stats_period * rte_get_timer_hz(); 3952 3953 while (f_quit == 0) { 3954 cur_time = rte_get_timer_cycles(); 3955 diff_time += cur_time - prev_time; 3956 3957 if (diff_time >= timer_period) { 3958 print_stats(); 3959 /* Reset the timer */ 3960 diff_time = 0; 3961 } 3962 /* Sleep to avoid unnecessary checks */ 3963 prev_time = cur_time; 3964 sleep(1); 3965 } 3966 } 3967 3968 printf("Press enter to exit\n"); 3969 rc = read(0, &c, 1); 3970 pmd_test_exit(); 3971 if (rc < 0) 3972 return 1; 3973 } 3974 3975 ret = rte_eal_cleanup(); 3976 if (ret != 0) 3977 rte_exit(EXIT_FAILURE, 3978 "EAL cleanup failed: %s\n", strerror(-ret)); 3979 3980 return EXIT_SUCCESS; 3981 } 3982