1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIB_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIB_LATENCYSTATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use main core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 220 221 /* 222 * Configuration of packet segments used by the "txonly" processing engine. 223 */ 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 226 TXONLY_DEF_PACKET_LEN, 227 }; 228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 229 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 231 /**< Split policy for packets to TX. */ 232 233 uint8_t txonly_multi_flow; 234 /**< Whether multiple flows are generated in TXONLY mode. */ 235 236 uint32_t tx_pkt_times_inter; 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 238 239 uint32_t tx_pkt_times_intra; 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 241 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 245 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 247 uint8_t dcb_config = 0; 248 249 /* Whether the dcb is in testing status */ 250 uint8_t dcb_test = 0; 251 252 /* 253 * Configurable number of RX/TX queues. 254 */ 255 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 256 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 257 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 258 259 /* 260 * Configurable number of RX/TX ring descriptors. 261 * Defaults are supplied by drivers via ethdev. 262 */ 263 #define RTE_TEST_RX_DESC_DEFAULT 0 264 #define RTE_TEST_TX_DESC_DEFAULT 0 265 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 266 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 267 268 #define RTE_PMD_PARAM_UNSET -1 269 /* 270 * Configurable values of RX and TX ring threshold registers. 271 */ 272 273 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 274 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 275 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 276 277 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 278 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 279 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 280 281 /* 282 * Configurable value of RX free threshold. 283 */ 284 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 285 286 /* 287 * Configurable value of RX drop enable. 288 */ 289 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 290 291 /* 292 * Configurable value of TX free threshold. 293 */ 294 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 295 296 /* 297 * Configurable value of TX RS bit threshold. 298 */ 299 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 300 301 /* 302 * Configurable value of buffered packets before sending. 303 */ 304 uint16_t noisy_tx_sw_bufsz; 305 306 /* 307 * Configurable value of packet buffer timeout. 308 */ 309 uint16_t noisy_tx_sw_buf_flush_time; 310 311 /* 312 * Configurable value for size of VNF internal memory area 313 * used for simulating noisy neighbour behaviour 314 */ 315 uint64_t noisy_lkup_mem_sz; 316 317 /* 318 * Configurable value of number of random writes done in 319 * VNF simulation memory area. 320 */ 321 uint64_t noisy_lkup_num_writes; 322 323 /* 324 * Configurable value of number of random reads done in 325 * VNF simulation memory area. 326 */ 327 uint64_t noisy_lkup_num_reads; 328 329 /* 330 * Configurable value of number of random reads/writes done in 331 * VNF simulation memory area. 332 */ 333 uint64_t noisy_lkup_num_reads_writes; 334 335 /* 336 * Receive Side Scaling (RSS) configuration. 337 */ 338 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 339 340 /* 341 * Port topology configuration 342 */ 343 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 344 345 /* 346 * Avoids to flush all the RX streams before starts forwarding. 347 */ 348 uint8_t no_flush_rx = 0; /* flush by default */ 349 350 /* 351 * Flow API isolated mode. 352 */ 353 uint8_t flow_isolate_all; 354 355 /* 356 * Avoids to check link status when starting/stopping a port. 357 */ 358 uint8_t no_link_check = 0; /* check by default */ 359 360 /* 361 * Don't automatically start all ports in interactive mode. 362 */ 363 uint8_t no_device_start = 0; 364 365 /* 366 * Enable link status change notification 367 */ 368 uint8_t lsc_interrupt = 1; /* enabled by default */ 369 370 /* 371 * Enable device removal notification. 372 */ 373 uint8_t rmv_interrupt = 1; /* enabled by default */ 374 375 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 376 377 /* After attach, port setup is called on event or by iterator */ 378 bool setup_on_probe_event = true; 379 380 /* Clear ptypes on port initialization. */ 381 uint8_t clear_ptypes = true; 382 383 /* Hairpin ports configuration mode. */ 384 uint16_t hairpin_mode; 385 386 /* Pretty printing of ethdev events */ 387 static const char * const eth_event_desc[] = { 388 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 389 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 390 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 391 [RTE_ETH_EVENT_INTR_RESET] = "reset", 392 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 393 [RTE_ETH_EVENT_IPSEC] = "IPsec", 394 [RTE_ETH_EVENT_MACSEC] = "MACsec", 395 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 396 [RTE_ETH_EVENT_NEW] = "device probed", 397 [RTE_ETH_EVENT_DESTROY] = "device released", 398 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 399 [RTE_ETH_EVENT_MAX] = NULL, 400 }; 401 402 /* 403 * Display or mask ether events 404 * Default to all events except VF_MBOX 405 */ 406 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 407 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 408 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 410 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 411 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 412 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 413 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 414 /* 415 * Decide if all memory are locked for performance. 416 */ 417 int do_mlockall = 0; 418 419 /* 420 * NIC bypass mode configuration options. 421 */ 422 423 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 424 /* The NIC bypass watchdog timeout. */ 425 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 426 #endif 427 428 429 #ifdef RTE_LIB_LATENCYSTATS 430 431 /* 432 * Set when latency stats is enabled in the commandline 433 */ 434 uint8_t latencystats_enabled; 435 436 /* 437 * Lcore ID to serive latency statistics. 438 */ 439 lcoreid_t latencystats_lcore_id = -1; 440 441 #endif 442 443 /* 444 * Ethernet device configuration. 445 */ 446 struct rte_eth_rxmode rx_mode = { 447 /* Default maximum frame length. 448 * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead" 449 * in init_config(). 450 */ 451 .max_rx_pkt_len = 0, 452 }; 453 454 struct rte_eth_txmode tx_mode = { 455 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 456 }; 457 458 struct rte_fdir_conf fdir_conf = { 459 .mode = RTE_FDIR_MODE_NONE, 460 .pballoc = RTE_FDIR_PBALLOC_64K, 461 .status = RTE_FDIR_REPORT_STATUS, 462 .mask = { 463 .vlan_tci_mask = 0xFFEF, 464 .ipv4_mask = { 465 .src_ip = 0xFFFFFFFF, 466 .dst_ip = 0xFFFFFFFF, 467 }, 468 .ipv6_mask = { 469 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 470 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 471 }, 472 .src_port_mask = 0xFFFF, 473 .dst_port_mask = 0xFFFF, 474 .mac_addr_byte_mask = 0xFF, 475 .tunnel_type_mask = 1, 476 .tunnel_id_mask = 0xFFFFFFFF, 477 }, 478 .drop_queue = 127, 479 }; 480 481 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 482 483 /* 484 * Display zero values by default for xstats 485 */ 486 uint8_t xstats_hide_zero; 487 488 /* 489 * Measure of CPU cycles disabled by default 490 */ 491 uint8_t record_core_cycles; 492 493 /* 494 * Display of RX and TX bursts disabled by default 495 */ 496 uint8_t record_burst_stats; 497 498 unsigned int num_sockets = 0; 499 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 500 501 #ifdef RTE_LIB_BITRATESTATS 502 /* Bitrate statistics */ 503 struct rte_stats_bitrates *bitrate_data; 504 lcoreid_t bitrate_lcore_id; 505 uint8_t bitrate_enabled; 506 #endif 507 508 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 509 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 510 511 /* 512 * hexadecimal bitmask of RX mq mode can be enabled. 513 */ 514 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 515 516 /* Forward function declarations */ 517 static void setup_attached_port(portid_t pi); 518 static void check_all_ports_link_status(uint32_t port_mask); 519 static int eth_event_callback(portid_t port_id, 520 enum rte_eth_event_type type, 521 void *param, void *ret_param); 522 static void dev_event_callback(const char *device_name, 523 enum rte_dev_event_type type, 524 void *param); 525 526 /* 527 * Check if all the ports are started. 528 * If yes, return positive value. If not, return zero. 529 */ 530 static int all_ports_started(void); 531 532 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 533 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 534 535 /* Holds the registered mbuf dynamic flags names. */ 536 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 537 538 /* 539 * Helper function to check if socket is already discovered. 540 * If yes, return positive value. If not, return zero. 541 */ 542 int 543 new_socket_id(unsigned int socket_id) 544 { 545 unsigned int i; 546 547 for (i = 0; i < num_sockets; i++) { 548 if (socket_ids[i] == socket_id) 549 return 0; 550 } 551 return 1; 552 } 553 554 /* 555 * Setup default configuration. 556 */ 557 static void 558 set_default_fwd_lcores_config(void) 559 { 560 unsigned int i; 561 unsigned int nb_lc; 562 unsigned int sock_num; 563 564 nb_lc = 0; 565 for (i = 0; i < RTE_MAX_LCORE; i++) { 566 if (!rte_lcore_is_enabled(i)) 567 continue; 568 sock_num = rte_lcore_to_socket_id(i); 569 if (new_socket_id(sock_num)) { 570 if (num_sockets >= RTE_MAX_NUMA_NODES) { 571 rte_exit(EXIT_FAILURE, 572 "Total sockets greater than %u\n", 573 RTE_MAX_NUMA_NODES); 574 } 575 socket_ids[num_sockets++] = sock_num; 576 } 577 if (i == rte_get_main_lcore()) 578 continue; 579 fwd_lcores_cpuids[nb_lc++] = i; 580 } 581 nb_lcores = (lcoreid_t) nb_lc; 582 nb_cfg_lcores = nb_lcores; 583 nb_fwd_lcores = 1; 584 } 585 586 static void 587 set_def_peer_eth_addrs(void) 588 { 589 portid_t i; 590 591 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 592 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 593 peer_eth_addrs[i].addr_bytes[5] = i; 594 } 595 } 596 597 static void 598 set_default_fwd_ports_config(void) 599 { 600 portid_t pt_id; 601 int i = 0; 602 603 RTE_ETH_FOREACH_DEV(pt_id) { 604 fwd_ports_ids[i++] = pt_id; 605 606 /* Update sockets info according to the attached device */ 607 int socket_id = rte_eth_dev_socket_id(pt_id); 608 if (socket_id >= 0 && new_socket_id(socket_id)) { 609 if (num_sockets >= RTE_MAX_NUMA_NODES) { 610 rte_exit(EXIT_FAILURE, 611 "Total sockets greater than %u\n", 612 RTE_MAX_NUMA_NODES); 613 } 614 socket_ids[num_sockets++] = socket_id; 615 } 616 } 617 618 nb_cfg_ports = nb_ports; 619 nb_fwd_ports = nb_ports; 620 } 621 622 void 623 set_def_fwd_config(void) 624 { 625 set_default_fwd_lcores_config(); 626 set_def_peer_eth_addrs(); 627 set_default_fwd_ports_config(); 628 } 629 630 /* extremely pessimistic estimation of memory required to create a mempool */ 631 static int 632 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 633 { 634 unsigned int n_pages, mbuf_per_pg, leftover; 635 uint64_t total_mem, mbuf_mem, obj_sz; 636 637 /* there is no good way to predict how much space the mempool will 638 * occupy because it will allocate chunks on the fly, and some of those 639 * will come from default DPDK memory while some will come from our 640 * external memory, so just assume 128MB will be enough for everyone. 641 */ 642 uint64_t hdr_mem = 128 << 20; 643 644 /* account for possible non-contiguousness */ 645 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 646 if (obj_sz > pgsz) { 647 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 648 return -1; 649 } 650 651 mbuf_per_pg = pgsz / obj_sz; 652 leftover = (nb_mbufs % mbuf_per_pg) > 0; 653 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 654 655 mbuf_mem = n_pages * pgsz; 656 657 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 658 659 if (total_mem > SIZE_MAX) { 660 TESTPMD_LOG(ERR, "Memory size too big\n"); 661 return -1; 662 } 663 *out = (size_t)total_mem; 664 665 return 0; 666 } 667 668 static int 669 pagesz_flags(uint64_t page_sz) 670 { 671 /* as per mmap() manpage, all page sizes are log2 of page size 672 * shifted by MAP_HUGE_SHIFT 673 */ 674 int log2 = rte_log2_u64(page_sz); 675 676 return (log2 << HUGE_SHIFT); 677 } 678 679 static void * 680 alloc_mem(size_t memsz, size_t pgsz, bool huge) 681 { 682 void *addr; 683 int flags; 684 685 /* allocate anonymous hugepages */ 686 flags = MAP_ANONYMOUS | MAP_PRIVATE; 687 if (huge) 688 flags |= HUGE_FLAG | pagesz_flags(pgsz); 689 690 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 691 if (addr == MAP_FAILED) 692 return NULL; 693 694 return addr; 695 } 696 697 struct extmem_param { 698 void *addr; 699 size_t len; 700 size_t pgsz; 701 rte_iova_t *iova_table; 702 unsigned int iova_table_len; 703 }; 704 705 static int 706 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 707 bool huge) 708 { 709 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 710 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 711 unsigned int cur_page, n_pages, pgsz_idx; 712 size_t mem_sz, cur_pgsz; 713 rte_iova_t *iovas = NULL; 714 void *addr; 715 int ret; 716 717 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 718 /* skip anything that is too big */ 719 if (pgsizes[pgsz_idx] > SIZE_MAX) 720 continue; 721 722 cur_pgsz = pgsizes[pgsz_idx]; 723 724 /* if we were told not to allocate hugepages, override */ 725 if (!huge) 726 cur_pgsz = sysconf(_SC_PAGESIZE); 727 728 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 729 if (ret < 0) { 730 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 731 return -1; 732 } 733 734 /* allocate our memory */ 735 addr = alloc_mem(mem_sz, cur_pgsz, huge); 736 737 /* if we couldn't allocate memory with a specified page size, 738 * that doesn't mean we can't do it with other page sizes, so 739 * try another one. 740 */ 741 if (addr == NULL) 742 continue; 743 744 /* store IOVA addresses for every page in this memory area */ 745 n_pages = mem_sz / cur_pgsz; 746 747 iovas = malloc(sizeof(*iovas) * n_pages); 748 749 if (iovas == NULL) { 750 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 751 goto fail; 752 } 753 /* lock memory if it's not huge pages */ 754 if (!huge) 755 mlock(addr, mem_sz); 756 757 /* populate IOVA addresses */ 758 for (cur_page = 0; cur_page < n_pages; cur_page++) { 759 rte_iova_t iova; 760 size_t offset; 761 void *cur; 762 763 offset = cur_pgsz * cur_page; 764 cur = RTE_PTR_ADD(addr, offset); 765 766 /* touch the page before getting its IOVA */ 767 *(volatile char *)cur = 0; 768 769 iova = rte_mem_virt2iova(cur); 770 771 iovas[cur_page] = iova; 772 } 773 774 break; 775 } 776 /* if we couldn't allocate anything */ 777 if (iovas == NULL) 778 return -1; 779 780 param->addr = addr; 781 param->len = mem_sz; 782 param->pgsz = cur_pgsz; 783 param->iova_table = iovas; 784 param->iova_table_len = n_pages; 785 786 return 0; 787 fail: 788 if (iovas) 789 free(iovas); 790 if (addr) 791 munmap(addr, mem_sz); 792 793 return -1; 794 } 795 796 static int 797 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 798 { 799 struct extmem_param param; 800 int socket_id, ret; 801 802 memset(¶m, 0, sizeof(param)); 803 804 /* check if our heap exists */ 805 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 806 if (socket_id < 0) { 807 /* create our heap */ 808 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 809 if (ret < 0) { 810 TESTPMD_LOG(ERR, "Cannot create heap\n"); 811 return -1; 812 } 813 } 814 815 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 816 if (ret < 0) { 817 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 818 return -1; 819 } 820 821 /* we now have a valid memory area, so add it to heap */ 822 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 823 param.addr, param.len, param.iova_table, 824 param.iova_table_len, param.pgsz); 825 826 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 827 828 /* not needed any more */ 829 free(param.iova_table); 830 831 if (ret < 0) { 832 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 833 munmap(param.addr, param.len); 834 return -1; 835 } 836 837 /* success */ 838 839 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 840 param.len >> 20); 841 842 return 0; 843 } 844 static void 845 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 846 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 847 { 848 uint16_t pid = 0; 849 int ret; 850 851 RTE_ETH_FOREACH_DEV(pid) { 852 struct rte_eth_dev *dev = 853 &rte_eth_devices[pid]; 854 855 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 856 memhdr->len); 857 if (ret) { 858 TESTPMD_LOG(DEBUG, 859 "unable to DMA unmap addr 0x%p " 860 "for device %s\n", 861 memhdr->addr, dev->data->name); 862 } 863 } 864 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 865 if (ret) { 866 TESTPMD_LOG(DEBUG, 867 "unable to un-register addr 0x%p\n", memhdr->addr); 868 } 869 } 870 871 static void 872 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 873 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 874 { 875 uint16_t pid = 0; 876 size_t page_size = sysconf(_SC_PAGESIZE); 877 int ret; 878 879 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 880 page_size); 881 if (ret) { 882 TESTPMD_LOG(DEBUG, 883 "unable to register addr 0x%p\n", memhdr->addr); 884 return; 885 } 886 RTE_ETH_FOREACH_DEV(pid) { 887 struct rte_eth_dev *dev = 888 &rte_eth_devices[pid]; 889 890 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 891 memhdr->len); 892 if (ret) { 893 TESTPMD_LOG(DEBUG, 894 "unable to DMA map addr 0x%p " 895 "for device %s\n", 896 memhdr->addr, dev->data->name); 897 } 898 } 899 } 900 901 static unsigned int 902 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 903 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 904 { 905 struct rte_pktmbuf_extmem *xmem; 906 unsigned int ext_num, zone_num, elt_num; 907 uint16_t elt_size; 908 909 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 910 elt_num = EXTBUF_ZONE_SIZE / elt_size; 911 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 912 913 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 914 if (xmem == NULL) { 915 TESTPMD_LOG(ERR, "Cannot allocate memory for " 916 "external buffer descriptors\n"); 917 *ext_mem = NULL; 918 return 0; 919 } 920 for (ext_num = 0; ext_num < zone_num; ext_num++) { 921 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 922 const struct rte_memzone *mz; 923 char mz_name[RTE_MEMZONE_NAMESIZE]; 924 int ret; 925 926 ret = snprintf(mz_name, sizeof(mz_name), 927 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 928 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 929 errno = ENAMETOOLONG; 930 ext_num = 0; 931 break; 932 } 933 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 934 socket_id, 935 RTE_MEMZONE_IOVA_CONTIG | 936 RTE_MEMZONE_1GB | 937 RTE_MEMZONE_SIZE_HINT_ONLY, 938 EXTBUF_ZONE_SIZE); 939 if (mz == NULL) { 940 /* 941 * The caller exits on external buffer creation 942 * error, so there is no need to free memzones. 943 */ 944 errno = ENOMEM; 945 ext_num = 0; 946 break; 947 } 948 xseg->buf_ptr = mz->addr; 949 xseg->buf_iova = mz->iova; 950 xseg->buf_len = EXTBUF_ZONE_SIZE; 951 xseg->elt_size = elt_size; 952 } 953 if (ext_num == 0 && xmem != NULL) { 954 free(xmem); 955 xmem = NULL; 956 } 957 *ext_mem = xmem; 958 return ext_num; 959 } 960 961 /* 962 * Configuration initialisation done once at init time. 963 */ 964 static struct rte_mempool * 965 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 966 unsigned int socket_id, uint16_t size_idx) 967 { 968 char pool_name[RTE_MEMPOOL_NAMESIZE]; 969 struct rte_mempool *rte_mp = NULL; 970 uint32_t mb_size; 971 972 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 973 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 974 975 TESTPMD_LOG(INFO, 976 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 977 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 978 979 switch (mp_alloc_type) { 980 case MP_ALLOC_NATIVE: 981 { 982 /* wrapper to rte_mempool_create() */ 983 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 984 rte_mbuf_best_mempool_ops()); 985 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 986 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 987 break; 988 } 989 case MP_ALLOC_ANON: 990 { 991 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 992 mb_size, (unsigned int) mb_mempool_cache, 993 sizeof(struct rte_pktmbuf_pool_private), 994 socket_id, mempool_flags); 995 if (rte_mp == NULL) 996 goto err; 997 998 if (rte_mempool_populate_anon(rte_mp) == 0) { 999 rte_mempool_free(rte_mp); 1000 rte_mp = NULL; 1001 goto err; 1002 } 1003 rte_pktmbuf_pool_init(rte_mp, NULL); 1004 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1005 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1006 break; 1007 } 1008 case MP_ALLOC_XMEM: 1009 case MP_ALLOC_XMEM_HUGE: 1010 { 1011 int heap_socket; 1012 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1013 1014 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1015 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1016 1017 heap_socket = 1018 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1019 if (heap_socket < 0) 1020 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1021 1022 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1023 rte_mbuf_best_mempool_ops()); 1024 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1025 mb_mempool_cache, 0, mbuf_seg_size, 1026 heap_socket); 1027 break; 1028 } 1029 case MP_ALLOC_XBUF: 1030 { 1031 struct rte_pktmbuf_extmem *ext_mem; 1032 unsigned int ext_num; 1033 1034 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1035 socket_id, pool_name, &ext_mem); 1036 if (ext_num == 0) 1037 rte_exit(EXIT_FAILURE, 1038 "Can't create pinned data buffers\n"); 1039 1040 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1041 rte_mbuf_best_mempool_ops()); 1042 rte_mp = rte_pktmbuf_pool_create_extbuf 1043 (pool_name, nb_mbuf, mb_mempool_cache, 1044 0, mbuf_seg_size, socket_id, 1045 ext_mem, ext_num); 1046 free(ext_mem); 1047 break; 1048 } 1049 default: 1050 { 1051 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1052 } 1053 } 1054 1055 err: 1056 if (rte_mp == NULL) { 1057 rte_exit(EXIT_FAILURE, 1058 "Creation of mbuf pool for socket %u failed: %s\n", 1059 socket_id, rte_strerror(rte_errno)); 1060 } else if (verbose_level > 0) { 1061 rte_mempool_dump(stdout, rte_mp); 1062 } 1063 return rte_mp; 1064 } 1065 1066 /* 1067 * Check given socket id is valid or not with NUMA mode, 1068 * if valid, return 0, else return -1 1069 */ 1070 static int 1071 check_socket_id(const unsigned int socket_id) 1072 { 1073 static int warning_once = 0; 1074 1075 if (new_socket_id(socket_id)) { 1076 if (!warning_once && numa_support) 1077 printf("Warning: NUMA should be configured manually by" 1078 " using --port-numa-config and" 1079 " --ring-numa-config parameters along with" 1080 " --numa.\n"); 1081 warning_once = 1; 1082 return -1; 1083 } 1084 return 0; 1085 } 1086 1087 /* 1088 * Get the allowed maximum number of RX queues. 1089 * *pid return the port id which has minimal value of 1090 * max_rx_queues in all ports. 1091 */ 1092 queueid_t 1093 get_allowed_max_nb_rxq(portid_t *pid) 1094 { 1095 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1096 bool max_rxq_valid = false; 1097 portid_t pi; 1098 struct rte_eth_dev_info dev_info; 1099 1100 RTE_ETH_FOREACH_DEV(pi) { 1101 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1102 continue; 1103 1104 max_rxq_valid = true; 1105 if (dev_info.max_rx_queues < allowed_max_rxq) { 1106 allowed_max_rxq = dev_info.max_rx_queues; 1107 *pid = pi; 1108 } 1109 } 1110 return max_rxq_valid ? allowed_max_rxq : 0; 1111 } 1112 1113 /* 1114 * Check input rxq is valid or not. 1115 * If input rxq is not greater than any of maximum number 1116 * of RX queues of all ports, it is valid. 1117 * if valid, return 0, else return -1 1118 */ 1119 int 1120 check_nb_rxq(queueid_t rxq) 1121 { 1122 queueid_t allowed_max_rxq; 1123 portid_t pid = 0; 1124 1125 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1126 if (rxq > allowed_max_rxq) { 1127 printf("Fail: input rxq (%u) can't be greater " 1128 "than max_rx_queues (%u) of port %u\n", 1129 rxq, 1130 allowed_max_rxq, 1131 pid); 1132 return -1; 1133 } 1134 return 0; 1135 } 1136 1137 /* 1138 * Get the allowed maximum number of TX queues. 1139 * *pid return the port id which has minimal value of 1140 * max_tx_queues in all ports. 1141 */ 1142 queueid_t 1143 get_allowed_max_nb_txq(portid_t *pid) 1144 { 1145 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1146 bool max_txq_valid = false; 1147 portid_t pi; 1148 struct rte_eth_dev_info dev_info; 1149 1150 RTE_ETH_FOREACH_DEV(pi) { 1151 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1152 continue; 1153 1154 max_txq_valid = true; 1155 if (dev_info.max_tx_queues < allowed_max_txq) { 1156 allowed_max_txq = dev_info.max_tx_queues; 1157 *pid = pi; 1158 } 1159 } 1160 return max_txq_valid ? allowed_max_txq : 0; 1161 } 1162 1163 /* 1164 * Check input txq is valid or not. 1165 * If input txq is not greater than any of maximum number 1166 * of TX queues of all ports, it is valid. 1167 * if valid, return 0, else return -1 1168 */ 1169 int 1170 check_nb_txq(queueid_t txq) 1171 { 1172 queueid_t allowed_max_txq; 1173 portid_t pid = 0; 1174 1175 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1176 if (txq > allowed_max_txq) { 1177 printf("Fail: input txq (%u) can't be greater " 1178 "than max_tx_queues (%u) of port %u\n", 1179 txq, 1180 allowed_max_txq, 1181 pid); 1182 return -1; 1183 } 1184 return 0; 1185 } 1186 1187 /* 1188 * Get the allowed maximum number of RXDs of every rx queue. 1189 * *pid return the port id which has minimal value of 1190 * max_rxd in all queues of all ports. 1191 */ 1192 static uint16_t 1193 get_allowed_max_nb_rxd(portid_t *pid) 1194 { 1195 uint16_t allowed_max_rxd = UINT16_MAX; 1196 portid_t pi; 1197 struct rte_eth_dev_info dev_info; 1198 1199 RTE_ETH_FOREACH_DEV(pi) { 1200 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1201 continue; 1202 1203 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1204 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1205 *pid = pi; 1206 } 1207 } 1208 return allowed_max_rxd; 1209 } 1210 1211 /* 1212 * Get the allowed minimal number of RXDs of every rx queue. 1213 * *pid return the port id which has minimal value of 1214 * min_rxd in all queues of all ports. 1215 */ 1216 static uint16_t 1217 get_allowed_min_nb_rxd(portid_t *pid) 1218 { 1219 uint16_t allowed_min_rxd = 0; 1220 portid_t pi; 1221 struct rte_eth_dev_info dev_info; 1222 1223 RTE_ETH_FOREACH_DEV(pi) { 1224 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1225 continue; 1226 1227 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1228 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1229 *pid = pi; 1230 } 1231 } 1232 1233 return allowed_min_rxd; 1234 } 1235 1236 /* 1237 * Check input rxd is valid or not. 1238 * If input rxd is not greater than any of maximum number 1239 * of RXDs of every Rx queues and is not less than any of 1240 * minimal number of RXDs of every Rx queues, it is valid. 1241 * if valid, return 0, else return -1 1242 */ 1243 int 1244 check_nb_rxd(queueid_t rxd) 1245 { 1246 uint16_t allowed_max_rxd; 1247 uint16_t allowed_min_rxd; 1248 portid_t pid = 0; 1249 1250 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1251 if (rxd > allowed_max_rxd) { 1252 printf("Fail: input rxd (%u) can't be greater " 1253 "than max_rxds (%u) of port %u\n", 1254 rxd, 1255 allowed_max_rxd, 1256 pid); 1257 return -1; 1258 } 1259 1260 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1261 if (rxd < allowed_min_rxd) { 1262 printf("Fail: input rxd (%u) can't be less " 1263 "than min_rxds (%u) of port %u\n", 1264 rxd, 1265 allowed_min_rxd, 1266 pid); 1267 return -1; 1268 } 1269 1270 return 0; 1271 } 1272 1273 /* 1274 * Get the allowed maximum number of TXDs of every rx queues. 1275 * *pid return the port id which has minimal value of 1276 * max_txd in every tx queue. 1277 */ 1278 static uint16_t 1279 get_allowed_max_nb_txd(portid_t *pid) 1280 { 1281 uint16_t allowed_max_txd = UINT16_MAX; 1282 portid_t pi; 1283 struct rte_eth_dev_info dev_info; 1284 1285 RTE_ETH_FOREACH_DEV(pi) { 1286 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1287 continue; 1288 1289 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1290 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1291 *pid = pi; 1292 } 1293 } 1294 return allowed_max_txd; 1295 } 1296 1297 /* 1298 * Get the allowed maximum number of TXDs of every tx queues. 1299 * *pid return the port id which has minimal value of 1300 * min_txd in every tx queue. 1301 */ 1302 static uint16_t 1303 get_allowed_min_nb_txd(portid_t *pid) 1304 { 1305 uint16_t allowed_min_txd = 0; 1306 portid_t pi; 1307 struct rte_eth_dev_info dev_info; 1308 1309 RTE_ETH_FOREACH_DEV(pi) { 1310 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1311 continue; 1312 1313 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1314 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1315 *pid = pi; 1316 } 1317 } 1318 1319 return allowed_min_txd; 1320 } 1321 1322 /* 1323 * Check input txd is valid or not. 1324 * If input txd is not greater than any of maximum number 1325 * of TXDs of every Rx queues, it is valid. 1326 * if valid, return 0, else return -1 1327 */ 1328 int 1329 check_nb_txd(queueid_t txd) 1330 { 1331 uint16_t allowed_max_txd; 1332 uint16_t allowed_min_txd; 1333 portid_t pid = 0; 1334 1335 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1336 if (txd > allowed_max_txd) { 1337 printf("Fail: input txd (%u) can't be greater " 1338 "than max_txds (%u) of port %u\n", 1339 txd, 1340 allowed_max_txd, 1341 pid); 1342 return -1; 1343 } 1344 1345 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1346 if (txd < allowed_min_txd) { 1347 printf("Fail: input txd (%u) can't be less " 1348 "than min_txds (%u) of port %u\n", 1349 txd, 1350 allowed_min_txd, 1351 pid); 1352 return -1; 1353 } 1354 return 0; 1355 } 1356 1357 1358 /* 1359 * Get the allowed maximum number of hairpin queues. 1360 * *pid return the port id which has minimal value of 1361 * max_hairpin_queues in all ports. 1362 */ 1363 queueid_t 1364 get_allowed_max_nb_hairpinq(portid_t *pid) 1365 { 1366 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1367 portid_t pi; 1368 struct rte_eth_hairpin_cap cap; 1369 1370 RTE_ETH_FOREACH_DEV(pi) { 1371 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1372 *pid = pi; 1373 return 0; 1374 } 1375 if (cap.max_nb_queues < allowed_max_hairpinq) { 1376 allowed_max_hairpinq = cap.max_nb_queues; 1377 *pid = pi; 1378 } 1379 } 1380 return allowed_max_hairpinq; 1381 } 1382 1383 /* 1384 * Check input hairpin is valid or not. 1385 * If input hairpin is not greater than any of maximum number 1386 * of hairpin queues of all ports, it is valid. 1387 * if valid, return 0, else return -1 1388 */ 1389 int 1390 check_nb_hairpinq(queueid_t hairpinq) 1391 { 1392 queueid_t allowed_max_hairpinq; 1393 portid_t pid = 0; 1394 1395 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1396 if (hairpinq > allowed_max_hairpinq) { 1397 printf("Fail: input hairpin (%u) can't be greater " 1398 "than max_hairpin_queues (%u) of port %u\n", 1399 hairpinq, allowed_max_hairpinq, pid); 1400 return -1; 1401 } 1402 return 0; 1403 } 1404 1405 static void 1406 init_config(void) 1407 { 1408 portid_t pid; 1409 struct rte_port *port; 1410 struct rte_mempool *mbp; 1411 unsigned int nb_mbuf_per_pool; 1412 lcoreid_t lc_id; 1413 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1414 struct rte_gro_param gro_param; 1415 uint32_t gso_types; 1416 uint16_t data_size; 1417 bool warning = 0; 1418 int k; 1419 int ret; 1420 1421 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1422 1423 /* Configuration of logical cores. */ 1424 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1425 sizeof(struct fwd_lcore *) * nb_lcores, 1426 RTE_CACHE_LINE_SIZE); 1427 if (fwd_lcores == NULL) { 1428 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1429 "failed\n", nb_lcores); 1430 } 1431 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1432 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1433 sizeof(struct fwd_lcore), 1434 RTE_CACHE_LINE_SIZE); 1435 if (fwd_lcores[lc_id] == NULL) { 1436 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1437 "failed\n"); 1438 } 1439 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1440 } 1441 1442 RTE_ETH_FOREACH_DEV(pid) { 1443 port = &ports[pid]; 1444 /* Apply default TxRx configuration for all ports */ 1445 port->dev_conf.txmode = tx_mode; 1446 port->dev_conf.rxmode = rx_mode; 1447 1448 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1449 if (ret != 0) 1450 rte_exit(EXIT_FAILURE, 1451 "rte_eth_dev_info_get() failed\n"); 1452 1453 ret = update_jumbo_frame_offload(pid); 1454 if (ret != 0) 1455 printf("Updating jumbo frame offload failed for port %u\n", 1456 pid); 1457 1458 if (!(port->dev_info.tx_offload_capa & 1459 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1460 port->dev_conf.txmode.offloads &= 1461 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1462 if (numa_support) { 1463 if (port_numa[pid] != NUMA_NO_CONFIG) 1464 port_per_socket[port_numa[pid]]++; 1465 else { 1466 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1467 1468 /* 1469 * if socket_id is invalid, 1470 * set to the first available socket. 1471 */ 1472 if (check_socket_id(socket_id) < 0) 1473 socket_id = socket_ids[0]; 1474 port_per_socket[socket_id]++; 1475 } 1476 } 1477 1478 /* Apply Rx offloads configuration */ 1479 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1480 port->rx_conf[k].offloads = 1481 port->dev_conf.rxmode.offloads; 1482 /* Apply Tx offloads configuration */ 1483 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1484 port->tx_conf[k].offloads = 1485 port->dev_conf.txmode.offloads; 1486 1487 /* set flag to initialize port/queue */ 1488 port->need_reconfig = 1; 1489 port->need_reconfig_queues = 1; 1490 port->tx_metadata = 0; 1491 1492 /* Check for maximum number of segments per MTU. Accordingly 1493 * update the mbuf data size. 1494 */ 1495 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1496 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1497 data_size = rx_mode.max_rx_pkt_len / 1498 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1499 1500 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1501 mbuf_data_size[0]) { 1502 mbuf_data_size[0] = data_size + 1503 RTE_PKTMBUF_HEADROOM; 1504 warning = 1; 1505 } 1506 } 1507 } 1508 1509 if (warning) 1510 TESTPMD_LOG(WARNING, 1511 "Configured mbuf size of the first segment %hu\n", 1512 mbuf_data_size[0]); 1513 /* 1514 * Create pools of mbuf. 1515 * If NUMA support is disabled, create a single pool of mbuf in 1516 * socket 0 memory by default. 1517 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1518 * 1519 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1520 * nb_txd can be configured at run time. 1521 */ 1522 if (param_total_num_mbufs) 1523 nb_mbuf_per_pool = param_total_num_mbufs; 1524 else { 1525 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1526 (nb_lcores * mb_mempool_cache) + 1527 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1528 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1529 } 1530 1531 if (numa_support) { 1532 uint8_t i, j; 1533 1534 for (i = 0; i < num_sockets; i++) 1535 for (j = 0; j < mbuf_data_size_n; j++) 1536 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1537 mbuf_pool_create(mbuf_data_size[j], 1538 nb_mbuf_per_pool, 1539 socket_ids[i], j); 1540 } else { 1541 uint8_t i; 1542 1543 for (i = 0; i < mbuf_data_size_n; i++) 1544 mempools[i] = mbuf_pool_create 1545 (mbuf_data_size[i], 1546 nb_mbuf_per_pool, 1547 socket_num == UMA_NO_CONFIG ? 1548 0 : socket_num, i); 1549 } 1550 1551 init_port_config(); 1552 1553 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1554 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1555 /* 1556 * Records which Mbuf pool to use by each logical core, if needed. 1557 */ 1558 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1559 mbp = mbuf_pool_find( 1560 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1561 1562 if (mbp == NULL) 1563 mbp = mbuf_pool_find(0, 0); 1564 fwd_lcores[lc_id]->mbp = mbp; 1565 /* initialize GSO context */ 1566 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1567 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1568 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1569 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1570 RTE_ETHER_CRC_LEN; 1571 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1572 } 1573 1574 /* Configuration of packet forwarding streams. */ 1575 if (init_fwd_streams() < 0) 1576 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1577 1578 fwd_config_setup(); 1579 1580 /* create a gro context for each lcore */ 1581 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1582 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1583 gro_param.max_item_per_flow = MAX_PKT_BURST; 1584 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1585 gro_param.socket_id = rte_lcore_to_socket_id( 1586 fwd_lcores_cpuids[lc_id]); 1587 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1588 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1589 rte_exit(EXIT_FAILURE, 1590 "rte_gro_ctx_create() failed\n"); 1591 } 1592 } 1593 } 1594 1595 1596 void 1597 reconfig(portid_t new_port_id, unsigned socket_id) 1598 { 1599 struct rte_port *port; 1600 int ret; 1601 1602 /* Reconfiguration of Ethernet ports. */ 1603 port = &ports[new_port_id]; 1604 1605 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1606 if (ret != 0) 1607 return; 1608 1609 /* set flag to initialize port/queue */ 1610 port->need_reconfig = 1; 1611 port->need_reconfig_queues = 1; 1612 port->socket_id = socket_id; 1613 1614 init_port_config(); 1615 } 1616 1617 1618 int 1619 init_fwd_streams(void) 1620 { 1621 portid_t pid; 1622 struct rte_port *port; 1623 streamid_t sm_id, nb_fwd_streams_new; 1624 queueid_t q; 1625 1626 /* set socket id according to numa or not */ 1627 RTE_ETH_FOREACH_DEV(pid) { 1628 port = &ports[pid]; 1629 if (nb_rxq > port->dev_info.max_rx_queues) { 1630 printf("Fail: nb_rxq(%d) is greater than " 1631 "max_rx_queues(%d)\n", nb_rxq, 1632 port->dev_info.max_rx_queues); 1633 return -1; 1634 } 1635 if (nb_txq > port->dev_info.max_tx_queues) { 1636 printf("Fail: nb_txq(%d) is greater than " 1637 "max_tx_queues(%d)\n", nb_txq, 1638 port->dev_info.max_tx_queues); 1639 return -1; 1640 } 1641 if (numa_support) { 1642 if (port_numa[pid] != NUMA_NO_CONFIG) 1643 port->socket_id = port_numa[pid]; 1644 else { 1645 port->socket_id = rte_eth_dev_socket_id(pid); 1646 1647 /* 1648 * if socket_id is invalid, 1649 * set to the first available socket. 1650 */ 1651 if (check_socket_id(port->socket_id) < 0) 1652 port->socket_id = socket_ids[0]; 1653 } 1654 } 1655 else { 1656 if (socket_num == UMA_NO_CONFIG) 1657 port->socket_id = 0; 1658 else 1659 port->socket_id = socket_num; 1660 } 1661 } 1662 1663 q = RTE_MAX(nb_rxq, nb_txq); 1664 if (q == 0) { 1665 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1666 return -1; 1667 } 1668 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1669 if (nb_fwd_streams_new == nb_fwd_streams) 1670 return 0; 1671 /* clear the old */ 1672 if (fwd_streams != NULL) { 1673 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1674 if (fwd_streams[sm_id] == NULL) 1675 continue; 1676 rte_free(fwd_streams[sm_id]); 1677 fwd_streams[sm_id] = NULL; 1678 } 1679 rte_free(fwd_streams); 1680 fwd_streams = NULL; 1681 } 1682 1683 /* init new */ 1684 nb_fwd_streams = nb_fwd_streams_new; 1685 if (nb_fwd_streams) { 1686 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1687 sizeof(struct fwd_stream *) * nb_fwd_streams, 1688 RTE_CACHE_LINE_SIZE); 1689 if (fwd_streams == NULL) 1690 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1691 " (struct fwd_stream *)) failed\n", 1692 nb_fwd_streams); 1693 1694 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1695 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1696 " struct fwd_stream", sizeof(struct fwd_stream), 1697 RTE_CACHE_LINE_SIZE); 1698 if (fwd_streams[sm_id] == NULL) 1699 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1700 "(struct fwd_stream) failed\n"); 1701 } 1702 } 1703 1704 return 0; 1705 } 1706 1707 static void 1708 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1709 { 1710 uint64_t total_burst, sburst; 1711 uint64_t nb_burst; 1712 uint64_t burst_stats[4]; 1713 uint16_t pktnb_stats[4]; 1714 uint16_t nb_pkt; 1715 int burst_percent[4], sburstp; 1716 int i; 1717 1718 /* 1719 * First compute the total number of packet bursts and the 1720 * two highest numbers of bursts of the same number of packets. 1721 */ 1722 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1723 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1724 1725 /* Show stats for 0 burst size always */ 1726 total_burst = pbs->pkt_burst_spread[0]; 1727 burst_stats[0] = pbs->pkt_burst_spread[0]; 1728 pktnb_stats[0] = 0; 1729 1730 /* Find the next 2 burst sizes with highest occurrences. */ 1731 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1732 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1733 1734 if (nb_burst == 0) 1735 continue; 1736 1737 total_burst += nb_burst; 1738 1739 if (nb_burst > burst_stats[1]) { 1740 burst_stats[2] = burst_stats[1]; 1741 pktnb_stats[2] = pktnb_stats[1]; 1742 burst_stats[1] = nb_burst; 1743 pktnb_stats[1] = nb_pkt; 1744 } else if (nb_burst > burst_stats[2]) { 1745 burst_stats[2] = nb_burst; 1746 pktnb_stats[2] = nb_pkt; 1747 } 1748 } 1749 if (total_burst == 0) 1750 return; 1751 1752 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1753 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1754 if (i == 3) { 1755 printf("%d%% of other]\n", 100 - sburstp); 1756 return; 1757 } 1758 1759 sburst += burst_stats[i]; 1760 if (sburst == total_burst) { 1761 printf("%d%% of %d pkts]\n", 1762 100 - sburstp, (int) pktnb_stats[i]); 1763 return; 1764 } 1765 1766 burst_percent[i] = 1767 (double)burst_stats[i] / total_burst * 100; 1768 printf("%d%% of %d pkts + ", 1769 burst_percent[i], (int) pktnb_stats[i]); 1770 sburstp += burst_percent[i]; 1771 } 1772 } 1773 1774 static void 1775 fwd_stream_stats_display(streamid_t stream_id) 1776 { 1777 struct fwd_stream *fs; 1778 static const char *fwd_top_stats_border = "-------"; 1779 1780 fs = fwd_streams[stream_id]; 1781 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1782 (fs->fwd_dropped == 0)) 1783 return; 1784 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1785 "TX Port=%2d/Queue=%2d %s\n", 1786 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1787 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1788 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1789 " TX-dropped: %-14"PRIu64, 1790 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1791 1792 /* if checksum mode */ 1793 if (cur_fwd_eng == &csum_fwd_engine) { 1794 printf(" RX- bad IP checksum: %-14"PRIu64 1795 " Rx- bad L4 checksum: %-14"PRIu64 1796 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1797 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1798 fs->rx_bad_outer_l4_csum); 1799 } else { 1800 printf("\n"); 1801 } 1802 1803 if (record_burst_stats) { 1804 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1805 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1806 } 1807 } 1808 1809 void 1810 fwd_stats_display(void) 1811 { 1812 static const char *fwd_stats_border = "----------------------"; 1813 static const char *acc_stats_border = "+++++++++++++++"; 1814 struct { 1815 struct fwd_stream *rx_stream; 1816 struct fwd_stream *tx_stream; 1817 uint64_t tx_dropped; 1818 uint64_t rx_bad_ip_csum; 1819 uint64_t rx_bad_l4_csum; 1820 uint64_t rx_bad_outer_l4_csum; 1821 } ports_stats[RTE_MAX_ETHPORTS]; 1822 uint64_t total_rx_dropped = 0; 1823 uint64_t total_tx_dropped = 0; 1824 uint64_t total_rx_nombuf = 0; 1825 struct rte_eth_stats stats; 1826 uint64_t fwd_cycles = 0; 1827 uint64_t total_recv = 0; 1828 uint64_t total_xmit = 0; 1829 struct rte_port *port; 1830 streamid_t sm_id; 1831 portid_t pt_id; 1832 int i; 1833 1834 memset(ports_stats, 0, sizeof(ports_stats)); 1835 1836 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1837 struct fwd_stream *fs = fwd_streams[sm_id]; 1838 1839 if (cur_fwd_config.nb_fwd_streams > 1840 cur_fwd_config.nb_fwd_ports) { 1841 fwd_stream_stats_display(sm_id); 1842 } else { 1843 ports_stats[fs->tx_port].tx_stream = fs; 1844 ports_stats[fs->rx_port].rx_stream = fs; 1845 } 1846 1847 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1848 1849 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1850 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1851 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1852 fs->rx_bad_outer_l4_csum; 1853 1854 if (record_core_cycles) 1855 fwd_cycles += fs->core_cycles; 1856 } 1857 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1858 pt_id = fwd_ports_ids[i]; 1859 port = &ports[pt_id]; 1860 1861 rte_eth_stats_get(pt_id, &stats); 1862 stats.ipackets -= port->stats.ipackets; 1863 stats.opackets -= port->stats.opackets; 1864 stats.ibytes -= port->stats.ibytes; 1865 stats.obytes -= port->stats.obytes; 1866 stats.imissed -= port->stats.imissed; 1867 stats.oerrors -= port->stats.oerrors; 1868 stats.rx_nombuf -= port->stats.rx_nombuf; 1869 1870 total_recv += stats.ipackets; 1871 total_xmit += stats.opackets; 1872 total_rx_dropped += stats.imissed; 1873 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1874 total_tx_dropped += stats.oerrors; 1875 total_rx_nombuf += stats.rx_nombuf; 1876 1877 printf("\n %s Forward statistics for port %-2d %s\n", 1878 fwd_stats_border, pt_id, fwd_stats_border); 1879 1880 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 1881 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 1882 stats.ipackets + stats.imissed); 1883 1884 if (cur_fwd_eng == &csum_fwd_engine) 1885 printf(" Bad-ipcsum: %-14"PRIu64 1886 " Bad-l4csum: %-14"PRIu64 1887 "Bad-outer-l4csum: %-14"PRIu64"\n", 1888 ports_stats[pt_id].rx_bad_ip_csum, 1889 ports_stats[pt_id].rx_bad_l4_csum, 1890 ports_stats[pt_id].rx_bad_outer_l4_csum); 1891 if (stats.ierrors + stats.rx_nombuf > 0) { 1892 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 1893 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 1894 } 1895 1896 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 1897 "TX-total: %-"PRIu64"\n", 1898 stats.opackets, ports_stats[pt_id].tx_dropped, 1899 stats.opackets + ports_stats[pt_id].tx_dropped); 1900 1901 if (record_burst_stats) { 1902 if (ports_stats[pt_id].rx_stream) 1903 pkt_burst_stats_display("RX", 1904 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1905 if (ports_stats[pt_id].tx_stream) 1906 pkt_burst_stats_display("TX", 1907 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1908 } 1909 1910 printf(" %s--------------------------------%s\n", 1911 fwd_stats_border, fwd_stats_border); 1912 } 1913 1914 printf("\n %s Accumulated forward statistics for all ports" 1915 "%s\n", 1916 acc_stats_border, acc_stats_border); 1917 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1918 "%-"PRIu64"\n" 1919 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1920 "%-"PRIu64"\n", 1921 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1922 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1923 if (total_rx_nombuf > 0) 1924 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1925 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1926 "%s\n", 1927 acc_stats_border, acc_stats_border); 1928 if (record_core_cycles) { 1929 #define CYC_PER_MHZ 1E6 1930 if (total_recv > 0 || total_xmit > 0) { 1931 uint64_t total_pkts = 0; 1932 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1933 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1934 total_pkts = total_xmit; 1935 else 1936 total_pkts = total_recv; 1937 1938 printf("\n CPU cycles/packet=%.2F (total cycles=" 1939 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1940 " MHz Clock\n", 1941 (double) fwd_cycles / total_pkts, 1942 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1943 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1944 } 1945 } 1946 } 1947 1948 void 1949 fwd_stats_reset(void) 1950 { 1951 streamid_t sm_id; 1952 portid_t pt_id; 1953 int i; 1954 1955 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1956 pt_id = fwd_ports_ids[i]; 1957 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1958 } 1959 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1960 struct fwd_stream *fs = fwd_streams[sm_id]; 1961 1962 fs->rx_packets = 0; 1963 fs->tx_packets = 0; 1964 fs->fwd_dropped = 0; 1965 fs->rx_bad_ip_csum = 0; 1966 fs->rx_bad_l4_csum = 0; 1967 fs->rx_bad_outer_l4_csum = 0; 1968 1969 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1970 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1971 fs->core_cycles = 0; 1972 } 1973 } 1974 1975 static void 1976 flush_fwd_rx_queues(void) 1977 { 1978 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1979 portid_t rxp; 1980 portid_t port_id; 1981 queueid_t rxq; 1982 uint16_t nb_rx; 1983 uint16_t i; 1984 uint8_t j; 1985 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1986 uint64_t timer_period; 1987 1988 /* convert to number of cycles */ 1989 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1990 1991 for (j = 0; j < 2; j++) { 1992 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1993 for (rxq = 0; rxq < nb_rxq; rxq++) { 1994 port_id = fwd_ports_ids[rxp]; 1995 /** 1996 * testpmd can stuck in the below do while loop 1997 * if rte_eth_rx_burst() always returns nonzero 1998 * packets. So timer is added to exit this loop 1999 * after 1sec timer expiry. 2000 */ 2001 prev_tsc = rte_rdtsc(); 2002 do { 2003 nb_rx = rte_eth_rx_burst(port_id, rxq, 2004 pkts_burst, MAX_PKT_BURST); 2005 for (i = 0; i < nb_rx; i++) 2006 rte_pktmbuf_free(pkts_burst[i]); 2007 2008 cur_tsc = rte_rdtsc(); 2009 diff_tsc = cur_tsc - prev_tsc; 2010 timer_tsc += diff_tsc; 2011 } while ((nb_rx > 0) && 2012 (timer_tsc < timer_period)); 2013 timer_tsc = 0; 2014 } 2015 } 2016 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2017 } 2018 } 2019 2020 static void 2021 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2022 { 2023 struct fwd_stream **fsm; 2024 streamid_t nb_fs; 2025 streamid_t sm_id; 2026 #ifdef RTE_LIB_BITRATESTATS 2027 uint64_t tics_per_1sec; 2028 uint64_t tics_datum; 2029 uint64_t tics_current; 2030 uint16_t i, cnt_ports; 2031 2032 cnt_ports = nb_ports; 2033 tics_datum = rte_rdtsc(); 2034 tics_per_1sec = rte_get_timer_hz(); 2035 #endif 2036 fsm = &fwd_streams[fc->stream_idx]; 2037 nb_fs = fc->stream_nb; 2038 do { 2039 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2040 (*pkt_fwd)(fsm[sm_id]); 2041 #ifdef RTE_LIB_BITRATESTATS 2042 if (bitrate_enabled != 0 && 2043 bitrate_lcore_id == rte_lcore_id()) { 2044 tics_current = rte_rdtsc(); 2045 if (tics_current - tics_datum >= tics_per_1sec) { 2046 /* Periodic bitrate calculation */ 2047 for (i = 0; i < cnt_ports; i++) 2048 rte_stats_bitrate_calc(bitrate_data, 2049 ports_ids[i]); 2050 tics_datum = tics_current; 2051 } 2052 } 2053 #endif 2054 #ifdef RTE_LIB_LATENCYSTATS 2055 if (latencystats_enabled != 0 && 2056 latencystats_lcore_id == rte_lcore_id()) 2057 rte_latencystats_update(); 2058 #endif 2059 2060 } while (! fc->stopped); 2061 } 2062 2063 static int 2064 start_pkt_forward_on_core(void *fwd_arg) 2065 { 2066 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2067 cur_fwd_config.fwd_eng->packet_fwd); 2068 return 0; 2069 } 2070 2071 /* 2072 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2073 * Used to start communication flows in network loopback test configurations. 2074 */ 2075 static int 2076 run_one_txonly_burst_on_core(void *fwd_arg) 2077 { 2078 struct fwd_lcore *fwd_lc; 2079 struct fwd_lcore tmp_lcore; 2080 2081 fwd_lc = (struct fwd_lcore *) fwd_arg; 2082 tmp_lcore = *fwd_lc; 2083 tmp_lcore.stopped = 1; 2084 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2085 return 0; 2086 } 2087 2088 /* 2089 * Launch packet forwarding: 2090 * - Setup per-port forwarding context. 2091 * - launch logical cores with their forwarding configuration. 2092 */ 2093 static void 2094 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2095 { 2096 port_fwd_begin_t port_fwd_begin; 2097 unsigned int i; 2098 unsigned int lc_id; 2099 int diag; 2100 2101 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2102 if (port_fwd_begin != NULL) { 2103 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2104 (*port_fwd_begin)(fwd_ports_ids[i]); 2105 } 2106 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2107 lc_id = fwd_lcores_cpuids[i]; 2108 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2109 fwd_lcores[i]->stopped = 0; 2110 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2111 fwd_lcores[i], lc_id); 2112 if (diag != 0) 2113 printf("launch lcore %u failed - diag=%d\n", 2114 lc_id, diag); 2115 } 2116 } 2117 } 2118 2119 /* 2120 * Launch packet forwarding configuration. 2121 */ 2122 void 2123 start_packet_forwarding(int with_tx_first) 2124 { 2125 port_fwd_begin_t port_fwd_begin; 2126 port_fwd_end_t port_fwd_end; 2127 struct rte_port *port; 2128 unsigned int i; 2129 portid_t pt_id; 2130 2131 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2132 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2133 2134 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2135 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2136 2137 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2138 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2139 (!nb_rxq || !nb_txq)) 2140 rte_exit(EXIT_FAILURE, 2141 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2142 cur_fwd_eng->fwd_mode_name); 2143 2144 if (all_ports_started() == 0) { 2145 printf("Not all ports were started\n"); 2146 return; 2147 } 2148 if (test_done == 0) { 2149 printf("Packet forwarding already started\n"); 2150 return; 2151 } 2152 2153 2154 if(dcb_test) { 2155 for (i = 0; i < nb_fwd_ports; i++) { 2156 pt_id = fwd_ports_ids[i]; 2157 port = &ports[pt_id]; 2158 if (!port->dcb_flag) { 2159 printf("In DCB mode, all forwarding ports must " 2160 "be configured in this mode.\n"); 2161 return; 2162 } 2163 } 2164 if (nb_fwd_lcores == 1) { 2165 printf("In DCB mode,the nb forwarding cores " 2166 "should be larger than 1.\n"); 2167 return; 2168 } 2169 } 2170 test_done = 0; 2171 2172 fwd_config_setup(); 2173 2174 if(!no_flush_rx) 2175 flush_fwd_rx_queues(); 2176 2177 pkt_fwd_config_display(&cur_fwd_config); 2178 rxtx_config_display(); 2179 2180 fwd_stats_reset(); 2181 if (with_tx_first) { 2182 port_fwd_begin = tx_only_engine.port_fwd_begin; 2183 if (port_fwd_begin != NULL) { 2184 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2185 (*port_fwd_begin)(fwd_ports_ids[i]); 2186 } 2187 while (with_tx_first--) { 2188 launch_packet_forwarding( 2189 run_one_txonly_burst_on_core); 2190 rte_eal_mp_wait_lcore(); 2191 } 2192 port_fwd_end = tx_only_engine.port_fwd_end; 2193 if (port_fwd_end != NULL) { 2194 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2195 (*port_fwd_end)(fwd_ports_ids[i]); 2196 } 2197 } 2198 launch_packet_forwarding(start_pkt_forward_on_core); 2199 } 2200 2201 void 2202 stop_packet_forwarding(void) 2203 { 2204 port_fwd_end_t port_fwd_end; 2205 lcoreid_t lc_id; 2206 portid_t pt_id; 2207 int i; 2208 2209 if (test_done) { 2210 printf("Packet forwarding not started\n"); 2211 return; 2212 } 2213 printf("Telling cores to stop..."); 2214 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2215 fwd_lcores[lc_id]->stopped = 1; 2216 printf("\nWaiting for lcores to finish...\n"); 2217 rte_eal_mp_wait_lcore(); 2218 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2219 if (port_fwd_end != NULL) { 2220 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2221 pt_id = fwd_ports_ids[i]; 2222 (*port_fwd_end)(pt_id); 2223 } 2224 } 2225 2226 fwd_stats_display(); 2227 2228 printf("\nDone.\n"); 2229 test_done = 1; 2230 } 2231 2232 void 2233 dev_set_link_up(portid_t pid) 2234 { 2235 if (rte_eth_dev_set_link_up(pid) < 0) 2236 printf("\nSet link up fail.\n"); 2237 } 2238 2239 void 2240 dev_set_link_down(portid_t pid) 2241 { 2242 if (rte_eth_dev_set_link_down(pid) < 0) 2243 printf("\nSet link down fail.\n"); 2244 } 2245 2246 static int 2247 all_ports_started(void) 2248 { 2249 portid_t pi; 2250 struct rte_port *port; 2251 2252 RTE_ETH_FOREACH_DEV(pi) { 2253 port = &ports[pi]; 2254 /* Check if there is a port which is not started */ 2255 if ((port->port_status != RTE_PORT_STARTED) && 2256 (port->slave_flag == 0)) 2257 return 0; 2258 } 2259 2260 /* No port is not started */ 2261 return 1; 2262 } 2263 2264 int 2265 port_is_stopped(portid_t port_id) 2266 { 2267 struct rte_port *port = &ports[port_id]; 2268 2269 if ((port->port_status != RTE_PORT_STOPPED) && 2270 (port->slave_flag == 0)) 2271 return 0; 2272 return 1; 2273 } 2274 2275 int 2276 all_ports_stopped(void) 2277 { 2278 portid_t pi; 2279 2280 RTE_ETH_FOREACH_DEV(pi) { 2281 if (!port_is_stopped(pi)) 2282 return 0; 2283 } 2284 2285 return 1; 2286 } 2287 2288 int 2289 port_is_started(portid_t port_id) 2290 { 2291 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2292 return 0; 2293 2294 if (ports[port_id].port_status != RTE_PORT_STARTED) 2295 return 0; 2296 2297 return 1; 2298 } 2299 2300 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2301 static int 2302 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2303 { 2304 queueid_t qi; 2305 struct rte_eth_hairpin_conf hairpin_conf = { 2306 .peer_count = 1, 2307 }; 2308 int i; 2309 int diag; 2310 struct rte_port *port = &ports[pi]; 2311 uint16_t peer_rx_port = pi; 2312 uint16_t peer_tx_port = pi; 2313 uint32_t manual = 1; 2314 uint32_t tx_exp = hairpin_mode & 0x10; 2315 2316 if (!(hairpin_mode & 0xf)) { 2317 peer_rx_port = pi; 2318 peer_tx_port = pi; 2319 manual = 0; 2320 } else if (hairpin_mode & 0x1) { 2321 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2322 RTE_ETH_DEV_NO_OWNER); 2323 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2324 peer_tx_port = rte_eth_find_next_owned_by(0, 2325 RTE_ETH_DEV_NO_OWNER); 2326 if (p_pi != RTE_MAX_ETHPORTS) { 2327 peer_rx_port = p_pi; 2328 } else { 2329 uint16_t next_pi; 2330 2331 /* Last port will be the peer RX port of the first. */ 2332 RTE_ETH_FOREACH_DEV(next_pi) 2333 peer_rx_port = next_pi; 2334 } 2335 manual = 1; 2336 } else if (hairpin_mode & 0x2) { 2337 if (cnt_pi & 0x1) { 2338 peer_rx_port = p_pi; 2339 } else { 2340 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2341 RTE_ETH_DEV_NO_OWNER); 2342 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2343 peer_rx_port = pi; 2344 } 2345 peer_tx_port = peer_rx_port; 2346 manual = 1; 2347 } 2348 2349 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2350 hairpin_conf.peers[0].port = peer_rx_port; 2351 hairpin_conf.peers[0].queue = i + nb_rxq; 2352 hairpin_conf.manual_bind = !!manual; 2353 hairpin_conf.tx_explicit = !!tx_exp; 2354 diag = rte_eth_tx_hairpin_queue_setup 2355 (pi, qi, nb_txd, &hairpin_conf); 2356 i++; 2357 if (diag == 0) 2358 continue; 2359 2360 /* Fail to setup rx queue, return */ 2361 if (rte_atomic16_cmpset(&(port->port_status), 2362 RTE_PORT_HANDLING, 2363 RTE_PORT_STOPPED) == 0) 2364 printf("Port %d can not be set back " 2365 "to stopped\n", pi); 2366 printf("Fail to configure port %d hairpin " 2367 "queues\n", pi); 2368 /* try to reconfigure queues next time */ 2369 port->need_reconfig_queues = 1; 2370 return -1; 2371 } 2372 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2373 hairpin_conf.peers[0].port = peer_tx_port; 2374 hairpin_conf.peers[0].queue = i + nb_txq; 2375 hairpin_conf.manual_bind = !!manual; 2376 hairpin_conf.tx_explicit = !!tx_exp; 2377 diag = rte_eth_rx_hairpin_queue_setup 2378 (pi, qi, nb_rxd, &hairpin_conf); 2379 i++; 2380 if (diag == 0) 2381 continue; 2382 2383 /* Fail to setup rx queue, return */ 2384 if (rte_atomic16_cmpset(&(port->port_status), 2385 RTE_PORT_HANDLING, 2386 RTE_PORT_STOPPED) == 0) 2387 printf("Port %d can not be set back " 2388 "to stopped\n", pi); 2389 printf("Fail to configure port %d hairpin " 2390 "queues\n", pi); 2391 /* try to reconfigure queues next time */ 2392 port->need_reconfig_queues = 1; 2393 return -1; 2394 } 2395 return 0; 2396 } 2397 2398 /* Configure the Rx with optional split. */ 2399 int 2400 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2401 uint16_t nb_rx_desc, unsigned int socket_id, 2402 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2403 { 2404 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2405 unsigned int i, mp_n; 2406 int ret; 2407 2408 if (rx_pkt_nb_segs <= 1 || 2409 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2410 rx_conf->rx_seg = NULL; 2411 rx_conf->rx_nseg = 0; 2412 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2413 nb_rx_desc, socket_id, 2414 rx_conf, mp); 2415 return ret; 2416 } 2417 for (i = 0; i < rx_pkt_nb_segs; i++) { 2418 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2419 struct rte_mempool *mpx; 2420 /* 2421 * Use last valid pool for the segments with number 2422 * exceeding the pool index. 2423 */ 2424 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2425 mpx = mbuf_pool_find(socket_id, mp_n); 2426 /* Handle zero as mbuf data buffer size. */ 2427 rx_seg->length = rx_pkt_seg_lengths[i] ? 2428 rx_pkt_seg_lengths[i] : 2429 mbuf_data_size[mp_n]; 2430 rx_seg->offset = i < rx_pkt_nb_offs ? 2431 rx_pkt_seg_offsets[i] : 0; 2432 rx_seg->mp = mpx ? mpx : mp; 2433 } 2434 rx_conf->rx_nseg = rx_pkt_nb_segs; 2435 rx_conf->rx_seg = rx_useg; 2436 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2437 socket_id, rx_conf, NULL); 2438 rx_conf->rx_seg = NULL; 2439 rx_conf->rx_nseg = 0; 2440 return ret; 2441 } 2442 2443 int 2444 start_port(portid_t pid) 2445 { 2446 int diag, need_check_link_status = -1; 2447 portid_t pi; 2448 portid_t p_pi = RTE_MAX_ETHPORTS; 2449 portid_t pl[RTE_MAX_ETHPORTS]; 2450 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2451 uint16_t cnt_pi = 0; 2452 uint16_t cfg_pi = 0; 2453 int peer_pi; 2454 queueid_t qi; 2455 struct rte_port *port; 2456 struct rte_ether_addr mac_addr; 2457 struct rte_eth_hairpin_cap cap; 2458 2459 if (port_id_is_invalid(pid, ENABLED_WARN)) 2460 return 0; 2461 2462 if(dcb_config) 2463 dcb_test = 1; 2464 RTE_ETH_FOREACH_DEV(pi) { 2465 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2466 continue; 2467 2468 need_check_link_status = 0; 2469 port = &ports[pi]; 2470 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2471 RTE_PORT_HANDLING) == 0) { 2472 printf("Port %d is now not stopped\n", pi); 2473 continue; 2474 } 2475 2476 if (port->need_reconfig > 0) { 2477 port->need_reconfig = 0; 2478 2479 if (flow_isolate_all) { 2480 int ret = port_flow_isolate(pi, 1); 2481 if (ret) { 2482 printf("Failed to apply isolated" 2483 " mode on port %d\n", pi); 2484 return -1; 2485 } 2486 } 2487 configure_rxtx_dump_callbacks(0); 2488 printf("Configuring Port %d (socket %u)\n", pi, 2489 port->socket_id); 2490 if (nb_hairpinq > 0 && 2491 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2492 printf("Port %d doesn't support hairpin " 2493 "queues\n", pi); 2494 return -1; 2495 } 2496 /* configure port */ 2497 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2498 nb_txq + nb_hairpinq, 2499 &(port->dev_conf)); 2500 if (diag != 0) { 2501 if (rte_atomic16_cmpset(&(port->port_status), 2502 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2503 printf("Port %d can not be set back " 2504 "to stopped\n", pi); 2505 printf("Fail to configure port %d\n", pi); 2506 /* try to reconfigure port next time */ 2507 port->need_reconfig = 1; 2508 return -1; 2509 } 2510 } 2511 if (port->need_reconfig_queues > 0) { 2512 port->need_reconfig_queues = 0; 2513 /* setup tx queues */ 2514 for (qi = 0; qi < nb_txq; qi++) { 2515 if ((numa_support) && 2516 (txring_numa[pi] != NUMA_NO_CONFIG)) 2517 diag = rte_eth_tx_queue_setup(pi, qi, 2518 port->nb_tx_desc[qi], 2519 txring_numa[pi], 2520 &(port->tx_conf[qi])); 2521 else 2522 diag = rte_eth_tx_queue_setup(pi, qi, 2523 port->nb_tx_desc[qi], 2524 port->socket_id, 2525 &(port->tx_conf[qi])); 2526 2527 if (diag == 0) 2528 continue; 2529 2530 /* Fail to setup tx queue, return */ 2531 if (rte_atomic16_cmpset(&(port->port_status), 2532 RTE_PORT_HANDLING, 2533 RTE_PORT_STOPPED) == 0) 2534 printf("Port %d can not be set back " 2535 "to stopped\n", pi); 2536 printf("Fail to configure port %d tx queues\n", 2537 pi); 2538 /* try to reconfigure queues next time */ 2539 port->need_reconfig_queues = 1; 2540 return -1; 2541 } 2542 for (qi = 0; qi < nb_rxq; qi++) { 2543 /* setup rx queues */ 2544 if ((numa_support) && 2545 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2546 struct rte_mempool * mp = 2547 mbuf_pool_find 2548 (rxring_numa[pi], 0); 2549 if (mp == NULL) { 2550 printf("Failed to setup RX queue:" 2551 "No mempool allocation" 2552 " on the socket %d\n", 2553 rxring_numa[pi]); 2554 return -1; 2555 } 2556 2557 diag = rx_queue_setup(pi, qi, 2558 port->nb_rx_desc[qi], 2559 rxring_numa[pi], 2560 &(port->rx_conf[qi]), 2561 mp); 2562 } else { 2563 struct rte_mempool *mp = 2564 mbuf_pool_find 2565 (port->socket_id, 0); 2566 if (mp == NULL) { 2567 printf("Failed to setup RX queue:" 2568 "No mempool allocation" 2569 " on the socket %d\n", 2570 port->socket_id); 2571 return -1; 2572 } 2573 diag = rx_queue_setup(pi, qi, 2574 port->nb_rx_desc[qi], 2575 port->socket_id, 2576 &(port->rx_conf[qi]), 2577 mp); 2578 } 2579 if (diag == 0) 2580 continue; 2581 2582 /* Fail to setup rx queue, return */ 2583 if (rte_atomic16_cmpset(&(port->port_status), 2584 RTE_PORT_HANDLING, 2585 RTE_PORT_STOPPED) == 0) 2586 printf("Port %d can not be set back " 2587 "to stopped\n", pi); 2588 printf("Fail to configure port %d rx queues\n", 2589 pi); 2590 /* try to reconfigure queues next time */ 2591 port->need_reconfig_queues = 1; 2592 return -1; 2593 } 2594 /* setup hairpin queues */ 2595 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2596 return -1; 2597 } 2598 configure_rxtx_dump_callbacks(verbose_level); 2599 if (clear_ptypes) { 2600 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2601 NULL, 0); 2602 if (diag < 0) 2603 printf( 2604 "Port %d: Failed to disable Ptype parsing\n", 2605 pi); 2606 } 2607 2608 p_pi = pi; 2609 cnt_pi++; 2610 2611 /* start port */ 2612 if (rte_eth_dev_start(pi) < 0) { 2613 printf("Fail to start port %d\n", pi); 2614 2615 /* Fail to setup rx queue, return */ 2616 if (rte_atomic16_cmpset(&(port->port_status), 2617 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2618 printf("Port %d can not be set back to " 2619 "stopped\n", pi); 2620 continue; 2621 } 2622 2623 if (rte_atomic16_cmpset(&(port->port_status), 2624 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2625 printf("Port %d can not be set into started\n", pi); 2626 2627 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2628 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2629 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2630 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2631 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2632 2633 /* at least one port started, need checking link status */ 2634 need_check_link_status = 1; 2635 2636 pl[cfg_pi++] = pi; 2637 } 2638 2639 if (need_check_link_status == 1 && !no_link_check) 2640 check_all_ports_link_status(RTE_PORT_ALL); 2641 else if (need_check_link_status == 0) 2642 printf("Please stop the ports first\n"); 2643 2644 if (hairpin_mode & 0xf) { 2645 uint16_t i; 2646 int j; 2647 2648 /* bind all started hairpin ports */ 2649 for (i = 0; i < cfg_pi; i++) { 2650 pi = pl[i]; 2651 /* bind current Tx to all peer Rx */ 2652 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2653 RTE_MAX_ETHPORTS, 1); 2654 if (peer_pi < 0) 2655 return peer_pi; 2656 for (j = 0; j < peer_pi; j++) { 2657 if (!port_is_started(peer_pl[j])) 2658 continue; 2659 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2660 if (diag < 0) { 2661 printf("Error during binding hairpin" 2662 " Tx port %u to %u: %s\n", 2663 pi, peer_pl[j], 2664 rte_strerror(-diag)); 2665 return -1; 2666 } 2667 } 2668 /* bind all peer Tx to current Rx */ 2669 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2670 RTE_MAX_ETHPORTS, 0); 2671 if (peer_pi < 0) 2672 return peer_pi; 2673 for (j = 0; j < peer_pi; j++) { 2674 if (!port_is_started(peer_pl[j])) 2675 continue; 2676 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2677 if (diag < 0) { 2678 printf("Error during binding hairpin" 2679 " Tx port %u to %u: %s\n", 2680 peer_pl[j], pi, 2681 rte_strerror(-diag)); 2682 return -1; 2683 } 2684 } 2685 } 2686 } 2687 2688 printf("Done\n"); 2689 return 0; 2690 } 2691 2692 void 2693 stop_port(portid_t pid) 2694 { 2695 portid_t pi; 2696 struct rte_port *port; 2697 int need_check_link_status = 0; 2698 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2699 int peer_pi; 2700 2701 if (dcb_test) { 2702 dcb_test = 0; 2703 dcb_config = 0; 2704 } 2705 2706 if (port_id_is_invalid(pid, ENABLED_WARN)) 2707 return; 2708 2709 printf("Stopping ports...\n"); 2710 2711 RTE_ETH_FOREACH_DEV(pi) { 2712 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2713 continue; 2714 2715 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2716 printf("Please remove port %d from forwarding configuration.\n", pi); 2717 continue; 2718 } 2719 2720 if (port_is_bonding_slave(pi)) { 2721 printf("Please remove port %d from bonded device.\n", pi); 2722 continue; 2723 } 2724 2725 port = &ports[pi]; 2726 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2727 RTE_PORT_HANDLING) == 0) 2728 continue; 2729 2730 if (hairpin_mode & 0xf) { 2731 int j; 2732 2733 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2734 /* unbind all peer Tx from current Rx */ 2735 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2736 RTE_MAX_ETHPORTS, 0); 2737 if (peer_pi < 0) 2738 continue; 2739 for (j = 0; j < peer_pi; j++) { 2740 if (!port_is_started(peer_pl[j])) 2741 continue; 2742 rte_eth_hairpin_unbind(peer_pl[j], pi); 2743 } 2744 } 2745 2746 if (port->flow_list) 2747 port_flow_flush(pi); 2748 2749 if (rte_eth_dev_stop(pi) != 0) 2750 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2751 pi); 2752 2753 if (rte_atomic16_cmpset(&(port->port_status), 2754 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2755 printf("Port %d can not be set into stopped\n", pi); 2756 need_check_link_status = 1; 2757 } 2758 if (need_check_link_status && !no_link_check) 2759 check_all_ports_link_status(RTE_PORT_ALL); 2760 2761 printf("Done\n"); 2762 } 2763 2764 static void 2765 remove_invalid_ports_in(portid_t *array, portid_t *total) 2766 { 2767 portid_t i; 2768 portid_t new_total = 0; 2769 2770 for (i = 0; i < *total; i++) 2771 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2772 array[new_total] = array[i]; 2773 new_total++; 2774 } 2775 *total = new_total; 2776 } 2777 2778 static void 2779 remove_invalid_ports(void) 2780 { 2781 remove_invalid_ports_in(ports_ids, &nb_ports); 2782 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2783 nb_cfg_ports = nb_fwd_ports; 2784 } 2785 2786 void 2787 close_port(portid_t pid) 2788 { 2789 portid_t pi; 2790 struct rte_port *port; 2791 2792 if (port_id_is_invalid(pid, ENABLED_WARN)) 2793 return; 2794 2795 printf("Closing ports...\n"); 2796 2797 RTE_ETH_FOREACH_DEV(pi) { 2798 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2799 continue; 2800 2801 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2802 printf("Please remove port %d from forwarding configuration.\n", pi); 2803 continue; 2804 } 2805 2806 if (port_is_bonding_slave(pi)) { 2807 printf("Please remove port %d from bonded device.\n", pi); 2808 continue; 2809 } 2810 2811 port = &ports[pi]; 2812 if (rte_atomic16_cmpset(&(port->port_status), 2813 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2814 printf("Port %d is already closed\n", pi); 2815 continue; 2816 } 2817 2818 port_flow_flush(pi); 2819 rte_eth_dev_close(pi); 2820 } 2821 2822 remove_invalid_ports(); 2823 printf("Done\n"); 2824 } 2825 2826 void 2827 reset_port(portid_t pid) 2828 { 2829 int diag; 2830 portid_t pi; 2831 struct rte_port *port; 2832 2833 if (port_id_is_invalid(pid, ENABLED_WARN)) 2834 return; 2835 2836 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2837 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2838 printf("Can not reset port(s), please stop port(s) first.\n"); 2839 return; 2840 } 2841 2842 printf("Resetting ports...\n"); 2843 2844 RTE_ETH_FOREACH_DEV(pi) { 2845 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2846 continue; 2847 2848 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2849 printf("Please remove port %d from forwarding " 2850 "configuration.\n", pi); 2851 continue; 2852 } 2853 2854 if (port_is_bonding_slave(pi)) { 2855 printf("Please remove port %d from bonded device.\n", 2856 pi); 2857 continue; 2858 } 2859 2860 diag = rte_eth_dev_reset(pi); 2861 if (diag == 0) { 2862 port = &ports[pi]; 2863 port->need_reconfig = 1; 2864 port->need_reconfig_queues = 1; 2865 } else { 2866 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2867 } 2868 } 2869 2870 printf("Done\n"); 2871 } 2872 2873 void 2874 attach_port(char *identifier) 2875 { 2876 portid_t pi; 2877 struct rte_dev_iterator iterator; 2878 2879 printf("Attaching a new port...\n"); 2880 2881 if (identifier == NULL) { 2882 printf("Invalid parameters are specified\n"); 2883 return; 2884 } 2885 2886 if (rte_dev_probe(identifier) < 0) { 2887 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2888 return; 2889 } 2890 2891 /* first attach mode: event */ 2892 if (setup_on_probe_event) { 2893 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2894 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2895 if (ports[pi].port_status == RTE_PORT_HANDLING && 2896 ports[pi].need_setup != 0) 2897 setup_attached_port(pi); 2898 return; 2899 } 2900 2901 /* second attach mode: iterator */ 2902 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2903 /* setup ports matching the devargs used for probing */ 2904 if (port_is_forwarding(pi)) 2905 continue; /* port was already attached before */ 2906 setup_attached_port(pi); 2907 } 2908 } 2909 2910 static void 2911 setup_attached_port(portid_t pi) 2912 { 2913 unsigned int socket_id; 2914 int ret; 2915 2916 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2917 /* if socket_id is invalid, set to the first available socket. */ 2918 if (check_socket_id(socket_id) < 0) 2919 socket_id = socket_ids[0]; 2920 reconfig(pi, socket_id); 2921 ret = rte_eth_promiscuous_enable(pi); 2922 if (ret != 0) 2923 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2924 pi, rte_strerror(-ret)); 2925 2926 ports_ids[nb_ports++] = pi; 2927 fwd_ports_ids[nb_fwd_ports++] = pi; 2928 nb_cfg_ports = nb_fwd_ports; 2929 ports[pi].need_setup = 0; 2930 ports[pi].port_status = RTE_PORT_STOPPED; 2931 2932 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2933 printf("Done\n"); 2934 } 2935 2936 static void 2937 detach_device(struct rte_device *dev) 2938 { 2939 portid_t sibling; 2940 2941 if (dev == NULL) { 2942 printf("Device already removed\n"); 2943 return; 2944 } 2945 2946 printf("Removing a device...\n"); 2947 2948 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2949 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2950 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2951 printf("Port %u not stopped\n", sibling); 2952 return; 2953 } 2954 port_flow_flush(sibling); 2955 } 2956 } 2957 2958 if (rte_dev_remove(dev) < 0) { 2959 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2960 return; 2961 } 2962 remove_invalid_ports(); 2963 2964 printf("Device is detached\n"); 2965 printf("Now total ports is %d\n", nb_ports); 2966 printf("Done\n"); 2967 return; 2968 } 2969 2970 void 2971 detach_port_device(portid_t port_id) 2972 { 2973 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2974 return; 2975 2976 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2977 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2978 printf("Port not stopped\n"); 2979 return; 2980 } 2981 printf("Port was not closed\n"); 2982 } 2983 2984 detach_device(rte_eth_devices[port_id].device); 2985 } 2986 2987 void 2988 detach_devargs(char *identifier) 2989 { 2990 struct rte_dev_iterator iterator; 2991 struct rte_devargs da; 2992 portid_t port_id; 2993 2994 printf("Removing a device...\n"); 2995 2996 memset(&da, 0, sizeof(da)); 2997 if (rte_devargs_parsef(&da, "%s", identifier)) { 2998 printf("cannot parse identifier\n"); 2999 if (da.args) 3000 free(da.args); 3001 return; 3002 } 3003 3004 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3005 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3006 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3007 printf("Port %u not stopped\n", port_id); 3008 rte_eth_iterator_cleanup(&iterator); 3009 return; 3010 } 3011 port_flow_flush(port_id); 3012 } 3013 } 3014 3015 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3016 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3017 da.name, da.bus->name); 3018 return; 3019 } 3020 3021 remove_invalid_ports(); 3022 3023 printf("Device %s is detached\n", identifier); 3024 printf("Now total ports is %d\n", nb_ports); 3025 printf("Done\n"); 3026 } 3027 3028 void 3029 pmd_test_exit(void) 3030 { 3031 portid_t pt_id; 3032 unsigned int i; 3033 int ret; 3034 3035 if (test_done == 0) 3036 stop_packet_forwarding(); 3037 3038 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3039 if (mempools[i]) { 3040 if (mp_alloc_type == MP_ALLOC_ANON) 3041 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3042 NULL); 3043 } 3044 } 3045 if (ports != NULL) { 3046 no_link_check = 1; 3047 RTE_ETH_FOREACH_DEV(pt_id) { 3048 printf("\nStopping port %d...\n", pt_id); 3049 fflush(stdout); 3050 stop_port(pt_id); 3051 } 3052 RTE_ETH_FOREACH_DEV(pt_id) { 3053 printf("\nShutting down port %d...\n", pt_id); 3054 fflush(stdout); 3055 close_port(pt_id); 3056 } 3057 } 3058 3059 if (hot_plug) { 3060 ret = rte_dev_event_monitor_stop(); 3061 if (ret) { 3062 RTE_LOG(ERR, EAL, 3063 "fail to stop device event monitor."); 3064 return; 3065 } 3066 3067 ret = rte_dev_event_callback_unregister(NULL, 3068 dev_event_callback, NULL); 3069 if (ret < 0) { 3070 RTE_LOG(ERR, EAL, 3071 "fail to unregister device event callback.\n"); 3072 return; 3073 } 3074 3075 ret = rte_dev_hotplug_handle_disable(); 3076 if (ret) { 3077 RTE_LOG(ERR, EAL, 3078 "fail to disable hotplug handling.\n"); 3079 return; 3080 } 3081 } 3082 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3083 if (mempools[i]) 3084 rte_mempool_free(mempools[i]); 3085 } 3086 3087 printf("\nBye...\n"); 3088 } 3089 3090 typedef void (*cmd_func_t)(void); 3091 struct pmd_test_command { 3092 const char *cmd_name; 3093 cmd_func_t cmd_func; 3094 }; 3095 3096 /* Check the link status of all ports in up to 9s, and print them finally */ 3097 static void 3098 check_all_ports_link_status(uint32_t port_mask) 3099 { 3100 #define CHECK_INTERVAL 100 /* 100ms */ 3101 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3102 portid_t portid; 3103 uint8_t count, all_ports_up, print_flag = 0; 3104 struct rte_eth_link link; 3105 int ret; 3106 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3107 3108 printf("Checking link statuses...\n"); 3109 fflush(stdout); 3110 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3111 all_ports_up = 1; 3112 RTE_ETH_FOREACH_DEV(portid) { 3113 if ((port_mask & (1 << portid)) == 0) 3114 continue; 3115 memset(&link, 0, sizeof(link)); 3116 ret = rte_eth_link_get_nowait(portid, &link); 3117 if (ret < 0) { 3118 all_ports_up = 0; 3119 if (print_flag == 1) 3120 printf("Port %u link get failed: %s\n", 3121 portid, rte_strerror(-ret)); 3122 continue; 3123 } 3124 /* print link status if flag set */ 3125 if (print_flag == 1) { 3126 rte_eth_link_to_str(link_status, 3127 sizeof(link_status), &link); 3128 printf("Port %d %s\n", portid, link_status); 3129 continue; 3130 } 3131 /* clear all_ports_up flag if any link down */ 3132 if (link.link_status == ETH_LINK_DOWN) { 3133 all_ports_up = 0; 3134 break; 3135 } 3136 } 3137 /* after finally printing all link status, get out */ 3138 if (print_flag == 1) 3139 break; 3140 3141 if (all_ports_up == 0) { 3142 fflush(stdout); 3143 rte_delay_ms(CHECK_INTERVAL); 3144 } 3145 3146 /* set the print_flag if all ports up or timeout */ 3147 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3148 print_flag = 1; 3149 } 3150 3151 if (lsc_interrupt) 3152 break; 3153 } 3154 } 3155 3156 static void 3157 rmv_port_callback(void *arg) 3158 { 3159 int need_to_start = 0; 3160 int org_no_link_check = no_link_check; 3161 portid_t port_id = (intptr_t)arg; 3162 struct rte_device *dev; 3163 3164 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3165 3166 if (!test_done && port_is_forwarding(port_id)) { 3167 need_to_start = 1; 3168 stop_packet_forwarding(); 3169 } 3170 no_link_check = 1; 3171 stop_port(port_id); 3172 no_link_check = org_no_link_check; 3173 3174 /* Save rte_device pointer before closing ethdev port */ 3175 dev = rte_eth_devices[port_id].device; 3176 close_port(port_id); 3177 detach_device(dev); /* might be already removed or have more ports */ 3178 3179 if (need_to_start) 3180 start_packet_forwarding(0); 3181 } 3182 3183 /* This function is used by the interrupt thread */ 3184 static int 3185 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3186 void *ret_param) 3187 { 3188 RTE_SET_USED(param); 3189 RTE_SET_USED(ret_param); 3190 3191 if (type >= RTE_ETH_EVENT_MAX) { 3192 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3193 port_id, __func__, type); 3194 fflush(stderr); 3195 } else if (event_print_mask & (UINT32_C(1) << type)) { 3196 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3197 eth_event_desc[type]); 3198 fflush(stdout); 3199 } 3200 3201 switch (type) { 3202 case RTE_ETH_EVENT_NEW: 3203 ports[port_id].need_setup = 1; 3204 ports[port_id].port_status = RTE_PORT_HANDLING; 3205 break; 3206 case RTE_ETH_EVENT_INTR_RMV: 3207 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3208 break; 3209 if (rte_eal_alarm_set(100000, 3210 rmv_port_callback, (void *)(intptr_t)port_id)) 3211 fprintf(stderr, "Could not set up deferred device removal\n"); 3212 break; 3213 case RTE_ETH_EVENT_DESTROY: 3214 ports[port_id].port_status = RTE_PORT_CLOSED; 3215 printf("Port %u is closed\n", port_id); 3216 break; 3217 default: 3218 break; 3219 } 3220 return 0; 3221 } 3222 3223 static int 3224 register_eth_event_callback(void) 3225 { 3226 int ret; 3227 enum rte_eth_event_type event; 3228 3229 for (event = RTE_ETH_EVENT_UNKNOWN; 3230 event < RTE_ETH_EVENT_MAX; event++) { 3231 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3232 event, 3233 eth_event_callback, 3234 NULL); 3235 if (ret != 0) { 3236 TESTPMD_LOG(ERR, "Failed to register callback for " 3237 "%s event\n", eth_event_desc[event]); 3238 return -1; 3239 } 3240 } 3241 3242 return 0; 3243 } 3244 3245 /* This function is used by the interrupt thread */ 3246 static void 3247 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3248 __rte_unused void *arg) 3249 { 3250 uint16_t port_id; 3251 int ret; 3252 3253 if (type >= RTE_DEV_EVENT_MAX) { 3254 fprintf(stderr, "%s called upon invalid event %d\n", 3255 __func__, type); 3256 fflush(stderr); 3257 } 3258 3259 switch (type) { 3260 case RTE_DEV_EVENT_REMOVE: 3261 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3262 device_name); 3263 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3264 if (ret) { 3265 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3266 device_name); 3267 return; 3268 } 3269 /* 3270 * Because the user's callback is invoked in eal interrupt 3271 * callback, the interrupt callback need to be finished before 3272 * it can be unregistered when detaching device. So finish 3273 * callback soon and use a deferred removal to detach device 3274 * is need. It is a workaround, once the device detaching be 3275 * moved into the eal in the future, the deferred removal could 3276 * be deleted. 3277 */ 3278 if (rte_eal_alarm_set(100000, 3279 rmv_port_callback, (void *)(intptr_t)port_id)) 3280 RTE_LOG(ERR, EAL, 3281 "Could not set up deferred device removal\n"); 3282 break; 3283 case RTE_DEV_EVENT_ADD: 3284 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3285 device_name); 3286 /* TODO: After finish kernel driver binding, 3287 * begin to attach port. 3288 */ 3289 break; 3290 default: 3291 break; 3292 } 3293 } 3294 3295 static void 3296 rxtx_port_config(struct rte_port *port) 3297 { 3298 uint16_t qid; 3299 uint64_t offloads; 3300 3301 for (qid = 0; qid < nb_rxq; qid++) { 3302 offloads = port->rx_conf[qid].offloads; 3303 port->rx_conf[qid] = port->dev_info.default_rxconf; 3304 if (offloads != 0) 3305 port->rx_conf[qid].offloads = offloads; 3306 3307 /* Check if any Rx parameters have been passed */ 3308 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3309 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3310 3311 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3312 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3313 3314 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3315 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3316 3317 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3318 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3319 3320 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3321 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3322 3323 port->nb_rx_desc[qid] = nb_rxd; 3324 } 3325 3326 for (qid = 0; qid < nb_txq; qid++) { 3327 offloads = port->tx_conf[qid].offloads; 3328 port->tx_conf[qid] = port->dev_info.default_txconf; 3329 if (offloads != 0) 3330 port->tx_conf[qid].offloads = offloads; 3331 3332 /* Check if any Tx parameters have been passed */ 3333 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3334 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3335 3336 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3337 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3338 3339 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3340 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3341 3342 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3343 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3344 3345 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3346 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3347 3348 port->nb_tx_desc[qid] = nb_txd; 3349 } 3350 } 3351 3352 /* 3353 * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload, 3354 * MTU is also aligned if JUMBO_FRAME offload is not set. 3355 * 3356 * port->dev_info should be set before calling this function. 3357 * 3358 * return 0 on success, negative on error 3359 */ 3360 int 3361 update_jumbo_frame_offload(portid_t portid) 3362 { 3363 struct rte_port *port = &ports[portid]; 3364 uint32_t eth_overhead; 3365 uint64_t rx_offloads; 3366 int ret; 3367 bool on; 3368 3369 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */ 3370 if (port->dev_info.max_mtu != UINT16_MAX && 3371 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu) 3372 eth_overhead = port->dev_info.max_rx_pktlen - 3373 port->dev_info.max_mtu; 3374 else 3375 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 3376 3377 rx_offloads = port->dev_conf.rxmode.offloads; 3378 3379 /* Default config value is 0 to use PMD specific overhead */ 3380 if (port->dev_conf.rxmode.max_rx_pkt_len == 0) 3381 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead; 3382 3383 if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) { 3384 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3385 on = false; 3386 } else { 3387 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3388 printf("Frame size (%u) is not supported by port %u\n", 3389 port->dev_conf.rxmode.max_rx_pkt_len, 3390 portid); 3391 return -1; 3392 } 3393 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3394 on = true; 3395 } 3396 3397 if (rx_offloads != port->dev_conf.rxmode.offloads) { 3398 uint16_t qid; 3399 3400 port->dev_conf.rxmode.offloads = rx_offloads; 3401 3402 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */ 3403 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) { 3404 if (on) 3405 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3406 else 3407 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3408 } 3409 } 3410 3411 /* If JUMBO_FRAME is set MTU conversion done by ethdev layer, 3412 * if unset do it here 3413 */ 3414 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3415 ret = rte_eth_dev_set_mtu(portid, 3416 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead); 3417 if (ret) 3418 printf("Failed to set MTU to %u for port %u\n", 3419 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead, 3420 portid); 3421 } 3422 3423 return 0; 3424 } 3425 3426 void 3427 init_port_config(void) 3428 { 3429 portid_t pid; 3430 struct rte_port *port; 3431 int ret; 3432 3433 RTE_ETH_FOREACH_DEV(pid) { 3434 port = &ports[pid]; 3435 port->dev_conf.fdir_conf = fdir_conf; 3436 3437 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3438 if (ret != 0) 3439 return; 3440 3441 if (nb_rxq > 1) { 3442 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3443 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3444 rss_hf & port->dev_info.flow_type_rss_offloads; 3445 } else { 3446 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3447 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3448 } 3449 3450 if (port->dcb_flag == 0) { 3451 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3452 port->dev_conf.rxmode.mq_mode = 3453 (enum rte_eth_rx_mq_mode) 3454 (rx_mq_mode & ETH_MQ_RX_RSS); 3455 else 3456 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3457 } 3458 3459 rxtx_port_config(port); 3460 3461 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3462 if (ret != 0) 3463 return; 3464 3465 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3466 rte_pmd_ixgbe_bypass_init(pid); 3467 #endif 3468 3469 if (lsc_interrupt && 3470 (rte_eth_devices[pid].data->dev_flags & 3471 RTE_ETH_DEV_INTR_LSC)) 3472 port->dev_conf.intr_conf.lsc = 1; 3473 if (rmv_interrupt && 3474 (rte_eth_devices[pid].data->dev_flags & 3475 RTE_ETH_DEV_INTR_RMV)) 3476 port->dev_conf.intr_conf.rmv = 1; 3477 } 3478 } 3479 3480 void set_port_slave_flag(portid_t slave_pid) 3481 { 3482 struct rte_port *port; 3483 3484 port = &ports[slave_pid]; 3485 port->slave_flag = 1; 3486 } 3487 3488 void clear_port_slave_flag(portid_t slave_pid) 3489 { 3490 struct rte_port *port; 3491 3492 port = &ports[slave_pid]; 3493 port->slave_flag = 0; 3494 } 3495 3496 uint8_t port_is_bonding_slave(portid_t slave_pid) 3497 { 3498 struct rte_port *port; 3499 3500 port = &ports[slave_pid]; 3501 if ((rte_eth_devices[slave_pid].data->dev_flags & 3502 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3503 return 1; 3504 return 0; 3505 } 3506 3507 const uint16_t vlan_tags[] = { 3508 0, 1, 2, 3, 4, 5, 6, 7, 3509 8, 9, 10, 11, 12, 13, 14, 15, 3510 16, 17, 18, 19, 20, 21, 22, 23, 3511 24, 25, 26, 27, 28, 29, 30, 31 3512 }; 3513 3514 static int 3515 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3516 enum dcb_mode_enable dcb_mode, 3517 enum rte_eth_nb_tcs num_tcs, 3518 uint8_t pfc_en) 3519 { 3520 uint8_t i; 3521 int32_t rc; 3522 struct rte_eth_rss_conf rss_conf; 3523 3524 /* 3525 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3526 * given above, and the number of traffic classes available for use. 3527 */ 3528 if (dcb_mode == DCB_VT_ENABLED) { 3529 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3530 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3531 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3532 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3533 3534 /* VMDQ+DCB RX and TX configurations */ 3535 vmdq_rx_conf->enable_default_pool = 0; 3536 vmdq_rx_conf->default_pool = 0; 3537 vmdq_rx_conf->nb_queue_pools = 3538 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3539 vmdq_tx_conf->nb_queue_pools = 3540 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3541 3542 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3543 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3544 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3545 vmdq_rx_conf->pool_map[i].pools = 3546 1 << (i % vmdq_rx_conf->nb_queue_pools); 3547 } 3548 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3549 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3550 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3551 } 3552 3553 /* set DCB mode of RX and TX of multiple queues */ 3554 eth_conf->rxmode.mq_mode = 3555 (enum rte_eth_rx_mq_mode) 3556 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3557 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3558 } else { 3559 struct rte_eth_dcb_rx_conf *rx_conf = 3560 ð_conf->rx_adv_conf.dcb_rx_conf; 3561 struct rte_eth_dcb_tx_conf *tx_conf = 3562 ð_conf->tx_adv_conf.dcb_tx_conf; 3563 3564 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3565 3566 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3567 if (rc != 0) 3568 return rc; 3569 3570 rx_conf->nb_tcs = num_tcs; 3571 tx_conf->nb_tcs = num_tcs; 3572 3573 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3574 rx_conf->dcb_tc[i] = i % num_tcs; 3575 tx_conf->dcb_tc[i] = i % num_tcs; 3576 } 3577 3578 eth_conf->rxmode.mq_mode = 3579 (enum rte_eth_rx_mq_mode) 3580 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3581 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3582 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3583 } 3584 3585 if (pfc_en) 3586 eth_conf->dcb_capability_en = 3587 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3588 else 3589 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3590 3591 return 0; 3592 } 3593 3594 int 3595 init_port_dcb_config(portid_t pid, 3596 enum dcb_mode_enable dcb_mode, 3597 enum rte_eth_nb_tcs num_tcs, 3598 uint8_t pfc_en) 3599 { 3600 struct rte_eth_conf port_conf; 3601 struct rte_port *rte_port; 3602 int retval; 3603 uint16_t i; 3604 3605 rte_port = &ports[pid]; 3606 3607 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3608 /* Enter DCB configuration status */ 3609 dcb_config = 1; 3610 3611 port_conf.rxmode = rte_port->dev_conf.rxmode; 3612 port_conf.txmode = rte_port->dev_conf.txmode; 3613 3614 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3615 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3616 if (retval < 0) 3617 return retval; 3618 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3619 3620 /* re-configure the device . */ 3621 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3622 if (retval < 0) 3623 return retval; 3624 3625 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3626 if (retval != 0) 3627 return retval; 3628 3629 /* If dev_info.vmdq_pool_base is greater than 0, 3630 * the queue id of vmdq pools is started after pf queues. 3631 */ 3632 if (dcb_mode == DCB_VT_ENABLED && 3633 rte_port->dev_info.vmdq_pool_base > 0) { 3634 printf("VMDQ_DCB multi-queue mode is nonsensical" 3635 " for port %d.", pid); 3636 return -1; 3637 } 3638 3639 /* Assume the ports in testpmd have the same dcb capability 3640 * and has the same number of rxq and txq in dcb mode 3641 */ 3642 if (dcb_mode == DCB_VT_ENABLED) { 3643 if (rte_port->dev_info.max_vfs > 0) { 3644 nb_rxq = rte_port->dev_info.nb_rx_queues; 3645 nb_txq = rte_port->dev_info.nb_tx_queues; 3646 } else { 3647 nb_rxq = rte_port->dev_info.max_rx_queues; 3648 nb_txq = rte_port->dev_info.max_tx_queues; 3649 } 3650 } else { 3651 /*if vt is disabled, use all pf queues */ 3652 if (rte_port->dev_info.vmdq_pool_base == 0) { 3653 nb_rxq = rte_port->dev_info.max_rx_queues; 3654 nb_txq = rte_port->dev_info.max_tx_queues; 3655 } else { 3656 nb_rxq = (queueid_t)num_tcs; 3657 nb_txq = (queueid_t)num_tcs; 3658 3659 } 3660 } 3661 rx_free_thresh = 64; 3662 3663 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3664 3665 rxtx_port_config(rte_port); 3666 /* VLAN filter */ 3667 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3668 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3669 rx_vft_set(pid, vlan_tags[i], 1); 3670 3671 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3672 if (retval != 0) 3673 return retval; 3674 3675 rte_port->dcb_flag = 1; 3676 3677 return 0; 3678 } 3679 3680 static void 3681 init_port(void) 3682 { 3683 int i; 3684 3685 /* Configuration of Ethernet ports. */ 3686 ports = rte_zmalloc("testpmd: ports", 3687 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3688 RTE_CACHE_LINE_SIZE); 3689 if (ports == NULL) { 3690 rte_exit(EXIT_FAILURE, 3691 "rte_zmalloc(%d struct rte_port) failed\n", 3692 RTE_MAX_ETHPORTS); 3693 } 3694 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3695 LIST_INIT(&ports[i].flow_tunnel_list); 3696 /* Initialize ports NUMA structures */ 3697 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3698 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3699 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3700 } 3701 3702 static void 3703 force_quit(void) 3704 { 3705 pmd_test_exit(); 3706 prompt_exit(); 3707 } 3708 3709 static void 3710 print_stats(void) 3711 { 3712 uint8_t i; 3713 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3714 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3715 3716 /* Clear screen and move to top left */ 3717 printf("%s%s", clr, top_left); 3718 3719 printf("\nPort statistics ===================================="); 3720 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3721 nic_stats_display(fwd_ports_ids[i]); 3722 3723 fflush(stdout); 3724 } 3725 3726 static void 3727 signal_handler(int signum) 3728 { 3729 if (signum == SIGINT || signum == SIGTERM) { 3730 printf("\nSignal %d received, preparing to exit...\n", 3731 signum); 3732 #ifdef RTE_LIB_PDUMP 3733 /* uninitialize packet capture framework */ 3734 rte_pdump_uninit(); 3735 #endif 3736 #ifdef RTE_LIB_LATENCYSTATS 3737 if (latencystats_enabled != 0) 3738 rte_latencystats_uninit(); 3739 #endif 3740 force_quit(); 3741 /* Set flag to indicate the force termination. */ 3742 f_quit = 1; 3743 /* exit with the expected status */ 3744 signal(signum, SIG_DFL); 3745 kill(getpid(), signum); 3746 } 3747 } 3748 3749 int 3750 main(int argc, char** argv) 3751 { 3752 int diag; 3753 portid_t port_id; 3754 uint16_t count; 3755 int ret; 3756 3757 signal(SIGINT, signal_handler); 3758 signal(SIGTERM, signal_handler); 3759 3760 testpmd_logtype = rte_log_register("testpmd"); 3761 if (testpmd_logtype < 0) 3762 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3763 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3764 3765 diag = rte_eal_init(argc, argv); 3766 if (diag < 0) 3767 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3768 rte_strerror(rte_errno)); 3769 3770 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3771 rte_exit(EXIT_FAILURE, 3772 "Secondary process type not supported.\n"); 3773 3774 ret = register_eth_event_callback(); 3775 if (ret != 0) 3776 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3777 3778 #ifdef RTE_LIB_PDUMP 3779 /* initialize packet capture framework */ 3780 rte_pdump_init(); 3781 #endif 3782 3783 count = 0; 3784 RTE_ETH_FOREACH_DEV(port_id) { 3785 ports_ids[count] = port_id; 3786 count++; 3787 } 3788 nb_ports = (portid_t) count; 3789 if (nb_ports == 0) 3790 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3791 3792 /* allocate port structures, and init them */ 3793 init_port(); 3794 3795 set_def_fwd_config(); 3796 if (nb_lcores == 0) 3797 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3798 "Check the core mask argument\n"); 3799 3800 /* Bitrate/latency stats disabled by default */ 3801 #ifdef RTE_LIB_BITRATESTATS 3802 bitrate_enabled = 0; 3803 #endif 3804 #ifdef RTE_LIB_LATENCYSTATS 3805 latencystats_enabled = 0; 3806 #endif 3807 3808 /* on FreeBSD, mlockall() is disabled by default */ 3809 #ifdef RTE_EXEC_ENV_FREEBSD 3810 do_mlockall = 0; 3811 #else 3812 do_mlockall = 1; 3813 #endif 3814 3815 argc -= diag; 3816 argv += diag; 3817 if (argc > 1) 3818 launch_args_parse(argc, argv); 3819 3820 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3821 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3822 strerror(errno)); 3823 } 3824 3825 if (tx_first && interactive) 3826 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3827 "interactive mode.\n"); 3828 3829 if (tx_first && lsc_interrupt) { 3830 printf("Warning: lsc_interrupt needs to be off when " 3831 " using tx_first. Disabling.\n"); 3832 lsc_interrupt = 0; 3833 } 3834 3835 if (!nb_rxq && !nb_txq) 3836 printf("Warning: Either rx or tx queues should be non-zero\n"); 3837 3838 if (nb_rxq > 1 && nb_rxq > nb_txq) 3839 printf("Warning: nb_rxq=%d enables RSS configuration, " 3840 "but nb_txq=%d will prevent to fully test it.\n", 3841 nb_rxq, nb_txq); 3842 3843 init_config(); 3844 3845 if (hot_plug) { 3846 ret = rte_dev_hotplug_handle_enable(); 3847 if (ret) { 3848 RTE_LOG(ERR, EAL, 3849 "fail to enable hotplug handling."); 3850 return -1; 3851 } 3852 3853 ret = rte_dev_event_monitor_start(); 3854 if (ret) { 3855 RTE_LOG(ERR, EAL, 3856 "fail to start device event monitoring."); 3857 return -1; 3858 } 3859 3860 ret = rte_dev_event_callback_register(NULL, 3861 dev_event_callback, NULL); 3862 if (ret) { 3863 RTE_LOG(ERR, EAL, 3864 "fail to register device event callback\n"); 3865 return -1; 3866 } 3867 } 3868 3869 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3870 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3871 3872 /* set all ports to promiscuous mode by default */ 3873 RTE_ETH_FOREACH_DEV(port_id) { 3874 ret = rte_eth_promiscuous_enable(port_id); 3875 if (ret != 0) 3876 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3877 port_id, rte_strerror(-ret)); 3878 } 3879 3880 /* Init metrics library */ 3881 rte_metrics_init(rte_socket_id()); 3882 3883 #ifdef RTE_LIB_LATENCYSTATS 3884 if (latencystats_enabled != 0) { 3885 int ret = rte_latencystats_init(1, NULL); 3886 if (ret) 3887 printf("Warning: latencystats init()" 3888 " returned error %d\n", ret); 3889 printf("Latencystats running on lcore %d\n", 3890 latencystats_lcore_id); 3891 } 3892 #endif 3893 3894 /* Setup bitrate stats */ 3895 #ifdef RTE_LIB_BITRATESTATS 3896 if (bitrate_enabled != 0) { 3897 bitrate_data = rte_stats_bitrate_create(); 3898 if (bitrate_data == NULL) 3899 rte_exit(EXIT_FAILURE, 3900 "Could not allocate bitrate data.\n"); 3901 rte_stats_bitrate_reg(bitrate_data); 3902 } 3903 #endif 3904 3905 #ifdef RTE_LIB_CMDLINE 3906 if (strlen(cmdline_filename) != 0) 3907 cmdline_read_from_file(cmdline_filename); 3908 3909 if (interactive == 1) { 3910 if (auto_start) { 3911 printf("Start automatic packet forwarding\n"); 3912 start_packet_forwarding(0); 3913 } 3914 prompt(); 3915 pmd_test_exit(); 3916 } else 3917 #endif 3918 { 3919 char c; 3920 int rc; 3921 3922 f_quit = 0; 3923 3924 printf("No commandline core given, start packet forwarding\n"); 3925 start_packet_forwarding(tx_first); 3926 if (stats_period != 0) { 3927 uint64_t prev_time = 0, cur_time, diff_time = 0; 3928 uint64_t timer_period; 3929 3930 /* Convert to number of cycles */ 3931 timer_period = stats_period * rte_get_timer_hz(); 3932 3933 while (f_quit == 0) { 3934 cur_time = rte_get_timer_cycles(); 3935 diff_time += cur_time - prev_time; 3936 3937 if (diff_time >= timer_period) { 3938 print_stats(); 3939 /* Reset the timer */ 3940 diff_time = 0; 3941 } 3942 /* Sleep to avoid unnecessary checks */ 3943 prev_time = cur_time; 3944 sleep(1); 3945 } 3946 } 3947 3948 printf("Press enter to exit\n"); 3949 rc = read(0, &c, 1); 3950 pmd_test_exit(); 3951 if (rc < 0) 3952 return 1; 3953 } 3954 3955 ret = rte_eal_cleanup(); 3956 if (ret != 0) 3957 rte_exit(EXIT_FAILURE, 3958 "EAL cleanup failed: %s\n", strerror(-ret)); 3959 3960 return EXIT_SUCCESS; 3961 } 3962