1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIB_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIB_LATENCYSTATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use main core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 220 221 /* 222 * Configuration of packet segments used by the "txonly" processing engine. 223 */ 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 226 TXONLY_DEF_PACKET_LEN, 227 }; 228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 229 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 231 /**< Split policy for packets to TX. */ 232 233 uint8_t txonly_multi_flow; 234 /**< Whether multiple flows are generated in TXONLY mode. */ 235 236 uint32_t tx_pkt_times_inter; 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 238 239 uint32_t tx_pkt_times_intra; 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 241 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 245 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 247 uint8_t dcb_config = 0; 248 249 /* 250 * Configurable number of RX/TX queues. 251 */ 252 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 253 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 254 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 255 256 /* 257 * Configurable number of RX/TX ring descriptors. 258 * Defaults are supplied by drivers via ethdev. 259 */ 260 #define RTE_TEST_RX_DESC_DEFAULT 0 261 #define RTE_TEST_TX_DESC_DEFAULT 0 262 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 263 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 264 265 #define RTE_PMD_PARAM_UNSET -1 266 /* 267 * Configurable values of RX and TX ring threshold registers. 268 */ 269 270 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 271 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 272 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 273 274 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 275 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 276 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of RX free threshold. 280 */ 281 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of RX drop enable. 285 */ 286 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 287 288 /* 289 * Configurable value of TX free threshold. 290 */ 291 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 292 293 /* 294 * Configurable value of TX RS bit threshold. 295 */ 296 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 297 298 /* 299 * Configurable value of buffered packets before sending. 300 */ 301 uint16_t noisy_tx_sw_bufsz; 302 303 /* 304 * Configurable value of packet buffer timeout. 305 */ 306 uint16_t noisy_tx_sw_buf_flush_time; 307 308 /* 309 * Configurable value for size of VNF internal memory area 310 * used for simulating noisy neighbour behaviour 311 */ 312 uint64_t noisy_lkup_mem_sz; 313 314 /* 315 * Configurable value of number of random writes done in 316 * VNF simulation memory area. 317 */ 318 uint64_t noisy_lkup_num_writes; 319 320 /* 321 * Configurable value of number of random reads done in 322 * VNF simulation memory area. 323 */ 324 uint64_t noisy_lkup_num_reads; 325 326 /* 327 * Configurable value of number of random reads/writes done in 328 * VNF simulation memory area. 329 */ 330 uint64_t noisy_lkup_num_reads_writes; 331 332 /* 333 * Receive Side Scaling (RSS) configuration. 334 */ 335 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 336 337 /* 338 * Port topology configuration 339 */ 340 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 341 342 /* 343 * Avoids to flush all the RX streams before starts forwarding. 344 */ 345 uint8_t no_flush_rx = 0; /* flush by default */ 346 347 /* 348 * Flow API isolated mode. 349 */ 350 uint8_t flow_isolate_all; 351 352 /* 353 * Avoids to check link status when starting/stopping a port. 354 */ 355 uint8_t no_link_check = 0; /* check by default */ 356 357 /* 358 * Don't automatically start all ports in interactive mode. 359 */ 360 uint8_t no_device_start = 0; 361 362 /* 363 * Enable link status change notification 364 */ 365 uint8_t lsc_interrupt = 1; /* enabled by default */ 366 367 /* 368 * Enable device removal notification. 369 */ 370 uint8_t rmv_interrupt = 1; /* enabled by default */ 371 372 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 373 374 /* After attach, port setup is called on event or by iterator */ 375 bool setup_on_probe_event = true; 376 377 /* Clear ptypes on port initialization. */ 378 uint8_t clear_ptypes = true; 379 380 /* Hairpin ports configuration mode. */ 381 uint16_t hairpin_mode; 382 383 /* Pretty printing of ethdev events */ 384 static const char * const eth_event_desc[] = { 385 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 386 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 387 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 388 [RTE_ETH_EVENT_INTR_RESET] = "reset", 389 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 390 [RTE_ETH_EVENT_IPSEC] = "IPsec", 391 [RTE_ETH_EVENT_MACSEC] = "MACsec", 392 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 393 [RTE_ETH_EVENT_NEW] = "device probed", 394 [RTE_ETH_EVENT_DESTROY] = "device released", 395 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 396 [RTE_ETH_EVENT_MAX] = NULL, 397 }; 398 399 /* 400 * Display or mask ether events 401 * Default to all events except VF_MBOX 402 */ 403 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 404 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 405 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 406 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 407 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 408 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 410 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 411 /* 412 * Decide if all memory are locked for performance. 413 */ 414 int do_mlockall = 0; 415 416 /* 417 * NIC bypass mode configuration options. 418 */ 419 420 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 421 /* The NIC bypass watchdog timeout. */ 422 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 423 #endif 424 425 426 #ifdef RTE_LIB_LATENCYSTATS 427 428 /* 429 * Set when latency stats is enabled in the commandline 430 */ 431 uint8_t latencystats_enabled; 432 433 /* 434 * Lcore ID to serive latency statistics. 435 */ 436 lcoreid_t latencystats_lcore_id = -1; 437 438 #endif 439 440 /* 441 * Ethernet device configuration. 442 */ 443 struct rte_eth_rxmode rx_mode = { 444 /* Default maximum frame length. 445 * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead" 446 * in init_config(). 447 */ 448 .max_rx_pkt_len = 0, 449 }; 450 451 struct rte_eth_txmode tx_mode = { 452 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 453 }; 454 455 struct rte_fdir_conf fdir_conf = { 456 .mode = RTE_FDIR_MODE_NONE, 457 .pballoc = RTE_FDIR_PBALLOC_64K, 458 .status = RTE_FDIR_REPORT_STATUS, 459 .mask = { 460 .vlan_tci_mask = 0xFFEF, 461 .ipv4_mask = { 462 .src_ip = 0xFFFFFFFF, 463 .dst_ip = 0xFFFFFFFF, 464 }, 465 .ipv6_mask = { 466 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 467 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 468 }, 469 .src_port_mask = 0xFFFF, 470 .dst_port_mask = 0xFFFF, 471 .mac_addr_byte_mask = 0xFF, 472 .tunnel_type_mask = 1, 473 .tunnel_id_mask = 0xFFFFFFFF, 474 }, 475 .drop_queue = 127, 476 }; 477 478 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 479 480 /* 481 * Display zero values by default for xstats 482 */ 483 uint8_t xstats_hide_zero; 484 485 /* 486 * Measure of CPU cycles disabled by default 487 */ 488 uint8_t record_core_cycles; 489 490 /* 491 * Display of RX and TX bursts disabled by default 492 */ 493 uint8_t record_burst_stats; 494 495 unsigned int num_sockets = 0; 496 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 497 498 #ifdef RTE_LIB_BITRATESTATS 499 /* Bitrate statistics */ 500 struct rte_stats_bitrates *bitrate_data; 501 lcoreid_t bitrate_lcore_id; 502 uint8_t bitrate_enabled; 503 #endif 504 505 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 506 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 507 508 /* 509 * hexadecimal bitmask of RX mq mode can be enabled. 510 */ 511 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 512 513 /* 514 * Used to set forced link speed 515 */ 516 uint32_t eth_link_speed; 517 518 /* Forward function declarations */ 519 static void setup_attached_port(portid_t pi); 520 static void check_all_ports_link_status(uint32_t port_mask); 521 static int eth_event_callback(portid_t port_id, 522 enum rte_eth_event_type type, 523 void *param, void *ret_param); 524 static void dev_event_callback(const char *device_name, 525 enum rte_dev_event_type type, 526 void *param); 527 528 /* 529 * Check if all the ports are started. 530 * If yes, return positive value. If not, return zero. 531 */ 532 static int all_ports_started(void); 533 534 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 535 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 536 537 /* Holds the registered mbuf dynamic flags names. */ 538 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 539 540 /* 541 * Helper function to check if socket is already discovered. 542 * If yes, return positive value. If not, return zero. 543 */ 544 int 545 new_socket_id(unsigned int socket_id) 546 { 547 unsigned int i; 548 549 for (i = 0; i < num_sockets; i++) { 550 if (socket_ids[i] == socket_id) 551 return 0; 552 } 553 return 1; 554 } 555 556 /* 557 * Setup default configuration. 558 */ 559 static void 560 set_default_fwd_lcores_config(void) 561 { 562 unsigned int i; 563 unsigned int nb_lc; 564 unsigned int sock_num; 565 566 nb_lc = 0; 567 for (i = 0; i < RTE_MAX_LCORE; i++) { 568 if (!rte_lcore_is_enabled(i)) 569 continue; 570 sock_num = rte_lcore_to_socket_id(i); 571 if (new_socket_id(sock_num)) { 572 if (num_sockets >= RTE_MAX_NUMA_NODES) { 573 rte_exit(EXIT_FAILURE, 574 "Total sockets greater than %u\n", 575 RTE_MAX_NUMA_NODES); 576 } 577 socket_ids[num_sockets++] = sock_num; 578 } 579 if (i == rte_get_main_lcore()) 580 continue; 581 fwd_lcores_cpuids[nb_lc++] = i; 582 } 583 nb_lcores = (lcoreid_t) nb_lc; 584 nb_cfg_lcores = nb_lcores; 585 nb_fwd_lcores = 1; 586 } 587 588 static void 589 set_def_peer_eth_addrs(void) 590 { 591 portid_t i; 592 593 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 594 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 595 peer_eth_addrs[i].addr_bytes[5] = i; 596 } 597 } 598 599 static void 600 set_default_fwd_ports_config(void) 601 { 602 portid_t pt_id; 603 int i = 0; 604 605 RTE_ETH_FOREACH_DEV(pt_id) { 606 fwd_ports_ids[i++] = pt_id; 607 608 /* Update sockets info according to the attached device */ 609 int socket_id = rte_eth_dev_socket_id(pt_id); 610 if (socket_id >= 0 && new_socket_id(socket_id)) { 611 if (num_sockets >= RTE_MAX_NUMA_NODES) { 612 rte_exit(EXIT_FAILURE, 613 "Total sockets greater than %u\n", 614 RTE_MAX_NUMA_NODES); 615 } 616 socket_ids[num_sockets++] = socket_id; 617 } 618 } 619 620 nb_cfg_ports = nb_ports; 621 nb_fwd_ports = nb_ports; 622 } 623 624 void 625 set_def_fwd_config(void) 626 { 627 set_default_fwd_lcores_config(); 628 set_def_peer_eth_addrs(); 629 set_default_fwd_ports_config(); 630 } 631 632 /* extremely pessimistic estimation of memory required to create a mempool */ 633 static int 634 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 635 { 636 unsigned int n_pages, mbuf_per_pg, leftover; 637 uint64_t total_mem, mbuf_mem, obj_sz; 638 639 /* there is no good way to predict how much space the mempool will 640 * occupy because it will allocate chunks on the fly, and some of those 641 * will come from default DPDK memory while some will come from our 642 * external memory, so just assume 128MB will be enough for everyone. 643 */ 644 uint64_t hdr_mem = 128 << 20; 645 646 /* account for possible non-contiguousness */ 647 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 648 if (obj_sz > pgsz) { 649 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 650 return -1; 651 } 652 653 mbuf_per_pg = pgsz / obj_sz; 654 leftover = (nb_mbufs % mbuf_per_pg) > 0; 655 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 656 657 mbuf_mem = n_pages * pgsz; 658 659 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 660 661 if (total_mem > SIZE_MAX) { 662 TESTPMD_LOG(ERR, "Memory size too big\n"); 663 return -1; 664 } 665 *out = (size_t)total_mem; 666 667 return 0; 668 } 669 670 static int 671 pagesz_flags(uint64_t page_sz) 672 { 673 /* as per mmap() manpage, all page sizes are log2 of page size 674 * shifted by MAP_HUGE_SHIFT 675 */ 676 int log2 = rte_log2_u64(page_sz); 677 678 return (log2 << HUGE_SHIFT); 679 } 680 681 static void * 682 alloc_mem(size_t memsz, size_t pgsz, bool huge) 683 { 684 void *addr; 685 int flags; 686 687 /* allocate anonymous hugepages */ 688 flags = MAP_ANONYMOUS | MAP_PRIVATE; 689 if (huge) 690 flags |= HUGE_FLAG | pagesz_flags(pgsz); 691 692 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 693 if (addr == MAP_FAILED) 694 return NULL; 695 696 return addr; 697 } 698 699 struct extmem_param { 700 void *addr; 701 size_t len; 702 size_t pgsz; 703 rte_iova_t *iova_table; 704 unsigned int iova_table_len; 705 }; 706 707 static int 708 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 709 bool huge) 710 { 711 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 712 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 713 unsigned int cur_page, n_pages, pgsz_idx; 714 size_t mem_sz, cur_pgsz; 715 rte_iova_t *iovas = NULL; 716 void *addr; 717 int ret; 718 719 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 720 /* skip anything that is too big */ 721 if (pgsizes[pgsz_idx] > SIZE_MAX) 722 continue; 723 724 cur_pgsz = pgsizes[pgsz_idx]; 725 726 /* if we were told not to allocate hugepages, override */ 727 if (!huge) 728 cur_pgsz = sysconf(_SC_PAGESIZE); 729 730 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 731 if (ret < 0) { 732 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 733 return -1; 734 } 735 736 /* allocate our memory */ 737 addr = alloc_mem(mem_sz, cur_pgsz, huge); 738 739 /* if we couldn't allocate memory with a specified page size, 740 * that doesn't mean we can't do it with other page sizes, so 741 * try another one. 742 */ 743 if (addr == NULL) 744 continue; 745 746 /* store IOVA addresses for every page in this memory area */ 747 n_pages = mem_sz / cur_pgsz; 748 749 iovas = malloc(sizeof(*iovas) * n_pages); 750 751 if (iovas == NULL) { 752 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 753 goto fail; 754 } 755 /* lock memory if it's not huge pages */ 756 if (!huge) 757 mlock(addr, mem_sz); 758 759 /* populate IOVA addresses */ 760 for (cur_page = 0; cur_page < n_pages; cur_page++) { 761 rte_iova_t iova; 762 size_t offset; 763 void *cur; 764 765 offset = cur_pgsz * cur_page; 766 cur = RTE_PTR_ADD(addr, offset); 767 768 /* touch the page before getting its IOVA */ 769 *(volatile char *)cur = 0; 770 771 iova = rte_mem_virt2iova(cur); 772 773 iovas[cur_page] = iova; 774 } 775 776 break; 777 } 778 /* if we couldn't allocate anything */ 779 if (iovas == NULL) 780 return -1; 781 782 param->addr = addr; 783 param->len = mem_sz; 784 param->pgsz = cur_pgsz; 785 param->iova_table = iovas; 786 param->iova_table_len = n_pages; 787 788 return 0; 789 fail: 790 if (iovas) 791 free(iovas); 792 if (addr) 793 munmap(addr, mem_sz); 794 795 return -1; 796 } 797 798 static int 799 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 800 { 801 struct extmem_param param; 802 int socket_id, ret; 803 804 memset(¶m, 0, sizeof(param)); 805 806 /* check if our heap exists */ 807 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 808 if (socket_id < 0) { 809 /* create our heap */ 810 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 811 if (ret < 0) { 812 TESTPMD_LOG(ERR, "Cannot create heap\n"); 813 return -1; 814 } 815 } 816 817 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 818 if (ret < 0) { 819 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 820 return -1; 821 } 822 823 /* we now have a valid memory area, so add it to heap */ 824 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 825 param.addr, param.len, param.iova_table, 826 param.iova_table_len, param.pgsz); 827 828 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 829 830 /* not needed any more */ 831 free(param.iova_table); 832 833 if (ret < 0) { 834 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 835 munmap(param.addr, param.len); 836 return -1; 837 } 838 839 /* success */ 840 841 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 842 param.len >> 20); 843 844 return 0; 845 } 846 static void 847 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 848 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 849 { 850 uint16_t pid = 0; 851 int ret; 852 853 RTE_ETH_FOREACH_DEV(pid) { 854 struct rte_eth_dev *dev = 855 &rte_eth_devices[pid]; 856 857 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 858 memhdr->len); 859 if (ret) { 860 TESTPMD_LOG(DEBUG, 861 "unable to DMA unmap addr 0x%p " 862 "for device %s\n", 863 memhdr->addr, dev->data->name); 864 } 865 } 866 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 867 if (ret) { 868 TESTPMD_LOG(DEBUG, 869 "unable to un-register addr 0x%p\n", memhdr->addr); 870 } 871 } 872 873 static void 874 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 875 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 876 { 877 uint16_t pid = 0; 878 size_t page_size = sysconf(_SC_PAGESIZE); 879 int ret; 880 881 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 882 page_size); 883 if (ret) { 884 TESTPMD_LOG(DEBUG, 885 "unable to register addr 0x%p\n", memhdr->addr); 886 return; 887 } 888 RTE_ETH_FOREACH_DEV(pid) { 889 struct rte_eth_dev *dev = 890 &rte_eth_devices[pid]; 891 892 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 893 memhdr->len); 894 if (ret) { 895 TESTPMD_LOG(DEBUG, 896 "unable to DMA map addr 0x%p " 897 "for device %s\n", 898 memhdr->addr, dev->data->name); 899 } 900 } 901 } 902 903 static unsigned int 904 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 905 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 906 { 907 struct rte_pktmbuf_extmem *xmem; 908 unsigned int ext_num, zone_num, elt_num; 909 uint16_t elt_size; 910 911 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 912 elt_num = EXTBUF_ZONE_SIZE / elt_size; 913 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 914 915 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 916 if (xmem == NULL) { 917 TESTPMD_LOG(ERR, "Cannot allocate memory for " 918 "external buffer descriptors\n"); 919 *ext_mem = NULL; 920 return 0; 921 } 922 for (ext_num = 0; ext_num < zone_num; ext_num++) { 923 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 924 const struct rte_memzone *mz; 925 char mz_name[RTE_MEMZONE_NAMESIZE]; 926 int ret; 927 928 ret = snprintf(mz_name, sizeof(mz_name), 929 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 930 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 931 errno = ENAMETOOLONG; 932 ext_num = 0; 933 break; 934 } 935 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 936 socket_id, 937 RTE_MEMZONE_IOVA_CONTIG | 938 RTE_MEMZONE_1GB | 939 RTE_MEMZONE_SIZE_HINT_ONLY, 940 EXTBUF_ZONE_SIZE); 941 if (mz == NULL) { 942 /* 943 * The caller exits on external buffer creation 944 * error, so there is no need to free memzones. 945 */ 946 errno = ENOMEM; 947 ext_num = 0; 948 break; 949 } 950 xseg->buf_ptr = mz->addr; 951 xseg->buf_iova = mz->iova; 952 xseg->buf_len = EXTBUF_ZONE_SIZE; 953 xseg->elt_size = elt_size; 954 } 955 if (ext_num == 0 && xmem != NULL) { 956 free(xmem); 957 xmem = NULL; 958 } 959 *ext_mem = xmem; 960 return ext_num; 961 } 962 963 /* 964 * Configuration initialisation done once at init time. 965 */ 966 static struct rte_mempool * 967 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 968 unsigned int socket_id, uint16_t size_idx) 969 { 970 char pool_name[RTE_MEMPOOL_NAMESIZE]; 971 struct rte_mempool *rte_mp = NULL; 972 uint32_t mb_size; 973 974 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 975 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 976 977 TESTPMD_LOG(INFO, 978 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 979 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 980 981 switch (mp_alloc_type) { 982 case MP_ALLOC_NATIVE: 983 { 984 /* wrapper to rte_mempool_create() */ 985 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 986 rte_mbuf_best_mempool_ops()); 987 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 988 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 989 break; 990 } 991 case MP_ALLOC_ANON: 992 { 993 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 994 mb_size, (unsigned int) mb_mempool_cache, 995 sizeof(struct rte_pktmbuf_pool_private), 996 socket_id, mempool_flags); 997 if (rte_mp == NULL) 998 goto err; 999 1000 if (rte_mempool_populate_anon(rte_mp) == 0) { 1001 rte_mempool_free(rte_mp); 1002 rte_mp = NULL; 1003 goto err; 1004 } 1005 rte_pktmbuf_pool_init(rte_mp, NULL); 1006 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1007 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1008 break; 1009 } 1010 case MP_ALLOC_XMEM: 1011 case MP_ALLOC_XMEM_HUGE: 1012 { 1013 int heap_socket; 1014 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1015 1016 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1017 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1018 1019 heap_socket = 1020 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1021 if (heap_socket < 0) 1022 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1023 1024 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1025 rte_mbuf_best_mempool_ops()); 1026 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1027 mb_mempool_cache, 0, mbuf_seg_size, 1028 heap_socket); 1029 break; 1030 } 1031 case MP_ALLOC_XBUF: 1032 { 1033 struct rte_pktmbuf_extmem *ext_mem; 1034 unsigned int ext_num; 1035 1036 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1037 socket_id, pool_name, &ext_mem); 1038 if (ext_num == 0) 1039 rte_exit(EXIT_FAILURE, 1040 "Can't create pinned data buffers\n"); 1041 1042 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1043 rte_mbuf_best_mempool_ops()); 1044 rte_mp = rte_pktmbuf_pool_create_extbuf 1045 (pool_name, nb_mbuf, mb_mempool_cache, 1046 0, mbuf_seg_size, socket_id, 1047 ext_mem, ext_num); 1048 free(ext_mem); 1049 break; 1050 } 1051 default: 1052 { 1053 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1054 } 1055 } 1056 1057 err: 1058 if (rte_mp == NULL) { 1059 rte_exit(EXIT_FAILURE, 1060 "Creation of mbuf pool for socket %u failed: %s\n", 1061 socket_id, rte_strerror(rte_errno)); 1062 } else if (verbose_level > 0) { 1063 rte_mempool_dump(stdout, rte_mp); 1064 } 1065 return rte_mp; 1066 } 1067 1068 /* 1069 * Check given socket id is valid or not with NUMA mode, 1070 * if valid, return 0, else return -1 1071 */ 1072 static int 1073 check_socket_id(const unsigned int socket_id) 1074 { 1075 static int warning_once = 0; 1076 1077 if (new_socket_id(socket_id)) { 1078 if (!warning_once && numa_support) 1079 printf("Warning: NUMA should be configured manually by" 1080 " using --port-numa-config and" 1081 " --ring-numa-config parameters along with" 1082 " --numa.\n"); 1083 warning_once = 1; 1084 return -1; 1085 } 1086 return 0; 1087 } 1088 1089 /* 1090 * Get the allowed maximum number of RX queues. 1091 * *pid return the port id which has minimal value of 1092 * max_rx_queues in all ports. 1093 */ 1094 queueid_t 1095 get_allowed_max_nb_rxq(portid_t *pid) 1096 { 1097 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1098 bool max_rxq_valid = false; 1099 portid_t pi; 1100 struct rte_eth_dev_info dev_info; 1101 1102 RTE_ETH_FOREACH_DEV(pi) { 1103 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1104 continue; 1105 1106 max_rxq_valid = true; 1107 if (dev_info.max_rx_queues < allowed_max_rxq) { 1108 allowed_max_rxq = dev_info.max_rx_queues; 1109 *pid = pi; 1110 } 1111 } 1112 return max_rxq_valid ? allowed_max_rxq : 0; 1113 } 1114 1115 /* 1116 * Check input rxq is valid or not. 1117 * If input rxq is not greater than any of maximum number 1118 * of RX queues of all ports, it is valid. 1119 * if valid, return 0, else return -1 1120 */ 1121 int 1122 check_nb_rxq(queueid_t rxq) 1123 { 1124 queueid_t allowed_max_rxq; 1125 portid_t pid = 0; 1126 1127 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1128 if (rxq > allowed_max_rxq) { 1129 printf("Fail: input rxq (%u) can't be greater " 1130 "than max_rx_queues (%u) of port %u\n", 1131 rxq, 1132 allowed_max_rxq, 1133 pid); 1134 return -1; 1135 } 1136 return 0; 1137 } 1138 1139 /* 1140 * Get the allowed maximum number of TX queues. 1141 * *pid return the port id which has minimal value of 1142 * max_tx_queues in all ports. 1143 */ 1144 queueid_t 1145 get_allowed_max_nb_txq(portid_t *pid) 1146 { 1147 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1148 bool max_txq_valid = false; 1149 portid_t pi; 1150 struct rte_eth_dev_info dev_info; 1151 1152 RTE_ETH_FOREACH_DEV(pi) { 1153 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1154 continue; 1155 1156 max_txq_valid = true; 1157 if (dev_info.max_tx_queues < allowed_max_txq) { 1158 allowed_max_txq = dev_info.max_tx_queues; 1159 *pid = pi; 1160 } 1161 } 1162 return max_txq_valid ? allowed_max_txq : 0; 1163 } 1164 1165 /* 1166 * Check input txq is valid or not. 1167 * If input txq is not greater than any of maximum number 1168 * of TX queues of all ports, it is valid. 1169 * if valid, return 0, else return -1 1170 */ 1171 int 1172 check_nb_txq(queueid_t txq) 1173 { 1174 queueid_t allowed_max_txq; 1175 portid_t pid = 0; 1176 1177 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1178 if (txq > allowed_max_txq) { 1179 printf("Fail: input txq (%u) can't be greater " 1180 "than max_tx_queues (%u) of port %u\n", 1181 txq, 1182 allowed_max_txq, 1183 pid); 1184 return -1; 1185 } 1186 return 0; 1187 } 1188 1189 /* 1190 * Get the allowed maximum number of RXDs of every rx queue. 1191 * *pid return the port id which has minimal value of 1192 * max_rxd in all queues of all ports. 1193 */ 1194 static uint16_t 1195 get_allowed_max_nb_rxd(portid_t *pid) 1196 { 1197 uint16_t allowed_max_rxd = UINT16_MAX; 1198 portid_t pi; 1199 struct rte_eth_dev_info dev_info; 1200 1201 RTE_ETH_FOREACH_DEV(pi) { 1202 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1203 continue; 1204 1205 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1206 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1207 *pid = pi; 1208 } 1209 } 1210 return allowed_max_rxd; 1211 } 1212 1213 /* 1214 * Get the allowed minimal number of RXDs of every rx queue. 1215 * *pid return the port id which has minimal value of 1216 * min_rxd in all queues of all ports. 1217 */ 1218 static uint16_t 1219 get_allowed_min_nb_rxd(portid_t *pid) 1220 { 1221 uint16_t allowed_min_rxd = 0; 1222 portid_t pi; 1223 struct rte_eth_dev_info dev_info; 1224 1225 RTE_ETH_FOREACH_DEV(pi) { 1226 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1227 continue; 1228 1229 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1230 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1231 *pid = pi; 1232 } 1233 } 1234 1235 return allowed_min_rxd; 1236 } 1237 1238 /* 1239 * Check input rxd is valid or not. 1240 * If input rxd is not greater than any of maximum number 1241 * of RXDs of every Rx queues and is not less than any of 1242 * minimal number of RXDs of every Rx queues, it is valid. 1243 * if valid, return 0, else return -1 1244 */ 1245 int 1246 check_nb_rxd(queueid_t rxd) 1247 { 1248 uint16_t allowed_max_rxd; 1249 uint16_t allowed_min_rxd; 1250 portid_t pid = 0; 1251 1252 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1253 if (rxd > allowed_max_rxd) { 1254 printf("Fail: input rxd (%u) can't be greater " 1255 "than max_rxds (%u) of port %u\n", 1256 rxd, 1257 allowed_max_rxd, 1258 pid); 1259 return -1; 1260 } 1261 1262 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1263 if (rxd < allowed_min_rxd) { 1264 printf("Fail: input rxd (%u) can't be less " 1265 "than min_rxds (%u) of port %u\n", 1266 rxd, 1267 allowed_min_rxd, 1268 pid); 1269 return -1; 1270 } 1271 1272 return 0; 1273 } 1274 1275 /* 1276 * Get the allowed maximum number of TXDs of every rx queues. 1277 * *pid return the port id which has minimal value of 1278 * max_txd in every tx queue. 1279 */ 1280 static uint16_t 1281 get_allowed_max_nb_txd(portid_t *pid) 1282 { 1283 uint16_t allowed_max_txd = UINT16_MAX; 1284 portid_t pi; 1285 struct rte_eth_dev_info dev_info; 1286 1287 RTE_ETH_FOREACH_DEV(pi) { 1288 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1289 continue; 1290 1291 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1292 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1293 *pid = pi; 1294 } 1295 } 1296 return allowed_max_txd; 1297 } 1298 1299 /* 1300 * Get the allowed maximum number of TXDs of every tx queues. 1301 * *pid return the port id which has minimal value of 1302 * min_txd in every tx queue. 1303 */ 1304 static uint16_t 1305 get_allowed_min_nb_txd(portid_t *pid) 1306 { 1307 uint16_t allowed_min_txd = 0; 1308 portid_t pi; 1309 struct rte_eth_dev_info dev_info; 1310 1311 RTE_ETH_FOREACH_DEV(pi) { 1312 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1313 continue; 1314 1315 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1316 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1317 *pid = pi; 1318 } 1319 } 1320 1321 return allowed_min_txd; 1322 } 1323 1324 /* 1325 * Check input txd is valid or not. 1326 * If input txd is not greater than any of maximum number 1327 * of TXDs of every Rx queues, it is valid. 1328 * if valid, return 0, else return -1 1329 */ 1330 int 1331 check_nb_txd(queueid_t txd) 1332 { 1333 uint16_t allowed_max_txd; 1334 uint16_t allowed_min_txd; 1335 portid_t pid = 0; 1336 1337 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1338 if (txd > allowed_max_txd) { 1339 printf("Fail: input txd (%u) can't be greater " 1340 "than max_txds (%u) of port %u\n", 1341 txd, 1342 allowed_max_txd, 1343 pid); 1344 return -1; 1345 } 1346 1347 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1348 if (txd < allowed_min_txd) { 1349 printf("Fail: input txd (%u) can't be less " 1350 "than min_txds (%u) of port %u\n", 1351 txd, 1352 allowed_min_txd, 1353 pid); 1354 return -1; 1355 } 1356 return 0; 1357 } 1358 1359 1360 /* 1361 * Get the allowed maximum number of hairpin queues. 1362 * *pid return the port id which has minimal value of 1363 * max_hairpin_queues in all ports. 1364 */ 1365 queueid_t 1366 get_allowed_max_nb_hairpinq(portid_t *pid) 1367 { 1368 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1369 portid_t pi; 1370 struct rte_eth_hairpin_cap cap; 1371 1372 RTE_ETH_FOREACH_DEV(pi) { 1373 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1374 *pid = pi; 1375 return 0; 1376 } 1377 if (cap.max_nb_queues < allowed_max_hairpinq) { 1378 allowed_max_hairpinq = cap.max_nb_queues; 1379 *pid = pi; 1380 } 1381 } 1382 return allowed_max_hairpinq; 1383 } 1384 1385 /* 1386 * Check input hairpin is valid or not. 1387 * If input hairpin is not greater than any of maximum number 1388 * of hairpin queues of all ports, it is valid. 1389 * if valid, return 0, else return -1 1390 */ 1391 int 1392 check_nb_hairpinq(queueid_t hairpinq) 1393 { 1394 queueid_t allowed_max_hairpinq; 1395 portid_t pid = 0; 1396 1397 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1398 if (hairpinq > allowed_max_hairpinq) { 1399 printf("Fail: input hairpin (%u) can't be greater " 1400 "than max_hairpin_queues (%u) of port %u\n", 1401 hairpinq, allowed_max_hairpinq, pid); 1402 return -1; 1403 } 1404 return 0; 1405 } 1406 1407 static void 1408 init_config(void) 1409 { 1410 portid_t pid; 1411 struct rte_port *port; 1412 struct rte_mempool *mbp; 1413 unsigned int nb_mbuf_per_pool; 1414 lcoreid_t lc_id; 1415 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1416 struct rte_gro_param gro_param; 1417 uint32_t gso_types; 1418 uint16_t data_size; 1419 bool warning = 0; 1420 int k; 1421 int ret; 1422 1423 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1424 1425 /* Configuration of logical cores. */ 1426 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1427 sizeof(struct fwd_lcore *) * nb_lcores, 1428 RTE_CACHE_LINE_SIZE); 1429 if (fwd_lcores == NULL) { 1430 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1431 "failed\n", nb_lcores); 1432 } 1433 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1434 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1435 sizeof(struct fwd_lcore), 1436 RTE_CACHE_LINE_SIZE); 1437 if (fwd_lcores[lc_id] == NULL) { 1438 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1439 "failed\n"); 1440 } 1441 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1442 } 1443 1444 RTE_ETH_FOREACH_DEV(pid) { 1445 port = &ports[pid]; 1446 /* Apply default TxRx configuration for all ports */ 1447 port->dev_conf.txmode = tx_mode; 1448 port->dev_conf.rxmode = rx_mode; 1449 1450 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1451 if (ret != 0) 1452 rte_exit(EXIT_FAILURE, 1453 "rte_eth_dev_info_get() failed\n"); 1454 1455 ret = update_jumbo_frame_offload(pid); 1456 if (ret != 0) 1457 printf("Updating jumbo frame offload failed for port %u\n", 1458 pid); 1459 1460 if (!(port->dev_info.tx_offload_capa & 1461 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1462 port->dev_conf.txmode.offloads &= 1463 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1464 if (numa_support) { 1465 if (port_numa[pid] != NUMA_NO_CONFIG) 1466 port_per_socket[port_numa[pid]]++; 1467 else { 1468 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1469 1470 /* 1471 * if socket_id is invalid, 1472 * set to the first available socket. 1473 */ 1474 if (check_socket_id(socket_id) < 0) 1475 socket_id = socket_ids[0]; 1476 port_per_socket[socket_id]++; 1477 } 1478 } 1479 1480 /* Apply Rx offloads configuration */ 1481 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1482 port->rx_conf[k].offloads = 1483 port->dev_conf.rxmode.offloads; 1484 /* Apply Tx offloads configuration */ 1485 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1486 port->tx_conf[k].offloads = 1487 port->dev_conf.txmode.offloads; 1488 1489 if (eth_link_speed) 1490 port->dev_conf.link_speeds = eth_link_speed; 1491 1492 /* set flag to initialize port/queue */ 1493 port->need_reconfig = 1; 1494 port->need_reconfig_queues = 1; 1495 port->tx_metadata = 0; 1496 1497 /* Check for maximum number of segments per MTU. Accordingly 1498 * update the mbuf data size. 1499 */ 1500 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1501 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1502 data_size = rx_mode.max_rx_pkt_len / 1503 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1504 1505 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1506 mbuf_data_size[0]) { 1507 mbuf_data_size[0] = data_size + 1508 RTE_PKTMBUF_HEADROOM; 1509 warning = 1; 1510 } 1511 } 1512 } 1513 1514 if (warning) 1515 TESTPMD_LOG(WARNING, 1516 "Configured mbuf size of the first segment %hu\n", 1517 mbuf_data_size[0]); 1518 /* 1519 * Create pools of mbuf. 1520 * If NUMA support is disabled, create a single pool of mbuf in 1521 * socket 0 memory by default. 1522 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1523 * 1524 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1525 * nb_txd can be configured at run time. 1526 */ 1527 if (param_total_num_mbufs) 1528 nb_mbuf_per_pool = param_total_num_mbufs; 1529 else { 1530 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1531 (nb_lcores * mb_mempool_cache) + 1532 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1533 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1534 } 1535 1536 if (numa_support) { 1537 uint8_t i, j; 1538 1539 for (i = 0; i < num_sockets; i++) 1540 for (j = 0; j < mbuf_data_size_n; j++) 1541 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1542 mbuf_pool_create(mbuf_data_size[j], 1543 nb_mbuf_per_pool, 1544 socket_ids[i], j); 1545 } else { 1546 uint8_t i; 1547 1548 for (i = 0; i < mbuf_data_size_n; i++) 1549 mempools[i] = mbuf_pool_create 1550 (mbuf_data_size[i], 1551 nb_mbuf_per_pool, 1552 socket_num == UMA_NO_CONFIG ? 1553 0 : socket_num, i); 1554 } 1555 1556 init_port_config(); 1557 1558 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1559 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1560 /* 1561 * Records which Mbuf pool to use by each logical core, if needed. 1562 */ 1563 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1564 mbp = mbuf_pool_find( 1565 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1566 1567 if (mbp == NULL) 1568 mbp = mbuf_pool_find(0, 0); 1569 fwd_lcores[lc_id]->mbp = mbp; 1570 /* initialize GSO context */ 1571 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1572 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1573 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1574 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1575 RTE_ETHER_CRC_LEN; 1576 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1577 } 1578 1579 fwd_config_setup(); 1580 1581 /* create a gro context for each lcore */ 1582 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1583 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1584 gro_param.max_item_per_flow = MAX_PKT_BURST; 1585 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1586 gro_param.socket_id = rte_lcore_to_socket_id( 1587 fwd_lcores_cpuids[lc_id]); 1588 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1589 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1590 rte_exit(EXIT_FAILURE, 1591 "rte_gro_ctx_create() failed\n"); 1592 } 1593 } 1594 } 1595 1596 1597 void 1598 reconfig(portid_t new_port_id, unsigned socket_id) 1599 { 1600 struct rte_port *port; 1601 int ret; 1602 1603 /* Reconfiguration of Ethernet ports. */ 1604 port = &ports[new_port_id]; 1605 1606 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1607 if (ret != 0) 1608 return; 1609 1610 /* set flag to initialize port/queue */ 1611 port->need_reconfig = 1; 1612 port->need_reconfig_queues = 1; 1613 port->socket_id = socket_id; 1614 1615 init_port_config(); 1616 } 1617 1618 1619 int 1620 init_fwd_streams(void) 1621 { 1622 portid_t pid; 1623 struct rte_port *port; 1624 streamid_t sm_id, nb_fwd_streams_new; 1625 queueid_t q; 1626 1627 /* set socket id according to numa or not */ 1628 RTE_ETH_FOREACH_DEV(pid) { 1629 port = &ports[pid]; 1630 if (nb_rxq > port->dev_info.max_rx_queues) { 1631 printf("Fail: nb_rxq(%d) is greater than " 1632 "max_rx_queues(%d)\n", nb_rxq, 1633 port->dev_info.max_rx_queues); 1634 return -1; 1635 } 1636 if (nb_txq > port->dev_info.max_tx_queues) { 1637 printf("Fail: nb_txq(%d) is greater than " 1638 "max_tx_queues(%d)\n", nb_txq, 1639 port->dev_info.max_tx_queues); 1640 return -1; 1641 } 1642 if (numa_support) { 1643 if (port_numa[pid] != NUMA_NO_CONFIG) 1644 port->socket_id = port_numa[pid]; 1645 else { 1646 port->socket_id = rte_eth_dev_socket_id(pid); 1647 1648 /* 1649 * if socket_id is invalid, 1650 * set to the first available socket. 1651 */ 1652 if (check_socket_id(port->socket_id) < 0) 1653 port->socket_id = socket_ids[0]; 1654 } 1655 } 1656 else { 1657 if (socket_num == UMA_NO_CONFIG) 1658 port->socket_id = 0; 1659 else 1660 port->socket_id = socket_num; 1661 } 1662 } 1663 1664 q = RTE_MAX(nb_rxq, nb_txq); 1665 if (q == 0) { 1666 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1667 return -1; 1668 } 1669 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1670 if (nb_fwd_streams_new == nb_fwd_streams) 1671 return 0; 1672 /* clear the old */ 1673 if (fwd_streams != NULL) { 1674 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1675 if (fwd_streams[sm_id] == NULL) 1676 continue; 1677 rte_free(fwd_streams[sm_id]); 1678 fwd_streams[sm_id] = NULL; 1679 } 1680 rte_free(fwd_streams); 1681 fwd_streams = NULL; 1682 } 1683 1684 /* init new */ 1685 nb_fwd_streams = nb_fwd_streams_new; 1686 if (nb_fwd_streams) { 1687 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1688 sizeof(struct fwd_stream *) * nb_fwd_streams, 1689 RTE_CACHE_LINE_SIZE); 1690 if (fwd_streams == NULL) 1691 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1692 " (struct fwd_stream *)) failed\n", 1693 nb_fwd_streams); 1694 1695 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1696 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1697 " struct fwd_stream", sizeof(struct fwd_stream), 1698 RTE_CACHE_LINE_SIZE); 1699 if (fwd_streams[sm_id] == NULL) 1700 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1701 "(struct fwd_stream) failed\n"); 1702 } 1703 } 1704 1705 return 0; 1706 } 1707 1708 static void 1709 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1710 { 1711 uint64_t total_burst, sburst; 1712 uint64_t nb_burst; 1713 uint64_t burst_stats[4]; 1714 uint16_t pktnb_stats[4]; 1715 uint16_t nb_pkt; 1716 int burst_percent[4], sburstp; 1717 int i; 1718 1719 /* 1720 * First compute the total number of packet bursts and the 1721 * two highest numbers of bursts of the same number of packets. 1722 */ 1723 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1724 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1725 1726 /* Show stats for 0 burst size always */ 1727 total_burst = pbs->pkt_burst_spread[0]; 1728 burst_stats[0] = pbs->pkt_burst_spread[0]; 1729 pktnb_stats[0] = 0; 1730 1731 /* Find the next 2 burst sizes with highest occurrences. */ 1732 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1733 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1734 1735 if (nb_burst == 0) 1736 continue; 1737 1738 total_burst += nb_burst; 1739 1740 if (nb_burst > burst_stats[1]) { 1741 burst_stats[2] = burst_stats[1]; 1742 pktnb_stats[2] = pktnb_stats[1]; 1743 burst_stats[1] = nb_burst; 1744 pktnb_stats[1] = nb_pkt; 1745 } else if (nb_burst > burst_stats[2]) { 1746 burst_stats[2] = nb_burst; 1747 pktnb_stats[2] = nb_pkt; 1748 } 1749 } 1750 if (total_burst == 0) 1751 return; 1752 1753 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1754 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1755 if (i == 3) { 1756 printf("%d%% of other]\n", 100 - sburstp); 1757 return; 1758 } 1759 1760 sburst += burst_stats[i]; 1761 if (sburst == total_burst) { 1762 printf("%d%% of %d pkts]\n", 1763 100 - sburstp, (int) pktnb_stats[i]); 1764 return; 1765 } 1766 1767 burst_percent[i] = 1768 (double)burst_stats[i] / total_burst * 100; 1769 printf("%d%% of %d pkts + ", 1770 burst_percent[i], (int) pktnb_stats[i]); 1771 sburstp += burst_percent[i]; 1772 } 1773 } 1774 1775 static void 1776 fwd_stream_stats_display(streamid_t stream_id) 1777 { 1778 struct fwd_stream *fs; 1779 static const char *fwd_top_stats_border = "-------"; 1780 1781 fs = fwd_streams[stream_id]; 1782 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1783 (fs->fwd_dropped == 0)) 1784 return; 1785 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1786 "TX Port=%2d/Queue=%2d %s\n", 1787 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1788 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1789 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1790 " TX-dropped: %-14"PRIu64, 1791 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1792 1793 /* if checksum mode */ 1794 if (cur_fwd_eng == &csum_fwd_engine) { 1795 printf(" RX- bad IP checksum: %-14"PRIu64 1796 " Rx- bad L4 checksum: %-14"PRIu64 1797 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1798 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1799 fs->rx_bad_outer_l4_csum); 1800 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1801 fs->rx_bad_outer_ip_csum); 1802 } else { 1803 printf("\n"); 1804 } 1805 1806 if (record_burst_stats) { 1807 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1808 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1809 } 1810 } 1811 1812 void 1813 fwd_stats_display(void) 1814 { 1815 static const char *fwd_stats_border = "----------------------"; 1816 static const char *acc_stats_border = "+++++++++++++++"; 1817 struct { 1818 struct fwd_stream *rx_stream; 1819 struct fwd_stream *tx_stream; 1820 uint64_t tx_dropped; 1821 uint64_t rx_bad_ip_csum; 1822 uint64_t rx_bad_l4_csum; 1823 uint64_t rx_bad_outer_l4_csum; 1824 uint64_t rx_bad_outer_ip_csum; 1825 } ports_stats[RTE_MAX_ETHPORTS]; 1826 uint64_t total_rx_dropped = 0; 1827 uint64_t total_tx_dropped = 0; 1828 uint64_t total_rx_nombuf = 0; 1829 struct rte_eth_stats stats; 1830 uint64_t fwd_cycles = 0; 1831 uint64_t total_recv = 0; 1832 uint64_t total_xmit = 0; 1833 struct rte_port *port; 1834 streamid_t sm_id; 1835 portid_t pt_id; 1836 int i; 1837 1838 memset(ports_stats, 0, sizeof(ports_stats)); 1839 1840 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1841 struct fwd_stream *fs = fwd_streams[sm_id]; 1842 1843 if (cur_fwd_config.nb_fwd_streams > 1844 cur_fwd_config.nb_fwd_ports) { 1845 fwd_stream_stats_display(sm_id); 1846 } else { 1847 ports_stats[fs->tx_port].tx_stream = fs; 1848 ports_stats[fs->rx_port].rx_stream = fs; 1849 } 1850 1851 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1852 1853 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1854 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1855 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1856 fs->rx_bad_outer_l4_csum; 1857 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 1858 fs->rx_bad_outer_ip_csum; 1859 1860 if (record_core_cycles) 1861 fwd_cycles += fs->core_cycles; 1862 } 1863 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1864 pt_id = fwd_ports_ids[i]; 1865 port = &ports[pt_id]; 1866 1867 rte_eth_stats_get(pt_id, &stats); 1868 stats.ipackets -= port->stats.ipackets; 1869 stats.opackets -= port->stats.opackets; 1870 stats.ibytes -= port->stats.ibytes; 1871 stats.obytes -= port->stats.obytes; 1872 stats.imissed -= port->stats.imissed; 1873 stats.oerrors -= port->stats.oerrors; 1874 stats.rx_nombuf -= port->stats.rx_nombuf; 1875 1876 total_recv += stats.ipackets; 1877 total_xmit += stats.opackets; 1878 total_rx_dropped += stats.imissed; 1879 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1880 total_tx_dropped += stats.oerrors; 1881 total_rx_nombuf += stats.rx_nombuf; 1882 1883 printf("\n %s Forward statistics for port %-2d %s\n", 1884 fwd_stats_border, pt_id, fwd_stats_border); 1885 1886 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 1887 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 1888 stats.ipackets + stats.imissed); 1889 1890 if (cur_fwd_eng == &csum_fwd_engine) { 1891 printf(" Bad-ipcsum: %-14"PRIu64 1892 " Bad-l4csum: %-14"PRIu64 1893 "Bad-outer-l4csum: %-14"PRIu64"\n", 1894 ports_stats[pt_id].rx_bad_ip_csum, 1895 ports_stats[pt_id].rx_bad_l4_csum, 1896 ports_stats[pt_id].rx_bad_outer_l4_csum); 1897 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 1898 ports_stats[pt_id].rx_bad_outer_ip_csum); 1899 } 1900 if (stats.ierrors + stats.rx_nombuf > 0) { 1901 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 1902 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 1903 } 1904 1905 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 1906 "TX-total: %-"PRIu64"\n", 1907 stats.opackets, ports_stats[pt_id].tx_dropped, 1908 stats.opackets + ports_stats[pt_id].tx_dropped); 1909 1910 if (record_burst_stats) { 1911 if (ports_stats[pt_id].rx_stream) 1912 pkt_burst_stats_display("RX", 1913 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1914 if (ports_stats[pt_id].tx_stream) 1915 pkt_burst_stats_display("TX", 1916 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1917 } 1918 1919 printf(" %s--------------------------------%s\n", 1920 fwd_stats_border, fwd_stats_border); 1921 } 1922 1923 printf("\n %s Accumulated forward statistics for all ports" 1924 "%s\n", 1925 acc_stats_border, acc_stats_border); 1926 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1927 "%-"PRIu64"\n" 1928 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1929 "%-"PRIu64"\n", 1930 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1931 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1932 if (total_rx_nombuf > 0) 1933 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1934 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1935 "%s\n", 1936 acc_stats_border, acc_stats_border); 1937 if (record_core_cycles) { 1938 #define CYC_PER_MHZ 1E6 1939 if (total_recv > 0 || total_xmit > 0) { 1940 uint64_t total_pkts = 0; 1941 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1942 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1943 total_pkts = total_xmit; 1944 else 1945 total_pkts = total_recv; 1946 1947 printf("\n CPU cycles/packet=%.2F (total cycles=" 1948 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1949 " MHz Clock\n", 1950 (double) fwd_cycles / total_pkts, 1951 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1952 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1953 } 1954 } 1955 } 1956 1957 void 1958 fwd_stats_reset(void) 1959 { 1960 streamid_t sm_id; 1961 portid_t pt_id; 1962 int i; 1963 1964 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1965 pt_id = fwd_ports_ids[i]; 1966 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1967 } 1968 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1969 struct fwd_stream *fs = fwd_streams[sm_id]; 1970 1971 fs->rx_packets = 0; 1972 fs->tx_packets = 0; 1973 fs->fwd_dropped = 0; 1974 fs->rx_bad_ip_csum = 0; 1975 fs->rx_bad_l4_csum = 0; 1976 fs->rx_bad_outer_l4_csum = 0; 1977 fs->rx_bad_outer_ip_csum = 0; 1978 1979 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1980 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1981 fs->core_cycles = 0; 1982 } 1983 } 1984 1985 static void 1986 flush_fwd_rx_queues(void) 1987 { 1988 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1989 portid_t rxp; 1990 portid_t port_id; 1991 queueid_t rxq; 1992 uint16_t nb_rx; 1993 uint16_t i; 1994 uint8_t j; 1995 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1996 uint64_t timer_period; 1997 1998 /* convert to number of cycles */ 1999 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2000 2001 for (j = 0; j < 2; j++) { 2002 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2003 for (rxq = 0; rxq < nb_rxq; rxq++) { 2004 port_id = fwd_ports_ids[rxp]; 2005 /** 2006 * testpmd can stuck in the below do while loop 2007 * if rte_eth_rx_burst() always returns nonzero 2008 * packets. So timer is added to exit this loop 2009 * after 1sec timer expiry. 2010 */ 2011 prev_tsc = rte_rdtsc(); 2012 do { 2013 nb_rx = rte_eth_rx_burst(port_id, rxq, 2014 pkts_burst, MAX_PKT_BURST); 2015 for (i = 0; i < nb_rx; i++) 2016 rte_pktmbuf_free(pkts_burst[i]); 2017 2018 cur_tsc = rte_rdtsc(); 2019 diff_tsc = cur_tsc - prev_tsc; 2020 timer_tsc += diff_tsc; 2021 } while ((nb_rx > 0) && 2022 (timer_tsc < timer_period)); 2023 timer_tsc = 0; 2024 } 2025 } 2026 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2027 } 2028 } 2029 2030 static void 2031 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2032 { 2033 struct fwd_stream **fsm; 2034 streamid_t nb_fs; 2035 streamid_t sm_id; 2036 #ifdef RTE_LIB_BITRATESTATS 2037 uint64_t tics_per_1sec; 2038 uint64_t tics_datum; 2039 uint64_t tics_current; 2040 uint16_t i, cnt_ports; 2041 2042 cnt_ports = nb_ports; 2043 tics_datum = rte_rdtsc(); 2044 tics_per_1sec = rte_get_timer_hz(); 2045 #endif 2046 fsm = &fwd_streams[fc->stream_idx]; 2047 nb_fs = fc->stream_nb; 2048 do { 2049 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2050 (*pkt_fwd)(fsm[sm_id]); 2051 #ifdef RTE_LIB_BITRATESTATS 2052 if (bitrate_enabled != 0 && 2053 bitrate_lcore_id == rte_lcore_id()) { 2054 tics_current = rte_rdtsc(); 2055 if (tics_current - tics_datum >= tics_per_1sec) { 2056 /* Periodic bitrate calculation */ 2057 for (i = 0; i < cnt_ports; i++) 2058 rte_stats_bitrate_calc(bitrate_data, 2059 ports_ids[i]); 2060 tics_datum = tics_current; 2061 } 2062 } 2063 #endif 2064 #ifdef RTE_LIB_LATENCYSTATS 2065 if (latencystats_enabled != 0 && 2066 latencystats_lcore_id == rte_lcore_id()) 2067 rte_latencystats_update(); 2068 #endif 2069 2070 } while (! fc->stopped); 2071 } 2072 2073 static int 2074 start_pkt_forward_on_core(void *fwd_arg) 2075 { 2076 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2077 cur_fwd_config.fwd_eng->packet_fwd); 2078 return 0; 2079 } 2080 2081 /* 2082 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2083 * Used to start communication flows in network loopback test configurations. 2084 */ 2085 static int 2086 run_one_txonly_burst_on_core(void *fwd_arg) 2087 { 2088 struct fwd_lcore *fwd_lc; 2089 struct fwd_lcore tmp_lcore; 2090 2091 fwd_lc = (struct fwd_lcore *) fwd_arg; 2092 tmp_lcore = *fwd_lc; 2093 tmp_lcore.stopped = 1; 2094 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2095 return 0; 2096 } 2097 2098 /* 2099 * Launch packet forwarding: 2100 * - Setup per-port forwarding context. 2101 * - launch logical cores with their forwarding configuration. 2102 */ 2103 static void 2104 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2105 { 2106 port_fwd_begin_t port_fwd_begin; 2107 unsigned int i; 2108 unsigned int lc_id; 2109 int diag; 2110 2111 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2112 if (port_fwd_begin != NULL) { 2113 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2114 (*port_fwd_begin)(fwd_ports_ids[i]); 2115 } 2116 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2117 lc_id = fwd_lcores_cpuids[i]; 2118 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2119 fwd_lcores[i]->stopped = 0; 2120 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2121 fwd_lcores[i], lc_id); 2122 if (diag != 0) 2123 printf("launch lcore %u failed - diag=%d\n", 2124 lc_id, diag); 2125 } 2126 } 2127 } 2128 2129 /* 2130 * Launch packet forwarding configuration. 2131 */ 2132 void 2133 start_packet_forwarding(int with_tx_first) 2134 { 2135 port_fwd_begin_t port_fwd_begin; 2136 port_fwd_end_t port_fwd_end; 2137 unsigned int i; 2138 2139 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2140 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2141 2142 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2143 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2144 2145 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2146 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2147 (!nb_rxq || !nb_txq)) 2148 rte_exit(EXIT_FAILURE, 2149 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2150 cur_fwd_eng->fwd_mode_name); 2151 2152 if (all_ports_started() == 0) { 2153 printf("Not all ports were started\n"); 2154 return; 2155 } 2156 if (test_done == 0) { 2157 printf("Packet forwarding already started\n"); 2158 return; 2159 } 2160 test_done = 0; 2161 2162 fwd_config_setup(); 2163 2164 if(!no_flush_rx) 2165 flush_fwd_rx_queues(); 2166 2167 pkt_fwd_config_display(&cur_fwd_config); 2168 rxtx_config_display(); 2169 2170 fwd_stats_reset(); 2171 if (with_tx_first) { 2172 port_fwd_begin = tx_only_engine.port_fwd_begin; 2173 if (port_fwd_begin != NULL) { 2174 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2175 (*port_fwd_begin)(fwd_ports_ids[i]); 2176 } 2177 while (with_tx_first--) { 2178 launch_packet_forwarding( 2179 run_one_txonly_burst_on_core); 2180 rte_eal_mp_wait_lcore(); 2181 } 2182 port_fwd_end = tx_only_engine.port_fwd_end; 2183 if (port_fwd_end != NULL) { 2184 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2185 (*port_fwd_end)(fwd_ports_ids[i]); 2186 } 2187 } 2188 launch_packet_forwarding(start_pkt_forward_on_core); 2189 } 2190 2191 void 2192 stop_packet_forwarding(void) 2193 { 2194 port_fwd_end_t port_fwd_end; 2195 lcoreid_t lc_id; 2196 portid_t pt_id; 2197 int i; 2198 2199 if (test_done) { 2200 printf("Packet forwarding not started\n"); 2201 return; 2202 } 2203 printf("Telling cores to stop..."); 2204 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2205 fwd_lcores[lc_id]->stopped = 1; 2206 printf("\nWaiting for lcores to finish...\n"); 2207 rte_eal_mp_wait_lcore(); 2208 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2209 if (port_fwd_end != NULL) { 2210 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2211 pt_id = fwd_ports_ids[i]; 2212 (*port_fwd_end)(pt_id); 2213 } 2214 } 2215 2216 fwd_stats_display(); 2217 2218 printf("\nDone.\n"); 2219 test_done = 1; 2220 } 2221 2222 void 2223 dev_set_link_up(portid_t pid) 2224 { 2225 if (rte_eth_dev_set_link_up(pid) < 0) 2226 printf("\nSet link up fail.\n"); 2227 } 2228 2229 void 2230 dev_set_link_down(portid_t pid) 2231 { 2232 if (rte_eth_dev_set_link_down(pid) < 0) 2233 printf("\nSet link down fail.\n"); 2234 } 2235 2236 static int 2237 all_ports_started(void) 2238 { 2239 portid_t pi; 2240 struct rte_port *port; 2241 2242 RTE_ETH_FOREACH_DEV(pi) { 2243 port = &ports[pi]; 2244 /* Check if there is a port which is not started */ 2245 if ((port->port_status != RTE_PORT_STARTED) && 2246 (port->slave_flag == 0)) 2247 return 0; 2248 } 2249 2250 /* No port is not started */ 2251 return 1; 2252 } 2253 2254 int 2255 port_is_stopped(portid_t port_id) 2256 { 2257 struct rte_port *port = &ports[port_id]; 2258 2259 if ((port->port_status != RTE_PORT_STOPPED) && 2260 (port->slave_flag == 0)) 2261 return 0; 2262 return 1; 2263 } 2264 2265 int 2266 all_ports_stopped(void) 2267 { 2268 portid_t pi; 2269 2270 RTE_ETH_FOREACH_DEV(pi) { 2271 if (!port_is_stopped(pi)) 2272 return 0; 2273 } 2274 2275 return 1; 2276 } 2277 2278 int 2279 port_is_started(portid_t port_id) 2280 { 2281 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2282 return 0; 2283 2284 if (ports[port_id].port_status != RTE_PORT_STARTED) 2285 return 0; 2286 2287 return 1; 2288 } 2289 2290 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2291 static int 2292 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2293 { 2294 queueid_t qi; 2295 struct rte_eth_hairpin_conf hairpin_conf = { 2296 .peer_count = 1, 2297 }; 2298 int i; 2299 int diag; 2300 struct rte_port *port = &ports[pi]; 2301 uint16_t peer_rx_port = pi; 2302 uint16_t peer_tx_port = pi; 2303 uint32_t manual = 1; 2304 uint32_t tx_exp = hairpin_mode & 0x10; 2305 2306 if (!(hairpin_mode & 0xf)) { 2307 peer_rx_port = pi; 2308 peer_tx_port = pi; 2309 manual = 0; 2310 } else if (hairpin_mode & 0x1) { 2311 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2312 RTE_ETH_DEV_NO_OWNER); 2313 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2314 peer_tx_port = rte_eth_find_next_owned_by(0, 2315 RTE_ETH_DEV_NO_OWNER); 2316 if (p_pi != RTE_MAX_ETHPORTS) { 2317 peer_rx_port = p_pi; 2318 } else { 2319 uint16_t next_pi; 2320 2321 /* Last port will be the peer RX port of the first. */ 2322 RTE_ETH_FOREACH_DEV(next_pi) 2323 peer_rx_port = next_pi; 2324 } 2325 manual = 1; 2326 } else if (hairpin_mode & 0x2) { 2327 if (cnt_pi & 0x1) { 2328 peer_rx_port = p_pi; 2329 } else { 2330 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2331 RTE_ETH_DEV_NO_OWNER); 2332 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2333 peer_rx_port = pi; 2334 } 2335 peer_tx_port = peer_rx_port; 2336 manual = 1; 2337 } 2338 2339 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2340 hairpin_conf.peers[0].port = peer_rx_port; 2341 hairpin_conf.peers[0].queue = i + nb_rxq; 2342 hairpin_conf.manual_bind = !!manual; 2343 hairpin_conf.tx_explicit = !!tx_exp; 2344 diag = rte_eth_tx_hairpin_queue_setup 2345 (pi, qi, nb_txd, &hairpin_conf); 2346 i++; 2347 if (diag == 0) 2348 continue; 2349 2350 /* Fail to setup rx queue, return */ 2351 if (rte_atomic16_cmpset(&(port->port_status), 2352 RTE_PORT_HANDLING, 2353 RTE_PORT_STOPPED) == 0) 2354 printf("Port %d can not be set back " 2355 "to stopped\n", pi); 2356 printf("Fail to configure port %d hairpin " 2357 "queues\n", pi); 2358 /* try to reconfigure queues next time */ 2359 port->need_reconfig_queues = 1; 2360 return -1; 2361 } 2362 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2363 hairpin_conf.peers[0].port = peer_tx_port; 2364 hairpin_conf.peers[0].queue = i + nb_txq; 2365 hairpin_conf.manual_bind = !!manual; 2366 hairpin_conf.tx_explicit = !!tx_exp; 2367 diag = rte_eth_rx_hairpin_queue_setup 2368 (pi, qi, nb_rxd, &hairpin_conf); 2369 i++; 2370 if (diag == 0) 2371 continue; 2372 2373 /* Fail to setup rx queue, return */ 2374 if (rte_atomic16_cmpset(&(port->port_status), 2375 RTE_PORT_HANDLING, 2376 RTE_PORT_STOPPED) == 0) 2377 printf("Port %d can not be set back " 2378 "to stopped\n", pi); 2379 printf("Fail to configure port %d hairpin " 2380 "queues\n", pi); 2381 /* try to reconfigure queues next time */ 2382 port->need_reconfig_queues = 1; 2383 return -1; 2384 } 2385 return 0; 2386 } 2387 2388 /* Configure the Rx with optional split. */ 2389 int 2390 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2391 uint16_t nb_rx_desc, unsigned int socket_id, 2392 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2393 { 2394 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2395 unsigned int i, mp_n; 2396 int ret; 2397 2398 if (rx_pkt_nb_segs <= 1 || 2399 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2400 rx_conf->rx_seg = NULL; 2401 rx_conf->rx_nseg = 0; 2402 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2403 nb_rx_desc, socket_id, 2404 rx_conf, mp); 2405 return ret; 2406 } 2407 for (i = 0; i < rx_pkt_nb_segs; i++) { 2408 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2409 struct rte_mempool *mpx; 2410 /* 2411 * Use last valid pool for the segments with number 2412 * exceeding the pool index. 2413 */ 2414 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2415 mpx = mbuf_pool_find(socket_id, mp_n); 2416 /* Handle zero as mbuf data buffer size. */ 2417 rx_seg->length = rx_pkt_seg_lengths[i] ? 2418 rx_pkt_seg_lengths[i] : 2419 mbuf_data_size[mp_n]; 2420 rx_seg->offset = i < rx_pkt_nb_offs ? 2421 rx_pkt_seg_offsets[i] : 0; 2422 rx_seg->mp = mpx ? mpx : mp; 2423 } 2424 rx_conf->rx_nseg = rx_pkt_nb_segs; 2425 rx_conf->rx_seg = rx_useg; 2426 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2427 socket_id, rx_conf, NULL); 2428 rx_conf->rx_seg = NULL; 2429 rx_conf->rx_nseg = 0; 2430 return ret; 2431 } 2432 2433 int 2434 start_port(portid_t pid) 2435 { 2436 int diag, need_check_link_status = -1; 2437 portid_t pi; 2438 portid_t p_pi = RTE_MAX_ETHPORTS; 2439 portid_t pl[RTE_MAX_ETHPORTS]; 2440 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2441 uint16_t cnt_pi = 0; 2442 uint16_t cfg_pi = 0; 2443 int peer_pi; 2444 queueid_t qi; 2445 struct rte_port *port; 2446 struct rte_ether_addr mac_addr; 2447 struct rte_eth_hairpin_cap cap; 2448 2449 if (port_id_is_invalid(pid, ENABLED_WARN)) 2450 return 0; 2451 2452 RTE_ETH_FOREACH_DEV(pi) { 2453 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2454 continue; 2455 2456 need_check_link_status = 0; 2457 port = &ports[pi]; 2458 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2459 RTE_PORT_HANDLING) == 0) { 2460 printf("Port %d is now not stopped\n", pi); 2461 continue; 2462 } 2463 2464 if (port->need_reconfig > 0) { 2465 port->need_reconfig = 0; 2466 2467 if (flow_isolate_all) { 2468 int ret = port_flow_isolate(pi, 1); 2469 if (ret) { 2470 printf("Failed to apply isolated" 2471 " mode on port %d\n", pi); 2472 return -1; 2473 } 2474 } 2475 configure_rxtx_dump_callbacks(0); 2476 printf("Configuring Port %d (socket %u)\n", pi, 2477 port->socket_id); 2478 if (nb_hairpinq > 0 && 2479 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2480 printf("Port %d doesn't support hairpin " 2481 "queues\n", pi); 2482 return -1; 2483 } 2484 /* configure port */ 2485 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2486 nb_txq + nb_hairpinq, 2487 &(port->dev_conf)); 2488 if (diag != 0) { 2489 if (rte_atomic16_cmpset(&(port->port_status), 2490 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2491 printf("Port %d can not be set back " 2492 "to stopped\n", pi); 2493 printf("Fail to configure port %d\n", pi); 2494 /* try to reconfigure port next time */ 2495 port->need_reconfig = 1; 2496 return -1; 2497 } 2498 } 2499 if (port->need_reconfig_queues > 0) { 2500 port->need_reconfig_queues = 0; 2501 /* setup tx queues */ 2502 for (qi = 0; qi < nb_txq; qi++) { 2503 if ((numa_support) && 2504 (txring_numa[pi] != NUMA_NO_CONFIG)) 2505 diag = rte_eth_tx_queue_setup(pi, qi, 2506 port->nb_tx_desc[qi], 2507 txring_numa[pi], 2508 &(port->tx_conf[qi])); 2509 else 2510 diag = rte_eth_tx_queue_setup(pi, qi, 2511 port->nb_tx_desc[qi], 2512 port->socket_id, 2513 &(port->tx_conf[qi])); 2514 2515 if (diag == 0) 2516 continue; 2517 2518 /* Fail to setup tx queue, return */ 2519 if (rte_atomic16_cmpset(&(port->port_status), 2520 RTE_PORT_HANDLING, 2521 RTE_PORT_STOPPED) == 0) 2522 printf("Port %d can not be set back " 2523 "to stopped\n", pi); 2524 printf("Fail to configure port %d tx queues\n", 2525 pi); 2526 /* try to reconfigure queues next time */ 2527 port->need_reconfig_queues = 1; 2528 return -1; 2529 } 2530 for (qi = 0; qi < nb_rxq; qi++) { 2531 /* setup rx queues */ 2532 if ((numa_support) && 2533 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2534 struct rte_mempool * mp = 2535 mbuf_pool_find 2536 (rxring_numa[pi], 0); 2537 if (mp == NULL) { 2538 printf("Failed to setup RX queue:" 2539 "No mempool allocation" 2540 " on the socket %d\n", 2541 rxring_numa[pi]); 2542 return -1; 2543 } 2544 2545 diag = rx_queue_setup(pi, qi, 2546 port->nb_rx_desc[qi], 2547 rxring_numa[pi], 2548 &(port->rx_conf[qi]), 2549 mp); 2550 } else { 2551 struct rte_mempool *mp = 2552 mbuf_pool_find 2553 (port->socket_id, 0); 2554 if (mp == NULL) { 2555 printf("Failed to setup RX queue:" 2556 "No mempool allocation" 2557 " on the socket %d\n", 2558 port->socket_id); 2559 return -1; 2560 } 2561 diag = rx_queue_setup(pi, qi, 2562 port->nb_rx_desc[qi], 2563 port->socket_id, 2564 &(port->rx_conf[qi]), 2565 mp); 2566 } 2567 if (diag == 0) 2568 continue; 2569 2570 /* Fail to setup rx queue, return */ 2571 if (rte_atomic16_cmpset(&(port->port_status), 2572 RTE_PORT_HANDLING, 2573 RTE_PORT_STOPPED) == 0) 2574 printf("Port %d can not be set back " 2575 "to stopped\n", pi); 2576 printf("Fail to configure port %d rx queues\n", 2577 pi); 2578 /* try to reconfigure queues next time */ 2579 port->need_reconfig_queues = 1; 2580 return -1; 2581 } 2582 /* setup hairpin queues */ 2583 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2584 return -1; 2585 } 2586 configure_rxtx_dump_callbacks(verbose_level); 2587 if (clear_ptypes) { 2588 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2589 NULL, 0); 2590 if (diag < 0) 2591 printf( 2592 "Port %d: Failed to disable Ptype parsing\n", 2593 pi); 2594 } 2595 2596 p_pi = pi; 2597 cnt_pi++; 2598 2599 /* start port */ 2600 diag = rte_eth_dev_start(pi); 2601 if (diag < 0) { 2602 printf("Fail to start port %d: %s\n", pi, 2603 rte_strerror(-diag)); 2604 2605 /* Fail to setup rx queue, return */ 2606 if (rte_atomic16_cmpset(&(port->port_status), 2607 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2608 printf("Port %d can not be set back to " 2609 "stopped\n", pi); 2610 continue; 2611 } 2612 2613 if (rte_atomic16_cmpset(&(port->port_status), 2614 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2615 printf("Port %d can not be set into started\n", pi); 2616 2617 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2618 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2619 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2620 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2621 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2622 2623 /* at least one port started, need checking link status */ 2624 need_check_link_status = 1; 2625 2626 pl[cfg_pi++] = pi; 2627 } 2628 2629 if (need_check_link_status == 1 && !no_link_check) 2630 check_all_ports_link_status(RTE_PORT_ALL); 2631 else if (need_check_link_status == 0) 2632 printf("Please stop the ports first\n"); 2633 2634 if (hairpin_mode & 0xf) { 2635 uint16_t i; 2636 int j; 2637 2638 /* bind all started hairpin ports */ 2639 for (i = 0; i < cfg_pi; i++) { 2640 pi = pl[i]; 2641 /* bind current Tx to all peer Rx */ 2642 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2643 RTE_MAX_ETHPORTS, 1); 2644 if (peer_pi < 0) 2645 return peer_pi; 2646 for (j = 0; j < peer_pi; j++) { 2647 if (!port_is_started(peer_pl[j])) 2648 continue; 2649 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2650 if (diag < 0) { 2651 printf("Error during binding hairpin" 2652 " Tx port %u to %u: %s\n", 2653 pi, peer_pl[j], 2654 rte_strerror(-diag)); 2655 return -1; 2656 } 2657 } 2658 /* bind all peer Tx to current Rx */ 2659 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2660 RTE_MAX_ETHPORTS, 0); 2661 if (peer_pi < 0) 2662 return peer_pi; 2663 for (j = 0; j < peer_pi; j++) { 2664 if (!port_is_started(peer_pl[j])) 2665 continue; 2666 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2667 if (diag < 0) { 2668 printf("Error during binding hairpin" 2669 " Tx port %u to %u: %s\n", 2670 peer_pl[j], pi, 2671 rte_strerror(-diag)); 2672 return -1; 2673 } 2674 } 2675 } 2676 } 2677 2678 printf("Done\n"); 2679 return 0; 2680 } 2681 2682 void 2683 stop_port(portid_t pid) 2684 { 2685 portid_t pi; 2686 struct rte_port *port; 2687 int need_check_link_status = 0; 2688 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2689 int peer_pi; 2690 2691 if (port_id_is_invalid(pid, ENABLED_WARN)) 2692 return; 2693 2694 printf("Stopping ports...\n"); 2695 2696 RTE_ETH_FOREACH_DEV(pi) { 2697 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2698 continue; 2699 2700 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2701 printf("Please remove port %d from forwarding configuration.\n", pi); 2702 continue; 2703 } 2704 2705 if (port_is_bonding_slave(pi)) { 2706 printf("Please remove port %d from bonded device.\n", pi); 2707 continue; 2708 } 2709 2710 port = &ports[pi]; 2711 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2712 RTE_PORT_HANDLING) == 0) 2713 continue; 2714 2715 if (hairpin_mode & 0xf) { 2716 int j; 2717 2718 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2719 /* unbind all peer Tx from current Rx */ 2720 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2721 RTE_MAX_ETHPORTS, 0); 2722 if (peer_pi < 0) 2723 continue; 2724 for (j = 0; j < peer_pi; j++) { 2725 if (!port_is_started(peer_pl[j])) 2726 continue; 2727 rte_eth_hairpin_unbind(peer_pl[j], pi); 2728 } 2729 } 2730 2731 if (port->flow_list) 2732 port_flow_flush(pi); 2733 2734 if (rte_eth_dev_stop(pi) != 0) 2735 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2736 pi); 2737 2738 if (rte_atomic16_cmpset(&(port->port_status), 2739 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2740 printf("Port %d can not be set into stopped\n", pi); 2741 need_check_link_status = 1; 2742 } 2743 if (need_check_link_status && !no_link_check) 2744 check_all_ports_link_status(RTE_PORT_ALL); 2745 2746 printf("Done\n"); 2747 } 2748 2749 static void 2750 remove_invalid_ports_in(portid_t *array, portid_t *total) 2751 { 2752 portid_t i; 2753 portid_t new_total = 0; 2754 2755 for (i = 0; i < *total; i++) 2756 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2757 array[new_total] = array[i]; 2758 new_total++; 2759 } 2760 *total = new_total; 2761 } 2762 2763 static void 2764 remove_invalid_ports(void) 2765 { 2766 remove_invalid_ports_in(ports_ids, &nb_ports); 2767 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2768 nb_cfg_ports = nb_fwd_ports; 2769 } 2770 2771 void 2772 close_port(portid_t pid) 2773 { 2774 portid_t pi; 2775 struct rte_port *port; 2776 2777 if (port_id_is_invalid(pid, ENABLED_WARN)) 2778 return; 2779 2780 printf("Closing ports...\n"); 2781 2782 RTE_ETH_FOREACH_DEV(pi) { 2783 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2784 continue; 2785 2786 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2787 printf("Please remove port %d from forwarding configuration.\n", pi); 2788 continue; 2789 } 2790 2791 if (port_is_bonding_slave(pi)) { 2792 printf("Please remove port %d from bonded device.\n", pi); 2793 continue; 2794 } 2795 2796 port = &ports[pi]; 2797 if (rte_atomic16_cmpset(&(port->port_status), 2798 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2799 printf("Port %d is already closed\n", pi); 2800 continue; 2801 } 2802 2803 port_flow_flush(pi); 2804 rte_eth_dev_close(pi); 2805 } 2806 2807 remove_invalid_ports(); 2808 printf("Done\n"); 2809 } 2810 2811 void 2812 reset_port(portid_t pid) 2813 { 2814 int diag; 2815 portid_t pi; 2816 struct rte_port *port; 2817 2818 if (port_id_is_invalid(pid, ENABLED_WARN)) 2819 return; 2820 2821 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2822 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2823 printf("Can not reset port(s), please stop port(s) first.\n"); 2824 return; 2825 } 2826 2827 printf("Resetting ports...\n"); 2828 2829 RTE_ETH_FOREACH_DEV(pi) { 2830 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2831 continue; 2832 2833 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2834 printf("Please remove port %d from forwarding " 2835 "configuration.\n", pi); 2836 continue; 2837 } 2838 2839 if (port_is_bonding_slave(pi)) { 2840 printf("Please remove port %d from bonded device.\n", 2841 pi); 2842 continue; 2843 } 2844 2845 diag = rte_eth_dev_reset(pi); 2846 if (diag == 0) { 2847 port = &ports[pi]; 2848 port->need_reconfig = 1; 2849 port->need_reconfig_queues = 1; 2850 } else { 2851 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2852 } 2853 } 2854 2855 printf("Done\n"); 2856 } 2857 2858 void 2859 attach_port(char *identifier) 2860 { 2861 portid_t pi; 2862 struct rte_dev_iterator iterator; 2863 2864 printf("Attaching a new port...\n"); 2865 2866 if (identifier == NULL) { 2867 printf("Invalid parameters are specified\n"); 2868 return; 2869 } 2870 2871 if (rte_dev_probe(identifier) < 0) { 2872 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2873 return; 2874 } 2875 2876 /* first attach mode: event */ 2877 if (setup_on_probe_event) { 2878 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2879 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2880 if (ports[pi].port_status == RTE_PORT_HANDLING && 2881 ports[pi].need_setup != 0) 2882 setup_attached_port(pi); 2883 return; 2884 } 2885 2886 /* second attach mode: iterator */ 2887 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2888 /* setup ports matching the devargs used for probing */ 2889 if (port_is_forwarding(pi)) 2890 continue; /* port was already attached before */ 2891 setup_attached_port(pi); 2892 } 2893 } 2894 2895 static void 2896 setup_attached_port(portid_t pi) 2897 { 2898 unsigned int socket_id; 2899 int ret; 2900 2901 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2902 /* if socket_id is invalid, set to the first available socket. */ 2903 if (check_socket_id(socket_id) < 0) 2904 socket_id = socket_ids[0]; 2905 reconfig(pi, socket_id); 2906 ret = rte_eth_promiscuous_enable(pi); 2907 if (ret != 0) 2908 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2909 pi, rte_strerror(-ret)); 2910 2911 ports_ids[nb_ports++] = pi; 2912 fwd_ports_ids[nb_fwd_ports++] = pi; 2913 nb_cfg_ports = nb_fwd_ports; 2914 ports[pi].need_setup = 0; 2915 ports[pi].port_status = RTE_PORT_STOPPED; 2916 2917 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2918 printf("Done\n"); 2919 } 2920 2921 static void 2922 detach_device(struct rte_device *dev) 2923 { 2924 portid_t sibling; 2925 2926 if (dev == NULL) { 2927 printf("Device already removed\n"); 2928 return; 2929 } 2930 2931 printf("Removing a device...\n"); 2932 2933 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2934 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2935 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2936 printf("Port %u not stopped\n", sibling); 2937 return; 2938 } 2939 port_flow_flush(sibling); 2940 } 2941 } 2942 2943 if (rte_dev_remove(dev) < 0) { 2944 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2945 return; 2946 } 2947 remove_invalid_ports(); 2948 2949 printf("Device is detached\n"); 2950 printf("Now total ports is %d\n", nb_ports); 2951 printf("Done\n"); 2952 return; 2953 } 2954 2955 void 2956 detach_port_device(portid_t port_id) 2957 { 2958 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2959 return; 2960 2961 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2962 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2963 printf("Port not stopped\n"); 2964 return; 2965 } 2966 printf("Port was not closed\n"); 2967 } 2968 2969 detach_device(rte_eth_devices[port_id].device); 2970 } 2971 2972 void 2973 detach_devargs(char *identifier) 2974 { 2975 struct rte_dev_iterator iterator; 2976 struct rte_devargs da; 2977 portid_t port_id; 2978 2979 printf("Removing a device...\n"); 2980 2981 memset(&da, 0, sizeof(da)); 2982 if (rte_devargs_parsef(&da, "%s", identifier)) { 2983 printf("cannot parse identifier\n"); 2984 return; 2985 } 2986 2987 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2988 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2989 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2990 printf("Port %u not stopped\n", port_id); 2991 rte_eth_iterator_cleanup(&iterator); 2992 rte_devargs_reset(&da); 2993 return; 2994 } 2995 port_flow_flush(port_id); 2996 } 2997 } 2998 2999 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3000 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3001 da.name, da.bus->name); 3002 rte_devargs_reset(&da); 3003 return; 3004 } 3005 3006 remove_invalid_ports(); 3007 3008 printf("Device %s is detached\n", identifier); 3009 printf("Now total ports is %d\n", nb_ports); 3010 printf("Done\n"); 3011 rte_devargs_reset(&da); 3012 } 3013 3014 void 3015 pmd_test_exit(void) 3016 { 3017 portid_t pt_id; 3018 unsigned int i; 3019 int ret; 3020 3021 if (test_done == 0) 3022 stop_packet_forwarding(); 3023 3024 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3025 if (mempools[i]) { 3026 if (mp_alloc_type == MP_ALLOC_ANON) 3027 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3028 NULL); 3029 } 3030 } 3031 if (ports != NULL) { 3032 no_link_check = 1; 3033 RTE_ETH_FOREACH_DEV(pt_id) { 3034 printf("\nStopping port %d...\n", pt_id); 3035 fflush(stdout); 3036 stop_port(pt_id); 3037 } 3038 RTE_ETH_FOREACH_DEV(pt_id) { 3039 printf("\nShutting down port %d...\n", pt_id); 3040 fflush(stdout); 3041 close_port(pt_id); 3042 } 3043 } 3044 3045 if (hot_plug) { 3046 ret = rte_dev_event_monitor_stop(); 3047 if (ret) { 3048 RTE_LOG(ERR, EAL, 3049 "fail to stop device event monitor."); 3050 return; 3051 } 3052 3053 ret = rte_dev_event_callback_unregister(NULL, 3054 dev_event_callback, NULL); 3055 if (ret < 0) { 3056 RTE_LOG(ERR, EAL, 3057 "fail to unregister device event callback.\n"); 3058 return; 3059 } 3060 3061 ret = rte_dev_hotplug_handle_disable(); 3062 if (ret) { 3063 RTE_LOG(ERR, EAL, 3064 "fail to disable hotplug handling.\n"); 3065 return; 3066 } 3067 } 3068 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3069 if (mempools[i]) 3070 rte_mempool_free(mempools[i]); 3071 } 3072 3073 printf("\nBye...\n"); 3074 } 3075 3076 typedef void (*cmd_func_t)(void); 3077 struct pmd_test_command { 3078 const char *cmd_name; 3079 cmd_func_t cmd_func; 3080 }; 3081 3082 /* Check the link status of all ports in up to 9s, and print them finally */ 3083 static void 3084 check_all_ports_link_status(uint32_t port_mask) 3085 { 3086 #define CHECK_INTERVAL 100 /* 100ms */ 3087 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3088 portid_t portid; 3089 uint8_t count, all_ports_up, print_flag = 0; 3090 struct rte_eth_link link; 3091 int ret; 3092 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3093 3094 printf("Checking link statuses...\n"); 3095 fflush(stdout); 3096 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3097 all_ports_up = 1; 3098 RTE_ETH_FOREACH_DEV(portid) { 3099 if ((port_mask & (1 << portid)) == 0) 3100 continue; 3101 memset(&link, 0, sizeof(link)); 3102 ret = rte_eth_link_get_nowait(portid, &link); 3103 if (ret < 0) { 3104 all_ports_up = 0; 3105 if (print_flag == 1) 3106 printf("Port %u link get failed: %s\n", 3107 portid, rte_strerror(-ret)); 3108 continue; 3109 } 3110 /* print link status if flag set */ 3111 if (print_flag == 1) { 3112 rte_eth_link_to_str(link_status, 3113 sizeof(link_status), &link); 3114 printf("Port %d %s\n", portid, link_status); 3115 continue; 3116 } 3117 /* clear all_ports_up flag if any link down */ 3118 if (link.link_status == ETH_LINK_DOWN) { 3119 all_ports_up = 0; 3120 break; 3121 } 3122 } 3123 /* after finally printing all link status, get out */ 3124 if (print_flag == 1) 3125 break; 3126 3127 if (all_ports_up == 0) { 3128 fflush(stdout); 3129 rte_delay_ms(CHECK_INTERVAL); 3130 } 3131 3132 /* set the print_flag if all ports up or timeout */ 3133 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3134 print_flag = 1; 3135 } 3136 3137 if (lsc_interrupt) 3138 break; 3139 } 3140 } 3141 3142 static void 3143 rmv_port_callback(void *arg) 3144 { 3145 int need_to_start = 0; 3146 int org_no_link_check = no_link_check; 3147 portid_t port_id = (intptr_t)arg; 3148 struct rte_device *dev; 3149 3150 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3151 3152 if (!test_done && port_is_forwarding(port_id)) { 3153 need_to_start = 1; 3154 stop_packet_forwarding(); 3155 } 3156 no_link_check = 1; 3157 stop_port(port_id); 3158 no_link_check = org_no_link_check; 3159 3160 /* Save rte_device pointer before closing ethdev port */ 3161 dev = rte_eth_devices[port_id].device; 3162 close_port(port_id); 3163 detach_device(dev); /* might be already removed or have more ports */ 3164 3165 if (need_to_start) 3166 start_packet_forwarding(0); 3167 } 3168 3169 /* This function is used by the interrupt thread */ 3170 static int 3171 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3172 void *ret_param) 3173 { 3174 RTE_SET_USED(param); 3175 RTE_SET_USED(ret_param); 3176 3177 if (type >= RTE_ETH_EVENT_MAX) { 3178 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3179 port_id, __func__, type); 3180 fflush(stderr); 3181 } else if (event_print_mask & (UINT32_C(1) << type)) { 3182 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3183 eth_event_desc[type]); 3184 fflush(stdout); 3185 } 3186 3187 switch (type) { 3188 case RTE_ETH_EVENT_NEW: 3189 ports[port_id].need_setup = 1; 3190 ports[port_id].port_status = RTE_PORT_HANDLING; 3191 break; 3192 case RTE_ETH_EVENT_INTR_RMV: 3193 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3194 break; 3195 if (rte_eal_alarm_set(100000, 3196 rmv_port_callback, (void *)(intptr_t)port_id)) 3197 fprintf(stderr, "Could not set up deferred device removal\n"); 3198 break; 3199 case RTE_ETH_EVENT_DESTROY: 3200 ports[port_id].port_status = RTE_PORT_CLOSED; 3201 printf("Port %u is closed\n", port_id); 3202 break; 3203 default: 3204 break; 3205 } 3206 return 0; 3207 } 3208 3209 static int 3210 register_eth_event_callback(void) 3211 { 3212 int ret; 3213 enum rte_eth_event_type event; 3214 3215 for (event = RTE_ETH_EVENT_UNKNOWN; 3216 event < RTE_ETH_EVENT_MAX; event++) { 3217 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3218 event, 3219 eth_event_callback, 3220 NULL); 3221 if (ret != 0) { 3222 TESTPMD_LOG(ERR, "Failed to register callback for " 3223 "%s event\n", eth_event_desc[event]); 3224 return -1; 3225 } 3226 } 3227 3228 return 0; 3229 } 3230 3231 /* This function is used by the interrupt thread */ 3232 static void 3233 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3234 __rte_unused void *arg) 3235 { 3236 uint16_t port_id; 3237 int ret; 3238 3239 if (type >= RTE_DEV_EVENT_MAX) { 3240 fprintf(stderr, "%s called upon invalid event %d\n", 3241 __func__, type); 3242 fflush(stderr); 3243 } 3244 3245 switch (type) { 3246 case RTE_DEV_EVENT_REMOVE: 3247 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3248 device_name); 3249 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3250 if (ret) { 3251 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3252 device_name); 3253 return; 3254 } 3255 /* 3256 * Because the user's callback is invoked in eal interrupt 3257 * callback, the interrupt callback need to be finished before 3258 * it can be unregistered when detaching device. So finish 3259 * callback soon and use a deferred removal to detach device 3260 * is need. It is a workaround, once the device detaching be 3261 * moved into the eal in the future, the deferred removal could 3262 * be deleted. 3263 */ 3264 if (rte_eal_alarm_set(100000, 3265 rmv_port_callback, (void *)(intptr_t)port_id)) 3266 RTE_LOG(ERR, EAL, 3267 "Could not set up deferred device removal\n"); 3268 break; 3269 case RTE_DEV_EVENT_ADD: 3270 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3271 device_name); 3272 /* TODO: After finish kernel driver binding, 3273 * begin to attach port. 3274 */ 3275 break; 3276 default: 3277 break; 3278 } 3279 } 3280 3281 static void 3282 rxtx_port_config(struct rte_port *port) 3283 { 3284 uint16_t qid; 3285 uint64_t offloads; 3286 3287 for (qid = 0; qid < nb_rxq; qid++) { 3288 offloads = port->rx_conf[qid].offloads; 3289 port->rx_conf[qid] = port->dev_info.default_rxconf; 3290 if (offloads != 0) 3291 port->rx_conf[qid].offloads = offloads; 3292 3293 /* Check if any Rx parameters have been passed */ 3294 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3295 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3296 3297 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3298 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3299 3300 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3301 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3302 3303 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3304 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3305 3306 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3307 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3308 3309 port->nb_rx_desc[qid] = nb_rxd; 3310 } 3311 3312 for (qid = 0; qid < nb_txq; qid++) { 3313 offloads = port->tx_conf[qid].offloads; 3314 port->tx_conf[qid] = port->dev_info.default_txconf; 3315 if (offloads != 0) 3316 port->tx_conf[qid].offloads = offloads; 3317 3318 /* Check if any Tx parameters have been passed */ 3319 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3320 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3321 3322 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3323 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3324 3325 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3326 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3327 3328 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3329 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3330 3331 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3332 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3333 3334 port->nb_tx_desc[qid] = nb_txd; 3335 } 3336 } 3337 3338 /* 3339 * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload, 3340 * MTU is also aligned if JUMBO_FRAME offload is not set. 3341 * 3342 * port->dev_info should be set before calling this function. 3343 * 3344 * return 0 on success, negative on error 3345 */ 3346 int 3347 update_jumbo_frame_offload(portid_t portid) 3348 { 3349 struct rte_port *port = &ports[portid]; 3350 uint32_t eth_overhead; 3351 uint64_t rx_offloads; 3352 int ret; 3353 bool on; 3354 3355 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */ 3356 if (port->dev_info.max_mtu != UINT16_MAX && 3357 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu) 3358 eth_overhead = port->dev_info.max_rx_pktlen - 3359 port->dev_info.max_mtu; 3360 else 3361 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 3362 3363 rx_offloads = port->dev_conf.rxmode.offloads; 3364 3365 /* Default config value is 0 to use PMD specific overhead */ 3366 if (port->dev_conf.rxmode.max_rx_pkt_len == 0) 3367 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead; 3368 3369 if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) { 3370 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3371 on = false; 3372 } else { 3373 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3374 printf("Frame size (%u) is not supported by port %u\n", 3375 port->dev_conf.rxmode.max_rx_pkt_len, 3376 portid); 3377 return -1; 3378 } 3379 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3380 on = true; 3381 } 3382 3383 if (rx_offloads != port->dev_conf.rxmode.offloads) { 3384 uint16_t qid; 3385 3386 port->dev_conf.rxmode.offloads = rx_offloads; 3387 3388 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */ 3389 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) { 3390 if (on) 3391 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3392 else 3393 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3394 } 3395 } 3396 3397 /* If JUMBO_FRAME is set MTU conversion done by ethdev layer, 3398 * if unset do it here 3399 */ 3400 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3401 ret = rte_eth_dev_set_mtu(portid, 3402 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead); 3403 if (ret) 3404 printf("Failed to set MTU to %u for port %u\n", 3405 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead, 3406 portid); 3407 } 3408 3409 return 0; 3410 } 3411 3412 void 3413 init_port_config(void) 3414 { 3415 portid_t pid; 3416 struct rte_port *port; 3417 int ret; 3418 3419 RTE_ETH_FOREACH_DEV(pid) { 3420 port = &ports[pid]; 3421 port->dev_conf.fdir_conf = fdir_conf; 3422 3423 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3424 if (ret != 0) 3425 return; 3426 3427 if (nb_rxq > 1) { 3428 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3429 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3430 rss_hf & port->dev_info.flow_type_rss_offloads; 3431 } else { 3432 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3433 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3434 } 3435 3436 if (port->dcb_flag == 0) { 3437 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3438 port->dev_conf.rxmode.mq_mode = 3439 (enum rte_eth_rx_mq_mode) 3440 (rx_mq_mode & ETH_MQ_RX_RSS); 3441 else 3442 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3443 } 3444 3445 rxtx_port_config(port); 3446 3447 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3448 if (ret != 0) 3449 return; 3450 3451 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3452 rte_pmd_ixgbe_bypass_init(pid); 3453 #endif 3454 3455 if (lsc_interrupt && 3456 (rte_eth_devices[pid].data->dev_flags & 3457 RTE_ETH_DEV_INTR_LSC)) 3458 port->dev_conf.intr_conf.lsc = 1; 3459 if (rmv_interrupt && 3460 (rte_eth_devices[pid].data->dev_flags & 3461 RTE_ETH_DEV_INTR_RMV)) 3462 port->dev_conf.intr_conf.rmv = 1; 3463 } 3464 } 3465 3466 void set_port_slave_flag(portid_t slave_pid) 3467 { 3468 struct rte_port *port; 3469 3470 port = &ports[slave_pid]; 3471 port->slave_flag = 1; 3472 } 3473 3474 void clear_port_slave_flag(portid_t slave_pid) 3475 { 3476 struct rte_port *port; 3477 3478 port = &ports[slave_pid]; 3479 port->slave_flag = 0; 3480 } 3481 3482 uint8_t port_is_bonding_slave(portid_t slave_pid) 3483 { 3484 struct rte_port *port; 3485 3486 port = &ports[slave_pid]; 3487 if ((rte_eth_devices[slave_pid].data->dev_flags & 3488 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3489 return 1; 3490 return 0; 3491 } 3492 3493 const uint16_t vlan_tags[] = { 3494 0, 1, 2, 3, 4, 5, 6, 7, 3495 8, 9, 10, 11, 12, 13, 14, 15, 3496 16, 17, 18, 19, 20, 21, 22, 23, 3497 24, 25, 26, 27, 28, 29, 30, 31 3498 }; 3499 3500 static int 3501 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3502 enum dcb_mode_enable dcb_mode, 3503 enum rte_eth_nb_tcs num_tcs, 3504 uint8_t pfc_en) 3505 { 3506 uint8_t i; 3507 int32_t rc; 3508 struct rte_eth_rss_conf rss_conf; 3509 3510 /* 3511 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3512 * given above, and the number of traffic classes available for use. 3513 */ 3514 if (dcb_mode == DCB_VT_ENABLED) { 3515 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3516 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3517 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3518 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3519 3520 /* VMDQ+DCB RX and TX configurations */ 3521 vmdq_rx_conf->enable_default_pool = 0; 3522 vmdq_rx_conf->default_pool = 0; 3523 vmdq_rx_conf->nb_queue_pools = 3524 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3525 vmdq_tx_conf->nb_queue_pools = 3526 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3527 3528 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3529 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3530 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3531 vmdq_rx_conf->pool_map[i].pools = 3532 1 << (i % vmdq_rx_conf->nb_queue_pools); 3533 } 3534 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3535 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3536 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3537 } 3538 3539 /* set DCB mode of RX and TX of multiple queues */ 3540 eth_conf->rxmode.mq_mode = 3541 (enum rte_eth_rx_mq_mode) 3542 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3543 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3544 } else { 3545 struct rte_eth_dcb_rx_conf *rx_conf = 3546 ð_conf->rx_adv_conf.dcb_rx_conf; 3547 struct rte_eth_dcb_tx_conf *tx_conf = 3548 ð_conf->tx_adv_conf.dcb_tx_conf; 3549 3550 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3551 3552 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3553 if (rc != 0) 3554 return rc; 3555 3556 rx_conf->nb_tcs = num_tcs; 3557 tx_conf->nb_tcs = num_tcs; 3558 3559 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3560 rx_conf->dcb_tc[i] = i % num_tcs; 3561 tx_conf->dcb_tc[i] = i % num_tcs; 3562 } 3563 3564 eth_conf->rxmode.mq_mode = 3565 (enum rte_eth_rx_mq_mode) 3566 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3567 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3568 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3569 } 3570 3571 if (pfc_en) 3572 eth_conf->dcb_capability_en = 3573 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3574 else 3575 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3576 3577 return 0; 3578 } 3579 3580 int 3581 init_port_dcb_config(portid_t pid, 3582 enum dcb_mode_enable dcb_mode, 3583 enum rte_eth_nb_tcs num_tcs, 3584 uint8_t pfc_en) 3585 { 3586 struct rte_eth_conf port_conf; 3587 struct rte_port *rte_port; 3588 int retval; 3589 uint16_t i; 3590 3591 rte_port = &ports[pid]; 3592 3593 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3594 3595 port_conf.rxmode = rte_port->dev_conf.rxmode; 3596 port_conf.txmode = rte_port->dev_conf.txmode; 3597 3598 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3599 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3600 if (retval < 0) 3601 return retval; 3602 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3603 3604 /* re-configure the device . */ 3605 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3606 if (retval < 0) 3607 return retval; 3608 3609 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3610 if (retval != 0) 3611 return retval; 3612 3613 /* If dev_info.vmdq_pool_base is greater than 0, 3614 * the queue id of vmdq pools is started after pf queues. 3615 */ 3616 if (dcb_mode == DCB_VT_ENABLED && 3617 rte_port->dev_info.vmdq_pool_base > 0) { 3618 printf("VMDQ_DCB multi-queue mode is nonsensical" 3619 " for port %d.", pid); 3620 return -1; 3621 } 3622 3623 /* Assume the ports in testpmd have the same dcb capability 3624 * and has the same number of rxq and txq in dcb mode 3625 */ 3626 if (dcb_mode == DCB_VT_ENABLED) { 3627 if (rte_port->dev_info.max_vfs > 0) { 3628 nb_rxq = rte_port->dev_info.nb_rx_queues; 3629 nb_txq = rte_port->dev_info.nb_tx_queues; 3630 } else { 3631 nb_rxq = rte_port->dev_info.max_rx_queues; 3632 nb_txq = rte_port->dev_info.max_tx_queues; 3633 } 3634 } else { 3635 /*if vt is disabled, use all pf queues */ 3636 if (rte_port->dev_info.vmdq_pool_base == 0) { 3637 nb_rxq = rte_port->dev_info.max_rx_queues; 3638 nb_txq = rte_port->dev_info.max_tx_queues; 3639 } else { 3640 nb_rxq = (queueid_t)num_tcs; 3641 nb_txq = (queueid_t)num_tcs; 3642 3643 } 3644 } 3645 rx_free_thresh = 64; 3646 3647 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3648 3649 rxtx_port_config(rte_port); 3650 /* VLAN filter */ 3651 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3652 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3653 rx_vft_set(pid, vlan_tags[i], 1); 3654 3655 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3656 if (retval != 0) 3657 return retval; 3658 3659 rte_port->dcb_flag = 1; 3660 3661 /* Enter DCB configuration status */ 3662 dcb_config = 1; 3663 3664 return 0; 3665 } 3666 3667 static void 3668 init_port(void) 3669 { 3670 int i; 3671 3672 /* Configuration of Ethernet ports. */ 3673 ports = rte_zmalloc("testpmd: ports", 3674 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3675 RTE_CACHE_LINE_SIZE); 3676 if (ports == NULL) { 3677 rte_exit(EXIT_FAILURE, 3678 "rte_zmalloc(%d struct rte_port) failed\n", 3679 RTE_MAX_ETHPORTS); 3680 } 3681 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3682 LIST_INIT(&ports[i].flow_tunnel_list); 3683 /* Initialize ports NUMA structures */ 3684 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3685 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3686 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3687 } 3688 3689 static void 3690 force_quit(void) 3691 { 3692 pmd_test_exit(); 3693 prompt_exit(); 3694 } 3695 3696 static void 3697 print_stats(void) 3698 { 3699 uint8_t i; 3700 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3701 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3702 3703 /* Clear screen and move to top left */ 3704 printf("%s%s", clr, top_left); 3705 3706 printf("\nPort statistics ===================================="); 3707 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3708 nic_stats_display(fwd_ports_ids[i]); 3709 3710 fflush(stdout); 3711 } 3712 3713 static void 3714 signal_handler(int signum) 3715 { 3716 if (signum == SIGINT || signum == SIGTERM) { 3717 printf("\nSignal %d received, preparing to exit...\n", 3718 signum); 3719 #ifdef RTE_LIB_PDUMP 3720 /* uninitialize packet capture framework */ 3721 rte_pdump_uninit(); 3722 #endif 3723 #ifdef RTE_LIB_LATENCYSTATS 3724 if (latencystats_enabled != 0) 3725 rte_latencystats_uninit(); 3726 #endif 3727 force_quit(); 3728 /* Set flag to indicate the force termination. */ 3729 f_quit = 1; 3730 /* exit with the expected status */ 3731 signal(signum, SIG_DFL); 3732 kill(getpid(), signum); 3733 } 3734 } 3735 3736 int 3737 main(int argc, char** argv) 3738 { 3739 int diag; 3740 portid_t port_id; 3741 uint16_t count; 3742 int ret; 3743 3744 signal(SIGINT, signal_handler); 3745 signal(SIGTERM, signal_handler); 3746 3747 testpmd_logtype = rte_log_register("testpmd"); 3748 if (testpmd_logtype < 0) 3749 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3750 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3751 3752 diag = rte_eal_init(argc, argv); 3753 if (diag < 0) 3754 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3755 rte_strerror(rte_errno)); 3756 3757 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3758 rte_exit(EXIT_FAILURE, 3759 "Secondary process type not supported.\n"); 3760 3761 ret = register_eth_event_callback(); 3762 if (ret != 0) 3763 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3764 3765 #ifdef RTE_LIB_PDUMP 3766 /* initialize packet capture framework */ 3767 rte_pdump_init(); 3768 #endif 3769 3770 count = 0; 3771 RTE_ETH_FOREACH_DEV(port_id) { 3772 ports_ids[count] = port_id; 3773 count++; 3774 } 3775 nb_ports = (portid_t) count; 3776 if (nb_ports == 0) 3777 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3778 3779 /* allocate port structures, and init them */ 3780 init_port(); 3781 3782 set_def_fwd_config(); 3783 if (nb_lcores == 0) 3784 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3785 "Check the core mask argument\n"); 3786 3787 /* Bitrate/latency stats disabled by default */ 3788 #ifdef RTE_LIB_BITRATESTATS 3789 bitrate_enabled = 0; 3790 #endif 3791 #ifdef RTE_LIB_LATENCYSTATS 3792 latencystats_enabled = 0; 3793 #endif 3794 3795 /* on FreeBSD, mlockall() is disabled by default */ 3796 #ifdef RTE_EXEC_ENV_FREEBSD 3797 do_mlockall = 0; 3798 #else 3799 do_mlockall = 1; 3800 #endif 3801 3802 argc -= diag; 3803 argv += diag; 3804 if (argc > 1) 3805 launch_args_parse(argc, argv); 3806 3807 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3808 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3809 strerror(errno)); 3810 } 3811 3812 if (tx_first && interactive) 3813 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3814 "interactive mode.\n"); 3815 3816 if (tx_first && lsc_interrupt) { 3817 printf("Warning: lsc_interrupt needs to be off when " 3818 " using tx_first. Disabling.\n"); 3819 lsc_interrupt = 0; 3820 } 3821 3822 if (!nb_rxq && !nb_txq) 3823 printf("Warning: Either rx or tx queues should be non-zero\n"); 3824 3825 if (nb_rxq > 1 && nb_rxq > nb_txq) 3826 printf("Warning: nb_rxq=%d enables RSS configuration, " 3827 "but nb_txq=%d will prevent to fully test it.\n", 3828 nb_rxq, nb_txq); 3829 3830 init_config(); 3831 3832 if (hot_plug) { 3833 ret = rte_dev_hotplug_handle_enable(); 3834 if (ret) { 3835 RTE_LOG(ERR, EAL, 3836 "fail to enable hotplug handling."); 3837 return -1; 3838 } 3839 3840 ret = rte_dev_event_monitor_start(); 3841 if (ret) { 3842 RTE_LOG(ERR, EAL, 3843 "fail to start device event monitoring."); 3844 return -1; 3845 } 3846 3847 ret = rte_dev_event_callback_register(NULL, 3848 dev_event_callback, NULL); 3849 if (ret) { 3850 RTE_LOG(ERR, EAL, 3851 "fail to register device event callback\n"); 3852 return -1; 3853 } 3854 } 3855 3856 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3857 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3858 3859 /* set all ports to promiscuous mode by default */ 3860 RTE_ETH_FOREACH_DEV(port_id) { 3861 ret = rte_eth_promiscuous_enable(port_id); 3862 if (ret != 0) 3863 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3864 port_id, rte_strerror(-ret)); 3865 } 3866 3867 /* Init metrics library */ 3868 rte_metrics_init(rte_socket_id()); 3869 3870 #ifdef RTE_LIB_LATENCYSTATS 3871 if (latencystats_enabled != 0) { 3872 int ret = rte_latencystats_init(1, NULL); 3873 if (ret) 3874 printf("Warning: latencystats init()" 3875 " returned error %d\n", ret); 3876 printf("Latencystats running on lcore %d\n", 3877 latencystats_lcore_id); 3878 } 3879 #endif 3880 3881 /* Setup bitrate stats */ 3882 #ifdef RTE_LIB_BITRATESTATS 3883 if (bitrate_enabled != 0) { 3884 bitrate_data = rte_stats_bitrate_create(); 3885 if (bitrate_data == NULL) 3886 rte_exit(EXIT_FAILURE, 3887 "Could not allocate bitrate data.\n"); 3888 rte_stats_bitrate_reg(bitrate_data); 3889 } 3890 #endif 3891 3892 #ifdef RTE_LIB_CMDLINE 3893 if (strlen(cmdline_filename) != 0) 3894 cmdline_read_from_file(cmdline_filename); 3895 3896 if (interactive == 1) { 3897 if (auto_start) { 3898 printf("Start automatic packet forwarding\n"); 3899 start_packet_forwarding(0); 3900 } 3901 prompt(); 3902 pmd_test_exit(); 3903 } else 3904 #endif 3905 { 3906 char c; 3907 int rc; 3908 3909 f_quit = 0; 3910 3911 printf("No commandline core given, start packet forwarding\n"); 3912 start_packet_forwarding(tx_first); 3913 if (stats_period != 0) { 3914 uint64_t prev_time = 0, cur_time, diff_time = 0; 3915 uint64_t timer_period; 3916 3917 /* Convert to number of cycles */ 3918 timer_period = stats_period * rte_get_timer_hz(); 3919 3920 while (f_quit == 0) { 3921 cur_time = rte_get_timer_cycles(); 3922 diff_time += cur_time - prev_time; 3923 3924 if (diff_time >= timer_period) { 3925 print_stats(); 3926 /* Reset the timer */ 3927 diff_time = 0; 3928 } 3929 /* Sleep to avoid unnecessary checks */ 3930 prev_time = cur_time; 3931 sleep(1); 3932 } 3933 } 3934 3935 printf("Press enter to exit\n"); 3936 rc = read(0, &c, 1); 3937 pmd_test_exit(); 3938 if (rc < 0) 3939 return 1; 3940 } 3941 3942 ret = rte_eal_cleanup(); 3943 if (ret != 0) 3944 rte_exit(EXIT_FAILURE, 3945 "EAL cleanup failed: %s\n", strerror(-ret)); 3946 3947 return EXIT_SUCCESS; 3948 } 3949