1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_atomic.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_pci.h> 47 #include <rte_ether.h> 48 #include <rte_ethdev.h> 49 #include <rte_dev.h> 50 #include <rte_string_fns.h> 51 #ifdef RTE_NET_IXGBE 52 #include <rte_pmd_ixgbe.h> 53 #endif 54 #ifdef RTE_LIB_PDUMP 55 #include <rte_pdump.h> 56 #endif 57 #include <rte_flow.h> 58 #include <rte_metrics.h> 59 #ifdef RTE_LIB_BITRATESTATS 60 #include <rte_bitrate.h> 61 #endif 62 #ifdef RTE_LIB_LATENCYSTATS 63 #include <rte_latencystats.h> 64 #endif 65 #ifdef RTE_EXEC_ENV_WINDOWS 66 #include <process.h> 67 #endif 68 69 #include "testpmd.h" 70 71 #ifndef MAP_HUGETLB 72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 73 #define HUGE_FLAG (0x40000) 74 #else 75 #define HUGE_FLAG MAP_HUGETLB 76 #endif 77 78 #ifndef MAP_HUGE_SHIFT 79 /* older kernels (or FreeBSD) will not have this define */ 80 #define HUGE_SHIFT (26) 81 #else 82 #define HUGE_SHIFT MAP_HUGE_SHIFT 83 #endif 84 85 #define EXTMEM_HEAP_NAME "extmem" 86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 87 88 uint16_t verbose_level = 0; /**< Silent by default. */ 89 int testpmd_logtype; /**< Log type for testpmd logs */ 90 91 /* use main core for command line ? */ 92 uint8_t interactive = 0; 93 uint8_t auto_start = 0; 94 uint8_t tx_first; 95 char cmdline_filename[PATH_MAX] = {0}; 96 97 /* 98 * NUMA support configuration. 99 * When set, the NUMA support attempts to dispatch the allocation of the 100 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 101 * probed ports among the CPU sockets 0 and 1. 102 * Otherwise, all memory is allocated from CPU socket 0. 103 */ 104 uint8_t numa_support = 1; /**< numa enabled by default */ 105 106 /* 107 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 108 * not configured. 109 */ 110 uint8_t socket_num = UMA_NO_CONFIG; 111 112 /* 113 * Select mempool allocation type: 114 * - native: use regular DPDK memory 115 * - anon: use regular DPDK memory to create mempool, but populate using 116 * anonymous memory (may not be IOVA-contiguous) 117 * - xmem: use externally allocated hugepage memory 118 */ 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 120 121 /* 122 * Store specified sockets on which memory pool to be used by ports 123 * is allocated. 124 */ 125 uint8_t port_numa[RTE_MAX_ETHPORTS]; 126 127 /* 128 * Store specified sockets on which RX ring to be used by ports 129 * is allocated. 130 */ 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 132 133 /* 134 * Store specified sockets on which TX ring to be used by ports 135 * is allocated. 136 */ 137 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 138 139 /* 140 * Record the Ethernet address of peer target ports to which packets are 141 * forwarded. 142 * Must be instantiated with the ethernet addresses of peer traffic generator 143 * ports. 144 */ 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 146 portid_t nb_peer_eth_addrs = 0; 147 148 /* 149 * Probed Target Environment. 150 */ 151 struct rte_port *ports; /**< For all probed ethernet ports. */ 152 portid_t nb_ports; /**< Number of probed ethernet ports. */ 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 154 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 155 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 157 158 /* 159 * Test Forwarding Configuration. 160 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 161 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 162 */ 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 165 portid_t nb_cfg_ports; /**< Number of configured ports. */ 166 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 167 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 170 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 172 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 173 174 /* 175 * Forwarding engines. 176 */ 177 struct fwd_engine * fwd_engines[] = { 178 &io_fwd_engine, 179 &mac_fwd_engine, 180 &mac_swap_engine, 181 &flow_gen_engine, 182 &rx_only_engine, 183 &tx_only_engine, 184 &csum_fwd_engine, 185 &icmp_echo_engine, 186 &noisy_vnf_engine, 187 &five_tuple_swap_fwd_engine, 188 #ifdef RTE_LIBRTE_IEEE1588 189 &ieee1588_fwd_engine, 190 #endif 191 NULL, 192 }; 193 194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 195 uint16_t mempool_flags; 196 197 struct fwd_config cur_fwd_config; 198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 199 uint32_t retry_enabled; 200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 202 203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 205 DEFAULT_MBUF_DATA_SIZE 206 }; /**< Mbuf data space size. */ 207 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 208 * specified on command-line. */ 209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 210 211 /* 212 * In container, it cannot terminate the process which running with 'stats-period' 213 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 214 */ 215 uint8_t f_quit; 216 217 /* 218 * Configuration of packet segments used to scatter received packets 219 * if some of split features is configured. 220 */ 221 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 222 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 223 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 224 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 225 226 /* 227 * Configuration of packet segments used by the "txonly" processing engine. 228 */ 229 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 230 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 231 TXONLY_DEF_PACKET_LEN, 232 }; 233 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 234 235 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 236 /**< Split policy for packets to TX. */ 237 238 uint8_t txonly_multi_flow; 239 /**< Whether multiple flows are generated in TXONLY mode. */ 240 241 uint32_t tx_pkt_times_inter; 242 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 243 244 uint32_t tx_pkt_times_intra; 245 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 246 247 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 248 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 249 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 250 251 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 252 uint8_t dcb_config = 0; 253 254 /* 255 * Configurable number of RX/TX queues. 256 */ 257 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 258 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 259 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 260 261 /* 262 * Configurable number of RX/TX ring descriptors. 263 * Defaults are supplied by drivers via ethdev. 264 */ 265 #define RTE_TEST_RX_DESC_DEFAULT 0 266 #define RTE_TEST_TX_DESC_DEFAULT 0 267 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 268 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 269 270 #define RTE_PMD_PARAM_UNSET -1 271 /* 272 * Configurable values of RX and TX ring threshold registers. 273 */ 274 275 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 276 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 277 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 278 279 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 280 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 281 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of RX free threshold. 285 */ 286 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 287 288 /* 289 * Configurable value of RX drop enable. 290 */ 291 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 292 293 /* 294 * Configurable value of TX free threshold. 295 */ 296 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 297 298 /* 299 * Configurable value of TX RS bit threshold. 300 */ 301 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 302 303 /* 304 * Configurable value of buffered packets before sending. 305 */ 306 uint16_t noisy_tx_sw_bufsz; 307 308 /* 309 * Configurable value of packet buffer timeout. 310 */ 311 uint16_t noisy_tx_sw_buf_flush_time; 312 313 /* 314 * Configurable value for size of VNF internal memory area 315 * used for simulating noisy neighbour behaviour 316 */ 317 uint64_t noisy_lkup_mem_sz; 318 319 /* 320 * Configurable value of number of random writes done in 321 * VNF simulation memory area. 322 */ 323 uint64_t noisy_lkup_num_writes; 324 325 /* 326 * Configurable value of number of random reads done in 327 * VNF simulation memory area. 328 */ 329 uint64_t noisy_lkup_num_reads; 330 331 /* 332 * Configurable value of number of random reads/writes done in 333 * VNF simulation memory area. 334 */ 335 uint64_t noisy_lkup_num_reads_writes; 336 337 /* 338 * Receive Side Scaling (RSS) configuration. 339 */ 340 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 341 342 /* 343 * Port topology configuration 344 */ 345 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 346 347 /* 348 * Avoids to flush all the RX streams before starts forwarding. 349 */ 350 uint8_t no_flush_rx = 0; /* flush by default */ 351 352 /* 353 * Flow API isolated mode. 354 */ 355 uint8_t flow_isolate_all; 356 357 /* 358 * Avoids to check link status when starting/stopping a port. 359 */ 360 uint8_t no_link_check = 0; /* check by default */ 361 362 /* 363 * Don't automatically start all ports in interactive mode. 364 */ 365 uint8_t no_device_start = 0; 366 367 /* 368 * Enable link status change notification 369 */ 370 uint8_t lsc_interrupt = 1; /* enabled by default */ 371 372 /* 373 * Enable device removal notification. 374 */ 375 uint8_t rmv_interrupt = 1; /* enabled by default */ 376 377 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 378 379 /* After attach, port setup is called on event or by iterator */ 380 bool setup_on_probe_event = true; 381 382 /* Clear ptypes on port initialization. */ 383 uint8_t clear_ptypes = true; 384 385 /* Hairpin ports configuration mode. */ 386 uint16_t hairpin_mode; 387 388 /* Pretty printing of ethdev events */ 389 static const char * const eth_event_desc[] = { 390 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 391 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 392 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 393 [RTE_ETH_EVENT_INTR_RESET] = "reset", 394 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 395 [RTE_ETH_EVENT_IPSEC] = "IPsec", 396 [RTE_ETH_EVENT_MACSEC] = "MACsec", 397 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 398 [RTE_ETH_EVENT_NEW] = "device probed", 399 [RTE_ETH_EVENT_DESTROY] = "device released", 400 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 401 [RTE_ETH_EVENT_MAX] = NULL, 402 }; 403 404 /* 405 * Display or mask ether events 406 * Default to all events except VF_MBOX 407 */ 408 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 410 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 411 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 412 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 413 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 414 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 415 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 416 /* 417 * Decide if all memory are locked for performance. 418 */ 419 int do_mlockall = 0; 420 421 /* 422 * NIC bypass mode configuration options. 423 */ 424 425 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 426 /* The NIC bypass watchdog timeout. */ 427 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 428 #endif 429 430 431 #ifdef RTE_LIB_LATENCYSTATS 432 433 /* 434 * Set when latency stats is enabled in the commandline 435 */ 436 uint8_t latencystats_enabled; 437 438 /* 439 * Lcore ID to serive latency statistics. 440 */ 441 lcoreid_t latencystats_lcore_id = -1; 442 443 #endif 444 445 /* 446 * Ethernet device configuration. 447 */ 448 struct rte_eth_rxmode rx_mode = { 449 /* Default maximum frame length. 450 * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead" 451 * in init_config(). 452 */ 453 .max_rx_pkt_len = 0, 454 }; 455 456 struct rte_eth_txmode tx_mode = { 457 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 458 }; 459 460 struct rte_fdir_conf fdir_conf = { 461 .mode = RTE_FDIR_MODE_NONE, 462 .pballoc = RTE_FDIR_PBALLOC_64K, 463 .status = RTE_FDIR_REPORT_STATUS, 464 .mask = { 465 .vlan_tci_mask = 0xFFEF, 466 .ipv4_mask = { 467 .src_ip = 0xFFFFFFFF, 468 .dst_ip = 0xFFFFFFFF, 469 }, 470 .ipv6_mask = { 471 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 472 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 473 }, 474 .src_port_mask = 0xFFFF, 475 .dst_port_mask = 0xFFFF, 476 .mac_addr_byte_mask = 0xFF, 477 .tunnel_type_mask = 1, 478 .tunnel_id_mask = 0xFFFFFFFF, 479 }, 480 .drop_queue = 127, 481 }; 482 483 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 484 485 /* 486 * Display zero values by default for xstats 487 */ 488 uint8_t xstats_hide_zero; 489 490 /* 491 * Measure of CPU cycles disabled by default 492 */ 493 uint8_t record_core_cycles; 494 495 /* 496 * Display of RX and TX bursts disabled by default 497 */ 498 uint8_t record_burst_stats; 499 500 unsigned int num_sockets = 0; 501 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 502 503 #ifdef RTE_LIB_BITRATESTATS 504 /* Bitrate statistics */ 505 struct rte_stats_bitrates *bitrate_data; 506 lcoreid_t bitrate_lcore_id; 507 uint8_t bitrate_enabled; 508 #endif 509 510 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 511 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 512 513 /* 514 * hexadecimal bitmask of RX mq mode can be enabled. 515 */ 516 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 517 518 /* 519 * Used to set forced link speed 520 */ 521 uint32_t eth_link_speed; 522 523 /* Forward function declarations */ 524 static void setup_attached_port(portid_t pi); 525 static void check_all_ports_link_status(uint32_t port_mask); 526 static int eth_event_callback(portid_t port_id, 527 enum rte_eth_event_type type, 528 void *param, void *ret_param); 529 static void dev_event_callback(const char *device_name, 530 enum rte_dev_event_type type, 531 void *param); 532 533 /* 534 * Check if all the ports are started. 535 * If yes, return positive value. If not, return zero. 536 */ 537 static int all_ports_started(void); 538 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 541 542 /* Holds the registered mbuf dynamic flags names. */ 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 544 545 /* 546 * Helper function to check if socket is already discovered. 547 * If yes, return positive value. If not, return zero. 548 */ 549 int 550 new_socket_id(unsigned int socket_id) 551 { 552 unsigned int i; 553 554 for (i = 0; i < num_sockets; i++) { 555 if (socket_ids[i] == socket_id) 556 return 0; 557 } 558 return 1; 559 } 560 561 /* 562 * Setup default configuration. 563 */ 564 static void 565 set_default_fwd_lcores_config(void) 566 { 567 unsigned int i; 568 unsigned int nb_lc; 569 unsigned int sock_num; 570 571 nb_lc = 0; 572 for (i = 0; i < RTE_MAX_LCORE; i++) { 573 if (!rte_lcore_is_enabled(i)) 574 continue; 575 sock_num = rte_lcore_to_socket_id(i); 576 if (new_socket_id(sock_num)) { 577 if (num_sockets >= RTE_MAX_NUMA_NODES) { 578 rte_exit(EXIT_FAILURE, 579 "Total sockets greater than %u\n", 580 RTE_MAX_NUMA_NODES); 581 } 582 socket_ids[num_sockets++] = sock_num; 583 } 584 if (i == rte_get_main_lcore()) 585 continue; 586 fwd_lcores_cpuids[nb_lc++] = i; 587 } 588 nb_lcores = (lcoreid_t) nb_lc; 589 nb_cfg_lcores = nb_lcores; 590 nb_fwd_lcores = 1; 591 } 592 593 static void 594 set_def_peer_eth_addrs(void) 595 { 596 portid_t i; 597 598 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 599 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 600 peer_eth_addrs[i].addr_bytes[5] = i; 601 } 602 } 603 604 static void 605 set_default_fwd_ports_config(void) 606 { 607 portid_t pt_id; 608 int i = 0; 609 610 RTE_ETH_FOREACH_DEV(pt_id) { 611 fwd_ports_ids[i++] = pt_id; 612 613 /* Update sockets info according to the attached device */ 614 int socket_id = rte_eth_dev_socket_id(pt_id); 615 if (socket_id >= 0 && new_socket_id(socket_id)) { 616 if (num_sockets >= RTE_MAX_NUMA_NODES) { 617 rte_exit(EXIT_FAILURE, 618 "Total sockets greater than %u\n", 619 RTE_MAX_NUMA_NODES); 620 } 621 socket_ids[num_sockets++] = socket_id; 622 } 623 } 624 625 nb_cfg_ports = nb_ports; 626 nb_fwd_ports = nb_ports; 627 } 628 629 void 630 set_def_fwd_config(void) 631 { 632 set_default_fwd_lcores_config(); 633 set_def_peer_eth_addrs(); 634 set_default_fwd_ports_config(); 635 } 636 637 #ifndef RTE_EXEC_ENV_WINDOWS 638 /* extremely pessimistic estimation of memory required to create a mempool */ 639 static int 640 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 641 { 642 unsigned int n_pages, mbuf_per_pg, leftover; 643 uint64_t total_mem, mbuf_mem, obj_sz; 644 645 /* there is no good way to predict how much space the mempool will 646 * occupy because it will allocate chunks on the fly, and some of those 647 * will come from default DPDK memory while some will come from our 648 * external memory, so just assume 128MB will be enough for everyone. 649 */ 650 uint64_t hdr_mem = 128 << 20; 651 652 /* account for possible non-contiguousness */ 653 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 654 if (obj_sz > pgsz) { 655 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 656 return -1; 657 } 658 659 mbuf_per_pg = pgsz / obj_sz; 660 leftover = (nb_mbufs % mbuf_per_pg) > 0; 661 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 662 663 mbuf_mem = n_pages * pgsz; 664 665 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 666 667 if (total_mem > SIZE_MAX) { 668 TESTPMD_LOG(ERR, "Memory size too big\n"); 669 return -1; 670 } 671 *out = (size_t)total_mem; 672 673 return 0; 674 } 675 676 static int 677 pagesz_flags(uint64_t page_sz) 678 { 679 /* as per mmap() manpage, all page sizes are log2 of page size 680 * shifted by MAP_HUGE_SHIFT 681 */ 682 int log2 = rte_log2_u64(page_sz); 683 684 return (log2 << HUGE_SHIFT); 685 } 686 687 static void * 688 alloc_mem(size_t memsz, size_t pgsz, bool huge) 689 { 690 void *addr; 691 int flags; 692 693 /* allocate anonymous hugepages */ 694 flags = MAP_ANONYMOUS | MAP_PRIVATE; 695 if (huge) 696 flags |= HUGE_FLAG | pagesz_flags(pgsz); 697 698 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 699 if (addr == MAP_FAILED) 700 return NULL; 701 702 return addr; 703 } 704 705 struct extmem_param { 706 void *addr; 707 size_t len; 708 size_t pgsz; 709 rte_iova_t *iova_table; 710 unsigned int iova_table_len; 711 }; 712 713 static int 714 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 715 bool huge) 716 { 717 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 718 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 719 unsigned int cur_page, n_pages, pgsz_idx; 720 size_t mem_sz, cur_pgsz; 721 rte_iova_t *iovas = NULL; 722 void *addr; 723 int ret; 724 725 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 726 /* skip anything that is too big */ 727 if (pgsizes[pgsz_idx] > SIZE_MAX) 728 continue; 729 730 cur_pgsz = pgsizes[pgsz_idx]; 731 732 /* if we were told not to allocate hugepages, override */ 733 if (!huge) 734 cur_pgsz = sysconf(_SC_PAGESIZE); 735 736 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 737 if (ret < 0) { 738 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 739 return -1; 740 } 741 742 /* allocate our memory */ 743 addr = alloc_mem(mem_sz, cur_pgsz, huge); 744 745 /* if we couldn't allocate memory with a specified page size, 746 * that doesn't mean we can't do it with other page sizes, so 747 * try another one. 748 */ 749 if (addr == NULL) 750 continue; 751 752 /* store IOVA addresses for every page in this memory area */ 753 n_pages = mem_sz / cur_pgsz; 754 755 iovas = malloc(sizeof(*iovas) * n_pages); 756 757 if (iovas == NULL) { 758 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 759 goto fail; 760 } 761 /* lock memory if it's not huge pages */ 762 if (!huge) 763 mlock(addr, mem_sz); 764 765 /* populate IOVA addresses */ 766 for (cur_page = 0; cur_page < n_pages; cur_page++) { 767 rte_iova_t iova; 768 size_t offset; 769 void *cur; 770 771 offset = cur_pgsz * cur_page; 772 cur = RTE_PTR_ADD(addr, offset); 773 774 /* touch the page before getting its IOVA */ 775 *(volatile char *)cur = 0; 776 777 iova = rte_mem_virt2iova(cur); 778 779 iovas[cur_page] = iova; 780 } 781 782 break; 783 } 784 /* if we couldn't allocate anything */ 785 if (iovas == NULL) 786 return -1; 787 788 param->addr = addr; 789 param->len = mem_sz; 790 param->pgsz = cur_pgsz; 791 param->iova_table = iovas; 792 param->iova_table_len = n_pages; 793 794 return 0; 795 fail: 796 if (iovas) 797 free(iovas); 798 if (addr) 799 munmap(addr, mem_sz); 800 801 return -1; 802 } 803 804 static int 805 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 806 { 807 struct extmem_param param; 808 int socket_id, ret; 809 810 memset(¶m, 0, sizeof(param)); 811 812 /* check if our heap exists */ 813 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 814 if (socket_id < 0) { 815 /* create our heap */ 816 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 817 if (ret < 0) { 818 TESTPMD_LOG(ERR, "Cannot create heap\n"); 819 return -1; 820 } 821 } 822 823 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 824 if (ret < 0) { 825 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 826 return -1; 827 } 828 829 /* we now have a valid memory area, so add it to heap */ 830 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 831 param.addr, param.len, param.iova_table, 832 param.iova_table_len, param.pgsz); 833 834 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 835 836 /* not needed any more */ 837 free(param.iova_table); 838 839 if (ret < 0) { 840 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 841 munmap(param.addr, param.len); 842 return -1; 843 } 844 845 /* success */ 846 847 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 848 param.len >> 20); 849 850 return 0; 851 } 852 static void 853 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 854 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 855 { 856 uint16_t pid = 0; 857 int ret; 858 859 RTE_ETH_FOREACH_DEV(pid) { 860 struct rte_eth_dev *dev = 861 &rte_eth_devices[pid]; 862 863 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 864 memhdr->len); 865 if (ret) { 866 TESTPMD_LOG(DEBUG, 867 "unable to DMA unmap addr 0x%p " 868 "for device %s\n", 869 memhdr->addr, dev->data->name); 870 } 871 } 872 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 873 if (ret) { 874 TESTPMD_LOG(DEBUG, 875 "unable to un-register addr 0x%p\n", memhdr->addr); 876 } 877 } 878 879 static void 880 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 881 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 882 { 883 uint16_t pid = 0; 884 size_t page_size = sysconf(_SC_PAGESIZE); 885 int ret; 886 887 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 888 page_size); 889 if (ret) { 890 TESTPMD_LOG(DEBUG, 891 "unable to register addr 0x%p\n", memhdr->addr); 892 return; 893 } 894 RTE_ETH_FOREACH_DEV(pid) { 895 struct rte_eth_dev *dev = 896 &rte_eth_devices[pid]; 897 898 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 899 memhdr->len); 900 if (ret) { 901 TESTPMD_LOG(DEBUG, 902 "unable to DMA map addr 0x%p " 903 "for device %s\n", 904 memhdr->addr, dev->data->name); 905 } 906 } 907 } 908 #endif 909 910 static unsigned int 911 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 912 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 913 { 914 struct rte_pktmbuf_extmem *xmem; 915 unsigned int ext_num, zone_num, elt_num; 916 uint16_t elt_size; 917 918 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 919 elt_num = EXTBUF_ZONE_SIZE / elt_size; 920 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 921 922 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 923 if (xmem == NULL) { 924 TESTPMD_LOG(ERR, "Cannot allocate memory for " 925 "external buffer descriptors\n"); 926 *ext_mem = NULL; 927 return 0; 928 } 929 for (ext_num = 0; ext_num < zone_num; ext_num++) { 930 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 931 const struct rte_memzone *mz; 932 char mz_name[RTE_MEMZONE_NAMESIZE]; 933 int ret; 934 935 ret = snprintf(mz_name, sizeof(mz_name), 936 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 937 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 938 errno = ENAMETOOLONG; 939 ext_num = 0; 940 break; 941 } 942 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 943 socket_id, 944 RTE_MEMZONE_IOVA_CONTIG | 945 RTE_MEMZONE_1GB | 946 RTE_MEMZONE_SIZE_HINT_ONLY, 947 EXTBUF_ZONE_SIZE); 948 if (mz == NULL) { 949 /* 950 * The caller exits on external buffer creation 951 * error, so there is no need to free memzones. 952 */ 953 errno = ENOMEM; 954 ext_num = 0; 955 break; 956 } 957 xseg->buf_ptr = mz->addr; 958 xseg->buf_iova = mz->iova; 959 xseg->buf_len = EXTBUF_ZONE_SIZE; 960 xseg->elt_size = elt_size; 961 } 962 if (ext_num == 0 && xmem != NULL) { 963 free(xmem); 964 xmem = NULL; 965 } 966 *ext_mem = xmem; 967 return ext_num; 968 } 969 970 /* 971 * Configuration initialisation done once at init time. 972 */ 973 static struct rte_mempool * 974 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 975 unsigned int socket_id, uint16_t size_idx) 976 { 977 char pool_name[RTE_MEMPOOL_NAMESIZE]; 978 struct rte_mempool *rte_mp = NULL; 979 #ifndef RTE_EXEC_ENV_WINDOWS 980 uint32_t mb_size; 981 982 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 983 #endif 984 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 985 986 TESTPMD_LOG(INFO, 987 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 988 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 989 990 switch (mp_alloc_type) { 991 case MP_ALLOC_NATIVE: 992 { 993 /* wrapper to rte_mempool_create() */ 994 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 995 rte_mbuf_best_mempool_ops()); 996 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 997 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 998 break; 999 } 1000 #ifndef RTE_EXEC_ENV_WINDOWS 1001 case MP_ALLOC_ANON: 1002 { 1003 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1004 mb_size, (unsigned int) mb_mempool_cache, 1005 sizeof(struct rte_pktmbuf_pool_private), 1006 socket_id, mempool_flags); 1007 if (rte_mp == NULL) 1008 goto err; 1009 1010 if (rte_mempool_populate_anon(rte_mp) == 0) { 1011 rte_mempool_free(rte_mp); 1012 rte_mp = NULL; 1013 goto err; 1014 } 1015 rte_pktmbuf_pool_init(rte_mp, NULL); 1016 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1017 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1018 break; 1019 } 1020 case MP_ALLOC_XMEM: 1021 case MP_ALLOC_XMEM_HUGE: 1022 { 1023 int heap_socket; 1024 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1025 1026 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1027 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1028 1029 heap_socket = 1030 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1031 if (heap_socket < 0) 1032 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1033 1034 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1035 rte_mbuf_best_mempool_ops()); 1036 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1037 mb_mempool_cache, 0, mbuf_seg_size, 1038 heap_socket); 1039 break; 1040 } 1041 #endif 1042 case MP_ALLOC_XBUF: 1043 { 1044 struct rte_pktmbuf_extmem *ext_mem; 1045 unsigned int ext_num; 1046 1047 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1048 socket_id, pool_name, &ext_mem); 1049 if (ext_num == 0) 1050 rte_exit(EXIT_FAILURE, 1051 "Can't create pinned data buffers\n"); 1052 1053 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1054 rte_mbuf_best_mempool_ops()); 1055 rte_mp = rte_pktmbuf_pool_create_extbuf 1056 (pool_name, nb_mbuf, mb_mempool_cache, 1057 0, mbuf_seg_size, socket_id, 1058 ext_mem, ext_num); 1059 free(ext_mem); 1060 break; 1061 } 1062 default: 1063 { 1064 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1065 } 1066 } 1067 1068 #ifndef RTE_EXEC_ENV_WINDOWS 1069 err: 1070 #endif 1071 if (rte_mp == NULL) { 1072 rte_exit(EXIT_FAILURE, 1073 "Creation of mbuf pool for socket %u failed: %s\n", 1074 socket_id, rte_strerror(rte_errno)); 1075 } else if (verbose_level > 0) { 1076 rte_mempool_dump(stdout, rte_mp); 1077 } 1078 return rte_mp; 1079 } 1080 1081 /* 1082 * Check given socket id is valid or not with NUMA mode, 1083 * if valid, return 0, else return -1 1084 */ 1085 static int 1086 check_socket_id(const unsigned int socket_id) 1087 { 1088 static int warning_once = 0; 1089 1090 if (new_socket_id(socket_id)) { 1091 if (!warning_once && numa_support) 1092 printf("Warning: NUMA should be configured manually by" 1093 " using --port-numa-config and" 1094 " --ring-numa-config parameters along with" 1095 " --numa.\n"); 1096 warning_once = 1; 1097 return -1; 1098 } 1099 return 0; 1100 } 1101 1102 /* 1103 * Get the allowed maximum number of RX queues. 1104 * *pid return the port id which has minimal value of 1105 * max_rx_queues in all ports. 1106 */ 1107 queueid_t 1108 get_allowed_max_nb_rxq(portid_t *pid) 1109 { 1110 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1111 bool max_rxq_valid = false; 1112 portid_t pi; 1113 struct rte_eth_dev_info dev_info; 1114 1115 RTE_ETH_FOREACH_DEV(pi) { 1116 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1117 continue; 1118 1119 max_rxq_valid = true; 1120 if (dev_info.max_rx_queues < allowed_max_rxq) { 1121 allowed_max_rxq = dev_info.max_rx_queues; 1122 *pid = pi; 1123 } 1124 } 1125 return max_rxq_valid ? allowed_max_rxq : 0; 1126 } 1127 1128 /* 1129 * Check input rxq is valid or not. 1130 * If input rxq is not greater than any of maximum number 1131 * of RX queues of all ports, it is valid. 1132 * if valid, return 0, else return -1 1133 */ 1134 int 1135 check_nb_rxq(queueid_t rxq) 1136 { 1137 queueid_t allowed_max_rxq; 1138 portid_t pid = 0; 1139 1140 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1141 if (rxq > allowed_max_rxq) { 1142 printf("Fail: input rxq (%u) can't be greater " 1143 "than max_rx_queues (%u) of port %u\n", 1144 rxq, 1145 allowed_max_rxq, 1146 pid); 1147 return -1; 1148 } 1149 return 0; 1150 } 1151 1152 /* 1153 * Get the allowed maximum number of TX queues. 1154 * *pid return the port id which has minimal value of 1155 * max_tx_queues in all ports. 1156 */ 1157 queueid_t 1158 get_allowed_max_nb_txq(portid_t *pid) 1159 { 1160 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1161 bool max_txq_valid = false; 1162 portid_t pi; 1163 struct rte_eth_dev_info dev_info; 1164 1165 RTE_ETH_FOREACH_DEV(pi) { 1166 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1167 continue; 1168 1169 max_txq_valid = true; 1170 if (dev_info.max_tx_queues < allowed_max_txq) { 1171 allowed_max_txq = dev_info.max_tx_queues; 1172 *pid = pi; 1173 } 1174 } 1175 return max_txq_valid ? allowed_max_txq : 0; 1176 } 1177 1178 /* 1179 * Check input txq is valid or not. 1180 * If input txq is not greater than any of maximum number 1181 * of TX queues of all ports, it is valid. 1182 * if valid, return 0, else return -1 1183 */ 1184 int 1185 check_nb_txq(queueid_t txq) 1186 { 1187 queueid_t allowed_max_txq; 1188 portid_t pid = 0; 1189 1190 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1191 if (txq > allowed_max_txq) { 1192 printf("Fail: input txq (%u) can't be greater " 1193 "than max_tx_queues (%u) of port %u\n", 1194 txq, 1195 allowed_max_txq, 1196 pid); 1197 return -1; 1198 } 1199 return 0; 1200 } 1201 1202 /* 1203 * Get the allowed maximum number of RXDs of every rx queue. 1204 * *pid return the port id which has minimal value of 1205 * max_rxd in all queues of all ports. 1206 */ 1207 static uint16_t 1208 get_allowed_max_nb_rxd(portid_t *pid) 1209 { 1210 uint16_t allowed_max_rxd = UINT16_MAX; 1211 portid_t pi; 1212 struct rte_eth_dev_info dev_info; 1213 1214 RTE_ETH_FOREACH_DEV(pi) { 1215 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1216 continue; 1217 1218 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1219 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1220 *pid = pi; 1221 } 1222 } 1223 return allowed_max_rxd; 1224 } 1225 1226 /* 1227 * Get the allowed minimal number of RXDs of every rx queue. 1228 * *pid return the port id which has minimal value of 1229 * min_rxd in all queues of all ports. 1230 */ 1231 static uint16_t 1232 get_allowed_min_nb_rxd(portid_t *pid) 1233 { 1234 uint16_t allowed_min_rxd = 0; 1235 portid_t pi; 1236 struct rte_eth_dev_info dev_info; 1237 1238 RTE_ETH_FOREACH_DEV(pi) { 1239 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1240 continue; 1241 1242 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1243 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1244 *pid = pi; 1245 } 1246 } 1247 1248 return allowed_min_rxd; 1249 } 1250 1251 /* 1252 * Check input rxd is valid or not. 1253 * If input rxd is not greater than any of maximum number 1254 * of RXDs of every Rx queues and is not less than any of 1255 * minimal number of RXDs of every Rx queues, it is valid. 1256 * if valid, return 0, else return -1 1257 */ 1258 int 1259 check_nb_rxd(queueid_t rxd) 1260 { 1261 uint16_t allowed_max_rxd; 1262 uint16_t allowed_min_rxd; 1263 portid_t pid = 0; 1264 1265 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1266 if (rxd > allowed_max_rxd) { 1267 printf("Fail: input rxd (%u) can't be greater " 1268 "than max_rxds (%u) of port %u\n", 1269 rxd, 1270 allowed_max_rxd, 1271 pid); 1272 return -1; 1273 } 1274 1275 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1276 if (rxd < allowed_min_rxd) { 1277 printf("Fail: input rxd (%u) can't be less " 1278 "than min_rxds (%u) of port %u\n", 1279 rxd, 1280 allowed_min_rxd, 1281 pid); 1282 return -1; 1283 } 1284 1285 return 0; 1286 } 1287 1288 /* 1289 * Get the allowed maximum number of TXDs of every rx queues. 1290 * *pid return the port id which has minimal value of 1291 * max_txd in every tx queue. 1292 */ 1293 static uint16_t 1294 get_allowed_max_nb_txd(portid_t *pid) 1295 { 1296 uint16_t allowed_max_txd = UINT16_MAX; 1297 portid_t pi; 1298 struct rte_eth_dev_info dev_info; 1299 1300 RTE_ETH_FOREACH_DEV(pi) { 1301 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1302 continue; 1303 1304 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1305 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1306 *pid = pi; 1307 } 1308 } 1309 return allowed_max_txd; 1310 } 1311 1312 /* 1313 * Get the allowed maximum number of TXDs of every tx queues. 1314 * *pid return the port id which has minimal value of 1315 * min_txd in every tx queue. 1316 */ 1317 static uint16_t 1318 get_allowed_min_nb_txd(portid_t *pid) 1319 { 1320 uint16_t allowed_min_txd = 0; 1321 portid_t pi; 1322 struct rte_eth_dev_info dev_info; 1323 1324 RTE_ETH_FOREACH_DEV(pi) { 1325 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1326 continue; 1327 1328 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1329 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1330 *pid = pi; 1331 } 1332 } 1333 1334 return allowed_min_txd; 1335 } 1336 1337 /* 1338 * Check input txd is valid or not. 1339 * If input txd is not greater than any of maximum number 1340 * of TXDs of every Rx queues, it is valid. 1341 * if valid, return 0, else return -1 1342 */ 1343 int 1344 check_nb_txd(queueid_t txd) 1345 { 1346 uint16_t allowed_max_txd; 1347 uint16_t allowed_min_txd; 1348 portid_t pid = 0; 1349 1350 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1351 if (txd > allowed_max_txd) { 1352 printf("Fail: input txd (%u) can't be greater " 1353 "than max_txds (%u) of port %u\n", 1354 txd, 1355 allowed_max_txd, 1356 pid); 1357 return -1; 1358 } 1359 1360 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1361 if (txd < allowed_min_txd) { 1362 printf("Fail: input txd (%u) can't be less " 1363 "than min_txds (%u) of port %u\n", 1364 txd, 1365 allowed_min_txd, 1366 pid); 1367 return -1; 1368 } 1369 return 0; 1370 } 1371 1372 1373 /* 1374 * Get the allowed maximum number of hairpin queues. 1375 * *pid return the port id which has minimal value of 1376 * max_hairpin_queues in all ports. 1377 */ 1378 queueid_t 1379 get_allowed_max_nb_hairpinq(portid_t *pid) 1380 { 1381 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1382 portid_t pi; 1383 struct rte_eth_hairpin_cap cap; 1384 1385 RTE_ETH_FOREACH_DEV(pi) { 1386 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1387 *pid = pi; 1388 return 0; 1389 } 1390 if (cap.max_nb_queues < allowed_max_hairpinq) { 1391 allowed_max_hairpinq = cap.max_nb_queues; 1392 *pid = pi; 1393 } 1394 } 1395 return allowed_max_hairpinq; 1396 } 1397 1398 /* 1399 * Check input hairpin is valid or not. 1400 * If input hairpin is not greater than any of maximum number 1401 * of hairpin queues of all ports, it is valid. 1402 * if valid, return 0, else return -1 1403 */ 1404 int 1405 check_nb_hairpinq(queueid_t hairpinq) 1406 { 1407 queueid_t allowed_max_hairpinq; 1408 portid_t pid = 0; 1409 1410 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1411 if (hairpinq > allowed_max_hairpinq) { 1412 printf("Fail: input hairpin (%u) can't be greater " 1413 "than max_hairpin_queues (%u) of port %u\n", 1414 hairpinq, allowed_max_hairpinq, pid); 1415 return -1; 1416 } 1417 return 0; 1418 } 1419 1420 static void 1421 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1422 { 1423 struct rte_port *port = &ports[pid]; 1424 uint16_t data_size; 1425 int ret; 1426 int i; 1427 1428 port->dev_conf.txmode = tx_mode; 1429 port->dev_conf.rxmode = rx_mode; 1430 1431 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1432 if (ret != 0) 1433 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1434 1435 ret = update_jumbo_frame_offload(pid); 1436 if (ret != 0) 1437 printf("Updating jumbo frame offload failed for port %u\n", 1438 pid); 1439 1440 if (!(port->dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1441 port->dev_conf.txmode.offloads &= 1442 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1443 1444 /* Apply Rx offloads configuration */ 1445 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1446 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads; 1447 /* Apply Tx offloads configuration */ 1448 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1449 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads; 1450 1451 if (eth_link_speed) 1452 port->dev_conf.link_speeds = eth_link_speed; 1453 1454 /* set flag to initialize port/queue */ 1455 port->need_reconfig = 1; 1456 port->need_reconfig_queues = 1; 1457 port->socket_id = socket_id; 1458 port->tx_metadata = 0; 1459 1460 /* 1461 * Check for maximum number of segments per MTU. 1462 * Accordingly update the mbuf data size. 1463 */ 1464 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1465 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1466 data_size = rx_mode.max_rx_pkt_len / 1467 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1468 1469 if ((data_size + RTE_PKTMBUF_HEADROOM) > mbuf_data_size[0]) { 1470 mbuf_data_size[0] = data_size + RTE_PKTMBUF_HEADROOM; 1471 TESTPMD_LOG(WARNING, 1472 "Configured mbuf size of the first segment %hu\n", 1473 mbuf_data_size[0]); 1474 } 1475 } 1476 } 1477 1478 static void 1479 init_config(void) 1480 { 1481 portid_t pid; 1482 struct rte_mempool *mbp; 1483 unsigned int nb_mbuf_per_pool; 1484 lcoreid_t lc_id; 1485 struct rte_gro_param gro_param; 1486 uint32_t gso_types; 1487 1488 /* Configuration of logical cores. */ 1489 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1490 sizeof(struct fwd_lcore *) * nb_lcores, 1491 RTE_CACHE_LINE_SIZE); 1492 if (fwd_lcores == NULL) { 1493 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1494 "failed\n", nb_lcores); 1495 } 1496 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1497 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1498 sizeof(struct fwd_lcore), 1499 RTE_CACHE_LINE_SIZE); 1500 if (fwd_lcores[lc_id] == NULL) { 1501 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1502 "failed\n"); 1503 } 1504 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1505 } 1506 1507 RTE_ETH_FOREACH_DEV(pid) { 1508 uint32_t socket_id; 1509 1510 if (numa_support) { 1511 socket_id = port_numa[pid]; 1512 if (port_numa[pid] == NUMA_NO_CONFIG) { 1513 socket_id = rte_eth_dev_socket_id(pid); 1514 1515 /* 1516 * if socket_id is invalid, 1517 * set to the first available socket. 1518 */ 1519 if (check_socket_id(socket_id) < 0) 1520 socket_id = socket_ids[0]; 1521 } 1522 } else { 1523 socket_id = (socket_num == UMA_NO_CONFIG) ? 1524 0 : socket_num; 1525 } 1526 /* Apply default TxRx configuration for all ports */ 1527 init_config_port_offloads(pid, socket_id); 1528 } 1529 /* 1530 * Create pools of mbuf. 1531 * If NUMA support is disabled, create a single pool of mbuf in 1532 * socket 0 memory by default. 1533 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1534 * 1535 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1536 * nb_txd can be configured at run time. 1537 */ 1538 if (param_total_num_mbufs) 1539 nb_mbuf_per_pool = param_total_num_mbufs; 1540 else { 1541 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1542 (nb_lcores * mb_mempool_cache) + 1543 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1544 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1545 } 1546 1547 if (numa_support) { 1548 uint8_t i, j; 1549 1550 for (i = 0; i < num_sockets; i++) 1551 for (j = 0; j < mbuf_data_size_n; j++) 1552 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1553 mbuf_pool_create(mbuf_data_size[j], 1554 nb_mbuf_per_pool, 1555 socket_ids[i], j); 1556 } else { 1557 uint8_t i; 1558 1559 for (i = 0; i < mbuf_data_size_n; i++) 1560 mempools[i] = mbuf_pool_create 1561 (mbuf_data_size[i], 1562 nb_mbuf_per_pool, 1563 socket_num == UMA_NO_CONFIG ? 1564 0 : socket_num, i); 1565 } 1566 1567 init_port_config(); 1568 1569 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1570 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1571 /* 1572 * Records which Mbuf pool to use by each logical core, if needed. 1573 */ 1574 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1575 mbp = mbuf_pool_find( 1576 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1577 1578 if (mbp == NULL) 1579 mbp = mbuf_pool_find(0, 0); 1580 fwd_lcores[lc_id]->mbp = mbp; 1581 /* initialize GSO context */ 1582 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1583 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1584 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1585 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1586 RTE_ETHER_CRC_LEN; 1587 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1588 } 1589 1590 fwd_config_setup(); 1591 1592 /* create a gro context for each lcore */ 1593 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1594 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1595 gro_param.max_item_per_flow = MAX_PKT_BURST; 1596 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1597 gro_param.socket_id = rte_lcore_to_socket_id( 1598 fwd_lcores_cpuids[lc_id]); 1599 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1600 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1601 rte_exit(EXIT_FAILURE, 1602 "rte_gro_ctx_create() failed\n"); 1603 } 1604 } 1605 } 1606 1607 1608 void 1609 reconfig(portid_t new_port_id, unsigned socket_id) 1610 { 1611 /* Reconfiguration of Ethernet ports. */ 1612 init_config_port_offloads(new_port_id, socket_id); 1613 init_port_config(); 1614 } 1615 1616 1617 int 1618 init_fwd_streams(void) 1619 { 1620 portid_t pid; 1621 struct rte_port *port; 1622 streamid_t sm_id, nb_fwd_streams_new; 1623 queueid_t q; 1624 1625 /* set socket id according to numa or not */ 1626 RTE_ETH_FOREACH_DEV(pid) { 1627 port = &ports[pid]; 1628 if (nb_rxq > port->dev_info.max_rx_queues) { 1629 printf("Fail: nb_rxq(%d) is greater than " 1630 "max_rx_queues(%d)\n", nb_rxq, 1631 port->dev_info.max_rx_queues); 1632 return -1; 1633 } 1634 if (nb_txq > port->dev_info.max_tx_queues) { 1635 printf("Fail: nb_txq(%d) is greater than " 1636 "max_tx_queues(%d)\n", nb_txq, 1637 port->dev_info.max_tx_queues); 1638 return -1; 1639 } 1640 if (numa_support) { 1641 if (port_numa[pid] != NUMA_NO_CONFIG) 1642 port->socket_id = port_numa[pid]; 1643 else { 1644 port->socket_id = rte_eth_dev_socket_id(pid); 1645 1646 /* 1647 * if socket_id is invalid, 1648 * set to the first available socket. 1649 */ 1650 if (check_socket_id(port->socket_id) < 0) 1651 port->socket_id = socket_ids[0]; 1652 } 1653 } 1654 else { 1655 if (socket_num == UMA_NO_CONFIG) 1656 port->socket_id = 0; 1657 else 1658 port->socket_id = socket_num; 1659 } 1660 } 1661 1662 q = RTE_MAX(nb_rxq, nb_txq); 1663 if (q == 0) { 1664 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1665 return -1; 1666 } 1667 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1668 if (nb_fwd_streams_new == nb_fwd_streams) 1669 return 0; 1670 /* clear the old */ 1671 if (fwd_streams != NULL) { 1672 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1673 if (fwd_streams[sm_id] == NULL) 1674 continue; 1675 rte_free(fwd_streams[sm_id]); 1676 fwd_streams[sm_id] = NULL; 1677 } 1678 rte_free(fwd_streams); 1679 fwd_streams = NULL; 1680 } 1681 1682 /* init new */ 1683 nb_fwd_streams = nb_fwd_streams_new; 1684 if (nb_fwd_streams) { 1685 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1686 sizeof(struct fwd_stream *) * nb_fwd_streams, 1687 RTE_CACHE_LINE_SIZE); 1688 if (fwd_streams == NULL) 1689 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1690 " (struct fwd_stream *)) failed\n", 1691 nb_fwd_streams); 1692 1693 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1694 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1695 " struct fwd_stream", sizeof(struct fwd_stream), 1696 RTE_CACHE_LINE_SIZE); 1697 if (fwd_streams[sm_id] == NULL) 1698 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1699 "(struct fwd_stream) failed\n"); 1700 } 1701 } 1702 1703 return 0; 1704 } 1705 1706 static void 1707 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1708 { 1709 uint64_t total_burst, sburst; 1710 uint64_t nb_burst; 1711 uint64_t burst_stats[4]; 1712 uint16_t pktnb_stats[4]; 1713 uint16_t nb_pkt; 1714 int burst_percent[4], sburstp; 1715 int i; 1716 1717 /* 1718 * First compute the total number of packet bursts and the 1719 * two highest numbers of bursts of the same number of packets. 1720 */ 1721 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1722 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1723 1724 /* Show stats for 0 burst size always */ 1725 total_burst = pbs->pkt_burst_spread[0]; 1726 burst_stats[0] = pbs->pkt_burst_spread[0]; 1727 pktnb_stats[0] = 0; 1728 1729 /* Find the next 2 burst sizes with highest occurrences. */ 1730 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1731 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1732 1733 if (nb_burst == 0) 1734 continue; 1735 1736 total_burst += nb_burst; 1737 1738 if (nb_burst > burst_stats[1]) { 1739 burst_stats[2] = burst_stats[1]; 1740 pktnb_stats[2] = pktnb_stats[1]; 1741 burst_stats[1] = nb_burst; 1742 pktnb_stats[1] = nb_pkt; 1743 } else if (nb_burst > burst_stats[2]) { 1744 burst_stats[2] = nb_burst; 1745 pktnb_stats[2] = nb_pkt; 1746 } 1747 } 1748 if (total_burst == 0) 1749 return; 1750 1751 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1752 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1753 if (i == 3) { 1754 printf("%d%% of other]\n", 100 - sburstp); 1755 return; 1756 } 1757 1758 sburst += burst_stats[i]; 1759 if (sburst == total_burst) { 1760 printf("%d%% of %d pkts]\n", 1761 100 - sburstp, (int) pktnb_stats[i]); 1762 return; 1763 } 1764 1765 burst_percent[i] = 1766 (double)burst_stats[i] / total_burst * 100; 1767 printf("%d%% of %d pkts + ", 1768 burst_percent[i], (int) pktnb_stats[i]); 1769 sburstp += burst_percent[i]; 1770 } 1771 } 1772 1773 static void 1774 fwd_stream_stats_display(streamid_t stream_id) 1775 { 1776 struct fwd_stream *fs; 1777 static const char *fwd_top_stats_border = "-------"; 1778 1779 fs = fwd_streams[stream_id]; 1780 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1781 (fs->fwd_dropped == 0)) 1782 return; 1783 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1784 "TX Port=%2d/Queue=%2d %s\n", 1785 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1786 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1787 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1788 " TX-dropped: %-14"PRIu64, 1789 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1790 1791 /* if checksum mode */ 1792 if (cur_fwd_eng == &csum_fwd_engine) { 1793 printf(" RX- bad IP checksum: %-14"PRIu64 1794 " Rx- bad L4 checksum: %-14"PRIu64 1795 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1796 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1797 fs->rx_bad_outer_l4_csum); 1798 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1799 fs->rx_bad_outer_ip_csum); 1800 } else { 1801 printf("\n"); 1802 } 1803 1804 if (record_burst_stats) { 1805 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1806 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1807 } 1808 } 1809 1810 void 1811 fwd_stats_display(void) 1812 { 1813 static const char *fwd_stats_border = "----------------------"; 1814 static const char *acc_stats_border = "+++++++++++++++"; 1815 struct { 1816 struct fwd_stream *rx_stream; 1817 struct fwd_stream *tx_stream; 1818 uint64_t tx_dropped; 1819 uint64_t rx_bad_ip_csum; 1820 uint64_t rx_bad_l4_csum; 1821 uint64_t rx_bad_outer_l4_csum; 1822 uint64_t rx_bad_outer_ip_csum; 1823 } ports_stats[RTE_MAX_ETHPORTS]; 1824 uint64_t total_rx_dropped = 0; 1825 uint64_t total_tx_dropped = 0; 1826 uint64_t total_rx_nombuf = 0; 1827 struct rte_eth_stats stats; 1828 uint64_t fwd_cycles = 0; 1829 uint64_t total_recv = 0; 1830 uint64_t total_xmit = 0; 1831 struct rte_port *port; 1832 streamid_t sm_id; 1833 portid_t pt_id; 1834 int i; 1835 1836 memset(ports_stats, 0, sizeof(ports_stats)); 1837 1838 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1839 struct fwd_stream *fs = fwd_streams[sm_id]; 1840 1841 if (cur_fwd_config.nb_fwd_streams > 1842 cur_fwd_config.nb_fwd_ports) { 1843 fwd_stream_stats_display(sm_id); 1844 } else { 1845 ports_stats[fs->tx_port].tx_stream = fs; 1846 ports_stats[fs->rx_port].rx_stream = fs; 1847 } 1848 1849 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1850 1851 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1852 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1853 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1854 fs->rx_bad_outer_l4_csum; 1855 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 1856 fs->rx_bad_outer_ip_csum; 1857 1858 if (record_core_cycles) 1859 fwd_cycles += fs->core_cycles; 1860 } 1861 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1862 pt_id = fwd_ports_ids[i]; 1863 port = &ports[pt_id]; 1864 1865 rte_eth_stats_get(pt_id, &stats); 1866 stats.ipackets -= port->stats.ipackets; 1867 stats.opackets -= port->stats.opackets; 1868 stats.ibytes -= port->stats.ibytes; 1869 stats.obytes -= port->stats.obytes; 1870 stats.imissed -= port->stats.imissed; 1871 stats.oerrors -= port->stats.oerrors; 1872 stats.rx_nombuf -= port->stats.rx_nombuf; 1873 1874 total_recv += stats.ipackets; 1875 total_xmit += stats.opackets; 1876 total_rx_dropped += stats.imissed; 1877 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1878 total_tx_dropped += stats.oerrors; 1879 total_rx_nombuf += stats.rx_nombuf; 1880 1881 printf("\n %s Forward statistics for port %-2d %s\n", 1882 fwd_stats_border, pt_id, fwd_stats_border); 1883 1884 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 1885 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 1886 stats.ipackets + stats.imissed); 1887 1888 if (cur_fwd_eng == &csum_fwd_engine) { 1889 printf(" Bad-ipcsum: %-14"PRIu64 1890 " Bad-l4csum: %-14"PRIu64 1891 "Bad-outer-l4csum: %-14"PRIu64"\n", 1892 ports_stats[pt_id].rx_bad_ip_csum, 1893 ports_stats[pt_id].rx_bad_l4_csum, 1894 ports_stats[pt_id].rx_bad_outer_l4_csum); 1895 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 1896 ports_stats[pt_id].rx_bad_outer_ip_csum); 1897 } 1898 if (stats.ierrors + stats.rx_nombuf > 0) { 1899 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 1900 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 1901 } 1902 1903 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 1904 "TX-total: %-"PRIu64"\n", 1905 stats.opackets, ports_stats[pt_id].tx_dropped, 1906 stats.opackets + ports_stats[pt_id].tx_dropped); 1907 1908 if (record_burst_stats) { 1909 if (ports_stats[pt_id].rx_stream) 1910 pkt_burst_stats_display("RX", 1911 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1912 if (ports_stats[pt_id].tx_stream) 1913 pkt_burst_stats_display("TX", 1914 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1915 } 1916 1917 printf(" %s--------------------------------%s\n", 1918 fwd_stats_border, fwd_stats_border); 1919 } 1920 1921 printf("\n %s Accumulated forward statistics for all ports" 1922 "%s\n", 1923 acc_stats_border, acc_stats_border); 1924 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1925 "%-"PRIu64"\n" 1926 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1927 "%-"PRIu64"\n", 1928 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1929 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1930 if (total_rx_nombuf > 0) 1931 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1932 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1933 "%s\n", 1934 acc_stats_border, acc_stats_border); 1935 if (record_core_cycles) { 1936 #define CYC_PER_MHZ 1E6 1937 if (total_recv > 0 || total_xmit > 0) { 1938 uint64_t total_pkts = 0; 1939 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1940 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1941 total_pkts = total_xmit; 1942 else 1943 total_pkts = total_recv; 1944 1945 printf("\n CPU cycles/packet=%.2F (total cycles=" 1946 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1947 " MHz Clock\n", 1948 (double) fwd_cycles / total_pkts, 1949 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1950 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1951 } 1952 } 1953 } 1954 1955 void 1956 fwd_stats_reset(void) 1957 { 1958 streamid_t sm_id; 1959 portid_t pt_id; 1960 int i; 1961 1962 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1963 pt_id = fwd_ports_ids[i]; 1964 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1965 } 1966 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1967 struct fwd_stream *fs = fwd_streams[sm_id]; 1968 1969 fs->rx_packets = 0; 1970 fs->tx_packets = 0; 1971 fs->fwd_dropped = 0; 1972 fs->rx_bad_ip_csum = 0; 1973 fs->rx_bad_l4_csum = 0; 1974 fs->rx_bad_outer_l4_csum = 0; 1975 fs->rx_bad_outer_ip_csum = 0; 1976 1977 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1978 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1979 fs->core_cycles = 0; 1980 } 1981 } 1982 1983 static void 1984 flush_fwd_rx_queues(void) 1985 { 1986 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1987 portid_t rxp; 1988 portid_t port_id; 1989 queueid_t rxq; 1990 uint16_t nb_rx; 1991 uint16_t i; 1992 uint8_t j; 1993 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1994 uint64_t timer_period; 1995 1996 /* convert to number of cycles */ 1997 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1998 1999 for (j = 0; j < 2; j++) { 2000 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2001 for (rxq = 0; rxq < nb_rxq; rxq++) { 2002 port_id = fwd_ports_ids[rxp]; 2003 /** 2004 * testpmd can stuck in the below do while loop 2005 * if rte_eth_rx_burst() always returns nonzero 2006 * packets. So timer is added to exit this loop 2007 * after 1sec timer expiry. 2008 */ 2009 prev_tsc = rte_rdtsc(); 2010 do { 2011 nb_rx = rte_eth_rx_burst(port_id, rxq, 2012 pkts_burst, MAX_PKT_BURST); 2013 for (i = 0; i < nb_rx; i++) 2014 rte_pktmbuf_free(pkts_burst[i]); 2015 2016 cur_tsc = rte_rdtsc(); 2017 diff_tsc = cur_tsc - prev_tsc; 2018 timer_tsc += diff_tsc; 2019 } while ((nb_rx > 0) && 2020 (timer_tsc < timer_period)); 2021 timer_tsc = 0; 2022 } 2023 } 2024 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2025 } 2026 } 2027 2028 static void 2029 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2030 { 2031 struct fwd_stream **fsm; 2032 streamid_t nb_fs; 2033 streamid_t sm_id; 2034 #ifdef RTE_LIB_BITRATESTATS 2035 uint64_t tics_per_1sec; 2036 uint64_t tics_datum; 2037 uint64_t tics_current; 2038 uint16_t i, cnt_ports; 2039 2040 cnt_ports = nb_ports; 2041 tics_datum = rte_rdtsc(); 2042 tics_per_1sec = rte_get_timer_hz(); 2043 #endif 2044 fsm = &fwd_streams[fc->stream_idx]; 2045 nb_fs = fc->stream_nb; 2046 do { 2047 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2048 (*pkt_fwd)(fsm[sm_id]); 2049 #ifdef RTE_LIB_BITRATESTATS 2050 if (bitrate_enabled != 0 && 2051 bitrate_lcore_id == rte_lcore_id()) { 2052 tics_current = rte_rdtsc(); 2053 if (tics_current - tics_datum >= tics_per_1sec) { 2054 /* Periodic bitrate calculation */ 2055 for (i = 0; i < cnt_ports; i++) 2056 rte_stats_bitrate_calc(bitrate_data, 2057 ports_ids[i]); 2058 tics_datum = tics_current; 2059 } 2060 } 2061 #endif 2062 #ifdef RTE_LIB_LATENCYSTATS 2063 if (latencystats_enabled != 0 && 2064 latencystats_lcore_id == rte_lcore_id()) 2065 rte_latencystats_update(); 2066 #endif 2067 2068 } while (! fc->stopped); 2069 } 2070 2071 static int 2072 start_pkt_forward_on_core(void *fwd_arg) 2073 { 2074 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2075 cur_fwd_config.fwd_eng->packet_fwd); 2076 return 0; 2077 } 2078 2079 /* 2080 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2081 * Used to start communication flows in network loopback test configurations. 2082 */ 2083 static int 2084 run_one_txonly_burst_on_core(void *fwd_arg) 2085 { 2086 struct fwd_lcore *fwd_lc; 2087 struct fwd_lcore tmp_lcore; 2088 2089 fwd_lc = (struct fwd_lcore *) fwd_arg; 2090 tmp_lcore = *fwd_lc; 2091 tmp_lcore.stopped = 1; 2092 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2093 return 0; 2094 } 2095 2096 /* 2097 * Launch packet forwarding: 2098 * - Setup per-port forwarding context. 2099 * - launch logical cores with their forwarding configuration. 2100 */ 2101 static void 2102 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2103 { 2104 port_fwd_begin_t port_fwd_begin; 2105 unsigned int i; 2106 unsigned int lc_id; 2107 int diag; 2108 2109 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2110 if (port_fwd_begin != NULL) { 2111 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2112 (*port_fwd_begin)(fwd_ports_ids[i]); 2113 } 2114 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2115 lc_id = fwd_lcores_cpuids[i]; 2116 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2117 fwd_lcores[i]->stopped = 0; 2118 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2119 fwd_lcores[i], lc_id); 2120 if (diag != 0) 2121 printf("launch lcore %u failed - diag=%d\n", 2122 lc_id, diag); 2123 } 2124 } 2125 } 2126 2127 /* 2128 * Launch packet forwarding configuration. 2129 */ 2130 void 2131 start_packet_forwarding(int with_tx_first) 2132 { 2133 port_fwd_begin_t port_fwd_begin; 2134 port_fwd_end_t port_fwd_end; 2135 unsigned int i; 2136 2137 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2138 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2139 2140 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2141 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2142 2143 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2144 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2145 (!nb_rxq || !nb_txq)) 2146 rte_exit(EXIT_FAILURE, 2147 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2148 cur_fwd_eng->fwd_mode_name); 2149 2150 if (all_ports_started() == 0) { 2151 printf("Not all ports were started\n"); 2152 return; 2153 } 2154 if (test_done == 0) { 2155 printf("Packet forwarding already started\n"); 2156 return; 2157 } 2158 test_done = 0; 2159 2160 fwd_config_setup(); 2161 2162 if(!no_flush_rx) 2163 flush_fwd_rx_queues(); 2164 2165 pkt_fwd_config_display(&cur_fwd_config); 2166 rxtx_config_display(); 2167 2168 fwd_stats_reset(); 2169 if (with_tx_first) { 2170 port_fwd_begin = tx_only_engine.port_fwd_begin; 2171 if (port_fwd_begin != NULL) { 2172 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2173 (*port_fwd_begin)(fwd_ports_ids[i]); 2174 } 2175 while (with_tx_first--) { 2176 launch_packet_forwarding( 2177 run_one_txonly_burst_on_core); 2178 rte_eal_mp_wait_lcore(); 2179 } 2180 port_fwd_end = tx_only_engine.port_fwd_end; 2181 if (port_fwd_end != NULL) { 2182 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2183 (*port_fwd_end)(fwd_ports_ids[i]); 2184 } 2185 } 2186 launch_packet_forwarding(start_pkt_forward_on_core); 2187 } 2188 2189 void 2190 stop_packet_forwarding(void) 2191 { 2192 port_fwd_end_t port_fwd_end; 2193 lcoreid_t lc_id; 2194 portid_t pt_id; 2195 int i; 2196 2197 if (test_done) { 2198 printf("Packet forwarding not started\n"); 2199 return; 2200 } 2201 printf("Telling cores to stop..."); 2202 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2203 fwd_lcores[lc_id]->stopped = 1; 2204 printf("\nWaiting for lcores to finish...\n"); 2205 rte_eal_mp_wait_lcore(); 2206 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2207 if (port_fwd_end != NULL) { 2208 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2209 pt_id = fwd_ports_ids[i]; 2210 (*port_fwd_end)(pt_id); 2211 } 2212 } 2213 2214 fwd_stats_display(); 2215 2216 printf("\nDone.\n"); 2217 test_done = 1; 2218 } 2219 2220 void 2221 dev_set_link_up(portid_t pid) 2222 { 2223 if (rte_eth_dev_set_link_up(pid) < 0) 2224 printf("\nSet link up fail.\n"); 2225 } 2226 2227 void 2228 dev_set_link_down(portid_t pid) 2229 { 2230 if (rte_eth_dev_set_link_down(pid) < 0) 2231 printf("\nSet link down fail.\n"); 2232 } 2233 2234 static int 2235 all_ports_started(void) 2236 { 2237 portid_t pi; 2238 struct rte_port *port; 2239 2240 RTE_ETH_FOREACH_DEV(pi) { 2241 port = &ports[pi]; 2242 /* Check if there is a port which is not started */ 2243 if ((port->port_status != RTE_PORT_STARTED) && 2244 (port->slave_flag == 0)) 2245 return 0; 2246 } 2247 2248 /* No port is not started */ 2249 return 1; 2250 } 2251 2252 int 2253 port_is_stopped(portid_t port_id) 2254 { 2255 struct rte_port *port = &ports[port_id]; 2256 2257 if ((port->port_status != RTE_PORT_STOPPED) && 2258 (port->slave_flag == 0)) 2259 return 0; 2260 return 1; 2261 } 2262 2263 int 2264 all_ports_stopped(void) 2265 { 2266 portid_t pi; 2267 2268 RTE_ETH_FOREACH_DEV(pi) { 2269 if (!port_is_stopped(pi)) 2270 return 0; 2271 } 2272 2273 return 1; 2274 } 2275 2276 int 2277 port_is_started(portid_t port_id) 2278 { 2279 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2280 return 0; 2281 2282 if (ports[port_id].port_status != RTE_PORT_STARTED) 2283 return 0; 2284 2285 return 1; 2286 } 2287 2288 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2289 static int 2290 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2291 { 2292 queueid_t qi; 2293 struct rte_eth_hairpin_conf hairpin_conf = { 2294 .peer_count = 1, 2295 }; 2296 int i; 2297 int diag; 2298 struct rte_port *port = &ports[pi]; 2299 uint16_t peer_rx_port = pi; 2300 uint16_t peer_tx_port = pi; 2301 uint32_t manual = 1; 2302 uint32_t tx_exp = hairpin_mode & 0x10; 2303 2304 if (!(hairpin_mode & 0xf)) { 2305 peer_rx_port = pi; 2306 peer_tx_port = pi; 2307 manual = 0; 2308 } else if (hairpin_mode & 0x1) { 2309 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2310 RTE_ETH_DEV_NO_OWNER); 2311 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2312 peer_tx_port = rte_eth_find_next_owned_by(0, 2313 RTE_ETH_DEV_NO_OWNER); 2314 if (p_pi != RTE_MAX_ETHPORTS) { 2315 peer_rx_port = p_pi; 2316 } else { 2317 uint16_t next_pi; 2318 2319 /* Last port will be the peer RX port of the first. */ 2320 RTE_ETH_FOREACH_DEV(next_pi) 2321 peer_rx_port = next_pi; 2322 } 2323 manual = 1; 2324 } else if (hairpin_mode & 0x2) { 2325 if (cnt_pi & 0x1) { 2326 peer_rx_port = p_pi; 2327 } else { 2328 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2329 RTE_ETH_DEV_NO_OWNER); 2330 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2331 peer_rx_port = pi; 2332 } 2333 peer_tx_port = peer_rx_port; 2334 manual = 1; 2335 } 2336 2337 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2338 hairpin_conf.peers[0].port = peer_rx_port; 2339 hairpin_conf.peers[0].queue = i + nb_rxq; 2340 hairpin_conf.manual_bind = !!manual; 2341 hairpin_conf.tx_explicit = !!tx_exp; 2342 diag = rte_eth_tx_hairpin_queue_setup 2343 (pi, qi, nb_txd, &hairpin_conf); 2344 i++; 2345 if (diag == 0) 2346 continue; 2347 2348 /* Fail to setup rx queue, return */ 2349 if (rte_atomic16_cmpset(&(port->port_status), 2350 RTE_PORT_HANDLING, 2351 RTE_PORT_STOPPED) == 0) 2352 printf("Port %d can not be set back " 2353 "to stopped\n", pi); 2354 printf("Fail to configure port %d hairpin " 2355 "queues\n", pi); 2356 /* try to reconfigure queues next time */ 2357 port->need_reconfig_queues = 1; 2358 return -1; 2359 } 2360 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2361 hairpin_conf.peers[0].port = peer_tx_port; 2362 hairpin_conf.peers[0].queue = i + nb_txq; 2363 hairpin_conf.manual_bind = !!manual; 2364 hairpin_conf.tx_explicit = !!tx_exp; 2365 diag = rte_eth_rx_hairpin_queue_setup 2366 (pi, qi, nb_rxd, &hairpin_conf); 2367 i++; 2368 if (diag == 0) 2369 continue; 2370 2371 /* Fail to setup rx queue, return */ 2372 if (rte_atomic16_cmpset(&(port->port_status), 2373 RTE_PORT_HANDLING, 2374 RTE_PORT_STOPPED) == 0) 2375 printf("Port %d can not be set back " 2376 "to stopped\n", pi); 2377 printf("Fail to configure port %d hairpin " 2378 "queues\n", pi); 2379 /* try to reconfigure queues next time */ 2380 port->need_reconfig_queues = 1; 2381 return -1; 2382 } 2383 return 0; 2384 } 2385 2386 /* Configure the Rx with optional split. */ 2387 int 2388 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2389 uint16_t nb_rx_desc, unsigned int socket_id, 2390 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2391 { 2392 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2393 unsigned int i, mp_n; 2394 int ret; 2395 2396 if (rx_pkt_nb_segs <= 1 || 2397 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2398 rx_conf->rx_seg = NULL; 2399 rx_conf->rx_nseg = 0; 2400 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2401 nb_rx_desc, socket_id, 2402 rx_conf, mp); 2403 return ret; 2404 } 2405 for (i = 0; i < rx_pkt_nb_segs; i++) { 2406 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2407 struct rte_mempool *mpx; 2408 /* 2409 * Use last valid pool for the segments with number 2410 * exceeding the pool index. 2411 */ 2412 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2413 mpx = mbuf_pool_find(socket_id, mp_n); 2414 /* Handle zero as mbuf data buffer size. */ 2415 rx_seg->length = rx_pkt_seg_lengths[i] ? 2416 rx_pkt_seg_lengths[i] : 2417 mbuf_data_size[mp_n]; 2418 rx_seg->offset = i < rx_pkt_nb_offs ? 2419 rx_pkt_seg_offsets[i] : 0; 2420 rx_seg->mp = mpx ? mpx : mp; 2421 } 2422 rx_conf->rx_nseg = rx_pkt_nb_segs; 2423 rx_conf->rx_seg = rx_useg; 2424 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2425 socket_id, rx_conf, NULL); 2426 rx_conf->rx_seg = NULL; 2427 rx_conf->rx_nseg = 0; 2428 return ret; 2429 } 2430 2431 int 2432 start_port(portid_t pid) 2433 { 2434 int diag, need_check_link_status = -1; 2435 portid_t pi; 2436 portid_t p_pi = RTE_MAX_ETHPORTS; 2437 portid_t pl[RTE_MAX_ETHPORTS]; 2438 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2439 uint16_t cnt_pi = 0; 2440 uint16_t cfg_pi = 0; 2441 int peer_pi; 2442 queueid_t qi; 2443 struct rte_port *port; 2444 struct rte_eth_hairpin_cap cap; 2445 2446 if (port_id_is_invalid(pid, ENABLED_WARN)) 2447 return 0; 2448 2449 RTE_ETH_FOREACH_DEV(pi) { 2450 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2451 continue; 2452 2453 need_check_link_status = 0; 2454 port = &ports[pi]; 2455 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2456 RTE_PORT_HANDLING) == 0) { 2457 printf("Port %d is now not stopped\n", pi); 2458 continue; 2459 } 2460 2461 if (port->need_reconfig > 0) { 2462 port->need_reconfig = 0; 2463 2464 if (flow_isolate_all) { 2465 int ret = port_flow_isolate(pi, 1); 2466 if (ret) { 2467 printf("Failed to apply isolated" 2468 " mode on port %d\n", pi); 2469 return -1; 2470 } 2471 } 2472 configure_rxtx_dump_callbacks(0); 2473 printf("Configuring Port %d (socket %u)\n", pi, 2474 port->socket_id); 2475 if (nb_hairpinq > 0 && 2476 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2477 printf("Port %d doesn't support hairpin " 2478 "queues\n", pi); 2479 return -1; 2480 } 2481 /* configure port */ 2482 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2483 nb_txq + nb_hairpinq, 2484 &(port->dev_conf)); 2485 if (diag != 0) { 2486 if (rte_atomic16_cmpset(&(port->port_status), 2487 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2488 printf("Port %d can not be set back " 2489 "to stopped\n", pi); 2490 printf("Fail to configure port %d\n", pi); 2491 /* try to reconfigure port next time */ 2492 port->need_reconfig = 1; 2493 return -1; 2494 } 2495 } 2496 if (port->need_reconfig_queues > 0) { 2497 port->need_reconfig_queues = 0; 2498 /* setup tx queues */ 2499 for (qi = 0; qi < nb_txq; qi++) { 2500 if ((numa_support) && 2501 (txring_numa[pi] != NUMA_NO_CONFIG)) 2502 diag = rte_eth_tx_queue_setup(pi, qi, 2503 port->nb_tx_desc[qi], 2504 txring_numa[pi], 2505 &(port->tx_conf[qi])); 2506 else 2507 diag = rte_eth_tx_queue_setup(pi, qi, 2508 port->nb_tx_desc[qi], 2509 port->socket_id, 2510 &(port->tx_conf[qi])); 2511 2512 if (diag == 0) 2513 continue; 2514 2515 /* Fail to setup tx queue, return */ 2516 if (rte_atomic16_cmpset(&(port->port_status), 2517 RTE_PORT_HANDLING, 2518 RTE_PORT_STOPPED) == 0) 2519 printf("Port %d can not be set back " 2520 "to stopped\n", pi); 2521 printf("Fail to configure port %d tx queues\n", 2522 pi); 2523 /* try to reconfigure queues next time */ 2524 port->need_reconfig_queues = 1; 2525 return -1; 2526 } 2527 for (qi = 0; qi < nb_rxq; qi++) { 2528 /* setup rx queues */ 2529 if ((numa_support) && 2530 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2531 struct rte_mempool * mp = 2532 mbuf_pool_find 2533 (rxring_numa[pi], 0); 2534 if (mp == NULL) { 2535 printf("Failed to setup RX queue:" 2536 "No mempool allocation" 2537 " on the socket %d\n", 2538 rxring_numa[pi]); 2539 return -1; 2540 } 2541 2542 diag = rx_queue_setup(pi, qi, 2543 port->nb_rx_desc[qi], 2544 rxring_numa[pi], 2545 &(port->rx_conf[qi]), 2546 mp); 2547 } else { 2548 struct rte_mempool *mp = 2549 mbuf_pool_find 2550 (port->socket_id, 0); 2551 if (mp == NULL) { 2552 printf("Failed to setup RX queue:" 2553 "No mempool allocation" 2554 " on the socket %d\n", 2555 port->socket_id); 2556 return -1; 2557 } 2558 diag = rx_queue_setup(pi, qi, 2559 port->nb_rx_desc[qi], 2560 port->socket_id, 2561 &(port->rx_conf[qi]), 2562 mp); 2563 } 2564 if (diag == 0) 2565 continue; 2566 2567 /* Fail to setup rx queue, return */ 2568 if (rte_atomic16_cmpset(&(port->port_status), 2569 RTE_PORT_HANDLING, 2570 RTE_PORT_STOPPED) == 0) 2571 printf("Port %d can not be set back " 2572 "to stopped\n", pi); 2573 printf("Fail to configure port %d rx queues\n", 2574 pi); 2575 /* try to reconfigure queues next time */ 2576 port->need_reconfig_queues = 1; 2577 return -1; 2578 } 2579 /* setup hairpin queues */ 2580 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2581 return -1; 2582 } 2583 configure_rxtx_dump_callbacks(verbose_level); 2584 if (clear_ptypes) { 2585 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2586 NULL, 0); 2587 if (diag < 0) 2588 printf( 2589 "Port %d: Failed to disable Ptype parsing\n", 2590 pi); 2591 } 2592 2593 p_pi = pi; 2594 cnt_pi++; 2595 2596 /* start port */ 2597 diag = rte_eth_dev_start(pi); 2598 if (diag < 0) { 2599 printf("Fail to start port %d: %s\n", pi, 2600 rte_strerror(-diag)); 2601 2602 /* Fail to setup rx queue, return */ 2603 if (rte_atomic16_cmpset(&(port->port_status), 2604 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2605 printf("Port %d can not be set back to " 2606 "stopped\n", pi); 2607 continue; 2608 } 2609 2610 if (rte_atomic16_cmpset(&(port->port_status), 2611 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2612 printf("Port %d can not be set into started\n", pi); 2613 2614 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 2615 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2616 port->eth_addr.addr_bytes[0], 2617 port->eth_addr.addr_bytes[1], 2618 port->eth_addr.addr_bytes[2], 2619 port->eth_addr.addr_bytes[3], 2620 port->eth_addr.addr_bytes[4], 2621 port->eth_addr.addr_bytes[5]); 2622 2623 /* at least one port started, need checking link status */ 2624 need_check_link_status = 1; 2625 2626 pl[cfg_pi++] = pi; 2627 } 2628 2629 if (need_check_link_status == 1 && !no_link_check) 2630 check_all_ports_link_status(RTE_PORT_ALL); 2631 else if (need_check_link_status == 0) 2632 printf("Please stop the ports first\n"); 2633 2634 if (hairpin_mode & 0xf) { 2635 uint16_t i; 2636 int j; 2637 2638 /* bind all started hairpin ports */ 2639 for (i = 0; i < cfg_pi; i++) { 2640 pi = pl[i]; 2641 /* bind current Tx to all peer Rx */ 2642 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2643 RTE_MAX_ETHPORTS, 1); 2644 if (peer_pi < 0) 2645 return peer_pi; 2646 for (j = 0; j < peer_pi; j++) { 2647 if (!port_is_started(peer_pl[j])) 2648 continue; 2649 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2650 if (diag < 0) { 2651 printf("Error during binding hairpin" 2652 " Tx port %u to %u: %s\n", 2653 pi, peer_pl[j], 2654 rte_strerror(-diag)); 2655 return -1; 2656 } 2657 } 2658 /* bind all peer Tx to current Rx */ 2659 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2660 RTE_MAX_ETHPORTS, 0); 2661 if (peer_pi < 0) 2662 return peer_pi; 2663 for (j = 0; j < peer_pi; j++) { 2664 if (!port_is_started(peer_pl[j])) 2665 continue; 2666 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2667 if (diag < 0) { 2668 printf("Error during binding hairpin" 2669 " Tx port %u to %u: %s\n", 2670 peer_pl[j], pi, 2671 rte_strerror(-diag)); 2672 return -1; 2673 } 2674 } 2675 } 2676 } 2677 2678 printf("Done\n"); 2679 return 0; 2680 } 2681 2682 void 2683 stop_port(portid_t pid) 2684 { 2685 portid_t pi; 2686 struct rte_port *port; 2687 int need_check_link_status = 0; 2688 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2689 int peer_pi; 2690 2691 if (port_id_is_invalid(pid, ENABLED_WARN)) 2692 return; 2693 2694 printf("Stopping ports...\n"); 2695 2696 RTE_ETH_FOREACH_DEV(pi) { 2697 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2698 continue; 2699 2700 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2701 printf("Please remove port %d from forwarding configuration.\n", pi); 2702 continue; 2703 } 2704 2705 if (port_is_bonding_slave(pi)) { 2706 printf("Please remove port %d from bonded device.\n", pi); 2707 continue; 2708 } 2709 2710 port = &ports[pi]; 2711 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2712 RTE_PORT_HANDLING) == 0) 2713 continue; 2714 2715 if (hairpin_mode & 0xf) { 2716 int j; 2717 2718 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2719 /* unbind all peer Tx from current Rx */ 2720 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2721 RTE_MAX_ETHPORTS, 0); 2722 if (peer_pi < 0) 2723 continue; 2724 for (j = 0; j < peer_pi; j++) { 2725 if (!port_is_started(peer_pl[j])) 2726 continue; 2727 rte_eth_hairpin_unbind(peer_pl[j], pi); 2728 } 2729 } 2730 2731 if (port->flow_list) 2732 port_flow_flush(pi); 2733 2734 if (rte_eth_dev_stop(pi) != 0) 2735 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2736 pi); 2737 2738 if (rte_atomic16_cmpset(&(port->port_status), 2739 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2740 printf("Port %d can not be set into stopped\n", pi); 2741 need_check_link_status = 1; 2742 } 2743 if (need_check_link_status && !no_link_check) 2744 check_all_ports_link_status(RTE_PORT_ALL); 2745 2746 printf("Done\n"); 2747 } 2748 2749 static void 2750 remove_invalid_ports_in(portid_t *array, portid_t *total) 2751 { 2752 portid_t i; 2753 portid_t new_total = 0; 2754 2755 for (i = 0; i < *total; i++) 2756 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2757 array[new_total] = array[i]; 2758 new_total++; 2759 } 2760 *total = new_total; 2761 } 2762 2763 static void 2764 remove_invalid_ports(void) 2765 { 2766 remove_invalid_ports_in(ports_ids, &nb_ports); 2767 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2768 nb_cfg_ports = nb_fwd_ports; 2769 } 2770 2771 void 2772 close_port(portid_t pid) 2773 { 2774 portid_t pi; 2775 struct rte_port *port; 2776 2777 if (port_id_is_invalid(pid, ENABLED_WARN)) 2778 return; 2779 2780 printf("Closing ports...\n"); 2781 2782 RTE_ETH_FOREACH_DEV(pi) { 2783 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2784 continue; 2785 2786 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2787 printf("Please remove port %d from forwarding configuration.\n", pi); 2788 continue; 2789 } 2790 2791 if (port_is_bonding_slave(pi)) { 2792 printf("Please remove port %d from bonded device.\n", pi); 2793 continue; 2794 } 2795 2796 port = &ports[pi]; 2797 if (rte_atomic16_cmpset(&(port->port_status), 2798 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2799 printf("Port %d is already closed\n", pi); 2800 continue; 2801 } 2802 2803 port_flow_flush(pi); 2804 rte_eth_dev_close(pi); 2805 } 2806 2807 remove_invalid_ports(); 2808 printf("Done\n"); 2809 } 2810 2811 void 2812 reset_port(portid_t pid) 2813 { 2814 int diag; 2815 portid_t pi; 2816 struct rte_port *port; 2817 2818 if (port_id_is_invalid(pid, ENABLED_WARN)) 2819 return; 2820 2821 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2822 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2823 printf("Can not reset port(s), please stop port(s) first.\n"); 2824 return; 2825 } 2826 2827 printf("Resetting ports...\n"); 2828 2829 RTE_ETH_FOREACH_DEV(pi) { 2830 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2831 continue; 2832 2833 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2834 printf("Please remove port %d from forwarding " 2835 "configuration.\n", pi); 2836 continue; 2837 } 2838 2839 if (port_is_bonding_slave(pi)) { 2840 printf("Please remove port %d from bonded device.\n", 2841 pi); 2842 continue; 2843 } 2844 2845 diag = rte_eth_dev_reset(pi); 2846 if (diag == 0) { 2847 port = &ports[pi]; 2848 port->need_reconfig = 1; 2849 port->need_reconfig_queues = 1; 2850 } else { 2851 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2852 } 2853 } 2854 2855 printf("Done\n"); 2856 } 2857 2858 void 2859 attach_port(char *identifier) 2860 { 2861 portid_t pi; 2862 struct rte_dev_iterator iterator; 2863 2864 printf("Attaching a new port...\n"); 2865 2866 if (identifier == NULL) { 2867 printf("Invalid parameters are specified\n"); 2868 return; 2869 } 2870 2871 if (rte_dev_probe(identifier) < 0) { 2872 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2873 return; 2874 } 2875 2876 /* first attach mode: event */ 2877 if (setup_on_probe_event) { 2878 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2879 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2880 if (ports[pi].port_status == RTE_PORT_HANDLING && 2881 ports[pi].need_setup != 0) 2882 setup_attached_port(pi); 2883 return; 2884 } 2885 2886 /* second attach mode: iterator */ 2887 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2888 /* setup ports matching the devargs used for probing */ 2889 if (port_is_forwarding(pi)) 2890 continue; /* port was already attached before */ 2891 setup_attached_port(pi); 2892 } 2893 } 2894 2895 static void 2896 setup_attached_port(portid_t pi) 2897 { 2898 unsigned int socket_id; 2899 int ret; 2900 2901 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2902 /* if socket_id is invalid, set to the first available socket. */ 2903 if (check_socket_id(socket_id) < 0) 2904 socket_id = socket_ids[0]; 2905 reconfig(pi, socket_id); 2906 ret = rte_eth_promiscuous_enable(pi); 2907 if (ret != 0) 2908 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2909 pi, rte_strerror(-ret)); 2910 2911 ports_ids[nb_ports++] = pi; 2912 fwd_ports_ids[nb_fwd_ports++] = pi; 2913 nb_cfg_ports = nb_fwd_ports; 2914 ports[pi].need_setup = 0; 2915 ports[pi].port_status = RTE_PORT_STOPPED; 2916 2917 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2918 printf("Done\n"); 2919 } 2920 2921 static void 2922 detach_device(struct rte_device *dev) 2923 { 2924 portid_t sibling; 2925 2926 if (dev == NULL) { 2927 printf("Device already removed\n"); 2928 return; 2929 } 2930 2931 printf("Removing a device...\n"); 2932 2933 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2934 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2935 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2936 printf("Port %u not stopped\n", sibling); 2937 return; 2938 } 2939 port_flow_flush(sibling); 2940 } 2941 } 2942 2943 if (rte_dev_remove(dev) < 0) { 2944 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2945 return; 2946 } 2947 remove_invalid_ports(); 2948 2949 printf("Device is detached\n"); 2950 printf("Now total ports is %d\n", nb_ports); 2951 printf("Done\n"); 2952 return; 2953 } 2954 2955 void 2956 detach_port_device(portid_t port_id) 2957 { 2958 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2959 return; 2960 2961 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2962 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2963 printf("Port not stopped\n"); 2964 return; 2965 } 2966 printf("Port was not closed\n"); 2967 } 2968 2969 detach_device(rte_eth_devices[port_id].device); 2970 } 2971 2972 void 2973 detach_devargs(char *identifier) 2974 { 2975 struct rte_dev_iterator iterator; 2976 struct rte_devargs da; 2977 portid_t port_id; 2978 2979 printf("Removing a device...\n"); 2980 2981 memset(&da, 0, sizeof(da)); 2982 if (rte_devargs_parsef(&da, "%s", identifier)) { 2983 printf("cannot parse identifier\n"); 2984 return; 2985 } 2986 2987 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2988 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2989 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2990 printf("Port %u not stopped\n", port_id); 2991 rte_eth_iterator_cleanup(&iterator); 2992 rte_devargs_reset(&da); 2993 return; 2994 } 2995 port_flow_flush(port_id); 2996 } 2997 } 2998 2999 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3000 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3001 da.name, da.bus->name); 3002 rte_devargs_reset(&da); 3003 return; 3004 } 3005 3006 remove_invalid_ports(); 3007 3008 printf("Device %s is detached\n", identifier); 3009 printf("Now total ports is %d\n", nb_ports); 3010 printf("Done\n"); 3011 rte_devargs_reset(&da); 3012 } 3013 3014 void 3015 pmd_test_exit(void) 3016 { 3017 portid_t pt_id; 3018 unsigned int i; 3019 int ret; 3020 3021 if (test_done == 0) 3022 stop_packet_forwarding(); 3023 3024 #ifndef RTE_EXEC_ENV_WINDOWS 3025 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3026 if (mempools[i]) { 3027 if (mp_alloc_type == MP_ALLOC_ANON) 3028 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3029 NULL); 3030 } 3031 } 3032 #endif 3033 if (ports != NULL) { 3034 no_link_check = 1; 3035 RTE_ETH_FOREACH_DEV(pt_id) { 3036 printf("\nStopping port %d...\n", pt_id); 3037 fflush(stdout); 3038 stop_port(pt_id); 3039 } 3040 RTE_ETH_FOREACH_DEV(pt_id) { 3041 printf("\nShutting down port %d...\n", pt_id); 3042 fflush(stdout); 3043 close_port(pt_id); 3044 } 3045 } 3046 3047 if (hot_plug) { 3048 ret = rte_dev_event_monitor_stop(); 3049 if (ret) { 3050 RTE_LOG(ERR, EAL, 3051 "fail to stop device event monitor."); 3052 return; 3053 } 3054 3055 ret = rte_dev_event_callback_unregister(NULL, 3056 dev_event_callback, NULL); 3057 if (ret < 0) { 3058 RTE_LOG(ERR, EAL, 3059 "fail to unregister device event callback.\n"); 3060 return; 3061 } 3062 3063 ret = rte_dev_hotplug_handle_disable(); 3064 if (ret) { 3065 RTE_LOG(ERR, EAL, 3066 "fail to disable hotplug handling.\n"); 3067 return; 3068 } 3069 } 3070 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3071 if (mempools[i]) 3072 rte_mempool_free(mempools[i]); 3073 } 3074 3075 printf("\nBye...\n"); 3076 } 3077 3078 typedef void (*cmd_func_t)(void); 3079 struct pmd_test_command { 3080 const char *cmd_name; 3081 cmd_func_t cmd_func; 3082 }; 3083 3084 /* Check the link status of all ports in up to 9s, and print them finally */ 3085 static void 3086 check_all_ports_link_status(uint32_t port_mask) 3087 { 3088 #define CHECK_INTERVAL 100 /* 100ms */ 3089 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3090 portid_t portid; 3091 uint8_t count, all_ports_up, print_flag = 0; 3092 struct rte_eth_link link; 3093 int ret; 3094 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3095 3096 printf("Checking link statuses...\n"); 3097 fflush(stdout); 3098 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3099 all_ports_up = 1; 3100 RTE_ETH_FOREACH_DEV(portid) { 3101 if ((port_mask & (1 << portid)) == 0) 3102 continue; 3103 memset(&link, 0, sizeof(link)); 3104 ret = rte_eth_link_get_nowait(portid, &link); 3105 if (ret < 0) { 3106 all_ports_up = 0; 3107 if (print_flag == 1) 3108 printf("Port %u link get failed: %s\n", 3109 portid, rte_strerror(-ret)); 3110 continue; 3111 } 3112 /* print link status if flag set */ 3113 if (print_flag == 1) { 3114 rte_eth_link_to_str(link_status, 3115 sizeof(link_status), &link); 3116 printf("Port %d %s\n", portid, link_status); 3117 continue; 3118 } 3119 /* clear all_ports_up flag if any link down */ 3120 if (link.link_status == ETH_LINK_DOWN) { 3121 all_ports_up = 0; 3122 break; 3123 } 3124 } 3125 /* after finally printing all link status, get out */ 3126 if (print_flag == 1) 3127 break; 3128 3129 if (all_ports_up == 0) { 3130 fflush(stdout); 3131 rte_delay_ms(CHECK_INTERVAL); 3132 } 3133 3134 /* set the print_flag if all ports up or timeout */ 3135 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3136 print_flag = 1; 3137 } 3138 3139 if (lsc_interrupt) 3140 break; 3141 } 3142 } 3143 3144 static void 3145 rmv_port_callback(void *arg) 3146 { 3147 int need_to_start = 0; 3148 int org_no_link_check = no_link_check; 3149 portid_t port_id = (intptr_t)arg; 3150 struct rte_device *dev; 3151 3152 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3153 3154 if (!test_done && port_is_forwarding(port_id)) { 3155 need_to_start = 1; 3156 stop_packet_forwarding(); 3157 } 3158 no_link_check = 1; 3159 stop_port(port_id); 3160 no_link_check = org_no_link_check; 3161 3162 /* Save rte_device pointer before closing ethdev port */ 3163 dev = rte_eth_devices[port_id].device; 3164 close_port(port_id); 3165 detach_device(dev); /* might be already removed or have more ports */ 3166 3167 if (need_to_start) 3168 start_packet_forwarding(0); 3169 } 3170 3171 /* This function is used by the interrupt thread */ 3172 static int 3173 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3174 void *ret_param) 3175 { 3176 RTE_SET_USED(param); 3177 RTE_SET_USED(ret_param); 3178 3179 if (type >= RTE_ETH_EVENT_MAX) { 3180 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3181 port_id, __func__, type); 3182 fflush(stderr); 3183 } else if (event_print_mask & (UINT32_C(1) << type)) { 3184 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3185 eth_event_desc[type]); 3186 fflush(stdout); 3187 } 3188 3189 switch (type) { 3190 case RTE_ETH_EVENT_NEW: 3191 ports[port_id].need_setup = 1; 3192 ports[port_id].port_status = RTE_PORT_HANDLING; 3193 break; 3194 case RTE_ETH_EVENT_INTR_RMV: 3195 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3196 break; 3197 if (rte_eal_alarm_set(100000, 3198 rmv_port_callback, (void *)(intptr_t)port_id)) 3199 fprintf(stderr, "Could not set up deferred device removal\n"); 3200 break; 3201 case RTE_ETH_EVENT_DESTROY: 3202 ports[port_id].port_status = RTE_PORT_CLOSED; 3203 printf("Port %u is closed\n", port_id); 3204 break; 3205 default: 3206 break; 3207 } 3208 return 0; 3209 } 3210 3211 static int 3212 register_eth_event_callback(void) 3213 { 3214 int ret; 3215 enum rte_eth_event_type event; 3216 3217 for (event = RTE_ETH_EVENT_UNKNOWN; 3218 event < RTE_ETH_EVENT_MAX; event++) { 3219 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3220 event, 3221 eth_event_callback, 3222 NULL); 3223 if (ret != 0) { 3224 TESTPMD_LOG(ERR, "Failed to register callback for " 3225 "%s event\n", eth_event_desc[event]); 3226 return -1; 3227 } 3228 } 3229 3230 return 0; 3231 } 3232 3233 /* This function is used by the interrupt thread */ 3234 static void 3235 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3236 __rte_unused void *arg) 3237 { 3238 uint16_t port_id; 3239 int ret; 3240 3241 if (type >= RTE_DEV_EVENT_MAX) { 3242 fprintf(stderr, "%s called upon invalid event %d\n", 3243 __func__, type); 3244 fflush(stderr); 3245 } 3246 3247 switch (type) { 3248 case RTE_DEV_EVENT_REMOVE: 3249 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3250 device_name); 3251 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3252 if (ret) { 3253 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3254 device_name); 3255 return; 3256 } 3257 /* 3258 * Because the user's callback is invoked in eal interrupt 3259 * callback, the interrupt callback need to be finished before 3260 * it can be unregistered when detaching device. So finish 3261 * callback soon and use a deferred removal to detach device 3262 * is need. It is a workaround, once the device detaching be 3263 * moved into the eal in the future, the deferred removal could 3264 * be deleted. 3265 */ 3266 if (rte_eal_alarm_set(100000, 3267 rmv_port_callback, (void *)(intptr_t)port_id)) 3268 RTE_LOG(ERR, EAL, 3269 "Could not set up deferred device removal\n"); 3270 break; 3271 case RTE_DEV_EVENT_ADD: 3272 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3273 device_name); 3274 /* TODO: After finish kernel driver binding, 3275 * begin to attach port. 3276 */ 3277 break; 3278 default: 3279 break; 3280 } 3281 } 3282 3283 static void 3284 rxtx_port_config(struct rte_port *port) 3285 { 3286 uint16_t qid; 3287 uint64_t offloads; 3288 3289 for (qid = 0; qid < nb_rxq; qid++) { 3290 offloads = port->rx_conf[qid].offloads; 3291 port->rx_conf[qid] = port->dev_info.default_rxconf; 3292 if (offloads != 0) 3293 port->rx_conf[qid].offloads = offloads; 3294 3295 /* Check if any Rx parameters have been passed */ 3296 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3297 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3298 3299 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3300 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3301 3302 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3303 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3304 3305 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3306 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3307 3308 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3309 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3310 3311 port->nb_rx_desc[qid] = nb_rxd; 3312 } 3313 3314 for (qid = 0; qid < nb_txq; qid++) { 3315 offloads = port->tx_conf[qid].offloads; 3316 port->tx_conf[qid] = port->dev_info.default_txconf; 3317 if (offloads != 0) 3318 port->tx_conf[qid].offloads = offloads; 3319 3320 /* Check if any Tx parameters have been passed */ 3321 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3322 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3323 3324 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3325 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3326 3327 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3328 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3329 3330 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3331 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3332 3333 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3334 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3335 3336 port->nb_tx_desc[qid] = nb_txd; 3337 } 3338 } 3339 3340 /* 3341 * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload, 3342 * MTU is also aligned if JUMBO_FRAME offload is not set. 3343 * 3344 * port->dev_info should be set before calling this function. 3345 * 3346 * return 0 on success, negative on error 3347 */ 3348 int 3349 update_jumbo_frame_offload(portid_t portid) 3350 { 3351 struct rte_port *port = &ports[portid]; 3352 uint32_t eth_overhead; 3353 uint64_t rx_offloads; 3354 int ret; 3355 bool on; 3356 3357 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */ 3358 if (port->dev_info.max_mtu != UINT16_MAX && 3359 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu) 3360 eth_overhead = port->dev_info.max_rx_pktlen - 3361 port->dev_info.max_mtu; 3362 else 3363 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 3364 3365 rx_offloads = port->dev_conf.rxmode.offloads; 3366 3367 /* Default config value is 0 to use PMD specific overhead */ 3368 if (port->dev_conf.rxmode.max_rx_pkt_len == 0) 3369 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead; 3370 3371 if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) { 3372 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3373 on = false; 3374 } else { 3375 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3376 printf("Frame size (%u) is not supported by port %u\n", 3377 port->dev_conf.rxmode.max_rx_pkt_len, 3378 portid); 3379 return -1; 3380 } 3381 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3382 on = true; 3383 } 3384 3385 if (rx_offloads != port->dev_conf.rxmode.offloads) { 3386 uint16_t qid; 3387 3388 port->dev_conf.rxmode.offloads = rx_offloads; 3389 3390 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */ 3391 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) { 3392 if (on) 3393 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 3394 else 3395 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 3396 } 3397 } 3398 3399 /* If JUMBO_FRAME is set MTU conversion done by ethdev layer, 3400 * if unset do it here 3401 */ 3402 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) { 3403 ret = rte_eth_dev_set_mtu(portid, 3404 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead); 3405 if (ret) 3406 printf("Failed to set MTU to %u for port %u\n", 3407 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead, 3408 portid); 3409 } 3410 3411 return 0; 3412 } 3413 3414 void 3415 init_port_config(void) 3416 { 3417 portid_t pid; 3418 struct rte_port *port; 3419 int ret; 3420 3421 RTE_ETH_FOREACH_DEV(pid) { 3422 port = &ports[pid]; 3423 port->dev_conf.fdir_conf = fdir_conf; 3424 3425 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3426 if (ret != 0) 3427 return; 3428 3429 if (nb_rxq > 1) { 3430 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3431 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3432 rss_hf & port->dev_info.flow_type_rss_offloads; 3433 } else { 3434 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3435 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3436 } 3437 3438 if (port->dcb_flag == 0) { 3439 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3440 port->dev_conf.rxmode.mq_mode = 3441 (enum rte_eth_rx_mq_mode) 3442 (rx_mq_mode & ETH_MQ_RX_RSS); 3443 else 3444 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3445 } 3446 3447 rxtx_port_config(port); 3448 3449 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3450 if (ret != 0) 3451 return; 3452 3453 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3454 rte_pmd_ixgbe_bypass_init(pid); 3455 #endif 3456 3457 if (lsc_interrupt && 3458 (rte_eth_devices[pid].data->dev_flags & 3459 RTE_ETH_DEV_INTR_LSC)) 3460 port->dev_conf.intr_conf.lsc = 1; 3461 if (rmv_interrupt && 3462 (rte_eth_devices[pid].data->dev_flags & 3463 RTE_ETH_DEV_INTR_RMV)) 3464 port->dev_conf.intr_conf.rmv = 1; 3465 } 3466 } 3467 3468 void set_port_slave_flag(portid_t slave_pid) 3469 { 3470 struct rte_port *port; 3471 3472 port = &ports[slave_pid]; 3473 port->slave_flag = 1; 3474 } 3475 3476 void clear_port_slave_flag(portid_t slave_pid) 3477 { 3478 struct rte_port *port; 3479 3480 port = &ports[slave_pid]; 3481 port->slave_flag = 0; 3482 } 3483 3484 uint8_t port_is_bonding_slave(portid_t slave_pid) 3485 { 3486 struct rte_port *port; 3487 3488 port = &ports[slave_pid]; 3489 if ((rte_eth_devices[slave_pid].data->dev_flags & 3490 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3491 return 1; 3492 return 0; 3493 } 3494 3495 const uint16_t vlan_tags[] = { 3496 0, 1, 2, 3, 4, 5, 6, 7, 3497 8, 9, 10, 11, 12, 13, 14, 15, 3498 16, 17, 18, 19, 20, 21, 22, 23, 3499 24, 25, 26, 27, 28, 29, 30, 31 3500 }; 3501 3502 static int 3503 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3504 enum dcb_mode_enable dcb_mode, 3505 enum rte_eth_nb_tcs num_tcs, 3506 uint8_t pfc_en) 3507 { 3508 uint8_t i; 3509 int32_t rc; 3510 struct rte_eth_rss_conf rss_conf; 3511 3512 /* 3513 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3514 * given above, and the number of traffic classes available for use. 3515 */ 3516 if (dcb_mode == DCB_VT_ENABLED) { 3517 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3518 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3519 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3520 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3521 3522 /* VMDQ+DCB RX and TX configurations */ 3523 vmdq_rx_conf->enable_default_pool = 0; 3524 vmdq_rx_conf->default_pool = 0; 3525 vmdq_rx_conf->nb_queue_pools = 3526 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3527 vmdq_tx_conf->nb_queue_pools = 3528 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3529 3530 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3531 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3532 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3533 vmdq_rx_conf->pool_map[i].pools = 3534 1 << (i % vmdq_rx_conf->nb_queue_pools); 3535 } 3536 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3537 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3538 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3539 } 3540 3541 /* set DCB mode of RX and TX of multiple queues */ 3542 eth_conf->rxmode.mq_mode = 3543 (enum rte_eth_rx_mq_mode) 3544 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3545 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3546 } else { 3547 struct rte_eth_dcb_rx_conf *rx_conf = 3548 ð_conf->rx_adv_conf.dcb_rx_conf; 3549 struct rte_eth_dcb_tx_conf *tx_conf = 3550 ð_conf->tx_adv_conf.dcb_tx_conf; 3551 3552 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3553 3554 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3555 if (rc != 0) 3556 return rc; 3557 3558 rx_conf->nb_tcs = num_tcs; 3559 tx_conf->nb_tcs = num_tcs; 3560 3561 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3562 rx_conf->dcb_tc[i] = i % num_tcs; 3563 tx_conf->dcb_tc[i] = i % num_tcs; 3564 } 3565 3566 eth_conf->rxmode.mq_mode = 3567 (enum rte_eth_rx_mq_mode) 3568 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3569 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3570 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3571 } 3572 3573 if (pfc_en) 3574 eth_conf->dcb_capability_en = 3575 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3576 else 3577 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3578 3579 return 0; 3580 } 3581 3582 int 3583 init_port_dcb_config(portid_t pid, 3584 enum dcb_mode_enable dcb_mode, 3585 enum rte_eth_nb_tcs num_tcs, 3586 uint8_t pfc_en) 3587 { 3588 struct rte_eth_conf port_conf; 3589 struct rte_port *rte_port; 3590 int retval; 3591 uint16_t i; 3592 3593 rte_port = &ports[pid]; 3594 3595 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3596 3597 port_conf.rxmode = rte_port->dev_conf.rxmode; 3598 port_conf.txmode = rte_port->dev_conf.txmode; 3599 3600 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3601 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3602 if (retval < 0) 3603 return retval; 3604 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3605 3606 /* re-configure the device . */ 3607 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3608 if (retval < 0) 3609 return retval; 3610 3611 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3612 if (retval != 0) 3613 return retval; 3614 3615 /* If dev_info.vmdq_pool_base is greater than 0, 3616 * the queue id of vmdq pools is started after pf queues. 3617 */ 3618 if (dcb_mode == DCB_VT_ENABLED && 3619 rte_port->dev_info.vmdq_pool_base > 0) { 3620 printf("VMDQ_DCB multi-queue mode is nonsensical" 3621 " for port %d.", pid); 3622 return -1; 3623 } 3624 3625 /* Assume the ports in testpmd have the same dcb capability 3626 * and has the same number of rxq and txq in dcb mode 3627 */ 3628 if (dcb_mode == DCB_VT_ENABLED) { 3629 if (rte_port->dev_info.max_vfs > 0) { 3630 nb_rxq = rte_port->dev_info.nb_rx_queues; 3631 nb_txq = rte_port->dev_info.nb_tx_queues; 3632 } else { 3633 nb_rxq = rte_port->dev_info.max_rx_queues; 3634 nb_txq = rte_port->dev_info.max_tx_queues; 3635 } 3636 } else { 3637 /*if vt is disabled, use all pf queues */ 3638 if (rte_port->dev_info.vmdq_pool_base == 0) { 3639 nb_rxq = rte_port->dev_info.max_rx_queues; 3640 nb_txq = rte_port->dev_info.max_tx_queues; 3641 } else { 3642 nb_rxq = (queueid_t)num_tcs; 3643 nb_txq = (queueid_t)num_tcs; 3644 3645 } 3646 } 3647 rx_free_thresh = 64; 3648 3649 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3650 3651 rxtx_port_config(rte_port); 3652 /* VLAN filter */ 3653 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3654 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3655 rx_vft_set(pid, vlan_tags[i], 1); 3656 3657 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3658 if (retval != 0) 3659 return retval; 3660 3661 rte_port->dcb_flag = 1; 3662 3663 /* Enter DCB configuration status */ 3664 dcb_config = 1; 3665 3666 return 0; 3667 } 3668 3669 static void 3670 init_port(void) 3671 { 3672 int i; 3673 3674 /* Configuration of Ethernet ports. */ 3675 ports = rte_zmalloc("testpmd: ports", 3676 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3677 RTE_CACHE_LINE_SIZE); 3678 if (ports == NULL) { 3679 rte_exit(EXIT_FAILURE, 3680 "rte_zmalloc(%d struct rte_port) failed\n", 3681 RTE_MAX_ETHPORTS); 3682 } 3683 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3684 LIST_INIT(&ports[i].flow_tunnel_list); 3685 /* Initialize ports NUMA structures */ 3686 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3687 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3688 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3689 } 3690 3691 static void 3692 force_quit(void) 3693 { 3694 pmd_test_exit(); 3695 prompt_exit(); 3696 } 3697 3698 static void 3699 print_stats(void) 3700 { 3701 uint8_t i; 3702 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3703 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3704 3705 /* Clear screen and move to top left */ 3706 printf("%s%s", clr, top_left); 3707 3708 printf("\nPort statistics ===================================="); 3709 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3710 nic_stats_display(fwd_ports_ids[i]); 3711 3712 fflush(stdout); 3713 } 3714 3715 static void 3716 signal_handler(int signum) 3717 { 3718 if (signum == SIGINT || signum == SIGTERM) { 3719 printf("\nSignal %d received, preparing to exit...\n", 3720 signum); 3721 #ifdef RTE_LIB_PDUMP 3722 /* uninitialize packet capture framework */ 3723 rte_pdump_uninit(); 3724 #endif 3725 #ifdef RTE_LIB_LATENCYSTATS 3726 if (latencystats_enabled != 0) 3727 rte_latencystats_uninit(); 3728 #endif 3729 force_quit(); 3730 /* Set flag to indicate the force termination. */ 3731 f_quit = 1; 3732 /* exit with the expected status */ 3733 #ifndef RTE_EXEC_ENV_WINDOWS 3734 signal(signum, SIG_DFL); 3735 kill(getpid(), signum); 3736 #endif 3737 } 3738 } 3739 3740 int 3741 main(int argc, char** argv) 3742 { 3743 int diag; 3744 portid_t port_id; 3745 uint16_t count; 3746 int ret; 3747 3748 signal(SIGINT, signal_handler); 3749 signal(SIGTERM, signal_handler); 3750 3751 testpmd_logtype = rte_log_register("testpmd"); 3752 if (testpmd_logtype < 0) 3753 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3754 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3755 3756 diag = rte_eal_init(argc, argv); 3757 if (diag < 0) 3758 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3759 rte_strerror(rte_errno)); 3760 3761 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3762 rte_exit(EXIT_FAILURE, 3763 "Secondary process type not supported.\n"); 3764 3765 ret = register_eth_event_callback(); 3766 if (ret != 0) 3767 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3768 3769 #ifdef RTE_LIB_PDUMP 3770 /* initialize packet capture framework */ 3771 rte_pdump_init(); 3772 #endif 3773 3774 count = 0; 3775 RTE_ETH_FOREACH_DEV(port_id) { 3776 ports_ids[count] = port_id; 3777 count++; 3778 } 3779 nb_ports = (portid_t) count; 3780 if (nb_ports == 0) 3781 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3782 3783 /* allocate port structures, and init them */ 3784 init_port(); 3785 3786 set_def_fwd_config(); 3787 if (nb_lcores == 0) 3788 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3789 "Check the core mask argument\n"); 3790 3791 /* Bitrate/latency stats disabled by default */ 3792 #ifdef RTE_LIB_BITRATESTATS 3793 bitrate_enabled = 0; 3794 #endif 3795 #ifdef RTE_LIB_LATENCYSTATS 3796 latencystats_enabled = 0; 3797 #endif 3798 3799 /* on FreeBSD, mlockall() is disabled by default */ 3800 #ifdef RTE_EXEC_ENV_FREEBSD 3801 do_mlockall = 0; 3802 #else 3803 do_mlockall = 1; 3804 #endif 3805 3806 argc -= diag; 3807 argv += diag; 3808 if (argc > 1) 3809 launch_args_parse(argc, argv); 3810 3811 #ifndef RTE_EXEC_ENV_WINDOWS 3812 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3813 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3814 strerror(errno)); 3815 } 3816 #endif 3817 3818 if (tx_first && interactive) 3819 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3820 "interactive mode.\n"); 3821 3822 if (tx_first && lsc_interrupt) { 3823 printf("Warning: lsc_interrupt needs to be off when " 3824 " using tx_first. Disabling.\n"); 3825 lsc_interrupt = 0; 3826 } 3827 3828 if (!nb_rxq && !nb_txq) 3829 printf("Warning: Either rx or tx queues should be non-zero\n"); 3830 3831 if (nb_rxq > 1 && nb_rxq > nb_txq) 3832 printf("Warning: nb_rxq=%d enables RSS configuration, " 3833 "but nb_txq=%d will prevent to fully test it.\n", 3834 nb_rxq, nb_txq); 3835 3836 init_config(); 3837 3838 if (hot_plug) { 3839 ret = rte_dev_hotplug_handle_enable(); 3840 if (ret) { 3841 RTE_LOG(ERR, EAL, 3842 "fail to enable hotplug handling."); 3843 return -1; 3844 } 3845 3846 ret = rte_dev_event_monitor_start(); 3847 if (ret) { 3848 RTE_LOG(ERR, EAL, 3849 "fail to start device event monitoring."); 3850 return -1; 3851 } 3852 3853 ret = rte_dev_event_callback_register(NULL, 3854 dev_event_callback, NULL); 3855 if (ret) { 3856 RTE_LOG(ERR, EAL, 3857 "fail to register device event callback\n"); 3858 return -1; 3859 } 3860 } 3861 3862 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3863 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3864 3865 /* set all ports to promiscuous mode by default */ 3866 RTE_ETH_FOREACH_DEV(port_id) { 3867 ret = rte_eth_promiscuous_enable(port_id); 3868 if (ret != 0) 3869 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3870 port_id, rte_strerror(-ret)); 3871 } 3872 3873 /* Init metrics library */ 3874 rte_metrics_init(rte_socket_id()); 3875 3876 #ifdef RTE_LIB_LATENCYSTATS 3877 if (latencystats_enabled != 0) { 3878 int ret = rte_latencystats_init(1, NULL); 3879 if (ret) 3880 printf("Warning: latencystats init()" 3881 " returned error %d\n", ret); 3882 printf("Latencystats running on lcore %d\n", 3883 latencystats_lcore_id); 3884 } 3885 #endif 3886 3887 /* Setup bitrate stats */ 3888 #ifdef RTE_LIB_BITRATESTATS 3889 if (bitrate_enabled != 0) { 3890 bitrate_data = rte_stats_bitrate_create(); 3891 if (bitrate_data == NULL) 3892 rte_exit(EXIT_FAILURE, 3893 "Could not allocate bitrate data.\n"); 3894 rte_stats_bitrate_reg(bitrate_data); 3895 } 3896 #endif 3897 3898 #ifdef RTE_LIB_CMDLINE 3899 if (strlen(cmdline_filename) != 0) 3900 cmdline_read_from_file(cmdline_filename); 3901 3902 if (interactive == 1) { 3903 if (auto_start) { 3904 printf("Start automatic packet forwarding\n"); 3905 start_packet_forwarding(0); 3906 } 3907 prompt(); 3908 pmd_test_exit(); 3909 } else 3910 #endif 3911 { 3912 char c; 3913 int rc; 3914 3915 f_quit = 0; 3916 3917 printf("No commandline core given, start packet forwarding\n"); 3918 start_packet_forwarding(tx_first); 3919 if (stats_period != 0) { 3920 uint64_t prev_time = 0, cur_time, diff_time = 0; 3921 uint64_t timer_period; 3922 3923 /* Convert to number of cycles */ 3924 timer_period = stats_period * rte_get_timer_hz(); 3925 3926 while (f_quit == 0) { 3927 cur_time = rte_get_timer_cycles(); 3928 diff_time += cur_time - prev_time; 3929 3930 if (diff_time >= timer_period) { 3931 print_stats(); 3932 /* Reset the timer */ 3933 diff_time = 0; 3934 } 3935 /* Sleep to avoid unnecessary checks */ 3936 prev_time = cur_time; 3937 rte_delay_us_sleep(US_PER_S); 3938 } 3939 } 3940 3941 printf("Press enter to exit\n"); 3942 rc = read(0, &c, 1); 3943 pmd_test_exit(); 3944 if (rc < 0) 3945 return 1; 3946 } 3947 3948 ret = rte_eal_cleanup(); 3949 if (ret != 0) 3950 rte_exit(EXIT_FAILURE, 3951 "EAL cleanup failed: %s\n", strerror(-ret)); 3952 3953 return EXIT_SUCCESS; 3954 } 3955