1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 219 /* 220 * Configuration of packet segments used by the "txonly" processing engine. 221 */ 222 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 223 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 224 TXONLY_DEF_PACKET_LEN, 225 }; 226 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 227 228 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 229 /**< Split policy for packets to TX. */ 230 231 uint8_t txonly_multi_flow; 232 /**< Whether multiple flows are generated in TXONLY mode. */ 233 234 uint32_t tx_pkt_times_inter; 235 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 236 237 uint32_t tx_pkt_times_intra; 238 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 239 240 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 241 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 242 243 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 244 uint8_t dcb_config = 0; 245 246 /* Whether the dcb is in testing status */ 247 uint8_t dcb_test = 0; 248 249 /* 250 * Configurable number of RX/TX queues. 251 */ 252 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 253 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 254 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 255 256 /* 257 * Configurable number of RX/TX ring descriptors. 258 * Defaults are supplied by drivers via ethdev. 259 */ 260 #define RTE_TEST_RX_DESC_DEFAULT 0 261 #define RTE_TEST_TX_DESC_DEFAULT 0 262 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 263 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 264 265 #define RTE_PMD_PARAM_UNSET -1 266 /* 267 * Configurable values of RX and TX ring threshold registers. 268 */ 269 270 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 271 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 272 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 273 274 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 275 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 276 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of RX free threshold. 280 */ 281 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of RX drop enable. 285 */ 286 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 287 288 /* 289 * Configurable value of TX free threshold. 290 */ 291 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 292 293 /* 294 * Configurable value of TX RS bit threshold. 295 */ 296 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 297 298 /* 299 * Configurable value of buffered packets before sending. 300 */ 301 uint16_t noisy_tx_sw_bufsz; 302 303 /* 304 * Configurable value of packet buffer timeout. 305 */ 306 uint16_t noisy_tx_sw_buf_flush_time; 307 308 /* 309 * Configurable value for size of VNF internal memory area 310 * used for simulating noisy neighbour behaviour 311 */ 312 uint64_t noisy_lkup_mem_sz; 313 314 /* 315 * Configurable value of number of random writes done in 316 * VNF simulation memory area. 317 */ 318 uint64_t noisy_lkup_num_writes; 319 320 /* 321 * Configurable value of number of random reads done in 322 * VNF simulation memory area. 323 */ 324 uint64_t noisy_lkup_num_reads; 325 326 /* 327 * Configurable value of number of random reads/writes done in 328 * VNF simulation memory area. 329 */ 330 uint64_t noisy_lkup_num_reads_writes; 331 332 /* 333 * Receive Side Scaling (RSS) configuration. 334 */ 335 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 336 337 /* 338 * Port topology configuration 339 */ 340 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 341 342 /* 343 * Avoids to flush all the RX streams before starts forwarding. 344 */ 345 uint8_t no_flush_rx = 0; /* flush by default */ 346 347 /* 348 * Flow API isolated mode. 349 */ 350 uint8_t flow_isolate_all; 351 352 /* 353 * Avoids to check link status when starting/stopping a port. 354 */ 355 uint8_t no_link_check = 0; /* check by default */ 356 357 /* 358 * Don't automatically start all ports in interactive mode. 359 */ 360 uint8_t no_device_start = 0; 361 362 /* 363 * Enable link status change notification 364 */ 365 uint8_t lsc_interrupt = 1; /* enabled by default */ 366 367 /* 368 * Enable device removal notification. 369 */ 370 uint8_t rmv_interrupt = 1; /* enabled by default */ 371 372 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 373 374 /* After attach, port setup is called on event or by iterator */ 375 bool setup_on_probe_event = true; 376 377 /* Clear ptypes on port initialization. */ 378 uint8_t clear_ptypes = true; 379 380 /* Hairpin ports configuration mode. */ 381 uint16_t hairpin_mode; 382 383 /* Pretty printing of ethdev events */ 384 static const char * const eth_event_desc[] = { 385 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 386 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 387 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 388 [RTE_ETH_EVENT_INTR_RESET] = "reset", 389 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 390 [RTE_ETH_EVENT_IPSEC] = "IPsec", 391 [RTE_ETH_EVENT_MACSEC] = "MACsec", 392 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 393 [RTE_ETH_EVENT_NEW] = "device probed", 394 [RTE_ETH_EVENT_DESTROY] = "device released", 395 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 396 [RTE_ETH_EVENT_MAX] = NULL, 397 }; 398 399 /* 400 * Display or mask ether events 401 * Default to all events except VF_MBOX 402 */ 403 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 404 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 405 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 406 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 407 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 408 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 410 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 411 /* 412 * Decide if all memory are locked for performance. 413 */ 414 int do_mlockall = 0; 415 416 /* 417 * NIC bypass mode configuration options. 418 */ 419 420 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 421 /* The NIC bypass watchdog timeout. */ 422 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 423 #endif 424 425 426 #ifdef RTE_LIBRTE_LATENCY_STATS 427 428 /* 429 * Set when latency stats is enabled in the commandline 430 */ 431 uint8_t latencystats_enabled; 432 433 /* 434 * Lcore ID to serive latency statistics. 435 */ 436 lcoreid_t latencystats_lcore_id = -1; 437 438 #endif 439 440 /* 441 * Ethernet device configuration. 442 */ 443 struct rte_eth_rxmode rx_mode = { 444 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 445 /**< Default maximum frame length. */ 446 }; 447 448 struct rte_eth_txmode tx_mode = { 449 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 450 }; 451 452 struct rte_fdir_conf fdir_conf = { 453 .mode = RTE_FDIR_MODE_NONE, 454 .pballoc = RTE_FDIR_PBALLOC_64K, 455 .status = RTE_FDIR_REPORT_STATUS, 456 .mask = { 457 .vlan_tci_mask = 0xFFEF, 458 .ipv4_mask = { 459 .src_ip = 0xFFFFFFFF, 460 .dst_ip = 0xFFFFFFFF, 461 }, 462 .ipv6_mask = { 463 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 464 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 465 }, 466 .src_port_mask = 0xFFFF, 467 .dst_port_mask = 0xFFFF, 468 .mac_addr_byte_mask = 0xFF, 469 .tunnel_type_mask = 1, 470 .tunnel_id_mask = 0xFFFFFFFF, 471 }, 472 .drop_queue = 127, 473 }; 474 475 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 476 477 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 478 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 479 480 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 481 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 482 483 uint16_t nb_tx_queue_stats_mappings = 0; 484 uint16_t nb_rx_queue_stats_mappings = 0; 485 486 /* 487 * Display zero values by default for xstats 488 */ 489 uint8_t xstats_hide_zero; 490 491 /* 492 * Measure of CPU cycles disabled by default 493 */ 494 uint8_t record_core_cycles; 495 496 /* 497 * Display of RX and TX bursts disabled by default 498 */ 499 uint8_t record_burst_stats; 500 501 unsigned int num_sockets = 0; 502 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 503 504 #ifdef RTE_LIBRTE_BITRATESTATS 505 /* Bitrate statistics */ 506 struct rte_stats_bitrates *bitrate_data; 507 lcoreid_t bitrate_lcore_id; 508 uint8_t bitrate_enabled; 509 #endif 510 511 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 512 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 513 514 /* 515 * hexadecimal bitmask of RX mq mode can be enabled. 516 */ 517 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 518 519 /* Forward function declarations */ 520 static void setup_attached_port(portid_t pi); 521 static void map_port_queue_stats_mapping_registers(portid_t pi, 522 struct rte_port *port); 523 static void check_all_ports_link_status(uint32_t port_mask); 524 static int eth_event_callback(portid_t port_id, 525 enum rte_eth_event_type type, 526 void *param, void *ret_param); 527 static void dev_event_callback(const char *device_name, 528 enum rte_dev_event_type type, 529 void *param); 530 531 /* 532 * Check if all the ports are started. 533 * If yes, return positive value. If not, return zero. 534 */ 535 static int all_ports_started(void); 536 537 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 538 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 539 540 /* Holds the registered mbuf dynamic flags names. */ 541 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 542 543 /* 544 * Helper function to check if socket is already discovered. 545 * If yes, return positive value. If not, return zero. 546 */ 547 int 548 new_socket_id(unsigned int socket_id) 549 { 550 unsigned int i; 551 552 for (i = 0; i < num_sockets; i++) { 553 if (socket_ids[i] == socket_id) 554 return 0; 555 } 556 return 1; 557 } 558 559 /* 560 * Setup default configuration. 561 */ 562 static void 563 set_default_fwd_lcores_config(void) 564 { 565 unsigned int i; 566 unsigned int nb_lc; 567 unsigned int sock_num; 568 569 nb_lc = 0; 570 for (i = 0; i < RTE_MAX_LCORE; i++) { 571 if (!rte_lcore_is_enabled(i)) 572 continue; 573 sock_num = rte_lcore_to_socket_id(i); 574 if (new_socket_id(sock_num)) { 575 if (num_sockets >= RTE_MAX_NUMA_NODES) { 576 rte_exit(EXIT_FAILURE, 577 "Total sockets greater than %u\n", 578 RTE_MAX_NUMA_NODES); 579 } 580 socket_ids[num_sockets++] = sock_num; 581 } 582 if (i == rte_get_master_lcore()) 583 continue; 584 fwd_lcores_cpuids[nb_lc++] = i; 585 } 586 nb_lcores = (lcoreid_t) nb_lc; 587 nb_cfg_lcores = nb_lcores; 588 nb_fwd_lcores = 1; 589 } 590 591 static void 592 set_def_peer_eth_addrs(void) 593 { 594 portid_t i; 595 596 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 597 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 598 peer_eth_addrs[i].addr_bytes[5] = i; 599 } 600 } 601 602 static void 603 set_default_fwd_ports_config(void) 604 { 605 portid_t pt_id; 606 int i = 0; 607 608 RTE_ETH_FOREACH_DEV(pt_id) { 609 fwd_ports_ids[i++] = pt_id; 610 611 /* Update sockets info according to the attached device */ 612 int socket_id = rte_eth_dev_socket_id(pt_id); 613 if (socket_id >= 0 && new_socket_id(socket_id)) { 614 if (num_sockets >= RTE_MAX_NUMA_NODES) { 615 rte_exit(EXIT_FAILURE, 616 "Total sockets greater than %u\n", 617 RTE_MAX_NUMA_NODES); 618 } 619 socket_ids[num_sockets++] = socket_id; 620 } 621 } 622 623 nb_cfg_ports = nb_ports; 624 nb_fwd_ports = nb_ports; 625 } 626 627 void 628 set_def_fwd_config(void) 629 { 630 set_default_fwd_lcores_config(); 631 set_def_peer_eth_addrs(); 632 set_default_fwd_ports_config(); 633 } 634 635 /* extremely pessimistic estimation of memory required to create a mempool */ 636 static int 637 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 638 { 639 unsigned int n_pages, mbuf_per_pg, leftover; 640 uint64_t total_mem, mbuf_mem, obj_sz; 641 642 /* there is no good way to predict how much space the mempool will 643 * occupy because it will allocate chunks on the fly, and some of those 644 * will come from default DPDK memory while some will come from our 645 * external memory, so just assume 128MB will be enough for everyone. 646 */ 647 uint64_t hdr_mem = 128 << 20; 648 649 /* account for possible non-contiguousness */ 650 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 651 if (obj_sz > pgsz) { 652 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 653 return -1; 654 } 655 656 mbuf_per_pg = pgsz / obj_sz; 657 leftover = (nb_mbufs % mbuf_per_pg) > 0; 658 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 659 660 mbuf_mem = n_pages * pgsz; 661 662 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 663 664 if (total_mem > SIZE_MAX) { 665 TESTPMD_LOG(ERR, "Memory size too big\n"); 666 return -1; 667 } 668 *out = (size_t)total_mem; 669 670 return 0; 671 } 672 673 static int 674 pagesz_flags(uint64_t page_sz) 675 { 676 /* as per mmap() manpage, all page sizes are log2 of page size 677 * shifted by MAP_HUGE_SHIFT 678 */ 679 int log2 = rte_log2_u64(page_sz); 680 681 return (log2 << HUGE_SHIFT); 682 } 683 684 static void * 685 alloc_mem(size_t memsz, size_t pgsz, bool huge) 686 { 687 void *addr; 688 int flags; 689 690 /* allocate anonymous hugepages */ 691 flags = MAP_ANONYMOUS | MAP_PRIVATE; 692 if (huge) 693 flags |= HUGE_FLAG | pagesz_flags(pgsz); 694 695 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 696 if (addr == MAP_FAILED) 697 return NULL; 698 699 return addr; 700 } 701 702 struct extmem_param { 703 void *addr; 704 size_t len; 705 size_t pgsz; 706 rte_iova_t *iova_table; 707 unsigned int iova_table_len; 708 }; 709 710 static int 711 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 712 bool huge) 713 { 714 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 715 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 716 unsigned int cur_page, n_pages, pgsz_idx; 717 size_t mem_sz, cur_pgsz; 718 rte_iova_t *iovas = NULL; 719 void *addr; 720 int ret; 721 722 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 723 /* skip anything that is too big */ 724 if (pgsizes[pgsz_idx] > SIZE_MAX) 725 continue; 726 727 cur_pgsz = pgsizes[pgsz_idx]; 728 729 /* if we were told not to allocate hugepages, override */ 730 if (!huge) 731 cur_pgsz = sysconf(_SC_PAGESIZE); 732 733 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 734 if (ret < 0) { 735 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 736 return -1; 737 } 738 739 /* allocate our memory */ 740 addr = alloc_mem(mem_sz, cur_pgsz, huge); 741 742 /* if we couldn't allocate memory with a specified page size, 743 * that doesn't mean we can't do it with other page sizes, so 744 * try another one. 745 */ 746 if (addr == NULL) 747 continue; 748 749 /* store IOVA addresses for every page in this memory area */ 750 n_pages = mem_sz / cur_pgsz; 751 752 iovas = malloc(sizeof(*iovas) * n_pages); 753 754 if (iovas == NULL) { 755 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 756 goto fail; 757 } 758 /* lock memory if it's not huge pages */ 759 if (!huge) 760 mlock(addr, mem_sz); 761 762 /* populate IOVA addresses */ 763 for (cur_page = 0; cur_page < n_pages; cur_page++) { 764 rte_iova_t iova; 765 size_t offset; 766 void *cur; 767 768 offset = cur_pgsz * cur_page; 769 cur = RTE_PTR_ADD(addr, offset); 770 771 /* touch the page before getting its IOVA */ 772 *(volatile char *)cur = 0; 773 774 iova = rte_mem_virt2iova(cur); 775 776 iovas[cur_page] = iova; 777 } 778 779 break; 780 } 781 /* if we couldn't allocate anything */ 782 if (iovas == NULL) 783 return -1; 784 785 param->addr = addr; 786 param->len = mem_sz; 787 param->pgsz = cur_pgsz; 788 param->iova_table = iovas; 789 param->iova_table_len = n_pages; 790 791 return 0; 792 fail: 793 if (iovas) 794 free(iovas); 795 if (addr) 796 munmap(addr, mem_sz); 797 798 return -1; 799 } 800 801 static int 802 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 803 { 804 struct extmem_param param; 805 int socket_id, ret; 806 807 memset(¶m, 0, sizeof(param)); 808 809 /* check if our heap exists */ 810 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 811 if (socket_id < 0) { 812 /* create our heap */ 813 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 814 if (ret < 0) { 815 TESTPMD_LOG(ERR, "Cannot create heap\n"); 816 return -1; 817 } 818 } 819 820 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 821 if (ret < 0) { 822 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 823 return -1; 824 } 825 826 /* we now have a valid memory area, so add it to heap */ 827 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 828 param.addr, param.len, param.iova_table, 829 param.iova_table_len, param.pgsz); 830 831 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 832 833 /* not needed any more */ 834 free(param.iova_table); 835 836 if (ret < 0) { 837 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 838 munmap(param.addr, param.len); 839 return -1; 840 } 841 842 /* success */ 843 844 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 845 param.len >> 20); 846 847 return 0; 848 } 849 static void 850 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 851 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 852 { 853 uint16_t pid = 0; 854 int ret; 855 856 RTE_ETH_FOREACH_DEV(pid) { 857 struct rte_eth_dev *dev = 858 &rte_eth_devices[pid]; 859 860 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 861 memhdr->len); 862 if (ret) { 863 TESTPMD_LOG(DEBUG, 864 "unable to DMA unmap addr 0x%p " 865 "for device %s\n", 866 memhdr->addr, dev->data->name); 867 } 868 } 869 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 870 if (ret) { 871 TESTPMD_LOG(DEBUG, 872 "unable to un-register addr 0x%p\n", memhdr->addr); 873 } 874 } 875 876 static void 877 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 878 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 879 { 880 uint16_t pid = 0; 881 size_t page_size = sysconf(_SC_PAGESIZE); 882 int ret; 883 884 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 885 page_size); 886 if (ret) { 887 TESTPMD_LOG(DEBUG, 888 "unable to register addr 0x%p\n", memhdr->addr); 889 return; 890 } 891 RTE_ETH_FOREACH_DEV(pid) { 892 struct rte_eth_dev *dev = 893 &rte_eth_devices[pid]; 894 895 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 896 memhdr->len); 897 if (ret) { 898 TESTPMD_LOG(DEBUG, 899 "unable to DMA map addr 0x%p " 900 "for device %s\n", 901 memhdr->addr, dev->data->name); 902 } 903 } 904 } 905 906 static unsigned int 907 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 908 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 909 { 910 struct rte_pktmbuf_extmem *xmem; 911 unsigned int ext_num, zone_num, elt_num; 912 uint16_t elt_size; 913 914 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 915 elt_num = EXTBUF_ZONE_SIZE / elt_size; 916 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 917 918 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 919 if (xmem == NULL) { 920 TESTPMD_LOG(ERR, "Cannot allocate memory for " 921 "external buffer descriptors\n"); 922 *ext_mem = NULL; 923 return 0; 924 } 925 for (ext_num = 0; ext_num < zone_num; ext_num++) { 926 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 927 const struct rte_memzone *mz; 928 char mz_name[RTE_MEMZONE_NAMESIZE]; 929 int ret; 930 931 ret = snprintf(mz_name, sizeof(mz_name), 932 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 933 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 934 errno = ENAMETOOLONG; 935 ext_num = 0; 936 break; 937 } 938 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 939 socket_id, 940 RTE_MEMZONE_IOVA_CONTIG | 941 RTE_MEMZONE_1GB | 942 RTE_MEMZONE_SIZE_HINT_ONLY, 943 EXTBUF_ZONE_SIZE); 944 if (mz == NULL) { 945 /* 946 * The caller exits on external buffer creation 947 * error, so there is no need to free memzones. 948 */ 949 errno = ENOMEM; 950 ext_num = 0; 951 break; 952 } 953 xseg->buf_ptr = mz->addr; 954 xseg->buf_iova = mz->iova; 955 xseg->buf_len = EXTBUF_ZONE_SIZE; 956 xseg->elt_size = elt_size; 957 } 958 if (ext_num == 0 && xmem != NULL) { 959 free(xmem); 960 xmem = NULL; 961 } 962 *ext_mem = xmem; 963 return ext_num; 964 } 965 966 /* 967 * Configuration initialisation done once at init time. 968 */ 969 static struct rte_mempool * 970 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 971 unsigned int socket_id, uint16_t size_idx) 972 { 973 char pool_name[RTE_MEMPOOL_NAMESIZE]; 974 struct rte_mempool *rte_mp = NULL; 975 uint32_t mb_size; 976 977 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 978 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 979 980 TESTPMD_LOG(INFO, 981 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 982 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 983 984 switch (mp_alloc_type) { 985 case MP_ALLOC_NATIVE: 986 { 987 /* wrapper to rte_mempool_create() */ 988 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 989 rte_mbuf_best_mempool_ops()); 990 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 991 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 992 break; 993 } 994 case MP_ALLOC_ANON: 995 { 996 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 997 mb_size, (unsigned int) mb_mempool_cache, 998 sizeof(struct rte_pktmbuf_pool_private), 999 socket_id, mempool_flags); 1000 if (rte_mp == NULL) 1001 goto err; 1002 1003 if (rte_mempool_populate_anon(rte_mp) == 0) { 1004 rte_mempool_free(rte_mp); 1005 rte_mp = NULL; 1006 goto err; 1007 } 1008 rte_pktmbuf_pool_init(rte_mp, NULL); 1009 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1010 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1011 break; 1012 } 1013 case MP_ALLOC_XMEM: 1014 case MP_ALLOC_XMEM_HUGE: 1015 { 1016 int heap_socket; 1017 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1018 1019 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1020 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1021 1022 heap_socket = 1023 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1024 if (heap_socket < 0) 1025 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1026 1027 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1028 rte_mbuf_best_mempool_ops()); 1029 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1030 mb_mempool_cache, 0, mbuf_seg_size, 1031 heap_socket); 1032 break; 1033 } 1034 case MP_ALLOC_XBUF: 1035 { 1036 struct rte_pktmbuf_extmem *ext_mem; 1037 unsigned int ext_num; 1038 1039 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1040 socket_id, pool_name, &ext_mem); 1041 if (ext_num == 0) 1042 rte_exit(EXIT_FAILURE, 1043 "Can't create pinned data buffers\n"); 1044 1045 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1046 rte_mbuf_best_mempool_ops()); 1047 rte_mp = rte_pktmbuf_pool_create_extbuf 1048 (pool_name, nb_mbuf, mb_mempool_cache, 1049 0, mbuf_seg_size, socket_id, 1050 ext_mem, ext_num); 1051 free(ext_mem); 1052 break; 1053 } 1054 default: 1055 { 1056 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1057 } 1058 } 1059 1060 err: 1061 if (rte_mp == NULL) { 1062 rte_exit(EXIT_FAILURE, 1063 "Creation of mbuf pool for socket %u failed: %s\n", 1064 socket_id, rte_strerror(rte_errno)); 1065 } else if (verbose_level > 0) { 1066 rte_mempool_dump(stdout, rte_mp); 1067 } 1068 return rte_mp; 1069 } 1070 1071 /* 1072 * Check given socket id is valid or not with NUMA mode, 1073 * if valid, return 0, else return -1 1074 */ 1075 static int 1076 check_socket_id(const unsigned int socket_id) 1077 { 1078 static int warning_once = 0; 1079 1080 if (new_socket_id(socket_id)) { 1081 if (!warning_once && numa_support) 1082 printf("Warning: NUMA should be configured manually by" 1083 " using --port-numa-config and" 1084 " --ring-numa-config parameters along with" 1085 " --numa.\n"); 1086 warning_once = 1; 1087 return -1; 1088 } 1089 return 0; 1090 } 1091 1092 /* 1093 * Get the allowed maximum number of RX queues. 1094 * *pid return the port id which has minimal value of 1095 * max_rx_queues in all ports. 1096 */ 1097 queueid_t 1098 get_allowed_max_nb_rxq(portid_t *pid) 1099 { 1100 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1101 bool max_rxq_valid = false; 1102 portid_t pi; 1103 struct rte_eth_dev_info dev_info; 1104 1105 RTE_ETH_FOREACH_DEV(pi) { 1106 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1107 continue; 1108 1109 max_rxq_valid = true; 1110 if (dev_info.max_rx_queues < allowed_max_rxq) { 1111 allowed_max_rxq = dev_info.max_rx_queues; 1112 *pid = pi; 1113 } 1114 } 1115 return max_rxq_valid ? allowed_max_rxq : 0; 1116 } 1117 1118 /* 1119 * Check input rxq is valid or not. 1120 * If input rxq is not greater than any of maximum number 1121 * of RX queues of all ports, it is valid. 1122 * if valid, return 0, else return -1 1123 */ 1124 int 1125 check_nb_rxq(queueid_t rxq) 1126 { 1127 queueid_t allowed_max_rxq; 1128 portid_t pid = 0; 1129 1130 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1131 if (rxq > allowed_max_rxq) { 1132 printf("Fail: input rxq (%u) can't be greater " 1133 "than max_rx_queues (%u) of port %u\n", 1134 rxq, 1135 allowed_max_rxq, 1136 pid); 1137 return -1; 1138 } 1139 return 0; 1140 } 1141 1142 /* 1143 * Get the allowed maximum number of TX queues. 1144 * *pid return the port id which has minimal value of 1145 * max_tx_queues in all ports. 1146 */ 1147 queueid_t 1148 get_allowed_max_nb_txq(portid_t *pid) 1149 { 1150 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1151 bool max_txq_valid = false; 1152 portid_t pi; 1153 struct rte_eth_dev_info dev_info; 1154 1155 RTE_ETH_FOREACH_DEV(pi) { 1156 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1157 continue; 1158 1159 max_txq_valid = true; 1160 if (dev_info.max_tx_queues < allowed_max_txq) { 1161 allowed_max_txq = dev_info.max_tx_queues; 1162 *pid = pi; 1163 } 1164 } 1165 return max_txq_valid ? allowed_max_txq : 0; 1166 } 1167 1168 /* 1169 * Check input txq is valid or not. 1170 * If input txq is not greater than any of maximum number 1171 * of TX queues of all ports, it is valid. 1172 * if valid, return 0, else return -1 1173 */ 1174 int 1175 check_nb_txq(queueid_t txq) 1176 { 1177 queueid_t allowed_max_txq; 1178 portid_t pid = 0; 1179 1180 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1181 if (txq > allowed_max_txq) { 1182 printf("Fail: input txq (%u) can't be greater " 1183 "than max_tx_queues (%u) of port %u\n", 1184 txq, 1185 allowed_max_txq, 1186 pid); 1187 return -1; 1188 } 1189 return 0; 1190 } 1191 1192 /* 1193 * Get the allowed maximum number of RXDs of every rx queue. 1194 * *pid return the port id which has minimal value of 1195 * max_rxd in all queues of all ports. 1196 */ 1197 static uint16_t 1198 get_allowed_max_nb_rxd(portid_t *pid) 1199 { 1200 uint16_t allowed_max_rxd = UINT16_MAX; 1201 portid_t pi; 1202 struct rte_eth_dev_info dev_info; 1203 1204 RTE_ETH_FOREACH_DEV(pi) { 1205 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1206 continue; 1207 1208 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1209 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1210 *pid = pi; 1211 } 1212 } 1213 return allowed_max_rxd; 1214 } 1215 1216 /* 1217 * Get the allowed minimal number of RXDs of every rx queue. 1218 * *pid return the port id which has minimal value of 1219 * min_rxd in all queues of all ports. 1220 */ 1221 static uint16_t 1222 get_allowed_min_nb_rxd(portid_t *pid) 1223 { 1224 uint16_t allowed_min_rxd = 0; 1225 portid_t pi; 1226 struct rte_eth_dev_info dev_info; 1227 1228 RTE_ETH_FOREACH_DEV(pi) { 1229 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1230 continue; 1231 1232 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1233 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1234 *pid = pi; 1235 } 1236 } 1237 1238 return allowed_min_rxd; 1239 } 1240 1241 /* 1242 * Check input rxd is valid or not. 1243 * If input rxd is not greater than any of maximum number 1244 * of RXDs of every Rx queues and is not less than any of 1245 * minimal number of RXDs of every Rx queues, it is valid. 1246 * if valid, return 0, else return -1 1247 */ 1248 int 1249 check_nb_rxd(queueid_t rxd) 1250 { 1251 uint16_t allowed_max_rxd; 1252 uint16_t allowed_min_rxd; 1253 portid_t pid = 0; 1254 1255 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1256 if (rxd > allowed_max_rxd) { 1257 printf("Fail: input rxd (%u) can't be greater " 1258 "than max_rxds (%u) of port %u\n", 1259 rxd, 1260 allowed_max_rxd, 1261 pid); 1262 return -1; 1263 } 1264 1265 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1266 if (rxd < allowed_min_rxd) { 1267 printf("Fail: input rxd (%u) can't be less " 1268 "than min_rxds (%u) of port %u\n", 1269 rxd, 1270 allowed_min_rxd, 1271 pid); 1272 return -1; 1273 } 1274 1275 return 0; 1276 } 1277 1278 /* 1279 * Get the allowed maximum number of TXDs of every rx queues. 1280 * *pid return the port id which has minimal value of 1281 * max_txd in every tx queue. 1282 */ 1283 static uint16_t 1284 get_allowed_max_nb_txd(portid_t *pid) 1285 { 1286 uint16_t allowed_max_txd = UINT16_MAX; 1287 portid_t pi; 1288 struct rte_eth_dev_info dev_info; 1289 1290 RTE_ETH_FOREACH_DEV(pi) { 1291 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1292 continue; 1293 1294 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1295 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1296 *pid = pi; 1297 } 1298 } 1299 return allowed_max_txd; 1300 } 1301 1302 /* 1303 * Get the allowed maximum number of TXDs of every tx queues. 1304 * *pid return the port id which has minimal value of 1305 * min_txd in every tx queue. 1306 */ 1307 static uint16_t 1308 get_allowed_min_nb_txd(portid_t *pid) 1309 { 1310 uint16_t allowed_min_txd = 0; 1311 portid_t pi; 1312 struct rte_eth_dev_info dev_info; 1313 1314 RTE_ETH_FOREACH_DEV(pi) { 1315 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1316 continue; 1317 1318 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1319 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1320 *pid = pi; 1321 } 1322 } 1323 1324 return allowed_min_txd; 1325 } 1326 1327 /* 1328 * Check input txd is valid or not. 1329 * If input txd is not greater than any of maximum number 1330 * of TXDs of every Rx queues, it is valid. 1331 * if valid, return 0, else return -1 1332 */ 1333 int 1334 check_nb_txd(queueid_t txd) 1335 { 1336 uint16_t allowed_max_txd; 1337 uint16_t allowed_min_txd; 1338 portid_t pid = 0; 1339 1340 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1341 if (txd > allowed_max_txd) { 1342 printf("Fail: input txd (%u) can't be greater " 1343 "than max_txds (%u) of port %u\n", 1344 txd, 1345 allowed_max_txd, 1346 pid); 1347 return -1; 1348 } 1349 1350 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1351 if (txd < allowed_min_txd) { 1352 printf("Fail: input txd (%u) can't be less " 1353 "than min_txds (%u) of port %u\n", 1354 txd, 1355 allowed_min_txd, 1356 pid); 1357 return -1; 1358 } 1359 return 0; 1360 } 1361 1362 1363 /* 1364 * Get the allowed maximum number of hairpin queues. 1365 * *pid return the port id which has minimal value of 1366 * max_hairpin_queues in all ports. 1367 */ 1368 queueid_t 1369 get_allowed_max_nb_hairpinq(portid_t *pid) 1370 { 1371 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1372 portid_t pi; 1373 struct rte_eth_hairpin_cap cap; 1374 1375 RTE_ETH_FOREACH_DEV(pi) { 1376 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1377 *pid = pi; 1378 return 0; 1379 } 1380 if (cap.max_nb_queues < allowed_max_hairpinq) { 1381 allowed_max_hairpinq = cap.max_nb_queues; 1382 *pid = pi; 1383 } 1384 } 1385 return allowed_max_hairpinq; 1386 } 1387 1388 /* 1389 * Check input hairpin is valid or not. 1390 * If input hairpin is not greater than any of maximum number 1391 * of hairpin queues of all ports, it is valid. 1392 * if valid, return 0, else return -1 1393 */ 1394 int 1395 check_nb_hairpinq(queueid_t hairpinq) 1396 { 1397 queueid_t allowed_max_hairpinq; 1398 portid_t pid = 0; 1399 1400 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1401 if (hairpinq > allowed_max_hairpinq) { 1402 printf("Fail: input hairpin (%u) can't be greater " 1403 "than max_hairpin_queues (%u) of port %u\n", 1404 hairpinq, allowed_max_hairpinq, pid); 1405 return -1; 1406 } 1407 return 0; 1408 } 1409 1410 static void 1411 init_config(void) 1412 { 1413 portid_t pid; 1414 struct rte_port *port; 1415 struct rte_mempool *mbp; 1416 unsigned int nb_mbuf_per_pool; 1417 lcoreid_t lc_id; 1418 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1419 struct rte_gro_param gro_param; 1420 uint32_t gso_types; 1421 uint16_t data_size; 1422 bool warning = 0; 1423 int k; 1424 int ret; 1425 1426 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1427 1428 /* Configuration of logical cores. */ 1429 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1430 sizeof(struct fwd_lcore *) * nb_lcores, 1431 RTE_CACHE_LINE_SIZE); 1432 if (fwd_lcores == NULL) { 1433 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1434 "failed\n", nb_lcores); 1435 } 1436 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1437 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1438 sizeof(struct fwd_lcore), 1439 RTE_CACHE_LINE_SIZE); 1440 if (fwd_lcores[lc_id] == NULL) { 1441 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1442 "failed\n"); 1443 } 1444 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1445 } 1446 1447 RTE_ETH_FOREACH_DEV(pid) { 1448 port = &ports[pid]; 1449 /* Apply default TxRx configuration for all ports */ 1450 port->dev_conf.txmode = tx_mode; 1451 port->dev_conf.rxmode = rx_mode; 1452 1453 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1454 if (ret != 0) 1455 rte_exit(EXIT_FAILURE, 1456 "rte_eth_dev_info_get() failed\n"); 1457 1458 if (!(port->dev_info.tx_offload_capa & 1459 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1460 port->dev_conf.txmode.offloads &= 1461 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1462 if (numa_support) { 1463 if (port_numa[pid] != NUMA_NO_CONFIG) 1464 port_per_socket[port_numa[pid]]++; 1465 else { 1466 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1467 1468 /* 1469 * if socket_id is invalid, 1470 * set to the first available socket. 1471 */ 1472 if (check_socket_id(socket_id) < 0) 1473 socket_id = socket_ids[0]; 1474 port_per_socket[socket_id]++; 1475 } 1476 } 1477 1478 /* Apply Rx offloads configuration */ 1479 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1480 port->rx_conf[k].offloads = 1481 port->dev_conf.rxmode.offloads; 1482 /* Apply Tx offloads configuration */ 1483 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1484 port->tx_conf[k].offloads = 1485 port->dev_conf.txmode.offloads; 1486 1487 /* set flag to initialize port/queue */ 1488 port->need_reconfig = 1; 1489 port->need_reconfig_queues = 1; 1490 port->tx_metadata = 0; 1491 1492 /* Check for maximum number of segments per MTU. Accordingly 1493 * update the mbuf data size. 1494 */ 1495 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1496 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1497 data_size = rx_mode.max_rx_pkt_len / 1498 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1499 1500 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1501 mbuf_data_size[0]) { 1502 mbuf_data_size[0] = data_size + 1503 RTE_PKTMBUF_HEADROOM; 1504 warning = 1; 1505 } 1506 } 1507 } 1508 1509 if (warning) 1510 TESTPMD_LOG(WARNING, 1511 "Configured mbuf size of the first segment %hu\n", 1512 mbuf_data_size[0]); 1513 /* 1514 * Create pools of mbuf. 1515 * If NUMA support is disabled, create a single pool of mbuf in 1516 * socket 0 memory by default. 1517 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1518 * 1519 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1520 * nb_txd can be configured at run time. 1521 */ 1522 if (param_total_num_mbufs) 1523 nb_mbuf_per_pool = param_total_num_mbufs; 1524 else { 1525 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1526 (nb_lcores * mb_mempool_cache) + 1527 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1528 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1529 } 1530 1531 if (numa_support) { 1532 uint8_t i, j; 1533 1534 for (i = 0; i < num_sockets; i++) 1535 for (j = 0; j < mbuf_data_size_n; j++) 1536 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1537 mbuf_pool_create(mbuf_data_size[j], 1538 nb_mbuf_per_pool, 1539 socket_ids[i], j); 1540 } else { 1541 uint8_t i; 1542 1543 for (i = 0; i < mbuf_data_size_n; i++) 1544 mempools[i] = mbuf_pool_create 1545 (mbuf_data_size[i], 1546 nb_mbuf_per_pool, 1547 socket_num == UMA_NO_CONFIG ? 1548 0 : socket_num, i); 1549 } 1550 1551 init_port_config(); 1552 1553 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1554 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1555 /* 1556 * Records which Mbuf pool to use by each logical core, if needed. 1557 */ 1558 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1559 mbp = mbuf_pool_find( 1560 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1561 1562 if (mbp == NULL) 1563 mbp = mbuf_pool_find(0, 0); 1564 fwd_lcores[lc_id]->mbp = mbp; 1565 /* initialize GSO context */ 1566 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1567 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1568 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1569 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1570 RTE_ETHER_CRC_LEN; 1571 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1572 } 1573 1574 /* Configuration of packet forwarding streams. */ 1575 if (init_fwd_streams() < 0) 1576 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1577 1578 fwd_config_setup(); 1579 1580 /* create a gro context for each lcore */ 1581 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1582 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1583 gro_param.max_item_per_flow = MAX_PKT_BURST; 1584 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1585 gro_param.socket_id = rte_lcore_to_socket_id( 1586 fwd_lcores_cpuids[lc_id]); 1587 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1588 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1589 rte_exit(EXIT_FAILURE, 1590 "rte_gro_ctx_create() failed\n"); 1591 } 1592 } 1593 } 1594 1595 1596 void 1597 reconfig(portid_t new_port_id, unsigned socket_id) 1598 { 1599 struct rte_port *port; 1600 int ret; 1601 1602 /* Reconfiguration of Ethernet ports. */ 1603 port = &ports[new_port_id]; 1604 1605 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1606 if (ret != 0) 1607 return; 1608 1609 /* set flag to initialize port/queue */ 1610 port->need_reconfig = 1; 1611 port->need_reconfig_queues = 1; 1612 port->socket_id = socket_id; 1613 1614 init_port_config(); 1615 } 1616 1617 1618 int 1619 init_fwd_streams(void) 1620 { 1621 portid_t pid; 1622 struct rte_port *port; 1623 streamid_t sm_id, nb_fwd_streams_new; 1624 queueid_t q; 1625 1626 /* set socket id according to numa or not */ 1627 RTE_ETH_FOREACH_DEV(pid) { 1628 port = &ports[pid]; 1629 if (nb_rxq > port->dev_info.max_rx_queues) { 1630 printf("Fail: nb_rxq(%d) is greater than " 1631 "max_rx_queues(%d)\n", nb_rxq, 1632 port->dev_info.max_rx_queues); 1633 return -1; 1634 } 1635 if (nb_txq > port->dev_info.max_tx_queues) { 1636 printf("Fail: nb_txq(%d) is greater than " 1637 "max_tx_queues(%d)\n", nb_txq, 1638 port->dev_info.max_tx_queues); 1639 return -1; 1640 } 1641 if (numa_support) { 1642 if (port_numa[pid] != NUMA_NO_CONFIG) 1643 port->socket_id = port_numa[pid]; 1644 else { 1645 port->socket_id = rte_eth_dev_socket_id(pid); 1646 1647 /* 1648 * if socket_id is invalid, 1649 * set to the first available socket. 1650 */ 1651 if (check_socket_id(port->socket_id) < 0) 1652 port->socket_id = socket_ids[0]; 1653 } 1654 } 1655 else { 1656 if (socket_num == UMA_NO_CONFIG) 1657 port->socket_id = 0; 1658 else 1659 port->socket_id = socket_num; 1660 } 1661 } 1662 1663 q = RTE_MAX(nb_rxq, nb_txq); 1664 if (q == 0) { 1665 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1666 return -1; 1667 } 1668 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1669 if (nb_fwd_streams_new == nb_fwd_streams) 1670 return 0; 1671 /* clear the old */ 1672 if (fwd_streams != NULL) { 1673 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1674 if (fwd_streams[sm_id] == NULL) 1675 continue; 1676 rte_free(fwd_streams[sm_id]); 1677 fwd_streams[sm_id] = NULL; 1678 } 1679 rte_free(fwd_streams); 1680 fwd_streams = NULL; 1681 } 1682 1683 /* init new */ 1684 nb_fwd_streams = nb_fwd_streams_new; 1685 if (nb_fwd_streams) { 1686 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1687 sizeof(struct fwd_stream *) * nb_fwd_streams, 1688 RTE_CACHE_LINE_SIZE); 1689 if (fwd_streams == NULL) 1690 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1691 " (struct fwd_stream *)) failed\n", 1692 nb_fwd_streams); 1693 1694 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1695 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1696 " struct fwd_stream", sizeof(struct fwd_stream), 1697 RTE_CACHE_LINE_SIZE); 1698 if (fwd_streams[sm_id] == NULL) 1699 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1700 "(struct fwd_stream) failed\n"); 1701 } 1702 } 1703 1704 return 0; 1705 } 1706 1707 static void 1708 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1709 { 1710 uint64_t total_burst, sburst; 1711 uint64_t nb_burst; 1712 uint64_t burst_stats[4]; 1713 uint16_t pktnb_stats[4]; 1714 uint16_t nb_pkt; 1715 int burst_percent[4], sburstp; 1716 int i; 1717 1718 /* 1719 * First compute the total number of packet bursts and the 1720 * two highest numbers of bursts of the same number of packets. 1721 */ 1722 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1723 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1724 1725 /* Show stats for 0 burst size always */ 1726 total_burst = pbs->pkt_burst_spread[0]; 1727 burst_stats[0] = pbs->pkt_burst_spread[0]; 1728 pktnb_stats[0] = 0; 1729 1730 /* Find the next 2 burst sizes with highest occurrences. */ 1731 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1732 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1733 1734 if (nb_burst == 0) 1735 continue; 1736 1737 total_burst += nb_burst; 1738 1739 if (nb_burst > burst_stats[1]) { 1740 burst_stats[2] = burst_stats[1]; 1741 pktnb_stats[2] = pktnb_stats[1]; 1742 burst_stats[1] = nb_burst; 1743 pktnb_stats[1] = nb_pkt; 1744 } else if (nb_burst > burst_stats[2]) { 1745 burst_stats[2] = nb_burst; 1746 pktnb_stats[2] = nb_pkt; 1747 } 1748 } 1749 if (total_burst == 0) 1750 return; 1751 1752 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1753 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1754 if (i == 3) { 1755 printf("%d%% of other]\n", 100 - sburstp); 1756 return; 1757 } 1758 1759 sburst += burst_stats[i]; 1760 if (sburst == total_burst) { 1761 printf("%d%% of %d pkts]\n", 1762 100 - sburstp, (int) pktnb_stats[i]); 1763 return; 1764 } 1765 1766 burst_percent[i] = 1767 (double)burst_stats[i] / total_burst * 100; 1768 printf("%d%% of %d pkts + ", 1769 burst_percent[i], (int) pktnb_stats[i]); 1770 sburstp += burst_percent[i]; 1771 } 1772 } 1773 1774 static void 1775 fwd_stream_stats_display(streamid_t stream_id) 1776 { 1777 struct fwd_stream *fs; 1778 static const char *fwd_top_stats_border = "-------"; 1779 1780 fs = fwd_streams[stream_id]; 1781 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1782 (fs->fwd_dropped == 0)) 1783 return; 1784 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1785 "TX Port=%2d/Queue=%2d %s\n", 1786 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1787 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1788 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1789 " TX-dropped: %-14"PRIu64, 1790 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1791 1792 /* if checksum mode */ 1793 if (cur_fwd_eng == &csum_fwd_engine) { 1794 printf(" RX- bad IP checksum: %-14"PRIu64 1795 " Rx- bad L4 checksum: %-14"PRIu64 1796 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1797 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1798 fs->rx_bad_outer_l4_csum); 1799 } else { 1800 printf("\n"); 1801 } 1802 1803 if (record_burst_stats) { 1804 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1805 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1806 } 1807 } 1808 1809 void 1810 fwd_stats_display(void) 1811 { 1812 static const char *fwd_stats_border = "----------------------"; 1813 static const char *acc_stats_border = "+++++++++++++++"; 1814 struct { 1815 struct fwd_stream *rx_stream; 1816 struct fwd_stream *tx_stream; 1817 uint64_t tx_dropped; 1818 uint64_t rx_bad_ip_csum; 1819 uint64_t rx_bad_l4_csum; 1820 uint64_t rx_bad_outer_l4_csum; 1821 } ports_stats[RTE_MAX_ETHPORTS]; 1822 uint64_t total_rx_dropped = 0; 1823 uint64_t total_tx_dropped = 0; 1824 uint64_t total_rx_nombuf = 0; 1825 struct rte_eth_stats stats; 1826 uint64_t fwd_cycles = 0; 1827 uint64_t total_recv = 0; 1828 uint64_t total_xmit = 0; 1829 struct rte_port *port; 1830 streamid_t sm_id; 1831 portid_t pt_id; 1832 int i; 1833 1834 memset(ports_stats, 0, sizeof(ports_stats)); 1835 1836 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1837 struct fwd_stream *fs = fwd_streams[sm_id]; 1838 1839 if (cur_fwd_config.nb_fwd_streams > 1840 cur_fwd_config.nb_fwd_ports) { 1841 fwd_stream_stats_display(sm_id); 1842 } else { 1843 ports_stats[fs->tx_port].tx_stream = fs; 1844 ports_stats[fs->rx_port].rx_stream = fs; 1845 } 1846 1847 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1848 1849 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1850 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1851 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1852 fs->rx_bad_outer_l4_csum; 1853 1854 if (record_core_cycles) 1855 fwd_cycles += fs->core_cycles; 1856 } 1857 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1858 uint8_t j; 1859 1860 pt_id = fwd_ports_ids[i]; 1861 port = &ports[pt_id]; 1862 1863 rte_eth_stats_get(pt_id, &stats); 1864 stats.ipackets -= port->stats.ipackets; 1865 stats.opackets -= port->stats.opackets; 1866 stats.ibytes -= port->stats.ibytes; 1867 stats.obytes -= port->stats.obytes; 1868 stats.imissed -= port->stats.imissed; 1869 stats.oerrors -= port->stats.oerrors; 1870 stats.rx_nombuf -= port->stats.rx_nombuf; 1871 1872 total_recv += stats.ipackets; 1873 total_xmit += stats.opackets; 1874 total_rx_dropped += stats.imissed; 1875 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1876 total_tx_dropped += stats.oerrors; 1877 total_rx_nombuf += stats.rx_nombuf; 1878 1879 printf("\n %s Forward statistics for port %-2d %s\n", 1880 fwd_stats_border, pt_id, fwd_stats_border); 1881 1882 if (!port->rx_queue_stats_mapping_enabled && 1883 !port->tx_queue_stats_mapping_enabled) { 1884 printf(" RX-packets: %-14"PRIu64 1885 " RX-dropped: %-14"PRIu64 1886 "RX-total: %-"PRIu64"\n", 1887 stats.ipackets, stats.imissed, 1888 stats.ipackets + stats.imissed); 1889 1890 if (cur_fwd_eng == &csum_fwd_engine) 1891 printf(" Bad-ipcsum: %-14"PRIu64 1892 " Bad-l4csum: %-14"PRIu64 1893 "Bad-outer-l4csum: %-14"PRIu64"\n", 1894 ports_stats[pt_id].rx_bad_ip_csum, 1895 ports_stats[pt_id].rx_bad_l4_csum, 1896 ports_stats[pt_id].rx_bad_outer_l4_csum); 1897 if (stats.ierrors + stats.rx_nombuf > 0) { 1898 printf(" RX-error: %-"PRIu64"\n", 1899 stats.ierrors); 1900 printf(" RX-nombufs: %-14"PRIu64"\n", 1901 stats.rx_nombuf); 1902 } 1903 1904 printf(" TX-packets: %-14"PRIu64 1905 " TX-dropped: %-14"PRIu64 1906 "TX-total: %-"PRIu64"\n", 1907 stats.opackets, ports_stats[pt_id].tx_dropped, 1908 stats.opackets + ports_stats[pt_id].tx_dropped); 1909 } else { 1910 printf(" RX-packets: %14"PRIu64 1911 " RX-dropped:%14"PRIu64 1912 " RX-total:%14"PRIu64"\n", 1913 stats.ipackets, stats.imissed, 1914 stats.ipackets + stats.imissed); 1915 1916 if (cur_fwd_eng == &csum_fwd_engine) 1917 printf(" Bad-ipcsum:%14"PRIu64 1918 " Bad-l4csum:%14"PRIu64 1919 " Bad-outer-l4csum: %-14"PRIu64"\n", 1920 ports_stats[pt_id].rx_bad_ip_csum, 1921 ports_stats[pt_id].rx_bad_l4_csum, 1922 ports_stats[pt_id].rx_bad_outer_l4_csum); 1923 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1924 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1925 printf(" RX-nombufs: %14"PRIu64"\n", 1926 stats.rx_nombuf); 1927 } 1928 1929 printf(" TX-packets: %14"PRIu64 1930 " TX-dropped:%14"PRIu64 1931 " TX-total:%14"PRIu64"\n", 1932 stats.opackets, ports_stats[pt_id].tx_dropped, 1933 stats.opackets + ports_stats[pt_id].tx_dropped); 1934 } 1935 1936 if (record_burst_stats) { 1937 if (ports_stats[pt_id].rx_stream) 1938 pkt_burst_stats_display("RX", 1939 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1940 if (ports_stats[pt_id].tx_stream) 1941 pkt_burst_stats_display("TX", 1942 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1943 } 1944 1945 if (port->rx_queue_stats_mapping_enabled) { 1946 printf("\n"); 1947 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1948 printf(" Stats reg %2d RX-packets:%14"PRIu64 1949 " RX-errors:%14"PRIu64 1950 " RX-bytes:%14"PRIu64"\n", 1951 j, stats.q_ipackets[j], 1952 stats.q_errors[j], stats.q_ibytes[j]); 1953 } 1954 printf("\n"); 1955 } 1956 if (port->tx_queue_stats_mapping_enabled) { 1957 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1958 printf(" Stats reg %2d TX-packets:%14"PRIu64 1959 " TX-bytes:%14" 1960 PRIu64"\n", 1961 j, stats.q_opackets[j], 1962 stats.q_obytes[j]); 1963 } 1964 } 1965 1966 printf(" %s--------------------------------%s\n", 1967 fwd_stats_border, fwd_stats_border); 1968 } 1969 1970 printf("\n %s Accumulated forward statistics for all ports" 1971 "%s\n", 1972 acc_stats_border, acc_stats_border); 1973 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1974 "%-"PRIu64"\n" 1975 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1976 "%-"PRIu64"\n", 1977 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1978 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1979 if (total_rx_nombuf > 0) 1980 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1981 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1982 "%s\n", 1983 acc_stats_border, acc_stats_border); 1984 if (record_core_cycles) { 1985 #define CYC_PER_MHZ 1E6 1986 if (total_recv > 0 || total_xmit > 0) { 1987 uint64_t total_pkts = 0; 1988 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1989 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1990 total_pkts = total_xmit; 1991 else 1992 total_pkts = total_recv; 1993 1994 printf("\n CPU cycles/packet=%.2F (total cycles=" 1995 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1996 " MHz Clock\n", 1997 (double) fwd_cycles / total_pkts, 1998 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1999 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2000 } 2001 } 2002 } 2003 2004 void 2005 fwd_stats_reset(void) 2006 { 2007 streamid_t sm_id; 2008 portid_t pt_id; 2009 int i; 2010 2011 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2012 pt_id = fwd_ports_ids[i]; 2013 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2014 } 2015 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2016 struct fwd_stream *fs = fwd_streams[sm_id]; 2017 2018 fs->rx_packets = 0; 2019 fs->tx_packets = 0; 2020 fs->fwd_dropped = 0; 2021 fs->rx_bad_ip_csum = 0; 2022 fs->rx_bad_l4_csum = 0; 2023 fs->rx_bad_outer_l4_csum = 0; 2024 2025 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2026 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2027 fs->core_cycles = 0; 2028 } 2029 } 2030 2031 static void 2032 flush_fwd_rx_queues(void) 2033 { 2034 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2035 portid_t rxp; 2036 portid_t port_id; 2037 queueid_t rxq; 2038 uint16_t nb_rx; 2039 uint16_t i; 2040 uint8_t j; 2041 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2042 uint64_t timer_period; 2043 2044 /* convert to number of cycles */ 2045 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2046 2047 for (j = 0; j < 2; j++) { 2048 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2049 for (rxq = 0; rxq < nb_rxq; rxq++) { 2050 port_id = fwd_ports_ids[rxp]; 2051 /** 2052 * testpmd can stuck in the below do while loop 2053 * if rte_eth_rx_burst() always returns nonzero 2054 * packets. So timer is added to exit this loop 2055 * after 1sec timer expiry. 2056 */ 2057 prev_tsc = rte_rdtsc(); 2058 do { 2059 nb_rx = rte_eth_rx_burst(port_id, rxq, 2060 pkts_burst, MAX_PKT_BURST); 2061 for (i = 0; i < nb_rx; i++) 2062 rte_pktmbuf_free(pkts_burst[i]); 2063 2064 cur_tsc = rte_rdtsc(); 2065 diff_tsc = cur_tsc - prev_tsc; 2066 timer_tsc += diff_tsc; 2067 } while ((nb_rx > 0) && 2068 (timer_tsc < timer_period)); 2069 timer_tsc = 0; 2070 } 2071 } 2072 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2073 } 2074 } 2075 2076 static void 2077 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2078 { 2079 struct fwd_stream **fsm; 2080 streamid_t nb_fs; 2081 streamid_t sm_id; 2082 #ifdef RTE_LIBRTE_BITRATESTATS 2083 uint64_t tics_per_1sec; 2084 uint64_t tics_datum; 2085 uint64_t tics_current; 2086 uint16_t i, cnt_ports; 2087 2088 cnt_ports = nb_ports; 2089 tics_datum = rte_rdtsc(); 2090 tics_per_1sec = rte_get_timer_hz(); 2091 #endif 2092 fsm = &fwd_streams[fc->stream_idx]; 2093 nb_fs = fc->stream_nb; 2094 do { 2095 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2096 (*pkt_fwd)(fsm[sm_id]); 2097 #ifdef RTE_LIBRTE_BITRATESTATS 2098 if (bitrate_enabled != 0 && 2099 bitrate_lcore_id == rte_lcore_id()) { 2100 tics_current = rte_rdtsc(); 2101 if (tics_current - tics_datum >= tics_per_1sec) { 2102 /* Periodic bitrate calculation */ 2103 for (i = 0; i < cnt_ports; i++) 2104 rte_stats_bitrate_calc(bitrate_data, 2105 ports_ids[i]); 2106 tics_datum = tics_current; 2107 } 2108 } 2109 #endif 2110 #ifdef RTE_LIBRTE_LATENCY_STATS 2111 if (latencystats_enabled != 0 && 2112 latencystats_lcore_id == rte_lcore_id()) 2113 rte_latencystats_update(); 2114 #endif 2115 2116 } while (! fc->stopped); 2117 } 2118 2119 static int 2120 start_pkt_forward_on_core(void *fwd_arg) 2121 { 2122 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2123 cur_fwd_config.fwd_eng->packet_fwd); 2124 return 0; 2125 } 2126 2127 /* 2128 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2129 * Used to start communication flows in network loopback test configurations. 2130 */ 2131 static int 2132 run_one_txonly_burst_on_core(void *fwd_arg) 2133 { 2134 struct fwd_lcore *fwd_lc; 2135 struct fwd_lcore tmp_lcore; 2136 2137 fwd_lc = (struct fwd_lcore *) fwd_arg; 2138 tmp_lcore = *fwd_lc; 2139 tmp_lcore.stopped = 1; 2140 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2141 return 0; 2142 } 2143 2144 /* 2145 * Launch packet forwarding: 2146 * - Setup per-port forwarding context. 2147 * - launch logical cores with their forwarding configuration. 2148 */ 2149 static void 2150 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2151 { 2152 port_fwd_begin_t port_fwd_begin; 2153 unsigned int i; 2154 unsigned int lc_id; 2155 int diag; 2156 2157 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2158 if (port_fwd_begin != NULL) { 2159 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2160 (*port_fwd_begin)(fwd_ports_ids[i]); 2161 } 2162 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2163 lc_id = fwd_lcores_cpuids[i]; 2164 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2165 fwd_lcores[i]->stopped = 0; 2166 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2167 fwd_lcores[i], lc_id); 2168 if (diag != 0) 2169 printf("launch lcore %u failed - diag=%d\n", 2170 lc_id, diag); 2171 } 2172 } 2173 } 2174 2175 /* 2176 * Launch packet forwarding configuration. 2177 */ 2178 void 2179 start_packet_forwarding(int with_tx_first) 2180 { 2181 port_fwd_begin_t port_fwd_begin; 2182 port_fwd_end_t port_fwd_end; 2183 struct rte_port *port; 2184 unsigned int i; 2185 portid_t pt_id; 2186 2187 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2188 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2189 2190 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2191 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2192 2193 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2194 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2195 (!nb_rxq || !nb_txq)) 2196 rte_exit(EXIT_FAILURE, 2197 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2198 cur_fwd_eng->fwd_mode_name); 2199 2200 if (all_ports_started() == 0) { 2201 printf("Not all ports were started\n"); 2202 return; 2203 } 2204 if (test_done == 0) { 2205 printf("Packet forwarding already started\n"); 2206 return; 2207 } 2208 2209 2210 if(dcb_test) { 2211 for (i = 0; i < nb_fwd_ports; i++) { 2212 pt_id = fwd_ports_ids[i]; 2213 port = &ports[pt_id]; 2214 if (!port->dcb_flag) { 2215 printf("In DCB mode, all forwarding ports must " 2216 "be configured in this mode.\n"); 2217 return; 2218 } 2219 } 2220 if (nb_fwd_lcores == 1) { 2221 printf("In DCB mode,the nb forwarding cores " 2222 "should be larger than 1.\n"); 2223 return; 2224 } 2225 } 2226 test_done = 0; 2227 2228 fwd_config_setup(); 2229 2230 if(!no_flush_rx) 2231 flush_fwd_rx_queues(); 2232 2233 pkt_fwd_config_display(&cur_fwd_config); 2234 rxtx_config_display(); 2235 2236 fwd_stats_reset(); 2237 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2238 pt_id = fwd_ports_ids[i]; 2239 port = &ports[pt_id]; 2240 map_port_queue_stats_mapping_registers(pt_id, port); 2241 } 2242 if (with_tx_first) { 2243 port_fwd_begin = tx_only_engine.port_fwd_begin; 2244 if (port_fwd_begin != NULL) { 2245 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2246 (*port_fwd_begin)(fwd_ports_ids[i]); 2247 } 2248 while (with_tx_first--) { 2249 launch_packet_forwarding( 2250 run_one_txonly_burst_on_core); 2251 rte_eal_mp_wait_lcore(); 2252 } 2253 port_fwd_end = tx_only_engine.port_fwd_end; 2254 if (port_fwd_end != NULL) { 2255 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2256 (*port_fwd_end)(fwd_ports_ids[i]); 2257 } 2258 } 2259 launch_packet_forwarding(start_pkt_forward_on_core); 2260 } 2261 2262 void 2263 stop_packet_forwarding(void) 2264 { 2265 port_fwd_end_t port_fwd_end; 2266 lcoreid_t lc_id; 2267 portid_t pt_id; 2268 int i; 2269 2270 if (test_done) { 2271 printf("Packet forwarding not started\n"); 2272 return; 2273 } 2274 printf("Telling cores to stop..."); 2275 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2276 fwd_lcores[lc_id]->stopped = 1; 2277 printf("\nWaiting for lcores to finish...\n"); 2278 rte_eal_mp_wait_lcore(); 2279 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2280 if (port_fwd_end != NULL) { 2281 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2282 pt_id = fwd_ports_ids[i]; 2283 (*port_fwd_end)(pt_id); 2284 } 2285 } 2286 2287 fwd_stats_display(); 2288 2289 printf("\nDone.\n"); 2290 test_done = 1; 2291 } 2292 2293 void 2294 dev_set_link_up(portid_t pid) 2295 { 2296 if (rte_eth_dev_set_link_up(pid) < 0) 2297 printf("\nSet link up fail.\n"); 2298 } 2299 2300 void 2301 dev_set_link_down(portid_t pid) 2302 { 2303 if (rte_eth_dev_set_link_down(pid) < 0) 2304 printf("\nSet link down fail.\n"); 2305 } 2306 2307 static int 2308 all_ports_started(void) 2309 { 2310 portid_t pi; 2311 struct rte_port *port; 2312 2313 RTE_ETH_FOREACH_DEV(pi) { 2314 port = &ports[pi]; 2315 /* Check if there is a port which is not started */ 2316 if ((port->port_status != RTE_PORT_STARTED) && 2317 (port->slave_flag == 0)) 2318 return 0; 2319 } 2320 2321 /* No port is not started */ 2322 return 1; 2323 } 2324 2325 int 2326 port_is_stopped(portid_t port_id) 2327 { 2328 struct rte_port *port = &ports[port_id]; 2329 2330 if ((port->port_status != RTE_PORT_STOPPED) && 2331 (port->slave_flag == 0)) 2332 return 0; 2333 return 1; 2334 } 2335 2336 int 2337 all_ports_stopped(void) 2338 { 2339 portid_t pi; 2340 2341 RTE_ETH_FOREACH_DEV(pi) { 2342 if (!port_is_stopped(pi)) 2343 return 0; 2344 } 2345 2346 return 1; 2347 } 2348 2349 int 2350 port_is_started(portid_t port_id) 2351 { 2352 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2353 return 0; 2354 2355 if (ports[port_id].port_status != RTE_PORT_STARTED) 2356 return 0; 2357 2358 return 1; 2359 } 2360 2361 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2362 static int 2363 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2364 { 2365 queueid_t qi; 2366 struct rte_eth_hairpin_conf hairpin_conf = { 2367 .peer_count = 1, 2368 }; 2369 int i; 2370 int diag; 2371 struct rte_port *port = &ports[pi]; 2372 uint16_t peer_rx_port = pi; 2373 uint16_t peer_tx_port = pi; 2374 uint32_t manual = 1; 2375 uint32_t tx_exp = hairpin_mode & 0x10; 2376 2377 if (!(hairpin_mode & 0xf)) { 2378 peer_rx_port = pi; 2379 peer_tx_port = pi; 2380 manual = 0; 2381 } else if (hairpin_mode & 0x1) { 2382 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2383 RTE_ETH_DEV_NO_OWNER); 2384 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2385 peer_tx_port = rte_eth_find_next_owned_by(0, 2386 RTE_ETH_DEV_NO_OWNER); 2387 if (p_pi != RTE_MAX_ETHPORTS) { 2388 peer_rx_port = p_pi; 2389 } else { 2390 uint16_t next_pi; 2391 2392 /* Last port will be the peer RX port of the first. */ 2393 RTE_ETH_FOREACH_DEV(next_pi) 2394 peer_rx_port = next_pi; 2395 } 2396 manual = 1; 2397 } else if (hairpin_mode & 0x2) { 2398 if (cnt_pi & 0x1) { 2399 peer_rx_port = p_pi; 2400 } else { 2401 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2402 RTE_ETH_DEV_NO_OWNER); 2403 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2404 peer_rx_port = pi; 2405 } 2406 peer_tx_port = peer_rx_port; 2407 manual = 1; 2408 } 2409 2410 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2411 hairpin_conf.peers[0].port = peer_rx_port; 2412 hairpin_conf.peers[0].queue = i + nb_rxq; 2413 hairpin_conf.manual_bind = !!manual; 2414 hairpin_conf.tx_explicit = !!tx_exp; 2415 diag = rte_eth_tx_hairpin_queue_setup 2416 (pi, qi, nb_txd, &hairpin_conf); 2417 i++; 2418 if (diag == 0) 2419 continue; 2420 2421 /* Fail to setup rx queue, return */ 2422 if (rte_atomic16_cmpset(&(port->port_status), 2423 RTE_PORT_HANDLING, 2424 RTE_PORT_STOPPED) == 0) 2425 printf("Port %d can not be set back " 2426 "to stopped\n", pi); 2427 printf("Fail to configure port %d hairpin " 2428 "queues\n", pi); 2429 /* try to reconfigure queues next time */ 2430 port->need_reconfig_queues = 1; 2431 return -1; 2432 } 2433 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2434 hairpin_conf.peers[0].port = peer_tx_port; 2435 hairpin_conf.peers[0].queue = i + nb_txq; 2436 hairpin_conf.manual_bind = !!manual; 2437 hairpin_conf.tx_explicit = !!tx_exp; 2438 diag = rte_eth_rx_hairpin_queue_setup 2439 (pi, qi, nb_rxd, &hairpin_conf); 2440 i++; 2441 if (diag == 0) 2442 continue; 2443 2444 /* Fail to setup rx queue, return */ 2445 if (rte_atomic16_cmpset(&(port->port_status), 2446 RTE_PORT_HANDLING, 2447 RTE_PORT_STOPPED) == 0) 2448 printf("Port %d can not be set back " 2449 "to stopped\n", pi); 2450 printf("Fail to configure port %d hairpin " 2451 "queues\n", pi); 2452 /* try to reconfigure queues next time */ 2453 port->need_reconfig_queues = 1; 2454 return -1; 2455 } 2456 return 0; 2457 } 2458 2459 int 2460 start_port(portid_t pid) 2461 { 2462 int diag, need_check_link_status = -1; 2463 portid_t pi; 2464 portid_t p_pi = RTE_MAX_ETHPORTS; 2465 portid_t pl[RTE_MAX_ETHPORTS]; 2466 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2467 uint16_t cnt_pi = 0; 2468 uint16_t cfg_pi = 0; 2469 int peer_pi; 2470 queueid_t qi; 2471 struct rte_port *port; 2472 struct rte_ether_addr mac_addr; 2473 struct rte_eth_hairpin_cap cap; 2474 2475 if (port_id_is_invalid(pid, ENABLED_WARN)) 2476 return 0; 2477 2478 if(dcb_config) 2479 dcb_test = 1; 2480 RTE_ETH_FOREACH_DEV(pi) { 2481 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2482 continue; 2483 2484 need_check_link_status = 0; 2485 port = &ports[pi]; 2486 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2487 RTE_PORT_HANDLING) == 0) { 2488 printf("Port %d is now not stopped\n", pi); 2489 continue; 2490 } 2491 2492 if (port->need_reconfig > 0) { 2493 port->need_reconfig = 0; 2494 2495 if (flow_isolate_all) { 2496 int ret = port_flow_isolate(pi, 1); 2497 if (ret) { 2498 printf("Failed to apply isolated" 2499 " mode on port %d\n", pi); 2500 return -1; 2501 } 2502 } 2503 configure_rxtx_dump_callbacks(0); 2504 printf("Configuring Port %d (socket %u)\n", pi, 2505 port->socket_id); 2506 if (nb_hairpinq > 0 && 2507 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2508 printf("Port %d doesn't support hairpin " 2509 "queues\n", pi); 2510 return -1; 2511 } 2512 /* configure port */ 2513 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2514 nb_txq + nb_hairpinq, 2515 &(port->dev_conf)); 2516 if (diag != 0) { 2517 if (rte_atomic16_cmpset(&(port->port_status), 2518 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2519 printf("Port %d can not be set back " 2520 "to stopped\n", pi); 2521 printf("Fail to configure port %d\n", pi); 2522 /* try to reconfigure port next time */ 2523 port->need_reconfig = 1; 2524 return -1; 2525 } 2526 } 2527 if (port->need_reconfig_queues > 0) { 2528 port->need_reconfig_queues = 0; 2529 /* setup tx queues */ 2530 for (qi = 0; qi < nb_txq; qi++) { 2531 if ((numa_support) && 2532 (txring_numa[pi] != NUMA_NO_CONFIG)) 2533 diag = rte_eth_tx_queue_setup(pi, qi, 2534 port->nb_tx_desc[qi], 2535 txring_numa[pi], 2536 &(port->tx_conf[qi])); 2537 else 2538 diag = rte_eth_tx_queue_setup(pi, qi, 2539 port->nb_tx_desc[qi], 2540 port->socket_id, 2541 &(port->tx_conf[qi])); 2542 2543 if (diag == 0) 2544 continue; 2545 2546 /* Fail to setup tx queue, return */ 2547 if (rte_atomic16_cmpset(&(port->port_status), 2548 RTE_PORT_HANDLING, 2549 RTE_PORT_STOPPED) == 0) 2550 printf("Port %d can not be set back " 2551 "to stopped\n", pi); 2552 printf("Fail to configure port %d tx queues\n", 2553 pi); 2554 /* try to reconfigure queues next time */ 2555 port->need_reconfig_queues = 1; 2556 return -1; 2557 } 2558 for (qi = 0; qi < nb_rxq; qi++) { 2559 /* setup rx queues */ 2560 if ((numa_support) && 2561 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2562 struct rte_mempool * mp = 2563 mbuf_pool_find 2564 (rxring_numa[pi], 0); 2565 if (mp == NULL) { 2566 printf("Failed to setup RX queue:" 2567 "No mempool allocation" 2568 " on the socket %d\n", 2569 rxring_numa[pi]); 2570 return -1; 2571 } 2572 2573 diag = rte_eth_rx_queue_setup(pi, qi, 2574 port->nb_rx_desc[qi], 2575 rxring_numa[pi], 2576 &(port->rx_conf[qi]), 2577 mp); 2578 } else { 2579 struct rte_mempool *mp = 2580 mbuf_pool_find 2581 (port->socket_id, 0); 2582 if (mp == NULL) { 2583 printf("Failed to setup RX queue:" 2584 "No mempool allocation" 2585 " on the socket %d\n", 2586 port->socket_id); 2587 return -1; 2588 } 2589 diag = rte_eth_rx_queue_setup(pi, qi, 2590 port->nb_rx_desc[qi], 2591 port->socket_id, 2592 &(port->rx_conf[qi]), 2593 mp); 2594 } 2595 if (diag == 0) 2596 continue; 2597 2598 /* Fail to setup rx queue, return */ 2599 if (rte_atomic16_cmpset(&(port->port_status), 2600 RTE_PORT_HANDLING, 2601 RTE_PORT_STOPPED) == 0) 2602 printf("Port %d can not be set back " 2603 "to stopped\n", pi); 2604 printf("Fail to configure port %d rx queues\n", 2605 pi); 2606 /* try to reconfigure queues next time */ 2607 port->need_reconfig_queues = 1; 2608 return -1; 2609 } 2610 /* setup hairpin queues */ 2611 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2612 return -1; 2613 } 2614 configure_rxtx_dump_callbacks(verbose_level); 2615 if (clear_ptypes) { 2616 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2617 NULL, 0); 2618 if (diag < 0) 2619 printf( 2620 "Port %d: Failed to disable Ptype parsing\n", 2621 pi); 2622 } 2623 2624 p_pi = pi; 2625 cnt_pi++; 2626 2627 /* start port */ 2628 if (rte_eth_dev_start(pi) < 0) { 2629 printf("Fail to start port %d\n", pi); 2630 2631 /* Fail to setup rx queue, return */ 2632 if (rte_atomic16_cmpset(&(port->port_status), 2633 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2634 printf("Port %d can not be set back to " 2635 "stopped\n", pi); 2636 continue; 2637 } 2638 2639 if (rte_atomic16_cmpset(&(port->port_status), 2640 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2641 printf("Port %d can not be set into started\n", pi); 2642 2643 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2644 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2645 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2646 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2647 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2648 2649 /* at least one port started, need checking link status */ 2650 need_check_link_status = 1; 2651 2652 pl[cfg_pi++] = pi; 2653 } 2654 2655 if (need_check_link_status == 1 && !no_link_check) 2656 check_all_ports_link_status(RTE_PORT_ALL); 2657 else if (need_check_link_status == 0) 2658 printf("Please stop the ports first\n"); 2659 2660 if (hairpin_mode & 0xf) { 2661 uint16_t i; 2662 int j; 2663 2664 /* bind all started hairpin ports */ 2665 for (i = 0; i < cfg_pi; i++) { 2666 pi = pl[i]; 2667 /* bind current Tx to all peer Rx */ 2668 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2669 RTE_MAX_ETHPORTS, 1); 2670 if (peer_pi < 0) 2671 return peer_pi; 2672 for (j = 0; j < peer_pi; j++) { 2673 if (!port_is_started(peer_pl[j])) 2674 continue; 2675 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2676 if (diag < 0) { 2677 printf("Error during binding hairpin" 2678 " Tx port %u to %u: %s\n", 2679 pi, peer_pl[j], 2680 rte_strerror(-diag)); 2681 return -1; 2682 } 2683 } 2684 /* bind all peer Tx to current Rx */ 2685 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2686 RTE_MAX_ETHPORTS, 0); 2687 if (peer_pi < 0) 2688 return peer_pi; 2689 for (j = 0; j < peer_pi; j++) { 2690 if (!port_is_started(peer_pl[j])) 2691 continue; 2692 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2693 if (diag < 0) { 2694 printf("Error during binding hairpin" 2695 " Tx port %u to %u: %s\n", 2696 peer_pl[j], pi, 2697 rte_strerror(-diag)); 2698 return -1; 2699 } 2700 } 2701 } 2702 } 2703 2704 printf("Done\n"); 2705 return 0; 2706 } 2707 2708 void 2709 stop_port(portid_t pid) 2710 { 2711 portid_t pi; 2712 struct rte_port *port; 2713 int need_check_link_status = 0; 2714 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2715 int peer_pi; 2716 2717 if (dcb_test) { 2718 dcb_test = 0; 2719 dcb_config = 0; 2720 } 2721 2722 if (port_id_is_invalid(pid, ENABLED_WARN)) 2723 return; 2724 2725 printf("Stopping ports...\n"); 2726 2727 RTE_ETH_FOREACH_DEV(pi) { 2728 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2729 continue; 2730 2731 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2732 printf("Please remove port %d from forwarding configuration.\n", pi); 2733 continue; 2734 } 2735 2736 if (port_is_bonding_slave(pi)) { 2737 printf("Please remove port %d from bonded device.\n", pi); 2738 continue; 2739 } 2740 2741 port = &ports[pi]; 2742 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2743 RTE_PORT_HANDLING) == 0) 2744 continue; 2745 2746 if (hairpin_mode & 0xf) { 2747 int j; 2748 2749 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2750 /* unbind all peer Tx from current Rx */ 2751 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2752 RTE_MAX_ETHPORTS, 0); 2753 if (peer_pi < 0) 2754 continue; 2755 for (j = 0; j < peer_pi; j++) { 2756 if (!port_is_started(peer_pl[j])) 2757 continue; 2758 rte_eth_hairpin_unbind(peer_pl[j], pi); 2759 } 2760 } 2761 2762 rte_eth_dev_stop(pi); 2763 2764 if (rte_atomic16_cmpset(&(port->port_status), 2765 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2766 printf("Port %d can not be set into stopped\n", pi); 2767 need_check_link_status = 1; 2768 } 2769 if (need_check_link_status && !no_link_check) 2770 check_all_ports_link_status(RTE_PORT_ALL); 2771 2772 printf("Done\n"); 2773 } 2774 2775 static void 2776 remove_invalid_ports_in(portid_t *array, portid_t *total) 2777 { 2778 portid_t i; 2779 portid_t new_total = 0; 2780 2781 for (i = 0; i < *total; i++) 2782 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2783 array[new_total] = array[i]; 2784 new_total++; 2785 } 2786 *total = new_total; 2787 } 2788 2789 static void 2790 remove_invalid_ports(void) 2791 { 2792 remove_invalid_ports_in(ports_ids, &nb_ports); 2793 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2794 nb_cfg_ports = nb_fwd_ports; 2795 } 2796 2797 void 2798 close_port(portid_t pid) 2799 { 2800 portid_t pi; 2801 struct rte_port *port; 2802 2803 if (port_id_is_invalid(pid, ENABLED_WARN)) 2804 return; 2805 2806 printf("Closing ports...\n"); 2807 2808 RTE_ETH_FOREACH_DEV(pi) { 2809 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2810 continue; 2811 2812 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2813 printf("Please remove port %d from forwarding configuration.\n", pi); 2814 continue; 2815 } 2816 2817 if (port_is_bonding_slave(pi)) { 2818 printf("Please remove port %d from bonded device.\n", pi); 2819 continue; 2820 } 2821 2822 port = &ports[pi]; 2823 if (rte_atomic16_cmpset(&(port->port_status), 2824 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2825 printf("Port %d is already closed\n", pi); 2826 continue; 2827 } 2828 2829 port_flow_flush(pi); 2830 rte_eth_dev_close(pi); 2831 } 2832 2833 remove_invalid_ports(); 2834 printf("Done\n"); 2835 } 2836 2837 void 2838 reset_port(portid_t pid) 2839 { 2840 int diag; 2841 portid_t pi; 2842 struct rte_port *port; 2843 2844 if (port_id_is_invalid(pid, ENABLED_WARN)) 2845 return; 2846 2847 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2848 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2849 printf("Can not reset port(s), please stop port(s) first.\n"); 2850 return; 2851 } 2852 2853 printf("Resetting ports...\n"); 2854 2855 RTE_ETH_FOREACH_DEV(pi) { 2856 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2857 continue; 2858 2859 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2860 printf("Please remove port %d from forwarding " 2861 "configuration.\n", pi); 2862 continue; 2863 } 2864 2865 if (port_is_bonding_slave(pi)) { 2866 printf("Please remove port %d from bonded device.\n", 2867 pi); 2868 continue; 2869 } 2870 2871 diag = rte_eth_dev_reset(pi); 2872 if (diag == 0) { 2873 port = &ports[pi]; 2874 port->need_reconfig = 1; 2875 port->need_reconfig_queues = 1; 2876 } else { 2877 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2878 } 2879 } 2880 2881 printf("Done\n"); 2882 } 2883 2884 void 2885 attach_port(char *identifier) 2886 { 2887 portid_t pi; 2888 struct rte_dev_iterator iterator; 2889 2890 printf("Attaching a new port...\n"); 2891 2892 if (identifier == NULL) { 2893 printf("Invalid parameters are specified\n"); 2894 return; 2895 } 2896 2897 if (rte_dev_probe(identifier) < 0) { 2898 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2899 return; 2900 } 2901 2902 /* first attach mode: event */ 2903 if (setup_on_probe_event) { 2904 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2905 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2906 if (ports[pi].port_status == RTE_PORT_HANDLING && 2907 ports[pi].need_setup != 0) 2908 setup_attached_port(pi); 2909 return; 2910 } 2911 2912 /* second attach mode: iterator */ 2913 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2914 /* setup ports matching the devargs used for probing */ 2915 if (port_is_forwarding(pi)) 2916 continue; /* port was already attached before */ 2917 setup_attached_port(pi); 2918 } 2919 } 2920 2921 static void 2922 setup_attached_port(portid_t pi) 2923 { 2924 unsigned int socket_id; 2925 int ret; 2926 2927 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2928 /* if socket_id is invalid, set to the first available socket. */ 2929 if (check_socket_id(socket_id) < 0) 2930 socket_id = socket_ids[0]; 2931 reconfig(pi, socket_id); 2932 ret = rte_eth_promiscuous_enable(pi); 2933 if (ret != 0) 2934 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2935 pi, rte_strerror(-ret)); 2936 2937 ports_ids[nb_ports++] = pi; 2938 fwd_ports_ids[nb_fwd_ports++] = pi; 2939 nb_cfg_ports = nb_fwd_ports; 2940 ports[pi].need_setup = 0; 2941 ports[pi].port_status = RTE_PORT_STOPPED; 2942 2943 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2944 printf("Done\n"); 2945 } 2946 2947 static void 2948 detach_device(struct rte_device *dev) 2949 { 2950 portid_t sibling; 2951 2952 if (dev == NULL) { 2953 printf("Device already removed\n"); 2954 return; 2955 } 2956 2957 printf("Removing a device...\n"); 2958 2959 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2960 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2961 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2962 printf("Port %u not stopped\n", sibling); 2963 return; 2964 } 2965 port_flow_flush(sibling); 2966 } 2967 } 2968 2969 if (rte_dev_remove(dev) < 0) { 2970 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2971 return; 2972 } 2973 remove_invalid_ports(); 2974 2975 printf("Device is detached\n"); 2976 printf("Now total ports is %d\n", nb_ports); 2977 printf("Done\n"); 2978 return; 2979 } 2980 2981 void 2982 detach_port_device(portid_t port_id) 2983 { 2984 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2985 return; 2986 2987 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2988 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2989 printf("Port not stopped\n"); 2990 return; 2991 } 2992 printf("Port was not closed\n"); 2993 } 2994 2995 detach_device(rte_eth_devices[port_id].device); 2996 } 2997 2998 void 2999 detach_devargs(char *identifier) 3000 { 3001 struct rte_dev_iterator iterator; 3002 struct rte_devargs da; 3003 portid_t port_id; 3004 3005 printf("Removing a device...\n"); 3006 3007 memset(&da, 0, sizeof(da)); 3008 if (rte_devargs_parsef(&da, "%s", identifier)) { 3009 printf("cannot parse identifier\n"); 3010 if (da.args) 3011 free(da.args); 3012 return; 3013 } 3014 3015 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3016 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3017 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3018 printf("Port %u not stopped\n", port_id); 3019 rte_eth_iterator_cleanup(&iterator); 3020 return; 3021 } 3022 port_flow_flush(port_id); 3023 } 3024 } 3025 3026 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3027 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3028 da.name, da.bus->name); 3029 return; 3030 } 3031 3032 remove_invalid_ports(); 3033 3034 printf("Device %s is detached\n", identifier); 3035 printf("Now total ports is %d\n", nb_ports); 3036 printf("Done\n"); 3037 } 3038 3039 void 3040 pmd_test_exit(void) 3041 { 3042 portid_t pt_id; 3043 unsigned int i; 3044 int ret; 3045 3046 if (test_done == 0) 3047 stop_packet_forwarding(); 3048 3049 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3050 if (mempools[i]) { 3051 if (mp_alloc_type == MP_ALLOC_ANON) 3052 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3053 NULL); 3054 } 3055 } 3056 if (ports != NULL) { 3057 no_link_check = 1; 3058 RTE_ETH_FOREACH_DEV(pt_id) { 3059 printf("\nStopping port %d...\n", pt_id); 3060 fflush(stdout); 3061 stop_port(pt_id); 3062 } 3063 RTE_ETH_FOREACH_DEV(pt_id) { 3064 printf("\nShutting down port %d...\n", pt_id); 3065 fflush(stdout); 3066 close_port(pt_id); 3067 } 3068 } 3069 3070 if (hot_plug) { 3071 ret = rte_dev_event_monitor_stop(); 3072 if (ret) { 3073 RTE_LOG(ERR, EAL, 3074 "fail to stop device event monitor."); 3075 return; 3076 } 3077 3078 ret = rte_dev_event_callback_unregister(NULL, 3079 dev_event_callback, NULL); 3080 if (ret < 0) { 3081 RTE_LOG(ERR, EAL, 3082 "fail to unregister device event callback.\n"); 3083 return; 3084 } 3085 3086 ret = rte_dev_hotplug_handle_disable(); 3087 if (ret) { 3088 RTE_LOG(ERR, EAL, 3089 "fail to disable hotplug handling.\n"); 3090 return; 3091 } 3092 } 3093 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3094 if (mempools[i]) 3095 rte_mempool_free(mempools[i]); 3096 } 3097 3098 printf("\nBye...\n"); 3099 } 3100 3101 typedef void (*cmd_func_t)(void); 3102 struct pmd_test_command { 3103 const char *cmd_name; 3104 cmd_func_t cmd_func; 3105 }; 3106 3107 /* Check the link status of all ports in up to 9s, and print them finally */ 3108 static void 3109 check_all_ports_link_status(uint32_t port_mask) 3110 { 3111 #define CHECK_INTERVAL 100 /* 100ms */ 3112 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3113 portid_t portid; 3114 uint8_t count, all_ports_up, print_flag = 0; 3115 struct rte_eth_link link; 3116 int ret; 3117 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3118 3119 printf("Checking link statuses...\n"); 3120 fflush(stdout); 3121 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3122 all_ports_up = 1; 3123 RTE_ETH_FOREACH_DEV(portid) { 3124 if ((port_mask & (1 << portid)) == 0) 3125 continue; 3126 memset(&link, 0, sizeof(link)); 3127 ret = rte_eth_link_get_nowait(portid, &link); 3128 if (ret < 0) { 3129 all_ports_up = 0; 3130 if (print_flag == 1) 3131 printf("Port %u link get failed: %s\n", 3132 portid, rte_strerror(-ret)); 3133 continue; 3134 } 3135 /* print link status if flag set */ 3136 if (print_flag == 1) { 3137 rte_eth_link_to_str(link_status, 3138 sizeof(link_status), &link); 3139 printf("Port %d %s\n", portid, link_status); 3140 continue; 3141 } 3142 /* clear all_ports_up flag if any link down */ 3143 if (link.link_status == ETH_LINK_DOWN) { 3144 all_ports_up = 0; 3145 break; 3146 } 3147 } 3148 /* after finally printing all link status, get out */ 3149 if (print_flag == 1) 3150 break; 3151 3152 if (all_ports_up == 0) { 3153 fflush(stdout); 3154 rte_delay_ms(CHECK_INTERVAL); 3155 } 3156 3157 /* set the print_flag if all ports up or timeout */ 3158 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3159 print_flag = 1; 3160 } 3161 3162 if (lsc_interrupt) 3163 break; 3164 } 3165 } 3166 3167 static void 3168 rmv_port_callback(void *arg) 3169 { 3170 int need_to_start = 0; 3171 int org_no_link_check = no_link_check; 3172 portid_t port_id = (intptr_t)arg; 3173 struct rte_device *dev; 3174 3175 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3176 3177 if (!test_done && port_is_forwarding(port_id)) { 3178 need_to_start = 1; 3179 stop_packet_forwarding(); 3180 } 3181 no_link_check = 1; 3182 stop_port(port_id); 3183 no_link_check = org_no_link_check; 3184 3185 /* Save rte_device pointer before closing ethdev port */ 3186 dev = rte_eth_devices[port_id].device; 3187 close_port(port_id); 3188 detach_device(dev); /* might be already removed or have more ports */ 3189 3190 if (need_to_start) 3191 start_packet_forwarding(0); 3192 } 3193 3194 /* This function is used by the interrupt thread */ 3195 static int 3196 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3197 void *ret_param) 3198 { 3199 RTE_SET_USED(param); 3200 RTE_SET_USED(ret_param); 3201 3202 if (type >= RTE_ETH_EVENT_MAX) { 3203 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3204 port_id, __func__, type); 3205 fflush(stderr); 3206 } else if (event_print_mask & (UINT32_C(1) << type)) { 3207 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3208 eth_event_desc[type]); 3209 fflush(stdout); 3210 } 3211 3212 switch (type) { 3213 case RTE_ETH_EVENT_NEW: 3214 ports[port_id].need_setup = 1; 3215 ports[port_id].port_status = RTE_PORT_HANDLING; 3216 break; 3217 case RTE_ETH_EVENT_INTR_RMV: 3218 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3219 break; 3220 if (rte_eal_alarm_set(100000, 3221 rmv_port_callback, (void *)(intptr_t)port_id)) 3222 fprintf(stderr, "Could not set up deferred device removal\n"); 3223 break; 3224 case RTE_ETH_EVENT_DESTROY: 3225 ports[port_id].port_status = RTE_PORT_CLOSED; 3226 printf("Port %u is closed\n", port_id); 3227 break; 3228 default: 3229 break; 3230 } 3231 return 0; 3232 } 3233 3234 static int 3235 register_eth_event_callback(void) 3236 { 3237 int ret; 3238 enum rte_eth_event_type event; 3239 3240 for (event = RTE_ETH_EVENT_UNKNOWN; 3241 event < RTE_ETH_EVENT_MAX; event++) { 3242 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3243 event, 3244 eth_event_callback, 3245 NULL); 3246 if (ret != 0) { 3247 TESTPMD_LOG(ERR, "Failed to register callback for " 3248 "%s event\n", eth_event_desc[event]); 3249 return -1; 3250 } 3251 } 3252 3253 return 0; 3254 } 3255 3256 /* This function is used by the interrupt thread */ 3257 static void 3258 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3259 __rte_unused void *arg) 3260 { 3261 uint16_t port_id; 3262 int ret; 3263 3264 if (type >= RTE_DEV_EVENT_MAX) { 3265 fprintf(stderr, "%s called upon invalid event %d\n", 3266 __func__, type); 3267 fflush(stderr); 3268 } 3269 3270 switch (type) { 3271 case RTE_DEV_EVENT_REMOVE: 3272 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3273 device_name); 3274 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3275 if (ret) { 3276 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3277 device_name); 3278 return; 3279 } 3280 /* 3281 * Because the user's callback is invoked in eal interrupt 3282 * callback, the interrupt callback need to be finished before 3283 * it can be unregistered when detaching device. So finish 3284 * callback soon and use a deferred removal to detach device 3285 * is need. It is a workaround, once the device detaching be 3286 * moved into the eal in the future, the deferred removal could 3287 * be deleted. 3288 */ 3289 if (rte_eal_alarm_set(100000, 3290 rmv_port_callback, (void *)(intptr_t)port_id)) 3291 RTE_LOG(ERR, EAL, 3292 "Could not set up deferred device removal\n"); 3293 break; 3294 case RTE_DEV_EVENT_ADD: 3295 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3296 device_name); 3297 /* TODO: After finish kernel driver binding, 3298 * begin to attach port. 3299 */ 3300 break; 3301 default: 3302 break; 3303 } 3304 } 3305 3306 static int 3307 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3308 { 3309 uint16_t i; 3310 int diag; 3311 uint8_t mapping_found = 0; 3312 3313 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3314 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3315 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3316 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3317 tx_queue_stats_mappings[i].queue_id, 3318 tx_queue_stats_mappings[i].stats_counter_id); 3319 if (diag != 0) 3320 return diag; 3321 mapping_found = 1; 3322 } 3323 } 3324 if (mapping_found) 3325 port->tx_queue_stats_mapping_enabled = 1; 3326 return 0; 3327 } 3328 3329 static int 3330 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3331 { 3332 uint16_t i; 3333 int diag; 3334 uint8_t mapping_found = 0; 3335 3336 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3337 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3338 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3339 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3340 rx_queue_stats_mappings[i].queue_id, 3341 rx_queue_stats_mappings[i].stats_counter_id); 3342 if (diag != 0) 3343 return diag; 3344 mapping_found = 1; 3345 } 3346 } 3347 if (mapping_found) 3348 port->rx_queue_stats_mapping_enabled = 1; 3349 return 0; 3350 } 3351 3352 static void 3353 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3354 { 3355 int diag = 0; 3356 3357 diag = set_tx_queue_stats_mapping_registers(pi, port); 3358 if (diag != 0) { 3359 if (diag == -ENOTSUP) { 3360 port->tx_queue_stats_mapping_enabled = 0; 3361 printf("TX queue stats mapping not supported port id=%d\n", pi); 3362 } 3363 else 3364 rte_exit(EXIT_FAILURE, 3365 "set_tx_queue_stats_mapping_registers " 3366 "failed for port id=%d diag=%d\n", 3367 pi, diag); 3368 } 3369 3370 diag = set_rx_queue_stats_mapping_registers(pi, port); 3371 if (diag != 0) { 3372 if (diag == -ENOTSUP) { 3373 port->rx_queue_stats_mapping_enabled = 0; 3374 printf("RX queue stats mapping not supported port id=%d\n", pi); 3375 } 3376 else 3377 rte_exit(EXIT_FAILURE, 3378 "set_rx_queue_stats_mapping_registers " 3379 "failed for port id=%d diag=%d\n", 3380 pi, diag); 3381 } 3382 } 3383 3384 static void 3385 rxtx_port_config(struct rte_port *port) 3386 { 3387 uint16_t qid; 3388 uint64_t offloads; 3389 3390 for (qid = 0; qid < nb_rxq; qid++) { 3391 offloads = port->rx_conf[qid].offloads; 3392 port->rx_conf[qid] = port->dev_info.default_rxconf; 3393 if (offloads != 0) 3394 port->rx_conf[qid].offloads = offloads; 3395 3396 /* Check if any Rx parameters have been passed */ 3397 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3398 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3399 3400 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3401 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3402 3403 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3404 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3405 3406 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3407 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3408 3409 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3410 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3411 3412 port->nb_rx_desc[qid] = nb_rxd; 3413 } 3414 3415 for (qid = 0; qid < nb_txq; qid++) { 3416 offloads = port->tx_conf[qid].offloads; 3417 port->tx_conf[qid] = port->dev_info.default_txconf; 3418 if (offloads != 0) 3419 port->tx_conf[qid].offloads = offloads; 3420 3421 /* Check if any Tx parameters have been passed */ 3422 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3423 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3424 3425 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3426 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3427 3428 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3429 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3430 3431 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3432 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3433 3434 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3435 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3436 3437 port->nb_tx_desc[qid] = nb_txd; 3438 } 3439 } 3440 3441 void 3442 init_port_config(void) 3443 { 3444 portid_t pid; 3445 struct rte_port *port; 3446 int ret; 3447 3448 RTE_ETH_FOREACH_DEV(pid) { 3449 port = &ports[pid]; 3450 port->dev_conf.fdir_conf = fdir_conf; 3451 3452 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3453 if (ret != 0) 3454 return; 3455 3456 if (nb_rxq > 1) { 3457 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3458 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3459 rss_hf & port->dev_info.flow_type_rss_offloads; 3460 } else { 3461 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3462 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3463 } 3464 3465 if (port->dcb_flag == 0) { 3466 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3467 port->dev_conf.rxmode.mq_mode = 3468 (enum rte_eth_rx_mq_mode) 3469 (rx_mq_mode & ETH_MQ_RX_RSS); 3470 else 3471 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3472 } 3473 3474 rxtx_port_config(port); 3475 3476 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3477 if (ret != 0) 3478 return; 3479 3480 map_port_queue_stats_mapping_registers(pid, port); 3481 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3482 rte_pmd_ixgbe_bypass_init(pid); 3483 #endif 3484 3485 if (lsc_interrupt && 3486 (rte_eth_devices[pid].data->dev_flags & 3487 RTE_ETH_DEV_INTR_LSC)) 3488 port->dev_conf.intr_conf.lsc = 1; 3489 if (rmv_interrupt && 3490 (rte_eth_devices[pid].data->dev_flags & 3491 RTE_ETH_DEV_INTR_RMV)) 3492 port->dev_conf.intr_conf.rmv = 1; 3493 } 3494 } 3495 3496 void set_port_slave_flag(portid_t slave_pid) 3497 { 3498 struct rte_port *port; 3499 3500 port = &ports[slave_pid]; 3501 port->slave_flag = 1; 3502 } 3503 3504 void clear_port_slave_flag(portid_t slave_pid) 3505 { 3506 struct rte_port *port; 3507 3508 port = &ports[slave_pid]; 3509 port->slave_flag = 0; 3510 } 3511 3512 uint8_t port_is_bonding_slave(portid_t slave_pid) 3513 { 3514 struct rte_port *port; 3515 3516 port = &ports[slave_pid]; 3517 if ((rte_eth_devices[slave_pid].data->dev_flags & 3518 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3519 return 1; 3520 return 0; 3521 } 3522 3523 const uint16_t vlan_tags[] = { 3524 0, 1, 2, 3, 4, 5, 6, 7, 3525 8, 9, 10, 11, 12, 13, 14, 15, 3526 16, 17, 18, 19, 20, 21, 22, 23, 3527 24, 25, 26, 27, 28, 29, 30, 31 3528 }; 3529 3530 static int 3531 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3532 enum dcb_mode_enable dcb_mode, 3533 enum rte_eth_nb_tcs num_tcs, 3534 uint8_t pfc_en) 3535 { 3536 uint8_t i; 3537 int32_t rc; 3538 struct rte_eth_rss_conf rss_conf; 3539 3540 /* 3541 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3542 * given above, and the number of traffic classes available for use. 3543 */ 3544 if (dcb_mode == DCB_VT_ENABLED) { 3545 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3546 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3547 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3548 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3549 3550 /* VMDQ+DCB RX and TX configurations */ 3551 vmdq_rx_conf->enable_default_pool = 0; 3552 vmdq_rx_conf->default_pool = 0; 3553 vmdq_rx_conf->nb_queue_pools = 3554 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3555 vmdq_tx_conf->nb_queue_pools = 3556 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3557 3558 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3559 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3560 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3561 vmdq_rx_conf->pool_map[i].pools = 3562 1 << (i % vmdq_rx_conf->nb_queue_pools); 3563 } 3564 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3565 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3566 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3567 } 3568 3569 /* set DCB mode of RX and TX of multiple queues */ 3570 eth_conf->rxmode.mq_mode = 3571 (enum rte_eth_rx_mq_mode) 3572 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3573 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3574 } else { 3575 struct rte_eth_dcb_rx_conf *rx_conf = 3576 ð_conf->rx_adv_conf.dcb_rx_conf; 3577 struct rte_eth_dcb_tx_conf *tx_conf = 3578 ð_conf->tx_adv_conf.dcb_tx_conf; 3579 3580 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3581 3582 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3583 if (rc != 0) 3584 return rc; 3585 3586 rx_conf->nb_tcs = num_tcs; 3587 tx_conf->nb_tcs = num_tcs; 3588 3589 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3590 rx_conf->dcb_tc[i] = i % num_tcs; 3591 tx_conf->dcb_tc[i] = i % num_tcs; 3592 } 3593 3594 eth_conf->rxmode.mq_mode = 3595 (enum rte_eth_rx_mq_mode) 3596 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3597 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3598 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3599 } 3600 3601 if (pfc_en) 3602 eth_conf->dcb_capability_en = 3603 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3604 else 3605 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3606 3607 return 0; 3608 } 3609 3610 int 3611 init_port_dcb_config(portid_t pid, 3612 enum dcb_mode_enable dcb_mode, 3613 enum rte_eth_nb_tcs num_tcs, 3614 uint8_t pfc_en) 3615 { 3616 struct rte_eth_conf port_conf; 3617 struct rte_port *rte_port; 3618 int retval; 3619 uint16_t i; 3620 3621 rte_port = &ports[pid]; 3622 3623 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3624 /* Enter DCB configuration status */ 3625 dcb_config = 1; 3626 3627 port_conf.rxmode = rte_port->dev_conf.rxmode; 3628 port_conf.txmode = rte_port->dev_conf.txmode; 3629 3630 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3631 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3632 if (retval < 0) 3633 return retval; 3634 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3635 3636 /* re-configure the device . */ 3637 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3638 if (retval < 0) 3639 return retval; 3640 3641 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3642 if (retval != 0) 3643 return retval; 3644 3645 /* If dev_info.vmdq_pool_base is greater than 0, 3646 * the queue id of vmdq pools is started after pf queues. 3647 */ 3648 if (dcb_mode == DCB_VT_ENABLED && 3649 rte_port->dev_info.vmdq_pool_base > 0) { 3650 printf("VMDQ_DCB multi-queue mode is nonsensical" 3651 " for port %d.", pid); 3652 return -1; 3653 } 3654 3655 /* Assume the ports in testpmd have the same dcb capability 3656 * and has the same number of rxq and txq in dcb mode 3657 */ 3658 if (dcb_mode == DCB_VT_ENABLED) { 3659 if (rte_port->dev_info.max_vfs > 0) { 3660 nb_rxq = rte_port->dev_info.nb_rx_queues; 3661 nb_txq = rte_port->dev_info.nb_tx_queues; 3662 } else { 3663 nb_rxq = rte_port->dev_info.max_rx_queues; 3664 nb_txq = rte_port->dev_info.max_tx_queues; 3665 } 3666 } else { 3667 /*if vt is disabled, use all pf queues */ 3668 if (rte_port->dev_info.vmdq_pool_base == 0) { 3669 nb_rxq = rte_port->dev_info.max_rx_queues; 3670 nb_txq = rte_port->dev_info.max_tx_queues; 3671 } else { 3672 nb_rxq = (queueid_t)num_tcs; 3673 nb_txq = (queueid_t)num_tcs; 3674 3675 } 3676 } 3677 rx_free_thresh = 64; 3678 3679 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3680 3681 rxtx_port_config(rte_port); 3682 /* VLAN filter */ 3683 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3684 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3685 rx_vft_set(pid, vlan_tags[i], 1); 3686 3687 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3688 if (retval != 0) 3689 return retval; 3690 3691 map_port_queue_stats_mapping_registers(pid, rte_port); 3692 3693 rte_port->dcb_flag = 1; 3694 3695 return 0; 3696 } 3697 3698 static void 3699 init_port(void) 3700 { 3701 int i; 3702 3703 /* Configuration of Ethernet ports. */ 3704 ports = rte_zmalloc("testpmd: ports", 3705 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3706 RTE_CACHE_LINE_SIZE); 3707 if (ports == NULL) { 3708 rte_exit(EXIT_FAILURE, 3709 "rte_zmalloc(%d struct rte_port) failed\n", 3710 RTE_MAX_ETHPORTS); 3711 } 3712 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3713 LIST_INIT(&ports[i].flow_tunnel_list); 3714 /* Initialize ports NUMA structures */ 3715 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3716 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3717 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3718 } 3719 3720 static void 3721 force_quit(void) 3722 { 3723 pmd_test_exit(); 3724 prompt_exit(); 3725 } 3726 3727 static void 3728 print_stats(void) 3729 { 3730 uint8_t i; 3731 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3732 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3733 3734 /* Clear screen and move to top left */ 3735 printf("%s%s", clr, top_left); 3736 3737 printf("\nPort statistics ===================================="); 3738 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3739 nic_stats_display(fwd_ports_ids[i]); 3740 3741 fflush(stdout); 3742 } 3743 3744 static void 3745 signal_handler(int signum) 3746 { 3747 if (signum == SIGINT || signum == SIGTERM) { 3748 printf("\nSignal %d received, preparing to exit...\n", 3749 signum); 3750 #ifdef RTE_LIBRTE_PDUMP 3751 /* uninitialize packet capture framework */ 3752 rte_pdump_uninit(); 3753 #endif 3754 #ifdef RTE_LIBRTE_LATENCY_STATS 3755 if (latencystats_enabled != 0) 3756 rte_latencystats_uninit(); 3757 #endif 3758 force_quit(); 3759 /* Set flag to indicate the force termination. */ 3760 f_quit = 1; 3761 /* exit with the expected status */ 3762 signal(signum, SIG_DFL); 3763 kill(getpid(), signum); 3764 } 3765 } 3766 3767 int 3768 main(int argc, char** argv) 3769 { 3770 int diag; 3771 portid_t port_id; 3772 uint16_t count; 3773 int ret; 3774 3775 signal(SIGINT, signal_handler); 3776 signal(SIGTERM, signal_handler); 3777 3778 testpmd_logtype = rte_log_register("testpmd"); 3779 if (testpmd_logtype < 0) 3780 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3781 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3782 3783 diag = rte_eal_init(argc, argv); 3784 if (diag < 0) 3785 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3786 rte_strerror(rte_errno)); 3787 3788 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3789 rte_exit(EXIT_FAILURE, 3790 "Secondary process type not supported.\n"); 3791 3792 ret = register_eth_event_callback(); 3793 if (ret != 0) 3794 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3795 3796 #ifdef RTE_LIBRTE_PDUMP 3797 /* initialize packet capture framework */ 3798 rte_pdump_init(); 3799 #endif 3800 3801 count = 0; 3802 RTE_ETH_FOREACH_DEV(port_id) { 3803 ports_ids[count] = port_id; 3804 count++; 3805 } 3806 nb_ports = (portid_t) count; 3807 if (nb_ports == 0) 3808 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3809 3810 /* allocate port structures, and init them */ 3811 init_port(); 3812 3813 set_def_fwd_config(); 3814 if (nb_lcores == 0) 3815 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3816 "Check the core mask argument\n"); 3817 3818 /* Bitrate/latency stats disabled by default */ 3819 #ifdef RTE_LIBRTE_BITRATESTATS 3820 bitrate_enabled = 0; 3821 #endif 3822 #ifdef RTE_LIBRTE_LATENCY_STATS 3823 latencystats_enabled = 0; 3824 #endif 3825 3826 /* on FreeBSD, mlockall() is disabled by default */ 3827 #ifdef RTE_EXEC_ENV_FREEBSD 3828 do_mlockall = 0; 3829 #else 3830 do_mlockall = 1; 3831 #endif 3832 3833 argc -= diag; 3834 argv += diag; 3835 if (argc > 1) 3836 launch_args_parse(argc, argv); 3837 3838 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3839 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3840 strerror(errno)); 3841 } 3842 3843 if (tx_first && interactive) 3844 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3845 "interactive mode.\n"); 3846 3847 if (tx_first && lsc_interrupt) { 3848 printf("Warning: lsc_interrupt needs to be off when " 3849 " using tx_first. Disabling.\n"); 3850 lsc_interrupt = 0; 3851 } 3852 3853 if (!nb_rxq && !nb_txq) 3854 printf("Warning: Either rx or tx queues should be non-zero\n"); 3855 3856 if (nb_rxq > 1 && nb_rxq > nb_txq) 3857 printf("Warning: nb_rxq=%d enables RSS configuration, " 3858 "but nb_txq=%d will prevent to fully test it.\n", 3859 nb_rxq, nb_txq); 3860 3861 init_config(); 3862 3863 if (hot_plug) { 3864 ret = rte_dev_hotplug_handle_enable(); 3865 if (ret) { 3866 RTE_LOG(ERR, EAL, 3867 "fail to enable hotplug handling."); 3868 return -1; 3869 } 3870 3871 ret = rte_dev_event_monitor_start(); 3872 if (ret) { 3873 RTE_LOG(ERR, EAL, 3874 "fail to start device event monitoring."); 3875 return -1; 3876 } 3877 3878 ret = rte_dev_event_callback_register(NULL, 3879 dev_event_callback, NULL); 3880 if (ret) { 3881 RTE_LOG(ERR, EAL, 3882 "fail to register device event callback\n"); 3883 return -1; 3884 } 3885 } 3886 3887 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3888 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3889 3890 /* set all ports to promiscuous mode by default */ 3891 RTE_ETH_FOREACH_DEV(port_id) { 3892 ret = rte_eth_promiscuous_enable(port_id); 3893 if (ret != 0) 3894 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3895 port_id, rte_strerror(-ret)); 3896 } 3897 3898 /* Init metrics library */ 3899 rte_metrics_init(rte_socket_id()); 3900 3901 #ifdef RTE_LIBRTE_LATENCY_STATS 3902 if (latencystats_enabled != 0) { 3903 int ret = rte_latencystats_init(1, NULL); 3904 if (ret) 3905 printf("Warning: latencystats init()" 3906 " returned error %d\n", ret); 3907 printf("Latencystats running on lcore %d\n", 3908 latencystats_lcore_id); 3909 } 3910 #endif 3911 3912 /* Setup bitrate stats */ 3913 #ifdef RTE_LIBRTE_BITRATESTATS 3914 if (bitrate_enabled != 0) { 3915 bitrate_data = rte_stats_bitrate_create(); 3916 if (bitrate_data == NULL) 3917 rte_exit(EXIT_FAILURE, 3918 "Could not allocate bitrate data.\n"); 3919 rte_stats_bitrate_reg(bitrate_data); 3920 } 3921 #endif 3922 3923 #ifdef RTE_LIBRTE_CMDLINE 3924 if (strlen(cmdline_filename) != 0) 3925 cmdline_read_from_file(cmdline_filename); 3926 3927 if (interactive == 1) { 3928 if (auto_start) { 3929 printf("Start automatic packet forwarding\n"); 3930 start_packet_forwarding(0); 3931 } 3932 prompt(); 3933 pmd_test_exit(); 3934 } else 3935 #endif 3936 { 3937 char c; 3938 int rc; 3939 3940 f_quit = 0; 3941 3942 printf("No commandline core given, start packet forwarding\n"); 3943 start_packet_forwarding(tx_first); 3944 if (stats_period != 0) { 3945 uint64_t prev_time = 0, cur_time, diff_time = 0; 3946 uint64_t timer_period; 3947 3948 /* Convert to number of cycles */ 3949 timer_period = stats_period * rte_get_timer_hz(); 3950 3951 while (f_quit == 0) { 3952 cur_time = rte_get_timer_cycles(); 3953 diff_time += cur_time - prev_time; 3954 3955 if (diff_time >= timer_period) { 3956 print_stats(); 3957 /* Reset the timer */ 3958 diff_time = 0; 3959 } 3960 /* Sleep to avoid unnecessary checks */ 3961 prev_time = cur_time; 3962 sleep(1); 3963 } 3964 } 3965 3966 printf("Press enter to exit\n"); 3967 rc = read(0, &c, 1); 3968 pmd_test_exit(); 3969 if (rc < 0) 3970 return 1; 3971 } 3972 3973 ret = rte_eal_cleanup(); 3974 if (ret != 0) 3975 rte_exit(EXIT_FAILURE, 3976 "EAL cleanup failed: %s\n", strerror(-ret)); 3977 3978 return EXIT_SUCCESS; 3979 } 3980