1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 192 uint16_t mempool_flags; 193 194 struct fwd_config cur_fwd_config; 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 196 uint32_t retry_enabled; 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 199 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 201 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 202 * specified on command-line. */ 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 204 205 /* 206 * In container, it cannot terminate the process which running with 'stats-period' 207 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 208 */ 209 uint8_t f_quit; 210 211 /* 212 * Configuration of packet segments used by the "txonly" processing engine. 213 */ 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 216 TXONLY_DEF_PACKET_LEN, 217 }; 218 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 219 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 221 /**< Split policy for packets to TX. */ 222 223 uint8_t txonly_multi_flow; 224 /**< Whether multiple flows are generated in TXONLY mode. */ 225 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 228 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 230 uint8_t dcb_config = 0; 231 232 /* Whether the dcb is in testing status */ 233 uint8_t dcb_test = 0; 234 235 /* 236 * Configurable number of RX/TX queues. 237 */ 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 240 241 /* 242 * Configurable number of RX/TX ring descriptors. 243 * Defaults are supplied by drivers via ethdev. 244 */ 245 #define RTE_TEST_RX_DESC_DEFAULT 0 246 #define RTE_TEST_TX_DESC_DEFAULT 0 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 249 250 #define RTE_PMD_PARAM_UNSET -1 251 /* 252 * Configurable values of RX and TX ring threshold registers. 253 */ 254 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 258 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 262 263 /* 264 * Configurable value of RX free threshold. 265 */ 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX drop enable. 270 */ 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of TX free threshold. 275 */ 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX RS bit threshold. 280 */ 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of buffered packets before sending. 285 */ 286 uint16_t noisy_tx_sw_bufsz; 287 288 /* 289 * Configurable value of packet buffer timeout. 290 */ 291 uint16_t noisy_tx_sw_buf_flush_time; 292 293 /* 294 * Configurable value for size of VNF internal memory area 295 * used for simulating noisy neighbour behaviour 296 */ 297 uint64_t noisy_lkup_mem_sz; 298 299 /* 300 * Configurable value of number of random writes done in 301 * VNF simulation memory area. 302 */ 303 uint64_t noisy_lkup_num_writes; 304 305 /* 306 * Configurable value of number of random reads done in 307 * VNF simulation memory area. 308 */ 309 uint64_t noisy_lkup_num_reads; 310 311 /* 312 * Configurable value of number of random reads/writes done in 313 * VNF simulation memory area. 314 */ 315 uint64_t noisy_lkup_num_reads_writes; 316 317 /* 318 * Receive Side Scaling (RSS) configuration. 319 */ 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 321 322 /* 323 * Port topology configuration 324 */ 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 326 327 /* 328 * Avoids to flush all the RX streams before starts forwarding. 329 */ 330 uint8_t no_flush_rx = 0; /* flush by default */ 331 332 /* 333 * Flow API isolated mode. 334 */ 335 uint8_t flow_isolate_all; 336 337 /* 338 * Avoids to check link status when starting/stopping a port. 339 */ 340 uint8_t no_link_check = 0; /* check by default */ 341 342 /* 343 * Enable link status change notification 344 */ 345 uint8_t lsc_interrupt = 1; /* enabled by default */ 346 347 /* 348 * Enable device removal notification. 349 */ 350 uint8_t rmv_interrupt = 1; /* enabled by default */ 351 352 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 353 354 /* After attach, port setup is called on event or by iterator */ 355 bool setup_on_probe_event = true; 356 357 /* Pretty printing of ethdev events */ 358 static const char * const eth_event_desc[] = { 359 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 360 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 361 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 362 [RTE_ETH_EVENT_INTR_RESET] = "reset", 363 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 364 [RTE_ETH_EVENT_IPSEC] = "IPsec", 365 [RTE_ETH_EVENT_MACSEC] = "MACsec", 366 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 367 [RTE_ETH_EVENT_NEW] = "device probed", 368 [RTE_ETH_EVENT_DESTROY] = "device released", 369 [RTE_ETH_EVENT_MAX] = NULL, 370 }; 371 372 /* 373 * Display or mask ether events 374 * Default to all events except VF_MBOX 375 */ 376 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 377 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 378 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 379 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 380 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 381 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 382 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 383 /* 384 * Decide if all memory are locked for performance. 385 */ 386 int do_mlockall = 0; 387 388 /* 389 * NIC bypass mode configuration options. 390 */ 391 392 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 393 /* The NIC bypass watchdog timeout. */ 394 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 395 #endif 396 397 398 #ifdef RTE_LIBRTE_LATENCY_STATS 399 400 /* 401 * Set when latency stats is enabled in the commandline 402 */ 403 uint8_t latencystats_enabled; 404 405 /* 406 * Lcore ID to serive latency statistics. 407 */ 408 lcoreid_t latencystats_lcore_id = -1; 409 410 #endif 411 412 /* 413 * Ethernet device configuration. 414 */ 415 struct rte_eth_rxmode rx_mode = { 416 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 417 }; 418 419 struct rte_eth_txmode tx_mode = { 420 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 421 }; 422 423 struct rte_fdir_conf fdir_conf = { 424 .mode = RTE_FDIR_MODE_NONE, 425 .pballoc = RTE_FDIR_PBALLOC_64K, 426 .status = RTE_FDIR_REPORT_STATUS, 427 .mask = { 428 .vlan_tci_mask = 0xFFEF, 429 .ipv4_mask = { 430 .src_ip = 0xFFFFFFFF, 431 .dst_ip = 0xFFFFFFFF, 432 }, 433 .ipv6_mask = { 434 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 435 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 436 }, 437 .src_port_mask = 0xFFFF, 438 .dst_port_mask = 0xFFFF, 439 .mac_addr_byte_mask = 0xFF, 440 .tunnel_type_mask = 1, 441 .tunnel_id_mask = 0xFFFFFFFF, 442 }, 443 .drop_queue = 127, 444 }; 445 446 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 447 448 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 449 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 450 451 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 452 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 453 454 uint16_t nb_tx_queue_stats_mappings = 0; 455 uint16_t nb_rx_queue_stats_mappings = 0; 456 457 /* 458 * Display zero values by default for xstats 459 */ 460 uint8_t xstats_hide_zero; 461 462 unsigned int num_sockets = 0; 463 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 464 465 #ifdef RTE_LIBRTE_BITRATE 466 /* Bitrate statistics */ 467 struct rte_stats_bitrates *bitrate_data; 468 lcoreid_t bitrate_lcore_id; 469 uint8_t bitrate_enabled; 470 #endif 471 472 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 473 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 474 475 struct vxlan_encap_conf vxlan_encap_conf = { 476 .select_ipv4 = 1, 477 .select_vlan = 0, 478 .select_tos_ttl = 0, 479 .vni = "\x00\x00\x00", 480 .udp_src = 0, 481 .udp_dst = RTE_BE16(4789), 482 .ipv4_src = IPv4(127, 0, 0, 1), 483 .ipv4_dst = IPv4(255, 255, 255, 255), 484 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 485 "\x00\x00\x00\x00\x00\x00\x00\x01", 486 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 487 "\x00\x00\x00\x00\x00\x00\x11\x11", 488 .vlan_tci = 0, 489 .ip_tos = 0, 490 .ip_ttl = 255, 491 .eth_src = "\x00\x00\x00\x00\x00\x00", 492 .eth_dst = "\xff\xff\xff\xff\xff\xff", 493 }; 494 495 struct nvgre_encap_conf nvgre_encap_conf = { 496 .select_ipv4 = 1, 497 .select_vlan = 0, 498 .tni = "\x00\x00\x00", 499 .ipv4_src = IPv4(127, 0, 0, 1), 500 .ipv4_dst = IPv4(255, 255, 255, 255), 501 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 502 "\x00\x00\x00\x00\x00\x00\x00\x01", 503 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 504 "\x00\x00\x00\x00\x00\x00\x11\x11", 505 .vlan_tci = 0, 506 .eth_src = "\x00\x00\x00\x00\x00\x00", 507 .eth_dst = "\xff\xff\xff\xff\xff\xff", 508 }; 509 510 /* Forward function declarations */ 511 static void setup_attached_port(portid_t pi); 512 static void map_port_queue_stats_mapping_registers(portid_t pi, 513 struct rte_port *port); 514 static void check_all_ports_link_status(uint32_t port_mask); 515 static int eth_event_callback(portid_t port_id, 516 enum rte_eth_event_type type, 517 void *param, void *ret_param); 518 static void dev_event_callback(const char *device_name, 519 enum rte_dev_event_type type, 520 void *param); 521 522 /* 523 * Check if all the ports are started. 524 * If yes, return positive value. If not, return zero. 525 */ 526 static int all_ports_started(void); 527 528 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 529 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN; 530 531 /* 532 * Helper function to check if socket is already discovered. 533 * If yes, return positive value. If not, return zero. 534 */ 535 int 536 new_socket_id(unsigned int socket_id) 537 { 538 unsigned int i; 539 540 for (i = 0; i < num_sockets; i++) { 541 if (socket_ids[i] == socket_id) 542 return 0; 543 } 544 return 1; 545 } 546 547 /* 548 * Setup default configuration. 549 */ 550 static void 551 set_default_fwd_lcores_config(void) 552 { 553 unsigned int i; 554 unsigned int nb_lc; 555 unsigned int sock_num; 556 557 nb_lc = 0; 558 for (i = 0; i < RTE_MAX_LCORE; i++) { 559 if (!rte_lcore_is_enabled(i)) 560 continue; 561 sock_num = rte_lcore_to_socket_id(i); 562 if (new_socket_id(sock_num)) { 563 if (num_sockets >= RTE_MAX_NUMA_NODES) { 564 rte_exit(EXIT_FAILURE, 565 "Total sockets greater than %u\n", 566 RTE_MAX_NUMA_NODES); 567 } 568 socket_ids[num_sockets++] = sock_num; 569 } 570 if (i == rte_get_master_lcore()) 571 continue; 572 fwd_lcores_cpuids[nb_lc++] = i; 573 } 574 nb_lcores = (lcoreid_t) nb_lc; 575 nb_cfg_lcores = nb_lcores; 576 nb_fwd_lcores = 1; 577 } 578 579 static void 580 set_def_peer_eth_addrs(void) 581 { 582 portid_t i; 583 584 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 585 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 586 peer_eth_addrs[i].addr_bytes[5] = i; 587 } 588 } 589 590 static void 591 set_default_fwd_ports_config(void) 592 { 593 portid_t pt_id; 594 int i = 0; 595 596 RTE_ETH_FOREACH_DEV(pt_id) { 597 fwd_ports_ids[i++] = pt_id; 598 599 /* Update sockets info according to the attached device */ 600 int socket_id = rte_eth_dev_socket_id(pt_id); 601 if (socket_id >= 0 && new_socket_id(socket_id)) { 602 if (num_sockets >= RTE_MAX_NUMA_NODES) { 603 rte_exit(EXIT_FAILURE, 604 "Total sockets greater than %u\n", 605 RTE_MAX_NUMA_NODES); 606 } 607 socket_ids[num_sockets++] = socket_id; 608 } 609 } 610 611 nb_cfg_ports = nb_ports; 612 nb_fwd_ports = nb_ports; 613 } 614 615 void 616 set_def_fwd_config(void) 617 { 618 set_default_fwd_lcores_config(); 619 set_def_peer_eth_addrs(); 620 set_default_fwd_ports_config(); 621 } 622 623 /* extremely pessimistic estimation of memory required to create a mempool */ 624 static int 625 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 626 { 627 unsigned int n_pages, mbuf_per_pg, leftover; 628 uint64_t total_mem, mbuf_mem, obj_sz; 629 630 /* there is no good way to predict how much space the mempool will 631 * occupy because it will allocate chunks on the fly, and some of those 632 * will come from default DPDK memory while some will come from our 633 * external memory, so just assume 128MB will be enough for everyone. 634 */ 635 uint64_t hdr_mem = 128 << 20; 636 637 /* account for possible non-contiguousness */ 638 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 639 if (obj_sz > pgsz) { 640 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 641 return -1; 642 } 643 644 mbuf_per_pg = pgsz / obj_sz; 645 leftover = (nb_mbufs % mbuf_per_pg) > 0; 646 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 647 648 mbuf_mem = n_pages * pgsz; 649 650 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 651 652 if (total_mem > SIZE_MAX) { 653 TESTPMD_LOG(ERR, "Memory size too big\n"); 654 return -1; 655 } 656 *out = (size_t)total_mem; 657 658 return 0; 659 } 660 661 static int 662 pagesz_flags(uint64_t page_sz) 663 { 664 /* as per mmap() manpage, all page sizes are log2 of page size 665 * shifted by MAP_HUGE_SHIFT 666 */ 667 int log2 = rte_log2_u64(page_sz); 668 669 return (log2 << HUGE_SHIFT); 670 } 671 672 static void * 673 alloc_mem(size_t memsz, size_t pgsz, bool huge) 674 { 675 void *addr; 676 int flags; 677 678 /* allocate anonymous hugepages */ 679 flags = MAP_ANONYMOUS | MAP_PRIVATE; 680 if (huge) 681 flags |= HUGE_FLAG | pagesz_flags(pgsz); 682 683 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 684 if (addr == MAP_FAILED) 685 return NULL; 686 687 return addr; 688 } 689 690 struct extmem_param { 691 void *addr; 692 size_t len; 693 size_t pgsz; 694 rte_iova_t *iova_table; 695 unsigned int iova_table_len; 696 }; 697 698 static int 699 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 700 bool huge) 701 { 702 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 703 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 704 unsigned int cur_page, n_pages, pgsz_idx; 705 size_t mem_sz, cur_pgsz; 706 rte_iova_t *iovas = NULL; 707 void *addr; 708 int ret; 709 710 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 711 /* skip anything that is too big */ 712 if (pgsizes[pgsz_idx] > SIZE_MAX) 713 continue; 714 715 cur_pgsz = pgsizes[pgsz_idx]; 716 717 /* if we were told not to allocate hugepages, override */ 718 if (!huge) 719 cur_pgsz = sysconf(_SC_PAGESIZE); 720 721 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 722 if (ret < 0) { 723 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 724 return -1; 725 } 726 727 /* allocate our memory */ 728 addr = alloc_mem(mem_sz, cur_pgsz, huge); 729 730 /* if we couldn't allocate memory with a specified page size, 731 * that doesn't mean we can't do it with other page sizes, so 732 * try another one. 733 */ 734 if (addr == NULL) 735 continue; 736 737 /* store IOVA addresses for every page in this memory area */ 738 n_pages = mem_sz / cur_pgsz; 739 740 iovas = malloc(sizeof(*iovas) * n_pages); 741 742 if (iovas == NULL) { 743 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 744 goto fail; 745 } 746 /* lock memory if it's not huge pages */ 747 if (!huge) 748 mlock(addr, mem_sz); 749 750 /* populate IOVA addresses */ 751 for (cur_page = 0; cur_page < n_pages; cur_page++) { 752 rte_iova_t iova; 753 size_t offset; 754 void *cur; 755 756 offset = cur_pgsz * cur_page; 757 cur = RTE_PTR_ADD(addr, offset); 758 759 /* touch the page before getting its IOVA */ 760 *(volatile char *)cur = 0; 761 762 iova = rte_mem_virt2iova(cur); 763 764 iovas[cur_page] = iova; 765 } 766 767 break; 768 } 769 /* if we couldn't allocate anything */ 770 if (iovas == NULL) 771 return -1; 772 773 param->addr = addr; 774 param->len = mem_sz; 775 param->pgsz = cur_pgsz; 776 param->iova_table = iovas; 777 param->iova_table_len = n_pages; 778 779 return 0; 780 fail: 781 if (iovas) 782 free(iovas); 783 if (addr) 784 munmap(addr, mem_sz); 785 786 return -1; 787 } 788 789 static int 790 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 791 { 792 struct extmem_param param; 793 int socket_id, ret; 794 795 memset(¶m, 0, sizeof(param)); 796 797 /* check if our heap exists */ 798 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 799 if (socket_id < 0) { 800 /* create our heap */ 801 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 802 if (ret < 0) { 803 TESTPMD_LOG(ERR, "Cannot create heap\n"); 804 return -1; 805 } 806 } 807 808 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 809 if (ret < 0) { 810 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 811 return -1; 812 } 813 814 /* we now have a valid memory area, so add it to heap */ 815 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 816 param.addr, param.len, param.iova_table, 817 param.iova_table_len, param.pgsz); 818 819 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 820 821 /* not needed any more */ 822 free(param.iova_table); 823 824 if (ret < 0) { 825 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 826 munmap(param.addr, param.len); 827 return -1; 828 } 829 830 /* success */ 831 832 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 833 param.len >> 20); 834 835 return 0; 836 } 837 static void 838 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 839 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 840 { 841 uint16_t pid = 0; 842 int ret; 843 844 RTE_ETH_FOREACH_DEV(pid) { 845 struct rte_eth_dev *dev = 846 &rte_eth_devices[pid]; 847 848 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 849 memhdr->len); 850 if (ret) { 851 TESTPMD_LOG(DEBUG, 852 "unable to DMA unmap addr 0x%p " 853 "for device %s\n", 854 memhdr->addr, dev->data->name); 855 } 856 } 857 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 858 if (ret) { 859 TESTPMD_LOG(DEBUG, 860 "unable to un-register addr 0x%p\n", memhdr->addr); 861 } 862 } 863 864 static void 865 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 866 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 867 { 868 uint16_t pid = 0; 869 size_t page_size = sysconf(_SC_PAGESIZE); 870 int ret; 871 872 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 873 page_size); 874 if (ret) { 875 TESTPMD_LOG(DEBUG, 876 "unable to register addr 0x%p\n", memhdr->addr); 877 return; 878 } 879 RTE_ETH_FOREACH_DEV(pid) { 880 struct rte_eth_dev *dev = 881 &rte_eth_devices[pid]; 882 883 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 884 memhdr->len); 885 if (ret) { 886 TESTPMD_LOG(DEBUG, 887 "unable to DMA map addr 0x%p " 888 "for device %s\n", 889 memhdr->addr, dev->data->name); 890 } 891 } 892 } 893 894 /* 895 * Configuration initialisation done once at init time. 896 */ 897 static struct rte_mempool * 898 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 899 unsigned int socket_id) 900 { 901 char pool_name[RTE_MEMPOOL_NAMESIZE]; 902 struct rte_mempool *rte_mp = NULL; 903 uint32_t mb_size; 904 905 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 906 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 907 908 TESTPMD_LOG(INFO, 909 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 910 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 911 912 switch (mp_alloc_type) { 913 case MP_ALLOC_NATIVE: 914 { 915 /* wrapper to rte_mempool_create() */ 916 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 917 rte_mbuf_best_mempool_ops()); 918 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 919 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 920 break; 921 } 922 case MP_ALLOC_ANON: 923 { 924 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 925 mb_size, (unsigned int) mb_mempool_cache, 926 sizeof(struct rte_pktmbuf_pool_private), 927 socket_id, mempool_flags); 928 if (rte_mp == NULL) 929 goto err; 930 931 if (rte_mempool_populate_anon(rte_mp) == 0) { 932 rte_mempool_free(rte_mp); 933 rte_mp = NULL; 934 goto err; 935 } 936 rte_pktmbuf_pool_init(rte_mp, NULL); 937 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 938 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 939 break; 940 } 941 case MP_ALLOC_XMEM: 942 case MP_ALLOC_XMEM_HUGE: 943 { 944 int heap_socket; 945 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 946 947 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 948 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 949 950 heap_socket = 951 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 952 if (heap_socket < 0) 953 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 954 955 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 956 rte_mbuf_best_mempool_ops()); 957 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 958 mb_mempool_cache, 0, mbuf_seg_size, 959 heap_socket); 960 break; 961 } 962 default: 963 { 964 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 965 } 966 } 967 968 err: 969 if (rte_mp == NULL) { 970 rte_exit(EXIT_FAILURE, 971 "Creation of mbuf pool for socket %u failed: %s\n", 972 socket_id, rte_strerror(rte_errno)); 973 } else if (verbose_level > 0) { 974 rte_mempool_dump(stdout, rte_mp); 975 } 976 return rte_mp; 977 } 978 979 /* 980 * Check given socket id is valid or not with NUMA mode, 981 * if valid, return 0, else return -1 982 */ 983 static int 984 check_socket_id(const unsigned int socket_id) 985 { 986 static int warning_once = 0; 987 988 if (new_socket_id(socket_id)) { 989 if (!warning_once && numa_support) 990 printf("Warning: NUMA should be configured manually by" 991 " using --port-numa-config and" 992 " --ring-numa-config parameters along with" 993 " --numa.\n"); 994 warning_once = 1; 995 return -1; 996 } 997 return 0; 998 } 999 1000 /* 1001 * Get the allowed maximum number of RX queues. 1002 * *pid return the port id which has minimal value of 1003 * max_rx_queues in all ports. 1004 */ 1005 queueid_t 1006 get_allowed_max_nb_rxq(portid_t *pid) 1007 { 1008 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 1009 portid_t pi; 1010 struct rte_eth_dev_info dev_info; 1011 1012 RTE_ETH_FOREACH_DEV(pi) { 1013 rte_eth_dev_info_get(pi, &dev_info); 1014 if (dev_info.max_rx_queues < allowed_max_rxq) { 1015 allowed_max_rxq = dev_info.max_rx_queues; 1016 *pid = pi; 1017 } 1018 } 1019 return allowed_max_rxq; 1020 } 1021 1022 /* 1023 * Check input rxq is valid or not. 1024 * If input rxq is not greater than any of maximum number 1025 * of RX queues of all ports, it is valid. 1026 * if valid, return 0, else return -1 1027 */ 1028 int 1029 check_nb_rxq(queueid_t rxq) 1030 { 1031 queueid_t allowed_max_rxq; 1032 portid_t pid = 0; 1033 1034 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1035 if (rxq > allowed_max_rxq) { 1036 printf("Fail: input rxq (%u) can't be greater " 1037 "than max_rx_queues (%u) of port %u\n", 1038 rxq, 1039 allowed_max_rxq, 1040 pid); 1041 return -1; 1042 } 1043 return 0; 1044 } 1045 1046 /* 1047 * Get the allowed maximum number of TX queues. 1048 * *pid return the port id which has minimal value of 1049 * max_tx_queues in all ports. 1050 */ 1051 queueid_t 1052 get_allowed_max_nb_txq(portid_t *pid) 1053 { 1054 queueid_t allowed_max_txq = MAX_QUEUE_ID; 1055 portid_t pi; 1056 struct rte_eth_dev_info dev_info; 1057 1058 RTE_ETH_FOREACH_DEV(pi) { 1059 rte_eth_dev_info_get(pi, &dev_info); 1060 if (dev_info.max_tx_queues < allowed_max_txq) { 1061 allowed_max_txq = dev_info.max_tx_queues; 1062 *pid = pi; 1063 } 1064 } 1065 return allowed_max_txq; 1066 } 1067 1068 /* 1069 * Check input txq is valid or not. 1070 * If input txq is not greater than any of maximum number 1071 * of TX queues of all ports, it is valid. 1072 * if valid, return 0, else return -1 1073 */ 1074 int 1075 check_nb_txq(queueid_t txq) 1076 { 1077 queueid_t allowed_max_txq; 1078 portid_t pid = 0; 1079 1080 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1081 if (txq > allowed_max_txq) { 1082 printf("Fail: input txq (%u) can't be greater " 1083 "than max_tx_queues (%u) of port %u\n", 1084 txq, 1085 allowed_max_txq, 1086 pid); 1087 return -1; 1088 } 1089 return 0; 1090 } 1091 1092 static void 1093 init_config(void) 1094 { 1095 portid_t pid; 1096 struct rte_port *port; 1097 struct rte_mempool *mbp; 1098 unsigned int nb_mbuf_per_pool; 1099 lcoreid_t lc_id; 1100 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1101 struct rte_gro_param gro_param; 1102 uint32_t gso_types; 1103 int k; 1104 1105 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1106 1107 /* Configuration of logical cores. */ 1108 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1109 sizeof(struct fwd_lcore *) * nb_lcores, 1110 RTE_CACHE_LINE_SIZE); 1111 if (fwd_lcores == NULL) { 1112 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1113 "failed\n", nb_lcores); 1114 } 1115 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1116 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1117 sizeof(struct fwd_lcore), 1118 RTE_CACHE_LINE_SIZE); 1119 if (fwd_lcores[lc_id] == NULL) { 1120 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1121 "failed\n"); 1122 } 1123 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1124 } 1125 1126 RTE_ETH_FOREACH_DEV(pid) { 1127 port = &ports[pid]; 1128 /* Apply default TxRx configuration for all ports */ 1129 port->dev_conf.txmode = tx_mode; 1130 port->dev_conf.rxmode = rx_mode; 1131 rte_eth_dev_info_get(pid, &port->dev_info); 1132 1133 if (!(port->dev_info.tx_offload_capa & 1134 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1135 port->dev_conf.txmode.offloads &= 1136 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1137 if (!(port->dev_info.tx_offload_capa & 1138 DEV_TX_OFFLOAD_MATCH_METADATA)) 1139 port->dev_conf.txmode.offloads &= 1140 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1141 if (numa_support) { 1142 if (port_numa[pid] != NUMA_NO_CONFIG) 1143 port_per_socket[port_numa[pid]]++; 1144 else { 1145 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1146 1147 /* 1148 * if socket_id is invalid, 1149 * set to the first available socket. 1150 */ 1151 if (check_socket_id(socket_id) < 0) 1152 socket_id = socket_ids[0]; 1153 port_per_socket[socket_id]++; 1154 } 1155 } 1156 1157 /* Apply Rx offloads configuration */ 1158 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1159 port->rx_conf[k].offloads = 1160 port->dev_conf.rxmode.offloads; 1161 /* Apply Tx offloads configuration */ 1162 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1163 port->tx_conf[k].offloads = 1164 port->dev_conf.txmode.offloads; 1165 1166 /* set flag to initialize port/queue */ 1167 port->need_reconfig = 1; 1168 port->need_reconfig_queues = 1; 1169 port->tx_metadata = 0; 1170 } 1171 1172 /* 1173 * Create pools of mbuf. 1174 * If NUMA support is disabled, create a single pool of mbuf in 1175 * socket 0 memory by default. 1176 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1177 * 1178 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1179 * nb_txd can be configured at run time. 1180 */ 1181 if (param_total_num_mbufs) 1182 nb_mbuf_per_pool = param_total_num_mbufs; 1183 else { 1184 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1185 (nb_lcores * mb_mempool_cache) + 1186 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1187 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1188 } 1189 1190 if (numa_support) { 1191 uint8_t i; 1192 1193 for (i = 0; i < num_sockets; i++) 1194 mempools[i] = mbuf_pool_create(mbuf_data_size, 1195 nb_mbuf_per_pool, 1196 socket_ids[i]); 1197 } else { 1198 if (socket_num == UMA_NO_CONFIG) 1199 mempools[0] = mbuf_pool_create(mbuf_data_size, 1200 nb_mbuf_per_pool, 0); 1201 else 1202 mempools[socket_num] = mbuf_pool_create 1203 (mbuf_data_size, 1204 nb_mbuf_per_pool, 1205 socket_num); 1206 } 1207 1208 init_port_config(); 1209 1210 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1211 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1212 /* 1213 * Records which Mbuf pool to use by each logical core, if needed. 1214 */ 1215 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1216 mbp = mbuf_pool_find( 1217 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1218 1219 if (mbp == NULL) 1220 mbp = mbuf_pool_find(0); 1221 fwd_lcores[lc_id]->mbp = mbp; 1222 /* initialize GSO context */ 1223 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1224 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1225 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1226 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN - 1227 ETHER_CRC_LEN; 1228 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1229 } 1230 1231 /* Configuration of packet forwarding streams. */ 1232 if (init_fwd_streams() < 0) 1233 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1234 1235 fwd_config_setup(); 1236 1237 /* create a gro context for each lcore */ 1238 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1239 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1240 gro_param.max_item_per_flow = MAX_PKT_BURST; 1241 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1242 gro_param.socket_id = rte_lcore_to_socket_id( 1243 fwd_lcores_cpuids[lc_id]); 1244 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1245 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1246 rte_exit(EXIT_FAILURE, 1247 "rte_gro_ctx_create() failed\n"); 1248 } 1249 } 1250 1251 #if defined RTE_LIBRTE_PMD_SOFTNIC 1252 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1253 RTE_ETH_FOREACH_DEV(pid) { 1254 port = &ports[pid]; 1255 const char *driver = port->dev_info.driver_name; 1256 1257 if (strcmp(driver, "net_softnic") == 0) 1258 port->softport.fwd_lcore_arg = fwd_lcores; 1259 } 1260 } 1261 #endif 1262 1263 } 1264 1265 1266 void 1267 reconfig(portid_t new_port_id, unsigned socket_id) 1268 { 1269 struct rte_port *port; 1270 1271 /* Reconfiguration of Ethernet ports. */ 1272 port = &ports[new_port_id]; 1273 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1274 1275 /* set flag to initialize port/queue */ 1276 port->need_reconfig = 1; 1277 port->need_reconfig_queues = 1; 1278 port->socket_id = socket_id; 1279 1280 init_port_config(); 1281 } 1282 1283 1284 int 1285 init_fwd_streams(void) 1286 { 1287 portid_t pid; 1288 struct rte_port *port; 1289 streamid_t sm_id, nb_fwd_streams_new; 1290 queueid_t q; 1291 1292 /* set socket id according to numa or not */ 1293 RTE_ETH_FOREACH_DEV(pid) { 1294 port = &ports[pid]; 1295 if (nb_rxq > port->dev_info.max_rx_queues) { 1296 printf("Fail: nb_rxq(%d) is greater than " 1297 "max_rx_queues(%d)\n", nb_rxq, 1298 port->dev_info.max_rx_queues); 1299 return -1; 1300 } 1301 if (nb_txq > port->dev_info.max_tx_queues) { 1302 printf("Fail: nb_txq(%d) is greater than " 1303 "max_tx_queues(%d)\n", nb_txq, 1304 port->dev_info.max_tx_queues); 1305 return -1; 1306 } 1307 if (numa_support) { 1308 if (port_numa[pid] != NUMA_NO_CONFIG) 1309 port->socket_id = port_numa[pid]; 1310 else { 1311 port->socket_id = rte_eth_dev_socket_id(pid); 1312 1313 /* 1314 * if socket_id is invalid, 1315 * set to the first available socket. 1316 */ 1317 if (check_socket_id(port->socket_id) < 0) 1318 port->socket_id = socket_ids[0]; 1319 } 1320 } 1321 else { 1322 if (socket_num == UMA_NO_CONFIG) 1323 port->socket_id = 0; 1324 else 1325 port->socket_id = socket_num; 1326 } 1327 } 1328 1329 q = RTE_MAX(nb_rxq, nb_txq); 1330 if (q == 0) { 1331 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1332 return -1; 1333 } 1334 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1335 if (nb_fwd_streams_new == nb_fwd_streams) 1336 return 0; 1337 /* clear the old */ 1338 if (fwd_streams != NULL) { 1339 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1340 if (fwd_streams[sm_id] == NULL) 1341 continue; 1342 rte_free(fwd_streams[sm_id]); 1343 fwd_streams[sm_id] = NULL; 1344 } 1345 rte_free(fwd_streams); 1346 fwd_streams = NULL; 1347 } 1348 1349 /* init new */ 1350 nb_fwd_streams = nb_fwd_streams_new; 1351 if (nb_fwd_streams) { 1352 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1353 sizeof(struct fwd_stream *) * nb_fwd_streams, 1354 RTE_CACHE_LINE_SIZE); 1355 if (fwd_streams == NULL) 1356 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1357 " (struct fwd_stream *)) failed\n", 1358 nb_fwd_streams); 1359 1360 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1361 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1362 " struct fwd_stream", sizeof(struct fwd_stream), 1363 RTE_CACHE_LINE_SIZE); 1364 if (fwd_streams[sm_id] == NULL) 1365 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1366 "(struct fwd_stream) failed\n"); 1367 } 1368 } 1369 1370 return 0; 1371 } 1372 1373 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1374 static void 1375 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1376 { 1377 unsigned int total_burst; 1378 unsigned int nb_burst; 1379 unsigned int burst_stats[3]; 1380 uint16_t pktnb_stats[3]; 1381 uint16_t nb_pkt; 1382 int burst_percent[3]; 1383 1384 /* 1385 * First compute the total number of packet bursts and the 1386 * two highest numbers of bursts of the same number of packets. 1387 */ 1388 total_burst = 0; 1389 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1390 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1391 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1392 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1393 if (nb_burst == 0) 1394 continue; 1395 total_burst += nb_burst; 1396 if (nb_burst > burst_stats[0]) { 1397 burst_stats[1] = burst_stats[0]; 1398 pktnb_stats[1] = pktnb_stats[0]; 1399 burst_stats[0] = nb_burst; 1400 pktnb_stats[0] = nb_pkt; 1401 } else if (nb_burst > burst_stats[1]) { 1402 burst_stats[1] = nb_burst; 1403 pktnb_stats[1] = nb_pkt; 1404 } 1405 } 1406 if (total_burst == 0) 1407 return; 1408 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1409 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1410 burst_percent[0], (int) pktnb_stats[0]); 1411 if (burst_stats[0] == total_burst) { 1412 printf("]\n"); 1413 return; 1414 } 1415 if (burst_stats[0] + burst_stats[1] == total_burst) { 1416 printf(" + %d%% of %d pkts]\n", 1417 100 - burst_percent[0], pktnb_stats[1]); 1418 return; 1419 } 1420 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1421 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1422 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1423 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1424 return; 1425 } 1426 printf(" + %d%% of %d pkts + %d%% of others]\n", 1427 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1428 } 1429 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1430 1431 static void 1432 fwd_stream_stats_display(streamid_t stream_id) 1433 { 1434 struct fwd_stream *fs; 1435 static const char *fwd_top_stats_border = "-------"; 1436 1437 fs = fwd_streams[stream_id]; 1438 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1439 (fs->fwd_dropped == 0)) 1440 return; 1441 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1442 "TX Port=%2d/Queue=%2d %s\n", 1443 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1444 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1445 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1446 " TX-dropped: %-14"PRIu64, 1447 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1448 1449 /* if checksum mode */ 1450 if (cur_fwd_eng == &csum_fwd_engine) { 1451 printf(" RX- bad IP checksum: %-14"PRIu64 1452 " Rx- bad L4 checksum: %-14"PRIu64 1453 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1454 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1455 fs->rx_bad_outer_l4_csum); 1456 } else { 1457 printf("\n"); 1458 } 1459 1460 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1461 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1462 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1463 #endif 1464 } 1465 1466 void 1467 fwd_stats_display(void) 1468 { 1469 static const char *fwd_stats_border = "----------------------"; 1470 static const char *acc_stats_border = "+++++++++++++++"; 1471 struct { 1472 struct fwd_stream *rx_stream; 1473 struct fwd_stream *tx_stream; 1474 uint64_t tx_dropped; 1475 uint64_t rx_bad_ip_csum; 1476 uint64_t rx_bad_l4_csum; 1477 uint64_t rx_bad_outer_l4_csum; 1478 } ports_stats[RTE_MAX_ETHPORTS]; 1479 uint64_t total_rx_dropped = 0; 1480 uint64_t total_tx_dropped = 0; 1481 uint64_t total_rx_nombuf = 0; 1482 struct rte_eth_stats stats; 1483 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1484 uint64_t fwd_cycles = 0; 1485 #endif 1486 uint64_t total_recv = 0; 1487 uint64_t total_xmit = 0; 1488 struct rte_port *port; 1489 streamid_t sm_id; 1490 portid_t pt_id; 1491 int i; 1492 1493 memset(ports_stats, 0, sizeof(ports_stats)); 1494 1495 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1496 struct fwd_stream *fs = fwd_streams[sm_id]; 1497 1498 if (cur_fwd_config.nb_fwd_streams > 1499 cur_fwd_config.nb_fwd_ports) { 1500 fwd_stream_stats_display(sm_id); 1501 } else { 1502 ports_stats[fs->tx_port].tx_stream = fs; 1503 ports_stats[fs->rx_port].rx_stream = fs; 1504 } 1505 1506 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1507 1508 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1509 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1510 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1511 fs->rx_bad_outer_l4_csum; 1512 1513 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1514 fwd_cycles += fs->core_cycles; 1515 #endif 1516 } 1517 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1518 uint8_t j; 1519 1520 pt_id = fwd_ports_ids[i]; 1521 port = &ports[pt_id]; 1522 1523 rte_eth_stats_get(pt_id, &stats); 1524 stats.ipackets -= port->stats.ipackets; 1525 stats.opackets -= port->stats.opackets; 1526 stats.ibytes -= port->stats.ibytes; 1527 stats.obytes -= port->stats.obytes; 1528 stats.imissed -= port->stats.imissed; 1529 stats.oerrors -= port->stats.oerrors; 1530 stats.rx_nombuf -= port->stats.rx_nombuf; 1531 1532 total_recv += stats.ipackets; 1533 total_xmit += stats.opackets; 1534 total_rx_dropped += stats.imissed; 1535 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1536 total_tx_dropped += stats.oerrors; 1537 total_rx_nombuf += stats.rx_nombuf; 1538 1539 printf("\n %s Forward statistics for port %-2d %s\n", 1540 fwd_stats_border, pt_id, fwd_stats_border); 1541 1542 if (!port->rx_queue_stats_mapping_enabled && 1543 !port->tx_queue_stats_mapping_enabled) { 1544 printf(" RX-packets: %-14"PRIu64 1545 " RX-dropped: %-14"PRIu64 1546 "RX-total: %-"PRIu64"\n", 1547 stats.ipackets, stats.imissed, 1548 stats.ipackets + stats.imissed); 1549 1550 if (cur_fwd_eng == &csum_fwd_engine) 1551 printf(" Bad-ipcsum: %-14"PRIu64 1552 " Bad-l4csum: %-14"PRIu64 1553 "Bad-outer-l4csum: %-14"PRIu64"\n", 1554 ports_stats[pt_id].rx_bad_ip_csum, 1555 ports_stats[pt_id].rx_bad_l4_csum, 1556 ports_stats[pt_id].rx_bad_outer_l4_csum); 1557 if (stats.ierrors + stats.rx_nombuf > 0) { 1558 printf(" RX-error: %-"PRIu64"\n", 1559 stats.ierrors); 1560 printf(" RX-nombufs: %-14"PRIu64"\n", 1561 stats.rx_nombuf); 1562 } 1563 1564 printf(" TX-packets: %-14"PRIu64 1565 " TX-dropped: %-14"PRIu64 1566 "TX-total: %-"PRIu64"\n", 1567 stats.opackets, ports_stats[pt_id].tx_dropped, 1568 stats.opackets + ports_stats[pt_id].tx_dropped); 1569 } else { 1570 printf(" RX-packets: %14"PRIu64 1571 " RX-dropped:%14"PRIu64 1572 " RX-total:%14"PRIu64"\n", 1573 stats.ipackets, stats.imissed, 1574 stats.ipackets + stats.imissed); 1575 1576 if (cur_fwd_eng == &csum_fwd_engine) 1577 printf(" Bad-ipcsum:%14"PRIu64 1578 " Bad-l4csum:%14"PRIu64 1579 " Bad-outer-l4csum: %-14"PRIu64"\n", 1580 ports_stats[pt_id].rx_bad_ip_csum, 1581 ports_stats[pt_id].rx_bad_l4_csum, 1582 ports_stats[pt_id].rx_bad_outer_l4_csum); 1583 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1584 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1585 printf(" RX-nombufs: %14"PRIu64"\n", 1586 stats.rx_nombuf); 1587 } 1588 1589 printf(" TX-packets: %14"PRIu64 1590 " TX-dropped:%14"PRIu64 1591 " TX-total:%14"PRIu64"\n", 1592 stats.opackets, ports_stats[pt_id].tx_dropped, 1593 stats.opackets + ports_stats[pt_id].tx_dropped); 1594 } 1595 1596 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1597 if (ports_stats[pt_id].rx_stream) 1598 pkt_burst_stats_display("RX", 1599 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1600 if (ports_stats[pt_id].tx_stream) 1601 pkt_burst_stats_display("TX", 1602 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1603 #endif 1604 1605 if (port->rx_queue_stats_mapping_enabled) { 1606 printf("\n"); 1607 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1608 printf(" Stats reg %2d RX-packets:%14"PRIu64 1609 " RX-errors:%14"PRIu64 1610 " RX-bytes:%14"PRIu64"\n", 1611 j, stats.q_ipackets[j], 1612 stats.q_errors[j], stats.q_ibytes[j]); 1613 } 1614 printf("\n"); 1615 } 1616 if (port->tx_queue_stats_mapping_enabled) { 1617 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1618 printf(" Stats reg %2d TX-packets:%14"PRIu64 1619 " TX-bytes:%14" 1620 PRIu64"\n", 1621 j, stats.q_opackets[j], 1622 stats.q_obytes[j]); 1623 } 1624 } 1625 1626 printf(" %s--------------------------------%s\n", 1627 fwd_stats_border, fwd_stats_border); 1628 } 1629 1630 printf("\n %s Accumulated forward statistics for all ports" 1631 "%s\n", 1632 acc_stats_border, acc_stats_border); 1633 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1634 "%-"PRIu64"\n" 1635 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1636 "%-"PRIu64"\n", 1637 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1638 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1639 if (total_rx_nombuf > 0) 1640 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1641 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1642 "%s\n", 1643 acc_stats_border, acc_stats_border); 1644 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1645 if (total_recv > 0) 1646 printf("\n CPU cycles/packet=%u (total cycles=" 1647 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1648 (unsigned int)(fwd_cycles / total_recv), 1649 fwd_cycles, total_recv); 1650 #endif 1651 } 1652 1653 void 1654 fwd_stats_reset(void) 1655 { 1656 streamid_t sm_id; 1657 portid_t pt_id; 1658 int i; 1659 1660 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1661 pt_id = fwd_ports_ids[i]; 1662 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1663 } 1664 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1665 struct fwd_stream *fs = fwd_streams[sm_id]; 1666 1667 fs->rx_packets = 0; 1668 fs->tx_packets = 0; 1669 fs->fwd_dropped = 0; 1670 fs->rx_bad_ip_csum = 0; 1671 fs->rx_bad_l4_csum = 0; 1672 fs->rx_bad_outer_l4_csum = 0; 1673 1674 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1675 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1676 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1677 #endif 1678 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1679 fs->core_cycles = 0; 1680 #endif 1681 } 1682 } 1683 1684 static void 1685 flush_fwd_rx_queues(void) 1686 { 1687 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1688 portid_t rxp; 1689 portid_t port_id; 1690 queueid_t rxq; 1691 uint16_t nb_rx; 1692 uint16_t i; 1693 uint8_t j; 1694 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1695 uint64_t timer_period; 1696 1697 /* convert to number of cycles */ 1698 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1699 1700 for (j = 0; j < 2; j++) { 1701 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1702 for (rxq = 0; rxq < nb_rxq; rxq++) { 1703 port_id = fwd_ports_ids[rxp]; 1704 /** 1705 * testpmd can stuck in the below do while loop 1706 * if rte_eth_rx_burst() always returns nonzero 1707 * packets. So timer is added to exit this loop 1708 * after 1sec timer expiry. 1709 */ 1710 prev_tsc = rte_rdtsc(); 1711 do { 1712 nb_rx = rte_eth_rx_burst(port_id, rxq, 1713 pkts_burst, MAX_PKT_BURST); 1714 for (i = 0; i < nb_rx; i++) 1715 rte_pktmbuf_free(pkts_burst[i]); 1716 1717 cur_tsc = rte_rdtsc(); 1718 diff_tsc = cur_tsc - prev_tsc; 1719 timer_tsc += diff_tsc; 1720 } while ((nb_rx > 0) && 1721 (timer_tsc < timer_period)); 1722 timer_tsc = 0; 1723 } 1724 } 1725 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1726 } 1727 } 1728 1729 static void 1730 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1731 { 1732 struct fwd_stream **fsm; 1733 streamid_t nb_fs; 1734 streamid_t sm_id; 1735 #ifdef RTE_LIBRTE_BITRATE 1736 uint64_t tics_per_1sec; 1737 uint64_t tics_datum; 1738 uint64_t tics_current; 1739 uint16_t i, cnt_ports; 1740 1741 cnt_ports = nb_ports; 1742 tics_datum = rte_rdtsc(); 1743 tics_per_1sec = rte_get_timer_hz(); 1744 #endif 1745 fsm = &fwd_streams[fc->stream_idx]; 1746 nb_fs = fc->stream_nb; 1747 do { 1748 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1749 (*pkt_fwd)(fsm[sm_id]); 1750 #ifdef RTE_LIBRTE_BITRATE 1751 if (bitrate_enabled != 0 && 1752 bitrate_lcore_id == rte_lcore_id()) { 1753 tics_current = rte_rdtsc(); 1754 if (tics_current - tics_datum >= tics_per_1sec) { 1755 /* Periodic bitrate calculation */ 1756 for (i = 0; i < cnt_ports; i++) 1757 rte_stats_bitrate_calc(bitrate_data, 1758 ports_ids[i]); 1759 tics_datum = tics_current; 1760 } 1761 } 1762 #endif 1763 #ifdef RTE_LIBRTE_LATENCY_STATS 1764 if (latencystats_enabled != 0 && 1765 latencystats_lcore_id == rte_lcore_id()) 1766 rte_latencystats_update(); 1767 #endif 1768 1769 } while (! fc->stopped); 1770 } 1771 1772 static int 1773 start_pkt_forward_on_core(void *fwd_arg) 1774 { 1775 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1776 cur_fwd_config.fwd_eng->packet_fwd); 1777 return 0; 1778 } 1779 1780 /* 1781 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1782 * Used to start communication flows in network loopback test configurations. 1783 */ 1784 static int 1785 run_one_txonly_burst_on_core(void *fwd_arg) 1786 { 1787 struct fwd_lcore *fwd_lc; 1788 struct fwd_lcore tmp_lcore; 1789 1790 fwd_lc = (struct fwd_lcore *) fwd_arg; 1791 tmp_lcore = *fwd_lc; 1792 tmp_lcore.stopped = 1; 1793 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1794 return 0; 1795 } 1796 1797 /* 1798 * Launch packet forwarding: 1799 * - Setup per-port forwarding context. 1800 * - launch logical cores with their forwarding configuration. 1801 */ 1802 static void 1803 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1804 { 1805 port_fwd_begin_t port_fwd_begin; 1806 unsigned int i; 1807 unsigned int lc_id; 1808 int diag; 1809 1810 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1811 if (port_fwd_begin != NULL) { 1812 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1813 (*port_fwd_begin)(fwd_ports_ids[i]); 1814 } 1815 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1816 lc_id = fwd_lcores_cpuids[i]; 1817 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1818 fwd_lcores[i]->stopped = 0; 1819 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1820 fwd_lcores[i], lc_id); 1821 if (diag != 0) 1822 printf("launch lcore %u failed - diag=%d\n", 1823 lc_id, diag); 1824 } 1825 } 1826 } 1827 1828 /* 1829 * Launch packet forwarding configuration. 1830 */ 1831 void 1832 start_packet_forwarding(int with_tx_first) 1833 { 1834 port_fwd_begin_t port_fwd_begin; 1835 port_fwd_end_t port_fwd_end; 1836 struct rte_port *port; 1837 unsigned int i; 1838 portid_t pt_id; 1839 1840 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1841 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1842 1843 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1844 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1845 1846 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1847 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1848 (!nb_rxq || !nb_txq)) 1849 rte_exit(EXIT_FAILURE, 1850 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1851 cur_fwd_eng->fwd_mode_name); 1852 1853 if (all_ports_started() == 0) { 1854 printf("Not all ports were started\n"); 1855 return; 1856 } 1857 if (test_done == 0) { 1858 printf("Packet forwarding already started\n"); 1859 return; 1860 } 1861 1862 1863 if(dcb_test) { 1864 for (i = 0; i < nb_fwd_ports; i++) { 1865 pt_id = fwd_ports_ids[i]; 1866 port = &ports[pt_id]; 1867 if (!port->dcb_flag) { 1868 printf("In DCB mode, all forwarding ports must " 1869 "be configured in this mode.\n"); 1870 return; 1871 } 1872 } 1873 if (nb_fwd_lcores == 1) { 1874 printf("In DCB mode,the nb forwarding cores " 1875 "should be larger than 1.\n"); 1876 return; 1877 } 1878 } 1879 test_done = 0; 1880 1881 fwd_config_setup(); 1882 1883 if(!no_flush_rx) 1884 flush_fwd_rx_queues(); 1885 1886 pkt_fwd_config_display(&cur_fwd_config); 1887 rxtx_config_display(); 1888 1889 fwd_stats_reset(); 1890 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1891 pt_id = fwd_ports_ids[i]; 1892 port = &ports[pt_id]; 1893 map_port_queue_stats_mapping_registers(pt_id, port); 1894 } 1895 if (with_tx_first) { 1896 port_fwd_begin = tx_only_engine.port_fwd_begin; 1897 if (port_fwd_begin != NULL) { 1898 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1899 (*port_fwd_begin)(fwd_ports_ids[i]); 1900 } 1901 while (with_tx_first--) { 1902 launch_packet_forwarding( 1903 run_one_txonly_burst_on_core); 1904 rte_eal_mp_wait_lcore(); 1905 } 1906 port_fwd_end = tx_only_engine.port_fwd_end; 1907 if (port_fwd_end != NULL) { 1908 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1909 (*port_fwd_end)(fwd_ports_ids[i]); 1910 } 1911 } 1912 launch_packet_forwarding(start_pkt_forward_on_core); 1913 } 1914 1915 void 1916 stop_packet_forwarding(void) 1917 { 1918 port_fwd_end_t port_fwd_end; 1919 lcoreid_t lc_id; 1920 portid_t pt_id; 1921 int i; 1922 1923 if (test_done) { 1924 printf("Packet forwarding not started\n"); 1925 return; 1926 } 1927 printf("Telling cores to stop..."); 1928 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1929 fwd_lcores[lc_id]->stopped = 1; 1930 printf("\nWaiting for lcores to finish...\n"); 1931 rte_eal_mp_wait_lcore(); 1932 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1933 if (port_fwd_end != NULL) { 1934 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1935 pt_id = fwd_ports_ids[i]; 1936 (*port_fwd_end)(pt_id); 1937 } 1938 } 1939 1940 fwd_stats_display(); 1941 1942 printf("\nDone.\n"); 1943 test_done = 1; 1944 } 1945 1946 void 1947 dev_set_link_up(portid_t pid) 1948 { 1949 if (rte_eth_dev_set_link_up(pid) < 0) 1950 printf("\nSet link up fail.\n"); 1951 } 1952 1953 void 1954 dev_set_link_down(portid_t pid) 1955 { 1956 if (rte_eth_dev_set_link_down(pid) < 0) 1957 printf("\nSet link down fail.\n"); 1958 } 1959 1960 static int 1961 all_ports_started(void) 1962 { 1963 portid_t pi; 1964 struct rte_port *port; 1965 1966 RTE_ETH_FOREACH_DEV(pi) { 1967 port = &ports[pi]; 1968 /* Check if there is a port which is not started */ 1969 if ((port->port_status != RTE_PORT_STARTED) && 1970 (port->slave_flag == 0)) 1971 return 0; 1972 } 1973 1974 /* No port is not started */ 1975 return 1; 1976 } 1977 1978 int 1979 port_is_stopped(portid_t port_id) 1980 { 1981 struct rte_port *port = &ports[port_id]; 1982 1983 if ((port->port_status != RTE_PORT_STOPPED) && 1984 (port->slave_flag == 0)) 1985 return 0; 1986 return 1; 1987 } 1988 1989 int 1990 all_ports_stopped(void) 1991 { 1992 portid_t pi; 1993 1994 RTE_ETH_FOREACH_DEV(pi) { 1995 if (!port_is_stopped(pi)) 1996 return 0; 1997 } 1998 1999 return 1; 2000 } 2001 2002 int 2003 port_is_started(portid_t port_id) 2004 { 2005 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2006 return 0; 2007 2008 if (ports[port_id].port_status != RTE_PORT_STARTED) 2009 return 0; 2010 2011 return 1; 2012 } 2013 2014 int 2015 start_port(portid_t pid) 2016 { 2017 int diag, need_check_link_status = -1; 2018 portid_t pi; 2019 queueid_t qi; 2020 struct rte_port *port; 2021 struct ether_addr mac_addr; 2022 2023 if (port_id_is_invalid(pid, ENABLED_WARN)) 2024 return 0; 2025 2026 if(dcb_config) 2027 dcb_test = 1; 2028 RTE_ETH_FOREACH_DEV(pi) { 2029 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2030 continue; 2031 2032 need_check_link_status = 0; 2033 port = &ports[pi]; 2034 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2035 RTE_PORT_HANDLING) == 0) { 2036 printf("Port %d is now not stopped\n", pi); 2037 continue; 2038 } 2039 2040 if (port->need_reconfig > 0) { 2041 port->need_reconfig = 0; 2042 2043 if (flow_isolate_all) { 2044 int ret = port_flow_isolate(pi, 1); 2045 if (ret) { 2046 printf("Failed to apply isolated" 2047 " mode on port %d\n", pi); 2048 return -1; 2049 } 2050 } 2051 configure_rxtx_dump_callbacks(0); 2052 printf("Configuring Port %d (socket %u)\n", pi, 2053 port->socket_id); 2054 /* configure port */ 2055 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 2056 &(port->dev_conf)); 2057 if (diag != 0) { 2058 if (rte_atomic16_cmpset(&(port->port_status), 2059 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2060 printf("Port %d can not be set back " 2061 "to stopped\n", pi); 2062 printf("Fail to configure port %d\n", pi); 2063 /* try to reconfigure port next time */ 2064 port->need_reconfig = 1; 2065 return -1; 2066 } 2067 } 2068 if (port->need_reconfig_queues > 0) { 2069 port->need_reconfig_queues = 0; 2070 /* setup tx queues */ 2071 for (qi = 0; qi < nb_txq; qi++) { 2072 if ((numa_support) && 2073 (txring_numa[pi] != NUMA_NO_CONFIG)) 2074 diag = rte_eth_tx_queue_setup(pi, qi, 2075 port->nb_tx_desc[qi], 2076 txring_numa[pi], 2077 &(port->tx_conf[qi])); 2078 else 2079 diag = rte_eth_tx_queue_setup(pi, qi, 2080 port->nb_tx_desc[qi], 2081 port->socket_id, 2082 &(port->tx_conf[qi])); 2083 2084 if (diag == 0) 2085 continue; 2086 2087 /* Fail to setup tx queue, return */ 2088 if (rte_atomic16_cmpset(&(port->port_status), 2089 RTE_PORT_HANDLING, 2090 RTE_PORT_STOPPED) == 0) 2091 printf("Port %d can not be set back " 2092 "to stopped\n", pi); 2093 printf("Fail to configure port %d tx queues\n", 2094 pi); 2095 /* try to reconfigure queues next time */ 2096 port->need_reconfig_queues = 1; 2097 return -1; 2098 } 2099 for (qi = 0; qi < nb_rxq; qi++) { 2100 /* setup rx queues */ 2101 if ((numa_support) && 2102 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2103 struct rte_mempool * mp = 2104 mbuf_pool_find(rxring_numa[pi]); 2105 if (mp == NULL) { 2106 printf("Failed to setup RX queue:" 2107 "No mempool allocation" 2108 " on the socket %d\n", 2109 rxring_numa[pi]); 2110 return -1; 2111 } 2112 2113 diag = rte_eth_rx_queue_setup(pi, qi, 2114 port->nb_rx_desc[qi], 2115 rxring_numa[pi], 2116 &(port->rx_conf[qi]), 2117 mp); 2118 } else { 2119 struct rte_mempool *mp = 2120 mbuf_pool_find(port->socket_id); 2121 if (mp == NULL) { 2122 printf("Failed to setup RX queue:" 2123 "No mempool allocation" 2124 " on the socket %d\n", 2125 port->socket_id); 2126 return -1; 2127 } 2128 diag = rte_eth_rx_queue_setup(pi, qi, 2129 port->nb_rx_desc[qi], 2130 port->socket_id, 2131 &(port->rx_conf[qi]), 2132 mp); 2133 } 2134 if (diag == 0) 2135 continue; 2136 2137 /* Fail to setup rx queue, return */ 2138 if (rte_atomic16_cmpset(&(port->port_status), 2139 RTE_PORT_HANDLING, 2140 RTE_PORT_STOPPED) == 0) 2141 printf("Port %d can not be set back " 2142 "to stopped\n", pi); 2143 printf("Fail to configure port %d rx queues\n", 2144 pi); 2145 /* try to reconfigure queues next time */ 2146 port->need_reconfig_queues = 1; 2147 return -1; 2148 } 2149 } 2150 configure_rxtx_dump_callbacks(verbose_level); 2151 /* start port */ 2152 if (rte_eth_dev_start(pi) < 0) { 2153 printf("Fail to start port %d\n", pi); 2154 2155 /* Fail to setup rx queue, return */ 2156 if (rte_atomic16_cmpset(&(port->port_status), 2157 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2158 printf("Port %d can not be set back to " 2159 "stopped\n", pi); 2160 continue; 2161 } 2162 2163 if (rte_atomic16_cmpset(&(port->port_status), 2164 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2165 printf("Port %d can not be set into started\n", pi); 2166 2167 rte_eth_macaddr_get(pi, &mac_addr); 2168 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2169 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2170 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2171 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2172 2173 /* at least one port started, need checking link status */ 2174 need_check_link_status = 1; 2175 } 2176 2177 if (need_check_link_status == 1 && !no_link_check) 2178 check_all_ports_link_status(RTE_PORT_ALL); 2179 else if (need_check_link_status == 0) 2180 printf("Please stop the ports first\n"); 2181 2182 printf("Done\n"); 2183 return 0; 2184 } 2185 2186 void 2187 stop_port(portid_t pid) 2188 { 2189 portid_t pi; 2190 struct rte_port *port; 2191 int need_check_link_status = 0; 2192 2193 if (dcb_test) { 2194 dcb_test = 0; 2195 dcb_config = 0; 2196 } 2197 2198 if (port_id_is_invalid(pid, ENABLED_WARN)) 2199 return; 2200 2201 printf("Stopping ports...\n"); 2202 2203 RTE_ETH_FOREACH_DEV(pi) { 2204 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2205 continue; 2206 2207 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2208 printf("Please remove port %d from forwarding configuration.\n", pi); 2209 continue; 2210 } 2211 2212 if (port_is_bonding_slave(pi)) { 2213 printf("Please remove port %d from bonded device.\n", pi); 2214 continue; 2215 } 2216 2217 port = &ports[pi]; 2218 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2219 RTE_PORT_HANDLING) == 0) 2220 continue; 2221 2222 rte_eth_dev_stop(pi); 2223 2224 if (rte_atomic16_cmpset(&(port->port_status), 2225 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2226 printf("Port %d can not be set into stopped\n", pi); 2227 need_check_link_status = 1; 2228 } 2229 if (need_check_link_status && !no_link_check) 2230 check_all_ports_link_status(RTE_PORT_ALL); 2231 2232 printf("Done\n"); 2233 } 2234 2235 static void 2236 remove_invalid_ports_in(portid_t *array, portid_t *total) 2237 { 2238 portid_t i; 2239 portid_t new_total = 0; 2240 2241 for (i = 0; i < *total; i++) 2242 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2243 array[new_total] = array[i]; 2244 new_total++; 2245 } 2246 *total = new_total; 2247 } 2248 2249 static void 2250 remove_invalid_ports(void) 2251 { 2252 remove_invalid_ports_in(ports_ids, &nb_ports); 2253 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2254 nb_cfg_ports = nb_fwd_ports; 2255 } 2256 2257 void 2258 close_port(portid_t pid) 2259 { 2260 portid_t pi; 2261 struct rte_port *port; 2262 2263 if (port_id_is_invalid(pid, ENABLED_WARN)) 2264 return; 2265 2266 printf("Closing ports...\n"); 2267 2268 RTE_ETH_FOREACH_DEV(pi) { 2269 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2270 continue; 2271 2272 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2273 printf("Please remove port %d from forwarding configuration.\n", pi); 2274 continue; 2275 } 2276 2277 if (port_is_bonding_slave(pi)) { 2278 printf("Please remove port %d from bonded device.\n", pi); 2279 continue; 2280 } 2281 2282 port = &ports[pi]; 2283 if (rte_atomic16_cmpset(&(port->port_status), 2284 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2285 printf("Port %d is already closed\n", pi); 2286 continue; 2287 } 2288 2289 if (rte_atomic16_cmpset(&(port->port_status), 2290 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2291 printf("Port %d is now not stopped\n", pi); 2292 continue; 2293 } 2294 2295 if (port->flow_list) 2296 port_flow_flush(pi); 2297 rte_eth_dev_close(pi); 2298 2299 remove_invalid_ports(); 2300 2301 if (rte_atomic16_cmpset(&(port->port_status), 2302 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2303 printf("Port %d cannot be set to closed\n", pi); 2304 } 2305 2306 printf("Done\n"); 2307 } 2308 2309 void 2310 reset_port(portid_t pid) 2311 { 2312 int diag; 2313 portid_t pi; 2314 struct rte_port *port; 2315 2316 if (port_id_is_invalid(pid, ENABLED_WARN)) 2317 return; 2318 2319 printf("Resetting ports...\n"); 2320 2321 RTE_ETH_FOREACH_DEV(pi) { 2322 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2323 continue; 2324 2325 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2326 printf("Please remove port %d from forwarding " 2327 "configuration.\n", pi); 2328 continue; 2329 } 2330 2331 if (port_is_bonding_slave(pi)) { 2332 printf("Please remove port %d from bonded device.\n", 2333 pi); 2334 continue; 2335 } 2336 2337 diag = rte_eth_dev_reset(pi); 2338 if (diag == 0) { 2339 port = &ports[pi]; 2340 port->need_reconfig = 1; 2341 port->need_reconfig_queues = 1; 2342 } else { 2343 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2344 } 2345 } 2346 2347 printf("Done\n"); 2348 } 2349 2350 void 2351 attach_port(char *identifier) 2352 { 2353 portid_t pi; 2354 struct rte_dev_iterator iterator; 2355 2356 printf("Attaching a new port...\n"); 2357 2358 if (identifier == NULL) { 2359 printf("Invalid parameters are specified\n"); 2360 return; 2361 } 2362 2363 if (rte_dev_probe(identifier) != 0) { 2364 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2365 return; 2366 } 2367 2368 /* first attach mode: event */ 2369 if (setup_on_probe_event) { 2370 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2371 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2372 if (ports[pi].port_status == RTE_PORT_HANDLING && 2373 ports[pi].need_setup != 0) 2374 setup_attached_port(pi); 2375 return; 2376 } 2377 2378 /* second attach mode: iterator */ 2379 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2380 /* setup ports matching the devargs used for probing */ 2381 if (port_is_forwarding(pi)) 2382 continue; /* port was already attached before */ 2383 setup_attached_port(pi); 2384 } 2385 } 2386 2387 static void 2388 setup_attached_port(portid_t pi) 2389 { 2390 unsigned int socket_id; 2391 2392 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2393 /* if socket_id is invalid, set to the first available socket. */ 2394 if (check_socket_id(socket_id) < 0) 2395 socket_id = socket_ids[0]; 2396 reconfig(pi, socket_id); 2397 rte_eth_promiscuous_enable(pi); 2398 2399 ports_ids[nb_ports++] = pi; 2400 fwd_ports_ids[nb_fwd_ports++] = pi; 2401 nb_cfg_ports = nb_fwd_ports; 2402 ports[pi].need_setup = 0; 2403 ports[pi].port_status = RTE_PORT_STOPPED; 2404 2405 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2406 printf("Done\n"); 2407 } 2408 2409 void 2410 detach_port_device(portid_t port_id) 2411 { 2412 struct rte_device *dev; 2413 portid_t sibling; 2414 2415 printf("Removing a device...\n"); 2416 2417 dev = rte_eth_devices[port_id].device; 2418 if (dev == NULL) { 2419 printf("Device already removed\n"); 2420 return; 2421 } 2422 2423 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2424 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2425 printf("Port not stopped\n"); 2426 return; 2427 } 2428 printf("Port was not closed\n"); 2429 if (ports[port_id].flow_list) 2430 port_flow_flush(port_id); 2431 } 2432 2433 if (rte_dev_remove(dev) != 0) { 2434 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2435 return; 2436 } 2437 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2438 /* reset mapping between old ports and removed device */ 2439 rte_eth_devices[sibling].device = NULL; 2440 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2441 /* sibling ports are forced to be closed */ 2442 ports[sibling].port_status = RTE_PORT_CLOSED; 2443 printf("Port %u is closed\n", sibling); 2444 } 2445 } 2446 2447 remove_invalid_ports(); 2448 2449 printf("Device of port %u is detached\n", port_id); 2450 printf("Now total ports is %d\n", nb_ports); 2451 printf("Done\n"); 2452 return; 2453 } 2454 2455 void 2456 pmd_test_exit(void) 2457 { 2458 struct rte_device *device; 2459 portid_t pt_id; 2460 int ret; 2461 int i; 2462 2463 if (test_done == 0) 2464 stop_packet_forwarding(); 2465 2466 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2467 if (mempools[i]) { 2468 if (mp_alloc_type == MP_ALLOC_ANON) 2469 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2470 NULL); 2471 } 2472 } 2473 if (ports != NULL) { 2474 no_link_check = 1; 2475 RTE_ETH_FOREACH_DEV(pt_id) { 2476 printf("\nStopping port %d...\n", pt_id); 2477 fflush(stdout); 2478 stop_port(pt_id); 2479 } 2480 RTE_ETH_FOREACH_DEV(pt_id) { 2481 printf("\nShutting down port %d...\n", pt_id); 2482 fflush(stdout); 2483 close_port(pt_id); 2484 2485 /* 2486 * This is a workaround to fix a virtio-user issue that 2487 * requires to call clean-up routine to remove existing 2488 * socket. 2489 * This workaround valid only for testpmd, needs a fix 2490 * valid for all applications. 2491 * TODO: Implement proper resource cleanup 2492 */ 2493 device = rte_eth_devices[pt_id].device; 2494 if (device && !strcmp(device->driver->name, "net_virtio_user")) 2495 detach_port_device(pt_id); 2496 } 2497 } 2498 2499 if (hot_plug) { 2500 ret = rte_dev_event_monitor_stop(); 2501 if (ret) { 2502 RTE_LOG(ERR, EAL, 2503 "fail to stop device event monitor."); 2504 return; 2505 } 2506 2507 ret = rte_dev_event_callback_unregister(NULL, 2508 dev_event_callback, NULL); 2509 if (ret < 0) { 2510 RTE_LOG(ERR, EAL, 2511 "fail to unregister device event callback.\n"); 2512 return; 2513 } 2514 2515 ret = rte_dev_hotplug_handle_disable(); 2516 if (ret) { 2517 RTE_LOG(ERR, EAL, 2518 "fail to disable hotplug handling.\n"); 2519 return; 2520 } 2521 } 2522 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2523 if (mempools[i]) 2524 rte_mempool_free(mempools[i]); 2525 } 2526 2527 printf("\nBye...\n"); 2528 } 2529 2530 typedef void (*cmd_func_t)(void); 2531 struct pmd_test_command { 2532 const char *cmd_name; 2533 cmd_func_t cmd_func; 2534 }; 2535 2536 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2537 2538 /* Check the link status of all ports in up to 9s, and print them finally */ 2539 static void 2540 check_all_ports_link_status(uint32_t port_mask) 2541 { 2542 #define CHECK_INTERVAL 100 /* 100ms */ 2543 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2544 portid_t portid; 2545 uint8_t count, all_ports_up, print_flag = 0; 2546 struct rte_eth_link link; 2547 2548 printf("Checking link statuses...\n"); 2549 fflush(stdout); 2550 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2551 all_ports_up = 1; 2552 RTE_ETH_FOREACH_DEV(portid) { 2553 if ((port_mask & (1 << portid)) == 0) 2554 continue; 2555 memset(&link, 0, sizeof(link)); 2556 rte_eth_link_get_nowait(portid, &link); 2557 /* print link status if flag set */ 2558 if (print_flag == 1) { 2559 if (link.link_status) 2560 printf( 2561 "Port%d Link Up. speed %u Mbps- %s\n", 2562 portid, link.link_speed, 2563 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2564 ("full-duplex") : ("half-duplex\n")); 2565 else 2566 printf("Port %d Link Down\n", portid); 2567 continue; 2568 } 2569 /* clear all_ports_up flag if any link down */ 2570 if (link.link_status == ETH_LINK_DOWN) { 2571 all_ports_up = 0; 2572 break; 2573 } 2574 } 2575 /* after finally printing all link status, get out */ 2576 if (print_flag == 1) 2577 break; 2578 2579 if (all_ports_up == 0) { 2580 fflush(stdout); 2581 rte_delay_ms(CHECK_INTERVAL); 2582 } 2583 2584 /* set the print_flag if all ports up or timeout */ 2585 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2586 print_flag = 1; 2587 } 2588 2589 if (lsc_interrupt) 2590 break; 2591 } 2592 } 2593 2594 /* 2595 * This callback is for remove a port for a device. It has limitation because 2596 * it is not for multiple port removal for a device. 2597 * TODO: the device detach invoke will plan to be removed from user side to 2598 * eal. And convert all PMDs to free port resources on ether device closing. 2599 */ 2600 static void 2601 rmv_port_callback(void *arg) 2602 { 2603 int need_to_start = 0; 2604 int org_no_link_check = no_link_check; 2605 portid_t port_id = (intptr_t)arg; 2606 2607 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2608 2609 if (!test_done && port_is_forwarding(port_id)) { 2610 need_to_start = 1; 2611 stop_packet_forwarding(); 2612 } 2613 no_link_check = 1; 2614 stop_port(port_id); 2615 no_link_check = org_no_link_check; 2616 close_port(port_id); 2617 detach_port_device(port_id); 2618 if (need_to_start) 2619 start_packet_forwarding(0); 2620 } 2621 2622 /* This function is used by the interrupt thread */ 2623 static int 2624 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2625 void *ret_param) 2626 { 2627 RTE_SET_USED(param); 2628 RTE_SET_USED(ret_param); 2629 2630 if (type >= RTE_ETH_EVENT_MAX) { 2631 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2632 port_id, __func__, type); 2633 fflush(stderr); 2634 } else if (event_print_mask & (UINT32_C(1) << type)) { 2635 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2636 eth_event_desc[type]); 2637 fflush(stdout); 2638 } 2639 2640 switch (type) { 2641 case RTE_ETH_EVENT_NEW: 2642 ports[port_id].need_setup = 1; 2643 ports[port_id].port_status = RTE_PORT_HANDLING; 2644 break; 2645 case RTE_ETH_EVENT_INTR_RMV: 2646 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2647 break; 2648 if (rte_eal_alarm_set(100000, 2649 rmv_port_callback, (void *)(intptr_t)port_id)) 2650 fprintf(stderr, "Could not set up deferred device removal\n"); 2651 break; 2652 default: 2653 break; 2654 } 2655 return 0; 2656 } 2657 2658 static int 2659 register_eth_event_callback(void) 2660 { 2661 int ret; 2662 enum rte_eth_event_type event; 2663 2664 for (event = RTE_ETH_EVENT_UNKNOWN; 2665 event < RTE_ETH_EVENT_MAX; event++) { 2666 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2667 event, 2668 eth_event_callback, 2669 NULL); 2670 if (ret != 0) { 2671 TESTPMD_LOG(ERR, "Failed to register callback for " 2672 "%s event\n", eth_event_desc[event]); 2673 return -1; 2674 } 2675 } 2676 2677 return 0; 2678 } 2679 2680 /* This function is used by the interrupt thread */ 2681 static void 2682 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2683 __rte_unused void *arg) 2684 { 2685 uint16_t port_id; 2686 int ret; 2687 2688 if (type >= RTE_DEV_EVENT_MAX) { 2689 fprintf(stderr, "%s called upon invalid event %d\n", 2690 __func__, type); 2691 fflush(stderr); 2692 } 2693 2694 switch (type) { 2695 case RTE_DEV_EVENT_REMOVE: 2696 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2697 device_name); 2698 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2699 if (ret) { 2700 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2701 device_name); 2702 return; 2703 } 2704 /* 2705 * Because the user's callback is invoked in eal interrupt 2706 * callback, the interrupt callback need to be finished before 2707 * it can be unregistered when detaching device. So finish 2708 * callback soon and use a deferred removal to detach device 2709 * is need. It is a workaround, once the device detaching be 2710 * moved into the eal in the future, the deferred removal could 2711 * be deleted. 2712 */ 2713 if (rte_eal_alarm_set(100000, 2714 rmv_port_callback, (void *)(intptr_t)port_id)) 2715 RTE_LOG(ERR, EAL, 2716 "Could not set up deferred device removal\n"); 2717 break; 2718 case RTE_DEV_EVENT_ADD: 2719 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2720 device_name); 2721 /* TODO: After finish kernel driver binding, 2722 * begin to attach port. 2723 */ 2724 break; 2725 default: 2726 break; 2727 } 2728 } 2729 2730 static int 2731 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2732 { 2733 uint16_t i; 2734 int diag; 2735 uint8_t mapping_found = 0; 2736 2737 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2738 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2739 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2740 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2741 tx_queue_stats_mappings[i].queue_id, 2742 tx_queue_stats_mappings[i].stats_counter_id); 2743 if (diag != 0) 2744 return diag; 2745 mapping_found = 1; 2746 } 2747 } 2748 if (mapping_found) 2749 port->tx_queue_stats_mapping_enabled = 1; 2750 return 0; 2751 } 2752 2753 static int 2754 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2755 { 2756 uint16_t i; 2757 int diag; 2758 uint8_t mapping_found = 0; 2759 2760 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2761 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2762 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2763 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2764 rx_queue_stats_mappings[i].queue_id, 2765 rx_queue_stats_mappings[i].stats_counter_id); 2766 if (diag != 0) 2767 return diag; 2768 mapping_found = 1; 2769 } 2770 } 2771 if (mapping_found) 2772 port->rx_queue_stats_mapping_enabled = 1; 2773 return 0; 2774 } 2775 2776 static void 2777 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2778 { 2779 int diag = 0; 2780 2781 diag = set_tx_queue_stats_mapping_registers(pi, port); 2782 if (diag != 0) { 2783 if (diag == -ENOTSUP) { 2784 port->tx_queue_stats_mapping_enabled = 0; 2785 printf("TX queue stats mapping not supported port id=%d\n", pi); 2786 } 2787 else 2788 rte_exit(EXIT_FAILURE, 2789 "set_tx_queue_stats_mapping_registers " 2790 "failed for port id=%d diag=%d\n", 2791 pi, diag); 2792 } 2793 2794 diag = set_rx_queue_stats_mapping_registers(pi, port); 2795 if (diag != 0) { 2796 if (diag == -ENOTSUP) { 2797 port->rx_queue_stats_mapping_enabled = 0; 2798 printf("RX queue stats mapping not supported port id=%d\n", pi); 2799 } 2800 else 2801 rte_exit(EXIT_FAILURE, 2802 "set_rx_queue_stats_mapping_registers " 2803 "failed for port id=%d diag=%d\n", 2804 pi, diag); 2805 } 2806 } 2807 2808 static void 2809 rxtx_port_config(struct rte_port *port) 2810 { 2811 uint16_t qid; 2812 2813 for (qid = 0; qid < nb_rxq; qid++) { 2814 port->rx_conf[qid] = port->dev_info.default_rxconf; 2815 2816 /* Check if any Rx parameters have been passed */ 2817 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2818 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2819 2820 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2821 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2822 2823 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2824 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2825 2826 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2827 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2828 2829 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2830 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2831 2832 port->nb_rx_desc[qid] = nb_rxd; 2833 } 2834 2835 for (qid = 0; qid < nb_txq; qid++) { 2836 port->tx_conf[qid] = port->dev_info.default_txconf; 2837 2838 /* Check if any Tx parameters have been passed */ 2839 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2840 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2841 2842 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2843 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2844 2845 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2846 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2847 2848 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2849 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2850 2851 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2852 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2853 2854 port->nb_tx_desc[qid] = nb_txd; 2855 } 2856 } 2857 2858 void 2859 init_port_config(void) 2860 { 2861 portid_t pid; 2862 struct rte_port *port; 2863 2864 RTE_ETH_FOREACH_DEV(pid) { 2865 port = &ports[pid]; 2866 port->dev_conf.fdir_conf = fdir_conf; 2867 rte_eth_dev_info_get(pid, &port->dev_info); 2868 if (nb_rxq > 1) { 2869 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2870 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2871 rss_hf & port->dev_info.flow_type_rss_offloads; 2872 } else { 2873 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2874 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2875 } 2876 2877 if (port->dcb_flag == 0) { 2878 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2879 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2880 else 2881 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2882 } 2883 2884 rxtx_port_config(port); 2885 2886 rte_eth_macaddr_get(pid, &port->eth_addr); 2887 2888 map_port_queue_stats_mapping_registers(pid, port); 2889 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2890 rte_pmd_ixgbe_bypass_init(pid); 2891 #endif 2892 2893 if (lsc_interrupt && 2894 (rte_eth_devices[pid].data->dev_flags & 2895 RTE_ETH_DEV_INTR_LSC)) 2896 port->dev_conf.intr_conf.lsc = 1; 2897 if (rmv_interrupt && 2898 (rte_eth_devices[pid].data->dev_flags & 2899 RTE_ETH_DEV_INTR_RMV)) 2900 port->dev_conf.intr_conf.rmv = 1; 2901 } 2902 } 2903 2904 void set_port_slave_flag(portid_t slave_pid) 2905 { 2906 struct rte_port *port; 2907 2908 port = &ports[slave_pid]; 2909 port->slave_flag = 1; 2910 } 2911 2912 void clear_port_slave_flag(portid_t slave_pid) 2913 { 2914 struct rte_port *port; 2915 2916 port = &ports[slave_pid]; 2917 port->slave_flag = 0; 2918 } 2919 2920 uint8_t port_is_bonding_slave(portid_t slave_pid) 2921 { 2922 struct rte_port *port; 2923 2924 port = &ports[slave_pid]; 2925 if ((rte_eth_devices[slave_pid].data->dev_flags & 2926 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2927 return 1; 2928 return 0; 2929 } 2930 2931 const uint16_t vlan_tags[] = { 2932 0, 1, 2, 3, 4, 5, 6, 7, 2933 8, 9, 10, 11, 12, 13, 14, 15, 2934 16, 17, 18, 19, 20, 21, 22, 23, 2935 24, 25, 26, 27, 28, 29, 30, 31 2936 }; 2937 2938 static int 2939 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 2940 enum dcb_mode_enable dcb_mode, 2941 enum rte_eth_nb_tcs num_tcs, 2942 uint8_t pfc_en) 2943 { 2944 uint8_t i; 2945 int32_t rc; 2946 struct rte_eth_rss_conf rss_conf; 2947 2948 /* 2949 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2950 * given above, and the number of traffic classes available for use. 2951 */ 2952 if (dcb_mode == DCB_VT_ENABLED) { 2953 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2954 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2955 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2956 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2957 2958 /* VMDQ+DCB RX and TX configurations */ 2959 vmdq_rx_conf->enable_default_pool = 0; 2960 vmdq_rx_conf->default_pool = 0; 2961 vmdq_rx_conf->nb_queue_pools = 2962 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2963 vmdq_tx_conf->nb_queue_pools = 2964 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2965 2966 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2967 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2968 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2969 vmdq_rx_conf->pool_map[i].pools = 2970 1 << (i % vmdq_rx_conf->nb_queue_pools); 2971 } 2972 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2973 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 2974 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 2975 } 2976 2977 /* set DCB mode of RX and TX of multiple queues */ 2978 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2979 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2980 } else { 2981 struct rte_eth_dcb_rx_conf *rx_conf = 2982 ð_conf->rx_adv_conf.dcb_rx_conf; 2983 struct rte_eth_dcb_tx_conf *tx_conf = 2984 ð_conf->tx_adv_conf.dcb_tx_conf; 2985 2986 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 2987 if (rc != 0) 2988 return rc; 2989 2990 rx_conf->nb_tcs = num_tcs; 2991 tx_conf->nb_tcs = num_tcs; 2992 2993 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2994 rx_conf->dcb_tc[i] = i % num_tcs; 2995 tx_conf->dcb_tc[i] = i % num_tcs; 2996 } 2997 2998 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2999 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3000 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3001 } 3002 3003 if (pfc_en) 3004 eth_conf->dcb_capability_en = 3005 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3006 else 3007 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3008 3009 return 0; 3010 } 3011 3012 int 3013 init_port_dcb_config(portid_t pid, 3014 enum dcb_mode_enable dcb_mode, 3015 enum rte_eth_nb_tcs num_tcs, 3016 uint8_t pfc_en) 3017 { 3018 struct rte_eth_conf port_conf; 3019 struct rte_port *rte_port; 3020 int retval; 3021 uint16_t i; 3022 3023 rte_port = &ports[pid]; 3024 3025 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3026 /* Enter DCB configuration status */ 3027 dcb_config = 1; 3028 3029 port_conf.rxmode = rte_port->dev_conf.rxmode; 3030 port_conf.txmode = rte_port->dev_conf.txmode; 3031 3032 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3033 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3034 if (retval < 0) 3035 return retval; 3036 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3037 3038 /* re-configure the device . */ 3039 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3040 if (retval < 0) 3041 return retval; 3042 rte_eth_dev_info_get(pid, &rte_port->dev_info); 3043 3044 /* If dev_info.vmdq_pool_base is greater than 0, 3045 * the queue id of vmdq pools is started after pf queues. 3046 */ 3047 if (dcb_mode == DCB_VT_ENABLED && 3048 rte_port->dev_info.vmdq_pool_base > 0) { 3049 printf("VMDQ_DCB multi-queue mode is nonsensical" 3050 " for port %d.", pid); 3051 return -1; 3052 } 3053 3054 /* Assume the ports in testpmd have the same dcb capability 3055 * and has the same number of rxq and txq in dcb mode 3056 */ 3057 if (dcb_mode == DCB_VT_ENABLED) { 3058 if (rte_port->dev_info.max_vfs > 0) { 3059 nb_rxq = rte_port->dev_info.nb_rx_queues; 3060 nb_txq = rte_port->dev_info.nb_tx_queues; 3061 } else { 3062 nb_rxq = rte_port->dev_info.max_rx_queues; 3063 nb_txq = rte_port->dev_info.max_tx_queues; 3064 } 3065 } else { 3066 /*if vt is disabled, use all pf queues */ 3067 if (rte_port->dev_info.vmdq_pool_base == 0) { 3068 nb_rxq = rte_port->dev_info.max_rx_queues; 3069 nb_txq = rte_port->dev_info.max_tx_queues; 3070 } else { 3071 nb_rxq = (queueid_t)num_tcs; 3072 nb_txq = (queueid_t)num_tcs; 3073 3074 } 3075 } 3076 rx_free_thresh = 64; 3077 3078 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3079 3080 rxtx_port_config(rte_port); 3081 /* VLAN filter */ 3082 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3083 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3084 rx_vft_set(pid, vlan_tags[i], 1); 3085 3086 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 3087 map_port_queue_stats_mapping_registers(pid, rte_port); 3088 3089 rte_port->dcb_flag = 1; 3090 3091 return 0; 3092 } 3093 3094 static void 3095 init_port(void) 3096 { 3097 /* Configuration of Ethernet ports. */ 3098 ports = rte_zmalloc("testpmd: ports", 3099 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3100 RTE_CACHE_LINE_SIZE); 3101 if (ports == NULL) { 3102 rte_exit(EXIT_FAILURE, 3103 "rte_zmalloc(%d struct rte_port) failed\n", 3104 RTE_MAX_ETHPORTS); 3105 } 3106 3107 /* Initialize ports NUMA structures */ 3108 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3109 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3110 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3111 } 3112 3113 static void 3114 force_quit(void) 3115 { 3116 pmd_test_exit(); 3117 prompt_exit(); 3118 } 3119 3120 static void 3121 print_stats(void) 3122 { 3123 uint8_t i; 3124 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3125 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3126 3127 /* Clear screen and move to top left */ 3128 printf("%s%s", clr, top_left); 3129 3130 printf("\nPort statistics ===================================="); 3131 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3132 nic_stats_display(fwd_ports_ids[i]); 3133 3134 fflush(stdout); 3135 } 3136 3137 static void 3138 signal_handler(int signum) 3139 { 3140 if (signum == SIGINT || signum == SIGTERM) { 3141 printf("\nSignal %d received, preparing to exit...\n", 3142 signum); 3143 #ifdef RTE_LIBRTE_PDUMP 3144 /* uninitialize packet capture framework */ 3145 rte_pdump_uninit(); 3146 #endif 3147 #ifdef RTE_LIBRTE_LATENCY_STATS 3148 rte_latencystats_uninit(); 3149 #endif 3150 force_quit(); 3151 /* Set flag to indicate the force termination. */ 3152 f_quit = 1; 3153 /* exit with the expected status */ 3154 signal(signum, SIG_DFL); 3155 kill(getpid(), signum); 3156 } 3157 } 3158 3159 int 3160 main(int argc, char** argv) 3161 { 3162 int diag; 3163 portid_t port_id; 3164 uint16_t count; 3165 int ret; 3166 3167 signal(SIGINT, signal_handler); 3168 signal(SIGTERM, signal_handler); 3169 3170 diag = rte_eal_init(argc, argv); 3171 if (diag < 0) 3172 rte_panic("Cannot init EAL\n"); 3173 3174 testpmd_logtype = rte_log_register("testpmd"); 3175 if (testpmd_logtype < 0) 3176 rte_panic("Cannot register log type"); 3177 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3178 3179 ret = register_eth_event_callback(); 3180 if (ret != 0) 3181 rte_panic("Cannot register for ethdev events"); 3182 3183 #ifdef RTE_LIBRTE_PDUMP 3184 /* initialize packet capture framework */ 3185 rte_pdump_init(); 3186 #endif 3187 3188 count = 0; 3189 RTE_ETH_FOREACH_DEV(port_id) { 3190 ports_ids[count] = port_id; 3191 count++; 3192 } 3193 nb_ports = (portid_t) count; 3194 if (nb_ports == 0) 3195 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3196 3197 /* allocate port structures, and init them */ 3198 init_port(); 3199 3200 set_def_fwd_config(); 3201 if (nb_lcores == 0) 3202 rte_panic("Empty set of forwarding logical cores - check the " 3203 "core mask supplied in the command parameters\n"); 3204 3205 /* Bitrate/latency stats disabled by default */ 3206 #ifdef RTE_LIBRTE_BITRATE 3207 bitrate_enabled = 0; 3208 #endif 3209 #ifdef RTE_LIBRTE_LATENCY_STATS 3210 latencystats_enabled = 0; 3211 #endif 3212 3213 /* on FreeBSD, mlockall() is disabled by default */ 3214 #ifdef RTE_EXEC_ENV_FREEBSD 3215 do_mlockall = 0; 3216 #else 3217 do_mlockall = 1; 3218 #endif 3219 3220 argc -= diag; 3221 argv += diag; 3222 if (argc > 1) 3223 launch_args_parse(argc, argv); 3224 3225 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3226 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3227 strerror(errno)); 3228 } 3229 3230 if (tx_first && interactive) 3231 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3232 "interactive mode.\n"); 3233 3234 if (tx_first && lsc_interrupt) { 3235 printf("Warning: lsc_interrupt needs to be off when " 3236 " using tx_first. Disabling.\n"); 3237 lsc_interrupt = 0; 3238 } 3239 3240 if (!nb_rxq && !nb_txq) 3241 printf("Warning: Either rx or tx queues should be non-zero\n"); 3242 3243 if (nb_rxq > 1 && nb_rxq > nb_txq) 3244 printf("Warning: nb_rxq=%d enables RSS configuration, " 3245 "but nb_txq=%d will prevent to fully test it.\n", 3246 nb_rxq, nb_txq); 3247 3248 init_config(); 3249 3250 if (hot_plug) { 3251 ret = rte_dev_hotplug_handle_enable(); 3252 if (ret) { 3253 RTE_LOG(ERR, EAL, 3254 "fail to enable hotplug handling."); 3255 return -1; 3256 } 3257 3258 ret = rte_dev_event_monitor_start(); 3259 if (ret) { 3260 RTE_LOG(ERR, EAL, 3261 "fail to start device event monitoring."); 3262 return -1; 3263 } 3264 3265 ret = rte_dev_event_callback_register(NULL, 3266 dev_event_callback, NULL); 3267 if (ret) { 3268 RTE_LOG(ERR, EAL, 3269 "fail to register device event callback\n"); 3270 return -1; 3271 } 3272 } 3273 3274 if (start_port(RTE_PORT_ALL) != 0) 3275 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3276 3277 /* set all ports to promiscuous mode by default */ 3278 RTE_ETH_FOREACH_DEV(port_id) 3279 rte_eth_promiscuous_enable(port_id); 3280 3281 /* Init metrics library */ 3282 rte_metrics_init(rte_socket_id()); 3283 3284 #ifdef RTE_LIBRTE_LATENCY_STATS 3285 if (latencystats_enabled != 0) { 3286 int ret = rte_latencystats_init(1, NULL); 3287 if (ret) 3288 printf("Warning: latencystats init()" 3289 " returned error %d\n", ret); 3290 printf("Latencystats running on lcore %d\n", 3291 latencystats_lcore_id); 3292 } 3293 #endif 3294 3295 /* Setup bitrate stats */ 3296 #ifdef RTE_LIBRTE_BITRATE 3297 if (bitrate_enabled != 0) { 3298 bitrate_data = rte_stats_bitrate_create(); 3299 if (bitrate_data == NULL) 3300 rte_exit(EXIT_FAILURE, 3301 "Could not allocate bitrate data.\n"); 3302 rte_stats_bitrate_reg(bitrate_data); 3303 } 3304 #endif 3305 3306 #ifdef RTE_LIBRTE_CMDLINE 3307 if (strlen(cmdline_filename) != 0) 3308 cmdline_read_from_file(cmdline_filename); 3309 3310 if (interactive == 1) { 3311 if (auto_start) { 3312 printf("Start automatic packet forwarding\n"); 3313 start_packet_forwarding(0); 3314 } 3315 prompt(); 3316 pmd_test_exit(); 3317 } else 3318 #endif 3319 { 3320 char c; 3321 int rc; 3322 3323 f_quit = 0; 3324 3325 printf("No commandline core given, start packet forwarding\n"); 3326 start_packet_forwarding(tx_first); 3327 if (stats_period != 0) { 3328 uint64_t prev_time = 0, cur_time, diff_time = 0; 3329 uint64_t timer_period; 3330 3331 /* Convert to number of cycles */ 3332 timer_period = stats_period * rte_get_timer_hz(); 3333 3334 while (f_quit == 0) { 3335 cur_time = rte_get_timer_cycles(); 3336 diff_time += cur_time - prev_time; 3337 3338 if (diff_time >= timer_period) { 3339 print_stats(); 3340 /* Reset the timer */ 3341 diff_time = 0; 3342 } 3343 /* Sleep to avoid unnecessary checks */ 3344 prev_time = cur_time; 3345 sleep(1); 3346 } 3347 } 3348 3349 printf("Press enter to exit\n"); 3350 rc = read(0, &c, 1); 3351 pmd_test_exit(); 3352 if (rc < 0) 3353 return 1; 3354 } 3355 3356 return 0; 3357 } 3358