1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 192 uint16_t mempool_flags; 193 194 struct fwd_config cur_fwd_config; 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 196 uint32_t retry_enabled; 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 199 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 201 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 202 * specified on command-line. */ 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 204 205 /* 206 * In container, it cannot terminate the process which running with 'stats-period' 207 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 208 */ 209 uint8_t f_quit; 210 211 /* 212 * Configuration of packet segments used by the "txonly" processing engine. 213 */ 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 216 TXONLY_DEF_PACKET_LEN, 217 }; 218 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 219 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 221 /**< Split policy for packets to TX. */ 222 223 uint8_t txonly_multi_flow; 224 /**< Whether multiple flows are generated in TXONLY mode. */ 225 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 228 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 230 uint8_t dcb_config = 0; 231 232 /* Whether the dcb is in testing status */ 233 uint8_t dcb_test = 0; 234 235 /* 236 * Configurable number of RX/TX queues. 237 */ 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 240 241 /* 242 * Configurable number of RX/TX ring descriptors. 243 * Defaults are supplied by drivers via ethdev. 244 */ 245 #define RTE_TEST_RX_DESC_DEFAULT 0 246 #define RTE_TEST_TX_DESC_DEFAULT 0 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 249 250 #define RTE_PMD_PARAM_UNSET -1 251 /* 252 * Configurable values of RX and TX ring threshold registers. 253 */ 254 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 258 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 262 263 /* 264 * Configurable value of RX free threshold. 265 */ 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX drop enable. 270 */ 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of TX free threshold. 275 */ 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX RS bit threshold. 280 */ 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of buffered packets before sending. 285 */ 286 uint16_t noisy_tx_sw_bufsz; 287 288 /* 289 * Configurable value of packet buffer timeout. 290 */ 291 uint16_t noisy_tx_sw_buf_flush_time; 292 293 /* 294 * Configurable value for size of VNF internal memory area 295 * used for simulating noisy neighbour behaviour 296 */ 297 uint64_t noisy_lkup_mem_sz; 298 299 /* 300 * Configurable value of number of random writes done in 301 * VNF simulation memory area. 302 */ 303 uint64_t noisy_lkup_num_writes; 304 305 /* 306 * Configurable value of number of random reads done in 307 * VNF simulation memory area. 308 */ 309 uint64_t noisy_lkup_num_reads; 310 311 /* 312 * Configurable value of number of random reads/writes done in 313 * VNF simulation memory area. 314 */ 315 uint64_t noisy_lkup_num_reads_writes; 316 317 /* 318 * Receive Side Scaling (RSS) configuration. 319 */ 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 321 322 /* 323 * Port topology configuration 324 */ 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 326 327 /* 328 * Avoids to flush all the RX streams before starts forwarding. 329 */ 330 uint8_t no_flush_rx = 0; /* flush by default */ 331 332 /* 333 * Flow API isolated mode. 334 */ 335 uint8_t flow_isolate_all; 336 337 /* 338 * Avoids to check link status when starting/stopping a port. 339 */ 340 uint8_t no_link_check = 0; /* check by default */ 341 342 /* 343 * Enable link status change notification 344 */ 345 uint8_t lsc_interrupt = 1; /* enabled by default */ 346 347 /* 348 * Enable device removal notification. 349 */ 350 uint8_t rmv_interrupt = 1; /* enabled by default */ 351 352 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 353 354 /* After attach, port setup is called on event or by iterator */ 355 bool setup_on_probe_event = true; 356 357 /* Pretty printing of ethdev events */ 358 static const char * const eth_event_desc[] = { 359 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 360 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 361 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 362 [RTE_ETH_EVENT_INTR_RESET] = "reset", 363 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 364 [RTE_ETH_EVENT_IPSEC] = "IPsec", 365 [RTE_ETH_EVENT_MACSEC] = "MACsec", 366 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 367 [RTE_ETH_EVENT_NEW] = "device probed", 368 [RTE_ETH_EVENT_DESTROY] = "device released", 369 [RTE_ETH_EVENT_MAX] = NULL, 370 }; 371 372 /* 373 * Display or mask ether events 374 * Default to all events except VF_MBOX 375 */ 376 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 377 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 378 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 379 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 380 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 381 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 382 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 383 /* 384 * Decide if all memory are locked for performance. 385 */ 386 int do_mlockall = 0; 387 388 /* 389 * NIC bypass mode configuration options. 390 */ 391 392 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 393 /* The NIC bypass watchdog timeout. */ 394 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 395 #endif 396 397 398 #ifdef RTE_LIBRTE_LATENCY_STATS 399 400 /* 401 * Set when latency stats is enabled in the commandline 402 */ 403 uint8_t latencystats_enabled; 404 405 /* 406 * Lcore ID to serive latency statistics. 407 */ 408 lcoreid_t latencystats_lcore_id = -1; 409 410 #endif 411 412 /* 413 * Ethernet device configuration. 414 */ 415 struct rte_eth_rxmode rx_mode = { 416 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 417 /**< Default maximum frame length. */ 418 }; 419 420 struct rte_eth_txmode tx_mode = { 421 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 422 }; 423 424 struct rte_fdir_conf fdir_conf = { 425 .mode = RTE_FDIR_MODE_NONE, 426 .pballoc = RTE_FDIR_PBALLOC_64K, 427 .status = RTE_FDIR_REPORT_STATUS, 428 .mask = { 429 .vlan_tci_mask = 0xFFEF, 430 .ipv4_mask = { 431 .src_ip = 0xFFFFFFFF, 432 .dst_ip = 0xFFFFFFFF, 433 }, 434 .ipv6_mask = { 435 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 436 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 437 }, 438 .src_port_mask = 0xFFFF, 439 .dst_port_mask = 0xFFFF, 440 .mac_addr_byte_mask = 0xFF, 441 .tunnel_type_mask = 1, 442 .tunnel_id_mask = 0xFFFFFFFF, 443 }, 444 .drop_queue = 127, 445 }; 446 447 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 448 449 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 450 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 451 452 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 453 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 454 455 uint16_t nb_tx_queue_stats_mappings = 0; 456 uint16_t nb_rx_queue_stats_mappings = 0; 457 458 /* 459 * Display zero values by default for xstats 460 */ 461 uint8_t xstats_hide_zero; 462 463 unsigned int num_sockets = 0; 464 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 465 466 #ifdef RTE_LIBRTE_BITRATE 467 /* Bitrate statistics */ 468 struct rte_stats_bitrates *bitrate_data; 469 lcoreid_t bitrate_lcore_id; 470 uint8_t bitrate_enabled; 471 #endif 472 473 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 474 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 475 476 struct vxlan_encap_conf vxlan_encap_conf = { 477 .select_ipv4 = 1, 478 .select_vlan = 0, 479 .select_tos_ttl = 0, 480 .vni = "\x00\x00\x00", 481 .udp_src = 0, 482 .udp_dst = RTE_BE16(4789), 483 .ipv4_src = RTE_IPv4(127, 0, 0, 1), 484 .ipv4_dst = RTE_IPv4(255, 255, 255, 255), 485 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 486 "\x00\x00\x00\x00\x00\x00\x00\x01", 487 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 488 "\x00\x00\x00\x00\x00\x00\x11\x11", 489 .vlan_tci = 0, 490 .ip_tos = 0, 491 .ip_ttl = 255, 492 .eth_src = "\x00\x00\x00\x00\x00\x00", 493 .eth_dst = "\xff\xff\xff\xff\xff\xff", 494 }; 495 496 struct nvgre_encap_conf nvgre_encap_conf = { 497 .select_ipv4 = 1, 498 .select_vlan = 0, 499 .tni = "\x00\x00\x00", 500 .ipv4_src = RTE_IPv4(127, 0, 0, 1), 501 .ipv4_dst = RTE_IPv4(255, 255, 255, 255), 502 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 503 "\x00\x00\x00\x00\x00\x00\x00\x01", 504 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 505 "\x00\x00\x00\x00\x00\x00\x11\x11", 506 .vlan_tci = 0, 507 .eth_src = "\x00\x00\x00\x00\x00\x00", 508 .eth_dst = "\xff\xff\xff\xff\xff\xff", 509 }; 510 511 /* Forward function declarations */ 512 static void setup_attached_port(portid_t pi); 513 static void map_port_queue_stats_mapping_registers(portid_t pi, 514 struct rte_port *port); 515 static void check_all_ports_link_status(uint32_t port_mask); 516 static int eth_event_callback(portid_t port_id, 517 enum rte_eth_event_type type, 518 void *param, void *ret_param); 519 static void dev_event_callback(const char *device_name, 520 enum rte_dev_event_type type, 521 void *param); 522 523 /* 524 * Check if all the ports are started. 525 * If yes, return positive value. If not, return zero. 526 */ 527 static int all_ports_started(void); 528 529 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 530 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 531 532 /* 533 * Helper function to check if socket is already discovered. 534 * If yes, return positive value. If not, return zero. 535 */ 536 int 537 new_socket_id(unsigned int socket_id) 538 { 539 unsigned int i; 540 541 for (i = 0; i < num_sockets; i++) { 542 if (socket_ids[i] == socket_id) 543 return 0; 544 } 545 return 1; 546 } 547 548 /* 549 * Setup default configuration. 550 */ 551 static void 552 set_default_fwd_lcores_config(void) 553 { 554 unsigned int i; 555 unsigned int nb_lc; 556 unsigned int sock_num; 557 558 nb_lc = 0; 559 for (i = 0; i < RTE_MAX_LCORE; i++) { 560 if (!rte_lcore_is_enabled(i)) 561 continue; 562 sock_num = rte_lcore_to_socket_id(i); 563 if (new_socket_id(sock_num)) { 564 if (num_sockets >= RTE_MAX_NUMA_NODES) { 565 rte_exit(EXIT_FAILURE, 566 "Total sockets greater than %u\n", 567 RTE_MAX_NUMA_NODES); 568 } 569 socket_ids[num_sockets++] = sock_num; 570 } 571 if (i == rte_get_master_lcore()) 572 continue; 573 fwd_lcores_cpuids[nb_lc++] = i; 574 } 575 nb_lcores = (lcoreid_t) nb_lc; 576 nb_cfg_lcores = nb_lcores; 577 nb_fwd_lcores = 1; 578 } 579 580 static void 581 set_def_peer_eth_addrs(void) 582 { 583 portid_t i; 584 585 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 586 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 587 peer_eth_addrs[i].addr_bytes[5] = i; 588 } 589 } 590 591 static void 592 set_default_fwd_ports_config(void) 593 { 594 portid_t pt_id; 595 int i = 0; 596 597 RTE_ETH_FOREACH_DEV(pt_id) { 598 fwd_ports_ids[i++] = pt_id; 599 600 /* Update sockets info according to the attached device */ 601 int socket_id = rte_eth_dev_socket_id(pt_id); 602 if (socket_id >= 0 && new_socket_id(socket_id)) { 603 if (num_sockets >= RTE_MAX_NUMA_NODES) { 604 rte_exit(EXIT_FAILURE, 605 "Total sockets greater than %u\n", 606 RTE_MAX_NUMA_NODES); 607 } 608 socket_ids[num_sockets++] = socket_id; 609 } 610 } 611 612 nb_cfg_ports = nb_ports; 613 nb_fwd_ports = nb_ports; 614 } 615 616 void 617 set_def_fwd_config(void) 618 { 619 set_default_fwd_lcores_config(); 620 set_def_peer_eth_addrs(); 621 set_default_fwd_ports_config(); 622 } 623 624 /* extremely pessimistic estimation of memory required to create a mempool */ 625 static int 626 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 627 { 628 unsigned int n_pages, mbuf_per_pg, leftover; 629 uint64_t total_mem, mbuf_mem, obj_sz; 630 631 /* there is no good way to predict how much space the mempool will 632 * occupy because it will allocate chunks on the fly, and some of those 633 * will come from default DPDK memory while some will come from our 634 * external memory, so just assume 128MB will be enough for everyone. 635 */ 636 uint64_t hdr_mem = 128 << 20; 637 638 /* account for possible non-contiguousness */ 639 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 640 if (obj_sz > pgsz) { 641 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 642 return -1; 643 } 644 645 mbuf_per_pg = pgsz / obj_sz; 646 leftover = (nb_mbufs % mbuf_per_pg) > 0; 647 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 648 649 mbuf_mem = n_pages * pgsz; 650 651 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 652 653 if (total_mem > SIZE_MAX) { 654 TESTPMD_LOG(ERR, "Memory size too big\n"); 655 return -1; 656 } 657 *out = (size_t)total_mem; 658 659 return 0; 660 } 661 662 static int 663 pagesz_flags(uint64_t page_sz) 664 { 665 /* as per mmap() manpage, all page sizes are log2 of page size 666 * shifted by MAP_HUGE_SHIFT 667 */ 668 int log2 = rte_log2_u64(page_sz); 669 670 return (log2 << HUGE_SHIFT); 671 } 672 673 static void * 674 alloc_mem(size_t memsz, size_t pgsz, bool huge) 675 { 676 void *addr; 677 int flags; 678 679 /* allocate anonymous hugepages */ 680 flags = MAP_ANONYMOUS | MAP_PRIVATE; 681 if (huge) 682 flags |= HUGE_FLAG | pagesz_flags(pgsz); 683 684 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 685 if (addr == MAP_FAILED) 686 return NULL; 687 688 return addr; 689 } 690 691 struct extmem_param { 692 void *addr; 693 size_t len; 694 size_t pgsz; 695 rte_iova_t *iova_table; 696 unsigned int iova_table_len; 697 }; 698 699 static int 700 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 701 bool huge) 702 { 703 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 704 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 705 unsigned int cur_page, n_pages, pgsz_idx; 706 size_t mem_sz, cur_pgsz; 707 rte_iova_t *iovas = NULL; 708 void *addr; 709 int ret; 710 711 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 712 /* skip anything that is too big */ 713 if (pgsizes[pgsz_idx] > SIZE_MAX) 714 continue; 715 716 cur_pgsz = pgsizes[pgsz_idx]; 717 718 /* if we were told not to allocate hugepages, override */ 719 if (!huge) 720 cur_pgsz = sysconf(_SC_PAGESIZE); 721 722 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 723 if (ret < 0) { 724 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 725 return -1; 726 } 727 728 /* allocate our memory */ 729 addr = alloc_mem(mem_sz, cur_pgsz, huge); 730 731 /* if we couldn't allocate memory with a specified page size, 732 * that doesn't mean we can't do it with other page sizes, so 733 * try another one. 734 */ 735 if (addr == NULL) 736 continue; 737 738 /* store IOVA addresses for every page in this memory area */ 739 n_pages = mem_sz / cur_pgsz; 740 741 iovas = malloc(sizeof(*iovas) * n_pages); 742 743 if (iovas == NULL) { 744 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 745 goto fail; 746 } 747 /* lock memory if it's not huge pages */ 748 if (!huge) 749 mlock(addr, mem_sz); 750 751 /* populate IOVA addresses */ 752 for (cur_page = 0; cur_page < n_pages; cur_page++) { 753 rte_iova_t iova; 754 size_t offset; 755 void *cur; 756 757 offset = cur_pgsz * cur_page; 758 cur = RTE_PTR_ADD(addr, offset); 759 760 /* touch the page before getting its IOVA */ 761 *(volatile char *)cur = 0; 762 763 iova = rte_mem_virt2iova(cur); 764 765 iovas[cur_page] = iova; 766 } 767 768 break; 769 } 770 /* if we couldn't allocate anything */ 771 if (iovas == NULL) 772 return -1; 773 774 param->addr = addr; 775 param->len = mem_sz; 776 param->pgsz = cur_pgsz; 777 param->iova_table = iovas; 778 param->iova_table_len = n_pages; 779 780 return 0; 781 fail: 782 if (iovas) 783 free(iovas); 784 if (addr) 785 munmap(addr, mem_sz); 786 787 return -1; 788 } 789 790 static int 791 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 792 { 793 struct extmem_param param; 794 int socket_id, ret; 795 796 memset(¶m, 0, sizeof(param)); 797 798 /* check if our heap exists */ 799 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 800 if (socket_id < 0) { 801 /* create our heap */ 802 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 803 if (ret < 0) { 804 TESTPMD_LOG(ERR, "Cannot create heap\n"); 805 return -1; 806 } 807 } 808 809 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 810 if (ret < 0) { 811 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 812 return -1; 813 } 814 815 /* we now have a valid memory area, so add it to heap */ 816 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 817 param.addr, param.len, param.iova_table, 818 param.iova_table_len, param.pgsz); 819 820 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 821 822 /* not needed any more */ 823 free(param.iova_table); 824 825 if (ret < 0) { 826 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 827 munmap(param.addr, param.len); 828 return -1; 829 } 830 831 /* success */ 832 833 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 834 param.len >> 20); 835 836 return 0; 837 } 838 static void 839 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 840 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 841 { 842 uint16_t pid = 0; 843 int ret; 844 845 RTE_ETH_FOREACH_DEV(pid) { 846 struct rte_eth_dev *dev = 847 &rte_eth_devices[pid]; 848 849 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 850 memhdr->len); 851 if (ret) { 852 TESTPMD_LOG(DEBUG, 853 "unable to DMA unmap addr 0x%p " 854 "for device %s\n", 855 memhdr->addr, dev->data->name); 856 } 857 } 858 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 859 if (ret) { 860 TESTPMD_LOG(DEBUG, 861 "unable to un-register addr 0x%p\n", memhdr->addr); 862 } 863 } 864 865 static void 866 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 867 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 868 { 869 uint16_t pid = 0; 870 size_t page_size = sysconf(_SC_PAGESIZE); 871 int ret; 872 873 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 874 page_size); 875 if (ret) { 876 TESTPMD_LOG(DEBUG, 877 "unable to register addr 0x%p\n", memhdr->addr); 878 return; 879 } 880 RTE_ETH_FOREACH_DEV(pid) { 881 struct rte_eth_dev *dev = 882 &rte_eth_devices[pid]; 883 884 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 885 memhdr->len); 886 if (ret) { 887 TESTPMD_LOG(DEBUG, 888 "unable to DMA map addr 0x%p " 889 "for device %s\n", 890 memhdr->addr, dev->data->name); 891 } 892 } 893 } 894 895 /* 896 * Configuration initialisation done once at init time. 897 */ 898 static struct rte_mempool * 899 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 900 unsigned int socket_id) 901 { 902 char pool_name[RTE_MEMPOOL_NAMESIZE]; 903 struct rte_mempool *rte_mp = NULL; 904 uint32_t mb_size; 905 906 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 907 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 908 909 TESTPMD_LOG(INFO, 910 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 911 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 912 913 switch (mp_alloc_type) { 914 case MP_ALLOC_NATIVE: 915 { 916 /* wrapper to rte_mempool_create() */ 917 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 918 rte_mbuf_best_mempool_ops()); 919 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 920 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 921 break; 922 } 923 case MP_ALLOC_ANON: 924 { 925 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 926 mb_size, (unsigned int) mb_mempool_cache, 927 sizeof(struct rte_pktmbuf_pool_private), 928 socket_id, mempool_flags); 929 if (rte_mp == NULL) 930 goto err; 931 932 if (rte_mempool_populate_anon(rte_mp) == 0) { 933 rte_mempool_free(rte_mp); 934 rte_mp = NULL; 935 goto err; 936 } 937 rte_pktmbuf_pool_init(rte_mp, NULL); 938 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 939 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 940 break; 941 } 942 case MP_ALLOC_XMEM: 943 case MP_ALLOC_XMEM_HUGE: 944 { 945 int heap_socket; 946 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 947 948 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 949 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 950 951 heap_socket = 952 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 953 if (heap_socket < 0) 954 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 955 956 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 957 rte_mbuf_best_mempool_ops()); 958 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 959 mb_mempool_cache, 0, mbuf_seg_size, 960 heap_socket); 961 break; 962 } 963 default: 964 { 965 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 966 } 967 } 968 969 err: 970 if (rte_mp == NULL) { 971 rte_exit(EXIT_FAILURE, 972 "Creation of mbuf pool for socket %u failed: %s\n", 973 socket_id, rte_strerror(rte_errno)); 974 } else if (verbose_level > 0) { 975 rte_mempool_dump(stdout, rte_mp); 976 } 977 return rte_mp; 978 } 979 980 /* 981 * Check given socket id is valid or not with NUMA mode, 982 * if valid, return 0, else return -1 983 */ 984 static int 985 check_socket_id(const unsigned int socket_id) 986 { 987 static int warning_once = 0; 988 989 if (new_socket_id(socket_id)) { 990 if (!warning_once && numa_support) 991 printf("Warning: NUMA should be configured manually by" 992 " using --port-numa-config and" 993 " --ring-numa-config parameters along with" 994 " --numa.\n"); 995 warning_once = 1; 996 return -1; 997 } 998 return 0; 999 } 1000 1001 /* 1002 * Get the allowed maximum number of RX queues. 1003 * *pid return the port id which has minimal value of 1004 * max_rx_queues in all ports. 1005 */ 1006 queueid_t 1007 get_allowed_max_nb_rxq(portid_t *pid) 1008 { 1009 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 1010 portid_t pi; 1011 struct rte_eth_dev_info dev_info; 1012 1013 RTE_ETH_FOREACH_DEV(pi) { 1014 rte_eth_dev_info_get(pi, &dev_info); 1015 if (dev_info.max_rx_queues < allowed_max_rxq) { 1016 allowed_max_rxq = dev_info.max_rx_queues; 1017 *pid = pi; 1018 } 1019 } 1020 return allowed_max_rxq; 1021 } 1022 1023 /* 1024 * Check input rxq is valid or not. 1025 * If input rxq is not greater than any of maximum number 1026 * of RX queues of all ports, it is valid. 1027 * if valid, return 0, else return -1 1028 */ 1029 int 1030 check_nb_rxq(queueid_t rxq) 1031 { 1032 queueid_t allowed_max_rxq; 1033 portid_t pid = 0; 1034 1035 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1036 if (rxq > allowed_max_rxq) { 1037 printf("Fail: input rxq (%u) can't be greater " 1038 "than max_rx_queues (%u) of port %u\n", 1039 rxq, 1040 allowed_max_rxq, 1041 pid); 1042 return -1; 1043 } 1044 return 0; 1045 } 1046 1047 /* 1048 * Get the allowed maximum number of TX queues. 1049 * *pid return the port id which has minimal value of 1050 * max_tx_queues in all ports. 1051 */ 1052 queueid_t 1053 get_allowed_max_nb_txq(portid_t *pid) 1054 { 1055 queueid_t allowed_max_txq = MAX_QUEUE_ID; 1056 portid_t pi; 1057 struct rte_eth_dev_info dev_info; 1058 1059 RTE_ETH_FOREACH_DEV(pi) { 1060 rte_eth_dev_info_get(pi, &dev_info); 1061 if (dev_info.max_tx_queues < allowed_max_txq) { 1062 allowed_max_txq = dev_info.max_tx_queues; 1063 *pid = pi; 1064 } 1065 } 1066 return allowed_max_txq; 1067 } 1068 1069 /* 1070 * Check input txq is valid or not. 1071 * If input txq is not greater than any of maximum number 1072 * of TX queues of all ports, it is valid. 1073 * if valid, return 0, else return -1 1074 */ 1075 int 1076 check_nb_txq(queueid_t txq) 1077 { 1078 queueid_t allowed_max_txq; 1079 portid_t pid = 0; 1080 1081 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1082 if (txq > allowed_max_txq) { 1083 printf("Fail: input txq (%u) can't be greater " 1084 "than max_tx_queues (%u) of port %u\n", 1085 txq, 1086 allowed_max_txq, 1087 pid); 1088 return -1; 1089 } 1090 return 0; 1091 } 1092 1093 static void 1094 init_config(void) 1095 { 1096 portid_t pid; 1097 struct rte_port *port; 1098 struct rte_mempool *mbp; 1099 unsigned int nb_mbuf_per_pool; 1100 lcoreid_t lc_id; 1101 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1102 struct rte_gro_param gro_param; 1103 uint32_t gso_types; 1104 int k; 1105 1106 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1107 1108 /* Configuration of logical cores. */ 1109 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1110 sizeof(struct fwd_lcore *) * nb_lcores, 1111 RTE_CACHE_LINE_SIZE); 1112 if (fwd_lcores == NULL) { 1113 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1114 "failed\n", nb_lcores); 1115 } 1116 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1117 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1118 sizeof(struct fwd_lcore), 1119 RTE_CACHE_LINE_SIZE); 1120 if (fwd_lcores[lc_id] == NULL) { 1121 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1122 "failed\n"); 1123 } 1124 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1125 } 1126 1127 RTE_ETH_FOREACH_DEV(pid) { 1128 port = &ports[pid]; 1129 /* Apply default TxRx configuration for all ports */ 1130 port->dev_conf.txmode = tx_mode; 1131 port->dev_conf.rxmode = rx_mode; 1132 rte_eth_dev_info_get(pid, &port->dev_info); 1133 1134 if (!(port->dev_info.tx_offload_capa & 1135 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1136 port->dev_conf.txmode.offloads &= 1137 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1138 if (!(port->dev_info.tx_offload_capa & 1139 DEV_TX_OFFLOAD_MATCH_METADATA)) 1140 port->dev_conf.txmode.offloads &= 1141 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1142 if (numa_support) { 1143 if (port_numa[pid] != NUMA_NO_CONFIG) 1144 port_per_socket[port_numa[pid]]++; 1145 else { 1146 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1147 1148 /* 1149 * if socket_id is invalid, 1150 * set to the first available socket. 1151 */ 1152 if (check_socket_id(socket_id) < 0) 1153 socket_id = socket_ids[0]; 1154 port_per_socket[socket_id]++; 1155 } 1156 } 1157 1158 /* Apply Rx offloads configuration */ 1159 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1160 port->rx_conf[k].offloads = 1161 port->dev_conf.rxmode.offloads; 1162 /* Apply Tx offloads configuration */ 1163 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1164 port->tx_conf[k].offloads = 1165 port->dev_conf.txmode.offloads; 1166 1167 /* set flag to initialize port/queue */ 1168 port->need_reconfig = 1; 1169 port->need_reconfig_queues = 1; 1170 port->tx_metadata = 0; 1171 } 1172 1173 /* 1174 * Create pools of mbuf. 1175 * If NUMA support is disabled, create a single pool of mbuf in 1176 * socket 0 memory by default. 1177 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1178 * 1179 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1180 * nb_txd can be configured at run time. 1181 */ 1182 if (param_total_num_mbufs) 1183 nb_mbuf_per_pool = param_total_num_mbufs; 1184 else { 1185 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1186 (nb_lcores * mb_mempool_cache) + 1187 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1188 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1189 } 1190 1191 if (numa_support) { 1192 uint8_t i; 1193 1194 for (i = 0; i < num_sockets; i++) 1195 mempools[i] = mbuf_pool_create(mbuf_data_size, 1196 nb_mbuf_per_pool, 1197 socket_ids[i]); 1198 } else { 1199 if (socket_num == UMA_NO_CONFIG) 1200 mempools[0] = mbuf_pool_create(mbuf_data_size, 1201 nb_mbuf_per_pool, 0); 1202 else 1203 mempools[socket_num] = mbuf_pool_create 1204 (mbuf_data_size, 1205 nb_mbuf_per_pool, 1206 socket_num); 1207 } 1208 1209 init_port_config(); 1210 1211 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1212 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1213 /* 1214 * Records which Mbuf pool to use by each logical core, if needed. 1215 */ 1216 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1217 mbp = mbuf_pool_find( 1218 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1219 1220 if (mbp == NULL) 1221 mbp = mbuf_pool_find(0); 1222 fwd_lcores[lc_id]->mbp = mbp; 1223 /* initialize GSO context */ 1224 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1225 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1226 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1227 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1228 RTE_ETHER_CRC_LEN; 1229 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1230 } 1231 1232 /* Configuration of packet forwarding streams. */ 1233 if (init_fwd_streams() < 0) 1234 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1235 1236 fwd_config_setup(); 1237 1238 /* create a gro context for each lcore */ 1239 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1240 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1241 gro_param.max_item_per_flow = MAX_PKT_BURST; 1242 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1243 gro_param.socket_id = rte_lcore_to_socket_id( 1244 fwd_lcores_cpuids[lc_id]); 1245 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1246 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1247 rte_exit(EXIT_FAILURE, 1248 "rte_gro_ctx_create() failed\n"); 1249 } 1250 } 1251 1252 #if defined RTE_LIBRTE_PMD_SOFTNIC 1253 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1254 RTE_ETH_FOREACH_DEV(pid) { 1255 port = &ports[pid]; 1256 const char *driver = port->dev_info.driver_name; 1257 1258 if (strcmp(driver, "net_softnic") == 0) 1259 port->softport.fwd_lcore_arg = fwd_lcores; 1260 } 1261 } 1262 #endif 1263 1264 } 1265 1266 1267 void 1268 reconfig(portid_t new_port_id, unsigned socket_id) 1269 { 1270 struct rte_port *port; 1271 1272 /* Reconfiguration of Ethernet ports. */ 1273 port = &ports[new_port_id]; 1274 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1275 1276 /* set flag to initialize port/queue */ 1277 port->need_reconfig = 1; 1278 port->need_reconfig_queues = 1; 1279 port->socket_id = socket_id; 1280 1281 init_port_config(); 1282 } 1283 1284 1285 int 1286 init_fwd_streams(void) 1287 { 1288 portid_t pid; 1289 struct rte_port *port; 1290 streamid_t sm_id, nb_fwd_streams_new; 1291 queueid_t q; 1292 1293 /* set socket id according to numa or not */ 1294 RTE_ETH_FOREACH_DEV(pid) { 1295 port = &ports[pid]; 1296 if (nb_rxq > port->dev_info.max_rx_queues) { 1297 printf("Fail: nb_rxq(%d) is greater than " 1298 "max_rx_queues(%d)\n", nb_rxq, 1299 port->dev_info.max_rx_queues); 1300 return -1; 1301 } 1302 if (nb_txq > port->dev_info.max_tx_queues) { 1303 printf("Fail: nb_txq(%d) is greater than " 1304 "max_tx_queues(%d)\n", nb_txq, 1305 port->dev_info.max_tx_queues); 1306 return -1; 1307 } 1308 if (numa_support) { 1309 if (port_numa[pid] != NUMA_NO_CONFIG) 1310 port->socket_id = port_numa[pid]; 1311 else { 1312 port->socket_id = rte_eth_dev_socket_id(pid); 1313 1314 /* 1315 * if socket_id is invalid, 1316 * set to the first available socket. 1317 */ 1318 if (check_socket_id(port->socket_id) < 0) 1319 port->socket_id = socket_ids[0]; 1320 } 1321 } 1322 else { 1323 if (socket_num == UMA_NO_CONFIG) 1324 port->socket_id = 0; 1325 else 1326 port->socket_id = socket_num; 1327 } 1328 } 1329 1330 q = RTE_MAX(nb_rxq, nb_txq); 1331 if (q == 0) { 1332 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1333 return -1; 1334 } 1335 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1336 if (nb_fwd_streams_new == nb_fwd_streams) 1337 return 0; 1338 /* clear the old */ 1339 if (fwd_streams != NULL) { 1340 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1341 if (fwd_streams[sm_id] == NULL) 1342 continue; 1343 rte_free(fwd_streams[sm_id]); 1344 fwd_streams[sm_id] = NULL; 1345 } 1346 rte_free(fwd_streams); 1347 fwd_streams = NULL; 1348 } 1349 1350 /* init new */ 1351 nb_fwd_streams = nb_fwd_streams_new; 1352 if (nb_fwd_streams) { 1353 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1354 sizeof(struct fwd_stream *) * nb_fwd_streams, 1355 RTE_CACHE_LINE_SIZE); 1356 if (fwd_streams == NULL) 1357 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1358 " (struct fwd_stream *)) failed\n", 1359 nb_fwd_streams); 1360 1361 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1362 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1363 " struct fwd_stream", sizeof(struct fwd_stream), 1364 RTE_CACHE_LINE_SIZE); 1365 if (fwd_streams[sm_id] == NULL) 1366 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1367 "(struct fwd_stream) failed\n"); 1368 } 1369 } 1370 1371 return 0; 1372 } 1373 1374 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1375 static void 1376 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1377 { 1378 unsigned int total_burst; 1379 unsigned int nb_burst; 1380 unsigned int burst_stats[3]; 1381 uint16_t pktnb_stats[3]; 1382 uint16_t nb_pkt; 1383 int burst_percent[3]; 1384 1385 /* 1386 * First compute the total number of packet bursts and the 1387 * two highest numbers of bursts of the same number of packets. 1388 */ 1389 total_burst = 0; 1390 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1391 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1392 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1393 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1394 if (nb_burst == 0) 1395 continue; 1396 total_burst += nb_burst; 1397 if (nb_burst > burst_stats[0]) { 1398 burst_stats[1] = burst_stats[0]; 1399 pktnb_stats[1] = pktnb_stats[0]; 1400 burst_stats[0] = nb_burst; 1401 pktnb_stats[0] = nb_pkt; 1402 } else if (nb_burst > burst_stats[1]) { 1403 burst_stats[1] = nb_burst; 1404 pktnb_stats[1] = nb_pkt; 1405 } 1406 } 1407 if (total_burst == 0) 1408 return; 1409 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1410 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1411 burst_percent[0], (int) pktnb_stats[0]); 1412 if (burst_stats[0] == total_burst) { 1413 printf("]\n"); 1414 return; 1415 } 1416 if (burst_stats[0] + burst_stats[1] == total_burst) { 1417 printf(" + %d%% of %d pkts]\n", 1418 100 - burst_percent[0], pktnb_stats[1]); 1419 return; 1420 } 1421 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1422 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1423 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1424 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1425 return; 1426 } 1427 printf(" + %d%% of %d pkts + %d%% of others]\n", 1428 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1429 } 1430 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1431 1432 static void 1433 fwd_stream_stats_display(streamid_t stream_id) 1434 { 1435 struct fwd_stream *fs; 1436 static const char *fwd_top_stats_border = "-------"; 1437 1438 fs = fwd_streams[stream_id]; 1439 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1440 (fs->fwd_dropped == 0)) 1441 return; 1442 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1443 "TX Port=%2d/Queue=%2d %s\n", 1444 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1445 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1446 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1447 " TX-dropped: %-14"PRIu64, 1448 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1449 1450 /* if checksum mode */ 1451 if (cur_fwd_eng == &csum_fwd_engine) { 1452 printf(" RX- bad IP checksum: %-14"PRIu64 1453 " Rx- bad L4 checksum: %-14"PRIu64 1454 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1455 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1456 fs->rx_bad_outer_l4_csum); 1457 } else { 1458 printf("\n"); 1459 } 1460 1461 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1462 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1463 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1464 #endif 1465 } 1466 1467 void 1468 fwd_stats_display(void) 1469 { 1470 static const char *fwd_stats_border = "----------------------"; 1471 static const char *acc_stats_border = "+++++++++++++++"; 1472 struct { 1473 struct fwd_stream *rx_stream; 1474 struct fwd_stream *tx_stream; 1475 uint64_t tx_dropped; 1476 uint64_t rx_bad_ip_csum; 1477 uint64_t rx_bad_l4_csum; 1478 uint64_t rx_bad_outer_l4_csum; 1479 } ports_stats[RTE_MAX_ETHPORTS]; 1480 uint64_t total_rx_dropped = 0; 1481 uint64_t total_tx_dropped = 0; 1482 uint64_t total_rx_nombuf = 0; 1483 struct rte_eth_stats stats; 1484 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1485 uint64_t fwd_cycles = 0; 1486 #endif 1487 uint64_t total_recv = 0; 1488 uint64_t total_xmit = 0; 1489 struct rte_port *port; 1490 streamid_t sm_id; 1491 portid_t pt_id; 1492 int i; 1493 1494 memset(ports_stats, 0, sizeof(ports_stats)); 1495 1496 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1497 struct fwd_stream *fs = fwd_streams[sm_id]; 1498 1499 if (cur_fwd_config.nb_fwd_streams > 1500 cur_fwd_config.nb_fwd_ports) { 1501 fwd_stream_stats_display(sm_id); 1502 } else { 1503 ports_stats[fs->tx_port].tx_stream = fs; 1504 ports_stats[fs->rx_port].rx_stream = fs; 1505 } 1506 1507 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1508 1509 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1510 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1511 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1512 fs->rx_bad_outer_l4_csum; 1513 1514 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1515 fwd_cycles += fs->core_cycles; 1516 #endif 1517 } 1518 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1519 uint8_t j; 1520 1521 pt_id = fwd_ports_ids[i]; 1522 port = &ports[pt_id]; 1523 1524 rte_eth_stats_get(pt_id, &stats); 1525 stats.ipackets -= port->stats.ipackets; 1526 stats.opackets -= port->stats.opackets; 1527 stats.ibytes -= port->stats.ibytes; 1528 stats.obytes -= port->stats.obytes; 1529 stats.imissed -= port->stats.imissed; 1530 stats.oerrors -= port->stats.oerrors; 1531 stats.rx_nombuf -= port->stats.rx_nombuf; 1532 1533 total_recv += stats.ipackets; 1534 total_xmit += stats.opackets; 1535 total_rx_dropped += stats.imissed; 1536 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1537 total_tx_dropped += stats.oerrors; 1538 total_rx_nombuf += stats.rx_nombuf; 1539 1540 printf("\n %s Forward statistics for port %-2d %s\n", 1541 fwd_stats_border, pt_id, fwd_stats_border); 1542 1543 if (!port->rx_queue_stats_mapping_enabled && 1544 !port->tx_queue_stats_mapping_enabled) { 1545 printf(" RX-packets: %-14"PRIu64 1546 " RX-dropped: %-14"PRIu64 1547 "RX-total: %-"PRIu64"\n", 1548 stats.ipackets, stats.imissed, 1549 stats.ipackets + stats.imissed); 1550 1551 if (cur_fwd_eng == &csum_fwd_engine) 1552 printf(" Bad-ipcsum: %-14"PRIu64 1553 " Bad-l4csum: %-14"PRIu64 1554 "Bad-outer-l4csum: %-14"PRIu64"\n", 1555 ports_stats[pt_id].rx_bad_ip_csum, 1556 ports_stats[pt_id].rx_bad_l4_csum, 1557 ports_stats[pt_id].rx_bad_outer_l4_csum); 1558 if (stats.ierrors + stats.rx_nombuf > 0) { 1559 printf(" RX-error: %-"PRIu64"\n", 1560 stats.ierrors); 1561 printf(" RX-nombufs: %-14"PRIu64"\n", 1562 stats.rx_nombuf); 1563 } 1564 1565 printf(" TX-packets: %-14"PRIu64 1566 " TX-dropped: %-14"PRIu64 1567 "TX-total: %-"PRIu64"\n", 1568 stats.opackets, ports_stats[pt_id].tx_dropped, 1569 stats.opackets + ports_stats[pt_id].tx_dropped); 1570 } else { 1571 printf(" RX-packets: %14"PRIu64 1572 " RX-dropped:%14"PRIu64 1573 " RX-total:%14"PRIu64"\n", 1574 stats.ipackets, stats.imissed, 1575 stats.ipackets + stats.imissed); 1576 1577 if (cur_fwd_eng == &csum_fwd_engine) 1578 printf(" Bad-ipcsum:%14"PRIu64 1579 " Bad-l4csum:%14"PRIu64 1580 " Bad-outer-l4csum: %-14"PRIu64"\n", 1581 ports_stats[pt_id].rx_bad_ip_csum, 1582 ports_stats[pt_id].rx_bad_l4_csum, 1583 ports_stats[pt_id].rx_bad_outer_l4_csum); 1584 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1585 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1586 printf(" RX-nombufs: %14"PRIu64"\n", 1587 stats.rx_nombuf); 1588 } 1589 1590 printf(" TX-packets: %14"PRIu64 1591 " TX-dropped:%14"PRIu64 1592 " TX-total:%14"PRIu64"\n", 1593 stats.opackets, ports_stats[pt_id].tx_dropped, 1594 stats.opackets + ports_stats[pt_id].tx_dropped); 1595 } 1596 1597 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1598 if (ports_stats[pt_id].rx_stream) 1599 pkt_burst_stats_display("RX", 1600 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1601 if (ports_stats[pt_id].tx_stream) 1602 pkt_burst_stats_display("TX", 1603 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1604 #endif 1605 1606 if (port->rx_queue_stats_mapping_enabled) { 1607 printf("\n"); 1608 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1609 printf(" Stats reg %2d RX-packets:%14"PRIu64 1610 " RX-errors:%14"PRIu64 1611 " RX-bytes:%14"PRIu64"\n", 1612 j, stats.q_ipackets[j], 1613 stats.q_errors[j], stats.q_ibytes[j]); 1614 } 1615 printf("\n"); 1616 } 1617 if (port->tx_queue_stats_mapping_enabled) { 1618 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1619 printf(" Stats reg %2d TX-packets:%14"PRIu64 1620 " TX-bytes:%14" 1621 PRIu64"\n", 1622 j, stats.q_opackets[j], 1623 stats.q_obytes[j]); 1624 } 1625 } 1626 1627 printf(" %s--------------------------------%s\n", 1628 fwd_stats_border, fwd_stats_border); 1629 } 1630 1631 printf("\n %s Accumulated forward statistics for all ports" 1632 "%s\n", 1633 acc_stats_border, acc_stats_border); 1634 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1635 "%-"PRIu64"\n" 1636 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1637 "%-"PRIu64"\n", 1638 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1639 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1640 if (total_rx_nombuf > 0) 1641 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1642 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1643 "%s\n", 1644 acc_stats_border, acc_stats_border); 1645 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1646 if (total_recv > 0) 1647 printf("\n CPU cycles/packet=%u (total cycles=" 1648 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1649 (unsigned int)(fwd_cycles / total_recv), 1650 fwd_cycles, total_recv); 1651 #endif 1652 } 1653 1654 void 1655 fwd_stats_reset(void) 1656 { 1657 streamid_t sm_id; 1658 portid_t pt_id; 1659 int i; 1660 1661 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1662 pt_id = fwd_ports_ids[i]; 1663 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1664 } 1665 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1666 struct fwd_stream *fs = fwd_streams[sm_id]; 1667 1668 fs->rx_packets = 0; 1669 fs->tx_packets = 0; 1670 fs->fwd_dropped = 0; 1671 fs->rx_bad_ip_csum = 0; 1672 fs->rx_bad_l4_csum = 0; 1673 fs->rx_bad_outer_l4_csum = 0; 1674 1675 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1676 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1677 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1678 #endif 1679 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1680 fs->core_cycles = 0; 1681 #endif 1682 } 1683 } 1684 1685 static void 1686 flush_fwd_rx_queues(void) 1687 { 1688 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1689 portid_t rxp; 1690 portid_t port_id; 1691 queueid_t rxq; 1692 uint16_t nb_rx; 1693 uint16_t i; 1694 uint8_t j; 1695 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1696 uint64_t timer_period; 1697 1698 /* convert to number of cycles */ 1699 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1700 1701 for (j = 0; j < 2; j++) { 1702 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1703 for (rxq = 0; rxq < nb_rxq; rxq++) { 1704 port_id = fwd_ports_ids[rxp]; 1705 /** 1706 * testpmd can stuck in the below do while loop 1707 * if rte_eth_rx_burst() always returns nonzero 1708 * packets. So timer is added to exit this loop 1709 * after 1sec timer expiry. 1710 */ 1711 prev_tsc = rte_rdtsc(); 1712 do { 1713 nb_rx = rte_eth_rx_burst(port_id, rxq, 1714 pkts_burst, MAX_PKT_BURST); 1715 for (i = 0; i < nb_rx; i++) 1716 rte_pktmbuf_free(pkts_burst[i]); 1717 1718 cur_tsc = rte_rdtsc(); 1719 diff_tsc = cur_tsc - prev_tsc; 1720 timer_tsc += diff_tsc; 1721 } while ((nb_rx > 0) && 1722 (timer_tsc < timer_period)); 1723 timer_tsc = 0; 1724 } 1725 } 1726 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1727 } 1728 } 1729 1730 static void 1731 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1732 { 1733 struct fwd_stream **fsm; 1734 streamid_t nb_fs; 1735 streamid_t sm_id; 1736 #ifdef RTE_LIBRTE_BITRATE 1737 uint64_t tics_per_1sec; 1738 uint64_t tics_datum; 1739 uint64_t tics_current; 1740 uint16_t i, cnt_ports; 1741 1742 cnt_ports = nb_ports; 1743 tics_datum = rte_rdtsc(); 1744 tics_per_1sec = rte_get_timer_hz(); 1745 #endif 1746 fsm = &fwd_streams[fc->stream_idx]; 1747 nb_fs = fc->stream_nb; 1748 do { 1749 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1750 (*pkt_fwd)(fsm[sm_id]); 1751 #ifdef RTE_LIBRTE_BITRATE 1752 if (bitrate_enabled != 0 && 1753 bitrate_lcore_id == rte_lcore_id()) { 1754 tics_current = rte_rdtsc(); 1755 if (tics_current - tics_datum >= tics_per_1sec) { 1756 /* Periodic bitrate calculation */ 1757 for (i = 0; i < cnt_ports; i++) 1758 rte_stats_bitrate_calc(bitrate_data, 1759 ports_ids[i]); 1760 tics_datum = tics_current; 1761 } 1762 } 1763 #endif 1764 #ifdef RTE_LIBRTE_LATENCY_STATS 1765 if (latencystats_enabled != 0 && 1766 latencystats_lcore_id == rte_lcore_id()) 1767 rte_latencystats_update(); 1768 #endif 1769 1770 } while (! fc->stopped); 1771 } 1772 1773 static int 1774 start_pkt_forward_on_core(void *fwd_arg) 1775 { 1776 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1777 cur_fwd_config.fwd_eng->packet_fwd); 1778 return 0; 1779 } 1780 1781 /* 1782 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1783 * Used to start communication flows in network loopback test configurations. 1784 */ 1785 static int 1786 run_one_txonly_burst_on_core(void *fwd_arg) 1787 { 1788 struct fwd_lcore *fwd_lc; 1789 struct fwd_lcore tmp_lcore; 1790 1791 fwd_lc = (struct fwd_lcore *) fwd_arg; 1792 tmp_lcore = *fwd_lc; 1793 tmp_lcore.stopped = 1; 1794 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1795 return 0; 1796 } 1797 1798 /* 1799 * Launch packet forwarding: 1800 * - Setup per-port forwarding context. 1801 * - launch logical cores with their forwarding configuration. 1802 */ 1803 static void 1804 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1805 { 1806 port_fwd_begin_t port_fwd_begin; 1807 unsigned int i; 1808 unsigned int lc_id; 1809 int diag; 1810 1811 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1812 if (port_fwd_begin != NULL) { 1813 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1814 (*port_fwd_begin)(fwd_ports_ids[i]); 1815 } 1816 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1817 lc_id = fwd_lcores_cpuids[i]; 1818 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1819 fwd_lcores[i]->stopped = 0; 1820 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1821 fwd_lcores[i], lc_id); 1822 if (diag != 0) 1823 printf("launch lcore %u failed - diag=%d\n", 1824 lc_id, diag); 1825 } 1826 } 1827 } 1828 1829 /* 1830 * Launch packet forwarding configuration. 1831 */ 1832 void 1833 start_packet_forwarding(int with_tx_first) 1834 { 1835 port_fwd_begin_t port_fwd_begin; 1836 port_fwd_end_t port_fwd_end; 1837 struct rte_port *port; 1838 unsigned int i; 1839 portid_t pt_id; 1840 1841 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1842 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1843 1844 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1845 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1846 1847 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1848 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1849 (!nb_rxq || !nb_txq)) 1850 rte_exit(EXIT_FAILURE, 1851 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1852 cur_fwd_eng->fwd_mode_name); 1853 1854 if (all_ports_started() == 0) { 1855 printf("Not all ports were started\n"); 1856 return; 1857 } 1858 if (test_done == 0) { 1859 printf("Packet forwarding already started\n"); 1860 return; 1861 } 1862 1863 1864 if(dcb_test) { 1865 for (i = 0; i < nb_fwd_ports; i++) { 1866 pt_id = fwd_ports_ids[i]; 1867 port = &ports[pt_id]; 1868 if (!port->dcb_flag) { 1869 printf("In DCB mode, all forwarding ports must " 1870 "be configured in this mode.\n"); 1871 return; 1872 } 1873 } 1874 if (nb_fwd_lcores == 1) { 1875 printf("In DCB mode,the nb forwarding cores " 1876 "should be larger than 1.\n"); 1877 return; 1878 } 1879 } 1880 test_done = 0; 1881 1882 fwd_config_setup(); 1883 1884 if(!no_flush_rx) 1885 flush_fwd_rx_queues(); 1886 1887 pkt_fwd_config_display(&cur_fwd_config); 1888 rxtx_config_display(); 1889 1890 fwd_stats_reset(); 1891 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1892 pt_id = fwd_ports_ids[i]; 1893 port = &ports[pt_id]; 1894 map_port_queue_stats_mapping_registers(pt_id, port); 1895 } 1896 if (with_tx_first) { 1897 port_fwd_begin = tx_only_engine.port_fwd_begin; 1898 if (port_fwd_begin != NULL) { 1899 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1900 (*port_fwd_begin)(fwd_ports_ids[i]); 1901 } 1902 while (with_tx_first--) { 1903 launch_packet_forwarding( 1904 run_one_txonly_burst_on_core); 1905 rte_eal_mp_wait_lcore(); 1906 } 1907 port_fwd_end = tx_only_engine.port_fwd_end; 1908 if (port_fwd_end != NULL) { 1909 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1910 (*port_fwd_end)(fwd_ports_ids[i]); 1911 } 1912 } 1913 launch_packet_forwarding(start_pkt_forward_on_core); 1914 } 1915 1916 void 1917 stop_packet_forwarding(void) 1918 { 1919 port_fwd_end_t port_fwd_end; 1920 lcoreid_t lc_id; 1921 portid_t pt_id; 1922 int i; 1923 1924 if (test_done) { 1925 printf("Packet forwarding not started\n"); 1926 return; 1927 } 1928 printf("Telling cores to stop..."); 1929 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1930 fwd_lcores[lc_id]->stopped = 1; 1931 printf("\nWaiting for lcores to finish...\n"); 1932 rte_eal_mp_wait_lcore(); 1933 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1934 if (port_fwd_end != NULL) { 1935 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1936 pt_id = fwd_ports_ids[i]; 1937 (*port_fwd_end)(pt_id); 1938 } 1939 } 1940 1941 fwd_stats_display(); 1942 1943 printf("\nDone.\n"); 1944 test_done = 1; 1945 } 1946 1947 void 1948 dev_set_link_up(portid_t pid) 1949 { 1950 if (rte_eth_dev_set_link_up(pid) < 0) 1951 printf("\nSet link up fail.\n"); 1952 } 1953 1954 void 1955 dev_set_link_down(portid_t pid) 1956 { 1957 if (rte_eth_dev_set_link_down(pid) < 0) 1958 printf("\nSet link down fail.\n"); 1959 } 1960 1961 static int 1962 all_ports_started(void) 1963 { 1964 portid_t pi; 1965 struct rte_port *port; 1966 1967 RTE_ETH_FOREACH_DEV(pi) { 1968 port = &ports[pi]; 1969 /* Check if there is a port which is not started */ 1970 if ((port->port_status != RTE_PORT_STARTED) && 1971 (port->slave_flag == 0)) 1972 return 0; 1973 } 1974 1975 /* No port is not started */ 1976 return 1; 1977 } 1978 1979 int 1980 port_is_stopped(portid_t port_id) 1981 { 1982 struct rte_port *port = &ports[port_id]; 1983 1984 if ((port->port_status != RTE_PORT_STOPPED) && 1985 (port->slave_flag == 0)) 1986 return 0; 1987 return 1; 1988 } 1989 1990 int 1991 all_ports_stopped(void) 1992 { 1993 portid_t pi; 1994 1995 RTE_ETH_FOREACH_DEV(pi) { 1996 if (!port_is_stopped(pi)) 1997 return 0; 1998 } 1999 2000 return 1; 2001 } 2002 2003 int 2004 port_is_started(portid_t port_id) 2005 { 2006 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2007 return 0; 2008 2009 if (ports[port_id].port_status != RTE_PORT_STARTED) 2010 return 0; 2011 2012 return 1; 2013 } 2014 2015 int 2016 start_port(portid_t pid) 2017 { 2018 int diag, need_check_link_status = -1; 2019 portid_t pi; 2020 queueid_t qi; 2021 struct rte_port *port; 2022 struct rte_ether_addr mac_addr; 2023 2024 if (port_id_is_invalid(pid, ENABLED_WARN)) 2025 return 0; 2026 2027 if(dcb_config) 2028 dcb_test = 1; 2029 RTE_ETH_FOREACH_DEV(pi) { 2030 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2031 continue; 2032 2033 need_check_link_status = 0; 2034 port = &ports[pi]; 2035 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2036 RTE_PORT_HANDLING) == 0) { 2037 printf("Port %d is now not stopped\n", pi); 2038 continue; 2039 } 2040 2041 if (port->need_reconfig > 0) { 2042 port->need_reconfig = 0; 2043 2044 if (flow_isolate_all) { 2045 int ret = port_flow_isolate(pi, 1); 2046 if (ret) { 2047 printf("Failed to apply isolated" 2048 " mode on port %d\n", pi); 2049 return -1; 2050 } 2051 } 2052 configure_rxtx_dump_callbacks(0); 2053 printf("Configuring Port %d (socket %u)\n", pi, 2054 port->socket_id); 2055 /* configure port */ 2056 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 2057 &(port->dev_conf)); 2058 if (diag != 0) { 2059 if (rte_atomic16_cmpset(&(port->port_status), 2060 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2061 printf("Port %d can not be set back " 2062 "to stopped\n", pi); 2063 printf("Fail to configure port %d\n", pi); 2064 /* try to reconfigure port next time */ 2065 port->need_reconfig = 1; 2066 return -1; 2067 } 2068 } 2069 if (port->need_reconfig_queues > 0) { 2070 port->need_reconfig_queues = 0; 2071 /* setup tx queues */ 2072 for (qi = 0; qi < nb_txq; qi++) { 2073 if ((numa_support) && 2074 (txring_numa[pi] != NUMA_NO_CONFIG)) 2075 diag = rte_eth_tx_queue_setup(pi, qi, 2076 port->nb_tx_desc[qi], 2077 txring_numa[pi], 2078 &(port->tx_conf[qi])); 2079 else 2080 diag = rte_eth_tx_queue_setup(pi, qi, 2081 port->nb_tx_desc[qi], 2082 port->socket_id, 2083 &(port->tx_conf[qi])); 2084 2085 if (diag == 0) 2086 continue; 2087 2088 /* Fail to setup tx queue, return */ 2089 if (rte_atomic16_cmpset(&(port->port_status), 2090 RTE_PORT_HANDLING, 2091 RTE_PORT_STOPPED) == 0) 2092 printf("Port %d can not be set back " 2093 "to stopped\n", pi); 2094 printf("Fail to configure port %d tx queues\n", 2095 pi); 2096 /* try to reconfigure queues next time */ 2097 port->need_reconfig_queues = 1; 2098 return -1; 2099 } 2100 for (qi = 0; qi < nb_rxq; qi++) { 2101 /* setup rx queues */ 2102 if ((numa_support) && 2103 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2104 struct rte_mempool * mp = 2105 mbuf_pool_find(rxring_numa[pi]); 2106 if (mp == NULL) { 2107 printf("Failed to setup RX queue:" 2108 "No mempool allocation" 2109 " on the socket %d\n", 2110 rxring_numa[pi]); 2111 return -1; 2112 } 2113 2114 diag = rte_eth_rx_queue_setup(pi, qi, 2115 port->nb_rx_desc[qi], 2116 rxring_numa[pi], 2117 &(port->rx_conf[qi]), 2118 mp); 2119 } else { 2120 struct rte_mempool *mp = 2121 mbuf_pool_find(port->socket_id); 2122 if (mp == NULL) { 2123 printf("Failed to setup RX queue:" 2124 "No mempool allocation" 2125 " on the socket %d\n", 2126 port->socket_id); 2127 return -1; 2128 } 2129 diag = rte_eth_rx_queue_setup(pi, qi, 2130 port->nb_rx_desc[qi], 2131 port->socket_id, 2132 &(port->rx_conf[qi]), 2133 mp); 2134 } 2135 if (diag == 0) 2136 continue; 2137 2138 /* Fail to setup rx queue, return */ 2139 if (rte_atomic16_cmpset(&(port->port_status), 2140 RTE_PORT_HANDLING, 2141 RTE_PORT_STOPPED) == 0) 2142 printf("Port %d can not be set back " 2143 "to stopped\n", pi); 2144 printf("Fail to configure port %d rx queues\n", 2145 pi); 2146 /* try to reconfigure queues next time */ 2147 port->need_reconfig_queues = 1; 2148 return -1; 2149 } 2150 } 2151 configure_rxtx_dump_callbacks(verbose_level); 2152 /* start port */ 2153 if (rte_eth_dev_start(pi) < 0) { 2154 printf("Fail to start port %d\n", pi); 2155 2156 /* Fail to setup rx queue, return */ 2157 if (rte_atomic16_cmpset(&(port->port_status), 2158 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2159 printf("Port %d can not be set back to " 2160 "stopped\n", pi); 2161 continue; 2162 } 2163 2164 if (rte_atomic16_cmpset(&(port->port_status), 2165 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2166 printf("Port %d can not be set into started\n", pi); 2167 2168 rte_eth_macaddr_get(pi, &mac_addr); 2169 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2170 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2171 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2172 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2173 2174 /* at least one port started, need checking link status */ 2175 need_check_link_status = 1; 2176 } 2177 2178 if (need_check_link_status == 1 && !no_link_check) 2179 check_all_ports_link_status(RTE_PORT_ALL); 2180 else if (need_check_link_status == 0) 2181 printf("Please stop the ports first\n"); 2182 2183 printf("Done\n"); 2184 return 0; 2185 } 2186 2187 void 2188 stop_port(portid_t pid) 2189 { 2190 portid_t pi; 2191 struct rte_port *port; 2192 int need_check_link_status = 0; 2193 2194 if (dcb_test) { 2195 dcb_test = 0; 2196 dcb_config = 0; 2197 } 2198 2199 if (port_id_is_invalid(pid, ENABLED_WARN)) 2200 return; 2201 2202 printf("Stopping ports...\n"); 2203 2204 RTE_ETH_FOREACH_DEV(pi) { 2205 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2206 continue; 2207 2208 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2209 printf("Please remove port %d from forwarding configuration.\n", pi); 2210 continue; 2211 } 2212 2213 if (port_is_bonding_slave(pi)) { 2214 printf("Please remove port %d from bonded device.\n", pi); 2215 continue; 2216 } 2217 2218 port = &ports[pi]; 2219 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2220 RTE_PORT_HANDLING) == 0) 2221 continue; 2222 2223 rte_eth_dev_stop(pi); 2224 2225 if (rte_atomic16_cmpset(&(port->port_status), 2226 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2227 printf("Port %d can not be set into stopped\n", pi); 2228 need_check_link_status = 1; 2229 } 2230 if (need_check_link_status && !no_link_check) 2231 check_all_ports_link_status(RTE_PORT_ALL); 2232 2233 printf("Done\n"); 2234 } 2235 2236 static void 2237 remove_invalid_ports_in(portid_t *array, portid_t *total) 2238 { 2239 portid_t i; 2240 portid_t new_total = 0; 2241 2242 for (i = 0; i < *total; i++) 2243 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2244 array[new_total] = array[i]; 2245 new_total++; 2246 } 2247 *total = new_total; 2248 } 2249 2250 static void 2251 remove_invalid_ports(void) 2252 { 2253 remove_invalid_ports_in(ports_ids, &nb_ports); 2254 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2255 nb_cfg_ports = nb_fwd_ports; 2256 } 2257 2258 void 2259 close_port(portid_t pid) 2260 { 2261 portid_t pi; 2262 struct rte_port *port; 2263 2264 if (port_id_is_invalid(pid, ENABLED_WARN)) 2265 return; 2266 2267 printf("Closing ports...\n"); 2268 2269 RTE_ETH_FOREACH_DEV(pi) { 2270 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2271 continue; 2272 2273 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2274 printf("Please remove port %d from forwarding configuration.\n", pi); 2275 continue; 2276 } 2277 2278 if (port_is_bonding_slave(pi)) { 2279 printf("Please remove port %d from bonded device.\n", pi); 2280 continue; 2281 } 2282 2283 port = &ports[pi]; 2284 if (rte_atomic16_cmpset(&(port->port_status), 2285 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2286 printf("Port %d is already closed\n", pi); 2287 continue; 2288 } 2289 2290 if (rte_atomic16_cmpset(&(port->port_status), 2291 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2292 printf("Port %d is now not stopped\n", pi); 2293 continue; 2294 } 2295 2296 if (port->flow_list) 2297 port_flow_flush(pi); 2298 rte_eth_dev_close(pi); 2299 2300 remove_invalid_ports(); 2301 2302 if (rte_atomic16_cmpset(&(port->port_status), 2303 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2304 printf("Port %d cannot be set to closed\n", pi); 2305 } 2306 2307 printf("Done\n"); 2308 } 2309 2310 void 2311 reset_port(portid_t pid) 2312 { 2313 int diag; 2314 portid_t pi; 2315 struct rte_port *port; 2316 2317 if (port_id_is_invalid(pid, ENABLED_WARN)) 2318 return; 2319 2320 printf("Resetting ports...\n"); 2321 2322 RTE_ETH_FOREACH_DEV(pi) { 2323 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2324 continue; 2325 2326 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2327 printf("Please remove port %d from forwarding " 2328 "configuration.\n", pi); 2329 continue; 2330 } 2331 2332 if (port_is_bonding_slave(pi)) { 2333 printf("Please remove port %d from bonded device.\n", 2334 pi); 2335 continue; 2336 } 2337 2338 diag = rte_eth_dev_reset(pi); 2339 if (diag == 0) { 2340 port = &ports[pi]; 2341 port->need_reconfig = 1; 2342 port->need_reconfig_queues = 1; 2343 } else { 2344 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2345 } 2346 } 2347 2348 printf("Done\n"); 2349 } 2350 2351 void 2352 attach_port(char *identifier) 2353 { 2354 portid_t pi; 2355 struct rte_dev_iterator iterator; 2356 2357 printf("Attaching a new port...\n"); 2358 2359 if (identifier == NULL) { 2360 printf("Invalid parameters are specified\n"); 2361 return; 2362 } 2363 2364 if (rte_dev_probe(identifier) != 0) { 2365 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2366 return; 2367 } 2368 2369 /* first attach mode: event */ 2370 if (setup_on_probe_event) { 2371 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2372 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2373 if (ports[pi].port_status == RTE_PORT_HANDLING && 2374 ports[pi].need_setup != 0) 2375 setup_attached_port(pi); 2376 return; 2377 } 2378 2379 /* second attach mode: iterator */ 2380 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2381 /* setup ports matching the devargs used for probing */ 2382 if (port_is_forwarding(pi)) 2383 continue; /* port was already attached before */ 2384 setup_attached_port(pi); 2385 } 2386 } 2387 2388 static void 2389 setup_attached_port(portid_t pi) 2390 { 2391 unsigned int socket_id; 2392 2393 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2394 /* if socket_id is invalid, set to the first available socket. */ 2395 if (check_socket_id(socket_id) < 0) 2396 socket_id = socket_ids[0]; 2397 reconfig(pi, socket_id); 2398 rte_eth_promiscuous_enable(pi); 2399 2400 ports_ids[nb_ports++] = pi; 2401 fwd_ports_ids[nb_fwd_ports++] = pi; 2402 nb_cfg_ports = nb_fwd_ports; 2403 ports[pi].need_setup = 0; 2404 ports[pi].port_status = RTE_PORT_STOPPED; 2405 2406 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2407 printf("Done\n"); 2408 } 2409 2410 void 2411 detach_port_device(portid_t port_id) 2412 { 2413 struct rte_device *dev; 2414 portid_t sibling; 2415 2416 printf("Removing a device...\n"); 2417 2418 dev = rte_eth_devices[port_id].device; 2419 if (dev == NULL) { 2420 printf("Device already removed\n"); 2421 return; 2422 } 2423 2424 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2425 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2426 printf("Port not stopped\n"); 2427 return; 2428 } 2429 printf("Port was not closed\n"); 2430 if (ports[port_id].flow_list) 2431 port_flow_flush(port_id); 2432 } 2433 2434 if (rte_dev_remove(dev) != 0) { 2435 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2436 return; 2437 } 2438 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2439 /* reset mapping between old ports and removed device */ 2440 rte_eth_devices[sibling].device = NULL; 2441 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2442 /* sibling ports are forced to be closed */ 2443 ports[sibling].port_status = RTE_PORT_CLOSED; 2444 printf("Port %u is closed\n", sibling); 2445 } 2446 } 2447 2448 remove_invalid_ports(); 2449 2450 printf("Device of port %u is detached\n", port_id); 2451 printf("Now total ports is %d\n", nb_ports); 2452 printf("Done\n"); 2453 return; 2454 } 2455 2456 void 2457 pmd_test_exit(void) 2458 { 2459 struct rte_device *device; 2460 portid_t pt_id; 2461 int ret; 2462 int i; 2463 2464 if (test_done == 0) 2465 stop_packet_forwarding(); 2466 2467 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2468 if (mempools[i]) { 2469 if (mp_alloc_type == MP_ALLOC_ANON) 2470 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2471 NULL); 2472 } 2473 } 2474 if (ports != NULL) { 2475 no_link_check = 1; 2476 RTE_ETH_FOREACH_DEV(pt_id) { 2477 printf("\nStopping port %d...\n", pt_id); 2478 fflush(stdout); 2479 stop_port(pt_id); 2480 } 2481 RTE_ETH_FOREACH_DEV(pt_id) { 2482 printf("\nShutting down port %d...\n", pt_id); 2483 fflush(stdout); 2484 close_port(pt_id); 2485 2486 /* 2487 * This is a workaround to fix a virtio-user issue that 2488 * requires to call clean-up routine to remove existing 2489 * socket. 2490 * This workaround valid only for testpmd, needs a fix 2491 * valid for all applications. 2492 * TODO: Implement proper resource cleanup 2493 */ 2494 device = rte_eth_devices[pt_id].device; 2495 if (device && !strcmp(device->driver->name, "net_virtio_user")) 2496 detach_port_device(pt_id); 2497 } 2498 } 2499 2500 if (hot_plug) { 2501 ret = rte_dev_event_monitor_stop(); 2502 if (ret) { 2503 RTE_LOG(ERR, EAL, 2504 "fail to stop device event monitor."); 2505 return; 2506 } 2507 2508 ret = rte_dev_event_callback_unregister(NULL, 2509 dev_event_callback, NULL); 2510 if (ret < 0) { 2511 RTE_LOG(ERR, EAL, 2512 "fail to unregister device event callback.\n"); 2513 return; 2514 } 2515 2516 ret = rte_dev_hotplug_handle_disable(); 2517 if (ret) { 2518 RTE_LOG(ERR, EAL, 2519 "fail to disable hotplug handling.\n"); 2520 return; 2521 } 2522 } 2523 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2524 if (mempools[i]) 2525 rte_mempool_free(mempools[i]); 2526 } 2527 2528 printf("\nBye...\n"); 2529 } 2530 2531 typedef void (*cmd_func_t)(void); 2532 struct pmd_test_command { 2533 const char *cmd_name; 2534 cmd_func_t cmd_func; 2535 }; 2536 2537 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2538 2539 /* Check the link status of all ports in up to 9s, and print them finally */ 2540 static void 2541 check_all_ports_link_status(uint32_t port_mask) 2542 { 2543 #define CHECK_INTERVAL 100 /* 100ms */ 2544 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2545 portid_t portid; 2546 uint8_t count, all_ports_up, print_flag = 0; 2547 struct rte_eth_link link; 2548 2549 printf("Checking link statuses...\n"); 2550 fflush(stdout); 2551 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2552 all_ports_up = 1; 2553 RTE_ETH_FOREACH_DEV(portid) { 2554 if ((port_mask & (1 << portid)) == 0) 2555 continue; 2556 memset(&link, 0, sizeof(link)); 2557 rte_eth_link_get_nowait(portid, &link); 2558 /* print link status if flag set */ 2559 if (print_flag == 1) { 2560 if (link.link_status) 2561 printf( 2562 "Port%d Link Up. speed %u Mbps- %s\n", 2563 portid, link.link_speed, 2564 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2565 ("full-duplex") : ("half-duplex\n")); 2566 else 2567 printf("Port %d Link Down\n", portid); 2568 continue; 2569 } 2570 /* clear all_ports_up flag if any link down */ 2571 if (link.link_status == ETH_LINK_DOWN) { 2572 all_ports_up = 0; 2573 break; 2574 } 2575 } 2576 /* after finally printing all link status, get out */ 2577 if (print_flag == 1) 2578 break; 2579 2580 if (all_ports_up == 0) { 2581 fflush(stdout); 2582 rte_delay_ms(CHECK_INTERVAL); 2583 } 2584 2585 /* set the print_flag if all ports up or timeout */ 2586 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2587 print_flag = 1; 2588 } 2589 2590 if (lsc_interrupt) 2591 break; 2592 } 2593 } 2594 2595 /* 2596 * This callback is for remove a port for a device. It has limitation because 2597 * it is not for multiple port removal for a device. 2598 * TODO: the device detach invoke will plan to be removed from user side to 2599 * eal. And convert all PMDs to free port resources on ether device closing. 2600 */ 2601 static void 2602 rmv_port_callback(void *arg) 2603 { 2604 int need_to_start = 0; 2605 int org_no_link_check = no_link_check; 2606 portid_t port_id = (intptr_t)arg; 2607 2608 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2609 2610 if (!test_done && port_is_forwarding(port_id)) { 2611 need_to_start = 1; 2612 stop_packet_forwarding(); 2613 } 2614 no_link_check = 1; 2615 stop_port(port_id); 2616 no_link_check = org_no_link_check; 2617 close_port(port_id); 2618 detach_port_device(port_id); 2619 if (need_to_start) 2620 start_packet_forwarding(0); 2621 } 2622 2623 /* This function is used by the interrupt thread */ 2624 static int 2625 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2626 void *ret_param) 2627 { 2628 RTE_SET_USED(param); 2629 RTE_SET_USED(ret_param); 2630 2631 if (type >= RTE_ETH_EVENT_MAX) { 2632 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2633 port_id, __func__, type); 2634 fflush(stderr); 2635 } else if (event_print_mask & (UINT32_C(1) << type)) { 2636 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2637 eth_event_desc[type]); 2638 fflush(stdout); 2639 } 2640 2641 switch (type) { 2642 case RTE_ETH_EVENT_NEW: 2643 ports[port_id].need_setup = 1; 2644 ports[port_id].port_status = RTE_PORT_HANDLING; 2645 break; 2646 case RTE_ETH_EVENT_INTR_RMV: 2647 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2648 break; 2649 if (rte_eal_alarm_set(100000, 2650 rmv_port_callback, (void *)(intptr_t)port_id)) 2651 fprintf(stderr, "Could not set up deferred device removal\n"); 2652 break; 2653 default: 2654 break; 2655 } 2656 return 0; 2657 } 2658 2659 static int 2660 register_eth_event_callback(void) 2661 { 2662 int ret; 2663 enum rte_eth_event_type event; 2664 2665 for (event = RTE_ETH_EVENT_UNKNOWN; 2666 event < RTE_ETH_EVENT_MAX; event++) { 2667 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2668 event, 2669 eth_event_callback, 2670 NULL); 2671 if (ret != 0) { 2672 TESTPMD_LOG(ERR, "Failed to register callback for " 2673 "%s event\n", eth_event_desc[event]); 2674 return -1; 2675 } 2676 } 2677 2678 return 0; 2679 } 2680 2681 /* This function is used by the interrupt thread */ 2682 static void 2683 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2684 __rte_unused void *arg) 2685 { 2686 uint16_t port_id; 2687 int ret; 2688 2689 if (type >= RTE_DEV_EVENT_MAX) { 2690 fprintf(stderr, "%s called upon invalid event %d\n", 2691 __func__, type); 2692 fflush(stderr); 2693 } 2694 2695 switch (type) { 2696 case RTE_DEV_EVENT_REMOVE: 2697 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2698 device_name); 2699 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2700 if (ret) { 2701 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2702 device_name); 2703 return; 2704 } 2705 /* 2706 * Because the user's callback is invoked in eal interrupt 2707 * callback, the interrupt callback need to be finished before 2708 * it can be unregistered when detaching device. So finish 2709 * callback soon and use a deferred removal to detach device 2710 * is need. It is a workaround, once the device detaching be 2711 * moved into the eal in the future, the deferred removal could 2712 * be deleted. 2713 */ 2714 if (rte_eal_alarm_set(100000, 2715 rmv_port_callback, (void *)(intptr_t)port_id)) 2716 RTE_LOG(ERR, EAL, 2717 "Could not set up deferred device removal\n"); 2718 break; 2719 case RTE_DEV_EVENT_ADD: 2720 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2721 device_name); 2722 /* TODO: After finish kernel driver binding, 2723 * begin to attach port. 2724 */ 2725 break; 2726 default: 2727 break; 2728 } 2729 } 2730 2731 static int 2732 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2733 { 2734 uint16_t i; 2735 int diag; 2736 uint8_t mapping_found = 0; 2737 2738 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2739 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2740 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2741 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2742 tx_queue_stats_mappings[i].queue_id, 2743 tx_queue_stats_mappings[i].stats_counter_id); 2744 if (diag != 0) 2745 return diag; 2746 mapping_found = 1; 2747 } 2748 } 2749 if (mapping_found) 2750 port->tx_queue_stats_mapping_enabled = 1; 2751 return 0; 2752 } 2753 2754 static int 2755 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2756 { 2757 uint16_t i; 2758 int diag; 2759 uint8_t mapping_found = 0; 2760 2761 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2762 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2763 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2764 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2765 rx_queue_stats_mappings[i].queue_id, 2766 rx_queue_stats_mappings[i].stats_counter_id); 2767 if (diag != 0) 2768 return diag; 2769 mapping_found = 1; 2770 } 2771 } 2772 if (mapping_found) 2773 port->rx_queue_stats_mapping_enabled = 1; 2774 return 0; 2775 } 2776 2777 static void 2778 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2779 { 2780 int diag = 0; 2781 2782 diag = set_tx_queue_stats_mapping_registers(pi, port); 2783 if (diag != 0) { 2784 if (diag == -ENOTSUP) { 2785 port->tx_queue_stats_mapping_enabled = 0; 2786 printf("TX queue stats mapping not supported port id=%d\n", pi); 2787 } 2788 else 2789 rte_exit(EXIT_FAILURE, 2790 "set_tx_queue_stats_mapping_registers " 2791 "failed for port id=%d diag=%d\n", 2792 pi, diag); 2793 } 2794 2795 diag = set_rx_queue_stats_mapping_registers(pi, port); 2796 if (diag != 0) { 2797 if (diag == -ENOTSUP) { 2798 port->rx_queue_stats_mapping_enabled = 0; 2799 printf("RX queue stats mapping not supported port id=%d\n", pi); 2800 } 2801 else 2802 rte_exit(EXIT_FAILURE, 2803 "set_rx_queue_stats_mapping_registers " 2804 "failed for port id=%d diag=%d\n", 2805 pi, diag); 2806 } 2807 } 2808 2809 static void 2810 rxtx_port_config(struct rte_port *port) 2811 { 2812 uint16_t qid; 2813 uint64_t offloads; 2814 2815 for (qid = 0; qid < nb_rxq; qid++) { 2816 offloads = port->rx_conf[qid].offloads; 2817 port->rx_conf[qid] = port->dev_info.default_rxconf; 2818 port->rx_conf[qid].offloads |= offloads; 2819 2820 /* Check if any Rx parameters have been passed */ 2821 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2822 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2823 2824 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2825 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2826 2827 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2828 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2829 2830 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2831 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2832 2833 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2834 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2835 2836 port->nb_rx_desc[qid] = nb_rxd; 2837 } 2838 2839 for (qid = 0; qid < nb_txq; qid++) { 2840 offloads = port->tx_conf[qid].offloads; 2841 port->tx_conf[qid] = port->dev_info.default_txconf; 2842 port->tx_conf[qid].offloads |= offloads; 2843 2844 /* Check if any Tx parameters have been passed */ 2845 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2846 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2847 2848 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2849 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2850 2851 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2852 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2853 2854 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2855 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2856 2857 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2858 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2859 2860 port->nb_tx_desc[qid] = nb_txd; 2861 } 2862 } 2863 2864 void 2865 init_port_config(void) 2866 { 2867 portid_t pid; 2868 struct rte_port *port; 2869 2870 RTE_ETH_FOREACH_DEV(pid) { 2871 port = &ports[pid]; 2872 port->dev_conf.fdir_conf = fdir_conf; 2873 rte_eth_dev_info_get(pid, &port->dev_info); 2874 if (nb_rxq > 1) { 2875 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2876 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2877 rss_hf & port->dev_info.flow_type_rss_offloads; 2878 } else { 2879 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2880 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2881 } 2882 2883 if (port->dcb_flag == 0) { 2884 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2885 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2886 else 2887 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2888 } 2889 2890 rxtx_port_config(port); 2891 2892 rte_eth_macaddr_get(pid, &port->eth_addr); 2893 2894 map_port_queue_stats_mapping_registers(pid, port); 2895 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2896 rte_pmd_ixgbe_bypass_init(pid); 2897 #endif 2898 2899 if (lsc_interrupt && 2900 (rte_eth_devices[pid].data->dev_flags & 2901 RTE_ETH_DEV_INTR_LSC)) 2902 port->dev_conf.intr_conf.lsc = 1; 2903 if (rmv_interrupt && 2904 (rte_eth_devices[pid].data->dev_flags & 2905 RTE_ETH_DEV_INTR_RMV)) 2906 port->dev_conf.intr_conf.rmv = 1; 2907 } 2908 } 2909 2910 void set_port_slave_flag(portid_t slave_pid) 2911 { 2912 struct rte_port *port; 2913 2914 port = &ports[slave_pid]; 2915 port->slave_flag = 1; 2916 } 2917 2918 void clear_port_slave_flag(portid_t slave_pid) 2919 { 2920 struct rte_port *port; 2921 2922 port = &ports[slave_pid]; 2923 port->slave_flag = 0; 2924 } 2925 2926 uint8_t port_is_bonding_slave(portid_t slave_pid) 2927 { 2928 struct rte_port *port; 2929 2930 port = &ports[slave_pid]; 2931 if ((rte_eth_devices[slave_pid].data->dev_flags & 2932 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2933 return 1; 2934 return 0; 2935 } 2936 2937 const uint16_t vlan_tags[] = { 2938 0, 1, 2, 3, 4, 5, 6, 7, 2939 8, 9, 10, 11, 12, 13, 14, 15, 2940 16, 17, 18, 19, 20, 21, 22, 23, 2941 24, 25, 26, 27, 28, 29, 30, 31 2942 }; 2943 2944 static int 2945 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 2946 enum dcb_mode_enable dcb_mode, 2947 enum rte_eth_nb_tcs num_tcs, 2948 uint8_t pfc_en) 2949 { 2950 uint8_t i; 2951 int32_t rc; 2952 struct rte_eth_rss_conf rss_conf; 2953 2954 /* 2955 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2956 * given above, and the number of traffic classes available for use. 2957 */ 2958 if (dcb_mode == DCB_VT_ENABLED) { 2959 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2960 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2961 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2962 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2963 2964 /* VMDQ+DCB RX and TX configurations */ 2965 vmdq_rx_conf->enable_default_pool = 0; 2966 vmdq_rx_conf->default_pool = 0; 2967 vmdq_rx_conf->nb_queue_pools = 2968 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2969 vmdq_tx_conf->nb_queue_pools = 2970 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2971 2972 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2973 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2974 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2975 vmdq_rx_conf->pool_map[i].pools = 2976 1 << (i % vmdq_rx_conf->nb_queue_pools); 2977 } 2978 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2979 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 2980 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 2981 } 2982 2983 /* set DCB mode of RX and TX of multiple queues */ 2984 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2985 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2986 } else { 2987 struct rte_eth_dcb_rx_conf *rx_conf = 2988 ð_conf->rx_adv_conf.dcb_rx_conf; 2989 struct rte_eth_dcb_tx_conf *tx_conf = 2990 ð_conf->tx_adv_conf.dcb_tx_conf; 2991 2992 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 2993 if (rc != 0) 2994 return rc; 2995 2996 rx_conf->nb_tcs = num_tcs; 2997 tx_conf->nb_tcs = num_tcs; 2998 2999 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3000 rx_conf->dcb_tc[i] = i % num_tcs; 3001 tx_conf->dcb_tc[i] = i % num_tcs; 3002 } 3003 3004 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 3005 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3006 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3007 } 3008 3009 if (pfc_en) 3010 eth_conf->dcb_capability_en = 3011 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3012 else 3013 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3014 3015 return 0; 3016 } 3017 3018 int 3019 init_port_dcb_config(portid_t pid, 3020 enum dcb_mode_enable dcb_mode, 3021 enum rte_eth_nb_tcs num_tcs, 3022 uint8_t pfc_en) 3023 { 3024 struct rte_eth_conf port_conf; 3025 struct rte_port *rte_port; 3026 int retval; 3027 uint16_t i; 3028 3029 rte_port = &ports[pid]; 3030 3031 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3032 /* Enter DCB configuration status */ 3033 dcb_config = 1; 3034 3035 port_conf.rxmode = rte_port->dev_conf.rxmode; 3036 port_conf.txmode = rte_port->dev_conf.txmode; 3037 3038 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3039 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3040 if (retval < 0) 3041 return retval; 3042 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3043 3044 /* re-configure the device . */ 3045 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3046 if (retval < 0) 3047 return retval; 3048 rte_eth_dev_info_get(pid, &rte_port->dev_info); 3049 3050 /* If dev_info.vmdq_pool_base is greater than 0, 3051 * the queue id of vmdq pools is started after pf queues. 3052 */ 3053 if (dcb_mode == DCB_VT_ENABLED && 3054 rte_port->dev_info.vmdq_pool_base > 0) { 3055 printf("VMDQ_DCB multi-queue mode is nonsensical" 3056 " for port %d.", pid); 3057 return -1; 3058 } 3059 3060 /* Assume the ports in testpmd have the same dcb capability 3061 * and has the same number of rxq and txq in dcb mode 3062 */ 3063 if (dcb_mode == DCB_VT_ENABLED) { 3064 if (rte_port->dev_info.max_vfs > 0) { 3065 nb_rxq = rte_port->dev_info.nb_rx_queues; 3066 nb_txq = rte_port->dev_info.nb_tx_queues; 3067 } else { 3068 nb_rxq = rte_port->dev_info.max_rx_queues; 3069 nb_txq = rte_port->dev_info.max_tx_queues; 3070 } 3071 } else { 3072 /*if vt is disabled, use all pf queues */ 3073 if (rte_port->dev_info.vmdq_pool_base == 0) { 3074 nb_rxq = rte_port->dev_info.max_rx_queues; 3075 nb_txq = rte_port->dev_info.max_tx_queues; 3076 } else { 3077 nb_rxq = (queueid_t)num_tcs; 3078 nb_txq = (queueid_t)num_tcs; 3079 3080 } 3081 } 3082 rx_free_thresh = 64; 3083 3084 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3085 3086 rxtx_port_config(rte_port); 3087 /* VLAN filter */ 3088 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3089 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3090 rx_vft_set(pid, vlan_tags[i], 1); 3091 3092 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 3093 map_port_queue_stats_mapping_registers(pid, rte_port); 3094 3095 rte_port->dcb_flag = 1; 3096 3097 return 0; 3098 } 3099 3100 static void 3101 init_port(void) 3102 { 3103 /* Configuration of Ethernet ports. */ 3104 ports = rte_zmalloc("testpmd: ports", 3105 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3106 RTE_CACHE_LINE_SIZE); 3107 if (ports == NULL) { 3108 rte_exit(EXIT_FAILURE, 3109 "rte_zmalloc(%d struct rte_port) failed\n", 3110 RTE_MAX_ETHPORTS); 3111 } 3112 3113 /* Initialize ports NUMA structures */ 3114 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3115 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3116 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3117 } 3118 3119 static void 3120 force_quit(void) 3121 { 3122 pmd_test_exit(); 3123 prompt_exit(); 3124 } 3125 3126 static void 3127 print_stats(void) 3128 { 3129 uint8_t i; 3130 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3131 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3132 3133 /* Clear screen and move to top left */ 3134 printf("%s%s", clr, top_left); 3135 3136 printf("\nPort statistics ===================================="); 3137 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3138 nic_stats_display(fwd_ports_ids[i]); 3139 3140 fflush(stdout); 3141 } 3142 3143 static void 3144 signal_handler(int signum) 3145 { 3146 if (signum == SIGINT || signum == SIGTERM) { 3147 printf("\nSignal %d received, preparing to exit...\n", 3148 signum); 3149 #ifdef RTE_LIBRTE_PDUMP 3150 /* uninitialize packet capture framework */ 3151 rte_pdump_uninit(); 3152 #endif 3153 #ifdef RTE_LIBRTE_LATENCY_STATS 3154 rte_latencystats_uninit(); 3155 #endif 3156 force_quit(); 3157 /* Set flag to indicate the force termination. */ 3158 f_quit = 1; 3159 /* exit with the expected status */ 3160 signal(signum, SIG_DFL); 3161 kill(getpid(), signum); 3162 } 3163 } 3164 3165 int 3166 main(int argc, char** argv) 3167 { 3168 int diag; 3169 portid_t port_id; 3170 uint16_t count; 3171 int ret; 3172 3173 signal(SIGINT, signal_handler); 3174 signal(SIGTERM, signal_handler); 3175 3176 diag = rte_eal_init(argc, argv); 3177 if (diag < 0) 3178 rte_panic("Cannot init EAL\n"); 3179 3180 testpmd_logtype = rte_log_register("testpmd"); 3181 if (testpmd_logtype < 0) 3182 rte_panic("Cannot register log type"); 3183 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3184 3185 ret = register_eth_event_callback(); 3186 if (ret != 0) 3187 rte_panic("Cannot register for ethdev events"); 3188 3189 #ifdef RTE_LIBRTE_PDUMP 3190 /* initialize packet capture framework */ 3191 rte_pdump_init(); 3192 #endif 3193 3194 count = 0; 3195 RTE_ETH_FOREACH_DEV(port_id) { 3196 ports_ids[count] = port_id; 3197 count++; 3198 } 3199 nb_ports = (portid_t) count; 3200 if (nb_ports == 0) 3201 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3202 3203 /* allocate port structures, and init them */ 3204 init_port(); 3205 3206 set_def_fwd_config(); 3207 if (nb_lcores == 0) 3208 rte_panic("Empty set of forwarding logical cores - check the " 3209 "core mask supplied in the command parameters\n"); 3210 3211 /* Bitrate/latency stats disabled by default */ 3212 #ifdef RTE_LIBRTE_BITRATE 3213 bitrate_enabled = 0; 3214 #endif 3215 #ifdef RTE_LIBRTE_LATENCY_STATS 3216 latencystats_enabled = 0; 3217 #endif 3218 3219 /* on FreeBSD, mlockall() is disabled by default */ 3220 #ifdef RTE_EXEC_ENV_FREEBSD 3221 do_mlockall = 0; 3222 #else 3223 do_mlockall = 1; 3224 #endif 3225 3226 argc -= diag; 3227 argv += diag; 3228 if (argc > 1) 3229 launch_args_parse(argc, argv); 3230 3231 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3232 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3233 strerror(errno)); 3234 } 3235 3236 if (tx_first && interactive) 3237 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3238 "interactive mode.\n"); 3239 3240 if (tx_first && lsc_interrupt) { 3241 printf("Warning: lsc_interrupt needs to be off when " 3242 " using tx_first. Disabling.\n"); 3243 lsc_interrupt = 0; 3244 } 3245 3246 if (!nb_rxq && !nb_txq) 3247 printf("Warning: Either rx or tx queues should be non-zero\n"); 3248 3249 if (nb_rxq > 1 && nb_rxq > nb_txq) 3250 printf("Warning: nb_rxq=%d enables RSS configuration, " 3251 "but nb_txq=%d will prevent to fully test it.\n", 3252 nb_rxq, nb_txq); 3253 3254 init_config(); 3255 3256 if (hot_plug) { 3257 ret = rte_dev_hotplug_handle_enable(); 3258 if (ret) { 3259 RTE_LOG(ERR, EAL, 3260 "fail to enable hotplug handling."); 3261 return -1; 3262 } 3263 3264 ret = rte_dev_event_monitor_start(); 3265 if (ret) { 3266 RTE_LOG(ERR, EAL, 3267 "fail to start device event monitoring."); 3268 return -1; 3269 } 3270 3271 ret = rte_dev_event_callback_register(NULL, 3272 dev_event_callback, NULL); 3273 if (ret) { 3274 RTE_LOG(ERR, EAL, 3275 "fail to register device event callback\n"); 3276 return -1; 3277 } 3278 } 3279 3280 if (start_port(RTE_PORT_ALL) != 0) 3281 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3282 3283 /* set all ports to promiscuous mode by default */ 3284 RTE_ETH_FOREACH_DEV(port_id) 3285 rte_eth_promiscuous_enable(port_id); 3286 3287 /* Init metrics library */ 3288 rte_metrics_init(rte_socket_id()); 3289 3290 #ifdef RTE_LIBRTE_LATENCY_STATS 3291 if (latencystats_enabled != 0) { 3292 int ret = rte_latencystats_init(1, NULL); 3293 if (ret) 3294 printf("Warning: latencystats init()" 3295 " returned error %d\n", ret); 3296 printf("Latencystats running on lcore %d\n", 3297 latencystats_lcore_id); 3298 } 3299 #endif 3300 3301 /* Setup bitrate stats */ 3302 #ifdef RTE_LIBRTE_BITRATE 3303 if (bitrate_enabled != 0) { 3304 bitrate_data = rte_stats_bitrate_create(); 3305 if (bitrate_data == NULL) 3306 rte_exit(EXIT_FAILURE, 3307 "Could not allocate bitrate data.\n"); 3308 rte_stats_bitrate_reg(bitrate_data); 3309 } 3310 #endif 3311 3312 #ifdef RTE_LIBRTE_CMDLINE 3313 if (strlen(cmdline_filename) != 0) 3314 cmdline_read_from_file(cmdline_filename); 3315 3316 if (interactive == 1) { 3317 if (auto_start) { 3318 printf("Start automatic packet forwarding\n"); 3319 start_packet_forwarding(0); 3320 } 3321 prompt(); 3322 pmd_test_exit(); 3323 } else 3324 #endif 3325 { 3326 char c; 3327 int rc; 3328 3329 f_quit = 0; 3330 3331 printf("No commandline core given, start packet forwarding\n"); 3332 start_packet_forwarding(tx_first); 3333 if (stats_period != 0) { 3334 uint64_t prev_time = 0, cur_time, diff_time = 0; 3335 uint64_t timer_period; 3336 3337 /* Convert to number of cycles */ 3338 timer_period = stats_period * rte_get_timer_hz(); 3339 3340 while (f_quit == 0) { 3341 cur_time = rte_get_timer_cycles(); 3342 diff_time += cur_time - prev_time; 3343 3344 if (diff_time >= timer_period) { 3345 print_stats(); 3346 /* Reset the timer */ 3347 diff_time = 0; 3348 } 3349 /* Sleep to avoid unnecessary checks */ 3350 prev_time = cur_time; 3351 sleep(1); 3352 } 3353 } 3354 3355 printf("Press enter to exit\n"); 3356 rc = read(0, &c, 1); 3357 pmd_test_exit(); 3358 if (rc < 0) 3359 return 1; 3360 } 3361 3362 return 0; 3363 } 3364