1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 #if defined RTE_LIBRTE_PMD_SOFTNIC 182 &softnic_fwd_engine, 183 #endif 184 #ifdef RTE_LIBRTE_IEEE1588 185 &ieee1588_fwd_engine, 186 #endif 187 NULL, 188 }; 189 190 struct fwd_config cur_fwd_config; 191 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 192 uint32_t retry_enabled; 193 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 194 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 195 196 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 197 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 198 * specified on command-line. */ 199 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 200 201 /* 202 * In container, it cannot terminate the process which running with 'stats-period' 203 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 204 */ 205 uint8_t f_quit; 206 207 /* 208 * Configuration of packet segments used by the "txonly" processing engine. 209 */ 210 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 211 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 212 TXONLY_DEF_PACKET_LEN, 213 }; 214 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 215 216 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 217 /**< Split policy for packets to TX. */ 218 219 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 220 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 221 222 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 223 uint8_t dcb_config = 0; 224 225 /* Whether the dcb is in testing status */ 226 uint8_t dcb_test = 0; 227 228 /* 229 * Configurable number of RX/TX queues. 230 */ 231 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 232 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 233 234 /* 235 * Configurable number of RX/TX ring descriptors. 236 * Defaults are supplied by drivers via ethdev. 237 */ 238 #define RTE_TEST_RX_DESC_DEFAULT 0 239 #define RTE_TEST_TX_DESC_DEFAULT 0 240 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 241 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 242 243 #define RTE_PMD_PARAM_UNSET -1 244 /* 245 * Configurable values of RX and TX ring threshold registers. 246 */ 247 248 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 249 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 250 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 251 252 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 253 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 254 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 255 256 /* 257 * Configurable value of RX free threshold. 258 */ 259 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 260 261 /* 262 * Configurable value of RX drop enable. 263 */ 264 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 265 266 /* 267 * Configurable value of TX free threshold. 268 */ 269 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 270 271 /* 272 * Configurable value of TX RS bit threshold. 273 */ 274 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 275 276 /* 277 * Receive Side Scaling (RSS) configuration. 278 */ 279 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 280 281 /* 282 * Port topology configuration 283 */ 284 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 285 286 /* 287 * Avoids to flush all the RX streams before starts forwarding. 288 */ 289 uint8_t no_flush_rx = 0; /* flush by default */ 290 291 /* 292 * Flow API isolated mode. 293 */ 294 uint8_t flow_isolate_all; 295 296 /* 297 * Avoids to check link status when starting/stopping a port. 298 */ 299 uint8_t no_link_check = 0; /* check by default */ 300 301 /* 302 * Enable link status change notification 303 */ 304 uint8_t lsc_interrupt = 1; /* enabled by default */ 305 306 /* 307 * Enable device removal notification. 308 */ 309 uint8_t rmv_interrupt = 1; /* enabled by default */ 310 311 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 312 313 /* 314 * Display or mask ether events 315 * Default to all events except VF_MBOX 316 */ 317 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 318 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 319 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 320 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 321 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 322 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 323 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 324 /* 325 * Decide if all memory are locked for performance. 326 */ 327 int do_mlockall = 0; 328 329 /* 330 * NIC bypass mode configuration options. 331 */ 332 333 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 334 /* The NIC bypass watchdog timeout. */ 335 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 336 #endif 337 338 339 #ifdef RTE_LIBRTE_LATENCY_STATS 340 341 /* 342 * Set when latency stats is enabled in the commandline 343 */ 344 uint8_t latencystats_enabled; 345 346 /* 347 * Lcore ID to serive latency statistics. 348 */ 349 lcoreid_t latencystats_lcore_id = -1; 350 351 #endif 352 353 /* 354 * Ethernet device configuration. 355 */ 356 struct rte_eth_rxmode rx_mode = { 357 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 358 }; 359 360 struct rte_eth_txmode tx_mode = { 361 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 362 }; 363 364 struct rte_fdir_conf fdir_conf = { 365 .mode = RTE_FDIR_MODE_NONE, 366 .pballoc = RTE_FDIR_PBALLOC_64K, 367 .status = RTE_FDIR_REPORT_STATUS, 368 .mask = { 369 .vlan_tci_mask = 0xFFEF, 370 .ipv4_mask = { 371 .src_ip = 0xFFFFFFFF, 372 .dst_ip = 0xFFFFFFFF, 373 }, 374 .ipv6_mask = { 375 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 376 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 377 }, 378 .src_port_mask = 0xFFFF, 379 .dst_port_mask = 0xFFFF, 380 .mac_addr_byte_mask = 0xFF, 381 .tunnel_type_mask = 1, 382 .tunnel_id_mask = 0xFFFFFFFF, 383 }, 384 .drop_queue = 127, 385 }; 386 387 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 388 389 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 390 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 391 392 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 393 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 394 395 uint16_t nb_tx_queue_stats_mappings = 0; 396 uint16_t nb_rx_queue_stats_mappings = 0; 397 398 /* 399 * Display zero values by default for xstats 400 */ 401 uint8_t xstats_hide_zero; 402 403 unsigned int num_sockets = 0; 404 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 405 406 #ifdef RTE_LIBRTE_BITRATE 407 /* Bitrate statistics */ 408 struct rte_stats_bitrates *bitrate_data; 409 lcoreid_t bitrate_lcore_id; 410 uint8_t bitrate_enabled; 411 #endif 412 413 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 414 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 415 416 struct vxlan_encap_conf vxlan_encap_conf = { 417 .select_ipv4 = 1, 418 .select_vlan = 0, 419 .vni = "\x00\x00\x00", 420 .udp_src = 0, 421 .udp_dst = RTE_BE16(4789), 422 .ipv4_src = IPv4(127, 0, 0, 1), 423 .ipv4_dst = IPv4(255, 255, 255, 255), 424 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 425 "\x00\x00\x00\x00\x00\x00\x00\x01", 426 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 427 "\x00\x00\x00\x00\x00\x00\x11\x11", 428 .vlan_tci = 0, 429 .eth_src = "\x00\x00\x00\x00\x00\x00", 430 .eth_dst = "\xff\xff\xff\xff\xff\xff", 431 }; 432 433 struct nvgre_encap_conf nvgre_encap_conf = { 434 .select_ipv4 = 1, 435 .select_vlan = 0, 436 .tni = "\x00\x00\x00", 437 .ipv4_src = IPv4(127, 0, 0, 1), 438 .ipv4_dst = IPv4(255, 255, 255, 255), 439 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 440 "\x00\x00\x00\x00\x00\x00\x00\x01", 441 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 442 "\x00\x00\x00\x00\x00\x00\x11\x11", 443 .vlan_tci = 0, 444 .eth_src = "\x00\x00\x00\x00\x00\x00", 445 .eth_dst = "\xff\xff\xff\xff\xff\xff", 446 }; 447 448 /* Forward function declarations */ 449 static void map_port_queue_stats_mapping_registers(portid_t pi, 450 struct rte_port *port); 451 static void check_all_ports_link_status(uint32_t port_mask); 452 static int eth_event_callback(portid_t port_id, 453 enum rte_eth_event_type type, 454 void *param, void *ret_param); 455 static void eth_dev_event_callback(char *device_name, 456 enum rte_dev_event_type type, 457 void *param); 458 static int eth_dev_event_callback_register(void); 459 static int eth_dev_event_callback_unregister(void); 460 461 462 /* 463 * Check if all the ports are started. 464 * If yes, return positive value. If not, return zero. 465 */ 466 static int all_ports_started(void); 467 468 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 469 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN; 470 471 /* 472 * Helper function to check if socket is already discovered. 473 * If yes, return positive value. If not, return zero. 474 */ 475 int 476 new_socket_id(unsigned int socket_id) 477 { 478 unsigned int i; 479 480 for (i = 0; i < num_sockets; i++) { 481 if (socket_ids[i] == socket_id) 482 return 0; 483 } 484 return 1; 485 } 486 487 /* 488 * Setup default configuration. 489 */ 490 static void 491 set_default_fwd_lcores_config(void) 492 { 493 unsigned int i; 494 unsigned int nb_lc; 495 unsigned int sock_num; 496 497 nb_lc = 0; 498 for (i = 0; i < RTE_MAX_LCORE; i++) { 499 if (!rte_lcore_is_enabled(i)) 500 continue; 501 sock_num = rte_lcore_to_socket_id(i); 502 if (new_socket_id(sock_num)) { 503 if (num_sockets >= RTE_MAX_NUMA_NODES) { 504 rte_exit(EXIT_FAILURE, 505 "Total sockets greater than %u\n", 506 RTE_MAX_NUMA_NODES); 507 } 508 socket_ids[num_sockets++] = sock_num; 509 } 510 if (i == rte_get_master_lcore()) 511 continue; 512 fwd_lcores_cpuids[nb_lc++] = i; 513 } 514 nb_lcores = (lcoreid_t) nb_lc; 515 nb_cfg_lcores = nb_lcores; 516 nb_fwd_lcores = 1; 517 } 518 519 static void 520 set_def_peer_eth_addrs(void) 521 { 522 portid_t i; 523 524 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 525 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 526 peer_eth_addrs[i].addr_bytes[5] = i; 527 } 528 } 529 530 static void 531 set_default_fwd_ports_config(void) 532 { 533 portid_t pt_id; 534 int i = 0; 535 536 RTE_ETH_FOREACH_DEV(pt_id) 537 fwd_ports_ids[i++] = pt_id; 538 539 nb_cfg_ports = nb_ports; 540 nb_fwd_ports = nb_ports; 541 } 542 543 void 544 set_def_fwd_config(void) 545 { 546 set_default_fwd_lcores_config(); 547 set_def_peer_eth_addrs(); 548 set_default_fwd_ports_config(); 549 } 550 551 /* extremely pessimistic estimation of memory required to create a mempool */ 552 static int 553 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 554 { 555 unsigned int n_pages, mbuf_per_pg, leftover; 556 uint64_t total_mem, mbuf_mem, obj_sz; 557 558 /* there is no good way to predict how much space the mempool will 559 * occupy because it will allocate chunks on the fly, and some of those 560 * will come from default DPDK memory while some will come from our 561 * external memory, so just assume 128MB will be enough for everyone. 562 */ 563 uint64_t hdr_mem = 128 << 20; 564 565 /* account for possible non-contiguousness */ 566 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 567 if (obj_sz > pgsz) { 568 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 569 return -1; 570 } 571 572 mbuf_per_pg = pgsz / obj_sz; 573 leftover = (nb_mbufs % mbuf_per_pg) > 0; 574 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 575 576 mbuf_mem = n_pages * pgsz; 577 578 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 579 580 if (total_mem > SIZE_MAX) { 581 TESTPMD_LOG(ERR, "Memory size too big\n"); 582 return -1; 583 } 584 *out = (size_t)total_mem; 585 586 return 0; 587 } 588 589 static inline uint32_t 590 bsf64(uint64_t v) 591 { 592 return (uint32_t)__builtin_ctzll(v); 593 } 594 595 static inline uint32_t 596 log2_u64(uint64_t v) 597 { 598 if (v == 0) 599 return 0; 600 v = rte_align64pow2(v); 601 return bsf64(v); 602 } 603 604 static int 605 pagesz_flags(uint64_t page_sz) 606 { 607 /* as per mmap() manpage, all page sizes are log2 of page size 608 * shifted by MAP_HUGE_SHIFT 609 */ 610 int log2 = log2_u64(page_sz); 611 612 return (log2 << HUGE_SHIFT); 613 } 614 615 static void * 616 alloc_mem(size_t memsz, size_t pgsz, bool huge) 617 { 618 void *addr; 619 int flags; 620 621 /* allocate anonymous hugepages */ 622 flags = MAP_ANONYMOUS | MAP_PRIVATE; 623 if (huge) 624 flags |= HUGE_FLAG | pagesz_flags(pgsz); 625 626 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 627 if (addr == MAP_FAILED) 628 return NULL; 629 630 return addr; 631 } 632 633 struct extmem_param { 634 void *addr; 635 size_t len; 636 size_t pgsz; 637 rte_iova_t *iova_table; 638 unsigned int iova_table_len; 639 }; 640 641 static int 642 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 643 bool huge) 644 { 645 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 646 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 647 unsigned int cur_page, n_pages, pgsz_idx; 648 size_t mem_sz, cur_pgsz; 649 rte_iova_t *iovas = NULL; 650 void *addr; 651 int ret; 652 653 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 654 /* skip anything that is too big */ 655 if (pgsizes[pgsz_idx] > SIZE_MAX) 656 continue; 657 658 cur_pgsz = pgsizes[pgsz_idx]; 659 660 /* if we were told not to allocate hugepages, override */ 661 if (!huge) 662 cur_pgsz = sysconf(_SC_PAGESIZE); 663 664 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 665 if (ret < 0) { 666 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 667 return -1; 668 } 669 670 /* allocate our memory */ 671 addr = alloc_mem(mem_sz, cur_pgsz, huge); 672 673 /* if we couldn't allocate memory with a specified page size, 674 * that doesn't mean we can't do it with other page sizes, so 675 * try another one. 676 */ 677 if (addr == NULL) 678 continue; 679 680 /* store IOVA addresses for every page in this memory area */ 681 n_pages = mem_sz / cur_pgsz; 682 683 iovas = malloc(sizeof(*iovas) * n_pages); 684 685 if (iovas == NULL) { 686 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 687 goto fail; 688 } 689 /* lock memory if it's not huge pages */ 690 if (!huge) 691 mlock(addr, mem_sz); 692 693 /* populate IOVA addresses */ 694 for (cur_page = 0; cur_page < n_pages; cur_page++) { 695 rte_iova_t iova; 696 size_t offset; 697 void *cur; 698 699 offset = cur_pgsz * cur_page; 700 cur = RTE_PTR_ADD(addr, offset); 701 702 /* touch the page before getting its IOVA */ 703 *(volatile char *)cur = 0; 704 705 iova = rte_mem_virt2iova(cur); 706 707 iovas[cur_page] = iova; 708 } 709 710 break; 711 } 712 /* if we couldn't allocate anything */ 713 if (iovas == NULL) 714 return -1; 715 716 param->addr = addr; 717 param->len = mem_sz; 718 param->pgsz = cur_pgsz; 719 param->iova_table = iovas; 720 param->iova_table_len = n_pages; 721 722 return 0; 723 fail: 724 if (iovas) 725 free(iovas); 726 if (addr) 727 munmap(addr, mem_sz); 728 729 return -1; 730 } 731 732 static int 733 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 734 { 735 struct extmem_param param; 736 int socket_id, ret; 737 738 memset(¶m, 0, sizeof(param)); 739 740 /* check if our heap exists */ 741 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 742 if (socket_id < 0) { 743 /* create our heap */ 744 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 745 if (ret < 0) { 746 TESTPMD_LOG(ERR, "Cannot create heap\n"); 747 return -1; 748 } 749 } 750 751 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 752 if (ret < 0) { 753 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 754 return -1; 755 } 756 757 /* we now have a valid memory area, so add it to heap */ 758 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 759 param.addr, param.len, param.iova_table, 760 param.iova_table_len, param.pgsz); 761 762 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 763 764 /* not needed any more */ 765 free(param.iova_table); 766 767 if (ret < 0) { 768 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 769 munmap(param.addr, param.len); 770 return -1; 771 } 772 773 /* success */ 774 775 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 776 param.len >> 20); 777 778 return 0; 779 } 780 781 /* 782 * Configuration initialisation done once at init time. 783 */ 784 static void 785 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 786 unsigned int socket_id) 787 { 788 char pool_name[RTE_MEMPOOL_NAMESIZE]; 789 struct rte_mempool *rte_mp = NULL; 790 uint32_t mb_size; 791 792 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 793 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 794 795 TESTPMD_LOG(INFO, 796 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 797 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 798 799 switch (mp_alloc_type) { 800 case MP_ALLOC_NATIVE: 801 { 802 /* wrapper to rte_mempool_create() */ 803 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 804 rte_mbuf_best_mempool_ops()); 805 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 806 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 807 break; 808 } 809 case MP_ALLOC_ANON: 810 { 811 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 812 mb_size, (unsigned int) mb_mempool_cache, 813 sizeof(struct rte_pktmbuf_pool_private), 814 socket_id, 0); 815 if (rte_mp == NULL) 816 goto err; 817 818 if (rte_mempool_populate_anon(rte_mp) == 0) { 819 rte_mempool_free(rte_mp); 820 rte_mp = NULL; 821 goto err; 822 } 823 rte_pktmbuf_pool_init(rte_mp, NULL); 824 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 825 break; 826 } 827 case MP_ALLOC_XMEM: 828 case MP_ALLOC_XMEM_HUGE: 829 { 830 int heap_socket; 831 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 832 833 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 834 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 835 836 heap_socket = 837 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 838 if (heap_socket < 0) 839 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 840 841 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 842 rte_mbuf_best_mempool_ops()); 843 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 844 mb_mempool_cache, 0, mbuf_seg_size, 845 heap_socket); 846 break; 847 } 848 default: 849 { 850 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 851 } 852 } 853 854 err: 855 if (rte_mp == NULL) { 856 rte_exit(EXIT_FAILURE, 857 "Creation of mbuf pool for socket %u failed: %s\n", 858 socket_id, rte_strerror(rte_errno)); 859 } else if (verbose_level > 0) { 860 rte_mempool_dump(stdout, rte_mp); 861 } 862 } 863 864 /* 865 * Check given socket id is valid or not with NUMA mode, 866 * if valid, return 0, else return -1 867 */ 868 static int 869 check_socket_id(const unsigned int socket_id) 870 { 871 static int warning_once = 0; 872 873 if (new_socket_id(socket_id)) { 874 if (!warning_once && numa_support) 875 printf("Warning: NUMA should be configured manually by" 876 " using --port-numa-config and" 877 " --ring-numa-config parameters along with" 878 " --numa.\n"); 879 warning_once = 1; 880 return -1; 881 } 882 return 0; 883 } 884 885 /* 886 * Get the allowed maximum number of RX queues. 887 * *pid return the port id which has minimal value of 888 * max_rx_queues in all ports. 889 */ 890 queueid_t 891 get_allowed_max_nb_rxq(portid_t *pid) 892 { 893 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 894 portid_t pi; 895 struct rte_eth_dev_info dev_info; 896 897 RTE_ETH_FOREACH_DEV(pi) { 898 rte_eth_dev_info_get(pi, &dev_info); 899 if (dev_info.max_rx_queues < allowed_max_rxq) { 900 allowed_max_rxq = dev_info.max_rx_queues; 901 *pid = pi; 902 } 903 } 904 return allowed_max_rxq; 905 } 906 907 /* 908 * Check input rxq is valid or not. 909 * If input rxq is not greater than any of maximum number 910 * of RX queues of all ports, it is valid. 911 * if valid, return 0, else return -1 912 */ 913 int 914 check_nb_rxq(queueid_t rxq) 915 { 916 queueid_t allowed_max_rxq; 917 portid_t pid = 0; 918 919 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 920 if (rxq > allowed_max_rxq) { 921 printf("Fail: input rxq (%u) can't be greater " 922 "than max_rx_queues (%u) of port %u\n", 923 rxq, 924 allowed_max_rxq, 925 pid); 926 return -1; 927 } 928 return 0; 929 } 930 931 /* 932 * Get the allowed maximum number of TX queues. 933 * *pid return the port id which has minimal value of 934 * max_tx_queues in all ports. 935 */ 936 queueid_t 937 get_allowed_max_nb_txq(portid_t *pid) 938 { 939 queueid_t allowed_max_txq = MAX_QUEUE_ID; 940 portid_t pi; 941 struct rte_eth_dev_info dev_info; 942 943 RTE_ETH_FOREACH_DEV(pi) { 944 rte_eth_dev_info_get(pi, &dev_info); 945 if (dev_info.max_tx_queues < allowed_max_txq) { 946 allowed_max_txq = dev_info.max_tx_queues; 947 *pid = pi; 948 } 949 } 950 return allowed_max_txq; 951 } 952 953 /* 954 * Check input txq is valid or not. 955 * If input txq is not greater than any of maximum number 956 * of TX queues of all ports, it is valid. 957 * if valid, return 0, else return -1 958 */ 959 int 960 check_nb_txq(queueid_t txq) 961 { 962 queueid_t allowed_max_txq; 963 portid_t pid = 0; 964 965 allowed_max_txq = get_allowed_max_nb_txq(&pid); 966 if (txq > allowed_max_txq) { 967 printf("Fail: input txq (%u) can't be greater " 968 "than max_tx_queues (%u) of port %u\n", 969 txq, 970 allowed_max_txq, 971 pid); 972 return -1; 973 } 974 return 0; 975 } 976 977 static void 978 init_config(void) 979 { 980 portid_t pid; 981 struct rte_port *port; 982 struct rte_mempool *mbp; 983 unsigned int nb_mbuf_per_pool; 984 lcoreid_t lc_id; 985 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 986 struct rte_gro_param gro_param; 987 uint32_t gso_types; 988 int k; 989 990 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 991 992 if (numa_support) { 993 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 994 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 995 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 996 } 997 998 /* Configuration of logical cores. */ 999 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1000 sizeof(struct fwd_lcore *) * nb_lcores, 1001 RTE_CACHE_LINE_SIZE); 1002 if (fwd_lcores == NULL) { 1003 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1004 "failed\n", nb_lcores); 1005 } 1006 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1007 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1008 sizeof(struct fwd_lcore), 1009 RTE_CACHE_LINE_SIZE); 1010 if (fwd_lcores[lc_id] == NULL) { 1011 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1012 "failed\n"); 1013 } 1014 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1015 } 1016 1017 RTE_ETH_FOREACH_DEV(pid) { 1018 port = &ports[pid]; 1019 /* Apply default TxRx configuration for all ports */ 1020 port->dev_conf.txmode = tx_mode; 1021 port->dev_conf.rxmode = rx_mode; 1022 rte_eth_dev_info_get(pid, &port->dev_info); 1023 1024 if (!(port->dev_info.tx_offload_capa & 1025 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1026 port->dev_conf.txmode.offloads &= 1027 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1028 if (numa_support) { 1029 if (port_numa[pid] != NUMA_NO_CONFIG) 1030 port_per_socket[port_numa[pid]]++; 1031 else { 1032 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1033 1034 /* if socket_id is invalid, set to 0 */ 1035 if (check_socket_id(socket_id) < 0) 1036 socket_id = 0; 1037 port_per_socket[socket_id]++; 1038 } 1039 } 1040 1041 /* Apply Rx offloads configuration */ 1042 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1043 port->rx_conf[k].offloads = 1044 port->dev_conf.rxmode.offloads; 1045 /* Apply Tx offloads configuration */ 1046 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1047 port->tx_conf[k].offloads = 1048 port->dev_conf.txmode.offloads; 1049 1050 /* set flag to initialize port/queue */ 1051 port->need_reconfig = 1; 1052 port->need_reconfig_queues = 1; 1053 } 1054 1055 /* 1056 * Create pools of mbuf. 1057 * If NUMA support is disabled, create a single pool of mbuf in 1058 * socket 0 memory by default. 1059 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1060 * 1061 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1062 * nb_txd can be configured at run time. 1063 */ 1064 if (param_total_num_mbufs) 1065 nb_mbuf_per_pool = param_total_num_mbufs; 1066 else { 1067 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1068 (nb_lcores * mb_mempool_cache) + 1069 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1070 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1071 } 1072 1073 if (numa_support) { 1074 uint8_t i; 1075 1076 for (i = 0; i < num_sockets; i++) 1077 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1078 socket_ids[i]); 1079 } else { 1080 if (socket_num == UMA_NO_CONFIG) 1081 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 1082 else 1083 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 1084 socket_num); 1085 } 1086 1087 init_port_config(); 1088 1089 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1090 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1091 /* 1092 * Records which Mbuf pool to use by each logical core, if needed. 1093 */ 1094 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1095 mbp = mbuf_pool_find( 1096 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1097 1098 if (mbp == NULL) 1099 mbp = mbuf_pool_find(0); 1100 fwd_lcores[lc_id]->mbp = mbp; 1101 /* initialize GSO context */ 1102 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1103 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1104 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1105 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN - 1106 ETHER_CRC_LEN; 1107 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1108 } 1109 1110 /* Configuration of packet forwarding streams. */ 1111 if (init_fwd_streams() < 0) 1112 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1113 1114 fwd_config_setup(); 1115 1116 /* create a gro context for each lcore */ 1117 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1118 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1119 gro_param.max_item_per_flow = MAX_PKT_BURST; 1120 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1121 gro_param.socket_id = rte_lcore_to_socket_id( 1122 fwd_lcores_cpuids[lc_id]); 1123 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1124 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1125 rte_exit(EXIT_FAILURE, 1126 "rte_gro_ctx_create() failed\n"); 1127 } 1128 } 1129 1130 #if defined RTE_LIBRTE_PMD_SOFTNIC 1131 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1132 RTE_ETH_FOREACH_DEV(pid) { 1133 port = &ports[pid]; 1134 const char *driver = port->dev_info.driver_name; 1135 1136 if (strcmp(driver, "net_softnic") == 0) 1137 port->softport.fwd_lcore_arg = fwd_lcores; 1138 } 1139 } 1140 #endif 1141 1142 } 1143 1144 1145 void 1146 reconfig(portid_t new_port_id, unsigned socket_id) 1147 { 1148 struct rte_port *port; 1149 1150 /* Reconfiguration of Ethernet ports. */ 1151 port = &ports[new_port_id]; 1152 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1153 1154 /* set flag to initialize port/queue */ 1155 port->need_reconfig = 1; 1156 port->need_reconfig_queues = 1; 1157 port->socket_id = socket_id; 1158 1159 init_port_config(); 1160 } 1161 1162 1163 int 1164 init_fwd_streams(void) 1165 { 1166 portid_t pid; 1167 struct rte_port *port; 1168 streamid_t sm_id, nb_fwd_streams_new; 1169 queueid_t q; 1170 1171 /* set socket id according to numa or not */ 1172 RTE_ETH_FOREACH_DEV(pid) { 1173 port = &ports[pid]; 1174 if (nb_rxq > port->dev_info.max_rx_queues) { 1175 printf("Fail: nb_rxq(%d) is greater than " 1176 "max_rx_queues(%d)\n", nb_rxq, 1177 port->dev_info.max_rx_queues); 1178 return -1; 1179 } 1180 if (nb_txq > port->dev_info.max_tx_queues) { 1181 printf("Fail: nb_txq(%d) is greater than " 1182 "max_tx_queues(%d)\n", nb_txq, 1183 port->dev_info.max_tx_queues); 1184 return -1; 1185 } 1186 if (numa_support) { 1187 if (port_numa[pid] != NUMA_NO_CONFIG) 1188 port->socket_id = port_numa[pid]; 1189 else { 1190 port->socket_id = rte_eth_dev_socket_id(pid); 1191 1192 /* if socket_id is invalid, set to 0 */ 1193 if (check_socket_id(port->socket_id) < 0) 1194 port->socket_id = 0; 1195 } 1196 } 1197 else { 1198 if (socket_num == UMA_NO_CONFIG) 1199 port->socket_id = 0; 1200 else 1201 port->socket_id = socket_num; 1202 } 1203 } 1204 1205 q = RTE_MAX(nb_rxq, nb_txq); 1206 if (q == 0) { 1207 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1208 return -1; 1209 } 1210 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1211 if (nb_fwd_streams_new == nb_fwd_streams) 1212 return 0; 1213 /* clear the old */ 1214 if (fwd_streams != NULL) { 1215 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1216 if (fwd_streams[sm_id] == NULL) 1217 continue; 1218 rte_free(fwd_streams[sm_id]); 1219 fwd_streams[sm_id] = NULL; 1220 } 1221 rte_free(fwd_streams); 1222 fwd_streams = NULL; 1223 } 1224 1225 /* init new */ 1226 nb_fwd_streams = nb_fwd_streams_new; 1227 if (nb_fwd_streams) { 1228 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1229 sizeof(struct fwd_stream *) * nb_fwd_streams, 1230 RTE_CACHE_LINE_SIZE); 1231 if (fwd_streams == NULL) 1232 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1233 " (struct fwd_stream *)) failed\n", 1234 nb_fwd_streams); 1235 1236 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1237 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1238 " struct fwd_stream", sizeof(struct fwd_stream), 1239 RTE_CACHE_LINE_SIZE); 1240 if (fwd_streams[sm_id] == NULL) 1241 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1242 "(struct fwd_stream) failed\n"); 1243 } 1244 } 1245 1246 return 0; 1247 } 1248 1249 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1250 static void 1251 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1252 { 1253 unsigned int total_burst; 1254 unsigned int nb_burst; 1255 unsigned int burst_stats[3]; 1256 uint16_t pktnb_stats[3]; 1257 uint16_t nb_pkt; 1258 int burst_percent[3]; 1259 1260 /* 1261 * First compute the total number of packet bursts and the 1262 * two highest numbers of bursts of the same number of packets. 1263 */ 1264 total_burst = 0; 1265 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1266 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1267 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1268 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1269 if (nb_burst == 0) 1270 continue; 1271 total_burst += nb_burst; 1272 if (nb_burst > burst_stats[0]) { 1273 burst_stats[1] = burst_stats[0]; 1274 pktnb_stats[1] = pktnb_stats[0]; 1275 burst_stats[0] = nb_burst; 1276 pktnb_stats[0] = nb_pkt; 1277 } else if (nb_burst > burst_stats[1]) { 1278 burst_stats[1] = nb_burst; 1279 pktnb_stats[1] = nb_pkt; 1280 } 1281 } 1282 if (total_burst == 0) 1283 return; 1284 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1285 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1286 burst_percent[0], (int) pktnb_stats[0]); 1287 if (burst_stats[0] == total_burst) { 1288 printf("]\n"); 1289 return; 1290 } 1291 if (burst_stats[0] + burst_stats[1] == total_burst) { 1292 printf(" + %d%% of %d pkts]\n", 1293 100 - burst_percent[0], pktnb_stats[1]); 1294 return; 1295 } 1296 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1297 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1298 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1299 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1300 return; 1301 } 1302 printf(" + %d%% of %d pkts + %d%% of others]\n", 1303 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1304 } 1305 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1306 1307 static void 1308 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats) 1309 { 1310 struct rte_port *port; 1311 uint8_t i; 1312 1313 static const char *fwd_stats_border = "----------------------"; 1314 1315 port = &ports[port_id]; 1316 printf("\n %s Forward statistics for port %-2d %s\n", 1317 fwd_stats_border, port_id, fwd_stats_border); 1318 1319 if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) { 1320 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1321 "%-"PRIu64"\n", 1322 stats->ipackets, stats->imissed, 1323 (uint64_t) (stats->ipackets + stats->imissed)); 1324 1325 if (cur_fwd_eng == &csum_fwd_engine) 1326 printf(" Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n", 1327 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 1328 if ((stats->ierrors + stats->rx_nombuf) > 0) { 1329 printf(" RX-error: %-"PRIu64"\n", stats->ierrors); 1330 printf(" RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf); 1331 } 1332 1333 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1334 "%-"PRIu64"\n", 1335 stats->opackets, port->tx_dropped, 1336 (uint64_t) (stats->opackets + port->tx_dropped)); 1337 } 1338 else { 1339 printf(" RX-packets: %14"PRIu64" RX-dropped:%14"PRIu64" RX-total:" 1340 "%14"PRIu64"\n", 1341 stats->ipackets, stats->imissed, 1342 (uint64_t) (stats->ipackets + stats->imissed)); 1343 1344 if (cur_fwd_eng == &csum_fwd_engine) 1345 printf(" Bad-ipcsum:%14"PRIu64" Bad-l4csum:%14"PRIu64"\n", 1346 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 1347 if ((stats->ierrors + stats->rx_nombuf) > 0) { 1348 printf(" RX-error:%"PRIu64"\n", stats->ierrors); 1349 printf(" RX-nombufs: %14"PRIu64"\n", 1350 stats->rx_nombuf); 1351 } 1352 1353 printf(" TX-packets: %14"PRIu64" TX-dropped:%14"PRIu64" TX-total:" 1354 "%14"PRIu64"\n", 1355 stats->opackets, port->tx_dropped, 1356 (uint64_t) (stats->opackets + port->tx_dropped)); 1357 } 1358 1359 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1360 if (port->rx_stream) 1361 pkt_burst_stats_display("RX", 1362 &port->rx_stream->rx_burst_stats); 1363 if (port->tx_stream) 1364 pkt_burst_stats_display("TX", 1365 &port->tx_stream->tx_burst_stats); 1366 #endif 1367 1368 if (port->rx_queue_stats_mapping_enabled) { 1369 printf("\n"); 1370 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 1371 printf(" Stats reg %2d RX-packets:%14"PRIu64 1372 " RX-errors:%14"PRIu64 1373 " RX-bytes:%14"PRIu64"\n", 1374 i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]); 1375 } 1376 printf("\n"); 1377 } 1378 if (port->tx_queue_stats_mapping_enabled) { 1379 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 1380 printf(" Stats reg %2d TX-packets:%14"PRIu64 1381 " TX-bytes:%14"PRIu64"\n", 1382 i, stats->q_opackets[i], stats->q_obytes[i]); 1383 } 1384 } 1385 1386 printf(" %s--------------------------------%s\n", 1387 fwd_stats_border, fwd_stats_border); 1388 } 1389 1390 static void 1391 fwd_stream_stats_display(streamid_t stream_id) 1392 { 1393 struct fwd_stream *fs; 1394 static const char *fwd_top_stats_border = "-------"; 1395 1396 fs = fwd_streams[stream_id]; 1397 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1398 (fs->fwd_dropped == 0)) 1399 return; 1400 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1401 "TX Port=%2d/Queue=%2d %s\n", 1402 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1403 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1404 printf(" RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u", 1405 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1406 1407 /* if checksum mode */ 1408 if (cur_fwd_eng == &csum_fwd_engine) { 1409 printf(" RX- bad IP checksum: %-14u Rx- bad L4 checksum: " 1410 "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum); 1411 } 1412 1413 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1414 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1415 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1416 #endif 1417 } 1418 1419 static void 1420 flush_fwd_rx_queues(void) 1421 { 1422 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1423 portid_t rxp; 1424 portid_t port_id; 1425 queueid_t rxq; 1426 uint16_t nb_rx; 1427 uint16_t i; 1428 uint8_t j; 1429 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1430 uint64_t timer_period; 1431 1432 /* convert to number of cycles */ 1433 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1434 1435 for (j = 0; j < 2; j++) { 1436 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1437 for (rxq = 0; rxq < nb_rxq; rxq++) { 1438 port_id = fwd_ports_ids[rxp]; 1439 /** 1440 * testpmd can stuck in the below do while loop 1441 * if rte_eth_rx_burst() always returns nonzero 1442 * packets. So timer is added to exit this loop 1443 * after 1sec timer expiry. 1444 */ 1445 prev_tsc = rte_rdtsc(); 1446 do { 1447 nb_rx = rte_eth_rx_burst(port_id, rxq, 1448 pkts_burst, MAX_PKT_BURST); 1449 for (i = 0; i < nb_rx; i++) 1450 rte_pktmbuf_free(pkts_burst[i]); 1451 1452 cur_tsc = rte_rdtsc(); 1453 diff_tsc = cur_tsc - prev_tsc; 1454 timer_tsc += diff_tsc; 1455 } while ((nb_rx > 0) && 1456 (timer_tsc < timer_period)); 1457 timer_tsc = 0; 1458 } 1459 } 1460 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1461 } 1462 } 1463 1464 static void 1465 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1466 { 1467 struct fwd_stream **fsm; 1468 streamid_t nb_fs; 1469 streamid_t sm_id; 1470 #ifdef RTE_LIBRTE_BITRATE 1471 uint64_t tics_per_1sec; 1472 uint64_t tics_datum; 1473 uint64_t tics_current; 1474 uint16_t i, cnt_ports; 1475 1476 cnt_ports = nb_ports; 1477 tics_datum = rte_rdtsc(); 1478 tics_per_1sec = rte_get_timer_hz(); 1479 #endif 1480 fsm = &fwd_streams[fc->stream_idx]; 1481 nb_fs = fc->stream_nb; 1482 do { 1483 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1484 (*pkt_fwd)(fsm[sm_id]); 1485 #ifdef RTE_LIBRTE_BITRATE 1486 if (bitrate_enabled != 0 && 1487 bitrate_lcore_id == rte_lcore_id()) { 1488 tics_current = rte_rdtsc(); 1489 if (tics_current - tics_datum >= tics_per_1sec) { 1490 /* Periodic bitrate calculation */ 1491 for (i = 0; i < cnt_ports; i++) 1492 rte_stats_bitrate_calc(bitrate_data, 1493 ports_ids[i]); 1494 tics_datum = tics_current; 1495 } 1496 } 1497 #endif 1498 #ifdef RTE_LIBRTE_LATENCY_STATS 1499 if (latencystats_enabled != 0 && 1500 latencystats_lcore_id == rte_lcore_id()) 1501 rte_latencystats_update(); 1502 #endif 1503 1504 } while (! fc->stopped); 1505 } 1506 1507 static int 1508 start_pkt_forward_on_core(void *fwd_arg) 1509 { 1510 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1511 cur_fwd_config.fwd_eng->packet_fwd); 1512 return 0; 1513 } 1514 1515 /* 1516 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1517 * Used to start communication flows in network loopback test configurations. 1518 */ 1519 static int 1520 run_one_txonly_burst_on_core(void *fwd_arg) 1521 { 1522 struct fwd_lcore *fwd_lc; 1523 struct fwd_lcore tmp_lcore; 1524 1525 fwd_lc = (struct fwd_lcore *) fwd_arg; 1526 tmp_lcore = *fwd_lc; 1527 tmp_lcore.stopped = 1; 1528 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1529 return 0; 1530 } 1531 1532 /* 1533 * Launch packet forwarding: 1534 * - Setup per-port forwarding context. 1535 * - launch logical cores with their forwarding configuration. 1536 */ 1537 static void 1538 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1539 { 1540 port_fwd_begin_t port_fwd_begin; 1541 unsigned int i; 1542 unsigned int lc_id; 1543 int diag; 1544 1545 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1546 if (port_fwd_begin != NULL) { 1547 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1548 (*port_fwd_begin)(fwd_ports_ids[i]); 1549 } 1550 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1551 lc_id = fwd_lcores_cpuids[i]; 1552 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1553 fwd_lcores[i]->stopped = 0; 1554 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1555 fwd_lcores[i], lc_id); 1556 if (diag != 0) 1557 printf("launch lcore %u failed - diag=%d\n", 1558 lc_id, diag); 1559 } 1560 } 1561 } 1562 1563 /* 1564 * Update the forward ports list. 1565 */ 1566 void 1567 update_fwd_ports(portid_t new_pid) 1568 { 1569 unsigned int i; 1570 unsigned int new_nb_fwd_ports = 0; 1571 int move = 0; 1572 1573 for (i = 0; i < nb_fwd_ports; ++i) { 1574 if (port_id_is_invalid(fwd_ports_ids[i], DISABLED_WARN)) 1575 move = 1; 1576 else if (move) 1577 fwd_ports_ids[new_nb_fwd_ports++] = fwd_ports_ids[i]; 1578 else 1579 new_nb_fwd_ports++; 1580 } 1581 if (new_pid < RTE_MAX_ETHPORTS) 1582 fwd_ports_ids[new_nb_fwd_ports++] = new_pid; 1583 1584 nb_fwd_ports = new_nb_fwd_ports; 1585 nb_cfg_ports = new_nb_fwd_ports; 1586 } 1587 1588 /* 1589 * Launch packet forwarding configuration. 1590 */ 1591 void 1592 start_packet_forwarding(int with_tx_first) 1593 { 1594 port_fwd_begin_t port_fwd_begin; 1595 port_fwd_end_t port_fwd_end; 1596 struct rte_port *port; 1597 unsigned int i; 1598 portid_t pt_id; 1599 streamid_t sm_id; 1600 1601 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1602 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1603 1604 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1605 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1606 1607 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1608 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1609 (!nb_rxq || !nb_txq)) 1610 rte_exit(EXIT_FAILURE, 1611 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1612 cur_fwd_eng->fwd_mode_name); 1613 1614 if (all_ports_started() == 0) { 1615 printf("Not all ports were started\n"); 1616 return; 1617 } 1618 if (test_done == 0) { 1619 printf("Packet forwarding already started\n"); 1620 return; 1621 } 1622 1623 1624 if(dcb_test) { 1625 for (i = 0; i < nb_fwd_ports; i++) { 1626 pt_id = fwd_ports_ids[i]; 1627 port = &ports[pt_id]; 1628 if (!port->dcb_flag) { 1629 printf("In DCB mode, all forwarding ports must " 1630 "be configured in this mode.\n"); 1631 return; 1632 } 1633 } 1634 if (nb_fwd_lcores == 1) { 1635 printf("In DCB mode,the nb forwarding cores " 1636 "should be larger than 1.\n"); 1637 return; 1638 } 1639 } 1640 test_done = 0; 1641 1642 fwd_config_setup(); 1643 1644 if(!no_flush_rx) 1645 flush_fwd_rx_queues(); 1646 1647 pkt_fwd_config_display(&cur_fwd_config); 1648 rxtx_config_display(); 1649 1650 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1651 pt_id = fwd_ports_ids[i]; 1652 port = &ports[pt_id]; 1653 rte_eth_stats_get(pt_id, &port->stats); 1654 port->tx_dropped = 0; 1655 1656 map_port_queue_stats_mapping_registers(pt_id, port); 1657 } 1658 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1659 fwd_streams[sm_id]->rx_packets = 0; 1660 fwd_streams[sm_id]->tx_packets = 0; 1661 fwd_streams[sm_id]->fwd_dropped = 0; 1662 fwd_streams[sm_id]->rx_bad_ip_csum = 0; 1663 fwd_streams[sm_id]->rx_bad_l4_csum = 0; 1664 1665 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1666 memset(&fwd_streams[sm_id]->rx_burst_stats, 0, 1667 sizeof(fwd_streams[sm_id]->rx_burst_stats)); 1668 memset(&fwd_streams[sm_id]->tx_burst_stats, 0, 1669 sizeof(fwd_streams[sm_id]->tx_burst_stats)); 1670 #endif 1671 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1672 fwd_streams[sm_id]->core_cycles = 0; 1673 #endif 1674 } 1675 if (with_tx_first) { 1676 port_fwd_begin = tx_only_engine.port_fwd_begin; 1677 if (port_fwd_begin != NULL) { 1678 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1679 (*port_fwd_begin)(fwd_ports_ids[i]); 1680 } 1681 while (with_tx_first--) { 1682 launch_packet_forwarding( 1683 run_one_txonly_burst_on_core); 1684 rte_eal_mp_wait_lcore(); 1685 } 1686 port_fwd_end = tx_only_engine.port_fwd_end; 1687 if (port_fwd_end != NULL) { 1688 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1689 (*port_fwd_end)(fwd_ports_ids[i]); 1690 } 1691 } 1692 launch_packet_forwarding(start_pkt_forward_on_core); 1693 } 1694 1695 void 1696 stop_packet_forwarding(void) 1697 { 1698 struct rte_eth_stats stats; 1699 struct rte_port *port; 1700 port_fwd_end_t port_fwd_end; 1701 int i; 1702 portid_t pt_id; 1703 streamid_t sm_id; 1704 lcoreid_t lc_id; 1705 uint64_t total_recv; 1706 uint64_t total_xmit; 1707 uint64_t total_rx_dropped; 1708 uint64_t total_tx_dropped; 1709 uint64_t total_rx_nombuf; 1710 uint64_t tx_dropped; 1711 uint64_t rx_bad_ip_csum; 1712 uint64_t rx_bad_l4_csum; 1713 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1714 uint64_t fwd_cycles; 1715 #endif 1716 1717 static const char *acc_stats_border = "+++++++++++++++"; 1718 1719 if (test_done) { 1720 printf("Packet forwarding not started\n"); 1721 return; 1722 } 1723 printf("Telling cores to stop..."); 1724 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1725 fwd_lcores[lc_id]->stopped = 1; 1726 printf("\nWaiting for lcores to finish...\n"); 1727 rte_eal_mp_wait_lcore(); 1728 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1729 if (port_fwd_end != NULL) { 1730 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1731 pt_id = fwd_ports_ids[i]; 1732 (*port_fwd_end)(pt_id); 1733 } 1734 } 1735 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1736 fwd_cycles = 0; 1737 #endif 1738 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1739 if (cur_fwd_config.nb_fwd_streams > 1740 cur_fwd_config.nb_fwd_ports) { 1741 fwd_stream_stats_display(sm_id); 1742 ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL; 1743 ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL; 1744 } else { 1745 ports[fwd_streams[sm_id]->tx_port].tx_stream = 1746 fwd_streams[sm_id]; 1747 ports[fwd_streams[sm_id]->rx_port].rx_stream = 1748 fwd_streams[sm_id]; 1749 } 1750 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped; 1751 tx_dropped = (uint64_t) (tx_dropped + 1752 fwd_streams[sm_id]->fwd_dropped); 1753 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped; 1754 1755 rx_bad_ip_csum = 1756 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum; 1757 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum + 1758 fwd_streams[sm_id]->rx_bad_ip_csum); 1759 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum = 1760 rx_bad_ip_csum; 1761 1762 rx_bad_l4_csum = 1763 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum; 1764 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum + 1765 fwd_streams[sm_id]->rx_bad_l4_csum); 1766 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum = 1767 rx_bad_l4_csum; 1768 1769 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1770 fwd_cycles = (uint64_t) (fwd_cycles + 1771 fwd_streams[sm_id]->core_cycles); 1772 #endif 1773 } 1774 total_recv = 0; 1775 total_xmit = 0; 1776 total_rx_dropped = 0; 1777 total_tx_dropped = 0; 1778 total_rx_nombuf = 0; 1779 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1780 pt_id = fwd_ports_ids[i]; 1781 1782 port = &ports[pt_id]; 1783 rte_eth_stats_get(pt_id, &stats); 1784 stats.ipackets -= port->stats.ipackets; 1785 port->stats.ipackets = 0; 1786 stats.opackets -= port->stats.opackets; 1787 port->stats.opackets = 0; 1788 stats.ibytes -= port->stats.ibytes; 1789 port->stats.ibytes = 0; 1790 stats.obytes -= port->stats.obytes; 1791 port->stats.obytes = 0; 1792 stats.imissed -= port->stats.imissed; 1793 port->stats.imissed = 0; 1794 stats.oerrors -= port->stats.oerrors; 1795 port->stats.oerrors = 0; 1796 stats.rx_nombuf -= port->stats.rx_nombuf; 1797 port->stats.rx_nombuf = 0; 1798 1799 total_recv += stats.ipackets; 1800 total_xmit += stats.opackets; 1801 total_rx_dropped += stats.imissed; 1802 total_tx_dropped += port->tx_dropped; 1803 total_rx_nombuf += stats.rx_nombuf; 1804 1805 fwd_port_stats_display(pt_id, &stats); 1806 } 1807 1808 printf("\n %s Accumulated forward statistics for all ports" 1809 "%s\n", 1810 acc_stats_border, acc_stats_border); 1811 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1812 "%-"PRIu64"\n" 1813 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1814 "%-"PRIu64"\n", 1815 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1816 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1817 if (total_rx_nombuf > 0) 1818 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1819 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1820 "%s\n", 1821 acc_stats_border, acc_stats_border); 1822 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1823 if (total_recv > 0) 1824 printf("\n CPU cycles/packet=%u (total cycles=" 1825 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1826 (unsigned int)(fwd_cycles / total_recv), 1827 fwd_cycles, total_recv); 1828 #endif 1829 printf("\nDone.\n"); 1830 test_done = 1; 1831 } 1832 1833 void 1834 dev_set_link_up(portid_t pid) 1835 { 1836 if (rte_eth_dev_set_link_up(pid) < 0) 1837 printf("\nSet link up fail.\n"); 1838 } 1839 1840 void 1841 dev_set_link_down(portid_t pid) 1842 { 1843 if (rte_eth_dev_set_link_down(pid) < 0) 1844 printf("\nSet link down fail.\n"); 1845 } 1846 1847 static int 1848 all_ports_started(void) 1849 { 1850 portid_t pi; 1851 struct rte_port *port; 1852 1853 RTE_ETH_FOREACH_DEV(pi) { 1854 port = &ports[pi]; 1855 /* Check if there is a port which is not started */ 1856 if ((port->port_status != RTE_PORT_STARTED) && 1857 (port->slave_flag == 0)) 1858 return 0; 1859 } 1860 1861 /* No port is not started */ 1862 return 1; 1863 } 1864 1865 int 1866 port_is_stopped(portid_t port_id) 1867 { 1868 struct rte_port *port = &ports[port_id]; 1869 1870 if ((port->port_status != RTE_PORT_STOPPED) && 1871 (port->slave_flag == 0)) 1872 return 0; 1873 return 1; 1874 } 1875 1876 int 1877 all_ports_stopped(void) 1878 { 1879 portid_t pi; 1880 1881 RTE_ETH_FOREACH_DEV(pi) { 1882 if (!port_is_stopped(pi)) 1883 return 0; 1884 } 1885 1886 return 1; 1887 } 1888 1889 int 1890 port_is_started(portid_t port_id) 1891 { 1892 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1893 return 0; 1894 1895 if (ports[port_id].port_status != RTE_PORT_STARTED) 1896 return 0; 1897 1898 return 1; 1899 } 1900 1901 static int 1902 port_is_closed(portid_t port_id) 1903 { 1904 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1905 return 0; 1906 1907 if (ports[port_id].port_status != RTE_PORT_CLOSED) 1908 return 0; 1909 1910 return 1; 1911 } 1912 1913 int 1914 start_port(portid_t pid) 1915 { 1916 int diag, need_check_link_status = -1; 1917 portid_t pi; 1918 queueid_t qi; 1919 struct rte_port *port; 1920 struct ether_addr mac_addr; 1921 enum rte_eth_event_type event_type; 1922 1923 if (port_id_is_invalid(pid, ENABLED_WARN)) 1924 return 0; 1925 1926 if(dcb_config) 1927 dcb_test = 1; 1928 RTE_ETH_FOREACH_DEV(pi) { 1929 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1930 continue; 1931 1932 need_check_link_status = 0; 1933 port = &ports[pi]; 1934 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1935 RTE_PORT_HANDLING) == 0) { 1936 printf("Port %d is now not stopped\n", pi); 1937 continue; 1938 } 1939 1940 if (port->need_reconfig > 0) { 1941 port->need_reconfig = 0; 1942 1943 if (flow_isolate_all) { 1944 int ret = port_flow_isolate(pi, 1); 1945 if (ret) { 1946 printf("Failed to apply isolated" 1947 " mode on port %d\n", pi); 1948 return -1; 1949 } 1950 } 1951 1952 printf("Configuring Port %d (socket %u)\n", pi, 1953 port->socket_id); 1954 /* configure port */ 1955 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1956 &(port->dev_conf)); 1957 if (diag != 0) { 1958 if (rte_atomic16_cmpset(&(port->port_status), 1959 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1960 printf("Port %d can not be set back " 1961 "to stopped\n", pi); 1962 printf("Fail to configure port %d\n", pi); 1963 /* try to reconfigure port next time */ 1964 port->need_reconfig = 1; 1965 return -1; 1966 } 1967 } 1968 if (port->need_reconfig_queues > 0) { 1969 port->need_reconfig_queues = 0; 1970 /* setup tx queues */ 1971 for (qi = 0; qi < nb_txq; qi++) { 1972 if ((numa_support) && 1973 (txring_numa[pi] != NUMA_NO_CONFIG)) 1974 diag = rte_eth_tx_queue_setup(pi, qi, 1975 port->nb_tx_desc[qi], 1976 txring_numa[pi], 1977 &(port->tx_conf[qi])); 1978 else 1979 diag = rte_eth_tx_queue_setup(pi, qi, 1980 port->nb_tx_desc[qi], 1981 port->socket_id, 1982 &(port->tx_conf[qi])); 1983 1984 if (diag == 0) 1985 continue; 1986 1987 /* Fail to setup tx queue, return */ 1988 if (rte_atomic16_cmpset(&(port->port_status), 1989 RTE_PORT_HANDLING, 1990 RTE_PORT_STOPPED) == 0) 1991 printf("Port %d can not be set back " 1992 "to stopped\n", pi); 1993 printf("Fail to configure port %d tx queues\n", 1994 pi); 1995 /* try to reconfigure queues next time */ 1996 port->need_reconfig_queues = 1; 1997 return -1; 1998 } 1999 for (qi = 0; qi < nb_rxq; qi++) { 2000 /* setup rx queues */ 2001 if ((numa_support) && 2002 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2003 struct rte_mempool * mp = 2004 mbuf_pool_find(rxring_numa[pi]); 2005 if (mp == NULL) { 2006 printf("Failed to setup RX queue:" 2007 "No mempool allocation" 2008 " on the socket %d\n", 2009 rxring_numa[pi]); 2010 return -1; 2011 } 2012 2013 diag = rte_eth_rx_queue_setup(pi, qi, 2014 port->nb_rx_desc[qi], 2015 rxring_numa[pi], 2016 &(port->rx_conf[qi]), 2017 mp); 2018 } else { 2019 struct rte_mempool *mp = 2020 mbuf_pool_find(port->socket_id); 2021 if (mp == NULL) { 2022 printf("Failed to setup RX queue:" 2023 "No mempool allocation" 2024 " on the socket %d\n", 2025 port->socket_id); 2026 return -1; 2027 } 2028 diag = rte_eth_rx_queue_setup(pi, qi, 2029 port->nb_rx_desc[qi], 2030 port->socket_id, 2031 &(port->rx_conf[qi]), 2032 mp); 2033 } 2034 if (diag == 0) 2035 continue; 2036 2037 /* Fail to setup rx queue, return */ 2038 if (rte_atomic16_cmpset(&(port->port_status), 2039 RTE_PORT_HANDLING, 2040 RTE_PORT_STOPPED) == 0) 2041 printf("Port %d can not be set back " 2042 "to stopped\n", pi); 2043 printf("Fail to configure port %d rx queues\n", 2044 pi); 2045 /* try to reconfigure queues next time */ 2046 port->need_reconfig_queues = 1; 2047 return -1; 2048 } 2049 } 2050 2051 /* start port */ 2052 if (rte_eth_dev_start(pi) < 0) { 2053 printf("Fail to start port %d\n", pi); 2054 2055 /* Fail to setup rx queue, return */ 2056 if (rte_atomic16_cmpset(&(port->port_status), 2057 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2058 printf("Port %d can not be set back to " 2059 "stopped\n", pi); 2060 continue; 2061 } 2062 2063 if (rte_atomic16_cmpset(&(port->port_status), 2064 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2065 printf("Port %d can not be set into started\n", pi); 2066 2067 rte_eth_macaddr_get(pi, &mac_addr); 2068 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2069 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2070 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2071 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2072 2073 /* at least one port started, need checking link status */ 2074 need_check_link_status = 1; 2075 } 2076 2077 for (event_type = RTE_ETH_EVENT_UNKNOWN; 2078 event_type < RTE_ETH_EVENT_MAX; 2079 event_type++) { 2080 diag = rte_eth_dev_callback_register(RTE_ETH_ALL, 2081 event_type, 2082 eth_event_callback, 2083 NULL); 2084 if (diag) { 2085 printf("Failed to setup even callback for event %d\n", 2086 event_type); 2087 return -1; 2088 } 2089 } 2090 2091 if (need_check_link_status == 1 && !no_link_check) 2092 check_all_ports_link_status(RTE_PORT_ALL); 2093 else if (need_check_link_status == 0) 2094 printf("Please stop the ports first\n"); 2095 2096 printf("Done\n"); 2097 return 0; 2098 } 2099 2100 void 2101 stop_port(portid_t pid) 2102 { 2103 portid_t pi; 2104 struct rte_port *port; 2105 int need_check_link_status = 0; 2106 2107 if (dcb_test) { 2108 dcb_test = 0; 2109 dcb_config = 0; 2110 } 2111 2112 if (port_id_is_invalid(pid, ENABLED_WARN)) 2113 return; 2114 2115 printf("Stopping ports...\n"); 2116 2117 RTE_ETH_FOREACH_DEV(pi) { 2118 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2119 continue; 2120 2121 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2122 printf("Please remove port %d from forwarding configuration.\n", pi); 2123 continue; 2124 } 2125 2126 if (port_is_bonding_slave(pi)) { 2127 printf("Please remove port %d from bonded device.\n", pi); 2128 continue; 2129 } 2130 2131 port = &ports[pi]; 2132 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2133 RTE_PORT_HANDLING) == 0) 2134 continue; 2135 2136 rte_eth_dev_stop(pi); 2137 2138 if (rte_atomic16_cmpset(&(port->port_status), 2139 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2140 printf("Port %d can not be set into stopped\n", pi); 2141 need_check_link_status = 1; 2142 } 2143 if (need_check_link_status && !no_link_check) 2144 check_all_ports_link_status(RTE_PORT_ALL); 2145 2146 printf("Done\n"); 2147 } 2148 2149 void 2150 close_port(portid_t pid) 2151 { 2152 portid_t pi; 2153 struct rte_port *port; 2154 2155 if (port_id_is_invalid(pid, ENABLED_WARN)) 2156 return; 2157 2158 printf("Closing ports...\n"); 2159 2160 RTE_ETH_FOREACH_DEV(pi) { 2161 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2162 continue; 2163 2164 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2165 printf("Please remove port %d from forwarding configuration.\n", pi); 2166 continue; 2167 } 2168 2169 if (port_is_bonding_slave(pi)) { 2170 printf("Please remove port %d from bonded device.\n", pi); 2171 continue; 2172 } 2173 2174 port = &ports[pi]; 2175 if (rte_atomic16_cmpset(&(port->port_status), 2176 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2177 printf("Port %d is already closed\n", pi); 2178 continue; 2179 } 2180 2181 if (rte_atomic16_cmpset(&(port->port_status), 2182 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2183 printf("Port %d is now not stopped\n", pi); 2184 continue; 2185 } 2186 2187 if (port->flow_list) 2188 port_flow_flush(pi); 2189 rte_eth_dev_close(pi); 2190 2191 if (rte_atomic16_cmpset(&(port->port_status), 2192 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2193 printf("Port %d cannot be set to closed\n", pi); 2194 } 2195 2196 printf("Done\n"); 2197 } 2198 2199 void 2200 reset_port(portid_t pid) 2201 { 2202 int diag; 2203 portid_t pi; 2204 struct rte_port *port; 2205 2206 if (port_id_is_invalid(pid, ENABLED_WARN)) 2207 return; 2208 2209 printf("Resetting ports...\n"); 2210 2211 RTE_ETH_FOREACH_DEV(pi) { 2212 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2213 continue; 2214 2215 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2216 printf("Please remove port %d from forwarding " 2217 "configuration.\n", pi); 2218 continue; 2219 } 2220 2221 if (port_is_bonding_slave(pi)) { 2222 printf("Please remove port %d from bonded device.\n", 2223 pi); 2224 continue; 2225 } 2226 2227 diag = rte_eth_dev_reset(pi); 2228 if (diag == 0) { 2229 port = &ports[pi]; 2230 port->need_reconfig = 1; 2231 port->need_reconfig_queues = 1; 2232 } else { 2233 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2234 } 2235 } 2236 2237 printf("Done\n"); 2238 } 2239 2240 static int 2241 eth_dev_event_callback_register(void) 2242 { 2243 int ret; 2244 2245 /* register the device event callback */ 2246 ret = rte_dev_event_callback_register(NULL, 2247 eth_dev_event_callback, NULL); 2248 if (ret) { 2249 printf("Failed to register device event callback\n"); 2250 return -1; 2251 } 2252 2253 return 0; 2254 } 2255 2256 2257 static int 2258 eth_dev_event_callback_unregister(void) 2259 { 2260 int ret; 2261 2262 /* unregister the device event callback */ 2263 ret = rte_dev_event_callback_unregister(NULL, 2264 eth_dev_event_callback, NULL); 2265 if (ret < 0) { 2266 printf("Failed to unregister device event callback\n"); 2267 return -1; 2268 } 2269 2270 return 0; 2271 } 2272 2273 void 2274 attach_port(char *identifier) 2275 { 2276 portid_t pi = 0; 2277 unsigned int socket_id; 2278 2279 printf("Attaching a new port...\n"); 2280 2281 if (identifier == NULL) { 2282 printf("Invalid parameters are specified\n"); 2283 return; 2284 } 2285 2286 if (rte_eth_dev_attach(identifier, &pi)) 2287 return; 2288 2289 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2290 /* if socket_id is invalid, set to 0 */ 2291 if (check_socket_id(socket_id) < 0) 2292 socket_id = 0; 2293 reconfig(pi, socket_id); 2294 rte_eth_promiscuous_enable(pi); 2295 2296 ports_ids[nb_ports] = pi; 2297 nb_ports = rte_eth_dev_count_avail(); 2298 2299 ports[pi].port_status = RTE_PORT_STOPPED; 2300 2301 update_fwd_ports(pi); 2302 2303 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2304 printf("Done\n"); 2305 } 2306 2307 void 2308 detach_port(portid_t port_id) 2309 { 2310 char name[RTE_ETH_NAME_MAX_LEN]; 2311 uint16_t i; 2312 2313 printf("Detaching a port...\n"); 2314 2315 if (!port_is_closed(port_id)) { 2316 printf("Please close port first\n"); 2317 return; 2318 } 2319 2320 if (ports[port_id].flow_list) 2321 port_flow_flush(port_id); 2322 2323 if (rte_eth_dev_detach(port_id, name)) { 2324 TESTPMD_LOG(ERR, "Failed to detach port %u\n", port_id); 2325 return; 2326 } 2327 2328 for (i = 0; i < nb_ports; i++) { 2329 if (ports_ids[i] == port_id) { 2330 ports_ids[i] = ports_ids[nb_ports-1]; 2331 ports_ids[nb_ports-1] = 0; 2332 break; 2333 } 2334 } 2335 nb_ports = rte_eth_dev_count_avail(); 2336 2337 update_fwd_ports(RTE_MAX_ETHPORTS); 2338 2339 printf("Port %u is detached. Now total ports is %d\n", 2340 port_id, nb_ports); 2341 printf("Done\n"); 2342 return; 2343 } 2344 2345 void 2346 pmd_test_exit(void) 2347 { 2348 struct rte_device *device; 2349 portid_t pt_id; 2350 int ret; 2351 2352 if (test_done == 0) 2353 stop_packet_forwarding(); 2354 2355 if (ports != NULL) { 2356 no_link_check = 1; 2357 RTE_ETH_FOREACH_DEV(pt_id) { 2358 printf("\nShutting down port %d...\n", pt_id); 2359 fflush(stdout); 2360 stop_port(pt_id); 2361 close_port(pt_id); 2362 2363 /* 2364 * This is a workaround to fix a virtio-user issue that 2365 * requires to call clean-up routine to remove existing 2366 * socket. 2367 * This workaround valid only for testpmd, needs a fix 2368 * valid for all applications. 2369 * TODO: Implement proper resource cleanup 2370 */ 2371 device = rte_eth_devices[pt_id].device; 2372 if (device && !strcmp(device->driver->name, "net_virtio_user")) 2373 detach_port(pt_id); 2374 } 2375 } 2376 2377 if (hot_plug) { 2378 ret = rte_dev_event_monitor_stop(); 2379 if (ret) 2380 RTE_LOG(ERR, EAL, 2381 "fail to stop device event monitor."); 2382 2383 ret = eth_dev_event_callback_unregister(); 2384 if (ret) 2385 RTE_LOG(ERR, EAL, 2386 "fail to unregister all event callbacks."); 2387 } 2388 2389 printf("\nBye...\n"); 2390 } 2391 2392 typedef void (*cmd_func_t)(void); 2393 struct pmd_test_command { 2394 const char *cmd_name; 2395 cmd_func_t cmd_func; 2396 }; 2397 2398 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2399 2400 /* Check the link status of all ports in up to 9s, and print them finally */ 2401 static void 2402 check_all_ports_link_status(uint32_t port_mask) 2403 { 2404 #define CHECK_INTERVAL 100 /* 100ms */ 2405 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2406 portid_t portid; 2407 uint8_t count, all_ports_up, print_flag = 0; 2408 struct rte_eth_link link; 2409 2410 printf("Checking link statuses...\n"); 2411 fflush(stdout); 2412 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2413 all_ports_up = 1; 2414 RTE_ETH_FOREACH_DEV(portid) { 2415 if ((port_mask & (1 << portid)) == 0) 2416 continue; 2417 memset(&link, 0, sizeof(link)); 2418 rte_eth_link_get_nowait(portid, &link); 2419 /* print link status if flag set */ 2420 if (print_flag == 1) { 2421 if (link.link_status) 2422 printf( 2423 "Port%d Link Up. speed %u Mbps- %s\n", 2424 portid, link.link_speed, 2425 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2426 ("full-duplex") : ("half-duplex\n")); 2427 else 2428 printf("Port %d Link Down\n", portid); 2429 continue; 2430 } 2431 /* clear all_ports_up flag if any link down */ 2432 if (link.link_status == ETH_LINK_DOWN) { 2433 all_ports_up = 0; 2434 break; 2435 } 2436 } 2437 /* after finally printing all link status, get out */ 2438 if (print_flag == 1) 2439 break; 2440 2441 if (all_ports_up == 0) { 2442 fflush(stdout); 2443 rte_delay_ms(CHECK_INTERVAL); 2444 } 2445 2446 /* set the print_flag if all ports up or timeout */ 2447 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2448 print_flag = 1; 2449 } 2450 2451 if (lsc_interrupt) 2452 break; 2453 } 2454 } 2455 2456 static void 2457 rmv_event_callback(void *arg) 2458 { 2459 int need_to_start = 0; 2460 int org_no_link_check = no_link_check; 2461 portid_t port_id = (intptr_t)arg; 2462 2463 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2464 2465 if (!test_done && port_is_forwarding(port_id)) { 2466 need_to_start = 1; 2467 stop_packet_forwarding(); 2468 } 2469 no_link_check = 1; 2470 stop_port(port_id); 2471 no_link_check = org_no_link_check; 2472 close_port(port_id); 2473 detach_port(port_id); 2474 if (need_to_start) 2475 start_packet_forwarding(0); 2476 } 2477 2478 /* This function is used by the interrupt thread */ 2479 static int 2480 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2481 void *ret_param) 2482 { 2483 static const char * const event_desc[] = { 2484 [RTE_ETH_EVENT_UNKNOWN] = "Unknown", 2485 [RTE_ETH_EVENT_INTR_LSC] = "LSC", 2486 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state", 2487 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset", 2488 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox", 2489 [RTE_ETH_EVENT_IPSEC] = "IPsec", 2490 [RTE_ETH_EVENT_MACSEC] = "MACsec", 2491 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 2492 [RTE_ETH_EVENT_NEW] = "device probed", 2493 [RTE_ETH_EVENT_DESTROY] = "device released", 2494 [RTE_ETH_EVENT_MAX] = NULL, 2495 }; 2496 2497 RTE_SET_USED(param); 2498 RTE_SET_USED(ret_param); 2499 2500 if (type >= RTE_ETH_EVENT_MAX) { 2501 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n", 2502 port_id, __func__, type); 2503 fflush(stderr); 2504 } else if (event_print_mask & (UINT32_C(1) << type)) { 2505 printf("\nPort %" PRIu8 ": %s event\n", port_id, 2506 event_desc[type]); 2507 fflush(stdout); 2508 } 2509 2510 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2511 return 0; 2512 2513 switch (type) { 2514 case RTE_ETH_EVENT_INTR_RMV: 2515 if (rte_eal_alarm_set(100000, 2516 rmv_event_callback, (void *)(intptr_t)port_id)) 2517 fprintf(stderr, "Could not set up deferred device removal\n"); 2518 break; 2519 default: 2520 break; 2521 } 2522 return 0; 2523 } 2524 2525 /* This function is used by the interrupt thread */ 2526 static void 2527 eth_dev_event_callback(char *device_name, enum rte_dev_event_type type, 2528 __rte_unused void *arg) 2529 { 2530 if (type >= RTE_DEV_EVENT_MAX) { 2531 fprintf(stderr, "%s called upon invalid event %d\n", 2532 __func__, type); 2533 fflush(stderr); 2534 } 2535 2536 switch (type) { 2537 case RTE_DEV_EVENT_REMOVE: 2538 RTE_LOG(ERR, EAL, "The device: %s has been removed!\n", 2539 device_name); 2540 /* TODO: After finish failure handle, begin to stop 2541 * packet forward, stop port, close port, detach port. 2542 */ 2543 break; 2544 case RTE_DEV_EVENT_ADD: 2545 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2546 device_name); 2547 /* TODO: After finish kernel driver binding, 2548 * begin to attach port. 2549 */ 2550 break; 2551 default: 2552 break; 2553 } 2554 } 2555 2556 static int 2557 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2558 { 2559 uint16_t i; 2560 int diag; 2561 uint8_t mapping_found = 0; 2562 2563 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2564 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2565 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2566 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2567 tx_queue_stats_mappings[i].queue_id, 2568 tx_queue_stats_mappings[i].stats_counter_id); 2569 if (diag != 0) 2570 return diag; 2571 mapping_found = 1; 2572 } 2573 } 2574 if (mapping_found) 2575 port->tx_queue_stats_mapping_enabled = 1; 2576 return 0; 2577 } 2578 2579 static int 2580 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2581 { 2582 uint16_t i; 2583 int diag; 2584 uint8_t mapping_found = 0; 2585 2586 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2587 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2588 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2589 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2590 rx_queue_stats_mappings[i].queue_id, 2591 rx_queue_stats_mappings[i].stats_counter_id); 2592 if (diag != 0) 2593 return diag; 2594 mapping_found = 1; 2595 } 2596 } 2597 if (mapping_found) 2598 port->rx_queue_stats_mapping_enabled = 1; 2599 return 0; 2600 } 2601 2602 static void 2603 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2604 { 2605 int diag = 0; 2606 2607 diag = set_tx_queue_stats_mapping_registers(pi, port); 2608 if (diag != 0) { 2609 if (diag == -ENOTSUP) { 2610 port->tx_queue_stats_mapping_enabled = 0; 2611 printf("TX queue stats mapping not supported port id=%d\n", pi); 2612 } 2613 else 2614 rte_exit(EXIT_FAILURE, 2615 "set_tx_queue_stats_mapping_registers " 2616 "failed for port id=%d diag=%d\n", 2617 pi, diag); 2618 } 2619 2620 diag = set_rx_queue_stats_mapping_registers(pi, port); 2621 if (diag != 0) { 2622 if (diag == -ENOTSUP) { 2623 port->rx_queue_stats_mapping_enabled = 0; 2624 printf("RX queue stats mapping not supported port id=%d\n", pi); 2625 } 2626 else 2627 rte_exit(EXIT_FAILURE, 2628 "set_rx_queue_stats_mapping_registers " 2629 "failed for port id=%d diag=%d\n", 2630 pi, diag); 2631 } 2632 } 2633 2634 static void 2635 rxtx_port_config(struct rte_port *port) 2636 { 2637 uint16_t qid; 2638 2639 for (qid = 0; qid < nb_rxq; qid++) { 2640 port->rx_conf[qid] = port->dev_info.default_rxconf; 2641 2642 /* Check if any Rx parameters have been passed */ 2643 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2644 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2645 2646 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2647 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2648 2649 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2650 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2651 2652 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2653 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2654 2655 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2656 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2657 2658 port->nb_rx_desc[qid] = nb_rxd; 2659 } 2660 2661 for (qid = 0; qid < nb_txq; qid++) { 2662 port->tx_conf[qid] = port->dev_info.default_txconf; 2663 2664 /* Check if any Tx parameters have been passed */ 2665 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2666 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2667 2668 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2669 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2670 2671 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2672 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2673 2674 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2675 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2676 2677 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2678 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2679 2680 port->nb_tx_desc[qid] = nb_txd; 2681 } 2682 } 2683 2684 void 2685 init_port_config(void) 2686 { 2687 portid_t pid; 2688 struct rte_port *port; 2689 2690 RTE_ETH_FOREACH_DEV(pid) { 2691 port = &ports[pid]; 2692 port->dev_conf.fdir_conf = fdir_conf; 2693 rte_eth_dev_info_get(pid, &port->dev_info); 2694 if (nb_rxq > 1) { 2695 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2696 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2697 rss_hf & port->dev_info.flow_type_rss_offloads; 2698 } else { 2699 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2700 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2701 } 2702 2703 if (port->dcb_flag == 0) { 2704 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2705 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2706 else 2707 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2708 } 2709 2710 rxtx_port_config(port); 2711 2712 rte_eth_macaddr_get(pid, &port->eth_addr); 2713 2714 map_port_queue_stats_mapping_registers(pid, port); 2715 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2716 rte_pmd_ixgbe_bypass_init(pid); 2717 #endif 2718 2719 if (lsc_interrupt && 2720 (rte_eth_devices[pid].data->dev_flags & 2721 RTE_ETH_DEV_INTR_LSC)) 2722 port->dev_conf.intr_conf.lsc = 1; 2723 if (rmv_interrupt && 2724 (rte_eth_devices[pid].data->dev_flags & 2725 RTE_ETH_DEV_INTR_RMV)) 2726 port->dev_conf.intr_conf.rmv = 1; 2727 } 2728 } 2729 2730 void set_port_slave_flag(portid_t slave_pid) 2731 { 2732 struct rte_port *port; 2733 2734 port = &ports[slave_pid]; 2735 port->slave_flag = 1; 2736 } 2737 2738 void clear_port_slave_flag(portid_t slave_pid) 2739 { 2740 struct rte_port *port; 2741 2742 port = &ports[slave_pid]; 2743 port->slave_flag = 0; 2744 } 2745 2746 uint8_t port_is_bonding_slave(portid_t slave_pid) 2747 { 2748 struct rte_port *port; 2749 2750 port = &ports[slave_pid]; 2751 if ((rte_eth_devices[slave_pid].data->dev_flags & 2752 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2753 return 1; 2754 return 0; 2755 } 2756 2757 const uint16_t vlan_tags[] = { 2758 0, 1, 2, 3, 4, 5, 6, 7, 2759 8, 9, 10, 11, 12, 13, 14, 15, 2760 16, 17, 18, 19, 20, 21, 22, 23, 2761 24, 25, 26, 27, 28, 29, 30, 31 2762 }; 2763 2764 static int 2765 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 2766 enum dcb_mode_enable dcb_mode, 2767 enum rte_eth_nb_tcs num_tcs, 2768 uint8_t pfc_en) 2769 { 2770 uint8_t i; 2771 int32_t rc; 2772 struct rte_eth_rss_conf rss_conf; 2773 2774 /* 2775 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2776 * given above, and the number of traffic classes available for use. 2777 */ 2778 if (dcb_mode == DCB_VT_ENABLED) { 2779 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2780 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2781 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2782 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2783 2784 /* VMDQ+DCB RX and TX configurations */ 2785 vmdq_rx_conf->enable_default_pool = 0; 2786 vmdq_rx_conf->default_pool = 0; 2787 vmdq_rx_conf->nb_queue_pools = 2788 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2789 vmdq_tx_conf->nb_queue_pools = 2790 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2791 2792 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2793 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2794 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2795 vmdq_rx_conf->pool_map[i].pools = 2796 1 << (i % vmdq_rx_conf->nb_queue_pools); 2797 } 2798 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2799 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 2800 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 2801 } 2802 2803 /* set DCB mode of RX and TX of multiple queues */ 2804 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2805 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2806 } else { 2807 struct rte_eth_dcb_rx_conf *rx_conf = 2808 ð_conf->rx_adv_conf.dcb_rx_conf; 2809 struct rte_eth_dcb_tx_conf *tx_conf = 2810 ð_conf->tx_adv_conf.dcb_tx_conf; 2811 2812 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 2813 if (rc != 0) 2814 return rc; 2815 2816 rx_conf->nb_tcs = num_tcs; 2817 tx_conf->nb_tcs = num_tcs; 2818 2819 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2820 rx_conf->dcb_tc[i] = i % num_tcs; 2821 tx_conf->dcb_tc[i] = i % num_tcs; 2822 } 2823 2824 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2825 eth_conf->rx_adv_conf.rss_conf = rss_conf; 2826 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2827 } 2828 2829 if (pfc_en) 2830 eth_conf->dcb_capability_en = 2831 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2832 else 2833 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2834 2835 return 0; 2836 } 2837 2838 int 2839 init_port_dcb_config(portid_t pid, 2840 enum dcb_mode_enable dcb_mode, 2841 enum rte_eth_nb_tcs num_tcs, 2842 uint8_t pfc_en) 2843 { 2844 struct rte_eth_conf port_conf; 2845 struct rte_port *rte_port; 2846 int retval; 2847 uint16_t i; 2848 2849 rte_port = &ports[pid]; 2850 2851 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2852 /* Enter DCB configuration status */ 2853 dcb_config = 1; 2854 2855 port_conf.rxmode = rte_port->dev_conf.rxmode; 2856 port_conf.txmode = rte_port->dev_conf.txmode; 2857 2858 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2859 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 2860 if (retval < 0) 2861 return retval; 2862 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2863 2864 /* re-configure the device . */ 2865 rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 2866 2867 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2868 2869 /* If dev_info.vmdq_pool_base is greater than 0, 2870 * the queue id of vmdq pools is started after pf queues. 2871 */ 2872 if (dcb_mode == DCB_VT_ENABLED && 2873 rte_port->dev_info.vmdq_pool_base > 0) { 2874 printf("VMDQ_DCB multi-queue mode is nonsensical" 2875 " for port %d.", pid); 2876 return -1; 2877 } 2878 2879 /* Assume the ports in testpmd have the same dcb capability 2880 * and has the same number of rxq and txq in dcb mode 2881 */ 2882 if (dcb_mode == DCB_VT_ENABLED) { 2883 if (rte_port->dev_info.max_vfs > 0) { 2884 nb_rxq = rte_port->dev_info.nb_rx_queues; 2885 nb_txq = rte_port->dev_info.nb_tx_queues; 2886 } else { 2887 nb_rxq = rte_port->dev_info.max_rx_queues; 2888 nb_txq = rte_port->dev_info.max_tx_queues; 2889 } 2890 } else { 2891 /*if vt is disabled, use all pf queues */ 2892 if (rte_port->dev_info.vmdq_pool_base == 0) { 2893 nb_rxq = rte_port->dev_info.max_rx_queues; 2894 nb_txq = rte_port->dev_info.max_tx_queues; 2895 } else { 2896 nb_rxq = (queueid_t)num_tcs; 2897 nb_txq = (queueid_t)num_tcs; 2898 2899 } 2900 } 2901 rx_free_thresh = 64; 2902 2903 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 2904 2905 rxtx_port_config(rte_port); 2906 /* VLAN filter */ 2907 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2908 for (i = 0; i < RTE_DIM(vlan_tags); i++) 2909 rx_vft_set(pid, vlan_tags[i], 1); 2910 2911 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 2912 map_port_queue_stats_mapping_registers(pid, rte_port); 2913 2914 rte_port->dcb_flag = 1; 2915 2916 return 0; 2917 } 2918 2919 static void 2920 init_port(void) 2921 { 2922 /* Configuration of Ethernet ports. */ 2923 ports = rte_zmalloc("testpmd: ports", 2924 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 2925 RTE_CACHE_LINE_SIZE); 2926 if (ports == NULL) { 2927 rte_exit(EXIT_FAILURE, 2928 "rte_zmalloc(%d struct rte_port) failed\n", 2929 RTE_MAX_ETHPORTS); 2930 } 2931 } 2932 2933 static void 2934 force_quit(void) 2935 { 2936 pmd_test_exit(); 2937 prompt_exit(); 2938 } 2939 2940 static void 2941 print_stats(void) 2942 { 2943 uint8_t i; 2944 const char clr[] = { 27, '[', '2', 'J', '\0' }; 2945 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 2946 2947 /* Clear screen and move to top left */ 2948 printf("%s%s", clr, top_left); 2949 2950 printf("\nPort statistics ===================================="); 2951 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2952 nic_stats_display(fwd_ports_ids[i]); 2953 } 2954 2955 static void 2956 signal_handler(int signum) 2957 { 2958 if (signum == SIGINT || signum == SIGTERM) { 2959 printf("\nSignal %d received, preparing to exit...\n", 2960 signum); 2961 #ifdef RTE_LIBRTE_PDUMP 2962 /* uninitialize packet capture framework */ 2963 rte_pdump_uninit(); 2964 #endif 2965 #ifdef RTE_LIBRTE_LATENCY_STATS 2966 rte_latencystats_uninit(); 2967 #endif 2968 force_quit(); 2969 /* Set flag to indicate the force termination. */ 2970 f_quit = 1; 2971 /* exit with the expected status */ 2972 signal(signum, SIG_DFL); 2973 kill(getpid(), signum); 2974 } 2975 } 2976 2977 int 2978 main(int argc, char** argv) 2979 { 2980 int diag; 2981 portid_t port_id; 2982 uint16_t count; 2983 int ret; 2984 2985 signal(SIGINT, signal_handler); 2986 signal(SIGTERM, signal_handler); 2987 2988 diag = rte_eal_init(argc, argv); 2989 if (diag < 0) 2990 rte_panic("Cannot init EAL\n"); 2991 2992 testpmd_logtype = rte_log_register("testpmd"); 2993 if (testpmd_logtype < 0) 2994 rte_panic("Cannot register log type"); 2995 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 2996 2997 #ifdef RTE_LIBRTE_PDUMP 2998 /* initialize packet capture framework */ 2999 rte_pdump_init(NULL); 3000 #endif 3001 3002 count = 0; 3003 RTE_ETH_FOREACH_DEV(port_id) { 3004 ports_ids[count] = port_id; 3005 count++; 3006 } 3007 nb_ports = (portid_t) count; 3008 if (nb_ports == 0) 3009 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3010 3011 /* allocate port structures, and init them */ 3012 init_port(); 3013 3014 set_def_fwd_config(); 3015 if (nb_lcores == 0) 3016 rte_panic("Empty set of forwarding logical cores - check the " 3017 "core mask supplied in the command parameters\n"); 3018 3019 /* Bitrate/latency stats disabled by default */ 3020 #ifdef RTE_LIBRTE_BITRATE 3021 bitrate_enabled = 0; 3022 #endif 3023 #ifdef RTE_LIBRTE_LATENCY_STATS 3024 latencystats_enabled = 0; 3025 #endif 3026 3027 /* on FreeBSD, mlockall() is disabled by default */ 3028 #ifdef RTE_EXEC_ENV_BSDAPP 3029 do_mlockall = 0; 3030 #else 3031 do_mlockall = 1; 3032 #endif 3033 3034 argc -= diag; 3035 argv += diag; 3036 if (argc > 1) 3037 launch_args_parse(argc, argv); 3038 3039 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3040 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3041 strerror(errno)); 3042 } 3043 3044 if (tx_first && interactive) 3045 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3046 "interactive mode.\n"); 3047 3048 if (tx_first && lsc_interrupt) { 3049 printf("Warning: lsc_interrupt needs to be off when " 3050 " using tx_first. Disabling.\n"); 3051 lsc_interrupt = 0; 3052 } 3053 3054 if (!nb_rxq && !nb_txq) 3055 printf("Warning: Either rx or tx queues should be non-zero\n"); 3056 3057 if (nb_rxq > 1 && nb_rxq > nb_txq) 3058 printf("Warning: nb_rxq=%d enables RSS configuration, " 3059 "but nb_txq=%d will prevent to fully test it.\n", 3060 nb_rxq, nb_txq); 3061 3062 init_config(); 3063 3064 if (hot_plug) { 3065 /* enable hot plug monitoring */ 3066 ret = rte_dev_event_monitor_start(); 3067 if (ret) { 3068 rte_errno = EINVAL; 3069 return -1; 3070 } 3071 eth_dev_event_callback_register(); 3072 3073 } 3074 3075 if (start_port(RTE_PORT_ALL) != 0) 3076 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3077 3078 /* set all ports to promiscuous mode by default */ 3079 RTE_ETH_FOREACH_DEV(port_id) 3080 rte_eth_promiscuous_enable(port_id); 3081 3082 /* Init metrics library */ 3083 rte_metrics_init(rte_socket_id()); 3084 3085 #ifdef RTE_LIBRTE_LATENCY_STATS 3086 if (latencystats_enabled != 0) { 3087 int ret = rte_latencystats_init(1, NULL); 3088 if (ret) 3089 printf("Warning: latencystats init()" 3090 " returned error %d\n", ret); 3091 printf("Latencystats running on lcore %d\n", 3092 latencystats_lcore_id); 3093 } 3094 #endif 3095 3096 /* Setup bitrate stats */ 3097 #ifdef RTE_LIBRTE_BITRATE 3098 if (bitrate_enabled != 0) { 3099 bitrate_data = rte_stats_bitrate_create(); 3100 if (bitrate_data == NULL) 3101 rte_exit(EXIT_FAILURE, 3102 "Could not allocate bitrate data.\n"); 3103 rte_stats_bitrate_reg(bitrate_data); 3104 } 3105 #endif 3106 3107 #ifdef RTE_LIBRTE_CMDLINE 3108 if (strlen(cmdline_filename) != 0) 3109 cmdline_read_from_file(cmdline_filename); 3110 3111 if (interactive == 1) { 3112 if (auto_start) { 3113 printf("Start automatic packet forwarding\n"); 3114 start_packet_forwarding(0); 3115 } 3116 prompt(); 3117 pmd_test_exit(); 3118 } else 3119 #endif 3120 { 3121 char c; 3122 int rc; 3123 3124 f_quit = 0; 3125 3126 printf("No commandline core given, start packet forwarding\n"); 3127 start_packet_forwarding(tx_first); 3128 if (stats_period != 0) { 3129 uint64_t prev_time = 0, cur_time, diff_time = 0; 3130 uint64_t timer_period; 3131 3132 /* Convert to number of cycles */ 3133 timer_period = stats_period * rte_get_timer_hz(); 3134 3135 while (f_quit == 0) { 3136 cur_time = rte_get_timer_cycles(); 3137 diff_time += cur_time - prev_time; 3138 3139 if (diff_time >= timer_period) { 3140 print_stats(); 3141 /* Reset the timer */ 3142 diff_time = 0; 3143 } 3144 /* Sleep to avoid unnecessary checks */ 3145 prev_time = cur_time; 3146 sleep(1); 3147 } 3148 } 3149 3150 printf("Press enter to exit\n"); 3151 rc = read(0, &c, 1); 3152 pmd_test_exit(); 3153 if (rc < 0) 3154 return 1; 3155 } 3156 3157 return 0; 3158 } 3159