1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 199 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 200 * specified on command-line. */ 201 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 202 203 /* 204 * In container, it cannot terminate the process which running with 'stats-period' 205 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 206 */ 207 uint8_t f_quit; 208 209 /* 210 * Configuration of packet segments used by the "txonly" processing engine. 211 */ 212 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 213 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 214 TXONLY_DEF_PACKET_LEN, 215 }; 216 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 217 218 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 219 /**< Split policy for packets to TX. */ 220 221 uint8_t txonly_multi_flow; 222 /**< Whether multiple flows are generated in TXONLY mode. */ 223 224 uint32_t tx_pkt_times_inter; 225 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 226 227 uint32_t tx_pkt_times_intra; 228 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 229 230 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 231 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 232 233 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 234 uint8_t dcb_config = 0; 235 236 /* Whether the dcb is in testing status */ 237 uint8_t dcb_test = 0; 238 239 /* 240 * Configurable number of RX/TX queues. 241 */ 242 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 243 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 244 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 245 246 /* 247 * Configurable number of RX/TX ring descriptors. 248 * Defaults are supplied by drivers via ethdev. 249 */ 250 #define RTE_TEST_RX_DESC_DEFAULT 0 251 #define RTE_TEST_TX_DESC_DEFAULT 0 252 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 253 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 254 255 #define RTE_PMD_PARAM_UNSET -1 256 /* 257 * Configurable values of RX and TX ring threshold registers. 258 */ 259 260 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 261 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 262 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 263 264 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 265 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 266 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX free threshold. 270 */ 271 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of RX drop enable. 275 */ 276 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX free threshold. 280 */ 281 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of TX RS bit threshold. 285 */ 286 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 287 288 /* 289 * Configurable value of buffered packets before sending. 290 */ 291 uint16_t noisy_tx_sw_bufsz; 292 293 /* 294 * Configurable value of packet buffer timeout. 295 */ 296 uint16_t noisy_tx_sw_buf_flush_time; 297 298 /* 299 * Configurable value for size of VNF internal memory area 300 * used for simulating noisy neighbour behaviour 301 */ 302 uint64_t noisy_lkup_mem_sz; 303 304 /* 305 * Configurable value of number of random writes done in 306 * VNF simulation memory area. 307 */ 308 uint64_t noisy_lkup_num_writes; 309 310 /* 311 * Configurable value of number of random reads done in 312 * VNF simulation memory area. 313 */ 314 uint64_t noisy_lkup_num_reads; 315 316 /* 317 * Configurable value of number of random reads/writes done in 318 * VNF simulation memory area. 319 */ 320 uint64_t noisy_lkup_num_reads_writes; 321 322 /* 323 * Receive Side Scaling (RSS) configuration. 324 */ 325 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 326 327 /* 328 * Port topology configuration 329 */ 330 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 331 332 /* 333 * Avoids to flush all the RX streams before starts forwarding. 334 */ 335 uint8_t no_flush_rx = 0; /* flush by default */ 336 337 /* 338 * Flow API isolated mode. 339 */ 340 uint8_t flow_isolate_all; 341 342 /* 343 * Avoids to check link status when starting/stopping a port. 344 */ 345 uint8_t no_link_check = 0; /* check by default */ 346 347 /* 348 * Don't automatically start all ports in interactive mode. 349 */ 350 uint8_t no_device_start = 0; 351 352 /* 353 * Enable link status change notification 354 */ 355 uint8_t lsc_interrupt = 1; /* enabled by default */ 356 357 /* 358 * Enable device removal notification. 359 */ 360 uint8_t rmv_interrupt = 1; /* enabled by default */ 361 362 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 363 364 /* After attach, port setup is called on event or by iterator */ 365 bool setup_on_probe_event = true; 366 367 /* Clear ptypes on port initialization. */ 368 uint8_t clear_ptypes = true; 369 370 /* Pretty printing of ethdev events */ 371 static const char * const eth_event_desc[] = { 372 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 373 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 374 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 375 [RTE_ETH_EVENT_INTR_RESET] = "reset", 376 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 377 [RTE_ETH_EVENT_IPSEC] = "IPsec", 378 [RTE_ETH_EVENT_MACSEC] = "MACsec", 379 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 380 [RTE_ETH_EVENT_NEW] = "device probed", 381 [RTE_ETH_EVENT_DESTROY] = "device released", 382 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 383 [RTE_ETH_EVENT_MAX] = NULL, 384 }; 385 386 /* 387 * Display or mask ether events 388 * Default to all events except VF_MBOX 389 */ 390 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 391 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 392 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 393 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 394 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 395 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 396 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 397 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 398 /* 399 * Decide if all memory are locked for performance. 400 */ 401 int do_mlockall = 0; 402 403 /* 404 * NIC bypass mode configuration options. 405 */ 406 407 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 408 /* The NIC bypass watchdog timeout. */ 409 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 410 #endif 411 412 413 #ifdef RTE_LIBRTE_LATENCY_STATS 414 415 /* 416 * Set when latency stats is enabled in the commandline 417 */ 418 uint8_t latencystats_enabled; 419 420 /* 421 * Lcore ID to serive latency statistics. 422 */ 423 lcoreid_t latencystats_lcore_id = -1; 424 425 #endif 426 427 /* 428 * Ethernet device configuration. 429 */ 430 struct rte_eth_rxmode rx_mode = { 431 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 432 /**< Default maximum frame length. */ 433 }; 434 435 struct rte_eth_txmode tx_mode = { 436 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 437 }; 438 439 struct rte_fdir_conf fdir_conf = { 440 .mode = RTE_FDIR_MODE_NONE, 441 .pballoc = RTE_FDIR_PBALLOC_64K, 442 .status = RTE_FDIR_REPORT_STATUS, 443 .mask = { 444 .vlan_tci_mask = 0xFFEF, 445 .ipv4_mask = { 446 .src_ip = 0xFFFFFFFF, 447 .dst_ip = 0xFFFFFFFF, 448 }, 449 .ipv6_mask = { 450 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 451 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 452 }, 453 .src_port_mask = 0xFFFF, 454 .dst_port_mask = 0xFFFF, 455 .mac_addr_byte_mask = 0xFF, 456 .tunnel_type_mask = 1, 457 .tunnel_id_mask = 0xFFFFFFFF, 458 }, 459 .drop_queue = 127, 460 }; 461 462 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 463 464 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 465 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 466 467 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 468 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 469 470 uint16_t nb_tx_queue_stats_mappings = 0; 471 uint16_t nb_rx_queue_stats_mappings = 0; 472 473 /* 474 * Display zero values by default for xstats 475 */ 476 uint8_t xstats_hide_zero; 477 478 /* 479 * Measure of CPU cycles disabled by default 480 */ 481 uint8_t record_core_cycles; 482 483 /* 484 * Display of RX and TX bursts disabled by default 485 */ 486 uint8_t record_burst_stats; 487 488 unsigned int num_sockets = 0; 489 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 490 491 #ifdef RTE_LIBRTE_BITRATESTATS 492 /* Bitrate statistics */ 493 struct rte_stats_bitrates *bitrate_data; 494 lcoreid_t bitrate_lcore_id; 495 uint8_t bitrate_enabled; 496 #endif 497 498 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 499 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 500 501 /* 502 * hexadecimal bitmask of RX mq mode can be enabled. 503 */ 504 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 505 506 /* Forward function declarations */ 507 static void setup_attached_port(portid_t pi); 508 static void map_port_queue_stats_mapping_registers(portid_t pi, 509 struct rte_port *port); 510 static void check_all_ports_link_status(uint32_t port_mask); 511 static int eth_event_callback(portid_t port_id, 512 enum rte_eth_event_type type, 513 void *param, void *ret_param); 514 static void dev_event_callback(const char *device_name, 515 enum rte_dev_event_type type, 516 void *param); 517 518 /* 519 * Check if all the ports are started. 520 * If yes, return positive value. If not, return zero. 521 */ 522 static int all_ports_started(void); 523 524 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 525 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 526 527 /* Holds the registered mbuf dynamic flags names. */ 528 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 529 530 /* 531 * Helper function to check if socket is already discovered. 532 * If yes, return positive value. If not, return zero. 533 */ 534 int 535 new_socket_id(unsigned int socket_id) 536 { 537 unsigned int i; 538 539 for (i = 0; i < num_sockets; i++) { 540 if (socket_ids[i] == socket_id) 541 return 0; 542 } 543 return 1; 544 } 545 546 /* 547 * Setup default configuration. 548 */ 549 static void 550 set_default_fwd_lcores_config(void) 551 { 552 unsigned int i; 553 unsigned int nb_lc; 554 unsigned int sock_num; 555 556 nb_lc = 0; 557 for (i = 0; i < RTE_MAX_LCORE; i++) { 558 if (!rte_lcore_is_enabled(i)) 559 continue; 560 sock_num = rte_lcore_to_socket_id(i); 561 if (new_socket_id(sock_num)) { 562 if (num_sockets >= RTE_MAX_NUMA_NODES) { 563 rte_exit(EXIT_FAILURE, 564 "Total sockets greater than %u\n", 565 RTE_MAX_NUMA_NODES); 566 } 567 socket_ids[num_sockets++] = sock_num; 568 } 569 if (i == rte_get_master_lcore()) 570 continue; 571 fwd_lcores_cpuids[nb_lc++] = i; 572 } 573 nb_lcores = (lcoreid_t) nb_lc; 574 nb_cfg_lcores = nb_lcores; 575 nb_fwd_lcores = 1; 576 } 577 578 static void 579 set_def_peer_eth_addrs(void) 580 { 581 portid_t i; 582 583 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 584 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 585 peer_eth_addrs[i].addr_bytes[5] = i; 586 } 587 } 588 589 static void 590 set_default_fwd_ports_config(void) 591 { 592 portid_t pt_id; 593 int i = 0; 594 595 RTE_ETH_FOREACH_DEV(pt_id) { 596 fwd_ports_ids[i++] = pt_id; 597 598 /* Update sockets info according to the attached device */ 599 int socket_id = rte_eth_dev_socket_id(pt_id); 600 if (socket_id >= 0 && new_socket_id(socket_id)) { 601 if (num_sockets >= RTE_MAX_NUMA_NODES) { 602 rte_exit(EXIT_FAILURE, 603 "Total sockets greater than %u\n", 604 RTE_MAX_NUMA_NODES); 605 } 606 socket_ids[num_sockets++] = socket_id; 607 } 608 } 609 610 nb_cfg_ports = nb_ports; 611 nb_fwd_ports = nb_ports; 612 } 613 614 void 615 set_def_fwd_config(void) 616 { 617 set_default_fwd_lcores_config(); 618 set_def_peer_eth_addrs(); 619 set_default_fwd_ports_config(); 620 } 621 622 /* extremely pessimistic estimation of memory required to create a mempool */ 623 static int 624 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 625 { 626 unsigned int n_pages, mbuf_per_pg, leftover; 627 uint64_t total_mem, mbuf_mem, obj_sz; 628 629 /* there is no good way to predict how much space the mempool will 630 * occupy because it will allocate chunks on the fly, and some of those 631 * will come from default DPDK memory while some will come from our 632 * external memory, so just assume 128MB will be enough for everyone. 633 */ 634 uint64_t hdr_mem = 128 << 20; 635 636 /* account for possible non-contiguousness */ 637 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 638 if (obj_sz > pgsz) { 639 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 640 return -1; 641 } 642 643 mbuf_per_pg = pgsz / obj_sz; 644 leftover = (nb_mbufs % mbuf_per_pg) > 0; 645 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 646 647 mbuf_mem = n_pages * pgsz; 648 649 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 650 651 if (total_mem > SIZE_MAX) { 652 TESTPMD_LOG(ERR, "Memory size too big\n"); 653 return -1; 654 } 655 *out = (size_t)total_mem; 656 657 return 0; 658 } 659 660 static int 661 pagesz_flags(uint64_t page_sz) 662 { 663 /* as per mmap() manpage, all page sizes are log2 of page size 664 * shifted by MAP_HUGE_SHIFT 665 */ 666 int log2 = rte_log2_u64(page_sz); 667 668 return (log2 << HUGE_SHIFT); 669 } 670 671 static void * 672 alloc_mem(size_t memsz, size_t pgsz, bool huge) 673 { 674 void *addr; 675 int flags; 676 677 /* allocate anonymous hugepages */ 678 flags = MAP_ANONYMOUS | MAP_PRIVATE; 679 if (huge) 680 flags |= HUGE_FLAG | pagesz_flags(pgsz); 681 682 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 683 if (addr == MAP_FAILED) 684 return NULL; 685 686 return addr; 687 } 688 689 struct extmem_param { 690 void *addr; 691 size_t len; 692 size_t pgsz; 693 rte_iova_t *iova_table; 694 unsigned int iova_table_len; 695 }; 696 697 static int 698 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 699 bool huge) 700 { 701 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 702 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 703 unsigned int cur_page, n_pages, pgsz_idx; 704 size_t mem_sz, cur_pgsz; 705 rte_iova_t *iovas = NULL; 706 void *addr; 707 int ret; 708 709 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 710 /* skip anything that is too big */ 711 if (pgsizes[pgsz_idx] > SIZE_MAX) 712 continue; 713 714 cur_pgsz = pgsizes[pgsz_idx]; 715 716 /* if we were told not to allocate hugepages, override */ 717 if (!huge) 718 cur_pgsz = sysconf(_SC_PAGESIZE); 719 720 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 721 if (ret < 0) { 722 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 723 return -1; 724 } 725 726 /* allocate our memory */ 727 addr = alloc_mem(mem_sz, cur_pgsz, huge); 728 729 /* if we couldn't allocate memory with a specified page size, 730 * that doesn't mean we can't do it with other page sizes, so 731 * try another one. 732 */ 733 if (addr == NULL) 734 continue; 735 736 /* store IOVA addresses for every page in this memory area */ 737 n_pages = mem_sz / cur_pgsz; 738 739 iovas = malloc(sizeof(*iovas) * n_pages); 740 741 if (iovas == NULL) { 742 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 743 goto fail; 744 } 745 /* lock memory if it's not huge pages */ 746 if (!huge) 747 mlock(addr, mem_sz); 748 749 /* populate IOVA addresses */ 750 for (cur_page = 0; cur_page < n_pages; cur_page++) { 751 rte_iova_t iova; 752 size_t offset; 753 void *cur; 754 755 offset = cur_pgsz * cur_page; 756 cur = RTE_PTR_ADD(addr, offset); 757 758 /* touch the page before getting its IOVA */ 759 *(volatile char *)cur = 0; 760 761 iova = rte_mem_virt2iova(cur); 762 763 iovas[cur_page] = iova; 764 } 765 766 break; 767 } 768 /* if we couldn't allocate anything */ 769 if (iovas == NULL) 770 return -1; 771 772 param->addr = addr; 773 param->len = mem_sz; 774 param->pgsz = cur_pgsz; 775 param->iova_table = iovas; 776 param->iova_table_len = n_pages; 777 778 return 0; 779 fail: 780 if (iovas) 781 free(iovas); 782 if (addr) 783 munmap(addr, mem_sz); 784 785 return -1; 786 } 787 788 static int 789 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 790 { 791 struct extmem_param param; 792 int socket_id, ret; 793 794 memset(¶m, 0, sizeof(param)); 795 796 /* check if our heap exists */ 797 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 798 if (socket_id < 0) { 799 /* create our heap */ 800 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 801 if (ret < 0) { 802 TESTPMD_LOG(ERR, "Cannot create heap\n"); 803 return -1; 804 } 805 } 806 807 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 808 if (ret < 0) { 809 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 810 return -1; 811 } 812 813 /* we now have a valid memory area, so add it to heap */ 814 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 815 param.addr, param.len, param.iova_table, 816 param.iova_table_len, param.pgsz); 817 818 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 819 820 /* not needed any more */ 821 free(param.iova_table); 822 823 if (ret < 0) { 824 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 825 munmap(param.addr, param.len); 826 return -1; 827 } 828 829 /* success */ 830 831 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 832 param.len >> 20); 833 834 return 0; 835 } 836 static void 837 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 838 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 839 { 840 uint16_t pid = 0; 841 int ret; 842 843 RTE_ETH_FOREACH_DEV(pid) { 844 struct rte_eth_dev *dev = 845 &rte_eth_devices[pid]; 846 847 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 848 memhdr->len); 849 if (ret) { 850 TESTPMD_LOG(DEBUG, 851 "unable to DMA unmap addr 0x%p " 852 "for device %s\n", 853 memhdr->addr, dev->data->name); 854 } 855 } 856 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 857 if (ret) { 858 TESTPMD_LOG(DEBUG, 859 "unable to un-register addr 0x%p\n", memhdr->addr); 860 } 861 } 862 863 static void 864 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 865 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 866 { 867 uint16_t pid = 0; 868 size_t page_size = sysconf(_SC_PAGESIZE); 869 int ret; 870 871 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 872 page_size); 873 if (ret) { 874 TESTPMD_LOG(DEBUG, 875 "unable to register addr 0x%p\n", memhdr->addr); 876 return; 877 } 878 RTE_ETH_FOREACH_DEV(pid) { 879 struct rte_eth_dev *dev = 880 &rte_eth_devices[pid]; 881 882 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 883 memhdr->len); 884 if (ret) { 885 TESTPMD_LOG(DEBUG, 886 "unable to DMA map addr 0x%p " 887 "for device %s\n", 888 memhdr->addr, dev->data->name); 889 } 890 } 891 } 892 893 static unsigned int 894 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 895 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 896 { 897 struct rte_pktmbuf_extmem *xmem; 898 unsigned int ext_num, zone_num, elt_num; 899 uint16_t elt_size; 900 901 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 902 elt_num = EXTBUF_ZONE_SIZE / elt_size; 903 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 904 905 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 906 if (xmem == NULL) { 907 TESTPMD_LOG(ERR, "Cannot allocate memory for " 908 "external buffer descriptors\n"); 909 *ext_mem = NULL; 910 return 0; 911 } 912 for (ext_num = 0; ext_num < zone_num; ext_num++) { 913 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 914 const struct rte_memzone *mz; 915 char mz_name[RTE_MEMZONE_NAMESIZE]; 916 int ret; 917 918 ret = snprintf(mz_name, sizeof(mz_name), 919 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 920 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 921 errno = ENAMETOOLONG; 922 ext_num = 0; 923 break; 924 } 925 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 926 socket_id, 927 RTE_MEMZONE_IOVA_CONTIG | 928 RTE_MEMZONE_1GB | 929 RTE_MEMZONE_SIZE_HINT_ONLY, 930 EXTBUF_ZONE_SIZE); 931 if (mz == NULL) { 932 /* 933 * The caller exits on external buffer creation 934 * error, so there is no need to free memzones. 935 */ 936 errno = ENOMEM; 937 ext_num = 0; 938 break; 939 } 940 xseg->buf_ptr = mz->addr; 941 xseg->buf_iova = mz->iova; 942 xseg->buf_len = EXTBUF_ZONE_SIZE; 943 xseg->elt_size = elt_size; 944 } 945 if (ext_num == 0 && xmem != NULL) { 946 free(xmem); 947 xmem = NULL; 948 } 949 *ext_mem = xmem; 950 return ext_num; 951 } 952 953 /* 954 * Configuration initialisation done once at init time. 955 */ 956 static struct rte_mempool * 957 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 958 unsigned int socket_id) 959 { 960 char pool_name[RTE_MEMPOOL_NAMESIZE]; 961 struct rte_mempool *rte_mp = NULL; 962 uint32_t mb_size; 963 964 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 965 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 966 967 TESTPMD_LOG(INFO, 968 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 969 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 970 971 switch (mp_alloc_type) { 972 case MP_ALLOC_NATIVE: 973 { 974 /* wrapper to rte_mempool_create() */ 975 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 976 rte_mbuf_best_mempool_ops()); 977 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 978 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 979 break; 980 } 981 case MP_ALLOC_ANON: 982 { 983 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 984 mb_size, (unsigned int) mb_mempool_cache, 985 sizeof(struct rte_pktmbuf_pool_private), 986 socket_id, mempool_flags); 987 if (rte_mp == NULL) 988 goto err; 989 990 if (rte_mempool_populate_anon(rte_mp) == 0) { 991 rte_mempool_free(rte_mp); 992 rte_mp = NULL; 993 goto err; 994 } 995 rte_pktmbuf_pool_init(rte_mp, NULL); 996 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 997 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 998 break; 999 } 1000 case MP_ALLOC_XMEM: 1001 case MP_ALLOC_XMEM_HUGE: 1002 { 1003 int heap_socket; 1004 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1005 1006 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1007 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1008 1009 heap_socket = 1010 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1011 if (heap_socket < 0) 1012 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1013 1014 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1015 rte_mbuf_best_mempool_ops()); 1016 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1017 mb_mempool_cache, 0, mbuf_seg_size, 1018 heap_socket); 1019 break; 1020 } 1021 case MP_ALLOC_XBUF: 1022 { 1023 struct rte_pktmbuf_extmem *ext_mem; 1024 unsigned int ext_num; 1025 1026 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1027 socket_id, pool_name, &ext_mem); 1028 if (ext_num == 0) 1029 rte_exit(EXIT_FAILURE, 1030 "Can't create pinned data buffers\n"); 1031 1032 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1033 rte_mbuf_best_mempool_ops()); 1034 rte_mp = rte_pktmbuf_pool_create_extbuf 1035 (pool_name, nb_mbuf, mb_mempool_cache, 1036 0, mbuf_seg_size, socket_id, 1037 ext_mem, ext_num); 1038 free(ext_mem); 1039 break; 1040 } 1041 default: 1042 { 1043 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1044 } 1045 } 1046 1047 err: 1048 if (rte_mp == NULL) { 1049 rte_exit(EXIT_FAILURE, 1050 "Creation of mbuf pool for socket %u failed: %s\n", 1051 socket_id, rte_strerror(rte_errno)); 1052 } else if (verbose_level > 0) { 1053 rte_mempool_dump(stdout, rte_mp); 1054 } 1055 return rte_mp; 1056 } 1057 1058 /* 1059 * Check given socket id is valid or not with NUMA mode, 1060 * if valid, return 0, else return -1 1061 */ 1062 static int 1063 check_socket_id(const unsigned int socket_id) 1064 { 1065 static int warning_once = 0; 1066 1067 if (new_socket_id(socket_id)) { 1068 if (!warning_once && numa_support) 1069 printf("Warning: NUMA should be configured manually by" 1070 " using --port-numa-config and" 1071 " --ring-numa-config parameters along with" 1072 " --numa.\n"); 1073 warning_once = 1; 1074 return -1; 1075 } 1076 return 0; 1077 } 1078 1079 /* 1080 * Get the allowed maximum number of RX queues. 1081 * *pid return the port id which has minimal value of 1082 * max_rx_queues in all ports. 1083 */ 1084 queueid_t 1085 get_allowed_max_nb_rxq(portid_t *pid) 1086 { 1087 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1088 bool max_rxq_valid = false; 1089 portid_t pi; 1090 struct rte_eth_dev_info dev_info; 1091 1092 RTE_ETH_FOREACH_DEV(pi) { 1093 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1094 continue; 1095 1096 max_rxq_valid = true; 1097 if (dev_info.max_rx_queues < allowed_max_rxq) { 1098 allowed_max_rxq = dev_info.max_rx_queues; 1099 *pid = pi; 1100 } 1101 } 1102 return max_rxq_valid ? allowed_max_rxq : 0; 1103 } 1104 1105 /* 1106 * Check input rxq is valid or not. 1107 * If input rxq is not greater than any of maximum number 1108 * of RX queues of all ports, it is valid. 1109 * if valid, return 0, else return -1 1110 */ 1111 int 1112 check_nb_rxq(queueid_t rxq) 1113 { 1114 queueid_t allowed_max_rxq; 1115 portid_t pid = 0; 1116 1117 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1118 if (rxq > allowed_max_rxq) { 1119 printf("Fail: input rxq (%u) can't be greater " 1120 "than max_rx_queues (%u) of port %u\n", 1121 rxq, 1122 allowed_max_rxq, 1123 pid); 1124 return -1; 1125 } 1126 return 0; 1127 } 1128 1129 /* 1130 * Get the allowed maximum number of TX queues. 1131 * *pid return the port id which has minimal value of 1132 * max_tx_queues in all ports. 1133 */ 1134 queueid_t 1135 get_allowed_max_nb_txq(portid_t *pid) 1136 { 1137 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1138 bool max_txq_valid = false; 1139 portid_t pi; 1140 struct rte_eth_dev_info dev_info; 1141 1142 RTE_ETH_FOREACH_DEV(pi) { 1143 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1144 continue; 1145 1146 max_txq_valid = true; 1147 if (dev_info.max_tx_queues < allowed_max_txq) { 1148 allowed_max_txq = dev_info.max_tx_queues; 1149 *pid = pi; 1150 } 1151 } 1152 return max_txq_valid ? allowed_max_txq : 0; 1153 } 1154 1155 /* 1156 * Check input txq is valid or not. 1157 * If input txq is not greater than any of maximum number 1158 * of TX queues of all ports, it is valid. 1159 * if valid, return 0, else return -1 1160 */ 1161 int 1162 check_nb_txq(queueid_t txq) 1163 { 1164 queueid_t allowed_max_txq; 1165 portid_t pid = 0; 1166 1167 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1168 if (txq > allowed_max_txq) { 1169 printf("Fail: input txq (%u) can't be greater " 1170 "than max_tx_queues (%u) of port %u\n", 1171 txq, 1172 allowed_max_txq, 1173 pid); 1174 return -1; 1175 } 1176 return 0; 1177 } 1178 1179 /* 1180 * Get the allowed maximum number of RXDs of every rx queue. 1181 * *pid return the port id which has minimal value of 1182 * max_rxd in all queues of all ports. 1183 */ 1184 static uint16_t 1185 get_allowed_max_nb_rxd(portid_t *pid) 1186 { 1187 uint16_t allowed_max_rxd = UINT16_MAX; 1188 portid_t pi; 1189 struct rte_eth_dev_info dev_info; 1190 1191 RTE_ETH_FOREACH_DEV(pi) { 1192 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1193 continue; 1194 1195 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1196 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1197 *pid = pi; 1198 } 1199 } 1200 return allowed_max_rxd; 1201 } 1202 1203 /* 1204 * Get the allowed minimal number of RXDs of every rx queue. 1205 * *pid return the port id which has minimal value of 1206 * min_rxd in all queues of all ports. 1207 */ 1208 static uint16_t 1209 get_allowed_min_nb_rxd(portid_t *pid) 1210 { 1211 uint16_t allowed_min_rxd = 0; 1212 portid_t pi; 1213 struct rte_eth_dev_info dev_info; 1214 1215 RTE_ETH_FOREACH_DEV(pi) { 1216 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1217 continue; 1218 1219 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1220 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1221 *pid = pi; 1222 } 1223 } 1224 1225 return allowed_min_rxd; 1226 } 1227 1228 /* 1229 * Check input rxd is valid or not. 1230 * If input rxd is not greater than any of maximum number 1231 * of RXDs of every Rx queues and is not less than any of 1232 * minimal number of RXDs of every Rx queues, it is valid. 1233 * if valid, return 0, else return -1 1234 */ 1235 int 1236 check_nb_rxd(queueid_t rxd) 1237 { 1238 uint16_t allowed_max_rxd; 1239 uint16_t allowed_min_rxd; 1240 portid_t pid = 0; 1241 1242 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1243 if (rxd > allowed_max_rxd) { 1244 printf("Fail: input rxd (%u) can't be greater " 1245 "than max_rxds (%u) of port %u\n", 1246 rxd, 1247 allowed_max_rxd, 1248 pid); 1249 return -1; 1250 } 1251 1252 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1253 if (rxd < allowed_min_rxd) { 1254 printf("Fail: input rxd (%u) can't be less " 1255 "than min_rxds (%u) of port %u\n", 1256 rxd, 1257 allowed_min_rxd, 1258 pid); 1259 return -1; 1260 } 1261 1262 return 0; 1263 } 1264 1265 /* 1266 * Get the allowed maximum number of TXDs of every rx queues. 1267 * *pid return the port id which has minimal value of 1268 * max_txd in every tx queue. 1269 */ 1270 static uint16_t 1271 get_allowed_max_nb_txd(portid_t *pid) 1272 { 1273 uint16_t allowed_max_txd = UINT16_MAX; 1274 portid_t pi; 1275 struct rte_eth_dev_info dev_info; 1276 1277 RTE_ETH_FOREACH_DEV(pi) { 1278 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1279 continue; 1280 1281 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1282 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1283 *pid = pi; 1284 } 1285 } 1286 return allowed_max_txd; 1287 } 1288 1289 /* 1290 * Get the allowed maximum number of TXDs of every tx queues. 1291 * *pid return the port id which has minimal value of 1292 * min_txd in every tx queue. 1293 */ 1294 static uint16_t 1295 get_allowed_min_nb_txd(portid_t *pid) 1296 { 1297 uint16_t allowed_min_txd = 0; 1298 portid_t pi; 1299 struct rte_eth_dev_info dev_info; 1300 1301 RTE_ETH_FOREACH_DEV(pi) { 1302 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1303 continue; 1304 1305 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1306 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1307 *pid = pi; 1308 } 1309 } 1310 1311 return allowed_min_txd; 1312 } 1313 1314 /* 1315 * Check input txd is valid or not. 1316 * If input txd is not greater than any of maximum number 1317 * of TXDs of every Rx queues, it is valid. 1318 * if valid, return 0, else return -1 1319 */ 1320 int 1321 check_nb_txd(queueid_t txd) 1322 { 1323 uint16_t allowed_max_txd; 1324 uint16_t allowed_min_txd; 1325 portid_t pid = 0; 1326 1327 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1328 if (txd > allowed_max_txd) { 1329 printf("Fail: input txd (%u) can't be greater " 1330 "than max_txds (%u) of port %u\n", 1331 txd, 1332 allowed_max_txd, 1333 pid); 1334 return -1; 1335 } 1336 1337 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1338 if (txd < allowed_min_txd) { 1339 printf("Fail: input txd (%u) can't be less " 1340 "than min_txds (%u) of port %u\n", 1341 txd, 1342 allowed_min_txd, 1343 pid); 1344 return -1; 1345 } 1346 return 0; 1347 } 1348 1349 1350 /* 1351 * Get the allowed maximum number of hairpin queues. 1352 * *pid return the port id which has minimal value of 1353 * max_hairpin_queues in all ports. 1354 */ 1355 queueid_t 1356 get_allowed_max_nb_hairpinq(portid_t *pid) 1357 { 1358 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1359 portid_t pi; 1360 struct rte_eth_hairpin_cap cap; 1361 1362 RTE_ETH_FOREACH_DEV(pi) { 1363 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1364 *pid = pi; 1365 return 0; 1366 } 1367 if (cap.max_nb_queues < allowed_max_hairpinq) { 1368 allowed_max_hairpinq = cap.max_nb_queues; 1369 *pid = pi; 1370 } 1371 } 1372 return allowed_max_hairpinq; 1373 } 1374 1375 /* 1376 * Check input hairpin is valid or not. 1377 * If input hairpin is not greater than any of maximum number 1378 * of hairpin queues of all ports, it is valid. 1379 * if valid, return 0, else return -1 1380 */ 1381 int 1382 check_nb_hairpinq(queueid_t hairpinq) 1383 { 1384 queueid_t allowed_max_hairpinq; 1385 portid_t pid = 0; 1386 1387 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1388 if (hairpinq > allowed_max_hairpinq) { 1389 printf("Fail: input hairpin (%u) can't be greater " 1390 "than max_hairpin_queues (%u) of port %u\n", 1391 hairpinq, allowed_max_hairpinq, pid); 1392 return -1; 1393 } 1394 return 0; 1395 } 1396 1397 static void 1398 init_config(void) 1399 { 1400 portid_t pid; 1401 struct rte_port *port; 1402 struct rte_mempool *mbp; 1403 unsigned int nb_mbuf_per_pool; 1404 lcoreid_t lc_id; 1405 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1406 struct rte_gro_param gro_param; 1407 uint32_t gso_types; 1408 uint16_t data_size; 1409 bool warning = 0; 1410 int k; 1411 int ret; 1412 1413 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1414 1415 /* Configuration of logical cores. */ 1416 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1417 sizeof(struct fwd_lcore *) * nb_lcores, 1418 RTE_CACHE_LINE_SIZE); 1419 if (fwd_lcores == NULL) { 1420 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1421 "failed\n", nb_lcores); 1422 } 1423 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1424 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1425 sizeof(struct fwd_lcore), 1426 RTE_CACHE_LINE_SIZE); 1427 if (fwd_lcores[lc_id] == NULL) { 1428 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1429 "failed\n"); 1430 } 1431 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1432 } 1433 1434 RTE_ETH_FOREACH_DEV(pid) { 1435 port = &ports[pid]; 1436 /* Apply default TxRx configuration for all ports */ 1437 port->dev_conf.txmode = tx_mode; 1438 port->dev_conf.rxmode = rx_mode; 1439 1440 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1441 if (ret != 0) 1442 rte_exit(EXIT_FAILURE, 1443 "rte_eth_dev_info_get() failed\n"); 1444 1445 if (!(port->dev_info.tx_offload_capa & 1446 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1447 port->dev_conf.txmode.offloads &= 1448 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1449 if (numa_support) { 1450 if (port_numa[pid] != NUMA_NO_CONFIG) 1451 port_per_socket[port_numa[pid]]++; 1452 else { 1453 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1454 1455 /* 1456 * if socket_id is invalid, 1457 * set to the first available socket. 1458 */ 1459 if (check_socket_id(socket_id) < 0) 1460 socket_id = socket_ids[0]; 1461 port_per_socket[socket_id]++; 1462 } 1463 } 1464 1465 /* Apply Rx offloads configuration */ 1466 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1467 port->rx_conf[k].offloads = 1468 port->dev_conf.rxmode.offloads; 1469 /* Apply Tx offloads configuration */ 1470 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1471 port->tx_conf[k].offloads = 1472 port->dev_conf.txmode.offloads; 1473 1474 /* set flag to initialize port/queue */ 1475 port->need_reconfig = 1; 1476 port->need_reconfig_queues = 1; 1477 port->tx_metadata = 0; 1478 1479 /* Check for maximum number of segments per MTU. Accordingly 1480 * update the mbuf data size. 1481 */ 1482 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1483 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1484 data_size = rx_mode.max_rx_pkt_len / 1485 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1486 1487 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1488 mbuf_data_size) { 1489 mbuf_data_size = data_size + 1490 RTE_PKTMBUF_HEADROOM; 1491 warning = 1; 1492 } 1493 } 1494 } 1495 1496 if (warning) 1497 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1498 mbuf_data_size); 1499 1500 /* 1501 * Create pools of mbuf. 1502 * If NUMA support is disabled, create a single pool of mbuf in 1503 * socket 0 memory by default. 1504 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1505 * 1506 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1507 * nb_txd can be configured at run time. 1508 */ 1509 if (param_total_num_mbufs) 1510 nb_mbuf_per_pool = param_total_num_mbufs; 1511 else { 1512 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1513 (nb_lcores * mb_mempool_cache) + 1514 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1515 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1516 } 1517 1518 if (numa_support) { 1519 uint8_t i; 1520 1521 for (i = 0; i < num_sockets; i++) 1522 mempools[i] = mbuf_pool_create(mbuf_data_size, 1523 nb_mbuf_per_pool, 1524 socket_ids[i]); 1525 } else { 1526 if (socket_num == UMA_NO_CONFIG) 1527 mempools[0] = mbuf_pool_create(mbuf_data_size, 1528 nb_mbuf_per_pool, 0); 1529 else 1530 mempools[socket_num] = mbuf_pool_create 1531 (mbuf_data_size, 1532 nb_mbuf_per_pool, 1533 socket_num); 1534 } 1535 1536 init_port_config(); 1537 1538 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1539 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1540 /* 1541 * Records which Mbuf pool to use by each logical core, if needed. 1542 */ 1543 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1544 mbp = mbuf_pool_find( 1545 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1546 1547 if (mbp == NULL) 1548 mbp = mbuf_pool_find(0); 1549 fwd_lcores[lc_id]->mbp = mbp; 1550 /* initialize GSO context */ 1551 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1552 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1553 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1554 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1555 RTE_ETHER_CRC_LEN; 1556 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1557 } 1558 1559 /* Configuration of packet forwarding streams. */ 1560 if (init_fwd_streams() < 0) 1561 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1562 1563 fwd_config_setup(); 1564 1565 /* create a gro context for each lcore */ 1566 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1567 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1568 gro_param.max_item_per_flow = MAX_PKT_BURST; 1569 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1570 gro_param.socket_id = rte_lcore_to_socket_id( 1571 fwd_lcores_cpuids[lc_id]); 1572 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1573 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1574 rte_exit(EXIT_FAILURE, 1575 "rte_gro_ctx_create() failed\n"); 1576 } 1577 } 1578 } 1579 1580 1581 void 1582 reconfig(portid_t new_port_id, unsigned socket_id) 1583 { 1584 struct rte_port *port; 1585 int ret; 1586 1587 /* Reconfiguration of Ethernet ports. */ 1588 port = &ports[new_port_id]; 1589 1590 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1591 if (ret != 0) 1592 return; 1593 1594 /* set flag to initialize port/queue */ 1595 port->need_reconfig = 1; 1596 port->need_reconfig_queues = 1; 1597 port->socket_id = socket_id; 1598 1599 init_port_config(); 1600 } 1601 1602 1603 int 1604 init_fwd_streams(void) 1605 { 1606 portid_t pid; 1607 struct rte_port *port; 1608 streamid_t sm_id, nb_fwd_streams_new; 1609 queueid_t q; 1610 1611 /* set socket id according to numa or not */ 1612 RTE_ETH_FOREACH_DEV(pid) { 1613 port = &ports[pid]; 1614 if (nb_rxq > port->dev_info.max_rx_queues) { 1615 printf("Fail: nb_rxq(%d) is greater than " 1616 "max_rx_queues(%d)\n", nb_rxq, 1617 port->dev_info.max_rx_queues); 1618 return -1; 1619 } 1620 if (nb_txq > port->dev_info.max_tx_queues) { 1621 printf("Fail: nb_txq(%d) is greater than " 1622 "max_tx_queues(%d)\n", nb_txq, 1623 port->dev_info.max_tx_queues); 1624 return -1; 1625 } 1626 if (numa_support) { 1627 if (port_numa[pid] != NUMA_NO_CONFIG) 1628 port->socket_id = port_numa[pid]; 1629 else { 1630 port->socket_id = rte_eth_dev_socket_id(pid); 1631 1632 /* 1633 * if socket_id is invalid, 1634 * set to the first available socket. 1635 */ 1636 if (check_socket_id(port->socket_id) < 0) 1637 port->socket_id = socket_ids[0]; 1638 } 1639 } 1640 else { 1641 if (socket_num == UMA_NO_CONFIG) 1642 port->socket_id = 0; 1643 else 1644 port->socket_id = socket_num; 1645 } 1646 } 1647 1648 q = RTE_MAX(nb_rxq, nb_txq); 1649 if (q == 0) { 1650 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1651 return -1; 1652 } 1653 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1654 if (nb_fwd_streams_new == nb_fwd_streams) 1655 return 0; 1656 /* clear the old */ 1657 if (fwd_streams != NULL) { 1658 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1659 if (fwd_streams[sm_id] == NULL) 1660 continue; 1661 rte_free(fwd_streams[sm_id]); 1662 fwd_streams[sm_id] = NULL; 1663 } 1664 rte_free(fwd_streams); 1665 fwd_streams = NULL; 1666 } 1667 1668 /* init new */ 1669 nb_fwd_streams = nb_fwd_streams_new; 1670 if (nb_fwd_streams) { 1671 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1672 sizeof(struct fwd_stream *) * nb_fwd_streams, 1673 RTE_CACHE_LINE_SIZE); 1674 if (fwd_streams == NULL) 1675 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1676 " (struct fwd_stream *)) failed\n", 1677 nb_fwd_streams); 1678 1679 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1680 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1681 " struct fwd_stream", sizeof(struct fwd_stream), 1682 RTE_CACHE_LINE_SIZE); 1683 if (fwd_streams[sm_id] == NULL) 1684 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1685 "(struct fwd_stream) failed\n"); 1686 } 1687 } 1688 1689 return 0; 1690 } 1691 1692 static void 1693 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1694 { 1695 uint64_t total_burst, sburst; 1696 uint64_t nb_burst; 1697 uint64_t burst_stats[4]; 1698 uint16_t pktnb_stats[4]; 1699 uint16_t nb_pkt; 1700 int burst_percent[4], sburstp; 1701 int i; 1702 1703 /* 1704 * First compute the total number of packet bursts and the 1705 * two highest numbers of bursts of the same number of packets. 1706 */ 1707 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1708 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1709 1710 /* Show stats for 0 burst size always */ 1711 total_burst = pbs->pkt_burst_spread[0]; 1712 burst_stats[0] = pbs->pkt_burst_spread[0]; 1713 pktnb_stats[0] = 0; 1714 1715 /* Find the next 2 burst sizes with highest occurrences. */ 1716 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1717 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1718 1719 if (nb_burst == 0) 1720 continue; 1721 1722 total_burst += nb_burst; 1723 1724 if (nb_burst > burst_stats[1]) { 1725 burst_stats[2] = burst_stats[1]; 1726 pktnb_stats[2] = pktnb_stats[1]; 1727 burst_stats[1] = nb_burst; 1728 pktnb_stats[1] = nb_pkt; 1729 } else if (nb_burst > burst_stats[2]) { 1730 burst_stats[2] = nb_burst; 1731 pktnb_stats[2] = nb_pkt; 1732 } 1733 } 1734 if (total_burst == 0) 1735 return; 1736 1737 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1738 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1739 if (i == 3) { 1740 printf("%d%% of other]\n", 100 - sburstp); 1741 return; 1742 } 1743 1744 sburst += burst_stats[i]; 1745 if (sburst == total_burst) { 1746 printf("%d%% of %d pkts]\n", 1747 100 - sburstp, (int) pktnb_stats[i]); 1748 return; 1749 } 1750 1751 burst_percent[i] = 1752 (double)burst_stats[i] / total_burst * 100; 1753 printf("%d%% of %d pkts + ", 1754 burst_percent[i], (int) pktnb_stats[i]); 1755 sburstp += burst_percent[i]; 1756 } 1757 } 1758 1759 static void 1760 fwd_stream_stats_display(streamid_t stream_id) 1761 { 1762 struct fwd_stream *fs; 1763 static const char *fwd_top_stats_border = "-------"; 1764 1765 fs = fwd_streams[stream_id]; 1766 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1767 (fs->fwd_dropped == 0)) 1768 return; 1769 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1770 "TX Port=%2d/Queue=%2d %s\n", 1771 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1772 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1773 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1774 " TX-dropped: %-14"PRIu64, 1775 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1776 1777 /* if checksum mode */ 1778 if (cur_fwd_eng == &csum_fwd_engine) { 1779 printf(" RX- bad IP checksum: %-14"PRIu64 1780 " Rx- bad L4 checksum: %-14"PRIu64 1781 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1782 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1783 fs->rx_bad_outer_l4_csum); 1784 } else { 1785 printf("\n"); 1786 } 1787 1788 if (record_burst_stats) { 1789 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1790 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1791 } 1792 } 1793 1794 void 1795 fwd_stats_display(void) 1796 { 1797 static const char *fwd_stats_border = "----------------------"; 1798 static const char *acc_stats_border = "+++++++++++++++"; 1799 struct { 1800 struct fwd_stream *rx_stream; 1801 struct fwd_stream *tx_stream; 1802 uint64_t tx_dropped; 1803 uint64_t rx_bad_ip_csum; 1804 uint64_t rx_bad_l4_csum; 1805 uint64_t rx_bad_outer_l4_csum; 1806 } ports_stats[RTE_MAX_ETHPORTS]; 1807 uint64_t total_rx_dropped = 0; 1808 uint64_t total_tx_dropped = 0; 1809 uint64_t total_rx_nombuf = 0; 1810 struct rte_eth_stats stats; 1811 uint64_t fwd_cycles = 0; 1812 uint64_t total_recv = 0; 1813 uint64_t total_xmit = 0; 1814 struct rte_port *port; 1815 streamid_t sm_id; 1816 portid_t pt_id; 1817 int i; 1818 1819 memset(ports_stats, 0, sizeof(ports_stats)); 1820 1821 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1822 struct fwd_stream *fs = fwd_streams[sm_id]; 1823 1824 if (cur_fwd_config.nb_fwd_streams > 1825 cur_fwd_config.nb_fwd_ports) { 1826 fwd_stream_stats_display(sm_id); 1827 } else { 1828 ports_stats[fs->tx_port].tx_stream = fs; 1829 ports_stats[fs->rx_port].rx_stream = fs; 1830 } 1831 1832 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1833 1834 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1835 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1836 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1837 fs->rx_bad_outer_l4_csum; 1838 1839 if (record_core_cycles) 1840 fwd_cycles += fs->core_cycles; 1841 } 1842 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1843 uint8_t j; 1844 1845 pt_id = fwd_ports_ids[i]; 1846 port = &ports[pt_id]; 1847 1848 rte_eth_stats_get(pt_id, &stats); 1849 stats.ipackets -= port->stats.ipackets; 1850 stats.opackets -= port->stats.opackets; 1851 stats.ibytes -= port->stats.ibytes; 1852 stats.obytes -= port->stats.obytes; 1853 stats.imissed -= port->stats.imissed; 1854 stats.oerrors -= port->stats.oerrors; 1855 stats.rx_nombuf -= port->stats.rx_nombuf; 1856 1857 total_recv += stats.ipackets; 1858 total_xmit += stats.opackets; 1859 total_rx_dropped += stats.imissed; 1860 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1861 total_tx_dropped += stats.oerrors; 1862 total_rx_nombuf += stats.rx_nombuf; 1863 1864 printf("\n %s Forward statistics for port %-2d %s\n", 1865 fwd_stats_border, pt_id, fwd_stats_border); 1866 1867 if (!port->rx_queue_stats_mapping_enabled && 1868 !port->tx_queue_stats_mapping_enabled) { 1869 printf(" RX-packets: %-14"PRIu64 1870 " RX-dropped: %-14"PRIu64 1871 "RX-total: %-"PRIu64"\n", 1872 stats.ipackets, stats.imissed, 1873 stats.ipackets + stats.imissed); 1874 1875 if (cur_fwd_eng == &csum_fwd_engine) 1876 printf(" Bad-ipcsum: %-14"PRIu64 1877 " Bad-l4csum: %-14"PRIu64 1878 "Bad-outer-l4csum: %-14"PRIu64"\n", 1879 ports_stats[pt_id].rx_bad_ip_csum, 1880 ports_stats[pt_id].rx_bad_l4_csum, 1881 ports_stats[pt_id].rx_bad_outer_l4_csum); 1882 if (stats.ierrors + stats.rx_nombuf > 0) { 1883 printf(" RX-error: %-"PRIu64"\n", 1884 stats.ierrors); 1885 printf(" RX-nombufs: %-14"PRIu64"\n", 1886 stats.rx_nombuf); 1887 } 1888 1889 printf(" TX-packets: %-14"PRIu64 1890 " TX-dropped: %-14"PRIu64 1891 "TX-total: %-"PRIu64"\n", 1892 stats.opackets, ports_stats[pt_id].tx_dropped, 1893 stats.opackets + ports_stats[pt_id].tx_dropped); 1894 } else { 1895 printf(" RX-packets: %14"PRIu64 1896 " RX-dropped:%14"PRIu64 1897 " RX-total:%14"PRIu64"\n", 1898 stats.ipackets, stats.imissed, 1899 stats.ipackets + stats.imissed); 1900 1901 if (cur_fwd_eng == &csum_fwd_engine) 1902 printf(" Bad-ipcsum:%14"PRIu64 1903 " Bad-l4csum:%14"PRIu64 1904 " Bad-outer-l4csum: %-14"PRIu64"\n", 1905 ports_stats[pt_id].rx_bad_ip_csum, 1906 ports_stats[pt_id].rx_bad_l4_csum, 1907 ports_stats[pt_id].rx_bad_outer_l4_csum); 1908 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1909 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1910 printf(" RX-nombufs: %14"PRIu64"\n", 1911 stats.rx_nombuf); 1912 } 1913 1914 printf(" TX-packets: %14"PRIu64 1915 " TX-dropped:%14"PRIu64 1916 " TX-total:%14"PRIu64"\n", 1917 stats.opackets, ports_stats[pt_id].tx_dropped, 1918 stats.opackets + ports_stats[pt_id].tx_dropped); 1919 } 1920 1921 if (record_burst_stats) { 1922 if (ports_stats[pt_id].rx_stream) 1923 pkt_burst_stats_display("RX", 1924 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1925 if (ports_stats[pt_id].tx_stream) 1926 pkt_burst_stats_display("TX", 1927 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1928 } 1929 1930 if (port->rx_queue_stats_mapping_enabled) { 1931 printf("\n"); 1932 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1933 printf(" Stats reg %2d RX-packets:%14"PRIu64 1934 " RX-errors:%14"PRIu64 1935 " RX-bytes:%14"PRIu64"\n", 1936 j, stats.q_ipackets[j], 1937 stats.q_errors[j], stats.q_ibytes[j]); 1938 } 1939 printf("\n"); 1940 } 1941 if (port->tx_queue_stats_mapping_enabled) { 1942 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1943 printf(" Stats reg %2d TX-packets:%14"PRIu64 1944 " TX-bytes:%14" 1945 PRIu64"\n", 1946 j, stats.q_opackets[j], 1947 stats.q_obytes[j]); 1948 } 1949 } 1950 1951 printf(" %s--------------------------------%s\n", 1952 fwd_stats_border, fwd_stats_border); 1953 } 1954 1955 printf("\n %s Accumulated forward statistics for all ports" 1956 "%s\n", 1957 acc_stats_border, acc_stats_border); 1958 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1959 "%-"PRIu64"\n" 1960 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1961 "%-"PRIu64"\n", 1962 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1963 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1964 if (total_rx_nombuf > 0) 1965 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1966 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1967 "%s\n", 1968 acc_stats_border, acc_stats_border); 1969 if (record_core_cycles) { 1970 #define CYC_PER_MHZ 1E6 1971 if (total_recv > 0 || total_xmit > 0) { 1972 uint64_t total_pkts = 0; 1973 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1974 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1975 total_pkts = total_xmit; 1976 else 1977 total_pkts = total_recv; 1978 1979 printf("\n CPU cycles/packet=%.2F (total cycles=" 1980 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1981 " MHz Clock\n", 1982 (double) fwd_cycles / total_pkts, 1983 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1984 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1985 } 1986 } 1987 } 1988 1989 void 1990 fwd_stats_reset(void) 1991 { 1992 streamid_t sm_id; 1993 portid_t pt_id; 1994 int i; 1995 1996 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1997 pt_id = fwd_ports_ids[i]; 1998 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1999 } 2000 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2001 struct fwd_stream *fs = fwd_streams[sm_id]; 2002 2003 fs->rx_packets = 0; 2004 fs->tx_packets = 0; 2005 fs->fwd_dropped = 0; 2006 fs->rx_bad_ip_csum = 0; 2007 fs->rx_bad_l4_csum = 0; 2008 fs->rx_bad_outer_l4_csum = 0; 2009 2010 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2011 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2012 fs->core_cycles = 0; 2013 } 2014 } 2015 2016 static void 2017 flush_fwd_rx_queues(void) 2018 { 2019 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2020 portid_t rxp; 2021 portid_t port_id; 2022 queueid_t rxq; 2023 uint16_t nb_rx; 2024 uint16_t i; 2025 uint8_t j; 2026 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2027 uint64_t timer_period; 2028 2029 /* convert to number of cycles */ 2030 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2031 2032 for (j = 0; j < 2; j++) { 2033 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2034 for (rxq = 0; rxq < nb_rxq; rxq++) { 2035 port_id = fwd_ports_ids[rxp]; 2036 /** 2037 * testpmd can stuck in the below do while loop 2038 * if rte_eth_rx_burst() always returns nonzero 2039 * packets. So timer is added to exit this loop 2040 * after 1sec timer expiry. 2041 */ 2042 prev_tsc = rte_rdtsc(); 2043 do { 2044 nb_rx = rte_eth_rx_burst(port_id, rxq, 2045 pkts_burst, MAX_PKT_BURST); 2046 for (i = 0; i < nb_rx; i++) 2047 rte_pktmbuf_free(pkts_burst[i]); 2048 2049 cur_tsc = rte_rdtsc(); 2050 diff_tsc = cur_tsc - prev_tsc; 2051 timer_tsc += diff_tsc; 2052 } while ((nb_rx > 0) && 2053 (timer_tsc < timer_period)); 2054 timer_tsc = 0; 2055 } 2056 } 2057 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2058 } 2059 } 2060 2061 static void 2062 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2063 { 2064 struct fwd_stream **fsm; 2065 streamid_t nb_fs; 2066 streamid_t sm_id; 2067 #ifdef RTE_LIBRTE_BITRATESTATS 2068 uint64_t tics_per_1sec; 2069 uint64_t tics_datum; 2070 uint64_t tics_current; 2071 uint16_t i, cnt_ports; 2072 2073 cnt_ports = nb_ports; 2074 tics_datum = rte_rdtsc(); 2075 tics_per_1sec = rte_get_timer_hz(); 2076 #endif 2077 fsm = &fwd_streams[fc->stream_idx]; 2078 nb_fs = fc->stream_nb; 2079 do { 2080 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2081 (*pkt_fwd)(fsm[sm_id]); 2082 #ifdef RTE_LIBRTE_BITRATESTATS 2083 if (bitrate_enabled != 0 && 2084 bitrate_lcore_id == rte_lcore_id()) { 2085 tics_current = rte_rdtsc(); 2086 if (tics_current - tics_datum >= tics_per_1sec) { 2087 /* Periodic bitrate calculation */ 2088 for (i = 0; i < cnt_ports; i++) 2089 rte_stats_bitrate_calc(bitrate_data, 2090 ports_ids[i]); 2091 tics_datum = tics_current; 2092 } 2093 } 2094 #endif 2095 #ifdef RTE_LIBRTE_LATENCY_STATS 2096 if (latencystats_enabled != 0 && 2097 latencystats_lcore_id == rte_lcore_id()) 2098 rte_latencystats_update(); 2099 #endif 2100 2101 } while (! fc->stopped); 2102 } 2103 2104 static int 2105 start_pkt_forward_on_core(void *fwd_arg) 2106 { 2107 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2108 cur_fwd_config.fwd_eng->packet_fwd); 2109 return 0; 2110 } 2111 2112 /* 2113 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2114 * Used to start communication flows in network loopback test configurations. 2115 */ 2116 static int 2117 run_one_txonly_burst_on_core(void *fwd_arg) 2118 { 2119 struct fwd_lcore *fwd_lc; 2120 struct fwd_lcore tmp_lcore; 2121 2122 fwd_lc = (struct fwd_lcore *) fwd_arg; 2123 tmp_lcore = *fwd_lc; 2124 tmp_lcore.stopped = 1; 2125 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2126 return 0; 2127 } 2128 2129 /* 2130 * Launch packet forwarding: 2131 * - Setup per-port forwarding context. 2132 * - launch logical cores with their forwarding configuration. 2133 */ 2134 static void 2135 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2136 { 2137 port_fwd_begin_t port_fwd_begin; 2138 unsigned int i; 2139 unsigned int lc_id; 2140 int diag; 2141 2142 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2143 if (port_fwd_begin != NULL) { 2144 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2145 (*port_fwd_begin)(fwd_ports_ids[i]); 2146 } 2147 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2148 lc_id = fwd_lcores_cpuids[i]; 2149 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2150 fwd_lcores[i]->stopped = 0; 2151 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2152 fwd_lcores[i], lc_id); 2153 if (diag != 0) 2154 printf("launch lcore %u failed - diag=%d\n", 2155 lc_id, diag); 2156 } 2157 } 2158 } 2159 2160 /* 2161 * Launch packet forwarding configuration. 2162 */ 2163 void 2164 start_packet_forwarding(int with_tx_first) 2165 { 2166 port_fwd_begin_t port_fwd_begin; 2167 port_fwd_end_t port_fwd_end; 2168 struct rte_port *port; 2169 unsigned int i; 2170 portid_t pt_id; 2171 2172 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2173 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2174 2175 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2176 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2177 2178 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2179 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2180 (!nb_rxq || !nb_txq)) 2181 rte_exit(EXIT_FAILURE, 2182 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2183 cur_fwd_eng->fwd_mode_name); 2184 2185 if (all_ports_started() == 0) { 2186 printf("Not all ports were started\n"); 2187 return; 2188 } 2189 if (test_done == 0) { 2190 printf("Packet forwarding already started\n"); 2191 return; 2192 } 2193 2194 2195 if(dcb_test) { 2196 for (i = 0; i < nb_fwd_ports; i++) { 2197 pt_id = fwd_ports_ids[i]; 2198 port = &ports[pt_id]; 2199 if (!port->dcb_flag) { 2200 printf("In DCB mode, all forwarding ports must " 2201 "be configured in this mode.\n"); 2202 return; 2203 } 2204 } 2205 if (nb_fwd_lcores == 1) { 2206 printf("In DCB mode,the nb forwarding cores " 2207 "should be larger than 1.\n"); 2208 return; 2209 } 2210 } 2211 test_done = 0; 2212 2213 fwd_config_setup(); 2214 2215 if(!no_flush_rx) 2216 flush_fwd_rx_queues(); 2217 2218 pkt_fwd_config_display(&cur_fwd_config); 2219 rxtx_config_display(); 2220 2221 fwd_stats_reset(); 2222 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2223 pt_id = fwd_ports_ids[i]; 2224 port = &ports[pt_id]; 2225 map_port_queue_stats_mapping_registers(pt_id, port); 2226 } 2227 if (with_tx_first) { 2228 port_fwd_begin = tx_only_engine.port_fwd_begin; 2229 if (port_fwd_begin != NULL) { 2230 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2231 (*port_fwd_begin)(fwd_ports_ids[i]); 2232 } 2233 while (with_tx_first--) { 2234 launch_packet_forwarding( 2235 run_one_txonly_burst_on_core); 2236 rte_eal_mp_wait_lcore(); 2237 } 2238 port_fwd_end = tx_only_engine.port_fwd_end; 2239 if (port_fwd_end != NULL) { 2240 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2241 (*port_fwd_end)(fwd_ports_ids[i]); 2242 } 2243 } 2244 launch_packet_forwarding(start_pkt_forward_on_core); 2245 } 2246 2247 void 2248 stop_packet_forwarding(void) 2249 { 2250 port_fwd_end_t port_fwd_end; 2251 lcoreid_t lc_id; 2252 portid_t pt_id; 2253 int i; 2254 2255 if (test_done) { 2256 printf("Packet forwarding not started\n"); 2257 return; 2258 } 2259 printf("Telling cores to stop..."); 2260 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2261 fwd_lcores[lc_id]->stopped = 1; 2262 printf("\nWaiting for lcores to finish...\n"); 2263 rte_eal_mp_wait_lcore(); 2264 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2265 if (port_fwd_end != NULL) { 2266 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2267 pt_id = fwd_ports_ids[i]; 2268 (*port_fwd_end)(pt_id); 2269 } 2270 } 2271 2272 fwd_stats_display(); 2273 2274 printf("\nDone.\n"); 2275 test_done = 1; 2276 } 2277 2278 void 2279 dev_set_link_up(portid_t pid) 2280 { 2281 if (rte_eth_dev_set_link_up(pid) < 0) 2282 printf("\nSet link up fail.\n"); 2283 } 2284 2285 void 2286 dev_set_link_down(portid_t pid) 2287 { 2288 if (rte_eth_dev_set_link_down(pid) < 0) 2289 printf("\nSet link down fail.\n"); 2290 } 2291 2292 static int 2293 all_ports_started(void) 2294 { 2295 portid_t pi; 2296 struct rte_port *port; 2297 2298 RTE_ETH_FOREACH_DEV(pi) { 2299 port = &ports[pi]; 2300 /* Check if there is a port which is not started */ 2301 if ((port->port_status != RTE_PORT_STARTED) && 2302 (port->slave_flag == 0)) 2303 return 0; 2304 } 2305 2306 /* No port is not started */ 2307 return 1; 2308 } 2309 2310 int 2311 port_is_stopped(portid_t port_id) 2312 { 2313 struct rte_port *port = &ports[port_id]; 2314 2315 if ((port->port_status != RTE_PORT_STOPPED) && 2316 (port->slave_flag == 0)) 2317 return 0; 2318 return 1; 2319 } 2320 2321 int 2322 all_ports_stopped(void) 2323 { 2324 portid_t pi; 2325 2326 RTE_ETH_FOREACH_DEV(pi) { 2327 if (!port_is_stopped(pi)) 2328 return 0; 2329 } 2330 2331 return 1; 2332 } 2333 2334 int 2335 port_is_started(portid_t port_id) 2336 { 2337 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2338 return 0; 2339 2340 if (ports[port_id].port_status != RTE_PORT_STARTED) 2341 return 0; 2342 2343 return 1; 2344 } 2345 2346 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2347 static int 2348 setup_hairpin_queues(portid_t pi) 2349 { 2350 queueid_t qi; 2351 struct rte_eth_hairpin_conf hairpin_conf = { 2352 .peer_count = 1, 2353 }; 2354 int i; 2355 int diag; 2356 struct rte_port *port = &ports[pi]; 2357 2358 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2359 hairpin_conf.peers[0].port = pi; 2360 hairpin_conf.peers[0].queue = i + nb_rxq; 2361 diag = rte_eth_tx_hairpin_queue_setup 2362 (pi, qi, nb_txd, &hairpin_conf); 2363 i++; 2364 if (diag == 0) 2365 continue; 2366 2367 /* Fail to setup rx queue, return */ 2368 if (rte_atomic16_cmpset(&(port->port_status), 2369 RTE_PORT_HANDLING, 2370 RTE_PORT_STOPPED) == 0) 2371 printf("Port %d can not be set back " 2372 "to stopped\n", pi); 2373 printf("Fail to configure port %d hairpin " 2374 "queues\n", pi); 2375 /* try to reconfigure queues next time */ 2376 port->need_reconfig_queues = 1; 2377 return -1; 2378 } 2379 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2380 hairpin_conf.peers[0].port = pi; 2381 hairpin_conf.peers[0].queue = i + nb_txq; 2382 diag = rte_eth_rx_hairpin_queue_setup 2383 (pi, qi, nb_rxd, &hairpin_conf); 2384 i++; 2385 if (diag == 0) 2386 continue; 2387 2388 /* Fail to setup rx queue, return */ 2389 if (rte_atomic16_cmpset(&(port->port_status), 2390 RTE_PORT_HANDLING, 2391 RTE_PORT_STOPPED) == 0) 2392 printf("Port %d can not be set back " 2393 "to stopped\n", pi); 2394 printf("Fail to configure port %d hairpin " 2395 "queues\n", pi); 2396 /* try to reconfigure queues next time */ 2397 port->need_reconfig_queues = 1; 2398 return -1; 2399 } 2400 return 0; 2401 } 2402 2403 int 2404 start_port(portid_t pid) 2405 { 2406 int diag, need_check_link_status = -1; 2407 portid_t pi; 2408 queueid_t qi; 2409 struct rte_port *port; 2410 struct rte_ether_addr mac_addr; 2411 struct rte_eth_hairpin_cap cap; 2412 2413 if (port_id_is_invalid(pid, ENABLED_WARN)) 2414 return 0; 2415 2416 if(dcb_config) 2417 dcb_test = 1; 2418 RTE_ETH_FOREACH_DEV(pi) { 2419 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2420 continue; 2421 2422 need_check_link_status = 0; 2423 port = &ports[pi]; 2424 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2425 RTE_PORT_HANDLING) == 0) { 2426 printf("Port %d is now not stopped\n", pi); 2427 continue; 2428 } 2429 2430 if (port->need_reconfig > 0) { 2431 port->need_reconfig = 0; 2432 2433 if (flow_isolate_all) { 2434 int ret = port_flow_isolate(pi, 1); 2435 if (ret) { 2436 printf("Failed to apply isolated" 2437 " mode on port %d\n", pi); 2438 return -1; 2439 } 2440 } 2441 configure_rxtx_dump_callbacks(0); 2442 printf("Configuring Port %d (socket %u)\n", pi, 2443 port->socket_id); 2444 if (nb_hairpinq > 0 && 2445 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2446 printf("Port %d doesn't support hairpin " 2447 "queues\n", pi); 2448 return -1; 2449 } 2450 /* configure port */ 2451 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2452 nb_txq + nb_hairpinq, 2453 &(port->dev_conf)); 2454 if (diag != 0) { 2455 if (rte_atomic16_cmpset(&(port->port_status), 2456 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2457 printf("Port %d can not be set back " 2458 "to stopped\n", pi); 2459 printf("Fail to configure port %d\n", pi); 2460 /* try to reconfigure port next time */ 2461 port->need_reconfig = 1; 2462 return -1; 2463 } 2464 } 2465 if (port->need_reconfig_queues > 0) { 2466 port->need_reconfig_queues = 0; 2467 /* setup tx queues */ 2468 for (qi = 0; qi < nb_txq; qi++) { 2469 if ((numa_support) && 2470 (txring_numa[pi] != NUMA_NO_CONFIG)) 2471 diag = rte_eth_tx_queue_setup(pi, qi, 2472 port->nb_tx_desc[qi], 2473 txring_numa[pi], 2474 &(port->tx_conf[qi])); 2475 else 2476 diag = rte_eth_tx_queue_setup(pi, qi, 2477 port->nb_tx_desc[qi], 2478 port->socket_id, 2479 &(port->tx_conf[qi])); 2480 2481 if (diag == 0) 2482 continue; 2483 2484 /* Fail to setup tx queue, return */ 2485 if (rte_atomic16_cmpset(&(port->port_status), 2486 RTE_PORT_HANDLING, 2487 RTE_PORT_STOPPED) == 0) 2488 printf("Port %d can not be set back " 2489 "to stopped\n", pi); 2490 printf("Fail to configure port %d tx queues\n", 2491 pi); 2492 /* try to reconfigure queues next time */ 2493 port->need_reconfig_queues = 1; 2494 return -1; 2495 } 2496 for (qi = 0; qi < nb_rxq; qi++) { 2497 /* setup rx queues */ 2498 if ((numa_support) && 2499 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2500 struct rte_mempool * mp = 2501 mbuf_pool_find(rxring_numa[pi]); 2502 if (mp == NULL) { 2503 printf("Failed to setup RX queue:" 2504 "No mempool allocation" 2505 " on the socket %d\n", 2506 rxring_numa[pi]); 2507 return -1; 2508 } 2509 2510 diag = rte_eth_rx_queue_setup(pi, qi, 2511 port->nb_rx_desc[qi], 2512 rxring_numa[pi], 2513 &(port->rx_conf[qi]), 2514 mp); 2515 } else { 2516 struct rte_mempool *mp = 2517 mbuf_pool_find(port->socket_id); 2518 if (mp == NULL) { 2519 printf("Failed to setup RX queue:" 2520 "No mempool allocation" 2521 " on the socket %d\n", 2522 port->socket_id); 2523 return -1; 2524 } 2525 diag = rte_eth_rx_queue_setup(pi, qi, 2526 port->nb_rx_desc[qi], 2527 port->socket_id, 2528 &(port->rx_conf[qi]), 2529 mp); 2530 } 2531 if (diag == 0) 2532 continue; 2533 2534 /* Fail to setup rx queue, return */ 2535 if (rte_atomic16_cmpset(&(port->port_status), 2536 RTE_PORT_HANDLING, 2537 RTE_PORT_STOPPED) == 0) 2538 printf("Port %d can not be set back " 2539 "to stopped\n", pi); 2540 printf("Fail to configure port %d rx queues\n", 2541 pi); 2542 /* try to reconfigure queues next time */ 2543 port->need_reconfig_queues = 1; 2544 return -1; 2545 } 2546 /* setup hairpin queues */ 2547 if (setup_hairpin_queues(pi) != 0) 2548 return -1; 2549 } 2550 configure_rxtx_dump_callbacks(verbose_level); 2551 if (clear_ptypes) { 2552 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2553 NULL, 0); 2554 if (diag < 0) 2555 printf( 2556 "Port %d: Failed to disable Ptype parsing\n", 2557 pi); 2558 } 2559 2560 /* start port */ 2561 if (rte_eth_dev_start(pi) < 0) { 2562 printf("Fail to start port %d\n", pi); 2563 2564 /* Fail to setup rx queue, return */ 2565 if (rte_atomic16_cmpset(&(port->port_status), 2566 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2567 printf("Port %d can not be set back to " 2568 "stopped\n", pi); 2569 continue; 2570 } 2571 2572 if (rte_atomic16_cmpset(&(port->port_status), 2573 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2574 printf("Port %d can not be set into started\n", pi); 2575 2576 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2577 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2578 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2579 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2580 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2581 2582 /* at least one port started, need checking link status */ 2583 need_check_link_status = 1; 2584 } 2585 2586 if (need_check_link_status == 1 && !no_link_check) 2587 check_all_ports_link_status(RTE_PORT_ALL); 2588 else if (need_check_link_status == 0) 2589 printf("Please stop the ports first\n"); 2590 2591 printf("Done\n"); 2592 return 0; 2593 } 2594 2595 void 2596 stop_port(portid_t pid) 2597 { 2598 portid_t pi; 2599 struct rte_port *port; 2600 int need_check_link_status = 0; 2601 2602 if (dcb_test) { 2603 dcb_test = 0; 2604 dcb_config = 0; 2605 } 2606 2607 if (port_id_is_invalid(pid, ENABLED_WARN)) 2608 return; 2609 2610 printf("Stopping ports...\n"); 2611 2612 RTE_ETH_FOREACH_DEV(pi) { 2613 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2614 continue; 2615 2616 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2617 printf("Please remove port %d from forwarding configuration.\n", pi); 2618 continue; 2619 } 2620 2621 if (port_is_bonding_slave(pi)) { 2622 printf("Please remove port %d from bonded device.\n", pi); 2623 continue; 2624 } 2625 2626 port = &ports[pi]; 2627 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2628 RTE_PORT_HANDLING) == 0) 2629 continue; 2630 2631 rte_eth_dev_stop(pi); 2632 2633 if (rte_atomic16_cmpset(&(port->port_status), 2634 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2635 printf("Port %d can not be set into stopped\n", pi); 2636 need_check_link_status = 1; 2637 } 2638 if (need_check_link_status && !no_link_check) 2639 check_all_ports_link_status(RTE_PORT_ALL); 2640 2641 printf("Done\n"); 2642 } 2643 2644 static void 2645 remove_invalid_ports_in(portid_t *array, portid_t *total) 2646 { 2647 portid_t i; 2648 portid_t new_total = 0; 2649 2650 for (i = 0; i < *total; i++) 2651 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2652 array[new_total] = array[i]; 2653 new_total++; 2654 } 2655 *total = new_total; 2656 } 2657 2658 static void 2659 remove_invalid_ports(void) 2660 { 2661 remove_invalid_ports_in(ports_ids, &nb_ports); 2662 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2663 nb_cfg_ports = nb_fwd_ports; 2664 } 2665 2666 void 2667 close_port(portid_t pid) 2668 { 2669 portid_t pi; 2670 struct rte_port *port; 2671 2672 if (port_id_is_invalid(pid, ENABLED_WARN)) 2673 return; 2674 2675 printf("Closing ports...\n"); 2676 2677 RTE_ETH_FOREACH_DEV(pi) { 2678 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2679 continue; 2680 2681 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2682 printf("Please remove port %d from forwarding configuration.\n", pi); 2683 continue; 2684 } 2685 2686 if (port_is_bonding_slave(pi)) { 2687 printf("Please remove port %d from bonded device.\n", pi); 2688 continue; 2689 } 2690 2691 port = &ports[pi]; 2692 if (rte_atomic16_cmpset(&(port->port_status), 2693 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2694 printf("Port %d is already closed\n", pi); 2695 continue; 2696 } 2697 2698 if (rte_atomic16_cmpset(&(port->port_status), 2699 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2700 printf("Port %d is now not stopped\n", pi); 2701 continue; 2702 } 2703 2704 if (port->flow_list) 2705 port_flow_flush(pi); 2706 rte_eth_dev_close(pi); 2707 2708 remove_invalid_ports(); 2709 2710 if (rte_atomic16_cmpset(&(port->port_status), 2711 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2712 printf("Port %d cannot be set to closed\n", pi); 2713 } 2714 2715 printf("Done\n"); 2716 } 2717 2718 void 2719 reset_port(portid_t pid) 2720 { 2721 int diag; 2722 portid_t pi; 2723 struct rte_port *port; 2724 2725 if (port_id_is_invalid(pid, ENABLED_WARN)) 2726 return; 2727 2728 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2729 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2730 printf("Can not reset port(s), please stop port(s) first.\n"); 2731 return; 2732 } 2733 2734 printf("Resetting ports...\n"); 2735 2736 RTE_ETH_FOREACH_DEV(pi) { 2737 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2738 continue; 2739 2740 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2741 printf("Please remove port %d from forwarding " 2742 "configuration.\n", pi); 2743 continue; 2744 } 2745 2746 if (port_is_bonding_slave(pi)) { 2747 printf("Please remove port %d from bonded device.\n", 2748 pi); 2749 continue; 2750 } 2751 2752 diag = rte_eth_dev_reset(pi); 2753 if (diag == 0) { 2754 port = &ports[pi]; 2755 port->need_reconfig = 1; 2756 port->need_reconfig_queues = 1; 2757 } else { 2758 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2759 } 2760 } 2761 2762 printf("Done\n"); 2763 } 2764 2765 void 2766 attach_port(char *identifier) 2767 { 2768 portid_t pi; 2769 struct rte_dev_iterator iterator; 2770 2771 printf("Attaching a new port...\n"); 2772 2773 if (identifier == NULL) { 2774 printf("Invalid parameters are specified\n"); 2775 return; 2776 } 2777 2778 if (rte_dev_probe(identifier) < 0) { 2779 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2780 return; 2781 } 2782 2783 /* first attach mode: event */ 2784 if (setup_on_probe_event) { 2785 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2786 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2787 if (ports[pi].port_status == RTE_PORT_HANDLING && 2788 ports[pi].need_setup != 0) 2789 setup_attached_port(pi); 2790 return; 2791 } 2792 2793 /* second attach mode: iterator */ 2794 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2795 /* setup ports matching the devargs used for probing */ 2796 if (port_is_forwarding(pi)) 2797 continue; /* port was already attached before */ 2798 setup_attached_port(pi); 2799 } 2800 } 2801 2802 static void 2803 setup_attached_port(portid_t pi) 2804 { 2805 unsigned int socket_id; 2806 int ret; 2807 2808 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2809 /* if socket_id is invalid, set to the first available socket. */ 2810 if (check_socket_id(socket_id) < 0) 2811 socket_id = socket_ids[0]; 2812 reconfig(pi, socket_id); 2813 ret = rte_eth_promiscuous_enable(pi); 2814 if (ret != 0) 2815 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2816 pi, rte_strerror(-ret)); 2817 2818 ports_ids[nb_ports++] = pi; 2819 fwd_ports_ids[nb_fwd_ports++] = pi; 2820 nb_cfg_ports = nb_fwd_ports; 2821 ports[pi].need_setup = 0; 2822 ports[pi].port_status = RTE_PORT_STOPPED; 2823 2824 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2825 printf("Done\n"); 2826 } 2827 2828 static void 2829 detach_device(struct rte_device *dev) 2830 { 2831 portid_t sibling; 2832 2833 if (dev == NULL) { 2834 printf("Device already removed\n"); 2835 return; 2836 } 2837 2838 printf("Removing a device...\n"); 2839 2840 if (rte_dev_remove(dev) < 0) { 2841 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2842 return; 2843 } 2844 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2845 /* reset mapping between old ports and removed device */ 2846 rte_eth_devices[sibling].device = NULL; 2847 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2848 /* sibling ports are forced to be closed */ 2849 ports[sibling].port_status = RTE_PORT_CLOSED; 2850 printf("Port %u is closed\n", sibling); 2851 } 2852 } 2853 2854 remove_invalid_ports(); 2855 2856 printf("Device is detached\n"); 2857 printf("Now total ports is %d\n", nb_ports); 2858 printf("Done\n"); 2859 return; 2860 } 2861 2862 void 2863 detach_port_device(portid_t port_id) 2864 { 2865 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2866 return; 2867 2868 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2869 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2870 printf("Port not stopped\n"); 2871 return; 2872 } 2873 printf("Port was not closed\n"); 2874 if (ports[port_id].flow_list) 2875 port_flow_flush(port_id); 2876 } 2877 2878 detach_device(rte_eth_devices[port_id].device); 2879 } 2880 2881 void 2882 detach_devargs(char *identifier) 2883 { 2884 struct rte_dev_iterator iterator; 2885 struct rte_devargs da; 2886 portid_t port_id; 2887 2888 printf("Removing a device...\n"); 2889 2890 memset(&da, 0, sizeof(da)); 2891 if (rte_devargs_parsef(&da, "%s", identifier)) { 2892 printf("cannot parse identifier\n"); 2893 if (da.args) 2894 free(da.args); 2895 return; 2896 } 2897 2898 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2899 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2900 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2901 printf("Port %u not stopped\n", port_id); 2902 rte_eth_iterator_cleanup(&iterator); 2903 return; 2904 } 2905 2906 /* sibling ports are forced to be closed */ 2907 if (ports[port_id].flow_list) 2908 port_flow_flush(port_id); 2909 ports[port_id].port_status = RTE_PORT_CLOSED; 2910 printf("Port %u is now closed\n", port_id); 2911 } 2912 } 2913 2914 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2915 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2916 da.name, da.bus->name); 2917 return; 2918 } 2919 2920 remove_invalid_ports(); 2921 2922 printf("Device %s is detached\n", identifier); 2923 printf("Now total ports is %d\n", nb_ports); 2924 printf("Done\n"); 2925 } 2926 2927 void 2928 pmd_test_exit(void) 2929 { 2930 portid_t pt_id; 2931 int ret; 2932 int i; 2933 2934 if (test_done == 0) 2935 stop_packet_forwarding(); 2936 2937 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2938 if (mempools[i]) { 2939 if (mp_alloc_type == MP_ALLOC_ANON) 2940 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2941 NULL); 2942 } 2943 } 2944 if (ports != NULL) { 2945 no_link_check = 1; 2946 RTE_ETH_FOREACH_DEV(pt_id) { 2947 printf("\nStopping port %d...\n", pt_id); 2948 fflush(stdout); 2949 stop_port(pt_id); 2950 } 2951 RTE_ETH_FOREACH_DEV(pt_id) { 2952 printf("\nShutting down port %d...\n", pt_id); 2953 fflush(stdout); 2954 close_port(pt_id); 2955 } 2956 } 2957 2958 if (hot_plug) { 2959 ret = rte_dev_event_monitor_stop(); 2960 if (ret) { 2961 RTE_LOG(ERR, EAL, 2962 "fail to stop device event monitor."); 2963 return; 2964 } 2965 2966 ret = rte_dev_event_callback_unregister(NULL, 2967 dev_event_callback, NULL); 2968 if (ret < 0) { 2969 RTE_LOG(ERR, EAL, 2970 "fail to unregister device event callback.\n"); 2971 return; 2972 } 2973 2974 ret = rte_dev_hotplug_handle_disable(); 2975 if (ret) { 2976 RTE_LOG(ERR, EAL, 2977 "fail to disable hotplug handling.\n"); 2978 return; 2979 } 2980 } 2981 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2982 if (mempools[i]) 2983 rte_mempool_free(mempools[i]); 2984 } 2985 2986 printf("\nBye...\n"); 2987 } 2988 2989 typedef void (*cmd_func_t)(void); 2990 struct pmd_test_command { 2991 const char *cmd_name; 2992 cmd_func_t cmd_func; 2993 }; 2994 2995 /* Check the link status of all ports in up to 9s, and print them finally */ 2996 static void 2997 check_all_ports_link_status(uint32_t port_mask) 2998 { 2999 #define CHECK_INTERVAL 100 /* 100ms */ 3000 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3001 portid_t portid; 3002 uint8_t count, all_ports_up, print_flag = 0; 3003 struct rte_eth_link link; 3004 int ret; 3005 3006 printf("Checking link statuses...\n"); 3007 fflush(stdout); 3008 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3009 all_ports_up = 1; 3010 RTE_ETH_FOREACH_DEV(portid) { 3011 if ((port_mask & (1 << portid)) == 0) 3012 continue; 3013 memset(&link, 0, sizeof(link)); 3014 ret = rte_eth_link_get_nowait(portid, &link); 3015 if (ret < 0) { 3016 all_ports_up = 0; 3017 if (print_flag == 1) 3018 printf("Port %u link get failed: %s\n", 3019 portid, rte_strerror(-ret)); 3020 continue; 3021 } 3022 /* print link status if flag set */ 3023 if (print_flag == 1) { 3024 if (link.link_status) 3025 printf( 3026 "Port%d Link Up. speed %u Mbps- %s\n", 3027 portid, link.link_speed, 3028 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 3029 ("full-duplex") : ("half-duplex")); 3030 else 3031 printf("Port %d Link Down\n", portid); 3032 continue; 3033 } 3034 /* clear all_ports_up flag if any link down */ 3035 if (link.link_status == ETH_LINK_DOWN) { 3036 all_ports_up = 0; 3037 break; 3038 } 3039 } 3040 /* after finally printing all link status, get out */ 3041 if (print_flag == 1) 3042 break; 3043 3044 if (all_ports_up == 0) { 3045 fflush(stdout); 3046 rte_delay_ms(CHECK_INTERVAL); 3047 } 3048 3049 /* set the print_flag if all ports up or timeout */ 3050 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3051 print_flag = 1; 3052 } 3053 3054 if (lsc_interrupt) 3055 break; 3056 } 3057 } 3058 3059 /* 3060 * This callback is for remove a port for a device. It has limitation because 3061 * it is not for multiple port removal for a device. 3062 * TODO: the device detach invoke will plan to be removed from user side to 3063 * eal. And convert all PMDs to free port resources on ether device closing. 3064 */ 3065 static void 3066 rmv_port_callback(void *arg) 3067 { 3068 int need_to_start = 0; 3069 int org_no_link_check = no_link_check; 3070 portid_t port_id = (intptr_t)arg; 3071 struct rte_device *dev; 3072 3073 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3074 3075 if (!test_done && port_is_forwarding(port_id)) { 3076 need_to_start = 1; 3077 stop_packet_forwarding(); 3078 } 3079 no_link_check = 1; 3080 stop_port(port_id); 3081 no_link_check = org_no_link_check; 3082 3083 /* Save rte_device pointer before closing ethdev port */ 3084 dev = rte_eth_devices[port_id].device; 3085 close_port(port_id); 3086 detach_device(dev); /* might be already removed or have more ports */ 3087 3088 if (need_to_start) 3089 start_packet_forwarding(0); 3090 } 3091 3092 /* This function is used by the interrupt thread */ 3093 static int 3094 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3095 void *ret_param) 3096 { 3097 RTE_SET_USED(param); 3098 RTE_SET_USED(ret_param); 3099 3100 if (type >= RTE_ETH_EVENT_MAX) { 3101 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3102 port_id, __func__, type); 3103 fflush(stderr); 3104 } else if (event_print_mask & (UINT32_C(1) << type)) { 3105 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3106 eth_event_desc[type]); 3107 fflush(stdout); 3108 } 3109 3110 switch (type) { 3111 case RTE_ETH_EVENT_NEW: 3112 ports[port_id].need_setup = 1; 3113 ports[port_id].port_status = RTE_PORT_HANDLING; 3114 break; 3115 case RTE_ETH_EVENT_INTR_RMV: 3116 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3117 break; 3118 if (rte_eal_alarm_set(100000, 3119 rmv_port_callback, (void *)(intptr_t)port_id)) 3120 fprintf(stderr, "Could not set up deferred device removal\n"); 3121 break; 3122 default: 3123 break; 3124 } 3125 return 0; 3126 } 3127 3128 static int 3129 register_eth_event_callback(void) 3130 { 3131 int ret; 3132 enum rte_eth_event_type event; 3133 3134 for (event = RTE_ETH_EVENT_UNKNOWN; 3135 event < RTE_ETH_EVENT_MAX; event++) { 3136 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3137 event, 3138 eth_event_callback, 3139 NULL); 3140 if (ret != 0) { 3141 TESTPMD_LOG(ERR, "Failed to register callback for " 3142 "%s event\n", eth_event_desc[event]); 3143 return -1; 3144 } 3145 } 3146 3147 return 0; 3148 } 3149 3150 /* This function is used by the interrupt thread */ 3151 static void 3152 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3153 __rte_unused void *arg) 3154 { 3155 uint16_t port_id; 3156 int ret; 3157 3158 if (type >= RTE_DEV_EVENT_MAX) { 3159 fprintf(stderr, "%s called upon invalid event %d\n", 3160 __func__, type); 3161 fflush(stderr); 3162 } 3163 3164 switch (type) { 3165 case RTE_DEV_EVENT_REMOVE: 3166 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3167 device_name); 3168 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3169 if (ret) { 3170 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3171 device_name); 3172 return; 3173 } 3174 /* 3175 * Because the user's callback is invoked in eal interrupt 3176 * callback, the interrupt callback need to be finished before 3177 * it can be unregistered when detaching device. So finish 3178 * callback soon and use a deferred removal to detach device 3179 * is need. It is a workaround, once the device detaching be 3180 * moved into the eal in the future, the deferred removal could 3181 * be deleted. 3182 */ 3183 if (rte_eal_alarm_set(100000, 3184 rmv_port_callback, (void *)(intptr_t)port_id)) 3185 RTE_LOG(ERR, EAL, 3186 "Could not set up deferred device removal\n"); 3187 break; 3188 case RTE_DEV_EVENT_ADD: 3189 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3190 device_name); 3191 /* TODO: After finish kernel driver binding, 3192 * begin to attach port. 3193 */ 3194 break; 3195 default: 3196 break; 3197 } 3198 } 3199 3200 static int 3201 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3202 { 3203 uint16_t i; 3204 int diag; 3205 uint8_t mapping_found = 0; 3206 3207 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3208 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3209 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3210 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3211 tx_queue_stats_mappings[i].queue_id, 3212 tx_queue_stats_mappings[i].stats_counter_id); 3213 if (diag != 0) 3214 return diag; 3215 mapping_found = 1; 3216 } 3217 } 3218 if (mapping_found) 3219 port->tx_queue_stats_mapping_enabled = 1; 3220 return 0; 3221 } 3222 3223 static int 3224 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3225 { 3226 uint16_t i; 3227 int diag; 3228 uint8_t mapping_found = 0; 3229 3230 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3231 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3232 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3233 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3234 rx_queue_stats_mappings[i].queue_id, 3235 rx_queue_stats_mappings[i].stats_counter_id); 3236 if (diag != 0) 3237 return diag; 3238 mapping_found = 1; 3239 } 3240 } 3241 if (mapping_found) 3242 port->rx_queue_stats_mapping_enabled = 1; 3243 return 0; 3244 } 3245 3246 static void 3247 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3248 { 3249 int diag = 0; 3250 3251 diag = set_tx_queue_stats_mapping_registers(pi, port); 3252 if (diag != 0) { 3253 if (diag == -ENOTSUP) { 3254 port->tx_queue_stats_mapping_enabled = 0; 3255 printf("TX queue stats mapping not supported port id=%d\n", pi); 3256 } 3257 else 3258 rte_exit(EXIT_FAILURE, 3259 "set_tx_queue_stats_mapping_registers " 3260 "failed for port id=%d diag=%d\n", 3261 pi, diag); 3262 } 3263 3264 diag = set_rx_queue_stats_mapping_registers(pi, port); 3265 if (diag != 0) { 3266 if (diag == -ENOTSUP) { 3267 port->rx_queue_stats_mapping_enabled = 0; 3268 printf("RX queue stats mapping not supported port id=%d\n", pi); 3269 } 3270 else 3271 rte_exit(EXIT_FAILURE, 3272 "set_rx_queue_stats_mapping_registers " 3273 "failed for port id=%d diag=%d\n", 3274 pi, diag); 3275 } 3276 } 3277 3278 static void 3279 rxtx_port_config(struct rte_port *port) 3280 { 3281 uint16_t qid; 3282 uint64_t offloads; 3283 3284 for (qid = 0; qid < nb_rxq; qid++) { 3285 offloads = port->rx_conf[qid].offloads; 3286 port->rx_conf[qid] = port->dev_info.default_rxconf; 3287 if (offloads != 0) 3288 port->rx_conf[qid].offloads = offloads; 3289 3290 /* Check if any Rx parameters have been passed */ 3291 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3292 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3293 3294 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3295 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3296 3297 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3298 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3299 3300 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3301 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3302 3303 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3304 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3305 3306 port->nb_rx_desc[qid] = nb_rxd; 3307 } 3308 3309 for (qid = 0; qid < nb_txq; qid++) { 3310 offloads = port->tx_conf[qid].offloads; 3311 port->tx_conf[qid] = port->dev_info.default_txconf; 3312 if (offloads != 0) 3313 port->tx_conf[qid].offloads = offloads; 3314 3315 /* Check if any Tx parameters have been passed */ 3316 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3317 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3318 3319 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3320 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3321 3322 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3323 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3324 3325 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3326 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3327 3328 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3329 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3330 3331 port->nb_tx_desc[qid] = nb_txd; 3332 } 3333 } 3334 3335 void 3336 init_port_config(void) 3337 { 3338 portid_t pid; 3339 struct rte_port *port; 3340 int ret; 3341 3342 RTE_ETH_FOREACH_DEV(pid) { 3343 port = &ports[pid]; 3344 port->dev_conf.fdir_conf = fdir_conf; 3345 3346 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3347 if (ret != 0) 3348 return; 3349 3350 if (nb_rxq > 1) { 3351 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3352 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3353 rss_hf & port->dev_info.flow_type_rss_offloads; 3354 } else { 3355 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3356 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3357 } 3358 3359 if (port->dcb_flag == 0) { 3360 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3361 port->dev_conf.rxmode.mq_mode = 3362 (enum rte_eth_rx_mq_mode) 3363 (rx_mq_mode & ETH_MQ_RX_RSS); 3364 else 3365 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3366 } 3367 3368 rxtx_port_config(port); 3369 3370 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3371 if (ret != 0) 3372 return; 3373 3374 map_port_queue_stats_mapping_registers(pid, port); 3375 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3376 rte_pmd_ixgbe_bypass_init(pid); 3377 #endif 3378 3379 if (lsc_interrupt && 3380 (rte_eth_devices[pid].data->dev_flags & 3381 RTE_ETH_DEV_INTR_LSC)) 3382 port->dev_conf.intr_conf.lsc = 1; 3383 if (rmv_interrupt && 3384 (rte_eth_devices[pid].data->dev_flags & 3385 RTE_ETH_DEV_INTR_RMV)) 3386 port->dev_conf.intr_conf.rmv = 1; 3387 } 3388 } 3389 3390 void set_port_slave_flag(portid_t slave_pid) 3391 { 3392 struct rte_port *port; 3393 3394 port = &ports[slave_pid]; 3395 port->slave_flag = 1; 3396 } 3397 3398 void clear_port_slave_flag(portid_t slave_pid) 3399 { 3400 struct rte_port *port; 3401 3402 port = &ports[slave_pid]; 3403 port->slave_flag = 0; 3404 } 3405 3406 uint8_t port_is_bonding_slave(portid_t slave_pid) 3407 { 3408 struct rte_port *port; 3409 3410 port = &ports[slave_pid]; 3411 if ((rte_eth_devices[slave_pid].data->dev_flags & 3412 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3413 return 1; 3414 return 0; 3415 } 3416 3417 const uint16_t vlan_tags[] = { 3418 0, 1, 2, 3, 4, 5, 6, 7, 3419 8, 9, 10, 11, 12, 13, 14, 15, 3420 16, 17, 18, 19, 20, 21, 22, 23, 3421 24, 25, 26, 27, 28, 29, 30, 31 3422 }; 3423 3424 static int 3425 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3426 enum dcb_mode_enable dcb_mode, 3427 enum rte_eth_nb_tcs num_tcs, 3428 uint8_t pfc_en) 3429 { 3430 uint8_t i; 3431 int32_t rc; 3432 struct rte_eth_rss_conf rss_conf; 3433 3434 /* 3435 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3436 * given above, and the number of traffic classes available for use. 3437 */ 3438 if (dcb_mode == DCB_VT_ENABLED) { 3439 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3440 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3441 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3442 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3443 3444 /* VMDQ+DCB RX and TX configurations */ 3445 vmdq_rx_conf->enable_default_pool = 0; 3446 vmdq_rx_conf->default_pool = 0; 3447 vmdq_rx_conf->nb_queue_pools = 3448 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3449 vmdq_tx_conf->nb_queue_pools = 3450 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3451 3452 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3453 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3454 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3455 vmdq_rx_conf->pool_map[i].pools = 3456 1 << (i % vmdq_rx_conf->nb_queue_pools); 3457 } 3458 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3459 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3460 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3461 } 3462 3463 /* set DCB mode of RX and TX of multiple queues */ 3464 eth_conf->rxmode.mq_mode = 3465 (enum rte_eth_rx_mq_mode) 3466 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3467 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3468 } else { 3469 struct rte_eth_dcb_rx_conf *rx_conf = 3470 ð_conf->rx_adv_conf.dcb_rx_conf; 3471 struct rte_eth_dcb_tx_conf *tx_conf = 3472 ð_conf->tx_adv_conf.dcb_tx_conf; 3473 3474 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3475 3476 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3477 if (rc != 0) 3478 return rc; 3479 3480 rx_conf->nb_tcs = num_tcs; 3481 tx_conf->nb_tcs = num_tcs; 3482 3483 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3484 rx_conf->dcb_tc[i] = i % num_tcs; 3485 tx_conf->dcb_tc[i] = i % num_tcs; 3486 } 3487 3488 eth_conf->rxmode.mq_mode = 3489 (enum rte_eth_rx_mq_mode) 3490 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3491 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3492 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3493 } 3494 3495 if (pfc_en) 3496 eth_conf->dcb_capability_en = 3497 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3498 else 3499 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3500 3501 return 0; 3502 } 3503 3504 int 3505 init_port_dcb_config(portid_t pid, 3506 enum dcb_mode_enable dcb_mode, 3507 enum rte_eth_nb_tcs num_tcs, 3508 uint8_t pfc_en) 3509 { 3510 struct rte_eth_conf port_conf; 3511 struct rte_port *rte_port; 3512 int retval; 3513 uint16_t i; 3514 3515 rte_port = &ports[pid]; 3516 3517 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3518 /* Enter DCB configuration status */ 3519 dcb_config = 1; 3520 3521 port_conf.rxmode = rte_port->dev_conf.rxmode; 3522 port_conf.txmode = rte_port->dev_conf.txmode; 3523 3524 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3525 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3526 if (retval < 0) 3527 return retval; 3528 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3529 3530 /* re-configure the device . */ 3531 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3532 if (retval < 0) 3533 return retval; 3534 3535 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3536 if (retval != 0) 3537 return retval; 3538 3539 /* If dev_info.vmdq_pool_base is greater than 0, 3540 * the queue id of vmdq pools is started after pf queues. 3541 */ 3542 if (dcb_mode == DCB_VT_ENABLED && 3543 rte_port->dev_info.vmdq_pool_base > 0) { 3544 printf("VMDQ_DCB multi-queue mode is nonsensical" 3545 " for port %d.", pid); 3546 return -1; 3547 } 3548 3549 /* Assume the ports in testpmd have the same dcb capability 3550 * and has the same number of rxq and txq in dcb mode 3551 */ 3552 if (dcb_mode == DCB_VT_ENABLED) { 3553 if (rte_port->dev_info.max_vfs > 0) { 3554 nb_rxq = rte_port->dev_info.nb_rx_queues; 3555 nb_txq = rte_port->dev_info.nb_tx_queues; 3556 } else { 3557 nb_rxq = rte_port->dev_info.max_rx_queues; 3558 nb_txq = rte_port->dev_info.max_tx_queues; 3559 } 3560 } else { 3561 /*if vt is disabled, use all pf queues */ 3562 if (rte_port->dev_info.vmdq_pool_base == 0) { 3563 nb_rxq = rte_port->dev_info.max_rx_queues; 3564 nb_txq = rte_port->dev_info.max_tx_queues; 3565 } else { 3566 nb_rxq = (queueid_t)num_tcs; 3567 nb_txq = (queueid_t)num_tcs; 3568 3569 } 3570 } 3571 rx_free_thresh = 64; 3572 3573 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3574 3575 rxtx_port_config(rte_port); 3576 /* VLAN filter */ 3577 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3578 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3579 rx_vft_set(pid, vlan_tags[i], 1); 3580 3581 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3582 if (retval != 0) 3583 return retval; 3584 3585 map_port_queue_stats_mapping_registers(pid, rte_port); 3586 3587 rte_port->dcb_flag = 1; 3588 3589 return 0; 3590 } 3591 3592 static void 3593 init_port(void) 3594 { 3595 /* Configuration of Ethernet ports. */ 3596 ports = rte_zmalloc("testpmd: ports", 3597 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3598 RTE_CACHE_LINE_SIZE); 3599 if (ports == NULL) { 3600 rte_exit(EXIT_FAILURE, 3601 "rte_zmalloc(%d struct rte_port) failed\n", 3602 RTE_MAX_ETHPORTS); 3603 } 3604 3605 /* Initialize ports NUMA structures */ 3606 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3607 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3608 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3609 } 3610 3611 static void 3612 force_quit(void) 3613 { 3614 pmd_test_exit(); 3615 prompt_exit(); 3616 } 3617 3618 static void 3619 print_stats(void) 3620 { 3621 uint8_t i; 3622 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3623 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3624 3625 /* Clear screen and move to top left */ 3626 printf("%s%s", clr, top_left); 3627 3628 printf("\nPort statistics ===================================="); 3629 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3630 nic_stats_display(fwd_ports_ids[i]); 3631 3632 fflush(stdout); 3633 } 3634 3635 static void 3636 signal_handler(int signum) 3637 { 3638 if (signum == SIGINT || signum == SIGTERM) { 3639 printf("\nSignal %d received, preparing to exit...\n", 3640 signum); 3641 #ifdef RTE_LIBRTE_PDUMP 3642 /* uninitialize packet capture framework */ 3643 rte_pdump_uninit(); 3644 #endif 3645 #ifdef RTE_LIBRTE_LATENCY_STATS 3646 if (latencystats_enabled != 0) 3647 rte_latencystats_uninit(); 3648 #endif 3649 force_quit(); 3650 /* Set flag to indicate the force termination. */ 3651 f_quit = 1; 3652 /* exit with the expected status */ 3653 signal(signum, SIG_DFL); 3654 kill(getpid(), signum); 3655 } 3656 } 3657 3658 int 3659 main(int argc, char** argv) 3660 { 3661 int diag; 3662 portid_t port_id; 3663 uint16_t count; 3664 int ret; 3665 3666 signal(SIGINT, signal_handler); 3667 signal(SIGTERM, signal_handler); 3668 3669 testpmd_logtype = rte_log_register("testpmd"); 3670 if (testpmd_logtype < 0) 3671 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3672 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3673 3674 diag = rte_eal_init(argc, argv); 3675 if (diag < 0) 3676 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3677 rte_strerror(rte_errno)); 3678 3679 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3680 rte_exit(EXIT_FAILURE, 3681 "Secondary process type not supported.\n"); 3682 3683 ret = register_eth_event_callback(); 3684 if (ret != 0) 3685 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3686 3687 #ifdef RTE_LIBRTE_PDUMP 3688 /* initialize packet capture framework */ 3689 rte_pdump_init(); 3690 #endif 3691 3692 count = 0; 3693 RTE_ETH_FOREACH_DEV(port_id) { 3694 ports_ids[count] = port_id; 3695 count++; 3696 } 3697 nb_ports = (portid_t) count; 3698 if (nb_ports == 0) 3699 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3700 3701 /* allocate port structures, and init them */ 3702 init_port(); 3703 3704 set_def_fwd_config(); 3705 if (nb_lcores == 0) 3706 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3707 "Check the core mask argument\n"); 3708 3709 /* Bitrate/latency stats disabled by default */ 3710 #ifdef RTE_LIBRTE_BITRATESTATS 3711 bitrate_enabled = 0; 3712 #endif 3713 #ifdef RTE_LIBRTE_LATENCY_STATS 3714 latencystats_enabled = 0; 3715 #endif 3716 3717 /* on FreeBSD, mlockall() is disabled by default */ 3718 #ifdef RTE_EXEC_ENV_FREEBSD 3719 do_mlockall = 0; 3720 #else 3721 do_mlockall = 1; 3722 #endif 3723 3724 argc -= diag; 3725 argv += diag; 3726 if (argc > 1) 3727 launch_args_parse(argc, argv); 3728 3729 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3730 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3731 strerror(errno)); 3732 } 3733 3734 if (tx_first && interactive) 3735 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3736 "interactive mode.\n"); 3737 3738 if (tx_first && lsc_interrupt) { 3739 printf("Warning: lsc_interrupt needs to be off when " 3740 " using tx_first. Disabling.\n"); 3741 lsc_interrupt = 0; 3742 } 3743 3744 if (!nb_rxq && !nb_txq) 3745 printf("Warning: Either rx or tx queues should be non-zero\n"); 3746 3747 if (nb_rxq > 1 && nb_rxq > nb_txq) 3748 printf("Warning: nb_rxq=%d enables RSS configuration, " 3749 "but nb_txq=%d will prevent to fully test it.\n", 3750 nb_rxq, nb_txq); 3751 3752 init_config(); 3753 3754 if (hot_plug) { 3755 ret = rte_dev_hotplug_handle_enable(); 3756 if (ret) { 3757 RTE_LOG(ERR, EAL, 3758 "fail to enable hotplug handling."); 3759 return -1; 3760 } 3761 3762 ret = rte_dev_event_monitor_start(); 3763 if (ret) { 3764 RTE_LOG(ERR, EAL, 3765 "fail to start device event monitoring."); 3766 return -1; 3767 } 3768 3769 ret = rte_dev_event_callback_register(NULL, 3770 dev_event_callback, NULL); 3771 if (ret) { 3772 RTE_LOG(ERR, EAL, 3773 "fail to register device event callback\n"); 3774 return -1; 3775 } 3776 } 3777 3778 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3779 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3780 3781 /* set all ports to promiscuous mode by default */ 3782 RTE_ETH_FOREACH_DEV(port_id) { 3783 ret = rte_eth_promiscuous_enable(port_id); 3784 if (ret != 0) 3785 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3786 port_id, rte_strerror(-ret)); 3787 } 3788 3789 /* Init metrics library */ 3790 rte_metrics_init(rte_socket_id()); 3791 3792 #ifdef RTE_LIBRTE_LATENCY_STATS 3793 if (latencystats_enabled != 0) { 3794 int ret = rte_latencystats_init(1, NULL); 3795 if (ret) 3796 printf("Warning: latencystats init()" 3797 " returned error %d\n", ret); 3798 printf("Latencystats running on lcore %d\n", 3799 latencystats_lcore_id); 3800 } 3801 #endif 3802 3803 /* Setup bitrate stats */ 3804 #ifdef RTE_LIBRTE_BITRATESTATS 3805 if (bitrate_enabled != 0) { 3806 bitrate_data = rte_stats_bitrate_create(); 3807 if (bitrate_data == NULL) 3808 rte_exit(EXIT_FAILURE, 3809 "Could not allocate bitrate data.\n"); 3810 rte_stats_bitrate_reg(bitrate_data); 3811 } 3812 #endif 3813 3814 #ifdef RTE_LIBRTE_CMDLINE 3815 if (strlen(cmdline_filename) != 0) 3816 cmdline_read_from_file(cmdline_filename); 3817 3818 if (interactive == 1) { 3819 if (auto_start) { 3820 printf("Start automatic packet forwarding\n"); 3821 start_packet_forwarding(0); 3822 } 3823 prompt(); 3824 pmd_test_exit(); 3825 } else 3826 #endif 3827 { 3828 char c; 3829 int rc; 3830 3831 f_quit = 0; 3832 3833 printf("No commandline core given, start packet forwarding\n"); 3834 start_packet_forwarding(tx_first); 3835 if (stats_period != 0) { 3836 uint64_t prev_time = 0, cur_time, diff_time = 0; 3837 uint64_t timer_period; 3838 3839 /* Convert to number of cycles */ 3840 timer_period = stats_period * rte_get_timer_hz(); 3841 3842 while (f_quit == 0) { 3843 cur_time = rte_get_timer_cycles(); 3844 diff_time += cur_time - prev_time; 3845 3846 if (diff_time >= timer_period) { 3847 print_stats(); 3848 /* Reset the timer */ 3849 diff_time = 0; 3850 } 3851 /* Sleep to avoid unnecessary checks */ 3852 prev_time = cur_time; 3853 sleep(1); 3854 } 3855 } 3856 3857 printf("Press enter to exit\n"); 3858 rc = read(0, &c, 1); 3859 pmd_test_exit(); 3860 if (rc < 0) 3861 return 1; 3862 } 3863 3864 ret = rte_eal_cleanup(); 3865 if (ret != 0) 3866 rte_exit(EXIT_FAILURE, 3867 "EAL cleanup failed: %s\n", strerror(-ret)); 3868 3869 return EXIT_SUCCESS; 3870 } 3871