1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATE 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 199 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 200 * specified on command-line. */ 201 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 202 203 /* 204 * In container, it cannot terminate the process which running with 'stats-period' 205 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 206 */ 207 uint8_t f_quit; 208 209 /* 210 * Configuration of packet segments used by the "txonly" processing engine. 211 */ 212 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 213 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 214 TXONLY_DEF_PACKET_LEN, 215 }; 216 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 217 218 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 219 /**< Split policy for packets to TX. */ 220 221 uint8_t txonly_multi_flow; 222 /**< Whether multiple flows are generated in TXONLY mode. */ 223 224 uint32_t tx_pkt_times_inter; 225 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 226 227 uint32_t tx_pkt_times_intra; 228 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 229 230 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 231 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 232 233 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 234 uint8_t dcb_config = 0; 235 236 /* Whether the dcb is in testing status */ 237 uint8_t dcb_test = 0; 238 239 /* 240 * Configurable number of RX/TX queues. 241 */ 242 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 243 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 244 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 245 246 /* 247 * Configurable number of RX/TX ring descriptors. 248 * Defaults are supplied by drivers via ethdev. 249 */ 250 #define RTE_TEST_RX_DESC_DEFAULT 0 251 #define RTE_TEST_TX_DESC_DEFAULT 0 252 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 253 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 254 255 #define RTE_PMD_PARAM_UNSET -1 256 /* 257 * Configurable values of RX and TX ring threshold registers. 258 */ 259 260 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 261 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 262 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 263 264 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 265 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 266 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX free threshold. 270 */ 271 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of RX drop enable. 275 */ 276 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX free threshold. 280 */ 281 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of TX RS bit threshold. 285 */ 286 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 287 288 /* 289 * Configurable value of buffered packets before sending. 290 */ 291 uint16_t noisy_tx_sw_bufsz; 292 293 /* 294 * Configurable value of packet buffer timeout. 295 */ 296 uint16_t noisy_tx_sw_buf_flush_time; 297 298 /* 299 * Configurable value for size of VNF internal memory area 300 * used for simulating noisy neighbour behaviour 301 */ 302 uint64_t noisy_lkup_mem_sz; 303 304 /* 305 * Configurable value of number of random writes done in 306 * VNF simulation memory area. 307 */ 308 uint64_t noisy_lkup_num_writes; 309 310 /* 311 * Configurable value of number of random reads done in 312 * VNF simulation memory area. 313 */ 314 uint64_t noisy_lkup_num_reads; 315 316 /* 317 * Configurable value of number of random reads/writes done in 318 * VNF simulation memory area. 319 */ 320 uint64_t noisy_lkup_num_reads_writes; 321 322 /* 323 * Receive Side Scaling (RSS) configuration. 324 */ 325 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 326 327 /* 328 * Port topology configuration 329 */ 330 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 331 332 /* 333 * Avoids to flush all the RX streams before starts forwarding. 334 */ 335 uint8_t no_flush_rx = 0; /* flush by default */ 336 337 /* 338 * Flow API isolated mode. 339 */ 340 uint8_t flow_isolate_all; 341 342 /* 343 * Avoids to check link status when starting/stopping a port. 344 */ 345 uint8_t no_link_check = 0; /* check by default */ 346 347 /* 348 * Don't automatically start all ports in interactive mode. 349 */ 350 uint8_t no_device_start = 0; 351 352 /* 353 * Enable link status change notification 354 */ 355 uint8_t lsc_interrupt = 1; /* enabled by default */ 356 357 /* 358 * Enable device removal notification. 359 */ 360 uint8_t rmv_interrupt = 1; /* enabled by default */ 361 362 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 363 364 /* After attach, port setup is called on event or by iterator */ 365 bool setup_on_probe_event = true; 366 367 /* Clear ptypes on port initialization. */ 368 uint8_t clear_ptypes = true; 369 370 /* Pretty printing of ethdev events */ 371 static const char * const eth_event_desc[] = { 372 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 373 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 374 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 375 [RTE_ETH_EVENT_INTR_RESET] = "reset", 376 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 377 [RTE_ETH_EVENT_IPSEC] = "IPsec", 378 [RTE_ETH_EVENT_MACSEC] = "MACsec", 379 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 380 [RTE_ETH_EVENT_NEW] = "device probed", 381 [RTE_ETH_EVENT_DESTROY] = "device released", 382 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 383 [RTE_ETH_EVENT_MAX] = NULL, 384 }; 385 386 /* 387 * Display or mask ether events 388 * Default to all events except VF_MBOX 389 */ 390 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 391 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 392 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 393 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 394 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 395 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 396 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 397 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 398 /* 399 * Decide if all memory are locked for performance. 400 */ 401 int do_mlockall = 0; 402 403 /* 404 * NIC bypass mode configuration options. 405 */ 406 407 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 408 /* The NIC bypass watchdog timeout. */ 409 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 410 #endif 411 412 413 #ifdef RTE_LIBRTE_LATENCY_STATS 414 415 /* 416 * Set when latency stats is enabled in the commandline 417 */ 418 uint8_t latencystats_enabled; 419 420 /* 421 * Lcore ID to serive latency statistics. 422 */ 423 lcoreid_t latencystats_lcore_id = -1; 424 425 #endif 426 427 /* 428 * Ethernet device configuration. 429 */ 430 struct rte_eth_rxmode rx_mode = { 431 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 432 /**< Default maximum frame length. */ 433 }; 434 435 struct rte_eth_txmode tx_mode = { 436 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 437 }; 438 439 struct rte_fdir_conf fdir_conf = { 440 .mode = RTE_FDIR_MODE_NONE, 441 .pballoc = RTE_FDIR_PBALLOC_64K, 442 .status = RTE_FDIR_REPORT_STATUS, 443 .mask = { 444 .vlan_tci_mask = 0xFFEF, 445 .ipv4_mask = { 446 .src_ip = 0xFFFFFFFF, 447 .dst_ip = 0xFFFFFFFF, 448 }, 449 .ipv6_mask = { 450 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 451 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 452 }, 453 .src_port_mask = 0xFFFF, 454 .dst_port_mask = 0xFFFF, 455 .mac_addr_byte_mask = 0xFF, 456 .tunnel_type_mask = 1, 457 .tunnel_id_mask = 0xFFFFFFFF, 458 }, 459 .drop_queue = 127, 460 }; 461 462 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 463 464 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 465 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 466 467 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 468 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 469 470 uint16_t nb_tx_queue_stats_mappings = 0; 471 uint16_t nb_rx_queue_stats_mappings = 0; 472 473 /* 474 * Display zero values by default for xstats 475 */ 476 uint8_t xstats_hide_zero; 477 478 /* 479 * Measure of CPU cycles disabled by default 480 */ 481 uint8_t record_core_cycles; 482 483 unsigned int num_sockets = 0; 484 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 485 486 #ifdef RTE_LIBRTE_BITRATE 487 /* Bitrate statistics */ 488 struct rte_stats_bitrates *bitrate_data; 489 lcoreid_t bitrate_lcore_id; 490 uint8_t bitrate_enabled; 491 #endif 492 493 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 494 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 495 496 /* 497 * hexadecimal bitmask of RX mq mode can be enabled. 498 */ 499 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 500 501 /* Forward function declarations */ 502 static void setup_attached_port(portid_t pi); 503 static void map_port_queue_stats_mapping_registers(portid_t pi, 504 struct rte_port *port); 505 static void check_all_ports_link_status(uint32_t port_mask); 506 static int eth_event_callback(portid_t port_id, 507 enum rte_eth_event_type type, 508 void *param, void *ret_param); 509 static void dev_event_callback(const char *device_name, 510 enum rte_dev_event_type type, 511 void *param); 512 513 /* 514 * Check if all the ports are started. 515 * If yes, return positive value. If not, return zero. 516 */ 517 static int all_ports_started(void); 518 519 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 520 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 521 522 /* Holds the registered mbuf dynamic flags names. */ 523 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 524 525 /* 526 * Helper function to check if socket is already discovered. 527 * If yes, return positive value. If not, return zero. 528 */ 529 int 530 new_socket_id(unsigned int socket_id) 531 { 532 unsigned int i; 533 534 for (i = 0; i < num_sockets; i++) { 535 if (socket_ids[i] == socket_id) 536 return 0; 537 } 538 return 1; 539 } 540 541 /* 542 * Setup default configuration. 543 */ 544 static void 545 set_default_fwd_lcores_config(void) 546 { 547 unsigned int i; 548 unsigned int nb_lc; 549 unsigned int sock_num; 550 551 nb_lc = 0; 552 for (i = 0; i < RTE_MAX_LCORE; i++) { 553 if (!rte_lcore_is_enabled(i)) 554 continue; 555 sock_num = rte_lcore_to_socket_id(i); 556 if (new_socket_id(sock_num)) { 557 if (num_sockets >= RTE_MAX_NUMA_NODES) { 558 rte_exit(EXIT_FAILURE, 559 "Total sockets greater than %u\n", 560 RTE_MAX_NUMA_NODES); 561 } 562 socket_ids[num_sockets++] = sock_num; 563 } 564 if (i == rte_get_master_lcore()) 565 continue; 566 fwd_lcores_cpuids[nb_lc++] = i; 567 } 568 nb_lcores = (lcoreid_t) nb_lc; 569 nb_cfg_lcores = nb_lcores; 570 nb_fwd_lcores = 1; 571 } 572 573 static void 574 set_def_peer_eth_addrs(void) 575 { 576 portid_t i; 577 578 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 579 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 580 peer_eth_addrs[i].addr_bytes[5] = i; 581 } 582 } 583 584 static void 585 set_default_fwd_ports_config(void) 586 { 587 portid_t pt_id; 588 int i = 0; 589 590 RTE_ETH_FOREACH_DEV(pt_id) { 591 fwd_ports_ids[i++] = pt_id; 592 593 /* Update sockets info according to the attached device */ 594 int socket_id = rte_eth_dev_socket_id(pt_id); 595 if (socket_id >= 0 && new_socket_id(socket_id)) { 596 if (num_sockets >= RTE_MAX_NUMA_NODES) { 597 rte_exit(EXIT_FAILURE, 598 "Total sockets greater than %u\n", 599 RTE_MAX_NUMA_NODES); 600 } 601 socket_ids[num_sockets++] = socket_id; 602 } 603 } 604 605 nb_cfg_ports = nb_ports; 606 nb_fwd_ports = nb_ports; 607 } 608 609 void 610 set_def_fwd_config(void) 611 { 612 set_default_fwd_lcores_config(); 613 set_def_peer_eth_addrs(); 614 set_default_fwd_ports_config(); 615 } 616 617 /* extremely pessimistic estimation of memory required to create a mempool */ 618 static int 619 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 620 { 621 unsigned int n_pages, mbuf_per_pg, leftover; 622 uint64_t total_mem, mbuf_mem, obj_sz; 623 624 /* there is no good way to predict how much space the mempool will 625 * occupy because it will allocate chunks on the fly, and some of those 626 * will come from default DPDK memory while some will come from our 627 * external memory, so just assume 128MB will be enough for everyone. 628 */ 629 uint64_t hdr_mem = 128 << 20; 630 631 /* account for possible non-contiguousness */ 632 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 633 if (obj_sz > pgsz) { 634 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 635 return -1; 636 } 637 638 mbuf_per_pg = pgsz / obj_sz; 639 leftover = (nb_mbufs % mbuf_per_pg) > 0; 640 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 641 642 mbuf_mem = n_pages * pgsz; 643 644 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 645 646 if (total_mem > SIZE_MAX) { 647 TESTPMD_LOG(ERR, "Memory size too big\n"); 648 return -1; 649 } 650 *out = (size_t)total_mem; 651 652 return 0; 653 } 654 655 static int 656 pagesz_flags(uint64_t page_sz) 657 { 658 /* as per mmap() manpage, all page sizes are log2 of page size 659 * shifted by MAP_HUGE_SHIFT 660 */ 661 int log2 = rte_log2_u64(page_sz); 662 663 return (log2 << HUGE_SHIFT); 664 } 665 666 static void * 667 alloc_mem(size_t memsz, size_t pgsz, bool huge) 668 { 669 void *addr; 670 int flags; 671 672 /* allocate anonymous hugepages */ 673 flags = MAP_ANONYMOUS | MAP_PRIVATE; 674 if (huge) 675 flags |= HUGE_FLAG | pagesz_flags(pgsz); 676 677 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 678 if (addr == MAP_FAILED) 679 return NULL; 680 681 return addr; 682 } 683 684 struct extmem_param { 685 void *addr; 686 size_t len; 687 size_t pgsz; 688 rte_iova_t *iova_table; 689 unsigned int iova_table_len; 690 }; 691 692 static int 693 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 694 bool huge) 695 { 696 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 697 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 698 unsigned int cur_page, n_pages, pgsz_idx; 699 size_t mem_sz, cur_pgsz; 700 rte_iova_t *iovas = NULL; 701 void *addr; 702 int ret; 703 704 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 705 /* skip anything that is too big */ 706 if (pgsizes[pgsz_idx] > SIZE_MAX) 707 continue; 708 709 cur_pgsz = pgsizes[pgsz_idx]; 710 711 /* if we were told not to allocate hugepages, override */ 712 if (!huge) 713 cur_pgsz = sysconf(_SC_PAGESIZE); 714 715 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 716 if (ret < 0) { 717 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 718 return -1; 719 } 720 721 /* allocate our memory */ 722 addr = alloc_mem(mem_sz, cur_pgsz, huge); 723 724 /* if we couldn't allocate memory with a specified page size, 725 * that doesn't mean we can't do it with other page sizes, so 726 * try another one. 727 */ 728 if (addr == NULL) 729 continue; 730 731 /* store IOVA addresses for every page in this memory area */ 732 n_pages = mem_sz / cur_pgsz; 733 734 iovas = malloc(sizeof(*iovas) * n_pages); 735 736 if (iovas == NULL) { 737 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 738 goto fail; 739 } 740 /* lock memory if it's not huge pages */ 741 if (!huge) 742 mlock(addr, mem_sz); 743 744 /* populate IOVA addresses */ 745 for (cur_page = 0; cur_page < n_pages; cur_page++) { 746 rte_iova_t iova; 747 size_t offset; 748 void *cur; 749 750 offset = cur_pgsz * cur_page; 751 cur = RTE_PTR_ADD(addr, offset); 752 753 /* touch the page before getting its IOVA */ 754 *(volatile char *)cur = 0; 755 756 iova = rte_mem_virt2iova(cur); 757 758 iovas[cur_page] = iova; 759 } 760 761 break; 762 } 763 /* if we couldn't allocate anything */ 764 if (iovas == NULL) 765 return -1; 766 767 param->addr = addr; 768 param->len = mem_sz; 769 param->pgsz = cur_pgsz; 770 param->iova_table = iovas; 771 param->iova_table_len = n_pages; 772 773 return 0; 774 fail: 775 if (iovas) 776 free(iovas); 777 if (addr) 778 munmap(addr, mem_sz); 779 780 return -1; 781 } 782 783 static int 784 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 785 { 786 struct extmem_param param; 787 int socket_id, ret; 788 789 memset(¶m, 0, sizeof(param)); 790 791 /* check if our heap exists */ 792 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 793 if (socket_id < 0) { 794 /* create our heap */ 795 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 796 if (ret < 0) { 797 TESTPMD_LOG(ERR, "Cannot create heap\n"); 798 return -1; 799 } 800 } 801 802 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 803 if (ret < 0) { 804 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 805 return -1; 806 } 807 808 /* we now have a valid memory area, so add it to heap */ 809 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 810 param.addr, param.len, param.iova_table, 811 param.iova_table_len, param.pgsz); 812 813 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 814 815 /* not needed any more */ 816 free(param.iova_table); 817 818 if (ret < 0) { 819 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 820 munmap(param.addr, param.len); 821 return -1; 822 } 823 824 /* success */ 825 826 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 827 param.len >> 20); 828 829 return 0; 830 } 831 static void 832 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 833 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 834 { 835 uint16_t pid = 0; 836 int ret; 837 838 RTE_ETH_FOREACH_DEV(pid) { 839 struct rte_eth_dev *dev = 840 &rte_eth_devices[pid]; 841 842 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 843 memhdr->len); 844 if (ret) { 845 TESTPMD_LOG(DEBUG, 846 "unable to DMA unmap addr 0x%p " 847 "for device %s\n", 848 memhdr->addr, dev->data->name); 849 } 850 } 851 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 852 if (ret) { 853 TESTPMD_LOG(DEBUG, 854 "unable to un-register addr 0x%p\n", memhdr->addr); 855 } 856 } 857 858 static void 859 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 860 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 861 { 862 uint16_t pid = 0; 863 size_t page_size = sysconf(_SC_PAGESIZE); 864 int ret; 865 866 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 867 page_size); 868 if (ret) { 869 TESTPMD_LOG(DEBUG, 870 "unable to register addr 0x%p\n", memhdr->addr); 871 return; 872 } 873 RTE_ETH_FOREACH_DEV(pid) { 874 struct rte_eth_dev *dev = 875 &rte_eth_devices[pid]; 876 877 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 878 memhdr->len); 879 if (ret) { 880 TESTPMD_LOG(DEBUG, 881 "unable to DMA map addr 0x%p " 882 "for device %s\n", 883 memhdr->addr, dev->data->name); 884 } 885 } 886 } 887 888 static unsigned int 889 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 890 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 891 { 892 struct rte_pktmbuf_extmem *xmem; 893 unsigned int ext_num, zone_num, elt_num; 894 uint16_t elt_size; 895 896 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 897 elt_num = EXTBUF_ZONE_SIZE / elt_size; 898 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 899 900 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 901 if (xmem == NULL) { 902 TESTPMD_LOG(ERR, "Cannot allocate memory for " 903 "external buffer descriptors\n"); 904 *ext_mem = NULL; 905 return 0; 906 } 907 for (ext_num = 0; ext_num < zone_num; ext_num++) { 908 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 909 const struct rte_memzone *mz; 910 char mz_name[RTE_MEMZONE_NAMESIZE]; 911 int ret; 912 913 ret = snprintf(mz_name, sizeof(mz_name), 914 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 915 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 916 errno = ENAMETOOLONG; 917 ext_num = 0; 918 break; 919 } 920 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 921 socket_id, 922 RTE_MEMZONE_IOVA_CONTIG | 923 RTE_MEMZONE_1GB | 924 RTE_MEMZONE_SIZE_HINT_ONLY, 925 EXTBUF_ZONE_SIZE); 926 if (mz == NULL) { 927 /* 928 * The caller exits on external buffer creation 929 * error, so there is no need to free memzones. 930 */ 931 errno = ENOMEM; 932 ext_num = 0; 933 break; 934 } 935 xseg->buf_ptr = mz->addr; 936 xseg->buf_iova = mz->iova; 937 xseg->buf_len = EXTBUF_ZONE_SIZE; 938 xseg->elt_size = elt_size; 939 } 940 if (ext_num == 0 && xmem != NULL) { 941 free(xmem); 942 xmem = NULL; 943 } 944 *ext_mem = xmem; 945 return ext_num; 946 } 947 948 /* 949 * Configuration initialisation done once at init time. 950 */ 951 static struct rte_mempool * 952 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 953 unsigned int socket_id) 954 { 955 char pool_name[RTE_MEMPOOL_NAMESIZE]; 956 struct rte_mempool *rte_mp = NULL; 957 uint32_t mb_size; 958 959 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 960 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 961 962 TESTPMD_LOG(INFO, 963 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 964 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 965 966 switch (mp_alloc_type) { 967 case MP_ALLOC_NATIVE: 968 { 969 /* wrapper to rte_mempool_create() */ 970 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 971 rte_mbuf_best_mempool_ops()); 972 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 973 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 974 break; 975 } 976 case MP_ALLOC_ANON: 977 { 978 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 979 mb_size, (unsigned int) mb_mempool_cache, 980 sizeof(struct rte_pktmbuf_pool_private), 981 socket_id, mempool_flags); 982 if (rte_mp == NULL) 983 goto err; 984 985 if (rte_mempool_populate_anon(rte_mp) == 0) { 986 rte_mempool_free(rte_mp); 987 rte_mp = NULL; 988 goto err; 989 } 990 rte_pktmbuf_pool_init(rte_mp, NULL); 991 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 992 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 993 break; 994 } 995 case MP_ALLOC_XMEM: 996 case MP_ALLOC_XMEM_HUGE: 997 { 998 int heap_socket; 999 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1000 1001 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1002 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1003 1004 heap_socket = 1005 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1006 if (heap_socket < 0) 1007 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1008 1009 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1010 rte_mbuf_best_mempool_ops()); 1011 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1012 mb_mempool_cache, 0, mbuf_seg_size, 1013 heap_socket); 1014 break; 1015 } 1016 case MP_ALLOC_XBUF: 1017 { 1018 struct rte_pktmbuf_extmem *ext_mem; 1019 unsigned int ext_num; 1020 1021 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1022 socket_id, pool_name, &ext_mem); 1023 if (ext_num == 0) 1024 rte_exit(EXIT_FAILURE, 1025 "Can't create pinned data buffers\n"); 1026 1027 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1028 rte_mbuf_best_mempool_ops()); 1029 rte_mp = rte_pktmbuf_pool_create_extbuf 1030 (pool_name, nb_mbuf, mb_mempool_cache, 1031 0, mbuf_seg_size, socket_id, 1032 ext_mem, ext_num); 1033 free(ext_mem); 1034 break; 1035 } 1036 default: 1037 { 1038 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1039 } 1040 } 1041 1042 err: 1043 if (rte_mp == NULL) { 1044 rte_exit(EXIT_FAILURE, 1045 "Creation of mbuf pool for socket %u failed: %s\n", 1046 socket_id, rte_strerror(rte_errno)); 1047 } else if (verbose_level > 0) { 1048 rte_mempool_dump(stdout, rte_mp); 1049 } 1050 return rte_mp; 1051 } 1052 1053 /* 1054 * Check given socket id is valid or not with NUMA mode, 1055 * if valid, return 0, else return -1 1056 */ 1057 static int 1058 check_socket_id(const unsigned int socket_id) 1059 { 1060 static int warning_once = 0; 1061 1062 if (new_socket_id(socket_id)) { 1063 if (!warning_once && numa_support) 1064 printf("Warning: NUMA should be configured manually by" 1065 " using --port-numa-config and" 1066 " --ring-numa-config parameters along with" 1067 " --numa.\n"); 1068 warning_once = 1; 1069 return -1; 1070 } 1071 return 0; 1072 } 1073 1074 /* 1075 * Get the allowed maximum number of RX queues. 1076 * *pid return the port id which has minimal value of 1077 * max_rx_queues in all ports. 1078 */ 1079 queueid_t 1080 get_allowed_max_nb_rxq(portid_t *pid) 1081 { 1082 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1083 bool max_rxq_valid = false; 1084 portid_t pi; 1085 struct rte_eth_dev_info dev_info; 1086 1087 RTE_ETH_FOREACH_DEV(pi) { 1088 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1089 continue; 1090 1091 max_rxq_valid = true; 1092 if (dev_info.max_rx_queues < allowed_max_rxq) { 1093 allowed_max_rxq = dev_info.max_rx_queues; 1094 *pid = pi; 1095 } 1096 } 1097 return max_rxq_valid ? allowed_max_rxq : 0; 1098 } 1099 1100 /* 1101 * Check input rxq is valid or not. 1102 * If input rxq is not greater than any of maximum number 1103 * of RX queues of all ports, it is valid. 1104 * if valid, return 0, else return -1 1105 */ 1106 int 1107 check_nb_rxq(queueid_t rxq) 1108 { 1109 queueid_t allowed_max_rxq; 1110 portid_t pid = 0; 1111 1112 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1113 if (rxq > allowed_max_rxq) { 1114 printf("Fail: input rxq (%u) can't be greater " 1115 "than max_rx_queues (%u) of port %u\n", 1116 rxq, 1117 allowed_max_rxq, 1118 pid); 1119 return -1; 1120 } 1121 return 0; 1122 } 1123 1124 /* 1125 * Get the allowed maximum number of TX queues. 1126 * *pid return the port id which has minimal value of 1127 * max_tx_queues in all ports. 1128 */ 1129 queueid_t 1130 get_allowed_max_nb_txq(portid_t *pid) 1131 { 1132 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1133 bool max_txq_valid = false; 1134 portid_t pi; 1135 struct rte_eth_dev_info dev_info; 1136 1137 RTE_ETH_FOREACH_DEV(pi) { 1138 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1139 continue; 1140 1141 max_txq_valid = true; 1142 if (dev_info.max_tx_queues < allowed_max_txq) { 1143 allowed_max_txq = dev_info.max_tx_queues; 1144 *pid = pi; 1145 } 1146 } 1147 return max_txq_valid ? allowed_max_txq : 0; 1148 } 1149 1150 /* 1151 * Check input txq is valid or not. 1152 * If input txq is not greater than any of maximum number 1153 * of TX queues of all ports, it is valid. 1154 * if valid, return 0, else return -1 1155 */ 1156 int 1157 check_nb_txq(queueid_t txq) 1158 { 1159 queueid_t allowed_max_txq; 1160 portid_t pid = 0; 1161 1162 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1163 if (txq > allowed_max_txq) { 1164 printf("Fail: input txq (%u) can't be greater " 1165 "than max_tx_queues (%u) of port %u\n", 1166 txq, 1167 allowed_max_txq, 1168 pid); 1169 return -1; 1170 } 1171 return 0; 1172 } 1173 1174 /* 1175 * Get the allowed maximum number of RXDs of every rx queue. 1176 * *pid return the port id which has minimal value of 1177 * max_rxd in all queues of all ports. 1178 */ 1179 static uint16_t 1180 get_allowed_max_nb_rxd(portid_t *pid) 1181 { 1182 uint16_t allowed_max_rxd = UINT16_MAX; 1183 portid_t pi; 1184 struct rte_eth_dev_info dev_info; 1185 1186 RTE_ETH_FOREACH_DEV(pi) { 1187 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1188 continue; 1189 1190 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1191 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1192 *pid = pi; 1193 } 1194 } 1195 return allowed_max_rxd; 1196 } 1197 1198 /* 1199 * Get the allowed minimal number of RXDs of every rx queue. 1200 * *pid return the port id which has minimal value of 1201 * min_rxd in all queues of all ports. 1202 */ 1203 static uint16_t 1204 get_allowed_min_nb_rxd(portid_t *pid) 1205 { 1206 uint16_t allowed_min_rxd = 0; 1207 portid_t pi; 1208 struct rte_eth_dev_info dev_info; 1209 1210 RTE_ETH_FOREACH_DEV(pi) { 1211 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1212 continue; 1213 1214 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1215 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1216 *pid = pi; 1217 } 1218 } 1219 1220 return allowed_min_rxd; 1221 } 1222 1223 /* 1224 * Check input rxd is valid or not. 1225 * If input rxd is not greater than any of maximum number 1226 * of RXDs of every Rx queues and is not less than any of 1227 * minimal number of RXDs of every Rx queues, it is valid. 1228 * if valid, return 0, else return -1 1229 */ 1230 int 1231 check_nb_rxd(queueid_t rxd) 1232 { 1233 uint16_t allowed_max_rxd; 1234 uint16_t allowed_min_rxd; 1235 portid_t pid = 0; 1236 1237 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1238 if (rxd > allowed_max_rxd) { 1239 printf("Fail: input rxd (%u) can't be greater " 1240 "than max_rxds (%u) of port %u\n", 1241 rxd, 1242 allowed_max_rxd, 1243 pid); 1244 return -1; 1245 } 1246 1247 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1248 if (rxd < allowed_min_rxd) { 1249 printf("Fail: input rxd (%u) can't be less " 1250 "than min_rxds (%u) of port %u\n", 1251 rxd, 1252 allowed_min_rxd, 1253 pid); 1254 return -1; 1255 } 1256 1257 return 0; 1258 } 1259 1260 /* 1261 * Get the allowed maximum number of TXDs of every rx queues. 1262 * *pid return the port id which has minimal value of 1263 * max_txd in every tx queue. 1264 */ 1265 static uint16_t 1266 get_allowed_max_nb_txd(portid_t *pid) 1267 { 1268 uint16_t allowed_max_txd = UINT16_MAX; 1269 portid_t pi; 1270 struct rte_eth_dev_info dev_info; 1271 1272 RTE_ETH_FOREACH_DEV(pi) { 1273 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1274 continue; 1275 1276 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1277 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1278 *pid = pi; 1279 } 1280 } 1281 return allowed_max_txd; 1282 } 1283 1284 /* 1285 * Get the allowed maximum number of TXDs of every tx queues. 1286 * *pid return the port id which has minimal value of 1287 * min_txd in every tx queue. 1288 */ 1289 static uint16_t 1290 get_allowed_min_nb_txd(portid_t *pid) 1291 { 1292 uint16_t allowed_min_txd = 0; 1293 portid_t pi; 1294 struct rte_eth_dev_info dev_info; 1295 1296 RTE_ETH_FOREACH_DEV(pi) { 1297 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1298 continue; 1299 1300 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1301 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1302 *pid = pi; 1303 } 1304 } 1305 1306 return allowed_min_txd; 1307 } 1308 1309 /* 1310 * Check input txd is valid or not. 1311 * If input txd is not greater than any of maximum number 1312 * of TXDs of every Rx queues, it is valid. 1313 * if valid, return 0, else return -1 1314 */ 1315 int 1316 check_nb_txd(queueid_t txd) 1317 { 1318 uint16_t allowed_max_txd; 1319 uint16_t allowed_min_txd; 1320 portid_t pid = 0; 1321 1322 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1323 if (txd > allowed_max_txd) { 1324 printf("Fail: input txd (%u) can't be greater " 1325 "than max_txds (%u) of port %u\n", 1326 txd, 1327 allowed_max_txd, 1328 pid); 1329 return -1; 1330 } 1331 1332 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1333 if (txd < allowed_min_txd) { 1334 printf("Fail: input txd (%u) can't be less " 1335 "than min_txds (%u) of port %u\n", 1336 txd, 1337 allowed_min_txd, 1338 pid); 1339 return -1; 1340 } 1341 return 0; 1342 } 1343 1344 1345 /* 1346 * Get the allowed maximum number of hairpin queues. 1347 * *pid return the port id which has minimal value of 1348 * max_hairpin_queues in all ports. 1349 */ 1350 queueid_t 1351 get_allowed_max_nb_hairpinq(portid_t *pid) 1352 { 1353 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1354 portid_t pi; 1355 struct rte_eth_hairpin_cap cap; 1356 1357 RTE_ETH_FOREACH_DEV(pi) { 1358 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1359 *pid = pi; 1360 return 0; 1361 } 1362 if (cap.max_nb_queues < allowed_max_hairpinq) { 1363 allowed_max_hairpinq = cap.max_nb_queues; 1364 *pid = pi; 1365 } 1366 } 1367 return allowed_max_hairpinq; 1368 } 1369 1370 /* 1371 * Check input hairpin is valid or not. 1372 * If input hairpin is not greater than any of maximum number 1373 * of hairpin queues of all ports, it is valid. 1374 * if valid, return 0, else return -1 1375 */ 1376 int 1377 check_nb_hairpinq(queueid_t hairpinq) 1378 { 1379 queueid_t allowed_max_hairpinq; 1380 portid_t pid = 0; 1381 1382 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1383 if (hairpinq > allowed_max_hairpinq) { 1384 printf("Fail: input hairpin (%u) can't be greater " 1385 "than max_hairpin_queues (%u) of port %u\n", 1386 hairpinq, allowed_max_hairpinq, pid); 1387 return -1; 1388 } 1389 return 0; 1390 } 1391 1392 static void 1393 init_config(void) 1394 { 1395 portid_t pid; 1396 struct rte_port *port; 1397 struct rte_mempool *mbp; 1398 unsigned int nb_mbuf_per_pool; 1399 lcoreid_t lc_id; 1400 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1401 struct rte_gro_param gro_param; 1402 uint32_t gso_types; 1403 uint16_t data_size; 1404 bool warning = 0; 1405 int k; 1406 int ret; 1407 1408 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1409 1410 /* Configuration of logical cores. */ 1411 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1412 sizeof(struct fwd_lcore *) * nb_lcores, 1413 RTE_CACHE_LINE_SIZE); 1414 if (fwd_lcores == NULL) { 1415 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1416 "failed\n", nb_lcores); 1417 } 1418 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1419 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1420 sizeof(struct fwd_lcore), 1421 RTE_CACHE_LINE_SIZE); 1422 if (fwd_lcores[lc_id] == NULL) { 1423 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1424 "failed\n"); 1425 } 1426 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1427 } 1428 1429 RTE_ETH_FOREACH_DEV(pid) { 1430 port = &ports[pid]; 1431 /* Apply default TxRx configuration for all ports */ 1432 port->dev_conf.txmode = tx_mode; 1433 port->dev_conf.rxmode = rx_mode; 1434 1435 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1436 if (ret != 0) 1437 rte_exit(EXIT_FAILURE, 1438 "rte_eth_dev_info_get() failed\n"); 1439 1440 if (!(port->dev_info.tx_offload_capa & 1441 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1442 port->dev_conf.txmode.offloads &= 1443 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1444 if (numa_support) { 1445 if (port_numa[pid] != NUMA_NO_CONFIG) 1446 port_per_socket[port_numa[pid]]++; 1447 else { 1448 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1449 1450 /* 1451 * if socket_id is invalid, 1452 * set to the first available socket. 1453 */ 1454 if (check_socket_id(socket_id) < 0) 1455 socket_id = socket_ids[0]; 1456 port_per_socket[socket_id]++; 1457 } 1458 } 1459 1460 /* Apply Rx offloads configuration */ 1461 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1462 port->rx_conf[k].offloads = 1463 port->dev_conf.rxmode.offloads; 1464 /* Apply Tx offloads configuration */ 1465 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1466 port->tx_conf[k].offloads = 1467 port->dev_conf.txmode.offloads; 1468 1469 /* set flag to initialize port/queue */ 1470 port->need_reconfig = 1; 1471 port->need_reconfig_queues = 1; 1472 port->tx_metadata = 0; 1473 1474 /* Check for maximum number of segments per MTU. Accordingly 1475 * update the mbuf data size. 1476 */ 1477 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1478 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1479 data_size = rx_mode.max_rx_pkt_len / 1480 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1481 1482 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1483 mbuf_data_size) { 1484 mbuf_data_size = data_size + 1485 RTE_PKTMBUF_HEADROOM; 1486 warning = 1; 1487 } 1488 } 1489 } 1490 1491 if (warning) 1492 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1493 mbuf_data_size); 1494 1495 /* 1496 * Create pools of mbuf. 1497 * If NUMA support is disabled, create a single pool of mbuf in 1498 * socket 0 memory by default. 1499 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1500 * 1501 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1502 * nb_txd can be configured at run time. 1503 */ 1504 if (param_total_num_mbufs) 1505 nb_mbuf_per_pool = param_total_num_mbufs; 1506 else { 1507 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1508 (nb_lcores * mb_mempool_cache) + 1509 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1510 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1511 } 1512 1513 if (numa_support) { 1514 uint8_t i; 1515 1516 for (i = 0; i < num_sockets; i++) 1517 mempools[i] = mbuf_pool_create(mbuf_data_size, 1518 nb_mbuf_per_pool, 1519 socket_ids[i]); 1520 } else { 1521 if (socket_num == UMA_NO_CONFIG) 1522 mempools[0] = mbuf_pool_create(mbuf_data_size, 1523 nb_mbuf_per_pool, 0); 1524 else 1525 mempools[socket_num] = mbuf_pool_create 1526 (mbuf_data_size, 1527 nb_mbuf_per_pool, 1528 socket_num); 1529 } 1530 1531 init_port_config(); 1532 1533 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1534 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1535 /* 1536 * Records which Mbuf pool to use by each logical core, if needed. 1537 */ 1538 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1539 mbp = mbuf_pool_find( 1540 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1541 1542 if (mbp == NULL) 1543 mbp = mbuf_pool_find(0); 1544 fwd_lcores[lc_id]->mbp = mbp; 1545 /* initialize GSO context */ 1546 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1547 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1548 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1549 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1550 RTE_ETHER_CRC_LEN; 1551 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1552 } 1553 1554 /* Configuration of packet forwarding streams. */ 1555 if (init_fwd_streams() < 0) 1556 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1557 1558 fwd_config_setup(); 1559 1560 /* create a gro context for each lcore */ 1561 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1562 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1563 gro_param.max_item_per_flow = MAX_PKT_BURST; 1564 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1565 gro_param.socket_id = rte_lcore_to_socket_id( 1566 fwd_lcores_cpuids[lc_id]); 1567 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1568 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1569 rte_exit(EXIT_FAILURE, 1570 "rte_gro_ctx_create() failed\n"); 1571 } 1572 } 1573 } 1574 1575 1576 void 1577 reconfig(portid_t new_port_id, unsigned socket_id) 1578 { 1579 struct rte_port *port; 1580 int ret; 1581 1582 /* Reconfiguration of Ethernet ports. */ 1583 port = &ports[new_port_id]; 1584 1585 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1586 if (ret != 0) 1587 return; 1588 1589 /* set flag to initialize port/queue */ 1590 port->need_reconfig = 1; 1591 port->need_reconfig_queues = 1; 1592 port->socket_id = socket_id; 1593 1594 init_port_config(); 1595 } 1596 1597 1598 int 1599 init_fwd_streams(void) 1600 { 1601 portid_t pid; 1602 struct rte_port *port; 1603 streamid_t sm_id, nb_fwd_streams_new; 1604 queueid_t q; 1605 1606 /* set socket id according to numa or not */ 1607 RTE_ETH_FOREACH_DEV(pid) { 1608 port = &ports[pid]; 1609 if (nb_rxq > port->dev_info.max_rx_queues) { 1610 printf("Fail: nb_rxq(%d) is greater than " 1611 "max_rx_queues(%d)\n", nb_rxq, 1612 port->dev_info.max_rx_queues); 1613 return -1; 1614 } 1615 if (nb_txq > port->dev_info.max_tx_queues) { 1616 printf("Fail: nb_txq(%d) is greater than " 1617 "max_tx_queues(%d)\n", nb_txq, 1618 port->dev_info.max_tx_queues); 1619 return -1; 1620 } 1621 if (numa_support) { 1622 if (port_numa[pid] != NUMA_NO_CONFIG) 1623 port->socket_id = port_numa[pid]; 1624 else { 1625 port->socket_id = rte_eth_dev_socket_id(pid); 1626 1627 /* 1628 * if socket_id is invalid, 1629 * set to the first available socket. 1630 */ 1631 if (check_socket_id(port->socket_id) < 0) 1632 port->socket_id = socket_ids[0]; 1633 } 1634 } 1635 else { 1636 if (socket_num == UMA_NO_CONFIG) 1637 port->socket_id = 0; 1638 else 1639 port->socket_id = socket_num; 1640 } 1641 } 1642 1643 q = RTE_MAX(nb_rxq, nb_txq); 1644 if (q == 0) { 1645 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1646 return -1; 1647 } 1648 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1649 if (nb_fwd_streams_new == nb_fwd_streams) 1650 return 0; 1651 /* clear the old */ 1652 if (fwd_streams != NULL) { 1653 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1654 if (fwd_streams[sm_id] == NULL) 1655 continue; 1656 rte_free(fwd_streams[sm_id]); 1657 fwd_streams[sm_id] = NULL; 1658 } 1659 rte_free(fwd_streams); 1660 fwd_streams = NULL; 1661 } 1662 1663 /* init new */ 1664 nb_fwd_streams = nb_fwd_streams_new; 1665 if (nb_fwd_streams) { 1666 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1667 sizeof(struct fwd_stream *) * nb_fwd_streams, 1668 RTE_CACHE_LINE_SIZE); 1669 if (fwd_streams == NULL) 1670 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1671 " (struct fwd_stream *)) failed\n", 1672 nb_fwd_streams); 1673 1674 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1675 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1676 " struct fwd_stream", sizeof(struct fwd_stream), 1677 RTE_CACHE_LINE_SIZE); 1678 if (fwd_streams[sm_id] == NULL) 1679 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1680 "(struct fwd_stream) failed\n"); 1681 } 1682 } 1683 1684 return 0; 1685 } 1686 1687 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1688 static void 1689 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1690 { 1691 uint64_t total_burst, sburst; 1692 uint64_t nb_burst; 1693 uint64_t burst_stats[4]; 1694 uint16_t pktnb_stats[4]; 1695 uint16_t nb_pkt; 1696 int burst_percent[4], sburstp; 1697 int i; 1698 1699 /* 1700 * First compute the total number of packet bursts and the 1701 * two highest numbers of bursts of the same number of packets. 1702 */ 1703 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1704 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1705 1706 /* Show stats for 0 burst size always */ 1707 total_burst = pbs->pkt_burst_spread[0]; 1708 burst_stats[0] = pbs->pkt_burst_spread[0]; 1709 pktnb_stats[0] = 0; 1710 1711 /* Find the next 2 burst sizes with highest occurrences. */ 1712 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1713 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1714 1715 if (nb_burst == 0) 1716 continue; 1717 1718 total_burst += nb_burst; 1719 1720 if (nb_burst > burst_stats[1]) { 1721 burst_stats[2] = burst_stats[1]; 1722 pktnb_stats[2] = pktnb_stats[1]; 1723 burst_stats[1] = nb_burst; 1724 pktnb_stats[1] = nb_pkt; 1725 } else if (nb_burst > burst_stats[2]) { 1726 burst_stats[2] = nb_burst; 1727 pktnb_stats[2] = nb_pkt; 1728 } 1729 } 1730 if (total_burst == 0) 1731 return; 1732 1733 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1734 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1735 if (i == 3) { 1736 printf("%d%% of other]\n", 100 - sburstp); 1737 return; 1738 } 1739 1740 sburst += burst_stats[i]; 1741 if (sburst == total_burst) { 1742 printf("%d%% of %d pkts]\n", 1743 100 - sburstp, (int) pktnb_stats[i]); 1744 return; 1745 } 1746 1747 burst_percent[i] = 1748 (double)burst_stats[i] / total_burst * 100; 1749 printf("%d%% of %d pkts + ", 1750 burst_percent[i], (int) pktnb_stats[i]); 1751 sburstp += burst_percent[i]; 1752 } 1753 } 1754 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1755 1756 static void 1757 fwd_stream_stats_display(streamid_t stream_id) 1758 { 1759 struct fwd_stream *fs; 1760 static const char *fwd_top_stats_border = "-------"; 1761 1762 fs = fwd_streams[stream_id]; 1763 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1764 (fs->fwd_dropped == 0)) 1765 return; 1766 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1767 "TX Port=%2d/Queue=%2d %s\n", 1768 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1769 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1770 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1771 " TX-dropped: %-14"PRIu64, 1772 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1773 1774 /* if checksum mode */ 1775 if (cur_fwd_eng == &csum_fwd_engine) { 1776 printf(" RX- bad IP checksum: %-14"PRIu64 1777 " Rx- bad L4 checksum: %-14"PRIu64 1778 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1779 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1780 fs->rx_bad_outer_l4_csum); 1781 } else { 1782 printf("\n"); 1783 } 1784 1785 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1786 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1787 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1788 #endif 1789 } 1790 1791 void 1792 fwd_stats_display(void) 1793 { 1794 static const char *fwd_stats_border = "----------------------"; 1795 static const char *acc_stats_border = "+++++++++++++++"; 1796 struct { 1797 struct fwd_stream *rx_stream; 1798 struct fwd_stream *tx_stream; 1799 uint64_t tx_dropped; 1800 uint64_t rx_bad_ip_csum; 1801 uint64_t rx_bad_l4_csum; 1802 uint64_t rx_bad_outer_l4_csum; 1803 } ports_stats[RTE_MAX_ETHPORTS]; 1804 uint64_t total_rx_dropped = 0; 1805 uint64_t total_tx_dropped = 0; 1806 uint64_t total_rx_nombuf = 0; 1807 struct rte_eth_stats stats; 1808 uint64_t fwd_cycles = 0; 1809 uint64_t total_recv = 0; 1810 uint64_t total_xmit = 0; 1811 struct rte_port *port; 1812 streamid_t sm_id; 1813 portid_t pt_id; 1814 int i; 1815 1816 memset(ports_stats, 0, sizeof(ports_stats)); 1817 1818 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1819 struct fwd_stream *fs = fwd_streams[sm_id]; 1820 1821 if (cur_fwd_config.nb_fwd_streams > 1822 cur_fwd_config.nb_fwd_ports) { 1823 fwd_stream_stats_display(sm_id); 1824 } else { 1825 ports_stats[fs->tx_port].tx_stream = fs; 1826 ports_stats[fs->rx_port].rx_stream = fs; 1827 } 1828 1829 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1830 1831 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1832 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1833 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1834 fs->rx_bad_outer_l4_csum; 1835 1836 if (record_core_cycles) 1837 fwd_cycles += fs->core_cycles; 1838 } 1839 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1840 uint8_t j; 1841 1842 pt_id = fwd_ports_ids[i]; 1843 port = &ports[pt_id]; 1844 1845 rte_eth_stats_get(pt_id, &stats); 1846 stats.ipackets -= port->stats.ipackets; 1847 stats.opackets -= port->stats.opackets; 1848 stats.ibytes -= port->stats.ibytes; 1849 stats.obytes -= port->stats.obytes; 1850 stats.imissed -= port->stats.imissed; 1851 stats.oerrors -= port->stats.oerrors; 1852 stats.rx_nombuf -= port->stats.rx_nombuf; 1853 1854 total_recv += stats.ipackets; 1855 total_xmit += stats.opackets; 1856 total_rx_dropped += stats.imissed; 1857 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1858 total_tx_dropped += stats.oerrors; 1859 total_rx_nombuf += stats.rx_nombuf; 1860 1861 printf("\n %s Forward statistics for port %-2d %s\n", 1862 fwd_stats_border, pt_id, fwd_stats_border); 1863 1864 if (!port->rx_queue_stats_mapping_enabled && 1865 !port->tx_queue_stats_mapping_enabled) { 1866 printf(" RX-packets: %-14"PRIu64 1867 " RX-dropped: %-14"PRIu64 1868 "RX-total: %-"PRIu64"\n", 1869 stats.ipackets, stats.imissed, 1870 stats.ipackets + stats.imissed); 1871 1872 if (cur_fwd_eng == &csum_fwd_engine) 1873 printf(" Bad-ipcsum: %-14"PRIu64 1874 " Bad-l4csum: %-14"PRIu64 1875 "Bad-outer-l4csum: %-14"PRIu64"\n", 1876 ports_stats[pt_id].rx_bad_ip_csum, 1877 ports_stats[pt_id].rx_bad_l4_csum, 1878 ports_stats[pt_id].rx_bad_outer_l4_csum); 1879 if (stats.ierrors + stats.rx_nombuf > 0) { 1880 printf(" RX-error: %-"PRIu64"\n", 1881 stats.ierrors); 1882 printf(" RX-nombufs: %-14"PRIu64"\n", 1883 stats.rx_nombuf); 1884 } 1885 1886 printf(" TX-packets: %-14"PRIu64 1887 " TX-dropped: %-14"PRIu64 1888 "TX-total: %-"PRIu64"\n", 1889 stats.opackets, ports_stats[pt_id].tx_dropped, 1890 stats.opackets + ports_stats[pt_id].tx_dropped); 1891 } else { 1892 printf(" RX-packets: %14"PRIu64 1893 " RX-dropped:%14"PRIu64 1894 " RX-total:%14"PRIu64"\n", 1895 stats.ipackets, stats.imissed, 1896 stats.ipackets + stats.imissed); 1897 1898 if (cur_fwd_eng == &csum_fwd_engine) 1899 printf(" Bad-ipcsum:%14"PRIu64 1900 " Bad-l4csum:%14"PRIu64 1901 " Bad-outer-l4csum: %-14"PRIu64"\n", 1902 ports_stats[pt_id].rx_bad_ip_csum, 1903 ports_stats[pt_id].rx_bad_l4_csum, 1904 ports_stats[pt_id].rx_bad_outer_l4_csum); 1905 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1906 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1907 printf(" RX-nombufs: %14"PRIu64"\n", 1908 stats.rx_nombuf); 1909 } 1910 1911 printf(" TX-packets: %14"PRIu64 1912 " TX-dropped:%14"PRIu64 1913 " TX-total:%14"PRIu64"\n", 1914 stats.opackets, ports_stats[pt_id].tx_dropped, 1915 stats.opackets + ports_stats[pt_id].tx_dropped); 1916 } 1917 1918 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1919 if (ports_stats[pt_id].rx_stream) 1920 pkt_burst_stats_display("RX", 1921 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1922 if (ports_stats[pt_id].tx_stream) 1923 pkt_burst_stats_display("TX", 1924 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1925 #endif 1926 1927 if (port->rx_queue_stats_mapping_enabled) { 1928 printf("\n"); 1929 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1930 printf(" Stats reg %2d RX-packets:%14"PRIu64 1931 " RX-errors:%14"PRIu64 1932 " RX-bytes:%14"PRIu64"\n", 1933 j, stats.q_ipackets[j], 1934 stats.q_errors[j], stats.q_ibytes[j]); 1935 } 1936 printf("\n"); 1937 } 1938 if (port->tx_queue_stats_mapping_enabled) { 1939 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1940 printf(" Stats reg %2d TX-packets:%14"PRIu64 1941 " TX-bytes:%14" 1942 PRIu64"\n", 1943 j, stats.q_opackets[j], 1944 stats.q_obytes[j]); 1945 } 1946 } 1947 1948 printf(" %s--------------------------------%s\n", 1949 fwd_stats_border, fwd_stats_border); 1950 } 1951 1952 printf("\n %s Accumulated forward statistics for all ports" 1953 "%s\n", 1954 acc_stats_border, acc_stats_border); 1955 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1956 "%-"PRIu64"\n" 1957 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1958 "%-"PRIu64"\n", 1959 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1960 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1961 if (total_rx_nombuf > 0) 1962 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1963 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1964 "%s\n", 1965 acc_stats_border, acc_stats_border); 1966 if (record_core_cycles) { 1967 #define CYC_PER_MHZ 1E6 1968 if (total_recv > 0 || total_xmit > 0) { 1969 uint64_t total_pkts = 0; 1970 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1971 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1972 total_pkts = total_xmit; 1973 else 1974 total_pkts = total_recv; 1975 1976 printf("\n CPU cycles/packet=%.2F (total cycles=" 1977 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1978 " MHz Clock\n", 1979 (double) fwd_cycles / total_pkts, 1980 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1981 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1982 } 1983 } 1984 } 1985 1986 void 1987 fwd_stats_reset(void) 1988 { 1989 streamid_t sm_id; 1990 portid_t pt_id; 1991 int i; 1992 1993 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1994 pt_id = fwd_ports_ids[i]; 1995 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1996 } 1997 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1998 struct fwd_stream *fs = fwd_streams[sm_id]; 1999 2000 fs->rx_packets = 0; 2001 fs->tx_packets = 0; 2002 fs->fwd_dropped = 0; 2003 fs->rx_bad_ip_csum = 0; 2004 fs->rx_bad_l4_csum = 0; 2005 fs->rx_bad_outer_l4_csum = 0; 2006 2007 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 2008 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2009 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2010 #endif 2011 fs->core_cycles = 0; 2012 } 2013 } 2014 2015 static void 2016 flush_fwd_rx_queues(void) 2017 { 2018 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2019 portid_t rxp; 2020 portid_t port_id; 2021 queueid_t rxq; 2022 uint16_t nb_rx; 2023 uint16_t i; 2024 uint8_t j; 2025 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2026 uint64_t timer_period; 2027 2028 /* convert to number of cycles */ 2029 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2030 2031 for (j = 0; j < 2; j++) { 2032 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2033 for (rxq = 0; rxq < nb_rxq; rxq++) { 2034 port_id = fwd_ports_ids[rxp]; 2035 /** 2036 * testpmd can stuck in the below do while loop 2037 * if rte_eth_rx_burst() always returns nonzero 2038 * packets. So timer is added to exit this loop 2039 * after 1sec timer expiry. 2040 */ 2041 prev_tsc = rte_rdtsc(); 2042 do { 2043 nb_rx = rte_eth_rx_burst(port_id, rxq, 2044 pkts_burst, MAX_PKT_BURST); 2045 for (i = 0; i < nb_rx; i++) 2046 rte_pktmbuf_free(pkts_burst[i]); 2047 2048 cur_tsc = rte_rdtsc(); 2049 diff_tsc = cur_tsc - prev_tsc; 2050 timer_tsc += diff_tsc; 2051 } while ((nb_rx > 0) && 2052 (timer_tsc < timer_period)); 2053 timer_tsc = 0; 2054 } 2055 } 2056 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2057 } 2058 } 2059 2060 static void 2061 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2062 { 2063 struct fwd_stream **fsm; 2064 streamid_t nb_fs; 2065 streamid_t sm_id; 2066 #ifdef RTE_LIBRTE_BITRATE 2067 uint64_t tics_per_1sec; 2068 uint64_t tics_datum; 2069 uint64_t tics_current; 2070 uint16_t i, cnt_ports; 2071 2072 cnt_ports = nb_ports; 2073 tics_datum = rte_rdtsc(); 2074 tics_per_1sec = rte_get_timer_hz(); 2075 #endif 2076 fsm = &fwd_streams[fc->stream_idx]; 2077 nb_fs = fc->stream_nb; 2078 do { 2079 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2080 (*pkt_fwd)(fsm[sm_id]); 2081 #ifdef RTE_LIBRTE_BITRATE 2082 if (bitrate_enabled != 0 && 2083 bitrate_lcore_id == rte_lcore_id()) { 2084 tics_current = rte_rdtsc(); 2085 if (tics_current - tics_datum >= tics_per_1sec) { 2086 /* Periodic bitrate calculation */ 2087 for (i = 0; i < cnt_ports; i++) 2088 rte_stats_bitrate_calc(bitrate_data, 2089 ports_ids[i]); 2090 tics_datum = tics_current; 2091 } 2092 } 2093 #endif 2094 #ifdef RTE_LIBRTE_LATENCY_STATS 2095 if (latencystats_enabled != 0 && 2096 latencystats_lcore_id == rte_lcore_id()) 2097 rte_latencystats_update(); 2098 #endif 2099 2100 } while (! fc->stopped); 2101 } 2102 2103 static int 2104 start_pkt_forward_on_core(void *fwd_arg) 2105 { 2106 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2107 cur_fwd_config.fwd_eng->packet_fwd); 2108 return 0; 2109 } 2110 2111 /* 2112 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2113 * Used to start communication flows in network loopback test configurations. 2114 */ 2115 static int 2116 run_one_txonly_burst_on_core(void *fwd_arg) 2117 { 2118 struct fwd_lcore *fwd_lc; 2119 struct fwd_lcore tmp_lcore; 2120 2121 fwd_lc = (struct fwd_lcore *) fwd_arg; 2122 tmp_lcore = *fwd_lc; 2123 tmp_lcore.stopped = 1; 2124 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2125 return 0; 2126 } 2127 2128 /* 2129 * Launch packet forwarding: 2130 * - Setup per-port forwarding context. 2131 * - launch logical cores with their forwarding configuration. 2132 */ 2133 static void 2134 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2135 { 2136 port_fwd_begin_t port_fwd_begin; 2137 unsigned int i; 2138 unsigned int lc_id; 2139 int diag; 2140 2141 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2142 if (port_fwd_begin != NULL) { 2143 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2144 (*port_fwd_begin)(fwd_ports_ids[i]); 2145 } 2146 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2147 lc_id = fwd_lcores_cpuids[i]; 2148 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2149 fwd_lcores[i]->stopped = 0; 2150 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2151 fwd_lcores[i], lc_id); 2152 if (diag != 0) 2153 printf("launch lcore %u failed - diag=%d\n", 2154 lc_id, diag); 2155 } 2156 } 2157 } 2158 2159 /* 2160 * Launch packet forwarding configuration. 2161 */ 2162 void 2163 start_packet_forwarding(int with_tx_first) 2164 { 2165 port_fwd_begin_t port_fwd_begin; 2166 port_fwd_end_t port_fwd_end; 2167 struct rte_port *port; 2168 unsigned int i; 2169 portid_t pt_id; 2170 2171 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2172 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2173 2174 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2175 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2176 2177 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2178 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2179 (!nb_rxq || !nb_txq)) 2180 rte_exit(EXIT_FAILURE, 2181 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2182 cur_fwd_eng->fwd_mode_name); 2183 2184 if (all_ports_started() == 0) { 2185 printf("Not all ports were started\n"); 2186 return; 2187 } 2188 if (test_done == 0) { 2189 printf("Packet forwarding already started\n"); 2190 return; 2191 } 2192 2193 2194 if(dcb_test) { 2195 for (i = 0; i < nb_fwd_ports; i++) { 2196 pt_id = fwd_ports_ids[i]; 2197 port = &ports[pt_id]; 2198 if (!port->dcb_flag) { 2199 printf("In DCB mode, all forwarding ports must " 2200 "be configured in this mode.\n"); 2201 return; 2202 } 2203 } 2204 if (nb_fwd_lcores == 1) { 2205 printf("In DCB mode,the nb forwarding cores " 2206 "should be larger than 1.\n"); 2207 return; 2208 } 2209 } 2210 test_done = 0; 2211 2212 fwd_config_setup(); 2213 2214 if(!no_flush_rx) 2215 flush_fwd_rx_queues(); 2216 2217 pkt_fwd_config_display(&cur_fwd_config); 2218 rxtx_config_display(); 2219 2220 fwd_stats_reset(); 2221 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2222 pt_id = fwd_ports_ids[i]; 2223 port = &ports[pt_id]; 2224 map_port_queue_stats_mapping_registers(pt_id, port); 2225 } 2226 if (with_tx_first) { 2227 port_fwd_begin = tx_only_engine.port_fwd_begin; 2228 if (port_fwd_begin != NULL) { 2229 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2230 (*port_fwd_begin)(fwd_ports_ids[i]); 2231 } 2232 while (with_tx_first--) { 2233 launch_packet_forwarding( 2234 run_one_txonly_burst_on_core); 2235 rte_eal_mp_wait_lcore(); 2236 } 2237 port_fwd_end = tx_only_engine.port_fwd_end; 2238 if (port_fwd_end != NULL) { 2239 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2240 (*port_fwd_end)(fwd_ports_ids[i]); 2241 } 2242 } 2243 launch_packet_forwarding(start_pkt_forward_on_core); 2244 } 2245 2246 void 2247 stop_packet_forwarding(void) 2248 { 2249 port_fwd_end_t port_fwd_end; 2250 lcoreid_t lc_id; 2251 portid_t pt_id; 2252 int i; 2253 2254 if (test_done) { 2255 printf("Packet forwarding not started\n"); 2256 return; 2257 } 2258 printf("Telling cores to stop..."); 2259 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2260 fwd_lcores[lc_id]->stopped = 1; 2261 printf("\nWaiting for lcores to finish...\n"); 2262 rte_eal_mp_wait_lcore(); 2263 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2264 if (port_fwd_end != NULL) { 2265 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2266 pt_id = fwd_ports_ids[i]; 2267 (*port_fwd_end)(pt_id); 2268 } 2269 } 2270 2271 fwd_stats_display(); 2272 2273 printf("\nDone.\n"); 2274 test_done = 1; 2275 } 2276 2277 void 2278 dev_set_link_up(portid_t pid) 2279 { 2280 if (rte_eth_dev_set_link_up(pid) < 0) 2281 printf("\nSet link up fail.\n"); 2282 } 2283 2284 void 2285 dev_set_link_down(portid_t pid) 2286 { 2287 if (rte_eth_dev_set_link_down(pid) < 0) 2288 printf("\nSet link down fail.\n"); 2289 } 2290 2291 static int 2292 all_ports_started(void) 2293 { 2294 portid_t pi; 2295 struct rte_port *port; 2296 2297 RTE_ETH_FOREACH_DEV(pi) { 2298 port = &ports[pi]; 2299 /* Check if there is a port which is not started */ 2300 if ((port->port_status != RTE_PORT_STARTED) && 2301 (port->slave_flag == 0)) 2302 return 0; 2303 } 2304 2305 /* No port is not started */ 2306 return 1; 2307 } 2308 2309 int 2310 port_is_stopped(portid_t port_id) 2311 { 2312 struct rte_port *port = &ports[port_id]; 2313 2314 if ((port->port_status != RTE_PORT_STOPPED) && 2315 (port->slave_flag == 0)) 2316 return 0; 2317 return 1; 2318 } 2319 2320 int 2321 all_ports_stopped(void) 2322 { 2323 portid_t pi; 2324 2325 RTE_ETH_FOREACH_DEV(pi) { 2326 if (!port_is_stopped(pi)) 2327 return 0; 2328 } 2329 2330 return 1; 2331 } 2332 2333 int 2334 port_is_started(portid_t port_id) 2335 { 2336 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2337 return 0; 2338 2339 if (ports[port_id].port_status != RTE_PORT_STARTED) 2340 return 0; 2341 2342 return 1; 2343 } 2344 2345 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2346 static int 2347 setup_hairpin_queues(portid_t pi) 2348 { 2349 queueid_t qi; 2350 struct rte_eth_hairpin_conf hairpin_conf = { 2351 .peer_count = 1, 2352 }; 2353 int i; 2354 int diag; 2355 struct rte_port *port = &ports[pi]; 2356 2357 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2358 hairpin_conf.peers[0].port = pi; 2359 hairpin_conf.peers[0].queue = i + nb_rxq; 2360 diag = rte_eth_tx_hairpin_queue_setup 2361 (pi, qi, nb_txd, &hairpin_conf); 2362 i++; 2363 if (diag == 0) 2364 continue; 2365 2366 /* Fail to setup rx queue, return */ 2367 if (rte_atomic16_cmpset(&(port->port_status), 2368 RTE_PORT_HANDLING, 2369 RTE_PORT_STOPPED) == 0) 2370 printf("Port %d can not be set back " 2371 "to stopped\n", pi); 2372 printf("Fail to configure port %d hairpin " 2373 "queues\n", pi); 2374 /* try to reconfigure queues next time */ 2375 port->need_reconfig_queues = 1; 2376 return -1; 2377 } 2378 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2379 hairpin_conf.peers[0].port = pi; 2380 hairpin_conf.peers[0].queue = i + nb_txq; 2381 diag = rte_eth_rx_hairpin_queue_setup 2382 (pi, qi, nb_rxd, &hairpin_conf); 2383 i++; 2384 if (diag == 0) 2385 continue; 2386 2387 /* Fail to setup rx queue, return */ 2388 if (rte_atomic16_cmpset(&(port->port_status), 2389 RTE_PORT_HANDLING, 2390 RTE_PORT_STOPPED) == 0) 2391 printf("Port %d can not be set back " 2392 "to stopped\n", pi); 2393 printf("Fail to configure port %d hairpin " 2394 "queues\n", pi); 2395 /* try to reconfigure queues next time */ 2396 port->need_reconfig_queues = 1; 2397 return -1; 2398 } 2399 return 0; 2400 } 2401 2402 int 2403 start_port(portid_t pid) 2404 { 2405 int diag, need_check_link_status = -1; 2406 portid_t pi; 2407 queueid_t qi; 2408 struct rte_port *port; 2409 struct rte_ether_addr mac_addr; 2410 struct rte_eth_hairpin_cap cap; 2411 2412 if (port_id_is_invalid(pid, ENABLED_WARN)) 2413 return 0; 2414 2415 if(dcb_config) 2416 dcb_test = 1; 2417 RTE_ETH_FOREACH_DEV(pi) { 2418 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2419 continue; 2420 2421 need_check_link_status = 0; 2422 port = &ports[pi]; 2423 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2424 RTE_PORT_HANDLING) == 0) { 2425 printf("Port %d is now not stopped\n", pi); 2426 continue; 2427 } 2428 2429 if (port->need_reconfig > 0) { 2430 port->need_reconfig = 0; 2431 2432 if (flow_isolate_all) { 2433 int ret = port_flow_isolate(pi, 1); 2434 if (ret) { 2435 printf("Failed to apply isolated" 2436 " mode on port %d\n", pi); 2437 return -1; 2438 } 2439 } 2440 configure_rxtx_dump_callbacks(0); 2441 printf("Configuring Port %d (socket %u)\n", pi, 2442 port->socket_id); 2443 if (nb_hairpinq > 0 && 2444 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2445 printf("Port %d doesn't support hairpin " 2446 "queues\n", pi); 2447 return -1; 2448 } 2449 /* configure port */ 2450 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2451 nb_txq + nb_hairpinq, 2452 &(port->dev_conf)); 2453 if (diag != 0) { 2454 if (rte_atomic16_cmpset(&(port->port_status), 2455 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2456 printf("Port %d can not be set back " 2457 "to stopped\n", pi); 2458 printf("Fail to configure port %d\n", pi); 2459 /* try to reconfigure port next time */ 2460 port->need_reconfig = 1; 2461 return -1; 2462 } 2463 } 2464 if (port->need_reconfig_queues > 0) { 2465 port->need_reconfig_queues = 0; 2466 /* setup tx queues */ 2467 for (qi = 0; qi < nb_txq; qi++) { 2468 if ((numa_support) && 2469 (txring_numa[pi] != NUMA_NO_CONFIG)) 2470 diag = rte_eth_tx_queue_setup(pi, qi, 2471 port->nb_tx_desc[qi], 2472 txring_numa[pi], 2473 &(port->tx_conf[qi])); 2474 else 2475 diag = rte_eth_tx_queue_setup(pi, qi, 2476 port->nb_tx_desc[qi], 2477 port->socket_id, 2478 &(port->tx_conf[qi])); 2479 2480 if (diag == 0) 2481 continue; 2482 2483 /* Fail to setup tx queue, return */ 2484 if (rte_atomic16_cmpset(&(port->port_status), 2485 RTE_PORT_HANDLING, 2486 RTE_PORT_STOPPED) == 0) 2487 printf("Port %d can not be set back " 2488 "to stopped\n", pi); 2489 printf("Fail to configure port %d tx queues\n", 2490 pi); 2491 /* try to reconfigure queues next time */ 2492 port->need_reconfig_queues = 1; 2493 return -1; 2494 } 2495 for (qi = 0; qi < nb_rxq; qi++) { 2496 /* setup rx queues */ 2497 if ((numa_support) && 2498 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2499 struct rte_mempool * mp = 2500 mbuf_pool_find(rxring_numa[pi]); 2501 if (mp == NULL) { 2502 printf("Failed to setup RX queue:" 2503 "No mempool allocation" 2504 " on the socket %d\n", 2505 rxring_numa[pi]); 2506 return -1; 2507 } 2508 2509 diag = rte_eth_rx_queue_setup(pi, qi, 2510 port->nb_rx_desc[qi], 2511 rxring_numa[pi], 2512 &(port->rx_conf[qi]), 2513 mp); 2514 } else { 2515 struct rte_mempool *mp = 2516 mbuf_pool_find(port->socket_id); 2517 if (mp == NULL) { 2518 printf("Failed to setup RX queue:" 2519 "No mempool allocation" 2520 " on the socket %d\n", 2521 port->socket_id); 2522 return -1; 2523 } 2524 diag = rte_eth_rx_queue_setup(pi, qi, 2525 port->nb_rx_desc[qi], 2526 port->socket_id, 2527 &(port->rx_conf[qi]), 2528 mp); 2529 } 2530 if (diag == 0) 2531 continue; 2532 2533 /* Fail to setup rx queue, return */ 2534 if (rte_atomic16_cmpset(&(port->port_status), 2535 RTE_PORT_HANDLING, 2536 RTE_PORT_STOPPED) == 0) 2537 printf("Port %d can not be set back " 2538 "to stopped\n", pi); 2539 printf("Fail to configure port %d rx queues\n", 2540 pi); 2541 /* try to reconfigure queues next time */ 2542 port->need_reconfig_queues = 1; 2543 return -1; 2544 } 2545 /* setup hairpin queues */ 2546 if (setup_hairpin_queues(pi) != 0) 2547 return -1; 2548 } 2549 configure_rxtx_dump_callbacks(verbose_level); 2550 if (clear_ptypes) { 2551 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2552 NULL, 0); 2553 if (diag < 0) 2554 printf( 2555 "Port %d: Failed to disable Ptype parsing\n", 2556 pi); 2557 } 2558 2559 /* start port */ 2560 if (rte_eth_dev_start(pi) < 0) { 2561 printf("Fail to start port %d\n", pi); 2562 2563 /* Fail to setup rx queue, return */ 2564 if (rte_atomic16_cmpset(&(port->port_status), 2565 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2566 printf("Port %d can not be set back to " 2567 "stopped\n", pi); 2568 continue; 2569 } 2570 2571 if (rte_atomic16_cmpset(&(port->port_status), 2572 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2573 printf("Port %d can not be set into started\n", pi); 2574 2575 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2576 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2577 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2578 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2579 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2580 2581 /* at least one port started, need checking link status */ 2582 need_check_link_status = 1; 2583 } 2584 2585 if (need_check_link_status == 1 && !no_link_check) 2586 check_all_ports_link_status(RTE_PORT_ALL); 2587 else if (need_check_link_status == 0) 2588 printf("Please stop the ports first\n"); 2589 2590 printf("Done\n"); 2591 return 0; 2592 } 2593 2594 void 2595 stop_port(portid_t pid) 2596 { 2597 portid_t pi; 2598 struct rte_port *port; 2599 int need_check_link_status = 0; 2600 2601 if (dcb_test) { 2602 dcb_test = 0; 2603 dcb_config = 0; 2604 } 2605 2606 if (port_id_is_invalid(pid, ENABLED_WARN)) 2607 return; 2608 2609 printf("Stopping ports...\n"); 2610 2611 RTE_ETH_FOREACH_DEV(pi) { 2612 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2613 continue; 2614 2615 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2616 printf("Please remove port %d from forwarding configuration.\n", pi); 2617 continue; 2618 } 2619 2620 if (port_is_bonding_slave(pi)) { 2621 printf("Please remove port %d from bonded device.\n", pi); 2622 continue; 2623 } 2624 2625 port = &ports[pi]; 2626 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2627 RTE_PORT_HANDLING) == 0) 2628 continue; 2629 2630 rte_eth_dev_stop(pi); 2631 2632 if (rte_atomic16_cmpset(&(port->port_status), 2633 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2634 printf("Port %d can not be set into stopped\n", pi); 2635 need_check_link_status = 1; 2636 } 2637 if (need_check_link_status && !no_link_check) 2638 check_all_ports_link_status(RTE_PORT_ALL); 2639 2640 printf("Done\n"); 2641 } 2642 2643 static void 2644 remove_invalid_ports_in(portid_t *array, portid_t *total) 2645 { 2646 portid_t i; 2647 portid_t new_total = 0; 2648 2649 for (i = 0; i < *total; i++) 2650 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2651 array[new_total] = array[i]; 2652 new_total++; 2653 } 2654 *total = new_total; 2655 } 2656 2657 static void 2658 remove_invalid_ports(void) 2659 { 2660 remove_invalid_ports_in(ports_ids, &nb_ports); 2661 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2662 nb_cfg_ports = nb_fwd_ports; 2663 } 2664 2665 void 2666 close_port(portid_t pid) 2667 { 2668 portid_t pi; 2669 struct rte_port *port; 2670 2671 if (port_id_is_invalid(pid, ENABLED_WARN)) 2672 return; 2673 2674 printf("Closing ports...\n"); 2675 2676 RTE_ETH_FOREACH_DEV(pi) { 2677 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2678 continue; 2679 2680 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2681 printf("Please remove port %d from forwarding configuration.\n", pi); 2682 continue; 2683 } 2684 2685 if (port_is_bonding_slave(pi)) { 2686 printf("Please remove port %d from bonded device.\n", pi); 2687 continue; 2688 } 2689 2690 port = &ports[pi]; 2691 if (rte_atomic16_cmpset(&(port->port_status), 2692 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2693 printf("Port %d is already closed\n", pi); 2694 continue; 2695 } 2696 2697 if (rte_atomic16_cmpset(&(port->port_status), 2698 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2699 printf("Port %d is now not stopped\n", pi); 2700 continue; 2701 } 2702 2703 if (port->flow_list) 2704 port_flow_flush(pi); 2705 rte_eth_dev_close(pi); 2706 2707 remove_invalid_ports(); 2708 2709 if (rte_atomic16_cmpset(&(port->port_status), 2710 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2711 printf("Port %d cannot be set to closed\n", pi); 2712 } 2713 2714 printf("Done\n"); 2715 } 2716 2717 void 2718 reset_port(portid_t pid) 2719 { 2720 int diag; 2721 portid_t pi; 2722 struct rte_port *port; 2723 2724 if (port_id_is_invalid(pid, ENABLED_WARN)) 2725 return; 2726 2727 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2728 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2729 printf("Can not reset port(s), please stop port(s) first.\n"); 2730 return; 2731 } 2732 2733 printf("Resetting ports...\n"); 2734 2735 RTE_ETH_FOREACH_DEV(pi) { 2736 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2737 continue; 2738 2739 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2740 printf("Please remove port %d from forwarding " 2741 "configuration.\n", pi); 2742 continue; 2743 } 2744 2745 if (port_is_bonding_slave(pi)) { 2746 printf("Please remove port %d from bonded device.\n", 2747 pi); 2748 continue; 2749 } 2750 2751 diag = rte_eth_dev_reset(pi); 2752 if (diag == 0) { 2753 port = &ports[pi]; 2754 port->need_reconfig = 1; 2755 port->need_reconfig_queues = 1; 2756 } else { 2757 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2758 } 2759 } 2760 2761 printf("Done\n"); 2762 } 2763 2764 void 2765 attach_port(char *identifier) 2766 { 2767 portid_t pi; 2768 struct rte_dev_iterator iterator; 2769 2770 printf("Attaching a new port...\n"); 2771 2772 if (identifier == NULL) { 2773 printf("Invalid parameters are specified\n"); 2774 return; 2775 } 2776 2777 if (rte_dev_probe(identifier) < 0) { 2778 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2779 return; 2780 } 2781 2782 /* first attach mode: event */ 2783 if (setup_on_probe_event) { 2784 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2785 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2786 if (ports[pi].port_status == RTE_PORT_HANDLING && 2787 ports[pi].need_setup != 0) 2788 setup_attached_port(pi); 2789 return; 2790 } 2791 2792 /* second attach mode: iterator */ 2793 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2794 /* setup ports matching the devargs used for probing */ 2795 if (port_is_forwarding(pi)) 2796 continue; /* port was already attached before */ 2797 setup_attached_port(pi); 2798 } 2799 } 2800 2801 static void 2802 setup_attached_port(portid_t pi) 2803 { 2804 unsigned int socket_id; 2805 int ret; 2806 2807 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2808 /* if socket_id is invalid, set to the first available socket. */ 2809 if (check_socket_id(socket_id) < 0) 2810 socket_id = socket_ids[0]; 2811 reconfig(pi, socket_id); 2812 ret = rte_eth_promiscuous_enable(pi); 2813 if (ret != 0) 2814 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2815 pi, rte_strerror(-ret)); 2816 2817 ports_ids[nb_ports++] = pi; 2818 fwd_ports_ids[nb_fwd_ports++] = pi; 2819 nb_cfg_ports = nb_fwd_ports; 2820 ports[pi].need_setup = 0; 2821 ports[pi].port_status = RTE_PORT_STOPPED; 2822 2823 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2824 printf("Done\n"); 2825 } 2826 2827 static void 2828 detach_device(struct rte_device *dev) 2829 { 2830 portid_t sibling; 2831 2832 if (dev == NULL) { 2833 printf("Device already removed\n"); 2834 return; 2835 } 2836 2837 printf("Removing a device...\n"); 2838 2839 if (rte_dev_remove(dev) < 0) { 2840 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2841 return; 2842 } 2843 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2844 /* reset mapping between old ports and removed device */ 2845 rte_eth_devices[sibling].device = NULL; 2846 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2847 /* sibling ports are forced to be closed */ 2848 ports[sibling].port_status = RTE_PORT_CLOSED; 2849 printf("Port %u is closed\n", sibling); 2850 } 2851 } 2852 2853 remove_invalid_ports(); 2854 2855 printf("Device is detached\n"); 2856 printf("Now total ports is %d\n", nb_ports); 2857 printf("Done\n"); 2858 return; 2859 } 2860 2861 void 2862 detach_port_device(portid_t port_id) 2863 { 2864 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2865 return; 2866 2867 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2868 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2869 printf("Port not stopped\n"); 2870 return; 2871 } 2872 printf("Port was not closed\n"); 2873 if (ports[port_id].flow_list) 2874 port_flow_flush(port_id); 2875 } 2876 2877 detach_device(rte_eth_devices[port_id].device); 2878 } 2879 2880 void 2881 detach_devargs(char *identifier) 2882 { 2883 struct rte_dev_iterator iterator; 2884 struct rte_devargs da; 2885 portid_t port_id; 2886 2887 printf("Removing a device...\n"); 2888 2889 memset(&da, 0, sizeof(da)); 2890 if (rte_devargs_parsef(&da, "%s", identifier)) { 2891 printf("cannot parse identifier\n"); 2892 if (da.args) 2893 free(da.args); 2894 return; 2895 } 2896 2897 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2898 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2899 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2900 printf("Port %u not stopped\n", port_id); 2901 rte_eth_iterator_cleanup(&iterator); 2902 return; 2903 } 2904 2905 /* sibling ports are forced to be closed */ 2906 if (ports[port_id].flow_list) 2907 port_flow_flush(port_id); 2908 ports[port_id].port_status = RTE_PORT_CLOSED; 2909 printf("Port %u is now closed\n", port_id); 2910 } 2911 } 2912 2913 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2914 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2915 da.name, da.bus->name); 2916 return; 2917 } 2918 2919 remove_invalid_ports(); 2920 2921 printf("Device %s is detached\n", identifier); 2922 printf("Now total ports is %d\n", nb_ports); 2923 printf("Done\n"); 2924 } 2925 2926 void 2927 pmd_test_exit(void) 2928 { 2929 portid_t pt_id; 2930 int ret; 2931 int i; 2932 2933 if (test_done == 0) 2934 stop_packet_forwarding(); 2935 2936 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2937 if (mempools[i]) { 2938 if (mp_alloc_type == MP_ALLOC_ANON) 2939 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2940 NULL); 2941 } 2942 } 2943 if (ports != NULL) { 2944 no_link_check = 1; 2945 RTE_ETH_FOREACH_DEV(pt_id) { 2946 printf("\nStopping port %d...\n", pt_id); 2947 fflush(stdout); 2948 stop_port(pt_id); 2949 } 2950 RTE_ETH_FOREACH_DEV(pt_id) { 2951 printf("\nShutting down port %d...\n", pt_id); 2952 fflush(stdout); 2953 close_port(pt_id); 2954 } 2955 } 2956 2957 if (hot_plug) { 2958 ret = rte_dev_event_monitor_stop(); 2959 if (ret) { 2960 RTE_LOG(ERR, EAL, 2961 "fail to stop device event monitor."); 2962 return; 2963 } 2964 2965 ret = rte_dev_event_callback_unregister(NULL, 2966 dev_event_callback, NULL); 2967 if (ret < 0) { 2968 RTE_LOG(ERR, EAL, 2969 "fail to unregister device event callback.\n"); 2970 return; 2971 } 2972 2973 ret = rte_dev_hotplug_handle_disable(); 2974 if (ret) { 2975 RTE_LOG(ERR, EAL, 2976 "fail to disable hotplug handling.\n"); 2977 return; 2978 } 2979 } 2980 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2981 if (mempools[i]) 2982 rte_mempool_free(mempools[i]); 2983 } 2984 2985 printf("\nBye...\n"); 2986 } 2987 2988 typedef void (*cmd_func_t)(void); 2989 struct pmd_test_command { 2990 const char *cmd_name; 2991 cmd_func_t cmd_func; 2992 }; 2993 2994 /* Check the link status of all ports in up to 9s, and print them finally */ 2995 static void 2996 check_all_ports_link_status(uint32_t port_mask) 2997 { 2998 #define CHECK_INTERVAL 100 /* 100ms */ 2999 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3000 portid_t portid; 3001 uint8_t count, all_ports_up, print_flag = 0; 3002 struct rte_eth_link link; 3003 int ret; 3004 3005 printf("Checking link statuses...\n"); 3006 fflush(stdout); 3007 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3008 all_ports_up = 1; 3009 RTE_ETH_FOREACH_DEV(portid) { 3010 if ((port_mask & (1 << portid)) == 0) 3011 continue; 3012 memset(&link, 0, sizeof(link)); 3013 ret = rte_eth_link_get_nowait(portid, &link); 3014 if (ret < 0) { 3015 all_ports_up = 0; 3016 if (print_flag == 1) 3017 printf("Port %u link get failed: %s\n", 3018 portid, rte_strerror(-ret)); 3019 continue; 3020 } 3021 /* print link status if flag set */ 3022 if (print_flag == 1) { 3023 if (link.link_status) 3024 printf( 3025 "Port%d Link Up. speed %u Mbps- %s\n", 3026 portid, link.link_speed, 3027 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 3028 ("full-duplex") : ("half-duplex")); 3029 else 3030 printf("Port %d Link Down\n", portid); 3031 continue; 3032 } 3033 /* clear all_ports_up flag if any link down */ 3034 if (link.link_status == ETH_LINK_DOWN) { 3035 all_ports_up = 0; 3036 break; 3037 } 3038 } 3039 /* after finally printing all link status, get out */ 3040 if (print_flag == 1) 3041 break; 3042 3043 if (all_ports_up == 0) { 3044 fflush(stdout); 3045 rte_delay_ms(CHECK_INTERVAL); 3046 } 3047 3048 /* set the print_flag if all ports up or timeout */ 3049 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3050 print_flag = 1; 3051 } 3052 3053 if (lsc_interrupt) 3054 break; 3055 } 3056 } 3057 3058 /* 3059 * This callback is for remove a port for a device. It has limitation because 3060 * it is not for multiple port removal for a device. 3061 * TODO: the device detach invoke will plan to be removed from user side to 3062 * eal. And convert all PMDs to free port resources on ether device closing. 3063 */ 3064 static void 3065 rmv_port_callback(void *arg) 3066 { 3067 int need_to_start = 0; 3068 int org_no_link_check = no_link_check; 3069 portid_t port_id = (intptr_t)arg; 3070 struct rte_device *dev; 3071 3072 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3073 3074 if (!test_done && port_is_forwarding(port_id)) { 3075 need_to_start = 1; 3076 stop_packet_forwarding(); 3077 } 3078 no_link_check = 1; 3079 stop_port(port_id); 3080 no_link_check = org_no_link_check; 3081 3082 /* Save rte_device pointer before closing ethdev port */ 3083 dev = rte_eth_devices[port_id].device; 3084 close_port(port_id); 3085 detach_device(dev); /* might be already removed or have more ports */ 3086 3087 if (need_to_start) 3088 start_packet_forwarding(0); 3089 } 3090 3091 /* This function is used by the interrupt thread */ 3092 static int 3093 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3094 void *ret_param) 3095 { 3096 RTE_SET_USED(param); 3097 RTE_SET_USED(ret_param); 3098 3099 if (type >= RTE_ETH_EVENT_MAX) { 3100 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3101 port_id, __func__, type); 3102 fflush(stderr); 3103 } else if (event_print_mask & (UINT32_C(1) << type)) { 3104 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3105 eth_event_desc[type]); 3106 fflush(stdout); 3107 } 3108 3109 switch (type) { 3110 case RTE_ETH_EVENT_NEW: 3111 ports[port_id].need_setup = 1; 3112 ports[port_id].port_status = RTE_PORT_HANDLING; 3113 break; 3114 case RTE_ETH_EVENT_INTR_RMV: 3115 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3116 break; 3117 if (rte_eal_alarm_set(100000, 3118 rmv_port_callback, (void *)(intptr_t)port_id)) 3119 fprintf(stderr, "Could not set up deferred device removal\n"); 3120 break; 3121 default: 3122 break; 3123 } 3124 return 0; 3125 } 3126 3127 static int 3128 register_eth_event_callback(void) 3129 { 3130 int ret; 3131 enum rte_eth_event_type event; 3132 3133 for (event = RTE_ETH_EVENT_UNKNOWN; 3134 event < RTE_ETH_EVENT_MAX; event++) { 3135 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3136 event, 3137 eth_event_callback, 3138 NULL); 3139 if (ret != 0) { 3140 TESTPMD_LOG(ERR, "Failed to register callback for " 3141 "%s event\n", eth_event_desc[event]); 3142 return -1; 3143 } 3144 } 3145 3146 return 0; 3147 } 3148 3149 /* This function is used by the interrupt thread */ 3150 static void 3151 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3152 __rte_unused void *arg) 3153 { 3154 uint16_t port_id; 3155 int ret; 3156 3157 if (type >= RTE_DEV_EVENT_MAX) { 3158 fprintf(stderr, "%s called upon invalid event %d\n", 3159 __func__, type); 3160 fflush(stderr); 3161 } 3162 3163 switch (type) { 3164 case RTE_DEV_EVENT_REMOVE: 3165 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3166 device_name); 3167 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3168 if (ret) { 3169 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3170 device_name); 3171 return; 3172 } 3173 /* 3174 * Because the user's callback is invoked in eal interrupt 3175 * callback, the interrupt callback need to be finished before 3176 * it can be unregistered when detaching device. So finish 3177 * callback soon and use a deferred removal to detach device 3178 * is need. It is a workaround, once the device detaching be 3179 * moved into the eal in the future, the deferred removal could 3180 * be deleted. 3181 */ 3182 if (rte_eal_alarm_set(100000, 3183 rmv_port_callback, (void *)(intptr_t)port_id)) 3184 RTE_LOG(ERR, EAL, 3185 "Could not set up deferred device removal\n"); 3186 break; 3187 case RTE_DEV_EVENT_ADD: 3188 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3189 device_name); 3190 /* TODO: After finish kernel driver binding, 3191 * begin to attach port. 3192 */ 3193 break; 3194 default: 3195 break; 3196 } 3197 } 3198 3199 static int 3200 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3201 { 3202 uint16_t i; 3203 int diag; 3204 uint8_t mapping_found = 0; 3205 3206 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3207 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3208 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3209 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3210 tx_queue_stats_mappings[i].queue_id, 3211 tx_queue_stats_mappings[i].stats_counter_id); 3212 if (diag != 0) 3213 return diag; 3214 mapping_found = 1; 3215 } 3216 } 3217 if (mapping_found) 3218 port->tx_queue_stats_mapping_enabled = 1; 3219 return 0; 3220 } 3221 3222 static int 3223 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3224 { 3225 uint16_t i; 3226 int diag; 3227 uint8_t mapping_found = 0; 3228 3229 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3230 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3231 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3232 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3233 rx_queue_stats_mappings[i].queue_id, 3234 rx_queue_stats_mappings[i].stats_counter_id); 3235 if (diag != 0) 3236 return diag; 3237 mapping_found = 1; 3238 } 3239 } 3240 if (mapping_found) 3241 port->rx_queue_stats_mapping_enabled = 1; 3242 return 0; 3243 } 3244 3245 static void 3246 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3247 { 3248 int diag = 0; 3249 3250 diag = set_tx_queue_stats_mapping_registers(pi, port); 3251 if (diag != 0) { 3252 if (diag == -ENOTSUP) { 3253 port->tx_queue_stats_mapping_enabled = 0; 3254 printf("TX queue stats mapping not supported port id=%d\n", pi); 3255 } 3256 else 3257 rte_exit(EXIT_FAILURE, 3258 "set_tx_queue_stats_mapping_registers " 3259 "failed for port id=%d diag=%d\n", 3260 pi, diag); 3261 } 3262 3263 diag = set_rx_queue_stats_mapping_registers(pi, port); 3264 if (diag != 0) { 3265 if (diag == -ENOTSUP) { 3266 port->rx_queue_stats_mapping_enabled = 0; 3267 printf("RX queue stats mapping not supported port id=%d\n", pi); 3268 } 3269 else 3270 rte_exit(EXIT_FAILURE, 3271 "set_rx_queue_stats_mapping_registers " 3272 "failed for port id=%d diag=%d\n", 3273 pi, diag); 3274 } 3275 } 3276 3277 static void 3278 rxtx_port_config(struct rte_port *port) 3279 { 3280 uint16_t qid; 3281 uint64_t offloads; 3282 3283 for (qid = 0; qid < nb_rxq; qid++) { 3284 offloads = port->rx_conf[qid].offloads; 3285 port->rx_conf[qid] = port->dev_info.default_rxconf; 3286 if (offloads != 0) 3287 port->rx_conf[qid].offloads = offloads; 3288 3289 /* Check if any Rx parameters have been passed */ 3290 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3291 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3292 3293 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3294 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3295 3296 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3297 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3298 3299 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3300 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3301 3302 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3303 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3304 3305 port->nb_rx_desc[qid] = nb_rxd; 3306 } 3307 3308 for (qid = 0; qid < nb_txq; qid++) { 3309 offloads = port->tx_conf[qid].offloads; 3310 port->tx_conf[qid] = port->dev_info.default_txconf; 3311 if (offloads != 0) 3312 port->tx_conf[qid].offloads = offloads; 3313 3314 /* Check if any Tx parameters have been passed */ 3315 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3316 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3317 3318 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3319 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3320 3321 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3322 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3323 3324 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3325 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3326 3327 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3328 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3329 3330 port->nb_tx_desc[qid] = nb_txd; 3331 } 3332 } 3333 3334 void 3335 init_port_config(void) 3336 { 3337 portid_t pid; 3338 struct rte_port *port; 3339 int ret; 3340 3341 RTE_ETH_FOREACH_DEV(pid) { 3342 port = &ports[pid]; 3343 port->dev_conf.fdir_conf = fdir_conf; 3344 3345 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3346 if (ret != 0) 3347 return; 3348 3349 if (nb_rxq > 1) { 3350 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3351 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3352 rss_hf & port->dev_info.flow_type_rss_offloads; 3353 } else { 3354 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3355 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3356 } 3357 3358 if (port->dcb_flag == 0) { 3359 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3360 port->dev_conf.rxmode.mq_mode = 3361 (enum rte_eth_rx_mq_mode) 3362 (rx_mq_mode & ETH_MQ_RX_RSS); 3363 else 3364 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3365 } 3366 3367 rxtx_port_config(port); 3368 3369 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3370 if (ret != 0) 3371 return; 3372 3373 map_port_queue_stats_mapping_registers(pid, port); 3374 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3375 rte_pmd_ixgbe_bypass_init(pid); 3376 #endif 3377 3378 if (lsc_interrupt && 3379 (rte_eth_devices[pid].data->dev_flags & 3380 RTE_ETH_DEV_INTR_LSC)) 3381 port->dev_conf.intr_conf.lsc = 1; 3382 if (rmv_interrupt && 3383 (rte_eth_devices[pid].data->dev_flags & 3384 RTE_ETH_DEV_INTR_RMV)) 3385 port->dev_conf.intr_conf.rmv = 1; 3386 } 3387 } 3388 3389 void set_port_slave_flag(portid_t slave_pid) 3390 { 3391 struct rte_port *port; 3392 3393 port = &ports[slave_pid]; 3394 port->slave_flag = 1; 3395 } 3396 3397 void clear_port_slave_flag(portid_t slave_pid) 3398 { 3399 struct rte_port *port; 3400 3401 port = &ports[slave_pid]; 3402 port->slave_flag = 0; 3403 } 3404 3405 uint8_t port_is_bonding_slave(portid_t slave_pid) 3406 { 3407 struct rte_port *port; 3408 3409 port = &ports[slave_pid]; 3410 if ((rte_eth_devices[slave_pid].data->dev_flags & 3411 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3412 return 1; 3413 return 0; 3414 } 3415 3416 const uint16_t vlan_tags[] = { 3417 0, 1, 2, 3, 4, 5, 6, 7, 3418 8, 9, 10, 11, 12, 13, 14, 15, 3419 16, 17, 18, 19, 20, 21, 22, 23, 3420 24, 25, 26, 27, 28, 29, 30, 31 3421 }; 3422 3423 static int 3424 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3425 enum dcb_mode_enable dcb_mode, 3426 enum rte_eth_nb_tcs num_tcs, 3427 uint8_t pfc_en) 3428 { 3429 uint8_t i; 3430 int32_t rc; 3431 struct rte_eth_rss_conf rss_conf; 3432 3433 /* 3434 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3435 * given above, and the number of traffic classes available for use. 3436 */ 3437 if (dcb_mode == DCB_VT_ENABLED) { 3438 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3439 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3440 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3441 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3442 3443 /* VMDQ+DCB RX and TX configurations */ 3444 vmdq_rx_conf->enable_default_pool = 0; 3445 vmdq_rx_conf->default_pool = 0; 3446 vmdq_rx_conf->nb_queue_pools = 3447 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3448 vmdq_tx_conf->nb_queue_pools = 3449 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3450 3451 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3452 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3453 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3454 vmdq_rx_conf->pool_map[i].pools = 3455 1 << (i % vmdq_rx_conf->nb_queue_pools); 3456 } 3457 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3458 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3459 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3460 } 3461 3462 /* set DCB mode of RX and TX of multiple queues */ 3463 eth_conf->rxmode.mq_mode = 3464 (enum rte_eth_rx_mq_mode) 3465 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3466 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3467 } else { 3468 struct rte_eth_dcb_rx_conf *rx_conf = 3469 ð_conf->rx_adv_conf.dcb_rx_conf; 3470 struct rte_eth_dcb_tx_conf *tx_conf = 3471 ð_conf->tx_adv_conf.dcb_tx_conf; 3472 3473 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3474 3475 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3476 if (rc != 0) 3477 return rc; 3478 3479 rx_conf->nb_tcs = num_tcs; 3480 tx_conf->nb_tcs = num_tcs; 3481 3482 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3483 rx_conf->dcb_tc[i] = i % num_tcs; 3484 tx_conf->dcb_tc[i] = i % num_tcs; 3485 } 3486 3487 eth_conf->rxmode.mq_mode = 3488 (enum rte_eth_rx_mq_mode) 3489 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3490 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3491 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3492 } 3493 3494 if (pfc_en) 3495 eth_conf->dcb_capability_en = 3496 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3497 else 3498 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3499 3500 return 0; 3501 } 3502 3503 int 3504 init_port_dcb_config(portid_t pid, 3505 enum dcb_mode_enable dcb_mode, 3506 enum rte_eth_nb_tcs num_tcs, 3507 uint8_t pfc_en) 3508 { 3509 struct rte_eth_conf port_conf; 3510 struct rte_port *rte_port; 3511 int retval; 3512 uint16_t i; 3513 3514 rte_port = &ports[pid]; 3515 3516 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3517 /* Enter DCB configuration status */ 3518 dcb_config = 1; 3519 3520 port_conf.rxmode = rte_port->dev_conf.rxmode; 3521 port_conf.txmode = rte_port->dev_conf.txmode; 3522 3523 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3524 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3525 if (retval < 0) 3526 return retval; 3527 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3528 3529 /* re-configure the device . */ 3530 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3531 if (retval < 0) 3532 return retval; 3533 3534 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3535 if (retval != 0) 3536 return retval; 3537 3538 /* If dev_info.vmdq_pool_base is greater than 0, 3539 * the queue id of vmdq pools is started after pf queues. 3540 */ 3541 if (dcb_mode == DCB_VT_ENABLED && 3542 rte_port->dev_info.vmdq_pool_base > 0) { 3543 printf("VMDQ_DCB multi-queue mode is nonsensical" 3544 " for port %d.", pid); 3545 return -1; 3546 } 3547 3548 /* Assume the ports in testpmd have the same dcb capability 3549 * and has the same number of rxq and txq in dcb mode 3550 */ 3551 if (dcb_mode == DCB_VT_ENABLED) { 3552 if (rte_port->dev_info.max_vfs > 0) { 3553 nb_rxq = rte_port->dev_info.nb_rx_queues; 3554 nb_txq = rte_port->dev_info.nb_tx_queues; 3555 } else { 3556 nb_rxq = rte_port->dev_info.max_rx_queues; 3557 nb_txq = rte_port->dev_info.max_tx_queues; 3558 } 3559 } else { 3560 /*if vt is disabled, use all pf queues */ 3561 if (rte_port->dev_info.vmdq_pool_base == 0) { 3562 nb_rxq = rte_port->dev_info.max_rx_queues; 3563 nb_txq = rte_port->dev_info.max_tx_queues; 3564 } else { 3565 nb_rxq = (queueid_t)num_tcs; 3566 nb_txq = (queueid_t)num_tcs; 3567 3568 } 3569 } 3570 rx_free_thresh = 64; 3571 3572 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3573 3574 rxtx_port_config(rte_port); 3575 /* VLAN filter */ 3576 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3577 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3578 rx_vft_set(pid, vlan_tags[i], 1); 3579 3580 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3581 if (retval != 0) 3582 return retval; 3583 3584 map_port_queue_stats_mapping_registers(pid, rte_port); 3585 3586 rte_port->dcb_flag = 1; 3587 3588 return 0; 3589 } 3590 3591 static void 3592 init_port(void) 3593 { 3594 /* Configuration of Ethernet ports. */ 3595 ports = rte_zmalloc("testpmd: ports", 3596 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3597 RTE_CACHE_LINE_SIZE); 3598 if (ports == NULL) { 3599 rte_exit(EXIT_FAILURE, 3600 "rte_zmalloc(%d struct rte_port) failed\n", 3601 RTE_MAX_ETHPORTS); 3602 } 3603 3604 /* Initialize ports NUMA structures */ 3605 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3606 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3607 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3608 } 3609 3610 static void 3611 force_quit(void) 3612 { 3613 pmd_test_exit(); 3614 prompt_exit(); 3615 } 3616 3617 static void 3618 print_stats(void) 3619 { 3620 uint8_t i; 3621 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3622 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3623 3624 /* Clear screen and move to top left */ 3625 printf("%s%s", clr, top_left); 3626 3627 printf("\nPort statistics ===================================="); 3628 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3629 nic_stats_display(fwd_ports_ids[i]); 3630 3631 fflush(stdout); 3632 } 3633 3634 static void 3635 signal_handler(int signum) 3636 { 3637 if (signum == SIGINT || signum == SIGTERM) { 3638 printf("\nSignal %d received, preparing to exit...\n", 3639 signum); 3640 #ifdef RTE_LIBRTE_PDUMP 3641 /* uninitialize packet capture framework */ 3642 rte_pdump_uninit(); 3643 #endif 3644 #ifdef RTE_LIBRTE_LATENCY_STATS 3645 if (latencystats_enabled != 0) 3646 rte_latencystats_uninit(); 3647 #endif 3648 force_quit(); 3649 /* Set flag to indicate the force termination. */ 3650 f_quit = 1; 3651 /* exit with the expected status */ 3652 signal(signum, SIG_DFL); 3653 kill(getpid(), signum); 3654 } 3655 } 3656 3657 int 3658 main(int argc, char** argv) 3659 { 3660 int diag; 3661 portid_t port_id; 3662 uint16_t count; 3663 int ret; 3664 3665 signal(SIGINT, signal_handler); 3666 signal(SIGTERM, signal_handler); 3667 3668 testpmd_logtype = rte_log_register("testpmd"); 3669 if (testpmd_logtype < 0) 3670 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3671 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3672 3673 diag = rte_eal_init(argc, argv); 3674 if (diag < 0) 3675 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3676 rte_strerror(rte_errno)); 3677 3678 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3679 rte_exit(EXIT_FAILURE, 3680 "Secondary process type not supported.\n"); 3681 3682 ret = register_eth_event_callback(); 3683 if (ret != 0) 3684 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3685 3686 #ifdef RTE_LIBRTE_PDUMP 3687 /* initialize packet capture framework */ 3688 rte_pdump_init(); 3689 #endif 3690 3691 count = 0; 3692 RTE_ETH_FOREACH_DEV(port_id) { 3693 ports_ids[count] = port_id; 3694 count++; 3695 } 3696 nb_ports = (portid_t) count; 3697 if (nb_ports == 0) 3698 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3699 3700 /* allocate port structures, and init them */ 3701 init_port(); 3702 3703 set_def_fwd_config(); 3704 if (nb_lcores == 0) 3705 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3706 "Check the core mask argument\n"); 3707 3708 /* Bitrate/latency stats disabled by default */ 3709 #ifdef RTE_LIBRTE_BITRATE 3710 bitrate_enabled = 0; 3711 #endif 3712 #ifdef RTE_LIBRTE_LATENCY_STATS 3713 latencystats_enabled = 0; 3714 #endif 3715 3716 /* on FreeBSD, mlockall() is disabled by default */ 3717 #ifdef RTE_EXEC_ENV_FREEBSD 3718 do_mlockall = 0; 3719 #else 3720 do_mlockall = 1; 3721 #endif 3722 3723 argc -= diag; 3724 argv += diag; 3725 if (argc > 1) 3726 launch_args_parse(argc, argv); 3727 3728 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3729 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3730 strerror(errno)); 3731 } 3732 3733 if (tx_first && interactive) 3734 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3735 "interactive mode.\n"); 3736 3737 if (tx_first && lsc_interrupt) { 3738 printf("Warning: lsc_interrupt needs to be off when " 3739 " using tx_first. Disabling.\n"); 3740 lsc_interrupt = 0; 3741 } 3742 3743 if (!nb_rxq && !nb_txq) 3744 printf("Warning: Either rx or tx queues should be non-zero\n"); 3745 3746 if (nb_rxq > 1 && nb_rxq > nb_txq) 3747 printf("Warning: nb_rxq=%d enables RSS configuration, " 3748 "but nb_txq=%d will prevent to fully test it.\n", 3749 nb_rxq, nb_txq); 3750 3751 init_config(); 3752 3753 if (hot_plug) { 3754 ret = rte_dev_hotplug_handle_enable(); 3755 if (ret) { 3756 RTE_LOG(ERR, EAL, 3757 "fail to enable hotplug handling."); 3758 return -1; 3759 } 3760 3761 ret = rte_dev_event_monitor_start(); 3762 if (ret) { 3763 RTE_LOG(ERR, EAL, 3764 "fail to start device event monitoring."); 3765 return -1; 3766 } 3767 3768 ret = rte_dev_event_callback_register(NULL, 3769 dev_event_callback, NULL); 3770 if (ret) { 3771 RTE_LOG(ERR, EAL, 3772 "fail to register device event callback\n"); 3773 return -1; 3774 } 3775 } 3776 3777 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3778 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3779 3780 /* set all ports to promiscuous mode by default */ 3781 RTE_ETH_FOREACH_DEV(port_id) { 3782 ret = rte_eth_promiscuous_enable(port_id); 3783 if (ret != 0) 3784 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3785 port_id, rte_strerror(-ret)); 3786 } 3787 3788 /* Init metrics library */ 3789 rte_metrics_init(rte_socket_id()); 3790 3791 #ifdef RTE_LIBRTE_LATENCY_STATS 3792 if (latencystats_enabled != 0) { 3793 int ret = rte_latencystats_init(1, NULL); 3794 if (ret) 3795 printf("Warning: latencystats init()" 3796 " returned error %d\n", ret); 3797 printf("Latencystats running on lcore %d\n", 3798 latencystats_lcore_id); 3799 } 3800 #endif 3801 3802 /* Setup bitrate stats */ 3803 #ifdef RTE_LIBRTE_BITRATE 3804 if (bitrate_enabled != 0) { 3805 bitrate_data = rte_stats_bitrate_create(); 3806 if (bitrate_data == NULL) 3807 rte_exit(EXIT_FAILURE, 3808 "Could not allocate bitrate data.\n"); 3809 rte_stats_bitrate_reg(bitrate_data); 3810 } 3811 #endif 3812 3813 #ifdef RTE_LIBRTE_CMDLINE 3814 if (strlen(cmdline_filename) != 0) 3815 cmdline_read_from_file(cmdline_filename); 3816 3817 if (interactive == 1) { 3818 if (auto_start) { 3819 printf("Start automatic packet forwarding\n"); 3820 start_packet_forwarding(0); 3821 } 3822 prompt(); 3823 pmd_test_exit(); 3824 } else 3825 #endif 3826 { 3827 char c; 3828 int rc; 3829 3830 f_quit = 0; 3831 3832 printf("No commandline core given, start packet forwarding\n"); 3833 start_packet_forwarding(tx_first); 3834 if (stats_period != 0) { 3835 uint64_t prev_time = 0, cur_time, diff_time = 0; 3836 uint64_t timer_period; 3837 3838 /* Convert to number of cycles */ 3839 timer_period = stats_period * rte_get_timer_hz(); 3840 3841 while (f_quit == 0) { 3842 cur_time = rte_get_timer_cycles(); 3843 diff_time += cur_time - prev_time; 3844 3845 if (diff_time >= timer_period) { 3846 print_stats(); 3847 /* Reset the timer */ 3848 diff_time = 0; 3849 } 3850 /* Sleep to avoid unnecessary checks */ 3851 prev_time = cur_time; 3852 sleep(1); 3853 } 3854 } 3855 3856 printf("Press enter to exit\n"); 3857 rc = read(0, &c, 1); 3858 pmd_test_exit(); 3859 if (rc < 0) 3860 return 1; 3861 } 3862 3863 ret = rte_eal_cleanup(); 3864 if (ret != 0) 3865 rte_exit(EXIT_FAILURE, 3866 "EAL cleanup failed: %s\n", strerror(-ret)); 3867 3868 return EXIT_SUCCESS; 3869 } 3870