1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 199 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 200 * specified on command-line. */ 201 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 202 203 /* 204 * In container, it cannot terminate the process which running with 'stats-period' 205 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 206 */ 207 uint8_t f_quit; 208 209 /* 210 * Configuration of packet segments used by the "txonly" processing engine. 211 */ 212 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 213 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 214 TXONLY_DEF_PACKET_LEN, 215 }; 216 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 217 218 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 219 /**< Split policy for packets to TX. */ 220 221 uint8_t txonly_multi_flow; 222 /**< Whether multiple flows are generated in TXONLY mode. */ 223 224 uint32_t tx_pkt_times_inter; 225 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 226 227 uint32_t tx_pkt_times_intra; 228 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 229 230 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 231 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 232 233 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 234 uint8_t dcb_config = 0; 235 236 /* Whether the dcb is in testing status */ 237 uint8_t dcb_test = 0; 238 239 /* 240 * Configurable number of RX/TX queues. 241 */ 242 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 243 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 244 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 245 246 /* 247 * Configurable number of RX/TX ring descriptors. 248 * Defaults are supplied by drivers via ethdev. 249 */ 250 #define RTE_TEST_RX_DESC_DEFAULT 0 251 #define RTE_TEST_TX_DESC_DEFAULT 0 252 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 253 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 254 255 #define RTE_PMD_PARAM_UNSET -1 256 /* 257 * Configurable values of RX and TX ring threshold registers. 258 */ 259 260 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 261 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 262 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 263 264 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 265 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 266 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX free threshold. 270 */ 271 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of RX drop enable. 275 */ 276 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX free threshold. 280 */ 281 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of TX RS bit threshold. 285 */ 286 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 287 288 /* 289 * Configurable value of buffered packets before sending. 290 */ 291 uint16_t noisy_tx_sw_bufsz; 292 293 /* 294 * Configurable value of packet buffer timeout. 295 */ 296 uint16_t noisy_tx_sw_buf_flush_time; 297 298 /* 299 * Configurable value for size of VNF internal memory area 300 * used for simulating noisy neighbour behaviour 301 */ 302 uint64_t noisy_lkup_mem_sz; 303 304 /* 305 * Configurable value of number of random writes done in 306 * VNF simulation memory area. 307 */ 308 uint64_t noisy_lkup_num_writes; 309 310 /* 311 * Configurable value of number of random reads done in 312 * VNF simulation memory area. 313 */ 314 uint64_t noisy_lkup_num_reads; 315 316 /* 317 * Configurable value of number of random reads/writes done in 318 * VNF simulation memory area. 319 */ 320 uint64_t noisy_lkup_num_reads_writes; 321 322 /* 323 * Receive Side Scaling (RSS) configuration. 324 */ 325 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 326 327 /* 328 * Port topology configuration 329 */ 330 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 331 332 /* 333 * Avoids to flush all the RX streams before starts forwarding. 334 */ 335 uint8_t no_flush_rx = 0; /* flush by default */ 336 337 /* 338 * Flow API isolated mode. 339 */ 340 uint8_t flow_isolate_all; 341 342 /* 343 * Avoids to check link status when starting/stopping a port. 344 */ 345 uint8_t no_link_check = 0; /* check by default */ 346 347 /* 348 * Don't automatically start all ports in interactive mode. 349 */ 350 uint8_t no_device_start = 0; 351 352 /* 353 * Enable link status change notification 354 */ 355 uint8_t lsc_interrupt = 1; /* enabled by default */ 356 357 /* 358 * Enable device removal notification. 359 */ 360 uint8_t rmv_interrupt = 1; /* enabled by default */ 361 362 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 363 364 /* After attach, port setup is called on event or by iterator */ 365 bool setup_on_probe_event = true; 366 367 /* Clear ptypes on port initialization. */ 368 uint8_t clear_ptypes = true; 369 370 /* Pretty printing of ethdev events */ 371 static const char * const eth_event_desc[] = { 372 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 373 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 374 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 375 [RTE_ETH_EVENT_INTR_RESET] = "reset", 376 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 377 [RTE_ETH_EVENT_IPSEC] = "IPsec", 378 [RTE_ETH_EVENT_MACSEC] = "MACsec", 379 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 380 [RTE_ETH_EVENT_NEW] = "device probed", 381 [RTE_ETH_EVENT_DESTROY] = "device released", 382 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 383 [RTE_ETH_EVENT_MAX] = NULL, 384 }; 385 386 /* 387 * Display or mask ether events 388 * Default to all events except VF_MBOX 389 */ 390 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 391 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 392 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 393 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 394 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 395 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 396 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 397 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 398 /* 399 * Decide if all memory are locked for performance. 400 */ 401 int do_mlockall = 0; 402 403 /* 404 * NIC bypass mode configuration options. 405 */ 406 407 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 408 /* The NIC bypass watchdog timeout. */ 409 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 410 #endif 411 412 413 #ifdef RTE_LIBRTE_LATENCY_STATS 414 415 /* 416 * Set when latency stats is enabled in the commandline 417 */ 418 uint8_t latencystats_enabled; 419 420 /* 421 * Lcore ID to serive latency statistics. 422 */ 423 lcoreid_t latencystats_lcore_id = -1; 424 425 #endif 426 427 /* 428 * Ethernet device configuration. 429 */ 430 struct rte_eth_rxmode rx_mode = { 431 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 432 /**< Default maximum frame length. */ 433 }; 434 435 struct rte_eth_txmode tx_mode = { 436 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 437 }; 438 439 struct rte_fdir_conf fdir_conf = { 440 .mode = RTE_FDIR_MODE_NONE, 441 .pballoc = RTE_FDIR_PBALLOC_64K, 442 .status = RTE_FDIR_REPORT_STATUS, 443 .mask = { 444 .vlan_tci_mask = 0xFFEF, 445 .ipv4_mask = { 446 .src_ip = 0xFFFFFFFF, 447 .dst_ip = 0xFFFFFFFF, 448 }, 449 .ipv6_mask = { 450 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 451 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 452 }, 453 .src_port_mask = 0xFFFF, 454 .dst_port_mask = 0xFFFF, 455 .mac_addr_byte_mask = 0xFF, 456 .tunnel_type_mask = 1, 457 .tunnel_id_mask = 0xFFFFFFFF, 458 }, 459 .drop_queue = 127, 460 }; 461 462 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 463 464 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 465 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 466 467 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 468 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 469 470 uint16_t nb_tx_queue_stats_mappings = 0; 471 uint16_t nb_rx_queue_stats_mappings = 0; 472 473 /* 474 * Display zero values by default for xstats 475 */ 476 uint8_t xstats_hide_zero; 477 478 /* 479 * Measure of CPU cycles disabled by default 480 */ 481 uint8_t record_core_cycles; 482 483 /* 484 * Display of RX and TX bursts disabled by default 485 */ 486 uint8_t record_burst_stats; 487 488 unsigned int num_sockets = 0; 489 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 490 491 #ifdef RTE_LIBRTE_BITRATESTATS 492 /* Bitrate statistics */ 493 struct rte_stats_bitrates *bitrate_data; 494 lcoreid_t bitrate_lcore_id; 495 uint8_t bitrate_enabled; 496 #endif 497 498 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 499 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 500 501 /* 502 * hexadecimal bitmask of RX mq mode can be enabled. 503 */ 504 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 505 506 /* Forward function declarations */ 507 static void setup_attached_port(portid_t pi); 508 static void map_port_queue_stats_mapping_registers(portid_t pi, 509 struct rte_port *port); 510 static void check_all_ports_link_status(uint32_t port_mask); 511 static int eth_event_callback(portid_t port_id, 512 enum rte_eth_event_type type, 513 void *param, void *ret_param); 514 static void dev_event_callback(const char *device_name, 515 enum rte_dev_event_type type, 516 void *param); 517 518 /* 519 * Check if all the ports are started. 520 * If yes, return positive value. If not, return zero. 521 */ 522 static int all_ports_started(void); 523 524 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 525 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 526 527 /* Holds the registered mbuf dynamic flags names. */ 528 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 529 530 /* 531 * Helper function to check if socket is already discovered. 532 * If yes, return positive value. If not, return zero. 533 */ 534 int 535 new_socket_id(unsigned int socket_id) 536 { 537 unsigned int i; 538 539 for (i = 0; i < num_sockets; i++) { 540 if (socket_ids[i] == socket_id) 541 return 0; 542 } 543 return 1; 544 } 545 546 /* 547 * Setup default configuration. 548 */ 549 static void 550 set_default_fwd_lcores_config(void) 551 { 552 unsigned int i; 553 unsigned int nb_lc; 554 unsigned int sock_num; 555 556 nb_lc = 0; 557 for (i = 0; i < RTE_MAX_LCORE; i++) { 558 if (!rte_lcore_is_enabled(i)) 559 continue; 560 sock_num = rte_lcore_to_socket_id(i); 561 if (new_socket_id(sock_num)) { 562 if (num_sockets >= RTE_MAX_NUMA_NODES) { 563 rte_exit(EXIT_FAILURE, 564 "Total sockets greater than %u\n", 565 RTE_MAX_NUMA_NODES); 566 } 567 socket_ids[num_sockets++] = sock_num; 568 } 569 if (i == rte_get_master_lcore()) 570 continue; 571 fwd_lcores_cpuids[nb_lc++] = i; 572 } 573 nb_lcores = (lcoreid_t) nb_lc; 574 nb_cfg_lcores = nb_lcores; 575 nb_fwd_lcores = 1; 576 } 577 578 static void 579 set_def_peer_eth_addrs(void) 580 { 581 portid_t i; 582 583 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 584 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 585 peer_eth_addrs[i].addr_bytes[5] = i; 586 } 587 } 588 589 static void 590 set_default_fwd_ports_config(void) 591 { 592 portid_t pt_id; 593 int i = 0; 594 595 RTE_ETH_FOREACH_DEV(pt_id) { 596 fwd_ports_ids[i++] = pt_id; 597 598 /* Update sockets info according to the attached device */ 599 int socket_id = rte_eth_dev_socket_id(pt_id); 600 if (socket_id >= 0 && new_socket_id(socket_id)) { 601 if (num_sockets >= RTE_MAX_NUMA_NODES) { 602 rte_exit(EXIT_FAILURE, 603 "Total sockets greater than %u\n", 604 RTE_MAX_NUMA_NODES); 605 } 606 socket_ids[num_sockets++] = socket_id; 607 } 608 } 609 610 nb_cfg_ports = nb_ports; 611 nb_fwd_ports = nb_ports; 612 } 613 614 void 615 set_def_fwd_config(void) 616 { 617 set_default_fwd_lcores_config(); 618 set_def_peer_eth_addrs(); 619 set_default_fwd_ports_config(); 620 } 621 622 /* extremely pessimistic estimation of memory required to create a mempool */ 623 static int 624 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 625 { 626 unsigned int n_pages, mbuf_per_pg, leftover; 627 uint64_t total_mem, mbuf_mem, obj_sz; 628 629 /* there is no good way to predict how much space the mempool will 630 * occupy because it will allocate chunks on the fly, and some of those 631 * will come from default DPDK memory while some will come from our 632 * external memory, so just assume 128MB will be enough for everyone. 633 */ 634 uint64_t hdr_mem = 128 << 20; 635 636 /* account for possible non-contiguousness */ 637 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 638 if (obj_sz > pgsz) { 639 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 640 return -1; 641 } 642 643 mbuf_per_pg = pgsz / obj_sz; 644 leftover = (nb_mbufs % mbuf_per_pg) > 0; 645 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 646 647 mbuf_mem = n_pages * pgsz; 648 649 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 650 651 if (total_mem > SIZE_MAX) { 652 TESTPMD_LOG(ERR, "Memory size too big\n"); 653 return -1; 654 } 655 *out = (size_t)total_mem; 656 657 return 0; 658 } 659 660 static int 661 pagesz_flags(uint64_t page_sz) 662 { 663 /* as per mmap() manpage, all page sizes are log2 of page size 664 * shifted by MAP_HUGE_SHIFT 665 */ 666 int log2 = rte_log2_u64(page_sz); 667 668 return (log2 << HUGE_SHIFT); 669 } 670 671 static void * 672 alloc_mem(size_t memsz, size_t pgsz, bool huge) 673 { 674 void *addr; 675 int flags; 676 677 /* allocate anonymous hugepages */ 678 flags = MAP_ANONYMOUS | MAP_PRIVATE; 679 if (huge) 680 flags |= HUGE_FLAG | pagesz_flags(pgsz); 681 682 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 683 if (addr == MAP_FAILED) 684 return NULL; 685 686 return addr; 687 } 688 689 struct extmem_param { 690 void *addr; 691 size_t len; 692 size_t pgsz; 693 rte_iova_t *iova_table; 694 unsigned int iova_table_len; 695 }; 696 697 static int 698 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 699 bool huge) 700 { 701 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 702 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 703 unsigned int cur_page, n_pages, pgsz_idx; 704 size_t mem_sz, cur_pgsz; 705 rte_iova_t *iovas = NULL; 706 void *addr; 707 int ret; 708 709 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 710 /* skip anything that is too big */ 711 if (pgsizes[pgsz_idx] > SIZE_MAX) 712 continue; 713 714 cur_pgsz = pgsizes[pgsz_idx]; 715 716 /* if we were told not to allocate hugepages, override */ 717 if (!huge) 718 cur_pgsz = sysconf(_SC_PAGESIZE); 719 720 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 721 if (ret < 0) { 722 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 723 return -1; 724 } 725 726 /* allocate our memory */ 727 addr = alloc_mem(mem_sz, cur_pgsz, huge); 728 729 /* if we couldn't allocate memory with a specified page size, 730 * that doesn't mean we can't do it with other page sizes, so 731 * try another one. 732 */ 733 if (addr == NULL) 734 continue; 735 736 /* store IOVA addresses for every page in this memory area */ 737 n_pages = mem_sz / cur_pgsz; 738 739 iovas = malloc(sizeof(*iovas) * n_pages); 740 741 if (iovas == NULL) { 742 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 743 goto fail; 744 } 745 /* lock memory if it's not huge pages */ 746 if (!huge) 747 mlock(addr, mem_sz); 748 749 /* populate IOVA addresses */ 750 for (cur_page = 0; cur_page < n_pages; cur_page++) { 751 rte_iova_t iova; 752 size_t offset; 753 void *cur; 754 755 offset = cur_pgsz * cur_page; 756 cur = RTE_PTR_ADD(addr, offset); 757 758 /* touch the page before getting its IOVA */ 759 *(volatile char *)cur = 0; 760 761 iova = rte_mem_virt2iova(cur); 762 763 iovas[cur_page] = iova; 764 } 765 766 break; 767 } 768 /* if we couldn't allocate anything */ 769 if (iovas == NULL) 770 return -1; 771 772 param->addr = addr; 773 param->len = mem_sz; 774 param->pgsz = cur_pgsz; 775 param->iova_table = iovas; 776 param->iova_table_len = n_pages; 777 778 return 0; 779 fail: 780 if (iovas) 781 free(iovas); 782 if (addr) 783 munmap(addr, mem_sz); 784 785 return -1; 786 } 787 788 static int 789 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 790 { 791 struct extmem_param param; 792 int socket_id, ret; 793 794 memset(¶m, 0, sizeof(param)); 795 796 /* check if our heap exists */ 797 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 798 if (socket_id < 0) { 799 /* create our heap */ 800 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 801 if (ret < 0) { 802 TESTPMD_LOG(ERR, "Cannot create heap\n"); 803 return -1; 804 } 805 } 806 807 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 808 if (ret < 0) { 809 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 810 return -1; 811 } 812 813 /* we now have a valid memory area, so add it to heap */ 814 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 815 param.addr, param.len, param.iova_table, 816 param.iova_table_len, param.pgsz); 817 818 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 819 820 /* not needed any more */ 821 free(param.iova_table); 822 823 if (ret < 0) { 824 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 825 munmap(param.addr, param.len); 826 return -1; 827 } 828 829 /* success */ 830 831 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 832 param.len >> 20); 833 834 return 0; 835 } 836 static void 837 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 838 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 839 { 840 uint16_t pid = 0; 841 int ret; 842 843 RTE_ETH_FOREACH_DEV(pid) { 844 struct rte_eth_dev *dev = 845 &rte_eth_devices[pid]; 846 847 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 848 memhdr->len); 849 if (ret) { 850 TESTPMD_LOG(DEBUG, 851 "unable to DMA unmap addr 0x%p " 852 "for device %s\n", 853 memhdr->addr, dev->data->name); 854 } 855 } 856 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 857 if (ret) { 858 TESTPMD_LOG(DEBUG, 859 "unable to un-register addr 0x%p\n", memhdr->addr); 860 } 861 } 862 863 static void 864 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 865 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 866 { 867 uint16_t pid = 0; 868 size_t page_size = sysconf(_SC_PAGESIZE); 869 int ret; 870 871 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 872 page_size); 873 if (ret) { 874 TESTPMD_LOG(DEBUG, 875 "unable to register addr 0x%p\n", memhdr->addr); 876 return; 877 } 878 RTE_ETH_FOREACH_DEV(pid) { 879 struct rte_eth_dev *dev = 880 &rte_eth_devices[pid]; 881 882 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 883 memhdr->len); 884 if (ret) { 885 TESTPMD_LOG(DEBUG, 886 "unable to DMA map addr 0x%p " 887 "for device %s\n", 888 memhdr->addr, dev->data->name); 889 } 890 } 891 } 892 893 static unsigned int 894 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 895 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 896 { 897 struct rte_pktmbuf_extmem *xmem; 898 unsigned int ext_num, zone_num, elt_num; 899 uint16_t elt_size; 900 901 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 902 elt_num = EXTBUF_ZONE_SIZE / elt_size; 903 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 904 905 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 906 if (xmem == NULL) { 907 TESTPMD_LOG(ERR, "Cannot allocate memory for " 908 "external buffer descriptors\n"); 909 *ext_mem = NULL; 910 return 0; 911 } 912 for (ext_num = 0; ext_num < zone_num; ext_num++) { 913 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 914 const struct rte_memzone *mz; 915 char mz_name[RTE_MEMZONE_NAMESIZE]; 916 int ret; 917 918 ret = snprintf(mz_name, sizeof(mz_name), 919 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 920 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 921 errno = ENAMETOOLONG; 922 ext_num = 0; 923 break; 924 } 925 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 926 socket_id, 927 RTE_MEMZONE_IOVA_CONTIG | 928 RTE_MEMZONE_1GB | 929 RTE_MEMZONE_SIZE_HINT_ONLY, 930 EXTBUF_ZONE_SIZE); 931 if (mz == NULL) { 932 /* 933 * The caller exits on external buffer creation 934 * error, so there is no need to free memzones. 935 */ 936 errno = ENOMEM; 937 ext_num = 0; 938 break; 939 } 940 xseg->buf_ptr = mz->addr; 941 xseg->buf_iova = mz->iova; 942 xseg->buf_len = EXTBUF_ZONE_SIZE; 943 xseg->elt_size = elt_size; 944 } 945 if (ext_num == 0 && xmem != NULL) { 946 free(xmem); 947 xmem = NULL; 948 } 949 *ext_mem = xmem; 950 return ext_num; 951 } 952 953 /* 954 * Configuration initialisation done once at init time. 955 */ 956 static struct rte_mempool * 957 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 958 unsigned int socket_id) 959 { 960 char pool_name[RTE_MEMPOOL_NAMESIZE]; 961 struct rte_mempool *rte_mp = NULL; 962 uint32_t mb_size; 963 964 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 965 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 966 967 TESTPMD_LOG(INFO, 968 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 969 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 970 971 switch (mp_alloc_type) { 972 case MP_ALLOC_NATIVE: 973 { 974 /* wrapper to rte_mempool_create() */ 975 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 976 rte_mbuf_best_mempool_ops()); 977 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 978 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 979 break; 980 } 981 case MP_ALLOC_ANON: 982 { 983 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 984 mb_size, (unsigned int) mb_mempool_cache, 985 sizeof(struct rte_pktmbuf_pool_private), 986 socket_id, mempool_flags); 987 if (rte_mp == NULL) 988 goto err; 989 990 if (rte_mempool_populate_anon(rte_mp) == 0) { 991 rte_mempool_free(rte_mp); 992 rte_mp = NULL; 993 goto err; 994 } 995 rte_pktmbuf_pool_init(rte_mp, NULL); 996 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 997 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 998 break; 999 } 1000 case MP_ALLOC_XMEM: 1001 case MP_ALLOC_XMEM_HUGE: 1002 { 1003 int heap_socket; 1004 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1005 1006 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1007 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1008 1009 heap_socket = 1010 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1011 if (heap_socket < 0) 1012 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1013 1014 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1015 rte_mbuf_best_mempool_ops()); 1016 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1017 mb_mempool_cache, 0, mbuf_seg_size, 1018 heap_socket); 1019 break; 1020 } 1021 case MP_ALLOC_XBUF: 1022 { 1023 struct rte_pktmbuf_extmem *ext_mem; 1024 unsigned int ext_num; 1025 1026 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1027 socket_id, pool_name, &ext_mem); 1028 if (ext_num == 0) 1029 rte_exit(EXIT_FAILURE, 1030 "Can't create pinned data buffers\n"); 1031 1032 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1033 rte_mbuf_best_mempool_ops()); 1034 rte_mp = rte_pktmbuf_pool_create_extbuf 1035 (pool_name, nb_mbuf, mb_mempool_cache, 1036 0, mbuf_seg_size, socket_id, 1037 ext_mem, ext_num); 1038 free(ext_mem); 1039 break; 1040 } 1041 default: 1042 { 1043 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1044 } 1045 } 1046 1047 err: 1048 if (rte_mp == NULL) { 1049 rte_exit(EXIT_FAILURE, 1050 "Creation of mbuf pool for socket %u failed: %s\n", 1051 socket_id, rte_strerror(rte_errno)); 1052 } else if (verbose_level > 0) { 1053 rte_mempool_dump(stdout, rte_mp); 1054 } 1055 return rte_mp; 1056 } 1057 1058 /* 1059 * Check given socket id is valid or not with NUMA mode, 1060 * if valid, return 0, else return -1 1061 */ 1062 static int 1063 check_socket_id(const unsigned int socket_id) 1064 { 1065 static int warning_once = 0; 1066 1067 if (new_socket_id(socket_id)) { 1068 if (!warning_once && numa_support) 1069 printf("Warning: NUMA should be configured manually by" 1070 " using --port-numa-config and" 1071 " --ring-numa-config parameters along with" 1072 " --numa.\n"); 1073 warning_once = 1; 1074 return -1; 1075 } 1076 return 0; 1077 } 1078 1079 /* 1080 * Get the allowed maximum number of RX queues. 1081 * *pid return the port id which has minimal value of 1082 * max_rx_queues in all ports. 1083 */ 1084 queueid_t 1085 get_allowed_max_nb_rxq(portid_t *pid) 1086 { 1087 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1088 bool max_rxq_valid = false; 1089 portid_t pi; 1090 struct rte_eth_dev_info dev_info; 1091 1092 RTE_ETH_FOREACH_DEV(pi) { 1093 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1094 continue; 1095 1096 max_rxq_valid = true; 1097 if (dev_info.max_rx_queues < allowed_max_rxq) { 1098 allowed_max_rxq = dev_info.max_rx_queues; 1099 *pid = pi; 1100 } 1101 } 1102 return max_rxq_valid ? allowed_max_rxq : 0; 1103 } 1104 1105 /* 1106 * Check input rxq is valid or not. 1107 * If input rxq is not greater than any of maximum number 1108 * of RX queues of all ports, it is valid. 1109 * if valid, return 0, else return -1 1110 */ 1111 int 1112 check_nb_rxq(queueid_t rxq) 1113 { 1114 queueid_t allowed_max_rxq; 1115 portid_t pid = 0; 1116 1117 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1118 if (rxq > allowed_max_rxq) { 1119 printf("Fail: input rxq (%u) can't be greater " 1120 "than max_rx_queues (%u) of port %u\n", 1121 rxq, 1122 allowed_max_rxq, 1123 pid); 1124 return -1; 1125 } 1126 return 0; 1127 } 1128 1129 /* 1130 * Get the allowed maximum number of TX queues. 1131 * *pid return the port id which has minimal value of 1132 * max_tx_queues in all ports. 1133 */ 1134 queueid_t 1135 get_allowed_max_nb_txq(portid_t *pid) 1136 { 1137 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1138 bool max_txq_valid = false; 1139 portid_t pi; 1140 struct rte_eth_dev_info dev_info; 1141 1142 RTE_ETH_FOREACH_DEV(pi) { 1143 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1144 continue; 1145 1146 max_txq_valid = true; 1147 if (dev_info.max_tx_queues < allowed_max_txq) { 1148 allowed_max_txq = dev_info.max_tx_queues; 1149 *pid = pi; 1150 } 1151 } 1152 return max_txq_valid ? allowed_max_txq : 0; 1153 } 1154 1155 /* 1156 * Check input txq is valid or not. 1157 * If input txq is not greater than any of maximum number 1158 * of TX queues of all ports, it is valid. 1159 * if valid, return 0, else return -1 1160 */ 1161 int 1162 check_nb_txq(queueid_t txq) 1163 { 1164 queueid_t allowed_max_txq; 1165 portid_t pid = 0; 1166 1167 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1168 if (txq > allowed_max_txq) { 1169 printf("Fail: input txq (%u) can't be greater " 1170 "than max_tx_queues (%u) of port %u\n", 1171 txq, 1172 allowed_max_txq, 1173 pid); 1174 return -1; 1175 } 1176 return 0; 1177 } 1178 1179 /* 1180 * Get the allowed maximum number of RXDs of every rx queue. 1181 * *pid return the port id which has minimal value of 1182 * max_rxd in all queues of all ports. 1183 */ 1184 static uint16_t 1185 get_allowed_max_nb_rxd(portid_t *pid) 1186 { 1187 uint16_t allowed_max_rxd = UINT16_MAX; 1188 portid_t pi; 1189 struct rte_eth_dev_info dev_info; 1190 1191 RTE_ETH_FOREACH_DEV(pi) { 1192 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1193 continue; 1194 1195 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1196 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1197 *pid = pi; 1198 } 1199 } 1200 return allowed_max_rxd; 1201 } 1202 1203 /* 1204 * Get the allowed minimal number of RXDs of every rx queue. 1205 * *pid return the port id which has minimal value of 1206 * min_rxd in all queues of all ports. 1207 */ 1208 static uint16_t 1209 get_allowed_min_nb_rxd(portid_t *pid) 1210 { 1211 uint16_t allowed_min_rxd = 0; 1212 portid_t pi; 1213 struct rte_eth_dev_info dev_info; 1214 1215 RTE_ETH_FOREACH_DEV(pi) { 1216 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1217 continue; 1218 1219 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1220 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1221 *pid = pi; 1222 } 1223 } 1224 1225 return allowed_min_rxd; 1226 } 1227 1228 /* 1229 * Check input rxd is valid or not. 1230 * If input rxd is not greater than any of maximum number 1231 * of RXDs of every Rx queues and is not less than any of 1232 * minimal number of RXDs of every Rx queues, it is valid. 1233 * if valid, return 0, else return -1 1234 */ 1235 int 1236 check_nb_rxd(queueid_t rxd) 1237 { 1238 uint16_t allowed_max_rxd; 1239 uint16_t allowed_min_rxd; 1240 portid_t pid = 0; 1241 1242 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1243 if (rxd > allowed_max_rxd) { 1244 printf("Fail: input rxd (%u) can't be greater " 1245 "than max_rxds (%u) of port %u\n", 1246 rxd, 1247 allowed_max_rxd, 1248 pid); 1249 return -1; 1250 } 1251 1252 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1253 if (rxd < allowed_min_rxd) { 1254 printf("Fail: input rxd (%u) can't be less " 1255 "than min_rxds (%u) of port %u\n", 1256 rxd, 1257 allowed_min_rxd, 1258 pid); 1259 return -1; 1260 } 1261 1262 return 0; 1263 } 1264 1265 /* 1266 * Get the allowed maximum number of TXDs of every rx queues. 1267 * *pid return the port id which has minimal value of 1268 * max_txd in every tx queue. 1269 */ 1270 static uint16_t 1271 get_allowed_max_nb_txd(portid_t *pid) 1272 { 1273 uint16_t allowed_max_txd = UINT16_MAX; 1274 portid_t pi; 1275 struct rte_eth_dev_info dev_info; 1276 1277 RTE_ETH_FOREACH_DEV(pi) { 1278 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1279 continue; 1280 1281 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1282 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1283 *pid = pi; 1284 } 1285 } 1286 return allowed_max_txd; 1287 } 1288 1289 /* 1290 * Get the allowed maximum number of TXDs of every tx queues. 1291 * *pid return the port id which has minimal value of 1292 * min_txd in every tx queue. 1293 */ 1294 static uint16_t 1295 get_allowed_min_nb_txd(portid_t *pid) 1296 { 1297 uint16_t allowed_min_txd = 0; 1298 portid_t pi; 1299 struct rte_eth_dev_info dev_info; 1300 1301 RTE_ETH_FOREACH_DEV(pi) { 1302 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1303 continue; 1304 1305 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1306 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1307 *pid = pi; 1308 } 1309 } 1310 1311 return allowed_min_txd; 1312 } 1313 1314 /* 1315 * Check input txd is valid or not. 1316 * If input txd is not greater than any of maximum number 1317 * of TXDs of every Rx queues, it is valid. 1318 * if valid, return 0, else return -1 1319 */ 1320 int 1321 check_nb_txd(queueid_t txd) 1322 { 1323 uint16_t allowed_max_txd; 1324 uint16_t allowed_min_txd; 1325 portid_t pid = 0; 1326 1327 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1328 if (txd > allowed_max_txd) { 1329 printf("Fail: input txd (%u) can't be greater " 1330 "than max_txds (%u) of port %u\n", 1331 txd, 1332 allowed_max_txd, 1333 pid); 1334 return -1; 1335 } 1336 1337 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1338 if (txd < allowed_min_txd) { 1339 printf("Fail: input txd (%u) can't be less " 1340 "than min_txds (%u) of port %u\n", 1341 txd, 1342 allowed_min_txd, 1343 pid); 1344 return -1; 1345 } 1346 return 0; 1347 } 1348 1349 1350 /* 1351 * Get the allowed maximum number of hairpin queues. 1352 * *pid return the port id which has minimal value of 1353 * max_hairpin_queues in all ports. 1354 */ 1355 queueid_t 1356 get_allowed_max_nb_hairpinq(portid_t *pid) 1357 { 1358 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1359 portid_t pi; 1360 struct rte_eth_hairpin_cap cap; 1361 1362 RTE_ETH_FOREACH_DEV(pi) { 1363 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1364 *pid = pi; 1365 return 0; 1366 } 1367 if (cap.max_nb_queues < allowed_max_hairpinq) { 1368 allowed_max_hairpinq = cap.max_nb_queues; 1369 *pid = pi; 1370 } 1371 } 1372 return allowed_max_hairpinq; 1373 } 1374 1375 /* 1376 * Check input hairpin is valid or not. 1377 * If input hairpin is not greater than any of maximum number 1378 * of hairpin queues of all ports, it is valid. 1379 * if valid, return 0, else return -1 1380 */ 1381 int 1382 check_nb_hairpinq(queueid_t hairpinq) 1383 { 1384 queueid_t allowed_max_hairpinq; 1385 portid_t pid = 0; 1386 1387 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1388 if (hairpinq > allowed_max_hairpinq) { 1389 printf("Fail: input hairpin (%u) can't be greater " 1390 "than max_hairpin_queues (%u) of port %u\n", 1391 hairpinq, allowed_max_hairpinq, pid); 1392 return -1; 1393 } 1394 return 0; 1395 } 1396 1397 static void 1398 init_config(void) 1399 { 1400 portid_t pid; 1401 struct rte_port *port; 1402 struct rte_mempool *mbp; 1403 unsigned int nb_mbuf_per_pool; 1404 lcoreid_t lc_id; 1405 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1406 struct rte_gro_param gro_param; 1407 uint32_t gso_types; 1408 uint16_t data_size; 1409 bool warning = 0; 1410 int k; 1411 int ret; 1412 1413 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1414 1415 /* Configuration of logical cores. */ 1416 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1417 sizeof(struct fwd_lcore *) * nb_lcores, 1418 RTE_CACHE_LINE_SIZE); 1419 if (fwd_lcores == NULL) { 1420 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1421 "failed\n", nb_lcores); 1422 } 1423 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1424 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1425 sizeof(struct fwd_lcore), 1426 RTE_CACHE_LINE_SIZE); 1427 if (fwd_lcores[lc_id] == NULL) { 1428 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1429 "failed\n"); 1430 } 1431 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1432 } 1433 1434 RTE_ETH_FOREACH_DEV(pid) { 1435 port = &ports[pid]; 1436 /* Apply default TxRx configuration for all ports */ 1437 port->dev_conf.txmode = tx_mode; 1438 port->dev_conf.rxmode = rx_mode; 1439 1440 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1441 if (ret != 0) 1442 rte_exit(EXIT_FAILURE, 1443 "rte_eth_dev_info_get() failed\n"); 1444 1445 if (!(port->dev_info.tx_offload_capa & 1446 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1447 port->dev_conf.txmode.offloads &= 1448 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1449 if (numa_support) { 1450 if (port_numa[pid] != NUMA_NO_CONFIG) 1451 port_per_socket[port_numa[pid]]++; 1452 else { 1453 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1454 1455 /* 1456 * if socket_id is invalid, 1457 * set to the first available socket. 1458 */ 1459 if (check_socket_id(socket_id) < 0) 1460 socket_id = socket_ids[0]; 1461 port_per_socket[socket_id]++; 1462 } 1463 } 1464 1465 /* Apply Rx offloads configuration */ 1466 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1467 port->rx_conf[k].offloads = 1468 port->dev_conf.rxmode.offloads; 1469 /* Apply Tx offloads configuration */ 1470 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1471 port->tx_conf[k].offloads = 1472 port->dev_conf.txmode.offloads; 1473 1474 /* set flag to initialize port/queue */ 1475 port->need_reconfig = 1; 1476 port->need_reconfig_queues = 1; 1477 port->tx_metadata = 0; 1478 1479 /* Check for maximum number of segments per MTU. Accordingly 1480 * update the mbuf data size. 1481 */ 1482 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1483 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1484 data_size = rx_mode.max_rx_pkt_len / 1485 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1486 1487 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1488 mbuf_data_size) { 1489 mbuf_data_size = data_size + 1490 RTE_PKTMBUF_HEADROOM; 1491 warning = 1; 1492 } 1493 } 1494 } 1495 1496 if (warning) 1497 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1498 mbuf_data_size); 1499 1500 /* 1501 * Create pools of mbuf. 1502 * If NUMA support is disabled, create a single pool of mbuf in 1503 * socket 0 memory by default. 1504 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1505 * 1506 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1507 * nb_txd can be configured at run time. 1508 */ 1509 if (param_total_num_mbufs) 1510 nb_mbuf_per_pool = param_total_num_mbufs; 1511 else { 1512 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1513 (nb_lcores * mb_mempool_cache) + 1514 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1515 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1516 } 1517 1518 if (numa_support) { 1519 uint8_t i; 1520 1521 for (i = 0; i < num_sockets; i++) 1522 mempools[i] = mbuf_pool_create(mbuf_data_size, 1523 nb_mbuf_per_pool, 1524 socket_ids[i]); 1525 } else { 1526 if (socket_num == UMA_NO_CONFIG) 1527 mempools[0] = mbuf_pool_create(mbuf_data_size, 1528 nb_mbuf_per_pool, 0); 1529 else 1530 mempools[socket_num] = mbuf_pool_create 1531 (mbuf_data_size, 1532 nb_mbuf_per_pool, 1533 socket_num); 1534 } 1535 1536 init_port_config(); 1537 1538 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1539 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1540 /* 1541 * Records which Mbuf pool to use by each logical core, if needed. 1542 */ 1543 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1544 mbp = mbuf_pool_find( 1545 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1546 1547 if (mbp == NULL) 1548 mbp = mbuf_pool_find(0); 1549 fwd_lcores[lc_id]->mbp = mbp; 1550 /* initialize GSO context */ 1551 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1552 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1553 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1554 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1555 RTE_ETHER_CRC_LEN; 1556 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1557 } 1558 1559 /* Configuration of packet forwarding streams. */ 1560 if (init_fwd_streams() < 0) 1561 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1562 1563 fwd_config_setup(); 1564 1565 /* create a gro context for each lcore */ 1566 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1567 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1568 gro_param.max_item_per_flow = MAX_PKT_BURST; 1569 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1570 gro_param.socket_id = rte_lcore_to_socket_id( 1571 fwd_lcores_cpuids[lc_id]); 1572 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1573 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1574 rte_exit(EXIT_FAILURE, 1575 "rte_gro_ctx_create() failed\n"); 1576 } 1577 } 1578 } 1579 1580 1581 void 1582 reconfig(portid_t new_port_id, unsigned socket_id) 1583 { 1584 struct rte_port *port; 1585 int ret; 1586 1587 /* Reconfiguration of Ethernet ports. */ 1588 port = &ports[new_port_id]; 1589 1590 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1591 if (ret != 0) 1592 return; 1593 1594 /* set flag to initialize port/queue */ 1595 port->need_reconfig = 1; 1596 port->need_reconfig_queues = 1; 1597 port->socket_id = socket_id; 1598 1599 init_port_config(); 1600 } 1601 1602 1603 int 1604 init_fwd_streams(void) 1605 { 1606 portid_t pid; 1607 struct rte_port *port; 1608 streamid_t sm_id, nb_fwd_streams_new; 1609 queueid_t q; 1610 1611 /* set socket id according to numa or not */ 1612 RTE_ETH_FOREACH_DEV(pid) { 1613 port = &ports[pid]; 1614 if (nb_rxq > port->dev_info.max_rx_queues) { 1615 printf("Fail: nb_rxq(%d) is greater than " 1616 "max_rx_queues(%d)\n", nb_rxq, 1617 port->dev_info.max_rx_queues); 1618 return -1; 1619 } 1620 if (nb_txq > port->dev_info.max_tx_queues) { 1621 printf("Fail: nb_txq(%d) is greater than " 1622 "max_tx_queues(%d)\n", nb_txq, 1623 port->dev_info.max_tx_queues); 1624 return -1; 1625 } 1626 if (numa_support) { 1627 if (port_numa[pid] != NUMA_NO_CONFIG) 1628 port->socket_id = port_numa[pid]; 1629 else { 1630 port->socket_id = rte_eth_dev_socket_id(pid); 1631 1632 /* 1633 * if socket_id is invalid, 1634 * set to the first available socket. 1635 */ 1636 if (check_socket_id(port->socket_id) < 0) 1637 port->socket_id = socket_ids[0]; 1638 } 1639 } 1640 else { 1641 if (socket_num == UMA_NO_CONFIG) 1642 port->socket_id = 0; 1643 else 1644 port->socket_id = socket_num; 1645 } 1646 } 1647 1648 q = RTE_MAX(nb_rxq, nb_txq); 1649 if (q == 0) { 1650 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1651 return -1; 1652 } 1653 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1654 if (nb_fwd_streams_new == nb_fwd_streams) 1655 return 0; 1656 /* clear the old */ 1657 if (fwd_streams != NULL) { 1658 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1659 if (fwd_streams[sm_id] == NULL) 1660 continue; 1661 rte_free(fwd_streams[sm_id]); 1662 fwd_streams[sm_id] = NULL; 1663 } 1664 rte_free(fwd_streams); 1665 fwd_streams = NULL; 1666 } 1667 1668 /* init new */ 1669 nb_fwd_streams = nb_fwd_streams_new; 1670 if (nb_fwd_streams) { 1671 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1672 sizeof(struct fwd_stream *) * nb_fwd_streams, 1673 RTE_CACHE_LINE_SIZE); 1674 if (fwd_streams == NULL) 1675 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1676 " (struct fwd_stream *)) failed\n", 1677 nb_fwd_streams); 1678 1679 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1680 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1681 " struct fwd_stream", sizeof(struct fwd_stream), 1682 RTE_CACHE_LINE_SIZE); 1683 if (fwd_streams[sm_id] == NULL) 1684 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1685 "(struct fwd_stream) failed\n"); 1686 } 1687 } 1688 1689 return 0; 1690 } 1691 1692 static void 1693 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1694 { 1695 uint64_t total_burst, sburst; 1696 uint64_t nb_burst; 1697 uint64_t burst_stats[4]; 1698 uint16_t pktnb_stats[4]; 1699 uint16_t nb_pkt; 1700 int burst_percent[4], sburstp; 1701 int i; 1702 1703 /* 1704 * First compute the total number of packet bursts and the 1705 * two highest numbers of bursts of the same number of packets. 1706 */ 1707 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1708 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1709 1710 /* Show stats for 0 burst size always */ 1711 total_burst = pbs->pkt_burst_spread[0]; 1712 burst_stats[0] = pbs->pkt_burst_spread[0]; 1713 pktnb_stats[0] = 0; 1714 1715 /* Find the next 2 burst sizes with highest occurrences. */ 1716 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1717 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1718 1719 if (nb_burst == 0) 1720 continue; 1721 1722 total_burst += nb_burst; 1723 1724 if (nb_burst > burst_stats[1]) { 1725 burst_stats[2] = burst_stats[1]; 1726 pktnb_stats[2] = pktnb_stats[1]; 1727 burst_stats[1] = nb_burst; 1728 pktnb_stats[1] = nb_pkt; 1729 } else if (nb_burst > burst_stats[2]) { 1730 burst_stats[2] = nb_burst; 1731 pktnb_stats[2] = nb_pkt; 1732 } 1733 } 1734 if (total_burst == 0) 1735 return; 1736 1737 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1738 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1739 if (i == 3) { 1740 printf("%d%% of other]\n", 100 - sburstp); 1741 return; 1742 } 1743 1744 sburst += burst_stats[i]; 1745 if (sburst == total_burst) { 1746 printf("%d%% of %d pkts]\n", 1747 100 - sburstp, (int) pktnb_stats[i]); 1748 return; 1749 } 1750 1751 burst_percent[i] = 1752 (double)burst_stats[i] / total_burst * 100; 1753 printf("%d%% of %d pkts + ", 1754 burst_percent[i], (int) pktnb_stats[i]); 1755 sburstp += burst_percent[i]; 1756 } 1757 } 1758 1759 static void 1760 fwd_stream_stats_display(streamid_t stream_id) 1761 { 1762 struct fwd_stream *fs; 1763 static const char *fwd_top_stats_border = "-------"; 1764 1765 fs = fwd_streams[stream_id]; 1766 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1767 (fs->fwd_dropped == 0)) 1768 return; 1769 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1770 "TX Port=%2d/Queue=%2d %s\n", 1771 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1772 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1773 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1774 " TX-dropped: %-14"PRIu64, 1775 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1776 1777 /* if checksum mode */ 1778 if (cur_fwd_eng == &csum_fwd_engine) { 1779 printf(" RX- bad IP checksum: %-14"PRIu64 1780 " Rx- bad L4 checksum: %-14"PRIu64 1781 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1782 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1783 fs->rx_bad_outer_l4_csum); 1784 } else { 1785 printf("\n"); 1786 } 1787 1788 if (record_burst_stats) { 1789 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1790 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1791 } 1792 } 1793 1794 void 1795 fwd_stats_display(void) 1796 { 1797 static const char *fwd_stats_border = "----------------------"; 1798 static const char *acc_stats_border = "+++++++++++++++"; 1799 struct { 1800 struct fwd_stream *rx_stream; 1801 struct fwd_stream *tx_stream; 1802 uint64_t tx_dropped; 1803 uint64_t rx_bad_ip_csum; 1804 uint64_t rx_bad_l4_csum; 1805 uint64_t rx_bad_outer_l4_csum; 1806 } ports_stats[RTE_MAX_ETHPORTS]; 1807 uint64_t total_rx_dropped = 0; 1808 uint64_t total_tx_dropped = 0; 1809 uint64_t total_rx_nombuf = 0; 1810 struct rte_eth_stats stats; 1811 uint64_t fwd_cycles = 0; 1812 uint64_t total_recv = 0; 1813 uint64_t total_xmit = 0; 1814 struct rte_port *port; 1815 streamid_t sm_id; 1816 portid_t pt_id; 1817 int i; 1818 1819 memset(ports_stats, 0, sizeof(ports_stats)); 1820 1821 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1822 struct fwd_stream *fs = fwd_streams[sm_id]; 1823 1824 if (cur_fwd_config.nb_fwd_streams > 1825 cur_fwd_config.nb_fwd_ports) { 1826 fwd_stream_stats_display(sm_id); 1827 } else { 1828 ports_stats[fs->tx_port].tx_stream = fs; 1829 ports_stats[fs->rx_port].rx_stream = fs; 1830 } 1831 1832 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1833 1834 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1835 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1836 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1837 fs->rx_bad_outer_l4_csum; 1838 1839 if (record_core_cycles) 1840 fwd_cycles += fs->core_cycles; 1841 } 1842 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1843 uint8_t j; 1844 1845 pt_id = fwd_ports_ids[i]; 1846 port = &ports[pt_id]; 1847 1848 rte_eth_stats_get(pt_id, &stats); 1849 stats.ipackets -= port->stats.ipackets; 1850 stats.opackets -= port->stats.opackets; 1851 stats.ibytes -= port->stats.ibytes; 1852 stats.obytes -= port->stats.obytes; 1853 stats.imissed -= port->stats.imissed; 1854 stats.oerrors -= port->stats.oerrors; 1855 stats.rx_nombuf -= port->stats.rx_nombuf; 1856 1857 total_recv += stats.ipackets; 1858 total_xmit += stats.opackets; 1859 total_rx_dropped += stats.imissed; 1860 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1861 total_tx_dropped += stats.oerrors; 1862 total_rx_nombuf += stats.rx_nombuf; 1863 1864 printf("\n %s Forward statistics for port %-2d %s\n", 1865 fwd_stats_border, pt_id, fwd_stats_border); 1866 1867 if (!port->rx_queue_stats_mapping_enabled && 1868 !port->tx_queue_stats_mapping_enabled) { 1869 printf(" RX-packets: %-14"PRIu64 1870 " RX-dropped: %-14"PRIu64 1871 "RX-total: %-"PRIu64"\n", 1872 stats.ipackets, stats.imissed, 1873 stats.ipackets + stats.imissed); 1874 1875 if (cur_fwd_eng == &csum_fwd_engine) 1876 printf(" Bad-ipcsum: %-14"PRIu64 1877 " Bad-l4csum: %-14"PRIu64 1878 "Bad-outer-l4csum: %-14"PRIu64"\n", 1879 ports_stats[pt_id].rx_bad_ip_csum, 1880 ports_stats[pt_id].rx_bad_l4_csum, 1881 ports_stats[pt_id].rx_bad_outer_l4_csum); 1882 if (stats.ierrors + stats.rx_nombuf > 0) { 1883 printf(" RX-error: %-"PRIu64"\n", 1884 stats.ierrors); 1885 printf(" RX-nombufs: %-14"PRIu64"\n", 1886 stats.rx_nombuf); 1887 } 1888 1889 printf(" TX-packets: %-14"PRIu64 1890 " TX-dropped: %-14"PRIu64 1891 "TX-total: %-"PRIu64"\n", 1892 stats.opackets, ports_stats[pt_id].tx_dropped, 1893 stats.opackets + ports_stats[pt_id].tx_dropped); 1894 } else { 1895 printf(" RX-packets: %14"PRIu64 1896 " RX-dropped:%14"PRIu64 1897 " RX-total:%14"PRIu64"\n", 1898 stats.ipackets, stats.imissed, 1899 stats.ipackets + stats.imissed); 1900 1901 if (cur_fwd_eng == &csum_fwd_engine) 1902 printf(" Bad-ipcsum:%14"PRIu64 1903 " Bad-l4csum:%14"PRIu64 1904 " Bad-outer-l4csum: %-14"PRIu64"\n", 1905 ports_stats[pt_id].rx_bad_ip_csum, 1906 ports_stats[pt_id].rx_bad_l4_csum, 1907 ports_stats[pt_id].rx_bad_outer_l4_csum); 1908 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1909 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1910 printf(" RX-nombufs: %14"PRIu64"\n", 1911 stats.rx_nombuf); 1912 } 1913 1914 printf(" TX-packets: %14"PRIu64 1915 " TX-dropped:%14"PRIu64 1916 " TX-total:%14"PRIu64"\n", 1917 stats.opackets, ports_stats[pt_id].tx_dropped, 1918 stats.opackets + ports_stats[pt_id].tx_dropped); 1919 } 1920 1921 if (record_burst_stats) { 1922 if (ports_stats[pt_id].rx_stream) 1923 pkt_burst_stats_display("RX", 1924 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1925 if (ports_stats[pt_id].tx_stream) 1926 pkt_burst_stats_display("TX", 1927 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1928 } 1929 1930 if (port->rx_queue_stats_mapping_enabled) { 1931 printf("\n"); 1932 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1933 printf(" Stats reg %2d RX-packets:%14"PRIu64 1934 " RX-errors:%14"PRIu64 1935 " RX-bytes:%14"PRIu64"\n", 1936 j, stats.q_ipackets[j], 1937 stats.q_errors[j], stats.q_ibytes[j]); 1938 } 1939 printf("\n"); 1940 } 1941 if (port->tx_queue_stats_mapping_enabled) { 1942 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1943 printf(" Stats reg %2d TX-packets:%14"PRIu64 1944 " TX-bytes:%14" 1945 PRIu64"\n", 1946 j, stats.q_opackets[j], 1947 stats.q_obytes[j]); 1948 } 1949 } 1950 1951 printf(" %s--------------------------------%s\n", 1952 fwd_stats_border, fwd_stats_border); 1953 } 1954 1955 printf("\n %s Accumulated forward statistics for all ports" 1956 "%s\n", 1957 acc_stats_border, acc_stats_border); 1958 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1959 "%-"PRIu64"\n" 1960 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1961 "%-"PRIu64"\n", 1962 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1963 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1964 if (total_rx_nombuf > 0) 1965 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1966 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1967 "%s\n", 1968 acc_stats_border, acc_stats_border); 1969 if (record_core_cycles) { 1970 #define CYC_PER_MHZ 1E6 1971 if (total_recv > 0 || total_xmit > 0) { 1972 uint64_t total_pkts = 0; 1973 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1974 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1975 total_pkts = total_xmit; 1976 else 1977 total_pkts = total_recv; 1978 1979 printf("\n CPU cycles/packet=%.2F (total cycles=" 1980 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1981 " MHz Clock\n", 1982 (double) fwd_cycles / total_pkts, 1983 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1984 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1985 } 1986 } 1987 } 1988 1989 void 1990 fwd_stats_reset(void) 1991 { 1992 streamid_t sm_id; 1993 portid_t pt_id; 1994 int i; 1995 1996 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1997 pt_id = fwd_ports_ids[i]; 1998 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1999 } 2000 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2001 struct fwd_stream *fs = fwd_streams[sm_id]; 2002 2003 fs->rx_packets = 0; 2004 fs->tx_packets = 0; 2005 fs->fwd_dropped = 0; 2006 fs->rx_bad_ip_csum = 0; 2007 fs->rx_bad_l4_csum = 0; 2008 fs->rx_bad_outer_l4_csum = 0; 2009 2010 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2011 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2012 fs->core_cycles = 0; 2013 } 2014 } 2015 2016 static void 2017 flush_fwd_rx_queues(void) 2018 { 2019 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2020 portid_t rxp; 2021 portid_t port_id; 2022 queueid_t rxq; 2023 uint16_t nb_rx; 2024 uint16_t i; 2025 uint8_t j; 2026 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2027 uint64_t timer_period; 2028 2029 /* convert to number of cycles */ 2030 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2031 2032 for (j = 0; j < 2; j++) { 2033 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2034 for (rxq = 0; rxq < nb_rxq; rxq++) { 2035 port_id = fwd_ports_ids[rxp]; 2036 /** 2037 * testpmd can stuck in the below do while loop 2038 * if rte_eth_rx_burst() always returns nonzero 2039 * packets. So timer is added to exit this loop 2040 * after 1sec timer expiry. 2041 */ 2042 prev_tsc = rte_rdtsc(); 2043 do { 2044 nb_rx = rte_eth_rx_burst(port_id, rxq, 2045 pkts_burst, MAX_PKT_BURST); 2046 for (i = 0; i < nb_rx; i++) 2047 rte_pktmbuf_free(pkts_burst[i]); 2048 2049 cur_tsc = rte_rdtsc(); 2050 diff_tsc = cur_tsc - prev_tsc; 2051 timer_tsc += diff_tsc; 2052 } while ((nb_rx > 0) && 2053 (timer_tsc < timer_period)); 2054 timer_tsc = 0; 2055 } 2056 } 2057 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2058 } 2059 } 2060 2061 static void 2062 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2063 { 2064 struct fwd_stream **fsm; 2065 streamid_t nb_fs; 2066 streamid_t sm_id; 2067 #ifdef RTE_LIBRTE_BITRATESTATS 2068 uint64_t tics_per_1sec; 2069 uint64_t tics_datum; 2070 uint64_t tics_current; 2071 uint16_t i, cnt_ports; 2072 2073 cnt_ports = nb_ports; 2074 tics_datum = rte_rdtsc(); 2075 tics_per_1sec = rte_get_timer_hz(); 2076 #endif 2077 fsm = &fwd_streams[fc->stream_idx]; 2078 nb_fs = fc->stream_nb; 2079 do { 2080 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2081 (*pkt_fwd)(fsm[sm_id]); 2082 #ifdef RTE_LIBRTE_BITRATESTATS 2083 if (bitrate_enabled != 0 && 2084 bitrate_lcore_id == rte_lcore_id()) { 2085 tics_current = rte_rdtsc(); 2086 if (tics_current - tics_datum >= tics_per_1sec) { 2087 /* Periodic bitrate calculation */ 2088 for (i = 0; i < cnt_ports; i++) 2089 rte_stats_bitrate_calc(bitrate_data, 2090 ports_ids[i]); 2091 tics_datum = tics_current; 2092 } 2093 } 2094 #endif 2095 #ifdef RTE_LIBRTE_LATENCY_STATS 2096 if (latencystats_enabled != 0 && 2097 latencystats_lcore_id == rte_lcore_id()) 2098 rte_latencystats_update(); 2099 #endif 2100 2101 } while (! fc->stopped); 2102 } 2103 2104 static int 2105 start_pkt_forward_on_core(void *fwd_arg) 2106 { 2107 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2108 cur_fwd_config.fwd_eng->packet_fwd); 2109 return 0; 2110 } 2111 2112 /* 2113 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2114 * Used to start communication flows in network loopback test configurations. 2115 */ 2116 static int 2117 run_one_txonly_burst_on_core(void *fwd_arg) 2118 { 2119 struct fwd_lcore *fwd_lc; 2120 struct fwd_lcore tmp_lcore; 2121 2122 fwd_lc = (struct fwd_lcore *) fwd_arg; 2123 tmp_lcore = *fwd_lc; 2124 tmp_lcore.stopped = 1; 2125 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2126 return 0; 2127 } 2128 2129 /* 2130 * Launch packet forwarding: 2131 * - Setup per-port forwarding context. 2132 * - launch logical cores with their forwarding configuration. 2133 */ 2134 static void 2135 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2136 { 2137 port_fwd_begin_t port_fwd_begin; 2138 unsigned int i; 2139 unsigned int lc_id; 2140 int diag; 2141 2142 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2143 if (port_fwd_begin != NULL) { 2144 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2145 (*port_fwd_begin)(fwd_ports_ids[i]); 2146 } 2147 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2148 lc_id = fwd_lcores_cpuids[i]; 2149 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2150 fwd_lcores[i]->stopped = 0; 2151 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2152 fwd_lcores[i], lc_id); 2153 if (diag != 0) 2154 printf("launch lcore %u failed - diag=%d\n", 2155 lc_id, diag); 2156 } 2157 } 2158 } 2159 2160 /* 2161 * Launch packet forwarding configuration. 2162 */ 2163 void 2164 start_packet_forwarding(int with_tx_first) 2165 { 2166 port_fwd_begin_t port_fwd_begin; 2167 port_fwd_end_t port_fwd_end; 2168 struct rte_port *port; 2169 unsigned int i; 2170 portid_t pt_id; 2171 2172 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2173 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2174 2175 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2176 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2177 2178 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2179 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2180 (!nb_rxq || !nb_txq)) 2181 rte_exit(EXIT_FAILURE, 2182 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2183 cur_fwd_eng->fwd_mode_name); 2184 2185 if (all_ports_started() == 0) { 2186 printf("Not all ports were started\n"); 2187 return; 2188 } 2189 if (test_done == 0) { 2190 printf("Packet forwarding already started\n"); 2191 return; 2192 } 2193 2194 2195 if(dcb_test) { 2196 for (i = 0; i < nb_fwd_ports; i++) { 2197 pt_id = fwd_ports_ids[i]; 2198 port = &ports[pt_id]; 2199 if (!port->dcb_flag) { 2200 printf("In DCB mode, all forwarding ports must " 2201 "be configured in this mode.\n"); 2202 return; 2203 } 2204 } 2205 if (nb_fwd_lcores == 1) { 2206 printf("In DCB mode,the nb forwarding cores " 2207 "should be larger than 1.\n"); 2208 return; 2209 } 2210 } 2211 test_done = 0; 2212 2213 fwd_config_setup(); 2214 2215 if(!no_flush_rx) 2216 flush_fwd_rx_queues(); 2217 2218 pkt_fwd_config_display(&cur_fwd_config); 2219 rxtx_config_display(); 2220 2221 fwd_stats_reset(); 2222 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2223 pt_id = fwd_ports_ids[i]; 2224 port = &ports[pt_id]; 2225 map_port_queue_stats_mapping_registers(pt_id, port); 2226 } 2227 if (with_tx_first) { 2228 port_fwd_begin = tx_only_engine.port_fwd_begin; 2229 if (port_fwd_begin != NULL) { 2230 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2231 (*port_fwd_begin)(fwd_ports_ids[i]); 2232 } 2233 while (with_tx_first--) { 2234 launch_packet_forwarding( 2235 run_one_txonly_burst_on_core); 2236 rte_eal_mp_wait_lcore(); 2237 } 2238 port_fwd_end = tx_only_engine.port_fwd_end; 2239 if (port_fwd_end != NULL) { 2240 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2241 (*port_fwd_end)(fwd_ports_ids[i]); 2242 } 2243 } 2244 launch_packet_forwarding(start_pkt_forward_on_core); 2245 } 2246 2247 void 2248 stop_packet_forwarding(void) 2249 { 2250 port_fwd_end_t port_fwd_end; 2251 lcoreid_t lc_id; 2252 portid_t pt_id; 2253 int i; 2254 2255 if (test_done) { 2256 printf("Packet forwarding not started\n"); 2257 return; 2258 } 2259 printf("Telling cores to stop..."); 2260 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2261 fwd_lcores[lc_id]->stopped = 1; 2262 printf("\nWaiting for lcores to finish...\n"); 2263 rte_eal_mp_wait_lcore(); 2264 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2265 if (port_fwd_end != NULL) { 2266 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2267 pt_id = fwd_ports_ids[i]; 2268 (*port_fwd_end)(pt_id); 2269 } 2270 } 2271 2272 fwd_stats_display(); 2273 2274 printf("\nDone.\n"); 2275 test_done = 1; 2276 } 2277 2278 void 2279 dev_set_link_up(portid_t pid) 2280 { 2281 if (rte_eth_dev_set_link_up(pid) < 0) 2282 printf("\nSet link up fail.\n"); 2283 } 2284 2285 void 2286 dev_set_link_down(portid_t pid) 2287 { 2288 if (rte_eth_dev_set_link_down(pid) < 0) 2289 printf("\nSet link down fail.\n"); 2290 } 2291 2292 static int 2293 all_ports_started(void) 2294 { 2295 portid_t pi; 2296 struct rte_port *port; 2297 2298 RTE_ETH_FOREACH_DEV(pi) { 2299 port = &ports[pi]; 2300 /* Check if there is a port which is not started */ 2301 if ((port->port_status != RTE_PORT_STARTED) && 2302 (port->slave_flag == 0)) 2303 return 0; 2304 } 2305 2306 /* No port is not started */ 2307 return 1; 2308 } 2309 2310 int 2311 port_is_stopped(portid_t port_id) 2312 { 2313 struct rte_port *port = &ports[port_id]; 2314 2315 if ((port->port_status != RTE_PORT_STOPPED) && 2316 (port->slave_flag == 0)) 2317 return 0; 2318 return 1; 2319 } 2320 2321 int 2322 all_ports_stopped(void) 2323 { 2324 portid_t pi; 2325 2326 RTE_ETH_FOREACH_DEV(pi) { 2327 if (!port_is_stopped(pi)) 2328 return 0; 2329 } 2330 2331 return 1; 2332 } 2333 2334 int 2335 port_is_started(portid_t port_id) 2336 { 2337 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2338 return 0; 2339 2340 if (ports[port_id].port_status != RTE_PORT_STARTED) 2341 return 0; 2342 2343 return 1; 2344 } 2345 2346 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2347 static int 2348 setup_hairpin_queues(portid_t pi) 2349 { 2350 queueid_t qi; 2351 struct rte_eth_hairpin_conf hairpin_conf = { 2352 .peer_count = 1, 2353 }; 2354 int i; 2355 int diag; 2356 struct rte_port *port = &ports[pi]; 2357 2358 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2359 hairpin_conf.peers[0].port = pi; 2360 hairpin_conf.peers[0].queue = i + nb_rxq; 2361 diag = rte_eth_tx_hairpin_queue_setup 2362 (pi, qi, nb_txd, &hairpin_conf); 2363 i++; 2364 if (diag == 0) 2365 continue; 2366 2367 /* Fail to setup rx queue, return */ 2368 if (rte_atomic16_cmpset(&(port->port_status), 2369 RTE_PORT_HANDLING, 2370 RTE_PORT_STOPPED) == 0) 2371 printf("Port %d can not be set back " 2372 "to stopped\n", pi); 2373 printf("Fail to configure port %d hairpin " 2374 "queues\n", pi); 2375 /* try to reconfigure queues next time */ 2376 port->need_reconfig_queues = 1; 2377 return -1; 2378 } 2379 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2380 hairpin_conf.peers[0].port = pi; 2381 hairpin_conf.peers[0].queue = i + nb_txq; 2382 diag = rte_eth_rx_hairpin_queue_setup 2383 (pi, qi, nb_rxd, &hairpin_conf); 2384 i++; 2385 if (diag == 0) 2386 continue; 2387 2388 /* Fail to setup rx queue, return */ 2389 if (rte_atomic16_cmpset(&(port->port_status), 2390 RTE_PORT_HANDLING, 2391 RTE_PORT_STOPPED) == 0) 2392 printf("Port %d can not be set back " 2393 "to stopped\n", pi); 2394 printf("Fail to configure port %d hairpin " 2395 "queues\n", pi); 2396 /* try to reconfigure queues next time */ 2397 port->need_reconfig_queues = 1; 2398 return -1; 2399 } 2400 return 0; 2401 } 2402 2403 int 2404 start_port(portid_t pid) 2405 { 2406 int diag, need_check_link_status = -1; 2407 portid_t pi; 2408 queueid_t qi; 2409 struct rte_port *port; 2410 struct rte_ether_addr mac_addr; 2411 struct rte_eth_hairpin_cap cap; 2412 2413 if (port_id_is_invalid(pid, ENABLED_WARN)) 2414 return 0; 2415 2416 if(dcb_config) 2417 dcb_test = 1; 2418 RTE_ETH_FOREACH_DEV(pi) { 2419 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2420 continue; 2421 2422 need_check_link_status = 0; 2423 port = &ports[pi]; 2424 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2425 RTE_PORT_HANDLING) == 0) { 2426 printf("Port %d is now not stopped\n", pi); 2427 continue; 2428 } 2429 2430 if (port->need_reconfig > 0) { 2431 port->need_reconfig = 0; 2432 2433 if (flow_isolate_all) { 2434 int ret = port_flow_isolate(pi, 1); 2435 if (ret) { 2436 printf("Failed to apply isolated" 2437 " mode on port %d\n", pi); 2438 return -1; 2439 } 2440 } 2441 configure_rxtx_dump_callbacks(0); 2442 printf("Configuring Port %d (socket %u)\n", pi, 2443 port->socket_id); 2444 if (nb_hairpinq > 0 && 2445 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2446 printf("Port %d doesn't support hairpin " 2447 "queues\n", pi); 2448 return -1; 2449 } 2450 /* configure port */ 2451 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2452 nb_txq + nb_hairpinq, 2453 &(port->dev_conf)); 2454 if (diag != 0) { 2455 if (rte_atomic16_cmpset(&(port->port_status), 2456 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2457 printf("Port %d can not be set back " 2458 "to stopped\n", pi); 2459 printf("Fail to configure port %d\n", pi); 2460 /* try to reconfigure port next time */ 2461 port->need_reconfig = 1; 2462 return -1; 2463 } 2464 } 2465 if (port->need_reconfig_queues > 0) { 2466 port->need_reconfig_queues = 0; 2467 /* setup tx queues */ 2468 for (qi = 0; qi < nb_txq; qi++) { 2469 if ((numa_support) && 2470 (txring_numa[pi] != NUMA_NO_CONFIG)) 2471 diag = rte_eth_tx_queue_setup(pi, qi, 2472 port->nb_tx_desc[qi], 2473 txring_numa[pi], 2474 &(port->tx_conf[qi])); 2475 else 2476 diag = rte_eth_tx_queue_setup(pi, qi, 2477 port->nb_tx_desc[qi], 2478 port->socket_id, 2479 &(port->tx_conf[qi])); 2480 2481 if (diag == 0) 2482 continue; 2483 2484 /* Fail to setup tx queue, return */ 2485 if (rte_atomic16_cmpset(&(port->port_status), 2486 RTE_PORT_HANDLING, 2487 RTE_PORT_STOPPED) == 0) 2488 printf("Port %d can not be set back " 2489 "to stopped\n", pi); 2490 printf("Fail to configure port %d tx queues\n", 2491 pi); 2492 /* try to reconfigure queues next time */ 2493 port->need_reconfig_queues = 1; 2494 return -1; 2495 } 2496 for (qi = 0; qi < nb_rxq; qi++) { 2497 /* setup rx queues */ 2498 if ((numa_support) && 2499 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2500 struct rte_mempool * mp = 2501 mbuf_pool_find(rxring_numa[pi]); 2502 if (mp == NULL) { 2503 printf("Failed to setup RX queue:" 2504 "No mempool allocation" 2505 " on the socket %d\n", 2506 rxring_numa[pi]); 2507 return -1; 2508 } 2509 2510 diag = rte_eth_rx_queue_setup(pi, qi, 2511 port->nb_rx_desc[qi], 2512 rxring_numa[pi], 2513 &(port->rx_conf[qi]), 2514 mp); 2515 } else { 2516 struct rte_mempool *mp = 2517 mbuf_pool_find(port->socket_id); 2518 if (mp == NULL) { 2519 printf("Failed to setup RX queue:" 2520 "No mempool allocation" 2521 " on the socket %d\n", 2522 port->socket_id); 2523 return -1; 2524 } 2525 diag = rte_eth_rx_queue_setup(pi, qi, 2526 port->nb_rx_desc[qi], 2527 port->socket_id, 2528 &(port->rx_conf[qi]), 2529 mp); 2530 } 2531 if (diag == 0) 2532 continue; 2533 2534 /* Fail to setup rx queue, return */ 2535 if (rte_atomic16_cmpset(&(port->port_status), 2536 RTE_PORT_HANDLING, 2537 RTE_PORT_STOPPED) == 0) 2538 printf("Port %d can not be set back " 2539 "to stopped\n", pi); 2540 printf("Fail to configure port %d rx queues\n", 2541 pi); 2542 /* try to reconfigure queues next time */ 2543 port->need_reconfig_queues = 1; 2544 return -1; 2545 } 2546 /* setup hairpin queues */ 2547 if (setup_hairpin_queues(pi) != 0) 2548 return -1; 2549 } 2550 configure_rxtx_dump_callbacks(verbose_level); 2551 if (clear_ptypes) { 2552 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2553 NULL, 0); 2554 if (diag < 0) 2555 printf( 2556 "Port %d: Failed to disable Ptype parsing\n", 2557 pi); 2558 } 2559 2560 /* start port */ 2561 if (rte_eth_dev_start(pi) < 0) { 2562 printf("Fail to start port %d\n", pi); 2563 2564 /* Fail to setup rx queue, return */ 2565 if (rte_atomic16_cmpset(&(port->port_status), 2566 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2567 printf("Port %d can not be set back to " 2568 "stopped\n", pi); 2569 continue; 2570 } 2571 2572 if (rte_atomic16_cmpset(&(port->port_status), 2573 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2574 printf("Port %d can not be set into started\n", pi); 2575 2576 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2577 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2578 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2579 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2580 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2581 2582 /* at least one port started, need checking link status */ 2583 need_check_link_status = 1; 2584 } 2585 2586 if (need_check_link_status == 1 && !no_link_check) 2587 check_all_ports_link_status(RTE_PORT_ALL); 2588 else if (need_check_link_status == 0) 2589 printf("Please stop the ports first\n"); 2590 2591 printf("Done\n"); 2592 return 0; 2593 } 2594 2595 void 2596 stop_port(portid_t pid) 2597 { 2598 portid_t pi; 2599 struct rte_port *port; 2600 int need_check_link_status = 0; 2601 2602 if (dcb_test) { 2603 dcb_test = 0; 2604 dcb_config = 0; 2605 } 2606 2607 if (port_id_is_invalid(pid, ENABLED_WARN)) 2608 return; 2609 2610 printf("Stopping ports...\n"); 2611 2612 RTE_ETH_FOREACH_DEV(pi) { 2613 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2614 continue; 2615 2616 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2617 printf("Please remove port %d from forwarding configuration.\n", pi); 2618 continue; 2619 } 2620 2621 if (port_is_bonding_slave(pi)) { 2622 printf("Please remove port %d from bonded device.\n", pi); 2623 continue; 2624 } 2625 2626 port = &ports[pi]; 2627 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2628 RTE_PORT_HANDLING) == 0) 2629 continue; 2630 2631 rte_eth_dev_stop(pi); 2632 2633 if (rte_atomic16_cmpset(&(port->port_status), 2634 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2635 printf("Port %d can not be set into stopped\n", pi); 2636 need_check_link_status = 1; 2637 } 2638 if (need_check_link_status && !no_link_check) 2639 check_all_ports_link_status(RTE_PORT_ALL); 2640 2641 printf("Done\n"); 2642 } 2643 2644 static void 2645 remove_invalid_ports_in(portid_t *array, portid_t *total) 2646 { 2647 portid_t i; 2648 portid_t new_total = 0; 2649 2650 for (i = 0; i < *total; i++) 2651 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2652 array[new_total] = array[i]; 2653 new_total++; 2654 } 2655 *total = new_total; 2656 } 2657 2658 static void 2659 remove_invalid_ports(void) 2660 { 2661 remove_invalid_ports_in(ports_ids, &nb_ports); 2662 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2663 nb_cfg_ports = nb_fwd_ports; 2664 } 2665 2666 void 2667 close_port(portid_t pid) 2668 { 2669 portid_t pi; 2670 struct rte_port *port; 2671 2672 if (port_id_is_invalid(pid, ENABLED_WARN)) 2673 return; 2674 2675 printf("Closing ports...\n"); 2676 2677 RTE_ETH_FOREACH_DEV(pi) { 2678 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2679 continue; 2680 2681 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2682 printf("Please remove port %d from forwarding configuration.\n", pi); 2683 continue; 2684 } 2685 2686 if (port_is_bonding_slave(pi)) { 2687 printf("Please remove port %d from bonded device.\n", pi); 2688 continue; 2689 } 2690 2691 port = &ports[pi]; 2692 if (rte_atomic16_cmpset(&(port->port_status), 2693 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2694 printf("Port %d is already closed\n", pi); 2695 continue; 2696 } 2697 2698 if (rte_atomic16_cmpset(&(port->port_status), 2699 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2700 printf("Port %d is now not stopped\n", pi); 2701 continue; 2702 } 2703 2704 if (port->flow_list) 2705 port_flow_flush(pi); 2706 rte_eth_dev_close(pi); 2707 2708 remove_invalid_ports(); 2709 2710 if (rte_atomic16_cmpset(&(port->port_status), 2711 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2712 printf("Port %d cannot be set to closed\n", pi); 2713 } 2714 2715 printf("Done\n"); 2716 } 2717 2718 void 2719 reset_port(portid_t pid) 2720 { 2721 int diag; 2722 portid_t pi; 2723 struct rte_port *port; 2724 2725 if (port_id_is_invalid(pid, ENABLED_WARN)) 2726 return; 2727 2728 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2729 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2730 printf("Can not reset port(s), please stop port(s) first.\n"); 2731 return; 2732 } 2733 2734 printf("Resetting ports...\n"); 2735 2736 RTE_ETH_FOREACH_DEV(pi) { 2737 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2738 continue; 2739 2740 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2741 printf("Please remove port %d from forwarding " 2742 "configuration.\n", pi); 2743 continue; 2744 } 2745 2746 if (port_is_bonding_slave(pi)) { 2747 printf("Please remove port %d from bonded device.\n", 2748 pi); 2749 continue; 2750 } 2751 2752 diag = rte_eth_dev_reset(pi); 2753 if (diag == 0) { 2754 port = &ports[pi]; 2755 port->need_reconfig = 1; 2756 port->need_reconfig_queues = 1; 2757 } else { 2758 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2759 } 2760 } 2761 2762 printf("Done\n"); 2763 } 2764 2765 void 2766 attach_port(char *identifier) 2767 { 2768 portid_t pi; 2769 struct rte_dev_iterator iterator; 2770 2771 printf("Attaching a new port...\n"); 2772 2773 if (identifier == NULL) { 2774 printf("Invalid parameters are specified\n"); 2775 return; 2776 } 2777 2778 if (rte_dev_probe(identifier) < 0) { 2779 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2780 return; 2781 } 2782 2783 /* first attach mode: event */ 2784 if (setup_on_probe_event) { 2785 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2786 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2787 if (ports[pi].port_status == RTE_PORT_HANDLING && 2788 ports[pi].need_setup != 0) 2789 setup_attached_port(pi); 2790 return; 2791 } 2792 2793 /* second attach mode: iterator */ 2794 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2795 /* setup ports matching the devargs used for probing */ 2796 if (port_is_forwarding(pi)) 2797 continue; /* port was already attached before */ 2798 setup_attached_port(pi); 2799 } 2800 } 2801 2802 static void 2803 setup_attached_port(portid_t pi) 2804 { 2805 unsigned int socket_id; 2806 int ret; 2807 2808 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2809 /* if socket_id is invalid, set to the first available socket. */ 2810 if (check_socket_id(socket_id) < 0) 2811 socket_id = socket_ids[0]; 2812 reconfig(pi, socket_id); 2813 ret = rte_eth_promiscuous_enable(pi); 2814 if (ret != 0) 2815 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2816 pi, rte_strerror(-ret)); 2817 2818 ports_ids[nb_ports++] = pi; 2819 fwd_ports_ids[nb_fwd_ports++] = pi; 2820 nb_cfg_ports = nb_fwd_ports; 2821 ports[pi].need_setup = 0; 2822 ports[pi].port_status = RTE_PORT_STOPPED; 2823 2824 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2825 printf("Done\n"); 2826 } 2827 2828 static void 2829 detach_device(struct rte_device *dev) 2830 { 2831 portid_t sibling; 2832 2833 if (dev == NULL) { 2834 printf("Device already removed\n"); 2835 return; 2836 } 2837 2838 printf("Removing a device...\n"); 2839 2840 if (rte_dev_remove(dev) < 0) { 2841 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2842 return; 2843 } 2844 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2845 /* reset mapping between old ports and removed device */ 2846 rte_eth_devices[sibling].device = NULL; 2847 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2848 /* sibling ports are forced to be closed */ 2849 ports[sibling].port_status = RTE_PORT_CLOSED; 2850 printf("Port %u is closed\n", sibling); 2851 } 2852 } 2853 2854 remove_invalid_ports(); 2855 2856 printf("Device is detached\n"); 2857 printf("Now total ports is %d\n", nb_ports); 2858 printf("Done\n"); 2859 return; 2860 } 2861 2862 void 2863 detach_port_device(portid_t port_id) 2864 { 2865 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2866 return; 2867 2868 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2869 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2870 printf("Port not stopped\n"); 2871 return; 2872 } 2873 printf("Port was not closed\n"); 2874 if (ports[port_id].flow_list) 2875 port_flow_flush(port_id); 2876 } 2877 2878 detach_device(rte_eth_devices[port_id].device); 2879 } 2880 2881 void 2882 detach_devargs(char *identifier) 2883 { 2884 struct rte_dev_iterator iterator; 2885 struct rte_devargs da; 2886 portid_t port_id; 2887 2888 printf("Removing a device...\n"); 2889 2890 memset(&da, 0, sizeof(da)); 2891 if (rte_devargs_parsef(&da, "%s", identifier)) { 2892 printf("cannot parse identifier\n"); 2893 if (da.args) 2894 free(da.args); 2895 return; 2896 } 2897 2898 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2899 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2900 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2901 printf("Port %u not stopped\n", port_id); 2902 rte_eth_iterator_cleanup(&iterator); 2903 return; 2904 } 2905 2906 /* sibling ports are forced to be closed */ 2907 if (ports[port_id].flow_list) 2908 port_flow_flush(port_id); 2909 ports[port_id].port_status = RTE_PORT_CLOSED; 2910 printf("Port %u is now closed\n", port_id); 2911 } 2912 } 2913 2914 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2915 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2916 da.name, da.bus->name); 2917 return; 2918 } 2919 2920 remove_invalid_ports(); 2921 2922 printf("Device %s is detached\n", identifier); 2923 printf("Now total ports is %d\n", nb_ports); 2924 printf("Done\n"); 2925 } 2926 2927 void 2928 pmd_test_exit(void) 2929 { 2930 portid_t pt_id; 2931 int ret; 2932 int i; 2933 2934 if (test_done == 0) 2935 stop_packet_forwarding(); 2936 2937 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2938 if (mempools[i]) { 2939 if (mp_alloc_type == MP_ALLOC_ANON) 2940 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2941 NULL); 2942 } 2943 } 2944 if (ports != NULL) { 2945 no_link_check = 1; 2946 RTE_ETH_FOREACH_DEV(pt_id) { 2947 printf("\nStopping port %d...\n", pt_id); 2948 fflush(stdout); 2949 stop_port(pt_id); 2950 } 2951 RTE_ETH_FOREACH_DEV(pt_id) { 2952 printf("\nShutting down port %d...\n", pt_id); 2953 fflush(stdout); 2954 close_port(pt_id); 2955 } 2956 } 2957 2958 if (hot_plug) { 2959 ret = rte_dev_event_monitor_stop(); 2960 if (ret) { 2961 RTE_LOG(ERR, EAL, 2962 "fail to stop device event monitor."); 2963 return; 2964 } 2965 2966 ret = rte_dev_event_callback_unregister(NULL, 2967 dev_event_callback, NULL); 2968 if (ret < 0) { 2969 RTE_LOG(ERR, EAL, 2970 "fail to unregister device event callback.\n"); 2971 return; 2972 } 2973 2974 ret = rte_dev_hotplug_handle_disable(); 2975 if (ret) { 2976 RTE_LOG(ERR, EAL, 2977 "fail to disable hotplug handling.\n"); 2978 return; 2979 } 2980 } 2981 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2982 if (mempools[i]) 2983 rte_mempool_free(mempools[i]); 2984 } 2985 2986 printf("\nBye...\n"); 2987 } 2988 2989 typedef void (*cmd_func_t)(void); 2990 struct pmd_test_command { 2991 const char *cmd_name; 2992 cmd_func_t cmd_func; 2993 }; 2994 2995 /* Check the link status of all ports in up to 9s, and print them finally */ 2996 static void 2997 check_all_ports_link_status(uint32_t port_mask) 2998 { 2999 #define CHECK_INTERVAL 100 /* 100ms */ 3000 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3001 portid_t portid; 3002 uint8_t count, all_ports_up, print_flag = 0; 3003 struct rte_eth_link link; 3004 int ret; 3005 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3006 3007 printf("Checking link statuses...\n"); 3008 fflush(stdout); 3009 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3010 all_ports_up = 1; 3011 RTE_ETH_FOREACH_DEV(portid) { 3012 if ((port_mask & (1 << portid)) == 0) 3013 continue; 3014 memset(&link, 0, sizeof(link)); 3015 ret = rte_eth_link_get_nowait(portid, &link); 3016 if (ret < 0) { 3017 all_ports_up = 0; 3018 if (print_flag == 1) 3019 printf("Port %u link get failed: %s\n", 3020 portid, rte_strerror(-ret)); 3021 continue; 3022 } 3023 /* print link status if flag set */ 3024 if (print_flag == 1) { 3025 rte_eth_link_to_str(link_status, 3026 sizeof(link_status), &link); 3027 printf("Port %d %s\n", portid, link_status); 3028 continue; 3029 } 3030 /* clear all_ports_up flag if any link down */ 3031 if (link.link_status == ETH_LINK_DOWN) { 3032 all_ports_up = 0; 3033 break; 3034 } 3035 } 3036 /* after finally printing all link status, get out */ 3037 if (print_flag == 1) 3038 break; 3039 3040 if (all_ports_up == 0) { 3041 fflush(stdout); 3042 rte_delay_ms(CHECK_INTERVAL); 3043 } 3044 3045 /* set the print_flag if all ports up or timeout */ 3046 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3047 print_flag = 1; 3048 } 3049 3050 if (lsc_interrupt) 3051 break; 3052 } 3053 } 3054 3055 /* 3056 * This callback is for remove a port for a device. It has limitation because 3057 * it is not for multiple port removal for a device. 3058 * TODO: the device detach invoke will plan to be removed from user side to 3059 * eal. And convert all PMDs to free port resources on ether device closing. 3060 */ 3061 static void 3062 rmv_port_callback(void *arg) 3063 { 3064 int need_to_start = 0; 3065 int org_no_link_check = no_link_check; 3066 portid_t port_id = (intptr_t)arg; 3067 struct rte_device *dev; 3068 3069 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3070 3071 if (!test_done && port_is_forwarding(port_id)) { 3072 need_to_start = 1; 3073 stop_packet_forwarding(); 3074 } 3075 no_link_check = 1; 3076 stop_port(port_id); 3077 no_link_check = org_no_link_check; 3078 3079 /* Save rte_device pointer before closing ethdev port */ 3080 dev = rte_eth_devices[port_id].device; 3081 close_port(port_id); 3082 detach_device(dev); /* might be already removed or have more ports */ 3083 3084 if (need_to_start) 3085 start_packet_forwarding(0); 3086 } 3087 3088 /* This function is used by the interrupt thread */ 3089 static int 3090 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3091 void *ret_param) 3092 { 3093 RTE_SET_USED(param); 3094 RTE_SET_USED(ret_param); 3095 3096 if (type >= RTE_ETH_EVENT_MAX) { 3097 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3098 port_id, __func__, type); 3099 fflush(stderr); 3100 } else if (event_print_mask & (UINT32_C(1) << type)) { 3101 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3102 eth_event_desc[type]); 3103 fflush(stdout); 3104 } 3105 3106 switch (type) { 3107 case RTE_ETH_EVENT_NEW: 3108 ports[port_id].need_setup = 1; 3109 ports[port_id].port_status = RTE_PORT_HANDLING; 3110 break; 3111 case RTE_ETH_EVENT_INTR_RMV: 3112 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3113 break; 3114 if (rte_eal_alarm_set(100000, 3115 rmv_port_callback, (void *)(intptr_t)port_id)) 3116 fprintf(stderr, "Could not set up deferred device removal\n"); 3117 break; 3118 default: 3119 break; 3120 } 3121 return 0; 3122 } 3123 3124 static int 3125 register_eth_event_callback(void) 3126 { 3127 int ret; 3128 enum rte_eth_event_type event; 3129 3130 for (event = RTE_ETH_EVENT_UNKNOWN; 3131 event < RTE_ETH_EVENT_MAX; event++) { 3132 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3133 event, 3134 eth_event_callback, 3135 NULL); 3136 if (ret != 0) { 3137 TESTPMD_LOG(ERR, "Failed to register callback for " 3138 "%s event\n", eth_event_desc[event]); 3139 return -1; 3140 } 3141 } 3142 3143 return 0; 3144 } 3145 3146 /* This function is used by the interrupt thread */ 3147 static void 3148 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3149 __rte_unused void *arg) 3150 { 3151 uint16_t port_id; 3152 int ret; 3153 3154 if (type >= RTE_DEV_EVENT_MAX) { 3155 fprintf(stderr, "%s called upon invalid event %d\n", 3156 __func__, type); 3157 fflush(stderr); 3158 } 3159 3160 switch (type) { 3161 case RTE_DEV_EVENT_REMOVE: 3162 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3163 device_name); 3164 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3165 if (ret) { 3166 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3167 device_name); 3168 return; 3169 } 3170 /* 3171 * Because the user's callback is invoked in eal interrupt 3172 * callback, the interrupt callback need to be finished before 3173 * it can be unregistered when detaching device. So finish 3174 * callback soon and use a deferred removal to detach device 3175 * is need. It is a workaround, once the device detaching be 3176 * moved into the eal in the future, the deferred removal could 3177 * be deleted. 3178 */ 3179 if (rte_eal_alarm_set(100000, 3180 rmv_port_callback, (void *)(intptr_t)port_id)) 3181 RTE_LOG(ERR, EAL, 3182 "Could not set up deferred device removal\n"); 3183 break; 3184 case RTE_DEV_EVENT_ADD: 3185 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3186 device_name); 3187 /* TODO: After finish kernel driver binding, 3188 * begin to attach port. 3189 */ 3190 break; 3191 default: 3192 break; 3193 } 3194 } 3195 3196 static int 3197 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3198 { 3199 uint16_t i; 3200 int diag; 3201 uint8_t mapping_found = 0; 3202 3203 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3204 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3205 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3206 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3207 tx_queue_stats_mappings[i].queue_id, 3208 tx_queue_stats_mappings[i].stats_counter_id); 3209 if (diag != 0) 3210 return diag; 3211 mapping_found = 1; 3212 } 3213 } 3214 if (mapping_found) 3215 port->tx_queue_stats_mapping_enabled = 1; 3216 return 0; 3217 } 3218 3219 static int 3220 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3221 { 3222 uint16_t i; 3223 int diag; 3224 uint8_t mapping_found = 0; 3225 3226 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3227 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3228 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3229 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3230 rx_queue_stats_mappings[i].queue_id, 3231 rx_queue_stats_mappings[i].stats_counter_id); 3232 if (diag != 0) 3233 return diag; 3234 mapping_found = 1; 3235 } 3236 } 3237 if (mapping_found) 3238 port->rx_queue_stats_mapping_enabled = 1; 3239 return 0; 3240 } 3241 3242 static void 3243 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3244 { 3245 int diag = 0; 3246 3247 diag = set_tx_queue_stats_mapping_registers(pi, port); 3248 if (diag != 0) { 3249 if (diag == -ENOTSUP) { 3250 port->tx_queue_stats_mapping_enabled = 0; 3251 printf("TX queue stats mapping not supported port id=%d\n", pi); 3252 } 3253 else 3254 rte_exit(EXIT_FAILURE, 3255 "set_tx_queue_stats_mapping_registers " 3256 "failed for port id=%d diag=%d\n", 3257 pi, diag); 3258 } 3259 3260 diag = set_rx_queue_stats_mapping_registers(pi, port); 3261 if (diag != 0) { 3262 if (diag == -ENOTSUP) { 3263 port->rx_queue_stats_mapping_enabled = 0; 3264 printf("RX queue stats mapping not supported port id=%d\n", pi); 3265 } 3266 else 3267 rte_exit(EXIT_FAILURE, 3268 "set_rx_queue_stats_mapping_registers " 3269 "failed for port id=%d diag=%d\n", 3270 pi, diag); 3271 } 3272 } 3273 3274 static void 3275 rxtx_port_config(struct rte_port *port) 3276 { 3277 uint16_t qid; 3278 uint64_t offloads; 3279 3280 for (qid = 0; qid < nb_rxq; qid++) { 3281 offloads = port->rx_conf[qid].offloads; 3282 port->rx_conf[qid] = port->dev_info.default_rxconf; 3283 if (offloads != 0) 3284 port->rx_conf[qid].offloads = offloads; 3285 3286 /* Check if any Rx parameters have been passed */ 3287 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3288 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3289 3290 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3291 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3292 3293 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3294 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3295 3296 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3297 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3298 3299 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3300 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3301 3302 port->nb_rx_desc[qid] = nb_rxd; 3303 } 3304 3305 for (qid = 0; qid < nb_txq; qid++) { 3306 offloads = port->tx_conf[qid].offloads; 3307 port->tx_conf[qid] = port->dev_info.default_txconf; 3308 if (offloads != 0) 3309 port->tx_conf[qid].offloads = offloads; 3310 3311 /* Check if any Tx parameters have been passed */ 3312 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3313 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3314 3315 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3316 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3317 3318 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3319 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3320 3321 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3322 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3323 3324 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3325 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3326 3327 port->nb_tx_desc[qid] = nb_txd; 3328 } 3329 } 3330 3331 void 3332 init_port_config(void) 3333 { 3334 portid_t pid; 3335 struct rte_port *port; 3336 int ret; 3337 3338 RTE_ETH_FOREACH_DEV(pid) { 3339 port = &ports[pid]; 3340 port->dev_conf.fdir_conf = fdir_conf; 3341 3342 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3343 if (ret != 0) 3344 return; 3345 3346 if (nb_rxq > 1) { 3347 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3348 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3349 rss_hf & port->dev_info.flow_type_rss_offloads; 3350 } else { 3351 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3352 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3353 } 3354 3355 if (port->dcb_flag == 0) { 3356 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3357 port->dev_conf.rxmode.mq_mode = 3358 (enum rte_eth_rx_mq_mode) 3359 (rx_mq_mode & ETH_MQ_RX_RSS); 3360 else 3361 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3362 } 3363 3364 rxtx_port_config(port); 3365 3366 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3367 if (ret != 0) 3368 return; 3369 3370 map_port_queue_stats_mapping_registers(pid, port); 3371 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3372 rte_pmd_ixgbe_bypass_init(pid); 3373 #endif 3374 3375 if (lsc_interrupt && 3376 (rte_eth_devices[pid].data->dev_flags & 3377 RTE_ETH_DEV_INTR_LSC)) 3378 port->dev_conf.intr_conf.lsc = 1; 3379 if (rmv_interrupt && 3380 (rte_eth_devices[pid].data->dev_flags & 3381 RTE_ETH_DEV_INTR_RMV)) 3382 port->dev_conf.intr_conf.rmv = 1; 3383 } 3384 } 3385 3386 void set_port_slave_flag(portid_t slave_pid) 3387 { 3388 struct rte_port *port; 3389 3390 port = &ports[slave_pid]; 3391 port->slave_flag = 1; 3392 } 3393 3394 void clear_port_slave_flag(portid_t slave_pid) 3395 { 3396 struct rte_port *port; 3397 3398 port = &ports[slave_pid]; 3399 port->slave_flag = 0; 3400 } 3401 3402 uint8_t port_is_bonding_slave(portid_t slave_pid) 3403 { 3404 struct rte_port *port; 3405 3406 port = &ports[slave_pid]; 3407 if ((rte_eth_devices[slave_pid].data->dev_flags & 3408 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3409 return 1; 3410 return 0; 3411 } 3412 3413 const uint16_t vlan_tags[] = { 3414 0, 1, 2, 3, 4, 5, 6, 7, 3415 8, 9, 10, 11, 12, 13, 14, 15, 3416 16, 17, 18, 19, 20, 21, 22, 23, 3417 24, 25, 26, 27, 28, 29, 30, 31 3418 }; 3419 3420 static int 3421 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3422 enum dcb_mode_enable dcb_mode, 3423 enum rte_eth_nb_tcs num_tcs, 3424 uint8_t pfc_en) 3425 { 3426 uint8_t i; 3427 int32_t rc; 3428 struct rte_eth_rss_conf rss_conf; 3429 3430 /* 3431 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3432 * given above, and the number of traffic classes available for use. 3433 */ 3434 if (dcb_mode == DCB_VT_ENABLED) { 3435 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3436 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3437 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3438 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3439 3440 /* VMDQ+DCB RX and TX configurations */ 3441 vmdq_rx_conf->enable_default_pool = 0; 3442 vmdq_rx_conf->default_pool = 0; 3443 vmdq_rx_conf->nb_queue_pools = 3444 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3445 vmdq_tx_conf->nb_queue_pools = 3446 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3447 3448 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3449 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3450 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3451 vmdq_rx_conf->pool_map[i].pools = 3452 1 << (i % vmdq_rx_conf->nb_queue_pools); 3453 } 3454 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3455 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3456 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3457 } 3458 3459 /* set DCB mode of RX and TX of multiple queues */ 3460 eth_conf->rxmode.mq_mode = 3461 (enum rte_eth_rx_mq_mode) 3462 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3463 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3464 } else { 3465 struct rte_eth_dcb_rx_conf *rx_conf = 3466 ð_conf->rx_adv_conf.dcb_rx_conf; 3467 struct rte_eth_dcb_tx_conf *tx_conf = 3468 ð_conf->tx_adv_conf.dcb_tx_conf; 3469 3470 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3471 3472 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3473 if (rc != 0) 3474 return rc; 3475 3476 rx_conf->nb_tcs = num_tcs; 3477 tx_conf->nb_tcs = num_tcs; 3478 3479 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3480 rx_conf->dcb_tc[i] = i % num_tcs; 3481 tx_conf->dcb_tc[i] = i % num_tcs; 3482 } 3483 3484 eth_conf->rxmode.mq_mode = 3485 (enum rte_eth_rx_mq_mode) 3486 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3487 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3488 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3489 } 3490 3491 if (pfc_en) 3492 eth_conf->dcb_capability_en = 3493 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3494 else 3495 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3496 3497 return 0; 3498 } 3499 3500 int 3501 init_port_dcb_config(portid_t pid, 3502 enum dcb_mode_enable dcb_mode, 3503 enum rte_eth_nb_tcs num_tcs, 3504 uint8_t pfc_en) 3505 { 3506 struct rte_eth_conf port_conf; 3507 struct rte_port *rte_port; 3508 int retval; 3509 uint16_t i; 3510 3511 rte_port = &ports[pid]; 3512 3513 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3514 /* Enter DCB configuration status */ 3515 dcb_config = 1; 3516 3517 port_conf.rxmode = rte_port->dev_conf.rxmode; 3518 port_conf.txmode = rte_port->dev_conf.txmode; 3519 3520 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3521 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3522 if (retval < 0) 3523 return retval; 3524 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3525 3526 /* re-configure the device . */ 3527 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3528 if (retval < 0) 3529 return retval; 3530 3531 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3532 if (retval != 0) 3533 return retval; 3534 3535 /* If dev_info.vmdq_pool_base is greater than 0, 3536 * the queue id of vmdq pools is started after pf queues. 3537 */ 3538 if (dcb_mode == DCB_VT_ENABLED && 3539 rte_port->dev_info.vmdq_pool_base > 0) { 3540 printf("VMDQ_DCB multi-queue mode is nonsensical" 3541 " for port %d.", pid); 3542 return -1; 3543 } 3544 3545 /* Assume the ports in testpmd have the same dcb capability 3546 * and has the same number of rxq and txq in dcb mode 3547 */ 3548 if (dcb_mode == DCB_VT_ENABLED) { 3549 if (rte_port->dev_info.max_vfs > 0) { 3550 nb_rxq = rte_port->dev_info.nb_rx_queues; 3551 nb_txq = rte_port->dev_info.nb_tx_queues; 3552 } else { 3553 nb_rxq = rte_port->dev_info.max_rx_queues; 3554 nb_txq = rte_port->dev_info.max_tx_queues; 3555 } 3556 } else { 3557 /*if vt is disabled, use all pf queues */ 3558 if (rte_port->dev_info.vmdq_pool_base == 0) { 3559 nb_rxq = rte_port->dev_info.max_rx_queues; 3560 nb_txq = rte_port->dev_info.max_tx_queues; 3561 } else { 3562 nb_rxq = (queueid_t)num_tcs; 3563 nb_txq = (queueid_t)num_tcs; 3564 3565 } 3566 } 3567 rx_free_thresh = 64; 3568 3569 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3570 3571 rxtx_port_config(rte_port); 3572 /* VLAN filter */ 3573 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3574 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3575 rx_vft_set(pid, vlan_tags[i], 1); 3576 3577 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3578 if (retval != 0) 3579 return retval; 3580 3581 map_port_queue_stats_mapping_registers(pid, rte_port); 3582 3583 rte_port->dcb_flag = 1; 3584 3585 return 0; 3586 } 3587 3588 static void 3589 init_port(void) 3590 { 3591 /* Configuration of Ethernet ports. */ 3592 ports = rte_zmalloc("testpmd: ports", 3593 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3594 RTE_CACHE_LINE_SIZE); 3595 if (ports == NULL) { 3596 rte_exit(EXIT_FAILURE, 3597 "rte_zmalloc(%d struct rte_port) failed\n", 3598 RTE_MAX_ETHPORTS); 3599 } 3600 3601 /* Initialize ports NUMA structures */ 3602 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3603 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3604 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3605 } 3606 3607 static void 3608 force_quit(void) 3609 { 3610 pmd_test_exit(); 3611 prompt_exit(); 3612 } 3613 3614 static void 3615 print_stats(void) 3616 { 3617 uint8_t i; 3618 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3619 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3620 3621 /* Clear screen and move to top left */ 3622 printf("%s%s", clr, top_left); 3623 3624 printf("\nPort statistics ===================================="); 3625 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3626 nic_stats_display(fwd_ports_ids[i]); 3627 3628 fflush(stdout); 3629 } 3630 3631 static void 3632 signal_handler(int signum) 3633 { 3634 if (signum == SIGINT || signum == SIGTERM) { 3635 printf("\nSignal %d received, preparing to exit...\n", 3636 signum); 3637 #ifdef RTE_LIBRTE_PDUMP 3638 /* uninitialize packet capture framework */ 3639 rte_pdump_uninit(); 3640 #endif 3641 #ifdef RTE_LIBRTE_LATENCY_STATS 3642 if (latencystats_enabled != 0) 3643 rte_latencystats_uninit(); 3644 #endif 3645 force_quit(); 3646 /* Set flag to indicate the force termination. */ 3647 f_quit = 1; 3648 /* exit with the expected status */ 3649 signal(signum, SIG_DFL); 3650 kill(getpid(), signum); 3651 } 3652 } 3653 3654 int 3655 main(int argc, char** argv) 3656 { 3657 int diag; 3658 portid_t port_id; 3659 uint16_t count; 3660 int ret; 3661 3662 signal(SIGINT, signal_handler); 3663 signal(SIGTERM, signal_handler); 3664 3665 testpmd_logtype = rte_log_register("testpmd"); 3666 if (testpmd_logtype < 0) 3667 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3668 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3669 3670 diag = rte_eal_init(argc, argv); 3671 if (diag < 0) 3672 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3673 rte_strerror(rte_errno)); 3674 3675 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3676 rte_exit(EXIT_FAILURE, 3677 "Secondary process type not supported.\n"); 3678 3679 ret = register_eth_event_callback(); 3680 if (ret != 0) 3681 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3682 3683 #ifdef RTE_LIBRTE_PDUMP 3684 /* initialize packet capture framework */ 3685 rte_pdump_init(); 3686 #endif 3687 3688 count = 0; 3689 RTE_ETH_FOREACH_DEV(port_id) { 3690 ports_ids[count] = port_id; 3691 count++; 3692 } 3693 nb_ports = (portid_t) count; 3694 if (nb_ports == 0) 3695 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3696 3697 /* allocate port structures, and init them */ 3698 init_port(); 3699 3700 set_def_fwd_config(); 3701 if (nb_lcores == 0) 3702 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3703 "Check the core mask argument\n"); 3704 3705 /* Bitrate/latency stats disabled by default */ 3706 #ifdef RTE_LIBRTE_BITRATESTATS 3707 bitrate_enabled = 0; 3708 #endif 3709 #ifdef RTE_LIBRTE_LATENCY_STATS 3710 latencystats_enabled = 0; 3711 #endif 3712 3713 /* on FreeBSD, mlockall() is disabled by default */ 3714 #ifdef RTE_EXEC_ENV_FREEBSD 3715 do_mlockall = 0; 3716 #else 3717 do_mlockall = 1; 3718 #endif 3719 3720 argc -= diag; 3721 argv += diag; 3722 if (argc > 1) 3723 launch_args_parse(argc, argv); 3724 3725 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3726 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3727 strerror(errno)); 3728 } 3729 3730 if (tx_first && interactive) 3731 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3732 "interactive mode.\n"); 3733 3734 if (tx_first && lsc_interrupt) { 3735 printf("Warning: lsc_interrupt needs to be off when " 3736 " using tx_first. Disabling.\n"); 3737 lsc_interrupt = 0; 3738 } 3739 3740 if (!nb_rxq && !nb_txq) 3741 printf("Warning: Either rx or tx queues should be non-zero\n"); 3742 3743 if (nb_rxq > 1 && nb_rxq > nb_txq) 3744 printf("Warning: nb_rxq=%d enables RSS configuration, " 3745 "but nb_txq=%d will prevent to fully test it.\n", 3746 nb_rxq, nb_txq); 3747 3748 init_config(); 3749 3750 if (hot_plug) { 3751 ret = rte_dev_hotplug_handle_enable(); 3752 if (ret) { 3753 RTE_LOG(ERR, EAL, 3754 "fail to enable hotplug handling."); 3755 return -1; 3756 } 3757 3758 ret = rte_dev_event_monitor_start(); 3759 if (ret) { 3760 RTE_LOG(ERR, EAL, 3761 "fail to start device event monitoring."); 3762 return -1; 3763 } 3764 3765 ret = rte_dev_event_callback_register(NULL, 3766 dev_event_callback, NULL); 3767 if (ret) { 3768 RTE_LOG(ERR, EAL, 3769 "fail to register device event callback\n"); 3770 return -1; 3771 } 3772 } 3773 3774 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3775 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3776 3777 /* set all ports to promiscuous mode by default */ 3778 RTE_ETH_FOREACH_DEV(port_id) { 3779 ret = rte_eth_promiscuous_enable(port_id); 3780 if (ret != 0) 3781 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3782 port_id, rte_strerror(-ret)); 3783 } 3784 3785 /* Init metrics library */ 3786 rte_metrics_init(rte_socket_id()); 3787 3788 #ifdef RTE_LIBRTE_LATENCY_STATS 3789 if (latencystats_enabled != 0) { 3790 int ret = rte_latencystats_init(1, NULL); 3791 if (ret) 3792 printf("Warning: latencystats init()" 3793 " returned error %d\n", ret); 3794 printf("Latencystats running on lcore %d\n", 3795 latencystats_lcore_id); 3796 } 3797 #endif 3798 3799 /* Setup bitrate stats */ 3800 #ifdef RTE_LIBRTE_BITRATESTATS 3801 if (bitrate_enabled != 0) { 3802 bitrate_data = rte_stats_bitrate_create(); 3803 if (bitrate_data == NULL) 3804 rte_exit(EXIT_FAILURE, 3805 "Could not allocate bitrate data.\n"); 3806 rte_stats_bitrate_reg(bitrate_data); 3807 } 3808 #endif 3809 3810 #ifdef RTE_LIBRTE_CMDLINE 3811 if (strlen(cmdline_filename) != 0) 3812 cmdline_read_from_file(cmdline_filename); 3813 3814 if (interactive == 1) { 3815 if (auto_start) { 3816 printf("Start automatic packet forwarding\n"); 3817 start_packet_forwarding(0); 3818 } 3819 prompt(); 3820 pmd_test_exit(); 3821 } else 3822 #endif 3823 { 3824 char c; 3825 int rc; 3826 3827 f_quit = 0; 3828 3829 printf("No commandline core given, start packet forwarding\n"); 3830 start_packet_forwarding(tx_first); 3831 if (stats_period != 0) { 3832 uint64_t prev_time = 0, cur_time, diff_time = 0; 3833 uint64_t timer_period; 3834 3835 /* Convert to number of cycles */ 3836 timer_period = stats_period * rte_get_timer_hz(); 3837 3838 while (f_quit == 0) { 3839 cur_time = rte_get_timer_cycles(); 3840 diff_time += cur_time - prev_time; 3841 3842 if (diff_time >= timer_period) { 3843 print_stats(); 3844 /* Reset the timer */ 3845 diff_time = 0; 3846 } 3847 /* Sleep to avoid unnecessary checks */ 3848 prev_time = cur_time; 3849 sleep(1); 3850 } 3851 } 3852 3853 printf("Press enter to exit\n"); 3854 rc = read(0, &c, 1); 3855 pmd_test_exit(); 3856 if (rc < 0) 3857 return 1; 3858 } 3859 3860 ret = rte_eal_cleanup(); 3861 if (ret != 0) 3862 rte_exit(EXIT_FAILURE, 3863 "EAL cleanup failed: %s\n", strerror(-ret)); 3864 3865 return EXIT_SUCCESS; 3866 } 3867