1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATE 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 199 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 200 * specified on command-line. */ 201 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 202 203 /* 204 * In container, it cannot terminate the process which running with 'stats-period' 205 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 206 */ 207 uint8_t f_quit; 208 209 /* 210 * Configuration of packet segments used by the "txonly" processing engine. 211 */ 212 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 213 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 214 TXONLY_DEF_PACKET_LEN, 215 }; 216 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 217 218 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 219 /**< Split policy for packets to TX. */ 220 221 uint8_t txonly_multi_flow; 222 /**< Whether multiple flows are generated in TXONLY mode. */ 223 224 uint32_t tx_pkt_times_inter; 225 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 226 227 uint32_t tx_pkt_times_intra; 228 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 229 230 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 231 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 232 233 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 234 uint8_t dcb_config = 0; 235 236 /* Whether the dcb is in testing status */ 237 uint8_t dcb_test = 0; 238 239 /* 240 * Configurable number of RX/TX queues. 241 */ 242 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 243 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 244 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 245 246 /* 247 * Configurable number of RX/TX ring descriptors. 248 * Defaults are supplied by drivers via ethdev. 249 */ 250 #define RTE_TEST_RX_DESC_DEFAULT 0 251 #define RTE_TEST_TX_DESC_DEFAULT 0 252 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 253 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 254 255 #define RTE_PMD_PARAM_UNSET -1 256 /* 257 * Configurable values of RX and TX ring threshold registers. 258 */ 259 260 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 261 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 262 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 263 264 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 265 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 266 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX free threshold. 270 */ 271 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of RX drop enable. 275 */ 276 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX free threshold. 280 */ 281 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of TX RS bit threshold. 285 */ 286 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 287 288 /* 289 * Configurable value of buffered packets before sending. 290 */ 291 uint16_t noisy_tx_sw_bufsz; 292 293 /* 294 * Configurable value of packet buffer timeout. 295 */ 296 uint16_t noisy_tx_sw_buf_flush_time; 297 298 /* 299 * Configurable value for size of VNF internal memory area 300 * used for simulating noisy neighbour behaviour 301 */ 302 uint64_t noisy_lkup_mem_sz; 303 304 /* 305 * Configurable value of number of random writes done in 306 * VNF simulation memory area. 307 */ 308 uint64_t noisy_lkup_num_writes; 309 310 /* 311 * Configurable value of number of random reads done in 312 * VNF simulation memory area. 313 */ 314 uint64_t noisy_lkup_num_reads; 315 316 /* 317 * Configurable value of number of random reads/writes done in 318 * VNF simulation memory area. 319 */ 320 uint64_t noisy_lkup_num_reads_writes; 321 322 /* 323 * Receive Side Scaling (RSS) configuration. 324 */ 325 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 326 327 /* 328 * Port topology configuration 329 */ 330 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 331 332 /* 333 * Avoids to flush all the RX streams before starts forwarding. 334 */ 335 uint8_t no_flush_rx = 0; /* flush by default */ 336 337 /* 338 * Flow API isolated mode. 339 */ 340 uint8_t flow_isolate_all; 341 342 /* 343 * Avoids to check link status when starting/stopping a port. 344 */ 345 uint8_t no_link_check = 0; /* check by default */ 346 347 /* 348 * Don't automatically start all ports in interactive mode. 349 */ 350 uint8_t no_device_start = 0; 351 352 /* 353 * Enable link status change notification 354 */ 355 uint8_t lsc_interrupt = 1; /* enabled by default */ 356 357 /* 358 * Enable device removal notification. 359 */ 360 uint8_t rmv_interrupt = 1; /* enabled by default */ 361 362 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 363 364 /* After attach, port setup is called on event or by iterator */ 365 bool setup_on_probe_event = true; 366 367 /* Clear ptypes on port initialization. */ 368 uint8_t clear_ptypes = true; 369 370 /* Pretty printing of ethdev events */ 371 static const char * const eth_event_desc[] = { 372 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 373 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 374 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 375 [RTE_ETH_EVENT_INTR_RESET] = "reset", 376 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 377 [RTE_ETH_EVENT_IPSEC] = "IPsec", 378 [RTE_ETH_EVENT_MACSEC] = "MACsec", 379 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 380 [RTE_ETH_EVENT_NEW] = "device probed", 381 [RTE_ETH_EVENT_DESTROY] = "device released", 382 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 383 [RTE_ETH_EVENT_MAX] = NULL, 384 }; 385 386 /* 387 * Display or mask ether events 388 * Default to all events except VF_MBOX 389 */ 390 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 391 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 392 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 393 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 394 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 395 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 396 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 397 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 398 /* 399 * Decide if all memory are locked for performance. 400 */ 401 int do_mlockall = 0; 402 403 /* 404 * NIC bypass mode configuration options. 405 */ 406 407 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 408 /* The NIC bypass watchdog timeout. */ 409 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 410 #endif 411 412 413 #ifdef RTE_LIBRTE_LATENCY_STATS 414 415 /* 416 * Set when latency stats is enabled in the commandline 417 */ 418 uint8_t latencystats_enabled; 419 420 /* 421 * Lcore ID to serive latency statistics. 422 */ 423 lcoreid_t latencystats_lcore_id = -1; 424 425 #endif 426 427 /* 428 * Ethernet device configuration. 429 */ 430 struct rte_eth_rxmode rx_mode = { 431 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 432 /**< Default maximum frame length. */ 433 }; 434 435 struct rte_eth_txmode tx_mode = { 436 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 437 }; 438 439 struct rte_fdir_conf fdir_conf = { 440 .mode = RTE_FDIR_MODE_NONE, 441 .pballoc = RTE_FDIR_PBALLOC_64K, 442 .status = RTE_FDIR_REPORT_STATUS, 443 .mask = { 444 .vlan_tci_mask = 0xFFEF, 445 .ipv4_mask = { 446 .src_ip = 0xFFFFFFFF, 447 .dst_ip = 0xFFFFFFFF, 448 }, 449 .ipv6_mask = { 450 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 451 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 452 }, 453 .src_port_mask = 0xFFFF, 454 .dst_port_mask = 0xFFFF, 455 .mac_addr_byte_mask = 0xFF, 456 .tunnel_type_mask = 1, 457 .tunnel_id_mask = 0xFFFFFFFF, 458 }, 459 .drop_queue = 127, 460 }; 461 462 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 463 464 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 465 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 466 467 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 468 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 469 470 uint16_t nb_tx_queue_stats_mappings = 0; 471 uint16_t nb_rx_queue_stats_mappings = 0; 472 473 /* 474 * Display zero values by default for xstats 475 */ 476 uint8_t xstats_hide_zero; 477 478 unsigned int num_sockets = 0; 479 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 480 481 #ifdef RTE_LIBRTE_BITRATE 482 /* Bitrate statistics */ 483 struct rte_stats_bitrates *bitrate_data; 484 lcoreid_t bitrate_lcore_id; 485 uint8_t bitrate_enabled; 486 #endif 487 488 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 489 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 490 491 /* 492 * hexadecimal bitmask of RX mq mode can be enabled. 493 */ 494 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 495 496 /* Forward function declarations */ 497 static void setup_attached_port(portid_t pi); 498 static void map_port_queue_stats_mapping_registers(portid_t pi, 499 struct rte_port *port); 500 static void check_all_ports_link_status(uint32_t port_mask); 501 static int eth_event_callback(portid_t port_id, 502 enum rte_eth_event_type type, 503 void *param, void *ret_param); 504 static void dev_event_callback(const char *device_name, 505 enum rte_dev_event_type type, 506 void *param); 507 508 /* 509 * Check if all the ports are started. 510 * If yes, return positive value. If not, return zero. 511 */ 512 static int all_ports_started(void); 513 514 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 515 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 516 517 /* Holds the registered mbuf dynamic flags names. */ 518 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 519 520 /* 521 * Helper function to check if socket is already discovered. 522 * If yes, return positive value. If not, return zero. 523 */ 524 int 525 new_socket_id(unsigned int socket_id) 526 { 527 unsigned int i; 528 529 for (i = 0; i < num_sockets; i++) { 530 if (socket_ids[i] == socket_id) 531 return 0; 532 } 533 return 1; 534 } 535 536 /* 537 * Setup default configuration. 538 */ 539 static void 540 set_default_fwd_lcores_config(void) 541 { 542 unsigned int i; 543 unsigned int nb_lc; 544 unsigned int sock_num; 545 546 nb_lc = 0; 547 for (i = 0; i < RTE_MAX_LCORE; i++) { 548 if (!rte_lcore_is_enabled(i)) 549 continue; 550 sock_num = rte_lcore_to_socket_id(i); 551 if (new_socket_id(sock_num)) { 552 if (num_sockets >= RTE_MAX_NUMA_NODES) { 553 rte_exit(EXIT_FAILURE, 554 "Total sockets greater than %u\n", 555 RTE_MAX_NUMA_NODES); 556 } 557 socket_ids[num_sockets++] = sock_num; 558 } 559 if (i == rte_get_master_lcore()) 560 continue; 561 fwd_lcores_cpuids[nb_lc++] = i; 562 } 563 nb_lcores = (lcoreid_t) nb_lc; 564 nb_cfg_lcores = nb_lcores; 565 nb_fwd_lcores = 1; 566 } 567 568 static void 569 set_def_peer_eth_addrs(void) 570 { 571 portid_t i; 572 573 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 574 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 575 peer_eth_addrs[i].addr_bytes[5] = i; 576 } 577 } 578 579 static void 580 set_default_fwd_ports_config(void) 581 { 582 portid_t pt_id; 583 int i = 0; 584 585 RTE_ETH_FOREACH_DEV(pt_id) { 586 fwd_ports_ids[i++] = pt_id; 587 588 /* Update sockets info according to the attached device */ 589 int socket_id = rte_eth_dev_socket_id(pt_id); 590 if (socket_id >= 0 && new_socket_id(socket_id)) { 591 if (num_sockets >= RTE_MAX_NUMA_NODES) { 592 rte_exit(EXIT_FAILURE, 593 "Total sockets greater than %u\n", 594 RTE_MAX_NUMA_NODES); 595 } 596 socket_ids[num_sockets++] = socket_id; 597 } 598 } 599 600 nb_cfg_ports = nb_ports; 601 nb_fwd_ports = nb_ports; 602 } 603 604 void 605 set_def_fwd_config(void) 606 { 607 set_default_fwd_lcores_config(); 608 set_def_peer_eth_addrs(); 609 set_default_fwd_ports_config(); 610 } 611 612 /* extremely pessimistic estimation of memory required to create a mempool */ 613 static int 614 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 615 { 616 unsigned int n_pages, mbuf_per_pg, leftover; 617 uint64_t total_mem, mbuf_mem, obj_sz; 618 619 /* there is no good way to predict how much space the mempool will 620 * occupy because it will allocate chunks on the fly, and some of those 621 * will come from default DPDK memory while some will come from our 622 * external memory, so just assume 128MB will be enough for everyone. 623 */ 624 uint64_t hdr_mem = 128 << 20; 625 626 /* account for possible non-contiguousness */ 627 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 628 if (obj_sz > pgsz) { 629 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 630 return -1; 631 } 632 633 mbuf_per_pg = pgsz / obj_sz; 634 leftover = (nb_mbufs % mbuf_per_pg) > 0; 635 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 636 637 mbuf_mem = n_pages * pgsz; 638 639 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 640 641 if (total_mem > SIZE_MAX) { 642 TESTPMD_LOG(ERR, "Memory size too big\n"); 643 return -1; 644 } 645 *out = (size_t)total_mem; 646 647 return 0; 648 } 649 650 static int 651 pagesz_flags(uint64_t page_sz) 652 { 653 /* as per mmap() manpage, all page sizes are log2 of page size 654 * shifted by MAP_HUGE_SHIFT 655 */ 656 int log2 = rte_log2_u64(page_sz); 657 658 return (log2 << HUGE_SHIFT); 659 } 660 661 static void * 662 alloc_mem(size_t memsz, size_t pgsz, bool huge) 663 { 664 void *addr; 665 int flags; 666 667 /* allocate anonymous hugepages */ 668 flags = MAP_ANONYMOUS | MAP_PRIVATE; 669 if (huge) 670 flags |= HUGE_FLAG | pagesz_flags(pgsz); 671 672 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 673 if (addr == MAP_FAILED) 674 return NULL; 675 676 return addr; 677 } 678 679 struct extmem_param { 680 void *addr; 681 size_t len; 682 size_t pgsz; 683 rte_iova_t *iova_table; 684 unsigned int iova_table_len; 685 }; 686 687 static int 688 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 689 bool huge) 690 { 691 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 692 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 693 unsigned int cur_page, n_pages, pgsz_idx; 694 size_t mem_sz, cur_pgsz; 695 rte_iova_t *iovas = NULL; 696 void *addr; 697 int ret; 698 699 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 700 /* skip anything that is too big */ 701 if (pgsizes[pgsz_idx] > SIZE_MAX) 702 continue; 703 704 cur_pgsz = pgsizes[pgsz_idx]; 705 706 /* if we were told not to allocate hugepages, override */ 707 if (!huge) 708 cur_pgsz = sysconf(_SC_PAGESIZE); 709 710 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 711 if (ret < 0) { 712 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 713 return -1; 714 } 715 716 /* allocate our memory */ 717 addr = alloc_mem(mem_sz, cur_pgsz, huge); 718 719 /* if we couldn't allocate memory with a specified page size, 720 * that doesn't mean we can't do it with other page sizes, so 721 * try another one. 722 */ 723 if (addr == NULL) 724 continue; 725 726 /* store IOVA addresses for every page in this memory area */ 727 n_pages = mem_sz / cur_pgsz; 728 729 iovas = malloc(sizeof(*iovas) * n_pages); 730 731 if (iovas == NULL) { 732 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 733 goto fail; 734 } 735 /* lock memory if it's not huge pages */ 736 if (!huge) 737 mlock(addr, mem_sz); 738 739 /* populate IOVA addresses */ 740 for (cur_page = 0; cur_page < n_pages; cur_page++) { 741 rte_iova_t iova; 742 size_t offset; 743 void *cur; 744 745 offset = cur_pgsz * cur_page; 746 cur = RTE_PTR_ADD(addr, offset); 747 748 /* touch the page before getting its IOVA */ 749 *(volatile char *)cur = 0; 750 751 iova = rte_mem_virt2iova(cur); 752 753 iovas[cur_page] = iova; 754 } 755 756 break; 757 } 758 /* if we couldn't allocate anything */ 759 if (iovas == NULL) 760 return -1; 761 762 param->addr = addr; 763 param->len = mem_sz; 764 param->pgsz = cur_pgsz; 765 param->iova_table = iovas; 766 param->iova_table_len = n_pages; 767 768 return 0; 769 fail: 770 if (iovas) 771 free(iovas); 772 if (addr) 773 munmap(addr, mem_sz); 774 775 return -1; 776 } 777 778 static int 779 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 780 { 781 struct extmem_param param; 782 int socket_id, ret; 783 784 memset(¶m, 0, sizeof(param)); 785 786 /* check if our heap exists */ 787 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 788 if (socket_id < 0) { 789 /* create our heap */ 790 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 791 if (ret < 0) { 792 TESTPMD_LOG(ERR, "Cannot create heap\n"); 793 return -1; 794 } 795 } 796 797 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 798 if (ret < 0) { 799 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 800 return -1; 801 } 802 803 /* we now have a valid memory area, so add it to heap */ 804 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 805 param.addr, param.len, param.iova_table, 806 param.iova_table_len, param.pgsz); 807 808 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 809 810 /* not needed any more */ 811 free(param.iova_table); 812 813 if (ret < 0) { 814 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 815 munmap(param.addr, param.len); 816 return -1; 817 } 818 819 /* success */ 820 821 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 822 param.len >> 20); 823 824 return 0; 825 } 826 static void 827 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 828 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 829 { 830 uint16_t pid = 0; 831 int ret; 832 833 RTE_ETH_FOREACH_DEV(pid) { 834 struct rte_eth_dev *dev = 835 &rte_eth_devices[pid]; 836 837 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 838 memhdr->len); 839 if (ret) { 840 TESTPMD_LOG(DEBUG, 841 "unable to DMA unmap addr 0x%p " 842 "for device %s\n", 843 memhdr->addr, dev->data->name); 844 } 845 } 846 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 847 if (ret) { 848 TESTPMD_LOG(DEBUG, 849 "unable to un-register addr 0x%p\n", memhdr->addr); 850 } 851 } 852 853 static void 854 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 855 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 856 { 857 uint16_t pid = 0; 858 size_t page_size = sysconf(_SC_PAGESIZE); 859 int ret; 860 861 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 862 page_size); 863 if (ret) { 864 TESTPMD_LOG(DEBUG, 865 "unable to register addr 0x%p\n", memhdr->addr); 866 return; 867 } 868 RTE_ETH_FOREACH_DEV(pid) { 869 struct rte_eth_dev *dev = 870 &rte_eth_devices[pid]; 871 872 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 873 memhdr->len); 874 if (ret) { 875 TESTPMD_LOG(DEBUG, 876 "unable to DMA map addr 0x%p " 877 "for device %s\n", 878 memhdr->addr, dev->data->name); 879 } 880 } 881 } 882 883 static unsigned int 884 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 885 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 886 { 887 struct rte_pktmbuf_extmem *xmem; 888 unsigned int ext_num, zone_num, elt_num; 889 uint16_t elt_size; 890 891 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 892 elt_num = EXTBUF_ZONE_SIZE / elt_size; 893 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 894 895 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 896 if (xmem == NULL) { 897 TESTPMD_LOG(ERR, "Cannot allocate memory for " 898 "external buffer descriptors\n"); 899 *ext_mem = NULL; 900 return 0; 901 } 902 for (ext_num = 0; ext_num < zone_num; ext_num++) { 903 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 904 const struct rte_memzone *mz; 905 char mz_name[RTE_MEMZONE_NAMESIZE]; 906 int ret; 907 908 ret = snprintf(mz_name, sizeof(mz_name), 909 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 910 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 911 errno = ENAMETOOLONG; 912 ext_num = 0; 913 break; 914 } 915 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 916 socket_id, 917 RTE_MEMZONE_IOVA_CONTIG | 918 RTE_MEMZONE_1GB | 919 RTE_MEMZONE_SIZE_HINT_ONLY, 920 EXTBUF_ZONE_SIZE); 921 if (mz == NULL) { 922 /* 923 * The caller exits on external buffer creation 924 * error, so there is no need to free memzones. 925 */ 926 errno = ENOMEM; 927 ext_num = 0; 928 break; 929 } 930 xseg->buf_ptr = mz->addr; 931 xseg->buf_iova = mz->iova; 932 xseg->buf_len = EXTBUF_ZONE_SIZE; 933 xseg->elt_size = elt_size; 934 } 935 if (ext_num == 0 && xmem != NULL) { 936 free(xmem); 937 xmem = NULL; 938 } 939 *ext_mem = xmem; 940 return ext_num; 941 } 942 943 /* 944 * Configuration initialisation done once at init time. 945 */ 946 static struct rte_mempool * 947 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 948 unsigned int socket_id) 949 { 950 char pool_name[RTE_MEMPOOL_NAMESIZE]; 951 struct rte_mempool *rte_mp = NULL; 952 uint32_t mb_size; 953 954 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 955 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 956 957 TESTPMD_LOG(INFO, 958 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 959 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 960 961 switch (mp_alloc_type) { 962 case MP_ALLOC_NATIVE: 963 { 964 /* wrapper to rte_mempool_create() */ 965 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 966 rte_mbuf_best_mempool_ops()); 967 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 968 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 969 break; 970 } 971 case MP_ALLOC_ANON: 972 { 973 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 974 mb_size, (unsigned int) mb_mempool_cache, 975 sizeof(struct rte_pktmbuf_pool_private), 976 socket_id, mempool_flags); 977 if (rte_mp == NULL) 978 goto err; 979 980 if (rte_mempool_populate_anon(rte_mp) == 0) { 981 rte_mempool_free(rte_mp); 982 rte_mp = NULL; 983 goto err; 984 } 985 rte_pktmbuf_pool_init(rte_mp, NULL); 986 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 987 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 988 break; 989 } 990 case MP_ALLOC_XMEM: 991 case MP_ALLOC_XMEM_HUGE: 992 { 993 int heap_socket; 994 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 995 996 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 997 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 998 999 heap_socket = 1000 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1001 if (heap_socket < 0) 1002 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1003 1004 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1005 rte_mbuf_best_mempool_ops()); 1006 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1007 mb_mempool_cache, 0, mbuf_seg_size, 1008 heap_socket); 1009 break; 1010 } 1011 case MP_ALLOC_XBUF: 1012 { 1013 struct rte_pktmbuf_extmem *ext_mem; 1014 unsigned int ext_num; 1015 1016 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1017 socket_id, pool_name, &ext_mem); 1018 if (ext_num == 0) 1019 rte_exit(EXIT_FAILURE, 1020 "Can't create pinned data buffers\n"); 1021 1022 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1023 rte_mbuf_best_mempool_ops()); 1024 rte_mp = rte_pktmbuf_pool_create_extbuf 1025 (pool_name, nb_mbuf, mb_mempool_cache, 1026 0, mbuf_seg_size, socket_id, 1027 ext_mem, ext_num); 1028 free(ext_mem); 1029 break; 1030 } 1031 default: 1032 { 1033 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1034 } 1035 } 1036 1037 err: 1038 if (rte_mp == NULL) { 1039 rte_exit(EXIT_FAILURE, 1040 "Creation of mbuf pool for socket %u failed: %s\n", 1041 socket_id, rte_strerror(rte_errno)); 1042 } else if (verbose_level > 0) { 1043 rte_mempool_dump(stdout, rte_mp); 1044 } 1045 return rte_mp; 1046 } 1047 1048 /* 1049 * Check given socket id is valid or not with NUMA mode, 1050 * if valid, return 0, else return -1 1051 */ 1052 static int 1053 check_socket_id(const unsigned int socket_id) 1054 { 1055 static int warning_once = 0; 1056 1057 if (new_socket_id(socket_id)) { 1058 if (!warning_once && numa_support) 1059 printf("Warning: NUMA should be configured manually by" 1060 " using --port-numa-config and" 1061 " --ring-numa-config parameters along with" 1062 " --numa.\n"); 1063 warning_once = 1; 1064 return -1; 1065 } 1066 return 0; 1067 } 1068 1069 /* 1070 * Get the allowed maximum number of RX queues. 1071 * *pid return the port id which has minimal value of 1072 * max_rx_queues in all ports. 1073 */ 1074 queueid_t 1075 get_allowed_max_nb_rxq(portid_t *pid) 1076 { 1077 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1078 bool max_rxq_valid = false; 1079 portid_t pi; 1080 struct rte_eth_dev_info dev_info; 1081 1082 RTE_ETH_FOREACH_DEV(pi) { 1083 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1084 continue; 1085 1086 max_rxq_valid = true; 1087 if (dev_info.max_rx_queues < allowed_max_rxq) { 1088 allowed_max_rxq = dev_info.max_rx_queues; 1089 *pid = pi; 1090 } 1091 } 1092 return max_rxq_valid ? allowed_max_rxq : 0; 1093 } 1094 1095 /* 1096 * Check input rxq is valid or not. 1097 * If input rxq is not greater than any of maximum number 1098 * of RX queues of all ports, it is valid. 1099 * if valid, return 0, else return -1 1100 */ 1101 int 1102 check_nb_rxq(queueid_t rxq) 1103 { 1104 queueid_t allowed_max_rxq; 1105 portid_t pid = 0; 1106 1107 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1108 if (rxq > allowed_max_rxq) { 1109 printf("Fail: input rxq (%u) can't be greater " 1110 "than max_rx_queues (%u) of port %u\n", 1111 rxq, 1112 allowed_max_rxq, 1113 pid); 1114 return -1; 1115 } 1116 return 0; 1117 } 1118 1119 /* 1120 * Get the allowed maximum number of TX queues. 1121 * *pid return the port id which has minimal value of 1122 * max_tx_queues in all ports. 1123 */ 1124 queueid_t 1125 get_allowed_max_nb_txq(portid_t *pid) 1126 { 1127 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1128 bool max_txq_valid = false; 1129 portid_t pi; 1130 struct rte_eth_dev_info dev_info; 1131 1132 RTE_ETH_FOREACH_DEV(pi) { 1133 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1134 continue; 1135 1136 max_txq_valid = true; 1137 if (dev_info.max_tx_queues < allowed_max_txq) { 1138 allowed_max_txq = dev_info.max_tx_queues; 1139 *pid = pi; 1140 } 1141 } 1142 return max_txq_valid ? allowed_max_txq : 0; 1143 } 1144 1145 /* 1146 * Check input txq is valid or not. 1147 * If input txq is not greater than any of maximum number 1148 * of TX queues of all ports, it is valid. 1149 * if valid, return 0, else return -1 1150 */ 1151 int 1152 check_nb_txq(queueid_t txq) 1153 { 1154 queueid_t allowed_max_txq; 1155 portid_t pid = 0; 1156 1157 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1158 if (txq > allowed_max_txq) { 1159 printf("Fail: input txq (%u) can't be greater " 1160 "than max_tx_queues (%u) of port %u\n", 1161 txq, 1162 allowed_max_txq, 1163 pid); 1164 return -1; 1165 } 1166 return 0; 1167 } 1168 1169 /* 1170 * Get the allowed maximum number of RXDs of every rx queue. 1171 * *pid return the port id which has minimal value of 1172 * max_rxd in all queues of all ports. 1173 */ 1174 static uint16_t 1175 get_allowed_max_nb_rxd(portid_t *pid) 1176 { 1177 uint16_t allowed_max_rxd = UINT16_MAX; 1178 portid_t pi; 1179 struct rte_eth_dev_info dev_info; 1180 1181 RTE_ETH_FOREACH_DEV(pi) { 1182 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1183 continue; 1184 1185 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1186 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1187 *pid = pi; 1188 } 1189 } 1190 return allowed_max_rxd; 1191 } 1192 1193 /* 1194 * Get the allowed minimal number of RXDs of every rx queue. 1195 * *pid return the port id which has minimal value of 1196 * min_rxd in all queues of all ports. 1197 */ 1198 static uint16_t 1199 get_allowed_min_nb_rxd(portid_t *pid) 1200 { 1201 uint16_t allowed_min_rxd = 0; 1202 portid_t pi; 1203 struct rte_eth_dev_info dev_info; 1204 1205 RTE_ETH_FOREACH_DEV(pi) { 1206 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1207 continue; 1208 1209 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1210 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1211 *pid = pi; 1212 } 1213 } 1214 1215 return allowed_min_rxd; 1216 } 1217 1218 /* 1219 * Check input rxd is valid or not. 1220 * If input rxd is not greater than any of maximum number 1221 * of RXDs of every Rx queues and is not less than any of 1222 * minimal number of RXDs of every Rx queues, it is valid. 1223 * if valid, return 0, else return -1 1224 */ 1225 int 1226 check_nb_rxd(queueid_t rxd) 1227 { 1228 uint16_t allowed_max_rxd; 1229 uint16_t allowed_min_rxd; 1230 portid_t pid = 0; 1231 1232 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1233 if (rxd > allowed_max_rxd) { 1234 printf("Fail: input rxd (%u) can't be greater " 1235 "than max_rxds (%u) of port %u\n", 1236 rxd, 1237 allowed_max_rxd, 1238 pid); 1239 return -1; 1240 } 1241 1242 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1243 if (rxd < allowed_min_rxd) { 1244 printf("Fail: input rxd (%u) can't be less " 1245 "than min_rxds (%u) of port %u\n", 1246 rxd, 1247 allowed_min_rxd, 1248 pid); 1249 return -1; 1250 } 1251 1252 return 0; 1253 } 1254 1255 /* 1256 * Get the allowed maximum number of TXDs of every rx queues. 1257 * *pid return the port id which has minimal value of 1258 * max_txd in every tx queue. 1259 */ 1260 static uint16_t 1261 get_allowed_max_nb_txd(portid_t *pid) 1262 { 1263 uint16_t allowed_max_txd = UINT16_MAX; 1264 portid_t pi; 1265 struct rte_eth_dev_info dev_info; 1266 1267 RTE_ETH_FOREACH_DEV(pi) { 1268 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1269 continue; 1270 1271 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1272 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1273 *pid = pi; 1274 } 1275 } 1276 return allowed_max_txd; 1277 } 1278 1279 /* 1280 * Get the allowed maximum number of TXDs of every tx queues. 1281 * *pid return the port id which has minimal value of 1282 * min_txd in every tx queue. 1283 */ 1284 static uint16_t 1285 get_allowed_min_nb_txd(portid_t *pid) 1286 { 1287 uint16_t allowed_min_txd = 0; 1288 portid_t pi; 1289 struct rte_eth_dev_info dev_info; 1290 1291 RTE_ETH_FOREACH_DEV(pi) { 1292 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1293 continue; 1294 1295 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1296 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1297 *pid = pi; 1298 } 1299 } 1300 1301 return allowed_min_txd; 1302 } 1303 1304 /* 1305 * Check input txd is valid or not. 1306 * If input txd is not greater than any of maximum number 1307 * of TXDs of every Rx queues, it is valid. 1308 * if valid, return 0, else return -1 1309 */ 1310 int 1311 check_nb_txd(queueid_t txd) 1312 { 1313 uint16_t allowed_max_txd; 1314 uint16_t allowed_min_txd; 1315 portid_t pid = 0; 1316 1317 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1318 if (txd > allowed_max_txd) { 1319 printf("Fail: input txd (%u) can't be greater " 1320 "than max_txds (%u) of port %u\n", 1321 txd, 1322 allowed_max_txd, 1323 pid); 1324 return -1; 1325 } 1326 1327 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1328 if (txd < allowed_min_txd) { 1329 printf("Fail: input txd (%u) can't be less " 1330 "than min_txds (%u) of port %u\n", 1331 txd, 1332 allowed_min_txd, 1333 pid); 1334 return -1; 1335 } 1336 return 0; 1337 } 1338 1339 1340 /* 1341 * Get the allowed maximum number of hairpin queues. 1342 * *pid return the port id which has minimal value of 1343 * max_hairpin_queues in all ports. 1344 */ 1345 queueid_t 1346 get_allowed_max_nb_hairpinq(portid_t *pid) 1347 { 1348 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1349 portid_t pi; 1350 struct rte_eth_hairpin_cap cap; 1351 1352 RTE_ETH_FOREACH_DEV(pi) { 1353 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1354 *pid = pi; 1355 return 0; 1356 } 1357 if (cap.max_nb_queues < allowed_max_hairpinq) { 1358 allowed_max_hairpinq = cap.max_nb_queues; 1359 *pid = pi; 1360 } 1361 } 1362 return allowed_max_hairpinq; 1363 } 1364 1365 /* 1366 * Check input hairpin is valid or not. 1367 * If input hairpin is not greater than any of maximum number 1368 * of hairpin queues of all ports, it is valid. 1369 * if valid, return 0, else return -1 1370 */ 1371 int 1372 check_nb_hairpinq(queueid_t hairpinq) 1373 { 1374 queueid_t allowed_max_hairpinq; 1375 portid_t pid = 0; 1376 1377 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1378 if (hairpinq > allowed_max_hairpinq) { 1379 printf("Fail: input hairpin (%u) can't be greater " 1380 "than max_hairpin_queues (%u) of port %u\n", 1381 hairpinq, allowed_max_hairpinq, pid); 1382 return -1; 1383 } 1384 return 0; 1385 } 1386 1387 static void 1388 init_config(void) 1389 { 1390 portid_t pid; 1391 struct rte_port *port; 1392 struct rte_mempool *mbp; 1393 unsigned int nb_mbuf_per_pool; 1394 lcoreid_t lc_id; 1395 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1396 struct rte_gro_param gro_param; 1397 uint32_t gso_types; 1398 uint16_t data_size; 1399 bool warning = 0; 1400 int k; 1401 int ret; 1402 1403 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1404 1405 /* Configuration of logical cores. */ 1406 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1407 sizeof(struct fwd_lcore *) * nb_lcores, 1408 RTE_CACHE_LINE_SIZE); 1409 if (fwd_lcores == NULL) { 1410 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1411 "failed\n", nb_lcores); 1412 } 1413 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1414 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1415 sizeof(struct fwd_lcore), 1416 RTE_CACHE_LINE_SIZE); 1417 if (fwd_lcores[lc_id] == NULL) { 1418 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1419 "failed\n"); 1420 } 1421 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1422 } 1423 1424 RTE_ETH_FOREACH_DEV(pid) { 1425 port = &ports[pid]; 1426 /* Apply default TxRx configuration for all ports */ 1427 port->dev_conf.txmode = tx_mode; 1428 port->dev_conf.rxmode = rx_mode; 1429 1430 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1431 if (ret != 0) 1432 rte_exit(EXIT_FAILURE, 1433 "rte_eth_dev_info_get() failed\n"); 1434 1435 if (!(port->dev_info.tx_offload_capa & 1436 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1437 port->dev_conf.txmode.offloads &= 1438 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1439 if (numa_support) { 1440 if (port_numa[pid] != NUMA_NO_CONFIG) 1441 port_per_socket[port_numa[pid]]++; 1442 else { 1443 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1444 1445 /* 1446 * if socket_id is invalid, 1447 * set to the first available socket. 1448 */ 1449 if (check_socket_id(socket_id) < 0) 1450 socket_id = socket_ids[0]; 1451 port_per_socket[socket_id]++; 1452 } 1453 } 1454 1455 /* Apply Rx offloads configuration */ 1456 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1457 port->rx_conf[k].offloads = 1458 port->dev_conf.rxmode.offloads; 1459 /* Apply Tx offloads configuration */ 1460 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1461 port->tx_conf[k].offloads = 1462 port->dev_conf.txmode.offloads; 1463 1464 /* set flag to initialize port/queue */ 1465 port->need_reconfig = 1; 1466 port->need_reconfig_queues = 1; 1467 port->tx_metadata = 0; 1468 1469 /* Check for maximum number of segments per MTU. Accordingly 1470 * update the mbuf data size. 1471 */ 1472 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1473 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1474 data_size = rx_mode.max_rx_pkt_len / 1475 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1476 1477 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1478 mbuf_data_size) { 1479 mbuf_data_size = data_size + 1480 RTE_PKTMBUF_HEADROOM; 1481 warning = 1; 1482 } 1483 } 1484 } 1485 1486 if (warning) 1487 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1488 mbuf_data_size); 1489 1490 /* 1491 * Create pools of mbuf. 1492 * If NUMA support is disabled, create a single pool of mbuf in 1493 * socket 0 memory by default. 1494 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1495 * 1496 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1497 * nb_txd can be configured at run time. 1498 */ 1499 if (param_total_num_mbufs) 1500 nb_mbuf_per_pool = param_total_num_mbufs; 1501 else { 1502 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1503 (nb_lcores * mb_mempool_cache) + 1504 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1505 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1506 } 1507 1508 if (numa_support) { 1509 uint8_t i; 1510 1511 for (i = 0; i < num_sockets; i++) 1512 mempools[i] = mbuf_pool_create(mbuf_data_size, 1513 nb_mbuf_per_pool, 1514 socket_ids[i]); 1515 } else { 1516 if (socket_num == UMA_NO_CONFIG) 1517 mempools[0] = mbuf_pool_create(mbuf_data_size, 1518 nb_mbuf_per_pool, 0); 1519 else 1520 mempools[socket_num] = mbuf_pool_create 1521 (mbuf_data_size, 1522 nb_mbuf_per_pool, 1523 socket_num); 1524 } 1525 1526 init_port_config(); 1527 1528 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1529 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1530 /* 1531 * Records which Mbuf pool to use by each logical core, if needed. 1532 */ 1533 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1534 mbp = mbuf_pool_find( 1535 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1536 1537 if (mbp == NULL) 1538 mbp = mbuf_pool_find(0); 1539 fwd_lcores[lc_id]->mbp = mbp; 1540 /* initialize GSO context */ 1541 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1542 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1543 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1544 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1545 RTE_ETHER_CRC_LEN; 1546 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1547 } 1548 1549 /* Configuration of packet forwarding streams. */ 1550 if (init_fwd_streams() < 0) 1551 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1552 1553 fwd_config_setup(); 1554 1555 /* create a gro context for each lcore */ 1556 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1557 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1558 gro_param.max_item_per_flow = MAX_PKT_BURST; 1559 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1560 gro_param.socket_id = rte_lcore_to_socket_id( 1561 fwd_lcores_cpuids[lc_id]); 1562 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1563 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1564 rte_exit(EXIT_FAILURE, 1565 "rte_gro_ctx_create() failed\n"); 1566 } 1567 } 1568 } 1569 1570 1571 void 1572 reconfig(portid_t new_port_id, unsigned socket_id) 1573 { 1574 struct rte_port *port; 1575 int ret; 1576 1577 /* Reconfiguration of Ethernet ports. */ 1578 port = &ports[new_port_id]; 1579 1580 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1581 if (ret != 0) 1582 return; 1583 1584 /* set flag to initialize port/queue */ 1585 port->need_reconfig = 1; 1586 port->need_reconfig_queues = 1; 1587 port->socket_id = socket_id; 1588 1589 init_port_config(); 1590 } 1591 1592 1593 int 1594 init_fwd_streams(void) 1595 { 1596 portid_t pid; 1597 struct rte_port *port; 1598 streamid_t sm_id, nb_fwd_streams_new; 1599 queueid_t q; 1600 1601 /* set socket id according to numa or not */ 1602 RTE_ETH_FOREACH_DEV(pid) { 1603 port = &ports[pid]; 1604 if (nb_rxq > port->dev_info.max_rx_queues) { 1605 printf("Fail: nb_rxq(%d) is greater than " 1606 "max_rx_queues(%d)\n", nb_rxq, 1607 port->dev_info.max_rx_queues); 1608 return -1; 1609 } 1610 if (nb_txq > port->dev_info.max_tx_queues) { 1611 printf("Fail: nb_txq(%d) is greater than " 1612 "max_tx_queues(%d)\n", nb_txq, 1613 port->dev_info.max_tx_queues); 1614 return -1; 1615 } 1616 if (numa_support) { 1617 if (port_numa[pid] != NUMA_NO_CONFIG) 1618 port->socket_id = port_numa[pid]; 1619 else { 1620 port->socket_id = rte_eth_dev_socket_id(pid); 1621 1622 /* 1623 * if socket_id is invalid, 1624 * set to the first available socket. 1625 */ 1626 if (check_socket_id(port->socket_id) < 0) 1627 port->socket_id = socket_ids[0]; 1628 } 1629 } 1630 else { 1631 if (socket_num == UMA_NO_CONFIG) 1632 port->socket_id = 0; 1633 else 1634 port->socket_id = socket_num; 1635 } 1636 } 1637 1638 q = RTE_MAX(nb_rxq, nb_txq); 1639 if (q == 0) { 1640 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1641 return -1; 1642 } 1643 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1644 if (nb_fwd_streams_new == nb_fwd_streams) 1645 return 0; 1646 /* clear the old */ 1647 if (fwd_streams != NULL) { 1648 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1649 if (fwd_streams[sm_id] == NULL) 1650 continue; 1651 rte_free(fwd_streams[sm_id]); 1652 fwd_streams[sm_id] = NULL; 1653 } 1654 rte_free(fwd_streams); 1655 fwd_streams = NULL; 1656 } 1657 1658 /* init new */ 1659 nb_fwd_streams = nb_fwd_streams_new; 1660 if (nb_fwd_streams) { 1661 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1662 sizeof(struct fwd_stream *) * nb_fwd_streams, 1663 RTE_CACHE_LINE_SIZE); 1664 if (fwd_streams == NULL) 1665 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1666 " (struct fwd_stream *)) failed\n", 1667 nb_fwd_streams); 1668 1669 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1670 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1671 " struct fwd_stream", sizeof(struct fwd_stream), 1672 RTE_CACHE_LINE_SIZE); 1673 if (fwd_streams[sm_id] == NULL) 1674 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1675 "(struct fwd_stream) failed\n"); 1676 } 1677 } 1678 1679 return 0; 1680 } 1681 1682 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1683 static void 1684 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1685 { 1686 uint64_t total_burst, sburst; 1687 uint64_t nb_burst; 1688 uint64_t burst_stats[4]; 1689 uint16_t pktnb_stats[4]; 1690 uint16_t nb_pkt; 1691 int burst_percent[4], sburstp; 1692 int i; 1693 1694 /* 1695 * First compute the total number of packet bursts and the 1696 * two highest numbers of bursts of the same number of packets. 1697 */ 1698 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1699 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1700 1701 /* Show stats for 0 burst size always */ 1702 total_burst = pbs->pkt_burst_spread[0]; 1703 burst_stats[0] = pbs->pkt_burst_spread[0]; 1704 pktnb_stats[0] = 0; 1705 1706 /* Find the next 2 burst sizes with highest occurrences. */ 1707 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1708 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1709 1710 if (nb_burst == 0) 1711 continue; 1712 1713 total_burst += nb_burst; 1714 1715 if (nb_burst > burst_stats[1]) { 1716 burst_stats[2] = burst_stats[1]; 1717 pktnb_stats[2] = pktnb_stats[1]; 1718 burst_stats[1] = nb_burst; 1719 pktnb_stats[1] = nb_pkt; 1720 } else if (nb_burst > burst_stats[2]) { 1721 burst_stats[2] = nb_burst; 1722 pktnb_stats[2] = nb_pkt; 1723 } 1724 } 1725 if (total_burst == 0) 1726 return; 1727 1728 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1729 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1730 if (i == 3) { 1731 printf("%d%% of other]\n", 100 - sburstp); 1732 return; 1733 } 1734 1735 sburst += burst_stats[i]; 1736 if (sburst == total_burst) { 1737 printf("%d%% of %d pkts]\n", 1738 100 - sburstp, (int) pktnb_stats[i]); 1739 return; 1740 } 1741 1742 burst_percent[i] = 1743 (double)burst_stats[i] / total_burst * 100; 1744 printf("%d%% of %d pkts + ", 1745 burst_percent[i], (int) pktnb_stats[i]); 1746 sburstp += burst_percent[i]; 1747 } 1748 } 1749 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1750 1751 static void 1752 fwd_stream_stats_display(streamid_t stream_id) 1753 { 1754 struct fwd_stream *fs; 1755 static const char *fwd_top_stats_border = "-------"; 1756 1757 fs = fwd_streams[stream_id]; 1758 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1759 (fs->fwd_dropped == 0)) 1760 return; 1761 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1762 "TX Port=%2d/Queue=%2d %s\n", 1763 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1764 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1765 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1766 " TX-dropped: %-14"PRIu64, 1767 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1768 1769 /* if checksum mode */ 1770 if (cur_fwd_eng == &csum_fwd_engine) { 1771 printf(" RX- bad IP checksum: %-14"PRIu64 1772 " Rx- bad L4 checksum: %-14"PRIu64 1773 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1774 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1775 fs->rx_bad_outer_l4_csum); 1776 } else { 1777 printf("\n"); 1778 } 1779 1780 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1781 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1782 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1783 #endif 1784 } 1785 1786 void 1787 fwd_stats_display(void) 1788 { 1789 static const char *fwd_stats_border = "----------------------"; 1790 static const char *acc_stats_border = "+++++++++++++++"; 1791 struct { 1792 struct fwd_stream *rx_stream; 1793 struct fwd_stream *tx_stream; 1794 uint64_t tx_dropped; 1795 uint64_t rx_bad_ip_csum; 1796 uint64_t rx_bad_l4_csum; 1797 uint64_t rx_bad_outer_l4_csum; 1798 } ports_stats[RTE_MAX_ETHPORTS]; 1799 uint64_t total_rx_dropped = 0; 1800 uint64_t total_tx_dropped = 0; 1801 uint64_t total_rx_nombuf = 0; 1802 struct rte_eth_stats stats; 1803 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1804 uint64_t fwd_cycles = 0; 1805 #endif 1806 uint64_t total_recv = 0; 1807 uint64_t total_xmit = 0; 1808 struct rte_port *port; 1809 streamid_t sm_id; 1810 portid_t pt_id; 1811 int i; 1812 1813 memset(ports_stats, 0, sizeof(ports_stats)); 1814 1815 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1816 struct fwd_stream *fs = fwd_streams[sm_id]; 1817 1818 if (cur_fwd_config.nb_fwd_streams > 1819 cur_fwd_config.nb_fwd_ports) { 1820 fwd_stream_stats_display(sm_id); 1821 } else { 1822 ports_stats[fs->tx_port].tx_stream = fs; 1823 ports_stats[fs->rx_port].rx_stream = fs; 1824 } 1825 1826 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1827 1828 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1829 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1830 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1831 fs->rx_bad_outer_l4_csum; 1832 1833 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1834 fwd_cycles += fs->core_cycles; 1835 #endif 1836 } 1837 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1838 uint8_t j; 1839 1840 pt_id = fwd_ports_ids[i]; 1841 port = &ports[pt_id]; 1842 1843 rte_eth_stats_get(pt_id, &stats); 1844 stats.ipackets -= port->stats.ipackets; 1845 stats.opackets -= port->stats.opackets; 1846 stats.ibytes -= port->stats.ibytes; 1847 stats.obytes -= port->stats.obytes; 1848 stats.imissed -= port->stats.imissed; 1849 stats.oerrors -= port->stats.oerrors; 1850 stats.rx_nombuf -= port->stats.rx_nombuf; 1851 1852 total_recv += stats.ipackets; 1853 total_xmit += stats.opackets; 1854 total_rx_dropped += stats.imissed; 1855 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1856 total_tx_dropped += stats.oerrors; 1857 total_rx_nombuf += stats.rx_nombuf; 1858 1859 printf("\n %s Forward statistics for port %-2d %s\n", 1860 fwd_stats_border, pt_id, fwd_stats_border); 1861 1862 if (!port->rx_queue_stats_mapping_enabled && 1863 !port->tx_queue_stats_mapping_enabled) { 1864 printf(" RX-packets: %-14"PRIu64 1865 " RX-dropped: %-14"PRIu64 1866 "RX-total: %-"PRIu64"\n", 1867 stats.ipackets, stats.imissed, 1868 stats.ipackets + stats.imissed); 1869 1870 if (cur_fwd_eng == &csum_fwd_engine) 1871 printf(" Bad-ipcsum: %-14"PRIu64 1872 " Bad-l4csum: %-14"PRIu64 1873 "Bad-outer-l4csum: %-14"PRIu64"\n", 1874 ports_stats[pt_id].rx_bad_ip_csum, 1875 ports_stats[pt_id].rx_bad_l4_csum, 1876 ports_stats[pt_id].rx_bad_outer_l4_csum); 1877 if (stats.ierrors + stats.rx_nombuf > 0) { 1878 printf(" RX-error: %-"PRIu64"\n", 1879 stats.ierrors); 1880 printf(" RX-nombufs: %-14"PRIu64"\n", 1881 stats.rx_nombuf); 1882 } 1883 1884 printf(" TX-packets: %-14"PRIu64 1885 " TX-dropped: %-14"PRIu64 1886 "TX-total: %-"PRIu64"\n", 1887 stats.opackets, ports_stats[pt_id].tx_dropped, 1888 stats.opackets + ports_stats[pt_id].tx_dropped); 1889 } else { 1890 printf(" RX-packets: %14"PRIu64 1891 " RX-dropped:%14"PRIu64 1892 " RX-total:%14"PRIu64"\n", 1893 stats.ipackets, stats.imissed, 1894 stats.ipackets + stats.imissed); 1895 1896 if (cur_fwd_eng == &csum_fwd_engine) 1897 printf(" Bad-ipcsum:%14"PRIu64 1898 " Bad-l4csum:%14"PRIu64 1899 " Bad-outer-l4csum: %-14"PRIu64"\n", 1900 ports_stats[pt_id].rx_bad_ip_csum, 1901 ports_stats[pt_id].rx_bad_l4_csum, 1902 ports_stats[pt_id].rx_bad_outer_l4_csum); 1903 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1904 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1905 printf(" RX-nombufs: %14"PRIu64"\n", 1906 stats.rx_nombuf); 1907 } 1908 1909 printf(" TX-packets: %14"PRIu64 1910 " TX-dropped:%14"PRIu64 1911 " TX-total:%14"PRIu64"\n", 1912 stats.opackets, ports_stats[pt_id].tx_dropped, 1913 stats.opackets + ports_stats[pt_id].tx_dropped); 1914 } 1915 1916 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1917 if (ports_stats[pt_id].rx_stream) 1918 pkt_burst_stats_display("RX", 1919 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1920 if (ports_stats[pt_id].tx_stream) 1921 pkt_burst_stats_display("TX", 1922 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1923 #endif 1924 1925 if (port->rx_queue_stats_mapping_enabled) { 1926 printf("\n"); 1927 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1928 printf(" Stats reg %2d RX-packets:%14"PRIu64 1929 " RX-errors:%14"PRIu64 1930 " RX-bytes:%14"PRIu64"\n", 1931 j, stats.q_ipackets[j], 1932 stats.q_errors[j], stats.q_ibytes[j]); 1933 } 1934 printf("\n"); 1935 } 1936 if (port->tx_queue_stats_mapping_enabled) { 1937 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1938 printf(" Stats reg %2d TX-packets:%14"PRIu64 1939 " TX-bytes:%14" 1940 PRIu64"\n", 1941 j, stats.q_opackets[j], 1942 stats.q_obytes[j]); 1943 } 1944 } 1945 1946 printf(" %s--------------------------------%s\n", 1947 fwd_stats_border, fwd_stats_border); 1948 } 1949 1950 printf("\n %s Accumulated forward statistics for all ports" 1951 "%s\n", 1952 acc_stats_border, acc_stats_border); 1953 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1954 "%-"PRIu64"\n" 1955 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1956 "%-"PRIu64"\n", 1957 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1958 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1959 if (total_rx_nombuf > 0) 1960 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1961 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1962 "%s\n", 1963 acc_stats_border, acc_stats_border); 1964 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1965 #define CYC_PER_MHZ 1E6 1966 if (total_recv > 0 || total_xmit > 0) { 1967 uint64_t total_pkts = 0; 1968 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1969 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1970 total_pkts = total_xmit; 1971 else 1972 total_pkts = total_recv; 1973 1974 printf("\n CPU cycles/packet=%.2F (total cycles=" 1975 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1976 " MHz Clock\n", 1977 (double) fwd_cycles / total_pkts, 1978 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1979 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1980 } 1981 #endif 1982 } 1983 1984 void 1985 fwd_stats_reset(void) 1986 { 1987 streamid_t sm_id; 1988 portid_t pt_id; 1989 int i; 1990 1991 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1992 pt_id = fwd_ports_ids[i]; 1993 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1994 } 1995 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1996 struct fwd_stream *fs = fwd_streams[sm_id]; 1997 1998 fs->rx_packets = 0; 1999 fs->tx_packets = 0; 2000 fs->fwd_dropped = 0; 2001 fs->rx_bad_ip_csum = 0; 2002 fs->rx_bad_l4_csum = 0; 2003 fs->rx_bad_outer_l4_csum = 0; 2004 2005 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 2006 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2007 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2008 #endif 2009 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 2010 fs->core_cycles = 0; 2011 #endif 2012 } 2013 } 2014 2015 static void 2016 flush_fwd_rx_queues(void) 2017 { 2018 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2019 portid_t rxp; 2020 portid_t port_id; 2021 queueid_t rxq; 2022 uint16_t nb_rx; 2023 uint16_t i; 2024 uint8_t j; 2025 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2026 uint64_t timer_period; 2027 2028 /* convert to number of cycles */ 2029 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2030 2031 for (j = 0; j < 2; j++) { 2032 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2033 for (rxq = 0; rxq < nb_rxq; rxq++) { 2034 port_id = fwd_ports_ids[rxp]; 2035 /** 2036 * testpmd can stuck in the below do while loop 2037 * if rte_eth_rx_burst() always returns nonzero 2038 * packets. So timer is added to exit this loop 2039 * after 1sec timer expiry. 2040 */ 2041 prev_tsc = rte_rdtsc(); 2042 do { 2043 nb_rx = rte_eth_rx_burst(port_id, rxq, 2044 pkts_burst, MAX_PKT_BURST); 2045 for (i = 0; i < nb_rx; i++) 2046 rte_pktmbuf_free(pkts_burst[i]); 2047 2048 cur_tsc = rte_rdtsc(); 2049 diff_tsc = cur_tsc - prev_tsc; 2050 timer_tsc += diff_tsc; 2051 } while ((nb_rx > 0) && 2052 (timer_tsc < timer_period)); 2053 timer_tsc = 0; 2054 } 2055 } 2056 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2057 } 2058 } 2059 2060 static void 2061 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2062 { 2063 struct fwd_stream **fsm; 2064 streamid_t nb_fs; 2065 streamid_t sm_id; 2066 #ifdef RTE_LIBRTE_BITRATE 2067 uint64_t tics_per_1sec; 2068 uint64_t tics_datum; 2069 uint64_t tics_current; 2070 uint16_t i, cnt_ports; 2071 2072 cnt_ports = nb_ports; 2073 tics_datum = rte_rdtsc(); 2074 tics_per_1sec = rte_get_timer_hz(); 2075 #endif 2076 fsm = &fwd_streams[fc->stream_idx]; 2077 nb_fs = fc->stream_nb; 2078 do { 2079 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2080 (*pkt_fwd)(fsm[sm_id]); 2081 #ifdef RTE_LIBRTE_BITRATE 2082 if (bitrate_enabled != 0 && 2083 bitrate_lcore_id == rte_lcore_id()) { 2084 tics_current = rte_rdtsc(); 2085 if (tics_current - tics_datum >= tics_per_1sec) { 2086 /* Periodic bitrate calculation */ 2087 for (i = 0; i < cnt_ports; i++) 2088 rte_stats_bitrate_calc(bitrate_data, 2089 ports_ids[i]); 2090 tics_datum = tics_current; 2091 } 2092 } 2093 #endif 2094 #ifdef RTE_LIBRTE_LATENCY_STATS 2095 if (latencystats_enabled != 0 && 2096 latencystats_lcore_id == rte_lcore_id()) 2097 rte_latencystats_update(); 2098 #endif 2099 2100 } while (! fc->stopped); 2101 } 2102 2103 static int 2104 start_pkt_forward_on_core(void *fwd_arg) 2105 { 2106 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2107 cur_fwd_config.fwd_eng->packet_fwd); 2108 return 0; 2109 } 2110 2111 /* 2112 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2113 * Used to start communication flows in network loopback test configurations. 2114 */ 2115 static int 2116 run_one_txonly_burst_on_core(void *fwd_arg) 2117 { 2118 struct fwd_lcore *fwd_lc; 2119 struct fwd_lcore tmp_lcore; 2120 2121 fwd_lc = (struct fwd_lcore *) fwd_arg; 2122 tmp_lcore = *fwd_lc; 2123 tmp_lcore.stopped = 1; 2124 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2125 return 0; 2126 } 2127 2128 /* 2129 * Launch packet forwarding: 2130 * - Setup per-port forwarding context. 2131 * - launch logical cores with their forwarding configuration. 2132 */ 2133 static void 2134 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2135 { 2136 port_fwd_begin_t port_fwd_begin; 2137 unsigned int i; 2138 unsigned int lc_id; 2139 int diag; 2140 2141 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2142 if (port_fwd_begin != NULL) { 2143 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2144 (*port_fwd_begin)(fwd_ports_ids[i]); 2145 } 2146 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2147 lc_id = fwd_lcores_cpuids[i]; 2148 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2149 fwd_lcores[i]->stopped = 0; 2150 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2151 fwd_lcores[i], lc_id); 2152 if (diag != 0) 2153 printf("launch lcore %u failed - diag=%d\n", 2154 lc_id, diag); 2155 } 2156 } 2157 } 2158 2159 /* 2160 * Launch packet forwarding configuration. 2161 */ 2162 void 2163 start_packet_forwarding(int with_tx_first) 2164 { 2165 port_fwd_begin_t port_fwd_begin; 2166 port_fwd_end_t port_fwd_end; 2167 struct rte_port *port; 2168 unsigned int i; 2169 portid_t pt_id; 2170 2171 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2172 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2173 2174 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2175 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2176 2177 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2178 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2179 (!nb_rxq || !nb_txq)) 2180 rte_exit(EXIT_FAILURE, 2181 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2182 cur_fwd_eng->fwd_mode_name); 2183 2184 if (all_ports_started() == 0) { 2185 printf("Not all ports were started\n"); 2186 return; 2187 } 2188 if (test_done == 0) { 2189 printf("Packet forwarding already started\n"); 2190 return; 2191 } 2192 2193 2194 if(dcb_test) { 2195 for (i = 0; i < nb_fwd_ports; i++) { 2196 pt_id = fwd_ports_ids[i]; 2197 port = &ports[pt_id]; 2198 if (!port->dcb_flag) { 2199 printf("In DCB mode, all forwarding ports must " 2200 "be configured in this mode.\n"); 2201 return; 2202 } 2203 } 2204 if (nb_fwd_lcores == 1) { 2205 printf("In DCB mode,the nb forwarding cores " 2206 "should be larger than 1.\n"); 2207 return; 2208 } 2209 } 2210 test_done = 0; 2211 2212 fwd_config_setup(); 2213 2214 if(!no_flush_rx) 2215 flush_fwd_rx_queues(); 2216 2217 pkt_fwd_config_display(&cur_fwd_config); 2218 rxtx_config_display(); 2219 2220 fwd_stats_reset(); 2221 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2222 pt_id = fwd_ports_ids[i]; 2223 port = &ports[pt_id]; 2224 map_port_queue_stats_mapping_registers(pt_id, port); 2225 } 2226 if (with_tx_first) { 2227 port_fwd_begin = tx_only_engine.port_fwd_begin; 2228 if (port_fwd_begin != NULL) { 2229 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2230 (*port_fwd_begin)(fwd_ports_ids[i]); 2231 } 2232 while (with_tx_first--) { 2233 launch_packet_forwarding( 2234 run_one_txonly_burst_on_core); 2235 rte_eal_mp_wait_lcore(); 2236 } 2237 port_fwd_end = tx_only_engine.port_fwd_end; 2238 if (port_fwd_end != NULL) { 2239 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2240 (*port_fwd_end)(fwd_ports_ids[i]); 2241 } 2242 } 2243 launch_packet_forwarding(start_pkt_forward_on_core); 2244 } 2245 2246 void 2247 stop_packet_forwarding(void) 2248 { 2249 port_fwd_end_t port_fwd_end; 2250 lcoreid_t lc_id; 2251 portid_t pt_id; 2252 int i; 2253 2254 if (test_done) { 2255 printf("Packet forwarding not started\n"); 2256 return; 2257 } 2258 printf("Telling cores to stop..."); 2259 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2260 fwd_lcores[lc_id]->stopped = 1; 2261 printf("\nWaiting for lcores to finish...\n"); 2262 rte_eal_mp_wait_lcore(); 2263 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2264 if (port_fwd_end != NULL) { 2265 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2266 pt_id = fwd_ports_ids[i]; 2267 (*port_fwd_end)(pt_id); 2268 } 2269 } 2270 2271 fwd_stats_display(); 2272 2273 printf("\nDone.\n"); 2274 test_done = 1; 2275 } 2276 2277 void 2278 dev_set_link_up(portid_t pid) 2279 { 2280 if (rte_eth_dev_set_link_up(pid) < 0) 2281 printf("\nSet link up fail.\n"); 2282 } 2283 2284 void 2285 dev_set_link_down(portid_t pid) 2286 { 2287 if (rte_eth_dev_set_link_down(pid) < 0) 2288 printf("\nSet link down fail.\n"); 2289 } 2290 2291 static int 2292 all_ports_started(void) 2293 { 2294 portid_t pi; 2295 struct rte_port *port; 2296 2297 RTE_ETH_FOREACH_DEV(pi) { 2298 port = &ports[pi]; 2299 /* Check if there is a port which is not started */ 2300 if ((port->port_status != RTE_PORT_STARTED) && 2301 (port->slave_flag == 0)) 2302 return 0; 2303 } 2304 2305 /* No port is not started */ 2306 return 1; 2307 } 2308 2309 int 2310 port_is_stopped(portid_t port_id) 2311 { 2312 struct rte_port *port = &ports[port_id]; 2313 2314 if ((port->port_status != RTE_PORT_STOPPED) && 2315 (port->slave_flag == 0)) 2316 return 0; 2317 return 1; 2318 } 2319 2320 int 2321 all_ports_stopped(void) 2322 { 2323 portid_t pi; 2324 2325 RTE_ETH_FOREACH_DEV(pi) { 2326 if (!port_is_stopped(pi)) 2327 return 0; 2328 } 2329 2330 return 1; 2331 } 2332 2333 int 2334 port_is_started(portid_t port_id) 2335 { 2336 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2337 return 0; 2338 2339 if (ports[port_id].port_status != RTE_PORT_STARTED) 2340 return 0; 2341 2342 return 1; 2343 } 2344 2345 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2346 static int 2347 setup_hairpin_queues(portid_t pi) 2348 { 2349 queueid_t qi; 2350 struct rte_eth_hairpin_conf hairpin_conf = { 2351 .peer_count = 1, 2352 }; 2353 int i; 2354 int diag; 2355 struct rte_port *port = &ports[pi]; 2356 2357 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2358 hairpin_conf.peers[0].port = pi; 2359 hairpin_conf.peers[0].queue = i + nb_rxq; 2360 diag = rte_eth_tx_hairpin_queue_setup 2361 (pi, qi, nb_txd, &hairpin_conf); 2362 i++; 2363 if (diag == 0) 2364 continue; 2365 2366 /* Fail to setup rx queue, return */ 2367 if (rte_atomic16_cmpset(&(port->port_status), 2368 RTE_PORT_HANDLING, 2369 RTE_PORT_STOPPED) == 0) 2370 printf("Port %d can not be set back " 2371 "to stopped\n", pi); 2372 printf("Fail to configure port %d hairpin " 2373 "queues\n", pi); 2374 /* try to reconfigure queues next time */ 2375 port->need_reconfig_queues = 1; 2376 return -1; 2377 } 2378 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2379 hairpin_conf.peers[0].port = pi; 2380 hairpin_conf.peers[0].queue = i + nb_txq; 2381 diag = rte_eth_rx_hairpin_queue_setup 2382 (pi, qi, nb_rxd, &hairpin_conf); 2383 i++; 2384 if (diag == 0) 2385 continue; 2386 2387 /* Fail to setup rx queue, return */ 2388 if (rte_atomic16_cmpset(&(port->port_status), 2389 RTE_PORT_HANDLING, 2390 RTE_PORT_STOPPED) == 0) 2391 printf("Port %d can not be set back " 2392 "to stopped\n", pi); 2393 printf("Fail to configure port %d hairpin " 2394 "queues\n", pi); 2395 /* try to reconfigure queues next time */ 2396 port->need_reconfig_queues = 1; 2397 return -1; 2398 } 2399 return 0; 2400 } 2401 2402 int 2403 start_port(portid_t pid) 2404 { 2405 int diag, need_check_link_status = -1; 2406 portid_t pi; 2407 queueid_t qi; 2408 struct rte_port *port; 2409 struct rte_ether_addr mac_addr; 2410 struct rte_eth_hairpin_cap cap; 2411 2412 if (port_id_is_invalid(pid, ENABLED_WARN)) 2413 return 0; 2414 2415 if(dcb_config) 2416 dcb_test = 1; 2417 RTE_ETH_FOREACH_DEV(pi) { 2418 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2419 continue; 2420 2421 need_check_link_status = 0; 2422 port = &ports[pi]; 2423 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2424 RTE_PORT_HANDLING) == 0) { 2425 printf("Port %d is now not stopped\n", pi); 2426 continue; 2427 } 2428 2429 if (port->need_reconfig > 0) { 2430 port->need_reconfig = 0; 2431 2432 if (flow_isolate_all) { 2433 int ret = port_flow_isolate(pi, 1); 2434 if (ret) { 2435 printf("Failed to apply isolated" 2436 " mode on port %d\n", pi); 2437 return -1; 2438 } 2439 } 2440 configure_rxtx_dump_callbacks(0); 2441 printf("Configuring Port %d (socket %u)\n", pi, 2442 port->socket_id); 2443 if (nb_hairpinq > 0 && 2444 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2445 printf("Port %d doesn't support hairpin " 2446 "queues\n", pi); 2447 return -1; 2448 } 2449 /* configure port */ 2450 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2451 nb_txq + nb_hairpinq, 2452 &(port->dev_conf)); 2453 if (diag != 0) { 2454 if (rte_atomic16_cmpset(&(port->port_status), 2455 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2456 printf("Port %d can not be set back " 2457 "to stopped\n", pi); 2458 printf("Fail to configure port %d\n", pi); 2459 /* try to reconfigure port next time */ 2460 port->need_reconfig = 1; 2461 return -1; 2462 } 2463 } 2464 if (port->need_reconfig_queues > 0) { 2465 port->need_reconfig_queues = 0; 2466 /* setup tx queues */ 2467 for (qi = 0; qi < nb_txq; qi++) { 2468 if ((numa_support) && 2469 (txring_numa[pi] != NUMA_NO_CONFIG)) 2470 diag = rte_eth_tx_queue_setup(pi, qi, 2471 port->nb_tx_desc[qi], 2472 txring_numa[pi], 2473 &(port->tx_conf[qi])); 2474 else 2475 diag = rte_eth_tx_queue_setup(pi, qi, 2476 port->nb_tx_desc[qi], 2477 port->socket_id, 2478 &(port->tx_conf[qi])); 2479 2480 if (diag == 0) 2481 continue; 2482 2483 /* Fail to setup tx queue, return */ 2484 if (rte_atomic16_cmpset(&(port->port_status), 2485 RTE_PORT_HANDLING, 2486 RTE_PORT_STOPPED) == 0) 2487 printf("Port %d can not be set back " 2488 "to stopped\n", pi); 2489 printf("Fail to configure port %d tx queues\n", 2490 pi); 2491 /* try to reconfigure queues next time */ 2492 port->need_reconfig_queues = 1; 2493 return -1; 2494 } 2495 for (qi = 0; qi < nb_rxq; qi++) { 2496 /* setup rx queues */ 2497 if ((numa_support) && 2498 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2499 struct rte_mempool * mp = 2500 mbuf_pool_find(rxring_numa[pi]); 2501 if (mp == NULL) { 2502 printf("Failed to setup RX queue:" 2503 "No mempool allocation" 2504 " on the socket %d\n", 2505 rxring_numa[pi]); 2506 return -1; 2507 } 2508 2509 diag = rte_eth_rx_queue_setup(pi, qi, 2510 port->nb_rx_desc[qi], 2511 rxring_numa[pi], 2512 &(port->rx_conf[qi]), 2513 mp); 2514 } else { 2515 struct rte_mempool *mp = 2516 mbuf_pool_find(port->socket_id); 2517 if (mp == NULL) { 2518 printf("Failed to setup RX queue:" 2519 "No mempool allocation" 2520 " on the socket %d\n", 2521 port->socket_id); 2522 return -1; 2523 } 2524 diag = rte_eth_rx_queue_setup(pi, qi, 2525 port->nb_rx_desc[qi], 2526 port->socket_id, 2527 &(port->rx_conf[qi]), 2528 mp); 2529 } 2530 if (diag == 0) 2531 continue; 2532 2533 /* Fail to setup rx queue, return */ 2534 if (rte_atomic16_cmpset(&(port->port_status), 2535 RTE_PORT_HANDLING, 2536 RTE_PORT_STOPPED) == 0) 2537 printf("Port %d can not be set back " 2538 "to stopped\n", pi); 2539 printf("Fail to configure port %d rx queues\n", 2540 pi); 2541 /* try to reconfigure queues next time */ 2542 port->need_reconfig_queues = 1; 2543 return -1; 2544 } 2545 /* setup hairpin queues */ 2546 if (setup_hairpin_queues(pi) != 0) 2547 return -1; 2548 } 2549 configure_rxtx_dump_callbacks(verbose_level); 2550 if (clear_ptypes) { 2551 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2552 NULL, 0); 2553 if (diag < 0) 2554 printf( 2555 "Port %d: Failed to disable Ptype parsing\n", 2556 pi); 2557 } 2558 2559 /* start port */ 2560 if (rte_eth_dev_start(pi) < 0) { 2561 printf("Fail to start port %d\n", pi); 2562 2563 /* Fail to setup rx queue, return */ 2564 if (rte_atomic16_cmpset(&(port->port_status), 2565 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2566 printf("Port %d can not be set back to " 2567 "stopped\n", pi); 2568 continue; 2569 } 2570 2571 if (rte_atomic16_cmpset(&(port->port_status), 2572 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2573 printf("Port %d can not be set into started\n", pi); 2574 2575 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2576 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2577 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2578 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2579 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2580 2581 /* at least one port started, need checking link status */ 2582 need_check_link_status = 1; 2583 } 2584 2585 if (need_check_link_status == 1 && !no_link_check) 2586 check_all_ports_link_status(RTE_PORT_ALL); 2587 else if (need_check_link_status == 0) 2588 printf("Please stop the ports first\n"); 2589 2590 printf("Done\n"); 2591 return 0; 2592 } 2593 2594 void 2595 stop_port(portid_t pid) 2596 { 2597 portid_t pi; 2598 struct rte_port *port; 2599 int need_check_link_status = 0; 2600 2601 if (dcb_test) { 2602 dcb_test = 0; 2603 dcb_config = 0; 2604 } 2605 2606 if (port_id_is_invalid(pid, ENABLED_WARN)) 2607 return; 2608 2609 printf("Stopping ports...\n"); 2610 2611 RTE_ETH_FOREACH_DEV(pi) { 2612 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2613 continue; 2614 2615 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2616 printf("Please remove port %d from forwarding configuration.\n", pi); 2617 continue; 2618 } 2619 2620 if (port_is_bonding_slave(pi)) { 2621 printf("Please remove port %d from bonded device.\n", pi); 2622 continue; 2623 } 2624 2625 port = &ports[pi]; 2626 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2627 RTE_PORT_HANDLING) == 0) 2628 continue; 2629 2630 rte_eth_dev_stop(pi); 2631 2632 if (rte_atomic16_cmpset(&(port->port_status), 2633 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2634 printf("Port %d can not be set into stopped\n", pi); 2635 need_check_link_status = 1; 2636 } 2637 if (need_check_link_status && !no_link_check) 2638 check_all_ports_link_status(RTE_PORT_ALL); 2639 2640 printf("Done\n"); 2641 } 2642 2643 static void 2644 remove_invalid_ports_in(portid_t *array, portid_t *total) 2645 { 2646 portid_t i; 2647 portid_t new_total = 0; 2648 2649 for (i = 0; i < *total; i++) 2650 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2651 array[new_total] = array[i]; 2652 new_total++; 2653 } 2654 *total = new_total; 2655 } 2656 2657 static void 2658 remove_invalid_ports(void) 2659 { 2660 remove_invalid_ports_in(ports_ids, &nb_ports); 2661 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2662 nb_cfg_ports = nb_fwd_ports; 2663 } 2664 2665 void 2666 close_port(portid_t pid) 2667 { 2668 portid_t pi; 2669 struct rte_port *port; 2670 2671 if (port_id_is_invalid(pid, ENABLED_WARN)) 2672 return; 2673 2674 printf("Closing ports...\n"); 2675 2676 RTE_ETH_FOREACH_DEV(pi) { 2677 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2678 continue; 2679 2680 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2681 printf("Please remove port %d from forwarding configuration.\n", pi); 2682 continue; 2683 } 2684 2685 if (port_is_bonding_slave(pi)) { 2686 printf("Please remove port %d from bonded device.\n", pi); 2687 continue; 2688 } 2689 2690 port = &ports[pi]; 2691 if (rte_atomic16_cmpset(&(port->port_status), 2692 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2693 printf("Port %d is already closed\n", pi); 2694 continue; 2695 } 2696 2697 if (rte_atomic16_cmpset(&(port->port_status), 2698 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2699 printf("Port %d is now not stopped\n", pi); 2700 continue; 2701 } 2702 2703 if (port->flow_list) 2704 port_flow_flush(pi); 2705 rte_eth_dev_close(pi); 2706 2707 remove_invalid_ports(); 2708 2709 if (rte_atomic16_cmpset(&(port->port_status), 2710 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2711 printf("Port %d cannot be set to closed\n", pi); 2712 } 2713 2714 printf("Done\n"); 2715 } 2716 2717 void 2718 reset_port(portid_t pid) 2719 { 2720 int diag; 2721 portid_t pi; 2722 struct rte_port *port; 2723 2724 if (port_id_is_invalid(pid, ENABLED_WARN)) 2725 return; 2726 2727 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2728 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2729 printf("Can not reset port(s), please stop port(s) first.\n"); 2730 return; 2731 } 2732 2733 printf("Resetting ports...\n"); 2734 2735 RTE_ETH_FOREACH_DEV(pi) { 2736 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2737 continue; 2738 2739 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2740 printf("Please remove port %d from forwarding " 2741 "configuration.\n", pi); 2742 continue; 2743 } 2744 2745 if (port_is_bonding_slave(pi)) { 2746 printf("Please remove port %d from bonded device.\n", 2747 pi); 2748 continue; 2749 } 2750 2751 diag = rte_eth_dev_reset(pi); 2752 if (diag == 0) { 2753 port = &ports[pi]; 2754 port->need_reconfig = 1; 2755 port->need_reconfig_queues = 1; 2756 } else { 2757 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2758 } 2759 } 2760 2761 printf("Done\n"); 2762 } 2763 2764 void 2765 attach_port(char *identifier) 2766 { 2767 portid_t pi; 2768 struct rte_dev_iterator iterator; 2769 2770 printf("Attaching a new port...\n"); 2771 2772 if (identifier == NULL) { 2773 printf("Invalid parameters are specified\n"); 2774 return; 2775 } 2776 2777 if (rte_dev_probe(identifier) < 0) { 2778 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2779 return; 2780 } 2781 2782 /* first attach mode: event */ 2783 if (setup_on_probe_event) { 2784 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2785 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2786 if (ports[pi].port_status == RTE_PORT_HANDLING && 2787 ports[pi].need_setup != 0) 2788 setup_attached_port(pi); 2789 return; 2790 } 2791 2792 /* second attach mode: iterator */ 2793 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2794 /* setup ports matching the devargs used for probing */ 2795 if (port_is_forwarding(pi)) 2796 continue; /* port was already attached before */ 2797 setup_attached_port(pi); 2798 } 2799 } 2800 2801 static void 2802 setup_attached_port(portid_t pi) 2803 { 2804 unsigned int socket_id; 2805 int ret; 2806 2807 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2808 /* if socket_id is invalid, set to the first available socket. */ 2809 if (check_socket_id(socket_id) < 0) 2810 socket_id = socket_ids[0]; 2811 reconfig(pi, socket_id); 2812 ret = rte_eth_promiscuous_enable(pi); 2813 if (ret != 0) 2814 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2815 pi, rte_strerror(-ret)); 2816 2817 ports_ids[nb_ports++] = pi; 2818 fwd_ports_ids[nb_fwd_ports++] = pi; 2819 nb_cfg_ports = nb_fwd_ports; 2820 ports[pi].need_setup = 0; 2821 ports[pi].port_status = RTE_PORT_STOPPED; 2822 2823 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2824 printf("Done\n"); 2825 } 2826 2827 static void 2828 detach_device(struct rte_device *dev) 2829 { 2830 portid_t sibling; 2831 2832 if (dev == NULL) { 2833 printf("Device already removed\n"); 2834 return; 2835 } 2836 2837 printf("Removing a device...\n"); 2838 2839 if (rte_dev_remove(dev) < 0) { 2840 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2841 return; 2842 } 2843 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2844 /* reset mapping between old ports and removed device */ 2845 rte_eth_devices[sibling].device = NULL; 2846 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2847 /* sibling ports are forced to be closed */ 2848 ports[sibling].port_status = RTE_PORT_CLOSED; 2849 printf("Port %u is closed\n", sibling); 2850 } 2851 } 2852 2853 remove_invalid_ports(); 2854 2855 printf("Device is detached\n"); 2856 printf("Now total ports is %d\n", nb_ports); 2857 printf("Done\n"); 2858 return; 2859 } 2860 2861 void 2862 detach_port_device(portid_t port_id) 2863 { 2864 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2865 return; 2866 2867 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2868 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2869 printf("Port not stopped\n"); 2870 return; 2871 } 2872 printf("Port was not closed\n"); 2873 if (ports[port_id].flow_list) 2874 port_flow_flush(port_id); 2875 } 2876 2877 detach_device(rte_eth_devices[port_id].device); 2878 } 2879 2880 void 2881 detach_devargs(char *identifier) 2882 { 2883 struct rte_dev_iterator iterator; 2884 struct rte_devargs da; 2885 portid_t port_id; 2886 2887 printf("Removing a device...\n"); 2888 2889 memset(&da, 0, sizeof(da)); 2890 if (rte_devargs_parsef(&da, "%s", identifier)) { 2891 printf("cannot parse identifier\n"); 2892 if (da.args) 2893 free(da.args); 2894 return; 2895 } 2896 2897 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2898 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2899 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2900 printf("Port %u not stopped\n", port_id); 2901 rte_eth_iterator_cleanup(&iterator); 2902 return; 2903 } 2904 2905 /* sibling ports are forced to be closed */ 2906 if (ports[port_id].flow_list) 2907 port_flow_flush(port_id); 2908 ports[port_id].port_status = RTE_PORT_CLOSED; 2909 printf("Port %u is now closed\n", port_id); 2910 } 2911 } 2912 2913 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2914 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2915 da.name, da.bus->name); 2916 return; 2917 } 2918 2919 remove_invalid_ports(); 2920 2921 printf("Device %s is detached\n", identifier); 2922 printf("Now total ports is %d\n", nb_ports); 2923 printf("Done\n"); 2924 } 2925 2926 void 2927 pmd_test_exit(void) 2928 { 2929 portid_t pt_id; 2930 int ret; 2931 int i; 2932 2933 if (test_done == 0) 2934 stop_packet_forwarding(); 2935 2936 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2937 if (mempools[i]) { 2938 if (mp_alloc_type == MP_ALLOC_ANON) 2939 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2940 NULL); 2941 } 2942 } 2943 if (ports != NULL) { 2944 no_link_check = 1; 2945 RTE_ETH_FOREACH_DEV(pt_id) { 2946 printf("\nStopping port %d...\n", pt_id); 2947 fflush(stdout); 2948 stop_port(pt_id); 2949 } 2950 RTE_ETH_FOREACH_DEV(pt_id) { 2951 printf("\nShutting down port %d...\n", pt_id); 2952 fflush(stdout); 2953 close_port(pt_id); 2954 } 2955 } 2956 2957 if (hot_plug) { 2958 ret = rte_dev_event_monitor_stop(); 2959 if (ret) { 2960 RTE_LOG(ERR, EAL, 2961 "fail to stop device event monitor."); 2962 return; 2963 } 2964 2965 ret = rte_dev_event_callback_unregister(NULL, 2966 dev_event_callback, NULL); 2967 if (ret < 0) { 2968 RTE_LOG(ERR, EAL, 2969 "fail to unregister device event callback.\n"); 2970 return; 2971 } 2972 2973 ret = rte_dev_hotplug_handle_disable(); 2974 if (ret) { 2975 RTE_LOG(ERR, EAL, 2976 "fail to disable hotplug handling.\n"); 2977 return; 2978 } 2979 } 2980 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2981 if (mempools[i]) 2982 rte_mempool_free(mempools[i]); 2983 } 2984 2985 printf("\nBye...\n"); 2986 } 2987 2988 typedef void (*cmd_func_t)(void); 2989 struct pmd_test_command { 2990 const char *cmd_name; 2991 cmd_func_t cmd_func; 2992 }; 2993 2994 /* Check the link status of all ports in up to 9s, and print them finally */ 2995 static void 2996 check_all_ports_link_status(uint32_t port_mask) 2997 { 2998 #define CHECK_INTERVAL 100 /* 100ms */ 2999 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3000 portid_t portid; 3001 uint8_t count, all_ports_up, print_flag = 0; 3002 struct rte_eth_link link; 3003 int ret; 3004 3005 printf("Checking link statuses...\n"); 3006 fflush(stdout); 3007 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3008 all_ports_up = 1; 3009 RTE_ETH_FOREACH_DEV(portid) { 3010 if ((port_mask & (1 << portid)) == 0) 3011 continue; 3012 memset(&link, 0, sizeof(link)); 3013 ret = rte_eth_link_get_nowait(portid, &link); 3014 if (ret < 0) { 3015 all_ports_up = 0; 3016 if (print_flag == 1) 3017 printf("Port %u link get failed: %s\n", 3018 portid, rte_strerror(-ret)); 3019 continue; 3020 } 3021 /* print link status if flag set */ 3022 if (print_flag == 1) { 3023 if (link.link_status) 3024 printf( 3025 "Port%d Link Up. speed %u Mbps- %s\n", 3026 portid, link.link_speed, 3027 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 3028 ("full-duplex") : ("half-duplex")); 3029 else 3030 printf("Port %d Link Down\n", portid); 3031 continue; 3032 } 3033 /* clear all_ports_up flag if any link down */ 3034 if (link.link_status == ETH_LINK_DOWN) { 3035 all_ports_up = 0; 3036 break; 3037 } 3038 } 3039 /* after finally printing all link status, get out */ 3040 if (print_flag == 1) 3041 break; 3042 3043 if (all_ports_up == 0) { 3044 fflush(stdout); 3045 rte_delay_ms(CHECK_INTERVAL); 3046 } 3047 3048 /* set the print_flag if all ports up or timeout */ 3049 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3050 print_flag = 1; 3051 } 3052 3053 if (lsc_interrupt) 3054 break; 3055 } 3056 } 3057 3058 /* 3059 * This callback is for remove a port for a device. It has limitation because 3060 * it is not for multiple port removal for a device. 3061 * TODO: the device detach invoke will plan to be removed from user side to 3062 * eal. And convert all PMDs to free port resources on ether device closing. 3063 */ 3064 static void 3065 rmv_port_callback(void *arg) 3066 { 3067 int need_to_start = 0; 3068 int org_no_link_check = no_link_check; 3069 portid_t port_id = (intptr_t)arg; 3070 struct rte_device *dev; 3071 3072 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3073 3074 if (!test_done && port_is_forwarding(port_id)) { 3075 need_to_start = 1; 3076 stop_packet_forwarding(); 3077 } 3078 no_link_check = 1; 3079 stop_port(port_id); 3080 no_link_check = org_no_link_check; 3081 3082 /* Save rte_device pointer before closing ethdev port */ 3083 dev = rte_eth_devices[port_id].device; 3084 close_port(port_id); 3085 detach_device(dev); /* might be already removed or have more ports */ 3086 3087 if (need_to_start) 3088 start_packet_forwarding(0); 3089 } 3090 3091 /* This function is used by the interrupt thread */ 3092 static int 3093 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3094 void *ret_param) 3095 { 3096 RTE_SET_USED(param); 3097 RTE_SET_USED(ret_param); 3098 3099 if (type >= RTE_ETH_EVENT_MAX) { 3100 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3101 port_id, __func__, type); 3102 fflush(stderr); 3103 } else if (event_print_mask & (UINT32_C(1) << type)) { 3104 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3105 eth_event_desc[type]); 3106 fflush(stdout); 3107 } 3108 3109 switch (type) { 3110 case RTE_ETH_EVENT_NEW: 3111 ports[port_id].need_setup = 1; 3112 ports[port_id].port_status = RTE_PORT_HANDLING; 3113 break; 3114 case RTE_ETH_EVENT_INTR_RMV: 3115 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3116 break; 3117 if (rte_eal_alarm_set(100000, 3118 rmv_port_callback, (void *)(intptr_t)port_id)) 3119 fprintf(stderr, "Could not set up deferred device removal\n"); 3120 break; 3121 default: 3122 break; 3123 } 3124 return 0; 3125 } 3126 3127 static int 3128 register_eth_event_callback(void) 3129 { 3130 int ret; 3131 enum rte_eth_event_type event; 3132 3133 for (event = RTE_ETH_EVENT_UNKNOWN; 3134 event < RTE_ETH_EVENT_MAX; event++) { 3135 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3136 event, 3137 eth_event_callback, 3138 NULL); 3139 if (ret != 0) { 3140 TESTPMD_LOG(ERR, "Failed to register callback for " 3141 "%s event\n", eth_event_desc[event]); 3142 return -1; 3143 } 3144 } 3145 3146 return 0; 3147 } 3148 3149 /* This function is used by the interrupt thread */ 3150 static void 3151 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3152 __rte_unused void *arg) 3153 { 3154 uint16_t port_id; 3155 int ret; 3156 3157 if (type >= RTE_DEV_EVENT_MAX) { 3158 fprintf(stderr, "%s called upon invalid event %d\n", 3159 __func__, type); 3160 fflush(stderr); 3161 } 3162 3163 switch (type) { 3164 case RTE_DEV_EVENT_REMOVE: 3165 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3166 device_name); 3167 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3168 if (ret) { 3169 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3170 device_name); 3171 return; 3172 } 3173 /* 3174 * Because the user's callback is invoked in eal interrupt 3175 * callback, the interrupt callback need to be finished before 3176 * it can be unregistered when detaching device. So finish 3177 * callback soon and use a deferred removal to detach device 3178 * is need. It is a workaround, once the device detaching be 3179 * moved into the eal in the future, the deferred removal could 3180 * be deleted. 3181 */ 3182 if (rte_eal_alarm_set(100000, 3183 rmv_port_callback, (void *)(intptr_t)port_id)) 3184 RTE_LOG(ERR, EAL, 3185 "Could not set up deferred device removal\n"); 3186 break; 3187 case RTE_DEV_EVENT_ADD: 3188 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3189 device_name); 3190 /* TODO: After finish kernel driver binding, 3191 * begin to attach port. 3192 */ 3193 break; 3194 default: 3195 break; 3196 } 3197 } 3198 3199 static int 3200 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3201 { 3202 uint16_t i; 3203 int diag; 3204 uint8_t mapping_found = 0; 3205 3206 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3207 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3208 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3209 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3210 tx_queue_stats_mappings[i].queue_id, 3211 tx_queue_stats_mappings[i].stats_counter_id); 3212 if (diag != 0) 3213 return diag; 3214 mapping_found = 1; 3215 } 3216 } 3217 if (mapping_found) 3218 port->tx_queue_stats_mapping_enabled = 1; 3219 return 0; 3220 } 3221 3222 static int 3223 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3224 { 3225 uint16_t i; 3226 int diag; 3227 uint8_t mapping_found = 0; 3228 3229 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3230 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3231 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3232 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3233 rx_queue_stats_mappings[i].queue_id, 3234 rx_queue_stats_mappings[i].stats_counter_id); 3235 if (diag != 0) 3236 return diag; 3237 mapping_found = 1; 3238 } 3239 } 3240 if (mapping_found) 3241 port->rx_queue_stats_mapping_enabled = 1; 3242 return 0; 3243 } 3244 3245 static void 3246 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3247 { 3248 int diag = 0; 3249 3250 diag = set_tx_queue_stats_mapping_registers(pi, port); 3251 if (diag != 0) { 3252 if (diag == -ENOTSUP) { 3253 port->tx_queue_stats_mapping_enabled = 0; 3254 printf("TX queue stats mapping not supported port id=%d\n", pi); 3255 } 3256 else 3257 rte_exit(EXIT_FAILURE, 3258 "set_tx_queue_stats_mapping_registers " 3259 "failed for port id=%d diag=%d\n", 3260 pi, diag); 3261 } 3262 3263 diag = set_rx_queue_stats_mapping_registers(pi, port); 3264 if (diag != 0) { 3265 if (diag == -ENOTSUP) { 3266 port->rx_queue_stats_mapping_enabled = 0; 3267 printf("RX queue stats mapping not supported port id=%d\n", pi); 3268 } 3269 else 3270 rte_exit(EXIT_FAILURE, 3271 "set_rx_queue_stats_mapping_registers " 3272 "failed for port id=%d diag=%d\n", 3273 pi, diag); 3274 } 3275 } 3276 3277 static void 3278 rxtx_port_config(struct rte_port *port) 3279 { 3280 uint16_t qid; 3281 uint64_t offloads; 3282 3283 for (qid = 0; qid < nb_rxq; qid++) { 3284 offloads = port->rx_conf[qid].offloads; 3285 port->rx_conf[qid] = port->dev_info.default_rxconf; 3286 if (offloads != 0) 3287 port->rx_conf[qid].offloads = offloads; 3288 3289 /* Check if any Rx parameters have been passed */ 3290 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3291 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3292 3293 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3294 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3295 3296 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3297 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3298 3299 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3300 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3301 3302 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3303 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3304 3305 port->nb_rx_desc[qid] = nb_rxd; 3306 } 3307 3308 for (qid = 0; qid < nb_txq; qid++) { 3309 offloads = port->tx_conf[qid].offloads; 3310 port->tx_conf[qid] = port->dev_info.default_txconf; 3311 if (offloads != 0) 3312 port->tx_conf[qid].offloads = offloads; 3313 3314 /* Check if any Tx parameters have been passed */ 3315 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3316 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3317 3318 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3319 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3320 3321 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3322 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3323 3324 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3325 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3326 3327 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3328 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3329 3330 port->nb_tx_desc[qid] = nb_txd; 3331 } 3332 } 3333 3334 void 3335 init_port_config(void) 3336 { 3337 portid_t pid; 3338 struct rte_port *port; 3339 int ret; 3340 3341 RTE_ETH_FOREACH_DEV(pid) { 3342 port = &ports[pid]; 3343 port->dev_conf.fdir_conf = fdir_conf; 3344 3345 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3346 if (ret != 0) 3347 return; 3348 3349 if (nb_rxq > 1) { 3350 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3351 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3352 rss_hf & port->dev_info.flow_type_rss_offloads; 3353 } else { 3354 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3355 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3356 } 3357 3358 if (port->dcb_flag == 0) { 3359 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3360 port->dev_conf.rxmode.mq_mode = 3361 (enum rte_eth_rx_mq_mode) 3362 (rx_mq_mode & ETH_MQ_RX_RSS); 3363 else 3364 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3365 } 3366 3367 rxtx_port_config(port); 3368 3369 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3370 if (ret != 0) 3371 return; 3372 3373 map_port_queue_stats_mapping_registers(pid, port); 3374 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3375 rte_pmd_ixgbe_bypass_init(pid); 3376 #endif 3377 3378 if (lsc_interrupt && 3379 (rte_eth_devices[pid].data->dev_flags & 3380 RTE_ETH_DEV_INTR_LSC)) 3381 port->dev_conf.intr_conf.lsc = 1; 3382 if (rmv_interrupt && 3383 (rte_eth_devices[pid].data->dev_flags & 3384 RTE_ETH_DEV_INTR_RMV)) 3385 port->dev_conf.intr_conf.rmv = 1; 3386 } 3387 } 3388 3389 void set_port_slave_flag(portid_t slave_pid) 3390 { 3391 struct rte_port *port; 3392 3393 port = &ports[slave_pid]; 3394 port->slave_flag = 1; 3395 } 3396 3397 void clear_port_slave_flag(portid_t slave_pid) 3398 { 3399 struct rte_port *port; 3400 3401 port = &ports[slave_pid]; 3402 port->slave_flag = 0; 3403 } 3404 3405 uint8_t port_is_bonding_slave(portid_t slave_pid) 3406 { 3407 struct rte_port *port; 3408 3409 port = &ports[slave_pid]; 3410 if ((rte_eth_devices[slave_pid].data->dev_flags & 3411 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3412 return 1; 3413 return 0; 3414 } 3415 3416 const uint16_t vlan_tags[] = { 3417 0, 1, 2, 3, 4, 5, 6, 7, 3418 8, 9, 10, 11, 12, 13, 14, 15, 3419 16, 17, 18, 19, 20, 21, 22, 23, 3420 24, 25, 26, 27, 28, 29, 30, 31 3421 }; 3422 3423 static int 3424 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3425 enum dcb_mode_enable dcb_mode, 3426 enum rte_eth_nb_tcs num_tcs, 3427 uint8_t pfc_en) 3428 { 3429 uint8_t i; 3430 int32_t rc; 3431 struct rte_eth_rss_conf rss_conf; 3432 3433 /* 3434 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3435 * given above, and the number of traffic classes available for use. 3436 */ 3437 if (dcb_mode == DCB_VT_ENABLED) { 3438 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3439 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3440 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3441 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3442 3443 /* VMDQ+DCB RX and TX configurations */ 3444 vmdq_rx_conf->enable_default_pool = 0; 3445 vmdq_rx_conf->default_pool = 0; 3446 vmdq_rx_conf->nb_queue_pools = 3447 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3448 vmdq_tx_conf->nb_queue_pools = 3449 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3450 3451 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3452 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3453 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3454 vmdq_rx_conf->pool_map[i].pools = 3455 1 << (i % vmdq_rx_conf->nb_queue_pools); 3456 } 3457 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3458 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3459 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3460 } 3461 3462 /* set DCB mode of RX and TX of multiple queues */ 3463 eth_conf->rxmode.mq_mode = 3464 (enum rte_eth_rx_mq_mode) 3465 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3466 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3467 } else { 3468 struct rte_eth_dcb_rx_conf *rx_conf = 3469 ð_conf->rx_adv_conf.dcb_rx_conf; 3470 struct rte_eth_dcb_tx_conf *tx_conf = 3471 ð_conf->tx_adv_conf.dcb_tx_conf; 3472 3473 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3474 3475 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3476 if (rc != 0) 3477 return rc; 3478 3479 rx_conf->nb_tcs = num_tcs; 3480 tx_conf->nb_tcs = num_tcs; 3481 3482 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3483 rx_conf->dcb_tc[i] = i % num_tcs; 3484 tx_conf->dcb_tc[i] = i % num_tcs; 3485 } 3486 3487 eth_conf->rxmode.mq_mode = 3488 (enum rte_eth_rx_mq_mode) 3489 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3490 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3491 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3492 } 3493 3494 if (pfc_en) 3495 eth_conf->dcb_capability_en = 3496 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3497 else 3498 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3499 3500 return 0; 3501 } 3502 3503 int 3504 init_port_dcb_config(portid_t pid, 3505 enum dcb_mode_enable dcb_mode, 3506 enum rte_eth_nb_tcs num_tcs, 3507 uint8_t pfc_en) 3508 { 3509 struct rte_eth_conf port_conf; 3510 struct rte_port *rte_port; 3511 int retval; 3512 uint16_t i; 3513 3514 rte_port = &ports[pid]; 3515 3516 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3517 /* Enter DCB configuration status */ 3518 dcb_config = 1; 3519 3520 port_conf.rxmode = rte_port->dev_conf.rxmode; 3521 port_conf.txmode = rte_port->dev_conf.txmode; 3522 3523 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3524 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3525 if (retval < 0) 3526 return retval; 3527 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3528 3529 /* re-configure the device . */ 3530 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3531 if (retval < 0) 3532 return retval; 3533 3534 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3535 if (retval != 0) 3536 return retval; 3537 3538 /* If dev_info.vmdq_pool_base is greater than 0, 3539 * the queue id of vmdq pools is started after pf queues. 3540 */ 3541 if (dcb_mode == DCB_VT_ENABLED && 3542 rte_port->dev_info.vmdq_pool_base > 0) { 3543 printf("VMDQ_DCB multi-queue mode is nonsensical" 3544 " for port %d.", pid); 3545 return -1; 3546 } 3547 3548 /* Assume the ports in testpmd have the same dcb capability 3549 * and has the same number of rxq and txq in dcb mode 3550 */ 3551 if (dcb_mode == DCB_VT_ENABLED) { 3552 if (rte_port->dev_info.max_vfs > 0) { 3553 nb_rxq = rte_port->dev_info.nb_rx_queues; 3554 nb_txq = rte_port->dev_info.nb_tx_queues; 3555 } else { 3556 nb_rxq = rte_port->dev_info.max_rx_queues; 3557 nb_txq = rte_port->dev_info.max_tx_queues; 3558 } 3559 } else { 3560 /*if vt is disabled, use all pf queues */ 3561 if (rte_port->dev_info.vmdq_pool_base == 0) { 3562 nb_rxq = rte_port->dev_info.max_rx_queues; 3563 nb_txq = rte_port->dev_info.max_tx_queues; 3564 } else { 3565 nb_rxq = (queueid_t)num_tcs; 3566 nb_txq = (queueid_t)num_tcs; 3567 3568 } 3569 } 3570 rx_free_thresh = 64; 3571 3572 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3573 3574 rxtx_port_config(rte_port); 3575 /* VLAN filter */ 3576 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3577 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3578 rx_vft_set(pid, vlan_tags[i], 1); 3579 3580 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3581 if (retval != 0) 3582 return retval; 3583 3584 map_port_queue_stats_mapping_registers(pid, rte_port); 3585 3586 rte_port->dcb_flag = 1; 3587 3588 return 0; 3589 } 3590 3591 static void 3592 init_port(void) 3593 { 3594 /* Configuration of Ethernet ports. */ 3595 ports = rte_zmalloc("testpmd: ports", 3596 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3597 RTE_CACHE_LINE_SIZE); 3598 if (ports == NULL) { 3599 rte_exit(EXIT_FAILURE, 3600 "rte_zmalloc(%d struct rte_port) failed\n", 3601 RTE_MAX_ETHPORTS); 3602 } 3603 3604 /* Initialize ports NUMA structures */ 3605 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3606 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3607 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3608 } 3609 3610 static void 3611 force_quit(void) 3612 { 3613 pmd_test_exit(); 3614 prompt_exit(); 3615 } 3616 3617 static void 3618 print_stats(void) 3619 { 3620 uint8_t i; 3621 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3622 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3623 3624 /* Clear screen and move to top left */ 3625 printf("%s%s", clr, top_left); 3626 3627 printf("\nPort statistics ===================================="); 3628 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3629 nic_stats_display(fwd_ports_ids[i]); 3630 3631 fflush(stdout); 3632 } 3633 3634 static void 3635 signal_handler(int signum) 3636 { 3637 if (signum == SIGINT || signum == SIGTERM) { 3638 printf("\nSignal %d received, preparing to exit...\n", 3639 signum); 3640 #ifdef RTE_LIBRTE_PDUMP 3641 /* uninitialize packet capture framework */ 3642 rte_pdump_uninit(); 3643 #endif 3644 #ifdef RTE_LIBRTE_LATENCY_STATS 3645 if (latencystats_enabled != 0) 3646 rte_latencystats_uninit(); 3647 #endif 3648 force_quit(); 3649 /* Set flag to indicate the force termination. */ 3650 f_quit = 1; 3651 /* exit with the expected status */ 3652 signal(signum, SIG_DFL); 3653 kill(getpid(), signum); 3654 } 3655 } 3656 3657 int 3658 main(int argc, char** argv) 3659 { 3660 int diag; 3661 portid_t port_id; 3662 uint16_t count; 3663 int ret; 3664 3665 signal(SIGINT, signal_handler); 3666 signal(SIGTERM, signal_handler); 3667 3668 testpmd_logtype = rte_log_register("testpmd"); 3669 if (testpmd_logtype < 0) 3670 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3671 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3672 3673 diag = rte_eal_init(argc, argv); 3674 if (diag < 0) 3675 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3676 rte_strerror(rte_errno)); 3677 3678 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3679 rte_exit(EXIT_FAILURE, 3680 "Secondary process type not supported.\n"); 3681 3682 ret = register_eth_event_callback(); 3683 if (ret != 0) 3684 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3685 3686 #ifdef RTE_LIBRTE_PDUMP 3687 /* initialize packet capture framework */ 3688 rte_pdump_init(); 3689 #endif 3690 3691 count = 0; 3692 RTE_ETH_FOREACH_DEV(port_id) { 3693 ports_ids[count] = port_id; 3694 count++; 3695 } 3696 nb_ports = (portid_t) count; 3697 if (nb_ports == 0) 3698 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3699 3700 /* allocate port structures, and init them */ 3701 init_port(); 3702 3703 set_def_fwd_config(); 3704 if (nb_lcores == 0) 3705 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3706 "Check the core mask argument\n"); 3707 3708 /* Bitrate/latency stats disabled by default */ 3709 #ifdef RTE_LIBRTE_BITRATE 3710 bitrate_enabled = 0; 3711 #endif 3712 #ifdef RTE_LIBRTE_LATENCY_STATS 3713 latencystats_enabled = 0; 3714 #endif 3715 3716 /* on FreeBSD, mlockall() is disabled by default */ 3717 #ifdef RTE_EXEC_ENV_FREEBSD 3718 do_mlockall = 0; 3719 #else 3720 do_mlockall = 1; 3721 #endif 3722 3723 argc -= diag; 3724 argv += diag; 3725 if (argc > 1) 3726 launch_args_parse(argc, argv); 3727 3728 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3729 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3730 strerror(errno)); 3731 } 3732 3733 if (tx_first && interactive) 3734 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3735 "interactive mode.\n"); 3736 3737 if (tx_first && lsc_interrupt) { 3738 printf("Warning: lsc_interrupt needs to be off when " 3739 " using tx_first. Disabling.\n"); 3740 lsc_interrupt = 0; 3741 } 3742 3743 if (!nb_rxq && !nb_txq) 3744 printf("Warning: Either rx or tx queues should be non-zero\n"); 3745 3746 if (nb_rxq > 1 && nb_rxq > nb_txq) 3747 printf("Warning: nb_rxq=%d enables RSS configuration, " 3748 "but nb_txq=%d will prevent to fully test it.\n", 3749 nb_rxq, nb_txq); 3750 3751 init_config(); 3752 3753 if (hot_plug) { 3754 ret = rte_dev_hotplug_handle_enable(); 3755 if (ret) { 3756 RTE_LOG(ERR, EAL, 3757 "fail to enable hotplug handling."); 3758 return -1; 3759 } 3760 3761 ret = rte_dev_event_monitor_start(); 3762 if (ret) { 3763 RTE_LOG(ERR, EAL, 3764 "fail to start device event monitoring."); 3765 return -1; 3766 } 3767 3768 ret = rte_dev_event_callback_register(NULL, 3769 dev_event_callback, NULL); 3770 if (ret) { 3771 RTE_LOG(ERR, EAL, 3772 "fail to register device event callback\n"); 3773 return -1; 3774 } 3775 } 3776 3777 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3778 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3779 3780 /* set all ports to promiscuous mode by default */ 3781 RTE_ETH_FOREACH_DEV(port_id) { 3782 ret = rte_eth_promiscuous_enable(port_id); 3783 if (ret != 0) 3784 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3785 port_id, rte_strerror(-ret)); 3786 } 3787 3788 /* Init metrics library */ 3789 rte_metrics_init(rte_socket_id()); 3790 3791 #ifdef RTE_LIBRTE_LATENCY_STATS 3792 if (latencystats_enabled != 0) { 3793 int ret = rte_latencystats_init(1, NULL); 3794 if (ret) 3795 printf("Warning: latencystats init()" 3796 " returned error %d\n", ret); 3797 printf("Latencystats running on lcore %d\n", 3798 latencystats_lcore_id); 3799 } 3800 #endif 3801 3802 /* Setup bitrate stats */ 3803 #ifdef RTE_LIBRTE_BITRATE 3804 if (bitrate_enabled != 0) { 3805 bitrate_data = rte_stats_bitrate_create(); 3806 if (bitrate_data == NULL) 3807 rte_exit(EXIT_FAILURE, 3808 "Could not allocate bitrate data.\n"); 3809 rte_stats_bitrate_reg(bitrate_data); 3810 } 3811 #endif 3812 3813 #ifdef RTE_LIBRTE_CMDLINE 3814 if (strlen(cmdline_filename) != 0) 3815 cmdline_read_from_file(cmdline_filename); 3816 3817 if (interactive == 1) { 3818 if (auto_start) { 3819 printf("Start automatic packet forwarding\n"); 3820 start_packet_forwarding(0); 3821 } 3822 prompt(); 3823 pmd_test_exit(); 3824 } else 3825 #endif 3826 { 3827 char c; 3828 int rc; 3829 3830 f_quit = 0; 3831 3832 printf("No commandline core given, start packet forwarding\n"); 3833 start_packet_forwarding(tx_first); 3834 if (stats_period != 0) { 3835 uint64_t prev_time = 0, cur_time, diff_time = 0; 3836 uint64_t timer_period; 3837 3838 /* Convert to number of cycles */ 3839 timer_period = stats_period * rte_get_timer_hz(); 3840 3841 while (f_quit == 0) { 3842 cur_time = rte_get_timer_cycles(); 3843 diff_time += cur_time - prev_time; 3844 3845 if (diff_time >= timer_period) { 3846 print_stats(); 3847 /* Reset the timer */ 3848 diff_time = 0; 3849 } 3850 /* Sleep to avoid unnecessary checks */ 3851 prev_time = cur_time; 3852 sleep(1); 3853 } 3854 } 3855 3856 printf("Press enter to exit\n"); 3857 rc = read(0, &c, 1); 3858 pmd_test_exit(); 3859 if (rc < 0) 3860 return 1; 3861 } 3862 3863 ret = rte_eal_cleanup(); 3864 if (ret != 0) 3865 rte_exit(EXIT_FAILURE, 3866 "EAL cleanup failed: %s\n", strerror(-ret)); 3867 3868 return EXIT_SUCCESS; 3869 } 3870