1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIB_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIB_LATENCYSTATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use main core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 220 221 /* 222 * Configuration of packet segments used by the "txonly" processing engine. 223 */ 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 226 TXONLY_DEF_PACKET_LEN, 227 }; 228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 229 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 231 /**< Split policy for packets to TX. */ 232 233 uint8_t txonly_multi_flow; 234 /**< Whether multiple flows are generated in TXONLY mode. */ 235 236 uint32_t tx_pkt_times_inter; 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 238 239 uint32_t tx_pkt_times_intra; 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 241 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 243 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 244 245 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 246 uint8_t dcb_config = 0; 247 248 /* Whether the dcb is in testing status */ 249 uint8_t dcb_test = 0; 250 251 /* 252 * Configurable number of RX/TX queues. 253 */ 254 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 255 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 256 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 257 258 /* 259 * Configurable number of RX/TX ring descriptors. 260 * Defaults are supplied by drivers via ethdev. 261 */ 262 #define RTE_TEST_RX_DESC_DEFAULT 0 263 #define RTE_TEST_TX_DESC_DEFAULT 0 264 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 265 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 266 267 #define RTE_PMD_PARAM_UNSET -1 268 /* 269 * Configurable values of RX and TX ring threshold registers. 270 */ 271 272 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 273 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 274 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 275 276 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 277 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 278 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 279 280 /* 281 * Configurable value of RX free threshold. 282 */ 283 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 284 285 /* 286 * Configurable value of RX drop enable. 287 */ 288 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 289 290 /* 291 * Configurable value of TX free threshold. 292 */ 293 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 294 295 /* 296 * Configurable value of TX RS bit threshold. 297 */ 298 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 299 300 /* 301 * Configurable value of buffered packets before sending. 302 */ 303 uint16_t noisy_tx_sw_bufsz; 304 305 /* 306 * Configurable value of packet buffer timeout. 307 */ 308 uint16_t noisy_tx_sw_buf_flush_time; 309 310 /* 311 * Configurable value for size of VNF internal memory area 312 * used for simulating noisy neighbour behaviour 313 */ 314 uint64_t noisy_lkup_mem_sz; 315 316 /* 317 * Configurable value of number of random writes done in 318 * VNF simulation memory area. 319 */ 320 uint64_t noisy_lkup_num_writes; 321 322 /* 323 * Configurable value of number of random reads done in 324 * VNF simulation memory area. 325 */ 326 uint64_t noisy_lkup_num_reads; 327 328 /* 329 * Configurable value of number of random reads/writes done in 330 * VNF simulation memory area. 331 */ 332 uint64_t noisy_lkup_num_reads_writes; 333 334 /* 335 * Receive Side Scaling (RSS) configuration. 336 */ 337 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 338 339 /* 340 * Port topology configuration 341 */ 342 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 343 344 /* 345 * Avoids to flush all the RX streams before starts forwarding. 346 */ 347 uint8_t no_flush_rx = 0; /* flush by default */ 348 349 /* 350 * Flow API isolated mode. 351 */ 352 uint8_t flow_isolate_all; 353 354 /* 355 * Avoids to check link status when starting/stopping a port. 356 */ 357 uint8_t no_link_check = 0; /* check by default */ 358 359 /* 360 * Don't automatically start all ports in interactive mode. 361 */ 362 uint8_t no_device_start = 0; 363 364 /* 365 * Enable link status change notification 366 */ 367 uint8_t lsc_interrupt = 1; /* enabled by default */ 368 369 /* 370 * Enable device removal notification. 371 */ 372 uint8_t rmv_interrupt = 1; /* enabled by default */ 373 374 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 375 376 /* After attach, port setup is called on event or by iterator */ 377 bool setup_on_probe_event = true; 378 379 /* Clear ptypes on port initialization. */ 380 uint8_t clear_ptypes = true; 381 382 /* Hairpin ports configuration mode. */ 383 uint16_t hairpin_mode; 384 385 /* Pretty printing of ethdev events */ 386 static const char * const eth_event_desc[] = { 387 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 388 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 389 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 390 [RTE_ETH_EVENT_INTR_RESET] = "reset", 391 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 392 [RTE_ETH_EVENT_IPSEC] = "IPsec", 393 [RTE_ETH_EVENT_MACSEC] = "MACsec", 394 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 395 [RTE_ETH_EVENT_NEW] = "device probed", 396 [RTE_ETH_EVENT_DESTROY] = "device released", 397 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 398 [RTE_ETH_EVENT_MAX] = NULL, 399 }; 400 401 /* 402 * Display or mask ether events 403 * Default to all events except VF_MBOX 404 */ 405 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 406 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 407 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 408 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 409 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 410 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 411 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 412 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 413 /* 414 * Decide if all memory are locked for performance. 415 */ 416 int do_mlockall = 0; 417 418 /* 419 * NIC bypass mode configuration options. 420 */ 421 422 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 423 /* The NIC bypass watchdog timeout. */ 424 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 425 #endif 426 427 428 #ifdef RTE_LIB_LATENCYSTATS 429 430 /* 431 * Set when latency stats is enabled in the commandline 432 */ 433 uint8_t latencystats_enabled; 434 435 /* 436 * Lcore ID to serive latency statistics. 437 */ 438 lcoreid_t latencystats_lcore_id = -1; 439 440 #endif 441 442 /* 443 * Ethernet device configuration. 444 */ 445 struct rte_eth_rxmode rx_mode = { 446 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 447 /**< Default maximum frame length. */ 448 }; 449 450 struct rte_eth_txmode tx_mode = { 451 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 452 }; 453 454 struct rte_fdir_conf fdir_conf = { 455 .mode = RTE_FDIR_MODE_NONE, 456 .pballoc = RTE_FDIR_PBALLOC_64K, 457 .status = RTE_FDIR_REPORT_STATUS, 458 .mask = { 459 .vlan_tci_mask = 0xFFEF, 460 .ipv4_mask = { 461 .src_ip = 0xFFFFFFFF, 462 .dst_ip = 0xFFFFFFFF, 463 }, 464 .ipv6_mask = { 465 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 466 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 467 }, 468 .src_port_mask = 0xFFFF, 469 .dst_port_mask = 0xFFFF, 470 .mac_addr_byte_mask = 0xFF, 471 .tunnel_type_mask = 1, 472 .tunnel_id_mask = 0xFFFFFFFF, 473 }, 474 .drop_queue = 127, 475 }; 476 477 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 478 479 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 480 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 481 482 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 483 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 484 485 uint16_t nb_tx_queue_stats_mappings = 0; 486 uint16_t nb_rx_queue_stats_mappings = 0; 487 488 /* 489 * Display zero values by default for xstats 490 */ 491 uint8_t xstats_hide_zero; 492 493 /* 494 * Measure of CPU cycles disabled by default 495 */ 496 uint8_t record_core_cycles; 497 498 /* 499 * Display of RX and TX bursts disabled by default 500 */ 501 uint8_t record_burst_stats; 502 503 unsigned int num_sockets = 0; 504 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 505 506 #ifdef RTE_LIB_BITRATESTATS 507 /* Bitrate statistics */ 508 struct rte_stats_bitrates *bitrate_data; 509 lcoreid_t bitrate_lcore_id; 510 uint8_t bitrate_enabled; 511 #endif 512 513 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 514 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 515 516 /* 517 * hexadecimal bitmask of RX mq mode can be enabled. 518 */ 519 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 520 521 /* Forward function declarations */ 522 static void setup_attached_port(portid_t pi); 523 static void map_port_queue_stats_mapping_registers(portid_t pi, 524 struct rte_port *port); 525 static void check_all_ports_link_status(uint32_t port_mask); 526 static int eth_event_callback(portid_t port_id, 527 enum rte_eth_event_type type, 528 void *param, void *ret_param); 529 static void dev_event_callback(const char *device_name, 530 enum rte_dev_event_type type, 531 void *param); 532 533 /* 534 * Check if all the ports are started. 535 * If yes, return positive value. If not, return zero. 536 */ 537 static int all_ports_started(void); 538 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 541 542 /* Holds the registered mbuf dynamic flags names. */ 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 544 545 /* 546 * Helper function to check if socket is already discovered. 547 * If yes, return positive value. If not, return zero. 548 */ 549 int 550 new_socket_id(unsigned int socket_id) 551 { 552 unsigned int i; 553 554 for (i = 0; i < num_sockets; i++) { 555 if (socket_ids[i] == socket_id) 556 return 0; 557 } 558 return 1; 559 } 560 561 /* 562 * Setup default configuration. 563 */ 564 static void 565 set_default_fwd_lcores_config(void) 566 { 567 unsigned int i; 568 unsigned int nb_lc; 569 unsigned int sock_num; 570 571 nb_lc = 0; 572 for (i = 0; i < RTE_MAX_LCORE; i++) { 573 if (!rte_lcore_is_enabled(i)) 574 continue; 575 sock_num = rte_lcore_to_socket_id(i); 576 if (new_socket_id(sock_num)) { 577 if (num_sockets >= RTE_MAX_NUMA_NODES) { 578 rte_exit(EXIT_FAILURE, 579 "Total sockets greater than %u\n", 580 RTE_MAX_NUMA_NODES); 581 } 582 socket_ids[num_sockets++] = sock_num; 583 } 584 if (i == rte_get_main_lcore()) 585 continue; 586 fwd_lcores_cpuids[nb_lc++] = i; 587 } 588 nb_lcores = (lcoreid_t) nb_lc; 589 nb_cfg_lcores = nb_lcores; 590 nb_fwd_lcores = 1; 591 } 592 593 static void 594 set_def_peer_eth_addrs(void) 595 { 596 portid_t i; 597 598 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 599 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 600 peer_eth_addrs[i].addr_bytes[5] = i; 601 } 602 } 603 604 static void 605 set_default_fwd_ports_config(void) 606 { 607 portid_t pt_id; 608 int i = 0; 609 610 RTE_ETH_FOREACH_DEV(pt_id) { 611 fwd_ports_ids[i++] = pt_id; 612 613 /* Update sockets info according to the attached device */ 614 int socket_id = rte_eth_dev_socket_id(pt_id); 615 if (socket_id >= 0 && new_socket_id(socket_id)) { 616 if (num_sockets >= RTE_MAX_NUMA_NODES) { 617 rte_exit(EXIT_FAILURE, 618 "Total sockets greater than %u\n", 619 RTE_MAX_NUMA_NODES); 620 } 621 socket_ids[num_sockets++] = socket_id; 622 } 623 } 624 625 nb_cfg_ports = nb_ports; 626 nb_fwd_ports = nb_ports; 627 } 628 629 void 630 set_def_fwd_config(void) 631 { 632 set_default_fwd_lcores_config(); 633 set_def_peer_eth_addrs(); 634 set_default_fwd_ports_config(); 635 } 636 637 /* extremely pessimistic estimation of memory required to create a mempool */ 638 static int 639 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 640 { 641 unsigned int n_pages, mbuf_per_pg, leftover; 642 uint64_t total_mem, mbuf_mem, obj_sz; 643 644 /* there is no good way to predict how much space the mempool will 645 * occupy because it will allocate chunks on the fly, and some of those 646 * will come from default DPDK memory while some will come from our 647 * external memory, so just assume 128MB will be enough for everyone. 648 */ 649 uint64_t hdr_mem = 128 << 20; 650 651 /* account for possible non-contiguousness */ 652 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 653 if (obj_sz > pgsz) { 654 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 655 return -1; 656 } 657 658 mbuf_per_pg = pgsz / obj_sz; 659 leftover = (nb_mbufs % mbuf_per_pg) > 0; 660 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 661 662 mbuf_mem = n_pages * pgsz; 663 664 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 665 666 if (total_mem > SIZE_MAX) { 667 TESTPMD_LOG(ERR, "Memory size too big\n"); 668 return -1; 669 } 670 *out = (size_t)total_mem; 671 672 return 0; 673 } 674 675 static int 676 pagesz_flags(uint64_t page_sz) 677 { 678 /* as per mmap() manpage, all page sizes are log2 of page size 679 * shifted by MAP_HUGE_SHIFT 680 */ 681 int log2 = rte_log2_u64(page_sz); 682 683 return (log2 << HUGE_SHIFT); 684 } 685 686 static void * 687 alloc_mem(size_t memsz, size_t pgsz, bool huge) 688 { 689 void *addr; 690 int flags; 691 692 /* allocate anonymous hugepages */ 693 flags = MAP_ANONYMOUS | MAP_PRIVATE; 694 if (huge) 695 flags |= HUGE_FLAG | pagesz_flags(pgsz); 696 697 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 698 if (addr == MAP_FAILED) 699 return NULL; 700 701 return addr; 702 } 703 704 struct extmem_param { 705 void *addr; 706 size_t len; 707 size_t pgsz; 708 rte_iova_t *iova_table; 709 unsigned int iova_table_len; 710 }; 711 712 static int 713 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 714 bool huge) 715 { 716 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 717 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 718 unsigned int cur_page, n_pages, pgsz_idx; 719 size_t mem_sz, cur_pgsz; 720 rte_iova_t *iovas = NULL; 721 void *addr; 722 int ret; 723 724 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 725 /* skip anything that is too big */ 726 if (pgsizes[pgsz_idx] > SIZE_MAX) 727 continue; 728 729 cur_pgsz = pgsizes[pgsz_idx]; 730 731 /* if we were told not to allocate hugepages, override */ 732 if (!huge) 733 cur_pgsz = sysconf(_SC_PAGESIZE); 734 735 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 736 if (ret < 0) { 737 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 738 return -1; 739 } 740 741 /* allocate our memory */ 742 addr = alloc_mem(mem_sz, cur_pgsz, huge); 743 744 /* if we couldn't allocate memory with a specified page size, 745 * that doesn't mean we can't do it with other page sizes, so 746 * try another one. 747 */ 748 if (addr == NULL) 749 continue; 750 751 /* store IOVA addresses for every page in this memory area */ 752 n_pages = mem_sz / cur_pgsz; 753 754 iovas = malloc(sizeof(*iovas) * n_pages); 755 756 if (iovas == NULL) { 757 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 758 goto fail; 759 } 760 /* lock memory if it's not huge pages */ 761 if (!huge) 762 mlock(addr, mem_sz); 763 764 /* populate IOVA addresses */ 765 for (cur_page = 0; cur_page < n_pages; cur_page++) { 766 rte_iova_t iova; 767 size_t offset; 768 void *cur; 769 770 offset = cur_pgsz * cur_page; 771 cur = RTE_PTR_ADD(addr, offset); 772 773 /* touch the page before getting its IOVA */ 774 *(volatile char *)cur = 0; 775 776 iova = rte_mem_virt2iova(cur); 777 778 iovas[cur_page] = iova; 779 } 780 781 break; 782 } 783 /* if we couldn't allocate anything */ 784 if (iovas == NULL) 785 return -1; 786 787 param->addr = addr; 788 param->len = mem_sz; 789 param->pgsz = cur_pgsz; 790 param->iova_table = iovas; 791 param->iova_table_len = n_pages; 792 793 return 0; 794 fail: 795 if (iovas) 796 free(iovas); 797 if (addr) 798 munmap(addr, mem_sz); 799 800 return -1; 801 } 802 803 static int 804 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 805 { 806 struct extmem_param param; 807 int socket_id, ret; 808 809 memset(¶m, 0, sizeof(param)); 810 811 /* check if our heap exists */ 812 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 813 if (socket_id < 0) { 814 /* create our heap */ 815 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 816 if (ret < 0) { 817 TESTPMD_LOG(ERR, "Cannot create heap\n"); 818 return -1; 819 } 820 } 821 822 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 823 if (ret < 0) { 824 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 825 return -1; 826 } 827 828 /* we now have a valid memory area, so add it to heap */ 829 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 830 param.addr, param.len, param.iova_table, 831 param.iova_table_len, param.pgsz); 832 833 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 834 835 /* not needed any more */ 836 free(param.iova_table); 837 838 if (ret < 0) { 839 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 840 munmap(param.addr, param.len); 841 return -1; 842 } 843 844 /* success */ 845 846 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 847 param.len >> 20); 848 849 return 0; 850 } 851 static void 852 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 853 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 854 { 855 uint16_t pid = 0; 856 int ret; 857 858 RTE_ETH_FOREACH_DEV(pid) { 859 struct rte_eth_dev *dev = 860 &rte_eth_devices[pid]; 861 862 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 863 memhdr->len); 864 if (ret) { 865 TESTPMD_LOG(DEBUG, 866 "unable to DMA unmap addr 0x%p " 867 "for device %s\n", 868 memhdr->addr, dev->data->name); 869 } 870 } 871 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 872 if (ret) { 873 TESTPMD_LOG(DEBUG, 874 "unable to un-register addr 0x%p\n", memhdr->addr); 875 } 876 } 877 878 static void 879 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 880 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 881 { 882 uint16_t pid = 0; 883 size_t page_size = sysconf(_SC_PAGESIZE); 884 int ret; 885 886 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 887 page_size); 888 if (ret) { 889 TESTPMD_LOG(DEBUG, 890 "unable to register addr 0x%p\n", memhdr->addr); 891 return; 892 } 893 RTE_ETH_FOREACH_DEV(pid) { 894 struct rte_eth_dev *dev = 895 &rte_eth_devices[pid]; 896 897 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 898 memhdr->len); 899 if (ret) { 900 TESTPMD_LOG(DEBUG, 901 "unable to DMA map addr 0x%p " 902 "for device %s\n", 903 memhdr->addr, dev->data->name); 904 } 905 } 906 } 907 908 static unsigned int 909 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 910 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 911 { 912 struct rte_pktmbuf_extmem *xmem; 913 unsigned int ext_num, zone_num, elt_num; 914 uint16_t elt_size; 915 916 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 917 elt_num = EXTBUF_ZONE_SIZE / elt_size; 918 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 919 920 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 921 if (xmem == NULL) { 922 TESTPMD_LOG(ERR, "Cannot allocate memory for " 923 "external buffer descriptors\n"); 924 *ext_mem = NULL; 925 return 0; 926 } 927 for (ext_num = 0; ext_num < zone_num; ext_num++) { 928 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 929 const struct rte_memzone *mz; 930 char mz_name[RTE_MEMZONE_NAMESIZE]; 931 int ret; 932 933 ret = snprintf(mz_name, sizeof(mz_name), 934 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 935 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 936 errno = ENAMETOOLONG; 937 ext_num = 0; 938 break; 939 } 940 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 941 socket_id, 942 RTE_MEMZONE_IOVA_CONTIG | 943 RTE_MEMZONE_1GB | 944 RTE_MEMZONE_SIZE_HINT_ONLY, 945 EXTBUF_ZONE_SIZE); 946 if (mz == NULL) { 947 /* 948 * The caller exits on external buffer creation 949 * error, so there is no need to free memzones. 950 */ 951 errno = ENOMEM; 952 ext_num = 0; 953 break; 954 } 955 xseg->buf_ptr = mz->addr; 956 xseg->buf_iova = mz->iova; 957 xseg->buf_len = EXTBUF_ZONE_SIZE; 958 xseg->elt_size = elt_size; 959 } 960 if (ext_num == 0 && xmem != NULL) { 961 free(xmem); 962 xmem = NULL; 963 } 964 *ext_mem = xmem; 965 return ext_num; 966 } 967 968 /* 969 * Configuration initialisation done once at init time. 970 */ 971 static struct rte_mempool * 972 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 973 unsigned int socket_id, uint16_t size_idx) 974 { 975 char pool_name[RTE_MEMPOOL_NAMESIZE]; 976 struct rte_mempool *rte_mp = NULL; 977 uint32_t mb_size; 978 979 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 980 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 981 982 TESTPMD_LOG(INFO, 983 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 984 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 985 986 switch (mp_alloc_type) { 987 case MP_ALLOC_NATIVE: 988 { 989 /* wrapper to rte_mempool_create() */ 990 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 991 rte_mbuf_best_mempool_ops()); 992 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 993 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 994 break; 995 } 996 case MP_ALLOC_ANON: 997 { 998 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 999 mb_size, (unsigned int) mb_mempool_cache, 1000 sizeof(struct rte_pktmbuf_pool_private), 1001 socket_id, mempool_flags); 1002 if (rte_mp == NULL) 1003 goto err; 1004 1005 if (rte_mempool_populate_anon(rte_mp) == 0) { 1006 rte_mempool_free(rte_mp); 1007 rte_mp = NULL; 1008 goto err; 1009 } 1010 rte_pktmbuf_pool_init(rte_mp, NULL); 1011 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1012 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1013 break; 1014 } 1015 case MP_ALLOC_XMEM: 1016 case MP_ALLOC_XMEM_HUGE: 1017 { 1018 int heap_socket; 1019 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1020 1021 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1022 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1023 1024 heap_socket = 1025 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1026 if (heap_socket < 0) 1027 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1028 1029 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1030 rte_mbuf_best_mempool_ops()); 1031 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1032 mb_mempool_cache, 0, mbuf_seg_size, 1033 heap_socket); 1034 break; 1035 } 1036 case MP_ALLOC_XBUF: 1037 { 1038 struct rte_pktmbuf_extmem *ext_mem; 1039 unsigned int ext_num; 1040 1041 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1042 socket_id, pool_name, &ext_mem); 1043 if (ext_num == 0) 1044 rte_exit(EXIT_FAILURE, 1045 "Can't create pinned data buffers\n"); 1046 1047 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1048 rte_mbuf_best_mempool_ops()); 1049 rte_mp = rte_pktmbuf_pool_create_extbuf 1050 (pool_name, nb_mbuf, mb_mempool_cache, 1051 0, mbuf_seg_size, socket_id, 1052 ext_mem, ext_num); 1053 free(ext_mem); 1054 break; 1055 } 1056 default: 1057 { 1058 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1059 } 1060 } 1061 1062 err: 1063 if (rte_mp == NULL) { 1064 rte_exit(EXIT_FAILURE, 1065 "Creation of mbuf pool for socket %u failed: %s\n", 1066 socket_id, rte_strerror(rte_errno)); 1067 } else if (verbose_level > 0) { 1068 rte_mempool_dump(stdout, rte_mp); 1069 } 1070 return rte_mp; 1071 } 1072 1073 /* 1074 * Check given socket id is valid or not with NUMA mode, 1075 * if valid, return 0, else return -1 1076 */ 1077 static int 1078 check_socket_id(const unsigned int socket_id) 1079 { 1080 static int warning_once = 0; 1081 1082 if (new_socket_id(socket_id)) { 1083 if (!warning_once && numa_support) 1084 printf("Warning: NUMA should be configured manually by" 1085 " using --port-numa-config and" 1086 " --ring-numa-config parameters along with" 1087 " --numa.\n"); 1088 warning_once = 1; 1089 return -1; 1090 } 1091 return 0; 1092 } 1093 1094 /* 1095 * Get the allowed maximum number of RX queues. 1096 * *pid return the port id which has minimal value of 1097 * max_rx_queues in all ports. 1098 */ 1099 queueid_t 1100 get_allowed_max_nb_rxq(portid_t *pid) 1101 { 1102 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1103 bool max_rxq_valid = false; 1104 portid_t pi; 1105 struct rte_eth_dev_info dev_info; 1106 1107 RTE_ETH_FOREACH_DEV(pi) { 1108 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1109 continue; 1110 1111 max_rxq_valid = true; 1112 if (dev_info.max_rx_queues < allowed_max_rxq) { 1113 allowed_max_rxq = dev_info.max_rx_queues; 1114 *pid = pi; 1115 } 1116 } 1117 return max_rxq_valid ? allowed_max_rxq : 0; 1118 } 1119 1120 /* 1121 * Check input rxq is valid or not. 1122 * If input rxq is not greater than any of maximum number 1123 * of RX queues of all ports, it is valid. 1124 * if valid, return 0, else return -1 1125 */ 1126 int 1127 check_nb_rxq(queueid_t rxq) 1128 { 1129 queueid_t allowed_max_rxq; 1130 portid_t pid = 0; 1131 1132 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1133 if (rxq > allowed_max_rxq) { 1134 printf("Fail: input rxq (%u) can't be greater " 1135 "than max_rx_queues (%u) of port %u\n", 1136 rxq, 1137 allowed_max_rxq, 1138 pid); 1139 return -1; 1140 } 1141 return 0; 1142 } 1143 1144 /* 1145 * Get the allowed maximum number of TX queues. 1146 * *pid return the port id which has minimal value of 1147 * max_tx_queues in all ports. 1148 */ 1149 queueid_t 1150 get_allowed_max_nb_txq(portid_t *pid) 1151 { 1152 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1153 bool max_txq_valid = false; 1154 portid_t pi; 1155 struct rte_eth_dev_info dev_info; 1156 1157 RTE_ETH_FOREACH_DEV(pi) { 1158 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1159 continue; 1160 1161 max_txq_valid = true; 1162 if (dev_info.max_tx_queues < allowed_max_txq) { 1163 allowed_max_txq = dev_info.max_tx_queues; 1164 *pid = pi; 1165 } 1166 } 1167 return max_txq_valid ? allowed_max_txq : 0; 1168 } 1169 1170 /* 1171 * Check input txq is valid or not. 1172 * If input txq is not greater than any of maximum number 1173 * of TX queues of all ports, it is valid. 1174 * if valid, return 0, else return -1 1175 */ 1176 int 1177 check_nb_txq(queueid_t txq) 1178 { 1179 queueid_t allowed_max_txq; 1180 portid_t pid = 0; 1181 1182 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1183 if (txq > allowed_max_txq) { 1184 printf("Fail: input txq (%u) can't be greater " 1185 "than max_tx_queues (%u) of port %u\n", 1186 txq, 1187 allowed_max_txq, 1188 pid); 1189 return -1; 1190 } 1191 return 0; 1192 } 1193 1194 /* 1195 * Get the allowed maximum number of RXDs of every rx queue. 1196 * *pid return the port id which has minimal value of 1197 * max_rxd in all queues of all ports. 1198 */ 1199 static uint16_t 1200 get_allowed_max_nb_rxd(portid_t *pid) 1201 { 1202 uint16_t allowed_max_rxd = UINT16_MAX; 1203 portid_t pi; 1204 struct rte_eth_dev_info dev_info; 1205 1206 RTE_ETH_FOREACH_DEV(pi) { 1207 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1208 continue; 1209 1210 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1211 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1212 *pid = pi; 1213 } 1214 } 1215 return allowed_max_rxd; 1216 } 1217 1218 /* 1219 * Get the allowed minimal number of RXDs of every rx queue. 1220 * *pid return the port id which has minimal value of 1221 * min_rxd in all queues of all ports. 1222 */ 1223 static uint16_t 1224 get_allowed_min_nb_rxd(portid_t *pid) 1225 { 1226 uint16_t allowed_min_rxd = 0; 1227 portid_t pi; 1228 struct rte_eth_dev_info dev_info; 1229 1230 RTE_ETH_FOREACH_DEV(pi) { 1231 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1232 continue; 1233 1234 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1235 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1236 *pid = pi; 1237 } 1238 } 1239 1240 return allowed_min_rxd; 1241 } 1242 1243 /* 1244 * Check input rxd is valid or not. 1245 * If input rxd is not greater than any of maximum number 1246 * of RXDs of every Rx queues and is not less than any of 1247 * minimal number of RXDs of every Rx queues, it is valid. 1248 * if valid, return 0, else return -1 1249 */ 1250 int 1251 check_nb_rxd(queueid_t rxd) 1252 { 1253 uint16_t allowed_max_rxd; 1254 uint16_t allowed_min_rxd; 1255 portid_t pid = 0; 1256 1257 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1258 if (rxd > allowed_max_rxd) { 1259 printf("Fail: input rxd (%u) can't be greater " 1260 "than max_rxds (%u) of port %u\n", 1261 rxd, 1262 allowed_max_rxd, 1263 pid); 1264 return -1; 1265 } 1266 1267 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1268 if (rxd < allowed_min_rxd) { 1269 printf("Fail: input rxd (%u) can't be less " 1270 "than min_rxds (%u) of port %u\n", 1271 rxd, 1272 allowed_min_rxd, 1273 pid); 1274 return -1; 1275 } 1276 1277 return 0; 1278 } 1279 1280 /* 1281 * Get the allowed maximum number of TXDs of every rx queues. 1282 * *pid return the port id which has minimal value of 1283 * max_txd in every tx queue. 1284 */ 1285 static uint16_t 1286 get_allowed_max_nb_txd(portid_t *pid) 1287 { 1288 uint16_t allowed_max_txd = UINT16_MAX; 1289 portid_t pi; 1290 struct rte_eth_dev_info dev_info; 1291 1292 RTE_ETH_FOREACH_DEV(pi) { 1293 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1294 continue; 1295 1296 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1297 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1298 *pid = pi; 1299 } 1300 } 1301 return allowed_max_txd; 1302 } 1303 1304 /* 1305 * Get the allowed maximum number of TXDs of every tx queues. 1306 * *pid return the port id which has minimal value of 1307 * min_txd in every tx queue. 1308 */ 1309 static uint16_t 1310 get_allowed_min_nb_txd(portid_t *pid) 1311 { 1312 uint16_t allowed_min_txd = 0; 1313 portid_t pi; 1314 struct rte_eth_dev_info dev_info; 1315 1316 RTE_ETH_FOREACH_DEV(pi) { 1317 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1318 continue; 1319 1320 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1321 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1322 *pid = pi; 1323 } 1324 } 1325 1326 return allowed_min_txd; 1327 } 1328 1329 /* 1330 * Check input txd is valid or not. 1331 * If input txd is not greater than any of maximum number 1332 * of TXDs of every Rx queues, it is valid. 1333 * if valid, return 0, else return -1 1334 */ 1335 int 1336 check_nb_txd(queueid_t txd) 1337 { 1338 uint16_t allowed_max_txd; 1339 uint16_t allowed_min_txd; 1340 portid_t pid = 0; 1341 1342 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1343 if (txd > allowed_max_txd) { 1344 printf("Fail: input txd (%u) can't be greater " 1345 "than max_txds (%u) of port %u\n", 1346 txd, 1347 allowed_max_txd, 1348 pid); 1349 return -1; 1350 } 1351 1352 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1353 if (txd < allowed_min_txd) { 1354 printf("Fail: input txd (%u) can't be less " 1355 "than min_txds (%u) of port %u\n", 1356 txd, 1357 allowed_min_txd, 1358 pid); 1359 return -1; 1360 } 1361 return 0; 1362 } 1363 1364 1365 /* 1366 * Get the allowed maximum number of hairpin queues. 1367 * *pid return the port id which has minimal value of 1368 * max_hairpin_queues in all ports. 1369 */ 1370 queueid_t 1371 get_allowed_max_nb_hairpinq(portid_t *pid) 1372 { 1373 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1374 portid_t pi; 1375 struct rte_eth_hairpin_cap cap; 1376 1377 RTE_ETH_FOREACH_DEV(pi) { 1378 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1379 *pid = pi; 1380 return 0; 1381 } 1382 if (cap.max_nb_queues < allowed_max_hairpinq) { 1383 allowed_max_hairpinq = cap.max_nb_queues; 1384 *pid = pi; 1385 } 1386 } 1387 return allowed_max_hairpinq; 1388 } 1389 1390 /* 1391 * Check input hairpin is valid or not. 1392 * If input hairpin is not greater than any of maximum number 1393 * of hairpin queues of all ports, it is valid. 1394 * if valid, return 0, else return -1 1395 */ 1396 int 1397 check_nb_hairpinq(queueid_t hairpinq) 1398 { 1399 queueid_t allowed_max_hairpinq; 1400 portid_t pid = 0; 1401 1402 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1403 if (hairpinq > allowed_max_hairpinq) { 1404 printf("Fail: input hairpin (%u) can't be greater " 1405 "than max_hairpin_queues (%u) of port %u\n", 1406 hairpinq, allowed_max_hairpinq, pid); 1407 return -1; 1408 } 1409 return 0; 1410 } 1411 1412 static void 1413 init_config(void) 1414 { 1415 portid_t pid; 1416 struct rte_port *port; 1417 struct rte_mempool *mbp; 1418 unsigned int nb_mbuf_per_pool; 1419 lcoreid_t lc_id; 1420 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1421 struct rte_gro_param gro_param; 1422 uint32_t gso_types; 1423 uint16_t data_size; 1424 uint16_t overhead_len; 1425 bool warning = 0; 1426 int k; 1427 int ret; 1428 1429 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1430 1431 /* Configuration of logical cores. */ 1432 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1433 sizeof(struct fwd_lcore *) * nb_lcores, 1434 RTE_CACHE_LINE_SIZE); 1435 if (fwd_lcores == NULL) { 1436 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1437 "failed\n", nb_lcores); 1438 } 1439 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1440 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1441 sizeof(struct fwd_lcore), 1442 RTE_CACHE_LINE_SIZE); 1443 if (fwd_lcores[lc_id] == NULL) { 1444 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1445 "failed\n"); 1446 } 1447 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1448 } 1449 1450 RTE_ETH_FOREACH_DEV(pid) { 1451 port = &ports[pid]; 1452 /* Apply default TxRx configuration for all ports */ 1453 port->dev_conf.txmode = tx_mode; 1454 port->dev_conf.rxmode = rx_mode; 1455 1456 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1457 if (ret != 0) 1458 rte_exit(EXIT_FAILURE, 1459 "rte_eth_dev_info_get() failed\n"); 1460 1461 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */ 1462 if (port->dev_info.max_rx_pktlen && port->dev_info.max_mtu) 1463 overhead_len = port->dev_info.max_rx_pktlen - 1464 port->dev_info.max_mtu; 1465 else 1466 overhead_len = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1467 1468 port->dev_conf.rxmode.max_rx_pkt_len = 1469 RTE_ETHER_MTU + overhead_len; 1470 1471 /* 1472 * This is workaround to avoid resize max rx packet len. 1473 * Ethdev assumes jumbo frame size must be greater than 1474 * RTE_ETHER_MAX_LEN, and will resize 'max_rx_pkt_len' to 1475 * default value when it is greater than RTE_ETHER_MAX_LEN 1476 * for normal frame. 1477 */ 1478 if (port->dev_conf.rxmode.max_rx_pkt_len > RTE_ETHER_MAX_LEN) { 1479 port->dev_conf.rxmode.offloads |= 1480 DEV_RX_OFFLOAD_JUMBO_FRAME; 1481 } 1482 1483 if (!(port->dev_info.tx_offload_capa & 1484 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1485 port->dev_conf.txmode.offloads &= 1486 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1487 if (numa_support) { 1488 if (port_numa[pid] != NUMA_NO_CONFIG) 1489 port_per_socket[port_numa[pid]]++; 1490 else { 1491 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1492 1493 /* 1494 * if socket_id is invalid, 1495 * set to the first available socket. 1496 */ 1497 if (check_socket_id(socket_id) < 0) 1498 socket_id = socket_ids[0]; 1499 port_per_socket[socket_id]++; 1500 } 1501 } 1502 1503 /* Apply Rx offloads configuration */ 1504 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1505 port->rx_conf[k].offloads = 1506 port->dev_conf.rxmode.offloads; 1507 /* Apply Tx offloads configuration */ 1508 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1509 port->tx_conf[k].offloads = 1510 port->dev_conf.txmode.offloads; 1511 1512 /* set flag to initialize port/queue */ 1513 port->need_reconfig = 1; 1514 port->need_reconfig_queues = 1; 1515 port->tx_metadata = 0; 1516 1517 /* Check for maximum number of segments per MTU. Accordingly 1518 * update the mbuf data size. 1519 */ 1520 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1521 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1522 data_size = rx_mode.max_rx_pkt_len / 1523 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1524 1525 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1526 mbuf_data_size[0]) { 1527 mbuf_data_size[0] = data_size + 1528 RTE_PKTMBUF_HEADROOM; 1529 warning = 1; 1530 } 1531 } 1532 } 1533 1534 if (warning) 1535 TESTPMD_LOG(WARNING, 1536 "Configured mbuf size of the first segment %hu\n", 1537 mbuf_data_size[0]); 1538 /* 1539 * Create pools of mbuf. 1540 * If NUMA support is disabled, create a single pool of mbuf in 1541 * socket 0 memory by default. 1542 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1543 * 1544 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1545 * nb_txd can be configured at run time. 1546 */ 1547 if (param_total_num_mbufs) 1548 nb_mbuf_per_pool = param_total_num_mbufs; 1549 else { 1550 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1551 (nb_lcores * mb_mempool_cache) + 1552 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1553 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1554 } 1555 1556 if (numa_support) { 1557 uint8_t i, j; 1558 1559 for (i = 0; i < num_sockets; i++) 1560 for (j = 0; j < mbuf_data_size_n; j++) 1561 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1562 mbuf_pool_create(mbuf_data_size[j], 1563 nb_mbuf_per_pool, 1564 socket_ids[i], j); 1565 } else { 1566 uint8_t i; 1567 1568 for (i = 0; i < mbuf_data_size_n; i++) 1569 mempools[i] = mbuf_pool_create 1570 (mbuf_data_size[i], 1571 nb_mbuf_per_pool, 1572 socket_num == UMA_NO_CONFIG ? 1573 0 : socket_num, i); 1574 } 1575 1576 init_port_config(); 1577 1578 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1579 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1580 /* 1581 * Records which Mbuf pool to use by each logical core, if needed. 1582 */ 1583 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1584 mbp = mbuf_pool_find( 1585 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1586 1587 if (mbp == NULL) 1588 mbp = mbuf_pool_find(0, 0); 1589 fwd_lcores[lc_id]->mbp = mbp; 1590 /* initialize GSO context */ 1591 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1592 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1593 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1594 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1595 RTE_ETHER_CRC_LEN; 1596 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1597 } 1598 1599 /* Configuration of packet forwarding streams. */ 1600 if (init_fwd_streams() < 0) 1601 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1602 1603 fwd_config_setup(); 1604 1605 /* create a gro context for each lcore */ 1606 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1607 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1608 gro_param.max_item_per_flow = MAX_PKT_BURST; 1609 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1610 gro_param.socket_id = rte_lcore_to_socket_id( 1611 fwd_lcores_cpuids[lc_id]); 1612 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1613 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1614 rte_exit(EXIT_FAILURE, 1615 "rte_gro_ctx_create() failed\n"); 1616 } 1617 } 1618 } 1619 1620 1621 void 1622 reconfig(portid_t new_port_id, unsigned socket_id) 1623 { 1624 struct rte_port *port; 1625 int ret; 1626 1627 /* Reconfiguration of Ethernet ports. */ 1628 port = &ports[new_port_id]; 1629 1630 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1631 if (ret != 0) 1632 return; 1633 1634 /* set flag to initialize port/queue */ 1635 port->need_reconfig = 1; 1636 port->need_reconfig_queues = 1; 1637 port->socket_id = socket_id; 1638 1639 init_port_config(); 1640 } 1641 1642 1643 int 1644 init_fwd_streams(void) 1645 { 1646 portid_t pid; 1647 struct rte_port *port; 1648 streamid_t sm_id, nb_fwd_streams_new; 1649 queueid_t q; 1650 1651 /* set socket id according to numa or not */ 1652 RTE_ETH_FOREACH_DEV(pid) { 1653 port = &ports[pid]; 1654 if (nb_rxq > port->dev_info.max_rx_queues) { 1655 printf("Fail: nb_rxq(%d) is greater than " 1656 "max_rx_queues(%d)\n", nb_rxq, 1657 port->dev_info.max_rx_queues); 1658 return -1; 1659 } 1660 if (nb_txq > port->dev_info.max_tx_queues) { 1661 printf("Fail: nb_txq(%d) is greater than " 1662 "max_tx_queues(%d)\n", nb_txq, 1663 port->dev_info.max_tx_queues); 1664 return -1; 1665 } 1666 if (numa_support) { 1667 if (port_numa[pid] != NUMA_NO_CONFIG) 1668 port->socket_id = port_numa[pid]; 1669 else { 1670 port->socket_id = rte_eth_dev_socket_id(pid); 1671 1672 /* 1673 * if socket_id is invalid, 1674 * set to the first available socket. 1675 */ 1676 if (check_socket_id(port->socket_id) < 0) 1677 port->socket_id = socket_ids[0]; 1678 } 1679 } 1680 else { 1681 if (socket_num == UMA_NO_CONFIG) 1682 port->socket_id = 0; 1683 else 1684 port->socket_id = socket_num; 1685 } 1686 } 1687 1688 q = RTE_MAX(nb_rxq, nb_txq); 1689 if (q == 0) { 1690 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1691 return -1; 1692 } 1693 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1694 if (nb_fwd_streams_new == nb_fwd_streams) 1695 return 0; 1696 /* clear the old */ 1697 if (fwd_streams != NULL) { 1698 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1699 if (fwd_streams[sm_id] == NULL) 1700 continue; 1701 rte_free(fwd_streams[sm_id]); 1702 fwd_streams[sm_id] = NULL; 1703 } 1704 rte_free(fwd_streams); 1705 fwd_streams = NULL; 1706 } 1707 1708 /* init new */ 1709 nb_fwd_streams = nb_fwd_streams_new; 1710 if (nb_fwd_streams) { 1711 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1712 sizeof(struct fwd_stream *) * nb_fwd_streams, 1713 RTE_CACHE_LINE_SIZE); 1714 if (fwd_streams == NULL) 1715 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1716 " (struct fwd_stream *)) failed\n", 1717 nb_fwd_streams); 1718 1719 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1720 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1721 " struct fwd_stream", sizeof(struct fwd_stream), 1722 RTE_CACHE_LINE_SIZE); 1723 if (fwd_streams[sm_id] == NULL) 1724 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1725 "(struct fwd_stream) failed\n"); 1726 } 1727 } 1728 1729 return 0; 1730 } 1731 1732 static void 1733 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1734 { 1735 uint64_t total_burst, sburst; 1736 uint64_t nb_burst; 1737 uint64_t burst_stats[4]; 1738 uint16_t pktnb_stats[4]; 1739 uint16_t nb_pkt; 1740 int burst_percent[4], sburstp; 1741 int i; 1742 1743 /* 1744 * First compute the total number of packet bursts and the 1745 * two highest numbers of bursts of the same number of packets. 1746 */ 1747 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1748 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1749 1750 /* Show stats for 0 burst size always */ 1751 total_burst = pbs->pkt_burst_spread[0]; 1752 burst_stats[0] = pbs->pkt_burst_spread[0]; 1753 pktnb_stats[0] = 0; 1754 1755 /* Find the next 2 burst sizes with highest occurrences. */ 1756 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1757 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1758 1759 if (nb_burst == 0) 1760 continue; 1761 1762 total_burst += nb_burst; 1763 1764 if (nb_burst > burst_stats[1]) { 1765 burst_stats[2] = burst_stats[1]; 1766 pktnb_stats[2] = pktnb_stats[1]; 1767 burst_stats[1] = nb_burst; 1768 pktnb_stats[1] = nb_pkt; 1769 } else if (nb_burst > burst_stats[2]) { 1770 burst_stats[2] = nb_burst; 1771 pktnb_stats[2] = nb_pkt; 1772 } 1773 } 1774 if (total_burst == 0) 1775 return; 1776 1777 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1778 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1779 if (i == 3) { 1780 printf("%d%% of other]\n", 100 - sburstp); 1781 return; 1782 } 1783 1784 sburst += burst_stats[i]; 1785 if (sburst == total_burst) { 1786 printf("%d%% of %d pkts]\n", 1787 100 - sburstp, (int) pktnb_stats[i]); 1788 return; 1789 } 1790 1791 burst_percent[i] = 1792 (double)burst_stats[i] / total_burst * 100; 1793 printf("%d%% of %d pkts + ", 1794 burst_percent[i], (int) pktnb_stats[i]); 1795 sburstp += burst_percent[i]; 1796 } 1797 } 1798 1799 static void 1800 fwd_stream_stats_display(streamid_t stream_id) 1801 { 1802 struct fwd_stream *fs; 1803 static const char *fwd_top_stats_border = "-------"; 1804 1805 fs = fwd_streams[stream_id]; 1806 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1807 (fs->fwd_dropped == 0)) 1808 return; 1809 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1810 "TX Port=%2d/Queue=%2d %s\n", 1811 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1812 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1813 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1814 " TX-dropped: %-14"PRIu64, 1815 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1816 1817 /* if checksum mode */ 1818 if (cur_fwd_eng == &csum_fwd_engine) { 1819 printf(" RX- bad IP checksum: %-14"PRIu64 1820 " Rx- bad L4 checksum: %-14"PRIu64 1821 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1822 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1823 fs->rx_bad_outer_l4_csum); 1824 } else { 1825 printf("\n"); 1826 } 1827 1828 if (record_burst_stats) { 1829 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1830 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1831 } 1832 } 1833 1834 void 1835 fwd_stats_display(void) 1836 { 1837 static const char *fwd_stats_border = "----------------------"; 1838 static const char *acc_stats_border = "+++++++++++++++"; 1839 struct { 1840 struct fwd_stream *rx_stream; 1841 struct fwd_stream *tx_stream; 1842 uint64_t tx_dropped; 1843 uint64_t rx_bad_ip_csum; 1844 uint64_t rx_bad_l4_csum; 1845 uint64_t rx_bad_outer_l4_csum; 1846 } ports_stats[RTE_MAX_ETHPORTS]; 1847 uint64_t total_rx_dropped = 0; 1848 uint64_t total_tx_dropped = 0; 1849 uint64_t total_rx_nombuf = 0; 1850 struct rte_eth_stats stats; 1851 uint64_t fwd_cycles = 0; 1852 uint64_t total_recv = 0; 1853 uint64_t total_xmit = 0; 1854 struct rte_port *port; 1855 streamid_t sm_id; 1856 portid_t pt_id; 1857 int i; 1858 1859 memset(ports_stats, 0, sizeof(ports_stats)); 1860 1861 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1862 struct fwd_stream *fs = fwd_streams[sm_id]; 1863 1864 if (cur_fwd_config.nb_fwd_streams > 1865 cur_fwd_config.nb_fwd_ports) { 1866 fwd_stream_stats_display(sm_id); 1867 } else { 1868 ports_stats[fs->tx_port].tx_stream = fs; 1869 ports_stats[fs->rx_port].rx_stream = fs; 1870 } 1871 1872 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1873 1874 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1875 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1876 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1877 fs->rx_bad_outer_l4_csum; 1878 1879 if (record_core_cycles) 1880 fwd_cycles += fs->core_cycles; 1881 } 1882 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1883 uint8_t j; 1884 1885 pt_id = fwd_ports_ids[i]; 1886 port = &ports[pt_id]; 1887 1888 rte_eth_stats_get(pt_id, &stats); 1889 stats.ipackets -= port->stats.ipackets; 1890 stats.opackets -= port->stats.opackets; 1891 stats.ibytes -= port->stats.ibytes; 1892 stats.obytes -= port->stats.obytes; 1893 stats.imissed -= port->stats.imissed; 1894 stats.oerrors -= port->stats.oerrors; 1895 stats.rx_nombuf -= port->stats.rx_nombuf; 1896 1897 total_recv += stats.ipackets; 1898 total_xmit += stats.opackets; 1899 total_rx_dropped += stats.imissed; 1900 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1901 total_tx_dropped += stats.oerrors; 1902 total_rx_nombuf += stats.rx_nombuf; 1903 1904 printf("\n %s Forward statistics for port %-2d %s\n", 1905 fwd_stats_border, pt_id, fwd_stats_border); 1906 1907 if (!port->rx_queue_stats_mapping_enabled && 1908 !port->tx_queue_stats_mapping_enabled) { 1909 printf(" RX-packets: %-14"PRIu64 1910 " RX-dropped: %-14"PRIu64 1911 "RX-total: %-"PRIu64"\n", 1912 stats.ipackets, stats.imissed, 1913 stats.ipackets + stats.imissed); 1914 1915 if (cur_fwd_eng == &csum_fwd_engine) 1916 printf(" Bad-ipcsum: %-14"PRIu64 1917 " Bad-l4csum: %-14"PRIu64 1918 "Bad-outer-l4csum: %-14"PRIu64"\n", 1919 ports_stats[pt_id].rx_bad_ip_csum, 1920 ports_stats[pt_id].rx_bad_l4_csum, 1921 ports_stats[pt_id].rx_bad_outer_l4_csum); 1922 if (stats.ierrors + stats.rx_nombuf > 0) { 1923 printf(" RX-error: %-"PRIu64"\n", 1924 stats.ierrors); 1925 printf(" RX-nombufs: %-14"PRIu64"\n", 1926 stats.rx_nombuf); 1927 } 1928 1929 printf(" TX-packets: %-14"PRIu64 1930 " TX-dropped: %-14"PRIu64 1931 "TX-total: %-"PRIu64"\n", 1932 stats.opackets, ports_stats[pt_id].tx_dropped, 1933 stats.opackets + ports_stats[pt_id].tx_dropped); 1934 } else { 1935 printf(" RX-packets: %14"PRIu64 1936 " RX-dropped:%14"PRIu64 1937 " RX-total:%14"PRIu64"\n", 1938 stats.ipackets, stats.imissed, 1939 stats.ipackets + stats.imissed); 1940 1941 if (cur_fwd_eng == &csum_fwd_engine) 1942 printf(" Bad-ipcsum:%14"PRIu64 1943 " Bad-l4csum:%14"PRIu64 1944 " Bad-outer-l4csum: %-14"PRIu64"\n", 1945 ports_stats[pt_id].rx_bad_ip_csum, 1946 ports_stats[pt_id].rx_bad_l4_csum, 1947 ports_stats[pt_id].rx_bad_outer_l4_csum); 1948 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1949 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1950 printf(" RX-nombufs: %14"PRIu64"\n", 1951 stats.rx_nombuf); 1952 } 1953 1954 printf(" TX-packets: %14"PRIu64 1955 " TX-dropped:%14"PRIu64 1956 " TX-total:%14"PRIu64"\n", 1957 stats.opackets, ports_stats[pt_id].tx_dropped, 1958 stats.opackets + ports_stats[pt_id].tx_dropped); 1959 } 1960 1961 if (record_burst_stats) { 1962 if (ports_stats[pt_id].rx_stream) 1963 pkt_burst_stats_display("RX", 1964 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1965 if (ports_stats[pt_id].tx_stream) 1966 pkt_burst_stats_display("TX", 1967 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1968 } 1969 1970 if (port->rx_queue_stats_mapping_enabled) { 1971 printf("\n"); 1972 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1973 printf(" Stats reg %2d RX-packets:%14"PRIu64 1974 " RX-errors:%14"PRIu64 1975 " RX-bytes:%14"PRIu64"\n", 1976 j, stats.q_ipackets[j], 1977 stats.q_errors[j], stats.q_ibytes[j]); 1978 } 1979 printf("\n"); 1980 } 1981 if (port->tx_queue_stats_mapping_enabled) { 1982 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1983 printf(" Stats reg %2d TX-packets:%14"PRIu64 1984 " TX-bytes:%14" 1985 PRIu64"\n", 1986 j, stats.q_opackets[j], 1987 stats.q_obytes[j]); 1988 } 1989 } 1990 1991 printf(" %s--------------------------------%s\n", 1992 fwd_stats_border, fwd_stats_border); 1993 } 1994 1995 printf("\n %s Accumulated forward statistics for all ports" 1996 "%s\n", 1997 acc_stats_border, acc_stats_border); 1998 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1999 "%-"PRIu64"\n" 2000 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2001 "%-"PRIu64"\n", 2002 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2003 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2004 if (total_rx_nombuf > 0) 2005 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2006 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2007 "%s\n", 2008 acc_stats_border, acc_stats_border); 2009 if (record_core_cycles) { 2010 #define CYC_PER_MHZ 1E6 2011 if (total_recv > 0 || total_xmit > 0) { 2012 uint64_t total_pkts = 0; 2013 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2014 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2015 total_pkts = total_xmit; 2016 else 2017 total_pkts = total_recv; 2018 2019 printf("\n CPU cycles/packet=%.2F (total cycles=" 2020 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2021 " MHz Clock\n", 2022 (double) fwd_cycles / total_pkts, 2023 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2024 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2025 } 2026 } 2027 } 2028 2029 void 2030 fwd_stats_reset(void) 2031 { 2032 streamid_t sm_id; 2033 portid_t pt_id; 2034 int i; 2035 2036 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2037 pt_id = fwd_ports_ids[i]; 2038 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2039 } 2040 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2041 struct fwd_stream *fs = fwd_streams[sm_id]; 2042 2043 fs->rx_packets = 0; 2044 fs->tx_packets = 0; 2045 fs->fwd_dropped = 0; 2046 fs->rx_bad_ip_csum = 0; 2047 fs->rx_bad_l4_csum = 0; 2048 fs->rx_bad_outer_l4_csum = 0; 2049 2050 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2051 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2052 fs->core_cycles = 0; 2053 } 2054 } 2055 2056 static void 2057 flush_fwd_rx_queues(void) 2058 { 2059 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2060 portid_t rxp; 2061 portid_t port_id; 2062 queueid_t rxq; 2063 uint16_t nb_rx; 2064 uint16_t i; 2065 uint8_t j; 2066 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2067 uint64_t timer_period; 2068 2069 /* convert to number of cycles */ 2070 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2071 2072 for (j = 0; j < 2; j++) { 2073 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2074 for (rxq = 0; rxq < nb_rxq; rxq++) { 2075 port_id = fwd_ports_ids[rxp]; 2076 /** 2077 * testpmd can stuck in the below do while loop 2078 * if rte_eth_rx_burst() always returns nonzero 2079 * packets. So timer is added to exit this loop 2080 * after 1sec timer expiry. 2081 */ 2082 prev_tsc = rte_rdtsc(); 2083 do { 2084 nb_rx = rte_eth_rx_burst(port_id, rxq, 2085 pkts_burst, MAX_PKT_BURST); 2086 for (i = 0; i < nb_rx; i++) 2087 rte_pktmbuf_free(pkts_burst[i]); 2088 2089 cur_tsc = rte_rdtsc(); 2090 diff_tsc = cur_tsc - prev_tsc; 2091 timer_tsc += diff_tsc; 2092 } while ((nb_rx > 0) && 2093 (timer_tsc < timer_period)); 2094 timer_tsc = 0; 2095 } 2096 } 2097 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2098 } 2099 } 2100 2101 static void 2102 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2103 { 2104 struct fwd_stream **fsm; 2105 streamid_t nb_fs; 2106 streamid_t sm_id; 2107 #ifdef RTE_LIB_BITRATESTATS 2108 uint64_t tics_per_1sec; 2109 uint64_t tics_datum; 2110 uint64_t tics_current; 2111 uint16_t i, cnt_ports; 2112 2113 cnt_ports = nb_ports; 2114 tics_datum = rte_rdtsc(); 2115 tics_per_1sec = rte_get_timer_hz(); 2116 #endif 2117 fsm = &fwd_streams[fc->stream_idx]; 2118 nb_fs = fc->stream_nb; 2119 do { 2120 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2121 (*pkt_fwd)(fsm[sm_id]); 2122 #ifdef RTE_LIB_BITRATESTATS 2123 if (bitrate_enabled != 0 && 2124 bitrate_lcore_id == rte_lcore_id()) { 2125 tics_current = rte_rdtsc(); 2126 if (tics_current - tics_datum >= tics_per_1sec) { 2127 /* Periodic bitrate calculation */ 2128 for (i = 0; i < cnt_ports; i++) 2129 rte_stats_bitrate_calc(bitrate_data, 2130 ports_ids[i]); 2131 tics_datum = tics_current; 2132 } 2133 } 2134 #endif 2135 #ifdef RTE_LIB_LATENCYSTATS 2136 if (latencystats_enabled != 0 && 2137 latencystats_lcore_id == rte_lcore_id()) 2138 rte_latencystats_update(); 2139 #endif 2140 2141 } while (! fc->stopped); 2142 } 2143 2144 static int 2145 start_pkt_forward_on_core(void *fwd_arg) 2146 { 2147 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2148 cur_fwd_config.fwd_eng->packet_fwd); 2149 return 0; 2150 } 2151 2152 /* 2153 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2154 * Used to start communication flows in network loopback test configurations. 2155 */ 2156 static int 2157 run_one_txonly_burst_on_core(void *fwd_arg) 2158 { 2159 struct fwd_lcore *fwd_lc; 2160 struct fwd_lcore tmp_lcore; 2161 2162 fwd_lc = (struct fwd_lcore *) fwd_arg; 2163 tmp_lcore = *fwd_lc; 2164 tmp_lcore.stopped = 1; 2165 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2166 return 0; 2167 } 2168 2169 /* 2170 * Launch packet forwarding: 2171 * - Setup per-port forwarding context. 2172 * - launch logical cores with their forwarding configuration. 2173 */ 2174 static void 2175 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2176 { 2177 port_fwd_begin_t port_fwd_begin; 2178 unsigned int i; 2179 unsigned int lc_id; 2180 int diag; 2181 2182 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2183 if (port_fwd_begin != NULL) { 2184 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2185 (*port_fwd_begin)(fwd_ports_ids[i]); 2186 } 2187 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2188 lc_id = fwd_lcores_cpuids[i]; 2189 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2190 fwd_lcores[i]->stopped = 0; 2191 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2192 fwd_lcores[i], lc_id); 2193 if (diag != 0) 2194 printf("launch lcore %u failed - diag=%d\n", 2195 lc_id, diag); 2196 } 2197 } 2198 } 2199 2200 /* 2201 * Launch packet forwarding configuration. 2202 */ 2203 void 2204 start_packet_forwarding(int with_tx_first) 2205 { 2206 port_fwd_begin_t port_fwd_begin; 2207 port_fwd_end_t port_fwd_end; 2208 struct rte_port *port; 2209 unsigned int i; 2210 portid_t pt_id; 2211 2212 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2213 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2214 2215 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2216 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2217 2218 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2219 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2220 (!nb_rxq || !nb_txq)) 2221 rte_exit(EXIT_FAILURE, 2222 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2223 cur_fwd_eng->fwd_mode_name); 2224 2225 if (all_ports_started() == 0) { 2226 printf("Not all ports were started\n"); 2227 return; 2228 } 2229 if (test_done == 0) { 2230 printf("Packet forwarding already started\n"); 2231 return; 2232 } 2233 2234 2235 if(dcb_test) { 2236 for (i = 0; i < nb_fwd_ports; i++) { 2237 pt_id = fwd_ports_ids[i]; 2238 port = &ports[pt_id]; 2239 if (!port->dcb_flag) { 2240 printf("In DCB mode, all forwarding ports must " 2241 "be configured in this mode.\n"); 2242 return; 2243 } 2244 } 2245 if (nb_fwd_lcores == 1) { 2246 printf("In DCB mode,the nb forwarding cores " 2247 "should be larger than 1.\n"); 2248 return; 2249 } 2250 } 2251 test_done = 0; 2252 2253 fwd_config_setup(); 2254 2255 if(!no_flush_rx) 2256 flush_fwd_rx_queues(); 2257 2258 pkt_fwd_config_display(&cur_fwd_config); 2259 rxtx_config_display(); 2260 2261 fwd_stats_reset(); 2262 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2263 pt_id = fwd_ports_ids[i]; 2264 port = &ports[pt_id]; 2265 map_port_queue_stats_mapping_registers(pt_id, port); 2266 } 2267 if (with_tx_first) { 2268 port_fwd_begin = tx_only_engine.port_fwd_begin; 2269 if (port_fwd_begin != NULL) { 2270 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2271 (*port_fwd_begin)(fwd_ports_ids[i]); 2272 } 2273 while (with_tx_first--) { 2274 launch_packet_forwarding( 2275 run_one_txonly_burst_on_core); 2276 rte_eal_mp_wait_lcore(); 2277 } 2278 port_fwd_end = tx_only_engine.port_fwd_end; 2279 if (port_fwd_end != NULL) { 2280 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2281 (*port_fwd_end)(fwd_ports_ids[i]); 2282 } 2283 } 2284 launch_packet_forwarding(start_pkt_forward_on_core); 2285 } 2286 2287 void 2288 stop_packet_forwarding(void) 2289 { 2290 port_fwd_end_t port_fwd_end; 2291 lcoreid_t lc_id; 2292 portid_t pt_id; 2293 int i; 2294 2295 if (test_done) { 2296 printf("Packet forwarding not started\n"); 2297 return; 2298 } 2299 printf("Telling cores to stop..."); 2300 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2301 fwd_lcores[lc_id]->stopped = 1; 2302 printf("\nWaiting for lcores to finish...\n"); 2303 rte_eal_mp_wait_lcore(); 2304 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2305 if (port_fwd_end != NULL) { 2306 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2307 pt_id = fwd_ports_ids[i]; 2308 (*port_fwd_end)(pt_id); 2309 } 2310 } 2311 2312 fwd_stats_display(); 2313 2314 printf("\nDone.\n"); 2315 test_done = 1; 2316 } 2317 2318 void 2319 dev_set_link_up(portid_t pid) 2320 { 2321 if (rte_eth_dev_set_link_up(pid) < 0) 2322 printf("\nSet link up fail.\n"); 2323 } 2324 2325 void 2326 dev_set_link_down(portid_t pid) 2327 { 2328 if (rte_eth_dev_set_link_down(pid) < 0) 2329 printf("\nSet link down fail.\n"); 2330 } 2331 2332 static int 2333 all_ports_started(void) 2334 { 2335 portid_t pi; 2336 struct rte_port *port; 2337 2338 RTE_ETH_FOREACH_DEV(pi) { 2339 port = &ports[pi]; 2340 /* Check if there is a port which is not started */ 2341 if ((port->port_status != RTE_PORT_STARTED) && 2342 (port->slave_flag == 0)) 2343 return 0; 2344 } 2345 2346 /* No port is not started */ 2347 return 1; 2348 } 2349 2350 int 2351 port_is_stopped(portid_t port_id) 2352 { 2353 struct rte_port *port = &ports[port_id]; 2354 2355 if ((port->port_status != RTE_PORT_STOPPED) && 2356 (port->slave_flag == 0)) 2357 return 0; 2358 return 1; 2359 } 2360 2361 int 2362 all_ports_stopped(void) 2363 { 2364 portid_t pi; 2365 2366 RTE_ETH_FOREACH_DEV(pi) { 2367 if (!port_is_stopped(pi)) 2368 return 0; 2369 } 2370 2371 return 1; 2372 } 2373 2374 int 2375 port_is_started(portid_t port_id) 2376 { 2377 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2378 return 0; 2379 2380 if (ports[port_id].port_status != RTE_PORT_STARTED) 2381 return 0; 2382 2383 return 1; 2384 } 2385 2386 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2387 static int 2388 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2389 { 2390 queueid_t qi; 2391 struct rte_eth_hairpin_conf hairpin_conf = { 2392 .peer_count = 1, 2393 }; 2394 int i; 2395 int diag; 2396 struct rte_port *port = &ports[pi]; 2397 uint16_t peer_rx_port = pi; 2398 uint16_t peer_tx_port = pi; 2399 uint32_t manual = 1; 2400 uint32_t tx_exp = hairpin_mode & 0x10; 2401 2402 if (!(hairpin_mode & 0xf)) { 2403 peer_rx_port = pi; 2404 peer_tx_port = pi; 2405 manual = 0; 2406 } else if (hairpin_mode & 0x1) { 2407 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2408 RTE_ETH_DEV_NO_OWNER); 2409 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2410 peer_tx_port = rte_eth_find_next_owned_by(0, 2411 RTE_ETH_DEV_NO_OWNER); 2412 if (p_pi != RTE_MAX_ETHPORTS) { 2413 peer_rx_port = p_pi; 2414 } else { 2415 uint16_t next_pi; 2416 2417 /* Last port will be the peer RX port of the first. */ 2418 RTE_ETH_FOREACH_DEV(next_pi) 2419 peer_rx_port = next_pi; 2420 } 2421 manual = 1; 2422 } else if (hairpin_mode & 0x2) { 2423 if (cnt_pi & 0x1) { 2424 peer_rx_port = p_pi; 2425 } else { 2426 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2427 RTE_ETH_DEV_NO_OWNER); 2428 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2429 peer_rx_port = pi; 2430 } 2431 peer_tx_port = peer_rx_port; 2432 manual = 1; 2433 } 2434 2435 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2436 hairpin_conf.peers[0].port = peer_rx_port; 2437 hairpin_conf.peers[0].queue = i + nb_rxq; 2438 hairpin_conf.manual_bind = !!manual; 2439 hairpin_conf.tx_explicit = !!tx_exp; 2440 diag = rte_eth_tx_hairpin_queue_setup 2441 (pi, qi, nb_txd, &hairpin_conf); 2442 i++; 2443 if (diag == 0) 2444 continue; 2445 2446 /* Fail to setup rx queue, return */ 2447 if (rte_atomic16_cmpset(&(port->port_status), 2448 RTE_PORT_HANDLING, 2449 RTE_PORT_STOPPED) == 0) 2450 printf("Port %d can not be set back " 2451 "to stopped\n", pi); 2452 printf("Fail to configure port %d hairpin " 2453 "queues\n", pi); 2454 /* try to reconfigure queues next time */ 2455 port->need_reconfig_queues = 1; 2456 return -1; 2457 } 2458 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2459 hairpin_conf.peers[0].port = peer_tx_port; 2460 hairpin_conf.peers[0].queue = i + nb_txq; 2461 hairpin_conf.manual_bind = !!manual; 2462 hairpin_conf.tx_explicit = !!tx_exp; 2463 diag = rte_eth_rx_hairpin_queue_setup 2464 (pi, qi, nb_rxd, &hairpin_conf); 2465 i++; 2466 if (diag == 0) 2467 continue; 2468 2469 /* Fail to setup rx queue, return */ 2470 if (rte_atomic16_cmpset(&(port->port_status), 2471 RTE_PORT_HANDLING, 2472 RTE_PORT_STOPPED) == 0) 2473 printf("Port %d can not be set back " 2474 "to stopped\n", pi); 2475 printf("Fail to configure port %d hairpin " 2476 "queues\n", pi); 2477 /* try to reconfigure queues next time */ 2478 port->need_reconfig_queues = 1; 2479 return -1; 2480 } 2481 return 0; 2482 } 2483 2484 /* Configure the Rx with optional split. */ 2485 int 2486 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2487 uint16_t nb_rx_desc, unsigned int socket_id, 2488 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2489 { 2490 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2491 unsigned int i, mp_n; 2492 int ret; 2493 2494 if (rx_pkt_nb_segs <= 1 || 2495 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2496 rx_conf->rx_seg = NULL; 2497 rx_conf->rx_nseg = 0; 2498 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2499 nb_rx_desc, socket_id, 2500 rx_conf, mp); 2501 return ret; 2502 } 2503 for (i = 0; i < rx_pkt_nb_segs; i++) { 2504 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2505 struct rte_mempool *mpx; 2506 /* 2507 * Use last valid pool for the segments with number 2508 * exceeding the pool index. 2509 */ 2510 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2511 mpx = mbuf_pool_find(socket_id, mp_n); 2512 /* Handle zero as mbuf data buffer size. */ 2513 rx_seg->length = rx_pkt_seg_lengths[i] ? 2514 rx_pkt_seg_lengths[i] : 2515 mbuf_data_size[mp_n]; 2516 rx_seg->offset = i < rx_pkt_nb_offs ? 2517 rx_pkt_seg_offsets[i] : 0; 2518 rx_seg->mp = mpx ? mpx : mp; 2519 } 2520 rx_conf->rx_nseg = rx_pkt_nb_segs; 2521 rx_conf->rx_seg = rx_useg; 2522 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2523 socket_id, rx_conf, NULL); 2524 rx_conf->rx_seg = NULL; 2525 rx_conf->rx_nseg = 0; 2526 return ret; 2527 } 2528 2529 int 2530 start_port(portid_t pid) 2531 { 2532 int diag, need_check_link_status = -1; 2533 portid_t pi; 2534 portid_t p_pi = RTE_MAX_ETHPORTS; 2535 portid_t pl[RTE_MAX_ETHPORTS]; 2536 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2537 uint16_t cnt_pi = 0; 2538 uint16_t cfg_pi = 0; 2539 int peer_pi; 2540 queueid_t qi; 2541 struct rte_port *port; 2542 struct rte_ether_addr mac_addr; 2543 struct rte_eth_hairpin_cap cap; 2544 2545 if (port_id_is_invalid(pid, ENABLED_WARN)) 2546 return 0; 2547 2548 if(dcb_config) 2549 dcb_test = 1; 2550 RTE_ETH_FOREACH_DEV(pi) { 2551 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2552 continue; 2553 2554 need_check_link_status = 0; 2555 port = &ports[pi]; 2556 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2557 RTE_PORT_HANDLING) == 0) { 2558 printf("Port %d is now not stopped\n", pi); 2559 continue; 2560 } 2561 2562 if (port->need_reconfig > 0) { 2563 port->need_reconfig = 0; 2564 2565 if (flow_isolate_all) { 2566 int ret = port_flow_isolate(pi, 1); 2567 if (ret) { 2568 printf("Failed to apply isolated" 2569 " mode on port %d\n", pi); 2570 return -1; 2571 } 2572 } 2573 configure_rxtx_dump_callbacks(0); 2574 printf("Configuring Port %d (socket %u)\n", pi, 2575 port->socket_id); 2576 if (nb_hairpinq > 0 && 2577 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2578 printf("Port %d doesn't support hairpin " 2579 "queues\n", pi); 2580 return -1; 2581 } 2582 /* configure port */ 2583 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2584 nb_txq + nb_hairpinq, 2585 &(port->dev_conf)); 2586 if (diag != 0) { 2587 if (rte_atomic16_cmpset(&(port->port_status), 2588 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2589 printf("Port %d can not be set back " 2590 "to stopped\n", pi); 2591 printf("Fail to configure port %d\n", pi); 2592 /* try to reconfigure port next time */ 2593 port->need_reconfig = 1; 2594 return -1; 2595 } 2596 } 2597 if (port->need_reconfig_queues > 0) { 2598 port->need_reconfig_queues = 0; 2599 /* setup tx queues */ 2600 for (qi = 0; qi < nb_txq; qi++) { 2601 if ((numa_support) && 2602 (txring_numa[pi] != NUMA_NO_CONFIG)) 2603 diag = rte_eth_tx_queue_setup(pi, qi, 2604 port->nb_tx_desc[qi], 2605 txring_numa[pi], 2606 &(port->tx_conf[qi])); 2607 else 2608 diag = rte_eth_tx_queue_setup(pi, qi, 2609 port->nb_tx_desc[qi], 2610 port->socket_id, 2611 &(port->tx_conf[qi])); 2612 2613 if (diag == 0) 2614 continue; 2615 2616 /* Fail to setup tx queue, return */ 2617 if (rte_atomic16_cmpset(&(port->port_status), 2618 RTE_PORT_HANDLING, 2619 RTE_PORT_STOPPED) == 0) 2620 printf("Port %d can not be set back " 2621 "to stopped\n", pi); 2622 printf("Fail to configure port %d tx queues\n", 2623 pi); 2624 /* try to reconfigure queues next time */ 2625 port->need_reconfig_queues = 1; 2626 return -1; 2627 } 2628 for (qi = 0; qi < nb_rxq; qi++) { 2629 /* setup rx queues */ 2630 if ((numa_support) && 2631 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2632 struct rte_mempool * mp = 2633 mbuf_pool_find 2634 (rxring_numa[pi], 0); 2635 if (mp == NULL) { 2636 printf("Failed to setup RX queue:" 2637 "No mempool allocation" 2638 " on the socket %d\n", 2639 rxring_numa[pi]); 2640 return -1; 2641 } 2642 2643 diag = rx_queue_setup(pi, qi, 2644 port->nb_rx_desc[qi], 2645 rxring_numa[pi], 2646 &(port->rx_conf[qi]), 2647 mp); 2648 } else { 2649 struct rte_mempool *mp = 2650 mbuf_pool_find 2651 (port->socket_id, 0); 2652 if (mp == NULL) { 2653 printf("Failed to setup RX queue:" 2654 "No mempool allocation" 2655 " on the socket %d\n", 2656 port->socket_id); 2657 return -1; 2658 } 2659 diag = rx_queue_setup(pi, qi, 2660 port->nb_rx_desc[qi], 2661 port->socket_id, 2662 &(port->rx_conf[qi]), 2663 mp); 2664 } 2665 if (diag == 0) 2666 continue; 2667 2668 /* Fail to setup rx queue, return */ 2669 if (rte_atomic16_cmpset(&(port->port_status), 2670 RTE_PORT_HANDLING, 2671 RTE_PORT_STOPPED) == 0) 2672 printf("Port %d can not be set back " 2673 "to stopped\n", pi); 2674 printf("Fail to configure port %d rx queues\n", 2675 pi); 2676 /* try to reconfigure queues next time */ 2677 port->need_reconfig_queues = 1; 2678 return -1; 2679 } 2680 /* setup hairpin queues */ 2681 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2682 return -1; 2683 } 2684 configure_rxtx_dump_callbacks(verbose_level); 2685 if (clear_ptypes) { 2686 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2687 NULL, 0); 2688 if (diag < 0) 2689 printf( 2690 "Port %d: Failed to disable Ptype parsing\n", 2691 pi); 2692 } 2693 2694 p_pi = pi; 2695 cnt_pi++; 2696 2697 /* start port */ 2698 if (rte_eth_dev_start(pi) < 0) { 2699 printf("Fail to start port %d\n", pi); 2700 2701 /* Fail to setup rx queue, return */ 2702 if (rte_atomic16_cmpset(&(port->port_status), 2703 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2704 printf("Port %d can not be set back to " 2705 "stopped\n", pi); 2706 continue; 2707 } 2708 2709 if (rte_atomic16_cmpset(&(port->port_status), 2710 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2711 printf("Port %d can not be set into started\n", pi); 2712 2713 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2714 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2715 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2716 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2717 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2718 2719 /* at least one port started, need checking link status */ 2720 need_check_link_status = 1; 2721 2722 pl[cfg_pi++] = pi; 2723 } 2724 2725 if (need_check_link_status == 1 && !no_link_check) 2726 check_all_ports_link_status(RTE_PORT_ALL); 2727 else if (need_check_link_status == 0) 2728 printf("Please stop the ports first\n"); 2729 2730 if (hairpin_mode & 0xf) { 2731 uint16_t i; 2732 int j; 2733 2734 /* bind all started hairpin ports */ 2735 for (i = 0; i < cfg_pi; i++) { 2736 pi = pl[i]; 2737 /* bind current Tx to all peer Rx */ 2738 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2739 RTE_MAX_ETHPORTS, 1); 2740 if (peer_pi < 0) 2741 return peer_pi; 2742 for (j = 0; j < peer_pi; j++) { 2743 if (!port_is_started(peer_pl[j])) 2744 continue; 2745 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2746 if (diag < 0) { 2747 printf("Error during binding hairpin" 2748 " Tx port %u to %u: %s\n", 2749 pi, peer_pl[j], 2750 rte_strerror(-diag)); 2751 return -1; 2752 } 2753 } 2754 /* bind all peer Tx to current Rx */ 2755 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2756 RTE_MAX_ETHPORTS, 0); 2757 if (peer_pi < 0) 2758 return peer_pi; 2759 for (j = 0; j < peer_pi; j++) { 2760 if (!port_is_started(peer_pl[j])) 2761 continue; 2762 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2763 if (diag < 0) { 2764 printf("Error during binding hairpin" 2765 " Tx port %u to %u: %s\n", 2766 peer_pl[j], pi, 2767 rte_strerror(-diag)); 2768 return -1; 2769 } 2770 } 2771 } 2772 } 2773 2774 printf("Done\n"); 2775 return 0; 2776 } 2777 2778 void 2779 stop_port(portid_t pid) 2780 { 2781 portid_t pi; 2782 struct rte_port *port; 2783 int need_check_link_status = 0; 2784 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2785 int peer_pi; 2786 2787 if (dcb_test) { 2788 dcb_test = 0; 2789 dcb_config = 0; 2790 } 2791 2792 if (port_id_is_invalid(pid, ENABLED_WARN)) 2793 return; 2794 2795 printf("Stopping ports...\n"); 2796 2797 RTE_ETH_FOREACH_DEV(pi) { 2798 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2799 continue; 2800 2801 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2802 printf("Please remove port %d from forwarding configuration.\n", pi); 2803 continue; 2804 } 2805 2806 if (port_is_bonding_slave(pi)) { 2807 printf("Please remove port %d from bonded device.\n", pi); 2808 continue; 2809 } 2810 2811 port = &ports[pi]; 2812 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2813 RTE_PORT_HANDLING) == 0) 2814 continue; 2815 2816 if (hairpin_mode & 0xf) { 2817 int j; 2818 2819 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2820 /* unbind all peer Tx from current Rx */ 2821 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2822 RTE_MAX_ETHPORTS, 0); 2823 if (peer_pi < 0) 2824 continue; 2825 for (j = 0; j < peer_pi; j++) { 2826 if (!port_is_started(peer_pl[j])) 2827 continue; 2828 rte_eth_hairpin_unbind(peer_pl[j], pi); 2829 } 2830 } 2831 2832 if (rte_eth_dev_stop(pi) != 0) 2833 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2834 pi); 2835 2836 if (rte_atomic16_cmpset(&(port->port_status), 2837 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2838 printf("Port %d can not be set into stopped\n", pi); 2839 need_check_link_status = 1; 2840 } 2841 if (need_check_link_status && !no_link_check) 2842 check_all_ports_link_status(RTE_PORT_ALL); 2843 2844 printf("Done\n"); 2845 } 2846 2847 static void 2848 remove_invalid_ports_in(portid_t *array, portid_t *total) 2849 { 2850 portid_t i; 2851 portid_t new_total = 0; 2852 2853 for (i = 0; i < *total; i++) 2854 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2855 array[new_total] = array[i]; 2856 new_total++; 2857 } 2858 *total = new_total; 2859 } 2860 2861 static void 2862 remove_invalid_ports(void) 2863 { 2864 remove_invalid_ports_in(ports_ids, &nb_ports); 2865 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2866 nb_cfg_ports = nb_fwd_ports; 2867 } 2868 2869 void 2870 close_port(portid_t pid) 2871 { 2872 portid_t pi; 2873 struct rte_port *port; 2874 2875 if (port_id_is_invalid(pid, ENABLED_WARN)) 2876 return; 2877 2878 printf("Closing ports...\n"); 2879 2880 RTE_ETH_FOREACH_DEV(pi) { 2881 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2882 continue; 2883 2884 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2885 printf("Please remove port %d from forwarding configuration.\n", pi); 2886 continue; 2887 } 2888 2889 if (port_is_bonding_slave(pi)) { 2890 printf("Please remove port %d from bonded device.\n", pi); 2891 continue; 2892 } 2893 2894 port = &ports[pi]; 2895 if (rte_atomic16_cmpset(&(port->port_status), 2896 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2897 printf("Port %d is already closed\n", pi); 2898 continue; 2899 } 2900 2901 port_flow_flush(pi); 2902 rte_eth_dev_close(pi); 2903 } 2904 2905 remove_invalid_ports(); 2906 printf("Done\n"); 2907 } 2908 2909 void 2910 reset_port(portid_t pid) 2911 { 2912 int diag; 2913 portid_t pi; 2914 struct rte_port *port; 2915 2916 if (port_id_is_invalid(pid, ENABLED_WARN)) 2917 return; 2918 2919 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2920 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2921 printf("Can not reset port(s), please stop port(s) first.\n"); 2922 return; 2923 } 2924 2925 printf("Resetting ports...\n"); 2926 2927 RTE_ETH_FOREACH_DEV(pi) { 2928 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2929 continue; 2930 2931 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2932 printf("Please remove port %d from forwarding " 2933 "configuration.\n", pi); 2934 continue; 2935 } 2936 2937 if (port_is_bonding_slave(pi)) { 2938 printf("Please remove port %d from bonded device.\n", 2939 pi); 2940 continue; 2941 } 2942 2943 diag = rte_eth_dev_reset(pi); 2944 if (diag == 0) { 2945 port = &ports[pi]; 2946 port->need_reconfig = 1; 2947 port->need_reconfig_queues = 1; 2948 } else { 2949 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2950 } 2951 } 2952 2953 printf("Done\n"); 2954 } 2955 2956 void 2957 attach_port(char *identifier) 2958 { 2959 portid_t pi; 2960 struct rte_dev_iterator iterator; 2961 2962 printf("Attaching a new port...\n"); 2963 2964 if (identifier == NULL) { 2965 printf("Invalid parameters are specified\n"); 2966 return; 2967 } 2968 2969 if (rte_dev_probe(identifier) < 0) { 2970 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2971 return; 2972 } 2973 2974 /* first attach mode: event */ 2975 if (setup_on_probe_event) { 2976 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2977 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2978 if (ports[pi].port_status == RTE_PORT_HANDLING && 2979 ports[pi].need_setup != 0) 2980 setup_attached_port(pi); 2981 return; 2982 } 2983 2984 /* second attach mode: iterator */ 2985 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2986 /* setup ports matching the devargs used for probing */ 2987 if (port_is_forwarding(pi)) 2988 continue; /* port was already attached before */ 2989 setup_attached_port(pi); 2990 } 2991 } 2992 2993 static void 2994 setup_attached_port(portid_t pi) 2995 { 2996 unsigned int socket_id; 2997 int ret; 2998 2999 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3000 /* if socket_id is invalid, set to the first available socket. */ 3001 if (check_socket_id(socket_id) < 0) 3002 socket_id = socket_ids[0]; 3003 reconfig(pi, socket_id); 3004 ret = rte_eth_promiscuous_enable(pi); 3005 if (ret != 0) 3006 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3007 pi, rte_strerror(-ret)); 3008 3009 ports_ids[nb_ports++] = pi; 3010 fwd_ports_ids[nb_fwd_ports++] = pi; 3011 nb_cfg_ports = nb_fwd_ports; 3012 ports[pi].need_setup = 0; 3013 ports[pi].port_status = RTE_PORT_STOPPED; 3014 3015 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3016 printf("Done\n"); 3017 } 3018 3019 static void 3020 detach_device(struct rte_device *dev) 3021 { 3022 portid_t sibling; 3023 3024 if (dev == NULL) { 3025 printf("Device already removed\n"); 3026 return; 3027 } 3028 3029 printf("Removing a device...\n"); 3030 3031 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3032 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3033 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3034 printf("Port %u not stopped\n", sibling); 3035 return; 3036 } 3037 port_flow_flush(sibling); 3038 } 3039 } 3040 3041 if (rte_dev_remove(dev) < 0) { 3042 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3043 return; 3044 } 3045 remove_invalid_ports(); 3046 3047 printf("Device is detached\n"); 3048 printf("Now total ports is %d\n", nb_ports); 3049 printf("Done\n"); 3050 return; 3051 } 3052 3053 void 3054 detach_port_device(portid_t port_id) 3055 { 3056 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3057 return; 3058 3059 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3060 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3061 printf("Port not stopped\n"); 3062 return; 3063 } 3064 printf("Port was not closed\n"); 3065 } 3066 3067 detach_device(rte_eth_devices[port_id].device); 3068 } 3069 3070 void 3071 detach_devargs(char *identifier) 3072 { 3073 struct rte_dev_iterator iterator; 3074 struct rte_devargs da; 3075 portid_t port_id; 3076 3077 printf("Removing a device...\n"); 3078 3079 memset(&da, 0, sizeof(da)); 3080 if (rte_devargs_parsef(&da, "%s", identifier)) { 3081 printf("cannot parse identifier\n"); 3082 if (da.args) 3083 free(da.args); 3084 return; 3085 } 3086 3087 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3088 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3089 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3090 printf("Port %u not stopped\n", port_id); 3091 rte_eth_iterator_cleanup(&iterator); 3092 return; 3093 } 3094 port_flow_flush(port_id); 3095 } 3096 } 3097 3098 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3099 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3100 da.name, da.bus->name); 3101 return; 3102 } 3103 3104 remove_invalid_ports(); 3105 3106 printf("Device %s is detached\n", identifier); 3107 printf("Now total ports is %d\n", nb_ports); 3108 printf("Done\n"); 3109 } 3110 3111 void 3112 pmd_test_exit(void) 3113 { 3114 portid_t pt_id; 3115 unsigned int i; 3116 int ret; 3117 3118 if (test_done == 0) 3119 stop_packet_forwarding(); 3120 3121 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3122 if (mempools[i]) { 3123 if (mp_alloc_type == MP_ALLOC_ANON) 3124 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3125 NULL); 3126 } 3127 } 3128 if (ports != NULL) { 3129 no_link_check = 1; 3130 RTE_ETH_FOREACH_DEV(pt_id) { 3131 printf("\nStopping port %d...\n", pt_id); 3132 fflush(stdout); 3133 stop_port(pt_id); 3134 } 3135 RTE_ETH_FOREACH_DEV(pt_id) { 3136 printf("\nShutting down port %d...\n", pt_id); 3137 fflush(stdout); 3138 close_port(pt_id); 3139 } 3140 } 3141 3142 if (hot_plug) { 3143 ret = rte_dev_event_monitor_stop(); 3144 if (ret) { 3145 RTE_LOG(ERR, EAL, 3146 "fail to stop device event monitor."); 3147 return; 3148 } 3149 3150 ret = rte_dev_event_callback_unregister(NULL, 3151 dev_event_callback, NULL); 3152 if (ret < 0) { 3153 RTE_LOG(ERR, EAL, 3154 "fail to unregister device event callback.\n"); 3155 return; 3156 } 3157 3158 ret = rte_dev_hotplug_handle_disable(); 3159 if (ret) { 3160 RTE_LOG(ERR, EAL, 3161 "fail to disable hotplug handling.\n"); 3162 return; 3163 } 3164 } 3165 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3166 if (mempools[i]) 3167 rte_mempool_free(mempools[i]); 3168 } 3169 3170 printf("\nBye...\n"); 3171 } 3172 3173 typedef void (*cmd_func_t)(void); 3174 struct pmd_test_command { 3175 const char *cmd_name; 3176 cmd_func_t cmd_func; 3177 }; 3178 3179 /* Check the link status of all ports in up to 9s, and print them finally */ 3180 static void 3181 check_all_ports_link_status(uint32_t port_mask) 3182 { 3183 #define CHECK_INTERVAL 100 /* 100ms */ 3184 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3185 portid_t portid; 3186 uint8_t count, all_ports_up, print_flag = 0; 3187 struct rte_eth_link link; 3188 int ret; 3189 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3190 3191 printf("Checking link statuses...\n"); 3192 fflush(stdout); 3193 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3194 all_ports_up = 1; 3195 RTE_ETH_FOREACH_DEV(portid) { 3196 if ((port_mask & (1 << portid)) == 0) 3197 continue; 3198 memset(&link, 0, sizeof(link)); 3199 ret = rte_eth_link_get_nowait(portid, &link); 3200 if (ret < 0) { 3201 all_ports_up = 0; 3202 if (print_flag == 1) 3203 printf("Port %u link get failed: %s\n", 3204 portid, rte_strerror(-ret)); 3205 continue; 3206 } 3207 /* print link status if flag set */ 3208 if (print_flag == 1) { 3209 rte_eth_link_to_str(link_status, 3210 sizeof(link_status), &link); 3211 printf("Port %d %s\n", portid, link_status); 3212 continue; 3213 } 3214 /* clear all_ports_up flag if any link down */ 3215 if (link.link_status == ETH_LINK_DOWN) { 3216 all_ports_up = 0; 3217 break; 3218 } 3219 } 3220 /* after finally printing all link status, get out */ 3221 if (print_flag == 1) 3222 break; 3223 3224 if (all_ports_up == 0) { 3225 fflush(stdout); 3226 rte_delay_ms(CHECK_INTERVAL); 3227 } 3228 3229 /* set the print_flag if all ports up or timeout */ 3230 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3231 print_flag = 1; 3232 } 3233 3234 if (lsc_interrupt) 3235 break; 3236 } 3237 } 3238 3239 static void 3240 rmv_port_callback(void *arg) 3241 { 3242 int need_to_start = 0; 3243 int org_no_link_check = no_link_check; 3244 portid_t port_id = (intptr_t)arg; 3245 struct rte_device *dev; 3246 3247 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3248 3249 if (!test_done && port_is_forwarding(port_id)) { 3250 need_to_start = 1; 3251 stop_packet_forwarding(); 3252 } 3253 no_link_check = 1; 3254 stop_port(port_id); 3255 no_link_check = org_no_link_check; 3256 3257 /* Save rte_device pointer before closing ethdev port */ 3258 dev = rte_eth_devices[port_id].device; 3259 close_port(port_id); 3260 detach_device(dev); /* might be already removed or have more ports */ 3261 3262 if (need_to_start) 3263 start_packet_forwarding(0); 3264 } 3265 3266 /* This function is used by the interrupt thread */ 3267 static int 3268 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3269 void *ret_param) 3270 { 3271 RTE_SET_USED(param); 3272 RTE_SET_USED(ret_param); 3273 3274 if (type >= RTE_ETH_EVENT_MAX) { 3275 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3276 port_id, __func__, type); 3277 fflush(stderr); 3278 } else if (event_print_mask & (UINT32_C(1) << type)) { 3279 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3280 eth_event_desc[type]); 3281 fflush(stdout); 3282 } 3283 3284 switch (type) { 3285 case RTE_ETH_EVENT_NEW: 3286 ports[port_id].need_setup = 1; 3287 ports[port_id].port_status = RTE_PORT_HANDLING; 3288 break; 3289 case RTE_ETH_EVENT_INTR_RMV: 3290 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3291 break; 3292 if (rte_eal_alarm_set(100000, 3293 rmv_port_callback, (void *)(intptr_t)port_id)) 3294 fprintf(stderr, "Could not set up deferred device removal\n"); 3295 break; 3296 case RTE_ETH_EVENT_DESTROY: 3297 ports[port_id].port_status = RTE_PORT_CLOSED; 3298 printf("Port %u is closed\n", port_id); 3299 break; 3300 default: 3301 break; 3302 } 3303 return 0; 3304 } 3305 3306 static int 3307 register_eth_event_callback(void) 3308 { 3309 int ret; 3310 enum rte_eth_event_type event; 3311 3312 for (event = RTE_ETH_EVENT_UNKNOWN; 3313 event < RTE_ETH_EVENT_MAX; event++) { 3314 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3315 event, 3316 eth_event_callback, 3317 NULL); 3318 if (ret != 0) { 3319 TESTPMD_LOG(ERR, "Failed to register callback for " 3320 "%s event\n", eth_event_desc[event]); 3321 return -1; 3322 } 3323 } 3324 3325 return 0; 3326 } 3327 3328 /* This function is used by the interrupt thread */ 3329 static void 3330 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3331 __rte_unused void *arg) 3332 { 3333 uint16_t port_id; 3334 int ret; 3335 3336 if (type >= RTE_DEV_EVENT_MAX) { 3337 fprintf(stderr, "%s called upon invalid event %d\n", 3338 __func__, type); 3339 fflush(stderr); 3340 } 3341 3342 switch (type) { 3343 case RTE_DEV_EVENT_REMOVE: 3344 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3345 device_name); 3346 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3347 if (ret) { 3348 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3349 device_name); 3350 return; 3351 } 3352 /* 3353 * Because the user's callback is invoked in eal interrupt 3354 * callback, the interrupt callback need to be finished before 3355 * it can be unregistered when detaching device. So finish 3356 * callback soon and use a deferred removal to detach device 3357 * is need. It is a workaround, once the device detaching be 3358 * moved into the eal in the future, the deferred removal could 3359 * be deleted. 3360 */ 3361 if (rte_eal_alarm_set(100000, 3362 rmv_port_callback, (void *)(intptr_t)port_id)) 3363 RTE_LOG(ERR, EAL, 3364 "Could not set up deferred device removal\n"); 3365 break; 3366 case RTE_DEV_EVENT_ADD: 3367 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3368 device_name); 3369 /* TODO: After finish kernel driver binding, 3370 * begin to attach port. 3371 */ 3372 break; 3373 default: 3374 break; 3375 } 3376 } 3377 3378 static int 3379 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3380 { 3381 uint16_t i; 3382 int diag; 3383 uint8_t mapping_found = 0; 3384 3385 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3386 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3387 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3388 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3389 tx_queue_stats_mappings[i].queue_id, 3390 tx_queue_stats_mappings[i].stats_counter_id); 3391 if (diag != 0) 3392 return diag; 3393 mapping_found = 1; 3394 } 3395 } 3396 if (mapping_found) 3397 port->tx_queue_stats_mapping_enabled = 1; 3398 return 0; 3399 } 3400 3401 static int 3402 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3403 { 3404 uint16_t i; 3405 int diag; 3406 uint8_t mapping_found = 0; 3407 3408 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3409 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3410 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3411 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3412 rx_queue_stats_mappings[i].queue_id, 3413 rx_queue_stats_mappings[i].stats_counter_id); 3414 if (diag != 0) 3415 return diag; 3416 mapping_found = 1; 3417 } 3418 } 3419 if (mapping_found) 3420 port->rx_queue_stats_mapping_enabled = 1; 3421 return 0; 3422 } 3423 3424 static void 3425 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3426 { 3427 int diag = 0; 3428 3429 diag = set_tx_queue_stats_mapping_registers(pi, port); 3430 if (diag != 0) { 3431 if (diag == -ENOTSUP) { 3432 port->tx_queue_stats_mapping_enabled = 0; 3433 printf("TX queue stats mapping not supported port id=%d\n", pi); 3434 } 3435 else 3436 rte_exit(EXIT_FAILURE, 3437 "set_tx_queue_stats_mapping_registers " 3438 "failed for port id=%d diag=%d\n", 3439 pi, diag); 3440 } 3441 3442 diag = set_rx_queue_stats_mapping_registers(pi, port); 3443 if (diag != 0) { 3444 if (diag == -ENOTSUP) { 3445 port->rx_queue_stats_mapping_enabled = 0; 3446 printf("RX queue stats mapping not supported port id=%d\n", pi); 3447 } 3448 else 3449 rte_exit(EXIT_FAILURE, 3450 "set_rx_queue_stats_mapping_registers " 3451 "failed for port id=%d diag=%d\n", 3452 pi, diag); 3453 } 3454 } 3455 3456 static void 3457 rxtx_port_config(struct rte_port *port) 3458 { 3459 uint16_t qid; 3460 uint64_t offloads; 3461 3462 for (qid = 0; qid < nb_rxq; qid++) { 3463 offloads = port->rx_conf[qid].offloads; 3464 port->rx_conf[qid] = port->dev_info.default_rxconf; 3465 if (offloads != 0) 3466 port->rx_conf[qid].offloads = offloads; 3467 3468 /* Check if any Rx parameters have been passed */ 3469 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3470 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3471 3472 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3473 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3474 3475 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3476 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3477 3478 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3479 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3480 3481 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3482 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3483 3484 port->nb_rx_desc[qid] = nb_rxd; 3485 } 3486 3487 for (qid = 0; qid < nb_txq; qid++) { 3488 offloads = port->tx_conf[qid].offloads; 3489 port->tx_conf[qid] = port->dev_info.default_txconf; 3490 if (offloads != 0) 3491 port->tx_conf[qid].offloads = offloads; 3492 3493 /* Check if any Tx parameters have been passed */ 3494 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3495 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3496 3497 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3498 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3499 3500 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3501 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3502 3503 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3504 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3505 3506 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3507 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3508 3509 port->nb_tx_desc[qid] = nb_txd; 3510 } 3511 } 3512 3513 void 3514 init_port_config(void) 3515 { 3516 portid_t pid; 3517 struct rte_port *port; 3518 int ret; 3519 3520 RTE_ETH_FOREACH_DEV(pid) { 3521 port = &ports[pid]; 3522 port->dev_conf.fdir_conf = fdir_conf; 3523 3524 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3525 if (ret != 0) 3526 return; 3527 3528 if (nb_rxq > 1) { 3529 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3530 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3531 rss_hf & port->dev_info.flow_type_rss_offloads; 3532 } else { 3533 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3534 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3535 } 3536 3537 if (port->dcb_flag == 0) { 3538 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3539 port->dev_conf.rxmode.mq_mode = 3540 (enum rte_eth_rx_mq_mode) 3541 (rx_mq_mode & ETH_MQ_RX_RSS); 3542 else 3543 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3544 } 3545 3546 rxtx_port_config(port); 3547 3548 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3549 if (ret != 0) 3550 return; 3551 3552 map_port_queue_stats_mapping_registers(pid, port); 3553 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3554 rte_pmd_ixgbe_bypass_init(pid); 3555 #endif 3556 3557 if (lsc_interrupt && 3558 (rte_eth_devices[pid].data->dev_flags & 3559 RTE_ETH_DEV_INTR_LSC)) 3560 port->dev_conf.intr_conf.lsc = 1; 3561 if (rmv_interrupt && 3562 (rte_eth_devices[pid].data->dev_flags & 3563 RTE_ETH_DEV_INTR_RMV)) 3564 port->dev_conf.intr_conf.rmv = 1; 3565 } 3566 } 3567 3568 void set_port_slave_flag(portid_t slave_pid) 3569 { 3570 struct rte_port *port; 3571 3572 port = &ports[slave_pid]; 3573 port->slave_flag = 1; 3574 } 3575 3576 void clear_port_slave_flag(portid_t slave_pid) 3577 { 3578 struct rte_port *port; 3579 3580 port = &ports[slave_pid]; 3581 port->slave_flag = 0; 3582 } 3583 3584 uint8_t port_is_bonding_slave(portid_t slave_pid) 3585 { 3586 struct rte_port *port; 3587 3588 port = &ports[slave_pid]; 3589 if ((rte_eth_devices[slave_pid].data->dev_flags & 3590 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3591 return 1; 3592 return 0; 3593 } 3594 3595 const uint16_t vlan_tags[] = { 3596 0, 1, 2, 3, 4, 5, 6, 7, 3597 8, 9, 10, 11, 12, 13, 14, 15, 3598 16, 17, 18, 19, 20, 21, 22, 23, 3599 24, 25, 26, 27, 28, 29, 30, 31 3600 }; 3601 3602 static int 3603 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3604 enum dcb_mode_enable dcb_mode, 3605 enum rte_eth_nb_tcs num_tcs, 3606 uint8_t pfc_en) 3607 { 3608 uint8_t i; 3609 int32_t rc; 3610 struct rte_eth_rss_conf rss_conf; 3611 3612 /* 3613 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3614 * given above, and the number of traffic classes available for use. 3615 */ 3616 if (dcb_mode == DCB_VT_ENABLED) { 3617 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3618 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3619 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3620 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3621 3622 /* VMDQ+DCB RX and TX configurations */ 3623 vmdq_rx_conf->enable_default_pool = 0; 3624 vmdq_rx_conf->default_pool = 0; 3625 vmdq_rx_conf->nb_queue_pools = 3626 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3627 vmdq_tx_conf->nb_queue_pools = 3628 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3629 3630 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3631 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3632 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3633 vmdq_rx_conf->pool_map[i].pools = 3634 1 << (i % vmdq_rx_conf->nb_queue_pools); 3635 } 3636 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3637 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3638 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3639 } 3640 3641 /* set DCB mode of RX and TX of multiple queues */ 3642 eth_conf->rxmode.mq_mode = 3643 (enum rte_eth_rx_mq_mode) 3644 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3645 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3646 } else { 3647 struct rte_eth_dcb_rx_conf *rx_conf = 3648 ð_conf->rx_adv_conf.dcb_rx_conf; 3649 struct rte_eth_dcb_tx_conf *tx_conf = 3650 ð_conf->tx_adv_conf.dcb_tx_conf; 3651 3652 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3653 3654 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3655 if (rc != 0) 3656 return rc; 3657 3658 rx_conf->nb_tcs = num_tcs; 3659 tx_conf->nb_tcs = num_tcs; 3660 3661 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3662 rx_conf->dcb_tc[i] = i % num_tcs; 3663 tx_conf->dcb_tc[i] = i % num_tcs; 3664 } 3665 3666 eth_conf->rxmode.mq_mode = 3667 (enum rte_eth_rx_mq_mode) 3668 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3669 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3670 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3671 } 3672 3673 if (pfc_en) 3674 eth_conf->dcb_capability_en = 3675 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3676 else 3677 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3678 3679 return 0; 3680 } 3681 3682 int 3683 init_port_dcb_config(portid_t pid, 3684 enum dcb_mode_enable dcb_mode, 3685 enum rte_eth_nb_tcs num_tcs, 3686 uint8_t pfc_en) 3687 { 3688 struct rte_eth_conf port_conf; 3689 struct rte_port *rte_port; 3690 int retval; 3691 uint16_t i; 3692 3693 rte_port = &ports[pid]; 3694 3695 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3696 /* Enter DCB configuration status */ 3697 dcb_config = 1; 3698 3699 port_conf.rxmode = rte_port->dev_conf.rxmode; 3700 port_conf.txmode = rte_port->dev_conf.txmode; 3701 3702 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3703 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3704 if (retval < 0) 3705 return retval; 3706 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3707 3708 /* re-configure the device . */ 3709 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3710 if (retval < 0) 3711 return retval; 3712 3713 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3714 if (retval != 0) 3715 return retval; 3716 3717 /* If dev_info.vmdq_pool_base is greater than 0, 3718 * the queue id of vmdq pools is started after pf queues. 3719 */ 3720 if (dcb_mode == DCB_VT_ENABLED && 3721 rte_port->dev_info.vmdq_pool_base > 0) { 3722 printf("VMDQ_DCB multi-queue mode is nonsensical" 3723 " for port %d.", pid); 3724 return -1; 3725 } 3726 3727 /* Assume the ports in testpmd have the same dcb capability 3728 * and has the same number of rxq and txq in dcb mode 3729 */ 3730 if (dcb_mode == DCB_VT_ENABLED) { 3731 if (rte_port->dev_info.max_vfs > 0) { 3732 nb_rxq = rte_port->dev_info.nb_rx_queues; 3733 nb_txq = rte_port->dev_info.nb_tx_queues; 3734 } else { 3735 nb_rxq = rte_port->dev_info.max_rx_queues; 3736 nb_txq = rte_port->dev_info.max_tx_queues; 3737 } 3738 } else { 3739 /*if vt is disabled, use all pf queues */ 3740 if (rte_port->dev_info.vmdq_pool_base == 0) { 3741 nb_rxq = rte_port->dev_info.max_rx_queues; 3742 nb_txq = rte_port->dev_info.max_tx_queues; 3743 } else { 3744 nb_rxq = (queueid_t)num_tcs; 3745 nb_txq = (queueid_t)num_tcs; 3746 3747 } 3748 } 3749 rx_free_thresh = 64; 3750 3751 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3752 3753 rxtx_port_config(rte_port); 3754 /* VLAN filter */ 3755 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3756 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3757 rx_vft_set(pid, vlan_tags[i], 1); 3758 3759 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3760 if (retval != 0) 3761 return retval; 3762 3763 map_port_queue_stats_mapping_registers(pid, rte_port); 3764 3765 rte_port->dcb_flag = 1; 3766 3767 return 0; 3768 } 3769 3770 static void 3771 init_port(void) 3772 { 3773 int i; 3774 3775 /* Configuration of Ethernet ports. */ 3776 ports = rte_zmalloc("testpmd: ports", 3777 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3778 RTE_CACHE_LINE_SIZE); 3779 if (ports == NULL) { 3780 rte_exit(EXIT_FAILURE, 3781 "rte_zmalloc(%d struct rte_port) failed\n", 3782 RTE_MAX_ETHPORTS); 3783 } 3784 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3785 LIST_INIT(&ports[i].flow_tunnel_list); 3786 /* Initialize ports NUMA structures */ 3787 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3788 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3789 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3790 } 3791 3792 static void 3793 force_quit(void) 3794 { 3795 pmd_test_exit(); 3796 prompt_exit(); 3797 } 3798 3799 static void 3800 print_stats(void) 3801 { 3802 uint8_t i; 3803 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3804 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3805 3806 /* Clear screen and move to top left */ 3807 printf("%s%s", clr, top_left); 3808 3809 printf("\nPort statistics ===================================="); 3810 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3811 nic_stats_display(fwd_ports_ids[i]); 3812 3813 fflush(stdout); 3814 } 3815 3816 static void 3817 signal_handler(int signum) 3818 { 3819 if (signum == SIGINT || signum == SIGTERM) { 3820 printf("\nSignal %d received, preparing to exit...\n", 3821 signum); 3822 #ifdef RTE_LIB_PDUMP 3823 /* uninitialize packet capture framework */ 3824 rte_pdump_uninit(); 3825 #endif 3826 #ifdef RTE_LIB_LATENCYSTATS 3827 if (latencystats_enabled != 0) 3828 rte_latencystats_uninit(); 3829 #endif 3830 force_quit(); 3831 /* Set flag to indicate the force termination. */ 3832 f_quit = 1; 3833 /* exit with the expected status */ 3834 signal(signum, SIG_DFL); 3835 kill(getpid(), signum); 3836 } 3837 } 3838 3839 int 3840 main(int argc, char** argv) 3841 { 3842 int diag; 3843 portid_t port_id; 3844 uint16_t count; 3845 int ret; 3846 3847 signal(SIGINT, signal_handler); 3848 signal(SIGTERM, signal_handler); 3849 3850 testpmd_logtype = rte_log_register("testpmd"); 3851 if (testpmd_logtype < 0) 3852 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3853 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3854 3855 diag = rte_eal_init(argc, argv); 3856 if (diag < 0) 3857 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3858 rte_strerror(rte_errno)); 3859 3860 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3861 rte_exit(EXIT_FAILURE, 3862 "Secondary process type not supported.\n"); 3863 3864 ret = register_eth_event_callback(); 3865 if (ret != 0) 3866 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3867 3868 #ifdef RTE_LIB_PDUMP 3869 /* initialize packet capture framework */ 3870 rte_pdump_init(); 3871 #endif 3872 3873 count = 0; 3874 RTE_ETH_FOREACH_DEV(port_id) { 3875 ports_ids[count] = port_id; 3876 count++; 3877 } 3878 nb_ports = (portid_t) count; 3879 if (nb_ports == 0) 3880 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3881 3882 /* allocate port structures, and init them */ 3883 init_port(); 3884 3885 set_def_fwd_config(); 3886 if (nb_lcores == 0) 3887 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3888 "Check the core mask argument\n"); 3889 3890 /* Bitrate/latency stats disabled by default */ 3891 #ifdef RTE_LIB_BITRATESTATS 3892 bitrate_enabled = 0; 3893 #endif 3894 #ifdef RTE_LIB_LATENCYSTATS 3895 latencystats_enabled = 0; 3896 #endif 3897 3898 /* on FreeBSD, mlockall() is disabled by default */ 3899 #ifdef RTE_EXEC_ENV_FREEBSD 3900 do_mlockall = 0; 3901 #else 3902 do_mlockall = 1; 3903 #endif 3904 3905 argc -= diag; 3906 argv += diag; 3907 if (argc > 1) 3908 launch_args_parse(argc, argv); 3909 3910 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3911 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3912 strerror(errno)); 3913 } 3914 3915 if (tx_first && interactive) 3916 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3917 "interactive mode.\n"); 3918 3919 if (tx_first && lsc_interrupt) { 3920 printf("Warning: lsc_interrupt needs to be off when " 3921 " using tx_first. Disabling.\n"); 3922 lsc_interrupt = 0; 3923 } 3924 3925 if (!nb_rxq && !nb_txq) 3926 printf("Warning: Either rx or tx queues should be non-zero\n"); 3927 3928 if (nb_rxq > 1 && nb_rxq > nb_txq) 3929 printf("Warning: nb_rxq=%d enables RSS configuration, " 3930 "but nb_txq=%d will prevent to fully test it.\n", 3931 nb_rxq, nb_txq); 3932 3933 init_config(); 3934 3935 if (hot_plug) { 3936 ret = rte_dev_hotplug_handle_enable(); 3937 if (ret) { 3938 RTE_LOG(ERR, EAL, 3939 "fail to enable hotplug handling."); 3940 return -1; 3941 } 3942 3943 ret = rte_dev_event_monitor_start(); 3944 if (ret) { 3945 RTE_LOG(ERR, EAL, 3946 "fail to start device event monitoring."); 3947 return -1; 3948 } 3949 3950 ret = rte_dev_event_callback_register(NULL, 3951 dev_event_callback, NULL); 3952 if (ret) { 3953 RTE_LOG(ERR, EAL, 3954 "fail to register device event callback\n"); 3955 return -1; 3956 } 3957 } 3958 3959 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3960 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3961 3962 /* set all ports to promiscuous mode by default */ 3963 RTE_ETH_FOREACH_DEV(port_id) { 3964 ret = rte_eth_promiscuous_enable(port_id); 3965 if (ret != 0) 3966 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3967 port_id, rte_strerror(-ret)); 3968 } 3969 3970 /* Init metrics library */ 3971 rte_metrics_init(rte_socket_id()); 3972 3973 #ifdef RTE_LIB_LATENCYSTATS 3974 if (latencystats_enabled != 0) { 3975 int ret = rte_latencystats_init(1, NULL); 3976 if (ret) 3977 printf("Warning: latencystats init()" 3978 " returned error %d\n", ret); 3979 printf("Latencystats running on lcore %d\n", 3980 latencystats_lcore_id); 3981 } 3982 #endif 3983 3984 /* Setup bitrate stats */ 3985 #ifdef RTE_LIB_BITRATESTATS 3986 if (bitrate_enabled != 0) { 3987 bitrate_data = rte_stats_bitrate_create(); 3988 if (bitrate_data == NULL) 3989 rte_exit(EXIT_FAILURE, 3990 "Could not allocate bitrate data.\n"); 3991 rte_stats_bitrate_reg(bitrate_data); 3992 } 3993 #endif 3994 3995 #ifdef RTE_LIB_CMDLINE 3996 if (strlen(cmdline_filename) != 0) 3997 cmdline_read_from_file(cmdline_filename); 3998 3999 if (interactive == 1) { 4000 if (auto_start) { 4001 printf("Start automatic packet forwarding\n"); 4002 start_packet_forwarding(0); 4003 } 4004 prompt(); 4005 pmd_test_exit(); 4006 } else 4007 #endif 4008 { 4009 char c; 4010 int rc; 4011 4012 f_quit = 0; 4013 4014 printf("No commandline core given, start packet forwarding\n"); 4015 start_packet_forwarding(tx_first); 4016 if (stats_period != 0) { 4017 uint64_t prev_time = 0, cur_time, diff_time = 0; 4018 uint64_t timer_period; 4019 4020 /* Convert to number of cycles */ 4021 timer_period = stats_period * rte_get_timer_hz(); 4022 4023 while (f_quit == 0) { 4024 cur_time = rte_get_timer_cycles(); 4025 diff_time += cur_time - prev_time; 4026 4027 if (diff_time >= timer_period) { 4028 print_stats(); 4029 /* Reset the timer */ 4030 diff_time = 0; 4031 } 4032 /* Sleep to avoid unnecessary checks */ 4033 prev_time = cur_time; 4034 sleep(1); 4035 } 4036 } 4037 4038 printf("Press enter to exit\n"); 4039 rc = read(0, &c, 1); 4040 pmd_test_exit(); 4041 if (rc < 0) 4042 return 1; 4043 } 4044 4045 ret = rte_eal_cleanup(); 4046 if (ret != 0) 4047 rte_exit(EXIT_FAILURE, 4048 "EAL cleanup failed: %s\n", strerror(-ret)); 4049 4050 return EXIT_SUCCESS; 4051 } 4052