1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIB_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIB_LATENCYSTATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use main core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 220 221 /* 222 * Configuration of packet segments used by the "txonly" processing engine. 223 */ 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 226 TXONLY_DEF_PACKET_LEN, 227 }; 228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 229 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 231 /**< Split policy for packets to TX. */ 232 233 uint8_t txonly_multi_flow; 234 /**< Whether multiple flows are generated in TXONLY mode. */ 235 236 uint32_t tx_pkt_times_inter; 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 238 239 uint32_t tx_pkt_times_intra; 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 241 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 243 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 244 245 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 246 uint8_t dcb_config = 0; 247 248 /* Whether the dcb is in testing status */ 249 uint8_t dcb_test = 0; 250 251 /* 252 * Configurable number of RX/TX queues. 253 */ 254 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 255 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 256 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 257 258 /* 259 * Configurable number of RX/TX ring descriptors. 260 * Defaults are supplied by drivers via ethdev. 261 */ 262 #define RTE_TEST_RX_DESC_DEFAULT 0 263 #define RTE_TEST_TX_DESC_DEFAULT 0 264 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 265 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 266 267 #define RTE_PMD_PARAM_UNSET -1 268 /* 269 * Configurable values of RX and TX ring threshold registers. 270 */ 271 272 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 273 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 274 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 275 276 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 277 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 278 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 279 280 /* 281 * Configurable value of RX free threshold. 282 */ 283 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 284 285 /* 286 * Configurable value of RX drop enable. 287 */ 288 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 289 290 /* 291 * Configurable value of TX free threshold. 292 */ 293 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 294 295 /* 296 * Configurable value of TX RS bit threshold. 297 */ 298 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 299 300 /* 301 * Configurable value of buffered packets before sending. 302 */ 303 uint16_t noisy_tx_sw_bufsz; 304 305 /* 306 * Configurable value of packet buffer timeout. 307 */ 308 uint16_t noisy_tx_sw_buf_flush_time; 309 310 /* 311 * Configurable value for size of VNF internal memory area 312 * used for simulating noisy neighbour behaviour 313 */ 314 uint64_t noisy_lkup_mem_sz; 315 316 /* 317 * Configurable value of number of random writes done in 318 * VNF simulation memory area. 319 */ 320 uint64_t noisy_lkup_num_writes; 321 322 /* 323 * Configurable value of number of random reads done in 324 * VNF simulation memory area. 325 */ 326 uint64_t noisy_lkup_num_reads; 327 328 /* 329 * Configurable value of number of random reads/writes done in 330 * VNF simulation memory area. 331 */ 332 uint64_t noisy_lkup_num_reads_writes; 333 334 /* 335 * Receive Side Scaling (RSS) configuration. 336 */ 337 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 338 339 /* 340 * Port topology configuration 341 */ 342 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 343 344 /* 345 * Avoids to flush all the RX streams before starts forwarding. 346 */ 347 uint8_t no_flush_rx = 0; /* flush by default */ 348 349 /* 350 * Flow API isolated mode. 351 */ 352 uint8_t flow_isolate_all; 353 354 /* 355 * Avoids to check link status when starting/stopping a port. 356 */ 357 uint8_t no_link_check = 0; /* check by default */ 358 359 /* 360 * Don't automatically start all ports in interactive mode. 361 */ 362 uint8_t no_device_start = 0; 363 364 /* 365 * Enable link status change notification 366 */ 367 uint8_t lsc_interrupt = 1; /* enabled by default */ 368 369 /* 370 * Enable device removal notification. 371 */ 372 uint8_t rmv_interrupt = 1; /* enabled by default */ 373 374 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 375 376 /* After attach, port setup is called on event or by iterator */ 377 bool setup_on_probe_event = true; 378 379 /* Clear ptypes on port initialization. */ 380 uint8_t clear_ptypes = true; 381 382 /* Hairpin ports configuration mode. */ 383 uint16_t hairpin_mode; 384 385 /* Pretty printing of ethdev events */ 386 static const char * const eth_event_desc[] = { 387 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 388 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 389 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 390 [RTE_ETH_EVENT_INTR_RESET] = "reset", 391 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 392 [RTE_ETH_EVENT_IPSEC] = "IPsec", 393 [RTE_ETH_EVENT_MACSEC] = "MACsec", 394 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 395 [RTE_ETH_EVENT_NEW] = "device probed", 396 [RTE_ETH_EVENT_DESTROY] = "device released", 397 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 398 [RTE_ETH_EVENT_MAX] = NULL, 399 }; 400 401 /* 402 * Display or mask ether events 403 * Default to all events except VF_MBOX 404 */ 405 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 406 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 407 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 408 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 409 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 410 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 411 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 412 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 413 /* 414 * Decide if all memory are locked for performance. 415 */ 416 int do_mlockall = 0; 417 418 /* 419 * NIC bypass mode configuration options. 420 */ 421 422 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 423 /* The NIC bypass watchdog timeout. */ 424 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 425 #endif 426 427 428 #ifdef RTE_LIB_LATENCYSTATS 429 430 /* 431 * Set when latency stats is enabled in the commandline 432 */ 433 uint8_t latencystats_enabled; 434 435 /* 436 * Lcore ID to serive latency statistics. 437 */ 438 lcoreid_t latencystats_lcore_id = -1; 439 440 #endif 441 442 /* 443 * Ethernet device configuration. 444 */ 445 struct rte_eth_rxmode rx_mode = { 446 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 447 /**< Default maximum frame length. */ 448 }; 449 450 struct rte_eth_txmode tx_mode = { 451 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 452 }; 453 454 struct rte_fdir_conf fdir_conf = { 455 .mode = RTE_FDIR_MODE_NONE, 456 .pballoc = RTE_FDIR_PBALLOC_64K, 457 .status = RTE_FDIR_REPORT_STATUS, 458 .mask = { 459 .vlan_tci_mask = 0xFFEF, 460 .ipv4_mask = { 461 .src_ip = 0xFFFFFFFF, 462 .dst_ip = 0xFFFFFFFF, 463 }, 464 .ipv6_mask = { 465 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 466 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 467 }, 468 .src_port_mask = 0xFFFF, 469 .dst_port_mask = 0xFFFF, 470 .mac_addr_byte_mask = 0xFF, 471 .tunnel_type_mask = 1, 472 .tunnel_id_mask = 0xFFFFFFFF, 473 }, 474 .drop_queue = 127, 475 }; 476 477 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 478 479 /* 480 * Display zero values by default for xstats 481 */ 482 uint8_t xstats_hide_zero; 483 484 /* 485 * Measure of CPU cycles disabled by default 486 */ 487 uint8_t record_core_cycles; 488 489 /* 490 * Display of RX and TX bursts disabled by default 491 */ 492 uint8_t record_burst_stats; 493 494 unsigned int num_sockets = 0; 495 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 496 497 #ifdef RTE_LIB_BITRATESTATS 498 /* Bitrate statistics */ 499 struct rte_stats_bitrates *bitrate_data; 500 lcoreid_t bitrate_lcore_id; 501 uint8_t bitrate_enabled; 502 #endif 503 504 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 505 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 506 507 /* 508 * hexadecimal bitmask of RX mq mode can be enabled. 509 */ 510 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 511 512 /* Forward function declarations */ 513 static void setup_attached_port(portid_t pi); 514 static void check_all_ports_link_status(uint32_t port_mask); 515 static int eth_event_callback(portid_t port_id, 516 enum rte_eth_event_type type, 517 void *param, void *ret_param); 518 static void dev_event_callback(const char *device_name, 519 enum rte_dev_event_type type, 520 void *param); 521 522 /* 523 * Check if all the ports are started. 524 * If yes, return positive value. If not, return zero. 525 */ 526 static int all_ports_started(void); 527 528 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 529 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 530 531 /* Holds the registered mbuf dynamic flags names. */ 532 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 533 534 /* 535 * Helper function to check if socket is already discovered. 536 * If yes, return positive value. If not, return zero. 537 */ 538 int 539 new_socket_id(unsigned int socket_id) 540 { 541 unsigned int i; 542 543 for (i = 0; i < num_sockets; i++) { 544 if (socket_ids[i] == socket_id) 545 return 0; 546 } 547 return 1; 548 } 549 550 /* 551 * Setup default configuration. 552 */ 553 static void 554 set_default_fwd_lcores_config(void) 555 { 556 unsigned int i; 557 unsigned int nb_lc; 558 unsigned int sock_num; 559 560 nb_lc = 0; 561 for (i = 0; i < RTE_MAX_LCORE; i++) { 562 if (!rte_lcore_is_enabled(i)) 563 continue; 564 sock_num = rte_lcore_to_socket_id(i); 565 if (new_socket_id(sock_num)) { 566 if (num_sockets >= RTE_MAX_NUMA_NODES) { 567 rte_exit(EXIT_FAILURE, 568 "Total sockets greater than %u\n", 569 RTE_MAX_NUMA_NODES); 570 } 571 socket_ids[num_sockets++] = sock_num; 572 } 573 if (i == rte_get_main_lcore()) 574 continue; 575 fwd_lcores_cpuids[nb_lc++] = i; 576 } 577 nb_lcores = (lcoreid_t) nb_lc; 578 nb_cfg_lcores = nb_lcores; 579 nb_fwd_lcores = 1; 580 } 581 582 static void 583 set_def_peer_eth_addrs(void) 584 { 585 portid_t i; 586 587 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 588 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 589 peer_eth_addrs[i].addr_bytes[5] = i; 590 } 591 } 592 593 static void 594 set_default_fwd_ports_config(void) 595 { 596 portid_t pt_id; 597 int i = 0; 598 599 RTE_ETH_FOREACH_DEV(pt_id) { 600 fwd_ports_ids[i++] = pt_id; 601 602 /* Update sockets info according to the attached device */ 603 int socket_id = rte_eth_dev_socket_id(pt_id); 604 if (socket_id >= 0 && new_socket_id(socket_id)) { 605 if (num_sockets >= RTE_MAX_NUMA_NODES) { 606 rte_exit(EXIT_FAILURE, 607 "Total sockets greater than %u\n", 608 RTE_MAX_NUMA_NODES); 609 } 610 socket_ids[num_sockets++] = socket_id; 611 } 612 } 613 614 nb_cfg_ports = nb_ports; 615 nb_fwd_ports = nb_ports; 616 } 617 618 void 619 set_def_fwd_config(void) 620 { 621 set_default_fwd_lcores_config(); 622 set_def_peer_eth_addrs(); 623 set_default_fwd_ports_config(); 624 } 625 626 /* extremely pessimistic estimation of memory required to create a mempool */ 627 static int 628 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 629 { 630 unsigned int n_pages, mbuf_per_pg, leftover; 631 uint64_t total_mem, mbuf_mem, obj_sz; 632 633 /* there is no good way to predict how much space the mempool will 634 * occupy because it will allocate chunks on the fly, and some of those 635 * will come from default DPDK memory while some will come from our 636 * external memory, so just assume 128MB will be enough for everyone. 637 */ 638 uint64_t hdr_mem = 128 << 20; 639 640 /* account for possible non-contiguousness */ 641 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 642 if (obj_sz > pgsz) { 643 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 644 return -1; 645 } 646 647 mbuf_per_pg = pgsz / obj_sz; 648 leftover = (nb_mbufs % mbuf_per_pg) > 0; 649 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 650 651 mbuf_mem = n_pages * pgsz; 652 653 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 654 655 if (total_mem > SIZE_MAX) { 656 TESTPMD_LOG(ERR, "Memory size too big\n"); 657 return -1; 658 } 659 *out = (size_t)total_mem; 660 661 return 0; 662 } 663 664 static int 665 pagesz_flags(uint64_t page_sz) 666 { 667 /* as per mmap() manpage, all page sizes are log2 of page size 668 * shifted by MAP_HUGE_SHIFT 669 */ 670 int log2 = rte_log2_u64(page_sz); 671 672 return (log2 << HUGE_SHIFT); 673 } 674 675 static void * 676 alloc_mem(size_t memsz, size_t pgsz, bool huge) 677 { 678 void *addr; 679 int flags; 680 681 /* allocate anonymous hugepages */ 682 flags = MAP_ANONYMOUS | MAP_PRIVATE; 683 if (huge) 684 flags |= HUGE_FLAG | pagesz_flags(pgsz); 685 686 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 687 if (addr == MAP_FAILED) 688 return NULL; 689 690 return addr; 691 } 692 693 struct extmem_param { 694 void *addr; 695 size_t len; 696 size_t pgsz; 697 rte_iova_t *iova_table; 698 unsigned int iova_table_len; 699 }; 700 701 static int 702 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 703 bool huge) 704 { 705 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 706 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 707 unsigned int cur_page, n_pages, pgsz_idx; 708 size_t mem_sz, cur_pgsz; 709 rte_iova_t *iovas = NULL; 710 void *addr; 711 int ret; 712 713 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 714 /* skip anything that is too big */ 715 if (pgsizes[pgsz_idx] > SIZE_MAX) 716 continue; 717 718 cur_pgsz = pgsizes[pgsz_idx]; 719 720 /* if we were told not to allocate hugepages, override */ 721 if (!huge) 722 cur_pgsz = sysconf(_SC_PAGESIZE); 723 724 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 725 if (ret < 0) { 726 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 727 return -1; 728 } 729 730 /* allocate our memory */ 731 addr = alloc_mem(mem_sz, cur_pgsz, huge); 732 733 /* if we couldn't allocate memory with a specified page size, 734 * that doesn't mean we can't do it with other page sizes, so 735 * try another one. 736 */ 737 if (addr == NULL) 738 continue; 739 740 /* store IOVA addresses for every page in this memory area */ 741 n_pages = mem_sz / cur_pgsz; 742 743 iovas = malloc(sizeof(*iovas) * n_pages); 744 745 if (iovas == NULL) { 746 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 747 goto fail; 748 } 749 /* lock memory if it's not huge pages */ 750 if (!huge) 751 mlock(addr, mem_sz); 752 753 /* populate IOVA addresses */ 754 for (cur_page = 0; cur_page < n_pages; cur_page++) { 755 rte_iova_t iova; 756 size_t offset; 757 void *cur; 758 759 offset = cur_pgsz * cur_page; 760 cur = RTE_PTR_ADD(addr, offset); 761 762 /* touch the page before getting its IOVA */ 763 *(volatile char *)cur = 0; 764 765 iova = rte_mem_virt2iova(cur); 766 767 iovas[cur_page] = iova; 768 } 769 770 break; 771 } 772 /* if we couldn't allocate anything */ 773 if (iovas == NULL) 774 return -1; 775 776 param->addr = addr; 777 param->len = mem_sz; 778 param->pgsz = cur_pgsz; 779 param->iova_table = iovas; 780 param->iova_table_len = n_pages; 781 782 return 0; 783 fail: 784 if (iovas) 785 free(iovas); 786 if (addr) 787 munmap(addr, mem_sz); 788 789 return -1; 790 } 791 792 static int 793 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 794 { 795 struct extmem_param param; 796 int socket_id, ret; 797 798 memset(¶m, 0, sizeof(param)); 799 800 /* check if our heap exists */ 801 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 802 if (socket_id < 0) { 803 /* create our heap */ 804 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 805 if (ret < 0) { 806 TESTPMD_LOG(ERR, "Cannot create heap\n"); 807 return -1; 808 } 809 } 810 811 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 812 if (ret < 0) { 813 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 814 return -1; 815 } 816 817 /* we now have a valid memory area, so add it to heap */ 818 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 819 param.addr, param.len, param.iova_table, 820 param.iova_table_len, param.pgsz); 821 822 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 823 824 /* not needed any more */ 825 free(param.iova_table); 826 827 if (ret < 0) { 828 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 829 munmap(param.addr, param.len); 830 return -1; 831 } 832 833 /* success */ 834 835 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 836 param.len >> 20); 837 838 return 0; 839 } 840 static void 841 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 842 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 843 { 844 uint16_t pid = 0; 845 int ret; 846 847 RTE_ETH_FOREACH_DEV(pid) { 848 struct rte_eth_dev *dev = 849 &rte_eth_devices[pid]; 850 851 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 852 memhdr->len); 853 if (ret) { 854 TESTPMD_LOG(DEBUG, 855 "unable to DMA unmap addr 0x%p " 856 "for device %s\n", 857 memhdr->addr, dev->data->name); 858 } 859 } 860 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 861 if (ret) { 862 TESTPMD_LOG(DEBUG, 863 "unable to un-register addr 0x%p\n", memhdr->addr); 864 } 865 } 866 867 static void 868 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 869 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 870 { 871 uint16_t pid = 0; 872 size_t page_size = sysconf(_SC_PAGESIZE); 873 int ret; 874 875 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 876 page_size); 877 if (ret) { 878 TESTPMD_LOG(DEBUG, 879 "unable to register addr 0x%p\n", memhdr->addr); 880 return; 881 } 882 RTE_ETH_FOREACH_DEV(pid) { 883 struct rte_eth_dev *dev = 884 &rte_eth_devices[pid]; 885 886 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 887 memhdr->len); 888 if (ret) { 889 TESTPMD_LOG(DEBUG, 890 "unable to DMA map addr 0x%p " 891 "for device %s\n", 892 memhdr->addr, dev->data->name); 893 } 894 } 895 } 896 897 static unsigned int 898 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 899 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 900 { 901 struct rte_pktmbuf_extmem *xmem; 902 unsigned int ext_num, zone_num, elt_num; 903 uint16_t elt_size; 904 905 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 906 elt_num = EXTBUF_ZONE_SIZE / elt_size; 907 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 908 909 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 910 if (xmem == NULL) { 911 TESTPMD_LOG(ERR, "Cannot allocate memory for " 912 "external buffer descriptors\n"); 913 *ext_mem = NULL; 914 return 0; 915 } 916 for (ext_num = 0; ext_num < zone_num; ext_num++) { 917 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 918 const struct rte_memzone *mz; 919 char mz_name[RTE_MEMZONE_NAMESIZE]; 920 int ret; 921 922 ret = snprintf(mz_name, sizeof(mz_name), 923 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 924 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 925 errno = ENAMETOOLONG; 926 ext_num = 0; 927 break; 928 } 929 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 930 socket_id, 931 RTE_MEMZONE_IOVA_CONTIG | 932 RTE_MEMZONE_1GB | 933 RTE_MEMZONE_SIZE_HINT_ONLY, 934 EXTBUF_ZONE_SIZE); 935 if (mz == NULL) { 936 /* 937 * The caller exits on external buffer creation 938 * error, so there is no need to free memzones. 939 */ 940 errno = ENOMEM; 941 ext_num = 0; 942 break; 943 } 944 xseg->buf_ptr = mz->addr; 945 xseg->buf_iova = mz->iova; 946 xseg->buf_len = EXTBUF_ZONE_SIZE; 947 xseg->elt_size = elt_size; 948 } 949 if (ext_num == 0 && xmem != NULL) { 950 free(xmem); 951 xmem = NULL; 952 } 953 *ext_mem = xmem; 954 return ext_num; 955 } 956 957 /* 958 * Configuration initialisation done once at init time. 959 */ 960 static struct rte_mempool * 961 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 962 unsigned int socket_id, uint16_t size_idx) 963 { 964 char pool_name[RTE_MEMPOOL_NAMESIZE]; 965 struct rte_mempool *rte_mp = NULL; 966 uint32_t mb_size; 967 968 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 969 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 970 971 TESTPMD_LOG(INFO, 972 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 973 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 974 975 switch (mp_alloc_type) { 976 case MP_ALLOC_NATIVE: 977 { 978 /* wrapper to rte_mempool_create() */ 979 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 980 rte_mbuf_best_mempool_ops()); 981 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 982 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 983 break; 984 } 985 case MP_ALLOC_ANON: 986 { 987 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 988 mb_size, (unsigned int) mb_mempool_cache, 989 sizeof(struct rte_pktmbuf_pool_private), 990 socket_id, mempool_flags); 991 if (rte_mp == NULL) 992 goto err; 993 994 if (rte_mempool_populate_anon(rte_mp) == 0) { 995 rte_mempool_free(rte_mp); 996 rte_mp = NULL; 997 goto err; 998 } 999 rte_pktmbuf_pool_init(rte_mp, NULL); 1000 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1001 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1002 break; 1003 } 1004 case MP_ALLOC_XMEM: 1005 case MP_ALLOC_XMEM_HUGE: 1006 { 1007 int heap_socket; 1008 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1009 1010 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1011 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1012 1013 heap_socket = 1014 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1015 if (heap_socket < 0) 1016 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1017 1018 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1019 rte_mbuf_best_mempool_ops()); 1020 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1021 mb_mempool_cache, 0, mbuf_seg_size, 1022 heap_socket); 1023 break; 1024 } 1025 case MP_ALLOC_XBUF: 1026 { 1027 struct rte_pktmbuf_extmem *ext_mem; 1028 unsigned int ext_num; 1029 1030 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1031 socket_id, pool_name, &ext_mem); 1032 if (ext_num == 0) 1033 rte_exit(EXIT_FAILURE, 1034 "Can't create pinned data buffers\n"); 1035 1036 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1037 rte_mbuf_best_mempool_ops()); 1038 rte_mp = rte_pktmbuf_pool_create_extbuf 1039 (pool_name, nb_mbuf, mb_mempool_cache, 1040 0, mbuf_seg_size, socket_id, 1041 ext_mem, ext_num); 1042 free(ext_mem); 1043 break; 1044 } 1045 default: 1046 { 1047 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1048 } 1049 } 1050 1051 err: 1052 if (rte_mp == NULL) { 1053 rte_exit(EXIT_FAILURE, 1054 "Creation of mbuf pool for socket %u failed: %s\n", 1055 socket_id, rte_strerror(rte_errno)); 1056 } else if (verbose_level > 0) { 1057 rte_mempool_dump(stdout, rte_mp); 1058 } 1059 return rte_mp; 1060 } 1061 1062 /* 1063 * Check given socket id is valid or not with NUMA mode, 1064 * if valid, return 0, else return -1 1065 */ 1066 static int 1067 check_socket_id(const unsigned int socket_id) 1068 { 1069 static int warning_once = 0; 1070 1071 if (new_socket_id(socket_id)) { 1072 if (!warning_once && numa_support) 1073 printf("Warning: NUMA should be configured manually by" 1074 " using --port-numa-config and" 1075 " --ring-numa-config parameters along with" 1076 " --numa.\n"); 1077 warning_once = 1; 1078 return -1; 1079 } 1080 return 0; 1081 } 1082 1083 /* 1084 * Get the allowed maximum number of RX queues. 1085 * *pid return the port id which has minimal value of 1086 * max_rx_queues in all ports. 1087 */ 1088 queueid_t 1089 get_allowed_max_nb_rxq(portid_t *pid) 1090 { 1091 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1092 bool max_rxq_valid = false; 1093 portid_t pi; 1094 struct rte_eth_dev_info dev_info; 1095 1096 RTE_ETH_FOREACH_DEV(pi) { 1097 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1098 continue; 1099 1100 max_rxq_valid = true; 1101 if (dev_info.max_rx_queues < allowed_max_rxq) { 1102 allowed_max_rxq = dev_info.max_rx_queues; 1103 *pid = pi; 1104 } 1105 } 1106 return max_rxq_valid ? allowed_max_rxq : 0; 1107 } 1108 1109 /* 1110 * Check input rxq is valid or not. 1111 * If input rxq is not greater than any of maximum number 1112 * of RX queues of all ports, it is valid. 1113 * if valid, return 0, else return -1 1114 */ 1115 int 1116 check_nb_rxq(queueid_t rxq) 1117 { 1118 queueid_t allowed_max_rxq; 1119 portid_t pid = 0; 1120 1121 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1122 if (rxq > allowed_max_rxq) { 1123 printf("Fail: input rxq (%u) can't be greater " 1124 "than max_rx_queues (%u) of port %u\n", 1125 rxq, 1126 allowed_max_rxq, 1127 pid); 1128 return -1; 1129 } 1130 return 0; 1131 } 1132 1133 /* 1134 * Get the allowed maximum number of TX queues. 1135 * *pid return the port id which has minimal value of 1136 * max_tx_queues in all ports. 1137 */ 1138 queueid_t 1139 get_allowed_max_nb_txq(portid_t *pid) 1140 { 1141 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1142 bool max_txq_valid = false; 1143 portid_t pi; 1144 struct rte_eth_dev_info dev_info; 1145 1146 RTE_ETH_FOREACH_DEV(pi) { 1147 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1148 continue; 1149 1150 max_txq_valid = true; 1151 if (dev_info.max_tx_queues < allowed_max_txq) { 1152 allowed_max_txq = dev_info.max_tx_queues; 1153 *pid = pi; 1154 } 1155 } 1156 return max_txq_valid ? allowed_max_txq : 0; 1157 } 1158 1159 /* 1160 * Check input txq is valid or not. 1161 * If input txq is not greater than any of maximum number 1162 * of TX queues of all ports, it is valid. 1163 * if valid, return 0, else return -1 1164 */ 1165 int 1166 check_nb_txq(queueid_t txq) 1167 { 1168 queueid_t allowed_max_txq; 1169 portid_t pid = 0; 1170 1171 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1172 if (txq > allowed_max_txq) { 1173 printf("Fail: input txq (%u) can't be greater " 1174 "than max_tx_queues (%u) of port %u\n", 1175 txq, 1176 allowed_max_txq, 1177 pid); 1178 return -1; 1179 } 1180 return 0; 1181 } 1182 1183 /* 1184 * Get the allowed maximum number of RXDs of every rx queue. 1185 * *pid return the port id which has minimal value of 1186 * max_rxd in all queues of all ports. 1187 */ 1188 static uint16_t 1189 get_allowed_max_nb_rxd(portid_t *pid) 1190 { 1191 uint16_t allowed_max_rxd = UINT16_MAX; 1192 portid_t pi; 1193 struct rte_eth_dev_info dev_info; 1194 1195 RTE_ETH_FOREACH_DEV(pi) { 1196 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1197 continue; 1198 1199 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1200 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1201 *pid = pi; 1202 } 1203 } 1204 return allowed_max_rxd; 1205 } 1206 1207 /* 1208 * Get the allowed minimal number of RXDs of every rx queue. 1209 * *pid return the port id which has minimal value of 1210 * min_rxd in all queues of all ports. 1211 */ 1212 static uint16_t 1213 get_allowed_min_nb_rxd(portid_t *pid) 1214 { 1215 uint16_t allowed_min_rxd = 0; 1216 portid_t pi; 1217 struct rte_eth_dev_info dev_info; 1218 1219 RTE_ETH_FOREACH_DEV(pi) { 1220 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1221 continue; 1222 1223 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1224 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1225 *pid = pi; 1226 } 1227 } 1228 1229 return allowed_min_rxd; 1230 } 1231 1232 /* 1233 * Check input rxd is valid or not. 1234 * If input rxd is not greater than any of maximum number 1235 * of RXDs of every Rx queues and is not less than any of 1236 * minimal number of RXDs of every Rx queues, it is valid. 1237 * if valid, return 0, else return -1 1238 */ 1239 int 1240 check_nb_rxd(queueid_t rxd) 1241 { 1242 uint16_t allowed_max_rxd; 1243 uint16_t allowed_min_rxd; 1244 portid_t pid = 0; 1245 1246 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1247 if (rxd > allowed_max_rxd) { 1248 printf("Fail: input rxd (%u) can't be greater " 1249 "than max_rxds (%u) of port %u\n", 1250 rxd, 1251 allowed_max_rxd, 1252 pid); 1253 return -1; 1254 } 1255 1256 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1257 if (rxd < allowed_min_rxd) { 1258 printf("Fail: input rxd (%u) can't be less " 1259 "than min_rxds (%u) of port %u\n", 1260 rxd, 1261 allowed_min_rxd, 1262 pid); 1263 return -1; 1264 } 1265 1266 return 0; 1267 } 1268 1269 /* 1270 * Get the allowed maximum number of TXDs of every rx queues. 1271 * *pid return the port id which has minimal value of 1272 * max_txd in every tx queue. 1273 */ 1274 static uint16_t 1275 get_allowed_max_nb_txd(portid_t *pid) 1276 { 1277 uint16_t allowed_max_txd = UINT16_MAX; 1278 portid_t pi; 1279 struct rte_eth_dev_info dev_info; 1280 1281 RTE_ETH_FOREACH_DEV(pi) { 1282 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1283 continue; 1284 1285 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1286 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1287 *pid = pi; 1288 } 1289 } 1290 return allowed_max_txd; 1291 } 1292 1293 /* 1294 * Get the allowed maximum number of TXDs of every tx queues. 1295 * *pid return the port id which has minimal value of 1296 * min_txd in every tx queue. 1297 */ 1298 static uint16_t 1299 get_allowed_min_nb_txd(portid_t *pid) 1300 { 1301 uint16_t allowed_min_txd = 0; 1302 portid_t pi; 1303 struct rte_eth_dev_info dev_info; 1304 1305 RTE_ETH_FOREACH_DEV(pi) { 1306 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1307 continue; 1308 1309 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1310 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1311 *pid = pi; 1312 } 1313 } 1314 1315 return allowed_min_txd; 1316 } 1317 1318 /* 1319 * Check input txd is valid or not. 1320 * If input txd is not greater than any of maximum number 1321 * of TXDs of every Rx queues, it is valid. 1322 * if valid, return 0, else return -1 1323 */ 1324 int 1325 check_nb_txd(queueid_t txd) 1326 { 1327 uint16_t allowed_max_txd; 1328 uint16_t allowed_min_txd; 1329 portid_t pid = 0; 1330 1331 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1332 if (txd > allowed_max_txd) { 1333 printf("Fail: input txd (%u) can't be greater " 1334 "than max_txds (%u) of port %u\n", 1335 txd, 1336 allowed_max_txd, 1337 pid); 1338 return -1; 1339 } 1340 1341 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1342 if (txd < allowed_min_txd) { 1343 printf("Fail: input txd (%u) can't be less " 1344 "than min_txds (%u) of port %u\n", 1345 txd, 1346 allowed_min_txd, 1347 pid); 1348 return -1; 1349 } 1350 return 0; 1351 } 1352 1353 1354 /* 1355 * Get the allowed maximum number of hairpin queues. 1356 * *pid return the port id which has minimal value of 1357 * max_hairpin_queues in all ports. 1358 */ 1359 queueid_t 1360 get_allowed_max_nb_hairpinq(portid_t *pid) 1361 { 1362 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1363 portid_t pi; 1364 struct rte_eth_hairpin_cap cap; 1365 1366 RTE_ETH_FOREACH_DEV(pi) { 1367 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1368 *pid = pi; 1369 return 0; 1370 } 1371 if (cap.max_nb_queues < allowed_max_hairpinq) { 1372 allowed_max_hairpinq = cap.max_nb_queues; 1373 *pid = pi; 1374 } 1375 } 1376 return allowed_max_hairpinq; 1377 } 1378 1379 /* 1380 * Check input hairpin is valid or not. 1381 * If input hairpin is not greater than any of maximum number 1382 * of hairpin queues of all ports, it is valid. 1383 * if valid, return 0, else return -1 1384 */ 1385 int 1386 check_nb_hairpinq(queueid_t hairpinq) 1387 { 1388 queueid_t allowed_max_hairpinq; 1389 portid_t pid = 0; 1390 1391 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1392 if (hairpinq > allowed_max_hairpinq) { 1393 printf("Fail: input hairpin (%u) can't be greater " 1394 "than max_hairpin_queues (%u) of port %u\n", 1395 hairpinq, allowed_max_hairpinq, pid); 1396 return -1; 1397 } 1398 return 0; 1399 } 1400 1401 static void 1402 init_config(void) 1403 { 1404 portid_t pid; 1405 struct rte_port *port; 1406 struct rte_mempool *mbp; 1407 unsigned int nb_mbuf_per_pool; 1408 lcoreid_t lc_id; 1409 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1410 struct rte_gro_param gro_param; 1411 uint32_t gso_types; 1412 uint16_t data_size; 1413 bool warning = 0; 1414 int k; 1415 int ret; 1416 1417 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1418 1419 /* Configuration of logical cores. */ 1420 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1421 sizeof(struct fwd_lcore *) * nb_lcores, 1422 RTE_CACHE_LINE_SIZE); 1423 if (fwd_lcores == NULL) { 1424 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1425 "failed\n", nb_lcores); 1426 } 1427 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1428 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1429 sizeof(struct fwd_lcore), 1430 RTE_CACHE_LINE_SIZE); 1431 if (fwd_lcores[lc_id] == NULL) { 1432 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1433 "failed\n"); 1434 } 1435 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1436 } 1437 1438 RTE_ETH_FOREACH_DEV(pid) { 1439 port = &ports[pid]; 1440 /* Apply default TxRx configuration for all ports */ 1441 port->dev_conf.txmode = tx_mode; 1442 port->dev_conf.rxmode = rx_mode; 1443 1444 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1445 if (ret != 0) 1446 rte_exit(EXIT_FAILURE, 1447 "rte_eth_dev_info_get() failed\n"); 1448 1449 if (!(port->dev_info.tx_offload_capa & 1450 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1451 port->dev_conf.txmode.offloads &= 1452 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1453 if (numa_support) { 1454 if (port_numa[pid] != NUMA_NO_CONFIG) 1455 port_per_socket[port_numa[pid]]++; 1456 else { 1457 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1458 1459 /* 1460 * if socket_id is invalid, 1461 * set to the first available socket. 1462 */ 1463 if (check_socket_id(socket_id) < 0) 1464 socket_id = socket_ids[0]; 1465 port_per_socket[socket_id]++; 1466 } 1467 } 1468 1469 /* Apply Rx offloads configuration */ 1470 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1471 port->rx_conf[k].offloads = 1472 port->dev_conf.rxmode.offloads; 1473 /* Apply Tx offloads configuration */ 1474 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1475 port->tx_conf[k].offloads = 1476 port->dev_conf.txmode.offloads; 1477 1478 /* set flag to initialize port/queue */ 1479 port->need_reconfig = 1; 1480 port->need_reconfig_queues = 1; 1481 port->tx_metadata = 0; 1482 1483 /* Check for maximum number of segments per MTU. Accordingly 1484 * update the mbuf data size. 1485 */ 1486 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1487 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1488 data_size = rx_mode.max_rx_pkt_len / 1489 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1490 1491 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1492 mbuf_data_size[0]) { 1493 mbuf_data_size[0] = data_size + 1494 RTE_PKTMBUF_HEADROOM; 1495 warning = 1; 1496 } 1497 } 1498 } 1499 1500 if (warning) 1501 TESTPMD_LOG(WARNING, 1502 "Configured mbuf size of the first segment %hu\n", 1503 mbuf_data_size[0]); 1504 /* 1505 * Create pools of mbuf. 1506 * If NUMA support is disabled, create a single pool of mbuf in 1507 * socket 0 memory by default. 1508 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1509 * 1510 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1511 * nb_txd can be configured at run time. 1512 */ 1513 if (param_total_num_mbufs) 1514 nb_mbuf_per_pool = param_total_num_mbufs; 1515 else { 1516 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1517 (nb_lcores * mb_mempool_cache) + 1518 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1519 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1520 } 1521 1522 if (numa_support) { 1523 uint8_t i, j; 1524 1525 for (i = 0; i < num_sockets; i++) 1526 for (j = 0; j < mbuf_data_size_n; j++) 1527 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1528 mbuf_pool_create(mbuf_data_size[j], 1529 nb_mbuf_per_pool, 1530 socket_ids[i], j); 1531 } else { 1532 uint8_t i; 1533 1534 for (i = 0; i < mbuf_data_size_n; i++) 1535 mempools[i] = mbuf_pool_create 1536 (mbuf_data_size[i], 1537 nb_mbuf_per_pool, 1538 socket_num == UMA_NO_CONFIG ? 1539 0 : socket_num, i); 1540 } 1541 1542 init_port_config(); 1543 1544 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1545 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1546 /* 1547 * Records which Mbuf pool to use by each logical core, if needed. 1548 */ 1549 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1550 mbp = mbuf_pool_find( 1551 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1552 1553 if (mbp == NULL) 1554 mbp = mbuf_pool_find(0, 0); 1555 fwd_lcores[lc_id]->mbp = mbp; 1556 /* initialize GSO context */ 1557 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1558 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1559 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1560 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1561 RTE_ETHER_CRC_LEN; 1562 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1563 } 1564 1565 /* Configuration of packet forwarding streams. */ 1566 if (init_fwd_streams() < 0) 1567 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1568 1569 fwd_config_setup(); 1570 1571 /* create a gro context for each lcore */ 1572 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1573 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1574 gro_param.max_item_per_flow = MAX_PKT_BURST; 1575 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1576 gro_param.socket_id = rte_lcore_to_socket_id( 1577 fwd_lcores_cpuids[lc_id]); 1578 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1579 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1580 rte_exit(EXIT_FAILURE, 1581 "rte_gro_ctx_create() failed\n"); 1582 } 1583 } 1584 } 1585 1586 1587 void 1588 reconfig(portid_t new_port_id, unsigned socket_id) 1589 { 1590 struct rte_port *port; 1591 int ret; 1592 1593 /* Reconfiguration of Ethernet ports. */ 1594 port = &ports[new_port_id]; 1595 1596 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1597 if (ret != 0) 1598 return; 1599 1600 /* set flag to initialize port/queue */ 1601 port->need_reconfig = 1; 1602 port->need_reconfig_queues = 1; 1603 port->socket_id = socket_id; 1604 1605 init_port_config(); 1606 } 1607 1608 1609 int 1610 init_fwd_streams(void) 1611 { 1612 portid_t pid; 1613 struct rte_port *port; 1614 streamid_t sm_id, nb_fwd_streams_new; 1615 queueid_t q; 1616 1617 /* set socket id according to numa or not */ 1618 RTE_ETH_FOREACH_DEV(pid) { 1619 port = &ports[pid]; 1620 if (nb_rxq > port->dev_info.max_rx_queues) { 1621 printf("Fail: nb_rxq(%d) is greater than " 1622 "max_rx_queues(%d)\n", nb_rxq, 1623 port->dev_info.max_rx_queues); 1624 return -1; 1625 } 1626 if (nb_txq > port->dev_info.max_tx_queues) { 1627 printf("Fail: nb_txq(%d) is greater than " 1628 "max_tx_queues(%d)\n", nb_txq, 1629 port->dev_info.max_tx_queues); 1630 return -1; 1631 } 1632 if (numa_support) { 1633 if (port_numa[pid] != NUMA_NO_CONFIG) 1634 port->socket_id = port_numa[pid]; 1635 else { 1636 port->socket_id = rte_eth_dev_socket_id(pid); 1637 1638 /* 1639 * if socket_id is invalid, 1640 * set to the first available socket. 1641 */ 1642 if (check_socket_id(port->socket_id) < 0) 1643 port->socket_id = socket_ids[0]; 1644 } 1645 } 1646 else { 1647 if (socket_num == UMA_NO_CONFIG) 1648 port->socket_id = 0; 1649 else 1650 port->socket_id = socket_num; 1651 } 1652 } 1653 1654 q = RTE_MAX(nb_rxq, nb_txq); 1655 if (q == 0) { 1656 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1657 return -1; 1658 } 1659 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1660 if (nb_fwd_streams_new == nb_fwd_streams) 1661 return 0; 1662 /* clear the old */ 1663 if (fwd_streams != NULL) { 1664 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1665 if (fwd_streams[sm_id] == NULL) 1666 continue; 1667 rte_free(fwd_streams[sm_id]); 1668 fwd_streams[sm_id] = NULL; 1669 } 1670 rte_free(fwd_streams); 1671 fwd_streams = NULL; 1672 } 1673 1674 /* init new */ 1675 nb_fwd_streams = nb_fwd_streams_new; 1676 if (nb_fwd_streams) { 1677 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1678 sizeof(struct fwd_stream *) * nb_fwd_streams, 1679 RTE_CACHE_LINE_SIZE); 1680 if (fwd_streams == NULL) 1681 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1682 " (struct fwd_stream *)) failed\n", 1683 nb_fwd_streams); 1684 1685 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1686 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1687 " struct fwd_stream", sizeof(struct fwd_stream), 1688 RTE_CACHE_LINE_SIZE); 1689 if (fwd_streams[sm_id] == NULL) 1690 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1691 "(struct fwd_stream) failed\n"); 1692 } 1693 } 1694 1695 return 0; 1696 } 1697 1698 static void 1699 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1700 { 1701 uint64_t total_burst, sburst; 1702 uint64_t nb_burst; 1703 uint64_t burst_stats[4]; 1704 uint16_t pktnb_stats[4]; 1705 uint16_t nb_pkt; 1706 int burst_percent[4], sburstp; 1707 int i; 1708 1709 /* 1710 * First compute the total number of packet bursts and the 1711 * two highest numbers of bursts of the same number of packets. 1712 */ 1713 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1714 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1715 1716 /* Show stats for 0 burst size always */ 1717 total_burst = pbs->pkt_burst_spread[0]; 1718 burst_stats[0] = pbs->pkt_burst_spread[0]; 1719 pktnb_stats[0] = 0; 1720 1721 /* Find the next 2 burst sizes with highest occurrences. */ 1722 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1723 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1724 1725 if (nb_burst == 0) 1726 continue; 1727 1728 total_burst += nb_burst; 1729 1730 if (nb_burst > burst_stats[1]) { 1731 burst_stats[2] = burst_stats[1]; 1732 pktnb_stats[2] = pktnb_stats[1]; 1733 burst_stats[1] = nb_burst; 1734 pktnb_stats[1] = nb_pkt; 1735 } else if (nb_burst > burst_stats[2]) { 1736 burst_stats[2] = nb_burst; 1737 pktnb_stats[2] = nb_pkt; 1738 } 1739 } 1740 if (total_burst == 0) 1741 return; 1742 1743 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1744 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1745 if (i == 3) { 1746 printf("%d%% of other]\n", 100 - sburstp); 1747 return; 1748 } 1749 1750 sburst += burst_stats[i]; 1751 if (sburst == total_burst) { 1752 printf("%d%% of %d pkts]\n", 1753 100 - sburstp, (int) pktnb_stats[i]); 1754 return; 1755 } 1756 1757 burst_percent[i] = 1758 (double)burst_stats[i] / total_burst * 100; 1759 printf("%d%% of %d pkts + ", 1760 burst_percent[i], (int) pktnb_stats[i]); 1761 sburstp += burst_percent[i]; 1762 } 1763 } 1764 1765 static void 1766 fwd_stream_stats_display(streamid_t stream_id) 1767 { 1768 struct fwd_stream *fs; 1769 static const char *fwd_top_stats_border = "-------"; 1770 1771 fs = fwd_streams[stream_id]; 1772 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1773 (fs->fwd_dropped == 0)) 1774 return; 1775 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1776 "TX Port=%2d/Queue=%2d %s\n", 1777 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1778 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1779 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1780 " TX-dropped: %-14"PRIu64, 1781 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1782 1783 /* if checksum mode */ 1784 if (cur_fwd_eng == &csum_fwd_engine) { 1785 printf(" RX- bad IP checksum: %-14"PRIu64 1786 " Rx- bad L4 checksum: %-14"PRIu64 1787 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1788 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1789 fs->rx_bad_outer_l4_csum); 1790 } else { 1791 printf("\n"); 1792 } 1793 1794 if (record_burst_stats) { 1795 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1796 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1797 } 1798 } 1799 1800 void 1801 fwd_stats_display(void) 1802 { 1803 static const char *fwd_stats_border = "----------------------"; 1804 static const char *acc_stats_border = "+++++++++++++++"; 1805 struct { 1806 struct fwd_stream *rx_stream; 1807 struct fwd_stream *tx_stream; 1808 uint64_t tx_dropped; 1809 uint64_t rx_bad_ip_csum; 1810 uint64_t rx_bad_l4_csum; 1811 uint64_t rx_bad_outer_l4_csum; 1812 } ports_stats[RTE_MAX_ETHPORTS]; 1813 uint64_t total_rx_dropped = 0; 1814 uint64_t total_tx_dropped = 0; 1815 uint64_t total_rx_nombuf = 0; 1816 struct rte_eth_stats stats; 1817 uint64_t fwd_cycles = 0; 1818 uint64_t total_recv = 0; 1819 uint64_t total_xmit = 0; 1820 struct rte_port *port; 1821 streamid_t sm_id; 1822 portid_t pt_id; 1823 int i; 1824 1825 memset(ports_stats, 0, sizeof(ports_stats)); 1826 1827 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1828 struct fwd_stream *fs = fwd_streams[sm_id]; 1829 1830 if (cur_fwd_config.nb_fwd_streams > 1831 cur_fwd_config.nb_fwd_ports) { 1832 fwd_stream_stats_display(sm_id); 1833 } else { 1834 ports_stats[fs->tx_port].tx_stream = fs; 1835 ports_stats[fs->rx_port].rx_stream = fs; 1836 } 1837 1838 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1839 1840 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1841 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1842 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1843 fs->rx_bad_outer_l4_csum; 1844 1845 if (record_core_cycles) 1846 fwd_cycles += fs->core_cycles; 1847 } 1848 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1849 pt_id = fwd_ports_ids[i]; 1850 port = &ports[pt_id]; 1851 1852 rte_eth_stats_get(pt_id, &stats); 1853 stats.ipackets -= port->stats.ipackets; 1854 stats.opackets -= port->stats.opackets; 1855 stats.ibytes -= port->stats.ibytes; 1856 stats.obytes -= port->stats.obytes; 1857 stats.imissed -= port->stats.imissed; 1858 stats.oerrors -= port->stats.oerrors; 1859 stats.rx_nombuf -= port->stats.rx_nombuf; 1860 1861 total_recv += stats.ipackets; 1862 total_xmit += stats.opackets; 1863 total_rx_dropped += stats.imissed; 1864 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1865 total_tx_dropped += stats.oerrors; 1866 total_rx_nombuf += stats.rx_nombuf; 1867 1868 printf("\n %s Forward statistics for port %-2d %s\n", 1869 fwd_stats_border, pt_id, fwd_stats_border); 1870 1871 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 1872 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 1873 stats.ipackets + stats.imissed); 1874 1875 if (cur_fwd_eng == &csum_fwd_engine) 1876 printf(" Bad-ipcsum: %-14"PRIu64 1877 " Bad-l4csum: %-14"PRIu64 1878 "Bad-outer-l4csum: %-14"PRIu64"\n", 1879 ports_stats[pt_id].rx_bad_ip_csum, 1880 ports_stats[pt_id].rx_bad_l4_csum, 1881 ports_stats[pt_id].rx_bad_outer_l4_csum); 1882 if (stats.ierrors + stats.rx_nombuf > 0) { 1883 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 1884 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 1885 } 1886 1887 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 1888 "TX-total: %-"PRIu64"\n", 1889 stats.opackets, ports_stats[pt_id].tx_dropped, 1890 stats.opackets + ports_stats[pt_id].tx_dropped); 1891 1892 if (record_burst_stats) { 1893 if (ports_stats[pt_id].rx_stream) 1894 pkt_burst_stats_display("RX", 1895 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1896 if (ports_stats[pt_id].tx_stream) 1897 pkt_burst_stats_display("TX", 1898 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1899 } 1900 1901 printf(" %s--------------------------------%s\n", 1902 fwd_stats_border, fwd_stats_border); 1903 } 1904 1905 printf("\n %s Accumulated forward statistics for all ports" 1906 "%s\n", 1907 acc_stats_border, acc_stats_border); 1908 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1909 "%-"PRIu64"\n" 1910 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1911 "%-"PRIu64"\n", 1912 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1913 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1914 if (total_rx_nombuf > 0) 1915 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1916 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1917 "%s\n", 1918 acc_stats_border, acc_stats_border); 1919 if (record_core_cycles) { 1920 #define CYC_PER_MHZ 1E6 1921 if (total_recv > 0 || total_xmit > 0) { 1922 uint64_t total_pkts = 0; 1923 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1924 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1925 total_pkts = total_xmit; 1926 else 1927 total_pkts = total_recv; 1928 1929 printf("\n CPU cycles/packet=%.2F (total cycles=" 1930 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1931 " MHz Clock\n", 1932 (double) fwd_cycles / total_pkts, 1933 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1934 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1935 } 1936 } 1937 } 1938 1939 void 1940 fwd_stats_reset(void) 1941 { 1942 streamid_t sm_id; 1943 portid_t pt_id; 1944 int i; 1945 1946 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1947 pt_id = fwd_ports_ids[i]; 1948 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1949 } 1950 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1951 struct fwd_stream *fs = fwd_streams[sm_id]; 1952 1953 fs->rx_packets = 0; 1954 fs->tx_packets = 0; 1955 fs->fwd_dropped = 0; 1956 fs->rx_bad_ip_csum = 0; 1957 fs->rx_bad_l4_csum = 0; 1958 fs->rx_bad_outer_l4_csum = 0; 1959 1960 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1961 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1962 fs->core_cycles = 0; 1963 } 1964 } 1965 1966 static void 1967 flush_fwd_rx_queues(void) 1968 { 1969 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1970 portid_t rxp; 1971 portid_t port_id; 1972 queueid_t rxq; 1973 uint16_t nb_rx; 1974 uint16_t i; 1975 uint8_t j; 1976 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1977 uint64_t timer_period; 1978 1979 /* convert to number of cycles */ 1980 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1981 1982 for (j = 0; j < 2; j++) { 1983 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1984 for (rxq = 0; rxq < nb_rxq; rxq++) { 1985 port_id = fwd_ports_ids[rxp]; 1986 /** 1987 * testpmd can stuck in the below do while loop 1988 * if rte_eth_rx_burst() always returns nonzero 1989 * packets. So timer is added to exit this loop 1990 * after 1sec timer expiry. 1991 */ 1992 prev_tsc = rte_rdtsc(); 1993 do { 1994 nb_rx = rte_eth_rx_burst(port_id, rxq, 1995 pkts_burst, MAX_PKT_BURST); 1996 for (i = 0; i < nb_rx; i++) 1997 rte_pktmbuf_free(pkts_burst[i]); 1998 1999 cur_tsc = rte_rdtsc(); 2000 diff_tsc = cur_tsc - prev_tsc; 2001 timer_tsc += diff_tsc; 2002 } while ((nb_rx > 0) && 2003 (timer_tsc < timer_period)); 2004 timer_tsc = 0; 2005 } 2006 } 2007 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2008 } 2009 } 2010 2011 static void 2012 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2013 { 2014 struct fwd_stream **fsm; 2015 streamid_t nb_fs; 2016 streamid_t sm_id; 2017 #ifdef RTE_LIB_BITRATESTATS 2018 uint64_t tics_per_1sec; 2019 uint64_t tics_datum; 2020 uint64_t tics_current; 2021 uint16_t i, cnt_ports; 2022 2023 cnt_ports = nb_ports; 2024 tics_datum = rte_rdtsc(); 2025 tics_per_1sec = rte_get_timer_hz(); 2026 #endif 2027 fsm = &fwd_streams[fc->stream_idx]; 2028 nb_fs = fc->stream_nb; 2029 do { 2030 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2031 (*pkt_fwd)(fsm[sm_id]); 2032 #ifdef RTE_LIB_BITRATESTATS 2033 if (bitrate_enabled != 0 && 2034 bitrate_lcore_id == rte_lcore_id()) { 2035 tics_current = rte_rdtsc(); 2036 if (tics_current - tics_datum >= tics_per_1sec) { 2037 /* Periodic bitrate calculation */ 2038 for (i = 0; i < cnt_ports; i++) 2039 rte_stats_bitrate_calc(bitrate_data, 2040 ports_ids[i]); 2041 tics_datum = tics_current; 2042 } 2043 } 2044 #endif 2045 #ifdef RTE_LIB_LATENCYSTATS 2046 if (latencystats_enabled != 0 && 2047 latencystats_lcore_id == rte_lcore_id()) 2048 rte_latencystats_update(); 2049 #endif 2050 2051 } while (! fc->stopped); 2052 } 2053 2054 static int 2055 start_pkt_forward_on_core(void *fwd_arg) 2056 { 2057 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2058 cur_fwd_config.fwd_eng->packet_fwd); 2059 return 0; 2060 } 2061 2062 /* 2063 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2064 * Used to start communication flows in network loopback test configurations. 2065 */ 2066 static int 2067 run_one_txonly_burst_on_core(void *fwd_arg) 2068 { 2069 struct fwd_lcore *fwd_lc; 2070 struct fwd_lcore tmp_lcore; 2071 2072 fwd_lc = (struct fwd_lcore *) fwd_arg; 2073 tmp_lcore = *fwd_lc; 2074 tmp_lcore.stopped = 1; 2075 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2076 return 0; 2077 } 2078 2079 /* 2080 * Launch packet forwarding: 2081 * - Setup per-port forwarding context. 2082 * - launch logical cores with their forwarding configuration. 2083 */ 2084 static void 2085 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2086 { 2087 port_fwd_begin_t port_fwd_begin; 2088 unsigned int i; 2089 unsigned int lc_id; 2090 int diag; 2091 2092 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2093 if (port_fwd_begin != NULL) { 2094 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2095 (*port_fwd_begin)(fwd_ports_ids[i]); 2096 } 2097 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2098 lc_id = fwd_lcores_cpuids[i]; 2099 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2100 fwd_lcores[i]->stopped = 0; 2101 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2102 fwd_lcores[i], lc_id); 2103 if (diag != 0) 2104 printf("launch lcore %u failed - diag=%d\n", 2105 lc_id, diag); 2106 } 2107 } 2108 } 2109 2110 /* 2111 * Launch packet forwarding configuration. 2112 */ 2113 void 2114 start_packet_forwarding(int with_tx_first) 2115 { 2116 port_fwd_begin_t port_fwd_begin; 2117 port_fwd_end_t port_fwd_end; 2118 struct rte_port *port; 2119 unsigned int i; 2120 portid_t pt_id; 2121 2122 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2123 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2124 2125 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2126 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2127 2128 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2129 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2130 (!nb_rxq || !nb_txq)) 2131 rte_exit(EXIT_FAILURE, 2132 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2133 cur_fwd_eng->fwd_mode_name); 2134 2135 if (all_ports_started() == 0) { 2136 printf("Not all ports were started\n"); 2137 return; 2138 } 2139 if (test_done == 0) { 2140 printf("Packet forwarding already started\n"); 2141 return; 2142 } 2143 2144 2145 if(dcb_test) { 2146 for (i = 0; i < nb_fwd_ports; i++) { 2147 pt_id = fwd_ports_ids[i]; 2148 port = &ports[pt_id]; 2149 if (!port->dcb_flag) { 2150 printf("In DCB mode, all forwarding ports must " 2151 "be configured in this mode.\n"); 2152 return; 2153 } 2154 } 2155 if (nb_fwd_lcores == 1) { 2156 printf("In DCB mode,the nb forwarding cores " 2157 "should be larger than 1.\n"); 2158 return; 2159 } 2160 } 2161 test_done = 0; 2162 2163 fwd_config_setup(); 2164 2165 if(!no_flush_rx) 2166 flush_fwd_rx_queues(); 2167 2168 pkt_fwd_config_display(&cur_fwd_config); 2169 rxtx_config_display(); 2170 2171 fwd_stats_reset(); 2172 if (with_tx_first) { 2173 port_fwd_begin = tx_only_engine.port_fwd_begin; 2174 if (port_fwd_begin != NULL) { 2175 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2176 (*port_fwd_begin)(fwd_ports_ids[i]); 2177 } 2178 while (with_tx_first--) { 2179 launch_packet_forwarding( 2180 run_one_txonly_burst_on_core); 2181 rte_eal_mp_wait_lcore(); 2182 } 2183 port_fwd_end = tx_only_engine.port_fwd_end; 2184 if (port_fwd_end != NULL) { 2185 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2186 (*port_fwd_end)(fwd_ports_ids[i]); 2187 } 2188 } 2189 launch_packet_forwarding(start_pkt_forward_on_core); 2190 } 2191 2192 void 2193 stop_packet_forwarding(void) 2194 { 2195 port_fwd_end_t port_fwd_end; 2196 lcoreid_t lc_id; 2197 portid_t pt_id; 2198 int i; 2199 2200 if (test_done) { 2201 printf("Packet forwarding not started\n"); 2202 return; 2203 } 2204 printf("Telling cores to stop..."); 2205 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2206 fwd_lcores[lc_id]->stopped = 1; 2207 printf("\nWaiting for lcores to finish...\n"); 2208 rte_eal_mp_wait_lcore(); 2209 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2210 if (port_fwd_end != NULL) { 2211 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2212 pt_id = fwd_ports_ids[i]; 2213 (*port_fwd_end)(pt_id); 2214 } 2215 } 2216 2217 fwd_stats_display(); 2218 2219 printf("\nDone.\n"); 2220 test_done = 1; 2221 } 2222 2223 void 2224 dev_set_link_up(portid_t pid) 2225 { 2226 if (rte_eth_dev_set_link_up(pid) < 0) 2227 printf("\nSet link up fail.\n"); 2228 } 2229 2230 void 2231 dev_set_link_down(portid_t pid) 2232 { 2233 if (rte_eth_dev_set_link_down(pid) < 0) 2234 printf("\nSet link down fail.\n"); 2235 } 2236 2237 static int 2238 all_ports_started(void) 2239 { 2240 portid_t pi; 2241 struct rte_port *port; 2242 2243 RTE_ETH_FOREACH_DEV(pi) { 2244 port = &ports[pi]; 2245 /* Check if there is a port which is not started */ 2246 if ((port->port_status != RTE_PORT_STARTED) && 2247 (port->slave_flag == 0)) 2248 return 0; 2249 } 2250 2251 /* No port is not started */ 2252 return 1; 2253 } 2254 2255 int 2256 port_is_stopped(portid_t port_id) 2257 { 2258 struct rte_port *port = &ports[port_id]; 2259 2260 if ((port->port_status != RTE_PORT_STOPPED) && 2261 (port->slave_flag == 0)) 2262 return 0; 2263 return 1; 2264 } 2265 2266 int 2267 all_ports_stopped(void) 2268 { 2269 portid_t pi; 2270 2271 RTE_ETH_FOREACH_DEV(pi) { 2272 if (!port_is_stopped(pi)) 2273 return 0; 2274 } 2275 2276 return 1; 2277 } 2278 2279 int 2280 port_is_started(portid_t port_id) 2281 { 2282 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2283 return 0; 2284 2285 if (ports[port_id].port_status != RTE_PORT_STARTED) 2286 return 0; 2287 2288 return 1; 2289 } 2290 2291 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2292 static int 2293 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2294 { 2295 queueid_t qi; 2296 struct rte_eth_hairpin_conf hairpin_conf = { 2297 .peer_count = 1, 2298 }; 2299 int i; 2300 int diag; 2301 struct rte_port *port = &ports[pi]; 2302 uint16_t peer_rx_port = pi; 2303 uint16_t peer_tx_port = pi; 2304 uint32_t manual = 1; 2305 uint32_t tx_exp = hairpin_mode & 0x10; 2306 2307 if (!(hairpin_mode & 0xf)) { 2308 peer_rx_port = pi; 2309 peer_tx_port = pi; 2310 manual = 0; 2311 } else if (hairpin_mode & 0x1) { 2312 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2313 RTE_ETH_DEV_NO_OWNER); 2314 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2315 peer_tx_port = rte_eth_find_next_owned_by(0, 2316 RTE_ETH_DEV_NO_OWNER); 2317 if (p_pi != RTE_MAX_ETHPORTS) { 2318 peer_rx_port = p_pi; 2319 } else { 2320 uint16_t next_pi; 2321 2322 /* Last port will be the peer RX port of the first. */ 2323 RTE_ETH_FOREACH_DEV(next_pi) 2324 peer_rx_port = next_pi; 2325 } 2326 manual = 1; 2327 } else if (hairpin_mode & 0x2) { 2328 if (cnt_pi & 0x1) { 2329 peer_rx_port = p_pi; 2330 } else { 2331 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2332 RTE_ETH_DEV_NO_OWNER); 2333 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2334 peer_rx_port = pi; 2335 } 2336 peer_tx_port = peer_rx_port; 2337 manual = 1; 2338 } 2339 2340 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2341 hairpin_conf.peers[0].port = peer_rx_port; 2342 hairpin_conf.peers[0].queue = i + nb_rxq; 2343 hairpin_conf.manual_bind = !!manual; 2344 hairpin_conf.tx_explicit = !!tx_exp; 2345 diag = rte_eth_tx_hairpin_queue_setup 2346 (pi, qi, nb_txd, &hairpin_conf); 2347 i++; 2348 if (diag == 0) 2349 continue; 2350 2351 /* Fail to setup rx queue, return */ 2352 if (rte_atomic16_cmpset(&(port->port_status), 2353 RTE_PORT_HANDLING, 2354 RTE_PORT_STOPPED) == 0) 2355 printf("Port %d can not be set back " 2356 "to stopped\n", pi); 2357 printf("Fail to configure port %d hairpin " 2358 "queues\n", pi); 2359 /* try to reconfigure queues next time */ 2360 port->need_reconfig_queues = 1; 2361 return -1; 2362 } 2363 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2364 hairpin_conf.peers[0].port = peer_tx_port; 2365 hairpin_conf.peers[0].queue = i + nb_txq; 2366 hairpin_conf.manual_bind = !!manual; 2367 hairpin_conf.tx_explicit = !!tx_exp; 2368 diag = rte_eth_rx_hairpin_queue_setup 2369 (pi, qi, nb_rxd, &hairpin_conf); 2370 i++; 2371 if (diag == 0) 2372 continue; 2373 2374 /* Fail to setup rx queue, return */ 2375 if (rte_atomic16_cmpset(&(port->port_status), 2376 RTE_PORT_HANDLING, 2377 RTE_PORT_STOPPED) == 0) 2378 printf("Port %d can not be set back " 2379 "to stopped\n", pi); 2380 printf("Fail to configure port %d hairpin " 2381 "queues\n", pi); 2382 /* try to reconfigure queues next time */ 2383 port->need_reconfig_queues = 1; 2384 return -1; 2385 } 2386 return 0; 2387 } 2388 2389 /* Configure the Rx with optional split. */ 2390 int 2391 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2392 uint16_t nb_rx_desc, unsigned int socket_id, 2393 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2394 { 2395 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2396 unsigned int i, mp_n; 2397 int ret; 2398 2399 if (rx_pkt_nb_segs <= 1 || 2400 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2401 rx_conf->rx_seg = NULL; 2402 rx_conf->rx_nseg = 0; 2403 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2404 nb_rx_desc, socket_id, 2405 rx_conf, mp); 2406 return ret; 2407 } 2408 for (i = 0; i < rx_pkt_nb_segs; i++) { 2409 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2410 struct rte_mempool *mpx; 2411 /* 2412 * Use last valid pool for the segments with number 2413 * exceeding the pool index. 2414 */ 2415 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2416 mpx = mbuf_pool_find(socket_id, mp_n); 2417 /* Handle zero as mbuf data buffer size. */ 2418 rx_seg->length = rx_pkt_seg_lengths[i] ? 2419 rx_pkt_seg_lengths[i] : 2420 mbuf_data_size[mp_n]; 2421 rx_seg->offset = i < rx_pkt_nb_offs ? 2422 rx_pkt_seg_offsets[i] : 0; 2423 rx_seg->mp = mpx ? mpx : mp; 2424 } 2425 rx_conf->rx_nseg = rx_pkt_nb_segs; 2426 rx_conf->rx_seg = rx_useg; 2427 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2428 socket_id, rx_conf, NULL); 2429 rx_conf->rx_seg = NULL; 2430 rx_conf->rx_nseg = 0; 2431 return ret; 2432 } 2433 2434 int 2435 start_port(portid_t pid) 2436 { 2437 int diag, need_check_link_status = -1; 2438 portid_t pi; 2439 portid_t p_pi = RTE_MAX_ETHPORTS; 2440 portid_t pl[RTE_MAX_ETHPORTS]; 2441 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2442 uint16_t cnt_pi = 0; 2443 uint16_t cfg_pi = 0; 2444 int peer_pi; 2445 queueid_t qi; 2446 struct rte_port *port; 2447 struct rte_ether_addr mac_addr; 2448 struct rte_eth_hairpin_cap cap; 2449 2450 if (port_id_is_invalid(pid, ENABLED_WARN)) 2451 return 0; 2452 2453 if(dcb_config) 2454 dcb_test = 1; 2455 RTE_ETH_FOREACH_DEV(pi) { 2456 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2457 continue; 2458 2459 need_check_link_status = 0; 2460 port = &ports[pi]; 2461 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2462 RTE_PORT_HANDLING) == 0) { 2463 printf("Port %d is now not stopped\n", pi); 2464 continue; 2465 } 2466 2467 if (port->need_reconfig > 0) { 2468 port->need_reconfig = 0; 2469 2470 if (flow_isolate_all) { 2471 int ret = port_flow_isolate(pi, 1); 2472 if (ret) { 2473 printf("Failed to apply isolated" 2474 " mode on port %d\n", pi); 2475 return -1; 2476 } 2477 } 2478 configure_rxtx_dump_callbacks(0); 2479 printf("Configuring Port %d (socket %u)\n", pi, 2480 port->socket_id); 2481 if (nb_hairpinq > 0 && 2482 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2483 printf("Port %d doesn't support hairpin " 2484 "queues\n", pi); 2485 return -1; 2486 } 2487 /* configure port */ 2488 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2489 nb_txq + nb_hairpinq, 2490 &(port->dev_conf)); 2491 if (diag != 0) { 2492 if (rte_atomic16_cmpset(&(port->port_status), 2493 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2494 printf("Port %d can not be set back " 2495 "to stopped\n", pi); 2496 printf("Fail to configure port %d\n", pi); 2497 /* try to reconfigure port next time */ 2498 port->need_reconfig = 1; 2499 return -1; 2500 } 2501 } 2502 if (port->need_reconfig_queues > 0) { 2503 port->need_reconfig_queues = 0; 2504 /* setup tx queues */ 2505 for (qi = 0; qi < nb_txq; qi++) { 2506 if ((numa_support) && 2507 (txring_numa[pi] != NUMA_NO_CONFIG)) 2508 diag = rte_eth_tx_queue_setup(pi, qi, 2509 port->nb_tx_desc[qi], 2510 txring_numa[pi], 2511 &(port->tx_conf[qi])); 2512 else 2513 diag = rte_eth_tx_queue_setup(pi, qi, 2514 port->nb_tx_desc[qi], 2515 port->socket_id, 2516 &(port->tx_conf[qi])); 2517 2518 if (diag == 0) 2519 continue; 2520 2521 /* Fail to setup tx queue, return */ 2522 if (rte_atomic16_cmpset(&(port->port_status), 2523 RTE_PORT_HANDLING, 2524 RTE_PORT_STOPPED) == 0) 2525 printf("Port %d can not be set back " 2526 "to stopped\n", pi); 2527 printf("Fail to configure port %d tx queues\n", 2528 pi); 2529 /* try to reconfigure queues next time */ 2530 port->need_reconfig_queues = 1; 2531 return -1; 2532 } 2533 for (qi = 0; qi < nb_rxq; qi++) { 2534 /* setup rx queues */ 2535 if ((numa_support) && 2536 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2537 struct rte_mempool * mp = 2538 mbuf_pool_find 2539 (rxring_numa[pi], 0); 2540 if (mp == NULL) { 2541 printf("Failed to setup RX queue:" 2542 "No mempool allocation" 2543 " on the socket %d\n", 2544 rxring_numa[pi]); 2545 return -1; 2546 } 2547 2548 diag = rx_queue_setup(pi, qi, 2549 port->nb_rx_desc[qi], 2550 rxring_numa[pi], 2551 &(port->rx_conf[qi]), 2552 mp); 2553 } else { 2554 struct rte_mempool *mp = 2555 mbuf_pool_find 2556 (port->socket_id, 0); 2557 if (mp == NULL) { 2558 printf("Failed to setup RX queue:" 2559 "No mempool allocation" 2560 " on the socket %d\n", 2561 port->socket_id); 2562 return -1; 2563 } 2564 diag = rx_queue_setup(pi, qi, 2565 port->nb_rx_desc[qi], 2566 port->socket_id, 2567 &(port->rx_conf[qi]), 2568 mp); 2569 } 2570 if (diag == 0) 2571 continue; 2572 2573 /* Fail to setup rx queue, return */ 2574 if (rte_atomic16_cmpset(&(port->port_status), 2575 RTE_PORT_HANDLING, 2576 RTE_PORT_STOPPED) == 0) 2577 printf("Port %d can not be set back " 2578 "to stopped\n", pi); 2579 printf("Fail to configure port %d rx queues\n", 2580 pi); 2581 /* try to reconfigure queues next time */ 2582 port->need_reconfig_queues = 1; 2583 return -1; 2584 } 2585 /* setup hairpin queues */ 2586 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2587 return -1; 2588 } 2589 configure_rxtx_dump_callbacks(verbose_level); 2590 if (clear_ptypes) { 2591 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2592 NULL, 0); 2593 if (diag < 0) 2594 printf( 2595 "Port %d: Failed to disable Ptype parsing\n", 2596 pi); 2597 } 2598 2599 p_pi = pi; 2600 cnt_pi++; 2601 2602 /* start port */ 2603 if (rte_eth_dev_start(pi) < 0) { 2604 printf("Fail to start port %d\n", pi); 2605 2606 /* Fail to setup rx queue, return */ 2607 if (rte_atomic16_cmpset(&(port->port_status), 2608 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2609 printf("Port %d can not be set back to " 2610 "stopped\n", pi); 2611 continue; 2612 } 2613 2614 if (rte_atomic16_cmpset(&(port->port_status), 2615 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2616 printf("Port %d can not be set into started\n", pi); 2617 2618 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2619 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2620 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2621 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2622 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2623 2624 /* at least one port started, need checking link status */ 2625 need_check_link_status = 1; 2626 2627 pl[cfg_pi++] = pi; 2628 } 2629 2630 if (need_check_link_status == 1 && !no_link_check) 2631 check_all_ports_link_status(RTE_PORT_ALL); 2632 else if (need_check_link_status == 0) 2633 printf("Please stop the ports first\n"); 2634 2635 if (hairpin_mode & 0xf) { 2636 uint16_t i; 2637 int j; 2638 2639 /* bind all started hairpin ports */ 2640 for (i = 0; i < cfg_pi; i++) { 2641 pi = pl[i]; 2642 /* bind current Tx to all peer Rx */ 2643 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2644 RTE_MAX_ETHPORTS, 1); 2645 if (peer_pi < 0) 2646 return peer_pi; 2647 for (j = 0; j < peer_pi; j++) { 2648 if (!port_is_started(peer_pl[j])) 2649 continue; 2650 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2651 if (diag < 0) { 2652 printf("Error during binding hairpin" 2653 " Tx port %u to %u: %s\n", 2654 pi, peer_pl[j], 2655 rte_strerror(-diag)); 2656 return -1; 2657 } 2658 } 2659 /* bind all peer Tx to current Rx */ 2660 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2661 RTE_MAX_ETHPORTS, 0); 2662 if (peer_pi < 0) 2663 return peer_pi; 2664 for (j = 0; j < peer_pi; j++) { 2665 if (!port_is_started(peer_pl[j])) 2666 continue; 2667 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2668 if (diag < 0) { 2669 printf("Error during binding hairpin" 2670 " Tx port %u to %u: %s\n", 2671 peer_pl[j], pi, 2672 rte_strerror(-diag)); 2673 return -1; 2674 } 2675 } 2676 } 2677 } 2678 2679 printf("Done\n"); 2680 return 0; 2681 } 2682 2683 void 2684 stop_port(portid_t pid) 2685 { 2686 portid_t pi; 2687 struct rte_port *port; 2688 int need_check_link_status = 0; 2689 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2690 int peer_pi; 2691 2692 if (dcb_test) { 2693 dcb_test = 0; 2694 dcb_config = 0; 2695 } 2696 2697 if (port_id_is_invalid(pid, ENABLED_WARN)) 2698 return; 2699 2700 printf("Stopping ports...\n"); 2701 2702 RTE_ETH_FOREACH_DEV(pi) { 2703 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2704 continue; 2705 2706 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2707 printf("Please remove port %d from forwarding configuration.\n", pi); 2708 continue; 2709 } 2710 2711 if (port_is_bonding_slave(pi)) { 2712 printf("Please remove port %d from bonded device.\n", pi); 2713 continue; 2714 } 2715 2716 port = &ports[pi]; 2717 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2718 RTE_PORT_HANDLING) == 0) 2719 continue; 2720 2721 if (hairpin_mode & 0xf) { 2722 int j; 2723 2724 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2725 /* unbind all peer Tx from current Rx */ 2726 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2727 RTE_MAX_ETHPORTS, 0); 2728 if (peer_pi < 0) 2729 continue; 2730 for (j = 0; j < peer_pi; j++) { 2731 if (!port_is_started(peer_pl[j])) 2732 continue; 2733 rte_eth_hairpin_unbind(peer_pl[j], pi); 2734 } 2735 } 2736 2737 if (port->flow_list) 2738 port_flow_flush(pi); 2739 2740 if (rte_eth_dev_stop(pi) != 0) 2741 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2742 pi); 2743 2744 if (rte_atomic16_cmpset(&(port->port_status), 2745 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2746 printf("Port %d can not be set into stopped\n", pi); 2747 need_check_link_status = 1; 2748 } 2749 if (need_check_link_status && !no_link_check) 2750 check_all_ports_link_status(RTE_PORT_ALL); 2751 2752 printf("Done\n"); 2753 } 2754 2755 static void 2756 remove_invalid_ports_in(portid_t *array, portid_t *total) 2757 { 2758 portid_t i; 2759 portid_t new_total = 0; 2760 2761 for (i = 0; i < *total; i++) 2762 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2763 array[new_total] = array[i]; 2764 new_total++; 2765 } 2766 *total = new_total; 2767 } 2768 2769 static void 2770 remove_invalid_ports(void) 2771 { 2772 remove_invalid_ports_in(ports_ids, &nb_ports); 2773 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2774 nb_cfg_ports = nb_fwd_ports; 2775 } 2776 2777 void 2778 close_port(portid_t pid) 2779 { 2780 portid_t pi; 2781 struct rte_port *port; 2782 2783 if (port_id_is_invalid(pid, ENABLED_WARN)) 2784 return; 2785 2786 printf("Closing ports...\n"); 2787 2788 RTE_ETH_FOREACH_DEV(pi) { 2789 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2790 continue; 2791 2792 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2793 printf("Please remove port %d from forwarding configuration.\n", pi); 2794 continue; 2795 } 2796 2797 if (port_is_bonding_slave(pi)) { 2798 printf("Please remove port %d from bonded device.\n", pi); 2799 continue; 2800 } 2801 2802 port = &ports[pi]; 2803 if (rte_atomic16_cmpset(&(port->port_status), 2804 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2805 printf("Port %d is already closed\n", pi); 2806 continue; 2807 } 2808 2809 port_flow_flush(pi); 2810 rte_eth_dev_close(pi); 2811 } 2812 2813 remove_invalid_ports(); 2814 printf("Done\n"); 2815 } 2816 2817 void 2818 reset_port(portid_t pid) 2819 { 2820 int diag; 2821 portid_t pi; 2822 struct rte_port *port; 2823 2824 if (port_id_is_invalid(pid, ENABLED_WARN)) 2825 return; 2826 2827 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2828 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2829 printf("Can not reset port(s), please stop port(s) first.\n"); 2830 return; 2831 } 2832 2833 printf("Resetting ports...\n"); 2834 2835 RTE_ETH_FOREACH_DEV(pi) { 2836 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2837 continue; 2838 2839 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2840 printf("Please remove port %d from forwarding " 2841 "configuration.\n", pi); 2842 continue; 2843 } 2844 2845 if (port_is_bonding_slave(pi)) { 2846 printf("Please remove port %d from bonded device.\n", 2847 pi); 2848 continue; 2849 } 2850 2851 diag = rte_eth_dev_reset(pi); 2852 if (diag == 0) { 2853 port = &ports[pi]; 2854 port->need_reconfig = 1; 2855 port->need_reconfig_queues = 1; 2856 } else { 2857 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2858 } 2859 } 2860 2861 printf("Done\n"); 2862 } 2863 2864 void 2865 attach_port(char *identifier) 2866 { 2867 portid_t pi; 2868 struct rte_dev_iterator iterator; 2869 2870 printf("Attaching a new port...\n"); 2871 2872 if (identifier == NULL) { 2873 printf("Invalid parameters are specified\n"); 2874 return; 2875 } 2876 2877 if (rte_dev_probe(identifier) < 0) { 2878 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2879 return; 2880 } 2881 2882 /* first attach mode: event */ 2883 if (setup_on_probe_event) { 2884 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2885 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2886 if (ports[pi].port_status == RTE_PORT_HANDLING && 2887 ports[pi].need_setup != 0) 2888 setup_attached_port(pi); 2889 return; 2890 } 2891 2892 /* second attach mode: iterator */ 2893 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2894 /* setup ports matching the devargs used for probing */ 2895 if (port_is_forwarding(pi)) 2896 continue; /* port was already attached before */ 2897 setup_attached_port(pi); 2898 } 2899 } 2900 2901 static void 2902 setup_attached_port(portid_t pi) 2903 { 2904 unsigned int socket_id; 2905 int ret; 2906 2907 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2908 /* if socket_id is invalid, set to the first available socket. */ 2909 if (check_socket_id(socket_id) < 0) 2910 socket_id = socket_ids[0]; 2911 reconfig(pi, socket_id); 2912 ret = rte_eth_promiscuous_enable(pi); 2913 if (ret != 0) 2914 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2915 pi, rte_strerror(-ret)); 2916 2917 ports_ids[nb_ports++] = pi; 2918 fwd_ports_ids[nb_fwd_ports++] = pi; 2919 nb_cfg_ports = nb_fwd_ports; 2920 ports[pi].need_setup = 0; 2921 ports[pi].port_status = RTE_PORT_STOPPED; 2922 2923 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2924 printf("Done\n"); 2925 } 2926 2927 static void 2928 detach_device(struct rte_device *dev) 2929 { 2930 portid_t sibling; 2931 2932 if (dev == NULL) { 2933 printf("Device already removed\n"); 2934 return; 2935 } 2936 2937 printf("Removing a device...\n"); 2938 2939 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2940 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2941 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2942 printf("Port %u not stopped\n", sibling); 2943 return; 2944 } 2945 port_flow_flush(sibling); 2946 } 2947 } 2948 2949 if (rte_dev_remove(dev) < 0) { 2950 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2951 return; 2952 } 2953 remove_invalid_ports(); 2954 2955 printf("Device is detached\n"); 2956 printf("Now total ports is %d\n", nb_ports); 2957 printf("Done\n"); 2958 return; 2959 } 2960 2961 void 2962 detach_port_device(portid_t port_id) 2963 { 2964 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2965 return; 2966 2967 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2968 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2969 printf("Port not stopped\n"); 2970 return; 2971 } 2972 printf("Port was not closed\n"); 2973 } 2974 2975 detach_device(rte_eth_devices[port_id].device); 2976 } 2977 2978 void 2979 detach_devargs(char *identifier) 2980 { 2981 struct rte_dev_iterator iterator; 2982 struct rte_devargs da; 2983 portid_t port_id; 2984 2985 printf("Removing a device...\n"); 2986 2987 memset(&da, 0, sizeof(da)); 2988 if (rte_devargs_parsef(&da, "%s", identifier)) { 2989 printf("cannot parse identifier\n"); 2990 if (da.args) 2991 free(da.args); 2992 return; 2993 } 2994 2995 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2996 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2997 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2998 printf("Port %u not stopped\n", port_id); 2999 rte_eth_iterator_cleanup(&iterator); 3000 return; 3001 } 3002 port_flow_flush(port_id); 3003 } 3004 } 3005 3006 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3007 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3008 da.name, da.bus->name); 3009 return; 3010 } 3011 3012 remove_invalid_ports(); 3013 3014 printf("Device %s is detached\n", identifier); 3015 printf("Now total ports is %d\n", nb_ports); 3016 printf("Done\n"); 3017 } 3018 3019 void 3020 pmd_test_exit(void) 3021 { 3022 portid_t pt_id; 3023 unsigned int i; 3024 int ret; 3025 3026 if (test_done == 0) 3027 stop_packet_forwarding(); 3028 3029 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3030 if (mempools[i]) { 3031 if (mp_alloc_type == MP_ALLOC_ANON) 3032 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3033 NULL); 3034 } 3035 } 3036 if (ports != NULL) { 3037 no_link_check = 1; 3038 RTE_ETH_FOREACH_DEV(pt_id) { 3039 printf("\nStopping port %d...\n", pt_id); 3040 fflush(stdout); 3041 stop_port(pt_id); 3042 } 3043 RTE_ETH_FOREACH_DEV(pt_id) { 3044 printf("\nShutting down port %d...\n", pt_id); 3045 fflush(stdout); 3046 close_port(pt_id); 3047 } 3048 } 3049 3050 if (hot_plug) { 3051 ret = rte_dev_event_monitor_stop(); 3052 if (ret) { 3053 RTE_LOG(ERR, EAL, 3054 "fail to stop device event monitor."); 3055 return; 3056 } 3057 3058 ret = rte_dev_event_callback_unregister(NULL, 3059 dev_event_callback, NULL); 3060 if (ret < 0) { 3061 RTE_LOG(ERR, EAL, 3062 "fail to unregister device event callback.\n"); 3063 return; 3064 } 3065 3066 ret = rte_dev_hotplug_handle_disable(); 3067 if (ret) { 3068 RTE_LOG(ERR, EAL, 3069 "fail to disable hotplug handling.\n"); 3070 return; 3071 } 3072 } 3073 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3074 if (mempools[i]) 3075 rte_mempool_free(mempools[i]); 3076 } 3077 3078 printf("\nBye...\n"); 3079 } 3080 3081 typedef void (*cmd_func_t)(void); 3082 struct pmd_test_command { 3083 const char *cmd_name; 3084 cmd_func_t cmd_func; 3085 }; 3086 3087 /* Check the link status of all ports in up to 9s, and print them finally */ 3088 static void 3089 check_all_ports_link_status(uint32_t port_mask) 3090 { 3091 #define CHECK_INTERVAL 100 /* 100ms */ 3092 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3093 portid_t portid; 3094 uint8_t count, all_ports_up, print_flag = 0; 3095 struct rte_eth_link link; 3096 int ret; 3097 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3098 3099 printf("Checking link statuses...\n"); 3100 fflush(stdout); 3101 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3102 all_ports_up = 1; 3103 RTE_ETH_FOREACH_DEV(portid) { 3104 if ((port_mask & (1 << portid)) == 0) 3105 continue; 3106 memset(&link, 0, sizeof(link)); 3107 ret = rte_eth_link_get_nowait(portid, &link); 3108 if (ret < 0) { 3109 all_ports_up = 0; 3110 if (print_flag == 1) 3111 printf("Port %u link get failed: %s\n", 3112 portid, rte_strerror(-ret)); 3113 continue; 3114 } 3115 /* print link status if flag set */ 3116 if (print_flag == 1) { 3117 rte_eth_link_to_str(link_status, 3118 sizeof(link_status), &link); 3119 printf("Port %d %s\n", portid, link_status); 3120 continue; 3121 } 3122 /* clear all_ports_up flag if any link down */ 3123 if (link.link_status == ETH_LINK_DOWN) { 3124 all_ports_up = 0; 3125 break; 3126 } 3127 } 3128 /* after finally printing all link status, get out */ 3129 if (print_flag == 1) 3130 break; 3131 3132 if (all_ports_up == 0) { 3133 fflush(stdout); 3134 rte_delay_ms(CHECK_INTERVAL); 3135 } 3136 3137 /* set the print_flag if all ports up or timeout */ 3138 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3139 print_flag = 1; 3140 } 3141 3142 if (lsc_interrupt) 3143 break; 3144 } 3145 } 3146 3147 static void 3148 rmv_port_callback(void *arg) 3149 { 3150 int need_to_start = 0; 3151 int org_no_link_check = no_link_check; 3152 portid_t port_id = (intptr_t)arg; 3153 struct rte_device *dev; 3154 3155 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3156 3157 if (!test_done && port_is_forwarding(port_id)) { 3158 need_to_start = 1; 3159 stop_packet_forwarding(); 3160 } 3161 no_link_check = 1; 3162 stop_port(port_id); 3163 no_link_check = org_no_link_check; 3164 3165 /* Save rte_device pointer before closing ethdev port */ 3166 dev = rte_eth_devices[port_id].device; 3167 close_port(port_id); 3168 detach_device(dev); /* might be already removed or have more ports */ 3169 3170 if (need_to_start) 3171 start_packet_forwarding(0); 3172 } 3173 3174 /* This function is used by the interrupt thread */ 3175 static int 3176 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3177 void *ret_param) 3178 { 3179 RTE_SET_USED(param); 3180 RTE_SET_USED(ret_param); 3181 3182 if (type >= RTE_ETH_EVENT_MAX) { 3183 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3184 port_id, __func__, type); 3185 fflush(stderr); 3186 } else if (event_print_mask & (UINT32_C(1) << type)) { 3187 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3188 eth_event_desc[type]); 3189 fflush(stdout); 3190 } 3191 3192 switch (type) { 3193 case RTE_ETH_EVENT_NEW: 3194 ports[port_id].need_setup = 1; 3195 ports[port_id].port_status = RTE_PORT_HANDLING; 3196 break; 3197 case RTE_ETH_EVENT_INTR_RMV: 3198 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3199 break; 3200 if (rte_eal_alarm_set(100000, 3201 rmv_port_callback, (void *)(intptr_t)port_id)) 3202 fprintf(stderr, "Could not set up deferred device removal\n"); 3203 break; 3204 case RTE_ETH_EVENT_DESTROY: 3205 ports[port_id].port_status = RTE_PORT_CLOSED; 3206 printf("Port %u is closed\n", port_id); 3207 break; 3208 default: 3209 break; 3210 } 3211 return 0; 3212 } 3213 3214 static int 3215 register_eth_event_callback(void) 3216 { 3217 int ret; 3218 enum rte_eth_event_type event; 3219 3220 for (event = RTE_ETH_EVENT_UNKNOWN; 3221 event < RTE_ETH_EVENT_MAX; event++) { 3222 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3223 event, 3224 eth_event_callback, 3225 NULL); 3226 if (ret != 0) { 3227 TESTPMD_LOG(ERR, "Failed to register callback for " 3228 "%s event\n", eth_event_desc[event]); 3229 return -1; 3230 } 3231 } 3232 3233 return 0; 3234 } 3235 3236 /* This function is used by the interrupt thread */ 3237 static void 3238 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3239 __rte_unused void *arg) 3240 { 3241 uint16_t port_id; 3242 int ret; 3243 3244 if (type >= RTE_DEV_EVENT_MAX) { 3245 fprintf(stderr, "%s called upon invalid event %d\n", 3246 __func__, type); 3247 fflush(stderr); 3248 } 3249 3250 switch (type) { 3251 case RTE_DEV_EVENT_REMOVE: 3252 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3253 device_name); 3254 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3255 if (ret) { 3256 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3257 device_name); 3258 return; 3259 } 3260 /* 3261 * Because the user's callback is invoked in eal interrupt 3262 * callback, the interrupt callback need to be finished before 3263 * it can be unregistered when detaching device. So finish 3264 * callback soon and use a deferred removal to detach device 3265 * is need. It is a workaround, once the device detaching be 3266 * moved into the eal in the future, the deferred removal could 3267 * be deleted. 3268 */ 3269 if (rte_eal_alarm_set(100000, 3270 rmv_port_callback, (void *)(intptr_t)port_id)) 3271 RTE_LOG(ERR, EAL, 3272 "Could not set up deferred device removal\n"); 3273 break; 3274 case RTE_DEV_EVENT_ADD: 3275 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3276 device_name); 3277 /* TODO: After finish kernel driver binding, 3278 * begin to attach port. 3279 */ 3280 break; 3281 default: 3282 break; 3283 } 3284 } 3285 3286 static void 3287 rxtx_port_config(struct rte_port *port) 3288 { 3289 uint16_t qid; 3290 uint64_t offloads; 3291 3292 for (qid = 0; qid < nb_rxq; qid++) { 3293 offloads = port->rx_conf[qid].offloads; 3294 port->rx_conf[qid] = port->dev_info.default_rxconf; 3295 if (offloads != 0) 3296 port->rx_conf[qid].offloads = offloads; 3297 3298 /* Check if any Rx parameters have been passed */ 3299 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3300 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3301 3302 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3303 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3304 3305 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3306 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3307 3308 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3309 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3310 3311 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3312 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3313 3314 port->nb_rx_desc[qid] = nb_rxd; 3315 } 3316 3317 for (qid = 0; qid < nb_txq; qid++) { 3318 offloads = port->tx_conf[qid].offloads; 3319 port->tx_conf[qid] = port->dev_info.default_txconf; 3320 if (offloads != 0) 3321 port->tx_conf[qid].offloads = offloads; 3322 3323 /* Check if any Tx parameters have been passed */ 3324 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3325 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3326 3327 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3328 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3329 3330 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3331 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3332 3333 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3334 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3335 3336 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3337 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3338 3339 port->nb_tx_desc[qid] = nb_txd; 3340 } 3341 } 3342 3343 void 3344 init_port_config(void) 3345 { 3346 portid_t pid; 3347 struct rte_port *port; 3348 int ret; 3349 3350 RTE_ETH_FOREACH_DEV(pid) { 3351 port = &ports[pid]; 3352 port->dev_conf.fdir_conf = fdir_conf; 3353 3354 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3355 if (ret != 0) 3356 return; 3357 3358 if (nb_rxq > 1) { 3359 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3360 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3361 rss_hf & port->dev_info.flow_type_rss_offloads; 3362 } else { 3363 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3364 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3365 } 3366 3367 if (port->dcb_flag == 0) { 3368 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3369 port->dev_conf.rxmode.mq_mode = 3370 (enum rte_eth_rx_mq_mode) 3371 (rx_mq_mode & ETH_MQ_RX_RSS); 3372 else 3373 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3374 } 3375 3376 rxtx_port_config(port); 3377 3378 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3379 if (ret != 0) 3380 return; 3381 3382 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3383 rte_pmd_ixgbe_bypass_init(pid); 3384 #endif 3385 3386 if (lsc_interrupt && 3387 (rte_eth_devices[pid].data->dev_flags & 3388 RTE_ETH_DEV_INTR_LSC)) 3389 port->dev_conf.intr_conf.lsc = 1; 3390 if (rmv_interrupt && 3391 (rte_eth_devices[pid].data->dev_flags & 3392 RTE_ETH_DEV_INTR_RMV)) 3393 port->dev_conf.intr_conf.rmv = 1; 3394 } 3395 } 3396 3397 void set_port_slave_flag(portid_t slave_pid) 3398 { 3399 struct rte_port *port; 3400 3401 port = &ports[slave_pid]; 3402 port->slave_flag = 1; 3403 } 3404 3405 void clear_port_slave_flag(portid_t slave_pid) 3406 { 3407 struct rte_port *port; 3408 3409 port = &ports[slave_pid]; 3410 port->slave_flag = 0; 3411 } 3412 3413 uint8_t port_is_bonding_slave(portid_t slave_pid) 3414 { 3415 struct rte_port *port; 3416 3417 port = &ports[slave_pid]; 3418 if ((rte_eth_devices[slave_pid].data->dev_flags & 3419 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3420 return 1; 3421 return 0; 3422 } 3423 3424 const uint16_t vlan_tags[] = { 3425 0, 1, 2, 3, 4, 5, 6, 7, 3426 8, 9, 10, 11, 12, 13, 14, 15, 3427 16, 17, 18, 19, 20, 21, 22, 23, 3428 24, 25, 26, 27, 28, 29, 30, 31 3429 }; 3430 3431 static int 3432 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3433 enum dcb_mode_enable dcb_mode, 3434 enum rte_eth_nb_tcs num_tcs, 3435 uint8_t pfc_en) 3436 { 3437 uint8_t i; 3438 int32_t rc; 3439 struct rte_eth_rss_conf rss_conf; 3440 3441 /* 3442 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3443 * given above, and the number of traffic classes available for use. 3444 */ 3445 if (dcb_mode == DCB_VT_ENABLED) { 3446 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3447 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3448 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3449 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3450 3451 /* VMDQ+DCB RX and TX configurations */ 3452 vmdq_rx_conf->enable_default_pool = 0; 3453 vmdq_rx_conf->default_pool = 0; 3454 vmdq_rx_conf->nb_queue_pools = 3455 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3456 vmdq_tx_conf->nb_queue_pools = 3457 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3458 3459 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3460 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3461 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3462 vmdq_rx_conf->pool_map[i].pools = 3463 1 << (i % vmdq_rx_conf->nb_queue_pools); 3464 } 3465 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3466 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3467 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3468 } 3469 3470 /* set DCB mode of RX and TX of multiple queues */ 3471 eth_conf->rxmode.mq_mode = 3472 (enum rte_eth_rx_mq_mode) 3473 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3474 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3475 } else { 3476 struct rte_eth_dcb_rx_conf *rx_conf = 3477 ð_conf->rx_adv_conf.dcb_rx_conf; 3478 struct rte_eth_dcb_tx_conf *tx_conf = 3479 ð_conf->tx_adv_conf.dcb_tx_conf; 3480 3481 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3482 3483 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3484 if (rc != 0) 3485 return rc; 3486 3487 rx_conf->nb_tcs = num_tcs; 3488 tx_conf->nb_tcs = num_tcs; 3489 3490 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3491 rx_conf->dcb_tc[i] = i % num_tcs; 3492 tx_conf->dcb_tc[i] = i % num_tcs; 3493 } 3494 3495 eth_conf->rxmode.mq_mode = 3496 (enum rte_eth_rx_mq_mode) 3497 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3498 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3499 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3500 } 3501 3502 if (pfc_en) 3503 eth_conf->dcb_capability_en = 3504 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3505 else 3506 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3507 3508 return 0; 3509 } 3510 3511 int 3512 init_port_dcb_config(portid_t pid, 3513 enum dcb_mode_enable dcb_mode, 3514 enum rte_eth_nb_tcs num_tcs, 3515 uint8_t pfc_en) 3516 { 3517 struct rte_eth_conf port_conf; 3518 struct rte_port *rte_port; 3519 int retval; 3520 uint16_t i; 3521 3522 rte_port = &ports[pid]; 3523 3524 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3525 /* Enter DCB configuration status */ 3526 dcb_config = 1; 3527 3528 port_conf.rxmode = rte_port->dev_conf.rxmode; 3529 port_conf.txmode = rte_port->dev_conf.txmode; 3530 3531 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3532 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3533 if (retval < 0) 3534 return retval; 3535 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3536 3537 /* re-configure the device . */ 3538 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3539 if (retval < 0) 3540 return retval; 3541 3542 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3543 if (retval != 0) 3544 return retval; 3545 3546 /* If dev_info.vmdq_pool_base is greater than 0, 3547 * the queue id of vmdq pools is started after pf queues. 3548 */ 3549 if (dcb_mode == DCB_VT_ENABLED && 3550 rte_port->dev_info.vmdq_pool_base > 0) { 3551 printf("VMDQ_DCB multi-queue mode is nonsensical" 3552 " for port %d.", pid); 3553 return -1; 3554 } 3555 3556 /* Assume the ports in testpmd have the same dcb capability 3557 * and has the same number of rxq and txq in dcb mode 3558 */ 3559 if (dcb_mode == DCB_VT_ENABLED) { 3560 if (rte_port->dev_info.max_vfs > 0) { 3561 nb_rxq = rte_port->dev_info.nb_rx_queues; 3562 nb_txq = rte_port->dev_info.nb_tx_queues; 3563 } else { 3564 nb_rxq = rte_port->dev_info.max_rx_queues; 3565 nb_txq = rte_port->dev_info.max_tx_queues; 3566 } 3567 } else { 3568 /*if vt is disabled, use all pf queues */ 3569 if (rte_port->dev_info.vmdq_pool_base == 0) { 3570 nb_rxq = rte_port->dev_info.max_rx_queues; 3571 nb_txq = rte_port->dev_info.max_tx_queues; 3572 } else { 3573 nb_rxq = (queueid_t)num_tcs; 3574 nb_txq = (queueid_t)num_tcs; 3575 3576 } 3577 } 3578 rx_free_thresh = 64; 3579 3580 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3581 3582 rxtx_port_config(rte_port); 3583 /* VLAN filter */ 3584 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3585 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3586 rx_vft_set(pid, vlan_tags[i], 1); 3587 3588 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3589 if (retval != 0) 3590 return retval; 3591 3592 rte_port->dcb_flag = 1; 3593 3594 return 0; 3595 } 3596 3597 static void 3598 init_port(void) 3599 { 3600 int i; 3601 3602 /* Configuration of Ethernet ports. */ 3603 ports = rte_zmalloc("testpmd: ports", 3604 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3605 RTE_CACHE_LINE_SIZE); 3606 if (ports == NULL) { 3607 rte_exit(EXIT_FAILURE, 3608 "rte_zmalloc(%d struct rte_port) failed\n", 3609 RTE_MAX_ETHPORTS); 3610 } 3611 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3612 LIST_INIT(&ports[i].flow_tunnel_list); 3613 /* Initialize ports NUMA structures */ 3614 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3615 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3616 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3617 } 3618 3619 static void 3620 force_quit(void) 3621 { 3622 pmd_test_exit(); 3623 prompt_exit(); 3624 } 3625 3626 static void 3627 print_stats(void) 3628 { 3629 uint8_t i; 3630 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3631 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3632 3633 /* Clear screen and move to top left */ 3634 printf("%s%s", clr, top_left); 3635 3636 printf("\nPort statistics ===================================="); 3637 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3638 nic_stats_display(fwd_ports_ids[i]); 3639 3640 fflush(stdout); 3641 } 3642 3643 static void 3644 signal_handler(int signum) 3645 { 3646 if (signum == SIGINT || signum == SIGTERM) { 3647 printf("\nSignal %d received, preparing to exit...\n", 3648 signum); 3649 #ifdef RTE_LIB_PDUMP 3650 /* uninitialize packet capture framework */ 3651 rte_pdump_uninit(); 3652 #endif 3653 #ifdef RTE_LIB_LATENCYSTATS 3654 if (latencystats_enabled != 0) 3655 rte_latencystats_uninit(); 3656 #endif 3657 force_quit(); 3658 /* Set flag to indicate the force termination. */ 3659 f_quit = 1; 3660 /* exit with the expected status */ 3661 signal(signum, SIG_DFL); 3662 kill(getpid(), signum); 3663 } 3664 } 3665 3666 int 3667 main(int argc, char** argv) 3668 { 3669 int diag; 3670 portid_t port_id; 3671 uint16_t count; 3672 int ret; 3673 3674 signal(SIGINT, signal_handler); 3675 signal(SIGTERM, signal_handler); 3676 3677 testpmd_logtype = rte_log_register("testpmd"); 3678 if (testpmd_logtype < 0) 3679 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3680 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3681 3682 diag = rte_eal_init(argc, argv); 3683 if (diag < 0) 3684 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3685 rte_strerror(rte_errno)); 3686 3687 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3688 rte_exit(EXIT_FAILURE, 3689 "Secondary process type not supported.\n"); 3690 3691 ret = register_eth_event_callback(); 3692 if (ret != 0) 3693 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3694 3695 #ifdef RTE_LIB_PDUMP 3696 /* initialize packet capture framework */ 3697 rte_pdump_init(); 3698 #endif 3699 3700 count = 0; 3701 RTE_ETH_FOREACH_DEV(port_id) { 3702 ports_ids[count] = port_id; 3703 count++; 3704 } 3705 nb_ports = (portid_t) count; 3706 if (nb_ports == 0) 3707 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3708 3709 /* allocate port structures, and init them */ 3710 init_port(); 3711 3712 set_def_fwd_config(); 3713 if (nb_lcores == 0) 3714 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3715 "Check the core mask argument\n"); 3716 3717 /* Bitrate/latency stats disabled by default */ 3718 #ifdef RTE_LIB_BITRATESTATS 3719 bitrate_enabled = 0; 3720 #endif 3721 #ifdef RTE_LIB_LATENCYSTATS 3722 latencystats_enabled = 0; 3723 #endif 3724 3725 /* on FreeBSD, mlockall() is disabled by default */ 3726 #ifdef RTE_EXEC_ENV_FREEBSD 3727 do_mlockall = 0; 3728 #else 3729 do_mlockall = 1; 3730 #endif 3731 3732 argc -= diag; 3733 argv += diag; 3734 if (argc > 1) 3735 launch_args_parse(argc, argv); 3736 3737 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3738 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3739 strerror(errno)); 3740 } 3741 3742 if (tx_first && interactive) 3743 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3744 "interactive mode.\n"); 3745 3746 if (tx_first && lsc_interrupt) { 3747 printf("Warning: lsc_interrupt needs to be off when " 3748 " using tx_first. Disabling.\n"); 3749 lsc_interrupt = 0; 3750 } 3751 3752 if (!nb_rxq && !nb_txq) 3753 printf("Warning: Either rx or tx queues should be non-zero\n"); 3754 3755 if (nb_rxq > 1 && nb_rxq > nb_txq) 3756 printf("Warning: nb_rxq=%d enables RSS configuration, " 3757 "but nb_txq=%d will prevent to fully test it.\n", 3758 nb_rxq, nb_txq); 3759 3760 init_config(); 3761 3762 if (hot_plug) { 3763 ret = rte_dev_hotplug_handle_enable(); 3764 if (ret) { 3765 RTE_LOG(ERR, EAL, 3766 "fail to enable hotplug handling."); 3767 return -1; 3768 } 3769 3770 ret = rte_dev_event_monitor_start(); 3771 if (ret) { 3772 RTE_LOG(ERR, EAL, 3773 "fail to start device event monitoring."); 3774 return -1; 3775 } 3776 3777 ret = rte_dev_event_callback_register(NULL, 3778 dev_event_callback, NULL); 3779 if (ret) { 3780 RTE_LOG(ERR, EAL, 3781 "fail to register device event callback\n"); 3782 return -1; 3783 } 3784 } 3785 3786 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3787 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3788 3789 /* set all ports to promiscuous mode by default */ 3790 RTE_ETH_FOREACH_DEV(port_id) { 3791 ret = rte_eth_promiscuous_enable(port_id); 3792 if (ret != 0) 3793 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3794 port_id, rte_strerror(-ret)); 3795 } 3796 3797 /* Init metrics library */ 3798 rte_metrics_init(rte_socket_id()); 3799 3800 #ifdef RTE_LIB_LATENCYSTATS 3801 if (latencystats_enabled != 0) { 3802 int ret = rte_latencystats_init(1, NULL); 3803 if (ret) 3804 printf("Warning: latencystats init()" 3805 " returned error %d\n", ret); 3806 printf("Latencystats running on lcore %d\n", 3807 latencystats_lcore_id); 3808 } 3809 #endif 3810 3811 /* Setup bitrate stats */ 3812 #ifdef RTE_LIB_BITRATESTATS 3813 if (bitrate_enabled != 0) { 3814 bitrate_data = rte_stats_bitrate_create(); 3815 if (bitrate_data == NULL) 3816 rte_exit(EXIT_FAILURE, 3817 "Could not allocate bitrate data.\n"); 3818 rte_stats_bitrate_reg(bitrate_data); 3819 } 3820 #endif 3821 3822 #ifdef RTE_LIB_CMDLINE 3823 if (strlen(cmdline_filename) != 0) 3824 cmdline_read_from_file(cmdline_filename); 3825 3826 if (interactive == 1) { 3827 if (auto_start) { 3828 printf("Start automatic packet forwarding\n"); 3829 start_packet_forwarding(0); 3830 } 3831 prompt(); 3832 pmd_test_exit(); 3833 } else 3834 #endif 3835 { 3836 char c; 3837 int rc; 3838 3839 f_quit = 0; 3840 3841 printf("No commandline core given, start packet forwarding\n"); 3842 start_packet_forwarding(tx_first); 3843 if (stats_period != 0) { 3844 uint64_t prev_time = 0, cur_time, diff_time = 0; 3845 uint64_t timer_period; 3846 3847 /* Convert to number of cycles */ 3848 timer_period = stats_period * rte_get_timer_hz(); 3849 3850 while (f_quit == 0) { 3851 cur_time = rte_get_timer_cycles(); 3852 diff_time += cur_time - prev_time; 3853 3854 if (diff_time >= timer_period) { 3855 print_stats(); 3856 /* Reset the timer */ 3857 diff_time = 0; 3858 } 3859 /* Sleep to avoid unnecessary checks */ 3860 prev_time = cur_time; 3861 sleep(1); 3862 } 3863 } 3864 3865 printf("Press enter to exit\n"); 3866 rc = read(0, &c, 1); 3867 pmd_test_exit(); 3868 if (rc < 0) 3869 return 1; 3870 } 3871 3872 ret = rte_eal_cleanup(); 3873 if (ret != 0) 3874 rte_exit(EXIT_FAILURE, 3875 "EAL cleanup failed: %s\n", strerror(-ret)); 3876 3877 return EXIT_SUCCESS; 3878 } 3879