1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_NET_IXGBE 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIB_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIB_BITRATESTATS 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIB_LATENCYSTATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use main core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 &five_tuple_swap_fwd_engine, 183 #ifdef RTE_LIBRTE_IEEE1588 184 &ieee1588_fwd_engine, 185 #endif 186 NULL, 187 }; 188 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 190 uint16_t mempool_flags; 191 192 struct fwd_config cur_fwd_config; 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 194 uint32_t retry_enabled; 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 197 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 200 DEFAULT_MBUF_DATA_SIZE 201 }; /**< Mbuf data space size. */ 202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 203 * specified on command-line. */ 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 205 206 /* 207 * In container, it cannot terminate the process which running with 'stats-period' 208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 209 */ 210 uint8_t f_quit; 211 212 /* 213 * Configuration of packet segments used to scatter received packets 214 * if some of split features is configured. 215 */ 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 220 221 /* 222 * Configuration of packet segments used by the "txonly" processing engine. 223 */ 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 226 TXONLY_DEF_PACKET_LEN, 227 }; 228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 229 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 231 /**< Split policy for packets to TX. */ 232 233 uint8_t txonly_multi_flow; 234 /**< Whether multiple flows are generated in TXONLY mode. */ 235 236 uint32_t tx_pkt_times_inter; 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 238 239 uint32_t tx_pkt_times_intra; 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 241 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 243 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 244 245 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 246 uint8_t dcb_config = 0; 247 248 /* Whether the dcb is in testing status */ 249 uint8_t dcb_test = 0; 250 251 /* 252 * Configurable number of RX/TX queues. 253 */ 254 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 255 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 256 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 257 258 /* 259 * Configurable number of RX/TX ring descriptors. 260 * Defaults are supplied by drivers via ethdev. 261 */ 262 #define RTE_TEST_RX_DESC_DEFAULT 0 263 #define RTE_TEST_TX_DESC_DEFAULT 0 264 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 265 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 266 267 #define RTE_PMD_PARAM_UNSET -1 268 /* 269 * Configurable values of RX and TX ring threshold registers. 270 */ 271 272 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 273 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 274 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 275 276 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 277 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 278 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 279 280 /* 281 * Configurable value of RX free threshold. 282 */ 283 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 284 285 /* 286 * Configurable value of RX drop enable. 287 */ 288 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 289 290 /* 291 * Configurable value of TX free threshold. 292 */ 293 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 294 295 /* 296 * Configurable value of TX RS bit threshold. 297 */ 298 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 299 300 /* 301 * Configurable value of buffered packets before sending. 302 */ 303 uint16_t noisy_tx_sw_bufsz; 304 305 /* 306 * Configurable value of packet buffer timeout. 307 */ 308 uint16_t noisy_tx_sw_buf_flush_time; 309 310 /* 311 * Configurable value for size of VNF internal memory area 312 * used for simulating noisy neighbour behaviour 313 */ 314 uint64_t noisy_lkup_mem_sz; 315 316 /* 317 * Configurable value of number of random writes done in 318 * VNF simulation memory area. 319 */ 320 uint64_t noisy_lkup_num_writes; 321 322 /* 323 * Configurable value of number of random reads done in 324 * VNF simulation memory area. 325 */ 326 uint64_t noisy_lkup_num_reads; 327 328 /* 329 * Configurable value of number of random reads/writes done in 330 * VNF simulation memory area. 331 */ 332 uint64_t noisy_lkup_num_reads_writes; 333 334 /* 335 * Receive Side Scaling (RSS) configuration. 336 */ 337 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 338 339 /* 340 * Port topology configuration 341 */ 342 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 343 344 /* 345 * Avoids to flush all the RX streams before starts forwarding. 346 */ 347 uint8_t no_flush_rx = 0; /* flush by default */ 348 349 /* 350 * Flow API isolated mode. 351 */ 352 uint8_t flow_isolate_all; 353 354 /* 355 * Avoids to check link status when starting/stopping a port. 356 */ 357 uint8_t no_link_check = 0; /* check by default */ 358 359 /* 360 * Don't automatically start all ports in interactive mode. 361 */ 362 uint8_t no_device_start = 0; 363 364 /* 365 * Enable link status change notification 366 */ 367 uint8_t lsc_interrupt = 1; /* enabled by default */ 368 369 /* 370 * Enable device removal notification. 371 */ 372 uint8_t rmv_interrupt = 1; /* enabled by default */ 373 374 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 375 376 /* After attach, port setup is called on event or by iterator */ 377 bool setup_on_probe_event = true; 378 379 /* Clear ptypes on port initialization. */ 380 uint8_t clear_ptypes = true; 381 382 /* Hairpin ports configuration mode. */ 383 uint16_t hairpin_mode; 384 385 /* Pretty printing of ethdev events */ 386 static const char * const eth_event_desc[] = { 387 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 388 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 389 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 390 [RTE_ETH_EVENT_INTR_RESET] = "reset", 391 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 392 [RTE_ETH_EVENT_IPSEC] = "IPsec", 393 [RTE_ETH_EVENT_MACSEC] = "MACsec", 394 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 395 [RTE_ETH_EVENT_NEW] = "device probed", 396 [RTE_ETH_EVENT_DESTROY] = "device released", 397 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 398 [RTE_ETH_EVENT_MAX] = NULL, 399 }; 400 401 /* 402 * Display or mask ether events 403 * Default to all events except VF_MBOX 404 */ 405 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 406 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 407 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 408 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 409 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 410 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 411 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 412 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 413 /* 414 * Decide if all memory are locked for performance. 415 */ 416 int do_mlockall = 0; 417 418 /* 419 * NIC bypass mode configuration options. 420 */ 421 422 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 423 /* The NIC bypass watchdog timeout. */ 424 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 425 #endif 426 427 428 #ifdef RTE_LIB_LATENCYSTATS 429 430 /* 431 * Set when latency stats is enabled in the commandline 432 */ 433 uint8_t latencystats_enabled; 434 435 /* 436 * Lcore ID to serive latency statistics. 437 */ 438 lcoreid_t latencystats_lcore_id = -1; 439 440 #endif 441 442 /* 443 * Ethernet device configuration. 444 */ 445 struct rte_eth_rxmode rx_mode = { 446 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 447 /**< Default maximum frame length. */ 448 }; 449 450 struct rte_eth_txmode tx_mode = { 451 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 452 }; 453 454 struct rte_fdir_conf fdir_conf = { 455 .mode = RTE_FDIR_MODE_NONE, 456 .pballoc = RTE_FDIR_PBALLOC_64K, 457 .status = RTE_FDIR_REPORT_STATUS, 458 .mask = { 459 .vlan_tci_mask = 0xFFEF, 460 .ipv4_mask = { 461 .src_ip = 0xFFFFFFFF, 462 .dst_ip = 0xFFFFFFFF, 463 }, 464 .ipv6_mask = { 465 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 466 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 467 }, 468 .src_port_mask = 0xFFFF, 469 .dst_port_mask = 0xFFFF, 470 .mac_addr_byte_mask = 0xFF, 471 .tunnel_type_mask = 1, 472 .tunnel_id_mask = 0xFFFFFFFF, 473 }, 474 .drop_queue = 127, 475 }; 476 477 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 478 479 /* 480 * Display zero values by default for xstats 481 */ 482 uint8_t xstats_hide_zero; 483 484 /* 485 * Measure of CPU cycles disabled by default 486 */ 487 uint8_t record_core_cycles; 488 489 /* 490 * Display of RX and TX bursts disabled by default 491 */ 492 uint8_t record_burst_stats; 493 494 unsigned int num_sockets = 0; 495 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 496 497 #ifdef RTE_LIB_BITRATESTATS 498 /* Bitrate statistics */ 499 struct rte_stats_bitrates *bitrate_data; 500 lcoreid_t bitrate_lcore_id; 501 uint8_t bitrate_enabled; 502 #endif 503 504 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 505 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 506 507 /* 508 * hexadecimal bitmask of RX mq mode can be enabled. 509 */ 510 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 511 512 /* Forward function declarations */ 513 static void setup_attached_port(portid_t pi); 514 static void check_all_ports_link_status(uint32_t port_mask); 515 static int eth_event_callback(portid_t port_id, 516 enum rte_eth_event_type type, 517 void *param, void *ret_param); 518 static void dev_event_callback(const char *device_name, 519 enum rte_dev_event_type type, 520 void *param); 521 522 /* 523 * Check if all the ports are started. 524 * If yes, return positive value. If not, return zero. 525 */ 526 static int all_ports_started(void); 527 528 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 529 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 530 531 /* Holds the registered mbuf dynamic flags names. */ 532 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 533 534 /* 535 * Helper function to check if socket is already discovered. 536 * If yes, return positive value. If not, return zero. 537 */ 538 int 539 new_socket_id(unsigned int socket_id) 540 { 541 unsigned int i; 542 543 for (i = 0; i < num_sockets; i++) { 544 if (socket_ids[i] == socket_id) 545 return 0; 546 } 547 return 1; 548 } 549 550 /* 551 * Setup default configuration. 552 */ 553 static void 554 set_default_fwd_lcores_config(void) 555 { 556 unsigned int i; 557 unsigned int nb_lc; 558 unsigned int sock_num; 559 560 nb_lc = 0; 561 for (i = 0; i < RTE_MAX_LCORE; i++) { 562 if (!rte_lcore_is_enabled(i)) 563 continue; 564 sock_num = rte_lcore_to_socket_id(i); 565 if (new_socket_id(sock_num)) { 566 if (num_sockets >= RTE_MAX_NUMA_NODES) { 567 rte_exit(EXIT_FAILURE, 568 "Total sockets greater than %u\n", 569 RTE_MAX_NUMA_NODES); 570 } 571 socket_ids[num_sockets++] = sock_num; 572 } 573 if (i == rte_get_main_lcore()) 574 continue; 575 fwd_lcores_cpuids[nb_lc++] = i; 576 } 577 nb_lcores = (lcoreid_t) nb_lc; 578 nb_cfg_lcores = nb_lcores; 579 nb_fwd_lcores = 1; 580 } 581 582 static void 583 set_def_peer_eth_addrs(void) 584 { 585 portid_t i; 586 587 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 588 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 589 peer_eth_addrs[i].addr_bytes[5] = i; 590 } 591 } 592 593 static void 594 set_default_fwd_ports_config(void) 595 { 596 portid_t pt_id; 597 int i = 0; 598 599 RTE_ETH_FOREACH_DEV(pt_id) { 600 fwd_ports_ids[i++] = pt_id; 601 602 /* Update sockets info according to the attached device */ 603 int socket_id = rte_eth_dev_socket_id(pt_id); 604 if (socket_id >= 0 && new_socket_id(socket_id)) { 605 if (num_sockets >= RTE_MAX_NUMA_NODES) { 606 rte_exit(EXIT_FAILURE, 607 "Total sockets greater than %u\n", 608 RTE_MAX_NUMA_NODES); 609 } 610 socket_ids[num_sockets++] = socket_id; 611 } 612 } 613 614 nb_cfg_ports = nb_ports; 615 nb_fwd_ports = nb_ports; 616 } 617 618 void 619 set_def_fwd_config(void) 620 { 621 set_default_fwd_lcores_config(); 622 set_def_peer_eth_addrs(); 623 set_default_fwd_ports_config(); 624 } 625 626 /* extremely pessimistic estimation of memory required to create a mempool */ 627 static int 628 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 629 { 630 unsigned int n_pages, mbuf_per_pg, leftover; 631 uint64_t total_mem, mbuf_mem, obj_sz; 632 633 /* there is no good way to predict how much space the mempool will 634 * occupy because it will allocate chunks on the fly, and some of those 635 * will come from default DPDK memory while some will come from our 636 * external memory, so just assume 128MB will be enough for everyone. 637 */ 638 uint64_t hdr_mem = 128 << 20; 639 640 /* account for possible non-contiguousness */ 641 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 642 if (obj_sz > pgsz) { 643 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 644 return -1; 645 } 646 647 mbuf_per_pg = pgsz / obj_sz; 648 leftover = (nb_mbufs % mbuf_per_pg) > 0; 649 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 650 651 mbuf_mem = n_pages * pgsz; 652 653 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 654 655 if (total_mem > SIZE_MAX) { 656 TESTPMD_LOG(ERR, "Memory size too big\n"); 657 return -1; 658 } 659 *out = (size_t)total_mem; 660 661 return 0; 662 } 663 664 static int 665 pagesz_flags(uint64_t page_sz) 666 { 667 /* as per mmap() manpage, all page sizes are log2 of page size 668 * shifted by MAP_HUGE_SHIFT 669 */ 670 int log2 = rte_log2_u64(page_sz); 671 672 return (log2 << HUGE_SHIFT); 673 } 674 675 static void * 676 alloc_mem(size_t memsz, size_t pgsz, bool huge) 677 { 678 void *addr; 679 int flags; 680 681 /* allocate anonymous hugepages */ 682 flags = MAP_ANONYMOUS | MAP_PRIVATE; 683 if (huge) 684 flags |= HUGE_FLAG | pagesz_flags(pgsz); 685 686 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 687 if (addr == MAP_FAILED) 688 return NULL; 689 690 return addr; 691 } 692 693 struct extmem_param { 694 void *addr; 695 size_t len; 696 size_t pgsz; 697 rte_iova_t *iova_table; 698 unsigned int iova_table_len; 699 }; 700 701 static int 702 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 703 bool huge) 704 { 705 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 706 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 707 unsigned int cur_page, n_pages, pgsz_idx; 708 size_t mem_sz, cur_pgsz; 709 rte_iova_t *iovas = NULL; 710 void *addr; 711 int ret; 712 713 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 714 /* skip anything that is too big */ 715 if (pgsizes[pgsz_idx] > SIZE_MAX) 716 continue; 717 718 cur_pgsz = pgsizes[pgsz_idx]; 719 720 /* if we were told not to allocate hugepages, override */ 721 if (!huge) 722 cur_pgsz = sysconf(_SC_PAGESIZE); 723 724 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 725 if (ret < 0) { 726 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 727 return -1; 728 } 729 730 /* allocate our memory */ 731 addr = alloc_mem(mem_sz, cur_pgsz, huge); 732 733 /* if we couldn't allocate memory with a specified page size, 734 * that doesn't mean we can't do it with other page sizes, so 735 * try another one. 736 */ 737 if (addr == NULL) 738 continue; 739 740 /* store IOVA addresses for every page in this memory area */ 741 n_pages = mem_sz / cur_pgsz; 742 743 iovas = malloc(sizeof(*iovas) * n_pages); 744 745 if (iovas == NULL) { 746 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 747 goto fail; 748 } 749 /* lock memory if it's not huge pages */ 750 if (!huge) 751 mlock(addr, mem_sz); 752 753 /* populate IOVA addresses */ 754 for (cur_page = 0; cur_page < n_pages; cur_page++) { 755 rte_iova_t iova; 756 size_t offset; 757 void *cur; 758 759 offset = cur_pgsz * cur_page; 760 cur = RTE_PTR_ADD(addr, offset); 761 762 /* touch the page before getting its IOVA */ 763 *(volatile char *)cur = 0; 764 765 iova = rte_mem_virt2iova(cur); 766 767 iovas[cur_page] = iova; 768 } 769 770 break; 771 } 772 /* if we couldn't allocate anything */ 773 if (iovas == NULL) 774 return -1; 775 776 param->addr = addr; 777 param->len = mem_sz; 778 param->pgsz = cur_pgsz; 779 param->iova_table = iovas; 780 param->iova_table_len = n_pages; 781 782 return 0; 783 fail: 784 if (iovas) 785 free(iovas); 786 if (addr) 787 munmap(addr, mem_sz); 788 789 return -1; 790 } 791 792 static int 793 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 794 { 795 struct extmem_param param; 796 int socket_id, ret; 797 798 memset(¶m, 0, sizeof(param)); 799 800 /* check if our heap exists */ 801 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 802 if (socket_id < 0) { 803 /* create our heap */ 804 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 805 if (ret < 0) { 806 TESTPMD_LOG(ERR, "Cannot create heap\n"); 807 return -1; 808 } 809 } 810 811 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 812 if (ret < 0) { 813 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 814 return -1; 815 } 816 817 /* we now have a valid memory area, so add it to heap */ 818 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 819 param.addr, param.len, param.iova_table, 820 param.iova_table_len, param.pgsz); 821 822 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 823 824 /* not needed any more */ 825 free(param.iova_table); 826 827 if (ret < 0) { 828 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 829 munmap(param.addr, param.len); 830 return -1; 831 } 832 833 /* success */ 834 835 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 836 param.len >> 20); 837 838 return 0; 839 } 840 static void 841 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 842 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 843 { 844 uint16_t pid = 0; 845 int ret; 846 847 RTE_ETH_FOREACH_DEV(pid) { 848 struct rte_eth_dev *dev = 849 &rte_eth_devices[pid]; 850 851 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 852 memhdr->len); 853 if (ret) { 854 TESTPMD_LOG(DEBUG, 855 "unable to DMA unmap addr 0x%p " 856 "for device %s\n", 857 memhdr->addr, dev->data->name); 858 } 859 } 860 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 861 if (ret) { 862 TESTPMD_LOG(DEBUG, 863 "unable to un-register addr 0x%p\n", memhdr->addr); 864 } 865 } 866 867 static void 868 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 869 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 870 { 871 uint16_t pid = 0; 872 size_t page_size = sysconf(_SC_PAGESIZE); 873 int ret; 874 875 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 876 page_size); 877 if (ret) { 878 TESTPMD_LOG(DEBUG, 879 "unable to register addr 0x%p\n", memhdr->addr); 880 return; 881 } 882 RTE_ETH_FOREACH_DEV(pid) { 883 struct rte_eth_dev *dev = 884 &rte_eth_devices[pid]; 885 886 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 887 memhdr->len); 888 if (ret) { 889 TESTPMD_LOG(DEBUG, 890 "unable to DMA map addr 0x%p " 891 "for device %s\n", 892 memhdr->addr, dev->data->name); 893 } 894 } 895 } 896 897 static unsigned int 898 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 899 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 900 { 901 struct rte_pktmbuf_extmem *xmem; 902 unsigned int ext_num, zone_num, elt_num; 903 uint16_t elt_size; 904 905 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 906 elt_num = EXTBUF_ZONE_SIZE / elt_size; 907 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 908 909 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 910 if (xmem == NULL) { 911 TESTPMD_LOG(ERR, "Cannot allocate memory for " 912 "external buffer descriptors\n"); 913 *ext_mem = NULL; 914 return 0; 915 } 916 for (ext_num = 0; ext_num < zone_num; ext_num++) { 917 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 918 const struct rte_memzone *mz; 919 char mz_name[RTE_MEMZONE_NAMESIZE]; 920 int ret; 921 922 ret = snprintf(mz_name, sizeof(mz_name), 923 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 924 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 925 errno = ENAMETOOLONG; 926 ext_num = 0; 927 break; 928 } 929 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 930 socket_id, 931 RTE_MEMZONE_IOVA_CONTIG | 932 RTE_MEMZONE_1GB | 933 RTE_MEMZONE_SIZE_HINT_ONLY, 934 EXTBUF_ZONE_SIZE); 935 if (mz == NULL) { 936 /* 937 * The caller exits on external buffer creation 938 * error, so there is no need to free memzones. 939 */ 940 errno = ENOMEM; 941 ext_num = 0; 942 break; 943 } 944 xseg->buf_ptr = mz->addr; 945 xseg->buf_iova = mz->iova; 946 xseg->buf_len = EXTBUF_ZONE_SIZE; 947 xseg->elt_size = elt_size; 948 } 949 if (ext_num == 0 && xmem != NULL) { 950 free(xmem); 951 xmem = NULL; 952 } 953 *ext_mem = xmem; 954 return ext_num; 955 } 956 957 /* 958 * Configuration initialisation done once at init time. 959 */ 960 static struct rte_mempool * 961 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 962 unsigned int socket_id, uint16_t size_idx) 963 { 964 char pool_name[RTE_MEMPOOL_NAMESIZE]; 965 struct rte_mempool *rte_mp = NULL; 966 uint32_t mb_size; 967 968 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 969 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 970 971 TESTPMD_LOG(INFO, 972 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 973 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 974 975 switch (mp_alloc_type) { 976 case MP_ALLOC_NATIVE: 977 { 978 /* wrapper to rte_mempool_create() */ 979 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 980 rte_mbuf_best_mempool_ops()); 981 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 982 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 983 break; 984 } 985 case MP_ALLOC_ANON: 986 { 987 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 988 mb_size, (unsigned int) mb_mempool_cache, 989 sizeof(struct rte_pktmbuf_pool_private), 990 socket_id, mempool_flags); 991 if (rte_mp == NULL) 992 goto err; 993 994 if (rte_mempool_populate_anon(rte_mp) == 0) { 995 rte_mempool_free(rte_mp); 996 rte_mp = NULL; 997 goto err; 998 } 999 rte_pktmbuf_pool_init(rte_mp, NULL); 1000 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1001 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1002 break; 1003 } 1004 case MP_ALLOC_XMEM: 1005 case MP_ALLOC_XMEM_HUGE: 1006 { 1007 int heap_socket; 1008 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1009 1010 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1011 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1012 1013 heap_socket = 1014 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1015 if (heap_socket < 0) 1016 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1017 1018 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1019 rte_mbuf_best_mempool_ops()); 1020 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1021 mb_mempool_cache, 0, mbuf_seg_size, 1022 heap_socket); 1023 break; 1024 } 1025 case MP_ALLOC_XBUF: 1026 { 1027 struct rte_pktmbuf_extmem *ext_mem; 1028 unsigned int ext_num; 1029 1030 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1031 socket_id, pool_name, &ext_mem); 1032 if (ext_num == 0) 1033 rte_exit(EXIT_FAILURE, 1034 "Can't create pinned data buffers\n"); 1035 1036 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1037 rte_mbuf_best_mempool_ops()); 1038 rte_mp = rte_pktmbuf_pool_create_extbuf 1039 (pool_name, nb_mbuf, mb_mempool_cache, 1040 0, mbuf_seg_size, socket_id, 1041 ext_mem, ext_num); 1042 free(ext_mem); 1043 break; 1044 } 1045 default: 1046 { 1047 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1048 } 1049 } 1050 1051 err: 1052 if (rte_mp == NULL) { 1053 rte_exit(EXIT_FAILURE, 1054 "Creation of mbuf pool for socket %u failed: %s\n", 1055 socket_id, rte_strerror(rte_errno)); 1056 } else if (verbose_level > 0) { 1057 rte_mempool_dump(stdout, rte_mp); 1058 } 1059 return rte_mp; 1060 } 1061 1062 /* 1063 * Check given socket id is valid or not with NUMA mode, 1064 * if valid, return 0, else return -1 1065 */ 1066 static int 1067 check_socket_id(const unsigned int socket_id) 1068 { 1069 static int warning_once = 0; 1070 1071 if (new_socket_id(socket_id)) { 1072 if (!warning_once && numa_support) 1073 printf("Warning: NUMA should be configured manually by" 1074 " using --port-numa-config and" 1075 " --ring-numa-config parameters along with" 1076 " --numa.\n"); 1077 warning_once = 1; 1078 return -1; 1079 } 1080 return 0; 1081 } 1082 1083 /* 1084 * Get the allowed maximum number of RX queues. 1085 * *pid return the port id which has minimal value of 1086 * max_rx_queues in all ports. 1087 */ 1088 queueid_t 1089 get_allowed_max_nb_rxq(portid_t *pid) 1090 { 1091 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1092 bool max_rxq_valid = false; 1093 portid_t pi; 1094 struct rte_eth_dev_info dev_info; 1095 1096 RTE_ETH_FOREACH_DEV(pi) { 1097 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1098 continue; 1099 1100 max_rxq_valid = true; 1101 if (dev_info.max_rx_queues < allowed_max_rxq) { 1102 allowed_max_rxq = dev_info.max_rx_queues; 1103 *pid = pi; 1104 } 1105 } 1106 return max_rxq_valid ? allowed_max_rxq : 0; 1107 } 1108 1109 /* 1110 * Check input rxq is valid or not. 1111 * If input rxq is not greater than any of maximum number 1112 * of RX queues of all ports, it is valid. 1113 * if valid, return 0, else return -1 1114 */ 1115 int 1116 check_nb_rxq(queueid_t rxq) 1117 { 1118 queueid_t allowed_max_rxq; 1119 portid_t pid = 0; 1120 1121 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1122 if (rxq > allowed_max_rxq) { 1123 printf("Fail: input rxq (%u) can't be greater " 1124 "than max_rx_queues (%u) of port %u\n", 1125 rxq, 1126 allowed_max_rxq, 1127 pid); 1128 return -1; 1129 } 1130 return 0; 1131 } 1132 1133 /* 1134 * Get the allowed maximum number of TX queues. 1135 * *pid return the port id which has minimal value of 1136 * max_tx_queues in all ports. 1137 */ 1138 queueid_t 1139 get_allowed_max_nb_txq(portid_t *pid) 1140 { 1141 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1142 bool max_txq_valid = false; 1143 portid_t pi; 1144 struct rte_eth_dev_info dev_info; 1145 1146 RTE_ETH_FOREACH_DEV(pi) { 1147 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1148 continue; 1149 1150 max_txq_valid = true; 1151 if (dev_info.max_tx_queues < allowed_max_txq) { 1152 allowed_max_txq = dev_info.max_tx_queues; 1153 *pid = pi; 1154 } 1155 } 1156 return max_txq_valid ? allowed_max_txq : 0; 1157 } 1158 1159 /* 1160 * Check input txq is valid or not. 1161 * If input txq is not greater than any of maximum number 1162 * of TX queues of all ports, it is valid. 1163 * if valid, return 0, else return -1 1164 */ 1165 int 1166 check_nb_txq(queueid_t txq) 1167 { 1168 queueid_t allowed_max_txq; 1169 portid_t pid = 0; 1170 1171 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1172 if (txq > allowed_max_txq) { 1173 printf("Fail: input txq (%u) can't be greater " 1174 "than max_tx_queues (%u) of port %u\n", 1175 txq, 1176 allowed_max_txq, 1177 pid); 1178 return -1; 1179 } 1180 return 0; 1181 } 1182 1183 /* 1184 * Get the allowed maximum number of RXDs of every rx queue. 1185 * *pid return the port id which has minimal value of 1186 * max_rxd in all queues of all ports. 1187 */ 1188 static uint16_t 1189 get_allowed_max_nb_rxd(portid_t *pid) 1190 { 1191 uint16_t allowed_max_rxd = UINT16_MAX; 1192 portid_t pi; 1193 struct rte_eth_dev_info dev_info; 1194 1195 RTE_ETH_FOREACH_DEV(pi) { 1196 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1197 continue; 1198 1199 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1200 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1201 *pid = pi; 1202 } 1203 } 1204 return allowed_max_rxd; 1205 } 1206 1207 /* 1208 * Get the allowed minimal number of RXDs of every rx queue. 1209 * *pid return the port id which has minimal value of 1210 * min_rxd in all queues of all ports. 1211 */ 1212 static uint16_t 1213 get_allowed_min_nb_rxd(portid_t *pid) 1214 { 1215 uint16_t allowed_min_rxd = 0; 1216 portid_t pi; 1217 struct rte_eth_dev_info dev_info; 1218 1219 RTE_ETH_FOREACH_DEV(pi) { 1220 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1221 continue; 1222 1223 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1224 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1225 *pid = pi; 1226 } 1227 } 1228 1229 return allowed_min_rxd; 1230 } 1231 1232 /* 1233 * Check input rxd is valid or not. 1234 * If input rxd is not greater than any of maximum number 1235 * of RXDs of every Rx queues and is not less than any of 1236 * minimal number of RXDs of every Rx queues, it is valid. 1237 * if valid, return 0, else return -1 1238 */ 1239 int 1240 check_nb_rxd(queueid_t rxd) 1241 { 1242 uint16_t allowed_max_rxd; 1243 uint16_t allowed_min_rxd; 1244 portid_t pid = 0; 1245 1246 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1247 if (rxd > allowed_max_rxd) { 1248 printf("Fail: input rxd (%u) can't be greater " 1249 "than max_rxds (%u) of port %u\n", 1250 rxd, 1251 allowed_max_rxd, 1252 pid); 1253 return -1; 1254 } 1255 1256 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1257 if (rxd < allowed_min_rxd) { 1258 printf("Fail: input rxd (%u) can't be less " 1259 "than min_rxds (%u) of port %u\n", 1260 rxd, 1261 allowed_min_rxd, 1262 pid); 1263 return -1; 1264 } 1265 1266 return 0; 1267 } 1268 1269 /* 1270 * Get the allowed maximum number of TXDs of every rx queues. 1271 * *pid return the port id which has minimal value of 1272 * max_txd in every tx queue. 1273 */ 1274 static uint16_t 1275 get_allowed_max_nb_txd(portid_t *pid) 1276 { 1277 uint16_t allowed_max_txd = UINT16_MAX; 1278 portid_t pi; 1279 struct rte_eth_dev_info dev_info; 1280 1281 RTE_ETH_FOREACH_DEV(pi) { 1282 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1283 continue; 1284 1285 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1286 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1287 *pid = pi; 1288 } 1289 } 1290 return allowed_max_txd; 1291 } 1292 1293 /* 1294 * Get the allowed maximum number of TXDs of every tx queues. 1295 * *pid return the port id which has minimal value of 1296 * min_txd in every tx queue. 1297 */ 1298 static uint16_t 1299 get_allowed_min_nb_txd(portid_t *pid) 1300 { 1301 uint16_t allowed_min_txd = 0; 1302 portid_t pi; 1303 struct rte_eth_dev_info dev_info; 1304 1305 RTE_ETH_FOREACH_DEV(pi) { 1306 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1307 continue; 1308 1309 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1310 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1311 *pid = pi; 1312 } 1313 } 1314 1315 return allowed_min_txd; 1316 } 1317 1318 /* 1319 * Check input txd is valid or not. 1320 * If input txd is not greater than any of maximum number 1321 * of TXDs of every Rx queues, it is valid. 1322 * if valid, return 0, else return -1 1323 */ 1324 int 1325 check_nb_txd(queueid_t txd) 1326 { 1327 uint16_t allowed_max_txd; 1328 uint16_t allowed_min_txd; 1329 portid_t pid = 0; 1330 1331 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1332 if (txd > allowed_max_txd) { 1333 printf("Fail: input txd (%u) can't be greater " 1334 "than max_txds (%u) of port %u\n", 1335 txd, 1336 allowed_max_txd, 1337 pid); 1338 return -1; 1339 } 1340 1341 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1342 if (txd < allowed_min_txd) { 1343 printf("Fail: input txd (%u) can't be less " 1344 "than min_txds (%u) of port %u\n", 1345 txd, 1346 allowed_min_txd, 1347 pid); 1348 return -1; 1349 } 1350 return 0; 1351 } 1352 1353 1354 /* 1355 * Get the allowed maximum number of hairpin queues. 1356 * *pid return the port id which has minimal value of 1357 * max_hairpin_queues in all ports. 1358 */ 1359 queueid_t 1360 get_allowed_max_nb_hairpinq(portid_t *pid) 1361 { 1362 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1363 portid_t pi; 1364 struct rte_eth_hairpin_cap cap; 1365 1366 RTE_ETH_FOREACH_DEV(pi) { 1367 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1368 *pid = pi; 1369 return 0; 1370 } 1371 if (cap.max_nb_queues < allowed_max_hairpinq) { 1372 allowed_max_hairpinq = cap.max_nb_queues; 1373 *pid = pi; 1374 } 1375 } 1376 return allowed_max_hairpinq; 1377 } 1378 1379 /* 1380 * Check input hairpin is valid or not. 1381 * If input hairpin is not greater than any of maximum number 1382 * of hairpin queues of all ports, it is valid. 1383 * if valid, return 0, else return -1 1384 */ 1385 int 1386 check_nb_hairpinq(queueid_t hairpinq) 1387 { 1388 queueid_t allowed_max_hairpinq; 1389 portid_t pid = 0; 1390 1391 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1392 if (hairpinq > allowed_max_hairpinq) { 1393 printf("Fail: input hairpin (%u) can't be greater " 1394 "than max_hairpin_queues (%u) of port %u\n", 1395 hairpinq, allowed_max_hairpinq, pid); 1396 return -1; 1397 } 1398 return 0; 1399 } 1400 1401 static void 1402 init_config(void) 1403 { 1404 portid_t pid; 1405 struct rte_port *port; 1406 struct rte_mempool *mbp; 1407 unsigned int nb_mbuf_per_pool; 1408 lcoreid_t lc_id; 1409 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1410 struct rte_gro_param gro_param; 1411 uint32_t gso_types; 1412 uint16_t data_size; 1413 bool warning = 0; 1414 int k; 1415 int ret; 1416 1417 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1418 1419 /* Configuration of logical cores. */ 1420 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1421 sizeof(struct fwd_lcore *) * nb_lcores, 1422 RTE_CACHE_LINE_SIZE); 1423 if (fwd_lcores == NULL) { 1424 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1425 "failed\n", nb_lcores); 1426 } 1427 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1428 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1429 sizeof(struct fwd_lcore), 1430 RTE_CACHE_LINE_SIZE); 1431 if (fwd_lcores[lc_id] == NULL) { 1432 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1433 "failed\n"); 1434 } 1435 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1436 } 1437 1438 RTE_ETH_FOREACH_DEV(pid) { 1439 port = &ports[pid]; 1440 /* Apply default TxRx configuration for all ports */ 1441 port->dev_conf.txmode = tx_mode; 1442 port->dev_conf.rxmode = rx_mode; 1443 1444 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1445 if (ret != 0) 1446 rte_exit(EXIT_FAILURE, 1447 "rte_eth_dev_info_get() failed\n"); 1448 1449 if (!(port->dev_info.tx_offload_capa & 1450 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1451 port->dev_conf.txmode.offloads &= 1452 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1453 if (numa_support) { 1454 if (port_numa[pid] != NUMA_NO_CONFIG) 1455 port_per_socket[port_numa[pid]]++; 1456 else { 1457 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1458 1459 /* 1460 * if socket_id is invalid, 1461 * set to the first available socket. 1462 */ 1463 if (check_socket_id(socket_id) < 0) 1464 socket_id = socket_ids[0]; 1465 port_per_socket[socket_id]++; 1466 } 1467 } 1468 1469 /* Apply Rx offloads configuration */ 1470 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1471 port->rx_conf[k].offloads = 1472 port->dev_conf.rxmode.offloads; 1473 /* Apply Tx offloads configuration */ 1474 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1475 port->tx_conf[k].offloads = 1476 port->dev_conf.txmode.offloads; 1477 1478 /* set flag to initialize port/queue */ 1479 port->need_reconfig = 1; 1480 port->need_reconfig_queues = 1; 1481 port->tx_metadata = 0; 1482 1483 /* Check for maximum number of segments per MTU. Accordingly 1484 * update the mbuf data size. 1485 */ 1486 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1487 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1488 data_size = rx_mode.max_rx_pkt_len / 1489 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1490 1491 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1492 mbuf_data_size[0]) { 1493 mbuf_data_size[0] = data_size + 1494 RTE_PKTMBUF_HEADROOM; 1495 warning = 1; 1496 } 1497 } 1498 } 1499 1500 if (warning) 1501 TESTPMD_LOG(WARNING, 1502 "Configured mbuf size of the first segment %hu\n", 1503 mbuf_data_size[0]); 1504 /* 1505 * Create pools of mbuf. 1506 * If NUMA support is disabled, create a single pool of mbuf in 1507 * socket 0 memory by default. 1508 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1509 * 1510 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1511 * nb_txd can be configured at run time. 1512 */ 1513 if (param_total_num_mbufs) 1514 nb_mbuf_per_pool = param_total_num_mbufs; 1515 else { 1516 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1517 (nb_lcores * mb_mempool_cache) + 1518 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1519 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1520 } 1521 1522 if (numa_support) { 1523 uint8_t i, j; 1524 1525 for (i = 0; i < num_sockets; i++) 1526 for (j = 0; j < mbuf_data_size_n; j++) 1527 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1528 mbuf_pool_create(mbuf_data_size[j], 1529 nb_mbuf_per_pool, 1530 socket_ids[i], j); 1531 } else { 1532 uint8_t i; 1533 1534 for (i = 0; i < mbuf_data_size_n; i++) 1535 mempools[i] = mbuf_pool_create 1536 (mbuf_data_size[i], 1537 nb_mbuf_per_pool, 1538 socket_num == UMA_NO_CONFIG ? 1539 0 : socket_num, i); 1540 } 1541 1542 init_port_config(); 1543 1544 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1545 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1546 /* 1547 * Records which Mbuf pool to use by each logical core, if needed. 1548 */ 1549 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1550 mbp = mbuf_pool_find( 1551 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1552 1553 if (mbp == NULL) 1554 mbp = mbuf_pool_find(0, 0); 1555 fwd_lcores[lc_id]->mbp = mbp; 1556 /* initialize GSO context */ 1557 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1558 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1559 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1560 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1561 RTE_ETHER_CRC_LEN; 1562 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1563 } 1564 1565 /* Configuration of packet forwarding streams. */ 1566 if (init_fwd_streams() < 0) 1567 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1568 1569 fwd_config_setup(); 1570 1571 /* create a gro context for each lcore */ 1572 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1573 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1574 gro_param.max_item_per_flow = MAX_PKT_BURST; 1575 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1576 gro_param.socket_id = rte_lcore_to_socket_id( 1577 fwd_lcores_cpuids[lc_id]); 1578 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1579 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1580 rte_exit(EXIT_FAILURE, 1581 "rte_gro_ctx_create() failed\n"); 1582 } 1583 } 1584 } 1585 1586 1587 void 1588 reconfig(portid_t new_port_id, unsigned socket_id) 1589 { 1590 struct rte_port *port; 1591 int ret; 1592 1593 /* Reconfiguration of Ethernet ports. */ 1594 port = &ports[new_port_id]; 1595 1596 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1597 if (ret != 0) 1598 return; 1599 1600 /* set flag to initialize port/queue */ 1601 port->need_reconfig = 1; 1602 port->need_reconfig_queues = 1; 1603 port->socket_id = socket_id; 1604 1605 init_port_config(); 1606 } 1607 1608 1609 int 1610 init_fwd_streams(void) 1611 { 1612 portid_t pid; 1613 struct rte_port *port; 1614 streamid_t sm_id, nb_fwd_streams_new; 1615 queueid_t q; 1616 1617 /* set socket id according to numa or not */ 1618 RTE_ETH_FOREACH_DEV(pid) { 1619 port = &ports[pid]; 1620 if (nb_rxq > port->dev_info.max_rx_queues) { 1621 printf("Fail: nb_rxq(%d) is greater than " 1622 "max_rx_queues(%d)\n", nb_rxq, 1623 port->dev_info.max_rx_queues); 1624 return -1; 1625 } 1626 if (nb_txq > port->dev_info.max_tx_queues) { 1627 printf("Fail: nb_txq(%d) is greater than " 1628 "max_tx_queues(%d)\n", nb_txq, 1629 port->dev_info.max_tx_queues); 1630 return -1; 1631 } 1632 if (numa_support) { 1633 if (port_numa[pid] != NUMA_NO_CONFIG) 1634 port->socket_id = port_numa[pid]; 1635 else { 1636 port->socket_id = rte_eth_dev_socket_id(pid); 1637 1638 /* 1639 * if socket_id is invalid, 1640 * set to the first available socket. 1641 */ 1642 if (check_socket_id(port->socket_id) < 0) 1643 port->socket_id = socket_ids[0]; 1644 } 1645 } 1646 else { 1647 if (socket_num == UMA_NO_CONFIG) 1648 port->socket_id = 0; 1649 else 1650 port->socket_id = socket_num; 1651 } 1652 } 1653 1654 q = RTE_MAX(nb_rxq, nb_txq); 1655 if (q == 0) { 1656 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1657 return -1; 1658 } 1659 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1660 if (nb_fwd_streams_new == nb_fwd_streams) 1661 return 0; 1662 /* clear the old */ 1663 if (fwd_streams != NULL) { 1664 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1665 if (fwd_streams[sm_id] == NULL) 1666 continue; 1667 rte_free(fwd_streams[sm_id]); 1668 fwd_streams[sm_id] = NULL; 1669 } 1670 rte_free(fwd_streams); 1671 fwd_streams = NULL; 1672 } 1673 1674 /* init new */ 1675 nb_fwd_streams = nb_fwd_streams_new; 1676 if (nb_fwd_streams) { 1677 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1678 sizeof(struct fwd_stream *) * nb_fwd_streams, 1679 RTE_CACHE_LINE_SIZE); 1680 if (fwd_streams == NULL) 1681 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1682 " (struct fwd_stream *)) failed\n", 1683 nb_fwd_streams); 1684 1685 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1686 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1687 " struct fwd_stream", sizeof(struct fwd_stream), 1688 RTE_CACHE_LINE_SIZE); 1689 if (fwd_streams[sm_id] == NULL) 1690 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1691 "(struct fwd_stream) failed\n"); 1692 } 1693 } 1694 1695 return 0; 1696 } 1697 1698 static void 1699 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1700 { 1701 uint64_t total_burst, sburst; 1702 uint64_t nb_burst; 1703 uint64_t burst_stats[4]; 1704 uint16_t pktnb_stats[4]; 1705 uint16_t nb_pkt; 1706 int burst_percent[4], sburstp; 1707 int i; 1708 1709 /* 1710 * First compute the total number of packet bursts and the 1711 * two highest numbers of bursts of the same number of packets. 1712 */ 1713 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1714 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1715 1716 /* Show stats for 0 burst size always */ 1717 total_burst = pbs->pkt_burst_spread[0]; 1718 burst_stats[0] = pbs->pkt_burst_spread[0]; 1719 pktnb_stats[0] = 0; 1720 1721 /* Find the next 2 burst sizes with highest occurrences. */ 1722 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1723 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1724 1725 if (nb_burst == 0) 1726 continue; 1727 1728 total_burst += nb_burst; 1729 1730 if (nb_burst > burst_stats[1]) { 1731 burst_stats[2] = burst_stats[1]; 1732 pktnb_stats[2] = pktnb_stats[1]; 1733 burst_stats[1] = nb_burst; 1734 pktnb_stats[1] = nb_pkt; 1735 } else if (nb_burst > burst_stats[2]) { 1736 burst_stats[2] = nb_burst; 1737 pktnb_stats[2] = nb_pkt; 1738 } 1739 } 1740 if (total_burst == 0) 1741 return; 1742 1743 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1744 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1745 if (i == 3) { 1746 printf("%d%% of other]\n", 100 - sburstp); 1747 return; 1748 } 1749 1750 sburst += burst_stats[i]; 1751 if (sburst == total_burst) { 1752 printf("%d%% of %d pkts]\n", 1753 100 - sburstp, (int) pktnb_stats[i]); 1754 return; 1755 } 1756 1757 burst_percent[i] = 1758 (double)burst_stats[i] / total_burst * 100; 1759 printf("%d%% of %d pkts + ", 1760 burst_percent[i], (int) pktnb_stats[i]); 1761 sburstp += burst_percent[i]; 1762 } 1763 } 1764 1765 static void 1766 fwd_stream_stats_display(streamid_t stream_id) 1767 { 1768 struct fwd_stream *fs; 1769 static const char *fwd_top_stats_border = "-------"; 1770 1771 fs = fwd_streams[stream_id]; 1772 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1773 (fs->fwd_dropped == 0)) 1774 return; 1775 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1776 "TX Port=%2d/Queue=%2d %s\n", 1777 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1778 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1779 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1780 " TX-dropped: %-14"PRIu64, 1781 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1782 1783 /* if checksum mode */ 1784 if (cur_fwd_eng == &csum_fwd_engine) { 1785 printf(" RX- bad IP checksum: %-14"PRIu64 1786 " Rx- bad L4 checksum: %-14"PRIu64 1787 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1788 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1789 fs->rx_bad_outer_l4_csum); 1790 } else { 1791 printf("\n"); 1792 } 1793 1794 if (record_burst_stats) { 1795 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1796 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1797 } 1798 } 1799 1800 void 1801 fwd_stats_display(void) 1802 { 1803 static const char *fwd_stats_border = "----------------------"; 1804 static const char *acc_stats_border = "+++++++++++++++"; 1805 struct { 1806 struct fwd_stream *rx_stream; 1807 struct fwd_stream *tx_stream; 1808 uint64_t tx_dropped; 1809 uint64_t rx_bad_ip_csum; 1810 uint64_t rx_bad_l4_csum; 1811 uint64_t rx_bad_outer_l4_csum; 1812 } ports_stats[RTE_MAX_ETHPORTS]; 1813 uint64_t total_rx_dropped = 0; 1814 uint64_t total_tx_dropped = 0; 1815 uint64_t total_rx_nombuf = 0; 1816 struct rte_eth_stats stats; 1817 uint64_t fwd_cycles = 0; 1818 uint64_t total_recv = 0; 1819 uint64_t total_xmit = 0; 1820 struct rte_port *port; 1821 streamid_t sm_id; 1822 portid_t pt_id; 1823 int i; 1824 1825 memset(ports_stats, 0, sizeof(ports_stats)); 1826 1827 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1828 struct fwd_stream *fs = fwd_streams[sm_id]; 1829 1830 if (cur_fwd_config.nb_fwd_streams > 1831 cur_fwd_config.nb_fwd_ports) { 1832 fwd_stream_stats_display(sm_id); 1833 } else { 1834 ports_stats[fs->tx_port].tx_stream = fs; 1835 ports_stats[fs->rx_port].rx_stream = fs; 1836 } 1837 1838 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1839 1840 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1841 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1842 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1843 fs->rx_bad_outer_l4_csum; 1844 1845 if (record_core_cycles) 1846 fwd_cycles += fs->core_cycles; 1847 } 1848 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1849 pt_id = fwd_ports_ids[i]; 1850 port = &ports[pt_id]; 1851 1852 rte_eth_stats_get(pt_id, &stats); 1853 stats.ipackets -= port->stats.ipackets; 1854 stats.opackets -= port->stats.opackets; 1855 stats.ibytes -= port->stats.ibytes; 1856 stats.obytes -= port->stats.obytes; 1857 stats.imissed -= port->stats.imissed; 1858 stats.oerrors -= port->stats.oerrors; 1859 stats.rx_nombuf -= port->stats.rx_nombuf; 1860 1861 total_recv += stats.ipackets; 1862 total_xmit += stats.opackets; 1863 total_rx_dropped += stats.imissed; 1864 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1865 total_tx_dropped += stats.oerrors; 1866 total_rx_nombuf += stats.rx_nombuf; 1867 1868 printf("\n %s Forward statistics for port %-2d %s\n", 1869 fwd_stats_border, pt_id, fwd_stats_border); 1870 1871 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 1872 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 1873 stats.ipackets + stats.imissed); 1874 1875 if (cur_fwd_eng == &csum_fwd_engine) 1876 printf(" Bad-ipcsum: %-14"PRIu64 1877 " Bad-l4csum: %-14"PRIu64 1878 "Bad-outer-l4csum: %-14"PRIu64"\n", 1879 ports_stats[pt_id].rx_bad_ip_csum, 1880 ports_stats[pt_id].rx_bad_l4_csum, 1881 ports_stats[pt_id].rx_bad_outer_l4_csum); 1882 if (stats.ierrors + stats.rx_nombuf > 0) { 1883 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 1884 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 1885 } 1886 1887 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 1888 "TX-total: %-"PRIu64"\n", 1889 stats.opackets, ports_stats[pt_id].tx_dropped, 1890 stats.opackets + ports_stats[pt_id].tx_dropped); 1891 1892 if (record_burst_stats) { 1893 if (ports_stats[pt_id].rx_stream) 1894 pkt_burst_stats_display("RX", 1895 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1896 if (ports_stats[pt_id].tx_stream) 1897 pkt_burst_stats_display("TX", 1898 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1899 } 1900 1901 printf(" %s--------------------------------%s\n", 1902 fwd_stats_border, fwd_stats_border); 1903 } 1904 1905 printf("\n %s Accumulated forward statistics for all ports" 1906 "%s\n", 1907 acc_stats_border, acc_stats_border); 1908 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1909 "%-"PRIu64"\n" 1910 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1911 "%-"PRIu64"\n", 1912 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1913 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1914 if (total_rx_nombuf > 0) 1915 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1916 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1917 "%s\n", 1918 acc_stats_border, acc_stats_border); 1919 if (record_core_cycles) { 1920 #define CYC_PER_MHZ 1E6 1921 if (total_recv > 0 || total_xmit > 0) { 1922 uint64_t total_pkts = 0; 1923 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 1924 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 1925 total_pkts = total_xmit; 1926 else 1927 total_pkts = total_recv; 1928 1929 printf("\n CPU cycles/packet=%.2F (total cycles=" 1930 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 1931 " MHz Clock\n", 1932 (double) fwd_cycles / total_pkts, 1933 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 1934 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1935 } 1936 } 1937 } 1938 1939 void 1940 fwd_stats_reset(void) 1941 { 1942 streamid_t sm_id; 1943 portid_t pt_id; 1944 int i; 1945 1946 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1947 pt_id = fwd_ports_ids[i]; 1948 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1949 } 1950 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1951 struct fwd_stream *fs = fwd_streams[sm_id]; 1952 1953 fs->rx_packets = 0; 1954 fs->tx_packets = 0; 1955 fs->fwd_dropped = 0; 1956 fs->rx_bad_ip_csum = 0; 1957 fs->rx_bad_l4_csum = 0; 1958 fs->rx_bad_outer_l4_csum = 0; 1959 1960 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1961 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1962 fs->core_cycles = 0; 1963 } 1964 } 1965 1966 static void 1967 flush_fwd_rx_queues(void) 1968 { 1969 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1970 portid_t rxp; 1971 portid_t port_id; 1972 queueid_t rxq; 1973 uint16_t nb_rx; 1974 uint16_t i; 1975 uint8_t j; 1976 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1977 uint64_t timer_period; 1978 1979 /* convert to number of cycles */ 1980 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1981 1982 for (j = 0; j < 2; j++) { 1983 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1984 for (rxq = 0; rxq < nb_rxq; rxq++) { 1985 port_id = fwd_ports_ids[rxp]; 1986 /** 1987 * testpmd can stuck in the below do while loop 1988 * if rte_eth_rx_burst() always returns nonzero 1989 * packets. So timer is added to exit this loop 1990 * after 1sec timer expiry. 1991 */ 1992 prev_tsc = rte_rdtsc(); 1993 do { 1994 nb_rx = rte_eth_rx_burst(port_id, rxq, 1995 pkts_burst, MAX_PKT_BURST); 1996 for (i = 0; i < nb_rx; i++) 1997 rte_pktmbuf_free(pkts_burst[i]); 1998 1999 cur_tsc = rte_rdtsc(); 2000 diff_tsc = cur_tsc - prev_tsc; 2001 timer_tsc += diff_tsc; 2002 } while ((nb_rx > 0) && 2003 (timer_tsc < timer_period)); 2004 timer_tsc = 0; 2005 } 2006 } 2007 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2008 } 2009 } 2010 2011 static void 2012 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2013 { 2014 struct fwd_stream **fsm; 2015 streamid_t nb_fs; 2016 streamid_t sm_id; 2017 #ifdef RTE_LIB_BITRATESTATS 2018 uint64_t tics_per_1sec; 2019 uint64_t tics_datum; 2020 uint64_t tics_current; 2021 uint16_t i, cnt_ports; 2022 2023 cnt_ports = nb_ports; 2024 tics_datum = rte_rdtsc(); 2025 tics_per_1sec = rte_get_timer_hz(); 2026 #endif 2027 fsm = &fwd_streams[fc->stream_idx]; 2028 nb_fs = fc->stream_nb; 2029 do { 2030 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2031 (*pkt_fwd)(fsm[sm_id]); 2032 #ifdef RTE_LIB_BITRATESTATS 2033 if (bitrate_enabled != 0 && 2034 bitrate_lcore_id == rte_lcore_id()) { 2035 tics_current = rte_rdtsc(); 2036 if (tics_current - tics_datum >= tics_per_1sec) { 2037 /* Periodic bitrate calculation */ 2038 for (i = 0; i < cnt_ports; i++) 2039 rte_stats_bitrate_calc(bitrate_data, 2040 ports_ids[i]); 2041 tics_datum = tics_current; 2042 } 2043 } 2044 #endif 2045 #ifdef RTE_LIB_LATENCYSTATS 2046 if (latencystats_enabled != 0 && 2047 latencystats_lcore_id == rte_lcore_id()) 2048 rte_latencystats_update(); 2049 #endif 2050 2051 } while (! fc->stopped); 2052 } 2053 2054 static int 2055 start_pkt_forward_on_core(void *fwd_arg) 2056 { 2057 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2058 cur_fwd_config.fwd_eng->packet_fwd); 2059 return 0; 2060 } 2061 2062 /* 2063 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2064 * Used to start communication flows in network loopback test configurations. 2065 */ 2066 static int 2067 run_one_txonly_burst_on_core(void *fwd_arg) 2068 { 2069 struct fwd_lcore *fwd_lc; 2070 struct fwd_lcore tmp_lcore; 2071 2072 fwd_lc = (struct fwd_lcore *) fwd_arg; 2073 tmp_lcore = *fwd_lc; 2074 tmp_lcore.stopped = 1; 2075 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2076 return 0; 2077 } 2078 2079 /* 2080 * Launch packet forwarding: 2081 * - Setup per-port forwarding context. 2082 * - launch logical cores with their forwarding configuration. 2083 */ 2084 static void 2085 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2086 { 2087 port_fwd_begin_t port_fwd_begin; 2088 unsigned int i; 2089 unsigned int lc_id; 2090 int diag; 2091 2092 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2093 if (port_fwd_begin != NULL) { 2094 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2095 (*port_fwd_begin)(fwd_ports_ids[i]); 2096 } 2097 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2098 lc_id = fwd_lcores_cpuids[i]; 2099 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2100 fwd_lcores[i]->stopped = 0; 2101 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2102 fwd_lcores[i], lc_id); 2103 if (diag != 0) 2104 printf("launch lcore %u failed - diag=%d\n", 2105 lc_id, diag); 2106 } 2107 } 2108 } 2109 2110 /* 2111 * Launch packet forwarding configuration. 2112 */ 2113 void 2114 start_packet_forwarding(int with_tx_first) 2115 { 2116 port_fwd_begin_t port_fwd_begin; 2117 port_fwd_end_t port_fwd_end; 2118 struct rte_port *port; 2119 unsigned int i; 2120 portid_t pt_id; 2121 2122 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2123 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2124 2125 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2126 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2127 2128 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2129 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2130 (!nb_rxq || !nb_txq)) 2131 rte_exit(EXIT_FAILURE, 2132 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2133 cur_fwd_eng->fwd_mode_name); 2134 2135 if (all_ports_started() == 0) { 2136 printf("Not all ports were started\n"); 2137 return; 2138 } 2139 if (test_done == 0) { 2140 printf("Packet forwarding already started\n"); 2141 return; 2142 } 2143 2144 2145 if(dcb_test) { 2146 for (i = 0; i < nb_fwd_ports; i++) { 2147 pt_id = fwd_ports_ids[i]; 2148 port = &ports[pt_id]; 2149 if (!port->dcb_flag) { 2150 printf("In DCB mode, all forwarding ports must " 2151 "be configured in this mode.\n"); 2152 return; 2153 } 2154 } 2155 if (nb_fwd_lcores == 1) { 2156 printf("In DCB mode,the nb forwarding cores " 2157 "should be larger than 1.\n"); 2158 return; 2159 } 2160 } 2161 test_done = 0; 2162 2163 fwd_config_setup(); 2164 2165 if(!no_flush_rx) 2166 flush_fwd_rx_queues(); 2167 2168 pkt_fwd_config_display(&cur_fwd_config); 2169 rxtx_config_display(); 2170 2171 fwd_stats_reset(); 2172 if (with_tx_first) { 2173 port_fwd_begin = tx_only_engine.port_fwd_begin; 2174 if (port_fwd_begin != NULL) { 2175 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2176 (*port_fwd_begin)(fwd_ports_ids[i]); 2177 } 2178 while (with_tx_first--) { 2179 launch_packet_forwarding( 2180 run_one_txonly_burst_on_core); 2181 rte_eal_mp_wait_lcore(); 2182 } 2183 port_fwd_end = tx_only_engine.port_fwd_end; 2184 if (port_fwd_end != NULL) { 2185 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2186 (*port_fwd_end)(fwd_ports_ids[i]); 2187 } 2188 } 2189 launch_packet_forwarding(start_pkt_forward_on_core); 2190 } 2191 2192 void 2193 stop_packet_forwarding(void) 2194 { 2195 port_fwd_end_t port_fwd_end; 2196 lcoreid_t lc_id; 2197 portid_t pt_id; 2198 int i; 2199 2200 if (test_done) { 2201 printf("Packet forwarding not started\n"); 2202 return; 2203 } 2204 printf("Telling cores to stop..."); 2205 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2206 fwd_lcores[lc_id]->stopped = 1; 2207 printf("\nWaiting for lcores to finish...\n"); 2208 rte_eal_mp_wait_lcore(); 2209 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2210 if (port_fwd_end != NULL) { 2211 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2212 pt_id = fwd_ports_ids[i]; 2213 (*port_fwd_end)(pt_id); 2214 } 2215 } 2216 2217 fwd_stats_display(); 2218 2219 printf("\nDone.\n"); 2220 test_done = 1; 2221 } 2222 2223 void 2224 dev_set_link_up(portid_t pid) 2225 { 2226 if (rte_eth_dev_set_link_up(pid) < 0) 2227 printf("\nSet link up fail.\n"); 2228 } 2229 2230 void 2231 dev_set_link_down(portid_t pid) 2232 { 2233 if (rte_eth_dev_set_link_down(pid) < 0) 2234 printf("\nSet link down fail.\n"); 2235 } 2236 2237 static int 2238 all_ports_started(void) 2239 { 2240 portid_t pi; 2241 struct rte_port *port; 2242 2243 RTE_ETH_FOREACH_DEV(pi) { 2244 port = &ports[pi]; 2245 /* Check if there is a port which is not started */ 2246 if ((port->port_status != RTE_PORT_STARTED) && 2247 (port->slave_flag == 0)) 2248 return 0; 2249 } 2250 2251 /* No port is not started */ 2252 return 1; 2253 } 2254 2255 int 2256 port_is_stopped(portid_t port_id) 2257 { 2258 struct rte_port *port = &ports[port_id]; 2259 2260 if ((port->port_status != RTE_PORT_STOPPED) && 2261 (port->slave_flag == 0)) 2262 return 0; 2263 return 1; 2264 } 2265 2266 int 2267 all_ports_stopped(void) 2268 { 2269 portid_t pi; 2270 2271 RTE_ETH_FOREACH_DEV(pi) { 2272 if (!port_is_stopped(pi)) 2273 return 0; 2274 } 2275 2276 return 1; 2277 } 2278 2279 int 2280 port_is_started(portid_t port_id) 2281 { 2282 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2283 return 0; 2284 2285 if (ports[port_id].port_status != RTE_PORT_STARTED) 2286 return 0; 2287 2288 return 1; 2289 } 2290 2291 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2292 static int 2293 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2294 { 2295 queueid_t qi; 2296 struct rte_eth_hairpin_conf hairpin_conf = { 2297 .peer_count = 1, 2298 }; 2299 int i; 2300 int diag; 2301 struct rte_port *port = &ports[pi]; 2302 uint16_t peer_rx_port = pi; 2303 uint16_t peer_tx_port = pi; 2304 uint32_t manual = 1; 2305 uint32_t tx_exp = hairpin_mode & 0x10; 2306 2307 if (!(hairpin_mode & 0xf)) { 2308 peer_rx_port = pi; 2309 peer_tx_port = pi; 2310 manual = 0; 2311 } else if (hairpin_mode & 0x1) { 2312 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2313 RTE_ETH_DEV_NO_OWNER); 2314 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2315 peer_tx_port = rte_eth_find_next_owned_by(0, 2316 RTE_ETH_DEV_NO_OWNER); 2317 if (p_pi != RTE_MAX_ETHPORTS) { 2318 peer_rx_port = p_pi; 2319 } else { 2320 uint16_t next_pi; 2321 2322 /* Last port will be the peer RX port of the first. */ 2323 RTE_ETH_FOREACH_DEV(next_pi) 2324 peer_rx_port = next_pi; 2325 } 2326 manual = 1; 2327 } else if (hairpin_mode & 0x2) { 2328 if (cnt_pi & 0x1) { 2329 peer_rx_port = p_pi; 2330 } else { 2331 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2332 RTE_ETH_DEV_NO_OWNER); 2333 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2334 peer_rx_port = pi; 2335 } 2336 peer_tx_port = peer_rx_port; 2337 manual = 1; 2338 } 2339 2340 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2341 hairpin_conf.peers[0].port = peer_rx_port; 2342 hairpin_conf.peers[0].queue = i + nb_rxq; 2343 hairpin_conf.manual_bind = !!manual; 2344 hairpin_conf.tx_explicit = !!tx_exp; 2345 diag = rte_eth_tx_hairpin_queue_setup 2346 (pi, qi, nb_txd, &hairpin_conf); 2347 i++; 2348 if (diag == 0) 2349 continue; 2350 2351 /* Fail to setup rx queue, return */ 2352 if (rte_atomic16_cmpset(&(port->port_status), 2353 RTE_PORT_HANDLING, 2354 RTE_PORT_STOPPED) == 0) 2355 printf("Port %d can not be set back " 2356 "to stopped\n", pi); 2357 printf("Fail to configure port %d hairpin " 2358 "queues\n", pi); 2359 /* try to reconfigure queues next time */ 2360 port->need_reconfig_queues = 1; 2361 return -1; 2362 } 2363 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2364 hairpin_conf.peers[0].port = peer_tx_port; 2365 hairpin_conf.peers[0].queue = i + nb_txq; 2366 hairpin_conf.manual_bind = !!manual; 2367 hairpin_conf.tx_explicit = !!tx_exp; 2368 diag = rte_eth_rx_hairpin_queue_setup 2369 (pi, qi, nb_rxd, &hairpin_conf); 2370 i++; 2371 if (diag == 0) 2372 continue; 2373 2374 /* Fail to setup rx queue, return */ 2375 if (rte_atomic16_cmpset(&(port->port_status), 2376 RTE_PORT_HANDLING, 2377 RTE_PORT_STOPPED) == 0) 2378 printf("Port %d can not be set back " 2379 "to stopped\n", pi); 2380 printf("Fail to configure port %d hairpin " 2381 "queues\n", pi); 2382 /* try to reconfigure queues next time */ 2383 port->need_reconfig_queues = 1; 2384 return -1; 2385 } 2386 return 0; 2387 } 2388 2389 /* Configure the Rx with optional split. */ 2390 int 2391 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2392 uint16_t nb_rx_desc, unsigned int socket_id, 2393 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2394 { 2395 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2396 unsigned int i, mp_n; 2397 int ret; 2398 2399 if (rx_pkt_nb_segs <= 1 || 2400 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2401 rx_conf->rx_seg = NULL; 2402 rx_conf->rx_nseg = 0; 2403 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2404 nb_rx_desc, socket_id, 2405 rx_conf, mp); 2406 return ret; 2407 } 2408 for (i = 0; i < rx_pkt_nb_segs; i++) { 2409 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2410 struct rte_mempool *mpx; 2411 /* 2412 * Use last valid pool for the segments with number 2413 * exceeding the pool index. 2414 */ 2415 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2416 mpx = mbuf_pool_find(socket_id, mp_n); 2417 /* Handle zero as mbuf data buffer size. */ 2418 rx_seg->length = rx_pkt_seg_lengths[i] ? 2419 rx_pkt_seg_lengths[i] : 2420 mbuf_data_size[mp_n]; 2421 rx_seg->offset = i < rx_pkt_nb_offs ? 2422 rx_pkt_seg_offsets[i] : 0; 2423 rx_seg->mp = mpx ? mpx : mp; 2424 } 2425 rx_conf->rx_nseg = rx_pkt_nb_segs; 2426 rx_conf->rx_seg = rx_useg; 2427 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2428 socket_id, rx_conf, NULL); 2429 rx_conf->rx_seg = NULL; 2430 rx_conf->rx_nseg = 0; 2431 return ret; 2432 } 2433 2434 int 2435 start_port(portid_t pid) 2436 { 2437 int diag, need_check_link_status = -1; 2438 portid_t pi; 2439 portid_t p_pi = RTE_MAX_ETHPORTS; 2440 portid_t pl[RTE_MAX_ETHPORTS]; 2441 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2442 uint16_t cnt_pi = 0; 2443 uint16_t cfg_pi = 0; 2444 int peer_pi; 2445 queueid_t qi; 2446 struct rte_port *port; 2447 struct rte_ether_addr mac_addr; 2448 struct rte_eth_hairpin_cap cap; 2449 2450 if (port_id_is_invalid(pid, ENABLED_WARN)) 2451 return 0; 2452 2453 if(dcb_config) 2454 dcb_test = 1; 2455 RTE_ETH_FOREACH_DEV(pi) { 2456 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2457 continue; 2458 2459 need_check_link_status = 0; 2460 port = &ports[pi]; 2461 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2462 RTE_PORT_HANDLING) == 0) { 2463 printf("Port %d is now not stopped\n", pi); 2464 continue; 2465 } 2466 2467 if (port->need_reconfig > 0) { 2468 port->need_reconfig = 0; 2469 2470 if (flow_isolate_all) { 2471 int ret = port_flow_isolate(pi, 1); 2472 if (ret) { 2473 printf("Failed to apply isolated" 2474 " mode on port %d\n", pi); 2475 return -1; 2476 } 2477 } 2478 configure_rxtx_dump_callbacks(0); 2479 printf("Configuring Port %d (socket %u)\n", pi, 2480 port->socket_id); 2481 if (nb_hairpinq > 0 && 2482 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2483 printf("Port %d doesn't support hairpin " 2484 "queues\n", pi); 2485 return -1; 2486 } 2487 /* configure port */ 2488 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2489 nb_txq + nb_hairpinq, 2490 &(port->dev_conf)); 2491 if (diag != 0) { 2492 if (rte_atomic16_cmpset(&(port->port_status), 2493 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2494 printf("Port %d can not be set back " 2495 "to stopped\n", pi); 2496 printf("Fail to configure port %d\n", pi); 2497 /* try to reconfigure port next time */ 2498 port->need_reconfig = 1; 2499 return -1; 2500 } 2501 } 2502 if (port->need_reconfig_queues > 0) { 2503 port->need_reconfig_queues = 0; 2504 /* setup tx queues */ 2505 for (qi = 0; qi < nb_txq; qi++) { 2506 if ((numa_support) && 2507 (txring_numa[pi] != NUMA_NO_CONFIG)) 2508 diag = rte_eth_tx_queue_setup(pi, qi, 2509 port->nb_tx_desc[qi], 2510 txring_numa[pi], 2511 &(port->tx_conf[qi])); 2512 else 2513 diag = rte_eth_tx_queue_setup(pi, qi, 2514 port->nb_tx_desc[qi], 2515 port->socket_id, 2516 &(port->tx_conf[qi])); 2517 2518 if (diag == 0) 2519 continue; 2520 2521 /* Fail to setup tx queue, return */ 2522 if (rte_atomic16_cmpset(&(port->port_status), 2523 RTE_PORT_HANDLING, 2524 RTE_PORT_STOPPED) == 0) 2525 printf("Port %d can not be set back " 2526 "to stopped\n", pi); 2527 printf("Fail to configure port %d tx queues\n", 2528 pi); 2529 /* try to reconfigure queues next time */ 2530 port->need_reconfig_queues = 1; 2531 return -1; 2532 } 2533 for (qi = 0; qi < nb_rxq; qi++) { 2534 /* setup rx queues */ 2535 if ((numa_support) && 2536 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2537 struct rte_mempool * mp = 2538 mbuf_pool_find 2539 (rxring_numa[pi], 0); 2540 if (mp == NULL) { 2541 printf("Failed to setup RX queue:" 2542 "No mempool allocation" 2543 " on the socket %d\n", 2544 rxring_numa[pi]); 2545 return -1; 2546 } 2547 2548 diag = rx_queue_setup(pi, qi, 2549 port->nb_rx_desc[qi], 2550 rxring_numa[pi], 2551 &(port->rx_conf[qi]), 2552 mp); 2553 } else { 2554 struct rte_mempool *mp = 2555 mbuf_pool_find 2556 (port->socket_id, 0); 2557 if (mp == NULL) { 2558 printf("Failed to setup RX queue:" 2559 "No mempool allocation" 2560 " on the socket %d\n", 2561 port->socket_id); 2562 return -1; 2563 } 2564 diag = rx_queue_setup(pi, qi, 2565 port->nb_rx_desc[qi], 2566 port->socket_id, 2567 &(port->rx_conf[qi]), 2568 mp); 2569 } 2570 if (diag == 0) 2571 continue; 2572 2573 /* Fail to setup rx queue, return */ 2574 if (rte_atomic16_cmpset(&(port->port_status), 2575 RTE_PORT_HANDLING, 2576 RTE_PORT_STOPPED) == 0) 2577 printf("Port %d can not be set back " 2578 "to stopped\n", pi); 2579 printf("Fail to configure port %d rx queues\n", 2580 pi); 2581 /* try to reconfigure queues next time */ 2582 port->need_reconfig_queues = 1; 2583 return -1; 2584 } 2585 /* setup hairpin queues */ 2586 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2587 return -1; 2588 } 2589 configure_rxtx_dump_callbacks(verbose_level); 2590 if (clear_ptypes) { 2591 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2592 NULL, 0); 2593 if (diag < 0) 2594 printf( 2595 "Port %d: Failed to disable Ptype parsing\n", 2596 pi); 2597 } 2598 2599 p_pi = pi; 2600 cnt_pi++; 2601 2602 /* start port */ 2603 if (rte_eth_dev_start(pi) < 0) { 2604 printf("Fail to start port %d\n", pi); 2605 2606 /* Fail to setup rx queue, return */ 2607 if (rte_atomic16_cmpset(&(port->port_status), 2608 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2609 printf("Port %d can not be set back to " 2610 "stopped\n", pi); 2611 continue; 2612 } 2613 2614 if (rte_atomic16_cmpset(&(port->port_status), 2615 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2616 printf("Port %d can not be set into started\n", pi); 2617 2618 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2619 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2620 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2621 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2622 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2623 2624 /* at least one port started, need checking link status */ 2625 need_check_link_status = 1; 2626 2627 pl[cfg_pi++] = pi; 2628 } 2629 2630 if (need_check_link_status == 1 && !no_link_check) 2631 check_all_ports_link_status(RTE_PORT_ALL); 2632 else if (need_check_link_status == 0) 2633 printf("Please stop the ports first\n"); 2634 2635 if (hairpin_mode & 0xf) { 2636 uint16_t i; 2637 int j; 2638 2639 /* bind all started hairpin ports */ 2640 for (i = 0; i < cfg_pi; i++) { 2641 pi = pl[i]; 2642 /* bind current Tx to all peer Rx */ 2643 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2644 RTE_MAX_ETHPORTS, 1); 2645 if (peer_pi < 0) 2646 return peer_pi; 2647 for (j = 0; j < peer_pi; j++) { 2648 if (!port_is_started(peer_pl[j])) 2649 continue; 2650 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2651 if (diag < 0) { 2652 printf("Error during binding hairpin" 2653 " Tx port %u to %u: %s\n", 2654 pi, peer_pl[j], 2655 rte_strerror(-diag)); 2656 return -1; 2657 } 2658 } 2659 /* bind all peer Tx to current Rx */ 2660 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2661 RTE_MAX_ETHPORTS, 0); 2662 if (peer_pi < 0) 2663 return peer_pi; 2664 for (j = 0; j < peer_pi; j++) { 2665 if (!port_is_started(peer_pl[j])) 2666 continue; 2667 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2668 if (diag < 0) { 2669 printf("Error during binding hairpin" 2670 " Tx port %u to %u: %s\n", 2671 peer_pl[j], pi, 2672 rte_strerror(-diag)); 2673 return -1; 2674 } 2675 } 2676 } 2677 } 2678 2679 printf("Done\n"); 2680 return 0; 2681 } 2682 2683 void 2684 stop_port(portid_t pid) 2685 { 2686 portid_t pi; 2687 struct rte_port *port; 2688 int need_check_link_status = 0; 2689 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2690 int peer_pi; 2691 2692 if (dcb_test) { 2693 dcb_test = 0; 2694 dcb_config = 0; 2695 } 2696 2697 if (port_id_is_invalid(pid, ENABLED_WARN)) 2698 return; 2699 2700 printf("Stopping ports...\n"); 2701 2702 RTE_ETH_FOREACH_DEV(pi) { 2703 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2704 continue; 2705 2706 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2707 printf("Please remove port %d from forwarding configuration.\n", pi); 2708 continue; 2709 } 2710 2711 if (port_is_bonding_slave(pi)) { 2712 printf("Please remove port %d from bonded device.\n", pi); 2713 continue; 2714 } 2715 2716 port = &ports[pi]; 2717 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2718 RTE_PORT_HANDLING) == 0) 2719 continue; 2720 2721 if (hairpin_mode & 0xf) { 2722 int j; 2723 2724 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 2725 /* unbind all peer Tx from current Rx */ 2726 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2727 RTE_MAX_ETHPORTS, 0); 2728 if (peer_pi < 0) 2729 continue; 2730 for (j = 0; j < peer_pi; j++) { 2731 if (!port_is_started(peer_pl[j])) 2732 continue; 2733 rte_eth_hairpin_unbind(peer_pl[j], pi); 2734 } 2735 } 2736 2737 if (rte_eth_dev_stop(pi) != 0) 2738 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 2739 pi); 2740 2741 if (rte_atomic16_cmpset(&(port->port_status), 2742 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2743 printf("Port %d can not be set into stopped\n", pi); 2744 need_check_link_status = 1; 2745 } 2746 if (need_check_link_status && !no_link_check) 2747 check_all_ports_link_status(RTE_PORT_ALL); 2748 2749 printf("Done\n"); 2750 } 2751 2752 static void 2753 remove_invalid_ports_in(portid_t *array, portid_t *total) 2754 { 2755 portid_t i; 2756 portid_t new_total = 0; 2757 2758 for (i = 0; i < *total; i++) 2759 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2760 array[new_total] = array[i]; 2761 new_total++; 2762 } 2763 *total = new_total; 2764 } 2765 2766 static void 2767 remove_invalid_ports(void) 2768 { 2769 remove_invalid_ports_in(ports_ids, &nb_ports); 2770 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2771 nb_cfg_ports = nb_fwd_ports; 2772 } 2773 2774 void 2775 close_port(portid_t pid) 2776 { 2777 portid_t pi; 2778 struct rte_port *port; 2779 2780 if (port_id_is_invalid(pid, ENABLED_WARN)) 2781 return; 2782 2783 printf("Closing ports...\n"); 2784 2785 RTE_ETH_FOREACH_DEV(pi) { 2786 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2787 continue; 2788 2789 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2790 printf("Please remove port %d from forwarding configuration.\n", pi); 2791 continue; 2792 } 2793 2794 if (port_is_bonding_slave(pi)) { 2795 printf("Please remove port %d from bonded device.\n", pi); 2796 continue; 2797 } 2798 2799 port = &ports[pi]; 2800 if (rte_atomic16_cmpset(&(port->port_status), 2801 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2802 printf("Port %d is already closed\n", pi); 2803 continue; 2804 } 2805 2806 port_flow_flush(pi); 2807 rte_eth_dev_close(pi); 2808 } 2809 2810 remove_invalid_ports(); 2811 printf("Done\n"); 2812 } 2813 2814 void 2815 reset_port(portid_t pid) 2816 { 2817 int diag; 2818 portid_t pi; 2819 struct rte_port *port; 2820 2821 if (port_id_is_invalid(pid, ENABLED_WARN)) 2822 return; 2823 2824 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2825 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2826 printf("Can not reset port(s), please stop port(s) first.\n"); 2827 return; 2828 } 2829 2830 printf("Resetting ports...\n"); 2831 2832 RTE_ETH_FOREACH_DEV(pi) { 2833 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2834 continue; 2835 2836 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2837 printf("Please remove port %d from forwarding " 2838 "configuration.\n", pi); 2839 continue; 2840 } 2841 2842 if (port_is_bonding_slave(pi)) { 2843 printf("Please remove port %d from bonded device.\n", 2844 pi); 2845 continue; 2846 } 2847 2848 diag = rte_eth_dev_reset(pi); 2849 if (diag == 0) { 2850 port = &ports[pi]; 2851 port->need_reconfig = 1; 2852 port->need_reconfig_queues = 1; 2853 } else { 2854 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2855 } 2856 } 2857 2858 printf("Done\n"); 2859 } 2860 2861 void 2862 attach_port(char *identifier) 2863 { 2864 portid_t pi; 2865 struct rte_dev_iterator iterator; 2866 2867 printf("Attaching a new port...\n"); 2868 2869 if (identifier == NULL) { 2870 printf("Invalid parameters are specified\n"); 2871 return; 2872 } 2873 2874 if (rte_dev_probe(identifier) < 0) { 2875 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2876 return; 2877 } 2878 2879 /* first attach mode: event */ 2880 if (setup_on_probe_event) { 2881 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2882 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2883 if (ports[pi].port_status == RTE_PORT_HANDLING && 2884 ports[pi].need_setup != 0) 2885 setup_attached_port(pi); 2886 return; 2887 } 2888 2889 /* second attach mode: iterator */ 2890 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2891 /* setup ports matching the devargs used for probing */ 2892 if (port_is_forwarding(pi)) 2893 continue; /* port was already attached before */ 2894 setup_attached_port(pi); 2895 } 2896 } 2897 2898 static void 2899 setup_attached_port(portid_t pi) 2900 { 2901 unsigned int socket_id; 2902 int ret; 2903 2904 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2905 /* if socket_id is invalid, set to the first available socket. */ 2906 if (check_socket_id(socket_id) < 0) 2907 socket_id = socket_ids[0]; 2908 reconfig(pi, socket_id); 2909 ret = rte_eth_promiscuous_enable(pi); 2910 if (ret != 0) 2911 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2912 pi, rte_strerror(-ret)); 2913 2914 ports_ids[nb_ports++] = pi; 2915 fwd_ports_ids[nb_fwd_ports++] = pi; 2916 nb_cfg_ports = nb_fwd_ports; 2917 ports[pi].need_setup = 0; 2918 ports[pi].port_status = RTE_PORT_STOPPED; 2919 2920 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2921 printf("Done\n"); 2922 } 2923 2924 static void 2925 detach_device(struct rte_device *dev) 2926 { 2927 portid_t sibling; 2928 2929 if (dev == NULL) { 2930 printf("Device already removed\n"); 2931 return; 2932 } 2933 2934 printf("Removing a device...\n"); 2935 2936 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2937 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2938 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 2939 printf("Port %u not stopped\n", sibling); 2940 return; 2941 } 2942 port_flow_flush(sibling); 2943 } 2944 } 2945 2946 if (rte_dev_remove(dev) < 0) { 2947 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2948 return; 2949 } 2950 remove_invalid_ports(); 2951 2952 printf("Device is detached\n"); 2953 printf("Now total ports is %d\n", nb_ports); 2954 printf("Done\n"); 2955 return; 2956 } 2957 2958 void 2959 detach_port_device(portid_t port_id) 2960 { 2961 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2962 return; 2963 2964 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2965 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2966 printf("Port not stopped\n"); 2967 return; 2968 } 2969 printf("Port was not closed\n"); 2970 } 2971 2972 detach_device(rte_eth_devices[port_id].device); 2973 } 2974 2975 void 2976 detach_devargs(char *identifier) 2977 { 2978 struct rte_dev_iterator iterator; 2979 struct rte_devargs da; 2980 portid_t port_id; 2981 2982 printf("Removing a device...\n"); 2983 2984 memset(&da, 0, sizeof(da)); 2985 if (rte_devargs_parsef(&da, "%s", identifier)) { 2986 printf("cannot parse identifier\n"); 2987 if (da.args) 2988 free(da.args); 2989 return; 2990 } 2991 2992 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2993 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2994 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2995 printf("Port %u not stopped\n", port_id); 2996 rte_eth_iterator_cleanup(&iterator); 2997 return; 2998 } 2999 port_flow_flush(port_id); 3000 } 3001 } 3002 3003 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3004 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3005 da.name, da.bus->name); 3006 return; 3007 } 3008 3009 remove_invalid_ports(); 3010 3011 printf("Device %s is detached\n", identifier); 3012 printf("Now total ports is %d\n", nb_ports); 3013 printf("Done\n"); 3014 } 3015 3016 void 3017 pmd_test_exit(void) 3018 { 3019 portid_t pt_id; 3020 unsigned int i; 3021 int ret; 3022 3023 if (test_done == 0) 3024 stop_packet_forwarding(); 3025 3026 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3027 if (mempools[i]) { 3028 if (mp_alloc_type == MP_ALLOC_ANON) 3029 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3030 NULL); 3031 } 3032 } 3033 if (ports != NULL) { 3034 no_link_check = 1; 3035 RTE_ETH_FOREACH_DEV(pt_id) { 3036 printf("\nStopping port %d...\n", pt_id); 3037 fflush(stdout); 3038 stop_port(pt_id); 3039 } 3040 RTE_ETH_FOREACH_DEV(pt_id) { 3041 printf("\nShutting down port %d...\n", pt_id); 3042 fflush(stdout); 3043 close_port(pt_id); 3044 } 3045 } 3046 3047 if (hot_plug) { 3048 ret = rte_dev_event_monitor_stop(); 3049 if (ret) { 3050 RTE_LOG(ERR, EAL, 3051 "fail to stop device event monitor."); 3052 return; 3053 } 3054 3055 ret = rte_dev_event_callback_unregister(NULL, 3056 dev_event_callback, NULL); 3057 if (ret < 0) { 3058 RTE_LOG(ERR, EAL, 3059 "fail to unregister device event callback.\n"); 3060 return; 3061 } 3062 3063 ret = rte_dev_hotplug_handle_disable(); 3064 if (ret) { 3065 RTE_LOG(ERR, EAL, 3066 "fail to disable hotplug handling.\n"); 3067 return; 3068 } 3069 } 3070 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3071 if (mempools[i]) 3072 rte_mempool_free(mempools[i]); 3073 } 3074 3075 printf("\nBye...\n"); 3076 } 3077 3078 typedef void (*cmd_func_t)(void); 3079 struct pmd_test_command { 3080 const char *cmd_name; 3081 cmd_func_t cmd_func; 3082 }; 3083 3084 /* Check the link status of all ports in up to 9s, and print them finally */ 3085 static void 3086 check_all_ports_link_status(uint32_t port_mask) 3087 { 3088 #define CHECK_INTERVAL 100 /* 100ms */ 3089 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3090 portid_t portid; 3091 uint8_t count, all_ports_up, print_flag = 0; 3092 struct rte_eth_link link; 3093 int ret; 3094 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3095 3096 printf("Checking link statuses...\n"); 3097 fflush(stdout); 3098 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3099 all_ports_up = 1; 3100 RTE_ETH_FOREACH_DEV(portid) { 3101 if ((port_mask & (1 << portid)) == 0) 3102 continue; 3103 memset(&link, 0, sizeof(link)); 3104 ret = rte_eth_link_get_nowait(portid, &link); 3105 if (ret < 0) { 3106 all_ports_up = 0; 3107 if (print_flag == 1) 3108 printf("Port %u link get failed: %s\n", 3109 portid, rte_strerror(-ret)); 3110 continue; 3111 } 3112 /* print link status if flag set */ 3113 if (print_flag == 1) { 3114 rte_eth_link_to_str(link_status, 3115 sizeof(link_status), &link); 3116 printf("Port %d %s\n", portid, link_status); 3117 continue; 3118 } 3119 /* clear all_ports_up flag if any link down */ 3120 if (link.link_status == ETH_LINK_DOWN) { 3121 all_ports_up = 0; 3122 break; 3123 } 3124 } 3125 /* after finally printing all link status, get out */ 3126 if (print_flag == 1) 3127 break; 3128 3129 if (all_ports_up == 0) { 3130 fflush(stdout); 3131 rte_delay_ms(CHECK_INTERVAL); 3132 } 3133 3134 /* set the print_flag if all ports up or timeout */ 3135 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3136 print_flag = 1; 3137 } 3138 3139 if (lsc_interrupt) 3140 break; 3141 } 3142 } 3143 3144 static void 3145 rmv_port_callback(void *arg) 3146 { 3147 int need_to_start = 0; 3148 int org_no_link_check = no_link_check; 3149 portid_t port_id = (intptr_t)arg; 3150 struct rte_device *dev; 3151 3152 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3153 3154 if (!test_done && port_is_forwarding(port_id)) { 3155 need_to_start = 1; 3156 stop_packet_forwarding(); 3157 } 3158 no_link_check = 1; 3159 stop_port(port_id); 3160 no_link_check = org_no_link_check; 3161 3162 /* Save rte_device pointer before closing ethdev port */ 3163 dev = rte_eth_devices[port_id].device; 3164 close_port(port_id); 3165 detach_device(dev); /* might be already removed or have more ports */ 3166 3167 if (need_to_start) 3168 start_packet_forwarding(0); 3169 } 3170 3171 /* This function is used by the interrupt thread */ 3172 static int 3173 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3174 void *ret_param) 3175 { 3176 RTE_SET_USED(param); 3177 RTE_SET_USED(ret_param); 3178 3179 if (type >= RTE_ETH_EVENT_MAX) { 3180 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3181 port_id, __func__, type); 3182 fflush(stderr); 3183 } else if (event_print_mask & (UINT32_C(1) << type)) { 3184 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3185 eth_event_desc[type]); 3186 fflush(stdout); 3187 } 3188 3189 switch (type) { 3190 case RTE_ETH_EVENT_NEW: 3191 ports[port_id].need_setup = 1; 3192 ports[port_id].port_status = RTE_PORT_HANDLING; 3193 break; 3194 case RTE_ETH_EVENT_INTR_RMV: 3195 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3196 break; 3197 if (rte_eal_alarm_set(100000, 3198 rmv_port_callback, (void *)(intptr_t)port_id)) 3199 fprintf(stderr, "Could not set up deferred device removal\n"); 3200 break; 3201 case RTE_ETH_EVENT_DESTROY: 3202 ports[port_id].port_status = RTE_PORT_CLOSED; 3203 printf("Port %u is closed\n", port_id); 3204 break; 3205 default: 3206 break; 3207 } 3208 return 0; 3209 } 3210 3211 static int 3212 register_eth_event_callback(void) 3213 { 3214 int ret; 3215 enum rte_eth_event_type event; 3216 3217 for (event = RTE_ETH_EVENT_UNKNOWN; 3218 event < RTE_ETH_EVENT_MAX; event++) { 3219 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3220 event, 3221 eth_event_callback, 3222 NULL); 3223 if (ret != 0) { 3224 TESTPMD_LOG(ERR, "Failed to register callback for " 3225 "%s event\n", eth_event_desc[event]); 3226 return -1; 3227 } 3228 } 3229 3230 return 0; 3231 } 3232 3233 /* This function is used by the interrupt thread */ 3234 static void 3235 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3236 __rte_unused void *arg) 3237 { 3238 uint16_t port_id; 3239 int ret; 3240 3241 if (type >= RTE_DEV_EVENT_MAX) { 3242 fprintf(stderr, "%s called upon invalid event %d\n", 3243 __func__, type); 3244 fflush(stderr); 3245 } 3246 3247 switch (type) { 3248 case RTE_DEV_EVENT_REMOVE: 3249 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3250 device_name); 3251 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3252 if (ret) { 3253 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3254 device_name); 3255 return; 3256 } 3257 /* 3258 * Because the user's callback is invoked in eal interrupt 3259 * callback, the interrupt callback need to be finished before 3260 * it can be unregistered when detaching device. So finish 3261 * callback soon and use a deferred removal to detach device 3262 * is need. It is a workaround, once the device detaching be 3263 * moved into the eal in the future, the deferred removal could 3264 * be deleted. 3265 */ 3266 if (rte_eal_alarm_set(100000, 3267 rmv_port_callback, (void *)(intptr_t)port_id)) 3268 RTE_LOG(ERR, EAL, 3269 "Could not set up deferred device removal\n"); 3270 break; 3271 case RTE_DEV_EVENT_ADD: 3272 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3273 device_name); 3274 /* TODO: After finish kernel driver binding, 3275 * begin to attach port. 3276 */ 3277 break; 3278 default: 3279 break; 3280 } 3281 } 3282 3283 static void 3284 rxtx_port_config(struct rte_port *port) 3285 { 3286 uint16_t qid; 3287 uint64_t offloads; 3288 3289 for (qid = 0; qid < nb_rxq; qid++) { 3290 offloads = port->rx_conf[qid].offloads; 3291 port->rx_conf[qid] = port->dev_info.default_rxconf; 3292 if (offloads != 0) 3293 port->rx_conf[qid].offloads = offloads; 3294 3295 /* Check if any Rx parameters have been passed */ 3296 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3297 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3298 3299 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3300 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3301 3302 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3303 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3304 3305 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3306 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3307 3308 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3309 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3310 3311 port->nb_rx_desc[qid] = nb_rxd; 3312 } 3313 3314 for (qid = 0; qid < nb_txq; qid++) { 3315 offloads = port->tx_conf[qid].offloads; 3316 port->tx_conf[qid] = port->dev_info.default_txconf; 3317 if (offloads != 0) 3318 port->tx_conf[qid].offloads = offloads; 3319 3320 /* Check if any Tx parameters have been passed */ 3321 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3322 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3323 3324 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3325 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3326 3327 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3328 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3329 3330 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3331 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3332 3333 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3334 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3335 3336 port->nb_tx_desc[qid] = nb_txd; 3337 } 3338 } 3339 3340 void 3341 init_port_config(void) 3342 { 3343 portid_t pid; 3344 struct rte_port *port; 3345 int ret; 3346 3347 RTE_ETH_FOREACH_DEV(pid) { 3348 port = &ports[pid]; 3349 port->dev_conf.fdir_conf = fdir_conf; 3350 3351 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3352 if (ret != 0) 3353 return; 3354 3355 if (nb_rxq > 1) { 3356 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3357 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3358 rss_hf & port->dev_info.flow_type_rss_offloads; 3359 } else { 3360 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3361 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3362 } 3363 3364 if (port->dcb_flag == 0) { 3365 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3366 port->dev_conf.rxmode.mq_mode = 3367 (enum rte_eth_rx_mq_mode) 3368 (rx_mq_mode & ETH_MQ_RX_RSS); 3369 else 3370 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3371 } 3372 3373 rxtx_port_config(port); 3374 3375 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3376 if (ret != 0) 3377 return; 3378 3379 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3380 rte_pmd_ixgbe_bypass_init(pid); 3381 #endif 3382 3383 if (lsc_interrupt && 3384 (rte_eth_devices[pid].data->dev_flags & 3385 RTE_ETH_DEV_INTR_LSC)) 3386 port->dev_conf.intr_conf.lsc = 1; 3387 if (rmv_interrupt && 3388 (rte_eth_devices[pid].data->dev_flags & 3389 RTE_ETH_DEV_INTR_RMV)) 3390 port->dev_conf.intr_conf.rmv = 1; 3391 } 3392 } 3393 3394 void set_port_slave_flag(portid_t slave_pid) 3395 { 3396 struct rte_port *port; 3397 3398 port = &ports[slave_pid]; 3399 port->slave_flag = 1; 3400 } 3401 3402 void clear_port_slave_flag(portid_t slave_pid) 3403 { 3404 struct rte_port *port; 3405 3406 port = &ports[slave_pid]; 3407 port->slave_flag = 0; 3408 } 3409 3410 uint8_t port_is_bonding_slave(portid_t slave_pid) 3411 { 3412 struct rte_port *port; 3413 3414 port = &ports[slave_pid]; 3415 if ((rte_eth_devices[slave_pid].data->dev_flags & 3416 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3417 return 1; 3418 return 0; 3419 } 3420 3421 const uint16_t vlan_tags[] = { 3422 0, 1, 2, 3, 4, 5, 6, 7, 3423 8, 9, 10, 11, 12, 13, 14, 15, 3424 16, 17, 18, 19, 20, 21, 22, 23, 3425 24, 25, 26, 27, 28, 29, 30, 31 3426 }; 3427 3428 static int 3429 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3430 enum dcb_mode_enable dcb_mode, 3431 enum rte_eth_nb_tcs num_tcs, 3432 uint8_t pfc_en) 3433 { 3434 uint8_t i; 3435 int32_t rc; 3436 struct rte_eth_rss_conf rss_conf; 3437 3438 /* 3439 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3440 * given above, and the number of traffic classes available for use. 3441 */ 3442 if (dcb_mode == DCB_VT_ENABLED) { 3443 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3444 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3445 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3446 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3447 3448 /* VMDQ+DCB RX and TX configurations */ 3449 vmdq_rx_conf->enable_default_pool = 0; 3450 vmdq_rx_conf->default_pool = 0; 3451 vmdq_rx_conf->nb_queue_pools = 3452 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3453 vmdq_tx_conf->nb_queue_pools = 3454 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3455 3456 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3457 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3458 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3459 vmdq_rx_conf->pool_map[i].pools = 3460 1 << (i % vmdq_rx_conf->nb_queue_pools); 3461 } 3462 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3463 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3464 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3465 } 3466 3467 /* set DCB mode of RX and TX of multiple queues */ 3468 eth_conf->rxmode.mq_mode = 3469 (enum rte_eth_rx_mq_mode) 3470 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3471 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3472 } else { 3473 struct rte_eth_dcb_rx_conf *rx_conf = 3474 ð_conf->rx_adv_conf.dcb_rx_conf; 3475 struct rte_eth_dcb_tx_conf *tx_conf = 3476 ð_conf->tx_adv_conf.dcb_tx_conf; 3477 3478 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3479 3480 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3481 if (rc != 0) 3482 return rc; 3483 3484 rx_conf->nb_tcs = num_tcs; 3485 tx_conf->nb_tcs = num_tcs; 3486 3487 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3488 rx_conf->dcb_tc[i] = i % num_tcs; 3489 tx_conf->dcb_tc[i] = i % num_tcs; 3490 } 3491 3492 eth_conf->rxmode.mq_mode = 3493 (enum rte_eth_rx_mq_mode) 3494 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3495 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3496 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3497 } 3498 3499 if (pfc_en) 3500 eth_conf->dcb_capability_en = 3501 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3502 else 3503 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3504 3505 return 0; 3506 } 3507 3508 int 3509 init_port_dcb_config(portid_t pid, 3510 enum dcb_mode_enable dcb_mode, 3511 enum rte_eth_nb_tcs num_tcs, 3512 uint8_t pfc_en) 3513 { 3514 struct rte_eth_conf port_conf; 3515 struct rte_port *rte_port; 3516 int retval; 3517 uint16_t i; 3518 3519 rte_port = &ports[pid]; 3520 3521 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3522 /* Enter DCB configuration status */ 3523 dcb_config = 1; 3524 3525 port_conf.rxmode = rte_port->dev_conf.rxmode; 3526 port_conf.txmode = rte_port->dev_conf.txmode; 3527 3528 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3529 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3530 if (retval < 0) 3531 return retval; 3532 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3533 3534 /* re-configure the device . */ 3535 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3536 if (retval < 0) 3537 return retval; 3538 3539 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3540 if (retval != 0) 3541 return retval; 3542 3543 /* If dev_info.vmdq_pool_base is greater than 0, 3544 * the queue id of vmdq pools is started after pf queues. 3545 */ 3546 if (dcb_mode == DCB_VT_ENABLED && 3547 rte_port->dev_info.vmdq_pool_base > 0) { 3548 printf("VMDQ_DCB multi-queue mode is nonsensical" 3549 " for port %d.", pid); 3550 return -1; 3551 } 3552 3553 /* Assume the ports in testpmd have the same dcb capability 3554 * and has the same number of rxq and txq in dcb mode 3555 */ 3556 if (dcb_mode == DCB_VT_ENABLED) { 3557 if (rte_port->dev_info.max_vfs > 0) { 3558 nb_rxq = rte_port->dev_info.nb_rx_queues; 3559 nb_txq = rte_port->dev_info.nb_tx_queues; 3560 } else { 3561 nb_rxq = rte_port->dev_info.max_rx_queues; 3562 nb_txq = rte_port->dev_info.max_tx_queues; 3563 } 3564 } else { 3565 /*if vt is disabled, use all pf queues */ 3566 if (rte_port->dev_info.vmdq_pool_base == 0) { 3567 nb_rxq = rte_port->dev_info.max_rx_queues; 3568 nb_txq = rte_port->dev_info.max_tx_queues; 3569 } else { 3570 nb_rxq = (queueid_t)num_tcs; 3571 nb_txq = (queueid_t)num_tcs; 3572 3573 } 3574 } 3575 rx_free_thresh = 64; 3576 3577 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3578 3579 rxtx_port_config(rte_port); 3580 /* VLAN filter */ 3581 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3582 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3583 rx_vft_set(pid, vlan_tags[i], 1); 3584 3585 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3586 if (retval != 0) 3587 return retval; 3588 3589 rte_port->dcb_flag = 1; 3590 3591 return 0; 3592 } 3593 3594 static void 3595 init_port(void) 3596 { 3597 int i; 3598 3599 /* Configuration of Ethernet ports. */ 3600 ports = rte_zmalloc("testpmd: ports", 3601 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3602 RTE_CACHE_LINE_SIZE); 3603 if (ports == NULL) { 3604 rte_exit(EXIT_FAILURE, 3605 "rte_zmalloc(%d struct rte_port) failed\n", 3606 RTE_MAX_ETHPORTS); 3607 } 3608 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 3609 LIST_INIT(&ports[i].flow_tunnel_list); 3610 /* Initialize ports NUMA structures */ 3611 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3612 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3613 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3614 } 3615 3616 static void 3617 force_quit(void) 3618 { 3619 pmd_test_exit(); 3620 prompt_exit(); 3621 } 3622 3623 static void 3624 print_stats(void) 3625 { 3626 uint8_t i; 3627 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3628 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3629 3630 /* Clear screen and move to top left */ 3631 printf("%s%s", clr, top_left); 3632 3633 printf("\nPort statistics ===================================="); 3634 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3635 nic_stats_display(fwd_ports_ids[i]); 3636 3637 fflush(stdout); 3638 } 3639 3640 static void 3641 signal_handler(int signum) 3642 { 3643 if (signum == SIGINT || signum == SIGTERM) { 3644 printf("\nSignal %d received, preparing to exit...\n", 3645 signum); 3646 #ifdef RTE_LIB_PDUMP 3647 /* uninitialize packet capture framework */ 3648 rte_pdump_uninit(); 3649 #endif 3650 #ifdef RTE_LIB_LATENCYSTATS 3651 if (latencystats_enabled != 0) 3652 rte_latencystats_uninit(); 3653 #endif 3654 force_quit(); 3655 /* Set flag to indicate the force termination. */ 3656 f_quit = 1; 3657 /* exit with the expected status */ 3658 signal(signum, SIG_DFL); 3659 kill(getpid(), signum); 3660 } 3661 } 3662 3663 int 3664 main(int argc, char** argv) 3665 { 3666 int diag; 3667 portid_t port_id; 3668 uint16_t count; 3669 int ret; 3670 3671 signal(SIGINT, signal_handler); 3672 signal(SIGTERM, signal_handler); 3673 3674 testpmd_logtype = rte_log_register("testpmd"); 3675 if (testpmd_logtype < 0) 3676 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3677 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3678 3679 diag = rte_eal_init(argc, argv); 3680 if (diag < 0) 3681 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3682 rte_strerror(rte_errno)); 3683 3684 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3685 rte_exit(EXIT_FAILURE, 3686 "Secondary process type not supported.\n"); 3687 3688 ret = register_eth_event_callback(); 3689 if (ret != 0) 3690 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3691 3692 #ifdef RTE_LIB_PDUMP 3693 /* initialize packet capture framework */ 3694 rte_pdump_init(); 3695 #endif 3696 3697 count = 0; 3698 RTE_ETH_FOREACH_DEV(port_id) { 3699 ports_ids[count] = port_id; 3700 count++; 3701 } 3702 nb_ports = (portid_t) count; 3703 if (nb_ports == 0) 3704 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3705 3706 /* allocate port structures, and init them */ 3707 init_port(); 3708 3709 set_def_fwd_config(); 3710 if (nb_lcores == 0) 3711 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3712 "Check the core mask argument\n"); 3713 3714 /* Bitrate/latency stats disabled by default */ 3715 #ifdef RTE_LIB_BITRATESTATS 3716 bitrate_enabled = 0; 3717 #endif 3718 #ifdef RTE_LIB_LATENCYSTATS 3719 latencystats_enabled = 0; 3720 #endif 3721 3722 /* on FreeBSD, mlockall() is disabled by default */ 3723 #ifdef RTE_EXEC_ENV_FREEBSD 3724 do_mlockall = 0; 3725 #else 3726 do_mlockall = 1; 3727 #endif 3728 3729 argc -= diag; 3730 argv += diag; 3731 if (argc > 1) 3732 launch_args_parse(argc, argv); 3733 3734 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3735 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3736 strerror(errno)); 3737 } 3738 3739 if (tx_first && interactive) 3740 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3741 "interactive mode.\n"); 3742 3743 if (tx_first && lsc_interrupt) { 3744 printf("Warning: lsc_interrupt needs to be off when " 3745 " using tx_first. Disabling.\n"); 3746 lsc_interrupt = 0; 3747 } 3748 3749 if (!nb_rxq && !nb_txq) 3750 printf("Warning: Either rx or tx queues should be non-zero\n"); 3751 3752 if (nb_rxq > 1 && nb_rxq > nb_txq) 3753 printf("Warning: nb_rxq=%d enables RSS configuration, " 3754 "but nb_txq=%d will prevent to fully test it.\n", 3755 nb_rxq, nb_txq); 3756 3757 init_config(); 3758 3759 if (hot_plug) { 3760 ret = rte_dev_hotplug_handle_enable(); 3761 if (ret) { 3762 RTE_LOG(ERR, EAL, 3763 "fail to enable hotplug handling."); 3764 return -1; 3765 } 3766 3767 ret = rte_dev_event_monitor_start(); 3768 if (ret) { 3769 RTE_LOG(ERR, EAL, 3770 "fail to start device event monitoring."); 3771 return -1; 3772 } 3773 3774 ret = rte_dev_event_callback_register(NULL, 3775 dev_event_callback, NULL); 3776 if (ret) { 3777 RTE_LOG(ERR, EAL, 3778 "fail to register device event callback\n"); 3779 return -1; 3780 } 3781 } 3782 3783 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3784 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3785 3786 /* set all ports to promiscuous mode by default */ 3787 RTE_ETH_FOREACH_DEV(port_id) { 3788 ret = rte_eth_promiscuous_enable(port_id); 3789 if (ret != 0) 3790 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3791 port_id, rte_strerror(-ret)); 3792 } 3793 3794 /* Init metrics library */ 3795 rte_metrics_init(rte_socket_id()); 3796 3797 #ifdef RTE_LIB_LATENCYSTATS 3798 if (latencystats_enabled != 0) { 3799 int ret = rte_latencystats_init(1, NULL); 3800 if (ret) 3801 printf("Warning: latencystats init()" 3802 " returned error %d\n", ret); 3803 printf("Latencystats running on lcore %d\n", 3804 latencystats_lcore_id); 3805 } 3806 #endif 3807 3808 /* Setup bitrate stats */ 3809 #ifdef RTE_LIB_BITRATESTATS 3810 if (bitrate_enabled != 0) { 3811 bitrate_data = rte_stats_bitrate_create(); 3812 if (bitrate_data == NULL) 3813 rte_exit(EXIT_FAILURE, 3814 "Could not allocate bitrate data.\n"); 3815 rte_stats_bitrate_reg(bitrate_data); 3816 } 3817 #endif 3818 3819 #ifdef RTE_LIB_CMDLINE 3820 if (strlen(cmdline_filename) != 0) 3821 cmdline_read_from_file(cmdline_filename); 3822 3823 if (interactive == 1) { 3824 if (auto_start) { 3825 printf("Start automatic packet forwarding\n"); 3826 start_packet_forwarding(0); 3827 } 3828 prompt(); 3829 pmd_test_exit(); 3830 } else 3831 #endif 3832 { 3833 char c; 3834 int rc; 3835 3836 f_quit = 0; 3837 3838 printf("No commandline core given, start packet forwarding\n"); 3839 start_packet_forwarding(tx_first); 3840 if (stats_period != 0) { 3841 uint64_t prev_time = 0, cur_time, diff_time = 0; 3842 uint64_t timer_period; 3843 3844 /* Convert to number of cycles */ 3845 timer_period = stats_period * rte_get_timer_hz(); 3846 3847 while (f_quit == 0) { 3848 cur_time = rte_get_timer_cycles(); 3849 diff_time += cur_time - prev_time; 3850 3851 if (diff_time >= timer_period) { 3852 print_stats(); 3853 /* Reset the timer */ 3854 diff_time = 0; 3855 } 3856 /* Sleep to avoid unnecessary checks */ 3857 prev_time = cur_time; 3858 sleep(1); 3859 } 3860 } 3861 3862 printf("Press enter to exit\n"); 3863 rc = read(0, &c, 1); 3864 pmd_test_exit(); 3865 if (rc < 0) 3866 return 1; 3867 } 3868 3869 ret = rte_eal_cleanup(); 3870 if (ret != 0) 3871 rte_exit(EXIT_FAILURE, 3872 "EAL cleanup failed: %s\n", strerror(-ret)); 3873 3874 return EXIT_SUCCESS; 3875 } 3876