1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATE 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 192 uint16_t mempool_flags; 193 194 struct fwd_config cur_fwd_config; 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 196 uint32_t retry_enabled; 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 199 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 201 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 202 * specified on command-line. */ 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 204 205 /* 206 * In container, it cannot terminate the process which running with 'stats-period' 207 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 208 */ 209 uint8_t f_quit; 210 211 /* 212 * Configuration of packet segments used by the "txonly" processing engine. 213 */ 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 216 TXONLY_DEF_PACKET_LEN, 217 }; 218 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 219 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 221 /**< Split policy for packets to TX. */ 222 223 uint8_t txonly_multi_flow; 224 /**< Whether multiple flows are generated in TXONLY mode. */ 225 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 228 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 230 uint8_t dcb_config = 0; 231 232 /* Whether the dcb is in testing status */ 233 uint8_t dcb_test = 0; 234 235 /* 236 * Configurable number of RX/TX queues. 237 */ 238 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 239 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 240 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 241 242 /* 243 * Configurable number of RX/TX ring descriptors. 244 * Defaults are supplied by drivers via ethdev. 245 */ 246 #define RTE_TEST_RX_DESC_DEFAULT 0 247 #define RTE_TEST_TX_DESC_DEFAULT 0 248 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 249 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 250 251 #define RTE_PMD_PARAM_UNSET -1 252 /* 253 * Configurable values of RX and TX ring threshold registers. 254 */ 255 256 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 258 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 259 260 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 262 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 263 264 /* 265 * Configurable value of RX free threshold. 266 */ 267 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 268 269 /* 270 * Configurable value of RX drop enable. 271 */ 272 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 273 274 /* 275 * Configurable value of TX free threshold. 276 */ 277 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 278 279 /* 280 * Configurable value of TX RS bit threshold. 281 */ 282 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 283 284 /* 285 * Configurable value of buffered packets before sending. 286 */ 287 uint16_t noisy_tx_sw_bufsz; 288 289 /* 290 * Configurable value of packet buffer timeout. 291 */ 292 uint16_t noisy_tx_sw_buf_flush_time; 293 294 /* 295 * Configurable value for size of VNF internal memory area 296 * used for simulating noisy neighbour behaviour 297 */ 298 uint64_t noisy_lkup_mem_sz; 299 300 /* 301 * Configurable value of number of random writes done in 302 * VNF simulation memory area. 303 */ 304 uint64_t noisy_lkup_num_writes; 305 306 /* 307 * Configurable value of number of random reads done in 308 * VNF simulation memory area. 309 */ 310 uint64_t noisy_lkup_num_reads; 311 312 /* 313 * Configurable value of number of random reads/writes done in 314 * VNF simulation memory area. 315 */ 316 uint64_t noisy_lkup_num_reads_writes; 317 318 /* 319 * Receive Side Scaling (RSS) configuration. 320 */ 321 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 322 323 /* 324 * Port topology configuration 325 */ 326 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 327 328 /* 329 * Avoids to flush all the RX streams before starts forwarding. 330 */ 331 uint8_t no_flush_rx = 0; /* flush by default */ 332 333 /* 334 * Flow API isolated mode. 335 */ 336 uint8_t flow_isolate_all; 337 338 /* 339 * Avoids to check link status when starting/stopping a port. 340 */ 341 uint8_t no_link_check = 0; /* check by default */ 342 343 /* 344 * Don't automatically start all ports in interactive mode. 345 */ 346 uint8_t no_device_start = 0; 347 348 /* 349 * Enable link status change notification 350 */ 351 uint8_t lsc_interrupt = 1; /* enabled by default */ 352 353 /* 354 * Enable device removal notification. 355 */ 356 uint8_t rmv_interrupt = 1; /* enabled by default */ 357 358 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 359 360 /* After attach, port setup is called on event or by iterator */ 361 bool setup_on_probe_event = true; 362 363 /* Clear ptypes on port initialization. */ 364 uint8_t clear_ptypes = true; 365 366 /* Pretty printing of ethdev events */ 367 static const char * const eth_event_desc[] = { 368 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 369 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 370 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 371 [RTE_ETH_EVENT_INTR_RESET] = "reset", 372 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 373 [RTE_ETH_EVENT_IPSEC] = "IPsec", 374 [RTE_ETH_EVENT_MACSEC] = "MACsec", 375 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 376 [RTE_ETH_EVENT_NEW] = "device probed", 377 [RTE_ETH_EVENT_DESTROY] = "device released", 378 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 379 [RTE_ETH_EVENT_MAX] = NULL, 380 }; 381 382 /* 383 * Display or mask ether events 384 * Default to all events except VF_MBOX 385 */ 386 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 387 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 388 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 389 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 390 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 391 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 392 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 393 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 394 /* 395 * Decide if all memory are locked for performance. 396 */ 397 int do_mlockall = 0; 398 399 /* 400 * NIC bypass mode configuration options. 401 */ 402 403 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 404 /* The NIC bypass watchdog timeout. */ 405 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 406 #endif 407 408 409 #ifdef RTE_LIBRTE_LATENCY_STATS 410 411 /* 412 * Set when latency stats is enabled in the commandline 413 */ 414 uint8_t latencystats_enabled; 415 416 /* 417 * Lcore ID to serive latency statistics. 418 */ 419 lcoreid_t latencystats_lcore_id = -1; 420 421 #endif 422 423 /* 424 * Ethernet device configuration. 425 */ 426 struct rte_eth_rxmode rx_mode = { 427 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 428 /**< Default maximum frame length. */ 429 }; 430 431 struct rte_eth_txmode tx_mode = { 432 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 433 }; 434 435 struct rte_fdir_conf fdir_conf = { 436 .mode = RTE_FDIR_MODE_NONE, 437 .pballoc = RTE_FDIR_PBALLOC_64K, 438 .status = RTE_FDIR_REPORT_STATUS, 439 .mask = { 440 .vlan_tci_mask = 0xFFEF, 441 .ipv4_mask = { 442 .src_ip = 0xFFFFFFFF, 443 .dst_ip = 0xFFFFFFFF, 444 }, 445 .ipv6_mask = { 446 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 447 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 448 }, 449 .src_port_mask = 0xFFFF, 450 .dst_port_mask = 0xFFFF, 451 .mac_addr_byte_mask = 0xFF, 452 .tunnel_type_mask = 1, 453 .tunnel_id_mask = 0xFFFFFFFF, 454 }, 455 .drop_queue = 127, 456 }; 457 458 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 459 460 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 461 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 462 463 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 464 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 465 466 uint16_t nb_tx_queue_stats_mappings = 0; 467 uint16_t nb_rx_queue_stats_mappings = 0; 468 469 /* 470 * Display zero values by default for xstats 471 */ 472 uint8_t xstats_hide_zero; 473 474 unsigned int num_sockets = 0; 475 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 476 477 #ifdef RTE_LIBRTE_BITRATE 478 /* Bitrate statistics */ 479 struct rte_stats_bitrates *bitrate_data; 480 lcoreid_t bitrate_lcore_id; 481 uint8_t bitrate_enabled; 482 #endif 483 484 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 485 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 486 487 /* 488 * hexadecimal bitmask of RX mq mode can be enabled. 489 */ 490 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 491 492 /* Forward function declarations */ 493 static void setup_attached_port(portid_t pi); 494 static void map_port_queue_stats_mapping_registers(portid_t pi, 495 struct rte_port *port); 496 static void check_all_ports_link_status(uint32_t port_mask); 497 static int eth_event_callback(portid_t port_id, 498 enum rte_eth_event_type type, 499 void *param, void *ret_param); 500 static void dev_event_callback(const char *device_name, 501 enum rte_dev_event_type type, 502 void *param); 503 504 /* 505 * Check if all the ports are started. 506 * If yes, return positive value. If not, return zero. 507 */ 508 static int all_ports_started(void); 509 510 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 511 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 512 513 /* Holds the registered mbuf dynamic flags names. */ 514 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 515 516 /* 517 * Helper function to check if socket is already discovered. 518 * If yes, return positive value. If not, return zero. 519 */ 520 int 521 new_socket_id(unsigned int socket_id) 522 { 523 unsigned int i; 524 525 for (i = 0; i < num_sockets; i++) { 526 if (socket_ids[i] == socket_id) 527 return 0; 528 } 529 return 1; 530 } 531 532 /* 533 * Setup default configuration. 534 */ 535 static void 536 set_default_fwd_lcores_config(void) 537 { 538 unsigned int i; 539 unsigned int nb_lc; 540 unsigned int sock_num; 541 542 nb_lc = 0; 543 for (i = 0; i < RTE_MAX_LCORE; i++) { 544 if (!rte_lcore_is_enabled(i)) 545 continue; 546 sock_num = rte_lcore_to_socket_id(i); 547 if (new_socket_id(sock_num)) { 548 if (num_sockets >= RTE_MAX_NUMA_NODES) { 549 rte_exit(EXIT_FAILURE, 550 "Total sockets greater than %u\n", 551 RTE_MAX_NUMA_NODES); 552 } 553 socket_ids[num_sockets++] = sock_num; 554 } 555 if (i == rte_get_master_lcore()) 556 continue; 557 fwd_lcores_cpuids[nb_lc++] = i; 558 } 559 nb_lcores = (lcoreid_t) nb_lc; 560 nb_cfg_lcores = nb_lcores; 561 nb_fwd_lcores = 1; 562 } 563 564 static void 565 set_def_peer_eth_addrs(void) 566 { 567 portid_t i; 568 569 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 570 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 571 peer_eth_addrs[i].addr_bytes[5] = i; 572 } 573 } 574 575 static void 576 set_default_fwd_ports_config(void) 577 { 578 portid_t pt_id; 579 int i = 0; 580 581 RTE_ETH_FOREACH_DEV(pt_id) { 582 fwd_ports_ids[i++] = pt_id; 583 584 /* Update sockets info according to the attached device */ 585 int socket_id = rte_eth_dev_socket_id(pt_id); 586 if (socket_id >= 0 && new_socket_id(socket_id)) { 587 if (num_sockets >= RTE_MAX_NUMA_NODES) { 588 rte_exit(EXIT_FAILURE, 589 "Total sockets greater than %u\n", 590 RTE_MAX_NUMA_NODES); 591 } 592 socket_ids[num_sockets++] = socket_id; 593 } 594 } 595 596 nb_cfg_ports = nb_ports; 597 nb_fwd_ports = nb_ports; 598 } 599 600 void 601 set_def_fwd_config(void) 602 { 603 set_default_fwd_lcores_config(); 604 set_def_peer_eth_addrs(); 605 set_default_fwd_ports_config(); 606 } 607 608 /* extremely pessimistic estimation of memory required to create a mempool */ 609 static int 610 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 611 { 612 unsigned int n_pages, mbuf_per_pg, leftover; 613 uint64_t total_mem, mbuf_mem, obj_sz; 614 615 /* there is no good way to predict how much space the mempool will 616 * occupy because it will allocate chunks on the fly, and some of those 617 * will come from default DPDK memory while some will come from our 618 * external memory, so just assume 128MB will be enough for everyone. 619 */ 620 uint64_t hdr_mem = 128 << 20; 621 622 /* account for possible non-contiguousness */ 623 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 624 if (obj_sz > pgsz) { 625 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 626 return -1; 627 } 628 629 mbuf_per_pg = pgsz / obj_sz; 630 leftover = (nb_mbufs % mbuf_per_pg) > 0; 631 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 632 633 mbuf_mem = n_pages * pgsz; 634 635 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 636 637 if (total_mem > SIZE_MAX) { 638 TESTPMD_LOG(ERR, "Memory size too big\n"); 639 return -1; 640 } 641 *out = (size_t)total_mem; 642 643 return 0; 644 } 645 646 static int 647 pagesz_flags(uint64_t page_sz) 648 { 649 /* as per mmap() manpage, all page sizes are log2 of page size 650 * shifted by MAP_HUGE_SHIFT 651 */ 652 int log2 = rte_log2_u64(page_sz); 653 654 return (log2 << HUGE_SHIFT); 655 } 656 657 static void * 658 alloc_mem(size_t memsz, size_t pgsz, bool huge) 659 { 660 void *addr; 661 int flags; 662 663 /* allocate anonymous hugepages */ 664 flags = MAP_ANONYMOUS | MAP_PRIVATE; 665 if (huge) 666 flags |= HUGE_FLAG | pagesz_flags(pgsz); 667 668 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 669 if (addr == MAP_FAILED) 670 return NULL; 671 672 return addr; 673 } 674 675 struct extmem_param { 676 void *addr; 677 size_t len; 678 size_t pgsz; 679 rte_iova_t *iova_table; 680 unsigned int iova_table_len; 681 }; 682 683 static int 684 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 685 bool huge) 686 { 687 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 688 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 689 unsigned int cur_page, n_pages, pgsz_idx; 690 size_t mem_sz, cur_pgsz; 691 rte_iova_t *iovas = NULL; 692 void *addr; 693 int ret; 694 695 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 696 /* skip anything that is too big */ 697 if (pgsizes[pgsz_idx] > SIZE_MAX) 698 continue; 699 700 cur_pgsz = pgsizes[pgsz_idx]; 701 702 /* if we were told not to allocate hugepages, override */ 703 if (!huge) 704 cur_pgsz = sysconf(_SC_PAGESIZE); 705 706 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 707 if (ret < 0) { 708 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 709 return -1; 710 } 711 712 /* allocate our memory */ 713 addr = alloc_mem(mem_sz, cur_pgsz, huge); 714 715 /* if we couldn't allocate memory with a specified page size, 716 * that doesn't mean we can't do it with other page sizes, so 717 * try another one. 718 */ 719 if (addr == NULL) 720 continue; 721 722 /* store IOVA addresses for every page in this memory area */ 723 n_pages = mem_sz / cur_pgsz; 724 725 iovas = malloc(sizeof(*iovas) * n_pages); 726 727 if (iovas == NULL) { 728 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 729 goto fail; 730 } 731 /* lock memory if it's not huge pages */ 732 if (!huge) 733 mlock(addr, mem_sz); 734 735 /* populate IOVA addresses */ 736 for (cur_page = 0; cur_page < n_pages; cur_page++) { 737 rte_iova_t iova; 738 size_t offset; 739 void *cur; 740 741 offset = cur_pgsz * cur_page; 742 cur = RTE_PTR_ADD(addr, offset); 743 744 /* touch the page before getting its IOVA */ 745 *(volatile char *)cur = 0; 746 747 iova = rte_mem_virt2iova(cur); 748 749 iovas[cur_page] = iova; 750 } 751 752 break; 753 } 754 /* if we couldn't allocate anything */ 755 if (iovas == NULL) 756 return -1; 757 758 param->addr = addr; 759 param->len = mem_sz; 760 param->pgsz = cur_pgsz; 761 param->iova_table = iovas; 762 param->iova_table_len = n_pages; 763 764 return 0; 765 fail: 766 if (iovas) 767 free(iovas); 768 if (addr) 769 munmap(addr, mem_sz); 770 771 return -1; 772 } 773 774 static int 775 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 776 { 777 struct extmem_param param; 778 int socket_id, ret; 779 780 memset(¶m, 0, sizeof(param)); 781 782 /* check if our heap exists */ 783 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 784 if (socket_id < 0) { 785 /* create our heap */ 786 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 787 if (ret < 0) { 788 TESTPMD_LOG(ERR, "Cannot create heap\n"); 789 return -1; 790 } 791 } 792 793 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 794 if (ret < 0) { 795 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 796 return -1; 797 } 798 799 /* we now have a valid memory area, so add it to heap */ 800 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 801 param.addr, param.len, param.iova_table, 802 param.iova_table_len, param.pgsz); 803 804 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 805 806 /* not needed any more */ 807 free(param.iova_table); 808 809 if (ret < 0) { 810 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 811 munmap(param.addr, param.len); 812 return -1; 813 } 814 815 /* success */ 816 817 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 818 param.len >> 20); 819 820 return 0; 821 } 822 static void 823 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 824 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 825 { 826 uint16_t pid = 0; 827 int ret; 828 829 RTE_ETH_FOREACH_DEV(pid) { 830 struct rte_eth_dev *dev = 831 &rte_eth_devices[pid]; 832 833 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 834 memhdr->len); 835 if (ret) { 836 TESTPMD_LOG(DEBUG, 837 "unable to DMA unmap addr 0x%p " 838 "for device %s\n", 839 memhdr->addr, dev->data->name); 840 } 841 } 842 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 843 if (ret) { 844 TESTPMD_LOG(DEBUG, 845 "unable to un-register addr 0x%p\n", memhdr->addr); 846 } 847 } 848 849 static void 850 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 851 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 852 { 853 uint16_t pid = 0; 854 size_t page_size = sysconf(_SC_PAGESIZE); 855 int ret; 856 857 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 858 page_size); 859 if (ret) { 860 TESTPMD_LOG(DEBUG, 861 "unable to register addr 0x%p\n", memhdr->addr); 862 return; 863 } 864 RTE_ETH_FOREACH_DEV(pid) { 865 struct rte_eth_dev *dev = 866 &rte_eth_devices[pid]; 867 868 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 869 memhdr->len); 870 if (ret) { 871 TESTPMD_LOG(DEBUG, 872 "unable to DMA map addr 0x%p " 873 "for device %s\n", 874 memhdr->addr, dev->data->name); 875 } 876 } 877 } 878 879 static unsigned int 880 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 881 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 882 { 883 struct rte_pktmbuf_extmem *xmem; 884 unsigned int ext_num, zone_num, elt_num; 885 uint16_t elt_size; 886 887 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 888 elt_num = EXTBUF_ZONE_SIZE / elt_size; 889 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 890 891 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 892 if (xmem == NULL) { 893 TESTPMD_LOG(ERR, "Cannot allocate memory for " 894 "external buffer descriptors\n"); 895 *ext_mem = NULL; 896 return 0; 897 } 898 for (ext_num = 0; ext_num < zone_num; ext_num++) { 899 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 900 const struct rte_memzone *mz; 901 char mz_name[RTE_MEMZONE_NAMESIZE]; 902 int ret; 903 904 ret = snprintf(mz_name, sizeof(mz_name), 905 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 906 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 907 errno = ENAMETOOLONG; 908 ext_num = 0; 909 break; 910 } 911 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 912 socket_id, 913 RTE_MEMZONE_IOVA_CONTIG | 914 RTE_MEMZONE_1GB | 915 RTE_MEMZONE_SIZE_HINT_ONLY, 916 EXTBUF_ZONE_SIZE); 917 if (mz == NULL) { 918 /* 919 * The caller exits on external buffer creation 920 * error, so there is no need to free memzones. 921 */ 922 errno = ENOMEM; 923 ext_num = 0; 924 break; 925 } 926 xseg->buf_ptr = mz->addr; 927 xseg->buf_iova = mz->iova; 928 xseg->buf_len = EXTBUF_ZONE_SIZE; 929 xseg->elt_size = elt_size; 930 } 931 if (ext_num == 0 && xmem != NULL) { 932 free(xmem); 933 xmem = NULL; 934 } 935 *ext_mem = xmem; 936 return ext_num; 937 } 938 939 /* 940 * Configuration initialisation done once at init time. 941 */ 942 static struct rte_mempool * 943 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 944 unsigned int socket_id) 945 { 946 char pool_name[RTE_MEMPOOL_NAMESIZE]; 947 struct rte_mempool *rte_mp = NULL; 948 uint32_t mb_size; 949 950 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 951 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 952 953 TESTPMD_LOG(INFO, 954 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 955 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 956 957 switch (mp_alloc_type) { 958 case MP_ALLOC_NATIVE: 959 { 960 /* wrapper to rte_mempool_create() */ 961 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 962 rte_mbuf_best_mempool_ops()); 963 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 964 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 965 break; 966 } 967 case MP_ALLOC_ANON: 968 { 969 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 970 mb_size, (unsigned int) mb_mempool_cache, 971 sizeof(struct rte_pktmbuf_pool_private), 972 socket_id, mempool_flags); 973 if (rte_mp == NULL) 974 goto err; 975 976 if (rte_mempool_populate_anon(rte_mp) == 0) { 977 rte_mempool_free(rte_mp); 978 rte_mp = NULL; 979 goto err; 980 } 981 rte_pktmbuf_pool_init(rte_mp, NULL); 982 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 983 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 984 break; 985 } 986 case MP_ALLOC_XMEM: 987 case MP_ALLOC_XMEM_HUGE: 988 { 989 int heap_socket; 990 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 991 992 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 993 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 994 995 heap_socket = 996 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 997 if (heap_socket < 0) 998 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 999 1000 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1001 rte_mbuf_best_mempool_ops()); 1002 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1003 mb_mempool_cache, 0, mbuf_seg_size, 1004 heap_socket); 1005 break; 1006 } 1007 case MP_ALLOC_XBUF: 1008 { 1009 struct rte_pktmbuf_extmem *ext_mem; 1010 unsigned int ext_num; 1011 1012 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1013 socket_id, pool_name, &ext_mem); 1014 if (ext_num == 0) 1015 rte_exit(EXIT_FAILURE, 1016 "Can't create pinned data buffers\n"); 1017 1018 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1019 rte_mbuf_best_mempool_ops()); 1020 rte_mp = rte_pktmbuf_pool_create_extbuf 1021 (pool_name, nb_mbuf, mb_mempool_cache, 1022 0, mbuf_seg_size, socket_id, 1023 ext_mem, ext_num); 1024 free(ext_mem); 1025 break; 1026 } 1027 default: 1028 { 1029 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1030 } 1031 } 1032 1033 err: 1034 if (rte_mp == NULL) { 1035 rte_exit(EXIT_FAILURE, 1036 "Creation of mbuf pool for socket %u failed: %s\n", 1037 socket_id, rte_strerror(rte_errno)); 1038 } else if (verbose_level > 0) { 1039 rte_mempool_dump(stdout, rte_mp); 1040 } 1041 return rte_mp; 1042 } 1043 1044 /* 1045 * Check given socket id is valid or not with NUMA mode, 1046 * if valid, return 0, else return -1 1047 */ 1048 static int 1049 check_socket_id(const unsigned int socket_id) 1050 { 1051 static int warning_once = 0; 1052 1053 if (new_socket_id(socket_id)) { 1054 if (!warning_once && numa_support) 1055 printf("Warning: NUMA should be configured manually by" 1056 " using --port-numa-config and" 1057 " --ring-numa-config parameters along with" 1058 " --numa.\n"); 1059 warning_once = 1; 1060 return -1; 1061 } 1062 return 0; 1063 } 1064 1065 /* 1066 * Get the allowed maximum number of RX queues. 1067 * *pid return the port id which has minimal value of 1068 * max_rx_queues in all ports. 1069 */ 1070 queueid_t 1071 get_allowed_max_nb_rxq(portid_t *pid) 1072 { 1073 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1074 bool max_rxq_valid = false; 1075 portid_t pi; 1076 struct rte_eth_dev_info dev_info; 1077 1078 RTE_ETH_FOREACH_DEV(pi) { 1079 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1080 continue; 1081 1082 max_rxq_valid = true; 1083 if (dev_info.max_rx_queues < allowed_max_rxq) { 1084 allowed_max_rxq = dev_info.max_rx_queues; 1085 *pid = pi; 1086 } 1087 } 1088 return max_rxq_valid ? allowed_max_rxq : 0; 1089 } 1090 1091 /* 1092 * Check input rxq is valid or not. 1093 * If input rxq is not greater than any of maximum number 1094 * of RX queues of all ports, it is valid. 1095 * if valid, return 0, else return -1 1096 */ 1097 int 1098 check_nb_rxq(queueid_t rxq) 1099 { 1100 queueid_t allowed_max_rxq; 1101 portid_t pid = 0; 1102 1103 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1104 if (rxq > allowed_max_rxq) { 1105 printf("Fail: input rxq (%u) can't be greater " 1106 "than max_rx_queues (%u) of port %u\n", 1107 rxq, 1108 allowed_max_rxq, 1109 pid); 1110 return -1; 1111 } 1112 return 0; 1113 } 1114 1115 /* 1116 * Get the allowed maximum number of TX queues. 1117 * *pid return the port id which has minimal value of 1118 * max_tx_queues in all ports. 1119 */ 1120 queueid_t 1121 get_allowed_max_nb_txq(portid_t *pid) 1122 { 1123 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1124 bool max_txq_valid = false; 1125 portid_t pi; 1126 struct rte_eth_dev_info dev_info; 1127 1128 RTE_ETH_FOREACH_DEV(pi) { 1129 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1130 continue; 1131 1132 max_txq_valid = true; 1133 if (dev_info.max_tx_queues < allowed_max_txq) { 1134 allowed_max_txq = dev_info.max_tx_queues; 1135 *pid = pi; 1136 } 1137 } 1138 return max_txq_valid ? allowed_max_txq : 0; 1139 } 1140 1141 /* 1142 * Check input txq is valid or not. 1143 * If input txq is not greater than any of maximum number 1144 * of TX queues of all ports, it is valid. 1145 * if valid, return 0, else return -1 1146 */ 1147 int 1148 check_nb_txq(queueid_t txq) 1149 { 1150 queueid_t allowed_max_txq; 1151 portid_t pid = 0; 1152 1153 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1154 if (txq > allowed_max_txq) { 1155 printf("Fail: input txq (%u) can't be greater " 1156 "than max_tx_queues (%u) of port %u\n", 1157 txq, 1158 allowed_max_txq, 1159 pid); 1160 return -1; 1161 } 1162 return 0; 1163 } 1164 1165 /* 1166 * Get the allowed maximum number of RXDs of every rx queue. 1167 * *pid return the port id which has minimal value of 1168 * max_rxd in all queues of all ports. 1169 */ 1170 static uint16_t 1171 get_allowed_max_nb_rxd(portid_t *pid) 1172 { 1173 uint16_t allowed_max_rxd = UINT16_MAX; 1174 portid_t pi; 1175 struct rte_eth_dev_info dev_info; 1176 1177 RTE_ETH_FOREACH_DEV(pi) { 1178 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1179 continue; 1180 1181 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1182 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1183 *pid = pi; 1184 } 1185 } 1186 return allowed_max_rxd; 1187 } 1188 1189 /* 1190 * Get the allowed minimal number of RXDs of every rx queue. 1191 * *pid return the port id which has minimal value of 1192 * min_rxd in all queues of all ports. 1193 */ 1194 static uint16_t 1195 get_allowed_min_nb_rxd(portid_t *pid) 1196 { 1197 uint16_t allowed_min_rxd = 0; 1198 portid_t pi; 1199 struct rte_eth_dev_info dev_info; 1200 1201 RTE_ETH_FOREACH_DEV(pi) { 1202 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1203 continue; 1204 1205 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1206 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1207 *pid = pi; 1208 } 1209 } 1210 1211 return allowed_min_rxd; 1212 } 1213 1214 /* 1215 * Check input rxd is valid or not. 1216 * If input rxd is not greater than any of maximum number 1217 * of RXDs of every Rx queues and is not less than any of 1218 * minimal number of RXDs of every Rx queues, it is valid. 1219 * if valid, return 0, else return -1 1220 */ 1221 int 1222 check_nb_rxd(queueid_t rxd) 1223 { 1224 uint16_t allowed_max_rxd; 1225 uint16_t allowed_min_rxd; 1226 portid_t pid = 0; 1227 1228 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1229 if (rxd > allowed_max_rxd) { 1230 printf("Fail: input rxd (%u) can't be greater " 1231 "than max_rxds (%u) of port %u\n", 1232 rxd, 1233 allowed_max_rxd, 1234 pid); 1235 return -1; 1236 } 1237 1238 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1239 if (rxd < allowed_min_rxd) { 1240 printf("Fail: input rxd (%u) can't be less " 1241 "than min_rxds (%u) of port %u\n", 1242 rxd, 1243 allowed_min_rxd, 1244 pid); 1245 return -1; 1246 } 1247 1248 return 0; 1249 } 1250 1251 /* 1252 * Get the allowed maximum number of TXDs of every rx queues. 1253 * *pid return the port id which has minimal value of 1254 * max_txd in every tx queue. 1255 */ 1256 static uint16_t 1257 get_allowed_max_nb_txd(portid_t *pid) 1258 { 1259 uint16_t allowed_max_txd = UINT16_MAX; 1260 portid_t pi; 1261 struct rte_eth_dev_info dev_info; 1262 1263 RTE_ETH_FOREACH_DEV(pi) { 1264 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1265 continue; 1266 1267 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1268 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1269 *pid = pi; 1270 } 1271 } 1272 return allowed_max_txd; 1273 } 1274 1275 /* 1276 * Get the allowed maximum number of TXDs of every tx queues. 1277 * *pid return the port id which has minimal value of 1278 * min_txd in every tx queue. 1279 */ 1280 static uint16_t 1281 get_allowed_min_nb_txd(portid_t *pid) 1282 { 1283 uint16_t allowed_min_txd = 0; 1284 portid_t pi; 1285 struct rte_eth_dev_info dev_info; 1286 1287 RTE_ETH_FOREACH_DEV(pi) { 1288 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1289 continue; 1290 1291 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1292 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1293 *pid = pi; 1294 } 1295 } 1296 1297 return allowed_min_txd; 1298 } 1299 1300 /* 1301 * Check input txd is valid or not. 1302 * If input txd is not greater than any of maximum number 1303 * of TXDs of every Rx queues, it is valid. 1304 * if valid, return 0, else return -1 1305 */ 1306 int 1307 check_nb_txd(queueid_t txd) 1308 { 1309 uint16_t allowed_max_txd; 1310 uint16_t allowed_min_txd; 1311 portid_t pid = 0; 1312 1313 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1314 if (txd > allowed_max_txd) { 1315 printf("Fail: input txd (%u) can't be greater " 1316 "than max_txds (%u) of port %u\n", 1317 txd, 1318 allowed_max_txd, 1319 pid); 1320 return -1; 1321 } 1322 1323 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1324 if (txd < allowed_min_txd) { 1325 printf("Fail: input txd (%u) can't be less " 1326 "than min_txds (%u) of port %u\n", 1327 txd, 1328 allowed_min_txd, 1329 pid); 1330 return -1; 1331 } 1332 return 0; 1333 } 1334 1335 1336 /* 1337 * Get the allowed maximum number of hairpin queues. 1338 * *pid return the port id which has minimal value of 1339 * max_hairpin_queues in all ports. 1340 */ 1341 queueid_t 1342 get_allowed_max_nb_hairpinq(portid_t *pid) 1343 { 1344 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1345 portid_t pi; 1346 struct rte_eth_hairpin_cap cap; 1347 1348 RTE_ETH_FOREACH_DEV(pi) { 1349 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1350 *pid = pi; 1351 return 0; 1352 } 1353 if (cap.max_nb_queues < allowed_max_hairpinq) { 1354 allowed_max_hairpinq = cap.max_nb_queues; 1355 *pid = pi; 1356 } 1357 } 1358 return allowed_max_hairpinq; 1359 } 1360 1361 /* 1362 * Check input hairpin is valid or not. 1363 * If input hairpin is not greater than any of maximum number 1364 * of hairpin queues of all ports, it is valid. 1365 * if valid, return 0, else return -1 1366 */ 1367 int 1368 check_nb_hairpinq(queueid_t hairpinq) 1369 { 1370 queueid_t allowed_max_hairpinq; 1371 portid_t pid = 0; 1372 1373 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1374 if (hairpinq > allowed_max_hairpinq) { 1375 printf("Fail: input hairpin (%u) can't be greater " 1376 "than max_hairpin_queues (%u) of port %u\n", 1377 hairpinq, allowed_max_hairpinq, pid); 1378 return -1; 1379 } 1380 return 0; 1381 } 1382 1383 static void 1384 init_config(void) 1385 { 1386 portid_t pid; 1387 struct rte_port *port; 1388 struct rte_mempool *mbp; 1389 unsigned int nb_mbuf_per_pool; 1390 lcoreid_t lc_id; 1391 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1392 struct rte_gro_param gro_param; 1393 uint32_t gso_types; 1394 uint16_t data_size; 1395 bool warning = 0; 1396 int k; 1397 int ret; 1398 1399 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1400 1401 /* Configuration of logical cores. */ 1402 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1403 sizeof(struct fwd_lcore *) * nb_lcores, 1404 RTE_CACHE_LINE_SIZE); 1405 if (fwd_lcores == NULL) { 1406 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1407 "failed\n", nb_lcores); 1408 } 1409 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1410 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1411 sizeof(struct fwd_lcore), 1412 RTE_CACHE_LINE_SIZE); 1413 if (fwd_lcores[lc_id] == NULL) { 1414 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1415 "failed\n"); 1416 } 1417 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1418 } 1419 1420 RTE_ETH_FOREACH_DEV(pid) { 1421 port = &ports[pid]; 1422 /* Apply default TxRx configuration for all ports */ 1423 port->dev_conf.txmode = tx_mode; 1424 port->dev_conf.rxmode = rx_mode; 1425 1426 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1427 if (ret != 0) 1428 rte_exit(EXIT_FAILURE, 1429 "rte_eth_dev_info_get() failed\n"); 1430 1431 if (!(port->dev_info.tx_offload_capa & 1432 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1433 port->dev_conf.txmode.offloads &= 1434 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1435 if (numa_support) { 1436 if (port_numa[pid] != NUMA_NO_CONFIG) 1437 port_per_socket[port_numa[pid]]++; 1438 else { 1439 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1440 1441 /* 1442 * if socket_id is invalid, 1443 * set to the first available socket. 1444 */ 1445 if (check_socket_id(socket_id) < 0) 1446 socket_id = socket_ids[0]; 1447 port_per_socket[socket_id]++; 1448 } 1449 } 1450 1451 /* Apply Rx offloads configuration */ 1452 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1453 port->rx_conf[k].offloads = 1454 port->dev_conf.rxmode.offloads; 1455 /* Apply Tx offloads configuration */ 1456 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1457 port->tx_conf[k].offloads = 1458 port->dev_conf.txmode.offloads; 1459 1460 /* set flag to initialize port/queue */ 1461 port->need_reconfig = 1; 1462 port->need_reconfig_queues = 1; 1463 port->tx_metadata = 0; 1464 1465 /* Check for maximum number of segments per MTU. Accordingly 1466 * update the mbuf data size. 1467 */ 1468 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1469 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1470 data_size = rx_mode.max_rx_pkt_len / 1471 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1472 1473 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1474 mbuf_data_size) { 1475 mbuf_data_size = data_size + 1476 RTE_PKTMBUF_HEADROOM; 1477 warning = 1; 1478 } 1479 } 1480 } 1481 1482 if (warning) 1483 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1484 mbuf_data_size); 1485 1486 /* 1487 * Create pools of mbuf. 1488 * If NUMA support is disabled, create a single pool of mbuf in 1489 * socket 0 memory by default. 1490 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1491 * 1492 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1493 * nb_txd can be configured at run time. 1494 */ 1495 if (param_total_num_mbufs) 1496 nb_mbuf_per_pool = param_total_num_mbufs; 1497 else { 1498 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1499 (nb_lcores * mb_mempool_cache) + 1500 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1501 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1502 } 1503 1504 if (numa_support) { 1505 uint8_t i; 1506 1507 for (i = 0; i < num_sockets; i++) 1508 mempools[i] = mbuf_pool_create(mbuf_data_size, 1509 nb_mbuf_per_pool, 1510 socket_ids[i]); 1511 } else { 1512 if (socket_num == UMA_NO_CONFIG) 1513 mempools[0] = mbuf_pool_create(mbuf_data_size, 1514 nb_mbuf_per_pool, 0); 1515 else 1516 mempools[socket_num] = mbuf_pool_create 1517 (mbuf_data_size, 1518 nb_mbuf_per_pool, 1519 socket_num); 1520 } 1521 1522 init_port_config(); 1523 1524 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1525 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1526 /* 1527 * Records which Mbuf pool to use by each logical core, if needed. 1528 */ 1529 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1530 mbp = mbuf_pool_find( 1531 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1532 1533 if (mbp == NULL) 1534 mbp = mbuf_pool_find(0); 1535 fwd_lcores[lc_id]->mbp = mbp; 1536 /* initialize GSO context */ 1537 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1538 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1539 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1540 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1541 RTE_ETHER_CRC_LEN; 1542 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1543 } 1544 1545 /* Configuration of packet forwarding streams. */ 1546 if (init_fwd_streams() < 0) 1547 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1548 1549 fwd_config_setup(); 1550 1551 /* create a gro context for each lcore */ 1552 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1553 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1554 gro_param.max_item_per_flow = MAX_PKT_BURST; 1555 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1556 gro_param.socket_id = rte_lcore_to_socket_id( 1557 fwd_lcores_cpuids[lc_id]); 1558 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1559 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1560 rte_exit(EXIT_FAILURE, 1561 "rte_gro_ctx_create() failed\n"); 1562 } 1563 } 1564 1565 #if defined RTE_LIBRTE_PMD_SOFTNIC 1566 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1567 RTE_ETH_FOREACH_DEV(pid) { 1568 port = &ports[pid]; 1569 const char *driver = port->dev_info.driver_name; 1570 1571 if (strcmp(driver, "net_softnic") == 0) 1572 port->softport.fwd_lcore_arg = fwd_lcores; 1573 } 1574 } 1575 #endif 1576 1577 } 1578 1579 1580 void 1581 reconfig(portid_t new_port_id, unsigned socket_id) 1582 { 1583 struct rte_port *port; 1584 int ret; 1585 1586 /* Reconfiguration of Ethernet ports. */ 1587 port = &ports[new_port_id]; 1588 1589 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1590 if (ret != 0) 1591 return; 1592 1593 /* set flag to initialize port/queue */ 1594 port->need_reconfig = 1; 1595 port->need_reconfig_queues = 1; 1596 port->socket_id = socket_id; 1597 1598 init_port_config(); 1599 } 1600 1601 1602 int 1603 init_fwd_streams(void) 1604 { 1605 portid_t pid; 1606 struct rte_port *port; 1607 streamid_t sm_id, nb_fwd_streams_new; 1608 queueid_t q; 1609 1610 /* set socket id according to numa or not */ 1611 RTE_ETH_FOREACH_DEV(pid) { 1612 port = &ports[pid]; 1613 if (nb_rxq > port->dev_info.max_rx_queues) { 1614 printf("Fail: nb_rxq(%d) is greater than " 1615 "max_rx_queues(%d)\n", nb_rxq, 1616 port->dev_info.max_rx_queues); 1617 return -1; 1618 } 1619 if (nb_txq > port->dev_info.max_tx_queues) { 1620 printf("Fail: nb_txq(%d) is greater than " 1621 "max_tx_queues(%d)\n", nb_txq, 1622 port->dev_info.max_tx_queues); 1623 return -1; 1624 } 1625 if (numa_support) { 1626 if (port_numa[pid] != NUMA_NO_CONFIG) 1627 port->socket_id = port_numa[pid]; 1628 else { 1629 port->socket_id = rte_eth_dev_socket_id(pid); 1630 1631 /* 1632 * if socket_id is invalid, 1633 * set to the first available socket. 1634 */ 1635 if (check_socket_id(port->socket_id) < 0) 1636 port->socket_id = socket_ids[0]; 1637 } 1638 } 1639 else { 1640 if (socket_num == UMA_NO_CONFIG) 1641 port->socket_id = 0; 1642 else 1643 port->socket_id = socket_num; 1644 } 1645 } 1646 1647 q = RTE_MAX(nb_rxq, nb_txq); 1648 if (q == 0) { 1649 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1650 return -1; 1651 } 1652 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1653 if (nb_fwd_streams_new == nb_fwd_streams) 1654 return 0; 1655 /* clear the old */ 1656 if (fwd_streams != NULL) { 1657 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1658 if (fwd_streams[sm_id] == NULL) 1659 continue; 1660 rte_free(fwd_streams[sm_id]); 1661 fwd_streams[sm_id] = NULL; 1662 } 1663 rte_free(fwd_streams); 1664 fwd_streams = NULL; 1665 } 1666 1667 /* init new */ 1668 nb_fwd_streams = nb_fwd_streams_new; 1669 if (nb_fwd_streams) { 1670 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1671 sizeof(struct fwd_stream *) * nb_fwd_streams, 1672 RTE_CACHE_LINE_SIZE); 1673 if (fwd_streams == NULL) 1674 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1675 " (struct fwd_stream *)) failed\n", 1676 nb_fwd_streams); 1677 1678 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1679 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1680 " struct fwd_stream", sizeof(struct fwd_stream), 1681 RTE_CACHE_LINE_SIZE); 1682 if (fwd_streams[sm_id] == NULL) 1683 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1684 "(struct fwd_stream) failed\n"); 1685 } 1686 } 1687 1688 return 0; 1689 } 1690 1691 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1692 static void 1693 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1694 { 1695 unsigned int total_burst; 1696 unsigned int nb_burst; 1697 unsigned int burst_stats[3]; 1698 uint16_t pktnb_stats[3]; 1699 uint16_t nb_pkt; 1700 int burst_percent[3]; 1701 1702 /* 1703 * First compute the total number of packet bursts and the 1704 * two highest numbers of bursts of the same number of packets. 1705 */ 1706 total_burst = 0; 1707 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1708 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1709 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1710 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1711 if (nb_burst == 0) 1712 continue; 1713 total_burst += nb_burst; 1714 if (nb_burst > burst_stats[0]) { 1715 burst_stats[1] = burst_stats[0]; 1716 pktnb_stats[1] = pktnb_stats[0]; 1717 burst_stats[0] = nb_burst; 1718 pktnb_stats[0] = nb_pkt; 1719 } else if (nb_burst > burst_stats[1]) { 1720 burst_stats[1] = nb_burst; 1721 pktnb_stats[1] = nb_pkt; 1722 } 1723 } 1724 if (total_burst == 0) 1725 return; 1726 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1727 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1728 burst_percent[0], (int) pktnb_stats[0]); 1729 if (burst_stats[0] == total_burst) { 1730 printf("]\n"); 1731 return; 1732 } 1733 if (burst_stats[0] + burst_stats[1] == total_burst) { 1734 printf(" + %d%% of %d pkts]\n", 1735 100 - burst_percent[0], pktnb_stats[1]); 1736 return; 1737 } 1738 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1739 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1740 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1741 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1742 return; 1743 } 1744 printf(" + %d%% of %d pkts + %d%% of others]\n", 1745 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1746 } 1747 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1748 1749 static void 1750 fwd_stream_stats_display(streamid_t stream_id) 1751 { 1752 struct fwd_stream *fs; 1753 static const char *fwd_top_stats_border = "-------"; 1754 1755 fs = fwd_streams[stream_id]; 1756 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1757 (fs->fwd_dropped == 0)) 1758 return; 1759 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1760 "TX Port=%2d/Queue=%2d %s\n", 1761 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1762 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1763 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1764 " TX-dropped: %-14"PRIu64, 1765 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1766 1767 /* if checksum mode */ 1768 if (cur_fwd_eng == &csum_fwd_engine) { 1769 printf(" RX- bad IP checksum: %-14"PRIu64 1770 " Rx- bad L4 checksum: %-14"PRIu64 1771 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1772 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1773 fs->rx_bad_outer_l4_csum); 1774 } else { 1775 printf("\n"); 1776 } 1777 1778 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1779 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1780 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1781 #endif 1782 } 1783 1784 void 1785 fwd_stats_display(void) 1786 { 1787 static const char *fwd_stats_border = "----------------------"; 1788 static const char *acc_stats_border = "+++++++++++++++"; 1789 struct { 1790 struct fwd_stream *rx_stream; 1791 struct fwd_stream *tx_stream; 1792 uint64_t tx_dropped; 1793 uint64_t rx_bad_ip_csum; 1794 uint64_t rx_bad_l4_csum; 1795 uint64_t rx_bad_outer_l4_csum; 1796 } ports_stats[RTE_MAX_ETHPORTS]; 1797 uint64_t total_rx_dropped = 0; 1798 uint64_t total_tx_dropped = 0; 1799 uint64_t total_rx_nombuf = 0; 1800 struct rte_eth_stats stats; 1801 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1802 uint64_t fwd_cycles = 0; 1803 #endif 1804 uint64_t total_recv = 0; 1805 uint64_t total_xmit = 0; 1806 struct rte_port *port; 1807 streamid_t sm_id; 1808 portid_t pt_id; 1809 int i; 1810 1811 memset(ports_stats, 0, sizeof(ports_stats)); 1812 1813 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1814 struct fwd_stream *fs = fwd_streams[sm_id]; 1815 1816 if (cur_fwd_config.nb_fwd_streams > 1817 cur_fwd_config.nb_fwd_ports) { 1818 fwd_stream_stats_display(sm_id); 1819 } else { 1820 ports_stats[fs->tx_port].tx_stream = fs; 1821 ports_stats[fs->rx_port].rx_stream = fs; 1822 } 1823 1824 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1825 1826 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1827 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1828 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1829 fs->rx_bad_outer_l4_csum; 1830 1831 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1832 fwd_cycles += fs->core_cycles; 1833 #endif 1834 } 1835 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1836 uint8_t j; 1837 1838 pt_id = fwd_ports_ids[i]; 1839 port = &ports[pt_id]; 1840 1841 rte_eth_stats_get(pt_id, &stats); 1842 stats.ipackets -= port->stats.ipackets; 1843 stats.opackets -= port->stats.opackets; 1844 stats.ibytes -= port->stats.ibytes; 1845 stats.obytes -= port->stats.obytes; 1846 stats.imissed -= port->stats.imissed; 1847 stats.oerrors -= port->stats.oerrors; 1848 stats.rx_nombuf -= port->stats.rx_nombuf; 1849 1850 total_recv += stats.ipackets; 1851 total_xmit += stats.opackets; 1852 total_rx_dropped += stats.imissed; 1853 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1854 total_tx_dropped += stats.oerrors; 1855 total_rx_nombuf += stats.rx_nombuf; 1856 1857 printf("\n %s Forward statistics for port %-2d %s\n", 1858 fwd_stats_border, pt_id, fwd_stats_border); 1859 1860 if (!port->rx_queue_stats_mapping_enabled && 1861 !port->tx_queue_stats_mapping_enabled) { 1862 printf(" RX-packets: %-14"PRIu64 1863 " RX-dropped: %-14"PRIu64 1864 "RX-total: %-"PRIu64"\n", 1865 stats.ipackets, stats.imissed, 1866 stats.ipackets + stats.imissed); 1867 1868 if (cur_fwd_eng == &csum_fwd_engine) 1869 printf(" Bad-ipcsum: %-14"PRIu64 1870 " Bad-l4csum: %-14"PRIu64 1871 "Bad-outer-l4csum: %-14"PRIu64"\n", 1872 ports_stats[pt_id].rx_bad_ip_csum, 1873 ports_stats[pt_id].rx_bad_l4_csum, 1874 ports_stats[pt_id].rx_bad_outer_l4_csum); 1875 if (stats.ierrors + stats.rx_nombuf > 0) { 1876 printf(" RX-error: %-"PRIu64"\n", 1877 stats.ierrors); 1878 printf(" RX-nombufs: %-14"PRIu64"\n", 1879 stats.rx_nombuf); 1880 } 1881 1882 printf(" TX-packets: %-14"PRIu64 1883 " TX-dropped: %-14"PRIu64 1884 "TX-total: %-"PRIu64"\n", 1885 stats.opackets, ports_stats[pt_id].tx_dropped, 1886 stats.opackets + ports_stats[pt_id].tx_dropped); 1887 } else { 1888 printf(" RX-packets: %14"PRIu64 1889 " RX-dropped:%14"PRIu64 1890 " RX-total:%14"PRIu64"\n", 1891 stats.ipackets, stats.imissed, 1892 stats.ipackets + stats.imissed); 1893 1894 if (cur_fwd_eng == &csum_fwd_engine) 1895 printf(" Bad-ipcsum:%14"PRIu64 1896 " Bad-l4csum:%14"PRIu64 1897 " Bad-outer-l4csum: %-14"PRIu64"\n", 1898 ports_stats[pt_id].rx_bad_ip_csum, 1899 ports_stats[pt_id].rx_bad_l4_csum, 1900 ports_stats[pt_id].rx_bad_outer_l4_csum); 1901 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1902 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1903 printf(" RX-nombufs: %14"PRIu64"\n", 1904 stats.rx_nombuf); 1905 } 1906 1907 printf(" TX-packets: %14"PRIu64 1908 " TX-dropped:%14"PRIu64 1909 " TX-total:%14"PRIu64"\n", 1910 stats.opackets, ports_stats[pt_id].tx_dropped, 1911 stats.opackets + ports_stats[pt_id].tx_dropped); 1912 } 1913 1914 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1915 if (ports_stats[pt_id].rx_stream) 1916 pkt_burst_stats_display("RX", 1917 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1918 if (ports_stats[pt_id].tx_stream) 1919 pkt_burst_stats_display("TX", 1920 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1921 #endif 1922 1923 if (port->rx_queue_stats_mapping_enabled) { 1924 printf("\n"); 1925 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1926 printf(" Stats reg %2d RX-packets:%14"PRIu64 1927 " RX-errors:%14"PRIu64 1928 " RX-bytes:%14"PRIu64"\n", 1929 j, stats.q_ipackets[j], 1930 stats.q_errors[j], stats.q_ibytes[j]); 1931 } 1932 printf("\n"); 1933 } 1934 if (port->tx_queue_stats_mapping_enabled) { 1935 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1936 printf(" Stats reg %2d TX-packets:%14"PRIu64 1937 " TX-bytes:%14" 1938 PRIu64"\n", 1939 j, stats.q_opackets[j], 1940 stats.q_obytes[j]); 1941 } 1942 } 1943 1944 printf(" %s--------------------------------%s\n", 1945 fwd_stats_border, fwd_stats_border); 1946 } 1947 1948 printf("\n %s Accumulated forward statistics for all ports" 1949 "%s\n", 1950 acc_stats_border, acc_stats_border); 1951 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1952 "%-"PRIu64"\n" 1953 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1954 "%-"PRIu64"\n", 1955 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1956 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1957 if (total_rx_nombuf > 0) 1958 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1959 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1960 "%s\n", 1961 acc_stats_border, acc_stats_border); 1962 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1963 if (total_recv > 0) 1964 printf("\n CPU cycles/packet=%u (total cycles=" 1965 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1966 (unsigned int)(fwd_cycles / total_recv), 1967 fwd_cycles, total_recv); 1968 #endif 1969 } 1970 1971 void 1972 fwd_stats_reset(void) 1973 { 1974 streamid_t sm_id; 1975 portid_t pt_id; 1976 int i; 1977 1978 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1979 pt_id = fwd_ports_ids[i]; 1980 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1981 } 1982 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1983 struct fwd_stream *fs = fwd_streams[sm_id]; 1984 1985 fs->rx_packets = 0; 1986 fs->tx_packets = 0; 1987 fs->fwd_dropped = 0; 1988 fs->rx_bad_ip_csum = 0; 1989 fs->rx_bad_l4_csum = 0; 1990 fs->rx_bad_outer_l4_csum = 0; 1991 1992 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1993 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1994 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1995 #endif 1996 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1997 fs->core_cycles = 0; 1998 #endif 1999 } 2000 } 2001 2002 static void 2003 flush_fwd_rx_queues(void) 2004 { 2005 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2006 portid_t rxp; 2007 portid_t port_id; 2008 queueid_t rxq; 2009 uint16_t nb_rx; 2010 uint16_t i; 2011 uint8_t j; 2012 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2013 uint64_t timer_period; 2014 2015 /* convert to number of cycles */ 2016 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2017 2018 for (j = 0; j < 2; j++) { 2019 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2020 for (rxq = 0; rxq < nb_rxq; rxq++) { 2021 port_id = fwd_ports_ids[rxp]; 2022 /** 2023 * testpmd can stuck in the below do while loop 2024 * if rte_eth_rx_burst() always returns nonzero 2025 * packets. So timer is added to exit this loop 2026 * after 1sec timer expiry. 2027 */ 2028 prev_tsc = rte_rdtsc(); 2029 do { 2030 nb_rx = rte_eth_rx_burst(port_id, rxq, 2031 pkts_burst, MAX_PKT_BURST); 2032 for (i = 0; i < nb_rx; i++) 2033 rte_pktmbuf_free(pkts_burst[i]); 2034 2035 cur_tsc = rte_rdtsc(); 2036 diff_tsc = cur_tsc - prev_tsc; 2037 timer_tsc += diff_tsc; 2038 } while ((nb_rx > 0) && 2039 (timer_tsc < timer_period)); 2040 timer_tsc = 0; 2041 } 2042 } 2043 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2044 } 2045 } 2046 2047 static void 2048 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2049 { 2050 struct fwd_stream **fsm; 2051 streamid_t nb_fs; 2052 streamid_t sm_id; 2053 #ifdef RTE_LIBRTE_BITRATE 2054 uint64_t tics_per_1sec; 2055 uint64_t tics_datum; 2056 uint64_t tics_current; 2057 uint16_t i, cnt_ports; 2058 2059 cnt_ports = nb_ports; 2060 tics_datum = rte_rdtsc(); 2061 tics_per_1sec = rte_get_timer_hz(); 2062 #endif 2063 fsm = &fwd_streams[fc->stream_idx]; 2064 nb_fs = fc->stream_nb; 2065 do { 2066 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2067 (*pkt_fwd)(fsm[sm_id]); 2068 #ifdef RTE_LIBRTE_BITRATE 2069 if (bitrate_enabled != 0 && 2070 bitrate_lcore_id == rte_lcore_id()) { 2071 tics_current = rte_rdtsc(); 2072 if (tics_current - tics_datum >= tics_per_1sec) { 2073 /* Periodic bitrate calculation */ 2074 for (i = 0; i < cnt_ports; i++) 2075 rte_stats_bitrate_calc(bitrate_data, 2076 ports_ids[i]); 2077 tics_datum = tics_current; 2078 } 2079 } 2080 #endif 2081 #ifdef RTE_LIBRTE_LATENCY_STATS 2082 if (latencystats_enabled != 0 && 2083 latencystats_lcore_id == rte_lcore_id()) 2084 rte_latencystats_update(); 2085 #endif 2086 2087 } while (! fc->stopped); 2088 } 2089 2090 static int 2091 start_pkt_forward_on_core(void *fwd_arg) 2092 { 2093 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2094 cur_fwd_config.fwd_eng->packet_fwd); 2095 return 0; 2096 } 2097 2098 /* 2099 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2100 * Used to start communication flows in network loopback test configurations. 2101 */ 2102 static int 2103 run_one_txonly_burst_on_core(void *fwd_arg) 2104 { 2105 struct fwd_lcore *fwd_lc; 2106 struct fwd_lcore tmp_lcore; 2107 2108 fwd_lc = (struct fwd_lcore *) fwd_arg; 2109 tmp_lcore = *fwd_lc; 2110 tmp_lcore.stopped = 1; 2111 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2112 return 0; 2113 } 2114 2115 /* 2116 * Launch packet forwarding: 2117 * - Setup per-port forwarding context. 2118 * - launch logical cores with their forwarding configuration. 2119 */ 2120 static void 2121 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2122 { 2123 port_fwd_begin_t port_fwd_begin; 2124 unsigned int i; 2125 unsigned int lc_id; 2126 int diag; 2127 2128 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2129 if (port_fwd_begin != NULL) { 2130 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2131 (*port_fwd_begin)(fwd_ports_ids[i]); 2132 } 2133 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2134 lc_id = fwd_lcores_cpuids[i]; 2135 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2136 fwd_lcores[i]->stopped = 0; 2137 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2138 fwd_lcores[i], lc_id); 2139 if (diag != 0) 2140 printf("launch lcore %u failed - diag=%d\n", 2141 lc_id, diag); 2142 } 2143 } 2144 } 2145 2146 /* 2147 * Launch packet forwarding configuration. 2148 */ 2149 void 2150 start_packet_forwarding(int with_tx_first) 2151 { 2152 port_fwd_begin_t port_fwd_begin; 2153 port_fwd_end_t port_fwd_end; 2154 struct rte_port *port; 2155 unsigned int i; 2156 portid_t pt_id; 2157 2158 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2159 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2160 2161 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2162 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2163 2164 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2165 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2166 (!nb_rxq || !nb_txq)) 2167 rte_exit(EXIT_FAILURE, 2168 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2169 cur_fwd_eng->fwd_mode_name); 2170 2171 if (all_ports_started() == 0) { 2172 printf("Not all ports were started\n"); 2173 return; 2174 } 2175 if (test_done == 0) { 2176 printf("Packet forwarding already started\n"); 2177 return; 2178 } 2179 2180 2181 if(dcb_test) { 2182 for (i = 0; i < nb_fwd_ports; i++) { 2183 pt_id = fwd_ports_ids[i]; 2184 port = &ports[pt_id]; 2185 if (!port->dcb_flag) { 2186 printf("In DCB mode, all forwarding ports must " 2187 "be configured in this mode.\n"); 2188 return; 2189 } 2190 } 2191 if (nb_fwd_lcores == 1) { 2192 printf("In DCB mode,the nb forwarding cores " 2193 "should be larger than 1.\n"); 2194 return; 2195 } 2196 } 2197 test_done = 0; 2198 2199 fwd_config_setup(); 2200 2201 if(!no_flush_rx) 2202 flush_fwd_rx_queues(); 2203 2204 pkt_fwd_config_display(&cur_fwd_config); 2205 rxtx_config_display(); 2206 2207 fwd_stats_reset(); 2208 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2209 pt_id = fwd_ports_ids[i]; 2210 port = &ports[pt_id]; 2211 map_port_queue_stats_mapping_registers(pt_id, port); 2212 } 2213 if (with_tx_first) { 2214 port_fwd_begin = tx_only_engine.port_fwd_begin; 2215 if (port_fwd_begin != NULL) { 2216 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2217 (*port_fwd_begin)(fwd_ports_ids[i]); 2218 } 2219 while (with_tx_first--) { 2220 launch_packet_forwarding( 2221 run_one_txonly_burst_on_core); 2222 rte_eal_mp_wait_lcore(); 2223 } 2224 port_fwd_end = tx_only_engine.port_fwd_end; 2225 if (port_fwd_end != NULL) { 2226 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2227 (*port_fwd_end)(fwd_ports_ids[i]); 2228 } 2229 } 2230 launch_packet_forwarding(start_pkt_forward_on_core); 2231 } 2232 2233 void 2234 stop_packet_forwarding(void) 2235 { 2236 port_fwd_end_t port_fwd_end; 2237 lcoreid_t lc_id; 2238 portid_t pt_id; 2239 int i; 2240 2241 if (test_done) { 2242 printf("Packet forwarding not started\n"); 2243 return; 2244 } 2245 printf("Telling cores to stop..."); 2246 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2247 fwd_lcores[lc_id]->stopped = 1; 2248 printf("\nWaiting for lcores to finish...\n"); 2249 rte_eal_mp_wait_lcore(); 2250 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2251 if (port_fwd_end != NULL) { 2252 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2253 pt_id = fwd_ports_ids[i]; 2254 (*port_fwd_end)(pt_id); 2255 } 2256 } 2257 2258 fwd_stats_display(); 2259 2260 printf("\nDone.\n"); 2261 test_done = 1; 2262 } 2263 2264 void 2265 dev_set_link_up(portid_t pid) 2266 { 2267 if (rte_eth_dev_set_link_up(pid) < 0) 2268 printf("\nSet link up fail.\n"); 2269 } 2270 2271 void 2272 dev_set_link_down(portid_t pid) 2273 { 2274 if (rte_eth_dev_set_link_down(pid) < 0) 2275 printf("\nSet link down fail.\n"); 2276 } 2277 2278 static int 2279 all_ports_started(void) 2280 { 2281 portid_t pi; 2282 struct rte_port *port; 2283 2284 RTE_ETH_FOREACH_DEV(pi) { 2285 port = &ports[pi]; 2286 /* Check if there is a port which is not started */ 2287 if ((port->port_status != RTE_PORT_STARTED) && 2288 (port->slave_flag == 0)) 2289 return 0; 2290 } 2291 2292 /* No port is not started */ 2293 return 1; 2294 } 2295 2296 int 2297 port_is_stopped(portid_t port_id) 2298 { 2299 struct rte_port *port = &ports[port_id]; 2300 2301 if ((port->port_status != RTE_PORT_STOPPED) && 2302 (port->slave_flag == 0)) 2303 return 0; 2304 return 1; 2305 } 2306 2307 int 2308 all_ports_stopped(void) 2309 { 2310 portid_t pi; 2311 2312 RTE_ETH_FOREACH_DEV(pi) { 2313 if (!port_is_stopped(pi)) 2314 return 0; 2315 } 2316 2317 return 1; 2318 } 2319 2320 int 2321 port_is_started(portid_t port_id) 2322 { 2323 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2324 return 0; 2325 2326 if (ports[port_id].port_status != RTE_PORT_STARTED) 2327 return 0; 2328 2329 return 1; 2330 } 2331 2332 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2333 static int 2334 setup_hairpin_queues(portid_t pi) 2335 { 2336 queueid_t qi; 2337 struct rte_eth_hairpin_conf hairpin_conf = { 2338 .peer_count = 1, 2339 }; 2340 int i; 2341 int diag; 2342 struct rte_port *port = &ports[pi]; 2343 2344 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2345 hairpin_conf.peers[0].port = pi; 2346 hairpin_conf.peers[0].queue = i + nb_rxq; 2347 diag = rte_eth_tx_hairpin_queue_setup 2348 (pi, qi, nb_txd, &hairpin_conf); 2349 i++; 2350 if (diag == 0) 2351 continue; 2352 2353 /* Fail to setup rx queue, return */ 2354 if (rte_atomic16_cmpset(&(port->port_status), 2355 RTE_PORT_HANDLING, 2356 RTE_PORT_STOPPED) == 0) 2357 printf("Port %d can not be set back " 2358 "to stopped\n", pi); 2359 printf("Fail to configure port %d hairpin " 2360 "queues\n", pi); 2361 /* try to reconfigure queues next time */ 2362 port->need_reconfig_queues = 1; 2363 return -1; 2364 } 2365 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2366 hairpin_conf.peers[0].port = pi; 2367 hairpin_conf.peers[0].queue = i + nb_txq; 2368 diag = rte_eth_rx_hairpin_queue_setup 2369 (pi, qi, nb_rxd, &hairpin_conf); 2370 i++; 2371 if (diag == 0) 2372 continue; 2373 2374 /* Fail to setup rx queue, return */ 2375 if (rte_atomic16_cmpset(&(port->port_status), 2376 RTE_PORT_HANDLING, 2377 RTE_PORT_STOPPED) == 0) 2378 printf("Port %d can not be set back " 2379 "to stopped\n", pi); 2380 printf("Fail to configure port %d hairpin " 2381 "queues\n", pi); 2382 /* try to reconfigure queues next time */ 2383 port->need_reconfig_queues = 1; 2384 return -1; 2385 } 2386 return 0; 2387 } 2388 2389 int 2390 start_port(portid_t pid) 2391 { 2392 int diag, need_check_link_status = -1; 2393 portid_t pi; 2394 queueid_t qi; 2395 struct rte_port *port; 2396 struct rte_ether_addr mac_addr; 2397 struct rte_eth_hairpin_cap cap; 2398 2399 if (port_id_is_invalid(pid, ENABLED_WARN)) 2400 return 0; 2401 2402 if(dcb_config) 2403 dcb_test = 1; 2404 RTE_ETH_FOREACH_DEV(pi) { 2405 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2406 continue; 2407 2408 need_check_link_status = 0; 2409 port = &ports[pi]; 2410 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2411 RTE_PORT_HANDLING) == 0) { 2412 printf("Port %d is now not stopped\n", pi); 2413 continue; 2414 } 2415 2416 if (port->need_reconfig > 0) { 2417 port->need_reconfig = 0; 2418 2419 if (flow_isolate_all) { 2420 int ret = port_flow_isolate(pi, 1); 2421 if (ret) { 2422 printf("Failed to apply isolated" 2423 " mode on port %d\n", pi); 2424 return -1; 2425 } 2426 } 2427 configure_rxtx_dump_callbacks(0); 2428 printf("Configuring Port %d (socket %u)\n", pi, 2429 port->socket_id); 2430 if (nb_hairpinq > 0 && 2431 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2432 printf("Port %d doesn't support hairpin " 2433 "queues\n", pi); 2434 return -1; 2435 } 2436 /* configure port */ 2437 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2438 nb_txq + nb_hairpinq, 2439 &(port->dev_conf)); 2440 if (diag != 0) { 2441 if (rte_atomic16_cmpset(&(port->port_status), 2442 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2443 printf("Port %d can not be set back " 2444 "to stopped\n", pi); 2445 printf("Fail to configure port %d\n", pi); 2446 /* try to reconfigure port next time */ 2447 port->need_reconfig = 1; 2448 return -1; 2449 } 2450 } 2451 if (port->need_reconfig_queues > 0) { 2452 port->need_reconfig_queues = 0; 2453 /* setup tx queues */ 2454 for (qi = 0; qi < nb_txq; qi++) { 2455 if ((numa_support) && 2456 (txring_numa[pi] != NUMA_NO_CONFIG)) 2457 diag = rte_eth_tx_queue_setup(pi, qi, 2458 port->nb_tx_desc[qi], 2459 txring_numa[pi], 2460 &(port->tx_conf[qi])); 2461 else 2462 diag = rte_eth_tx_queue_setup(pi, qi, 2463 port->nb_tx_desc[qi], 2464 port->socket_id, 2465 &(port->tx_conf[qi])); 2466 2467 if (diag == 0) 2468 continue; 2469 2470 /* Fail to setup tx queue, return */ 2471 if (rte_atomic16_cmpset(&(port->port_status), 2472 RTE_PORT_HANDLING, 2473 RTE_PORT_STOPPED) == 0) 2474 printf("Port %d can not be set back " 2475 "to stopped\n", pi); 2476 printf("Fail to configure port %d tx queues\n", 2477 pi); 2478 /* try to reconfigure queues next time */ 2479 port->need_reconfig_queues = 1; 2480 return -1; 2481 } 2482 for (qi = 0; qi < nb_rxq; qi++) { 2483 /* setup rx queues */ 2484 if ((numa_support) && 2485 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2486 struct rte_mempool * mp = 2487 mbuf_pool_find(rxring_numa[pi]); 2488 if (mp == NULL) { 2489 printf("Failed to setup RX queue:" 2490 "No mempool allocation" 2491 " on the socket %d\n", 2492 rxring_numa[pi]); 2493 return -1; 2494 } 2495 2496 diag = rte_eth_rx_queue_setup(pi, qi, 2497 port->nb_rx_desc[qi], 2498 rxring_numa[pi], 2499 &(port->rx_conf[qi]), 2500 mp); 2501 } else { 2502 struct rte_mempool *mp = 2503 mbuf_pool_find(port->socket_id); 2504 if (mp == NULL) { 2505 printf("Failed to setup RX queue:" 2506 "No mempool allocation" 2507 " on the socket %d\n", 2508 port->socket_id); 2509 return -1; 2510 } 2511 diag = rte_eth_rx_queue_setup(pi, qi, 2512 port->nb_rx_desc[qi], 2513 port->socket_id, 2514 &(port->rx_conf[qi]), 2515 mp); 2516 } 2517 if (diag == 0) 2518 continue; 2519 2520 /* Fail to setup rx queue, return */ 2521 if (rte_atomic16_cmpset(&(port->port_status), 2522 RTE_PORT_HANDLING, 2523 RTE_PORT_STOPPED) == 0) 2524 printf("Port %d can not be set back " 2525 "to stopped\n", pi); 2526 printf("Fail to configure port %d rx queues\n", 2527 pi); 2528 /* try to reconfigure queues next time */ 2529 port->need_reconfig_queues = 1; 2530 return -1; 2531 } 2532 /* setup hairpin queues */ 2533 if (setup_hairpin_queues(pi) != 0) 2534 return -1; 2535 } 2536 configure_rxtx_dump_callbacks(verbose_level); 2537 if (clear_ptypes) { 2538 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2539 NULL, 0); 2540 if (diag < 0) 2541 printf( 2542 "Port %d: Failed to disable Ptype parsing\n", 2543 pi); 2544 } 2545 2546 /* start port */ 2547 if (rte_eth_dev_start(pi) < 0) { 2548 printf("Fail to start port %d\n", pi); 2549 2550 /* Fail to setup rx queue, return */ 2551 if (rte_atomic16_cmpset(&(port->port_status), 2552 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2553 printf("Port %d can not be set back to " 2554 "stopped\n", pi); 2555 continue; 2556 } 2557 2558 if (rte_atomic16_cmpset(&(port->port_status), 2559 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2560 printf("Port %d can not be set into started\n", pi); 2561 2562 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2563 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2564 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2565 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2566 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2567 2568 /* at least one port started, need checking link status */ 2569 need_check_link_status = 1; 2570 } 2571 2572 if (need_check_link_status == 1 && !no_link_check) 2573 check_all_ports_link_status(RTE_PORT_ALL); 2574 else if (need_check_link_status == 0) 2575 printf("Please stop the ports first\n"); 2576 2577 printf("Done\n"); 2578 return 0; 2579 } 2580 2581 void 2582 stop_port(portid_t pid) 2583 { 2584 portid_t pi; 2585 struct rte_port *port; 2586 int need_check_link_status = 0; 2587 2588 if (dcb_test) { 2589 dcb_test = 0; 2590 dcb_config = 0; 2591 } 2592 2593 if (port_id_is_invalid(pid, ENABLED_WARN)) 2594 return; 2595 2596 printf("Stopping ports...\n"); 2597 2598 RTE_ETH_FOREACH_DEV(pi) { 2599 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2600 continue; 2601 2602 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2603 printf("Please remove port %d from forwarding configuration.\n", pi); 2604 continue; 2605 } 2606 2607 if (port_is_bonding_slave(pi)) { 2608 printf("Please remove port %d from bonded device.\n", pi); 2609 continue; 2610 } 2611 2612 port = &ports[pi]; 2613 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2614 RTE_PORT_HANDLING) == 0) 2615 continue; 2616 2617 rte_eth_dev_stop(pi); 2618 2619 if (rte_atomic16_cmpset(&(port->port_status), 2620 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2621 printf("Port %d can not be set into stopped\n", pi); 2622 need_check_link_status = 1; 2623 } 2624 if (need_check_link_status && !no_link_check) 2625 check_all_ports_link_status(RTE_PORT_ALL); 2626 2627 printf("Done\n"); 2628 } 2629 2630 static void 2631 remove_invalid_ports_in(portid_t *array, portid_t *total) 2632 { 2633 portid_t i; 2634 portid_t new_total = 0; 2635 2636 for (i = 0; i < *total; i++) 2637 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2638 array[new_total] = array[i]; 2639 new_total++; 2640 } 2641 *total = new_total; 2642 } 2643 2644 static void 2645 remove_invalid_ports(void) 2646 { 2647 remove_invalid_ports_in(ports_ids, &nb_ports); 2648 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2649 nb_cfg_ports = nb_fwd_ports; 2650 } 2651 2652 void 2653 close_port(portid_t pid) 2654 { 2655 portid_t pi; 2656 struct rte_port *port; 2657 2658 if (port_id_is_invalid(pid, ENABLED_WARN)) 2659 return; 2660 2661 printf("Closing ports...\n"); 2662 2663 RTE_ETH_FOREACH_DEV(pi) { 2664 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2665 continue; 2666 2667 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2668 printf("Please remove port %d from forwarding configuration.\n", pi); 2669 continue; 2670 } 2671 2672 if (port_is_bonding_slave(pi)) { 2673 printf("Please remove port %d from bonded device.\n", pi); 2674 continue; 2675 } 2676 2677 port = &ports[pi]; 2678 if (rte_atomic16_cmpset(&(port->port_status), 2679 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2680 printf("Port %d is already closed\n", pi); 2681 continue; 2682 } 2683 2684 if (rte_atomic16_cmpset(&(port->port_status), 2685 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2686 printf("Port %d is now not stopped\n", pi); 2687 continue; 2688 } 2689 2690 if (port->flow_list) 2691 port_flow_flush(pi); 2692 rte_eth_dev_close(pi); 2693 2694 remove_invalid_ports(); 2695 2696 if (rte_atomic16_cmpset(&(port->port_status), 2697 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2698 printf("Port %d cannot be set to closed\n", pi); 2699 } 2700 2701 printf("Done\n"); 2702 } 2703 2704 void 2705 reset_port(portid_t pid) 2706 { 2707 int diag; 2708 portid_t pi; 2709 struct rte_port *port; 2710 2711 if (port_id_is_invalid(pid, ENABLED_WARN)) 2712 return; 2713 2714 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2715 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2716 printf("Can not reset port(s), please stop port(s) first.\n"); 2717 return; 2718 } 2719 2720 printf("Resetting ports...\n"); 2721 2722 RTE_ETH_FOREACH_DEV(pi) { 2723 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2724 continue; 2725 2726 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2727 printf("Please remove port %d from forwarding " 2728 "configuration.\n", pi); 2729 continue; 2730 } 2731 2732 if (port_is_bonding_slave(pi)) { 2733 printf("Please remove port %d from bonded device.\n", 2734 pi); 2735 continue; 2736 } 2737 2738 diag = rte_eth_dev_reset(pi); 2739 if (diag == 0) { 2740 port = &ports[pi]; 2741 port->need_reconfig = 1; 2742 port->need_reconfig_queues = 1; 2743 } else { 2744 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2745 } 2746 } 2747 2748 printf("Done\n"); 2749 } 2750 2751 void 2752 attach_port(char *identifier) 2753 { 2754 portid_t pi; 2755 struct rte_dev_iterator iterator; 2756 2757 printf("Attaching a new port...\n"); 2758 2759 if (identifier == NULL) { 2760 printf("Invalid parameters are specified\n"); 2761 return; 2762 } 2763 2764 if (rte_dev_probe(identifier) < 0) { 2765 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2766 return; 2767 } 2768 2769 /* first attach mode: event */ 2770 if (setup_on_probe_event) { 2771 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2772 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2773 if (ports[pi].port_status == RTE_PORT_HANDLING && 2774 ports[pi].need_setup != 0) 2775 setup_attached_port(pi); 2776 return; 2777 } 2778 2779 /* second attach mode: iterator */ 2780 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2781 /* setup ports matching the devargs used for probing */ 2782 if (port_is_forwarding(pi)) 2783 continue; /* port was already attached before */ 2784 setup_attached_port(pi); 2785 } 2786 } 2787 2788 static void 2789 setup_attached_port(portid_t pi) 2790 { 2791 unsigned int socket_id; 2792 int ret; 2793 2794 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2795 /* if socket_id is invalid, set to the first available socket. */ 2796 if (check_socket_id(socket_id) < 0) 2797 socket_id = socket_ids[0]; 2798 reconfig(pi, socket_id); 2799 ret = rte_eth_promiscuous_enable(pi); 2800 if (ret != 0) 2801 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2802 pi, rte_strerror(-ret)); 2803 2804 ports_ids[nb_ports++] = pi; 2805 fwd_ports_ids[nb_fwd_ports++] = pi; 2806 nb_cfg_ports = nb_fwd_ports; 2807 ports[pi].need_setup = 0; 2808 ports[pi].port_status = RTE_PORT_STOPPED; 2809 2810 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2811 printf("Done\n"); 2812 } 2813 2814 static void 2815 detach_device(struct rte_device *dev) 2816 { 2817 portid_t sibling; 2818 2819 if (dev == NULL) { 2820 printf("Device already removed\n"); 2821 return; 2822 } 2823 2824 printf("Removing a device...\n"); 2825 2826 if (rte_dev_remove(dev) < 0) { 2827 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2828 return; 2829 } 2830 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2831 /* reset mapping between old ports and removed device */ 2832 rte_eth_devices[sibling].device = NULL; 2833 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2834 /* sibling ports are forced to be closed */ 2835 ports[sibling].port_status = RTE_PORT_CLOSED; 2836 printf("Port %u is closed\n", sibling); 2837 } 2838 } 2839 2840 remove_invalid_ports(); 2841 2842 printf("Device is detached\n"); 2843 printf("Now total ports is %d\n", nb_ports); 2844 printf("Done\n"); 2845 return; 2846 } 2847 2848 void 2849 detach_port_device(portid_t port_id) 2850 { 2851 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2852 return; 2853 2854 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2855 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2856 printf("Port not stopped\n"); 2857 return; 2858 } 2859 printf("Port was not closed\n"); 2860 if (ports[port_id].flow_list) 2861 port_flow_flush(port_id); 2862 } 2863 2864 detach_device(rte_eth_devices[port_id].device); 2865 } 2866 2867 void 2868 detach_devargs(char *identifier) 2869 { 2870 struct rte_dev_iterator iterator; 2871 struct rte_devargs da; 2872 portid_t port_id; 2873 2874 printf("Removing a device...\n"); 2875 2876 memset(&da, 0, sizeof(da)); 2877 if (rte_devargs_parsef(&da, "%s", identifier)) { 2878 printf("cannot parse identifier\n"); 2879 if (da.args) 2880 free(da.args); 2881 return; 2882 } 2883 2884 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2885 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2886 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2887 printf("Port %u not stopped\n", port_id); 2888 rte_eth_iterator_cleanup(&iterator); 2889 return; 2890 } 2891 2892 /* sibling ports are forced to be closed */ 2893 if (ports[port_id].flow_list) 2894 port_flow_flush(port_id); 2895 ports[port_id].port_status = RTE_PORT_CLOSED; 2896 printf("Port %u is now closed\n", port_id); 2897 } 2898 } 2899 2900 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2901 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2902 da.name, da.bus->name); 2903 return; 2904 } 2905 2906 remove_invalid_ports(); 2907 2908 printf("Device %s is detached\n", identifier); 2909 printf("Now total ports is %d\n", nb_ports); 2910 printf("Done\n"); 2911 } 2912 2913 void 2914 pmd_test_exit(void) 2915 { 2916 portid_t pt_id; 2917 int ret; 2918 int i; 2919 2920 if (test_done == 0) 2921 stop_packet_forwarding(); 2922 2923 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2924 if (mempools[i]) { 2925 if (mp_alloc_type == MP_ALLOC_ANON) 2926 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2927 NULL); 2928 } 2929 } 2930 if (ports != NULL) { 2931 no_link_check = 1; 2932 RTE_ETH_FOREACH_DEV(pt_id) { 2933 printf("\nStopping port %d...\n", pt_id); 2934 fflush(stdout); 2935 stop_port(pt_id); 2936 } 2937 RTE_ETH_FOREACH_DEV(pt_id) { 2938 printf("\nShutting down port %d...\n", pt_id); 2939 fflush(stdout); 2940 close_port(pt_id); 2941 } 2942 } 2943 2944 if (hot_plug) { 2945 ret = rte_dev_event_monitor_stop(); 2946 if (ret) { 2947 RTE_LOG(ERR, EAL, 2948 "fail to stop device event monitor."); 2949 return; 2950 } 2951 2952 ret = rte_dev_event_callback_unregister(NULL, 2953 dev_event_callback, NULL); 2954 if (ret < 0) { 2955 RTE_LOG(ERR, EAL, 2956 "fail to unregister device event callback.\n"); 2957 return; 2958 } 2959 2960 ret = rte_dev_hotplug_handle_disable(); 2961 if (ret) { 2962 RTE_LOG(ERR, EAL, 2963 "fail to disable hotplug handling.\n"); 2964 return; 2965 } 2966 } 2967 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2968 if (mempools[i]) 2969 rte_mempool_free(mempools[i]); 2970 } 2971 2972 printf("\nBye...\n"); 2973 } 2974 2975 typedef void (*cmd_func_t)(void); 2976 struct pmd_test_command { 2977 const char *cmd_name; 2978 cmd_func_t cmd_func; 2979 }; 2980 2981 /* Check the link status of all ports in up to 9s, and print them finally */ 2982 static void 2983 check_all_ports_link_status(uint32_t port_mask) 2984 { 2985 #define CHECK_INTERVAL 100 /* 100ms */ 2986 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2987 portid_t portid; 2988 uint8_t count, all_ports_up, print_flag = 0; 2989 struct rte_eth_link link; 2990 int ret; 2991 2992 printf("Checking link statuses...\n"); 2993 fflush(stdout); 2994 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2995 all_ports_up = 1; 2996 RTE_ETH_FOREACH_DEV(portid) { 2997 if ((port_mask & (1 << portid)) == 0) 2998 continue; 2999 memset(&link, 0, sizeof(link)); 3000 ret = rte_eth_link_get_nowait(portid, &link); 3001 if (ret < 0) { 3002 all_ports_up = 0; 3003 if (print_flag == 1) 3004 printf("Port %u link get failed: %s\n", 3005 portid, rte_strerror(-ret)); 3006 continue; 3007 } 3008 /* print link status if flag set */ 3009 if (print_flag == 1) { 3010 if (link.link_status) 3011 printf( 3012 "Port%d Link Up. speed %u Mbps- %s\n", 3013 portid, link.link_speed, 3014 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 3015 ("full-duplex") : ("half-duplex")); 3016 else 3017 printf("Port %d Link Down\n", portid); 3018 continue; 3019 } 3020 /* clear all_ports_up flag if any link down */ 3021 if (link.link_status == ETH_LINK_DOWN) { 3022 all_ports_up = 0; 3023 break; 3024 } 3025 } 3026 /* after finally printing all link status, get out */ 3027 if (print_flag == 1) 3028 break; 3029 3030 if (all_ports_up == 0) { 3031 fflush(stdout); 3032 rte_delay_ms(CHECK_INTERVAL); 3033 } 3034 3035 /* set the print_flag if all ports up or timeout */ 3036 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3037 print_flag = 1; 3038 } 3039 3040 if (lsc_interrupt) 3041 break; 3042 } 3043 } 3044 3045 /* 3046 * This callback is for remove a port for a device. It has limitation because 3047 * it is not for multiple port removal for a device. 3048 * TODO: the device detach invoke will plan to be removed from user side to 3049 * eal. And convert all PMDs to free port resources on ether device closing. 3050 */ 3051 static void 3052 rmv_port_callback(void *arg) 3053 { 3054 int need_to_start = 0; 3055 int org_no_link_check = no_link_check; 3056 portid_t port_id = (intptr_t)arg; 3057 struct rte_device *dev; 3058 3059 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3060 3061 if (!test_done && port_is_forwarding(port_id)) { 3062 need_to_start = 1; 3063 stop_packet_forwarding(); 3064 } 3065 no_link_check = 1; 3066 stop_port(port_id); 3067 no_link_check = org_no_link_check; 3068 3069 /* Save rte_device pointer before closing ethdev port */ 3070 dev = rte_eth_devices[port_id].device; 3071 close_port(port_id); 3072 detach_device(dev); /* might be already removed or have more ports */ 3073 3074 if (need_to_start) 3075 start_packet_forwarding(0); 3076 } 3077 3078 /* This function is used by the interrupt thread */ 3079 static int 3080 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3081 void *ret_param) 3082 { 3083 RTE_SET_USED(param); 3084 RTE_SET_USED(ret_param); 3085 3086 if (type >= RTE_ETH_EVENT_MAX) { 3087 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3088 port_id, __func__, type); 3089 fflush(stderr); 3090 } else if (event_print_mask & (UINT32_C(1) << type)) { 3091 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3092 eth_event_desc[type]); 3093 fflush(stdout); 3094 } 3095 3096 switch (type) { 3097 case RTE_ETH_EVENT_NEW: 3098 ports[port_id].need_setup = 1; 3099 ports[port_id].port_status = RTE_PORT_HANDLING; 3100 break; 3101 case RTE_ETH_EVENT_INTR_RMV: 3102 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3103 break; 3104 if (rte_eal_alarm_set(100000, 3105 rmv_port_callback, (void *)(intptr_t)port_id)) 3106 fprintf(stderr, "Could not set up deferred device removal\n"); 3107 break; 3108 default: 3109 break; 3110 } 3111 return 0; 3112 } 3113 3114 static int 3115 register_eth_event_callback(void) 3116 { 3117 int ret; 3118 enum rte_eth_event_type event; 3119 3120 for (event = RTE_ETH_EVENT_UNKNOWN; 3121 event < RTE_ETH_EVENT_MAX; event++) { 3122 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3123 event, 3124 eth_event_callback, 3125 NULL); 3126 if (ret != 0) { 3127 TESTPMD_LOG(ERR, "Failed to register callback for " 3128 "%s event\n", eth_event_desc[event]); 3129 return -1; 3130 } 3131 } 3132 3133 return 0; 3134 } 3135 3136 /* This function is used by the interrupt thread */ 3137 static void 3138 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3139 __rte_unused void *arg) 3140 { 3141 uint16_t port_id; 3142 int ret; 3143 3144 if (type >= RTE_DEV_EVENT_MAX) { 3145 fprintf(stderr, "%s called upon invalid event %d\n", 3146 __func__, type); 3147 fflush(stderr); 3148 } 3149 3150 switch (type) { 3151 case RTE_DEV_EVENT_REMOVE: 3152 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3153 device_name); 3154 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3155 if (ret) { 3156 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3157 device_name); 3158 return; 3159 } 3160 /* 3161 * Because the user's callback is invoked in eal interrupt 3162 * callback, the interrupt callback need to be finished before 3163 * it can be unregistered when detaching device. So finish 3164 * callback soon and use a deferred removal to detach device 3165 * is need. It is a workaround, once the device detaching be 3166 * moved into the eal in the future, the deferred removal could 3167 * be deleted. 3168 */ 3169 if (rte_eal_alarm_set(100000, 3170 rmv_port_callback, (void *)(intptr_t)port_id)) 3171 RTE_LOG(ERR, EAL, 3172 "Could not set up deferred device removal\n"); 3173 break; 3174 case RTE_DEV_EVENT_ADD: 3175 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3176 device_name); 3177 /* TODO: After finish kernel driver binding, 3178 * begin to attach port. 3179 */ 3180 break; 3181 default: 3182 break; 3183 } 3184 } 3185 3186 static int 3187 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3188 { 3189 uint16_t i; 3190 int diag; 3191 uint8_t mapping_found = 0; 3192 3193 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3194 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3195 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3196 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3197 tx_queue_stats_mappings[i].queue_id, 3198 tx_queue_stats_mappings[i].stats_counter_id); 3199 if (diag != 0) 3200 return diag; 3201 mapping_found = 1; 3202 } 3203 } 3204 if (mapping_found) 3205 port->tx_queue_stats_mapping_enabled = 1; 3206 return 0; 3207 } 3208 3209 static int 3210 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3211 { 3212 uint16_t i; 3213 int diag; 3214 uint8_t mapping_found = 0; 3215 3216 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3217 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3218 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3219 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3220 rx_queue_stats_mappings[i].queue_id, 3221 rx_queue_stats_mappings[i].stats_counter_id); 3222 if (diag != 0) 3223 return diag; 3224 mapping_found = 1; 3225 } 3226 } 3227 if (mapping_found) 3228 port->rx_queue_stats_mapping_enabled = 1; 3229 return 0; 3230 } 3231 3232 static void 3233 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3234 { 3235 int diag = 0; 3236 3237 diag = set_tx_queue_stats_mapping_registers(pi, port); 3238 if (diag != 0) { 3239 if (diag == -ENOTSUP) { 3240 port->tx_queue_stats_mapping_enabled = 0; 3241 printf("TX queue stats mapping not supported port id=%d\n", pi); 3242 } 3243 else 3244 rte_exit(EXIT_FAILURE, 3245 "set_tx_queue_stats_mapping_registers " 3246 "failed for port id=%d diag=%d\n", 3247 pi, diag); 3248 } 3249 3250 diag = set_rx_queue_stats_mapping_registers(pi, port); 3251 if (diag != 0) { 3252 if (diag == -ENOTSUP) { 3253 port->rx_queue_stats_mapping_enabled = 0; 3254 printf("RX queue stats mapping not supported port id=%d\n", pi); 3255 } 3256 else 3257 rte_exit(EXIT_FAILURE, 3258 "set_rx_queue_stats_mapping_registers " 3259 "failed for port id=%d diag=%d\n", 3260 pi, diag); 3261 } 3262 } 3263 3264 static void 3265 rxtx_port_config(struct rte_port *port) 3266 { 3267 uint16_t qid; 3268 uint64_t offloads; 3269 3270 for (qid = 0; qid < nb_rxq; qid++) { 3271 offloads = port->rx_conf[qid].offloads; 3272 port->rx_conf[qid] = port->dev_info.default_rxconf; 3273 if (offloads != 0) 3274 port->rx_conf[qid].offloads = offloads; 3275 3276 /* Check if any Rx parameters have been passed */ 3277 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3278 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3279 3280 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3281 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3282 3283 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3284 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3285 3286 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3287 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3288 3289 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3290 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3291 3292 port->nb_rx_desc[qid] = nb_rxd; 3293 } 3294 3295 for (qid = 0; qid < nb_txq; qid++) { 3296 offloads = port->tx_conf[qid].offloads; 3297 port->tx_conf[qid] = port->dev_info.default_txconf; 3298 if (offloads != 0) 3299 port->tx_conf[qid].offloads = offloads; 3300 3301 /* Check if any Tx parameters have been passed */ 3302 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3303 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3304 3305 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3306 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3307 3308 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3309 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3310 3311 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3312 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3313 3314 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3315 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3316 3317 port->nb_tx_desc[qid] = nb_txd; 3318 } 3319 } 3320 3321 void 3322 init_port_config(void) 3323 { 3324 portid_t pid; 3325 struct rte_port *port; 3326 int ret; 3327 3328 RTE_ETH_FOREACH_DEV(pid) { 3329 port = &ports[pid]; 3330 port->dev_conf.fdir_conf = fdir_conf; 3331 3332 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3333 if (ret != 0) 3334 return; 3335 3336 if (nb_rxq > 1) { 3337 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3338 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3339 rss_hf & port->dev_info.flow_type_rss_offloads; 3340 } else { 3341 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3342 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3343 } 3344 3345 if (port->dcb_flag == 0) { 3346 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3347 port->dev_conf.rxmode.mq_mode = 3348 (enum rte_eth_rx_mq_mode) 3349 (rx_mq_mode & ETH_MQ_RX_RSS); 3350 else 3351 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3352 } 3353 3354 rxtx_port_config(port); 3355 3356 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3357 if (ret != 0) 3358 return; 3359 3360 map_port_queue_stats_mapping_registers(pid, port); 3361 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3362 rte_pmd_ixgbe_bypass_init(pid); 3363 #endif 3364 3365 if (lsc_interrupt && 3366 (rte_eth_devices[pid].data->dev_flags & 3367 RTE_ETH_DEV_INTR_LSC)) 3368 port->dev_conf.intr_conf.lsc = 1; 3369 if (rmv_interrupt && 3370 (rte_eth_devices[pid].data->dev_flags & 3371 RTE_ETH_DEV_INTR_RMV)) 3372 port->dev_conf.intr_conf.rmv = 1; 3373 } 3374 } 3375 3376 void set_port_slave_flag(portid_t slave_pid) 3377 { 3378 struct rte_port *port; 3379 3380 port = &ports[slave_pid]; 3381 port->slave_flag = 1; 3382 } 3383 3384 void clear_port_slave_flag(portid_t slave_pid) 3385 { 3386 struct rte_port *port; 3387 3388 port = &ports[slave_pid]; 3389 port->slave_flag = 0; 3390 } 3391 3392 uint8_t port_is_bonding_slave(portid_t slave_pid) 3393 { 3394 struct rte_port *port; 3395 3396 port = &ports[slave_pid]; 3397 if ((rte_eth_devices[slave_pid].data->dev_flags & 3398 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3399 return 1; 3400 return 0; 3401 } 3402 3403 const uint16_t vlan_tags[] = { 3404 0, 1, 2, 3, 4, 5, 6, 7, 3405 8, 9, 10, 11, 12, 13, 14, 15, 3406 16, 17, 18, 19, 20, 21, 22, 23, 3407 24, 25, 26, 27, 28, 29, 30, 31 3408 }; 3409 3410 static int 3411 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3412 enum dcb_mode_enable dcb_mode, 3413 enum rte_eth_nb_tcs num_tcs, 3414 uint8_t pfc_en) 3415 { 3416 uint8_t i; 3417 int32_t rc; 3418 struct rte_eth_rss_conf rss_conf; 3419 3420 /* 3421 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3422 * given above, and the number of traffic classes available for use. 3423 */ 3424 if (dcb_mode == DCB_VT_ENABLED) { 3425 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3426 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3427 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3428 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3429 3430 /* VMDQ+DCB RX and TX configurations */ 3431 vmdq_rx_conf->enable_default_pool = 0; 3432 vmdq_rx_conf->default_pool = 0; 3433 vmdq_rx_conf->nb_queue_pools = 3434 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3435 vmdq_tx_conf->nb_queue_pools = 3436 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3437 3438 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3439 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3440 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3441 vmdq_rx_conf->pool_map[i].pools = 3442 1 << (i % vmdq_rx_conf->nb_queue_pools); 3443 } 3444 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3445 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3446 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3447 } 3448 3449 /* set DCB mode of RX and TX of multiple queues */ 3450 eth_conf->rxmode.mq_mode = 3451 (enum rte_eth_rx_mq_mode) 3452 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3453 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3454 } else { 3455 struct rte_eth_dcb_rx_conf *rx_conf = 3456 ð_conf->rx_adv_conf.dcb_rx_conf; 3457 struct rte_eth_dcb_tx_conf *tx_conf = 3458 ð_conf->tx_adv_conf.dcb_tx_conf; 3459 3460 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3461 if (rc != 0) 3462 return rc; 3463 3464 rx_conf->nb_tcs = num_tcs; 3465 tx_conf->nb_tcs = num_tcs; 3466 3467 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3468 rx_conf->dcb_tc[i] = i % num_tcs; 3469 tx_conf->dcb_tc[i] = i % num_tcs; 3470 } 3471 3472 eth_conf->rxmode.mq_mode = 3473 (enum rte_eth_rx_mq_mode) 3474 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3475 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3476 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3477 } 3478 3479 if (pfc_en) 3480 eth_conf->dcb_capability_en = 3481 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3482 else 3483 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3484 3485 return 0; 3486 } 3487 3488 int 3489 init_port_dcb_config(portid_t pid, 3490 enum dcb_mode_enable dcb_mode, 3491 enum rte_eth_nb_tcs num_tcs, 3492 uint8_t pfc_en) 3493 { 3494 struct rte_eth_conf port_conf; 3495 struct rte_port *rte_port; 3496 int retval; 3497 uint16_t i; 3498 3499 rte_port = &ports[pid]; 3500 3501 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3502 /* Enter DCB configuration status */ 3503 dcb_config = 1; 3504 3505 port_conf.rxmode = rte_port->dev_conf.rxmode; 3506 port_conf.txmode = rte_port->dev_conf.txmode; 3507 3508 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3509 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3510 if (retval < 0) 3511 return retval; 3512 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3513 3514 /* re-configure the device . */ 3515 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3516 if (retval < 0) 3517 return retval; 3518 3519 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3520 if (retval != 0) 3521 return retval; 3522 3523 /* If dev_info.vmdq_pool_base is greater than 0, 3524 * the queue id of vmdq pools is started after pf queues. 3525 */ 3526 if (dcb_mode == DCB_VT_ENABLED && 3527 rte_port->dev_info.vmdq_pool_base > 0) { 3528 printf("VMDQ_DCB multi-queue mode is nonsensical" 3529 " for port %d.", pid); 3530 return -1; 3531 } 3532 3533 /* Assume the ports in testpmd have the same dcb capability 3534 * and has the same number of rxq and txq in dcb mode 3535 */ 3536 if (dcb_mode == DCB_VT_ENABLED) { 3537 if (rte_port->dev_info.max_vfs > 0) { 3538 nb_rxq = rte_port->dev_info.nb_rx_queues; 3539 nb_txq = rte_port->dev_info.nb_tx_queues; 3540 } else { 3541 nb_rxq = rte_port->dev_info.max_rx_queues; 3542 nb_txq = rte_port->dev_info.max_tx_queues; 3543 } 3544 } else { 3545 /*if vt is disabled, use all pf queues */ 3546 if (rte_port->dev_info.vmdq_pool_base == 0) { 3547 nb_rxq = rte_port->dev_info.max_rx_queues; 3548 nb_txq = rte_port->dev_info.max_tx_queues; 3549 } else { 3550 nb_rxq = (queueid_t)num_tcs; 3551 nb_txq = (queueid_t)num_tcs; 3552 3553 } 3554 } 3555 rx_free_thresh = 64; 3556 3557 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3558 3559 rxtx_port_config(rte_port); 3560 /* VLAN filter */ 3561 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3562 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3563 rx_vft_set(pid, vlan_tags[i], 1); 3564 3565 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3566 if (retval != 0) 3567 return retval; 3568 3569 map_port_queue_stats_mapping_registers(pid, rte_port); 3570 3571 rte_port->dcb_flag = 1; 3572 3573 return 0; 3574 } 3575 3576 static void 3577 init_port(void) 3578 { 3579 /* Configuration of Ethernet ports. */ 3580 ports = rte_zmalloc("testpmd: ports", 3581 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3582 RTE_CACHE_LINE_SIZE); 3583 if (ports == NULL) { 3584 rte_exit(EXIT_FAILURE, 3585 "rte_zmalloc(%d struct rte_port) failed\n", 3586 RTE_MAX_ETHPORTS); 3587 } 3588 3589 /* Initialize ports NUMA structures */ 3590 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3591 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3592 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3593 } 3594 3595 static void 3596 force_quit(void) 3597 { 3598 pmd_test_exit(); 3599 prompt_exit(); 3600 } 3601 3602 static void 3603 print_stats(void) 3604 { 3605 uint8_t i; 3606 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3607 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3608 3609 /* Clear screen and move to top left */ 3610 printf("%s%s", clr, top_left); 3611 3612 printf("\nPort statistics ===================================="); 3613 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3614 nic_stats_display(fwd_ports_ids[i]); 3615 3616 fflush(stdout); 3617 } 3618 3619 static void 3620 signal_handler(int signum) 3621 { 3622 if (signum == SIGINT || signum == SIGTERM) { 3623 printf("\nSignal %d received, preparing to exit...\n", 3624 signum); 3625 #ifdef RTE_LIBRTE_PDUMP 3626 /* uninitialize packet capture framework */ 3627 rte_pdump_uninit(); 3628 #endif 3629 #ifdef RTE_LIBRTE_LATENCY_STATS 3630 if (latencystats_enabled != 0) 3631 rte_latencystats_uninit(); 3632 #endif 3633 force_quit(); 3634 /* Set flag to indicate the force termination. */ 3635 f_quit = 1; 3636 /* exit with the expected status */ 3637 signal(signum, SIG_DFL); 3638 kill(getpid(), signum); 3639 } 3640 } 3641 3642 int 3643 main(int argc, char** argv) 3644 { 3645 int diag; 3646 portid_t port_id; 3647 uint16_t count; 3648 int ret; 3649 3650 signal(SIGINT, signal_handler); 3651 signal(SIGTERM, signal_handler); 3652 3653 testpmd_logtype = rte_log_register("testpmd"); 3654 if (testpmd_logtype < 0) 3655 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3656 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3657 3658 diag = rte_eal_init(argc, argv); 3659 if (diag < 0) 3660 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3661 rte_strerror(rte_errno)); 3662 3663 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3664 rte_exit(EXIT_FAILURE, 3665 "Secondary process type not supported.\n"); 3666 3667 ret = register_eth_event_callback(); 3668 if (ret != 0) 3669 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3670 3671 #ifdef RTE_LIBRTE_PDUMP 3672 /* initialize packet capture framework */ 3673 rte_pdump_init(); 3674 #endif 3675 3676 count = 0; 3677 RTE_ETH_FOREACH_DEV(port_id) { 3678 ports_ids[count] = port_id; 3679 count++; 3680 } 3681 nb_ports = (portid_t) count; 3682 if (nb_ports == 0) 3683 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3684 3685 /* allocate port structures, and init them */ 3686 init_port(); 3687 3688 set_def_fwd_config(); 3689 if (nb_lcores == 0) 3690 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3691 "Check the core mask argument\n"); 3692 3693 /* Bitrate/latency stats disabled by default */ 3694 #ifdef RTE_LIBRTE_BITRATE 3695 bitrate_enabled = 0; 3696 #endif 3697 #ifdef RTE_LIBRTE_LATENCY_STATS 3698 latencystats_enabled = 0; 3699 #endif 3700 3701 /* on FreeBSD, mlockall() is disabled by default */ 3702 #ifdef RTE_EXEC_ENV_FREEBSD 3703 do_mlockall = 0; 3704 #else 3705 do_mlockall = 1; 3706 #endif 3707 3708 argc -= diag; 3709 argv += diag; 3710 if (argc > 1) 3711 launch_args_parse(argc, argv); 3712 3713 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3714 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3715 strerror(errno)); 3716 } 3717 3718 if (tx_first && interactive) 3719 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3720 "interactive mode.\n"); 3721 3722 if (tx_first && lsc_interrupt) { 3723 printf("Warning: lsc_interrupt needs to be off when " 3724 " using tx_first. Disabling.\n"); 3725 lsc_interrupt = 0; 3726 } 3727 3728 if (!nb_rxq && !nb_txq) 3729 printf("Warning: Either rx or tx queues should be non-zero\n"); 3730 3731 if (nb_rxq > 1 && nb_rxq > nb_txq) 3732 printf("Warning: nb_rxq=%d enables RSS configuration, " 3733 "but nb_txq=%d will prevent to fully test it.\n", 3734 nb_rxq, nb_txq); 3735 3736 init_config(); 3737 3738 if (hot_plug) { 3739 ret = rte_dev_hotplug_handle_enable(); 3740 if (ret) { 3741 RTE_LOG(ERR, EAL, 3742 "fail to enable hotplug handling."); 3743 return -1; 3744 } 3745 3746 ret = rte_dev_event_monitor_start(); 3747 if (ret) { 3748 RTE_LOG(ERR, EAL, 3749 "fail to start device event monitoring."); 3750 return -1; 3751 } 3752 3753 ret = rte_dev_event_callback_register(NULL, 3754 dev_event_callback, NULL); 3755 if (ret) { 3756 RTE_LOG(ERR, EAL, 3757 "fail to register device event callback\n"); 3758 return -1; 3759 } 3760 } 3761 3762 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3763 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3764 3765 /* set all ports to promiscuous mode by default */ 3766 RTE_ETH_FOREACH_DEV(port_id) { 3767 ret = rte_eth_promiscuous_enable(port_id); 3768 if (ret != 0) 3769 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3770 port_id, rte_strerror(-ret)); 3771 } 3772 3773 /* Init metrics library */ 3774 rte_metrics_init(rte_socket_id()); 3775 3776 #ifdef RTE_LIBRTE_LATENCY_STATS 3777 if (latencystats_enabled != 0) { 3778 int ret = rte_latencystats_init(1, NULL); 3779 if (ret) 3780 printf("Warning: latencystats init()" 3781 " returned error %d\n", ret); 3782 printf("Latencystats running on lcore %d\n", 3783 latencystats_lcore_id); 3784 } 3785 #endif 3786 3787 /* Setup bitrate stats */ 3788 #ifdef RTE_LIBRTE_BITRATE 3789 if (bitrate_enabled != 0) { 3790 bitrate_data = rte_stats_bitrate_create(); 3791 if (bitrate_data == NULL) 3792 rte_exit(EXIT_FAILURE, 3793 "Could not allocate bitrate data.\n"); 3794 rte_stats_bitrate_reg(bitrate_data); 3795 } 3796 #endif 3797 3798 #ifdef RTE_LIBRTE_CMDLINE 3799 if (strlen(cmdline_filename) != 0) 3800 cmdline_read_from_file(cmdline_filename); 3801 3802 if (interactive == 1) { 3803 if (auto_start) { 3804 printf("Start automatic packet forwarding\n"); 3805 start_packet_forwarding(0); 3806 } 3807 prompt(); 3808 pmd_test_exit(); 3809 } else 3810 #endif 3811 { 3812 char c; 3813 int rc; 3814 3815 f_quit = 0; 3816 3817 printf("No commandline core given, start packet forwarding\n"); 3818 start_packet_forwarding(tx_first); 3819 if (stats_period != 0) { 3820 uint64_t prev_time = 0, cur_time, diff_time = 0; 3821 uint64_t timer_period; 3822 3823 /* Convert to number of cycles */ 3824 timer_period = stats_period * rte_get_timer_hz(); 3825 3826 while (f_quit == 0) { 3827 cur_time = rte_get_timer_cycles(); 3828 diff_time += cur_time - prev_time; 3829 3830 if (diff_time >= timer_period) { 3831 print_stats(); 3832 /* Reset the timer */ 3833 diff_time = 0; 3834 } 3835 /* Sleep to avoid unnecessary checks */ 3836 prev_time = cur_time; 3837 sleep(1); 3838 } 3839 } 3840 3841 printf("Press enter to exit\n"); 3842 rc = read(0, &c, 1); 3843 pmd_test_exit(); 3844 if (rc < 0) 3845 return 1; 3846 } 3847 3848 ret = rte_eal_cleanup(); 3849 if (ret != 0) 3850 rte_exit(EXIT_FAILURE, 3851 "EAL cleanup failed: %s\n", strerror(-ret)); 3852 3853 return EXIT_SUCCESS; 3854 } 3855