1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATE 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 192 uint16_t mempool_flags; 193 194 struct fwd_config cur_fwd_config; 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 196 uint32_t retry_enabled; 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 199 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 201 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 202 * specified on command-line. */ 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 204 205 /* 206 * In container, it cannot terminate the process which running with 'stats-period' 207 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 208 */ 209 uint8_t f_quit; 210 211 /* 212 * Configuration of packet segments used by the "txonly" processing engine. 213 */ 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 216 TXONLY_DEF_PACKET_LEN, 217 }; 218 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 219 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 221 /**< Split policy for packets to TX. */ 222 223 uint8_t txonly_multi_flow; 224 /**< Whether multiple flows are generated in TXONLY mode. */ 225 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 228 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 230 uint8_t dcb_config = 0; 231 232 /* Whether the dcb is in testing status */ 233 uint8_t dcb_test = 0; 234 235 /* 236 * Configurable number of RX/TX queues. 237 */ 238 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 239 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 240 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 241 242 /* 243 * Configurable number of RX/TX ring descriptors. 244 * Defaults are supplied by drivers via ethdev. 245 */ 246 #define RTE_TEST_RX_DESC_DEFAULT 0 247 #define RTE_TEST_TX_DESC_DEFAULT 0 248 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 249 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 250 251 #define RTE_PMD_PARAM_UNSET -1 252 /* 253 * Configurable values of RX and TX ring threshold registers. 254 */ 255 256 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 258 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 259 260 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 262 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 263 264 /* 265 * Configurable value of RX free threshold. 266 */ 267 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 268 269 /* 270 * Configurable value of RX drop enable. 271 */ 272 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 273 274 /* 275 * Configurable value of TX free threshold. 276 */ 277 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 278 279 /* 280 * Configurable value of TX RS bit threshold. 281 */ 282 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 283 284 /* 285 * Configurable value of buffered packets before sending. 286 */ 287 uint16_t noisy_tx_sw_bufsz; 288 289 /* 290 * Configurable value of packet buffer timeout. 291 */ 292 uint16_t noisy_tx_sw_buf_flush_time; 293 294 /* 295 * Configurable value for size of VNF internal memory area 296 * used for simulating noisy neighbour behaviour 297 */ 298 uint64_t noisy_lkup_mem_sz; 299 300 /* 301 * Configurable value of number of random writes done in 302 * VNF simulation memory area. 303 */ 304 uint64_t noisy_lkup_num_writes; 305 306 /* 307 * Configurable value of number of random reads done in 308 * VNF simulation memory area. 309 */ 310 uint64_t noisy_lkup_num_reads; 311 312 /* 313 * Configurable value of number of random reads/writes done in 314 * VNF simulation memory area. 315 */ 316 uint64_t noisy_lkup_num_reads_writes; 317 318 /* 319 * Receive Side Scaling (RSS) configuration. 320 */ 321 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 322 323 /* 324 * Port topology configuration 325 */ 326 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 327 328 /* 329 * Avoids to flush all the RX streams before starts forwarding. 330 */ 331 uint8_t no_flush_rx = 0; /* flush by default */ 332 333 /* 334 * Flow API isolated mode. 335 */ 336 uint8_t flow_isolate_all; 337 338 /* 339 * Avoids to check link status when starting/stopping a port. 340 */ 341 uint8_t no_link_check = 0; /* check by default */ 342 343 /* 344 * Don't automatically start all ports in interactive mode. 345 */ 346 uint8_t no_device_start = 0; 347 348 /* 349 * Enable link status change notification 350 */ 351 uint8_t lsc_interrupt = 1; /* enabled by default */ 352 353 /* 354 * Enable device removal notification. 355 */ 356 uint8_t rmv_interrupt = 1; /* enabled by default */ 357 358 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 359 360 /* After attach, port setup is called on event or by iterator */ 361 bool setup_on_probe_event = true; 362 363 /* Clear ptypes on port initialization. */ 364 uint8_t clear_ptypes = true; 365 366 /* Pretty printing of ethdev events */ 367 static const char * const eth_event_desc[] = { 368 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 369 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 370 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 371 [RTE_ETH_EVENT_INTR_RESET] = "reset", 372 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 373 [RTE_ETH_EVENT_IPSEC] = "IPsec", 374 [RTE_ETH_EVENT_MACSEC] = "MACsec", 375 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 376 [RTE_ETH_EVENT_NEW] = "device probed", 377 [RTE_ETH_EVENT_DESTROY] = "device released", 378 [RTE_ETH_EVENT_MAX] = NULL, 379 }; 380 381 /* 382 * Display or mask ether events 383 * Default to all events except VF_MBOX 384 */ 385 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 386 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 387 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 388 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 389 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 390 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 391 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 392 /* 393 * Decide if all memory are locked for performance. 394 */ 395 int do_mlockall = 0; 396 397 /* 398 * NIC bypass mode configuration options. 399 */ 400 401 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 402 /* The NIC bypass watchdog timeout. */ 403 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 404 #endif 405 406 407 #ifdef RTE_LIBRTE_LATENCY_STATS 408 409 /* 410 * Set when latency stats is enabled in the commandline 411 */ 412 uint8_t latencystats_enabled; 413 414 /* 415 * Lcore ID to serive latency statistics. 416 */ 417 lcoreid_t latencystats_lcore_id = -1; 418 419 #endif 420 421 /* 422 * Ethernet device configuration. 423 */ 424 struct rte_eth_rxmode rx_mode = { 425 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 426 /**< Default maximum frame length. */ 427 }; 428 429 struct rte_eth_txmode tx_mode = { 430 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 431 }; 432 433 struct rte_fdir_conf fdir_conf = { 434 .mode = RTE_FDIR_MODE_NONE, 435 .pballoc = RTE_FDIR_PBALLOC_64K, 436 .status = RTE_FDIR_REPORT_STATUS, 437 .mask = { 438 .vlan_tci_mask = 0xFFEF, 439 .ipv4_mask = { 440 .src_ip = 0xFFFFFFFF, 441 .dst_ip = 0xFFFFFFFF, 442 }, 443 .ipv6_mask = { 444 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 445 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 446 }, 447 .src_port_mask = 0xFFFF, 448 .dst_port_mask = 0xFFFF, 449 .mac_addr_byte_mask = 0xFF, 450 .tunnel_type_mask = 1, 451 .tunnel_id_mask = 0xFFFFFFFF, 452 }, 453 .drop_queue = 127, 454 }; 455 456 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 457 458 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 459 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 460 461 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 462 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 463 464 uint16_t nb_tx_queue_stats_mappings = 0; 465 uint16_t nb_rx_queue_stats_mappings = 0; 466 467 /* 468 * Display zero values by default for xstats 469 */ 470 uint8_t xstats_hide_zero; 471 472 unsigned int num_sockets = 0; 473 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 474 475 #ifdef RTE_LIBRTE_BITRATE 476 /* Bitrate statistics */ 477 struct rte_stats_bitrates *bitrate_data; 478 lcoreid_t bitrate_lcore_id; 479 uint8_t bitrate_enabled; 480 #endif 481 482 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 483 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 484 485 /* Forward function declarations */ 486 static void setup_attached_port(portid_t pi); 487 static void map_port_queue_stats_mapping_registers(portid_t pi, 488 struct rte_port *port); 489 static void check_all_ports_link_status(uint32_t port_mask); 490 static int eth_event_callback(portid_t port_id, 491 enum rte_eth_event_type type, 492 void *param, void *ret_param); 493 static void dev_event_callback(const char *device_name, 494 enum rte_dev_event_type type, 495 void *param); 496 497 /* 498 * Check if all the ports are started. 499 * If yes, return positive value. If not, return zero. 500 */ 501 static int all_ports_started(void); 502 503 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 504 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 505 506 /* Holds the registered mbuf dynamic flags names. */ 507 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 508 509 /* 510 * Helper function to check if socket is already discovered. 511 * If yes, return positive value. If not, return zero. 512 */ 513 int 514 new_socket_id(unsigned int socket_id) 515 { 516 unsigned int i; 517 518 for (i = 0; i < num_sockets; i++) { 519 if (socket_ids[i] == socket_id) 520 return 0; 521 } 522 return 1; 523 } 524 525 /* 526 * Setup default configuration. 527 */ 528 static void 529 set_default_fwd_lcores_config(void) 530 { 531 unsigned int i; 532 unsigned int nb_lc; 533 unsigned int sock_num; 534 535 nb_lc = 0; 536 for (i = 0; i < RTE_MAX_LCORE; i++) { 537 if (!rte_lcore_is_enabled(i)) 538 continue; 539 sock_num = rte_lcore_to_socket_id(i); 540 if (new_socket_id(sock_num)) { 541 if (num_sockets >= RTE_MAX_NUMA_NODES) { 542 rte_exit(EXIT_FAILURE, 543 "Total sockets greater than %u\n", 544 RTE_MAX_NUMA_NODES); 545 } 546 socket_ids[num_sockets++] = sock_num; 547 } 548 if (i == rte_get_master_lcore()) 549 continue; 550 fwd_lcores_cpuids[nb_lc++] = i; 551 } 552 nb_lcores = (lcoreid_t) nb_lc; 553 nb_cfg_lcores = nb_lcores; 554 nb_fwd_lcores = 1; 555 } 556 557 static void 558 set_def_peer_eth_addrs(void) 559 { 560 portid_t i; 561 562 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 563 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 564 peer_eth_addrs[i].addr_bytes[5] = i; 565 } 566 } 567 568 static void 569 set_default_fwd_ports_config(void) 570 { 571 portid_t pt_id; 572 int i = 0; 573 574 RTE_ETH_FOREACH_DEV(pt_id) { 575 fwd_ports_ids[i++] = pt_id; 576 577 /* Update sockets info according to the attached device */ 578 int socket_id = rte_eth_dev_socket_id(pt_id); 579 if (socket_id >= 0 && new_socket_id(socket_id)) { 580 if (num_sockets >= RTE_MAX_NUMA_NODES) { 581 rte_exit(EXIT_FAILURE, 582 "Total sockets greater than %u\n", 583 RTE_MAX_NUMA_NODES); 584 } 585 socket_ids[num_sockets++] = socket_id; 586 } 587 } 588 589 nb_cfg_ports = nb_ports; 590 nb_fwd_ports = nb_ports; 591 } 592 593 void 594 set_def_fwd_config(void) 595 { 596 set_default_fwd_lcores_config(); 597 set_def_peer_eth_addrs(); 598 set_default_fwd_ports_config(); 599 } 600 601 /* extremely pessimistic estimation of memory required to create a mempool */ 602 static int 603 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 604 { 605 unsigned int n_pages, mbuf_per_pg, leftover; 606 uint64_t total_mem, mbuf_mem, obj_sz; 607 608 /* there is no good way to predict how much space the mempool will 609 * occupy because it will allocate chunks on the fly, and some of those 610 * will come from default DPDK memory while some will come from our 611 * external memory, so just assume 128MB will be enough for everyone. 612 */ 613 uint64_t hdr_mem = 128 << 20; 614 615 /* account for possible non-contiguousness */ 616 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 617 if (obj_sz > pgsz) { 618 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 619 return -1; 620 } 621 622 mbuf_per_pg = pgsz / obj_sz; 623 leftover = (nb_mbufs % mbuf_per_pg) > 0; 624 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 625 626 mbuf_mem = n_pages * pgsz; 627 628 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 629 630 if (total_mem > SIZE_MAX) { 631 TESTPMD_LOG(ERR, "Memory size too big\n"); 632 return -1; 633 } 634 *out = (size_t)total_mem; 635 636 return 0; 637 } 638 639 static int 640 pagesz_flags(uint64_t page_sz) 641 { 642 /* as per mmap() manpage, all page sizes are log2 of page size 643 * shifted by MAP_HUGE_SHIFT 644 */ 645 int log2 = rte_log2_u64(page_sz); 646 647 return (log2 << HUGE_SHIFT); 648 } 649 650 static void * 651 alloc_mem(size_t memsz, size_t pgsz, bool huge) 652 { 653 void *addr; 654 int flags; 655 656 /* allocate anonymous hugepages */ 657 flags = MAP_ANONYMOUS | MAP_PRIVATE; 658 if (huge) 659 flags |= HUGE_FLAG | pagesz_flags(pgsz); 660 661 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 662 if (addr == MAP_FAILED) 663 return NULL; 664 665 return addr; 666 } 667 668 struct extmem_param { 669 void *addr; 670 size_t len; 671 size_t pgsz; 672 rte_iova_t *iova_table; 673 unsigned int iova_table_len; 674 }; 675 676 static int 677 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 678 bool huge) 679 { 680 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 681 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 682 unsigned int cur_page, n_pages, pgsz_idx; 683 size_t mem_sz, cur_pgsz; 684 rte_iova_t *iovas = NULL; 685 void *addr; 686 int ret; 687 688 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 689 /* skip anything that is too big */ 690 if (pgsizes[pgsz_idx] > SIZE_MAX) 691 continue; 692 693 cur_pgsz = pgsizes[pgsz_idx]; 694 695 /* if we were told not to allocate hugepages, override */ 696 if (!huge) 697 cur_pgsz = sysconf(_SC_PAGESIZE); 698 699 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 700 if (ret < 0) { 701 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 702 return -1; 703 } 704 705 /* allocate our memory */ 706 addr = alloc_mem(mem_sz, cur_pgsz, huge); 707 708 /* if we couldn't allocate memory with a specified page size, 709 * that doesn't mean we can't do it with other page sizes, so 710 * try another one. 711 */ 712 if (addr == NULL) 713 continue; 714 715 /* store IOVA addresses for every page in this memory area */ 716 n_pages = mem_sz / cur_pgsz; 717 718 iovas = malloc(sizeof(*iovas) * n_pages); 719 720 if (iovas == NULL) { 721 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 722 goto fail; 723 } 724 /* lock memory if it's not huge pages */ 725 if (!huge) 726 mlock(addr, mem_sz); 727 728 /* populate IOVA addresses */ 729 for (cur_page = 0; cur_page < n_pages; cur_page++) { 730 rte_iova_t iova; 731 size_t offset; 732 void *cur; 733 734 offset = cur_pgsz * cur_page; 735 cur = RTE_PTR_ADD(addr, offset); 736 737 /* touch the page before getting its IOVA */ 738 *(volatile char *)cur = 0; 739 740 iova = rte_mem_virt2iova(cur); 741 742 iovas[cur_page] = iova; 743 } 744 745 break; 746 } 747 /* if we couldn't allocate anything */ 748 if (iovas == NULL) 749 return -1; 750 751 param->addr = addr; 752 param->len = mem_sz; 753 param->pgsz = cur_pgsz; 754 param->iova_table = iovas; 755 param->iova_table_len = n_pages; 756 757 return 0; 758 fail: 759 if (iovas) 760 free(iovas); 761 if (addr) 762 munmap(addr, mem_sz); 763 764 return -1; 765 } 766 767 static int 768 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 769 { 770 struct extmem_param param; 771 int socket_id, ret; 772 773 memset(¶m, 0, sizeof(param)); 774 775 /* check if our heap exists */ 776 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 777 if (socket_id < 0) { 778 /* create our heap */ 779 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 780 if (ret < 0) { 781 TESTPMD_LOG(ERR, "Cannot create heap\n"); 782 return -1; 783 } 784 } 785 786 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 787 if (ret < 0) { 788 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 789 return -1; 790 } 791 792 /* we now have a valid memory area, so add it to heap */ 793 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 794 param.addr, param.len, param.iova_table, 795 param.iova_table_len, param.pgsz); 796 797 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 798 799 /* not needed any more */ 800 free(param.iova_table); 801 802 if (ret < 0) { 803 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 804 munmap(param.addr, param.len); 805 return -1; 806 } 807 808 /* success */ 809 810 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 811 param.len >> 20); 812 813 return 0; 814 } 815 static void 816 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 817 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 818 { 819 uint16_t pid = 0; 820 int ret; 821 822 RTE_ETH_FOREACH_DEV(pid) { 823 struct rte_eth_dev *dev = 824 &rte_eth_devices[pid]; 825 826 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 827 memhdr->len); 828 if (ret) { 829 TESTPMD_LOG(DEBUG, 830 "unable to DMA unmap addr 0x%p " 831 "for device %s\n", 832 memhdr->addr, dev->data->name); 833 } 834 } 835 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 836 if (ret) { 837 TESTPMD_LOG(DEBUG, 838 "unable to un-register addr 0x%p\n", memhdr->addr); 839 } 840 } 841 842 static void 843 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 844 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 845 { 846 uint16_t pid = 0; 847 size_t page_size = sysconf(_SC_PAGESIZE); 848 int ret; 849 850 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 851 page_size); 852 if (ret) { 853 TESTPMD_LOG(DEBUG, 854 "unable to register addr 0x%p\n", memhdr->addr); 855 return; 856 } 857 RTE_ETH_FOREACH_DEV(pid) { 858 struct rte_eth_dev *dev = 859 &rte_eth_devices[pid]; 860 861 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 862 memhdr->len); 863 if (ret) { 864 TESTPMD_LOG(DEBUG, 865 "unable to DMA map addr 0x%p " 866 "for device %s\n", 867 memhdr->addr, dev->data->name); 868 } 869 } 870 } 871 872 static unsigned int 873 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 874 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 875 { 876 struct rte_pktmbuf_extmem *xmem; 877 unsigned int ext_num, zone_num, elt_num; 878 uint16_t elt_size; 879 880 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 881 elt_num = EXTBUF_ZONE_SIZE / elt_size; 882 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 883 884 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 885 if (xmem == NULL) { 886 TESTPMD_LOG(ERR, "Cannot allocate memory for " 887 "external buffer descriptors\n"); 888 *ext_mem = NULL; 889 return 0; 890 } 891 for (ext_num = 0; ext_num < zone_num; ext_num++) { 892 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 893 const struct rte_memzone *mz; 894 char mz_name[RTE_MEMZONE_NAMESIZE]; 895 int ret; 896 897 ret = snprintf(mz_name, sizeof(mz_name), 898 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 899 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 900 errno = ENAMETOOLONG; 901 ext_num = 0; 902 break; 903 } 904 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 905 socket_id, 906 RTE_MEMZONE_IOVA_CONTIG | 907 RTE_MEMZONE_1GB | 908 RTE_MEMZONE_SIZE_HINT_ONLY, 909 EXTBUF_ZONE_SIZE); 910 if (mz == NULL) { 911 /* 912 * The caller exits on external buffer creation 913 * error, so there is no need to free memzones. 914 */ 915 errno = ENOMEM; 916 ext_num = 0; 917 break; 918 } 919 xseg->buf_ptr = mz->addr; 920 xseg->buf_iova = mz->iova; 921 xseg->buf_len = EXTBUF_ZONE_SIZE; 922 xseg->elt_size = elt_size; 923 } 924 if (ext_num == 0 && xmem != NULL) { 925 free(xmem); 926 xmem = NULL; 927 } 928 *ext_mem = xmem; 929 return ext_num; 930 } 931 932 /* 933 * Configuration initialisation done once at init time. 934 */ 935 static struct rte_mempool * 936 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 937 unsigned int socket_id) 938 { 939 char pool_name[RTE_MEMPOOL_NAMESIZE]; 940 struct rte_mempool *rte_mp = NULL; 941 uint32_t mb_size; 942 943 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 944 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 945 946 TESTPMD_LOG(INFO, 947 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 948 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 949 950 switch (mp_alloc_type) { 951 case MP_ALLOC_NATIVE: 952 { 953 /* wrapper to rte_mempool_create() */ 954 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 955 rte_mbuf_best_mempool_ops()); 956 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 957 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 958 break; 959 } 960 case MP_ALLOC_ANON: 961 { 962 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 963 mb_size, (unsigned int) mb_mempool_cache, 964 sizeof(struct rte_pktmbuf_pool_private), 965 socket_id, mempool_flags); 966 if (rte_mp == NULL) 967 goto err; 968 969 if (rte_mempool_populate_anon(rte_mp) == 0) { 970 rte_mempool_free(rte_mp); 971 rte_mp = NULL; 972 goto err; 973 } 974 rte_pktmbuf_pool_init(rte_mp, NULL); 975 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 976 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 977 break; 978 } 979 case MP_ALLOC_XMEM: 980 case MP_ALLOC_XMEM_HUGE: 981 { 982 int heap_socket; 983 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 984 985 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 986 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 987 988 heap_socket = 989 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 990 if (heap_socket < 0) 991 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 992 993 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 994 rte_mbuf_best_mempool_ops()); 995 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 996 mb_mempool_cache, 0, mbuf_seg_size, 997 heap_socket); 998 break; 999 } 1000 case MP_ALLOC_XBUF: 1001 { 1002 struct rte_pktmbuf_extmem *ext_mem; 1003 unsigned int ext_num; 1004 1005 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1006 socket_id, pool_name, &ext_mem); 1007 if (ext_num == 0) 1008 rte_exit(EXIT_FAILURE, 1009 "Can't create pinned data buffers\n"); 1010 1011 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1012 rte_mbuf_best_mempool_ops()); 1013 rte_mp = rte_pktmbuf_pool_create_extbuf 1014 (pool_name, nb_mbuf, mb_mempool_cache, 1015 0, mbuf_seg_size, socket_id, 1016 ext_mem, ext_num); 1017 free(ext_mem); 1018 break; 1019 } 1020 default: 1021 { 1022 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1023 } 1024 } 1025 1026 err: 1027 if (rte_mp == NULL) { 1028 rte_exit(EXIT_FAILURE, 1029 "Creation of mbuf pool for socket %u failed: %s\n", 1030 socket_id, rte_strerror(rte_errno)); 1031 } else if (verbose_level > 0) { 1032 rte_mempool_dump(stdout, rte_mp); 1033 } 1034 return rte_mp; 1035 } 1036 1037 /* 1038 * Check given socket id is valid or not with NUMA mode, 1039 * if valid, return 0, else return -1 1040 */ 1041 static int 1042 check_socket_id(const unsigned int socket_id) 1043 { 1044 static int warning_once = 0; 1045 1046 if (new_socket_id(socket_id)) { 1047 if (!warning_once && numa_support) 1048 printf("Warning: NUMA should be configured manually by" 1049 " using --port-numa-config and" 1050 " --ring-numa-config parameters along with" 1051 " --numa.\n"); 1052 warning_once = 1; 1053 return -1; 1054 } 1055 return 0; 1056 } 1057 1058 /* 1059 * Get the allowed maximum number of RX queues. 1060 * *pid return the port id which has minimal value of 1061 * max_rx_queues in all ports. 1062 */ 1063 queueid_t 1064 get_allowed_max_nb_rxq(portid_t *pid) 1065 { 1066 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1067 bool max_rxq_valid = false; 1068 portid_t pi; 1069 struct rte_eth_dev_info dev_info; 1070 1071 RTE_ETH_FOREACH_DEV(pi) { 1072 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1073 continue; 1074 1075 max_rxq_valid = true; 1076 if (dev_info.max_rx_queues < allowed_max_rxq) { 1077 allowed_max_rxq = dev_info.max_rx_queues; 1078 *pid = pi; 1079 } 1080 } 1081 return max_rxq_valid ? allowed_max_rxq : 0; 1082 } 1083 1084 /* 1085 * Check input rxq is valid or not. 1086 * If input rxq is not greater than any of maximum number 1087 * of RX queues of all ports, it is valid. 1088 * if valid, return 0, else return -1 1089 */ 1090 int 1091 check_nb_rxq(queueid_t rxq) 1092 { 1093 queueid_t allowed_max_rxq; 1094 portid_t pid = 0; 1095 1096 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1097 if (rxq > allowed_max_rxq) { 1098 printf("Fail: input rxq (%u) can't be greater " 1099 "than max_rx_queues (%u) of port %u\n", 1100 rxq, 1101 allowed_max_rxq, 1102 pid); 1103 return -1; 1104 } 1105 return 0; 1106 } 1107 1108 /* 1109 * Get the allowed maximum number of TX queues. 1110 * *pid return the port id which has minimal value of 1111 * max_tx_queues in all ports. 1112 */ 1113 queueid_t 1114 get_allowed_max_nb_txq(portid_t *pid) 1115 { 1116 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1117 bool max_txq_valid = false; 1118 portid_t pi; 1119 struct rte_eth_dev_info dev_info; 1120 1121 RTE_ETH_FOREACH_DEV(pi) { 1122 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1123 continue; 1124 1125 max_txq_valid = true; 1126 if (dev_info.max_tx_queues < allowed_max_txq) { 1127 allowed_max_txq = dev_info.max_tx_queues; 1128 *pid = pi; 1129 } 1130 } 1131 return max_txq_valid ? allowed_max_txq : 0; 1132 } 1133 1134 /* 1135 * Check input txq is valid or not. 1136 * If input txq is not greater than any of maximum number 1137 * of TX queues of all ports, it is valid. 1138 * if valid, return 0, else return -1 1139 */ 1140 int 1141 check_nb_txq(queueid_t txq) 1142 { 1143 queueid_t allowed_max_txq; 1144 portid_t pid = 0; 1145 1146 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1147 if (txq > allowed_max_txq) { 1148 printf("Fail: input txq (%u) can't be greater " 1149 "than max_tx_queues (%u) of port %u\n", 1150 txq, 1151 allowed_max_txq, 1152 pid); 1153 return -1; 1154 } 1155 return 0; 1156 } 1157 1158 /* 1159 * Get the allowed maximum number of RXDs of every rx queue. 1160 * *pid return the port id which has minimal value of 1161 * max_rxd in all queues of all ports. 1162 */ 1163 static uint16_t 1164 get_allowed_max_nb_rxd(portid_t *pid) 1165 { 1166 uint16_t allowed_max_rxd = UINT16_MAX; 1167 portid_t pi; 1168 struct rte_eth_dev_info dev_info; 1169 1170 RTE_ETH_FOREACH_DEV(pi) { 1171 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1172 continue; 1173 1174 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1175 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1176 *pid = pi; 1177 } 1178 } 1179 return allowed_max_rxd; 1180 } 1181 1182 /* 1183 * Get the allowed minimal number of RXDs of every rx queue. 1184 * *pid return the port id which has minimal value of 1185 * min_rxd in all queues of all ports. 1186 */ 1187 static uint16_t 1188 get_allowed_min_nb_rxd(portid_t *pid) 1189 { 1190 uint16_t allowed_min_rxd = 0; 1191 portid_t pi; 1192 struct rte_eth_dev_info dev_info; 1193 1194 RTE_ETH_FOREACH_DEV(pi) { 1195 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1196 continue; 1197 1198 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1199 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1200 *pid = pi; 1201 } 1202 } 1203 1204 return allowed_min_rxd; 1205 } 1206 1207 /* 1208 * Check input rxd is valid or not. 1209 * If input rxd is not greater than any of maximum number 1210 * of RXDs of every Rx queues and is not less than any of 1211 * minimal number of RXDs of every Rx queues, it is valid. 1212 * if valid, return 0, else return -1 1213 */ 1214 int 1215 check_nb_rxd(queueid_t rxd) 1216 { 1217 uint16_t allowed_max_rxd; 1218 uint16_t allowed_min_rxd; 1219 portid_t pid = 0; 1220 1221 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1222 if (rxd > allowed_max_rxd) { 1223 printf("Fail: input rxd (%u) can't be greater " 1224 "than max_rxds (%u) of port %u\n", 1225 rxd, 1226 allowed_max_rxd, 1227 pid); 1228 return -1; 1229 } 1230 1231 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1232 if (rxd < allowed_min_rxd) { 1233 printf("Fail: input rxd (%u) can't be less " 1234 "than min_rxds (%u) of port %u\n", 1235 rxd, 1236 allowed_min_rxd, 1237 pid); 1238 return -1; 1239 } 1240 1241 return 0; 1242 } 1243 1244 /* 1245 * Get the allowed maximum number of TXDs of every rx queues. 1246 * *pid return the port id which has minimal value of 1247 * max_txd in every tx queue. 1248 */ 1249 static uint16_t 1250 get_allowed_max_nb_txd(portid_t *pid) 1251 { 1252 uint16_t allowed_max_txd = UINT16_MAX; 1253 portid_t pi; 1254 struct rte_eth_dev_info dev_info; 1255 1256 RTE_ETH_FOREACH_DEV(pi) { 1257 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1258 continue; 1259 1260 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1261 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1262 *pid = pi; 1263 } 1264 } 1265 return allowed_max_txd; 1266 } 1267 1268 /* 1269 * Get the allowed maximum number of TXDs of every tx queues. 1270 * *pid return the port id which has minimal value of 1271 * min_txd in every tx queue. 1272 */ 1273 static uint16_t 1274 get_allowed_min_nb_txd(portid_t *pid) 1275 { 1276 uint16_t allowed_min_txd = 0; 1277 portid_t pi; 1278 struct rte_eth_dev_info dev_info; 1279 1280 RTE_ETH_FOREACH_DEV(pi) { 1281 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1282 continue; 1283 1284 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1285 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1286 *pid = pi; 1287 } 1288 } 1289 1290 return allowed_min_txd; 1291 } 1292 1293 /* 1294 * Check input txd is valid or not. 1295 * If input txd is not greater than any of maximum number 1296 * of TXDs of every Rx queues, it is valid. 1297 * if valid, return 0, else return -1 1298 */ 1299 int 1300 check_nb_txd(queueid_t txd) 1301 { 1302 uint16_t allowed_max_txd; 1303 uint16_t allowed_min_txd; 1304 portid_t pid = 0; 1305 1306 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1307 if (txd > allowed_max_txd) { 1308 printf("Fail: input txd (%u) can't be greater " 1309 "than max_txds (%u) of port %u\n", 1310 txd, 1311 allowed_max_txd, 1312 pid); 1313 return -1; 1314 } 1315 1316 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1317 if (txd < allowed_min_txd) { 1318 printf("Fail: input txd (%u) can't be less " 1319 "than min_txds (%u) of port %u\n", 1320 txd, 1321 allowed_min_txd, 1322 pid); 1323 return -1; 1324 } 1325 return 0; 1326 } 1327 1328 1329 /* 1330 * Get the allowed maximum number of hairpin queues. 1331 * *pid return the port id which has minimal value of 1332 * max_hairpin_queues in all ports. 1333 */ 1334 queueid_t 1335 get_allowed_max_nb_hairpinq(portid_t *pid) 1336 { 1337 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1338 portid_t pi; 1339 struct rte_eth_hairpin_cap cap; 1340 1341 RTE_ETH_FOREACH_DEV(pi) { 1342 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1343 *pid = pi; 1344 return 0; 1345 } 1346 if (cap.max_nb_queues < allowed_max_hairpinq) { 1347 allowed_max_hairpinq = cap.max_nb_queues; 1348 *pid = pi; 1349 } 1350 } 1351 return allowed_max_hairpinq; 1352 } 1353 1354 /* 1355 * Check input hairpin is valid or not. 1356 * If input hairpin is not greater than any of maximum number 1357 * of hairpin queues of all ports, it is valid. 1358 * if valid, return 0, else return -1 1359 */ 1360 int 1361 check_nb_hairpinq(queueid_t hairpinq) 1362 { 1363 queueid_t allowed_max_hairpinq; 1364 portid_t pid = 0; 1365 1366 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1367 if (hairpinq > allowed_max_hairpinq) { 1368 printf("Fail: input hairpin (%u) can't be greater " 1369 "than max_hairpin_queues (%u) of port %u\n", 1370 hairpinq, allowed_max_hairpinq, pid); 1371 return -1; 1372 } 1373 return 0; 1374 } 1375 1376 static void 1377 init_config(void) 1378 { 1379 portid_t pid; 1380 struct rte_port *port; 1381 struct rte_mempool *mbp; 1382 unsigned int nb_mbuf_per_pool; 1383 lcoreid_t lc_id; 1384 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1385 struct rte_gro_param gro_param; 1386 uint32_t gso_types; 1387 uint16_t data_size; 1388 bool warning = 0; 1389 int k; 1390 int ret; 1391 1392 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1393 1394 /* Configuration of logical cores. */ 1395 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1396 sizeof(struct fwd_lcore *) * nb_lcores, 1397 RTE_CACHE_LINE_SIZE); 1398 if (fwd_lcores == NULL) { 1399 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1400 "failed\n", nb_lcores); 1401 } 1402 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1403 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1404 sizeof(struct fwd_lcore), 1405 RTE_CACHE_LINE_SIZE); 1406 if (fwd_lcores[lc_id] == NULL) { 1407 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1408 "failed\n"); 1409 } 1410 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1411 } 1412 1413 RTE_ETH_FOREACH_DEV(pid) { 1414 port = &ports[pid]; 1415 /* Apply default TxRx configuration for all ports */ 1416 port->dev_conf.txmode = tx_mode; 1417 port->dev_conf.rxmode = rx_mode; 1418 1419 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1420 if (ret != 0) 1421 rte_exit(EXIT_FAILURE, 1422 "rte_eth_dev_info_get() failed\n"); 1423 1424 if (!(port->dev_info.tx_offload_capa & 1425 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1426 port->dev_conf.txmode.offloads &= 1427 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1428 if (numa_support) { 1429 if (port_numa[pid] != NUMA_NO_CONFIG) 1430 port_per_socket[port_numa[pid]]++; 1431 else { 1432 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1433 1434 /* 1435 * if socket_id is invalid, 1436 * set to the first available socket. 1437 */ 1438 if (check_socket_id(socket_id) < 0) 1439 socket_id = socket_ids[0]; 1440 port_per_socket[socket_id]++; 1441 } 1442 } 1443 1444 /* Apply Rx offloads configuration */ 1445 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1446 port->rx_conf[k].offloads = 1447 port->dev_conf.rxmode.offloads; 1448 /* Apply Tx offloads configuration */ 1449 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1450 port->tx_conf[k].offloads = 1451 port->dev_conf.txmode.offloads; 1452 1453 /* set flag to initialize port/queue */ 1454 port->need_reconfig = 1; 1455 port->need_reconfig_queues = 1; 1456 port->tx_metadata = 0; 1457 1458 /* Check for maximum number of segments per MTU. Accordingly 1459 * update the mbuf data size. 1460 */ 1461 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1462 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1463 data_size = rx_mode.max_rx_pkt_len / 1464 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1465 1466 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1467 mbuf_data_size) { 1468 mbuf_data_size = data_size + 1469 RTE_PKTMBUF_HEADROOM; 1470 warning = 1; 1471 } 1472 } 1473 } 1474 1475 if (warning) 1476 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1477 mbuf_data_size); 1478 1479 /* 1480 * Create pools of mbuf. 1481 * If NUMA support is disabled, create a single pool of mbuf in 1482 * socket 0 memory by default. 1483 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1484 * 1485 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1486 * nb_txd can be configured at run time. 1487 */ 1488 if (param_total_num_mbufs) 1489 nb_mbuf_per_pool = param_total_num_mbufs; 1490 else { 1491 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1492 (nb_lcores * mb_mempool_cache) + 1493 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1494 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1495 } 1496 1497 if (numa_support) { 1498 uint8_t i; 1499 1500 for (i = 0; i < num_sockets; i++) 1501 mempools[i] = mbuf_pool_create(mbuf_data_size, 1502 nb_mbuf_per_pool, 1503 socket_ids[i]); 1504 } else { 1505 if (socket_num == UMA_NO_CONFIG) 1506 mempools[0] = mbuf_pool_create(mbuf_data_size, 1507 nb_mbuf_per_pool, 0); 1508 else 1509 mempools[socket_num] = mbuf_pool_create 1510 (mbuf_data_size, 1511 nb_mbuf_per_pool, 1512 socket_num); 1513 } 1514 1515 init_port_config(); 1516 1517 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1518 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1519 /* 1520 * Records which Mbuf pool to use by each logical core, if needed. 1521 */ 1522 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1523 mbp = mbuf_pool_find( 1524 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1525 1526 if (mbp == NULL) 1527 mbp = mbuf_pool_find(0); 1528 fwd_lcores[lc_id]->mbp = mbp; 1529 /* initialize GSO context */ 1530 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1531 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1532 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1533 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1534 RTE_ETHER_CRC_LEN; 1535 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1536 } 1537 1538 /* Configuration of packet forwarding streams. */ 1539 if (init_fwd_streams() < 0) 1540 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1541 1542 fwd_config_setup(); 1543 1544 /* create a gro context for each lcore */ 1545 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1546 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1547 gro_param.max_item_per_flow = MAX_PKT_BURST; 1548 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1549 gro_param.socket_id = rte_lcore_to_socket_id( 1550 fwd_lcores_cpuids[lc_id]); 1551 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1552 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1553 rte_exit(EXIT_FAILURE, 1554 "rte_gro_ctx_create() failed\n"); 1555 } 1556 } 1557 1558 #if defined RTE_LIBRTE_PMD_SOFTNIC 1559 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1560 RTE_ETH_FOREACH_DEV(pid) { 1561 port = &ports[pid]; 1562 const char *driver = port->dev_info.driver_name; 1563 1564 if (strcmp(driver, "net_softnic") == 0) 1565 port->softport.fwd_lcore_arg = fwd_lcores; 1566 } 1567 } 1568 #endif 1569 1570 } 1571 1572 1573 void 1574 reconfig(portid_t new_port_id, unsigned socket_id) 1575 { 1576 struct rte_port *port; 1577 int ret; 1578 1579 /* Reconfiguration of Ethernet ports. */ 1580 port = &ports[new_port_id]; 1581 1582 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1583 if (ret != 0) 1584 return; 1585 1586 /* set flag to initialize port/queue */ 1587 port->need_reconfig = 1; 1588 port->need_reconfig_queues = 1; 1589 port->socket_id = socket_id; 1590 1591 init_port_config(); 1592 } 1593 1594 1595 int 1596 init_fwd_streams(void) 1597 { 1598 portid_t pid; 1599 struct rte_port *port; 1600 streamid_t sm_id, nb_fwd_streams_new; 1601 queueid_t q; 1602 1603 /* set socket id according to numa or not */ 1604 RTE_ETH_FOREACH_DEV(pid) { 1605 port = &ports[pid]; 1606 if (nb_rxq > port->dev_info.max_rx_queues) { 1607 printf("Fail: nb_rxq(%d) is greater than " 1608 "max_rx_queues(%d)\n", nb_rxq, 1609 port->dev_info.max_rx_queues); 1610 return -1; 1611 } 1612 if (nb_txq > port->dev_info.max_tx_queues) { 1613 printf("Fail: nb_txq(%d) is greater than " 1614 "max_tx_queues(%d)\n", nb_txq, 1615 port->dev_info.max_tx_queues); 1616 return -1; 1617 } 1618 if (numa_support) { 1619 if (port_numa[pid] != NUMA_NO_CONFIG) 1620 port->socket_id = port_numa[pid]; 1621 else { 1622 port->socket_id = rte_eth_dev_socket_id(pid); 1623 1624 /* 1625 * if socket_id is invalid, 1626 * set to the first available socket. 1627 */ 1628 if (check_socket_id(port->socket_id) < 0) 1629 port->socket_id = socket_ids[0]; 1630 } 1631 } 1632 else { 1633 if (socket_num == UMA_NO_CONFIG) 1634 port->socket_id = 0; 1635 else 1636 port->socket_id = socket_num; 1637 } 1638 } 1639 1640 q = RTE_MAX(nb_rxq, nb_txq); 1641 if (q == 0) { 1642 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1643 return -1; 1644 } 1645 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1646 if (nb_fwd_streams_new == nb_fwd_streams) 1647 return 0; 1648 /* clear the old */ 1649 if (fwd_streams != NULL) { 1650 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1651 if (fwd_streams[sm_id] == NULL) 1652 continue; 1653 rte_free(fwd_streams[sm_id]); 1654 fwd_streams[sm_id] = NULL; 1655 } 1656 rte_free(fwd_streams); 1657 fwd_streams = NULL; 1658 } 1659 1660 /* init new */ 1661 nb_fwd_streams = nb_fwd_streams_new; 1662 if (nb_fwd_streams) { 1663 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1664 sizeof(struct fwd_stream *) * nb_fwd_streams, 1665 RTE_CACHE_LINE_SIZE); 1666 if (fwd_streams == NULL) 1667 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1668 " (struct fwd_stream *)) failed\n", 1669 nb_fwd_streams); 1670 1671 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1672 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1673 " struct fwd_stream", sizeof(struct fwd_stream), 1674 RTE_CACHE_LINE_SIZE); 1675 if (fwd_streams[sm_id] == NULL) 1676 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1677 "(struct fwd_stream) failed\n"); 1678 } 1679 } 1680 1681 return 0; 1682 } 1683 1684 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1685 static void 1686 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1687 { 1688 unsigned int total_burst; 1689 unsigned int nb_burst; 1690 unsigned int burst_stats[3]; 1691 uint16_t pktnb_stats[3]; 1692 uint16_t nb_pkt; 1693 int burst_percent[3]; 1694 1695 /* 1696 * First compute the total number of packet bursts and the 1697 * two highest numbers of bursts of the same number of packets. 1698 */ 1699 total_burst = 0; 1700 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1701 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1702 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1703 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1704 if (nb_burst == 0) 1705 continue; 1706 total_burst += nb_burst; 1707 if (nb_burst > burst_stats[0]) { 1708 burst_stats[1] = burst_stats[0]; 1709 pktnb_stats[1] = pktnb_stats[0]; 1710 burst_stats[0] = nb_burst; 1711 pktnb_stats[0] = nb_pkt; 1712 } else if (nb_burst > burst_stats[1]) { 1713 burst_stats[1] = nb_burst; 1714 pktnb_stats[1] = nb_pkt; 1715 } 1716 } 1717 if (total_burst == 0) 1718 return; 1719 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1720 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1721 burst_percent[0], (int) pktnb_stats[0]); 1722 if (burst_stats[0] == total_burst) { 1723 printf("]\n"); 1724 return; 1725 } 1726 if (burst_stats[0] + burst_stats[1] == total_burst) { 1727 printf(" + %d%% of %d pkts]\n", 1728 100 - burst_percent[0], pktnb_stats[1]); 1729 return; 1730 } 1731 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1732 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1733 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1734 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1735 return; 1736 } 1737 printf(" + %d%% of %d pkts + %d%% of others]\n", 1738 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1739 } 1740 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1741 1742 static void 1743 fwd_stream_stats_display(streamid_t stream_id) 1744 { 1745 struct fwd_stream *fs; 1746 static const char *fwd_top_stats_border = "-------"; 1747 1748 fs = fwd_streams[stream_id]; 1749 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1750 (fs->fwd_dropped == 0)) 1751 return; 1752 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1753 "TX Port=%2d/Queue=%2d %s\n", 1754 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1755 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1756 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1757 " TX-dropped: %-14"PRIu64, 1758 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1759 1760 /* if checksum mode */ 1761 if (cur_fwd_eng == &csum_fwd_engine) { 1762 printf(" RX- bad IP checksum: %-14"PRIu64 1763 " Rx- bad L4 checksum: %-14"PRIu64 1764 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1765 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1766 fs->rx_bad_outer_l4_csum); 1767 } else { 1768 printf("\n"); 1769 } 1770 1771 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1772 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1773 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1774 #endif 1775 } 1776 1777 void 1778 fwd_stats_display(void) 1779 { 1780 static const char *fwd_stats_border = "----------------------"; 1781 static const char *acc_stats_border = "+++++++++++++++"; 1782 struct { 1783 struct fwd_stream *rx_stream; 1784 struct fwd_stream *tx_stream; 1785 uint64_t tx_dropped; 1786 uint64_t rx_bad_ip_csum; 1787 uint64_t rx_bad_l4_csum; 1788 uint64_t rx_bad_outer_l4_csum; 1789 } ports_stats[RTE_MAX_ETHPORTS]; 1790 uint64_t total_rx_dropped = 0; 1791 uint64_t total_tx_dropped = 0; 1792 uint64_t total_rx_nombuf = 0; 1793 struct rte_eth_stats stats; 1794 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1795 uint64_t fwd_cycles = 0; 1796 #endif 1797 uint64_t total_recv = 0; 1798 uint64_t total_xmit = 0; 1799 struct rte_port *port; 1800 streamid_t sm_id; 1801 portid_t pt_id; 1802 int i; 1803 1804 memset(ports_stats, 0, sizeof(ports_stats)); 1805 1806 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1807 struct fwd_stream *fs = fwd_streams[sm_id]; 1808 1809 if (cur_fwd_config.nb_fwd_streams > 1810 cur_fwd_config.nb_fwd_ports) { 1811 fwd_stream_stats_display(sm_id); 1812 } else { 1813 ports_stats[fs->tx_port].tx_stream = fs; 1814 ports_stats[fs->rx_port].rx_stream = fs; 1815 } 1816 1817 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1818 1819 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1820 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1821 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1822 fs->rx_bad_outer_l4_csum; 1823 1824 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1825 fwd_cycles += fs->core_cycles; 1826 #endif 1827 } 1828 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1829 uint8_t j; 1830 1831 pt_id = fwd_ports_ids[i]; 1832 port = &ports[pt_id]; 1833 1834 rte_eth_stats_get(pt_id, &stats); 1835 stats.ipackets -= port->stats.ipackets; 1836 stats.opackets -= port->stats.opackets; 1837 stats.ibytes -= port->stats.ibytes; 1838 stats.obytes -= port->stats.obytes; 1839 stats.imissed -= port->stats.imissed; 1840 stats.oerrors -= port->stats.oerrors; 1841 stats.rx_nombuf -= port->stats.rx_nombuf; 1842 1843 total_recv += stats.ipackets; 1844 total_xmit += stats.opackets; 1845 total_rx_dropped += stats.imissed; 1846 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1847 total_tx_dropped += stats.oerrors; 1848 total_rx_nombuf += stats.rx_nombuf; 1849 1850 printf("\n %s Forward statistics for port %-2d %s\n", 1851 fwd_stats_border, pt_id, fwd_stats_border); 1852 1853 if (!port->rx_queue_stats_mapping_enabled && 1854 !port->tx_queue_stats_mapping_enabled) { 1855 printf(" RX-packets: %-14"PRIu64 1856 " RX-dropped: %-14"PRIu64 1857 "RX-total: %-"PRIu64"\n", 1858 stats.ipackets, stats.imissed, 1859 stats.ipackets + stats.imissed); 1860 1861 if (cur_fwd_eng == &csum_fwd_engine) 1862 printf(" Bad-ipcsum: %-14"PRIu64 1863 " Bad-l4csum: %-14"PRIu64 1864 "Bad-outer-l4csum: %-14"PRIu64"\n", 1865 ports_stats[pt_id].rx_bad_ip_csum, 1866 ports_stats[pt_id].rx_bad_l4_csum, 1867 ports_stats[pt_id].rx_bad_outer_l4_csum); 1868 if (stats.ierrors + stats.rx_nombuf > 0) { 1869 printf(" RX-error: %-"PRIu64"\n", 1870 stats.ierrors); 1871 printf(" RX-nombufs: %-14"PRIu64"\n", 1872 stats.rx_nombuf); 1873 } 1874 1875 printf(" TX-packets: %-14"PRIu64 1876 " TX-dropped: %-14"PRIu64 1877 "TX-total: %-"PRIu64"\n", 1878 stats.opackets, ports_stats[pt_id].tx_dropped, 1879 stats.opackets + ports_stats[pt_id].tx_dropped); 1880 } else { 1881 printf(" RX-packets: %14"PRIu64 1882 " RX-dropped:%14"PRIu64 1883 " RX-total:%14"PRIu64"\n", 1884 stats.ipackets, stats.imissed, 1885 stats.ipackets + stats.imissed); 1886 1887 if (cur_fwd_eng == &csum_fwd_engine) 1888 printf(" Bad-ipcsum:%14"PRIu64 1889 " Bad-l4csum:%14"PRIu64 1890 " Bad-outer-l4csum: %-14"PRIu64"\n", 1891 ports_stats[pt_id].rx_bad_ip_csum, 1892 ports_stats[pt_id].rx_bad_l4_csum, 1893 ports_stats[pt_id].rx_bad_outer_l4_csum); 1894 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1895 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1896 printf(" RX-nombufs: %14"PRIu64"\n", 1897 stats.rx_nombuf); 1898 } 1899 1900 printf(" TX-packets: %14"PRIu64 1901 " TX-dropped:%14"PRIu64 1902 " TX-total:%14"PRIu64"\n", 1903 stats.opackets, ports_stats[pt_id].tx_dropped, 1904 stats.opackets + ports_stats[pt_id].tx_dropped); 1905 } 1906 1907 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1908 if (ports_stats[pt_id].rx_stream) 1909 pkt_burst_stats_display("RX", 1910 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1911 if (ports_stats[pt_id].tx_stream) 1912 pkt_burst_stats_display("TX", 1913 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1914 #endif 1915 1916 if (port->rx_queue_stats_mapping_enabled) { 1917 printf("\n"); 1918 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1919 printf(" Stats reg %2d RX-packets:%14"PRIu64 1920 " RX-errors:%14"PRIu64 1921 " RX-bytes:%14"PRIu64"\n", 1922 j, stats.q_ipackets[j], 1923 stats.q_errors[j], stats.q_ibytes[j]); 1924 } 1925 printf("\n"); 1926 } 1927 if (port->tx_queue_stats_mapping_enabled) { 1928 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1929 printf(" Stats reg %2d TX-packets:%14"PRIu64 1930 " TX-bytes:%14" 1931 PRIu64"\n", 1932 j, stats.q_opackets[j], 1933 stats.q_obytes[j]); 1934 } 1935 } 1936 1937 printf(" %s--------------------------------%s\n", 1938 fwd_stats_border, fwd_stats_border); 1939 } 1940 1941 printf("\n %s Accumulated forward statistics for all ports" 1942 "%s\n", 1943 acc_stats_border, acc_stats_border); 1944 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1945 "%-"PRIu64"\n" 1946 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1947 "%-"PRIu64"\n", 1948 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1949 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1950 if (total_rx_nombuf > 0) 1951 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1952 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1953 "%s\n", 1954 acc_stats_border, acc_stats_border); 1955 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1956 if (total_recv > 0) 1957 printf("\n CPU cycles/packet=%u (total cycles=" 1958 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1959 (unsigned int)(fwd_cycles / total_recv), 1960 fwd_cycles, total_recv); 1961 #endif 1962 } 1963 1964 void 1965 fwd_stats_reset(void) 1966 { 1967 streamid_t sm_id; 1968 portid_t pt_id; 1969 int i; 1970 1971 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1972 pt_id = fwd_ports_ids[i]; 1973 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1974 } 1975 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1976 struct fwd_stream *fs = fwd_streams[sm_id]; 1977 1978 fs->rx_packets = 0; 1979 fs->tx_packets = 0; 1980 fs->fwd_dropped = 0; 1981 fs->rx_bad_ip_csum = 0; 1982 fs->rx_bad_l4_csum = 0; 1983 fs->rx_bad_outer_l4_csum = 0; 1984 1985 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1986 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1987 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1988 #endif 1989 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1990 fs->core_cycles = 0; 1991 #endif 1992 } 1993 } 1994 1995 static void 1996 flush_fwd_rx_queues(void) 1997 { 1998 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1999 portid_t rxp; 2000 portid_t port_id; 2001 queueid_t rxq; 2002 uint16_t nb_rx; 2003 uint16_t i; 2004 uint8_t j; 2005 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2006 uint64_t timer_period; 2007 2008 /* convert to number of cycles */ 2009 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2010 2011 for (j = 0; j < 2; j++) { 2012 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2013 for (rxq = 0; rxq < nb_rxq; rxq++) { 2014 port_id = fwd_ports_ids[rxp]; 2015 /** 2016 * testpmd can stuck in the below do while loop 2017 * if rte_eth_rx_burst() always returns nonzero 2018 * packets. So timer is added to exit this loop 2019 * after 1sec timer expiry. 2020 */ 2021 prev_tsc = rte_rdtsc(); 2022 do { 2023 nb_rx = rte_eth_rx_burst(port_id, rxq, 2024 pkts_burst, MAX_PKT_BURST); 2025 for (i = 0; i < nb_rx; i++) 2026 rte_pktmbuf_free(pkts_burst[i]); 2027 2028 cur_tsc = rte_rdtsc(); 2029 diff_tsc = cur_tsc - prev_tsc; 2030 timer_tsc += diff_tsc; 2031 } while ((nb_rx > 0) && 2032 (timer_tsc < timer_period)); 2033 timer_tsc = 0; 2034 } 2035 } 2036 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2037 } 2038 } 2039 2040 static void 2041 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2042 { 2043 struct fwd_stream **fsm; 2044 streamid_t nb_fs; 2045 streamid_t sm_id; 2046 #ifdef RTE_LIBRTE_BITRATE 2047 uint64_t tics_per_1sec; 2048 uint64_t tics_datum; 2049 uint64_t tics_current; 2050 uint16_t i, cnt_ports; 2051 2052 cnt_ports = nb_ports; 2053 tics_datum = rte_rdtsc(); 2054 tics_per_1sec = rte_get_timer_hz(); 2055 #endif 2056 fsm = &fwd_streams[fc->stream_idx]; 2057 nb_fs = fc->stream_nb; 2058 do { 2059 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2060 (*pkt_fwd)(fsm[sm_id]); 2061 #ifdef RTE_LIBRTE_BITRATE 2062 if (bitrate_enabled != 0 && 2063 bitrate_lcore_id == rte_lcore_id()) { 2064 tics_current = rte_rdtsc(); 2065 if (tics_current - tics_datum >= tics_per_1sec) { 2066 /* Periodic bitrate calculation */ 2067 for (i = 0; i < cnt_ports; i++) 2068 rte_stats_bitrate_calc(bitrate_data, 2069 ports_ids[i]); 2070 tics_datum = tics_current; 2071 } 2072 } 2073 #endif 2074 #ifdef RTE_LIBRTE_LATENCY_STATS 2075 if (latencystats_enabled != 0 && 2076 latencystats_lcore_id == rte_lcore_id()) 2077 rte_latencystats_update(); 2078 #endif 2079 2080 } while (! fc->stopped); 2081 } 2082 2083 static int 2084 start_pkt_forward_on_core(void *fwd_arg) 2085 { 2086 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2087 cur_fwd_config.fwd_eng->packet_fwd); 2088 return 0; 2089 } 2090 2091 /* 2092 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2093 * Used to start communication flows in network loopback test configurations. 2094 */ 2095 static int 2096 run_one_txonly_burst_on_core(void *fwd_arg) 2097 { 2098 struct fwd_lcore *fwd_lc; 2099 struct fwd_lcore tmp_lcore; 2100 2101 fwd_lc = (struct fwd_lcore *) fwd_arg; 2102 tmp_lcore = *fwd_lc; 2103 tmp_lcore.stopped = 1; 2104 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2105 return 0; 2106 } 2107 2108 /* 2109 * Launch packet forwarding: 2110 * - Setup per-port forwarding context. 2111 * - launch logical cores with their forwarding configuration. 2112 */ 2113 static void 2114 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2115 { 2116 port_fwd_begin_t port_fwd_begin; 2117 unsigned int i; 2118 unsigned int lc_id; 2119 int diag; 2120 2121 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2122 if (port_fwd_begin != NULL) { 2123 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2124 (*port_fwd_begin)(fwd_ports_ids[i]); 2125 } 2126 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2127 lc_id = fwd_lcores_cpuids[i]; 2128 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2129 fwd_lcores[i]->stopped = 0; 2130 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2131 fwd_lcores[i], lc_id); 2132 if (diag != 0) 2133 printf("launch lcore %u failed - diag=%d\n", 2134 lc_id, diag); 2135 } 2136 } 2137 } 2138 2139 /* 2140 * Launch packet forwarding configuration. 2141 */ 2142 void 2143 start_packet_forwarding(int with_tx_first) 2144 { 2145 port_fwd_begin_t port_fwd_begin; 2146 port_fwd_end_t port_fwd_end; 2147 struct rte_port *port; 2148 unsigned int i; 2149 portid_t pt_id; 2150 2151 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2152 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2153 2154 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2155 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2156 2157 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2158 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2159 (!nb_rxq || !nb_txq)) 2160 rte_exit(EXIT_FAILURE, 2161 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2162 cur_fwd_eng->fwd_mode_name); 2163 2164 if (all_ports_started() == 0) { 2165 printf("Not all ports were started\n"); 2166 return; 2167 } 2168 if (test_done == 0) { 2169 printf("Packet forwarding already started\n"); 2170 return; 2171 } 2172 2173 2174 if(dcb_test) { 2175 for (i = 0; i < nb_fwd_ports; i++) { 2176 pt_id = fwd_ports_ids[i]; 2177 port = &ports[pt_id]; 2178 if (!port->dcb_flag) { 2179 printf("In DCB mode, all forwarding ports must " 2180 "be configured in this mode.\n"); 2181 return; 2182 } 2183 } 2184 if (nb_fwd_lcores == 1) { 2185 printf("In DCB mode,the nb forwarding cores " 2186 "should be larger than 1.\n"); 2187 return; 2188 } 2189 } 2190 test_done = 0; 2191 2192 fwd_config_setup(); 2193 2194 if(!no_flush_rx) 2195 flush_fwd_rx_queues(); 2196 2197 pkt_fwd_config_display(&cur_fwd_config); 2198 rxtx_config_display(); 2199 2200 fwd_stats_reset(); 2201 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2202 pt_id = fwd_ports_ids[i]; 2203 port = &ports[pt_id]; 2204 map_port_queue_stats_mapping_registers(pt_id, port); 2205 } 2206 if (with_tx_first) { 2207 port_fwd_begin = tx_only_engine.port_fwd_begin; 2208 if (port_fwd_begin != NULL) { 2209 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2210 (*port_fwd_begin)(fwd_ports_ids[i]); 2211 } 2212 while (with_tx_first--) { 2213 launch_packet_forwarding( 2214 run_one_txonly_burst_on_core); 2215 rte_eal_mp_wait_lcore(); 2216 } 2217 port_fwd_end = tx_only_engine.port_fwd_end; 2218 if (port_fwd_end != NULL) { 2219 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2220 (*port_fwd_end)(fwd_ports_ids[i]); 2221 } 2222 } 2223 launch_packet_forwarding(start_pkt_forward_on_core); 2224 } 2225 2226 void 2227 stop_packet_forwarding(void) 2228 { 2229 port_fwd_end_t port_fwd_end; 2230 lcoreid_t lc_id; 2231 portid_t pt_id; 2232 int i; 2233 2234 if (test_done) { 2235 printf("Packet forwarding not started\n"); 2236 return; 2237 } 2238 printf("Telling cores to stop..."); 2239 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2240 fwd_lcores[lc_id]->stopped = 1; 2241 printf("\nWaiting for lcores to finish...\n"); 2242 rte_eal_mp_wait_lcore(); 2243 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2244 if (port_fwd_end != NULL) { 2245 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2246 pt_id = fwd_ports_ids[i]; 2247 (*port_fwd_end)(pt_id); 2248 } 2249 } 2250 2251 fwd_stats_display(); 2252 2253 printf("\nDone.\n"); 2254 test_done = 1; 2255 } 2256 2257 void 2258 dev_set_link_up(portid_t pid) 2259 { 2260 if (rte_eth_dev_set_link_up(pid) < 0) 2261 printf("\nSet link up fail.\n"); 2262 } 2263 2264 void 2265 dev_set_link_down(portid_t pid) 2266 { 2267 if (rte_eth_dev_set_link_down(pid) < 0) 2268 printf("\nSet link down fail.\n"); 2269 } 2270 2271 static int 2272 all_ports_started(void) 2273 { 2274 portid_t pi; 2275 struct rte_port *port; 2276 2277 RTE_ETH_FOREACH_DEV(pi) { 2278 port = &ports[pi]; 2279 /* Check if there is a port which is not started */ 2280 if ((port->port_status != RTE_PORT_STARTED) && 2281 (port->slave_flag == 0)) 2282 return 0; 2283 } 2284 2285 /* No port is not started */ 2286 return 1; 2287 } 2288 2289 int 2290 port_is_stopped(portid_t port_id) 2291 { 2292 struct rte_port *port = &ports[port_id]; 2293 2294 if ((port->port_status != RTE_PORT_STOPPED) && 2295 (port->slave_flag == 0)) 2296 return 0; 2297 return 1; 2298 } 2299 2300 int 2301 all_ports_stopped(void) 2302 { 2303 portid_t pi; 2304 2305 RTE_ETH_FOREACH_DEV(pi) { 2306 if (!port_is_stopped(pi)) 2307 return 0; 2308 } 2309 2310 return 1; 2311 } 2312 2313 int 2314 port_is_started(portid_t port_id) 2315 { 2316 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2317 return 0; 2318 2319 if (ports[port_id].port_status != RTE_PORT_STARTED) 2320 return 0; 2321 2322 return 1; 2323 } 2324 2325 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2326 static int 2327 setup_hairpin_queues(portid_t pi) 2328 { 2329 queueid_t qi; 2330 struct rte_eth_hairpin_conf hairpin_conf = { 2331 .peer_count = 1, 2332 }; 2333 int i; 2334 int diag; 2335 struct rte_port *port = &ports[pi]; 2336 2337 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2338 hairpin_conf.peers[0].port = pi; 2339 hairpin_conf.peers[0].queue = i + nb_rxq; 2340 diag = rte_eth_tx_hairpin_queue_setup 2341 (pi, qi, nb_txd, &hairpin_conf); 2342 i++; 2343 if (diag == 0) 2344 continue; 2345 2346 /* Fail to setup rx queue, return */ 2347 if (rte_atomic16_cmpset(&(port->port_status), 2348 RTE_PORT_HANDLING, 2349 RTE_PORT_STOPPED) == 0) 2350 printf("Port %d can not be set back " 2351 "to stopped\n", pi); 2352 printf("Fail to configure port %d hairpin " 2353 "queues\n", pi); 2354 /* try to reconfigure queues next time */ 2355 port->need_reconfig_queues = 1; 2356 return -1; 2357 } 2358 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2359 hairpin_conf.peers[0].port = pi; 2360 hairpin_conf.peers[0].queue = i + nb_txq; 2361 diag = rte_eth_rx_hairpin_queue_setup 2362 (pi, qi, nb_rxd, &hairpin_conf); 2363 i++; 2364 if (diag == 0) 2365 continue; 2366 2367 /* Fail to setup rx queue, return */ 2368 if (rte_atomic16_cmpset(&(port->port_status), 2369 RTE_PORT_HANDLING, 2370 RTE_PORT_STOPPED) == 0) 2371 printf("Port %d can not be set back " 2372 "to stopped\n", pi); 2373 printf("Fail to configure port %d hairpin " 2374 "queues\n", pi); 2375 /* try to reconfigure queues next time */ 2376 port->need_reconfig_queues = 1; 2377 return -1; 2378 } 2379 return 0; 2380 } 2381 2382 int 2383 start_port(portid_t pid) 2384 { 2385 int diag, need_check_link_status = -1; 2386 portid_t pi; 2387 queueid_t qi; 2388 struct rte_port *port; 2389 struct rte_ether_addr mac_addr; 2390 struct rte_eth_hairpin_cap cap; 2391 2392 if (port_id_is_invalid(pid, ENABLED_WARN)) 2393 return 0; 2394 2395 if(dcb_config) 2396 dcb_test = 1; 2397 RTE_ETH_FOREACH_DEV(pi) { 2398 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2399 continue; 2400 2401 need_check_link_status = 0; 2402 port = &ports[pi]; 2403 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2404 RTE_PORT_HANDLING) == 0) { 2405 printf("Port %d is now not stopped\n", pi); 2406 continue; 2407 } 2408 2409 if (port->need_reconfig > 0) { 2410 port->need_reconfig = 0; 2411 2412 if (flow_isolate_all) { 2413 int ret = port_flow_isolate(pi, 1); 2414 if (ret) { 2415 printf("Failed to apply isolated" 2416 " mode on port %d\n", pi); 2417 return -1; 2418 } 2419 } 2420 configure_rxtx_dump_callbacks(0); 2421 printf("Configuring Port %d (socket %u)\n", pi, 2422 port->socket_id); 2423 if (nb_hairpinq > 0 && 2424 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2425 printf("Port %d doesn't support hairpin " 2426 "queues\n", pi); 2427 return -1; 2428 } 2429 /* configure port */ 2430 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2431 nb_txq + nb_hairpinq, 2432 &(port->dev_conf)); 2433 if (diag != 0) { 2434 if (rte_atomic16_cmpset(&(port->port_status), 2435 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2436 printf("Port %d can not be set back " 2437 "to stopped\n", pi); 2438 printf("Fail to configure port %d\n", pi); 2439 /* try to reconfigure port next time */ 2440 port->need_reconfig = 1; 2441 return -1; 2442 } 2443 } 2444 if (port->need_reconfig_queues > 0) { 2445 port->need_reconfig_queues = 0; 2446 /* setup tx queues */ 2447 for (qi = 0; qi < nb_txq; qi++) { 2448 if ((numa_support) && 2449 (txring_numa[pi] != NUMA_NO_CONFIG)) 2450 diag = rte_eth_tx_queue_setup(pi, qi, 2451 port->nb_tx_desc[qi], 2452 txring_numa[pi], 2453 &(port->tx_conf[qi])); 2454 else 2455 diag = rte_eth_tx_queue_setup(pi, qi, 2456 port->nb_tx_desc[qi], 2457 port->socket_id, 2458 &(port->tx_conf[qi])); 2459 2460 if (diag == 0) 2461 continue; 2462 2463 /* Fail to setup tx queue, return */ 2464 if (rte_atomic16_cmpset(&(port->port_status), 2465 RTE_PORT_HANDLING, 2466 RTE_PORT_STOPPED) == 0) 2467 printf("Port %d can not be set back " 2468 "to stopped\n", pi); 2469 printf("Fail to configure port %d tx queues\n", 2470 pi); 2471 /* try to reconfigure queues next time */ 2472 port->need_reconfig_queues = 1; 2473 return -1; 2474 } 2475 for (qi = 0; qi < nb_rxq; qi++) { 2476 /* setup rx queues */ 2477 if ((numa_support) && 2478 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2479 struct rte_mempool * mp = 2480 mbuf_pool_find(rxring_numa[pi]); 2481 if (mp == NULL) { 2482 printf("Failed to setup RX queue:" 2483 "No mempool allocation" 2484 " on the socket %d\n", 2485 rxring_numa[pi]); 2486 return -1; 2487 } 2488 2489 diag = rte_eth_rx_queue_setup(pi, qi, 2490 port->nb_rx_desc[qi], 2491 rxring_numa[pi], 2492 &(port->rx_conf[qi]), 2493 mp); 2494 } else { 2495 struct rte_mempool *mp = 2496 mbuf_pool_find(port->socket_id); 2497 if (mp == NULL) { 2498 printf("Failed to setup RX queue:" 2499 "No mempool allocation" 2500 " on the socket %d\n", 2501 port->socket_id); 2502 return -1; 2503 } 2504 diag = rte_eth_rx_queue_setup(pi, qi, 2505 port->nb_rx_desc[qi], 2506 port->socket_id, 2507 &(port->rx_conf[qi]), 2508 mp); 2509 } 2510 if (diag == 0) 2511 continue; 2512 2513 /* Fail to setup rx queue, return */ 2514 if (rte_atomic16_cmpset(&(port->port_status), 2515 RTE_PORT_HANDLING, 2516 RTE_PORT_STOPPED) == 0) 2517 printf("Port %d can not be set back " 2518 "to stopped\n", pi); 2519 printf("Fail to configure port %d rx queues\n", 2520 pi); 2521 /* try to reconfigure queues next time */ 2522 port->need_reconfig_queues = 1; 2523 return -1; 2524 } 2525 /* setup hairpin queues */ 2526 if (setup_hairpin_queues(pi) != 0) 2527 return -1; 2528 } 2529 configure_rxtx_dump_callbacks(verbose_level); 2530 if (clear_ptypes) { 2531 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2532 NULL, 0); 2533 if (diag < 0) 2534 printf( 2535 "Port %d: Failed to disable Ptype parsing\n", 2536 pi); 2537 } 2538 2539 /* start port */ 2540 if (rte_eth_dev_start(pi) < 0) { 2541 printf("Fail to start port %d\n", pi); 2542 2543 /* Fail to setup rx queue, return */ 2544 if (rte_atomic16_cmpset(&(port->port_status), 2545 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2546 printf("Port %d can not be set back to " 2547 "stopped\n", pi); 2548 continue; 2549 } 2550 2551 if (rte_atomic16_cmpset(&(port->port_status), 2552 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2553 printf("Port %d can not be set into started\n", pi); 2554 2555 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2556 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2557 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2558 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2559 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2560 2561 /* at least one port started, need checking link status */ 2562 need_check_link_status = 1; 2563 } 2564 2565 if (need_check_link_status == 1 && !no_link_check) 2566 check_all_ports_link_status(RTE_PORT_ALL); 2567 else if (need_check_link_status == 0) 2568 printf("Please stop the ports first\n"); 2569 2570 printf("Done\n"); 2571 return 0; 2572 } 2573 2574 void 2575 stop_port(portid_t pid) 2576 { 2577 portid_t pi; 2578 struct rte_port *port; 2579 int need_check_link_status = 0; 2580 2581 if (dcb_test) { 2582 dcb_test = 0; 2583 dcb_config = 0; 2584 } 2585 2586 if (port_id_is_invalid(pid, ENABLED_WARN)) 2587 return; 2588 2589 printf("Stopping ports...\n"); 2590 2591 RTE_ETH_FOREACH_DEV(pi) { 2592 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2593 continue; 2594 2595 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2596 printf("Please remove port %d from forwarding configuration.\n", pi); 2597 continue; 2598 } 2599 2600 if (port_is_bonding_slave(pi)) { 2601 printf("Please remove port %d from bonded device.\n", pi); 2602 continue; 2603 } 2604 2605 port = &ports[pi]; 2606 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2607 RTE_PORT_HANDLING) == 0) 2608 continue; 2609 2610 rte_eth_dev_stop(pi); 2611 2612 if (rte_atomic16_cmpset(&(port->port_status), 2613 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2614 printf("Port %d can not be set into stopped\n", pi); 2615 need_check_link_status = 1; 2616 } 2617 if (need_check_link_status && !no_link_check) 2618 check_all_ports_link_status(RTE_PORT_ALL); 2619 2620 printf("Done\n"); 2621 } 2622 2623 static void 2624 remove_invalid_ports_in(portid_t *array, portid_t *total) 2625 { 2626 portid_t i; 2627 portid_t new_total = 0; 2628 2629 for (i = 0; i < *total; i++) 2630 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2631 array[new_total] = array[i]; 2632 new_total++; 2633 } 2634 *total = new_total; 2635 } 2636 2637 static void 2638 remove_invalid_ports(void) 2639 { 2640 remove_invalid_ports_in(ports_ids, &nb_ports); 2641 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2642 nb_cfg_ports = nb_fwd_ports; 2643 } 2644 2645 void 2646 close_port(portid_t pid) 2647 { 2648 portid_t pi; 2649 struct rte_port *port; 2650 2651 if (port_id_is_invalid(pid, ENABLED_WARN)) 2652 return; 2653 2654 printf("Closing ports...\n"); 2655 2656 RTE_ETH_FOREACH_DEV(pi) { 2657 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2658 continue; 2659 2660 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2661 printf("Please remove port %d from forwarding configuration.\n", pi); 2662 continue; 2663 } 2664 2665 if (port_is_bonding_slave(pi)) { 2666 printf("Please remove port %d from bonded device.\n", pi); 2667 continue; 2668 } 2669 2670 port = &ports[pi]; 2671 if (rte_atomic16_cmpset(&(port->port_status), 2672 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2673 printf("Port %d is already closed\n", pi); 2674 continue; 2675 } 2676 2677 if (rte_atomic16_cmpset(&(port->port_status), 2678 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2679 printf("Port %d is now not stopped\n", pi); 2680 continue; 2681 } 2682 2683 if (port->flow_list) 2684 port_flow_flush(pi); 2685 rte_eth_dev_close(pi); 2686 2687 remove_invalid_ports(); 2688 2689 if (rte_atomic16_cmpset(&(port->port_status), 2690 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2691 printf("Port %d cannot be set to closed\n", pi); 2692 } 2693 2694 printf("Done\n"); 2695 } 2696 2697 void 2698 reset_port(portid_t pid) 2699 { 2700 int diag; 2701 portid_t pi; 2702 struct rte_port *port; 2703 2704 if (port_id_is_invalid(pid, ENABLED_WARN)) 2705 return; 2706 2707 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2708 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2709 printf("Can not reset port(s), please stop port(s) first.\n"); 2710 return; 2711 } 2712 2713 printf("Resetting ports...\n"); 2714 2715 RTE_ETH_FOREACH_DEV(pi) { 2716 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2717 continue; 2718 2719 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2720 printf("Please remove port %d from forwarding " 2721 "configuration.\n", pi); 2722 continue; 2723 } 2724 2725 if (port_is_bonding_slave(pi)) { 2726 printf("Please remove port %d from bonded device.\n", 2727 pi); 2728 continue; 2729 } 2730 2731 diag = rte_eth_dev_reset(pi); 2732 if (diag == 0) { 2733 port = &ports[pi]; 2734 port->need_reconfig = 1; 2735 port->need_reconfig_queues = 1; 2736 } else { 2737 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2738 } 2739 } 2740 2741 printf("Done\n"); 2742 } 2743 2744 void 2745 attach_port(char *identifier) 2746 { 2747 portid_t pi; 2748 struct rte_dev_iterator iterator; 2749 2750 printf("Attaching a new port...\n"); 2751 2752 if (identifier == NULL) { 2753 printf("Invalid parameters are specified\n"); 2754 return; 2755 } 2756 2757 if (rte_dev_probe(identifier) < 0) { 2758 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2759 return; 2760 } 2761 2762 /* first attach mode: event */ 2763 if (setup_on_probe_event) { 2764 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2765 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2766 if (ports[pi].port_status == RTE_PORT_HANDLING && 2767 ports[pi].need_setup != 0) 2768 setup_attached_port(pi); 2769 return; 2770 } 2771 2772 /* second attach mode: iterator */ 2773 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2774 /* setup ports matching the devargs used for probing */ 2775 if (port_is_forwarding(pi)) 2776 continue; /* port was already attached before */ 2777 setup_attached_port(pi); 2778 } 2779 } 2780 2781 static void 2782 setup_attached_port(portid_t pi) 2783 { 2784 unsigned int socket_id; 2785 int ret; 2786 2787 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2788 /* if socket_id is invalid, set to the first available socket. */ 2789 if (check_socket_id(socket_id) < 0) 2790 socket_id = socket_ids[0]; 2791 reconfig(pi, socket_id); 2792 ret = rte_eth_promiscuous_enable(pi); 2793 if (ret != 0) 2794 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2795 pi, rte_strerror(-ret)); 2796 2797 ports_ids[nb_ports++] = pi; 2798 fwd_ports_ids[nb_fwd_ports++] = pi; 2799 nb_cfg_ports = nb_fwd_ports; 2800 ports[pi].need_setup = 0; 2801 ports[pi].port_status = RTE_PORT_STOPPED; 2802 2803 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2804 printf("Done\n"); 2805 } 2806 2807 static void 2808 detach_device(struct rte_device *dev) 2809 { 2810 portid_t sibling; 2811 2812 if (dev == NULL) { 2813 printf("Device already removed\n"); 2814 return; 2815 } 2816 2817 printf("Removing a device...\n"); 2818 2819 if (rte_dev_remove(dev) < 0) { 2820 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2821 return; 2822 } 2823 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2824 /* reset mapping between old ports and removed device */ 2825 rte_eth_devices[sibling].device = NULL; 2826 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2827 /* sibling ports are forced to be closed */ 2828 ports[sibling].port_status = RTE_PORT_CLOSED; 2829 printf("Port %u is closed\n", sibling); 2830 } 2831 } 2832 2833 remove_invalid_ports(); 2834 2835 printf("Device is detached\n"); 2836 printf("Now total ports is %d\n", nb_ports); 2837 printf("Done\n"); 2838 return; 2839 } 2840 2841 void 2842 detach_port_device(portid_t port_id) 2843 { 2844 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2845 return; 2846 2847 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2848 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2849 printf("Port not stopped\n"); 2850 return; 2851 } 2852 printf("Port was not closed\n"); 2853 if (ports[port_id].flow_list) 2854 port_flow_flush(port_id); 2855 } 2856 2857 detach_device(rte_eth_devices[port_id].device); 2858 } 2859 2860 void 2861 detach_devargs(char *identifier) 2862 { 2863 struct rte_dev_iterator iterator; 2864 struct rte_devargs da; 2865 portid_t port_id; 2866 2867 printf("Removing a device...\n"); 2868 2869 memset(&da, 0, sizeof(da)); 2870 if (rte_devargs_parsef(&da, "%s", identifier)) { 2871 printf("cannot parse identifier\n"); 2872 if (da.args) 2873 free(da.args); 2874 return; 2875 } 2876 2877 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2878 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2879 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2880 printf("Port %u not stopped\n", port_id); 2881 rte_eth_iterator_cleanup(&iterator); 2882 return; 2883 } 2884 2885 /* sibling ports are forced to be closed */ 2886 if (ports[port_id].flow_list) 2887 port_flow_flush(port_id); 2888 ports[port_id].port_status = RTE_PORT_CLOSED; 2889 printf("Port %u is now closed\n", port_id); 2890 } 2891 } 2892 2893 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2894 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2895 da.name, da.bus->name); 2896 return; 2897 } 2898 2899 remove_invalid_ports(); 2900 2901 printf("Device %s is detached\n", identifier); 2902 printf("Now total ports is %d\n", nb_ports); 2903 printf("Done\n"); 2904 } 2905 2906 void 2907 pmd_test_exit(void) 2908 { 2909 portid_t pt_id; 2910 int ret; 2911 int i; 2912 2913 if (test_done == 0) 2914 stop_packet_forwarding(); 2915 2916 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2917 if (mempools[i]) { 2918 if (mp_alloc_type == MP_ALLOC_ANON) 2919 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2920 NULL); 2921 } 2922 } 2923 if (ports != NULL) { 2924 no_link_check = 1; 2925 RTE_ETH_FOREACH_DEV(pt_id) { 2926 printf("\nStopping port %d...\n", pt_id); 2927 fflush(stdout); 2928 stop_port(pt_id); 2929 } 2930 RTE_ETH_FOREACH_DEV(pt_id) { 2931 printf("\nShutting down port %d...\n", pt_id); 2932 fflush(stdout); 2933 close_port(pt_id); 2934 } 2935 } 2936 2937 if (hot_plug) { 2938 ret = rte_dev_event_monitor_stop(); 2939 if (ret) { 2940 RTE_LOG(ERR, EAL, 2941 "fail to stop device event monitor."); 2942 return; 2943 } 2944 2945 ret = rte_dev_event_callback_unregister(NULL, 2946 dev_event_callback, NULL); 2947 if (ret < 0) { 2948 RTE_LOG(ERR, EAL, 2949 "fail to unregister device event callback.\n"); 2950 return; 2951 } 2952 2953 ret = rte_dev_hotplug_handle_disable(); 2954 if (ret) { 2955 RTE_LOG(ERR, EAL, 2956 "fail to disable hotplug handling.\n"); 2957 return; 2958 } 2959 } 2960 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2961 if (mempools[i]) 2962 rte_mempool_free(mempools[i]); 2963 } 2964 2965 printf("\nBye...\n"); 2966 } 2967 2968 typedef void (*cmd_func_t)(void); 2969 struct pmd_test_command { 2970 const char *cmd_name; 2971 cmd_func_t cmd_func; 2972 }; 2973 2974 /* Check the link status of all ports in up to 9s, and print them finally */ 2975 static void 2976 check_all_ports_link_status(uint32_t port_mask) 2977 { 2978 #define CHECK_INTERVAL 100 /* 100ms */ 2979 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2980 portid_t portid; 2981 uint8_t count, all_ports_up, print_flag = 0; 2982 struct rte_eth_link link; 2983 int ret; 2984 2985 printf("Checking link statuses...\n"); 2986 fflush(stdout); 2987 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2988 all_ports_up = 1; 2989 RTE_ETH_FOREACH_DEV(portid) { 2990 if ((port_mask & (1 << portid)) == 0) 2991 continue; 2992 memset(&link, 0, sizeof(link)); 2993 ret = rte_eth_link_get_nowait(portid, &link); 2994 if (ret < 0) { 2995 all_ports_up = 0; 2996 if (print_flag == 1) 2997 printf("Port %u link get failed: %s\n", 2998 portid, rte_strerror(-ret)); 2999 continue; 3000 } 3001 /* print link status if flag set */ 3002 if (print_flag == 1) { 3003 if (link.link_status) 3004 printf( 3005 "Port%d Link Up. speed %u Mbps- %s\n", 3006 portid, link.link_speed, 3007 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 3008 ("full-duplex") : ("half-duplex\n")); 3009 else 3010 printf("Port %d Link Down\n", portid); 3011 continue; 3012 } 3013 /* clear all_ports_up flag if any link down */ 3014 if (link.link_status == ETH_LINK_DOWN) { 3015 all_ports_up = 0; 3016 break; 3017 } 3018 } 3019 /* after finally printing all link status, get out */ 3020 if (print_flag == 1) 3021 break; 3022 3023 if (all_ports_up == 0) { 3024 fflush(stdout); 3025 rte_delay_ms(CHECK_INTERVAL); 3026 } 3027 3028 /* set the print_flag if all ports up or timeout */ 3029 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3030 print_flag = 1; 3031 } 3032 3033 if (lsc_interrupt) 3034 break; 3035 } 3036 } 3037 3038 /* 3039 * This callback is for remove a port for a device. It has limitation because 3040 * it is not for multiple port removal for a device. 3041 * TODO: the device detach invoke will plan to be removed from user side to 3042 * eal. And convert all PMDs to free port resources on ether device closing. 3043 */ 3044 static void 3045 rmv_port_callback(void *arg) 3046 { 3047 int need_to_start = 0; 3048 int org_no_link_check = no_link_check; 3049 portid_t port_id = (intptr_t)arg; 3050 struct rte_device *dev; 3051 3052 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3053 3054 if (!test_done && port_is_forwarding(port_id)) { 3055 need_to_start = 1; 3056 stop_packet_forwarding(); 3057 } 3058 no_link_check = 1; 3059 stop_port(port_id); 3060 no_link_check = org_no_link_check; 3061 3062 /* Save rte_device pointer before closing ethdev port */ 3063 dev = rte_eth_devices[port_id].device; 3064 close_port(port_id); 3065 detach_device(dev); /* might be already removed or have more ports */ 3066 3067 if (need_to_start) 3068 start_packet_forwarding(0); 3069 } 3070 3071 /* This function is used by the interrupt thread */ 3072 static int 3073 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3074 void *ret_param) 3075 { 3076 RTE_SET_USED(param); 3077 RTE_SET_USED(ret_param); 3078 3079 if (type >= RTE_ETH_EVENT_MAX) { 3080 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3081 port_id, __func__, type); 3082 fflush(stderr); 3083 } else if (event_print_mask & (UINT32_C(1) << type)) { 3084 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3085 eth_event_desc[type]); 3086 fflush(stdout); 3087 } 3088 3089 switch (type) { 3090 case RTE_ETH_EVENT_NEW: 3091 ports[port_id].need_setup = 1; 3092 ports[port_id].port_status = RTE_PORT_HANDLING; 3093 break; 3094 case RTE_ETH_EVENT_INTR_RMV: 3095 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3096 break; 3097 if (rte_eal_alarm_set(100000, 3098 rmv_port_callback, (void *)(intptr_t)port_id)) 3099 fprintf(stderr, "Could not set up deferred device removal\n"); 3100 break; 3101 default: 3102 break; 3103 } 3104 return 0; 3105 } 3106 3107 static int 3108 register_eth_event_callback(void) 3109 { 3110 int ret; 3111 enum rte_eth_event_type event; 3112 3113 for (event = RTE_ETH_EVENT_UNKNOWN; 3114 event < RTE_ETH_EVENT_MAX; event++) { 3115 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3116 event, 3117 eth_event_callback, 3118 NULL); 3119 if (ret != 0) { 3120 TESTPMD_LOG(ERR, "Failed to register callback for " 3121 "%s event\n", eth_event_desc[event]); 3122 return -1; 3123 } 3124 } 3125 3126 return 0; 3127 } 3128 3129 /* This function is used by the interrupt thread */ 3130 static void 3131 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3132 __rte_unused void *arg) 3133 { 3134 uint16_t port_id; 3135 int ret; 3136 3137 if (type >= RTE_DEV_EVENT_MAX) { 3138 fprintf(stderr, "%s called upon invalid event %d\n", 3139 __func__, type); 3140 fflush(stderr); 3141 } 3142 3143 switch (type) { 3144 case RTE_DEV_EVENT_REMOVE: 3145 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3146 device_name); 3147 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3148 if (ret) { 3149 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3150 device_name); 3151 return; 3152 } 3153 /* 3154 * Because the user's callback is invoked in eal interrupt 3155 * callback, the interrupt callback need to be finished before 3156 * it can be unregistered when detaching device. So finish 3157 * callback soon and use a deferred removal to detach device 3158 * is need. It is a workaround, once the device detaching be 3159 * moved into the eal in the future, the deferred removal could 3160 * be deleted. 3161 */ 3162 if (rte_eal_alarm_set(100000, 3163 rmv_port_callback, (void *)(intptr_t)port_id)) 3164 RTE_LOG(ERR, EAL, 3165 "Could not set up deferred device removal\n"); 3166 break; 3167 case RTE_DEV_EVENT_ADD: 3168 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3169 device_name); 3170 /* TODO: After finish kernel driver binding, 3171 * begin to attach port. 3172 */ 3173 break; 3174 default: 3175 break; 3176 } 3177 } 3178 3179 static int 3180 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3181 { 3182 uint16_t i; 3183 int diag; 3184 uint8_t mapping_found = 0; 3185 3186 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3187 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3188 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3189 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3190 tx_queue_stats_mappings[i].queue_id, 3191 tx_queue_stats_mappings[i].stats_counter_id); 3192 if (diag != 0) 3193 return diag; 3194 mapping_found = 1; 3195 } 3196 } 3197 if (mapping_found) 3198 port->tx_queue_stats_mapping_enabled = 1; 3199 return 0; 3200 } 3201 3202 static int 3203 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3204 { 3205 uint16_t i; 3206 int diag; 3207 uint8_t mapping_found = 0; 3208 3209 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3210 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3211 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3212 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3213 rx_queue_stats_mappings[i].queue_id, 3214 rx_queue_stats_mappings[i].stats_counter_id); 3215 if (diag != 0) 3216 return diag; 3217 mapping_found = 1; 3218 } 3219 } 3220 if (mapping_found) 3221 port->rx_queue_stats_mapping_enabled = 1; 3222 return 0; 3223 } 3224 3225 static void 3226 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3227 { 3228 int diag = 0; 3229 3230 diag = set_tx_queue_stats_mapping_registers(pi, port); 3231 if (diag != 0) { 3232 if (diag == -ENOTSUP) { 3233 port->tx_queue_stats_mapping_enabled = 0; 3234 printf("TX queue stats mapping not supported port id=%d\n", pi); 3235 } 3236 else 3237 rte_exit(EXIT_FAILURE, 3238 "set_tx_queue_stats_mapping_registers " 3239 "failed for port id=%d diag=%d\n", 3240 pi, diag); 3241 } 3242 3243 diag = set_rx_queue_stats_mapping_registers(pi, port); 3244 if (diag != 0) { 3245 if (diag == -ENOTSUP) { 3246 port->rx_queue_stats_mapping_enabled = 0; 3247 printf("RX queue stats mapping not supported port id=%d\n", pi); 3248 } 3249 else 3250 rte_exit(EXIT_FAILURE, 3251 "set_rx_queue_stats_mapping_registers " 3252 "failed for port id=%d diag=%d\n", 3253 pi, diag); 3254 } 3255 } 3256 3257 static void 3258 rxtx_port_config(struct rte_port *port) 3259 { 3260 uint16_t qid; 3261 uint64_t offloads; 3262 3263 for (qid = 0; qid < nb_rxq; qid++) { 3264 offloads = port->rx_conf[qid].offloads; 3265 port->rx_conf[qid] = port->dev_info.default_rxconf; 3266 if (offloads != 0) 3267 port->rx_conf[qid].offloads = offloads; 3268 3269 /* Check if any Rx parameters have been passed */ 3270 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3271 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3272 3273 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3274 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3275 3276 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3277 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3278 3279 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3280 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3281 3282 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3283 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3284 3285 port->nb_rx_desc[qid] = nb_rxd; 3286 } 3287 3288 for (qid = 0; qid < nb_txq; qid++) { 3289 offloads = port->tx_conf[qid].offloads; 3290 port->tx_conf[qid] = port->dev_info.default_txconf; 3291 if (offloads != 0) 3292 port->tx_conf[qid].offloads = offloads; 3293 3294 /* Check if any Tx parameters have been passed */ 3295 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3296 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3297 3298 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3299 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3300 3301 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3302 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3303 3304 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3305 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3306 3307 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3308 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3309 3310 port->nb_tx_desc[qid] = nb_txd; 3311 } 3312 } 3313 3314 void 3315 init_port_config(void) 3316 { 3317 portid_t pid; 3318 struct rte_port *port; 3319 int ret; 3320 3321 RTE_ETH_FOREACH_DEV(pid) { 3322 port = &ports[pid]; 3323 port->dev_conf.fdir_conf = fdir_conf; 3324 3325 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3326 if (ret != 0) 3327 return; 3328 3329 if (nb_rxq > 1) { 3330 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3331 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3332 rss_hf & port->dev_info.flow_type_rss_offloads; 3333 } else { 3334 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3335 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3336 } 3337 3338 if (port->dcb_flag == 0) { 3339 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3340 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 3341 else 3342 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3343 } 3344 3345 rxtx_port_config(port); 3346 3347 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3348 if (ret != 0) 3349 return; 3350 3351 map_port_queue_stats_mapping_registers(pid, port); 3352 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3353 rte_pmd_ixgbe_bypass_init(pid); 3354 #endif 3355 3356 if (lsc_interrupt && 3357 (rte_eth_devices[pid].data->dev_flags & 3358 RTE_ETH_DEV_INTR_LSC)) 3359 port->dev_conf.intr_conf.lsc = 1; 3360 if (rmv_interrupt && 3361 (rte_eth_devices[pid].data->dev_flags & 3362 RTE_ETH_DEV_INTR_RMV)) 3363 port->dev_conf.intr_conf.rmv = 1; 3364 } 3365 } 3366 3367 void set_port_slave_flag(portid_t slave_pid) 3368 { 3369 struct rte_port *port; 3370 3371 port = &ports[slave_pid]; 3372 port->slave_flag = 1; 3373 } 3374 3375 void clear_port_slave_flag(portid_t slave_pid) 3376 { 3377 struct rte_port *port; 3378 3379 port = &ports[slave_pid]; 3380 port->slave_flag = 0; 3381 } 3382 3383 uint8_t port_is_bonding_slave(portid_t slave_pid) 3384 { 3385 struct rte_port *port; 3386 3387 port = &ports[slave_pid]; 3388 if ((rte_eth_devices[slave_pid].data->dev_flags & 3389 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3390 return 1; 3391 return 0; 3392 } 3393 3394 const uint16_t vlan_tags[] = { 3395 0, 1, 2, 3, 4, 5, 6, 7, 3396 8, 9, 10, 11, 12, 13, 14, 15, 3397 16, 17, 18, 19, 20, 21, 22, 23, 3398 24, 25, 26, 27, 28, 29, 30, 31 3399 }; 3400 3401 static int 3402 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3403 enum dcb_mode_enable dcb_mode, 3404 enum rte_eth_nb_tcs num_tcs, 3405 uint8_t pfc_en) 3406 { 3407 uint8_t i; 3408 int32_t rc; 3409 struct rte_eth_rss_conf rss_conf; 3410 3411 /* 3412 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3413 * given above, and the number of traffic classes available for use. 3414 */ 3415 if (dcb_mode == DCB_VT_ENABLED) { 3416 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3417 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3418 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3419 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3420 3421 /* VMDQ+DCB RX and TX configurations */ 3422 vmdq_rx_conf->enable_default_pool = 0; 3423 vmdq_rx_conf->default_pool = 0; 3424 vmdq_rx_conf->nb_queue_pools = 3425 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3426 vmdq_tx_conf->nb_queue_pools = 3427 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3428 3429 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3430 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3431 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3432 vmdq_rx_conf->pool_map[i].pools = 3433 1 << (i % vmdq_rx_conf->nb_queue_pools); 3434 } 3435 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3436 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3437 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3438 } 3439 3440 /* set DCB mode of RX and TX of multiple queues */ 3441 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 3442 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3443 } else { 3444 struct rte_eth_dcb_rx_conf *rx_conf = 3445 ð_conf->rx_adv_conf.dcb_rx_conf; 3446 struct rte_eth_dcb_tx_conf *tx_conf = 3447 ð_conf->tx_adv_conf.dcb_tx_conf; 3448 3449 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3450 if (rc != 0) 3451 return rc; 3452 3453 rx_conf->nb_tcs = num_tcs; 3454 tx_conf->nb_tcs = num_tcs; 3455 3456 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3457 rx_conf->dcb_tc[i] = i % num_tcs; 3458 tx_conf->dcb_tc[i] = i % num_tcs; 3459 } 3460 3461 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 3462 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3463 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3464 } 3465 3466 if (pfc_en) 3467 eth_conf->dcb_capability_en = 3468 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3469 else 3470 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3471 3472 return 0; 3473 } 3474 3475 int 3476 init_port_dcb_config(portid_t pid, 3477 enum dcb_mode_enable dcb_mode, 3478 enum rte_eth_nb_tcs num_tcs, 3479 uint8_t pfc_en) 3480 { 3481 struct rte_eth_conf port_conf; 3482 struct rte_port *rte_port; 3483 int retval; 3484 uint16_t i; 3485 3486 rte_port = &ports[pid]; 3487 3488 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3489 /* Enter DCB configuration status */ 3490 dcb_config = 1; 3491 3492 port_conf.rxmode = rte_port->dev_conf.rxmode; 3493 port_conf.txmode = rte_port->dev_conf.txmode; 3494 3495 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3496 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3497 if (retval < 0) 3498 return retval; 3499 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3500 3501 /* re-configure the device . */ 3502 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3503 if (retval < 0) 3504 return retval; 3505 3506 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3507 if (retval != 0) 3508 return retval; 3509 3510 /* If dev_info.vmdq_pool_base is greater than 0, 3511 * the queue id of vmdq pools is started after pf queues. 3512 */ 3513 if (dcb_mode == DCB_VT_ENABLED && 3514 rte_port->dev_info.vmdq_pool_base > 0) { 3515 printf("VMDQ_DCB multi-queue mode is nonsensical" 3516 " for port %d.", pid); 3517 return -1; 3518 } 3519 3520 /* Assume the ports in testpmd have the same dcb capability 3521 * and has the same number of rxq and txq in dcb mode 3522 */ 3523 if (dcb_mode == DCB_VT_ENABLED) { 3524 if (rte_port->dev_info.max_vfs > 0) { 3525 nb_rxq = rte_port->dev_info.nb_rx_queues; 3526 nb_txq = rte_port->dev_info.nb_tx_queues; 3527 } else { 3528 nb_rxq = rte_port->dev_info.max_rx_queues; 3529 nb_txq = rte_port->dev_info.max_tx_queues; 3530 } 3531 } else { 3532 /*if vt is disabled, use all pf queues */ 3533 if (rte_port->dev_info.vmdq_pool_base == 0) { 3534 nb_rxq = rte_port->dev_info.max_rx_queues; 3535 nb_txq = rte_port->dev_info.max_tx_queues; 3536 } else { 3537 nb_rxq = (queueid_t)num_tcs; 3538 nb_txq = (queueid_t)num_tcs; 3539 3540 } 3541 } 3542 rx_free_thresh = 64; 3543 3544 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3545 3546 rxtx_port_config(rte_port); 3547 /* VLAN filter */ 3548 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3549 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3550 rx_vft_set(pid, vlan_tags[i], 1); 3551 3552 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3553 if (retval != 0) 3554 return retval; 3555 3556 map_port_queue_stats_mapping_registers(pid, rte_port); 3557 3558 rte_port->dcb_flag = 1; 3559 3560 return 0; 3561 } 3562 3563 static void 3564 init_port(void) 3565 { 3566 /* Configuration of Ethernet ports. */ 3567 ports = rte_zmalloc("testpmd: ports", 3568 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3569 RTE_CACHE_LINE_SIZE); 3570 if (ports == NULL) { 3571 rte_exit(EXIT_FAILURE, 3572 "rte_zmalloc(%d struct rte_port) failed\n", 3573 RTE_MAX_ETHPORTS); 3574 } 3575 3576 /* Initialize ports NUMA structures */ 3577 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3578 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3579 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3580 } 3581 3582 static void 3583 force_quit(void) 3584 { 3585 pmd_test_exit(); 3586 prompt_exit(); 3587 } 3588 3589 static void 3590 print_stats(void) 3591 { 3592 uint8_t i; 3593 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3594 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3595 3596 /* Clear screen and move to top left */ 3597 printf("%s%s", clr, top_left); 3598 3599 printf("\nPort statistics ===================================="); 3600 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3601 nic_stats_display(fwd_ports_ids[i]); 3602 3603 fflush(stdout); 3604 } 3605 3606 static void 3607 signal_handler(int signum) 3608 { 3609 if (signum == SIGINT || signum == SIGTERM) { 3610 printf("\nSignal %d received, preparing to exit...\n", 3611 signum); 3612 #ifdef RTE_LIBRTE_PDUMP 3613 /* uninitialize packet capture framework */ 3614 rte_pdump_uninit(); 3615 #endif 3616 #ifdef RTE_LIBRTE_LATENCY_STATS 3617 if (latencystats_enabled != 0) 3618 rte_latencystats_uninit(); 3619 #endif 3620 force_quit(); 3621 /* Set flag to indicate the force termination. */ 3622 f_quit = 1; 3623 /* exit with the expected status */ 3624 signal(signum, SIG_DFL); 3625 kill(getpid(), signum); 3626 } 3627 } 3628 3629 int 3630 main(int argc, char** argv) 3631 { 3632 int diag; 3633 portid_t port_id; 3634 uint16_t count; 3635 int ret; 3636 3637 signal(SIGINT, signal_handler); 3638 signal(SIGTERM, signal_handler); 3639 3640 testpmd_logtype = rte_log_register("testpmd"); 3641 if (testpmd_logtype < 0) 3642 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3643 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3644 3645 diag = rte_eal_init(argc, argv); 3646 if (diag < 0) 3647 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3648 rte_strerror(rte_errno)); 3649 3650 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3651 rte_exit(EXIT_FAILURE, 3652 "Secondary process type not supported.\n"); 3653 3654 ret = register_eth_event_callback(); 3655 if (ret != 0) 3656 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3657 3658 #ifdef RTE_LIBRTE_PDUMP 3659 /* initialize packet capture framework */ 3660 rte_pdump_init(); 3661 #endif 3662 3663 count = 0; 3664 RTE_ETH_FOREACH_DEV(port_id) { 3665 ports_ids[count] = port_id; 3666 count++; 3667 } 3668 nb_ports = (portid_t) count; 3669 if (nb_ports == 0) 3670 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3671 3672 /* allocate port structures, and init them */ 3673 init_port(); 3674 3675 set_def_fwd_config(); 3676 if (nb_lcores == 0) 3677 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3678 "Check the core mask argument\n"); 3679 3680 /* Bitrate/latency stats disabled by default */ 3681 #ifdef RTE_LIBRTE_BITRATE 3682 bitrate_enabled = 0; 3683 #endif 3684 #ifdef RTE_LIBRTE_LATENCY_STATS 3685 latencystats_enabled = 0; 3686 #endif 3687 3688 /* on FreeBSD, mlockall() is disabled by default */ 3689 #ifdef RTE_EXEC_ENV_FREEBSD 3690 do_mlockall = 0; 3691 #else 3692 do_mlockall = 1; 3693 #endif 3694 3695 argc -= diag; 3696 argv += diag; 3697 if (argc > 1) 3698 launch_args_parse(argc, argv); 3699 3700 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3701 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3702 strerror(errno)); 3703 } 3704 3705 if (tx_first && interactive) 3706 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3707 "interactive mode.\n"); 3708 3709 if (tx_first && lsc_interrupt) { 3710 printf("Warning: lsc_interrupt needs to be off when " 3711 " using tx_first. Disabling.\n"); 3712 lsc_interrupt = 0; 3713 } 3714 3715 if (!nb_rxq && !nb_txq) 3716 printf("Warning: Either rx or tx queues should be non-zero\n"); 3717 3718 if (nb_rxq > 1 && nb_rxq > nb_txq) 3719 printf("Warning: nb_rxq=%d enables RSS configuration, " 3720 "but nb_txq=%d will prevent to fully test it.\n", 3721 nb_rxq, nb_txq); 3722 3723 init_config(); 3724 3725 if (hot_plug) { 3726 ret = rte_dev_hotplug_handle_enable(); 3727 if (ret) { 3728 RTE_LOG(ERR, EAL, 3729 "fail to enable hotplug handling."); 3730 return -1; 3731 } 3732 3733 ret = rte_dev_event_monitor_start(); 3734 if (ret) { 3735 RTE_LOG(ERR, EAL, 3736 "fail to start device event monitoring."); 3737 return -1; 3738 } 3739 3740 ret = rte_dev_event_callback_register(NULL, 3741 dev_event_callback, NULL); 3742 if (ret) { 3743 RTE_LOG(ERR, EAL, 3744 "fail to register device event callback\n"); 3745 return -1; 3746 } 3747 } 3748 3749 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3750 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3751 3752 /* set all ports to promiscuous mode by default */ 3753 RTE_ETH_FOREACH_DEV(port_id) { 3754 ret = rte_eth_promiscuous_enable(port_id); 3755 if (ret != 0) 3756 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3757 port_id, rte_strerror(-ret)); 3758 } 3759 3760 /* Init metrics library */ 3761 rte_metrics_init(rte_socket_id()); 3762 3763 #ifdef RTE_LIBRTE_LATENCY_STATS 3764 if (latencystats_enabled != 0) { 3765 int ret = rte_latencystats_init(1, NULL); 3766 if (ret) 3767 printf("Warning: latencystats init()" 3768 " returned error %d\n", ret); 3769 printf("Latencystats running on lcore %d\n", 3770 latencystats_lcore_id); 3771 } 3772 #endif 3773 3774 /* Setup bitrate stats */ 3775 #ifdef RTE_LIBRTE_BITRATE 3776 if (bitrate_enabled != 0) { 3777 bitrate_data = rte_stats_bitrate_create(); 3778 if (bitrate_data == NULL) 3779 rte_exit(EXIT_FAILURE, 3780 "Could not allocate bitrate data.\n"); 3781 rte_stats_bitrate_reg(bitrate_data); 3782 } 3783 #endif 3784 3785 #ifdef RTE_LIBRTE_CMDLINE 3786 if (strlen(cmdline_filename) != 0) 3787 cmdline_read_from_file(cmdline_filename); 3788 3789 if (interactive == 1) { 3790 if (auto_start) { 3791 printf("Start automatic packet forwarding\n"); 3792 start_packet_forwarding(0); 3793 } 3794 prompt(); 3795 pmd_test_exit(); 3796 } else 3797 #endif 3798 { 3799 char c; 3800 int rc; 3801 3802 f_quit = 0; 3803 3804 printf("No commandline core given, start packet forwarding\n"); 3805 start_packet_forwarding(tx_first); 3806 if (stats_period != 0) { 3807 uint64_t prev_time = 0, cur_time, diff_time = 0; 3808 uint64_t timer_period; 3809 3810 /* Convert to number of cycles */ 3811 timer_period = stats_period * rte_get_timer_hz(); 3812 3813 while (f_quit == 0) { 3814 cur_time = rte_get_timer_cycles(); 3815 diff_time += cur_time - prev_time; 3816 3817 if (diff_time >= timer_period) { 3818 print_stats(); 3819 /* Reset the timer */ 3820 diff_time = 0; 3821 } 3822 /* Sleep to avoid unnecessary checks */ 3823 prev_time = cur_time; 3824 sleep(1); 3825 } 3826 } 3827 3828 printf("Press enter to exit\n"); 3829 rc = read(0, &c, 1); 3830 pmd_test_exit(); 3831 if (rc < 0) 3832 return 1; 3833 } 3834 3835 ret = rte_eal_cleanup(); 3836 if (ret != 0) 3837 rte_exit(EXIT_FAILURE, 3838 "EAL cleanup failed: %s\n", strerror(-ret)); 3839 3840 return EXIT_SUCCESS; 3841 } 3842