1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 192 uint16_t mempool_flags; 193 194 struct fwd_config cur_fwd_config; 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 196 uint32_t retry_enabled; 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 199 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 201 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 202 * specified on command-line. */ 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 204 205 /* 206 * In container, it cannot terminate the process which running with 'stats-period' 207 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 208 */ 209 uint8_t f_quit; 210 211 /* 212 * Configuration of packet segments used by the "txonly" processing engine. 213 */ 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 216 TXONLY_DEF_PACKET_LEN, 217 }; 218 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 219 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 221 /**< Split policy for packets to TX. */ 222 223 uint8_t txonly_multi_flow; 224 /**< Whether multiple flows are generated in TXONLY mode. */ 225 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 228 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 230 uint8_t dcb_config = 0; 231 232 /* Whether the dcb is in testing status */ 233 uint8_t dcb_test = 0; 234 235 /* 236 * Configurable number of RX/TX queues. 237 */ 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 240 241 /* 242 * Configurable number of RX/TX ring descriptors. 243 * Defaults are supplied by drivers via ethdev. 244 */ 245 #define RTE_TEST_RX_DESC_DEFAULT 0 246 #define RTE_TEST_TX_DESC_DEFAULT 0 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 249 250 #define RTE_PMD_PARAM_UNSET -1 251 /* 252 * Configurable values of RX and TX ring threshold registers. 253 */ 254 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 258 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 262 263 /* 264 * Configurable value of RX free threshold. 265 */ 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX drop enable. 270 */ 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of TX free threshold. 275 */ 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX RS bit threshold. 280 */ 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of buffered packets before sending. 285 */ 286 uint16_t noisy_tx_sw_bufsz; 287 288 /* 289 * Configurable value of packet buffer timeout. 290 */ 291 uint16_t noisy_tx_sw_buf_flush_time; 292 293 /* 294 * Configurable value for size of VNF internal memory area 295 * used for simulating noisy neighbour behaviour 296 */ 297 uint64_t noisy_lkup_mem_sz; 298 299 /* 300 * Configurable value of number of random writes done in 301 * VNF simulation memory area. 302 */ 303 uint64_t noisy_lkup_num_writes; 304 305 /* 306 * Configurable value of number of random reads done in 307 * VNF simulation memory area. 308 */ 309 uint64_t noisy_lkup_num_reads; 310 311 /* 312 * Configurable value of number of random reads/writes done in 313 * VNF simulation memory area. 314 */ 315 uint64_t noisy_lkup_num_reads_writes; 316 317 /* 318 * Receive Side Scaling (RSS) configuration. 319 */ 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 321 322 /* 323 * Port topology configuration 324 */ 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 326 327 /* 328 * Avoids to flush all the RX streams before starts forwarding. 329 */ 330 uint8_t no_flush_rx = 0; /* flush by default */ 331 332 /* 333 * Flow API isolated mode. 334 */ 335 uint8_t flow_isolate_all; 336 337 /* 338 * Avoids to check link status when starting/stopping a port. 339 */ 340 uint8_t no_link_check = 0; /* check by default */ 341 342 /* 343 * Enable link status change notification 344 */ 345 uint8_t lsc_interrupt = 1; /* enabled by default */ 346 347 /* 348 * Enable device removal notification. 349 */ 350 uint8_t rmv_interrupt = 1; /* enabled by default */ 351 352 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 353 354 /* After attach, port setup is called on event or by iterator */ 355 bool setup_on_probe_event = true; 356 357 /* Pretty printing of ethdev events */ 358 static const char * const eth_event_desc[] = { 359 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 360 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 361 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 362 [RTE_ETH_EVENT_INTR_RESET] = "reset", 363 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 364 [RTE_ETH_EVENT_IPSEC] = "IPsec", 365 [RTE_ETH_EVENT_MACSEC] = "MACsec", 366 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 367 [RTE_ETH_EVENT_NEW] = "device probed", 368 [RTE_ETH_EVENT_DESTROY] = "device released", 369 [RTE_ETH_EVENT_MAX] = NULL, 370 }; 371 372 /* 373 * Display or mask ether events 374 * Default to all events except VF_MBOX 375 */ 376 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 377 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 378 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 379 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 380 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 381 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 382 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 383 /* 384 * Decide if all memory are locked for performance. 385 */ 386 int do_mlockall = 0; 387 388 /* 389 * NIC bypass mode configuration options. 390 */ 391 392 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 393 /* The NIC bypass watchdog timeout. */ 394 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 395 #endif 396 397 398 #ifdef RTE_LIBRTE_LATENCY_STATS 399 400 /* 401 * Set when latency stats is enabled in the commandline 402 */ 403 uint8_t latencystats_enabled; 404 405 /* 406 * Lcore ID to serive latency statistics. 407 */ 408 lcoreid_t latencystats_lcore_id = -1; 409 410 #endif 411 412 /* 413 * Ethernet device configuration. 414 */ 415 struct rte_eth_rxmode rx_mode = { 416 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 417 /**< Default maximum frame length. */ 418 }; 419 420 struct rte_eth_txmode tx_mode = { 421 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 422 }; 423 424 struct rte_fdir_conf fdir_conf = { 425 .mode = RTE_FDIR_MODE_NONE, 426 .pballoc = RTE_FDIR_PBALLOC_64K, 427 .status = RTE_FDIR_REPORT_STATUS, 428 .mask = { 429 .vlan_tci_mask = 0xFFEF, 430 .ipv4_mask = { 431 .src_ip = 0xFFFFFFFF, 432 .dst_ip = 0xFFFFFFFF, 433 }, 434 .ipv6_mask = { 435 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 436 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 437 }, 438 .src_port_mask = 0xFFFF, 439 .dst_port_mask = 0xFFFF, 440 .mac_addr_byte_mask = 0xFF, 441 .tunnel_type_mask = 1, 442 .tunnel_id_mask = 0xFFFFFFFF, 443 }, 444 .drop_queue = 127, 445 }; 446 447 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 448 449 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 450 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 451 452 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 453 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 454 455 uint16_t nb_tx_queue_stats_mappings = 0; 456 uint16_t nb_rx_queue_stats_mappings = 0; 457 458 /* 459 * Display zero values by default for xstats 460 */ 461 uint8_t xstats_hide_zero; 462 463 unsigned int num_sockets = 0; 464 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 465 466 #ifdef RTE_LIBRTE_BITRATE 467 /* Bitrate statistics */ 468 struct rte_stats_bitrates *bitrate_data; 469 lcoreid_t bitrate_lcore_id; 470 uint8_t bitrate_enabled; 471 #endif 472 473 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 474 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 475 476 struct vxlan_encap_conf vxlan_encap_conf = { 477 .select_ipv4 = 1, 478 .select_vlan = 0, 479 .select_tos_ttl = 0, 480 .vni = "\x00\x00\x00", 481 .udp_src = 0, 482 .udp_dst = RTE_BE16(4789), 483 .ipv4_src = RTE_IPV4(127, 0, 0, 1), 484 .ipv4_dst = RTE_IPV4(255, 255, 255, 255), 485 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 486 "\x00\x00\x00\x00\x00\x00\x00\x01", 487 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 488 "\x00\x00\x00\x00\x00\x00\x11\x11", 489 .vlan_tci = 0, 490 .ip_tos = 0, 491 .ip_ttl = 255, 492 .eth_src = "\x00\x00\x00\x00\x00\x00", 493 .eth_dst = "\xff\xff\xff\xff\xff\xff", 494 }; 495 496 struct nvgre_encap_conf nvgre_encap_conf = { 497 .select_ipv4 = 1, 498 .select_vlan = 0, 499 .tni = "\x00\x00\x00", 500 .ipv4_src = RTE_IPV4(127, 0, 0, 1), 501 .ipv4_dst = RTE_IPV4(255, 255, 255, 255), 502 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 503 "\x00\x00\x00\x00\x00\x00\x00\x01", 504 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 505 "\x00\x00\x00\x00\x00\x00\x11\x11", 506 .vlan_tci = 0, 507 .eth_src = "\x00\x00\x00\x00\x00\x00", 508 .eth_dst = "\xff\xff\xff\xff\xff\xff", 509 }; 510 511 /* Forward function declarations */ 512 static void setup_attached_port(portid_t pi); 513 static void map_port_queue_stats_mapping_registers(portid_t pi, 514 struct rte_port *port); 515 static void check_all_ports_link_status(uint32_t port_mask); 516 static int eth_event_callback(portid_t port_id, 517 enum rte_eth_event_type type, 518 void *param, void *ret_param); 519 static void dev_event_callback(const char *device_name, 520 enum rte_dev_event_type type, 521 void *param); 522 523 /* 524 * Check if all the ports are started. 525 * If yes, return positive value. If not, return zero. 526 */ 527 static int all_ports_started(void); 528 529 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 530 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 531 532 /* 533 * Helper function to check if socket is already discovered. 534 * If yes, return positive value. If not, return zero. 535 */ 536 int 537 new_socket_id(unsigned int socket_id) 538 { 539 unsigned int i; 540 541 for (i = 0; i < num_sockets; i++) { 542 if (socket_ids[i] == socket_id) 543 return 0; 544 } 545 return 1; 546 } 547 548 /* 549 * Setup default configuration. 550 */ 551 static void 552 set_default_fwd_lcores_config(void) 553 { 554 unsigned int i; 555 unsigned int nb_lc; 556 unsigned int sock_num; 557 558 nb_lc = 0; 559 for (i = 0; i < RTE_MAX_LCORE; i++) { 560 if (!rte_lcore_is_enabled(i)) 561 continue; 562 sock_num = rte_lcore_to_socket_id(i); 563 if (new_socket_id(sock_num)) { 564 if (num_sockets >= RTE_MAX_NUMA_NODES) { 565 rte_exit(EXIT_FAILURE, 566 "Total sockets greater than %u\n", 567 RTE_MAX_NUMA_NODES); 568 } 569 socket_ids[num_sockets++] = sock_num; 570 } 571 if (i == rte_get_master_lcore()) 572 continue; 573 fwd_lcores_cpuids[nb_lc++] = i; 574 } 575 nb_lcores = (lcoreid_t) nb_lc; 576 nb_cfg_lcores = nb_lcores; 577 nb_fwd_lcores = 1; 578 } 579 580 static void 581 set_def_peer_eth_addrs(void) 582 { 583 portid_t i; 584 585 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 586 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 587 peer_eth_addrs[i].addr_bytes[5] = i; 588 } 589 } 590 591 static void 592 set_default_fwd_ports_config(void) 593 { 594 portid_t pt_id; 595 int i = 0; 596 597 RTE_ETH_FOREACH_DEV(pt_id) { 598 fwd_ports_ids[i++] = pt_id; 599 600 /* Update sockets info according to the attached device */ 601 int socket_id = rte_eth_dev_socket_id(pt_id); 602 if (socket_id >= 0 && new_socket_id(socket_id)) { 603 if (num_sockets >= RTE_MAX_NUMA_NODES) { 604 rte_exit(EXIT_FAILURE, 605 "Total sockets greater than %u\n", 606 RTE_MAX_NUMA_NODES); 607 } 608 socket_ids[num_sockets++] = socket_id; 609 } 610 } 611 612 nb_cfg_ports = nb_ports; 613 nb_fwd_ports = nb_ports; 614 } 615 616 void 617 set_def_fwd_config(void) 618 { 619 set_default_fwd_lcores_config(); 620 set_def_peer_eth_addrs(); 621 set_default_fwd_ports_config(); 622 } 623 624 /* extremely pessimistic estimation of memory required to create a mempool */ 625 static int 626 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 627 { 628 unsigned int n_pages, mbuf_per_pg, leftover; 629 uint64_t total_mem, mbuf_mem, obj_sz; 630 631 /* there is no good way to predict how much space the mempool will 632 * occupy because it will allocate chunks on the fly, and some of those 633 * will come from default DPDK memory while some will come from our 634 * external memory, so just assume 128MB will be enough for everyone. 635 */ 636 uint64_t hdr_mem = 128 << 20; 637 638 /* account for possible non-contiguousness */ 639 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 640 if (obj_sz > pgsz) { 641 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 642 return -1; 643 } 644 645 mbuf_per_pg = pgsz / obj_sz; 646 leftover = (nb_mbufs % mbuf_per_pg) > 0; 647 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 648 649 mbuf_mem = n_pages * pgsz; 650 651 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 652 653 if (total_mem > SIZE_MAX) { 654 TESTPMD_LOG(ERR, "Memory size too big\n"); 655 return -1; 656 } 657 *out = (size_t)total_mem; 658 659 return 0; 660 } 661 662 static int 663 pagesz_flags(uint64_t page_sz) 664 { 665 /* as per mmap() manpage, all page sizes are log2 of page size 666 * shifted by MAP_HUGE_SHIFT 667 */ 668 int log2 = rte_log2_u64(page_sz); 669 670 return (log2 << HUGE_SHIFT); 671 } 672 673 static void * 674 alloc_mem(size_t memsz, size_t pgsz, bool huge) 675 { 676 void *addr; 677 int flags; 678 679 /* allocate anonymous hugepages */ 680 flags = MAP_ANONYMOUS | MAP_PRIVATE; 681 if (huge) 682 flags |= HUGE_FLAG | pagesz_flags(pgsz); 683 684 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 685 if (addr == MAP_FAILED) 686 return NULL; 687 688 return addr; 689 } 690 691 struct extmem_param { 692 void *addr; 693 size_t len; 694 size_t pgsz; 695 rte_iova_t *iova_table; 696 unsigned int iova_table_len; 697 }; 698 699 static int 700 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 701 bool huge) 702 { 703 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 704 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 705 unsigned int cur_page, n_pages, pgsz_idx; 706 size_t mem_sz, cur_pgsz; 707 rte_iova_t *iovas = NULL; 708 void *addr; 709 int ret; 710 711 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 712 /* skip anything that is too big */ 713 if (pgsizes[pgsz_idx] > SIZE_MAX) 714 continue; 715 716 cur_pgsz = pgsizes[pgsz_idx]; 717 718 /* if we were told not to allocate hugepages, override */ 719 if (!huge) 720 cur_pgsz = sysconf(_SC_PAGESIZE); 721 722 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 723 if (ret < 0) { 724 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 725 return -1; 726 } 727 728 /* allocate our memory */ 729 addr = alloc_mem(mem_sz, cur_pgsz, huge); 730 731 /* if we couldn't allocate memory with a specified page size, 732 * that doesn't mean we can't do it with other page sizes, so 733 * try another one. 734 */ 735 if (addr == NULL) 736 continue; 737 738 /* store IOVA addresses for every page in this memory area */ 739 n_pages = mem_sz / cur_pgsz; 740 741 iovas = malloc(sizeof(*iovas) * n_pages); 742 743 if (iovas == NULL) { 744 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 745 goto fail; 746 } 747 /* lock memory if it's not huge pages */ 748 if (!huge) 749 mlock(addr, mem_sz); 750 751 /* populate IOVA addresses */ 752 for (cur_page = 0; cur_page < n_pages; cur_page++) { 753 rte_iova_t iova; 754 size_t offset; 755 void *cur; 756 757 offset = cur_pgsz * cur_page; 758 cur = RTE_PTR_ADD(addr, offset); 759 760 /* touch the page before getting its IOVA */ 761 *(volatile char *)cur = 0; 762 763 iova = rte_mem_virt2iova(cur); 764 765 iovas[cur_page] = iova; 766 } 767 768 break; 769 } 770 /* if we couldn't allocate anything */ 771 if (iovas == NULL) 772 return -1; 773 774 param->addr = addr; 775 param->len = mem_sz; 776 param->pgsz = cur_pgsz; 777 param->iova_table = iovas; 778 param->iova_table_len = n_pages; 779 780 return 0; 781 fail: 782 if (iovas) 783 free(iovas); 784 if (addr) 785 munmap(addr, mem_sz); 786 787 return -1; 788 } 789 790 static int 791 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 792 { 793 struct extmem_param param; 794 int socket_id, ret; 795 796 memset(¶m, 0, sizeof(param)); 797 798 /* check if our heap exists */ 799 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 800 if (socket_id < 0) { 801 /* create our heap */ 802 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 803 if (ret < 0) { 804 TESTPMD_LOG(ERR, "Cannot create heap\n"); 805 return -1; 806 } 807 } 808 809 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 810 if (ret < 0) { 811 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 812 return -1; 813 } 814 815 /* we now have a valid memory area, so add it to heap */ 816 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 817 param.addr, param.len, param.iova_table, 818 param.iova_table_len, param.pgsz); 819 820 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 821 822 /* not needed any more */ 823 free(param.iova_table); 824 825 if (ret < 0) { 826 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 827 munmap(param.addr, param.len); 828 return -1; 829 } 830 831 /* success */ 832 833 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 834 param.len >> 20); 835 836 return 0; 837 } 838 static void 839 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 840 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 841 { 842 uint16_t pid = 0; 843 int ret; 844 845 RTE_ETH_FOREACH_DEV(pid) { 846 struct rte_eth_dev *dev = 847 &rte_eth_devices[pid]; 848 849 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 850 memhdr->len); 851 if (ret) { 852 TESTPMD_LOG(DEBUG, 853 "unable to DMA unmap addr 0x%p " 854 "for device %s\n", 855 memhdr->addr, dev->data->name); 856 } 857 } 858 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 859 if (ret) { 860 TESTPMD_LOG(DEBUG, 861 "unable to un-register addr 0x%p\n", memhdr->addr); 862 } 863 } 864 865 static void 866 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 867 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 868 { 869 uint16_t pid = 0; 870 size_t page_size = sysconf(_SC_PAGESIZE); 871 int ret; 872 873 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 874 page_size); 875 if (ret) { 876 TESTPMD_LOG(DEBUG, 877 "unable to register addr 0x%p\n", memhdr->addr); 878 return; 879 } 880 RTE_ETH_FOREACH_DEV(pid) { 881 struct rte_eth_dev *dev = 882 &rte_eth_devices[pid]; 883 884 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 885 memhdr->len); 886 if (ret) { 887 TESTPMD_LOG(DEBUG, 888 "unable to DMA map addr 0x%p " 889 "for device %s\n", 890 memhdr->addr, dev->data->name); 891 } 892 } 893 } 894 895 /* 896 * Configuration initialisation done once at init time. 897 */ 898 static struct rte_mempool * 899 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 900 unsigned int socket_id) 901 { 902 char pool_name[RTE_MEMPOOL_NAMESIZE]; 903 struct rte_mempool *rte_mp = NULL; 904 uint32_t mb_size; 905 906 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 907 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 908 909 TESTPMD_LOG(INFO, 910 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 911 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 912 913 switch (mp_alloc_type) { 914 case MP_ALLOC_NATIVE: 915 { 916 /* wrapper to rte_mempool_create() */ 917 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 918 rte_mbuf_best_mempool_ops()); 919 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 920 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 921 break; 922 } 923 case MP_ALLOC_ANON: 924 { 925 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 926 mb_size, (unsigned int) mb_mempool_cache, 927 sizeof(struct rte_pktmbuf_pool_private), 928 socket_id, mempool_flags); 929 if (rte_mp == NULL) 930 goto err; 931 932 if (rte_mempool_populate_anon(rte_mp) == 0) { 933 rte_mempool_free(rte_mp); 934 rte_mp = NULL; 935 goto err; 936 } 937 rte_pktmbuf_pool_init(rte_mp, NULL); 938 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 939 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 940 break; 941 } 942 case MP_ALLOC_XMEM: 943 case MP_ALLOC_XMEM_HUGE: 944 { 945 int heap_socket; 946 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 947 948 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 949 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 950 951 heap_socket = 952 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 953 if (heap_socket < 0) 954 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 955 956 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 957 rte_mbuf_best_mempool_ops()); 958 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 959 mb_mempool_cache, 0, mbuf_seg_size, 960 heap_socket); 961 break; 962 } 963 default: 964 { 965 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 966 } 967 } 968 969 err: 970 if (rte_mp == NULL) { 971 rte_exit(EXIT_FAILURE, 972 "Creation of mbuf pool for socket %u failed: %s\n", 973 socket_id, rte_strerror(rte_errno)); 974 } else if (verbose_level > 0) { 975 rte_mempool_dump(stdout, rte_mp); 976 } 977 return rte_mp; 978 } 979 980 /* 981 * Check given socket id is valid or not with NUMA mode, 982 * if valid, return 0, else return -1 983 */ 984 static int 985 check_socket_id(const unsigned int socket_id) 986 { 987 static int warning_once = 0; 988 989 if (new_socket_id(socket_id)) { 990 if (!warning_once && numa_support) 991 printf("Warning: NUMA should be configured manually by" 992 " using --port-numa-config and" 993 " --ring-numa-config parameters along with" 994 " --numa.\n"); 995 warning_once = 1; 996 return -1; 997 } 998 return 0; 999 } 1000 1001 /* 1002 * Get the allowed maximum number of RX queues. 1003 * *pid return the port id which has minimal value of 1004 * max_rx_queues in all ports. 1005 */ 1006 queueid_t 1007 get_allowed_max_nb_rxq(portid_t *pid) 1008 { 1009 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 1010 portid_t pi; 1011 struct rte_eth_dev_info dev_info; 1012 1013 RTE_ETH_FOREACH_DEV(pi) { 1014 rte_eth_dev_info_get(pi, &dev_info); 1015 if (dev_info.max_rx_queues < allowed_max_rxq) { 1016 allowed_max_rxq = dev_info.max_rx_queues; 1017 *pid = pi; 1018 } 1019 } 1020 return allowed_max_rxq; 1021 } 1022 1023 /* 1024 * Check input rxq is valid or not. 1025 * If input rxq is not greater than any of maximum number 1026 * of RX queues of all ports, it is valid. 1027 * if valid, return 0, else return -1 1028 */ 1029 int 1030 check_nb_rxq(queueid_t rxq) 1031 { 1032 queueid_t allowed_max_rxq; 1033 portid_t pid = 0; 1034 1035 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1036 if (rxq > allowed_max_rxq) { 1037 printf("Fail: input rxq (%u) can't be greater " 1038 "than max_rx_queues (%u) of port %u\n", 1039 rxq, 1040 allowed_max_rxq, 1041 pid); 1042 return -1; 1043 } 1044 return 0; 1045 } 1046 1047 /* 1048 * Get the allowed maximum number of TX queues. 1049 * *pid return the port id which has minimal value of 1050 * max_tx_queues in all ports. 1051 */ 1052 queueid_t 1053 get_allowed_max_nb_txq(portid_t *pid) 1054 { 1055 queueid_t allowed_max_txq = MAX_QUEUE_ID; 1056 portid_t pi; 1057 struct rte_eth_dev_info dev_info; 1058 1059 RTE_ETH_FOREACH_DEV(pi) { 1060 rte_eth_dev_info_get(pi, &dev_info); 1061 if (dev_info.max_tx_queues < allowed_max_txq) { 1062 allowed_max_txq = dev_info.max_tx_queues; 1063 *pid = pi; 1064 } 1065 } 1066 return allowed_max_txq; 1067 } 1068 1069 /* 1070 * Check input txq is valid or not. 1071 * If input txq is not greater than any of maximum number 1072 * of TX queues of all ports, it is valid. 1073 * if valid, return 0, else return -1 1074 */ 1075 int 1076 check_nb_txq(queueid_t txq) 1077 { 1078 queueid_t allowed_max_txq; 1079 portid_t pid = 0; 1080 1081 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1082 if (txq > allowed_max_txq) { 1083 printf("Fail: input txq (%u) can't be greater " 1084 "than max_tx_queues (%u) of port %u\n", 1085 txq, 1086 allowed_max_txq, 1087 pid); 1088 return -1; 1089 } 1090 return 0; 1091 } 1092 1093 static void 1094 init_config(void) 1095 { 1096 portid_t pid; 1097 struct rte_port *port; 1098 struct rte_mempool *mbp; 1099 unsigned int nb_mbuf_per_pool; 1100 lcoreid_t lc_id; 1101 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1102 struct rte_gro_param gro_param; 1103 uint32_t gso_types; 1104 uint16_t data_size; 1105 bool warning = 0; 1106 int k; 1107 1108 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1109 1110 /* Configuration of logical cores. */ 1111 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1112 sizeof(struct fwd_lcore *) * nb_lcores, 1113 RTE_CACHE_LINE_SIZE); 1114 if (fwd_lcores == NULL) { 1115 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1116 "failed\n", nb_lcores); 1117 } 1118 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1119 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1120 sizeof(struct fwd_lcore), 1121 RTE_CACHE_LINE_SIZE); 1122 if (fwd_lcores[lc_id] == NULL) { 1123 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1124 "failed\n"); 1125 } 1126 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1127 } 1128 1129 RTE_ETH_FOREACH_DEV(pid) { 1130 port = &ports[pid]; 1131 /* Apply default TxRx configuration for all ports */ 1132 port->dev_conf.txmode = tx_mode; 1133 port->dev_conf.rxmode = rx_mode; 1134 rte_eth_dev_info_get(pid, &port->dev_info); 1135 1136 if (!(port->dev_info.tx_offload_capa & 1137 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1138 port->dev_conf.txmode.offloads &= 1139 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1140 if (!(port->dev_info.tx_offload_capa & 1141 DEV_TX_OFFLOAD_MATCH_METADATA)) 1142 port->dev_conf.txmode.offloads &= 1143 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1144 if (numa_support) { 1145 if (port_numa[pid] != NUMA_NO_CONFIG) 1146 port_per_socket[port_numa[pid]]++; 1147 else { 1148 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1149 1150 /* 1151 * if socket_id is invalid, 1152 * set to the first available socket. 1153 */ 1154 if (check_socket_id(socket_id) < 0) 1155 socket_id = socket_ids[0]; 1156 port_per_socket[socket_id]++; 1157 } 1158 } 1159 1160 /* Apply Rx offloads configuration */ 1161 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1162 port->rx_conf[k].offloads = 1163 port->dev_conf.rxmode.offloads; 1164 /* Apply Tx offloads configuration */ 1165 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1166 port->tx_conf[k].offloads = 1167 port->dev_conf.txmode.offloads; 1168 1169 /* set flag to initialize port/queue */ 1170 port->need_reconfig = 1; 1171 port->need_reconfig_queues = 1; 1172 port->tx_metadata = 0; 1173 1174 /* Check for maximum number of segments per MTU. Accordingly 1175 * update the mbuf data size. 1176 */ 1177 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1178 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1179 data_size = rx_mode.max_rx_pkt_len / 1180 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1181 1182 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1183 mbuf_data_size) { 1184 mbuf_data_size = data_size + 1185 RTE_PKTMBUF_HEADROOM; 1186 warning = 1; 1187 } 1188 } 1189 } 1190 1191 if (warning) 1192 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1193 mbuf_data_size); 1194 1195 /* 1196 * Create pools of mbuf. 1197 * If NUMA support is disabled, create a single pool of mbuf in 1198 * socket 0 memory by default. 1199 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1200 * 1201 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1202 * nb_txd can be configured at run time. 1203 */ 1204 if (param_total_num_mbufs) 1205 nb_mbuf_per_pool = param_total_num_mbufs; 1206 else { 1207 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1208 (nb_lcores * mb_mempool_cache) + 1209 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1210 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1211 } 1212 1213 if (numa_support) { 1214 uint8_t i; 1215 1216 for (i = 0; i < num_sockets; i++) 1217 mempools[i] = mbuf_pool_create(mbuf_data_size, 1218 nb_mbuf_per_pool, 1219 socket_ids[i]); 1220 } else { 1221 if (socket_num == UMA_NO_CONFIG) 1222 mempools[0] = mbuf_pool_create(mbuf_data_size, 1223 nb_mbuf_per_pool, 0); 1224 else 1225 mempools[socket_num] = mbuf_pool_create 1226 (mbuf_data_size, 1227 nb_mbuf_per_pool, 1228 socket_num); 1229 } 1230 1231 init_port_config(); 1232 1233 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1234 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1235 /* 1236 * Records which Mbuf pool to use by each logical core, if needed. 1237 */ 1238 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1239 mbp = mbuf_pool_find( 1240 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1241 1242 if (mbp == NULL) 1243 mbp = mbuf_pool_find(0); 1244 fwd_lcores[lc_id]->mbp = mbp; 1245 /* initialize GSO context */ 1246 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1247 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1248 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1249 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1250 RTE_ETHER_CRC_LEN; 1251 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1252 } 1253 1254 /* Configuration of packet forwarding streams. */ 1255 if (init_fwd_streams() < 0) 1256 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1257 1258 fwd_config_setup(); 1259 1260 /* create a gro context for each lcore */ 1261 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1262 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1263 gro_param.max_item_per_flow = MAX_PKT_BURST; 1264 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1265 gro_param.socket_id = rte_lcore_to_socket_id( 1266 fwd_lcores_cpuids[lc_id]); 1267 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1268 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1269 rte_exit(EXIT_FAILURE, 1270 "rte_gro_ctx_create() failed\n"); 1271 } 1272 } 1273 1274 #if defined RTE_LIBRTE_PMD_SOFTNIC 1275 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1276 RTE_ETH_FOREACH_DEV(pid) { 1277 port = &ports[pid]; 1278 const char *driver = port->dev_info.driver_name; 1279 1280 if (strcmp(driver, "net_softnic") == 0) 1281 port->softport.fwd_lcore_arg = fwd_lcores; 1282 } 1283 } 1284 #endif 1285 1286 } 1287 1288 1289 void 1290 reconfig(portid_t new_port_id, unsigned socket_id) 1291 { 1292 struct rte_port *port; 1293 1294 /* Reconfiguration of Ethernet ports. */ 1295 port = &ports[new_port_id]; 1296 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1297 1298 /* set flag to initialize port/queue */ 1299 port->need_reconfig = 1; 1300 port->need_reconfig_queues = 1; 1301 port->socket_id = socket_id; 1302 1303 init_port_config(); 1304 } 1305 1306 1307 int 1308 init_fwd_streams(void) 1309 { 1310 portid_t pid; 1311 struct rte_port *port; 1312 streamid_t sm_id, nb_fwd_streams_new; 1313 queueid_t q; 1314 1315 /* set socket id according to numa or not */ 1316 RTE_ETH_FOREACH_DEV(pid) { 1317 port = &ports[pid]; 1318 if (nb_rxq > port->dev_info.max_rx_queues) { 1319 printf("Fail: nb_rxq(%d) is greater than " 1320 "max_rx_queues(%d)\n", nb_rxq, 1321 port->dev_info.max_rx_queues); 1322 return -1; 1323 } 1324 if (nb_txq > port->dev_info.max_tx_queues) { 1325 printf("Fail: nb_txq(%d) is greater than " 1326 "max_tx_queues(%d)\n", nb_txq, 1327 port->dev_info.max_tx_queues); 1328 return -1; 1329 } 1330 if (numa_support) { 1331 if (port_numa[pid] != NUMA_NO_CONFIG) 1332 port->socket_id = port_numa[pid]; 1333 else { 1334 port->socket_id = rte_eth_dev_socket_id(pid); 1335 1336 /* 1337 * if socket_id is invalid, 1338 * set to the first available socket. 1339 */ 1340 if (check_socket_id(port->socket_id) < 0) 1341 port->socket_id = socket_ids[0]; 1342 } 1343 } 1344 else { 1345 if (socket_num == UMA_NO_CONFIG) 1346 port->socket_id = 0; 1347 else 1348 port->socket_id = socket_num; 1349 } 1350 } 1351 1352 q = RTE_MAX(nb_rxq, nb_txq); 1353 if (q == 0) { 1354 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1355 return -1; 1356 } 1357 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1358 if (nb_fwd_streams_new == nb_fwd_streams) 1359 return 0; 1360 /* clear the old */ 1361 if (fwd_streams != NULL) { 1362 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1363 if (fwd_streams[sm_id] == NULL) 1364 continue; 1365 rte_free(fwd_streams[sm_id]); 1366 fwd_streams[sm_id] = NULL; 1367 } 1368 rte_free(fwd_streams); 1369 fwd_streams = NULL; 1370 } 1371 1372 /* init new */ 1373 nb_fwd_streams = nb_fwd_streams_new; 1374 if (nb_fwd_streams) { 1375 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1376 sizeof(struct fwd_stream *) * nb_fwd_streams, 1377 RTE_CACHE_LINE_SIZE); 1378 if (fwd_streams == NULL) 1379 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1380 " (struct fwd_stream *)) failed\n", 1381 nb_fwd_streams); 1382 1383 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1384 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1385 " struct fwd_stream", sizeof(struct fwd_stream), 1386 RTE_CACHE_LINE_SIZE); 1387 if (fwd_streams[sm_id] == NULL) 1388 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1389 "(struct fwd_stream) failed\n"); 1390 } 1391 } 1392 1393 return 0; 1394 } 1395 1396 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1397 static void 1398 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1399 { 1400 unsigned int total_burst; 1401 unsigned int nb_burst; 1402 unsigned int burst_stats[3]; 1403 uint16_t pktnb_stats[3]; 1404 uint16_t nb_pkt; 1405 int burst_percent[3]; 1406 1407 /* 1408 * First compute the total number of packet bursts and the 1409 * two highest numbers of bursts of the same number of packets. 1410 */ 1411 total_burst = 0; 1412 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1413 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1414 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1415 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1416 if (nb_burst == 0) 1417 continue; 1418 total_burst += nb_burst; 1419 if (nb_burst > burst_stats[0]) { 1420 burst_stats[1] = burst_stats[0]; 1421 pktnb_stats[1] = pktnb_stats[0]; 1422 burst_stats[0] = nb_burst; 1423 pktnb_stats[0] = nb_pkt; 1424 } else if (nb_burst > burst_stats[1]) { 1425 burst_stats[1] = nb_burst; 1426 pktnb_stats[1] = nb_pkt; 1427 } 1428 } 1429 if (total_burst == 0) 1430 return; 1431 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1432 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1433 burst_percent[0], (int) pktnb_stats[0]); 1434 if (burst_stats[0] == total_burst) { 1435 printf("]\n"); 1436 return; 1437 } 1438 if (burst_stats[0] + burst_stats[1] == total_burst) { 1439 printf(" + %d%% of %d pkts]\n", 1440 100 - burst_percent[0], pktnb_stats[1]); 1441 return; 1442 } 1443 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1444 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1445 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1446 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1447 return; 1448 } 1449 printf(" + %d%% of %d pkts + %d%% of others]\n", 1450 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1451 } 1452 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1453 1454 static void 1455 fwd_stream_stats_display(streamid_t stream_id) 1456 { 1457 struct fwd_stream *fs; 1458 static const char *fwd_top_stats_border = "-------"; 1459 1460 fs = fwd_streams[stream_id]; 1461 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1462 (fs->fwd_dropped == 0)) 1463 return; 1464 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1465 "TX Port=%2d/Queue=%2d %s\n", 1466 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1467 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1468 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1469 " TX-dropped: %-14"PRIu64, 1470 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1471 1472 /* if checksum mode */ 1473 if (cur_fwd_eng == &csum_fwd_engine) { 1474 printf(" RX- bad IP checksum: %-14"PRIu64 1475 " Rx- bad L4 checksum: %-14"PRIu64 1476 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1477 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1478 fs->rx_bad_outer_l4_csum); 1479 } else { 1480 printf("\n"); 1481 } 1482 1483 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1484 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1485 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1486 #endif 1487 } 1488 1489 void 1490 fwd_stats_display(void) 1491 { 1492 static const char *fwd_stats_border = "----------------------"; 1493 static const char *acc_stats_border = "+++++++++++++++"; 1494 struct { 1495 struct fwd_stream *rx_stream; 1496 struct fwd_stream *tx_stream; 1497 uint64_t tx_dropped; 1498 uint64_t rx_bad_ip_csum; 1499 uint64_t rx_bad_l4_csum; 1500 uint64_t rx_bad_outer_l4_csum; 1501 } ports_stats[RTE_MAX_ETHPORTS]; 1502 uint64_t total_rx_dropped = 0; 1503 uint64_t total_tx_dropped = 0; 1504 uint64_t total_rx_nombuf = 0; 1505 struct rte_eth_stats stats; 1506 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1507 uint64_t fwd_cycles = 0; 1508 #endif 1509 uint64_t total_recv = 0; 1510 uint64_t total_xmit = 0; 1511 struct rte_port *port; 1512 streamid_t sm_id; 1513 portid_t pt_id; 1514 int i; 1515 1516 memset(ports_stats, 0, sizeof(ports_stats)); 1517 1518 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1519 struct fwd_stream *fs = fwd_streams[sm_id]; 1520 1521 if (cur_fwd_config.nb_fwd_streams > 1522 cur_fwd_config.nb_fwd_ports) { 1523 fwd_stream_stats_display(sm_id); 1524 } else { 1525 ports_stats[fs->tx_port].tx_stream = fs; 1526 ports_stats[fs->rx_port].rx_stream = fs; 1527 } 1528 1529 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1530 1531 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1532 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1533 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1534 fs->rx_bad_outer_l4_csum; 1535 1536 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1537 fwd_cycles += fs->core_cycles; 1538 #endif 1539 } 1540 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1541 uint8_t j; 1542 1543 pt_id = fwd_ports_ids[i]; 1544 port = &ports[pt_id]; 1545 1546 rte_eth_stats_get(pt_id, &stats); 1547 stats.ipackets -= port->stats.ipackets; 1548 stats.opackets -= port->stats.opackets; 1549 stats.ibytes -= port->stats.ibytes; 1550 stats.obytes -= port->stats.obytes; 1551 stats.imissed -= port->stats.imissed; 1552 stats.oerrors -= port->stats.oerrors; 1553 stats.rx_nombuf -= port->stats.rx_nombuf; 1554 1555 total_recv += stats.ipackets; 1556 total_xmit += stats.opackets; 1557 total_rx_dropped += stats.imissed; 1558 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1559 total_tx_dropped += stats.oerrors; 1560 total_rx_nombuf += stats.rx_nombuf; 1561 1562 printf("\n %s Forward statistics for port %-2d %s\n", 1563 fwd_stats_border, pt_id, fwd_stats_border); 1564 1565 if (!port->rx_queue_stats_mapping_enabled && 1566 !port->tx_queue_stats_mapping_enabled) { 1567 printf(" RX-packets: %-14"PRIu64 1568 " RX-dropped: %-14"PRIu64 1569 "RX-total: %-"PRIu64"\n", 1570 stats.ipackets, stats.imissed, 1571 stats.ipackets + stats.imissed); 1572 1573 if (cur_fwd_eng == &csum_fwd_engine) 1574 printf(" Bad-ipcsum: %-14"PRIu64 1575 " Bad-l4csum: %-14"PRIu64 1576 "Bad-outer-l4csum: %-14"PRIu64"\n", 1577 ports_stats[pt_id].rx_bad_ip_csum, 1578 ports_stats[pt_id].rx_bad_l4_csum, 1579 ports_stats[pt_id].rx_bad_outer_l4_csum); 1580 if (stats.ierrors + stats.rx_nombuf > 0) { 1581 printf(" RX-error: %-"PRIu64"\n", 1582 stats.ierrors); 1583 printf(" RX-nombufs: %-14"PRIu64"\n", 1584 stats.rx_nombuf); 1585 } 1586 1587 printf(" TX-packets: %-14"PRIu64 1588 " TX-dropped: %-14"PRIu64 1589 "TX-total: %-"PRIu64"\n", 1590 stats.opackets, ports_stats[pt_id].tx_dropped, 1591 stats.opackets + ports_stats[pt_id].tx_dropped); 1592 } else { 1593 printf(" RX-packets: %14"PRIu64 1594 " RX-dropped:%14"PRIu64 1595 " RX-total:%14"PRIu64"\n", 1596 stats.ipackets, stats.imissed, 1597 stats.ipackets + stats.imissed); 1598 1599 if (cur_fwd_eng == &csum_fwd_engine) 1600 printf(" Bad-ipcsum:%14"PRIu64 1601 " Bad-l4csum:%14"PRIu64 1602 " Bad-outer-l4csum: %-14"PRIu64"\n", 1603 ports_stats[pt_id].rx_bad_ip_csum, 1604 ports_stats[pt_id].rx_bad_l4_csum, 1605 ports_stats[pt_id].rx_bad_outer_l4_csum); 1606 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1607 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1608 printf(" RX-nombufs: %14"PRIu64"\n", 1609 stats.rx_nombuf); 1610 } 1611 1612 printf(" TX-packets: %14"PRIu64 1613 " TX-dropped:%14"PRIu64 1614 " TX-total:%14"PRIu64"\n", 1615 stats.opackets, ports_stats[pt_id].tx_dropped, 1616 stats.opackets + ports_stats[pt_id].tx_dropped); 1617 } 1618 1619 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1620 if (ports_stats[pt_id].rx_stream) 1621 pkt_burst_stats_display("RX", 1622 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1623 if (ports_stats[pt_id].tx_stream) 1624 pkt_burst_stats_display("TX", 1625 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1626 #endif 1627 1628 if (port->rx_queue_stats_mapping_enabled) { 1629 printf("\n"); 1630 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1631 printf(" Stats reg %2d RX-packets:%14"PRIu64 1632 " RX-errors:%14"PRIu64 1633 " RX-bytes:%14"PRIu64"\n", 1634 j, stats.q_ipackets[j], 1635 stats.q_errors[j], stats.q_ibytes[j]); 1636 } 1637 printf("\n"); 1638 } 1639 if (port->tx_queue_stats_mapping_enabled) { 1640 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1641 printf(" Stats reg %2d TX-packets:%14"PRIu64 1642 " TX-bytes:%14" 1643 PRIu64"\n", 1644 j, stats.q_opackets[j], 1645 stats.q_obytes[j]); 1646 } 1647 } 1648 1649 printf(" %s--------------------------------%s\n", 1650 fwd_stats_border, fwd_stats_border); 1651 } 1652 1653 printf("\n %s Accumulated forward statistics for all ports" 1654 "%s\n", 1655 acc_stats_border, acc_stats_border); 1656 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1657 "%-"PRIu64"\n" 1658 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1659 "%-"PRIu64"\n", 1660 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1661 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1662 if (total_rx_nombuf > 0) 1663 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1664 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1665 "%s\n", 1666 acc_stats_border, acc_stats_border); 1667 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1668 if (total_recv > 0) 1669 printf("\n CPU cycles/packet=%u (total cycles=" 1670 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1671 (unsigned int)(fwd_cycles / total_recv), 1672 fwd_cycles, total_recv); 1673 #endif 1674 } 1675 1676 void 1677 fwd_stats_reset(void) 1678 { 1679 streamid_t sm_id; 1680 portid_t pt_id; 1681 int i; 1682 1683 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1684 pt_id = fwd_ports_ids[i]; 1685 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1686 } 1687 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1688 struct fwd_stream *fs = fwd_streams[sm_id]; 1689 1690 fs->rx_packets = 0; 1691 fs->tx_packets = 0; 1692 fs->fwd_dropped = 0; 1693 fs->rx_bad_ip_csum = 0; 1694 fs->rx_bad_l4_csum = 0; 1695 fs->rx_bad_outer_l4_csum = 0; 1696 1697 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1698 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1699 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1700 #endif 1701 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1702 fs->core_cycles = 0; 1703 #endif 1704 } 1705 } 1706 1707 static void 1708 flush_fwd_rx_queues(void) 1709 { 1710 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1711 portid_t rxp; 1712 portid_t port_id; 1713 queueid_t rxq; 1714 uint16_t nb_rx; 1715 uint16_t i; 1716 uint8_t j; 1717 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1718 uint64_t timer_period; 1719 1720 /* convert to number of cycles */ 1721 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1722 1723 for (j = 0; j < 2; j++) { 1724 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1725 for (rxq = 0; rxq < nb_rxq; rxq++) { 1726 port_id = fwd_ports_ids[rxp]; 1727 /** 1728 * testpmd can stuck in the below do while loop 1729 * if rte_eth_rx_burst() always returns nonzero 1730 * packets. So timer is added to exit this loop 1731 * after 1sec timer expiry. 1732 */ 1733 prev_tsc = rte_rdtsc(); 1734 do { 1735 nb_rx = rte_eth_rx_burst(port_id, rxq, 1736 pkts_burst, MAX_PKT_BURST); 1737 for (i = 0; i < nb_rx; i++) 1738 rte_pktmbuf_free(pkts_burst[i]); 1739 1740 cur_tsc = rte_rdtsc(); 1741 diff_tsc = cur_tsc - prev_tsc; 1742 timer_tsc += diff_tsc; 1743 } while ((nb_rx > 0) && 1744 (timer_tsc < timer_period)); 1745 timer_tsc = 0; 1746 } 1747 } 1748 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1749 } 1750 } 1751 1752 static void 1753 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1754 { 1755 struct fwd_stream **fsm; 1756 streamid_t nb_fs; 1757 streamid_t sm_id; 1758 #ifdef RTE_LIBRTE_BITRATE 1759 uint64_t tics_per_1sec; 1760 uint64_t tics_datum; 1761 uint64_t tics_current; 1762 uint16_t i, cnt_ports; 1763 1764 cnt_ports = nb_ports; 1765 tics_datum = rte_rdtsc(); 1766 tics_per_1sec = rte_get_timer_hz(); 1767 #endif 1768 fsm = &fwd_streams[fc->stream_idx]; 1769 nb_fs = fc->stream_nb; 1770 do { 1771 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1772 (*pkt_fwd)(fsm[sm_id]); 1773 #ifdef RTE_LIBRTE_BITRATE 1774 if (bitrate_enabled != 0 && 1775 bitrate_lcore_id == rte_lcore_id()) { 1776 tics_current = rte_rdtsc(); 1777 if (tics_current - tics_datum >= tics_per_1sec) { 1778 /* Periodic bitrate calculation */ 1779 for (i = 0; i < cnt_ports; i++) 1780 rte_stats_bitrate_calc(bitrate_data, 1781 ports_ids[i]); 1782 tics_datum = tics_current; 1783 } 1784 } 1785 #endif 1786 #ifdef RTE_LIBRTE_LATENCY_STATS 1787 if (latencystats_enabled != 0 && 1788 latencystats_lcore_id == rte_lcore_id()) 1789 rte_latencystats_update(); 1790 #endif 1791 1792 } while (! fc->stopped); 1793 } 1794 1795 static int 1796 start_pkt_forward_on_core(void *fwd_arg) 1797 { 1798 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1799 cur_fwd_config.fwd_eng->packet_fwd); 1800 return 0; 1801 } 1802 1803 /* 1804 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1805 * Used to start communication flows in network loopback test configurations. 1806 */ 1807 static int 1808 run_one_txonly_burst_on_core(void *fwd_arg) 1809 { 1810 struct fwd_lcore *fwd_lc; 1811 struct fwd_lcore tmp_lcore; 1812 1813 fwd_lc = (struct fwd_lcore *) fwd_arg; 1814 tmp_lcore = *fwd_lc; 1815 tmp_lcore.stopped = 1; 1816 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1817 return 0; 1818 } 1819 1820 /* 1821 * Launch packet forwarding: 1822 * - Setup per-port forwarding context. 1823 * - launch logical cores with their forwarding configuration. 1824 */ 1825 static void 1826 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1827 { 1828 port_fwd_begin_t port_fwd_begin; 1829 unsigned int i; 1830 unsigned int lc_id; 1831 int diag; 1832 1833 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1834 if (port_fwd_begin != NULL) { 1835 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1836 (*port_fwd_begin)(fwd_ports_ids[i]); 1837 } 1838 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1839 lc_id = fwd_lcores_cpuids[i]; 1840 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1841 fwd_lcores[i]->stopped = 0; 1842 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1843 fwd_lcores[i], lc_id); 1844 if (diag != 0) 1845 printf("launch lcore %u failed - diag=%d\n", 1846 lc_id, diag); 1847 } 1848 } 1849 } 1850 1851 /* 1852 * Launch packet forwarding configuration. 1853 */ 1854 void 1855 start_packet_forwarding(int with_tx_first) 1856 { 1857 port_fwd_begin_t port_fwd_begin; 1858 port_fwd_end_t port_fwd_end; 1859 struct rte_port *port; 1860 unsigned int i; 1861 portid_t pt_id; 1862 1863 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1864 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1865 1866 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1867 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1868 1869 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1870 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1871 (!nb_rxq || !nb_txq)) 1872 rte_exit(EXIT_FAILURE, 1873 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1874 cur_fwd_eng->fwd_mode_name); 1875 1876 if (all_ports_started() == 0) { 1877 printf("Not all ports were started\n"); 1878 return; 1879 } 1880 if (test_done == 0) { 1881 printf("Packet forwarding already started\n"); 1882 return; 1883 } 1884 1885 1886 if(dcb_test) { 1887 for (i = 0; i < nb_fwd_ports; i++) { 1888 pt_id = fwd_ports_ids[i]; 1889 port = &ports[pt_id]; 1890 if (!port->dcb_flag) { 1891 printf("In DCB mode, all forwarding ports must " 1892 "be configured in this mode.\n"); 1893 return; 1894 } 1895 } 1896 if (nb_fwd_lcores == 1) { 1897 printf("In DCB mode,the nb forwarding cores " 1898 "should be larger than 1.\n"); 1899 return; 1900 } 1901 } 1902 test_done = 0; 1903 1904 fwd_config_setup(); 1905 1906 if(!no_flush_rx) 1907 flush_fwd_rx_queues(); 1908 1909 pkt_fwd_config_display(&cur_fwd_config); 1910 rxtx_config_display(); 1911 1912 fwd_stats_reset(); 1913 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1914 pt_id = fwd_ports_ids[i]; 1915 port = &ports[pt_id]; 1916 map_port_queue_stats_mapping_registers(pt_id, port); 1917 } 1918 if (with_tx_first) { 1919 port_fwd_begin = tx_only_engine.port_fwd_begin; 1920 if (port_fwd_begin != NULL) { 1921 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1922 (*port_fwd_begin)(fwd_ports_ids[i]); 1923 } 1924 while (with_tx_first--) { 1925 launch_packet_forwarding( 1926 run_one_txonly_burst_on_core); 1927 rte_eal_mp_wait_lcore(); 1928 } 1929 port_fwd_end = tx_only_engine.port_fwd_end; 1930 if (port_fwd_end != NULL) { 1931 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1932 (*port_fwd_end)(fwd_ports_ids[i]); 1933 } 1934 } 1935 launch_packet_forwarding(start_pkt_forward_on_core); 1936 } 1937 1938 void 1939 stop_packet_forwarding(void) 1940 { 1941 port_fwd_end_t port_fwd_end; 1942 lcoreid_t lc_id; 1943 portid_t pt_id; 1944 int i; 1945 1946 if (test_done) { 1947 printf("Packet forwarding not started\n"); 1948 return; 1949 } 1950 printf("Telling cores to stop..."); 1951 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1952 fwd_lcores[lc_id]->stopped = 1; 1953 printf("\nWaiting for lcores to finish...\n"); 1954 rte_eal_mp_wait_lcore(); 1955 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1956 if (port_fwd_end != NULL) { 1957 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1958 pt_id = fwd_ports_ids[i]; 1959 (*port_fwd_end)(pt_id); 1960 } 1961 } 1962 1963 fwd_stats_display(); 1964 1965 printf("\nDone.\n"); 1966 test_done = 1; 1967 } 1968 1969 void 1970 dev_set_link_up(portid_t pid) 1971 { 1972 if (rte_eth_dev_set_link_up(pid) < 0) 1973 printf("\nSet link up fail.\n"); 1974 } 1975 1976 void 1977 dev_set_link_down(portid_t pid) 1978 { 1979 if (rte_eth_dev_set_link_down(pid) < 0) 1980 printf("\nSet link down fail.\n"); 1981 } 1982 1983 static int 1984 all_ports_started(void) 1985 { 1986 portid_t pi; 1987 struct rte_port *port; 1988 1989 RTE_ETH_FOREACH_DEV(pi) { 1990 port = &ports[pi]; 1991 /* Check if there is a port which is not started */ 1992 if ((port->port_status != RTE_PORT_STARTED) && 1993 (port->slave_flag == 0)) 1994 return 0; 1995 } 1996 1997 /* No port is not started */ 1998 return 1; 1999 } 2000 2001 int 2002 port_is_stopped(portid_t port_id) 2003 { 2004 struct rte_port *port = &ports[port_id]; 2005 2006 if ((port->port_status != RTE_PORT_STOPPED) && 2007 (port->slave_flag == 0)) 2008 return 0; 2009 return 1; 2010 } 2011 2012 int 2013 all_ports_stopped(void) 2014 { 2015 portid_t pi; 2016 2017 RTE_ETH_FOREACH_DEV(pi) { 2018 if (!port_is_stopped(pi)) 2019 return 0; 2020 } 2021 2022 return 1; 2023 } 2024 2025 int 2026 port_is_started(portid_t port_id) 2027 { 2028 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2029 return 0; 2030 2031 if (ports[port_id].port_status != RTE_PORT_STARTED) 2032 return 0; 2033 2034 return 1; 2035 } 2036 2037 int 2038 start_port(portid_t pid) 2039 { 2040 int diag, need_check_link_status = -1; 2041 portid_t pi; 2042 queueid_t qi; 2043 struct rte_port *port; 2044 struct rte_ether_addr mac_addr; 2045 2046 if (port_id_is_invalid(pid, ENABLED_WARN)) 2047 return 0; 2048 2049 if(dcb_config) 2050 dcb_test = 1; 2051 RTE_ETH_FOREACH_DEV(pi) { 2052 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2053 continue; 2054 2055 need_check_link_status = 0; 2056 port = &ports[pi]; 2057 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2058 RTE_PORT_HANDLING) == 0) { 2059 printf("Port %d is now not stopped\n", pi); 2060 continue; 2061 } 2062 2063 if (port->need_reconfig > 0) { 2064 port->need_reconfig = 0; 2065 2066 if (flow_isolate_all) { 2067 int ret = port_flow_isolate(pi, 1); 2068 if (ret) { 2069 printf("Failed to apply isolated" 2070 " mode on port %d\n", pi); 2071 return -1; 2072 } 2073 } 2074 configure_rxtx_dump_callbacks(0); 2075 printf("Configuring Port %d (socket %u)\n", pi, 2076 port->socket_id); 2077 /* configure port */ 2078 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 2079 &(port->dev_conf)); 2080 if (diag != 0) { 2081 if (rte_atomic16_cmpset(&(port->port_status), 2082 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2083 printf("Port %d can not be set back " 2084 "to stopped\n", pi); 2085 printf("Fail to configure port %d\n", pi); 2086 /* try to reconfigure port next time */ 2087 port->need_reconfig = 1; 2088 return -1; 2089 } 2090 } 2091 if (port->need_reconfig_queues > 0) { 2092 port->need_reconfig_queues = 0; 2093 /* setup tx queues */ 2094 for (qi = 0; qi < nb_txq; qi++) { 2095 if ((numa_support) && 2096 (txring_numa[pi] != NUMA_NO_CONFIG)) 2097 diag = rte_eth_tx_queue_setup(pi, qi, 2098 port->nb_tx_desc[qi], 2099 txring_numa[pi], 2100 &(port->tx_conf[qi])); 2101 else 2102 diag = rte_eth_tx_queue_setup(pi, qi, 2103 port->nb_tx_desc[qi], 2104 port->socket_id, 2105 &(port->tx_conf[qi])); 2106 2107 if (diag == 0) 2108 continue; 2109 2110 /* Fail to setup tx queue, return */ 2111 if (rte_atomic16_cmpset(&(port->port_status), 2112 RTE_PORT_HANDLING, 2113 RTE_PORT_STOPPED) == 0) 2114 printf("Port %d can not be set back " 2115 "to stopped\n", pi); 2116 printf("Fail to configure port %d tx queues\n", 2117 pi); 2118 /* try to reconfigure queues next time */ 2119 port->need_reconfig_queues = 1; 2120 return -1; 2121 } 2122 for (qi = 0; qi < nb_rxq; qi++) { 2123 /* setup rx queues */ 2124 if ((numa_support) && 2125 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2126 struct rte_mempool * mp = 2127 mbuf_pool_find(rxring_numa[pi]); 2128 if (mp == NULL) { 2129 printf("Failed to setup RX queue:" 2130 "No mempool allocation" 2131 " on the socket %d\n", 2132 rxring_numa[pi]); 2133 return -1; 2134 } 2135 2136 diag = rte_eth_rx_queue_setup(pi, qi, 2137 port->nb_rx_desc[qi], 2138 rxring_numa[pi], 2139 &(port->rx_conf[qi]), 2140 mp); 2141 } else { 2142 struct rte_mempool *mp = 2143 mbuf_pool_find(port->socket_id); 2144 if (mp == NULL) { 2145 printf("Failed to setup RX queue:" 2146 "No mempool allocation" 2147 " on the socket %d\n", 2148 port->socket_id); 2149 return -1; 2150 } 2151 diag = rte_eth_rx_queue_setup(pi, qi, 2152 port->nb_rx_desc[qi], 2153 port->socket_id, 2154 &(port->rx_conf[qi]), 2155 mp); 2156 } 2157 if (diag == 0) 2158 continue; 2159 2160 /* Fail to setup rx queue, return */ 2161 if (rte_atomic16_cmpset(&(port->port_status), 2162 RTE_PORT_HANDLING, 2163 RTE_PORT_STOPPED) == 0) 2164 printf("Port %d can not be set back " 2165 "to stopped\n", pi); 2166 printf("Fail to configure port %d rx queues\n", 2167 pi); 2168 /* try to reconfigure queues next time */ 2169 port->need_reconfig_queues = 1; 2170 return -1; 2171 } 2172 } 2173 configure_rxtx_dump_callbacks(verbose_level); 2174 /* start port */ 2175 if (rte_eth_dev_start(pi) < 0) { 2176 printf("Fail to start port %d\n", pi); 2177 2178 /* Fail to setup rx queue, return */ 2179 if (rte_atomic16_cmpset(&(port->port_status), 2180 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2181 printf("Port %d can not be set back to " 2182 "stopped\n", pi); 2183 continue; 2184 } 2185 2186 if (rte_atomic16_cmpset(&(port->port_status), 2187 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2188 printf("Port %d can not be set into started\n", pi); 2189 2190 rte_eth_macaddr_get(pi, &mac_addr); 2191 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2192 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2193 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2194 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2195 2196 /* at least one port started, need checking link status */ 2197 need_check_link_status = 1; 2198 } 2199 2200 if (need_check_link_status == 1 && !no_link_check) 2201 check_all_ports_link_status(RTE_PORT_ALL); 2202 else if (need_check_link_status == 0) 2203 printf("Please stop the ports first\n"); 2204 2205 printf("Done\n"); 2206 return 0; 2207 } 2208 2209 void 2210 stop_port(portid_t pid) 2211 { 2212 portid_t pi; 2213 struct rte_port *port; 2214 int need_check_link_status = 0; 2215 2216 if (dcb_test) { 2217 dcb_test = 0; 2218 dcb_config = 0; 2219 } 2220 2221 if (port_id_is_invalid(pid, ENABLED_WARN)) 2222 return; 2223 2224 printf("Stopping ports...\n"); 2225 2226 RTE_ETH_FOREACH_DEV(pi) { 2227 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2228 continue; 2229 2230 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2231 printf("Please remove port %d from forwarding configuration.\n", pi); 2232 continue; 2233 } 2234 2235 if (port_is_bonding_slave(pi)) { 2236 printf("Please remove port %d from bonded device.\n", pi); 2237 continue; 2238 } 2239 2240 port = &ports[pi]; 2241 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2242 RTE_PORT_HANDLING) == 0) 2243 continue; 2244 2245 rte_eth_dev_stop(pi); 2246 2247 if (rte_atomic16_cmpset(&(port->port_status), 2248 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2249 printf("Port %d can not be set into stopped\n", pi); 2250 need_check_link_status = 1; 2251 } 2252 if (need_check_link_status && !no_link_check) 2253 check_all_ports_link_status(RTE_PORT_ALL); 2254 2255 printf("Done\n"); 2256 } 2257 2258 static void 2259 remove_invalid_ports_in(portid_t *array, portid_t *total) 2260 { 2261 portid_t i; 2262 portid_t new_total = 0; 2263 2264 for (i = 0; i < *total; i++) 2265 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2266 array[new_total] = array[i]; 2267 new_total++; 2268 } 2269 *total = new_total; 2270 } 2271 2272 static void 2273 remove_invalid_ports(void) 2274 { 2275 remove_invalid_ports_in(ports_ids, &nb_ports); 2276 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2277 nb_cfg_ports = nb_fwd_ports; 2278 } 2279 2280 void 2281 close_port(portid_t pid) 2282 { 2283 portid_t pi; 2284 struct rte_port *port; 2285 2286 if (port_id_is_invalid(pid, ENABLED_WARN)) 2287 return; 2288 2289 printf("Closing ports...\n"); 2290 2291 RTE_ETH_FOREACH_DEV(pi) { 2292 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2293 continue; 2294 2295 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2296 printf("Please remove port %d from forwarding configuration.\n", pi); 2297 continue; 2298 } 2299 2300 if (port_is_bonding_slave(pi)) { 2301 printf("Please remove port %d from bonded device.\n", pi); 2302 continue; 2303 } 2304 2305 port = &ports[pi]; 2306 if (rte_atomic16_cmpset(&(port->port_status), 2307 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2308 printf("Port %d is already closed\n", pi); 2309 continue; 2310 } 2311 2312 if (rte_atomic16_cmpset(&(port->port_status), 2313 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2314 printf("Port %d is now not stopped\n", pi); 2315 continue; 2316 } 2317 2318 if (port->flow_list) 2319 port_flow_flush(pi); 2320 rte_eth_dev_close(pi); 2321 2322 remove_invalid_ports(); 2323 2324 if (rte_atomic16_cmpset(&(port->port_status), 2325 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2326 printf("Port %d cannot be set to closed\n", pi); 2327 } 2328 2329 printf("Done\n"); 2330 } 2331 2332 void 2333 reset_port(portid_t pid) 2334 { 2335 int diag; 2336 portid_t pi; 2337 struct rte_port *port; 2338 2339 if (port_id_is_invalid(pid, ENABLED_WARN)) 2340 return; 2341 2342 printf("Resetting ports...\n"); 2343 2344 RTE_ETH_FOREACH_DEV(pi) { 2345 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2346 continue; 2347 2348 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2349 printf("Please remove port %d from forwarding " 2350 "configuration.\n", pi); 2351 continue; 2352 } 2353 2354 if (port_is_bonding_slave(pi)) { 2355 printf("Please remove port %d from bonded device.\n", 2356 pi); 2357 continue; 2358 } 2359 2360 diag = rte_eth_dev_reset(pi); 2361 if (diag == 0) { 2362 port = &ports[pi]; 2363 port->need_reconfig = 1; 2364 port->need_reconfig_queues = 1; 2365 } else { 2366 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2367 } 2368 } 2369 2370 printf("Done\n"); 2371 } 2372 2373 void 2374 attach_port(char *identifier) 2375 { 2376 portid_t pi; 2377 struct rte_dev_iterator iterator; 2378 2379 printf("Attaching a new port...\n"); 2380 2381 if (identifier == NULL) { 2382 printf("Invalid parameters are specified\n"); 2383 return; 2384 } 2385 2386 if (rte_dev_probe(identifier) < 0) { 2387 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2388 return; 2389 } 2390 2391 /* first attach mode: event */ 2392 if (setup_on_probe_event) { 2393 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2394 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2395 if (ports[pi].port_status == RTE_PORT_HANDLING && 2396 ports[pi].need_setup != 0) 2397 setup_attached_port(pi); 2398 return; 2399 } 2400 2401 /* second attach mode: iterator */ 2402 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2403 /* setup ports matching the devargs used for probing */ 2404 if (port_is_forwarding(pi)) 2405 continue; /* port was already attached before */ 2406 setup_attached_port(pi); 2407 } 2408 } 2409 2410 static void 2411 setup_attached_port(portid_t pi) 2412 { 2413 unsigned int socket_id; 2414 2415 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2416 /* if socket_id is invalid, set to the first available socket. */ 2417 if (check_socket_id(socket_id) < 0) 2418 socket_id = socket_ids[0]; 2419 reconfig(pi, socket_id); 2420 rte_eth_promiscuous_enable(pi); 2421 2422 ports_ids[nb_ports++] = pi; 2423 fwd_ports_ids[nb_fwd_ports++] = pi; 2424 nb_cfg_ports = nb_fwd_ports; 2425 ports[pi].need_setup = 0; 2426 ports[pi].port_status = RTE_PORT_STOPPED; 2427 2428 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2429 printf("Done\n"); 2430 } 2431 2432 void 2433 detach_port_device(portid_t port_id) 2434 { 2435 struct rte_device *dev; 2436 portid_t sibling; 2437 2438 printf("Removing a device...\n"); 2439 2440 dev = rte_eth_devices[port_id].device; 2441 if (dev == NULL) { 2442 printf("Device already removed\n"); 2443 return; 2444 } 2445 2446 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2447 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2448 printf("Port not stopped\n"); 2449 return; 2450 } 2451 printf("Port was not closed\n"); 2452 if (ports[port_id].flow_list) 2453 port_flow_flush(port_id); 2454 } 2455 2456 if (rte_dev_remove(dev) < 0) { 2457 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2458 return; 2459 } 2460 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2461 /* reset mapping between old ports and removed device */ 2462 rte_eth_devices[sibling].device = NULL; 2463 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2464 /* sibling ports are forced to be closed */ 2465 ports[sibling].port_status = RTE_PORT_CLOSED; 2466 printf("Port %u is closed\n", sibling); 2467 } 2468 } 2469 2470 remove_invalid_ports(); 2471 2472 printf("Device of port %u is detached\n", port_id); 2473 printf("Now total ports is %d\n", nb_ports); 2474 printf("Done\n"); 2475 return; 2476 } 2477 2478 void 2479 detach_device(char *identifier) 2480 { 2481 struct rte_dev_iterator iterator; 2482 struct rte_devargs da; 2483 portid_t port_id; 2484 2485 printf("Removing a device...\n"); 2486 2487 memset(&da, 0, sizeof(da)); 2488 if (rte_devargs_parsef(&da, "%s", identifier)) { 2489 printf("cannot parse identifier\n"); 2490 if (da.args) 2491 free(da.args); 2492 return; 2493 } 2494 2495 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2496 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2497 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2498 printf("Port %u not stopped\n", port_id); 2499 return; 2500 } 2501 2502 /* sibling ports are forced to be closed */ 2503 if (ports[port_id].flow_list) 2504 port_flow_flush(port_id); 2505 ports[port_id].port_status = RTE_PORT_CLOSED; 2506 printf("Port %u is now closed\n", port_id); 2507 } 2508 } 2509 2510 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2511 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2512 da.name, da.bus->name); 2513 return; 2514 } 2515 2516 remove_invalid_ports(); 2517 2518 printf("Device %s is detached\n", identifier); 2519 printf("Now total ports is %d\n", nb_ports); 2520 printf("Done\n"); 2521 } 2522 2523 void 2524 pmd_test_exit(void) 2525 { 2526 portid_t pt_id; 2527 int ret; 2528 int i; 2529 2530 if (test_done == 0) 2531 stop_packet_forwarding(); 2532 2533 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2534 if (mempools[i]) { 2535 if (mp_alloc_type == MP_ALLOC_ANON) 2536 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2537 NULL); 2538 } 2539 } 2540 if (ports != NULL) { 2541 no_link_check = 1; 2542 RTE_ETH_FOREACH_DEV(pt_id) { 2543 printf("\nStopping port %d...\n", pt_id); 2544 fflush(stdout); 2545 stop_port(pt_id); 2546 } 2547 RTE_ETH_FOREACH_DEV(pt_id) { 2548 printf("\nShutting down port %d...\n", pt_id); 2549 fflush(stdout); 2550 close_port(pt_id); 2551 } 2552 } 2553 2554 if (hot_plug) { 2555 ret = rte_dev_event_monitor_stop(); 2556 if (ret) { 2557 RTE_LOG(ERR, EAL, 2558 "fail to stop device event monitor."); 2559 return; 2560 } 2561 2562 ret = rte_dev_event_callback_unregister(NULL, 2563 dev_event_callback, NULL); 2564 if (ret < 0) { 2565 RTE_LOG(ERR, EAL, 2566 "fail to unregister device event callback.\n"); 2567 return; 2568 } 2569 2570 ret = rte_dev_hotplug_handle_disable(); 2571 if (ret) { 2572 RTE_LOG(ERR, EAL, 2573 "fail to disable hotplug handling.\n"); 2574 return; 2575 } 2576 } 2577 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2578 if (mempools[i]) 2579 rte_mempool_free(mempools[i]); 2580 } 2581 2582 printf("\nBye...\n"); 2583 } 2584 2585 typedef void (*cmd_func_t)(void); 2586 struct pmd_test_command { 2587 const char *cmd_name; 2588 cmd_func_t cmd_func; 2589 }; 2590 2591 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2592 2593 /* Check the link status of all ports in up to 9s, and print them finally */ 2594 static void 2595 check_all_ports_link_status(uint32_t port_mask) 2596 { 2597 #define CHECK_INTERVAL 100 /* 100ms */ 2598 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2599 portid_t portid; 2600 uint8_t count, all_ports_up, print_flag = 0; 2601 struct rte_eth_link link; 2602 2603 printf("Checking link statuses...\n"); 2604 fflush(stdout); 2605 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2606 all_ports_up = 1; 2607 RTE_ETH_FOREACH_DEV(portid) { 2608 if ((port_mask & (1 << portid)) == 0) 2609 continue; 2610 memset(&link, 0, sizeof(link)); 2611 rte_eth_link_get_nowait(portid, &link); 2612 /* print link status if flag set */ 2613 if (print_flag == 1) { 2614 if (link.link_status) 2615 printf( 2616 "Port%d Link Up. speed %u Mbps- %s\n", 2617 portid, link.link_speed, 2618 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2619 ("full-duplex") : ("half-duplex\n")); 2620 else 2621 printf("Port %d Link Down\n", portid); 2622 continue; 2623 } 2624 /* clear all_ports_up flag if any link down */ 2625 if (link.link_status == ETH_LINK_DOWN) { 2626 all_ports_up = 0; 2627 break; 2628 } 2629 } 2630 /* after finally printing all link status, get out */ 2631 if (print_flag == 1) 2632 break; 2633 2634 if (all_ports_up == 0) { 2635 fflush(stdout); 2636 rte_delay_ms(CHECK_INTERVAL); 2637 } 2638 2639 /* set the print_flag if all ports up or timeout */ 2640 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2641 print_flag = 1; 2642 } 2643 2644 if (lsc_interrupt) 2645 break; 2646 } 2647 } 2648 2649 /* 2650 * This callback is for remove a port for a device. It has limitation because 2651 * it is not for multiple port removal for a device. 2652 * TODO: the device detach invoke will plan to be removed from user side to 2653 * eal. And convert all PMDs to free port resources on ether device closing. 2654 */ 2655 static void 2656 rmv_port_callback(void *arg) 2657 { 2658 int need_to_start = 0; 2659 int org_no_link_check = no_link_check; 2660 portid_t port_id = (intptr_t)arg; 2661 2662 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2663 2664 if (!test_done && port_is_forwarding(port_id)) { 2665 need_to_start = 1; 2666 stop_packet_forwarding(); 2667 } 2668 no_link_check = 1; 2669 stop_port(port_id); 2670 no_link_check = org_no_link_check; 2671 close_port(port_id); 2672 detach_port_device(port_id); 2673 if (need_to_start) 2674 start_packet_forwarding(0); 2675 } 2676 2677 /* This function is used by the interrupt thread */ 2678 static int 2679 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2680 void *ret_param) 2681 { 2682 RTE_SET_USED(param); 2683 RTE_SET_USED(ret_param); 2684 2685 if (type >= RTE_ETH_EVENT_MAX) { 2686 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2687 port_id, __func__, type); 2688 fflush(stderr); 2689 } else if (event_print_mask & (UINT32_C(1) << type)) { 2690 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2691 eth_event_desc[type]); 2692 fflush(stdout); 2693 } 2694 2695 switch (type) { 2696 case RTE_ETH_EVENT_NEW: 2697 ports[port_id].need_setup = 1; 2698 ports[port_id].port_status = RTE_PORT_HANDLING; 2699 break; 2700 case RTE_ETH_EVENT_INTR_RMV: 2701 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2702 break; 2703 if (rte_eal_alarm_set(100000, 2704 rmv_port_callback, (void *)(intptr_t)port_id)) 2705 fprintf(stderr, "Could not set up deferred device removal\n"); 2706 break; 2707 default: 2708 break; 2709 } 2710 return 0; 2711 } 2712 2713 static int 2714 register_eth_event_callback(void) 2715 { 2716 int ret; 2717 enum rte_eth_event_type event; 2718 2719 for (event = RTE_ETH_EVENT_UNKNOWN; 2720 event < RTE_ETH_EVENT_MAX; event++) { 2721 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2722 event, 2723 eth_event_callback, 2724 NULL); 2725 if (ret != 0) { 2726 TESTPMD_LOG(ERR, "Failed to register callback for " 2727 "%s event\n", eth_event_desc[event]); 2728 return -1; 2729 } 2730 } 2731 2732 return 0; 2733 } 2734 2735 /* This function is used by the interrupt thread */ 2736 static void 2737 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2738 __rte_unused void *arg) 2739 { 2740 uint16_t port_id; 2741 int ret; 2742 2743 if (type >= RTE_DEV_EVENT_MAX) { 2744 fprintf(stderr, "%s called upon invalid event %d\n", 2745 __func__, type); 2746 fflush(stderr); 2747 } 2748 2749 switch (type) { 2750 case RTE_DEV_EVENT_REMOVE: 2751 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2752 device_name); 2753 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2754 if (ret) { 2755 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2756 device_name); 2757 return; 2758 } 2759 /* 2760 * Because the user's callback is invoked in eal interrupt 2761 * callback, the interrupt callback need to be finished before 2762 * it can be unregistered when detaching device. So finish 2763 * callback soon and use a deferred removal to detach device 2764 * is need. It is a workaround, once the device detaching be 2765 * moved into the eal in the future, the deferred removal could 2766 * be deleted. 2767 */ 2768 if (rte_eal_alarm_set(100000, 2769 rmv_port_callback, (void *)(intptr_t)port_id)) 2770 RTE_LOG(ERR, EAL, 2771 "Could not set up deferred device removal\n"); 2772 break; 2773 case RTE_DEV_EVENT_ADD: 2774 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2775 device_name); 2776 /* TODO: After finish kernel driver binding, 2777 * begin to attach port. 2778 */ 2779 break; 2780 default: 2781 break; 2782 } 2783 } 2784 2785 static int 2786 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2787 { 2788 uint16_t i; 2789 int diag; 2790 uint8_t mapping_found = 0; 2791 2792 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2793 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2794 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2795 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2796 tx_queue_stats_mappings[i].queue_id, 2797 tx_queue_stats_mappings[i].stats_counter_id); 2798 if (diag != 0) 2799 return diag; 2800 mapping_found = 1; 2801 } 2802 } 2803 if (mapping_found) 2804 port->tx_queue_stats_mapping_enabled = 1; 2805 return 0; 2806 } 2807 2808 static int 2809 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2810 { 2811 uint16_t i; 2812 int diag; 2813 uint8_t mapping_found = 0; 2814 2815 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2816 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2817 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2818 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2819 rx_queue_stats_mappings[i].queue_id, 2820 rx_queue_stats_mappings[i].stats_counter_id); 2821 if (diag != 0) 2822 return diag; 2823 mapping_found = 1; 2824 } 2825 } 2826 if (mapping_found) 2827 port->rx_queue_stats_mapping_enabled = 1; 2828 return 0; 2829 } 2830 2831 static void 2832 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2833 { 2834 int diag = 0; 2835 2836 diag = set_tx_queue_stats_mapping_registers(pi, port); 2837 if (diag != 0) { 2838 if (diag == -ENOTSUP) { 2839 port->tx_queue_stats_mapping_enabled = 0; 2840 printf("TX queue stats mapping not supported port id=%d\n", pi); 2841 } 2842 else 2843 rte_exit(EXIT_FAILURE, 2844 "set_tx_queue_stats_mapping_registers " 2845 "failed for port id=%d diag=%d\n", 2846 pi, diag); 2847 } 2848 2849 diag = set_rx_queue_stats_mapping_registers(pi, port); 2850 if (diag != 0) { 2851 if (diag == -ENOTSUP) { 2852 port->rx_queue_stats_mapping_enabled = 0; 2853 printf("RX queue stats mapping not supported port id=%d\n", pi); 2854 } 2855 else 2856 rte_exit(EXIT_FAILURE, 2857 "set_rx_queue_stats_mapping_registers " 2858 "failed for port id=%d diag=%d\n", 2859 pi, diag); 2860 } 2861 } 2862 2863 static void 2864 rxtx_port_config(struct rte_port *port) 2865 { 2866 uint16_t qid; 2867 uint64_t offloads; 2868 2869 for (qid = 0; qid < nb_rxq; qid++) { 2870 offloads = port->rx_conf[qid].offloads; 2871 port->rx_conf[qid] = port->dev_info.default_rxconf; 2872 if (offloads != 0) 2873 port->rx_conf[qid].offloads = offloads; 2874 2875 /* Check if any Rx parameters have been passed */ 2876 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2877 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2878 2879 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2880 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2881 2882 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2883 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2884 2885 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2886 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2887 2888 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2889 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2890 2891 port->nb_rx_desc[qid] = nb_rxd; 2892 } 2893 2894 for (qid = 0; qid < nb_txq; qid++) { 2895 offloads = port->tx_conf[qid].offloads; 2896 port->tx_conf[qid] = port->dev_info.default_txconf; 2897 if (offloads != 0) 2898 port->tx_conf[qid].offloads = offloads; 2899 2900 /* Check if any Tx parameters have been passed */ 2901 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2902 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2903 2904 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2905 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2906 2907 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2908 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2909 2910 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2911 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2912 2913 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2914 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2915 2916 port->nb_tx_desc[qid] = nb_txd; 2917 } 2918 } 2919 2920 void 2921 init_port_config(void) 2922 { 2923 portid_t pid; 2924 struct rte_port *port; 2925 2926 RTE_ETH_FOREACH_DEV(pid) { 2927 port = &ports[pid]; 2928 port->dev_conf.fdir_conf = fdir_conf; 2929 rte_eth_dev_info_get(pid, &port->dev_info); 2930 if (nb_rxq > 1) { 2931 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2932 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2933 rss_hf & port->dev_info.flow_type_rss_offloads; 2934 } else { 2935 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2936 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2937 } 2938 2939 if (port->dcb_flag == 0) { 2940 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2941 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2942 else 2943 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2944 } 2945 2946 rxtx_port_config(port); 2947 2948 rte_eth_macaddr_get(pid, &port->eth_addr); 2949 2950 map_port_queue_stats_mapping_registers(pid, port); 2951 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2952 rte_pmd_ixgbe_bypass_init(pid); 2953 #endif 2954 2955 if (lsc_interrupt && 2956 (rte_eth_devices[pid].data->dev_flags & 2957 RTE_ETH_DEV_INTR_LSC)) 2958 port->dev_conf.intr_conf.lsc = 1; 2959 if (rmv_interrupt && 2960 (rte_eth_devices[pid].data->dev_flags & 2961 RTE_ETH_DEV_INTR_RMV)) 2962 port->dev_conf.intr_conf.rmv = 1; 2963 } 2964 } 2965 2966 void set_port_slave_flag(portid_t slave_pid) 2967 { 2968 struct rte_port *port; 2969 2970 port = &ports[slave_pid]; 2971 port->slave_flag = 1; 2972 } 2973 2974 void clear_port_slave_flag(portid_t slave_pid) 2975 { 2976 struct rte_port *port; 2977 2978 port = &ports[slave_pid]; 2979 port->slave_flag = 0; 2980 } 2981 2982 uint8_t port_is_bonding_slave(portid_t slave_pid) 2983 { 2984 struct rte_port *port; 2985 2986 port = &ports[slave_pid]; 2987 if ((rte_eth_devices[slave_pid].data->dev_flags & 2988 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2989 return 1; 2990 return 0; 2991 } 2992 2993 const uint16_t vlan_tags[] = { 2994 0, 1, 2, 3, 4, 5, 6, 7, 2995 8, 9, 10, 11, 12, 13, 14, 15, 2996 16, 17, 18, 19, 20, 21, 22, 23, 2997 24, 25, 26, 27, 28, 29, 30, 31 2998 }; 2999 3000 static int 3001 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3002 enum dcb_mode_enable dcb_mode, 3003 enum rte_eth_nb_tcs num_tcs, 3004 uint8_t pfc_en) 3005 { 3006 uint8_t i; 3007 int32_t rc; 3008 struct rte_eth_rss_conf rss_conf; 3009 3010 /* 3011 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3012 * given above, and the number of traffic classes available for use. 3013 */ 3014 if (dcb_mode == DCB_VT_ENABLED) { 3015 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3016 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3017 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3018 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3019 3020 /* VMDQ+DCB RX and TX configurations */ 3021 vmdq_rx_conf->enable_default_pool = 0; 3022 vmdq_rx_conf->default_pool = 0; 3023 vmdq_rx_conf->nb_queue_pools = 3024 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3025 vmdq_tx_conf->nb_queue_pools = 3026 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3027 3028 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3029 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3030 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3031 vmdq_rx_conf->pool_map[i].pools = 3032 1 << (i % vmdq_rx_conf->nb_queue_pools); 3033 } 3034 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3035 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3036 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3037 } 3038 3039 /* set DCB mode of RX and TX of multiple queues */ 3040 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 3041 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3042 } else { 3043 struct rte_eth_dcb_rx_conf *rx_conf = 3044 ð_conf->rx_adv_conf.dcb_rx_conf; 3045 struct rte_eth_dcb_tx_conf *tx_conf = 3046 ð_conf->tx_adv_conf.dcb_tx_conf; 3047 3048 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3049 if (rc != 0) 3050 return rc; 3051 3052 rx_conf->nb_tcs = num_tcs; 3053 tx_conf->nb_tcs = num_tcs; 3054 3055 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3056 rx_conf->dcb_tc[i] = i % num_tcs; 3057 tx_conf->dcb_tc[i] = i % num_tcs; 3058 } 3059 3060 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 3061 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3062 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3063 } 3064 3065 if (pfc_en) 3066 eth_conf->dcb_capability_en = 3067 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3068 else 3069 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3070 3071 return 0; 3072 } 3073 3074 int 3075 init_port_dcb_config(portid_t pid, 3076 enum dcb_mode_enable dcb_mode, 3077 enum rte_eth_nb_tcs num_tcs, 3078 uint8_t pfc_en) 3079 { 3080 struct rte_eth_conf port_conf; 3081 struct rte_port *rte_port; 3082 int retval; 3083 uint16_t i; 3084 3085 rte_port = &ports[pid]; 3086 3087 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3088 /* Enter DCB configuration status */ 3089 dcb_config = 1; 3090 3091 port_conf.rxmode = rte_port->dev_conf.rxmode; 3092 port_conf.txmode = rte_port->dev_conf.txmode; 3093 3094 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3095 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3096 if (retval < 0) 3097 return retval; 3098 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3099 3100 /* re-configure the device . */ 3101 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3102 if (retval < 0) 3103 return retval; 3104 rte_eth_dev_info_get(pid, &rte_port->dev_info); 3105 3106 /* If dev_info.vmdq_pool_base is greater than 0, 3107 * the queue id of vmdq pools is started after pf queues. 3108 */ 3109 if (dcb_mode == DCB_VT_ENABLED && 3110 rte_port->dev_info.vmdq_pool_base > 0) { 3111 printf("VMDQ_DCB multi-queue mode is nonsensical" 3112 " for port %d.", pid); 3113 return -1; 3114 } 3115 3116 /* Assume the ports in testpmd have the same dcb capability 3117 * and has the same number of rxq and txq in dcb mode 3118 */ 3119 if (dcb_mode == DCB_VT_ENABLED) { 3120 if (rte_port->dev_info.max_vfs > 0) { 3121 nb_rxq = rte_port->dev_info.nb_rx_queues; 3122 nb_txq = rte_port->dev_info.nb_tx_queues; 3123 } else { 3124 nb_rxq = rte_port->dev_info.max_rx_queues; 3125 nb_txq = rte_port->dev_info.max_tx_queues; 3126 } 3127 } else { 3128 /*if vt is disabled, use all pf queues */ 3129 if (rte_port->dev_info.vmdq_pool_base == 0) { 3130 nb_rxq = rte_port->dev_info.max_rx_queues; 3131 nb_txq = rte_port->dev_info.max_tx_queues; 3132 } else { 3133 nb_rxq = (queueid_t)num_tcs; 3134 nb_txq = (queueid_t)num_tcs; 3135 3136 } 3137 } 3138 rx_free_thresh = 64; 3139 3140 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3141 3142 rxtx_port_config(rte_port); 3143 /* VLAN filter */ 3144 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3145 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3146 rx_vft_set(pid, vlan_tags[i], 1); 3147 3148 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 3149 map_port_queue_stats_mapping_registers(pid, rte_port); 3150 3151 rte_port->dcb_flag = 1; 3152 3153 return 0; 3154 } 3155 3156 static void 3157 init_port(void) 3158 { 3159 /* Configuration of Ethernet ports. */ 3160 ports = rte_zmalloc("testpmd: ports", 3161 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3162 RTE_CACHE_LINE_SIZE); 3163 if (ports == NULL) { 3164 rte_exit(EXIT_FAILURE, 3165 "rte_zmalloc(%d struct rte_port) failed\n", 3166 RTE_MAX_ETHPORTS); 3167 } 3168 3169 /* Initialize ports NUMA structures */ 3170 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3171 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3172 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3173 } 3174 3175 static void 3176 force_quit(void) 3177 { 3178 pmd_test_exit(); 3179 prompt_exit(); 3180 } 3181 3182 static void 3183 print_stats(void) 3184 { 3185 uint8_t i; 3186 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3187 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3188 3189 /* Clear screen and move to top left */ 3190 printf("%s%s", clr, top_left); 3191 3192 printf("\nPort statistics ===================================="); 3193 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3194 nic_stats_display(fwd_ports_ids[i]); 3195 3196 fflush(stdout); 3197 } 3198 3199 static void 3200 signal_handler(int signum) 3201 { 3202 if (signum == SIGINT || signum == SIGTERM) { 3203 printf("\nSignal %d received, preparing to exit...\n", 3204 signum); 3205 #ifdef RTE_LIBRTE_PDUMP 3206 /* uninitialize packet capture framework */ 3207 rte_pdump_uninit(); 3208 #endif 3209 #ifdef RTE_LIBRTE_LATENCY_STATS 3210 rte_latencystats_uninit(); 3211 #endif 3212 force_quit(); 3213 /* Set flag to indicate the force termination. */ 3214 f_quit = 1; 3215 /* exit with the expected status */ 3216 signal(signum, SIG_DFL); 3217 kill(getpid(), signum); 3218 } 3219 } 3220 3221 int 3222 main(int argc, char** argv) 3223 { 3224 int diag; 3225 portid_t port_id; 3226 uint16_t count; 3227 int ret; 3228 3229 signal(SIGINT, signal_handler); 3230 signal(SIGTERM, signal_handler); 3231 3232 diag = rte_eal_init(argc, argv); 3233 if (diag < 0) 3234 rte_panic("Cannot init EAL\n"); 3235 3236 testpmd_logtype = rte_log_register("testpmd"); 3237 if (testpmd_logtype < 0) 3238 rte_panic("Cannot register log type"); 3239 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3240 3241 ret = register_eth_event_callback(); 3242 if (ret != 0) 3243 rte_panic("Cannot register for ethdev events"); 3244 3245 #ifdef RTE_LIBRTE_PDUMP 3246 /* initialize packet capture framework */ 3247 rte_pdump_init(); 3248 #endif 3249 3250 count = 0; 3251 RTE_ETH_FOREACH_DEV(port_id) { 3252 ports_ids[count] = port_id; 3253 count++; 3254 } 3255 nb_ports = (portid_t) count; 3256 if (nb_ports == 0) 3257 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3258 3259 /* allocate port structures, and init them */ 3260 init_port(); 3261 3262 set_def_fwd_config(); 3263 if (nb_lcores == 0) 3264 rte_panic("Empty set of forwarding logical cores - check the " 3265 "core mask supplied in the command parameters\n"); 3266 3267 /* Bitrate/latency stats disabled by default */ 3268 #ifdef RTE_LIBRTE_BITRATE 3269 bitrate_enabled = 0; 3270 #endif 3271 #ifdef RTE_LIBRTE_LATENCY_STATS 3272 latencystats_enabled = 0; 3273 #endif 3274 3275 /* on FreeBSD, mlockall() is disabled by default */ 3276 #ifdef RTE_EXEC_ENV_FREEBSD 3277 do_mlockall = 0; 3278 #else 3279 do_mlockall = 1; 3280 #endif 3281 3282 argc -= diag; 3283 argv += diag; 3284 if (argc > 1) 3285 launch_args_parse(argc, argv); 3286 3287 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3288 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3289 strerror(errno)); 3290 } 3291 3292 if (tx_first && interactive) 3293 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3294 "interactive mode.\n"); 3295 3296 if (tx_first && lsc_interrupt) { 3297 printf("Warning: lsc_interrupt needs to be off when " 3298 " using tx_first. Disabling.\n"); 3299 lsc_interrupt = 0; 3300 } 3301 3302 if (!nb_rxq && !nb_txq) 3303 printf("Warning: Either rx or tx queues should be non-zero\n"); 3304 3305 if (nb_rxq > 1 && nb_rxq > nb_txq) 3306 printf("Warning: nb_rxq=%d enables RSS configuration, " 3307 "but nb_txq=%d will prevent to fully test it.\n", 3308 nb_rxq, nb_txq); 3309 3310 init_config(); 3311 3312 if (hot_plug) { 3313 ret = rte_dev_hotplug_handle_enable(); 3314 if (ret) { 3315 RTE_LOG(ERR, EAL, 3316 "fail to enable hotplug handling."); 3317 return -1; 3318 } 3319 3320 ret = rte_dev_event_monitor_start(); 3321 if (ret) { 3322 RTE_LOG(ERR, EAL, 3323 "fail to start device event monitoring."); 3324 return -1; 3325 } 3326 3327 ret = rte_dev_event_callback_register(NULL, 3328 dev_event_callback, NULL); 3329 if (ret) { 3330 RTE_LOG(ERR, EAL, 3331 "fail to register device event callback\n"); 3332 return -1; 3333 } 3334 } 3335 3336 if (start_port(RTE_PORT_ALL) != 0) 3337 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3338 3339 /* set all ports to promiscuous mode by default */ 3340 RTE_ETH_FOREACH_DEV(port_id) 3341 rte_eth_promiscuous_enable(port_id); 3342 3343 /* Init metrics library */ 3344 rte_metrics_init(rte_socket_id()); 3345 3346 #ifdef RTE_LIBRTE_LATENCY_STATS 3347 if (latencystats_enabled != 0) { 3348 int ret = rte_latencystats_init(1, NULL); 3349 if (ret) 3350 printf("Warning: latencystats init()" 3351 " returned error %d\n", ret); 3352 printf("Latencystats running on lcore %d\n", 3353 latencystats_lcore_id); 3354 } 3355 #endif 3356 3357 /* Setup bitrate stats */ 3358 #ifdef RTE_LIBRTE_BITRATE 3359 if (bitrate_enabled != 0) { 3360 bitrate_data = rte_stats_bitrate_create(); 3361 if (bitrate_data == NULL) 3362 rte_exit(EXIT_FAILURE, 3363 "Could not allocate bitrate data.\n"); 3364 rte_stats_bitrate_reg(bitrate_data); 3365 } 3366 #endif 3367 3368 #ifdef RTE_LIBRTE_CMDLINE 3369 if (strlen(cmdline_filename) != 0) 3370 cmdline_read_from_file(cmdline_filename); 3371 3372 if (interactive == 1) { 3373 if (auto_start) { 3374 printf("Start automatic packet forwarding\n"); 3375 start_packet_forwarding(0); 3376 } 3377 prompt(); 3378 pmd_test_exit(); 3379 } else 3380 #endif 3381 { 3382 char c; 3383 int rc; 3384 3385 f_quit = 0; 3386 3387 printf("No commandline core given, start packet forwarding\n"); 3388 start_packet_forwarding(tx_first); 3389 if (stats_period != 0) { 3390 uint64_t prev_time = 0, cur_time, diff_time = 0; 3391 uint64_t timer_period; 3392 3393 /* Convert to number of cycles */ 3394 timer_period = stats_period * rte_get_timer_hz(); 3395 3396 while (f_quit == 0) { 3397 cur_time = rte_get_timer_cycles(); 3398 diff_time += cur_time - prev_time; 3399 3400 if (diff_time >= timer_period) { 3401 print_stats(); 3402 /* Reset the timer */ 3403 diff_time = 0; 3404 } 3405 /* Sleep to avoid unnecessary checks */ 3406 prev_time = cur_time; 3407 sleep(1); 3408 } 3409 } 3410 3411 printf("Press enter to exit\n"); 3412 rc = read(0, &c, 1); 3413 pmd_test_exit(); 3414 if (rc < 0) 3415 return 1; 3416 } 3417 3418 return 0; 3419 } 3420