1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 192 uint16_t mempool_flags; 193 194 struct fwd_config cur_fwd_config; 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 196 uint32_t retry_enabled; 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 199 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 201 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 202 * specified on command-line. */ 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 204 205 /* 206 * In container, it cannot terminate the process which running with 'stats-period' 207 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 208 */ 209 uint8_t f_quit; 210 211 /* 212 * Configuration of packet segments used by the "txonly" processing engine. 213 */ 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 216 TXONLY_DEF_PACKET_LEN, 217 }; 218 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 219 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 221 /**< Split policy for packets to TX. */ 222 223 uint8_t txonly_multi_flow; 224 /**< Whether multiple flows are generated in TXONLY mode. */ 225 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 228 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 230 uint8_t dcb_config = 0; 231 232 /* Whether the dcb is in testing status */ 233 uint8_t dcb_test = 0; 234 235 /* 236 * Configurable number of RX/TX queues. 237 */ 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 240 241 /* 242 * Configurable number of RX/TX ring descriptors. 243 * Defaults are supplied by drivers via ethdev. 244 */ 245 #define RTE_TEST_RX_DESC_DEFAULT 0 246 #define RTE_TEST_TX_DESC_DEFAULT 0 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 249 250 #define RTE_PMD_PARAM_UNSET -1 251 /* 252 * Configurable values of RX and TX ring threshold registers. 253 */ 254 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 258 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 262 263 /* 264 * Configurable value of RX free threshold. 265 */ 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX drop enable. 270 */ 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of TX free threshold. 275 */ 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX RS bit threshold. 280 */ 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of buffered packets before sending. 285 */ 286 uint16_t noisy_tx_sw_bufsz; 287 288 /* 289 * Configurable value of packet buffer timeout. 290 */ 291 uint16_t noisy_tx_sw_buf_flush_time; 292 293 /* 294 * Configurable value for size of VNF internal memory area 295 * used for simulating noisy neighbour behaviour 296 */ 297 uint64_t noisy_lkup_mem_sz; 298 299 /* 300 * Configurable value of number of random writes done in 301 * VNF simulation memory area. 302 */ 303 uint64_t noisy_lkup_num_writes; 304 305 /* 306 * Configurable value of number of random reads done in 307 * VNF simulation memory area. 308 */ 309 uint64_t noisy_lkup_num_reads; 310 311 /* 312 * Configurable value of number of random reads/writes done in 313 * VNF simulation memory area. 314 */ 315 uint64_t noisy_lkup_num_reads_writes; 316 317 /* 318 * Receive Side Scaling (RSS) configuration. 319 */ 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 321 322 /* 323 * Port topology configuration 324 */ 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 326 327 /* 328 * Avoids to flush all the RX streams before starts forwarding. 329 */ 330 uint8_t no_flush_rx = 0; /* flush by default */ 331 332 /* 333 * Flow API isolated mode. 334 */ 335 uint8_t flow_isolate_all; 336 337 /* 338 * Avoids to check link status when starting/stopping a port. 339 */ 340 uint8_t no_link_check = 0; /* check by default */ 341 342 /* 343 * Don't automatically start all ports in interactive mode. 344 */ 345 uint8_t no_device_start = 0; 346 347 /* 348 * Enable link status change notification 349 */ 350 uint8_t lsc_interrupt = 1; /* enabled by default */ 351 352 /* 353 * Enable device removal notification. 354 */ 355 uint8_t rmv_interrupt = 1; /* enabled by default */ 356 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 358 359 /* After attach, port setup is called on event or by iterator */ 360 bool setup_on_probe_event = true; 361 362 /* Pretty printing of ethdev events */ 363 static const char * const eth_event_desc[] = { 364 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 365 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 366 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 367 [RTE_ETH_EVENT_INTR_RESET] = "reset", 368 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 369 [RTE_ETH_EVENT_IPSEC] = "IPsec", 370 [RTE_ETH_EVENT_MACSEC] = "MACsec", 371 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 372 [RTE_ETH_EVENT_NEW] = "device probed", 373 [RTE_ETH_EVENT_DESTROY] = "device released", 374 [RTE_ETH_EVENT_MAX] = NULL, 375 }; 376 377 /* 378 * Display or mask ether events 379 * Default to all events except VF_MBOX 380 */ 381 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 382 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 383 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 384 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 385 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 386 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 387 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 388 /* 389 * Decide if all memory are locked for performance. 390 */ 391 int do_mlockall = 0; 392 393 /* 394 * NIC bypass mode configuration options. 395 */ 396 397 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 398 /* The NIC bypass watchdog timeout. */ 399 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 400 #endif 401 402 403 #ifdef RTE_LIBRTE_LATENCY_STATS 404 405 /* 406 * Set when latency stats is enabled in the commandline 407 */ 408 uint8_t latencystats_enabled; 409 410 /* 411 * Lcore ID to serive latency statistics. 412 */ 413 lcoreid_t latencystats_lcore_id = -1; 414 415 #endif 416 417 /* 418 * Ethernet device configuration. 419 */ 420 struct rte_eth_rxmode rx_mode = { 421 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 422 /**< Default maximum frame length. */ 423 }; 424 425 struct rte_eth_txmode tx_mode = { 426 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 427 }; 428 429 struct rte_fdir_conf fdir_conf = { 430 .mode = RTE_FDIR_MODE_NONE, 431 .pballoc = RTE_FDIR_PBALLOC_64K, 432 .status = RTE_FDIR_REPORT_STATUS, 433 .mask = { 434 .vlan_tci_mask = 0xFFEF, 435 .ipv4_mask = { 436 .src_ip = 0xFFFFFFFF, 437 .dst_ip = 0xFFFFFFFF, 438 }, 439 .ipv6_mask = { 440 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 441 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 442 }, 443 .src_port_mask = 0xFFFF, 444 .dst_port_mask = 0xFFFF, 445 .mac_addr_byte_mask = 0xFF, 446 .tunnel_type_mask = 1, 447 .tunnel_id_mask = 0xFFFFFFFF, 448 }, 449 .drop_queue = 127, 450 }; 451 452 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 453 454 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 455 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 456 457 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 458 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 459 460 uint16_t nb_tx_queue_stats_mappings = 0; 461 uint16_t nb_rx_queue_stats_mappings = 0; 462 463 /* 464 * Display zero values by default for xstats 465 */ 466 uint8_t xstats_hide_zero; 467 468 unsigned int num_sockets = 0; 469 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 470 471 #ifdef RTE_LIBRTE_BITRATE 472 /* Bitrate statistics */ 473 struct rte_stats_bitrates *bitrate_data; 474 lcoreid_t bitrate_lcore_id; 475 uint8_t bitrate_enabled; 476 #endif 477 478 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 479 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 480 481 struct vxlan_encap_conf vxlan_encap_conf = { 482 .select_ipv4 = 1, 483 .select_vlan = 0, 484 .select_tos_ttl = 0, 485 .vni = "\x00\x00\x00", 486 .udp_src = 0, 487 .udp_dst = RTE_BE16(4789), 488 .ipv4_src = RTE_IPV4(127, 0, 0, 1), 489 .ipv4_dst = RTE_IPV4(255, 255, 255, 255), 490 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 491 "\x00\x00\x00\x00\x00\x00\x00\x01", 492 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 493 "\x00\x00\x00\x00\x00\x00\x11\x11", 494 .vlan_tci = 0, 495 .ip_tos = 0, 496 .ip_ttl = 255, 497 .eth_src = "\x00\x00\x00\x00\x00\x00", 498 .eth_dst = "\xff\xff\xff\xff\xff\xff", 499 }; 500 501 struct nvgre_encap_conf nvgre_encap_conf = { 502 .select_ipv4 = 1, 503 .select_vlan = 0, 504 .tni = "\x00\x00\x00", 505 .ipv4_src = RTE_IPV4(127, 0, 0, 1), 506 .ipv4_dst = RTE_IPV4(255, 255, 255, 255), 507 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 508 "\x00\x00\x00\x00\x00\x00\x00\x01", 509 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 510 "\x00\x00\x00\x00\x00\x00\x11\x11", 511 .vlan_tci = 0, 512 .eth_src = "\x00\x00\x00\x00\x00\x00", 513 .eth_dst = "\xff\xff\xff\xff\xff\xff", 514 }; 515 516 /* Forward function declarations */ 517 static void setup_attached_port(portid_t pi); 518 static void map_port_queue_stats_mapping_registers(portid_t pi, 519 struct rte_port *port); 520 static void check_all_ports_link_status(uint32_t port_mask); 521 static int eth_event_callback(portid_t port_id, 522 enum rte_eth_event_type type, 523 void *param, void *ret_param); 524 static void dev_event_callback(const char *device_name, 525 enum rte_dev_event_type type, 526 void *param); 527 528 /* 529 * Check if all the ports are started. 530 * If yes, return positive value. If not, return zero. 531 */ 532 static int all_ports_started(void); 533 534 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 535 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 536 537 /* 538 * Helper function to check if socket is already discovered. 539 * If yes, return positive value. If not, return zero. 540 */ 541 int 542 new_socket_id(unsigned int socket_id) 543 { 544 unsigned int i; 545 546 for (i = 0; i < num_sockets; i++) { 547 if (socket_ids[i] == socket_id) 548 return 0; 549 } 550 return 1; 551 } 552 553 /* 554 * Setup default configuration. 555 */ 556 static void 557 set_default_fwd_lcores_config(void) 558 { 559 unsigned int i; 560 unsigned int nb_lc; 561 unsigned int sock_num; 562 563 nb_lc = 0; 564 for (i = 0; i < RTE_MAX_LCORE; i++) { 565 if (!rte_lcore_is_enabled(i)) 566 continue; 567 sock_num = rte_lcore_to_socket_id(i); 568 if (new_socket_id(sock_num)) { 569 if (num_sockets >= RTE_MAX_NUMA_NODES) { 570 rte_exit(EXIT_FAILURE, 571 "Total sockets greater than %u\n", 572 RTE_MAX_NUMA_NODES); 573 } 574 socket_ids[num_sockets++] = sock_num; 575 } 576 if (i == rte_get_master_lcore()) 577 continue; 578 fwd_lcores_cpuids[nb_lc++] = i; 579 } 580 nb_lcores = (lcoreid_t) nb_lc; 581 nb_cfg_lcores = nb_lcores; 582 nb_fwd_lcores = 1; 583 } 584 585 static void 586 set_def_peer_eth_addrs(void) 587 { 588 portid_t i; 589 590 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 591 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 592 peer_eth_addrs[i].addr_bytes[5] = i; 593 } 594 } 595 596 static void 597 set_default_fwd_ports_config(void) 598 { 599 portid_t pt_id; 600 int i = 0; 601 602 RTE_ETH_FOREACH_DEV(pt_id) { 603 fwd_ports_ids[i++] = pt_id; 604 605 /* Update sockets info according to the attached device */ 606 int socket_id = rte_eth_dev_socket_id(pt_id); 607 if (socket_id >= 0 && new_socket_id(socket_id)) { 608 if (num_sockets >= RTE_MAX_NUMA_NODES) { 609 rte_exit(EXIT_FAILURE, 610 "Total sockets greater than %u\n", 611 RTE_MAX_NUMA_NODES); 612 } 613 socket_ids[num_sockets++] = socket_id; 614 } 615 } 616 617 nb_cfg_ports = nb_ports; 618 nb_fwd_ports = nb_ports; 619 } 620 621 void 622 set_def_fwd_config(void) 623 { 624 set_default_fwd_lcores_config(); 625 set_def_peer_eth_addrs(); 626 set_default_fwd_ports_config(); 627 } 628 629 /* extremely pessimistic estimation of memory required to create a mempool */ 630 static int 631 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 632 { 633 unsigned int n_pages, mbuf_per_pg, leftover; 634 uint64_t total_mem, mbuf_mem, obj_sz; 635 636 /* there is no good way to predict how much space the mempool will 637 * occupy because it will allocate chunks on the fly, and some of those 638 * will come from default DPDK memory while some will come from our 639 * external memory, so just assume 128MB will be enough for everyone. 640 */ 641 uint64_t hdr_mem = 128 << 20; 642 643 /* account for possible non-contiguousness */ 644 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 645 if (obj_sz > pgsz) { 646 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 647 return -1; 648 } 649 650 mbuf_per_pg = pgsz / obj_sz; 651 leftover = (nb_mbufs % mbuf_per_pg) > 0; 652 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 653 654 mbuf_mem = n_pages * pgsz; 655 656 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 657 658 if (total_mem > SIZE_MAX) { 659 TESTPMD_LOG(ERR, "Memory size too big\n"); 660 return -1; 661 } 662 *out = (size_t)total_mem; 663 664 return 0; 665 } 666 667 static int 668 pagesz_flags(uint64_t page_sz) 669 { 670 /* as per mmap() manpage, all page sizes are log2 of page size 671 * shifted by MAP_HUGE_SHIFT 672 */ 673 int log2 = rte_log2_u64(page_sz); 674 675 return (log2 << HUGE_SHIFT); 676 } 677 678 static void * 679 alloc_mem(size_t memsz, size_t pgsz, bool huge) 680 { 681 void *addr; 682 int flags; 683 684 /* allocate anonymous hugepages */ 685 flags = MAP_ANONYMOUS | MAP_PRIVATE; 686 if (huge) 687 flags |= HUGE_FLAG | pagesz_flags(pgsz); 688 689 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 690 if (addr == MAP_FAILED) 691 return NULL; 692 693 return addr; 694 } 695 696 struct extmem_param { 697 void *addr; 698 size_t len; 699 size_t pgsz; 700 rte_iova_t *iova_table; 701 unsigned int iova_table_len; 702 }; 703 704 static int 705 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 706 bool huge) 707 { 708 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 709 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 710 unsigned int cur_page, n_pages, pgsz_idx; 711 size_t mem_sz, cur_pgsz; 712 rte_iova_t *iovas = NULL; 713 void *addr; 714 int ret; 715 716 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 717 /* skip anything that is too big */ 718 if (pgsizes[pgsz_idx] > SIZE_MAX) 719 continue; 720 721 cur_pgsz = pgsizes[pgsz_idx]; 722 723 /* if we were told not to allocate hugepages, override */ 724 if (!huge) 725 cur_pgsz = sysconf(_SC_PAGESIZE); 726 727 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 728 if (ret < 0) { 729 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 730 return -1; 731 } 732 733 /* allocate our memory */ 734 addr = alloc_mem(mem_sz, cur_pgsz, huge); 735 736 /* if we couldn't allocate memory with a specified page size, 737 * that doesn't mean we can't do it with other page sizes, so 738 * try another one. 739 */ 740 if (addr == NULL) 741 continue; 742 743 /* store IOVA addresses for every page in this memory area */ 744 n_pages = mem_sz / cur_pgsz; 745 746 iovas = malloc(sizeof(*iovas) * n_pages); 747 748 if (iovas == NULL) { 749 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 750 goto fail; 751 } 752 /* lock memory if it's not huge pages */ 753 if (!huge) 754 mlock(addr, mem_sz); 755 756 /* populate IOVA addresses */ 757 for (cur_page = 0; cur_page < n_pages; cur_page++) { 758 rte_iova_t iova; 759 size_t offset; 760 void *cur; 761 762 offset = cur_pgsz * cur_page; 763 cur = RTE_PTR_ADD(addr, offset); 764 765 /* touch the page before getting its IOVA */ 766 *(volatile char *)cur = 0; 767 768 iova = rte_mem_virt2iova(cur); 769 770 iovas[cur_page] = iova; 771 } 772 773 break; 774 } 775 /* if we couldn't allocate anything */ 776 if (iovas == NULL) 777 return -1; 778 779 param->addr = addr; 780 param->len = mem_sz; 781 param->pgsz = cur_pgsz; 782 param->iova_table = iovas; 783 param->iova_table_len = n_pages; 784 785 return 0; 786 fail: 787 if (iovas) 788 free(iovas); 789 if (addr) 790 munmap(addr, mem_sz); 791 792 return -1; 793 } 794 795 static int 796 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 797 { 798 struct extmem_param param; 799 int socket_id, ret; 800 801 memset(¶m, 0, sizeof(param)); 802 803 /* check if our heap exists */ 804 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 805 if (socket_id < 0) { 806 /* create our heap */ 807 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 808 if (ret < 0) { 809 TESTPMD_LOG(ERR, "Cannot create heap\n"); 810 return -1; 811 } 812 } 813 814 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 815 if (ret < 0) { 816 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 817 return -1; 818 } 819 820 /* we now have a valid memory area, so add it to heap */ 821 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 822 param.addr, param.len, param.iova_table, 823 param.iova_table_len, param.pgsz); 824 825 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 826 827 /* not needed any more */ 828 free(param.iova_table); 829 830 if (ret < 0) { 831 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 832 munmap(param.addr, param.len); 833 return -1; 834 } 835 836 /* success */ 837 838 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 839 param.len >> 20); 840 841 return 0; 842 } 843 static void 844 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 845 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 846 { 847 uint16_t pid = 0; 848 int ret; 849 850 RTE_ETH_FOREACH_DEV(pid) { 851 struct rte_eth_dev *dev = 852 &rte_eth_devices[pid]; 853 854 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 855 memhdr->len); 856 if (ret) { 857 TESTPMD_LOG(DEBUG, 858 "unable to DMA unmap addr 0x%p " 859 "for device %s\n", 860 memhdr->addr, dev->data->name); 861 } 862 } 863 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 864 if (ret) { 865 TESTPMD_LOG(DEBUG, 866 "unable to un-register addr 0x%p\n", memhdr->addr); 867 } 868 } 869 870 static void 871 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 872 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 873 { 874 uint16_t pid = 0; 875 size_t page_size = sysconf(_SC_PAGESIZE); 876 int ret; 877 878 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 879 page_size); 880 if (ret) { 881 TESTPMD_LOG(DEBUG, 882 "unable to register addr 0x%p\n", memhdr->addr); 883 return; 884 } 885 RTE_ETH_FOREACH_DEV(pid) { 886 struct rte_eth_dev *dev = 887 &rte_eth_devices[pid]; 888 889 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 890 memhdr->len); 891 if (ret) { 892 TESTPMD_LOG(DEBUG, 893 "unable to DMA map addr 0x%p " 894 "for device %s\n", 895 memhdr->addr, dev->data->name); 896 } 897 } 898 } 899 900 /* 901 * Configuration initialisation done once at init time. 902 */ 903 static struct rte_mempool * 904 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 905 unsigned int socket_id) 906 { 907 char pool_name[RTE_MEMPOOL_NAMESIZE]; 908 struct rte_mempool *rte_mp = NULL; 909 uint32_t mb_size; 910 911 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 912 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 913 914 TESTPMD_LOG(INFO, 915 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 916 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 917 918 switch (mp_alloc_type) { 919 case MP_ALLOC_NATIVE: 920 { 921 /* wrapper to rte_mempool_create() */ 922 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 923 rte_mbuf_best_mempool_ops()); 924 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 925 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 926 break; 927 } 928 case MP_ALLOC_ANON: 929 { 930 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 931 mb_size, (unsigned int) mb_mempool_cache, 932 sizeof(struct rte_pktmbuf_pool_private), 933 socket_id, mempool_flags); 934 if (rte_mp == NULL) 935 goto err; 936 937 if (rte_mempool_populate_anon(rte_mp) == 0) { 938 rte_mempool_free(rte_mp); 939 rte_mp = NULL; 940 goto err; 941 } 942 rte_pktmbuf_pool_init(rte_mp, NULL); 943 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 944 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 945 break; 946 } 947 case MP_ALLOC_XMEM: 948 case MP_ALLOC_XMEM_HUGE: 949 { 950 int heap_socket; 951 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 952 953 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 954 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 955 956 heap_socket = 957 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 958 if (heap_socket < 0) 959 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 960 961 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 962 rte_mbuf_best_mempool_ops()); 963 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 964 mb_mempool_cache, 0, mbuf_seg_size, 965 heap_socket); 966 break; 967 } 968 default: 969 { 970 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 971 } 972 } 973 974 err: 975 if (rte_mp == NULL) { 976 rte_exit(EXIT_FAILURE, 977 "Creation of mbuf pool for socket %u failed: %s\n", 978 socket_id, rte_strerror(rte_errno)); 979 } else if (verbose_level > 0) { 980 rte_mempool_dump(stdout, rte_mp); 981 } 982 return rte_mp; 983 } 984 985 /* 986 * Check given socket id is valid or not with NUMA mode, 987 * if valid, return 0, else return -1 988 */ 989 static int 990 check_socket_id(const unsigned int socket_id) 991 { 992 static int warning_once = 0; 993 994 if (new_socket_id(socket_id)) { 995 if (!warning_once && numa_support) 996 printf("Warning: NUMA should be configured manually by" 997 " using --port-numa-config and" 998 " --ring-numa-config parameters along with" 999 " --numa.\n"); 1000 warning_once = 1; 1001 return -1; 1002 } 1003 return 0; 1004 } 1005 1006 /* 1007 * Get the allowed maximum number of RX queues. 1008 * *pid return the port id which has minimal value of 1009 * max_rx_queues in all ports. 1010 */ 1011 queueid_t 1012 get_allowed_max_nb_rxq(portid_t *pid) 1013 { 1014 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 1015 bool max_rxq_valid = false; 1016 portid_t pi; 1017 struct rte_eth_dev_info dev_info; 1018 1019 RTE_ETH_FOREACH_DEV(pi) { 1020 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1021 continue; 1022 1023 max_rxq_valid = true; 1024 if (dev_info.max_rx_queues < allowed_max_rxq) { 1025 allowed_max_rxq = dev_info.max_rx_queues; 1026 *pid = pi; 1027 } 1028 } 1029 return max_rxq_valid ? allowed_max_rxq : 0; 1030 } 1031 1032 /* 1033 * Check input rxq is valid or not. 1034 * If input rxq is not greater than any of maximum number 1035 * of RX queues of all ports, it is valid. 1036 * if valid, return 0, else return -1 1037 */ 1038 int 1039 check_nb_rxq(queueid_t rxq) 1040 { 1041 queueid_t allowed_max_rxq; 1042 portid_t pid = 0; 1043 1044 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1045 if (rxq > allowed_max_rxq) { 1046 printf("Fail: input rxq (%u) can't be greater " 1047 "than max_rx_queues (%u) of port %u\n", 1048 rxq, 1049 allowed_max_rxq, 1050 pid); 1051 return -1; 1052 } 1053 return 0; 1054 } 1055 1056 /* 1057 * Get the allowed maximum number of TX queues. 1058 * *pid return the port id which has minimal value of 1059 * max_tx_queues in all ports. 1060 */ 1061 queueid_t 1062 get_allowed_max_nb_txq(portid_t *pid) 1063 { 1064 queueid_t allowed_max_txq = MAX_QUEUE_ID; 1065 bool max_txq_valid = false; 1066 portid_t pi; 1067 struct rte_eth_dev_info dev_info; 1068 1069 RTE_ETH_FOREACH_DEV(pi) { 1070 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1071 continue; 1072 1073 max_txq_valid = true; 1074 if (dev_info.max_tx_queues < allowed_max_txq) { 1075 allowed_max_txq = dev_info.max_tx_queues; 1076 *pid = pi; 1077 } 1078 } 1079 return max_txq_valid ? allowed_max_txq : 0; 1080 } 1081 1082 /* 1083 * Check input txq is valid or not. 1084 * If input txq is not greater than any of maximum number 1085 * of TX queues of all ports, it is valid. 1086 * if valid, return 0, else return -1 1087 */ 1088 int 1089 check_nb_txq(queueid_t txq) 1090 { 1091 queueid_t allowed_max_txq; 1092 portid_t pid = 0; 1093 1094 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1095 if (txq > allowed_max_txq) { 1096 printf("Fail: input txq (%u) can't be greater " 1097 "than max_tx_queues (%u) of port %u\n", 1098 txq, 1099 allowed_max_txq, 1100 pid); 1101 return -1; 1102 } 1103 return 0; 1104 } 1105 1106 static void 1107 init_config(void) 1108 { 1109 portid_t pid; 1110 struct rte_port *port; 1111 struct rte_mempool *mbp; 1112 unsigned int nb_mbuf_per_pool; 1113 lcoreid_t lc_id; 1114 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1115 struct rte_gro_param gro_param; 1116 uint32_t gso_types; 1117 uint16_t data_size; 1118 bool warning = 0; 1119 int k; 1120 int ret; 1121 1122 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1123 1124 /* Configuration of logical cores. */ 1125 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1126 sizeof(struct fwd_lcore *) * nb_lcores, 1127 RTE_CACHE_LINE_SIZE); 1128 if (fwd_lcores == NULL) { 1129 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1130 "failed\n", nb_lcores); 1131 } 1132 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1133 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1134 sizeof(struct fwd_lcore), 1135 RTE_CACHE_LINE_SIZE); 1136 if (fwd_lcores[lc_id] == NULL) { 1137 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1138 "failed\n"); 1139 } 1140 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1141 } 1142 1143 RTE_ETH_FOREACH_DEV(pid) { 1144 port = &ports[pid]; 1145 /* Apply default TxRx configuration for all ports */ 1146 port->dev_conf.txmode = tx_mode; 1147 port->dev_conf.rxmode = rx_mode; 1148 1149 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1150 if (ret != 0) 1151 rte_exit(EXIT_FAILURE, 1152 "rte_eth_dev_info_get() failed\n"); 1153 1154 if (!(port->dev_info.tx_offload_capa & 1155 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1156 port->dev_conf.txmode.offloads &= 1157 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1158 if (!(port->dev_info.tx_offload_capa & 1159 DEV_TX_OFFLOAD_MATCH_METADATA)) 1160 port->dev_conf.txmode.offloads &= 1161 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1162 if (numa_support) { 1163 if (port_numa[pid] != NUMA_NO_CONFIG) 1164 port_per_socket[port_numa[pid]]++; 1165 else { 1166 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1167 1168 /* 1169 * if socket_id is invalid, 1170 * set to the first available socket. 1171 */ 1172 if (check_socket_id(socket_id) < 0) 1173 socket_id = socket_ids[0]; 1174 port_per_socket[socket_id]++; 1175 } 1176 } 1177 1178 /* Apply Rx offloads configuration */ 1179 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1180 port->rx_conf[k].offloads = 1181 port->dev_conf.rxmode.offloads; 1182 /* Apply Tx offloads configuration */ 1183 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1184 port->tx_conf[k].offloads = 1185 port->dev_conf.txmode.offloads; 1186 1187 /* set flag to initialize port/queue */ 1188 port->need_reconfig = 1; 1189 port->need_reconfig_queues = 1; 1190 port->tx_metadata = 0; 1191 1192 /* Check for maximum number of segments per MTU. Accordingly 1193 * update the mbuf data size. 1194 */ 1195 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1196 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1197 data_size = rx_mode.max_rx_pkt_len / 1198 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1199 1200 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1201 mbuf_data_size) { 1202 mbuf_data_size = data_size + 1203 RTE_PKTMBUF_HEADROOM; 1204 warning = 1; 1205 } 1206 } 1207 } 1208 1209 if (warning) 1210 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1211 mbuf_data_size); 1212 1213 /* 1214 * Create pools of mbuf. 1215 * If NUMA support is disabled, create a single pool of mbuf in 1216 * socket 0 memory by default. 1217 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1218 * 1219 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1220 * nb_txd can be configured at run time. 1221 */ 1222 if (param_total_num_mbufs) 1223 nb_mbuf_per_pool = param_total_num_mbufs; 1224 else { 1225 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1226 (nb_lcores * mb_mempool_cache) + 1227 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1228 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1229 } 1230 1231 if (numa_support) { 1232 uint8_t i; 1233 1234 for (i = 0; i < num_sockets; i++) 1235 mempools[i] = mbuf_pool_create(mbuf_data_size, 1236 nb_mbuf_per_pool, 1237 socket_ids[i]); 1238 } else { 1239 if (socket_num == UMA_NO_CONFIG) 1240 mempools[0] = mbuf_pool_create(mbuf_data_size, 1241 nb_mbuf_per_pool, 0); 1242 else 1243 mempools[socket_num] = mbuf_pool_create 1244 (mbuf_data_size, 1245 nb_mbuf_per_pool, 1246 socket_num); 1247 } 1248 1249 init_port_config(); 1250 1251 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1252 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1253 /* 1254 * Records which Mbuf pool to use by each logical core, if needed. 1255 */ 1256 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1257 mbp = mbuf_pool_find( 1258 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1259 1260 if (mbp == NULL) 1261 mbp = mbuf_pool_find(0); 1262 fwd_lcores[lc_id]->mbp = mbp; 1263 /* initialize GSO context */ 1264 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1265 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1266 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1267 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1268 RTE_ETHER_CRC_LEN; 1269 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1270 } 1271 1272 /* Configuration of packet forwarding streams. */ 1273 if (init_fwd_streams() < 0) 1274 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1275 1276 fwd_config_setup(); 1277 1278 /* create a gro context for each lcore */ 1279 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1280 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1281 gro_param.max_item_per_flow = MAX_PKT_BURST; 1282 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1283 gro_param.socket_id = rte_lcore_to_socket_id( 1284 fwd_lcores_cpuids[lc_id]); 1285 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1286 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1287 rte_exit(EXIT_FAILURE, 1288 "rte_gro_ctx_create() failed\n"); 1289 } 1290 } 1291 1292 #if defined RTE_LIBRTE_PMD_SOFTNIC 1293 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1294 RTE_ETH_FOREACH_DEV(pid) { 1295 port = &ports[pid]; 1296 const char *driver = port->dev_info.driver_name; 1297 1298 if (strcmp(driver, "net_softnic") == 0) 1299 port->softport.fwd_lcore_arg = fwd_lcores; 1300 } 1301 } 1302 #endif 1303 1304 } 1305 1306 1307 void 1308 reconfig(portid_t new_port_id, unsigned socket_id) 1309 { 1310 struct rte_port *port; 1311 int ret; 1312 1313 /* Reconfiguration of Ethernet ports. */ 1314 port = &ports[new_port_id]; 1315 1316 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1317 if (ret != 0) 1318 return; 1319 1320 /* set flag to initialize port/queue */ 1321 port->need_reconfig = 1; 1322 port->need_reconfig_queues = 1; 1323 port->socket_id = socket_id; 1324 1325 init_port_config(); 1326 } 1327 1328 1329 int 1330 init_fwd_streams(void) 1331 { 1332 portid_t pid; 1333 struct rte_port *port; 1334 streamid_t sm_id, nb_fwd_streams_new; 1335 queueid_t q; 1336 1337 /* set socket id according to numa or not */ 1338 RTE_ETH_FOREACH_DEV(pid) { 1339 port = &ports[pid]; 1340 if (nb_rxq > port->dev_info.max_rx_queues) { 1341 printf("Fail: nb_rxq(%d) is greater than " 1342 "max_rx_queues(%d)\n", nb_rxq, 1343 port->dev_info.max_rx_queues); 1344 return -1; 1345 } 1346 if (nb_txq > port->dev_info.max_tx_queues) { 1347 printf("Fail: nb_txq(%d) is greater than " 1348 "max_tx_queues(%d)\n", nb_txq, 1349 port->dev_info.max_tx_queues); 1350 return -1; 1351 } 1352 if (numa_support) { 1353 if (port_numa[pid] != NUMA_NO_CONFIG) 1354 port->socket_id = port_numa[pid]; 1355 else { 1356 port->socket_id = rte_eth_dev_socket_id(pid); 1357 1358 /* 1359 * if socket_id is invalid, 1360 * set to the first available socket. 1361 */ 1362 if (check_socket_id(port->socket_id) < 0) 1363 port->socket_id = socket_ids[0]; 1364 } 1365 } 1366 else { 1367 if (socket_num == UMA_NO_CONFIG) 1368 port->socket_id = 0; 1369 else 1370 port->socket_id = socket_num; 1371 } 1372 } 1373 1374 q = RTE_MAX(nb_rxq, nb_txq); 1375 if (q == 0) { 1376 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1377 return -1; 1378 } 1379 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1380 if (nb_fwd_streams_new == nb_fwd_streams) 1381 return 0; 1382 /* clear the old */ 1383 if (fwd_streams != NULL) { 1384 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1385 if (fwd_streams[sm_id] == NULL) 1386 continue; 1387 rte_free(fwd_streams[sm_id]); 1388 fwd_streams[sm_id] = NULL; 1389 } 1390 rte_free(fwd_streams); 1391 fwd_streams = NULL; 1392 } 1393 1394 /* init new */ 1395 nb_fwd_streams = nb_fwd_streams_new; 1396 if (nb_fwd_streams) { 1397 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1398 sizeof(struct fwd_stream *) * nb_fwd_streams, 1399 RTE_CACHE_LINE_SIZE); 1400 if (fwd_streams == NULL) 1401 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1402 " (struct fwd_stream *)) failed\n", 1403 nb_fwd_streams); 1404 1405 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1406 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1407 " struct fwd_stream", sizeof(struct fwd_stream), 1408 RTE_CACHE_LINE_SIZE); 1409 if (fwd_streams[sm_id] == NULL) 1410 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1411 "(struct fwd_stream) failed\n"); 1412 } 1413 } 1414 1415 return 0; 1416 } 1417 1418 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1419 static void 1420 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1421 { 1422 unsigned int total_burst; 1423 unsigned int nb_burst; 1424 unsigned int burst_stats[3]; 1425 uint16_t pktnb_stats[3]; 1426 uint16_t nb_pkt; 1427 int burst_percent[3]; 1428 1429 /* 1430 * First compute the total number of packet bursts and the 1431 * two highest numbers of bursts of the same number of packets. 1432 */ 1433 total_burst = 0; 1434 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1435 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1436 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1437 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1438 if (nb_burst == 0) 1439 continue; 1440 total_burst += nb_burst; 1441 if (nb_burst > burst_stats[0]) { 1442 burst_stats[1] = burst_stats[0]; 1443 pktnb_stats[1] = pktnb_stats[0]; 1444 burst_stats[0] = nb_burst; 1445 pktnb_stats[0] = nb_pkt; 1446 } else if (nb_burst > burst_stats[1]) { 1447 burst_stats[1] = nb_burst; 1448 pktnb_stats[1] = nb_pkt; 1449 } 1450 } 1451 if (total_burst == 0) 1452 return; 1453 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1454 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1455 burst_percent[0], (int) pktnb_stats[0]); 1456 if (burst_stats[0] == total_burst) { 1457 printf("]\n"); 1458 return; 1459 } 1460 if (burst_stats[0] + burst_stats[1] == total_burst) { 1461 printf(" + %d%% of %d pkts]\n", 1462 100 - burst_percent[0], pktnb_stats[1]); 1463 return; 1464 } 1465 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1466 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1467 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1468 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1469 return; 1470 } 1471 printf(" + %d%% of %d pkts + %d%% of others]\n", 1472 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1473 } 1474 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1475 1476 static void 1477 fwd_stream_stats_display(streamid_t stream_id) 1478 { 1479 struct fwd_stream *fs; 1480 static const char *fwd_top_stats_border = "-------"; 1481 1482 fs = fwd_streams[stream_id]; 1483 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1484 (fs->fwd_dropped == 0)) 1485 return; 1486 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1487 "TX Port=%2d/Queue=%2d %s\n", 1488 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1489 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1490 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1491 " TX-dropped: %-14"PRIu64, 1492 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1493 1494 /* if checksum mode */ 1495 if (cur_fwd_eng == &csum_fwd_engine) { 1496 printf(" RX- bad IP checksum: %-14"PRIu64 1497 " Rx- bad L4 checksum: %-14"PRIu64 1498 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1499 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1500 fs->rx_bad_outer_l4_csum); 1501 } else { 1502 printf("\n"); 1503 } 1504 1505 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1506 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1507 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1508 #endif 1509 } 1510 1511 void 1512 fwd_stats_display(void) 1513 { 1514 static const char *fwd_stats_border = "----------------------"; 1515 static const char *acc_stats_border = "+++++++++++++++"; 1516 struct { 1517 struct fwd_stream *rx_stream; 1518 struct fwd_stream *tx_stream; 1519 uint64_t tx_dropped; 1520 uint64_t rx_bad_ip_csum; 1521 uint64_t rx_bad_l4_csum; 1522 uint64_t rx_bad_outer_l4_csum; 1523 } ports_stats[RTE_MAX_ETHPORTS]; 1524 uint64_t total_rx_dropped = 0; 1525 uint64_t total_tx_dropped = 0; 1526 uint64_t total_rx_nombuf = 0; 1527 struct rte_eth_stats stats; 1528 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1529 uint64_t fwd_cycles = 0; 1530 #endif 1531 uint64_t total_recv = 0; 1532 uint64_t total_xmit = 0; 1533 struct rte_port *port; 1534 streamid_t sm_id; 1535 portid_t pt_id; 1536 int i; 1537 1538 memset(ports_stats, 0, sizeof(ports_stats)); 1539 1540 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1541 struct fwd_stream *fs = fwd_streams[sm_id]; 1542 1543 if (cur_fwd_config.nb_fwd_streams > 1544 cur_fwd_config.nb_fwd_ports) { 1545 fwd_stream_stats_display(sm_id); 1546 } else { 1547 ports_stats[fs->tx_port].tx_stream = fs; 1548 ports_stats[fs->rx_port].rx_stream = fs; 1549 } 1550 1551 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1552 1553 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1554 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1555 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1556 fs->rx_bad_outer_l4_csum; 1557 1558 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1559 fwd_cycles += fs->core_cycles; 1560 #endif 1561 } 1562 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1563 uint8_t j; 1564 1565 pt_id = fwd_ports_ids[i]; 1566 port = &ports[pt_id]; 1567 1568 rte_eth_stats_get(pt_id, &stats); 1569 stats.ipackets -= port->stats.ipackets; 1570 stats.opackets -= port->stats.opackets; 1571 stats.ibytes -= port->stats.ibytes; 1572 stats.obytes -= port->stats.obytes; 1573 stats.imissed -= port->stats.imissed; 1574 stats.oerrors -= port->stats.oerrors; 1575 stats.rx_nombuf -= port->stats.rx_nombuf; 1576 1577 total_recv += stats.ipackets; 1578 total_xmit += stats.opackets; 1579 total_rx_dropped += stats.imissed; 1580 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1581 total_tx_dropped += stats.oerrors; 1582 total_rx_nombuf += stats.rx_nombuf; 1583 1584 printf("\n %s Forward statistics for port %-2d %s\n", 1585 fwd_stats_border, pt_id, fwd_stats_border); 1586 1587 if (!port->rx_queue_stats_mapping_enabled && 1588 !port->tx_queue_stats_mapping_enabled) { 1589 printf(" RX-packets: %-14"PRIu64 1590 " RX-dropped: %-14"PRIu64 1591 "RX-total: %-"PRIu64"\n", 1592 stats.ipackets, stats.imissed, 1593 stats.ipackets + stats.imissed); 1594 1595 if (cur_fwd_eng == &csum_fwd_engine) 1596 printf(" Bad-ipcsum: %-14"PRIu64 1597 " Bad-l4csum: %-14"PRIu64 1598 "Bad-outer-l4csum: %-14"PRIu64"\n", 1599 ports_stats[pt_id].rx_bad_ip_csum, 1600 ports_stats[pt_id].rx_bad_l4_csum, 1601 ports_stats[pt_id].rx_bad_outer_l4_csum); 1602 if (stats.ierrors + stats.rx_nombuf > 0) { 1603 printf(" RX-error: %-"PRIu64"\n", 1604 stats.ierrors); 1605 printf(" RX-nombufs: %-14"PRIu64"\n", 1606 stats.rx_nombuf); 1607 } 1608 1609 printf(" TX-packets: %-14"PRIu64 1610 " TX-dropped: %-14"PRIu64 1611 "TX-total: %-"PRIu64"\n", 1612 stats.opackets, ports_stats[pt_id].tx_dropped, 1613 stats.opackets + ports_stats[pt_id].tx_dropped); 1614 } else { 1615 printf(" RX-packets: %14"PRIu64 1616 " RX-dropped:%14"PRIu64 1617 " RX-total:%14"PRIu64"\n", 1618 stats.ipackets, stats.imissed, 1619 stats.ipackets + stats.imissed); 1620 1621 if (cur_fwd_eng == &csum_fwd_engine) 1622 printf(" Bad-ipcsum:%14"PRIu64 1623 " Bad-l4csum:%14"PRIu64 1624 " Bad-outer-l4csum: %-14"PRIu64"\n", 1625 ports_stats[pt_id].rx_bad_ip_csum, 1626 ports_stats[pt_id].rx_bad_l4_csum, 1627 ports_stats[pt_id].rx_bad_outer_l4_csum); 1628 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1629 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1630 printf(" RX-nombufs: %14"PRIu64"\n", 1631 stats.rx_nombuf); 1632 } 1633 1634 printf(" TX-packets: %14"PRIu64 1635 " TX-dropped:%14"PRIu64 1636 " TX-total:%14"PRIu64"\n", 1637 stats.opackets, ports_stats[pt_id].tx_dropped, 1638 stats.opackets + ports_stats[pt_id].tx_dropped); 1639 } 1640 1641 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1642 if (ports_stats[pt_id].rx_stream) 1643 pkt_burst_stats_display("RX", 1644 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1645 if (ports_stats[pt_id].tx_stream) 1646 pkt_burst_stats_display("TX", 1647 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1648 #endif 1649 1650 if (port->rx_queue_stats_mapping_enabled) { 1651 printf("\n"); 1652 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1653 printf(" Stats reg %2d RX-packets:%14"PRIu64 1654 " RX-errors:%14"PRIu64 1655 " RX-bytes:%14"PRIu64"\n", 1656 j, stats.q_ipackets[j], 1657 stats.q_errors[j], stats.q_ibytes[j]); 1658 } 1659 printf("\n"); 1660 } 1661 if (port->tx_queue_stats_mapping_enabled) { 1662 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1663 printf(" Stats reg %2d TX-packets:%14"PRIu64 1664 " TX-bytes:%14" 1665 PRIu64"\n", 1666 j, stats.q_opackets[j], 1667 stats.q_obytes[j]); 1668 } 1669 } 1670 1671 printf(" %s--------------------------------%s\n", 1672 fwd_stats_border, fwd_stats_border); 1673 } 1674 1675 printf("\n %s Accumulated forward statistics for all ports" 1676 "%s\n", 1677 acc_stats_border, acc_stats_border); 1678 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1679 "%-"PRIu64"\n" 1680 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1681 "%-"PRIu64"\n", 1682 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1683 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1684 if (total_rx_nombuf > 0) 1685 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1686 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1687 "%s\n", 1688 acc_stats_border, acc_stats_border); 1689 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1690 if (total_recv > 0) 1691 printf("\n CPU cycles/packet=%u (total cycles=" 1692 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1693 (unsigned int)(fwd_cycles / total_recv), 1694 fwd_cycles, total_recv); 1695 #endif 1696 } 1697 1698 void 1699 fwd_stats_reset(void) 1700 { 1701 streamid_t sm_id; 1702 portid_t pt_id; 1703 int i; 1704 1705 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1706 pt_id = fwd_ports_ids[i]; 1707 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1708 } 1709 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1710 struct fwd_stream *fs = fwd_streams[sm_id]; 1711 1712 fs->rx_packets = 0; 1713 fs->tx_packets = 0; 1714 fs->fwd_dropped = 0; 1715 fs->rx_bad_ip_csum = 0; 1716 fs->rx_bad_l4_csum = 0; 1717 fs->rx_bad_outer_l4_csum = 0; 1718 1719 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1720 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1721 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1722 #endif 1723 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1724 fs->core_cycles = 0; 1725 #endif 1726 } 1727 } 1728 1729 static void 1730 flush_fwd_rx_queues(void) 1731 { 1732 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1733 portid_t rxp; 1734 portid_t port_id; 1735 queueid_t rxq; 1736 uint16_t nb_rx; 1737 uint16_t i; 1738 uint8_t j; 1739 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1740 uint64_t timer_period; 1741 1742 /* convert to number of cycles */ 1743 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1744 1745 for (j = 0; j < 2; j++) { 1746 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1747 for (rxq = 0; rxq < nb_rxq; rxq++) { 1748 port_id = fwd_ports_ids[rxp]; 1749 /** 1750 * testpmd can stuck in the below do while loop 1751 * if rte_eth_rx_burst() always returns nonzero 1752 * packets. So timer is added to exit this loop 1753 * after 1sec timer expiry. 1754 */ 1755 prev_tsc = rte_rdtsc(); 1756 do { 1757 nb_rx = rte_eth_rx_burst(port_id, rxq, 1758 pkts_burst, MAX_PKT_BURST); 1759 for (i = 0; i < nb_rx; i++) 1760 rte_pktmbuf_free(pkts_burst[i]); 1761 1762 cur_tsc = rte_rdtsc(); 1763 diff_tsc = cur_tsc - prev_tsc; 1764 timer_tsc += diff_tsc; 1765 } while ((nb_rx > 0) && 1766 (timer_tsc < timer_period)); 1767 timer_tsc = 0; 1768 } 1769 } 1770 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1771 } 1772 } 1773 1774 static void 1775 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1776 { 1777 struct fwd_stream **fsm; 1778 streamid_t nb_fs; 1779 streamid_t sm_id; 1780 #ifdef RTE_LIBRTE_BITRATE 1781 uint64_t tics_per_1sec; 1782 uint64_t tics_datum; 1783 uint64_t tics_current; 1784 uint16_t i, cnt_ports; 1785 1786 cnt_ports = nb_ports; 1787 tics_datum = rte_rdtsc(); 1788 tics_per_1sec = rte_get_timer_hz(); 1789 #endif 1790 fsm = &fwd_streams[fc->stream_idx]; 1791 nb_fs = fc->stream_nb; 1792 do { 1793 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1794 (*pkt_fwd)(fsm[sm_id]); 1795 #ifdef RTE_LIBRTE_BITRATE 1796 if (bitrate_enabled != 0 && 1797 bitrate_lcore_id == rte_lcore_id()) { 1798 tics_current = rte_rdtsc(); 1799 if (tics_current - tics_datum >= tics_per_1sec) { 1800 /* Periodic bitrate calculation */ 1801 for (i = 0; i < cnt_ports; i++) 1802 rte_stats_bitrate_calc(bitrate_data, 1803 ports_ids[i]); 1804 tics_datum = tics_current; 1805 } 1806 } 1807 #endif 1808 #ifdef RTE_LIBRTE_LATENCY_STATS 1809 if (latencystats_enabled != 0 && 1810 latencystats_lcore_id == rte_lcore_id()) 1811 rte_latencystats_update(); 1812 #endif 1813 1814 } while (! fc->stopped); 1815 } 1816 1817 static int 1818 start_pkt_forward_on_core(void *fwd_arg) 1819 { 1820 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1821 cur_fwd_config.fwd_eng->packet_fwd); 1822 return 0; 1823 } 1824 1825 /* 1826 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1827 * Used to start communication flows in network loopback test configurations. 1828 */ 1829 static int 1830 run_one_txonly_burst_on_core(void *fwd_arg) 1831 { 1832 struct fwd_lcore *fwd_lc; 1833 struct fwd_lcore tmp_lcore; 1834 1835 fwd_lc = (struct fwd_lcore *) fwd_arg; 1836 tmp_lcore = *fwd_lc; 1837 tmp_lcore.stopped = 1; 1838 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1839 return 0; 1840 } 1841 1842 /* 1843 * Launch packet forwarding: 1844 * - Setup per-port forwarding context. 1845 * - launch logical cores with their forwarding configuration. 1846 */ 1847 static void 1848 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1849 { 1850 port_fwd_begin_t port_fwd_begin; 1851 unsigned int i; 1852 unsigned int lc_id; 1853 int diag; 1854 1855 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1856 if (port_fwd_begin != NULL) { 1857 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1858 (*port_fwd_begin)(fwd_ports_ids[i]); 1859 } 1860 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1861 lc_id = fwd_lcores_cpuids[i]; 1862 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1863 fwd_lcores[i]->stopped = 0; 1864 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1865 fwd_lcores[i], lc_id); 1866 if (diag != 0) 1867 printf("launch lcore %u failed - diag=%d\n", 1868 lc_id, diag); 1869 } 1870 } 1871 } 1872 1873 /* 1874 * Launch packet forwarding configuration. 1875 */ 1876 void 1877 start_packet_forwarding(int with_tx_first) 1878 { 1879 port_fwd_begin_t port_fwd_begin; 1880 port_fwd_end_t port_fwd_end; 1881 struct rte_port *port; 1882 unsigned int i; 1883 portid_t pt_id; 1884 1885 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1886 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1887 1888 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1889 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1890 1891 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1892 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1893 (!nb_rxq || !nb_txq)) 1894 rte_exit(EXIT_FAILURE, 1895 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1896 cur_fwd_eng->fwd_mode_name); 1897 1898 if (all_ports_started() == 0) { 1899 printf("Not all ports were started\n"); 1900 return; 1901 } 1902 if (test_done == 0) { 1903 printf("Packet forwarding already started\n"); 1904 return; 1905 } 1906 1907 1908 if(dcb_test) { 1909 for (i = 0; i < nb_fwd_ports; i++) { 1910 pt_id = fwd_ports_ids[i]; 1911 port = &ports[pt_id]; 1912 if (!port->dcb_flag) { 1913 printf("In DCB mode, all forwarding ports must " 1914 "be configured in this mode.\n"); 1915 return; 1916 } 1917 } 1918 if (nb_fwd_lcores == 1) { 1919 printf("In DCB mode,the nb forwarding cores " 1920 "should be larger than 1.\n"); 1921 return; 1922 } 1923 } 1924 test_done = 0; 1925 1926 fwd_config_setup(); 1927 1928 if(!no_flush_rx) 1929 flush_fwd_rx_queues(); 1930 1931 pkt_fwd_config_display(&cur_fwd_config); 1932 rxtx_config_display(); 1933 1934 fwd_stats_reset(); 1935 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1936 pt_id = fwd_ports_ids[i]; 1937 port = &ports[pt_id]; 1938 map_port_queue_stats_mapping_registers(pt_id, port); 1939 } 1940 if (with_tx_first) { 1941 port_fwd_begin = tx_only_engine.port_fwd_begin; 1942 if (port_fwd_begin != NULL) { 1943 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1944 (*port_fwd_begin)(fwd_ports_ids[i]); 1945 } 1946 while (with_tx_first--) { 1947 launch_packet_forwarding( 1948 run_one_txonly_burst_on_core); 1949 rte_eal_mp_wait_lcore(); 1950 } 1951 port_fwd_end = tx_only_engine.port_fwd_end; 1952 if (port_fwd_end != NULL) { 1953 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1954 (*port_fwd_end)(fwd_ports_ids[i]); 1955 } 1956 } 1957 launch_packet_forwarding(start_pkt_forward_on_core); 1958 } 1959 1960 void 1961 stop_packet_forwarding(void) 1962 { 1963 port_fwd_end_t port_fwd_end; 1964 lcoreid_t lc_id; 1965 portid_t pt_id; 1966 int i; 1967 1968 if (test_done) { 1969 printf("Packet forwarding not started\n"); 1970 return; 1971 } 1972 printf("Telling cores to stop..."); 1973 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1974 fwd_lcores[lc_id]->stopped = 1; 1975 printf("\nWaiting for lcores to finish...\n"); 1976 rte_eal_mp_wait_lcore(); 1977 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1978 if (port_fwd_end != NULL) { 1979 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1980 pt_id = fwd_ports_ids[i]; 1981 (*port_fwd_end)(pt_id); 1982 } 1983 } 1984 1985 fwd_stats_display(); 1986 1987 printf("\nDone.\n"); 1988 test_done = 1; 1989 } 1990 1991 void 1992 dev_set_link_up(portid_t pid) 1993 { 1994 if (rte_eth_dev_set_link_up(pid) < 0) 1995 printf("\nSet link up fail.\n"); 1996 } 1997 1998 void 1999 dev_set_link_down(portid_t pid) 2000 { 2001 if (rte_eth_dev_set_link_down(pid) < 0) 2002 printf("\nSet link down fail.\n"); 2003 } 2004 2005 static int 2006 all_ports_started(void) 2007 { 2008 portid_t pi; 2009 struct rte_port *port; 2010 2011 RTE_ETH_FOREACH_DEV(pi) { 2012 port = &ports[pi]; 2013 /* Check if there is a port which is not started */ 2014 if ((port->port_status != RTE_PORT_STARTED) && 2015 (port->slave_flag == 0)) 2016 return 0; 2017 } 2018 2019 /* No port is not started */ 2020 return 1; 2021 } 2022 2023 int 2024 port_is_stopped(portid_t port_id) 2025 { 2026 struct rte_port *port = &ports[port_id]; 2027 2028 if ((port->port_status != RTE_PORT_STOPPED) && 2029 (port->slave_flag == 0)) 2030 return 0; 2031 return 1; 2032 } 2033 2034 int 2035 all_ports_stopped(void) 2036 { 2037 portid_t pi; 2038 2039 RTE_ETH_FOREACH_DEV(pi) { 2040 if (!port_is_stopped(pi)) 2041 return 0; 2042 } 2043 2044 return 1; 2045 } 2046 2047 int 2048 port_is_started(portid_t port_id) 2049 { 2050 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2051 return 0; 2052 2053 if (ports[port_id].port_status != RTE_PORT_STARTED) 2054 return 0; 2055 2056 return 1; 2057 } 2058 2059 int 2060 start_port(portid_t pid) 2061 { 2062 int diag, need_check_link_status = -1; 2063 portid_t pi; 2064 queueid_t qi; 2065 struct rte_port *port; 2066 struct rte_ether_addr mac_addr; 2067 2068 if (port_id_is_invalid(pid, ENABLED_WARN)) 2069 return 0; 2070 2071 if(dcb_config) 2072 dcb_test = 1; 2073 RTE_ETH_FOREACH_DEV(pi) { 2074 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2075 continue; 2076 2077 need_check_link_status = 0; 2078 port = &ports[pi]; 2079 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2080 RTE_PORT_HANDLING) == 0) { 2081 printf("Port %d is now not stopped\n", pi); 2082 continue; 2083 } 2084 2085 if (port->need_reconfig > 0) { 2086 port->need_reconfig = 0; 2087 2088 if (flow_isolate_all) { 2089 int ret = port_flow_isolate(pi, 1); 2090 if (ret) { 2091 printf("Failed to apply isolated" 2092 " mode on port %d\n", pi); 2093 return -1; 2094 } 2095 } 2096 configure_rxtx_dump_callbacks(0); 2097 printf("Configuring Port %d (socket %u)\n", pi, 2098 port->socket_id); 2099 /* configure port */ 2100 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 2101 &(port->dev_conf)); 2102 if (diag != 0) { 2103 if (rte_atomic16_cmpset(&(port->port_status), 2104 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2105 printf("Port %d can not be set back " 2106 "to stopped\n", pi); 2107 printf("Fail to configure port %d\n", pi); 2108 /* try to reconfigure port next time */ 2109 port->need_reconfig = 1; 2110 return -1; 2111 } 2112 } 2113 if (port->need_reconfig_queues > 0) { 2114 port->need_reconfig_queues = 0; 2115 /* setup tx queues */ 2116 for (qi = 0; qi < nb_txq; qi++) { 2117 if ((numa_support) && 2118 (txring_numa[pi] != NUMA_NO_CONFIG)) 2119 diag = rte_eth_tx_queue_setup(pi, qi, 2120 port->nb_tx_desc[qi], 2121 txring_numa[pi], 2122 &(port->tx_conf[qi])); 2123 else 2124 diag = rte_eth_tx_queue_setup(pi, qi, 2125 port->nb_tx_desc[qi], 2126 port->socket_id, 2127 &(port->tx_conf[qi])); 2128 2129 if (diag == 0) 2130 continue; 2131 2132 /* Fail to setup tx queue, return */ 2133 if (rte_atomic16_cmpset(&(port->port_status), 2134 RTE_PORT_HANDLING, 2135 RTE_PORT_STOPPED) == 0) 2136 printf("Port %d can not be set back " 2137 "to stopped\n", pi); 2138 printf("Fail to configure port %d tx queues\n", 2139 pi); 2140 /* try to reconfigure queues next time */ 2141 port->need_reconfig_queues = 1; 2142 return -1; 2143 } 2144 for (qi = 0; qi < nb_rxq; qi++) { 2145 /* setup rx queues */ 2146 if ((numa_support) && 2147 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2148 struct rte_mempool * mp = 2149 mbuf_pool_find(rxring_numa[pi]); 2150 if (mp == NULL) { 2151 printf("Failed to setup RX queue:" 2152 "No mempool allocation" 2153 " on the socket %d\n", 2154 rxring_numa[pi]); 2155 return -1; 2156 } 2157 2158 diag = rte_eth_rx_queue_setup(pi, qi, 2159 port->nb_rx_desc[qi], 2160 rxring_numa[pi], 2161 &(port->rx_conf[qi]), 2162 mp); 2163 } else { 2164 struct rte_mempool *mp = 2165 mbuf_pool_find(port->socket_id); 2166 if (mp == NULL) { 2167 printf("Failed to setup RX queue:" 2168 "No mempool allocation" 2169 " on the socket %d\n", 2170 port->socket_id); 2171 return -1; 2172 } 2173 diag = rte_eth_rx_queue_setup(pi, qi, 2174 port->nb_rx_desc[qi], 2175 port->socket_id, 2176 &(port->rx_conf[qi]), 2177 mp); 2178 } 2179 if (diag == 0) 2180 continue; 2181 2182 /* Fail to setup rx queue, return */ 2183 if (rte_atomic16_cmpset(&(port->port_status), 2184 RTE_PORT_HANDLING, 2185 RTE_PORT_STOPPED) == 0) 2186 printf("Port %d can not be set back " 2187 "to stopped\n", pi); 2188 printf("Fail to configure port %d rx queues\n", 2189 pi); 2190 /* try to reconfigure queues next time */ 2191 port->need_reconfig_queues = 1; 2192 return -1; 2193 } 2194 } 2195 configure_rxtx_dump_callbacks(verbose_level); 2196 /* start port */ 2197 if (rte_eth_dev_start(pi) < 0) { 2198 printf("Fail to start port %d\n", pi); 2199 2200 /* Fail to setup rx queue, return */ 2201 if (rte_atomic16_cmpset(&(port->port_status), 2202 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2203 printf("Port %d can not be set back to " 2204 "stopped\n", pi); 2205 continue; 2206 } 2207 2208 if (rte_atomic16_cmpset(&(port->port_status), 2209 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2210 printf("Port %d can not be set into started\n", pi); 2211 2212 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2213 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2214 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2215 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2216 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2217 2218 /* at least one port started, need checking link status */ 2219 need_check_link_status = 1; 2220 } 2221 2222 if (need_check_link_status == 1 && !no_link_check) 2223 check_all_ports_link_status(RTE_PORT_ALL); 2224 else if (need_check_link_status == 0) 2225 printf("Please stop the ports first\n"); 2226 2227 printf("Done\n"); 2228 return 0; 2229 } 2230 2231 void 2232 stop_port(portid_t pid) 2233 { 2234 portid_t pi; 2235 struct rte_port *port; 2236 int need_check_link_status = 0; 2237 2238 if (dcb_test) { 2239 dcb_test = 0; 2240 dcb_config = 0; 2241 } 2242 2243 if (port_id_is_invalid(pid, ENABLED_WARN)) 2244 return; 2245 2246 printf("Stopping ports...\n"); 2247 2248 RTE_ETH_FOREACH_DEV(pi) { 2249 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2250 continue; 2251 2252 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2253 printf("Please remove port %d from forwarding configuration.\n", pi); 2254 continue; 2255 } 2256 2257 if (port_is_bonding_slave(pi)) { 2258 printf("Please remove port %d from bonded device.\n", pi); 2259 continue; 2260 } 2261 2262 port = &ports[pi]; 2263 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2264 RTE_PORT_HANDLING) == 0) 2265 continue; 2266 2267 rte_eth_dev_stop(pi); 2268 2269 if (rte_atomic16_cmpset(&(port->port_status), 2270 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2271 printf("Port %d can not be set into stopped\n", pi); 2272 need_check_link_status = 1; 2273 } 2274 if (need_check_link_status && !no_link_check) 2275 check_all_ports_link_status(RTE_PORT_ALL); 2276 2277 printf("Done\n"); 2278 } 2279 2280 static void 2281 remove_invalid_ports_in(portid_t *array, portid_t *total) 2282 { 2283 portid_t i; 2284 portid_t new_total = 0; 2285 2286 for (i = 0; i < *total; i++) 2287 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2288 array[new_total] = array[i]; 2289 new_total++; 2290 } 2291 *total = new_total; 2292 } 2293 2294 static void 2295 remove_invalid_ports(void) 2296 { 2297 remove_invalid_ports_in(ports_ids, &nb_ports); 2298 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2299 nb_cfg_ports = nb_fwd_ports; 2300 } 2301 2302 void 2303 close_port(portid_t pid) 2304 { 2305 portid_t pi; 2306 struct rte_port *port; 2307 2308 if (port_id_is_invalid(pid, ENABLED_WARN)) 2309 return; 2310 2311 printf("Closing ports...\n"); 2312 2313 RTE_ETH_FOREACH_DEV(pi) { 2314 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2315 continue; 2316 2317 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2318 printf("Please remove port %d from forwarding configuration.\n", pi); 2319 continue; 2320 } 2321 2322 if (port_is_bonding_slave(pi)) { 2323 printf("Please remove port %d from bonded device.\n", pi); 2324 continue; 2325 } 2326 2327 port = &ports[pi]; 2328 if (rte_atomic16_cmpset(&(port->port_status), 2329 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2330 printf("Port %d is already closed\n", pi); 2331 continue; 2332 } 2333 2334 if (rte_atomic16_cmpset(&(port->port_status), 2335 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2336 printf("Port %d is now not stopped\n", pi); 2337 continue; 2338 } 2339 2340 if (port->flow_list) 2341 port_flow_flush(pi); 2342 rte_eth_dev_close(pi); 2343 2344 remove_invalid_ports(); 2345 2346 if (rte_atomic16_cmpset(&(port->port_status), 2347 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2348 printf("Port %d cannot be set to closed\n", pi); 2349 } 2350 2351 printf("Done\n"); 2352 } 2353 2354 void 2355 reset_port(portid_t pid) 2356 { 2357 int diag; 2358 portid_t pi; 2359 struct rte_port *port; 2360 2361 if (port_id_is_invalid(pid, ENABLED_WARN)) 2362 return; 2363 2364 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2365 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2366 printf("Can not reset port(s), please stop port(s) first.\n"); 2367 return; 2368 } 2369 2370 printf("Resetting ports...\n"); 2371 2372 RTE_ETH_FOREACH_DEV(pi) { 2373 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2374 continue; 2375 2376 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2377 printf("Please remove port %d from forwarding " 2378 "configuration.\n", pi); 2379 continue; 2380 } 2381 2382 if (port_is_bonding_slave(pi)) { 2383 printf("Please remove port %d from bonded device.\n", 2384 pi); 2385 continue; 2386 } 2387 2388 diag = rte_eth_dev_reset(pi); 2389 if (diag == 0) { 2390 port = &ports[pi]; 2391 port->need_reconfig = 1; 2392 port->need_reconfig_queues = 1; 2393 } else { 2394 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2395 } 2396 } 2397 2398 printf("Done\n"); 2399 } 2400 2401 void 2402 attach_port(char *identifier) 2403 { 2404 portid_t pi; 2405 struct rte_dev_iterator iterator; 2406 2407 printf("Attaching a new port...\n"); 2408 2409 if (identifier == NULL) { 2410 printf("Invalid parameters are specified\n"); 2411 return; 2412 } 2413 2414 if (rte_dev_probe(identifier) < 0) { 2415 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2416 return; 2417 } 2418 2419 /* first attach mode: event */ 2420 if (setup_on_probe_event) { 2421 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2422 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2423 if (ports[pi].port_status == RTE_PORT_HANDLING && 2424 ports[pi].need_setup != 0) 2425 setup_attached_port(pi); 2426 return; 2427 } 2428 2429 /* second attach mode: iterator */ 2430 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2431 /* setup ports matching the devargs used for probing */ 2432 if (port_is_forwarding(pi)) 2433 continue; /* port was already attached before */ 2434 setup_attached_port(pi); 2435 } 2436 } 2437 2438 static void 2439 setup_attached_port(portid_t pi) 2440 { 2441 unsigned int socket_id; 2442 int ret; 2443 2444 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2445 /* if socket_id is invalid, set to the first available socket. */ 2446 if (check_socket_id(socket_id) < 0) 2447 socket_id = socket_ids[0]; 2448 reconfig(pi, socket_id); 2449 ret = rte_eth_promiscuous_enable(pi); 2450 if (ret != 0) 2451 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2452 pi, rte_strerror(-ret)); 2453 2454 ports_ids[nb_ports++] = pi; 2455 fwd_ports_ids[nb_fwd_ports++] = pi; 2456 nb_cfg_ports = nb_fwd_ports; 2457 ports[pi].need_setup = 0; 2458 ports[pi].port_status = RTE_PORT_STOPPED; 2459 2460 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2461 printf("Done\n"); 2462 } 2463 2464 void 2465 detach_port_device(portid_t port_id) 2466 { 2467 struct rte_device *dev; 2468 portid_t sibling; 2469 2470 printf("Removing a device...\n"); 2471 2472 dev = rte_eth_devices[port_id].device; 2473 if (dev == NULL) { 2474 printf("Device already removed\n"); 2475 return; 2476 } 2477 2478 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2479 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2480 printf("Port not stopped\n"); 2481 return; 2482 } 2483 printf("Port was not closed\n"); 2484 if (ports[port_id].flow_list) 2485 port_flow_flush(port_id); 2486 } 2487 2488 if (rte_dev_remove(dev) < 0) { 2489 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2490 return; 2491 } 2492 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2493 /* reset mapping between old ports and removed device */ 2494 rte_eth_devices[sibling].device = NULL; 2495 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2496 /* sibling ports are forced to be closed */ 2497 ports[sibling].port_status = RTE_PORT_CLOSED; 2498 printf("Port %u is closed\n", sibling); 2499 } 2500 } 2501 2502 remove_invalid_ports(); 2503 2504 printf("Device of port %u is detached\n", port_id); 2505 printf("Now total ports is %d\n", nb_ports); 2506 printf("Done\n"); 2507 return; 2508 } 2509 2510 void 2511 detach_device(char *identifier) 2512 { 2513 struct rte_dev_iterator iterator; 2514 struct rte_devargs da; 2515 portid_t port_id; 2516 2517 printf("Removing a device...\n"); 2518 2519 memset(&da, 0, sizeof(da)); 2520 if (rte_devargs_parsef(&da, "%s", identifier)) { 2521 printf("cannot parse identifier\n"); 2522 if (da.args) 2523 free(da.args); 2524 return; 2525 } 2526 2527 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2528 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2529 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2530 printf("Port %u not stopped\n", port_id); 2531 return; 2532 } 2533 2534 /* sibling ports are forced to be closed */ 2535 if (ports[port_id].flow_list) 2536 port_flow_flush(port_id); 2537 ports[port_id].port_status = RTE_PORT_CLOSED; 2538 printf("Port %u is now closed\n", port_id); 2539 } 2540 } 2541 2542 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2543 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2544 da.name, da.bus->name); 2545 return; 2546 } 2547 2548 remove_invalid_ports(); 2549 2550 printf("Device %s is detached\n", identifier); 2551 printf("Now total ports is %d\n", nb_ports); 2552 printf("Done\n"); 2553 } 2554 2555 void 2556 pmd_test_exit(void) 2557 { 2558 portid_t pt_id; 2559 int ret; 2560 int i; 2561 2562 if (test_done == 0) 2563 stop_packet_forwarding(); 2564 2565 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2566 if (mempools[i]) { 2567 if (mp_alloc_type == MP_ALLOC_ANON) 2568 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2569 NULL); 2570 } 2571 } 2572 if (ports != NULL) { 2573 no_link_check = 1; 2574 RTE_ETH_FOREACH_DEV(pt_id) { 2575 printf("\nStopping port %d...\n", pt_id); 2576 fflush(stdout); 2577 stop_port(pt_id); 2578 } 2579 RTE_ETH_FOREACH_DEV(pt_id) { 2580 printf("\nShutting down port %d...\n", pt_id); 2581 fflush(stdout); 2582 close_port(pt_id); 2583 } 2584 } 2585 2586 if (hot_plug) { 2587 ret = rte_dev_event_monitor_stop(); 2588 if (ret) { 2589 RTE_LOG(ERR, EAL, 2590 "fail to stop device event monitor."); 2591 return; 2592 } 2593 2594 ret = rte_dev_event_callback_unregister(NULL, 2595 dev_event_callback, NULL); 2596 if (ret < 0) { 2597 RTE_LOG(ERR, EAL, 2598 "fail to unregister device event callback.\n"); 2599 return; 2600 } 2601 2602 ret = rte_dev_hotplug_handle_disable(); 2603 if (ret) { 2604 RTE_LOG(ERR, EAL, 2605 "fail to disable hotplug handling.\n"); 2606 return; 2607 } 2608 } 2609 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2610 if (mempools[i]) 2611 rte_mempool_free(mempools[i]); 2612 } 2613 2614 printf("\nBye...\n"); 2615 } 2616 2617 typedef void (*cmd_func_t)(void); 2618 struct pmd_test_command { 2619 const char *cmd_name; 2620 cmd_func_t cmd_func; 2621 }; 2622 2623 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2624 2625 /* Check the link status of all ports in up to 9s, and print them finally */ 2626 static void 2627 check_all_ports_link_status(uint32_t port_mask) 2628 { 2629 #define CHECK_INTERVAL 100 /* 100ms */ 2630 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2631 portid_t portid; 2632 uint8_t count, all_ports_up, print_flag = 0; 2633 struct rte_eth_link link; 2634 int ret; 2635 2636 printf("Checking link statuses...\n"); 2637 fflush(stdout); 2638 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2639 all_ports_up = 1; 2640 RTE_ETH_FOREACH_DEV(portid) { 2641 if ((port_mask & (1 << portid)) == 0) 2642 continue; 2643 memset(&link, 0, sizeof(link)); 2644 ret = rte_eth_link_get_nowait(portid, &link); 2645 if (ret < 0) { 2646 all_ports_up = 0; 2647 if (print_flag == 1) 2648 printf("Port %u link get failed: %s\n", 2649 portid, rte_strerror(-ret)); 2650 continue; 2651 } 2652 /* print link status if flag set */ 2653 if (print_flag == 1) { 2654 if (link.link_status) 2655 printf( 2656 "Port%d Link Up. speed %u Mbps- %s\n", 2657 portid, link.link_speed, 2658 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2659 ("full-duplex") : ("half-duplex\n")); 2660 else 2661 printf("Port %d Link Down\n", portid); 2662 continue; 2663 } 2664 /* clear all_ports_up flag if any link down */ 2665 if (link.link_status == ETH_LINK_DOWN) { 2666 all_ports_up = 0; 2667 break; 2668 } 2669 } 2670 /* after finally printing all link status, get out */ 2671 if (print_flag == 1) 2672 break; 2673 2674 if (all_ports_up == 0) { 2675 fflush(stdout); 2676 rte_delay_ms(CHECK_INTERVAL); 2677 } 2678 2679 /* set the print_flag if all ports up or timeout */ 2680 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2681 print_flag = 1; 2682 } 2683 2684 if (lsc_interrupt) 2685 break; 2686 } 2687 } 2688 2689 /* 2690 * This callback is for remove a port for a device. It has limitation because 2691 * it is not for multiple port removal for a device. 2692 * TODO: the device detach invoke will plan to be removed from user side to 2693 * eal. And convert all PMDs to free port resources on ether device closing. 2694 */ 2695 static void 2696 rmv_port_callback(void *arg) 2697 { 2698 int need_to_start = 0; 2699 int org_no_link_check = no_link_check; 2700 portid_t port_id = (intptr_t)arg; 2701 2702 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2703 2704 if (!test_done && port_is_forwarding(port_id)) { 2705 need_to_start = 1; 2706 stop_packet_forwarding(); 2707 } 2708 no_link_check = 1; 2709 stop_port(port_id); 2710 no_link_check = org_no_link_check; 2711 close_port(port_id); 2712 detach_port_device(port_id); 2713 if (need_to_start) 2714 start_packet_forwarding(0); 2715 } 2716 2717 /* This function is used by the interrupt thread */ 2718 static int 2719 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2720 void *ret_param) 2721 { 2722 RTE_SET_USED(param); 2723 RTE_SET_USED(ret_param); 2724 2725 if (type >= RTE_ETH_EVENT_MAX) { 2726 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2727 port_id, __func__, type); 2728 fflush(stderr); 2729 } else if (event_print_mask & (UINT32_C(1) << type)) { 2730 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2731 eth_event_desc[type]); 2732 fflush(stdout); 2733 } 2734 2735 switch (type) { 2736 case RTE_ETH_EVENT_NEW: 2737 ports[port_id].need_setup = 1; 2738 ports[port_id].port_status = RTE_PORT_HANDLING; 2739 break; 2740 case RTE_ETH_EVENT_INTR_RMV: 2741 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2742 break; 2743 if (rte_eal_alarm_set(100000, 2744 rmv_port_callback, (void *)(intptr_t)port_id)) 2745 fprintf(stderr, "Could not set up deferred device removal\n"); 2746 break; 2747 default: 2748 break; 2749 } 2750 return 0; 2751 } 2752 2753 static int 2754 register_eth_event_callback(void) 2755 { 2756 int ret; 2757 enum rte_eth_event_type event; 2758 2759 for (event = RTE_ETH_EVENT_UNKNOWN; 2760 event < RTE_ETH_EVENT_MAX; event++) { 2761 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2762 event, 2763 eth_event_callback, 2764 NULL); 2765 if (ret != 0) { 2766 TESTPMD_LOG(ERR, "Failed to register callback for " 2767 "%s event\n", eth_event_desc[event]); 2768 return -1; 2769 } 2770 } 2771 2772 return 0; 2773 } 2774 2775 /* This function is used by the interrupt thread */ 2776 static void 2777 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2778 __rte_unused void *arg) 2779 { 2780 uint16_t port_id; 2781 int ret; 2782 2783 if (type >= RTE_DEV_EVENT_MAX) { 2784 fprintf(stderr, "%s called upon invalid event %d\n", 2785 __func__, type); 2786 fflush(stderr); 2787 } 2788 2789 switch (type) { 2790 case RTE_DEV_EVENT_REMOVE: 2791 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2792 device_name); 2793 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2794 if (ret) { 2795 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2796 device_name); 2797 return; 2798 } 2799 /* 2800 * Because the user's callback is invoked in eal interrupt 2801 * callback, the interrupt callback need to be finished before 2802 * it can be unregistered when detaching device. So finish 2803 * callback soon and use a deferred removal to detach device 2804 * is need. It is a workaround, once the device detaching be 2805 * moved into the eal in the future, the deferred removal could 2806 * be deleted. 2807 */ 2808 if (rte_eal_alarm_set(100000, 2809 rmv_port_callback, (void *)(intptr_t)port_id)) 2810 RTE_LOG(ERR, EAL, 2811 "Could not set up deferred device removal\n"); 2812 break; 2813 case RTE_DEV_EVENT_ADD: 2814 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2815 device_name); 2816 /* TODO: After finish kernel driver binding, 2817 * begin to attach port. 2818 */ 2819 break; 2820 default: 2821 break; 2822 } 2823 } 2824 2825 static int 2826 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2827 { 2828 uint16_t i; 2829 int diag; 2830 uint8_t mapping_found = 0; 2831 2832 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2833 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2834 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2835 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2836 tx_queue_stats_mappings[i].queue_id, 2837 tx_queue_stats_mappings[i].stats_counter_id); 2838 if (diag != 0) 2839 return diag; 2840 mapping_found = 1; 2841 } 2842 } 2843 if (mapping_found) 2844 port->tx_queue_stats_mapping_enabled = 1; 2845 return 0; 2846 } 2847 2848 static int 2849 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2850 { 2851 uint16_t i; 2852 int diag; 2853 uint8_t mapping_found = 0; 2854 2855 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2856 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2857 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2858 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2859 rx_queue_stats_mappings[i].queue_id, 2860 rx_queue_stats_mappings[i].stats_counter_id); 2861 if (diag != 0) 2862 return diag; 2863 mapping_found = 1; 2864 } 2865 } 2866 if (mapping_found) 2867 port->rx_queue_stats_mapping_enabled = 1; 2868 return 0; 2869 } 2870 2871 static void 2872 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2873 { 2874 int diag = 0; 2875 2876 diag = set_tx_queue_stats_mapping_registers(pi, port); 2877 if (diag != 0) { 2878 if (diag == -ENOTSUP) { 2879 port->tx_queue_stats_mapping_enabled = 0; 2880 printf("TX queue stats mapping not supported port id=%d\n", pi); 2881 } 2882 else 2883 rte_exit(EXIT_FAILURE, 2884 "set_tx_queue_stats_mapping_registers " 2885 "failed for port id=%d diag=%d\n", 2886 pi, diag); 2887 } 2888 2889 diag = set_rx_queue_stats_mapping_registers(pi, port); 2890 if (diag != 0) { 2891 if (diag == -ENOTSUP) { 2892 port->rx_queue_stats_mapping_enabled = 0; 2893 printf("RX queue stats mapping not supported port id=%d\n", pi); 2894 } 2895 else 2896 rte_exit(EXIT_FAILURE, 2897 "set_rx_queue_stats_mapping_registers " 2898 "failed for port id=%d diag=%d\n", 2899 pi, diag); 2900 } 2901 } 2902 2903 static void 2904 rxtx_port_config(struct rte_port *port) 2905 { 2906 uint16_t qid; 2907 uint64_t offloads; 2908 2909 for (qid = 0; qid < nb_rxq; qid++) { 2910 offloads = port->rx_conf[qid].offloads; 2911 port->rx_conf[qid] = port->dev_info.default_rxconf; 2912 if (offloads != 0) 2913 port->rx_conf[qid].offloads = offloads; 2914 2915 /* Check if any Rx parameters have been passed */ 2916 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2917 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2918 2919 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2920 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2921 2922 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2923 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2924 2925 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2926 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2927 2928 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2929 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2930 2931 port->nb_rx_desc[qid] = nb_rxd; 2932 } 2933 2934 for (qid = 0; qid < nb_txq; qid++) { 2935 offloads = port->tx_conf[qid].offloads; 2936 port->tx_conf[qid] = port->dev_info.default_txconf; 2937 if (offloads != 0) 2938 port->tx_conf[qid].offloads = offloads; 2939 2940 /* Check if any Tx parameters have been passed */ 2941 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2942 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2943 2944 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2945 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2946 2947 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2948 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2949 2950 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2951 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2952 2953 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2954 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2955 2956 port->nb_tx_desc[qid] = nb_txd; 2957 } 2958 } 2959 2960 void 2961 init_port_config(void) 2962 { 2963 portid_t pid; 2964 struct rte_port *port; 2965 int ret; 2966 2967 RTE_ETH_FOREACH_DEV(pid) { 2968 port = &ports[pid]; 2969 port->dev_conf.fdir_conf = fdir_conf; 2970 2971 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 2972 if (ret != 0) 2973 return; 2974 2975 if (nb_rxq > 1) { 2976 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2977 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2978 rss_hf & port->dev_info.flow_type_rss_offloads; 2979 } else { 2980 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2981 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2982 } 2983 2984 if (port->dcb_flag == 0) { 2985 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2986 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2987 else 2988 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2989 } 2990 2991 rxtx_port_config(port); 2992 2993 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 2994 if (ret != 0) 2995 return; 2996 2997 map_port_queue_stats_mapping_registers(pid, port); 2998 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2999 rte_pmd_ixgbe_bypass_init(pid); 3000 #endif 3001 3002 if (lsc_interrupt && 3003 (rte_eth_devices[pid].data->dev_flags & 3004 RTE_ETH_DEV_INTR_LSC)) 3005 port->dev_conf.intr_conf.lsc = 1; 3006 if (rmv_interrupt && 3007 (rte_eth_devices[pid].data->dev_flags & 3008 RTE_ETH_DEV_INTR_RMV)) 3009 port->dev_conf.intr_conf.rmv = 1; 3010 } 3011 } 3012 3013 void set_port_slave_flag(portid_t slave_pid) 3014 { 3015 struct rte_port *port; 3016 3017 port = &ports[slave_pid]; 3018 port->slave_flag = 1; 3019 } 3020 3021 void clear_port_slave_flag(portid_t slave_pid) 3022 { 3023 struct rte_port *port; 3024 3025 port = &ports[slave_pid]; 3026 port->slave_flag = 0; 3027 } 3028 3029 uint8_t port_is_bonding_slave(portid_t slave_pid) 3030 { 3031 struct rte_port *port; 3032 3033 port = &ports[slave_pid]; 3034 if ((rte_eth_devices[slave_pid].data->dev_flags & 3035 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3036 return 1; 3037 return 0; 3038 } 3039 3040 const uint16_t vlan_tags[] = { 3041 0, 1, 2, 3, 4, 5, 6, 7, 3042 8, 9, 10, 11, 12, 13, 14, 15, 3043 16, 17, 18, 19, 20, 21, 22, 23, 3044 24, 25, 26, 27, 28, 29, 30, 31 3045 }; 3046 3047 static int 3048 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3049 enum dcb_mode_enable dcb_mode, 3050 enum rte_eth_nb_tcs num_tcs, 3051 uint8_t pfc_en) 3052 { 3053 uint8_t i; 3054 int32_t rc; 3055 struct rte_eth_rss_conf rss_conf; 3056 3057 /* 3058 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3059 * given above, and the number of traffic classes available for use. 3060 */ 3061 if (dcb_mode == DCB_VT_ENABLED) { 3062 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3063 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3064 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3065 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3066 3067 /* VMDQ+DCB RX and TX configurations */ 3068 vmdq_rx_conf->enable_default_pool = 0; 3069 vmdq_rx_conf->default_pool = 0; 3070 vmdq_rx_conf->nb_queue_pools = 3071 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3072 vmdq_tx_conf->nb_queue_pools = 3073 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3074 3075 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3076 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3077 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3078 vmdq_rx_conf->pool_map[i].pools = 3079 1 << (i % vmdq_rx_conf->nb_queue_pools); 3080 } 3081 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3082 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3083 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3084 } 3085 3086 /* set DCB mode of RX and TX of multiple queues */ 3087 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 3088 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3089 } else { 3090 struct rte_eth_dcb_rx_conf *rx_conf = 3091 ð_conf->rx_adv_conf.dcb_rx_conf; 3092 struct rte_eth_dcb_tx_conf *tx_conf = 3093 ð_conf->tx_adv_conf.dcb_tx_conf; 3094 3095 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3096 if (rc != 0) 3097 return rc; 3098 3099 rx_conf->nb_tcs = num_tcs; 3100 tx_conf->nb_tcs = num_tcs; 3101 3102 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3103 rx_conf->dcb_tc[i] = i % num_tcs; 3104 tx_conf->dcb_tc[i] = i % num_tcs; 3105 } 3106 3107 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 3108 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3109 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3110 } 3111 3112 if (pfc_en) 3113 eth_conf->dcb_capability_en = 3114 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3115 else 3116 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3117 3118 return 0; 3119 } 3120 3121 int 3122 init_port_dcb_config(portid_t pid, 3123 enum dcb_mode_enable dcb_mode, 3124 enum rte_eth_nb_tcs num_tcs, 3125 uint8_t pfc_en) 3126 { 3127 struct rte_eth_conf port_conf; 3128 struct rte_port *rte_port; 3129 int retval; 3130 uint16_t i; 3131 3132 rte_port = &ports[pid]; 3133 3134 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3135 /* Enter DCB configuration status */ 3136 dcb_config = 1; 3137 3138 port_conf.rxmode = rte_port->dev_conf.rxmode; 3139 port_conf.txmode = rte_port->dev_conf.txmode; 3140 3141 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3142 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3143 if (retval < 0) 3144 return retval; 3145 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3146 3147 /* re-configure the device . */ 3148 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3149 if (retval < 0) 3150 return retval; 3151 3152 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3153 if (retval != 0) 3154 return retval; 3155 3156 /* If dev_info.vmdq_pool_base is greater than 0, 3157 * the queue id of vmdq pools is started after pf queues. 3158 */ 3159 if (dcb_mode == DCB_VT_ENABLED && 3160 rte_port->dev_info.vmdq_pool_base > 0) { 3161 printf("VMDQ_DCB multi-queue mode is nonsensical" 3162 " for port %d.", pid); 3163 return -1; 3164 } 3165 3166 /* Assume the ports in testpmd have the same dcb capability 3167 * and has the same number of rxq and txq in dcb mode 3168 */ 3169 if (dcb_mode == DCB_VT_ENABLED) { 3170 if (rte_port->dev_info.max_vfs > 0) { 3171 nb_rxq = rte_port->dev_info.nb_rx_queues; 3172 nb_txq = rte_port->dev_info.nb_tx_queues; 3173 } else { 3174 nb_rxq = rte_port->dev_info.max_rx_queues; 3175 nb_txq = rte_port->dev_info.max_tx_queues; 3176 } 3177 } else { 3178 /*if vt is disabled, use all pf queues */ 3179 if (rte_port->dev_info.vmdq_pool_base == 0) { 3180 nb_rxq = rte_port->dev_info.max_rx_queues; 3181 nb_txq = rte_port->dev_info.max_tx_queues; 3182 } else { 3183 nb_rxq = (queueid_t)num_tcs; 3184 nb_txq = (queueid_t)num_tcs; 3185 3186 } 3187 } 3188 rx_free_thresh = 64; 3189 3190 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3191 3192 rxtx_port_config(rte_port); 3193 /* VLAN filter */ 3194 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3195 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3196 rx_vft_set(pid, vlan_tags[i], 1); 3197 3198 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3199 if (retval != 0) 3200 return retval; 3201 3202 map_port_queue_stats_mapping_registers(pid, rte_port); 3203 3204 rte_port->dcb_flag = 1; 3205 3206 return 0; 3207 } 3208 3209 static void 3210 init_port(void) 3211 { 3212 /* Configuration of Ethernet ports. */ 3213 ports = rte_zmalloc("testpmd: ports", 3214 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3215 RTE_CACHE_LINE_SIZE); 3216 if (ports == NULL) { 3217 rte_exit(EXIT_FAILURE, 3218 "rte_zmalloc(%d struct rte_port) failed\n", 3219 RTE_MAX_ETHPORTS); 3220 } 3221 3222 /* Initialize ports NUMA structures */ 3223 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3224 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3225 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3226 } 3227 3228 static void 3229 force_quit(void) 3230 { 3231 pmd_test_exit(); 3232 prompt_exit(); 3233 } 3234 3235 static void 3236 print_stats(void) 3237 { 3238 uint8_t i; 3239 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3240 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3241 3242 /* Clear screen and move to top left */ 3243 printf("%s%s", clr, top_left); 3244 3245 printf("\nPort statistics ===================================="); 3246 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3247 nic_stats_display(fwd_ports_ids[i]); 3248 3249 fflush(stdout); 3250 } 3251 3252 static void 3253 signal_handler(int signum) 3254 { 3255 if (signum == SIGINT || signum == SIGTERM) { 3256 printf("\nSignal %d received, preparing to exit...\n", 3257 signum); 3258 #ifdef RTE_LIBRTE_PDUMP 3259 /* uninitialize packet capture framework */ 3260 rte_pdump_uninit(); 3261 #endif 3262 #ifdef RTE_LIBRTE_LATENCY_STATS 3263 if (latencystats_enabled != 0) 3264 rte_latencystats_uninit(); 3265 #endif 3266 force_quit(); 3267 /* Set flag to indicate the force termination. */ 3268 f_quit = 1; 3269 /* exit with the expected status */ 3270 signal(signum, SIG_DFL); 3271 kill(getpid(), signum); 3272 } 3273 } 3274 3275 int 3276 main(int argc, char** argv) 3277 { 3278 int diag; 3279 portid_t port_id; 3280 uint16_t count; 3281 int ret; 3282 3283 signal(SIGINT, signal_handler); 3284 signal(SIGTERM, signal_handler); 3285 3286 testpmd_logtype = rte_log_register("testpmd"); 3287 if (testpmd_logtype < 0) 3288 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3289 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3290 3291 diag = rte_eal_init(argc, argv); 3292 if (diag < 0) 3293 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3294 rte_strerror(rte_errno)); 3295 3296 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3297 rte_exit(EXIT_FAILURE, 3298 "Secondary process type not supported.\n"); 3299 3300 ret = register_eth_event_callback(); 3301 if (ret != 0) 3302 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3303 3304 #ifdef RTE_LIBRTE_PDUMP 3305 /* initialize packet capture framework */ 3306 rte_pdump_init(); 3307 #endif 3308 3309 count = 0; 3310 RTE_ETH_FOREACH_DEV(port_id) { 3311 ports_ids[count] = port_id; 3312 count++; 3313 } 3314 nb_ports = (portid_t) count; 3315 if (nb_ports == 0) 3316 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3317 3318 /* allocate port structures, and init them */ 3319 init_port(); 3320 3321 set_def_fwd_config(); 3322 if (nb_lcores == 0) 3323 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3324 "Check the core mask argument\n"); 3325 3326 /* Bitrate/latency stats disabled by default */ 3327 #ifdef RTE_LIBRTE_BITRATE 3328 bitrate_enabled = 0; 3329 #endif 3330 #ifdef RTE_LIBRTE_LATENCY_STATS 3331 latencystats_enabled = 0; 3332 #endif 3333 3334 /* on FreeBSD, mlockall() is disabled by default */ 3335 #ifdef RTE_EXEC_ENV_FREEBSD 3336 do_mlockall = 0; 3337 #else 3338 do_mlockall = 1; 3339 #endif 3340 3341 argc -= diag; 3342 argv += diag; 3343 if (argc > 1) 3344 launch_args_parse(argc, argv); 3345 3346 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3347 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3348 strerror(errno)); 3349 } 3350 3351 if (tx_first && interactive) 3352 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3353 "interactive mode.\n"); 3354 3355 if (tx_first && lsc_interrupt) { 3356 printf("Warning: lsc_interrupt needs to be off when " 3357 " using tx_first. Disabling.\n"); 3358 lsc_interrupt = 0; 3359 } 3360 3361 if (!nb_rxq && !nb_txq) 3362 printf("Warning: Either rx or tx queues should be non-zero\n"); 3363 3364 if (nb_rxq > 1 && nb_rxq > nb_txq) 3365 printf("Warning: nb_rxq=%d enables RSS configuration, " 3366 "but nb_txq=%d will prevent to fully test it.\n", 3367 nb_rxq, nb_txq); 3368 3369 init_config(); 3370 3371 if (hot_plug) { 3372 ret = rte_dev_hotplug_handle_enable(); 3373 if (ret) { 3374 RTE_LOG(ERR, EAL, 3375 "fail to enable hotplug handling."); 3376 return -1; 3377 } 3378 3379 ret = rte_dev_event_monitor_start(); 3380 if (ret) { 3381 RTE_LOG(ERR, EAL, 3382 "fail to start device event monitoring."); 3383 return -1; 3384 } 3385 3386 ret = rte_dev_event_callback_register(NULL, 3387 dev_event_callback, NULL); 3388 if (ret) { 3389 RTE_LOG(ERR, EAL, 3390 "fail to register device event callback\n"); 3391 return -1; 3392 } 3393 } 3394 3395 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3396 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3397 3398 /* set all ports to promiscuous mode by default */ 3399 RTE_ETH_FOREACH_DEV(port_id) { 3400 ret = rte_eth_promiscuous_enable(port_id); 3401 if (ret != 0) 3402 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3403 port_id, rte_strerror(-ret)); 3404 } 3405 3406 /* Init metrics library */ 3407 rte_metrics_init(rte_socket_id()); 3408 3409 #ifdef RTE_LIBRTE_LATENCY_STATS 3410 if (latencystats_enabled != 0) { 3411 int ret = rte_latencystats_init(1, NULL); 3412 if (ret) 3413 printf("Warning: latencystats init()" 3414 " returned error %d\n", ret); 3415 printf("Latencystats running on lcore %d\n", 3416 latencystats_lcore_id); 3417 } 3418 #endif 3419 3420 /* Setup bitrate stats */ 3421 #ifdef RTE_LIBRTE_BITRATE 3422 if (bitrate_enabled != 0) { 3423 bitrate_data = rte_stats_bitrate_create(); 3424 if (bitrate_data == NULL) 3425 rte_exit(EXIT_FAILURE, 3426 "Could not allocate bitrate data.\n"); 3427 rte_stats_bitrate_reg(bitrate_data); 3428 } 3429 #endif 3430 3431 #ifdef RTE_LIBRTE_CMDLINE 3432 if (strlen(cmdline_filename) != 0) 3433 cmdline_read_from_file(cmdline_filename); 3434 3435 if (interactive == 1) { 3436 if (auto_start) { 3437 printf("Start automatic packet forwarding\n"); 3438 start_packet_forwarding(0); 3439 } 3440 prompt(); 3441 pmd_test_exit(); 3442 } else 3443 #endif 3444 { 3445 char c; 3446 int rc; 3447 3448 f_quit = 0; 3449 3450 printf("No commandline core given, start packet forwarding\n"); 3451 start_packet_forwarding(tx_first); 3452 if (stats_period != 0) { 3453 uint64_t prev_time = 0, cur_time, diff_time = 0; 3454 uint64_t timer_period; 3455 3456 /* Convert to number of cycles */ 3457 timer_period = stats_period * rte_get_timer_hz(); 3458 3459 while (f_quit == 0) { 3460 cur_time = rte_get_timer_cycles(); 3461 diff_time += cur_time - prev_time; 3462 3463 if (diff_time >= timer_period) { 3464 print_stats(); 3465 /* Reset the timer */ 3466 diff_time = 0; 3467 } 3468 /* Sleep to avoid unnecessary checks */ 3469 prev_time = cur_time; 3470 sleep(1); 3471 } 3472 } 3473 3474 printf("Press enter to exit\n"); 3475 rc = read(0, &c, 1); 3476 pmd_test_exit(); 3477 if (rc < 0) 3478 return 1; 3479 } 3480 3481 return 0; 3482 } 3483