1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdarg.h> 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <signal.h> 38 #include <string.h> 39 #include <time.h> 40 #include <fcntl.h> 41 #include <sys/types.h> 42 #include <errno.h> 43 44 #include <sys/queue.h> 45 #include <sys/stat.h> 46 47 #include <stdint.h> 48 #include <unistd.h> 49 #include <inttypes.h> 50 51 #include <rte_common.h> 52 #include <rte_errno.h> 53 #include <rte_byteorder.h> 54 #include <rte_log.h> 55 #include <rte_debug.h> 56 #include <rte_cycles.h> 57 #include <rte_memory.h> 58 #include <rte_memcpy.h> 59 #include <rte_memzone.h> 60 #include <rte_launch.h> 61 #include <rte_eal.h> 62 #include <rte_alarm.h> 63 #include <rte_per_lcore.h> 64 #include <rte_lcore.h> 65 #include <rte_atomic.h> 66 #include <rte_branch_prediction.h> 67 #include <rte_mempool.h> 68 #include <rte_malloc.h> 69 #include <rte_mbuf.h> 70 #include <rte_interrupts.h> 71 #include <rte_pci.h> 72 #include <rte_ether.h> 73 #include <rte_ethdev.h> 74 #include <rte_dev.h> 75 #include <rte_string_fns.h> 76 #ifdef RTE_LIBRTE_IXGBE_PMD 77 #include <rte_pmd_ixgbe.h> 78 #endif 79 #ifdef RTE_LIBRTE_PMD_XENVIRT 80 #include <rte_eth_xenvirt.h> 81 #endif 82 #ifdef RTE_LIBRTE_PDUMP 83 #include <rte_pdump.h> 84 #endif 85 #include <rte_flow.h> 86 #include <rte_metrics.h> 87 #ifdef RTE_LIBRTE_BITRATE 88 #include <rte_bitrate.h> 89 #endif 90 #ifdef RTE_LIBRTE_LATENCY_STATS 91 #include <rte_latencystats.h> 92 #endif 93 94 #include "testpmd.h" 95 96 uint16_t verbose_level = 0; /**< Silent by default. */ 97 98 /* use master core for command line ? */ 99 uint8_t interactive = 0; 100 uint8_t auto_start = 0; 101 uint8_t tx_first; 102 char cmdline_filename[PATH_MAX] = {0}; 103 104 /* 105 * NUMA support configuration. 106 * When set, the NUMA support attempts to dispatch the allocation of the 107 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 108 * probed ports among the CPU sockets 0 and 1. 109 * Otherwise, all memory is allocated from CPU socket 0. 110 */ 111 uint8_t numa_support = 1; /**< numa enabled by default */ 112 113 /* 114 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 115 * not configured. 116 */ 117 uint8_t socket_num = UMA_NO_CONFIG; 118 119 /* 120 * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs. 121 */ 122 uint8_t mp_anon = 0; 123 124 /* 125 * Record the Ethernet address of peer target ports to which packets are 126 * forwarded. 127 * Must be instantiated with the ethernet addresses of peer traffic generator 128 * ports. 129 */ 130 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 131 portid_t nb_peer_eth_addrs = 0; 132 133 /* 134 * Probed Target Environment. 135 */ 136 struct rte_port *ports; /**< For all probed ethernet ports. */ 137 portid_t nb_ports; /**< Number of probed ethernet ports. */ 138 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 139 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 140 141 /* 142 * Test Forwarding Configuration. 143 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 144 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 145 */ 146 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 147 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 148 portid_t nb_cfg_ports; /**< Number of configured ports. */ 149 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 150 151 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 152 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 153 154 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 155 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 156 157 /* 158 * Forwarding engines. 159 */ 160 struct fwd_engine * fwd_engines[] = { 161 &io_fwd_engine, 162 &mac_fwd_engine, 163 &mac_swap_engine, 164 &flow_gen_engine, 165 &rx_only_engine, 166 &tx_only_engine, 167 &csum_fwd_engine, 168 &icmp_echo_engine, 169 #ifdef RTE_LIBRTE_IEEE1588 170 &ieee1588_fwd_engine, 171 #endif 172 NULL, 173 }; 174 175 struct fwd_config cur_fwd_config; 176 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 177 uint32_t retry_enabled; 178 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 179 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 180 181 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 182 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 183 * specified on command-line. */ 184 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 185 /* 186 * Configuration of packet segments used by the "txonly" processing engine. 187 */ 188 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 189 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 190 TXONLY_DEF_PACKET_LEN, 191 }; 192 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 193 194 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 195 /**< Split policy for packets to TX. */ 196 197 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 198 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 199 200 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 201 uint8_t dcb_config = 0; 202 203 /* Whether the dcb is in testing status */ 204 uint8_t dcb_test = 0; 205 206 /* 207 * Configurable number of RX/TX queues. 208 */ 209 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 210 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 211 212 /* 213 * Configurable number of RX/TX ring descriptors. 214 */ 215 #define RTE_TEST_RX_DESC_DEFAULT 128 216 #define RTE_TEST_TX_DESC_DEFAULT 512 217 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 218 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 219 220 #define RTE_PMD_PARAM_UNSET -1 221 /* 222 * Configurable values of RX and TX ring threshold registers. 223 */ 224 225 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 226 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 227 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 228 229 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 230 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 231 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 232 233 /* 234 * Configurable value of RX free threshold. 235 */ 236 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 237 238 /* 239 * Configurable value of RX drop enable. 240 */ 241 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 242 243 /* 244 * Configurable value of TX free threshold. 245 */ 246 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 247 248 /* 249 * Configurable value of TX RS bit threshold. 250 */ 251 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 252 253 /* 254 * Configurable value of TX queue flags. 255 */ 256 int32_t txq_flags = RTE_PMD_PARAM_UNSET; 257 258 /* 259 * Receive Side Scaling (RSS) configuration. 260 */ 261 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 262 263 /* 264 * Port topology configuration 265 */ 266 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 267 268 /* 269 * Avoids to flush all the RX streams before starts forwarding. 270 */ 271 uint8_t no_flush_rx = 0; /* flush by default */ 272 273 /* 274 * Avoids to check link status when starting/stopping a port. 275 */ 276 uint8_t no_link_check = 0; /* check by default */ 277 278 /* 279 * Enable link status change notification 280 */ 281 uint8_t lsc_interrupt = 1; /* enabled by default */ 282 283 /* 284 * Enable device removal notification. 285 */ 286 uint8_t rmv_interrupt = 1; /* enabled by default */ 287 288 /* 289 * Display or mask ether events 290 * Default to all events except VF_MBOX 291 */ 292 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 293 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 294 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 295 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 296 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 297 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 298 299 /* 300 * NIC bypass mode configuration options. 301 */ 302 303 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 304 /* The NIC bypass watchdog timeout. */ 305 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 306 #endif 307 308 309 #ifdef RTE_LIBRTE_LATENCY_STATS 310 311 /* 312 * Set when latency stats is enabled in the commandline 313 */ 314 uint8_t latencystats_enabled; 315 316 /* 317 * Lcore ID to serive latency statistics. 318 */ 319 lcoreid_t latencystats_lcore_id = -1; 320 321 #endif 322 323 /* 324 * Ethernet device configuration. 325 */ 326 struct rte_eth_rxmode rx_mode = { 327 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 328 .split_hdr_size = 0, 329 .header_split = 0, /**< Header Split disabled. */ 330 .hw_ip_checksum = 0, /**< IP checksum offload disabled. */ 331 .hw_vlan_filter = 1, /**< VLAN filtering enabled. */ 332 .hw_vlan_strip = 1, /**< VLAN strip enabled. */ 333 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 334 .jumbo_frame = 0, /**< Jumbo Frame Support disabled. */ 335 .hw_strip_crc = 1, /**< CRC stripping by hardware enabled. */ 336 }; 337 338 struct rte_fdir_conf fdir_conf = { 339 .mode = RTE_FDIR_MODE_NONE, 340 .pballoc = RTE_FDIR_PBALLOC_64K, 341 .status = RTE_FDIR_REPORT_STATUS, 342 .mask = { 343 .vlan_tci_mask = 0x0, 344 .ipv4_mask = { 345 .src_ip = 0xFFFFFFFF, 346 .dst_ip = 0xFFFFFFFF, 347 }, 348 .ipv6_mask = { 349 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 350 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 351 }, 352 .src_port_mask = 0xFFFF, 353 .dst_port_mask = 0xFFFF, 354 .mac_addr_byte_mask = 0xFF, 355 .tunnel_type_mask = 1, 356 .tunnel_id_mask = 0xFFFFFFFF, 357 }, 358 .drop_queue = 127, 359 }; 360 361 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 362 363 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 364 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 365 366 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 367 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 368 369 uint16_t nb_tx_queue_stats_mappings = 0; 370 uint16_t nb_rx_queue_stats_mappings = 0; 371 372 unsigned int num_sockets = 0; 373 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 374 375 #ifdef RTE_LIBRTE_BITRATE 376 /* Bitrate statistics */ 377 struct rte_stats_bitrates *bitrate_data; 378 lcoreid_t bitrate_lcore_id; 379 uint8_t bitrate_enabled; 380 #endif 381 382 /* Forward function declarations */ 383 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port); 384 static void check_all_ports_link_status(uint32_t port_mask); 385 static int eth_event_callback(uint8_t port_id, 386 enum rte_eth_event_type type, 387 void *param, void *ret_param); 388 389 /* 390 * Check if all the ports are started. 391 * If yes, return positive value. If not, return zero. 392 */ 393 static int all_ports_started(void); 394 395 /* 396 * Helper function to check if socket is already discovered. 397 * If yes, return positive value. If not, return zero. 398 */ 399 int 400 new_socket_id(unsigned int socket_id) 401 { 402 unsigned int i; 403 404 for (i = 0; i < num_sockets; i++) { 405 if (socket_ids[i] == socket_id) 406 return 0; 407 } 408 return 1; 409 } 410 411 /* 412 * Setup default configuration. 413 */ 414 static void 415 set_default_fwd_lcores_config(void) 416 { 417 unsigned int i; 418 unsigned int nb_lc; 419 unsigned int sock_num; 420 421 nb_lc = 0; 422 for (i = 0; i < RTE_MAX_LCORE; i++) { 423 sock_num = rte_lcore_to_socket_id(i); 424 if (new_socket_id(sock_num)) { 425 if (num_sockets >= RTE_MAX_NUMA_NODES) { 426 rte_exit(EXIT_FAILURE, 427 "Total sockets greater than %u\n", 428 RTE_MAX_NUMA_NODES); 429 } 430 socket_ids[num_sockets++] = sock_num; 431 } 432 if (!rte_lcore_is_enabled(i)) 433 continue; 434 if (i == rte_get_master_lcore()) 435 continue; 436 fwd_lcores_cpuids[nb_lc++] = i; 437 } 438 nb_lcores = (lcoreid_t) nb_lc; 439 nb_cfg_lcores = nb_lcores; 440 nb_fwd_lcores = 1; 441 } 442 443 static void 444 set_def_peer_eth_addrs(void) 445 { 446 portid_t i; 447 448 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 449 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 450 peer_eth_addrs[i].addr_bytes[5] = i; 451 } 452 } 453 454 static void 455 set_default_fwd_ports_config(void) 456 { 457 portid_t pt_id; 458 459 for (pt_id = 0; pt_id < nb_ports; pt_id++) 460 fwd_ports_ids[pt_id] = pt_id; 461 462 nb_cfg_ports = nb_ports; 463 nb_fwd_ports = nb_ports; 464 } 465 466 void 467 set_def_fwd_config(void) 468 { 469 set_default_fwd_lcores_config(); 470 set_def_peer_eth_addrs(); 471 set_default_fwd_ports_config(); 472 } 473 474 /* 475 * Configuration initialisation done once at init time. 476 */ 477 static void 478 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 479 unsigned int socket_id) 480 { 481 char pool_name[RTE_MEMPOOL_NAMESIZE]; 482 struct rte_mempool *rte_mp = NULL; 483 uint32_t mb_size; 484 485 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 486 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 487 488 RTE_LOG(INFO, USER1, 489 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 490 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 491 492 #ifdef RTE_LIBRTE_PMD_XENVIRT 493 rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size, 494 (unsigned) mb_mempool_cache, 495 sizeof(struct rte_pktmbuf_pool_private), 496 rte_pktmbuf_pool_init, NULL, 497 rte_pktmbuf_init, NULL, 498 socket_id, 0); 499 #endif 500 501 /* if the former XEN allocation failed fall back to normal allocation */ 502 if (rte_mp == NULL) { 503 if (mp_anon != 0) { 504 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 505 mb_size, (unsigned) mb_mempool_cache, 506 sizeof(struct rte_pktmbuf_pool_private), 507 socket_id, 0); 508 if (rte_mp == NULL) 509 goto err; 510 511 if (rte_mempool_populate_anon(rte_mp) == 0) { 512 rte_mempool_free(rte_mp); 513 rte_mp = NULL; 514 goto err; 515 } 516 rte_pktmbuf_pool_init(rte_mp, NULL); 517 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 518 } else { 519 /* wrapper to rte_mempool_create() */ 520 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 521 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 522 } 523 } 524 525 err: 526 if (rte_mp == NULL) { 527 rte_exit(EXIT_FAILURE, 528 "Creation of mbuf pool for socket %u failed: %s\n", 529 socket_id, rte_strerror(rte_errno)); 530 } else if (verbose_level > 0) { 531 rte_mempool_dump(stdout, rte_mp); 532 } 533 } 534 535 /* 536 * Check given socket id is valid or not with NUMA mode, 537 * if valid, return 0, else return -1 538 */ 539 static int 540 check_socket_id(const unsigned int socket_id) 541 { 542 static int warning_once = 0; 543 544 if (new_socket_id(socket_id)) { 545 if (!warning_once && numa_support) 546 printf("Warning: NUMA should be configured manually by" 547 " using --port-numa-config and" 548 " --ring-numa-config parameters along with" 549 " --numa.\n"); 550 warning_once = 1; 551 return -1; 552 } 553 return 0; 554 } 555 556 static void 557 init_config(void) 558 { 559 portid_t pid; 560 struct rte_port *port; 561 struct rte_mempool *mbp; 562 unsigned int nb_mbuf_per_pool; 563 lcoreid_t lc_id; 564 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 565 566 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 567 568 if (numa_support) { 569 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 570 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 571 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 572 } 573 574 /* Configuration of logical cores. */ 575 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 576 sizeof(struct fwd_lcore *) * nb_lcores, 577 RTE_CACHE_LINE_SIZE); 578 if (fwd_lcores == NULL) { 579 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 580 "failed\n", nb_lcores); 581 } 582 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 583 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 584 sizeof(struct fwd_lcore), 585 RTE_CACHE_LINE_SIZE); 586 if (fwd_lcores[lc_id] == NULL) { 587 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 588 "failed\n"); 589 } 590 fwd_lcores[lc_id]->cpuid_idx = lc_id; 591 } 592 593 RTE_ETH_FOREACH_DEV(pid) { 594 port = &ports[pid]; 595 rte_eth_dev_info_get(pid, &port->dev_info); 596 597 if (numa_support) { 598 if (port_numa[pid] != NUMA_NO_CONFIG) 599 port_per_socket[port_numa[pid]]++; 600 else { 601 uint32_t socket_id = rte_eth_dev_socket_id(pid); 602 603 /* if socket_id is invalid, set to 0 */ 604 if (check_socket_id(socket_id) < 0) 605 socket_id = 0; 606 port_per_socket[socket_id]++; 607 } 608 } 609 610 /* set flag to initialize port/queue */ 611 port->need_reconfig = 1; 612 port->need_reconfig_queues = 1; 613 } 614 615 /* 616 * Create pools of mbuf. 617 * If NUMA support is disabled, create a single pool of mbuf in 618 * socket 0 memory by default. 619 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 620 * 621 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 622 * nb_txd can be configured at run time. 623 */ 624 if (param_total_num_mbufs) 625 nb_mbuf_per_pool = param_total_num_mbufs; 626 else { 627 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 628 (nb_lcores * mb_mempool_cache) + 629 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 630 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 631 } 632 633 if (numa_support) { 634 uint8_t i; 635 636 for (i = 0; i < num_sockets; i++) 637 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 638 socket_ids[i]); 639 } else { 640 if (socket_num == UMA_NO_CONFIG) 641 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 642 else 643 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 644 socket_num); 645 } 646 647 init_port_config(); 648 649 /* 650 * Records which Mbuf pool to use by each logical core, if needed. 651 */ 652 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 653 mbp = mbuf_pool_find( 654 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 655 656 if (mbp == NULL) 657 mbp = mbuf_pool_find(0); 658 fwd_lcores[lc_id]->mbp = mbp; 659 } 660 661 /* Configuration of packet forwarding streams. */ 662 if (init_fwd_streams() < 0) 663 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 664 665 fwd_config_setup(); 666 } 667 668 669 void 670 reconfig(portid_t new_port_id, unsigned socket_id) 671 { 672 struct rte_port *port; 673 674 /* Reconfiguration of Ethernet ports. */ 675 port = &ports[new_port_id]; 676 rte_eth_dev_info_get(new_port_id, &port->dev_info); 677 678 /* set flag to initialize port/queue */ 679 port->need_reconfig = 1; 680 port->need_reconfig_queues = 1; 681 port->socket_id = socket_id; 682 683 init_port_config(); 684 } 685 686 687 int 688 init_fwd_streams(void) 689 { 690 portid_t pid; 691 struct rte_port *port; 692 streamid_t sm_id, nb_fwd_streams_new; 693 queueid_t q; 694 695 /* set socket id according to numa or not */ 696 RTE_ETH_FOREACH_DEV(pid) { 697 port = &ports[pid]; 698 if (nb_rxq > port->dev_info.max_rx_queues) { 699 printf("Fail: nb_rxq(%d) is greater than " 700 "max_rx_queues(%d)\n", nb_rxq, 701 port->dev_info.max_rx_queues); 702 return -1; 703 } 704 if (nb_txq > port->dev_info.max_tx_queues) { 705 printf("Fail: nb_txq(%d) is greater than " 706 "max_tx_queues(%d)\n", nb_txq, 707 port->dev_info.max_tx_queues); 708 return -1; 709 } 710 if (numa_support) { 711 if (port_numa[pid] != NUMA_NO_CONFIG) 712 port->socket_id = port_numa[pid]; 713 else { 714 port->socket_id = rte_eth_dev_socket_id(pid); 715 716 /* if socket_id is invalid, set to 0 */ 717 if (check_socket_id(port->socket_id) < 0) 718 port->socket_id = 0; 719 } 720 } 721 else { 722 if (socket_num == UMA_NO_CONFIG) 723 port->socket_id = 0; 724 else 725 port->socket_id = socket_num; 726 } 727 } 728 729 q = RTE_MAX(nb_rxq, nb_txq); 730 if (q == 0) { 731 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 732 return -1; 733 } 734 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 735 if (nb_fwd_streams_new == nb_fwd_streams) 736 return 0; 737 /* clear the old */ 738 if (fwd_streams != NULL) { 739 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 740 if (fwd_streams[sm_id] == NULL) 741 continue; 742 rte_free(fwd_streams[sm_id]); 743 fwd_streams[sm_id] = NULL; 744 } 745 rte_free(fwd_streams); 746 fwd_streams = NULL; 747 } 748 749 /* init new */ 750 nb_fwd_streams = nb_fwd_streams_new; 751 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 752 sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE); 753 if (fwd_streams == NULL) 754 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) " 755 "failed\n", nb_fwd_streams); 756 757 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 758 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream", 759 sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE); 760 if (fwd_streams[sm_id] == NULL) 761 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)" 762 " failed\n"); 763 } 764 765 return 0; 766 } 767 768 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 769 static void 770 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 771 { 772 unsigned int total_burst; 773 unsigned int nb_burst; 774 unsigned int burst_stats[3]; 775 uint16_t pktnb_stats[3]; 776 uint16_t nb_pkt; 777 int burst_percent[3]; 778 779 /* 780 * First compute the total number of packet bursts and the 781 * two highest numbers of bursts of the same number of packets. 782 */ 783 total_burst = 0; 784 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 785 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 786 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 787 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 788 if (nb_burst == 0) 789 continue; 790 total_burst += nb_burst; 791 if (nb_burst > burst_stats[0]) { 792 burst_stats[1] = burst_stats[0]; 793 pktnb_stats[1] = pktnb_stats[0]; 794 burst_stats[0] = nb_burst; 795 pktnb_stats[0] = nb_pkt; 796 } 797 } 798 if (total_burst == 0) 799 return; 800 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 801 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 802 burst_percent[0], (int) pktnb_stats[0]); 803 if (burst_stats[0] == total_burst) { 804 printf("]\n"); 805 return; 806 } 807 if (burst_stats[0] + burst_stats[1] == total_burst) { 808 printf(" + %d%% of %d pkts]\n", 809 100 - burst_percent[0], pktnb_stats[1]); 810 return; 811 } 812 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 813 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 814 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 815 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 816 return; 817 } 818 printf(" + %d%% of %d pkts + %d%% of others]\n", 819 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 820 } 821 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 822 823 static void 824 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats) 825 { 826 struct rte_port *port; 827 uint8_t i; 828 829 static const char *fwd_stats_border = "----------------------"; 830 831 port = &ports[port_id]; 832 printf("\n %s Forward statistics for port %-2d %s\n", 833 fwd_stats_border, port_id, fwd_stats_border); 834 835 if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) { 836 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 837 "%-"PRIu64"\n", 838 stats->ipackets, stats->imissed, 839 (uint64_t) (stats->ipackets + stats->imissed)); 840 841 if (cur_fwd_eng == &csum_fwd_engine) 842 printf(" Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n", 843 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 844 if ((stats->ierrors + stats->rx_nombuf) > 0) { 845 printf(" RX-error: %-"PRIu64"\n", stats->ierrors); 846 printf(" RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf); 847 } 848 849 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 850 "%-"PRIu64"\n", 851 stats->opackets, port->tx_dropped, 852 (uint64_t) (stats->opackets + port->tx_dropped)); 853 } 854 else { 855 printf(" RX-packets: %14"PRIu64" RX-dropped:%14"PRIu64" RX-total:" 856 "%14"PRIu64"\n", 857 stats->ipackets, stats->imissed, 858 (uint64_t) (stats->ipackets + stats->imissed)); 859 860 if (cur_fwd_eng == &csum_fwd_engine) 861 printf(" Bad-ipcsum:%14"PRIu64" Bad-l4csum:%14"PRIu64"\n", 862 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 863 if ((stats->ierrors + stats->rx_nombuf) > 0) { 864 printf(" RX-error:%"PRIu64"\n", stats->ierrors); 865 printf(" RX-nombufs: %14"PRIu64"\n", 866 stats->rx_nombuf); 867 } 868 869 printf(" TX-packets: %14"PRIu64" TX-dropped:%14"PRIu64" TX-total:" 870 "%14"PRIu64"\n", 871 stats->opackets, port->tx_dropped, 872 (uint64_t) (stats->opackets + port->tx_dropped)); 873 } 874 875 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 876 if (port->rx_stream) 877 pkt_burst_stats_display("RX", 878 &port->rx_stream->rx_burst_stats); 879 if (port->tx_stream) 880 pkt_burst_stats_display("TX", 881 &port->tx_stream->tx_burst_stats); 882 #endif 883 884 if (port->rx_queue_stats_mapping_enabled) { 885 printf("\n"); 886 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 887 printf(" Stats reg %2d RX-packets:%14"PRIu64 888 " RX-errors:%14"PRIu64 889 " RX-bytes:%14"PRIu64"\n", 890 i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]); 891 } 892 printf("\n"); 893 } 894 if (port->tx_queue_stats_mapping_enabled) { 895 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 896 printf(" Stats reg %2d TX-packets:%14"PRIu64 897 " TX-bytes:%14"PRIu64"\n", 898 i, stats->q_opackets[i], stats->q_obytes[i]); 899 } 900 } 901 902 printf(" %s--------------------------------%s\n", 903 fwd_stats_border, fwd_stats_border); 904 } 905 906 static void 907 fwd_stream_stats_display(streamid_t stream_id) 908 { 909 struct fwd_stream *fs; 910 static const char *fwd_top_stats_border = "-------"; 911 912 fs = fwd_streams[stream_id]; 913 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 914 (fs->fwd_dropped == 0)) 915 return; 916 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 917 "TX Port=%2d/Queue=%2d %s\n", 918 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 919 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 920 printf(" RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u", 921 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 922 923 /* if checksum mode */ 924 if (cur_fwd_eng == &csum_fwd_engine) { 925 printf(" RX- bad IP checksum: %-14u Rx- bad L4 checksum: " 926 "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum); 927 } 928 929 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 930 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 931 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 932 #endif 933 } 934 935 static void 936 flush_fwd_rx_queues(void) 937 { 938 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 939 portid_t rxp; 940 portid_t port_id; 941 queueid_t rxq; 942 uint16_t nb_rx; 943 uint16_t i; 944 uint8_t j; 945 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 946 uint64_t timer_period; 947 948 /* convert to number of cycles */ 949 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 950 951 for (j = 0; j < 2; j++) { 952 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 953 for (rxq = 0; rxq < nb_rxq; rxq++) { 954 port_id = fwd_ports_ids[rxp]; 955 /** 956 * testpmd can stuck in the below do while loop 957 * if rte_eth_rx_burst() always returns nonzero 958 * packets. So timer is added to exit this loop 959 * after 1sec timer expiry. 960 */ 961 prev_tsc = rte_rdtsc(); 962 do { 963 nb_rx = rte_eth_rx_burst(port_id, rxq, 964 pkts_burst, MAX_PKT_BURST); 965 for (i = 0; i < nb_rx; i++) 966 rte_pktmbuf_free(pkts_burst[i]); 967 968 cur_tsc = rte_rdtsc(); 969 diff_tsc = cur_tsc - prev_tsc; 970 timer_tsc += diff_tsc; 971 } while ((nb_rx > 0) && 972 (timer_tsc < timer_period)); 973 timer_tsc = 0; 974 } 975 } 976 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 977 } 978 } 979 980 static void 981 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 982 { 983 struct fwd_stream **fsm; 984 streamid_t nb_fs; 985 streamid_t sm_id; 986 #ifdef RTE_LIBRTE_BITRATE 987 uint64_t tics_per_1sec; 988 uint64_t tics_datum; 989 uint64_t tics_current; 990 uint8_t idx_port, cnt_ports; 991 992 cnt_ports = rte_eth_dev_count(); 993 tics_datum = rte_rdtsc(); 994 tics_per_1sec = rte_get_timer_hz(); 995 #endif 996 fsm = &fwd_streams[fc->stream_idx]; 997 nb_fs = fc->stream_nb; 998 do { 999 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1000 (*pkt_fwd)(fsm[sm_id]); 1001 #ifdef RTE_LIBRTE_BITRATE 1002 if (bitrate_enabled != 0 && 1003 bitrate_lcore_id == rte_lcore_id()) { 1004 tics_current = rte_rdtsc(); 1005 if (tics_current - tics_datum >= tics_per_1sec) { 1006 /* Periodic bitrate calculation */ 1007 for (idx_port = 0; 1008 idx_port < cnt_ports; 1009 idx_port++) 1010 rte_stats_bitrate_calc(bitrate_data, 1011 idx_port); 1012 tics_datum = tics_current; 1013 } 1014 } 1015 #endif 1016 #ifdef RTE_LIBRTE_LATENCY_STATS 1017 if (latencystats_enabled != 0 && 1018 latencystats_lcore_id == rte_lcore_id()) 1019 rte_latencystats_update(); 1020 #endif 1021 1022 } while (! fc->stopped); 1023 } 1024 1025 static int 1026 start_pkt_forward_on_core(void *fwd_arg) 1027 { 1028 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1029 cur_fwd_config.fwd_eng->packet_fwd); 1030 return 0; 1031 } 1032 1033 /* 1034 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1035 * Used to start communication flows in network loopback test configurations. 1036 */ 1037 static int 1038 run_one_txonly_burst_on_core(void *fwd_arg) 1039 { 1040 struct fwd_lcore *fwd_lc; 1041 struct fwd_lcore tmp_lcore; 1042 1043 fwd_lc = (struct fwd_lcore *) fwd_arg; 1044 tmp_lcore = *fwd_lc; 1045 tmp_lcore.stopped = 1; 1046 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1047 return 0; 1048 } 1049 1050 /* 1051 * Launch packet forwarding: 1052 * - Setup per-port forwarding context. 1053 * - launch logical cores with their forwarding configuration. 1054 */ 1055 static void 1056 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1057 { 1058 port_fwd_begin_t port_fwd_begin; 1059 unsigned int i; 1060 unsigned int lc_id; 1061 int diag; 1062 1063 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1064 if (port_fwd_begin != NULL) { 1065 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1066 (*port_fwd_begin)(fwd_ports_ids[i]); 1067 } 1068 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1069 lc_id = fwd_lcores_cpuids[i]; 1070 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1071 fwd_lcores[i]->stopped = 0; 1072 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1073 fwd_lcores[i], lc_id); 1074 if (diag != 0) 1075 printf("launch lcore %u failed - diag=%d\n", 1076 lc_id, diag); 1077 } 1078 } 1079 } 1080 1081 /* 1082 * Launch packet forwarding configuration. 1083 */ 1084 void 1085 start_packet_forwarding(int with_tx_first) 1086 { 1087 port_fwd_begin_t port_fwd_begin; 1088 port_fwd_end_t port_fwd_end; 1089 struct rte_port *port; 1090 unsigned int i; 1091 portid_t pt_id; 1092 streamid_t sm_id; 1093 1094 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1095 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1096 1097 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1098 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1099 1100 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1101 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1102 (!nb_rxq || !nb_txq)) 1103 rte_exit(EXIT_FAILURE, 1104 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1105 cur_fwd_eng->fwd_mode_name); 1106 1107 if (all_ports_started() == 0) { 1108 printf("Not all ports were started\n"); 1109 return; 1110 } 1111 if (test_done == 0) { 1112 printf("Packet forwarding already started\n"); 1113 return; 1114 } 1115 1116 if (init_fwd_streams() < 0) { 1117 printf("Fail from init_fwd_streams()\n"); 1118 return; 1119 } 1120 1121 if(dcb_test) { 1122 for (i = 0; i < nb_fwd_ports; i++) { 1123 pt_id = fwd_ports_ids[i]; 1124 port = &ports[pt_id]; 1125 if (!port->dcb_flag) { 1126 printf("In DCB mode, all forwarding ports must " 1127 "be configured in this mode.\n"); 1128 return; 1129 } 1130 } 1131 if (nb_fwd_lcores == 1) { 1132 printf("In DCB mode,the nb forwarding cores " 1133 "should be larger than 1.\n"); 1134 return; 1135 } 1136 } 1137 test_done = 0; 1138 1139 if(!no_flush_rx) 1140 flush_fwd_rx_queues(); 1141 1142 fwd_config_setup(); 1143 pkt_fwd_config_display(&cur_fwd_config); 1144 rxtx_config_display(); 1145 1146 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1147 pt_id = fwd_ports_ids[i]; 1148 port = &ports[pt_id]; 1149 rte_eth_stats_get(pt_id, &port->stats); 1150 port->tx_dropped = 0; 1151 1152 map_port_queue_stats_mapping_registers(pt_id, port); 1153 } 1154 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1155 fwd_streams[sm_id]->rx_packets = 0; 1156 fwd_streams[sm_id]->tx_packets = 0; 1157 fwd_streams[sm_id]->fwd_dropped = 0; 1158 fwd_streams[sm_id]->rx_bad_ip_csum = 0; 1159 fwd_streams[sm_id]->rx_bad_l4_csum = 0; 1160 1161 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1162 memset(&fwd_streams[sm_id]->rx_burst_stats, 0, 1163 sizeof(fwd_streams[sm_id]->rx_burst_stats)); 1164 memset(&fwd_streams[sm_id]->tx_burst_stats, 0, 1165 sizeof(fwd_streams[sm_id]->tx_burst_stats)); 1166 #endif 1167 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1168 fwd_streams[sm_id]->core_cycles = 0; 1169 #endif 1170 } 1171 if (with_tx_first) { 1172 port_fwd_begin = tx_only_engine.port_fwd_begin; 1173 if (port_fwd_begin != NULL) { 1174 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1175 (*port_fwd_begin)(fwd_ports_ids[i]); 1176 } 1177 while (with_tx_first--) { 1178 launch_packet_forwarding( 1179 run_one_txonly_burst_on_core); 1180 rte_eal_mp_wait_lcore(); 1181 } 1182 port_fwd_end = tx_only_engine.port_fwd_end; 1183 if (port_fwd_end != NULL) { 1184 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1185 (*port_fwd_end)(fwd_ports_ids[i]); 1186 } 1187 } 1188 launch_packet_forwarding(start_pkt_forward_on_core); 1189 } 1190 1191 void 1192 stop_packet_forwarding(void) 1193 { 1194 struct rte_eth_stats stats; 1195 struct rte_port *port; 1196 port_fwd_end_t port_fwd_end; 1197 int i; 1198 portid_t pt_id; 1199 streamid_t sm_id; 1200 lcoreid_t lc_id; 1201 uint64_t total_recv; 1202 uint64_t total_xmit; 1203 uint64_t total_rx_dropped; 1204 uint64_t total_tx_dropped; 1205 uint64_t total_rx_nombuf; 1206 uint64_t tx_dropped; 1207 uint64_t rx_bad_ip_csum; 1208 uint64_t rx_bad_l4_csum; 1209 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1210 uint64_t fwd_cycles; 1211 #endif 1212 static const char *acc_stats_border = "+++++++++++++++"; 1213 1214 if (test_done) { 1215 printf("Packet forwarding not started\n"); 1216 return; 1217 } 1218 printf("Telling cores to stop..."); 1219 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1220 fwd_lcores[lc_id]->stopped = 1; 1221 printf("\nWaiting for lcores to finish...\n"); 1222 rte_eal_mp_wait_lcore(); 1223 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1224 if (port_fwd_end != NULL) { 1225 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1226 pt_id = fwd_ports_ids[i]; 1227 (*port_fwd_end)(pt_id); 1228 } 1229 } 1230 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1231 fwd_cycles = 0; 1232 #endif 1233 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1234 if (cur_fwd_config.nb_fwd_streams > 1235 cur_fwd_config.nb_fwd_ports) { 1236 fwd_stream_stats_display(sm_id); 1237 ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL; 1238 ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL; 1239 } else { 1240 ports[fwd_streams[sm_id]->tx_port].tx_stream = 1241 fwd_streams[sm_id]; 1242 ports[fwd_streams[sm_id]->rx_port].rx_stream = 1243 fwd_streams[sm_id]; 1244 } 1245 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped; 1246 tx_dropped = (uint64_t) (tx_dropped + 1247 fwd_streams[sm_id]->fwd_dropped); 1248 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped; 1249 1250 rx_bad_ip_csum = 1251 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum; 1252 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum + 1253 fwd_streams[sm_id]->rx_bad_ip_csum); 1254 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum = 1255 rx_bad_ip_csum; 1256 1257 rx_bad_l4_csum = 1258 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum; 1259 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum + 1260 fwd_streams[sm_id]->rx_bad_l4_csum); 1261 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum = 1262 rx_bad_l4_csum; 1263 1264 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1265 fwd_cycles = (uint64_t) (fwd_cycles + 1266 fwd_streams[sm_id]->core_cycles); 1267 #endif 1268 } 1269 total_recv = 0; 1270 total_xmit = 0; 1271 total_rx_dropped = 0; 1272 total_tx_dropped = 0; 1273 total_rx_nombuf = 0; 1274 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1275 pt_id = fwd_ports_ids[i]; 1276 1277 port = &ports[pt_id]; 1278 rte_eth_stats_get(pt_id, &stats); 1279 stats.ipackets -= port->stats.ipackets; 1280 port->stats.ipackets = 0; 1281 stats.opackets -= port->stats.opackets; 1282 port->stats.opackets = 0; 1283 stats.ibytes -= port->stats.ibytes; 1284 port->stats.ibytes = 0; 1285 stats.obytes -= port->stats.obytes; 1286 port->stats.obytes = 0; 1287 stats.imissed -= port->stats.imissed; 1288 port->stats.imissed = 0; 1289 stats.oerrors -= port->stats.oerrors; 1290 port->stats.oerrors = 0; 1291 stats.rx_nombuf -= port->stats.rx_nombuf; 1292 port->stats.rx_nombuf = 0; 1293 1294 total_recv += stats.ipackets; 1295 total_xmit += stats.opackets; 1296 total_rx_dropped += stats.imissed; 1297 total_tx_dropped += port->tx_dropped; 1298 total_rx_nombuf += stats.rx_nombuf; 1299 1300 fwd_port_stats_display(pt_id, &stats); 1301 } 1302 printf("\n %s Accumulated forward statistics for all ports" 1303 "%s\n", 1304 acc_stats_border, acc_stats_border); 1305 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1306 "%-"PRIu64"\n" 1307 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1308 "%-"PRIu64"\n", 1309 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1310 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1311 if (total_rx_nombuf > 0) 1312 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1313 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1314 "%s\n", 1315 acc_stats_border, acc_stats_border); 1316 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1317 if (total_recv > 0) 1318 printf("\n CPU cycles/packet=%u (total cycles=" 1319 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1320 (unsigned int)(fwd_cycles / total_recv), 1321 fwd_cycles, total_recv); 1322 #endif 1323 printf("\nDone.\n"); 1324 test_done = 1; 1325 } 1326 1327 void 1328 dev_set_link_up(portid_t pid) 1329 { 1330 if (rte_eth_dev_set_link_up((uint8_t)pid) < 0) 1331 printf("\nSet link up fail.\n"); 1332 } 1333 1334 void 1335 dev_set_link_down(portid_t pid) 1336 { 1337 if (rte_eth_dev_set_link_down((uint8_t)pid) < 0) 1338 printf("\nSet link down fail.\n"); 1339 } 1340 1341 static int 1342 all_ports_started(void) 1343 { 1344 portid_t pi; 1345 struct rte_port *port; 1346 1347 RTE_ETH_FOREACH_DEV(pi) { 1348 port = &ports[pi]; 1349 /* Check if there is a port which is not started */ 1350 if ((port->port_status != RTE_PORT_STARTED) && 1351 (port->slave_flag == 0)) 1352 return 0; 1353 } 1354 1355 /* No port is not started */ 1356 return 1; 1357 } 1358 1359 int 1360 all_ports_stopped(void) 1361 { 1362 portid_t pi; 1363 struct rte_port *port; 1364 1365 RTE_ETH_FOREACH_DEV(pi) { 1366 port = &ports[pi]; 1367 if ((port->port_status != RTE_PORT_STOPPED) && 1368 (port->slave_flag == 0)) 1369 return 0; 1370 } 1371 1372 return 1; 1373 } 1374 1375 int 1376 port_is_started(portid_t port_id) 1377 { 1378 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1379 return 0; 1380 1381 if (ports[port_id].port_status != RTE_PORT_STARTED) 1382 return 0; 1383 1384 return 1; 1385 } 1386 1387 static int 1388 port_is_closed(portid_t port_id) 1389 { 1390 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1391 return 0; 1392 1393 if (ports[port_id].port_status != RTE_PORT_CLOSED) 1394 return 0; 1395 1396 return 1; 1397 } 1398 1399 int 1400 start_port(portid_t pid) 1401 { 1402 int diag, need_check_link_status = -1; 1403 portid_t pi; 1404 queueid_t qi; 1405 struct rte_port *port; 1406 struct ether_addr mac_addr; 1407 enum rte_eth_event_type event_type; 1408 1409 if (port_id_is_invalid(pid, ENABLED_WARN)) 1410 return 0; 1411 1412 if(dcb_config) 1413 dcb_test = 1; 1414 RTE_ETH_FOREACH_DEV(pi) { 1415 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1416 continue; 1417 1418 need_check_link_status = 0; 1419 port = &ports[pi]; 1420 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1421 RTE_PORT_HANDLING) == 0) { 1422 printf("Port %d is now not stopped\n", pi); 1423 continue; 1424 } 1425 1426 if (port->need_reconfig > 0) { 1427 port->need_reconfig = 0; 1428 1429 printf("Configuring Port %d (socket %u)\n", pi, 1430 port->socket_id); 1431 /* configure port */ 1432 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1433 &(port->dev_conf)); 1434 if (diag != 0) { 1435 if (rte_atomic16_cmpset(&(port->port_status), 1436 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1437 printf("Port %d can not be set back " 1438 "to stopped\n", pi); 1439 printf("Fail to configure port %d\n", pi); 1440 /* try to reconfigure port next time */ 1441 port->need_reconfig = 1; 1442 return -1; 1443 } 1444 } 1445 if (port->need_reconfig_queues > 0) { 1446 port->need_reconfig_queues = 0; 1447 /* setup tx queues */ 1448 for (qi = 0; qi < nb_txq; qi++) { 1449 if ((numa_support) && 1450 (txring_numa[pi] != NUMA_NO_CONFIG)) 1451 diag = rte_eth_tx_queue_setup(pi, qi, 1452 nb_txd,txring_numa[pi], 1453 &(port->tx_conf)); 1454 else 1455 diag = rte_eth_tx_queue_setup(pi, qi, 1456 nb_txd,port->socket_id, 1457 &(port->tx_conf)); 1458 1459 if (diag == 0) 1460 continue; 1461 1462 /* Fail to setup tx queue, return */ 1463 if (rte_atomic16_cmpset(&(port->port_status), 1464 RTE_PORT_HANDLING, 1465 RTE_PORT_STOPPED) == 0) 1466 printf("Port %d can not be set back " 1467 "to stopped\n", pi); 1468 printf("Fail to configure port %d tx queues\n", pi); 1469 /* try to reconfigure queues next time */ 1470 port->need_reconfig_queues = 1; 1471 return -1; 1472 } 1473 /* setup rx queues */ 1474 for (qi = 0; qi < nb_rxq; qi++) { 1475 if ((numa_support) && 1476 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 1477 struct rte_mempool * mp = 1478 mbuf_pool_find(rxring_numa[pi]); 1479 if (mp == NULL) { 1480 printf("Failed to setup RX queue:" 1481 "No mempool allocation" 1482 " on the socket %d\n", 1483 rxring_numa[pi]); 1484 return -1; 1485 } 1486 1487 diag = rte_eth_rx_queue_setup(pi, qi, 1488 nb_rxd,rxring_numa[pi], 1489 &(port->rx_conf),mp); 1490 } else { 1491 struct rte_mempool *mp = 1492 mbuf_pool_find(port->socket_id); 1493 if (mp == NULL) { 1494 printf("Failed to setup RX queue:" 1495 "No mempool allocation" 1496 " on the socket %d\n", 1497 port->socket_id); 1498 return -1; 1499 } 1500 diag = rte_eth_rx_queue_setup(pi, qi, 1501 nb_rxd,port->socket_id, 1502 &(port->rx_conf), mp); 1503 } 1504 if (diag == 0) 1505 continue; 1506 1507 /* Fail to setup rx queue, return */ 1508 if (rte_atomic16_cmpset(&(port->port_status), 1509 RTE_PORT_HANDLING, 1510 RTE_PORT_STOPPED) == 0) 1511 printf("Port %d can not be set back " 1512 "to stopped\n", pi); 1513 printf("Fail to configure port %d rx queues\n", pi); 1514 /* try to reconfigure queues next time */ 1515 port->need_reconfig_queues = 1; 1516 return -1; 1517 } 1518 } 1519 1520 for (event_type = RTE_ETH_EVENT_UNKNOWN; 1521 event_type < RTE_ETH_EVENT_MAX; 1522 event_type++) { 1523 diag = rte_eth_dev_callback_register(pi, 1524 event_type, 1525 eth_event_callback, 1526 NULL); 1527 if (diag) { 1528 printf("Failed to setup even callback for event %d\n", 1529 event_type); 1530 return -1; 1531 } 1532 } 1533 1534 /* start port */ 1535 if (rte_eth_dev_start(pi) < 0) { 1536 printf("Fail to start port %d\n", pi); 1537 1538 /* Fail to setup rx queue, return */ 1539 if (rte_atomic16_cmpset(&(port->port_status), 1540 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1541 printf("Port %d can not be set back to " 1542 "stopped\n", pi); 1543 continue; 1544 } 1545 1546 if (rte_atomic16_cmpset(&(port->port_status), 1547 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 1548 printf("Port %d can not be set into started\n", pi); 1549 1550 rte_eth_macaddr_get(pi, &mac_addr); 1551 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 1552 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 1553 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 1554 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 1555 1556 /* at least one port started, need checking link status */ 1557 need_check_link_status = 1; 1558 } 1559 1560 if (need_check_link_status == 1 && !no_link_check) 1561 check_all_ports_link_status(RTE_PORT_ALL); 1562 else if (need_check_link_status == 0) 1563 printf("Please stop the ports first\n"); 1564 1565 printf("Done\n"); 1566 return 0; 1567 } 1568 1569 void 1570 stop_port(portid_t pid) 1571 { 1572 portid_t pi; 1573 struct rte_port *port; 1574 int need_check_link_status = 0; 1575 1576 if (dcb_test) { 1577 dcb_test = 0; 1578 dcb_config = 0; 1579 } 1580 1581 if (port_id_is_invalid(pid, ENABLED_WARN)) 1582 return; 1583 1584 printf("Stopping ports...\n"); 1585 1586 RTE_ETH_FOREACH_DEV(pi) { 1587 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1588 continue; 1589 1590 if (port_is_forwarding(pi) != 0 && test_done == 0) { 1591 printf("Please remove port %d from forwarding configuration.\n", pi); 1592 continue; 1593 } 1594 1595 if (port_is_bonding_slave(pi)) { 1596 printf("Please remove port %d from bonded device.\n", pi); 1597 continue; 1598 } 1599 1600 port = &ports[pi]; 1601 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 1602 RTE_PORT_HANDLING) == 0) 1603 continue; 1604 1605 rte_eth_dev_stop(pi); 1606 1607 if (rte_atomic16_cmpset(&(port->port_status), 1608 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1609 printf("Port %d can not be set into stopped\n", pi); 1610 need_check_link_status = 1; 1611 } 1612 if (need_check_link_status && !no_link_check) 1613 check_all_ports_link_status(RTE_PORT_ALL); 1614 1615 printf("Done\n"); 1616 } 1617 1618 void 1619 close_port(portid_t pid) 1620 { 1621 portid_t pi; 1622 struct rte_port *port; 1623 1624 if (port_id_is_invalid(pid, ENABLED_WARN)) 1625 return; 1626 1627 printf("Closing ports...\n"); 1628 1629 RTE_ETH_FOREACH_DEV(pi) { 1630 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1631 continue; 1632 1633 if (port_is_forwarding(pi) != 0 && test_done == 0) { 1634 printf("Please remove port %d from forwarding configuration.\n", pi); 1635 continue; 1636 } 1637 1638 if (port_is_bonding_slave(pi)) { 1639 printf("Please remove port %d from bonded device.\n", pi); 1640 continue; 1641 } 1642 1643 port = &ports[pi]; 1644 if (rte_atomic16_cmpset(&(port->port_status), 1645 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 1646 printf("Port %d is already closed\n", pi); 1647 continue; 1648 } 1649 1650 if (rte_atomic16_cmpset(&(port->port_status), 1651 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 1652 printf("Port %d is now not stopped\n", pi); 1653 continue; 1654 } 1655 1656 if (port->flow_list) 1657 port_flow_flush(pi); 1658 rte_eth_dev_close(pi); 1659 1660 if (rte_atomic16_cmpset(&(port->port_status), 1661 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 1662 printf("Port %d cannot be set to closed\n", pi); 1663 } 1664 1665 printf("Done\n"); 1666 } 1667 1668 void 1669 attach_port(char *identifier) 1670 { 1671 portid_t pi = 0; 1672 unsigned int socket_id; 1673 1674 printf("Attaching a new port...\n"); 1675 1676 if (identifier == NULL) { 1677 printf("Invalid parameters are specified\n"); 1678 return; 1679 } 1680 1681 if (rte_eth_dev_attach(identifier, &pi)) 1682 return; 1683 1684 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 1685 /* if socket_id is invalid, set to 0 */ 1686 if (check_socket_id(socket_id) < 0) 1687 socket_id = 0; 1688 reconfig(pi, socket_id); 1689 rte_eth_promiscuous_enable(pi); 1690 1691 nb_ports = rte_eth_dev_count(); 1692 1693 ports[pi].port_status = RTE_PORT_STOPPED; 1694 1695 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 1696 printf("Done\n"); 1697 } 1698 1699 void 1700 detach_port(uint8_t port_id) 1701 { 1702 char name[RTE_ETH_NAME_MAX_LEN]; 1703 1704 printf("Detaching a port...\n"); 1705 1706 if (!port_is_closed(port_id)) { 1707 printf("Please close port first\n"); 1708 return; 1709 } 1710 1711 if (ports[port_id].flow_list) 1712 port_flow_flush(port_id); 1713 1714 if (rte_eth_dev_detach(port_id, name)) 1715 return; 1716 1717 nb_ports = rte_eth_dev_count(); 1718 1719 printf("Port '%s' is detached. Now total ports is %d\n", 1720 name, nb_ports); 1721 printf("Done\n"); 1722 return; 1723 } 1724 1725 void 1726 pmd_test_exit(void) 1727 { 1728 portid_t pt_id; 1729 1730 if (test_done == 0) 1731 stop_packet_forwarding(); 1732 1733 if (ports != NULL) { 1734 no_link_check = 1; 1735 RTE_ETH_FOREACH_DEV(pt_id) { 1736 printf("\nShutting down port %d...\n", pt_id); 1737 fflush(stdout); 1738 stop_port(pt_id); 1739 close_port(pt_id); 1740 } 1741 } 1742 printf("\nBye...\n"); 1743 } 1744 1745 typedef void (*cmd_func_t)(void); 1746 struct pmd_test_command { 1747 const char *cmd_name; 1748 cmd_func_t cmd_func; 1749 }; 1750 1751 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 1752 1753 /* Check the link status of all ports in up to 9s, and print them finally */ 1754 static void 1755 check_all_ports_link_status(uint32_t port_mask) 1756 { 1757 #define CHECK_INTERVAL 100 /* 100ms */ 1758 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1759 uint8_t portid, count, all_ports_up, print_flag = 0; 1760 struct rte_eth_link link; 1761 1762 printf("Checking link statuses...\n"); 1763 fflush(stdout); 1764 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1765 all_ports_up = 1; 1766 RTE_ETH_FOREACH_DEV(portid) { 1767 if ((port_mask & (1 << portid)) == 0) 1768 continue; 1769 memset(&link, 0, sizeof(link)); 1770 rte_eth_link_get_nowait(portid, &link); 1771 /* print link status if flag set */ 1772 if (print_flag == 1) { 1773 if (link.link_status) 1774 printf("Port %d Link Up - speed %u " 1775 "Mbps - %s\n", (uint8_t)portid, 1776 (unsigned)link.link_speed, 1777 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1778 ("full-duplex") : ("half-duplex\n")); 1779 else 1780 printf("Port %d Link Down\n", 1781 (uint8_t)portid); 1782 continue; 1783 } 1784 /* clear all_ports_up flag if any link down */ 1785 if (link.link_status == ETH_LINK_DOWN) { 1786 all_ports_up = 0; 1787 break; 1788 } 1789 } 1790 /* after finally printing all link status, get out */ 1791 if (print_flag == 1) 1792 break; 1793 1794 if (all_ports_up == 0) { 1795 fflush(stdout); 1796 rte_delay_ms(CHECK_INTERVAL); 1797 } 1798 1799 /* set the print_flag if all ports up or timeout */ 1800 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1801 print_flag = 1; 1802 } 1803 1804 if (lsc_interrupt) 1805 break; 1806 } 1807 } 1808 1809 static void 1810 rmv_event_callback(void *arg) 1811 { 1812 struct rte_eth_dev *dev; 1813 uint8_t port_id = (intptr_t)arg; 1814 1815 RTE_ETH_VALID_PORTID_OR_RET(port_id); 1816 dev = &rte_eth_devices[port_id]; 1817 1818 stop_port(port_id); 1819 close_port(port_id); 1820 printf("removing device %s\n", dev->device->name); 1821 rte_eal_dev_detach(dev->device); 1822 dev->state = RTE_ETH_DEV_UNUSED; 1823 } 1824 1825 /* This function is used by the interrupt thread */ 1826 static int 1827 eth_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param, 1828 void *ret_param) 1829 { 1830 static const char * const event_desc[] = { 1831 [RTE_ETH_EVENT_UNKNOWN] = "Unknown", 1832 [RTE_ETH_EVENT_INTR_LSC] = "LSC", 1833 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state", 1834 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset", 1835 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox", 1836 [RTE_ETH_EVENT_MACSEC] = "MACsec", 1837 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 1838 [RTE_ETH_EVENT_MAX] = NULL, 1839 }; 1840 1841 RTE_SET_USED(param); 1842 RTE_SET_USED(ret_param); 1843 1844 if (type >= RTE_ETH_EVENT_MAX) { 1845 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n", 1846 port_id, __func__, type); 1847 fflush(stderr); 1848 } else if (event_print_mask & (UINT32_C(1) << type)) { 1849 printf("\nPort %" PRIu8 ": %s event\n", port_id, 1850 event_desc[type]); 1851 fflush(stdout); 1852 } 1853 1854 switch (type) { 1855 case RTE_ETH_EVENT_INTR_RMV: 1856 if (rte_eal_alarm_set(100000, 1857 rmv_event_callback, (void *)(intptr_t)port_id)) 1858 fprintf(stderr, "Could not set up deferred device removal\n"); 1859 break; 1860 default: 1861 break; 1862 } 1863 return 0; 1864 } 1865 1866 static int 1867 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) 1868 { 1869 uint16_t i; 1870 int diag; 1871 uint8_t mapping_found = 0; 1872 1873 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 1874 if ((tx_queue_stats_mappings[i].port_id == port_id) && 1875 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 1876 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 1877 tx_queue_stats_mappings[i].queue_id, 1878 tx_queue_stats_mappings[i].stats_counter_id); 1879 if (diag != 0) 1880 return diag; 1881 mapping_found = 1; 1882 } 1883 } 1884 if (mapping_found) 1885 port->tx_queue_stats_mapping_enabled = 1; 1886 return 0; 1887 } 1888 1889 static int 1890 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) 1891 { 1892 uint16_t i; 1893 int diag; 1894 uint8_t mapping_found = 0; 1895 1896 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 1897 if ((rx_queue_stats_mappings[i].port_id == port_id) && 1898 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 1899 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 1900 rx_queue_stats_mappings[i].queue_id, 1901 rx_queue_stats_mappings[i].stats_counter_id); 1902 if (diag != 0) 1903 return diag; 1904 mapping_found = 1; 1905 } 1906 } 1907 if (mapping_found) 1908 port->rx_queue_stats_mapping_enabled = 1; 1909 return 0; 1910 } 1911 1912 static void 1913 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port) 1914 { 1915 int diag = 0; 1916 1917 diag = set_tx_queue_stats_mapping_registers(pi, port); 1918 if (diag != 0) { 1919 if (diag == -ENOTSUP) { 1920 port->tx_queue_stats_mapping_enabled = 0; 1921 printf("TX queue stats mapping not supported port id=%d\n", pi); 1922 } 1923 else 1924 rte_exit(EXIT_FAILURE, 1925 "set_tx_queue_stats_mapping_registers " 1926 "failed for port id=%d diag=%d\n", 1927 pi, diag); 1928 } 1929 1930 diag = set_rx_queue_stats_mapping_registers(pi, port); 1931 if (diag != 0) { 1932 if (diag == -ENOTSUP) { 1933 port->rx_queue_stats_mapping_enabled = 0; 1934 printf("RX queue stats mapping not supported port id=%d\n", pi); 1935 } 1936 else 1937 rte_exit(EXIT_FAILURE, 1938 "set_rx_queue_stats_mapping_registers " 1939 "failed for port id=%d diag=%d\n", 1940 pi, diag); 1941 } 1942 } 1943 1944 static void 1945 rxtx_port_config(struct rte_port *port) 1946 { 1947 port->rx_conf = port->dev_info.default_rxconf; 1948 port->tx_conf = port->dev_info.default_txconf; 1949 1950 /* Check if any RX/TX parameters have been passed */ 1951 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 1952 port->rx_conf.rx_thresh.pthresh = rx_pthresh; 1953 1954 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 1955 port->rx_conf.rx_thresh.hthresh = rx_hthresh; 1956 1957 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 1958 port->rx_conf.rx_thresh.wthresh = rx_wthresh; 1959 1960 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 1961 port->rx_conf.rx_free_thresh = rx_free_thresh; 1962 1963 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 1964 port->rx_conf.rx_drop_en = rx_drop_en; 1965 1966 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 1967 port->tx_conf.tx_thresh.pthresh = tx_pthresh; 1968 1969 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 1970 port->tx_conf.tx_thresh.hthresh = tx_hthresh; 1971 1972 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 1973 port->tx_conf.tx_thresh.wthresh = tx_wthresh; 1974 1975 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 1976 port->tx_conf.tx_rs_thresh = tx_rs_thresh; 1977 1978 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 1979 port->tx_conf.tx_free_thresh = tx_free_thresh; 1980 1981 if (txq_flags != RTE_PMD_PARAM_UNSET) 1982 port->tx_conf.txq_flags = txq_flags; 1983 } 1984 1985 void 1986 init_port_config(void) 1987 { 1988 portid_t pid; 1989 struct rte_port *port; 1990 1991 RTE_ETH_FOREACH_DEV(pid) { 1992 port = &ports[pid]; 1993 port->dev_conf.rxmode = rx_mode; 1994 port->dev_conf.fdir_conf = fdir_conf; 1995 if (nb_rxq > 1) { 1996 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 1997 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf; 1998 } else { 1999 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2000 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2001 } 2002 2003 if (port->dcb_flag == 0) { 2004 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2005 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2006 else 2007 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2008 } 2009 2010 rxtx_port_config(port); 2011 2012 rte_eth_macaddr_get(pid, &port->eth_addr); 2013 2014 map_port_queue_stats_mapping_registers(pid, port); 2015 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2016 rte_pmd_ixgbe_bypass_init(pid); 2017 #endif 2018 2019 if (lsc_interrupt && 2020 (rte_eth_devices[pid].data->dev_flags & 2021 RTE_ETH_DEV_INTR_LSC)) 2022 port->dev_conf.intr_conf.lsc = 1; 2023 if (rmv_interrupt && 2024 (rte_eth_devices[pid].data->dev_flags & 2025 RTE_ETH_DEV_INTR_RMV)) 2026 port->dev_conf.intr_conf.rmv = 1; 2027 } 2028 } 2029 2030 void set_port_slave_flag(portid_t slave_pid) 2031 { 2032 struct rte_port *port; 2033 2034 port = &ports[slave_pid]; 2035 port->slave_flag = 1; 2036 } 2037 2038 void clear_port_slave_flag(portid_t slave_pid) 2039 { 2040 struct rte_port *port; 2041 2042 port = &ports[slave_pid]; 2043 port->slave_flag = 0; 2044 } 2045 2046 uint8_t port_is_bonding_slave(portid_t slave_pid) 2047 { 2048 struct rte_port *port; 2049 2050 port = &ports[slave_pid]; 2051 return port->slave_flag; 2052 } 2053 2054 const uint16_t vlan_tags[] = { 2055 0, 1, 2, 3, 4, 5, 6, 7, 2056 8, 9, 10, 11, 12, 13, 14, 15, 2057 16, 17, 18, 19, 20, 21, 22, 23, 2058 24, 25, 26, 27, 28, 29, 30, 31 2059 }; 2060 2061 static int 2062 get_eth_dcb_conf(struct rte_eth_conf *eth_conf, 2063 enum dcb_mode_enable dcb_mode, 2064 enum rte_eth_nb_tcs num_tcs, 2065 uint8_t pfc_en) 2066 { 2067 uint8_t i; 2068 2069 /* 2070 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2071 * given above, and the number of traffic classes available for use. 2072 */ 2073 if (dcb_mode == DCB_VT_ENABLED) { 2074 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2075 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2076 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2077 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2078 2079 /* VMDQ+DCB RX and TX configurations */ 2080 vmdq_rx_conf->enable_default_pool = 0; 2081 vmdq_rx_conf->default_pool = 0; 2082 vmdq_rx_conf->nb_queue_pools = 2083 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2084 vmdq_tx_conf->nb_queue_pools = 2085 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2086 2087 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2088 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2089 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2090 vmdq_rx_conf->pool_map[i].pools = 2091 1 << (i % vmdq_rx_conf->nb_queue_pools); 2092 } 2093 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2094 vmdq_rx_conf->dcb_tc[i] = i; 2095 vmdq_tx_conf->dcb_tc[i] = i; 2096 } 2097 2098 /* set DCB mode of RX and TX of multiple queues */ 2099 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2100 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2101 } else { 2102 struct rte_eth_dcb_rx_conf *rx_conf = 2103 ð_conf->rx_adv_conf.dcb_rx_conf; 2104 struct rte_eth_dcb_tx_conf *tx_conf = 2105 ð_conf->tx_adv_conf.dcb_tx_conf; 2106 2107 rx_conf->nb_tcs = num_tcs; 2108 tx_conf->nb_tcs = num_tcs; 2109 2110 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2111 rx_conf->dcb_tc[i] = i % num_tcs; 2112 tx_conf->dcb_tc[i] = i % num_tcs; 2113 } 2114 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2115 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf; 2116 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2117 } 2118 2119 if (pfc_en) 2120 eth_conf->dcb_capability_en = 2121 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2122 else 2123 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2124 2125 return 0; 2126 } 2127 2128 int 2129 init_port_dcb_config(portid_t pid, 2130 enum dcb_mode_enable dcb_mode, 2131 enum rte_eth_nb_tcs num_tcs, 2132 uint8_t pfc_en) 2133 { 2134 struct rte_eth_conf port_conf; 2135 struct rte_port *rte_port; 2136 int retval; 2137 uint16_t i; 2138 2139 rte_port = &ports[pid]; 2140 2141 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2142 /* Enter DCB configuration status */ 2143 dcb_config = 1; 2144 2145 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2146 retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en); 2147 if (retval < 0) 2148 return retval; 2149 port_conf.rxmode.hw_vlan_filter = 1; 2150 2151 /** 2152 * Write the configuration into the device. 2153 * Set the numbers of RX & TX queues to 0, so 2154 * the RX & TX queues will not be setup. 2155 */ 2156 (void)rte_eth_dev_configure(pid, 0, 0, &port_conf); 2157 2158 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2159 2160 /* If dev_info.vmdq_pool_base is greater than 0, 2161 * the queue id of vmdq pools is started after pf queues. 2162 */ 2163 if (dcb_mode == DCB_VT_ENABLED && 2164 rte_port->dev_info.vmdq_pool_base > 0) { 2165 printf("VMDQ_DCB multi-queue mode is nonsensical" 2166 " for port %d.", pid); 2167 return -1; 2168 } 2169 2170 /* Assume the ports in testpmd have the same dcb capability 2171 * and has the same number of rxq and txq in dcb mode 2172 */ 2173 if (dcb_mode == DCB_VT_ENABLED) { 2174 if (rte_port->dev_info.max_vfs > 0) { 2175 nb_rxq = rte_port->dev_info.nb_rx_queues; 2176 nb_txq = rte_port->dev_info.nb_tx_queues; 2177 } else { 2178 nb_rxq = rte_port->dev_info.max_rx_queues; 2179 nb_txq = rte_port->dev_info.max_tx_queues; 2180 } 2181 } else { 2182 /*if vt is disabled, use all pf queues */ 2183 if (rte_port->dev_info.vmdq_pool_base == 0) { 2184 nb_rxq = rte_port->dev_info.max_rx_queues; 2185 nb_txq = rte_port->dev_info.max_tx_queues; 2186 } else { 2187 nb_rxq = (queueid_t)num_tcs; 2188 nb_txq = (queueid_t)num_tcs; 2189 2190 } 2191 } 2192 rx_free_thresh = 64; 2193 2194 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 2195 2196 rxtx_port_config(rte_port); 2197 /* VLAN filter */ 2198 rte_port->dev_conf.rxmode.hw_vlan_filter = 1; 2199 for (i = 0; i < RTE_DIM(vlan_tags); i++) 2200 rx_vft_set(pid, vlan_tags[i], 1); 2201 2202 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 2203 map_port_queue_stats_mapping_registers(pid, rte_port); 2204 2205 rte_port->dcb_flag = 1; 2206 2207 return 0; 2208 } 2209 2210 static void 2211 init_port(void) 2212 { 2213 /* Configuration of Ethernet ports. */ 2214 ports = rte_zmalloc("testpmd: ports", 2215 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 2216 RTE_CACHE_LINE_SIZE); 2217 if (ports == NULL) { 2218 rte_exit(EXIT_FAILURE, 2219 "rte_zmalloc(%d struct rte_port) failed\n", 2220 RTE_MAX_ETHPORTS); 2221 } 2222 } 2223 2224 static void 2225 force_quit(void) 2226 { 2227 pmd_test_exit(); 2228 prompt_exit(); 2229 } 2230 2231 static void 2232 print_stats(void) 2233 { 2234 uint8_t i; 2235 const char clr[] = { 27, '[', '2', 'J', '\0' }; 2236 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 2237 2238 /* Clear screen and move to top left */ 2239 printf("%s%s", clr, top_left); 2240 2241 printf("\nPort statistics ===================================="); 2242 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2243 nic_stats_display(fwd_ports_ids[i]); 2244 } 2245 2246 static void 2247 signal_handler(int signum) 2248 { 2249 if (signum == SIGINT || signum == SIGTERM) { 2250 printf("\nSignal %d received, preparing to exit...\n", 2251 signum); 2252 #ifdef RTE_LIBRTE_PDUMP 2253 /* uninitialize packet capture framework */ 2254 rte_pdump_uninit(); 2255 #endif 2256 #ifdef RTE_LIBRTE_LATENCY_STATS 2257 rte_latencystats_uninit(); 2258 #endif 2259 force_quit(); 2260 /* exit with the expected status */ 2261 signal(signum, SIG_DFL); 2262 kill(getpid(), signum); 2263 } 2264 } 2265 2266 int 2267 main(int argc, char** argv) 2268 { 2269 int diag; 2270 uint8_t port_id; 2271 2272 signal(SIGINT, signal_handler); 2273 signal(SIGTERM, signal_handler); 2274 2275 diag = rte_eal_init(argc, argv); 2276 if (diag < 0) 2277 rte_panic("Cannot init EAL\n"); 2278 2279 #ifdef RTE_LIBRTE_PDUMP 2280 /* initialize packet capture framework */ 2281 rte_pdump_init(NULL); 2282 #endif 2283 2284 nb_ports = (portid_t) rte_eth_dev_count(); 2285 if (nb_ports == 0) 2286 RTE_LOG(WARNING, EAL, "No probed ethernet devices\n"); 2287 2288 /* allocate port structures, and init them */ 2289 init_port(); 2290 2291 set_def_fwd_config(); 2292 if (nb_lcores == 0) 2293 rte_panic("Empty set of forwarding logical cores - check the " 2294 "core mask supplied in the command parameters\n"); 2295 2296 /* Bitrate/latency stats disabled by default */ 2297 #ifdef RTE_LIBRTE_BITRATE 2298 bitrate_enabled = 0; 2299 #endif 2300 #ifdef RTE_LIBRTE_LATENCY_STATS 2301 latencystats_enabled = 0; 2302 #endif 2303 2304 argc -= diag; 2305 argv += diag; 2306 if (argc > 1) 2307 launch_args_parse(argc, argv); 2308 2309 if (tx_first && interactive) 2310 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 2311 "interactive mode.\n"); 2312 if (!nb_rxq && !nb_txq) 2313 printf("Warning: Either rx or tx queues should be non-zero\n"); 2314 2315 if (nb_rxq > 1 && nb_rxq > nb_txq) 2316 printf("Warning: nb_rxq=%d enables RSS configuration, " 2317 "but nb_txq=%d will prevent to fully test it.\n", 2318 nb_rxq, nb_txq); 2319 2320 init_config(); 2321 if (start_port(RTE_PORT_ALL) != 0) 2322 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 2323 2324 /* set all ports to promiscuous mode by default */ 2325 RTE_ETH_FOREACH_DEV(port_id) 2326 rte_eth_promiscuous_enable(port_id); 2327 2328 /* Init metrics library */ 2329 rte_metrics_init(rte_socket_id()); 2330 2331 #ifdef RTE_LIBRTE_LATENCY_STATS 2332 if (latencystats_enabled != 0) { 2333 int ret = rte_latencystats_init(1, NULL); 2334 if (ret) 2335 printf("Warning: latencystats init()" 2336 " returned error %d\n", ret); 2337 printf("Latencystats running on lcore %d\n", 2338 latencystats_lcore_id); 2339 } 2340 #endif 2341 2342 /* Setup bitrate stats */ 2343 #ifdef RTE_LIBRTE_BITRATE 2344 if (bitrate_enabled != 0) { 2345 bitrate_data = rte_stats_bitrate_create(); 2346 if (bitrate_data == NULL) 2347 rte_exit(EXIT_FAILURE, 2348 "Could not allocate bitrate data.\n"); 2349 rte_stats_bitrate_reg(bitrate_data); 2350 } 2351 #endif 2352 2353 #ifdef RTE_LIBRTE_CMDLINE 2354 if (strlen(cmdline_filename) != 0) 2355 cmdline_read_from_file(cmdline_filename); 2356 2357 if (interactive == 1) { 2358 if (auto_start) { 2359 printf("Start automatic packet forwarding\n"); 2360 start_packet_forwarding(0); 2361 } 2362 prompt(); 2363 pmd_test_exit(); 2364 } else 2365 #endif 2366 { 2367 char c; 2368 int rc; 2369 2370 printf("No commandline core given, start packet forwarding\n"); 2371 start_packet_forwarding(tx_first); 2372 if (stats_period != 0) { 2373 uint64_t prev_time = 0, cur_time, diff_time = 0; 2374 uint64_t timer_period; 2375 2376 /* Convert to number of cycles */ 2377 timer_period = stats_period * rte_get_timer_hz(); 2378 2379 while (1) { 2380 cur_time = rte_get_timer_cycles(); 2381 diff_time += cur_time - prev_time; 2382 2383 if (diff_time >= timer_period) { 2384 print_stats(); 2385 /* Reset the timer */ 2386 diff_time = 0; 2387 } 2388 /* Sleep to avoid unnecessary checks */ 2389 prev_time = cur_time; 2390 sleep(1); 2391 } 2392 } 2393 2394 printf("Press enter to exit\n"); 2395 rc = read(0, &c, 1); 2396 pmd_test_exit(); 2397 if (rc < 0) 2398 return 1; 2399 } 2400 2401 return 0; 2402 } 2403