1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdarg.h> 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <signal.h> 38 #include <string.h> 39 #include <time.h> 40 #include <fcntl.h> 41 #include <sys/types.h> 42 #include <errno.h> 43 44 #include <sys/queue.h> 45 #include <sys/stat.h> 46 47 #include <stdint.h> 48 #include <unistd.h> 49 #include <inttypes.h> 50 51 #include <rte_common.h> 52 #include <rte_errno.h> 53 #include <rte_byteorder.h> 54 #include <rte_log.h> 55 #include <rte_debug.h> 56 #include <rte_cycles.h> 57 #include <rte_memory.h> 58 #include <rte_memcpy.h> 59 #include <rte_memzone.h> 60 #include <rte_launch.h> 61 #include <rte_eal.h> 62 #include <rte_alarm.h> 63 #include <rte_per_lcore.h> 64 #include <rte_lcore.h> 65 #include <rte_atomic.h> 66 #include <rte_branch_prediction.h> 67 #include <rte_mempool.h> 68 #include <rte_malloc.h> 69 #include <rte_mbuf.h> 70 #include <rte_interrupts.h> 71 #include <rte_pci.h> 72 #include <rte_ether.h> 73 #include <rte_ethdev.h> 74 #include <rte_dev.h> 75 #include <rte_string_fns.h> 76 #ifdef RTE_LIBRTE_IXGBE_PMD 77 #include <rte_pmd_ixgbe.h> 78 #endif 79 #ifdef RTE_LIBRTE_PMD_XENVIRT 80 #include <rte_eth_xenvirt.h> 81 #endif 82 #ifdef RTE_LIBRTE_PDUMP 83 #include <rte_pdump.h> 84 #endif 85 #include <rte_flow.h> 86 #include <rte_metrics.h> 87 #ifdef RTE_LIBRTE_BITRATE 88 #include <rte_bitrate.h> 89 #endif 90 #ifdef RTE_LIBRTE_LATENCY_STATS 91 #include <rte_latencystats.h> 92 #endif 93 94 #include "testpmd.h" 95 96 uint16_t verbose_level = 0; /**< Silent by default. */ 97 98 /* use master core for command line ? */ 99 uint8_t interactive = 0; 100 uint8_t auto_start = 0; 101 char cmdline_filename[PATH_MAX] = {0}; 102 103 /* 104 * NUMA support configuration. 105 * When set, the NUMA support attempts to dispatch the allocation of the 106 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 107 * probed ports among the CPU sockets 0 and 1. 108 * Otherwise, all memory is allocated from CPU socket 0. 109 */ 110 uint8_t numa_support = 1; /**< numa enabled by default */ 111 112 /* 113 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 114 * not configured. 115 */ 116 uint8_t socket_num = UMA_NO_CONFIG; 117 118 /* 119 * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs. 120 */ 121 uint8_t mp_anon = 0; 122 123 /* 124 * Record the Ethernet address of peer target ports to which packets are 125 * forwarded. 126 * Must be instantiated with the ethernet addresses of peer traffic generator 127 * ports. 128 */ 129 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 130 portid_t nb_peer_eth_addrs = 0; 131 132 /* 133 * Probed Target Environment. 134 */ 135 struct rte_port *ports; /**< For all probed ethernet ports. */ 136 portid_t nb_ports; /**< Number of probed ethernet ports. */ 137 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 138 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 139 140 /* 141 * Test Forwarding Configuration. 142 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 143 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 144 */ 145 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 146 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 147 portid_t nb_cfg_ports; /**< Number of configured ports. */ 148 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 149 150 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 151 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 152 153 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 154 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 155 156 /* 157 * Forwarding engines. 158 */ 159 struct fwd_engine * fwd_engines[] = { 160 &io_fwd_engine, 161 &mac_fwd_engine, 162 &mac_swap_engine, 163 &flow_gen_engine, 164 &rx_only_engine, 165 &tx_only_engine, 166 &csum_fwd_engine, 167 &icmp_echo_engine, 168 #ifdef RTE_LIBRTE_IEEE1588 169 &ieee1588_fwd_engine, 170 #endif 171 NULL, 172 }; 173 174 struct fwd_config cur_fwd_config; 175 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 176 uint32_t retry_enabled; 177 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 178 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 179 180 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 181 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 182 * specified on command-line. */ 183 184 /* 185 * Configuration of packet segments used by the "txonly" processing engine. 186 */ 187 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 188 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 189 TXONLY_DEF_PACKET_LEN, 190 }; 191 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 192 193 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 194 /**< Split policy for packets to TX. */ 195 196 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 197 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 198 199 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 200 uint8_t dcb_config = 0; 201 202 /* Whether the dcb is in testing status */ 203 uint8_t dcb_test = 0; 204 205 /* 206 * Configurable number of RX/TX queues. 207 */ 208 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 209 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 210 211 /* 212 * Configurable number of RX/TX ring descriptors. 213 */ 214 #define RTE_TEST_RX_DESC_DEFAULT 128 215 #define RTE_TEST_TX_DESC_DEFAULT 512 216 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 217 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 218 219 #define RTE_PMD_PARAM_UNSET -1 220 /* 221 * Configurable values of RX and TX ring threshold registers. 222 */ 223 224 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 225 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 226 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 227 228 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 229 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 230 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 231 232 /* 233 * Configurable value of RX free threshold. 234 */ 235 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 236 237 /* 238 * Configurable value of RX drop enable. 239 */ 240 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 241 242 /* 243 * Configurable value of TX free threshold. 244 */ 245 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 246 247 /* 248 * Configurable value of TX RS bit threshold. 249 */ 250 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 251 252 /* 253 * Configurable value of TX queue flags. 254 */ 255 int32_t txq_flags = RTE_PMD_PARAM_UNSET; 256 257 /* 258 * Receive Side Scaling (RSS) configuration. 259 */ 260 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 261 262 /* 263 * Port topology configuration 264 */ 265 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 266 267 /* 268 * Avoids to flush all the RX streams before starts forwarding. 269 */ 270 uint8_t no_flush_rx = 0; /* flush by default */ 271 272 /* 273 * Avoids to check link status when starting/stopping a port. 274 */ 275 uint8_t no_link_check = 0; /* check by default */ 276 277 /* 278 * Enable link status change notification 279 */ 280 uint8_t lsc_interrupt = 1; /* enabled by default */ 281 282 /* 283 * Enable device removal notification. 284 */ 285 uint8_t rmv_interrupt = 1; /* enabled by default */ 286 287 /* 288 * Display or mask ether events 289 * Default to all events except VF_MBOX 290 */ 291 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 292 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 293 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 294 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 295 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 296 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 297 298 /* 299 * NIC bypass mode configuration options. 300 */ 301 302 #ifdef RTE_LIBRTE_IXGBE_BYPASS 303 /* The NIC bypass watchdog timeout. */ 304 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 305 #endif 306 307 308 #ifdef RTE_LIBRTE_LATENCY_STATS 309 310 /* 311 * Set when latency stats is enabled in the commandline 312 */ 313 uint8_t latencystats_enabled; 314 315 /* 316 * Lcore ID to serive latency statistics. 317 */ 318 lcoreid_t latencystats_lcore_id = -1; 319 320 #endif 321 322 /* 323 * Ethernet device configuration. 324 */ 325 struct rte_eth_rxmode rx_mode = { 326 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 327 .split_hdr_size = 0, 328 .header_split = 0, /**< Header Split disabled. */ 329 .hw_ip_checksum = 0, /**< IP checksum offload disabled. */ 330 .hw_vlan_filter = 1, /**< VLAN filtering enabled. */ 331 .hw_vlan_strip = 1, /**< VLAN strip enabled. */ 332 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 333 .jumbo_frame = 0, /**< Jumbo Frame Support disabled. */ 334 .hw_strip_crc = 1, /**< CRC stripping by hardware enabled. */ 335 }; 336 337 struct rte_fdir_conf fdir_conf = { 338 .mode = RTE_FDIR_MODE_NONE, 339 .pballoc = RTE_FDIR_PBALLOC_64K, 340 .status = RTE_FDIR_REPORT_STATUS, 341 .mask = { 342 .vlan_tci_mask = 0x0, 343 .ipv4_mask = { 344 .src_ip = 0xFFFFFFFF, 345 .dst_ip = 0xFFFFFFFF, 346 }, 347 .ipv6_mask = { 348 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 349 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 350 }, 351 .src_port_mask = 0xFFFF, 352 .dst_port_mask = 0xFFFF, 353 .mac_addr_byte_mask = 0xFF, 354 .tunnel_type_mask = 1, 355 .tunnel_id_mask = 0xFFFFFFFF, 356 }, 357 .drop_queue = 127, 358 }; 359 360 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 361 362 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 363 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 364 365 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 366 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 367 368 uint16_t nb_tx_queue_stats_mappings = 0; 369 uint16_t nb_rx_queue_stats_mappings = 0; 370 371 unsigned int num_sockets = 0; 372 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 373 374 #ifdef RTE_LIBRTE_BITRATE 375 /* Bitrate statistics */ 376 struct rte_stats_bitrates *bitrate_data; 377 lcoreid_t bitrate_lcore_id; 378 uint8_t bitrate_enabled; 379 #endif 380 381 /* Forward function declarations */ 382 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port); 383 static void check_all_ports_link_status(uint32_t port_mask); 384 static void eth_event_callback(uint8_t port_id, 385 enum rte_eth_event_type type, 386 void *param); 387 388 /* 389 * Check if all the ports are started. 390 * If yes, return positive value. If not, return zero. 391 */ 392 static int all_ports_started(void); 393 394 /* 395 * Helper function to check if socket is already discovered. 396 * If yes, return positive value. If not, return zero. 397 */ 398 int 399 new_socket_id(unsigned int socket_id) 400 { 401 unsigned int i; 402 403 for (i = 0; i < num_sockets; i++) { 404 if (socket_ids[i] == socket_id) 405 return 0; 406 } 407 return 1; 408 } 409 410 /* 411 * Setup default configuration. 412 */ 413 static void 414 set_default_fwd_lcores_config(void) 415 { 416 unsigned int i; 417 unsigned int nb_lc; 418 unsigned int sock_num; 419 420 nb_lc = 0; 421 for (i = 0; i < RTE_MAX_LCORE; i++) { 422 sock_num = rte_lcore_to_socket_id(i); 423 if (new_socket_id(sock_num)) { 424 if (num_sockets >= RTE_MAX_NUMA_NODES) { 425 rte_exit(EXIT_FAILURE, 426 "Total sockets greater than %u\n", 427 RTE_MAX_NUMA_NODES); 428 } 429 socket_ids[num_sockets++] = sock_num; 430 } 431 if (!rte_lcore_is_enabled(i)) 432 continue; 433 if (i == rte_get_master_lcore()) 434 continue; 435 fwd_lcores_cpuids[nb_lc++] = i; 436 } 437 nb_lcores = (lcoreid_t) nb_lc; 438 nb_cfg_lcores = nb_lcores; 439 nb_fwd_lcores = 1; 440 } 441 442 static void 443 set_def_peer_eth_addrs(void) 444 { 445 portid_t i; 446 447 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 448 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 449 peer_eth_addrs[i].addr_bytes[5] = i; 450 } 451 } 452 453 static void 454 set_default_fwd_ports_config(void) 455 { 456 portid_t pt_id; 457 458 for (pt_id = 0; pt_id < nb_ports; pt_id++) 459 fwd_ports_ids[pt_id] = pt_id; 460 461 nb_cfg_ports = nb_ports; 462 nb_fwd_ports = nb_ports; 463 } 464 465 void 466 set_def_fwd_config(void) 467 { 468 set_default_fwd_lcores_config(); 469 set_def_peer_eth_addrs(); 470 set_default_fwd_ports_config(); 471 } 472 473 /* 474 * Configuration initialisation done once at init time. 475 */ 476 static void 477 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 478 unsigned int socket_id) 479 { 480 char pool_name[RTE_MEMPOOL_NAMESIZE]; 481 struct rte_mempool *rte_mp = NULL; 482 uint32_t mb_size; 483 484 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 485 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 486 487 RTE_LOG(INFO, USER1, 488 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 489 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 490 491 #ifdef RTE_LIBRTE_PMD_XENVIRT 492 rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size, 493 (unsigned) mb_mempool_cache, 494 sizeof(struct rte_pktmbuf_pool_private), 495 rte_pktmbuf_pool_init, NULL, 496 rte_pktmbuf_init, NULL, 497 socket_id, 0); 498 #endif 499 500 /* if the former XEN allocation failed fall back to normal allocation */ 501 if (rte_mp == NULL) { 502 if (mp_anon != 0) { 503 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 504 mb_size, (unsigned) mb_mempool_cache, 505 sizeof(struct rte_pktmbuf_pool_private), 506 socket_id, 0); 507 if (rte_mp == NULL) 508 goto err; 509 510 if (rte_mempool_populate_anon(rte_mp) == 0) { 511 rte_mempool_free(rte_mp); 512 rte_mp = NULL; 513 goto err; 514 } 515 rte_pktmbuf_pool_init(rte_mp, NULL); 516 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 517 } else { 518 /* wrapper to rte_mempool_create() */ 519 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 520 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 521 } 522 } 523 524 err: 525 if (rte_mp == NULL) { 526 rte_exit(EXIT_FAILURE, 527 "Creation of mbuf pool for socket %u failed: %s\n", 528 socket_id, rte_strerror(rte_errno)); 529 } else if (verbose_level > 0) { 530 rte_mempool_dump(stdout, rte_mp); 531 } 532 } 533 534 /* 535 * Check given socket id is valid or not with NUMA mode, 536 * if valid, return 0, else return -1 537 */ 538 static int 539 check_socket_id(const unsigned int socket_id) 540 { 541 static int warning_once = 0; 542 543 if (new_socket_id(socket_id)) { 544 if (!warning_once && numa_support) 545 printf("Warning: NUMA should be configured manually by" 546 " using --port-numa-config and" 547 " --ring-numa-config parameters along with" 548 " --numa.\n"); 549 warning_once = 1; 550 return -1; 551 } 552 return 0; 553 } 554 555 static void 556 init_config(void) 557 { 558 portid_t pid; 559 struct rte_port *port; 560 struct rte_mempool *mbp; 561 unsigned int nb_mbuf_per_pool; 562 lcoreid_t lc_id; 563 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 564 565 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 566 567 if (numa_support) { 568 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 569 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 570 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 571 } 572 573 /* Configuration of logical cores. */ 574 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 575 sizeof(struct fwd_lcore *) * nb_lcores, 576 RTE_CACHE_LINE_SIZE); 577 if (fwd_lcores == NULL) { 578 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 579 "failed\n", nb_lcores); 580 } 581 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 582 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 583 sizeof(struct fwd_lcore), 584 RTE_CACHE_LINE_SIZE); 585 if (fwd_lcores[lc_id] == NULL) { 586 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 587 "failed\n"); 588 } 589 fwd_lcores[lc_id]->cpuid_idx = lc_id; 590 } 591 592 RTE_ETH_FOREACH_DEV(pid) { 593 port = &ports[pid]; 594 rte_eth_dev_info_get(pid, &port->dev_info); 595 596 if (numa_support) { 597 if (port_numa[pid] != NUMA_NO_CONFIG) 598 port_per_socket[port_numa[pid]]++; 599 else { 600 uint32_t socket_id = rte_eth_dev_socket_id(pid); 601 602 /* if socket_id is invalid, set to 0 */ 603 if (check_socket_id(socket_id) < 0) 604 socket_id = 0; 605 port_per_socket[socket_id]++; 606 } 607 } 608 609 /* set flag to initialize port/queue */ 610 port->need_reconfig = 1; 611 port->need_reconfig_queues = 1; 612 } 613 614 /* 615 * Create pools of mbuf. 616 * If NUMA support is disabled, create a single pool of mbuf in 617 * socket 0 memory by default. 618 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 619 * 620 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 621 * nb_txd can be configured at run time. 622 */ 623 if (param_total_num_mbufs) 624 nb_mbuf_per_pool = param_total_num_mbufs; 625 else { 626 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 627 (nb_lcores * mb_mempool_cache) + 628 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 629 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 630 } 631 632 if (numa_support) { 633 uint8_t i; 634 635 for (i = 0; i < num_sockets; i++) 636 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 637 socket_ids[i]); 638 } else { 639 if (socket_num == UMA_NO_CONFIG) 640 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 641 else 642 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 643 socket_num); 644 } 645 646 init_port_config(); 647 648 /* 649 * Records which Mbuf pool to use by each logical core, if needed. 650 */ 651 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 652 mbp = mbuf_pool_find( 653 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 654 655 if (mbp == NULL) 656 mbp = mbuf_pool_find(0); 657 fwd_lcores[lc_id]->mbp = mbp; 658 } 659 660 /* Configuration of packet forwarding streams. */ 661 if (init_fwd_streams() < 0) 662 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 663 664 fwd_config_setup(); 665 } 666 667 668 void 669 reconfig(portid_t new_port_id, unsigned socket_id) 670 { 671 struct rte_port *port; 672 673 /* Reconfiguration of Ethernet ports. */ 674 port = &ports[new_port_id]; 675 rte_eth_dev_info_get(new_port_id, &port->dev_info); 676 677 /* set flag to initialize port/queue */ 678 port->need_reconfig = 1; 679 port->need_reconfig_queues = 1; 680 port->socket_id = socket_id; 681 682 init_port_config(); 683 } 684 685 686 int 687 init_fwd_streams(void) 688 { 689 portid_t pid; 690 struct rte_port *port; 691 streamid_t sm_id, nb_fwd_streams_new; 692 queueid_t q; 693 694 /* set socket id according to numa or not */ 695 RTE_ETH_FOREACH_DEV(pid) { 696 port = &ports[pid]; 697 if (nb_rxq > port->dev_info.max_rx_queues) { 698 printf("Fail: nb_rxq(%d) is greater than " 699 "max_rx_queues(%d)\n", nb_rxq, 700 port->dev_info.max_rx_queues); 701 return -1; 702 } 703 if (nb_txq > port->dev_info.max_tx_queues) { 704 printf("Fail: nb_txq(%d) is greater than " 705 "max_tx_queues(%d)\n", nb_txq, 706 port->dev_info.max_tx_queues); 707 return -1; 708 } 709 if (numa_support) { 710 if (port_numa[pid] != NUMA_NO_CONFIG) 711 port->socket_id = port_numa[pid]; 712 else { 713 port->socket_id = rte_eth_dev_socket_id(pid); 714 715 /* if socket_id is invalid, set to 0 */ 716 if (check_socket_id(port->socket_id) < 0) 717 port->socket_id = 0; 718 } 719 } 720 else { 721 if (socket_num == UMA_NO_CONFIG) 722 port->socket_id = 0; 723 else 724 port->socket_id = socket_num; 725 } 726 } 727 728 q = RTE_MAX(nb_rxq, nb_txq); 729 if (q == 0) { 730 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 731 return -1; 732 } 733 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 734 if (nb_fwd_streams_new == nb_fwd_streams) 735 return 0; 736 /* clear the old */ 737 if (fwd_streams != NULL) { 738 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 739 if (fwd_streams[sm_id] == NULL) 740 continue; 741 rte_free(fwd_streams[sm_id]); 742 fwd_streams[sm_id] = NULL; 743 } 744 rte_free(fwd_streams); 745 fwd_streams = NULL; 746 } 747 748 /* init new */ 749 nb_fwd_streams = nb_fwd_streams_new; 750 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 751 sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE); 752 if (fwd_streams == NULL) 753 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) " 754 "failed\n", nb_fwd_streams); 755 756 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 757 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream", 758 sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE); 759 if (fwd_streams[sm_id] == NULL) 760 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)" 761 " failed\n"); 762 } 763 764 return 0; 765 } 766 767 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 768 static void 769 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 770 { 771 unsigned int total_burst; 772 unsigned int nb_burst; 773 unsigned int burst_stats[3]; 774 uint16_t pktnb_stats[3]; 775 uint16_t nb_pkt; 776 int burst_percent[3]; 777 778 /* 779 * First compute the total number of packet bursts and the 780 * two highest numbers of bursts of the same number of packets. 781 */ 782 total_burst = 0; 783 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 784 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 785 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 786 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 787 if (nb_burst == 0) 788 continue; 789 total_burst += nb_burst; 790 if (nb_burst > burst_stats[0]) { 791 burst_stats[1] = burst_stats[0]; 792 pktnb_stats[1] = pktnb_stats[0]; 793 burst_stats[0] = nb_burst; 794 pktnb_stats[0] = nb_pkt; 795 } 796 } 797 if (total_burst == 0) 798 return; 799 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 800 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 801 burst_percent[0], (int) pktnb_stats[0]); 802 if (burst_stats[0] == total_burst) { 803 printf("]\n"); 804 return; 805 } 806 if (burst_stats[0] + burst_stats[1] == total_burst) { 807 printf(" + %d%% of %d pkts]\n", 808 100 - burst_percent[0], pktnb_stats[1]); 809 return; 810 } 811 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 812 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 813 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 814 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 815 return; 816 } 817 printf(" + %d%% of %d pkts + %d%% of others]\n", 818 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 819 } 820 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 821 822 static void 823 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats) 824 { 825 struct rte_port *port; 826 uint8_t i; 827 828 static const char *fwd_stats_border = "----------------------"; 829 830 port = &ports[port_id]; 831 printf("\n %s Forward statistics for port %-2d %s\n", 832 fwd_stats_border, port_id, fwd_stats_border); 833 834 if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) { 835 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 836 "%-"PRIu64"\n", 837 stats->ipackets, stats->imissed, 838 (uint64_t) (stats->ipackets + stats->imissed)); 839 840 if (cur_fwd_eng == &csum_fwd_engine) 841 printf(" Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n", 842 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 843 if ((stats->ierrors + stats->rx_nombuf) > 0) { 844 printf(" RX-error: %-"PRIu64"\n", stats->ierrors); 845 printf(" RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf); 846 } 847 848 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 849 "%-"PRIu64"\n", 850 stats->opackets, port->tx_dropped, 851 (uint64_t) (stats->opackets + port->tx_dropped)); 852 } 853 else { 854 printf(" RX-packets: %14"PRIu64" RX-dropped:%14"PRIu64" RX-total:" 855 "%14"PRIu64"\n", 856 stats->ipackets, stats->imissed, 857 (uint64_t) (stats->ipackets + stats->imissed)); 858 859 if (cur_fwd_eng == &csum_fwd_engine) 860 printf(" Bad-ipcsum:%14"PRIu64" Bad-l4csum:%14"PRIu64"\n", 861 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 862 if ((stats->ierrors + stats->rx_nombuf) > 0) { 863 printf(" RX-error:%"PRIu64"\n", stats->ierrors); 864 printf(" RX-nombufs: %14"PRIu64"\n", 865 stats->rx_nombuf); 866 } 867 868 printf(" TX-packets: %14"PRIu64" TX-dropped:%14"PRIu64" TX-total:" 869 "%14"PRIu64"\n", 870 stats->opackets, port->tx_dropped, 871 (uint64_t) (stats->opackets + port->tx_dropped)); 872 } 873 874 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 875 if (port->rx_stream) 876 pkt_burst_stats_display("RX", 877 &port->rx_stream->rx_burst_stats); 878 if (port->tx_stream) 879 pkt_burst_stats_display("TX", 880 &port->tx_stream->tx_burst_stats); 881 #endif 882 883 if (port->rx_queue_stats_mapping_enabled) { 884 printf("\n"); 885 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 886 printf(" Stats reg %2d RX-packets:%14"PRIu64 887 " RX-errors:%14"PRIu64 888 " RX-bytes:%14"PRIu64"\n", 889 i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]); 890 } 891 printf("\n"); 892 } 893 if (port->tx_queue_stats_mapping_enabled) { 894 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 895 printf(" Stats reg %2d TX-packets:%14"PRIu64 896 " TX-bytes:%14"PRIu64"\n", 897 i, stats->q_opackets[i], stats->q_obytes[i]); 898 } 899 } 900 901 printf(" %s--------------------------------%s\n", 902 fwd_stats_border, fwd_stats_border); 903 } 904 905 static void 906 fwd_stream_stats_display(streamid_t stream_id) 907 { 908 struct fwd_stream *fs; 909 static const char *fwd_top_stats_border = "-------"; 910 911 fs = fwd_streams[stream_id]; 912 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 913 (fs->fwd_dropped == 0)) 914 return; 915 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 916 "TX Port=%2d/Queue=%2d %s\n", 917 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 918 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 919 printf(" RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u", 920 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 921 922 /* if checksum mode */ 923 if (cur_fwd_eng == &csum_fwd_engine) { 924 printf(" RX- bad IP checksum: %-14u Rx- bad L4 checksum: " 925 "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum); 926 } 927 928 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 929 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 930 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 931 #endif 932 } 933 934 static void 935 flush_fwd_rx_queues(void) 936 { 937 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 938 portid_t rxp; 939 portid_t port_id; 940 queueid_t rxq; 941 uint16_t nb_rx; 942 uint16_t i; 943 uint8_t j; 944 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 945 uint64_t timer_period; 946 947 /* convert to number of cycles */ 948 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 949 950 for (j = 0; j < 2; j++) { 951 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 952 for (rxq = 0; rxq < nb_rxq; rxq++) { 953 port_id = fwd_ports_ids[rxp]; 954 /** 955 * testpmd can stuck in the below do while loop 956 * if rte_eth_rx_burst() always returns nonzero 957 * packets. So timer is added to exit this loop 958 * after 1sec timer expiry. 959 */ 960 prev_tsc = rte_rdtsc(); 961 do { 962 nb_rx = rte_eth_rx_burst(port_id, rxq, 963 pkts_burst, MAX_PKT_BURST); 964 for (i = 0; i < nb_rx; i++) 965 rte_pktmbuf_free(pkts_burst[i]); 966 967 cur_tsc = rte_rdtsc(); 968 diff_tsc = cur_tsc - prev_tsc; 969 timer_tsc += diff_tsc; 970 } while ((nb_rx > 0) && 971 (timer_tsc < timer_period)); 972 timer_tsc = 0; 973 } 974 } 975 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 976 } 977 } 978 979 static void 980 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 981 { 982 struct fwd_stream **fsm; 983 streamid_t nb_fs; 984 streamid_t sm_id; 985 #ifdef RTE_LIBRTE_BITRATE 986 uint64_t tics_per_1sec; 987 uint64_t tics_datum; 988 uint64_t tics_current; 989 uint8_t idx_port, cnt_ports; 990 991 cnt_ports = rte_eth_dev_count(); 992 tics_datum = rte_rdtsc(); 993 tics_per_1sec = rte_get_timer_hz(); 994 #endif 995 fsm = &fwd_streams[fc->stream_idx]; 996 nb_fs = fc->stream_nb; 997 do { 998 for (sm_id = 0; sm_id < nb_fs; sm_id++) 999 (*pkt_fwd)(fsm[sm_id]); 1000 #ifdef RTE_LIBRTE_BITRATE 1001 if (bitrate_enabled != 0 && 1002 bitrate_lcore_id == rte_lcore_id()) { 1003 tics_current = rte_rdtsc(); 1004 if (tics_current - tics_datum >= tics_per_1sec) { 1005 /* Periodic bitrate calculation */ 1006 for (idx_port = 0; 1007 idx_port < cnt_ports; 1008 idx_port++) 1009 rte_stats_bitrate_calc(bitrate_data, 1010 idx_port); 1011 tics_datum = tics_current; 1012 } 1013 } 1014 #endif 1015 #ifdef RTE_LIBRTE_LATENCY_STATS 1016 if (latencystats_enabled != 0 && 1017 latencystats_lcore_id == rte_lcore_id()) 1018 rte_latencystats_update(); 1019 #endif 1020 1021 } while (! fc->stopped); 1022 } 1023 1024 static int 1025 start_pkt_forward_on_core(void *fwd_arg) 1026 { 1027 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1028 cur_fwd_config.fwd_eng->packet_fwd); 1029 return 0; 1030 } 1031 1032 /* 1033 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1034 * Used to start communication flows in network loopback test configurations. 1035 */ 1036 static int 1037 run_one_txonly_burst_on_core(void *fwd_arg) 1038 { 1039 struct fwd_lcore *fwd_lc; 1040 struct fwd_lcore tmp_lcore; 1041 1042 fwd_lc = (struct fwd_lcore *) fwd_arg; 1043 tmp_lcore = *fwd_lc; 1044 tmp_lcore.stopped = 1; 1045 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1046 return 0; 1047 } 1048 1049 /* 1050 * Launch packet forwarding: 1051 * - Setup per-port forwarding context. 1052 * - launch logical cores with their forwarding configuration. 1053 */ 1054 static void 1055 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1056 { 1057 port_fwd_begin_t port_fwd_begin; 1058 unsigned int i; 1059 unsigned int lc_id; 1060 int diag; 1061 1062 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1063 if (port_fwd_begin != NULL) { 1064 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1065 (*port_fwd_begin)(fwd_ports_ids[i]); 1066 } 1067 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1068 lc_id = fwd_lcores_cpuids[i]; 1069 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1070 fwd_lcores[i]->stopped = 0; 1071 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1072 fwd_lcores[i], lc_id); 1073 if (diag != 0) 1074 printf("launch lcore %u failed - diag=%d\n", 1075 lc_id, diag); 1076 } 1077 } 1078 } 1079 1080 /* 1081 * Launch packet forwarding configuration. 1082 */ 1083 void 1084 start_packet_forwarding(int with_tx_first) 1085 { 1086 port_fwd_begin_t port_fwd_begin; 1087 port_fwd_end_t port_fwd_end; 1088 struct rte_port *port; 1089 unsigned int i; 1090 portid_t pt_id; 1091 streamid_t sm_id; 1092 1093 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1094 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1095 1096 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1097 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1098 1099 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1100 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1101 (!nb_rxq || !nb_txq)) 1102 rte_exit(EXIT_FAILURE, 1103 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1104 cur_fwd_eng->fwd_mode_name); 1105 1106 if (all_ports_started() == 0) { 1107 printf("Not all ports were started\n"); 1108 return; 1109 } 1110 if (test_done == 0) { 1111 printf("Packet forwarding already started\n"); 1112 return; 1113 } 1114 1115 if (init_fwd_streams() < 0) { 1116 printf("Fail from init_fwd_streams()\n"); 1117 return; 1118 } 1119 1120 if(dcb_test) { 1121 for (i = 0; i < nb_fwd_ports; i++) { 1122 pt_id = fwd_ports_ids[i]; 1123 port = &ports[pt_id]; 1124 if (!port->dcb_flag) { 1125 printf("In DCB mode, all forwarding ports must " 1126 "be configured in this mode.\n"); 1127 return; 1128 } 1129 } 1130 if (nb_fwd_lcores == 1) { 1131 printf("In DCB mode,the nb forwarding cores " 1132 "should be larger than 1.\n"); 1133 return; 1134 } 1135 } 1136 test_done = 0; 1137 1138 if(!no_flush_rx) 1139 flush_fwd_rx_queues(); 1140 1141 fwd_config_setup(); 1142 pkt_fwd_config_display(&cur_fwd_config); 1143 rxtx_config_display(); 1144 1145 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1146 pt_id = fwd_ports_ids[i]; 1147 port = &ports[pt_id]; 1148 rte_eth_stats_get(pt_id, &port->stats); 1149 port->tx_dropped = 0; 1150 1151 map_port_queue_stats_mapping_registers(pt_id, port); 1152 } 1153 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1154 fwd_streams[sm_id]->rx_packets = 0; 1155 fwd_streams[sm_id]->tx_packets = 0; 1156 fwd_streams[sm_id]->fwd_dropped = 0; 1157 fwd_streams[sm_id]->rx_bad_ip_csum = 0; 1158 fwd_streams[sm_id]->rx_bad_l4_csum = 0; 1159 1160 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1161 memset(&fwd_streams[sm_id]->rx_burst_stats, 0, 1162 sizeof(fwd_streams[sm_id]->rx_burst_stats)); 1163 memset(&fwd_streams[sm_id]->tx_burst_stats, 0, 1164 sizeof(fwd_streams[sm_id]->tx_burst_stats)); 1165 #endif 1166 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1167 fwd_streams[sm_id]->core_cycles = 0; 1168 #endif 1169 } 1170 if (with_tx_first) { 1171 port_fwd_begin = tx_only_engine.port_fwd_begin; 1172 if (port_fwd_begin != NULL) { 1173 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1174 (*port_fwd_begin)(fwd_ports_ids[i]); 1175 } 1176 while (with_tx_first--) { 1177 launch_packet_forwarding( 1178 run_one_txonly_burst_on_core); 1179 rte_eal_mp_wait_lcore(); 1180 } 1181 port_fwd_end = tx_only_engine.port_fwd_end; 1182 if (port_fwd_end != NULL) { 1183 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1184 (*port_fwd_end)(fwd_ports_ids[i]); 1185 } 1186 } 1187 launch_packet_forwarding(start_pkt_forward_on_core); 1188 } 1189 1190 void 1191 stop_packet_forwarding(void) 1192 { 1193 struct rte_eth_stats stats; 1194 struct rte_port *port; 1195 port_fwd_end_t port_fwd_end; 1196 int i; 1197 portid_t pt_id; 1198 streamid_t sm_id; 1199 lcoreid_t lc_id; 1200 uint64_t total_recv; 1201 uint64_t total_xmit; 1202 uint64_t total_rx_dropped; 1203 uint64_t total_tx_dropped; 1204 uint64_t total_rx_nombuf; 1205 uint64_t tx_dropped; 1206 uint64_t rx_bad_ip_csum; 1207 uint64_t rx_bad_l4_csum; 1208 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1209 uint64_t fwd_cycles; 1210 #endif 1211 static const char *acc_stats_border = "+++++++++++++++"; 1212 1213 if (test_done) { 1214 printf("Packet forwarding not started\n"); 1215 return; 1216 } 1217 printf("Telling cores to stop..."); 1218 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1219 fwd_lcores[lc_id]->stopped = 1; 1220 printf("\nWaiting for lcores to finish...\n"); 1221 rte_eal_mp_wait_lcore(); 1222 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1223 if (port_fwd_end != NULL) { 1224 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1225 pt_id = fwd_ports_ids[i]; 1226 (*port_fwd_end)(pt_id); 1227 } 1228 } 1229 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1230 fwd_cycles = 0; 1231 #endif 1232 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1233 if (cur_fwd_config.nb_fwd_streams > 1234 cur_fwd_config.nb_fwd_ports) { 1235 fwd_stream_stats_display(sm_id); 1236 ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL; 1237 ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL; 1238 } else { 1239 ports[fwd_streams[sm_id]->tx_port].tx_stream = 1240 fwd_streams[sm_id]; 1241 ports[fwd_streams[sm_id]->rx_port].rx_stream = 1242 fwd_streams[sm_id]; 1243 } 1244 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped; 1245 tx_dropped = (uint64_t) (tx_dropped + 1246 fwd_streams[sm_id]->fwd_dropped); 1247 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped; 1248 1249 rx_bad_ip_csum = 1250 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum; 1251 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum + 1252 fwd_streams[sm_id]->rx_bad_ip_csum); 1253 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum = 1254 rx_bad_ip_csum; 1255 1256 rx_bad_l4_csum = 1257 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum; 1258 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum + 1259 fwd_streams[sm_id]->rx_bad_l4_csum); 1260 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum = 1261 rx_bad_l4_csum; 1262 1263 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1264 fwd_cycles = (uint64_t) (fwd_cycles + 1265 fwd_streams[sm_id]->core_cycles); 1266 #endif 1267 } 1268 total_recv = 0; 1269 total_xmit = 0; 1270 total_rx_dropped = 0; 1271 total_tx_dropped = 0; 1272 total_rx_nombuf = 0; 1273 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1274 pt_id = fwd_ports_ids[i]; 1275 1276 port = &ports[pt_id]; 1277 rte_eth_stats_get(pt_id, &stats); 1278 stats.ipackets -= port->stats.ipackets; 1279 port->stats.ipackets = 0; 1280 stats.opackets -= port->stats.opackets; 1281 port->stats.opackets = 0; 1282 stats.ibytes -= port->stats.ibytes; 1283 port->stats.ibytes = 0; 1284 stats.obytes -= port->stats.obytes; 1285 port->stats.obytes = 0; 1286 stats.imissed -= port->stats.imissed; 1287 port->stats.imissed = 0; 1288 stats.oerrors -= port->stats.oerrors; 1289 port->stats.oerrors = 0; 1290 stats.rx_nombuf -= port->stats.rx_nombuf; 1291 port->stats.rx_nombuf = 0; 1292 1293 total_recv += stats.ipackets; 1294 total_xmit += stats.opackets; 1295 total_rx_dropped += stats.imissed; 1296 total_tx_dropped += port->tx_dropped; 1297 total_rx_nombuf += stats.rx_nombuf; 1298 1299 fwd_port_stats_display(pt_id, &stats); 1300 } 1301 printf("\n %s Accumulated forward statistics for all ports" 1302 "%s\n", 1303 acc_stats_border, acc_stats_border); 1304 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1305 "%-"PRIu64"\n" 1306 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1307 "%-"PRIu64"\n", 1308 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1309 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1310 if (total_rx_nombuf > 0) 1311 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1312 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1313 "%s\n", 1314 acc_stats_border, acc_stats_border); 1315 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1316 if (total_recv > 0) 1317 printf("\n CPU cycles/packet=%u (total cycles=" 1318 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1319 (unsigned int)(fwd_cycles / total_recv), 1320 fwd_cycles, total_recv); 1321 #endif 1322 printf("\nDone.\n"); 1323 test_done = 1; 1324 } 1325 1326 void 1327 dev_set_link_up(portid_t pid) 1328 { 1329 if (rte_eth_dev_set_link_up((uint8_t)pid) < 0) 1330 printf("\nSet link up fail.\n"); 1331 } 1332 1333 void 1334 dev_set_link_down(portid_t pid) 1335 { 1336 if (rte_eth_dev_set_link_down((uint8_t)pid) < 0) 1337 printf("\nSet link down fail.\n"); 1338 } 1339 1340 static int 1341 all_ports_started(void) 1342 { 1343 portid_t pi; 1344 struct rte_port *port; 1345 1346 RTE_ETH_FOREACH_DEV(pi) { 1347 port = &ports[pi]; 1348 /* Check if there is a port which is not started */ 1349 if ((port->port_status != RTE_PORT_STARTED) && 1350 (port->slave_flag == 0)) 1351 return 0; 1352 } 1353 1354 /* No port is not started */ 1355 return 1; 1356 } 1357 1358 int 1359 all_ports_stopped(void) 1360 { 1361 portid_t pi; 1362 struct rte_port *port; 1363 1364 RTE_ETH_FOREACH_DEV(pi) { 1365 port = &ports[pi]; 1366 if ((port->port_status != RTE_PORT_STOPPED) && 1367 (port->slave_flag == 0)) 1368 return 0; 1369 } 1370 1371 return 1; 1372 } 1373 1374 int 1375 port_is_started(portid_t port_id) 1376 { 1377 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1378 return 0; 1379 1380 if (ports[port_id].port_status != RTE_PORT_STARTED) 1381 return 0; 1382 1383 return 1; 1384 } 1385 1386 static int 1387 port_is_closed(portid_t port_id) 1388 { 1389 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1390 return 0; 1391 1392 if (ports[port_id].port_status != RTE_PORT_CLOSED) 1393 return 0; 1394 1395 return 1; 1396 } 1397 1398 int 1399 start_port(portid_t pid) 1400 { 1401 int diag, need_check_link_status = -1; 1402 portid_t pi; 1403 queueid_t qi; 1404 struct rte_port *port; 1405 struct ether_addr mac_addr; 1406 enum rte_eth_event_type event_type; 1407 1408 if (port_id_is_invalid(pid, ENABLED_WARN)) 1409 return 0; 1410 1411 if(dcb_config) 1412 dcb_test = 1; 1413 RTE_ETH_FOREACH_DEV(pi) { 1414 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1415 continue; 1416 1417 need_check_link_status = 0; 1418 port = &ports[pi]; 1419 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1420 RTE_PORT_HANDLING) == 0) { 1421 printf("Port %d is now not stopped\n", pi); 1422 continue; 1423 } 1424 1425 if (port->need_reconfig > 0) { 1426 port->need_reconfig = 0; 1427 1428 printf("Configuring Port %d (socket %u)\n", pi, 1429 port->socket_id); 1430 /* configure port */ 1431 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1432 &(port->dev_conf)); 1433 if (diag != 0) { 1434 if (rte_atomic16_cmpset(&(port->port_status), 1435 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1436 printf("Port %d can not be set back " 1437 "to stopped\n", pi); 1438 printf("Fail to configure port %d\n", pi); 1439 /* try to reconfigure port next time */ 1440 port->need_reconfig = 1; 1441 return -1; 1442 } 1443 } 1444 if (port->need_reconfig_queues > 0) { 1445 port->need_reconfig_queues = 0; 1446 /* setup tx queues */ 1447 for (qi = 0; qi < nb_txq; qi++) { 1448 if ((numa_support) && 1449 (txring_numa[pi] != NUMA_NO_CONFIG)) 1450 diag = rte_eth_tx_queue_setup(pi, qi, 1451 nb_txd,txring_numa[pi], 1452 &(port->tx_conf)); 1453 else 1454 diag = rte_eth_tx_queue_setup(pi, qi, 1455 nb_txd,port->socket_id, 1456 &(port->tx_conf)); 1457 1458 if (diag == 0) 1459 continue; 1460 1461 /* Fail to setup tx queue, return */ 1462 if (rte_atomic16_cmpset(&(port->port_status), 1463 RTE_PORT_HANDLING, 1464 RTE_PORT_STOPPED) == 0) 1465 printf("Port %d can not be set back " 1466 "to stopped\n", pi); 1467 printf("Fail to configure port %d tx queues\n", pi); 1468 /* try to reconfigure queues next time */ 1469 port->need_reconfig_queues = 1; 1470 return -1; 1471 } 1472 /* setup rx queues */ 1473 for (qi = 0; qi < nb_rxq; qi++) { 1474 if ((numa_support) && 1475 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 1476 struct rte_mempool * mp = 1477 mbuf_pool_find(rxring_numa[pi]); 1478 if (mp == NULL) { 1479 printf("Failed to setup RX queue:" 1480 "No mempool allocation" 1481 " on the socket %d\n", 1482 rxring_numa[pi]); 1483 return -1; 1484 } 1485 1486 diag = rte_eth_rx_queue_setup(pi, qi, 1487 nb_rxd,rxring_numa[pi], 1488 &(port->rx_conf),mp); 1489 } else { 1490 struct rte_mempool *mp = 1491 mbuf_pool_find(port->socket_id); 1492 if (mp == NULL) { 1493 printf("Failed to setup RX queue:" 1494 "No mempool allocation" 1495 " on the socket %d\n", 1496 port->socket_id); 1497 return -1; 1498 } 1499 diag = rte_eth_rx_queue_setup(pi, qi, 1500 nb_rxd,port->socket_id, 1501 &(port->rx_conf), mp); 1502 } 1503 if (diag == 0) 1504 continue; 1505 1506 /* Fail to setup rx queue, return */ 1507 if (rte_atomic16_cmpset(&(port->port_status), 1508 RTE_PORT_HANDLING, 1509 RTE_PORT_STOPPED) == 0) 1510 printf("Port %d can not be set back " 1511 "to stopped\n", pi); 1512 printf("Fail to configure port %d rx queues\n", pi); 1513 /* try to reconfigure queues next time */ 1514 port->need_reconfig_queues = 1; 1515 return -1; 1516 } 1517 } 1518 1519 for (event_type = RTE_ETH_EVENT_UNKNOWN; 1520 event_type < RTE_ETH_EVENT_MAX; 1521 event_type++) { 1522 diag = rte_eth_dev_callback_register(pi, 1523 event_type, 1524 eth_event_callback, 1525 NULL); 1526 if (diag) { 1527 printf("Failed to setup even callback for event %d\n", 1528 event_type); 1529 return -1; 1530 } 1531 } 1532 1533 /* start port */ 1534 if (rte_eth_dev_start(pi) < 0) { 1535 printf("Fail to start port %d\n", pi); 1536 1537 /* Fail to setup rx queue, return */ 1538 if (rte_atomic16_cmpset(&(port->port_status), 1539 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1540 printf("Port %d can not be set back to " 1541 "stopped\n", pi); 1542 continue; 1543 } 1544 1545 if (rte_atomic16_cmpset(&(port->port_status), 1546 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 1547 printf("Port %d can not be set into started\n", pi); 1548 1549 rte_eth_macaddr_get(pi, &mac_addr); 1550 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 1551 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 1552 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 1553 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 1554 1555 /* at least one port started, need checking link status */ 1556 need_check_link_status = 1; 1557 } 1558 1559 if (need_check_link_status == 1 && !no_link_check) 1560 check_all_ports_link_status(RTE_PORT_ALL); 1561 else if (need_check_link_status == 0) 1562 printf("Please stop the ports first\n"); 1563 1564 printf("Done\n"); 1565 return 0; 1566 } 1567 1568 void 1569 stop_port(portid_t pid) 1570 { 1571 portid_t pi; 1572 struct rte_port *port; 1573 int need_check_link_status = 0; 1574 1575 if (dcb_test) { 1576 dcb_test = 0; 1577 dcb_config = 0; 1578 } 1579 1580 if (port_id_is_invalid(pid, ENABLED_WARN)) 1581 return; 1582 1583 printf("Stopping ports...\n"); 1584 1585 RTE_ETH_FOREACH_DEV(pi) { 1586 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1587 continue; 1588 1589 if (port_is_forwarding(pi) != 0 && test_done == 0) { 1590 printf("Please remove port %d from forwarding configuration.\n", pi); 1591 continue; 1592 } 1593 1594 if (port_is_bonding_slave(pi)) { 1595 printf("Please remove port %d from bonded device.\n", pi); 1596 continue; 1597 } 1598 1599 port = &ports[pi]; 1600 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 1601 RTE_PORT_HANDLING) == 0) 1602 continue; 1603 1604 rte_eth_dev_stop(pi); 1605 1606 if (rte_atomic16_cmpset(&(port->port_status), 1607 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1608 printf("Port %d can not be set into stopped\n", pi); 1609 need_check_link_status = 1; 1610 } 1611 if (need_check_link_status && !no_link_check) 1612 check_all_ports_link_status(RTE_PORT_ALL); 1613 1614 printf("Done\n"); 1615 } 1616 1617 void 1618 close_port(portid_t pid) 1619 { 1620 portid_t pi; 1621 struct rte_port *port; 1622 1623 if (port_id_is_invalid(pid, ENABLED_WARN)) 1624 return; 1625 1626 printf("Closing ports...\n"); 1627 1628 RTE_ETH_FOREACH_DEV(pi) { 1629 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1630 continue; 1631 1632 if (port_is_forwarding(pi) != 0 && test_done == 0) { 1633 printf("Please remove port %d from forwarding configuration.\n", pi); 1634 continue; 1635 } 1636 1637 if (port_is_bonding_slave(pi)) { 1638 printf("Please remove port %d from bonded device.\n", pi); 1639 continue; 1640 } 1641 1642 port = &ports[pi]; 1643 if (rte_atomic16_cmpset(&(port->port_status), 1644 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 1645 printf("Port %d is already closed\n", pi); 1646 continue; 1647 } 1648 1649 if (rte_atomic16_cmpset(&(port->port_status), 1650 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 1651 printf("Port %d is now not stopped\n", pi); 1652 continue; 1653 } 1654 1655 if (port->flow_list) 1656 port_flow_flush(pi); 1657 rte_eth_dev_close(pi); 1658 1659 if (rte_atomic16_cmpset(&(port->port_status), 1660 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 1661 printf("Port %d cannot be set to closed\n", pi); 1662 } 1663 1664 printf("Done\n"); 1665 } 1666 1667 void 1668 attach_port(char *identifier) 1669 { 1670 portid_t pi = 0; 1671 unsigned int socket_id; 1672 1673 printf("Attaching a new port...\n"); 1674 1675 if (identifier == NULL) { 1676 printf("Invalid parameters are specified\n"); 1677 return; 1678 } 1679 1680 if (rte_eth_dev_attach(identifier, &pi)) 1681 return; 1682 1683 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 1684 /* if socket_id is invalid, set to 0 */ 1685 if (check_socket_id(socket_id) < 0) 1686 socket_id = 0; 1687 reconfig(pi, socket_id); 1688 rte_eth_promiscuous_enable(pi); 1689 1690 nb_ports = rte_eth_dev_count(); 1691 1692 ports[pi].port_status = RTE_PORT_STOPPED; 1693 1694 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 1695 printf("Done\n"); 1696 } 1697 1698 void 1699 detach_port(uint8_t port_id) 1700 { 1701 char name[RTE_ETH_NAME_MAX_LEN]; 1702 1703 printf("Detaching a port...\n"); 1704 1705 if (!port_is_closed(port_id)) { 1706 printf("Please close port first\n"); 1707 return; 1708 } 1709 1710 if (ports[port_id].flow_list) 1711 port_flow_flush(port_id); 1712 1713 if (rte_eth_dev_detach(port_id, name)) 1714 return; 1715 1716 nb_ports = rte_eth_dev_count(); 1717 1718 printf("Port '%s' is detached. Now total ports is %d\n", 1719 name, nb_ports); 1720 printf("Done\n"); 1721 return; 1722 } 1723 1724 void 1725 pmd_test_exit(void) 1726 { 1727 portid_t pt_id; 1728 1729 if (test_done == 0) 1730 stop_packet_forwarding(); 1731 1732 if (ports != NULL) { 1733 no_link_check = 1; 1734 RTE_ETH_FOREACH_DEV(pt_id) { 1735 printf("\nShutting down port %d...\n", pt_id); 1736 fflush(stdout); 1737 stop_port(pt_id); 1738 close_port(pt_id); 1739 } 1740 } 1741 printf("\nBye...\n"); 1742 } 1743 1744 typedef void (*cmd_func_t)(void); 1745 struct pmd_test_command { 1746 const char *cmd_name; 1747 cmd_func_t cmd_func; 1748 }; 1749 1750 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 1751 1752 /* Check the link status of all ports in up to 9s, and print them finally */ 1753 static void 1754 check_all_ports_link_status(uint32_t port_mask) 1755 { 1756 #define CHECK_INTERVAL 100 /* 100ms */ 1757 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1758 uint8_t portid, count, all_ports_up, print_flag = 0; 1759 struct rte_eth_link link; 1760 1761 printf("Checking link statuses...\n"); 1762 fflush(stdout); 1763 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1764 all_ports_up = 1; 1765 RTE_ETH_FOREACH_DEV(portid) { 1766 if ((port_mask & (1 << portid)) == 0) 1767 continue; 1768 memset(&link, 0, sizeof(link)); 1769 rte_eth_link_get_nowait(portid, &link); 1770 /* print link status if flag set */ 1771 if (print_flag == 1) { 1772 if (link.link_status) 1773 printf("Port %d Link Up - speed %u " 1774 "Mbps - %s\n", (uint8_t)portid, 1775 (unsigned)link.link_speed, 1776 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1777 ("full-duplex") : ("half-duplex\n")); 1778 else 1779 printf("Port %d Link Down\n", 1780 (uint8_t)portid); 1781 continue; 1782 } 1783 /* clear all_ports_up flag if any link down */ 1784 if (link.link_status == ETH_LINK_DOWN) { 1785 all_ports_up = 0; 1786 break; 1787 } 1788 } 1789 /* after finally printing all link status, get out */ 1790 if (print_flag == 1) 1791 break; 1792 1793 if (all_ports_up == 0) { 1794 fflush(stdout); 1795 rte_delay_ms(CHECK_INTERVAL); 1796 } 1797 1798 /* set the print_flag if all ports up or timeout */ 1799 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1800 print_flag = 1; 1801 } 1802 1803 if (lsc_interrupt) 1804 break; 1805 } 1806 } 1807 1808 static void 1809 rmv_event_callback(void *arg) 1810 { 1811 struct rte_eth_dev *dev; 1812 struct rte_devargs *da; 1813 char name[32] = ""; 1814 uint8_t port_id = (intptr_t)arg; 1815 1816 RTE_ETH_VALID_PORTID_OR_RET(port_id); 1817 dev = &rte_eth_devices[port_id]; 1818 da = dev->device->devargs; 1819 1820 stop_port(port_id); 1821 close_port(port_id); 1822 if (da->type == RTE_DEVTYPE_VIRTUAL) 1823 snprintf(name, sizeof(name), "%s", da->virt.drv_name); 1824 else if (da->type == RTE_DEVTYPE_WHITELISTED_PCI) 1825 rte_pci_device_name(&da->pci.addr, name, sizeof(name)); 1826 printf("removing device %s\n", name); 1827 rte_eal_dev_detach(name); 1828 dev->state = RTE_ETH_DEV_UNUSED; 1829 } 1830 1831 /* This function is used by the interrupt thread */ 1832 static void 1833 eth_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param) 1834 { 1835 static const char * const event_desc[] = { 1836 [RTE_ETH_EVENT_UNKNOWN] = "Unknown", 1837 [RTE_ETH_EVENT_INTR_LSC] = "LSC", 1838 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state", 1839 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset", 1840 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox", 1841 [RTE_ETH_EVENT_MACSEC] = "MACsec", 1842 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 1843 [RTE_ETH_EVENT_MAX] = NULL, 1844 }; 1845 1846 RTE_SET_USED(param); 1847 1848 if (type >= RTE_ETH_EVENT_MAX) { 1849 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n", 1850 port_id, __func__, type); 1851 fflush(stderr); 1852 } else if (event_print_mask & (UINT32_C(1) << type)) { 1853 printf("\nPort %" PRIu8 ": %s event\n", port_id, 1854 event_desc[type]); 1855 fflush(stdout); 1856 } 1857 1858 switch (type) { 1859 case RTE_ETH_EVENT_INTR_RMV: 1860 if (rte_eal_alarm_set(100000, 1861 rmv_event_callback, (void *)(intptr_t)port_id)) 1862 fprintf(stderr, "Could not set up deferred device removal\n"); 1863 break; 1864 default: 1865 break; 1866 } 1867 } 1868 1869 static int 1870 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) 1871 { 1872 uint16_t i; 1873 int diag; 1874 uint8_t mapping_found = 0; 1875 1876 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 1877 if ((tx_queue_stats_mappings[i].port_id == port_id) && 1878 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 1879 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 1880 tx_queue_stats_mappings[i].queue_id, 1881 tx_queue_stats_mappings[i].stats_counter_id); 1882 if (diag != 0) 1883 return diag; 1884 mapping_found = 1; 1885 } 1886 } 1887 if (mapping_found) 1888 port->tx_queue_stats_mapping_enabled = 1; 1889 return 0; 1890 } 1891 1892 static int 1893 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) 1894 { 1895 uint16_t i; 1896 int diag; 1897 uint8_t mapping_found = 0; 1898 1899 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 1900 if ((rx_queue_stats_mappings[i].port_id == port_id) && 1901 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 1902 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 1903 rx_queue_stats_mappings[i].queue_id, 1904 rx_queue_stats_mappings[i].stats_counter_id); 1905 if (diag != 0) 1906 return diag; 1907 mapping_found = 1; 1908 } 1909 } 1910 if (mapping_found) 1911 port->rx_queue_stats_mapping_enabled = 1; 1912 return 0; 1913 } 1914 1915 static void 1916 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port) 1917 { 1918 int diag = 0; 1919 1920 diag = set_tx_queue_stats_mapping_registers(pi, port); 1921 if (diag != 0) { 1922 if (diag == -ENOTSUP) { 1923 port->tx_queue_stats_mapping_enabled = 0; 1924 printf("TX queue stats mapping not supported port id=%d\n", pi); 1925 } 1926 else 1927 rte_exit(EXIT_FAILURE, 1928 "set_tx_queue_stats_mapping_registers " 1929 "failed for port id=%d diag=%d\n", 1930 pi, diag); 1931 } 1932 1933 diag = set_rx_queue_stats_mapping_registers(pi, port); 1934 if (diag != 0) { 1935 if (diag == -ENOTSUP) { 1936 port->rx_queue_stats_mapping_enabled = 0; 1937 printf("RX queue stats mapping not supported port id=%d\n", pi); 1938 } 1939 else 1940 rte_exit(EXIT_FAILURE, 1941 "set_rx_queue_stats_mapping_registers " 1942 "failed for port id=%d diag=%d\n", 1943 pi, diag); 1944 } 1945 } 1946 1947 static void 1948 rxtx_port_config(struct rte_port *port) 1949 { 1950 port->rx_conf = port->dev_info.default_rxconf; 1951 port->tx_conf = port->dev_info.default_txconf; 1952 1953 /* Check if any RX/TX parameters have been passed */ 1954 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 1955 port->rx_conf.rx_thresh.pthresh = rx_pthresh; 1956 1957 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 1958 port->rx_conf.rx_thresh.hthresh = rx_hthresh; 1959 1960 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 1961 port->rx_conf.rx_thresh.wthresh = rx_wthresh; 1962 1963 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 1964 port->rx_conf.rx_free_thresh = rx_free_thresh; 1965 1966 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 1967 port->rx_conf.rx_drop_en = rx_drop_en; 1968 1969 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 1970 port->tx_conf.tx_thresh.pthresh = tx_pthresh; 1971 1972 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 1973 port->tx_conf.tx_thresh.hthresh = tx_hthresh; 1974 1975 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 1976 port->tx_conf.tx_thresh.wthresh = tx_wthresh; 1977 1978 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 1979 port->tx_conf.tx_rs_thresh = tx_rs_thresh; 1980 1981 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 1982 port->tx_conf.tx_free_thresh = tx_free_thresh; 1983 1984 if (txq_flags != RTE_PMD_PARAM_UNSET) 1985 port->tx_conf.txq_flags = txq_flags; 1986 } 1987 1988 void 1989 init_port_config(void) 1990 { 1991 portid_t pid; 1992 struct rte_port *port; 1993 1994 RTE_ETH_FOREACH_DEV(pid) { 1995 port = &ports[pid]; 1996 port->dev_conf.rxmode = rx_mode; 1997 port->dev_conf.fdir_conf = fdir_conf; 1998 if (nb_rxq > 1) { 1999 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2000 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf; 2001 } else { 2002 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2003 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2004 } 2005 2006 if (port->dcb_flag == 0) { 2007 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2008 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2009 else 2010 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2011 } 2012 2013 rxtx_port_config(port); 2014 2015 rte_eth_macaddr_get(pid, &port->eth_addr); 2016 2017 map_port_queue_stats_mapping_registers(pid, port); 2018 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2019 rte_pmd_ixgbe_bypass_init(pid); 2020 #endif 2021 2022 if (lsc_interrupt && 2023 (rte_eth_devices[pid].data->dev_flags & 2024 RTE_ETH_DEV_INTR_LSC)) 2025 port->dev_conf.intr_conf.lsc = 1; 2026 if (rmv_interrupt && 2027 (rte_eth_devices[pid].data->dev_flags & 2028 RTE_ETH_DEV_INTR_RMV)) 2029 port->dev_conf.intr_conf.rmv = 1; 2030 } 2031 } 2032 2033 void set_port_slave_flag(portid_t slave_pid) 2034 { 2035 struct rte_port *port; 2036 2037 port = &ports[slave_pid]; 2038 port->slave_flag = 1; 2039 } 2040 2041 void clear_port_slave_flag(portid_t slave_pid) 2042 { 2043 struct rte_port *port; 2044 2045 port = &ports[slave_pid]; 2046 port->slave_flag = 0; 2047 } 2048 2049 uint8_t port_is_bonding_slave(portid_t slave_pid) 2050 { 2051 struct rte_port *port; 2052 2053 port = &ports[slave_pid]; 2054 return port->slave_flag; 2055 } 2056 2057 const uint16_t vlan_tags[] = { 2058 0, 1, 2, 3, 4, 5, 6, 7, 2059 8, 9, 10, 11, 12, 13, 14, 15, 2060 16, 17, 18, 19, 20, 21, 22, 23, 2061 24, 25, 26, 27, 28, 29, 30, 31 2062 }; 2063 2064 static int 2065 get_eth_dcb_conf(struct rte_eth_conf *eth_conf, 2066 enum dcb_mode_enable dcb_mode, 2067 enum rte_eth_nb_tcs num_tcs, 2068 uint8_t pfc_en) 2069 { 2070 uint8_t i; 2071 2072 /* 2073 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2074 * given above, and the number of traffic classes available for use. 2075 */ 2076 if (dcb_mode == DCB_VT_ENABLED) { 2077 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2078 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2079 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2080 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2081 2082 /* VMDQ+DCB RX and TX configurations */ 2083 vmdq_rx_conf->enable_default_pool = 0; 2084 vmdq_rx_conf->default_pool = 0; 2085 vmdq_rx_conf->nb_queue_pools = 2086 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2087 vmdq_tx_conf->nb_queue_pools = 2088 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2089 2090 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2091 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2092 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2093 vmdq_rx_conf->pool_map[i].pools = 2094 1 << (i % vmdq_rx_conf->nb_queue_pools); 2095 } 2096 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2097 vmdq_rx_conf->dcb_tc[i] = i; 2098 vmdq_tx_conf->dcb_tc[i] = i; 2099 } 2100 2101 /* set DCB mode of RX and TX of multiple queues */ 2102 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2103 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2104 } else { 2105 struct rte_eth_dcb_rx_conf *rx_conf = 2106 ð_conf->rx_adv_conf.dcb_rx_conf; 2107 struct rte_eth_dcb_tx_conf *tx_conf = 2108 ð_conf->tx_adv_conf.dcb_tx_conf; 2109 2110 rx_conf->nb_tcs = num_tcs; 2111 tx_conf->nb_tcs = num_tcs; 2112 2113 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2114 rx_conf->dcb_tc[i] = i % num_tcs; 2115 tx_conf->dcb_tc[i] = i % num_tcs; 2116 } 2117 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2118 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf; 2119 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2120 } 2121 2122 if (pfc_en) 2123 eth_conf->dcb_capability_en = 2124 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2125 else 2126 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2127 2128 return 0; 2129 } 2130 2131 int 2132 init_port_dcb_config(portid_t pid, 2133 enum dcb_mode_enable dcb_mode, 2134 enum rte_eth_nb_tcs num_tcs, 2135 uint8_t pfc_en) 2136 { 2137 struct rte_eth_conf port_conf; 2138 struct rte_port *rte_port; 2139 int retval; 2140 uint16_t i; 2141 2142 rte_port = &ports[pid]; 2143 2144 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2145 /* Enter DCB configuration status */ 2146 dcb_config = 1; 2147 2148 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2149 retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en); 2150 if (retval < 0) 2151 return retval; 2152 port_conf.rxmode.hw_vlan_filter = 1; 2153 2154 /** 2155 * Write the configuration into the device. 2156 * Set the numbers of RX & TX queues to 0, so 2157 * the RX & TX queues will not be setup. 2158 */ 2159 (void)rte_eth_dev_configure(pid, 0, 0, &port_conf); 2160 2161 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2162 2163 /* If dev_info.vmdq_pool_base is greater than 0, 2164 * the queue id of vmdq pools is started after pf queues. 2165 */ 2166 if (dcb_mode == DCB_VT_ENABLED && 2167 rte_port->dev_info.vmdq_pool_base > 0) { 2168 printf("VMDQ_DCB multi-queue mode is nonsensical" 2169 " for port %d.", pid); 2170 return -1; 2171 } 2172 2173 /* Assume the ports in testpmd have the same dcb capability 2174 * and has the same number of rxq and txq in dcb mode 2175 */ 2176 if (dcb_mode == DCB_VT_ENABLED) { 2177 if (rte_port->dev_info.max_vfs > 0) { 2178 nb_rxq = rte_port->dev_info.nb_rx_queues; 2179 nb_txq = rte_port->dev_info.nb_tx_queues; 2180 } else { 2181 nb_rxq = rte_port->dev_info.max_rx_queues; 2182 nb_txq = rte_port->dev_info.max_tx_queues; 2183 } 2184 } else { 2185 /*if vt is disabled, use all pf queues */ 2186 if (rte_port->dev_info.vmdq_pool_base == 0) { 2187 nb_rxq = rte_port->dev_info.max_rx_queues; 2188 nb_txq = rte_port->dev_info.max_tx_queues; 2189 } else { 2190 nb_rxq = (queueid_t)num_tcs; 2191 nb_txq = (queueid_t)num_tcs; 2192 2193 } 2194 } 2195 rx_free_thresh = 64; 2196 2197 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 2198 2199 rxtx_port_config(rte_port); 2200 /* VLAN filter */ 2201 rte_port->dev_conf.rxmode.hw_vlan_filter = 1; 2202 for (i = 0; i < RTE_DIM(vlan_tags); i++) 2203 rx_vft_set(pid, vlan_tags[i], 1); 2204 2205 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 2206 map_port_queue_stats_mapping_registers(pid, rte_port); 2207 2208 rte_port->dcb_flag = 1; 2209 2210 return 0; 2211 } 2212 2213 static void 2214 init_port(void) 2215 { 2216 /* Configuration of Ethernet ports. */ 2217 ports = rte_zmalloc("testpmd: ports", 2218 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 2219 RTE_CACHE_LINE_SIZE); 2220 if (ports == NULL) { 2221 rte_exit(EXIT_FAILURE, 2222 "rte_zmalloc(%d struct rte_port) failed\n", 2223 RTE_MAX_ETHPORTS); 2224 } 2225 } 2226 2227 static void 2228 force_quit(void) 2229 { 2230 pmd_test_exit(); 2231 prompt_exit(); 2232 } 2233 2234 static void 2235 signal_handler(int signum) 2236 { 2237 if (signum == SIGINT || signum == SIGTERM) { 2238 printf("\nSignal %d received, preparing to exit...\n", 2239 signum); 2240 #ifdef RTE_LIBRTE_PDUMP 2241 /* uninitialize packet capture framework */ 2242 rte_pdump_uninit(); 2243 #endif 2244 #ifdef RTE_LIBRTE_LATENCY_STATS 2245 rte_latencystats_uninit(); 2246 #endif 2247 force_quit(); 2248 /* exit with the expected status */ 2249 signal(signum, SIG_DFL); 2250 kill(getpid(), signum); 2251 } 2252 } 2253 2254 int 2255 main(int argc, char** argv) 2256 { 2257 int diag; 2258 uint8_t port_id; 2259 2260 signal(SIGINT, signal_handler); 2261 signal(SIGTERM, signal_handler); 2262 2263 diag = rte_eal_init(argc, argv); 2264 if (diag < 0) 2265 rte_panic("Cannot init EAL\n"); 2266 2267 #ifdef RTE_LIBRTE_PDUMP 2268 /* initialize packet capture framework */ 2269 rte_pdump_init(NULL); 2270 #endif 2271 2272 nb_ports = (portid_t) rte_eth_dev_count(); 2273 if (nb_ports == 0) 2274 RTE_LOG(WARNING, EAL, "No probed ethernet devices\n"); 2275 2276 /* allocate port structures, and init them */ 2277 init_port(); 2278 2279 set_def_fwd_config(); 2280 if (nb_lcores == 0) 2281 rte_panic("Empty set of forwarding logical cores - check the " 2282 "core mask supplied in the command parameters\n"); 2283 2284 /* Bitrate/latency stats disabled by default */ 2285 #ifdef RTE_LIBRTE_BITRATE 2286 bitrate_enabled = 0; 2287 #endif 2288 #ifdef RTE_LIBRTE_LATENCY_STATS 2289 latencystats_enabled = 0; 2290 #endif 2291 2292 argc -= diag; 2293 argv += diag; 2294 if (argc > 1) 2295 launch_args_parse(argc, argv); 2296 2297 if (!nb_rxq && !nb_txq) 2298 printf("Warning: Either rx or tx queues should be non-zero\n"); 2299 2300 if (nb_rxq > 1 && nb_rxq > nb_txq) 2301 printf("Warning: nb_rxq=%d enables RSS configuration, " 2302 "but nb_txq=%d will prevent to fully test it.\n", 2303 nb_rxq, nb_txq); 2304 2305 init_config(); 2306 if (start_port(RTE_PORT_ALL) != 0) 2307 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 2308 2309 /* set all ports to promiscuous mode by default */ 2310 RTE_ETH_FOREACH_DEV(port_id) 2311 rte_eth_promiscuous_enable(port_id); 2312 2313 /* Init metrics library */ 2314 rte_metrics_init(rte_socket_id()); 2315 2316 #ifdef RTE_LIBRTE_LATENCY_STATS 2317 if (latencystats_enabled != 0) { 2318 int ret = rte_latencystats_init(1, NULL); 2319 if (ret) 2320 printf("Warning: latencystats init()" 2321 " returned error %d\n", ret); 2322 printf("Latencystats running on lcore %d\n", 2323 latencystats_lcore_id); 2324 } 2325 #endif 2326 2327 /* Setup bitrate stats */ 2328 #ifdef RTE_LIBRTE_BITRATE 2329 if (bitrate_enabled != 0) { 2330 bitrate_data = rte_stats_bitrate_create(); 2331 if (bitrate_data == NULL) 2332 rte_exit(EXIT_FAILURE, 2333 "Could not allocate bitrate data.\n"); 2334 rte_stats_bitrate_reg(bitrate_data); 2335 } 2336 #endif 2337 2338 #ifdef RTE_LIBRTE_CMDLINE 2339 if (strlen(cmdline_filename) != 0) 2340 cmdline_read_from_file(cmdline_filename); 2341 2342 if (interactive == 1) { 2343 if (auto_start) { 2344 printf("Start automatic packet forwarding\n"); 2345 start_packet_forwarding(0); 2346 } 2347 prompt(); 2348 pmd_test_exit(); 2349 } else 2350 #endif 2351 { 2352 char c; 2353 int rc; 2354 2355 printf("No commandline core given, start packet forwarding\n"); 2356 start_packet_forwarding(0); 2357 printf("Press enter to exit\n"); 2358 rc = read(0, &c, 1); 2359 pmd_test_exit(); 2360 if (rc < 0) 2361 return 1; 2362 } 2363 2364 return 0; 2365 } 2366