1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdarg.h> 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <signal.h> 38 #include <string.h> 39 #include <time.h> 40 #include <fcntl.h> 41 #include <sys/types.h> 42 #include <errno.h> 43 44 #include <sys/queue.h> 45 #include <sys/stat.h> 46 47 #include <stdint.h> 48 #include <unistd.h> 49 #include <inttypes.h> 50 51 #include <rte_common.h> 52 #include <rte_errno.h> 53 #include <rte_byteorder.h> 54 #include <rte_log.h> 55 #include <rte_debug.h> 56 #include <rte_cycles.h> 57 #include <rte_memory.h> 58 #include <rte_memcpy.h> 59 #include <rte_memzone.h> 60 #include <rte_launch.h> 61 #include <rte_eal.h> 62 #include <rte_alarm.h> 63 #include <rte_per_lcore.h> 64 #include <rte_lcore.h> 65 #include <rte_atomic.h> 66 #include <rte_branch_prediction.h> 67 #include <rte_mempool.h> 68 #include <rte_malloc.h> 69 #include <rte_mbuf.h> 70 #include <rte_interrupts.h> 71 #include <rte_pci.h> 72 #include <rte_ether.h> 73 #include <rte_ethdev.h> 74 #include <rte_dev.h> 75 #include <rte_string_fns.h> 76 #ifdef RTE_LIBRTE_PMD_XENVIRT 77 #include <rte_eth_xenvirt.h> 78 #endif 79 #ifdef RTE_LIBRTE_PDUMP 80 #include <rte_pdump.h> 81 #endif 82 #include <rte_flow.h> 83 #include <rte_metrics.h> 84 #ifdef RTE_LIBRTE_BITRATE 85 #include <rte_bitrate.h> 86 #endif 87 #ifdef RTE_LIBRTE_LATENCY_STATS 88 #include <rte_latencystats.h> 89 #endif 90 91 #include "testpmd.h" 92 93 uint16_t verbose_level = 0; /**< Silent by default. */ 94 95 /* use master core for command line ? */ 96 uint8_t interactive = 0; 97 uint8_t auto_start = 0; 98 char cmdline_filename[PATH_MAX] = {0}; 99 100 /* 101 * NUMA support configuration. 102 * When set, the NUMA support attempts to dispatch the allocation of the 103 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 104 * probed ports among the CPU sockets 0 and 1. 105 * Otherwise, all memory is allocated from CPU socket 0. 106 */ 107 uint8_t numa_support = 1; /**< numa enabled by default */ 108 109 /* 110 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 111 * not configured. 112 */ 113 uint8_t socket_num = UMA_NO_CONFIG; 114 115 /* 116 * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs. 117 */ 118 uint8_t mp_anon = 0; 119 120 /* 121 * Record the Ethernet address of peer target ports to which packets are 122 * forwarded. 123 * Must be instantiated with the ethernet addresses of peer traffic generator 124 * ports. 125 */ 126 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 127 portid_t nb_peer_eth_addrs = 0; 128 129 /* 130 * Probed Target Environment. 131 */ 132 struct rte_port *ports; /**< For all probed ethernet ports. */ 133 portid_t nb_ports; /**< Number of probed ethernet ports. */ 134 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 135 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 136 137 /* 138 * Test Forwarding Configuration. 139 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 140 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 141 */ 142 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 143 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 144 portid_t nb_cfg_ports; /**< Number of configured ports. */ 145 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 146 147 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 148 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 149 150 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 151 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 152 153 /* 154 * Forwarding engines. 155 */ 156 struct fwd_engine * fwd_engines[] = { 157 &io_fwd_engine, 158 &mac_fwd_engine, 159 &mac_swap_engine, 160 &flow_gen_engine, 161 &rx_only_engine, 162 &tx_only_engine, 163 &csum_fwd_engine, 164 &icmp_echo_engine, 165 #ifdef RTE_LIBRTE_IEEE1588 166 &ieee1588_fwd_engine, 167 #endif 168 NULL, 169 }; 170 171 struct fwd_config cur_fwd_config; 172 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 173 uint32_t retry_enabled; 174 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 175 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 176 177 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 178 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 179 * specified on command-line. */ 180 181 /* 182 * Configuration of packet segments used by the "txonly" processing engine. 183 */ 184 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 185 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 186 TXONLY_DEF_PACKET_LEN, 187 }; 188 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 189 190 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 191 /**< Split policy for packets to TX. */ 192 193 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 194 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 195 196 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 197 uint8_t dcb_config = 0; 198 199 /* Whether the dcb is in testing status */ 200 uint8_t dcb_test = 0; 201 202 /* 203 * Configurable number of RX/TX queues. 204 */ 205 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 206 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 207 208 /* 209 * Configurable number of RX/TX ring descriptors. 210 */ 211 #define RTE_TEST_RX_DESC_DEFAULT 128 212 #define RTE_TEST_TX_DESC_DEFAULT 512 213 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 214 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 215 216 #define RTE_PMD_PARAM_UNSET -1 217 /* 218 * Configurable values of RX and TX ring threshold registers. 219 */ 220 221 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 222 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 223 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 224 225 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 226 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 227 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 228 229 /* 230 * Configurable value of RX free threshold. 231 */ 232 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 233 234 /* 235 * Configurable value of RX drop enable. 236 */ 237 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 238 239 /* 240 * Configurable value of TX free threshold. 241 */ 242 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 243 244 /* 245 * Configurable value of TX RS bit threshold. 246 */ 247 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 248 249 /* 250 * Configurable value of TX queue flags. 251 */ 252 int32_t txq_flags = RTE_PMD_PARAM_UNSET; 253 254 /* 255 * Receive Side Scaling (RSS) configuration. 256 */ 257 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 258 259 /* 260 * Port topology configuration 261 */ 262 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 263 264 /* 265 * Avoids to flush all the RX streams before starts forwarding. 266 */ 267 uint8_t no_flush_rx = 0; /* flush by default */ 268 269 /* 270 * Avoids to check link status when starting/stopping a port. 271 */ 272 uint8_t no_link_check = 0; /* check by default */ 273 274 /* 275 * Enable link status change notification 276 */ 277 uint8_t lsc_interrupt = 1; /* enabled by default */ 278 279 /* 280 * Enable device removal notification. 281 */ 282 uint8_t rmv_interrupt = 1; /* enabled by default */ 283 284 /* 285 * NIC bypass mode configuration options. 286 */ 287 #ifdef RTE_NIC_BYPASS 288 289 /* The NIC bypass watchdog timeout. */ 290 uint32_t bypass_timeout = RTE_BYPASS_TMT_OFF; 291 292 #endif 293 294 #ifdef RTE_LIBRTE_LATENCY_STATS 295 296 /* 297 * Set when latency stats is enabled in the commandline 298 */ 299 uint8_t latencystats_enabled; 300 301 /* 302 * Lcore ID to serive latency statistics. 303 */ 304 lcoreid_t latencystats_lcore_id = -1; 305 306 #endif 307 308 /* 309 * Ethernet device configuration. 310 */ 311 struct rte_eth_rxmode rx_mode = { 312 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 313 .split_hdr_size = 0, 314 .header_split = 0, /**< Header Split disabled. */ 315 .hw_ip_checksum = 0, /**< IP checksum offload disabled. */ 316 .hw_vlan_filter = 1, /**< VLAN filtering enabled. */ 317 .hw_vlan_strip = 1, /**< VLAN strip enabled. */ 318 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 319 .jumbo_frame = 0, /**< Jumbo Frame Support disabled. */ 320 .hw_strip_crc = 1, /**< CRC stripping by hardware enabled. */ 321 }; 322 323 struct rte_fdir_conf fdir_conf = { 324 .mode = RTE_FDIR_MODE_NONE, 325 .pballoc = RTE_FDIR_PBALLOC_64K, 326 .status = RTE_FDIR_REPORT_STATUS, 327 .mask = { 328 .vlan_tci_mask = 0x0, 329 .ipv4_mask = { 330 .src_ip = 0xFFFFFFFF, 331 .dst_ip = 0xFFFFFFFF, 332 }, 333 .ipv6_mask = { 334 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 335 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 336 }, 337 .src_port_mask = 0xFFFF, 338 .dst_port_mask = 0xFFFF, 339 .mac_addr_byte_mask = 0xFF, 340 .tunnel_type_mask = 1, 341 .tunnel_id_mask = 0xFFFFFFFF, 342 }, 343 .drop_queue = 127, 344 }; 345 346 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 347 348 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 349 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 350 351 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 352 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 353 354 uint16_t nb_tx_queue_stats_mappings = 0; 355 uint16_t nb_rx_queue_stats_mappings = 0; 356 357 unsigned max_socket = 0; 358 359 #ifdef RTE_LIBRTE_BITRATE 360 /* Bitrate statistics */ 361 struct rte_stats_bitrates *bitrate_data; 362 lcoreid_t bitrate_lcore_id; 363 uint8_t bitrate_enabled; 364 #endif 365 366 /* Forward function declarations */ 367 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port); 368 static void check_all_ports_link_status(uint32_t port_mask); 369 static void eth_event_callback(uint8_t port_id, 370 enum rte_eth_event_type type, 371 void *param); 372 373 /* 374 * Check if all the ports are started. 375 * If yes, return positive value. If not, return zero. 376 */ 377 static int all_ports_started(void); 378 379 /* 380 * Setup default configuration. 381 */ 382 static void 383 set_default_fwd_lcores_config(void) 384 { 385 unsigned int i; 386 unsigned int nb_lc; 387 unsigned int sock_num; 388 389 nb_lc = 0; 390 for (i = 0; i < RTE_MAX_LCORE; i++) { 391 sock_num = rte_lcore_to_socket_id(i) + 1; 392 if (sock_num > max_socket) { 393 if (sock_num > RTE_MAX_NUMA_NODES) 394 rte_exit(EXIT_FAILURE, "Total sockets greater than %u\n", RTE_MAX_NUMA_NODES); 395 max_socket = sock_num; 396 } 397 if (!rte_lcore_is_enabled(i)) 398 continue; 399 if (i == rte_get_master_lcore()) 400 continue; 401 fwd_lcores_cpuids[nb_lc++] = i; 402 } 403 nb_lcores = (lcoreid_t) nb_lc; 404 nb_cfg_lcores = nb_lcores; 405 nb_fwd_lcores = 1; 406 } 407 408 static void 409 set_def_peer_eth_addrs(void) 410 { 411 portid_t i; 412 413 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 414 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 415 peer_eth_addrs[i].addr_bytes[5] = i; 416 } 417 } 418 419 static void 420 set_default_fwd_ports_config(void) 421 { 422 portid_t pt_id; 423 424 for (pt_id = 0; pt_id < nb_ports; pt_id++) 425 fwd_ports_ids[pt_id] = pt_id; 426 427 nb_cfg_ports = nb_ports; 428 nb_fwd_ports = nb_ports; 429 } 430 431 void 432 set_def_fwd_config(void) 433 { 434 set_default_fwd_lcores_config(); 435 set_def_peer_eth_addrs(); 436 set_default_fwd_ports_config(); 437 } 438 439 /* 440 * Configuration initialisation done once at init time. 441 */ 442 static void 443 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 444 unsigned int socket_id) 445 { 446 char pool_name[RTE_MEMPOOL_NAMESIZE]; 447 struct rte_mempool *rte_mp = NULL; 448 uint32_t mb_size; 449 450 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 451 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 452 453 RTE_LOG(INFO, USER1, 454 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 455 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 456 457 #ifdef RTE_LIBRTE_PMD_XENVIRT 458 rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size, 459 (unsigned) mb_mempool_cache, 460 sizeof(struct rte_pktmbuf_pool_private), 461 rte_pktmbuf_pool_init, NULL, 462 rte_pktmbuf_init, NULL, 463 socket_id, 0); 464 #endif 465 466 /* if the former XEN allocation failed fall back to normal allocation */ 467 if (rte_mp == NULL) { 468 if (mp_anon != 0) { 469 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 470 mb_size, (unsigned) mb_mempool_cache, 471 sizeof(struct rte_pktmbuf_pool_private), 472 socket_id, 0); 473 if (rte_mp == NULL) 474 goto err; 475 476 if (rte_mempool_populate_anon(rte_mp) == 0) { 477 rte_mempool_free(rte_mp); 478 rte_mp = NULL; 479 goto err; 480 } 481 rte_pktmbuf_pool_init(rte_mp, NULL); 482 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 483 } else { 484 /* wrapper to rte_mempool_create() */ 485 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 486 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 487 } 488 } 489 490 err: 491 if (rte_mp == NULL) { 492 rte_exit(EXIT_FAILURE, 493 "Creation of mbuf pool for socket %u failed: %s\n", 494 socket_id, rte_strerror(rte_errno)); 495 } else if (verbose_level > 0) { 496 rte_mempool_dump(stdout, rte_mp); 497 } 498 } 499 500 /* 501 * Check given socket id is valid or not with NUMA mode, 502 * if valid, return 0, else return -1 503 */ 504 static int 505 check_socket_id(const unsigned int socket_id) 506 { 507 static int warning_once = 0; 508 509 if (socket_id >= max_socket) { 510 if (!warning_once && numa_support) 511 printf("Warning: NUMA should be configured manually by" 512 " using --port-numa-config and" 513 " --ring-numa-config parameters along with" 514 " --numa.\n"); 515 warning_once = 1; 516 return -1; 517 } 518 return 0; 519 } 520 521 static void 522 init_config(void) 523 { 524 portid_t pid; 525 struct rte_port *port; 526 struct rte_mempool *mbp; 527 unsigned int nb_mbuf_per_pool; 528 lcoreid_t lc_id; 529 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 530 531 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 532 533 if (numa_support) { 534 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 535 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 536 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 537 } 538 539 /* Configuration of logical cores. */ 540 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 541 sizeof(struct fwd_lcore *) * nb_lcores, 542 RTE_CACHE_LINE_SIZE); 543 if (fwd_lcores == NULL) { 544 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 545 "failed\n", nb_lcores); 546 } 547 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 548 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 549 sizeof(struct fwd_lcore), 550 RTE_CACHE_LINE_SIZE); 551 if (fwd_lcores[lc_id] == NULL) { 552 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 553 "failed\n"); 554 } 555 fwd_lcores[lc_id]->cpuid_idx = lc_id; 556 } 557 558 RTE_ETH_FOREACH_DEV(pid) { 559 port = &ports[pid]; 560 rte_eth_dev_info_get(pid, &port->dev_info); 561 562 if (numa_support) { 563 if (port_numa[pid] != NUMA_NO_CONFIG) 564 port_per_socket[port_numa[pid]]++; 565 else { 566 uint32_t socket_id = rte_eth_dev_socket_id(pid); 567 568 /* if socket_id is invalid, set to 0 */ 569 if (check_socket_id(socket_id) < 0) 570 socket_id = 0; 571 port_per_socket[socket_id]++; 572 } 573 } 574 575 /* set flag to initialize port/queue */ 576 port->need_reconfig = 1; 577 port->need_reconfig_queues = 1; 578 } 579 580 /* 581 * Create pools of mbuf. 582 * If NUMA support is disabled, create a single pool of mbuf in 583 * socket 0 memory by default. 584 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 585 * 586 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 587 * nb_txd can be configured at run time. 588 */ 589 if (param_total_num_mbufs) 590 nb_mbuf_per_pool = param_total_num_mbufs; 591 else { 592 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 593 (nb_lcores * mb_mempool_cache) + 594 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 595 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 596 } 597 598 if (numa_support) { 599 uint8_t i; 600 601 for (i = 0; i < max_socket; i++) 602 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, i); 603 } else { 604 if (socket_num == UMA_NO_CONFIG) 605 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 606 else 607 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 608 socket_num); 609 } 610 611 init_port_config(); 612 613 /* 614 * Records which Mbuf pool to use by each logical core, if needed. 615 */ 616 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 617 mbp = mbuf_pool_find( 618 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 619 620 if (mbp == NULL) 621 mbp = mbuf_pool_find(0); 622 fwd_lcores[lc_id]->mbp = mbp; 623 } 624 625 /* Configuration of packet forwarding streams. */ 626 if (init_fwd_streams() < 0) 627 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 628 629 fwd_config_setup(); 630 } 631 632 633 void 634 reconfig(portid_t new_port_id, unsigned socket_id) 635 { 636 struct rte_port *port; 637 638 /* Reconfiguration of Ethernet ports. */ 639 port = &ports[new_port_id]; 640 rte_eth_dev_info_get(new_port_id, &port->dev_info); 641 642 /* set flag to initialize port/queue */ 643 port->need_reconfig = 1; 644 port->need_reconfig_queues = 1; 645 port->socket_id = socket_id; 646 647 init_port_config(); 648 } 649 650 651 int 652 init_fwd_streams(void) 653 { 654 portid_t pid; 655 struct rte_port *port; 656 streamid_t sm_id, nb_fwd_streams_new; 657 queueid_t q; 658 659 /* set socket id according to numa or not */ 660 RTE_ETH_FOREACH_DEV(pid) { 661 port = &ports[pid]; 662 if (nb_rxq > port->dev_info.max_rx_queues) { 663 printf("Fail: nb_rxq(%d) is greater than " 664 "max_rx_queues(%d)\n", nb_rxq, 665 port->dev_info.max_rx_queues); 666 return -1; 667 } 668 if (nb_txq > port->dev_info.max_tx_queues) { 669 printf("Fail: nb_txq(%d) is greater than " 670 "max_tx_queues(%d)\n", nb_txq, 671 port->dev_info.max_tx_queues); 672 return -1; 673 } 674 if (numa_support) { 675 if (port_numa[pid] != NUMA_NO_CONFIG) 676 port->socket_id = port_numa[pid]; 677 else { 678 port->socket_id = rte_eth_dev_socket_id(pid); 679 680 /* if socket_id is invalid, set to 0 */ 681 if (check_socket_id(port->socket_id) < 0) 682 port->socket_id = 0; 683 } 684 } 685 else { 686 if (socket_num == UMA_NO_CONFIG) 687 port->socket_id = 0; 688 else 689 port->socket_id = socket_num; 690 } 691 } 692 693 q = RTE_MAX(nb_rxq, nb_txq); 694 if (q == 0) { 695 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 696 return -1; 697 } 698 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 699 if (nb_fwd_streams_new == nb_fwd_streams) 700 return 0; 701 /* clear the old */ 702 if (fwd_streams != NULL) { 703 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 704 if (fwd_streams[sm_id] == NULL) 705 continue; 706 rte_free(fwd_streams[sm_id]); 707 fwd_streams[sm_id] = NULL; 708 } 709 rte_free(fwd_streams); 710 fwd_streams = NULL; 711 } 712 713 /* init new */ 714 nb_fwd_streams = nb_fwd_streams_new; 715 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 716 sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE); 717 if (fwd_streams == NULL) 718 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) " 719 "failed\n", nb_fwd_streams); 720 721 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 722 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream", 723 sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE); 724 if (fwd_streams[sm_id] == NULL) 725 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)" 726 " failed\n"); 727 } 728 729 return 0; 730 } 731 732 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 733 static void 734 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 735 { 736 unsigned int total_burst; 737 unsigned int nb_burst; 738 unsigned int burst_stats[3]; 739 uint16_t pktnb_stats[3]; 740 uint16_t nb_pkt; 741 int burst_percent[3]; 742 743 /* 744 * First compute the total number of packet bursts and the 745 * two highest numbers of bursts of the same number of packets. 746 */ 747 total_burst = 0; 748 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 749 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 750 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 751 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 752 if (nb_burst == 0) 753 continue; 754 total_burst += nb_burst; 755 if (nb_burst > burst_stats[0]) { 756 burst_stats[1] = burst_stats[0]; 757 pktnb_stats[1] = pktnb_stats[0]; 758 burst_stats[0] = nb_burst; 759 pktnb_stats[0] = nb_pkt; 760 } 761 } 762 if (total_burst == 0) 763 return; 764 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 765 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 766 burst_percent[0], (int) pktnb_stats[0]); 767 if (burst_stats[0] == total_burst) { 768 printf("]\n"); 769 return; 770 } 771 if (burst_stats[0] + burst_stats[1] == total_burst) { 772 printf(" + %d%% of %d pkts]\n", 773 100 - burst_percent[0], pktnb_stats[1]); 774 return; 775 } 776 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 777 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 778 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 779 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 780 return; 781 } 782 printf(" + %d%% of %d pkts + %d%% of others]\n", 783 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 784 } 785 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 786 787 static void 788 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats) 789 { 790 struct rte_port *port; 791 uint8_t i; 792 793 static const char *fwd_stats_border = "----------------------"; 794 795 port = &ports[port_id]; 796 printf("\n %s Forward statistics for port %-2d %s\n", 797 fwd_stats_border, port_id, fwd_stats_border); 798 799 if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) { 800 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 801 "%-"PRIu64"\n", 802 stats->ipackets, stats->imissed, 803 (uint64_t) (stats->ipackets + stats->imissed)); 804 805 if (cur_fwd_eng == &csum_fwd_engine) 806 printf(" Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n", 807 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 808 if ((stats->ierrors + stats->rx_nombuf) > 0) { 809 printf(" RX-error: %-"PRIu64"\n", stats->ierrors); 810 printf(" RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf); 811 } 812 813 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 814 "%-"PRIu64"\n", 815 stats->opackets, port->tx_dropped, 816 (uint64_t) (stats->opackets + port->tx_dropped)); 817 } 818 else { 819 printf(" RX-packets: %14"PRIu64" RX-dropped:%14"PRIu64" RX-total:" 820 "%14"PRIu64"\n", 821 stats->ipackets, stats->imissed, 822 (uint64_t) (stats->ipackets + stats->imissed)); 823 824 if (cur_fwd_eng == &csum_fwd_engine) 825 printf(" Bad-ipcsum:%14"PRIu64" Bad-l4csum:%14"PRIu64"\n", 826 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 827 if ((stats->ierrors + stats->rx_nombuf) > 0) { 828 printf(" RX-error:%"PRIu64"\n", stats->ierrors); 829 printf(" RX-nombufs: %14"PRIu64"\n", 830 stats->rx_nombuf); 831 } 832 833 printf(" TX-packets: %14"PRIu64" TX-dropped:%14"PRIu64" TX-total:" 834 "%14"PRIu64"\n", 835 stats->opackets, port->tx_dropped, 836 (uint64_t) (stats->opackets + port->tx_dropped)); 837 } 838 839 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 840 if (port->rx_stream) 841 pkt_burst_stats_display("RX", 842 &port->rx_stream->rx_burst_stats); 843 if (port->tx_stream) 844 pkt_burst_stats_display("TX", 845 &port->tx_stream->tx_burst_stats); 846 #endif 847 848 if (port->rx_queue_stats_mapping_enabled) { 849 printf("\n"); 850 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 851 printf(" Stats reg %2d RX-packets:%14"PRIu64 852 " RX-errors:%14"PRIu64 853 " RX-bytes:%14"PRIu64"\n", 854 i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]); 855 } 856 printf("\n"); 857 } 858 if (port->tx_queue_stats_mapping_enabled) { 859 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 860 printf(" Stats reg %2d TX-packets:%14"PRIu64 861 " TX-bytes:%14"PRIu64"\n", 862 i, stats->q_opackets[i], stats->q_obytes[i]); 863 } 864 } 865 866 printf(" %s--------------------------------%s\n", 867 fwd_stats_border, fwd_stats_border); 868 } 869 870 static void 871 fwd_stream_stats_display(streamid_t stream_id) 872 { 873 struct fwd_stream *fs; 874 static const char *fwd_top_stats_border = "-------"; 875 876 fs = fwd_streams[stream_id]; 877 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 878 (fs->fwd_dropped == 0)) 879 return; 880 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 881 "TX Port=%2d/Queue=%2d %s\n", 882 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 883 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 884 printf(" RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u", 885 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 886 887 /* if checksum mode */ 888 if (cur_fwd_eng == &csum_fwd_engine) { 889 printf(" RX- bad IP checksum: %-14u Rx- bad L4 checksum: " 890 "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum); 891 } 892 893 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 894 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 895 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 896 #endif 897 } 898 899 static void 900 flush_fwd_rx_queues(void) 901 { 902 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 903 portid_t rxp; 904 portid_t port_id; 905 queueid_t rxq; 906 uint16_t nb_rx; 907 uint16_t i; 908 uint8_t j; 909 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 910 uint64_t timer_period; 911 912 /* convert to number of cycles */ 913 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 914 915 for (j = 0; j < 2; j++) { 916 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 917 for (rxq = 0; rxq < nb_rxq; rxq++) { 918 port_id = fwd_ports_ids[rxp]; 919 /** 920 * testpmd can stuck in the below do while loop 921 * if rte_eth_rx_burst() always returns nonzero 922 * packets. So timer is added to exit this loop 923 * after 1sec timer expiry. 924 */ 925 prev_tsc = rte_rdtsc(); 926 do { 927 nb_rx = rte_eth_rx_burst(port_id, rxq, 928 pkts_burst, MAX_PKT_BURST); 929 for (i = 0; i < nb_rx; i++) 930 rte_pktmbuf_free(pkts_burst[i]); 931 932 cur_tsc = rte_rdtsc(); 933 diff_tsc = cur_tsc - prev_tsc; 934 timer_tsc += diff_tsc; 935 } while ((nb_rx > 0) && 936 (timer_tsc < timer_period)); 937 timer_tsc = 0; 938 } 939 } 940 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 941 } 942 } 943 944 static void 945 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 946 { 947 struct fwd_stream **fsm; 948 streamid_t nb_fs; 949 streamid_t sm_id; 950 #ifdef RTE_LIBRTE_BITRATE 951 uint64_t tics_per_1sec; 952 uint64_t tics_datum; 953 uint64_t tics_current; 954 uint8_t idx_port, cnt_ports; 955 956 cnt_ports = rte_eth_dev_count(); 957 tics_datum = rte_rdtsc(); 958 tics_per_1sec = rte_get_timer_hz(); 959 #endif 960 fsm = &fwd_streams[fc->stream_idx]; 961 nb_fs = fc->stream_nb; 962 do { 963 for (sm_id = 0; sm_id < nb_fs; sm_id++) 964 (*pkt_fwd)(fsm[sm_id]); 965 #ifdef RTE_LIBRTE_BITRATE 966 if (bitrate_enabled != 0 && 967 bitrate_lcore_id == rte_lcore_id()) { 968 tics_current = rte_rdtsc(); 969 if (tics_current - tics_datum >= tics_per_1sec) { 970 /* Periodic bitrate calculation */ 971 for (idx_port = 0; 972 idx_port < cnt_ports; 973 idx_port++) 974 rte_stats_bitrate_calc(bitrate_data, 975 idx_port); 976 tics_datum = tics_current; 977 } 978 } 979 #endif 980 #ifdef RTE_LIBRTE_LATENCY_STATS 981 if (latencystats_enabled != 0 && 982 latencystats_lcore_id == rte_lcore_id()) 983 rte_latencystats_update(); 984 #endif 985 986 } while (! fc->stopped); 987 } 988 989 static int 990 start_pkt_forward_on_core(void *fwd_arg) 991 { 992 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 993 cur_fwd_config.fwd_eng->packet_fwd); 994 return 0; 995 } 996 997 /* 998 * Run the TXONLY packet forwarding engine to send a single burst of packets. 999 * Used to start communication flows in network loopback test configurations. 1000 */ 1001 static int 1002 run_one_txonly_burst_on_core(void *fwd_arg) 1003 { 1004 struct fwd_lcore *fwd_lc; 1005 struct fwd_lcore tmp_lcore; 1006 1007 fwd_lc = (struct fwd_lcore *) fwd_arg; 1008 tmp_lcore = *fwd_lc; 1009 tmp_lcore.stopped = 1; 1010 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1011 return 0; 1012 } 1013 1014 /* 1015 * Launch packet forwarding: 1016 * - Setup per-port forwarding context. 1017 * - launch logical cores with their forwarding configuration. 1018 */ 1019 static void 1020 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1021 { 1022 port_fwd_begin_t port_fwd_begin; 1023 unsigned int i; 1024 unsigned int lc_id; 1025 int diag; 1026 1027 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1028 if (port_fwd_begin != NULL) { 1029 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1030 (*port_fwd_begin)(fwd_ports_ids[i]); 1031 } 1032 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1033 lc_id = fwd_lcores_cpuids[i]; 1034 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1035 fwd_lcores[i]->stopped = 0; 1036 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1037 fwd_lcores[i], lc_id); 1038 if (diag != 0) 1039 printf("launch lcore %u failed - diag=%d\n", 1040 lc_id, diag); 1041 } 1042 } 1043 } 1044 1045 /* 1046 * Launch packet forwarding configuration. 1047 */ 1048 void 1049 start_packet_forwarding(int with_tx_first) 1050 { 1051 port_fwd_begin_t port_fwd_begin; 1052 port_fwd_end_t port_fwd_end; 1053 struct rte_port *port; 1054 unsigned int i; 1055 portid_t pt_id; 1056 streamid_t sm_id; 1057 1058 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1059 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1060 1061 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1062 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1063 1064 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1065 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1066 (!nb_rxq || !nb_txq)) 1067 rte_exit(EXIT_FAILURE, 1068 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1069 cur_fwd_eng->fwd_mode_name); 1070 1071 if (all_ports_started() == 0) { 1072 printf("Not all ports were started\n"); 1073 return; 1074 } 1075 if (test_done == 0) { 1076 printf("Packet forwarding already started\n"); 1077 return; 1078 } 1079 1080 if (init_fwd_streams() < 0) { 1081 printf("Fail from init_fwd_streams()\n"); 1082 return; 1083 } 1084 1085 if(dcb_test) { 1086 for (i = 0; i < nb_fwd_ports; i++) { 1087 pt_id = fwd_ports_ids[i]; 1088 port = &ports[pt_id]; 1089 if (!port->dcb_flag) { 1090 printf("In DCB mode, all forwarding ports must " 1091 "be configured in this mode.\n"); 1092 return; 1093 } 1094 } 1095 if (nb_fwd_lcores == 1) { 1096 printf("In DCB mode,the nb forwarding cores " 1097 "should be larger than 1.\n"); 1098 return; 1099 } 1100 } 1101 test_done = 0; 1102 1103 if(!no_flush_rx) 1104 flush_fwd_rx_queues(); 1105 1106 fwd_config_setup(); 1107 pkt_fwd_config_display(&cur_fwd_config); 1108 rxtx_config_display(); 1109 1110 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1111 pt_id = fwd_ports_ids[i]; 1112 port = &ports[pt_id]; 1113 rte_eth_stats_get(pt_id, &port->stats); 1114 port->tx_dropped = 0; 1115 1116 map_port_queue_stats_mapping_registers(pt_id, port); 1117 } 1118 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1119 fwd_streams[sm_id]->rx_packets = 0; 1120 fwd_streams[sm_id]->tx_packets = 0; 1121 fwd_streams[sm_id]->fwd_dropped = 0; 1122 fwd_streams[sm_id]->rx_bad_ip_csum = 0; 1123 fwd_streams[sm_id]->rx_bad_l4_csum = 0; 1124 1125 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1126 memset(&fwd_streams[sm_id]->rx_burst_stats, 0, 1127 sizeof(fwd_streams[sm_id]->rx_burst_stats)); 1128 memset(&fwd_streams[sm_id]->tx_burst_stats, 0, 1129 sizeof(fwd_streams[sm_id]->tx_burst_stats)); 1130 #endif 1131 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1132 fwd_streams[sm_id]->core_cycles = 0; 1133 #endif 1134 } 1135 if (with_tx_first) { 1136 port_fwd_begin = tx_only_engine.port_fwd_begin; 1137 if (port_fwd_begin != NULL) { 1138 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1139 (*port_fwd_begin)(fwd_ports_ids[i]); 1140 } 1141 while (with_tx_first--) { 1142 launch_packet_forwarding( 1143 run_one_txonly_burst_on_core); 1144 rte_eal_mp_wait_lcore(); 1145 } 1146 port_fwd_end = tx_only_engine.port_fwd_end; 1147 if (port_fwd_end != NULL) { 1148 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1149 (*port_fwd_end)(fwd_ports_ids[i]); 1150 } 1151 } 1152 launch_packet_forwarding(start_pkt_forward_on_core); 1153 } 1154 1155 void 1156 stop_packet_forwarding(void) 1157 { 1158 struct rte_eth_stats stats; 1159 struct rte_port *port; 1160 port_fwd_end_t port_fwd_end; 1161 int i; 1162 portid_t pt_id; 1163 streamid_t sm_id; 1164 lcoreid_t lc_id; 1165 uint64_t total_recv; 1166 uint64_t total_xmit; 1167 uint64_t total_rx_dropped; 1168 uint64_t total_tx_dropped; 1169 uint64_t total_rx_nombuf; 1170 uint64_t tx_dropped; 1171 uint64_t rx_bad_ip_csum; 1172 uint64_t rx_bad_l4_csum; 1173 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1174 uint64_t fwd_cycles; 1175 #endif 1176 static const char *acc_stats_border = "+++++++++++++++"; 1177 1178 if (test_done) { 1179 printf("Packet forwarding not started\n"); 1180 return; 1181 } 1182 printf("Telling cores to stop..."); 1183 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1184 fwd_lcores[lc_id]->stopped = 1; 1185 printf("\nWaiting for lcores to finish...\n"); 1186 rte_eal_mp_wait_lcore(); 1187 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1188 if (port_fwd_end != NULL) { 1189 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1190 pt_id = fwd_ports_ids[i]; 1191 (*port_fwd_end)(pt_id); 1192 } 1193 } 1194 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1195 fwd_cycles = 0; 1196 #endif 1197 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1198 if (cur_fwd_config.nb_fwd_streams > 1199 cur_fwd_config.nb_fwd_ports) { 1200 fwd_stream_stats_display(sm_id); 1201 ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL; 1202 ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL; 1203 } else { 1204 ports[fwd_streams[sm_id]->tx_port].tx_stream = 1205 fwd_streams[sm_id]; 1206 ports[fwd_streams[sm_id]->rx_port].rx_stream = 1207 fwd_streams[sm_id]; 1208 } 1209 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped; 1210 tx_dropped = (uint64_t) (tx_dropped + 1211 fwd_streams[sm_id]->fwd_dropped); 1212 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped; 1213 1214 rx_bad_ip_csum = 1215 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum; 1216 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum + 1217 fwd_streams[sm_id]->rx_bad_ip_csum); 1218 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum = 1219 rx_bad_ip_csum; 1220 1221 rx_bad_l4_csum = 1222 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum; 1223 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum + 1224 fwd_streams[sm_id]->rx_bad_l4_csum); 1225 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum = 1226 rx_bad_l4_csum; 1227 1228 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1229 fwd_cycles = (uint64_t) (fwd_cycles + 1230 fwd_streams[sm_id]->core_cycles); 1231 #endif 1232 } 1233 total_recv = 0; 1234 total_xmit = 0; 1235 total_rx_dropped = 0; 1236 total_tx_dropped = 0; 1237 total_rx_nombuf = 0; 1238 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1239 pt_id = fwd_ports_ids[i]; 1240 1241 port = &ports[pt_id]; 1242 rte_eth_stats_get(pt_id, &stats); 1243 stats.ipackets -= port->stats.ipackets; 1244 port->stats.ipackets = 0; 1245 stats.opackets -= port->stats.opackets; 1246 port->stats.opackets = 0; 1247 stats.ibytes -= port->stats.ibytes; 1248 port->stats.ibytes = 0; 1249 stats.obytes -= port->stats.obytes; 1250 port->stats.obytes = 0; 1251 stats.imissed -= port->stats.imissed; 1252 port->stats.imissed = 0; 1253 stats.oerrors -= port->stats.oerrors; 1254 port->stats.oerrors = 0; 1255 stats.rx_nombuf -= port->stats.rx_nombuf; 1256 port->stats.rx_nombuf = 0; 1257 1258 total_recv += stats.ipackets; 1259 total_xmit += stats.opackets; 1260 total_rx_dropped += stats.imissed; 1261 total_tx_dropped += port->tx_dropped; 1262 total_rx_nombuf += stats.rx_nombuf; 1263 1264 fwd_port_stats_display(pt_id, &stats); 1265 } 1266 printf("\n %s Accumulated forward statistics for all ports" 1267 "%s\n", 1268 acc_stats_border, acc_stats_border); 1269 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1270 "%-"PRIu64"\n" 1271 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1272 "%-"PRIu64"\n", 1273 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1274 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1275 if (total_rx_nombuf > 0) 1276 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1277 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1278 "%s\n", 1279 acc_stats_border, acc_stats_border); 1280 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1281 if (total_recv > 0) 1282 printf("\n CPU cycles/packet=%u (total cycles=" 1283 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1284 (unsigned int)(fwd_cycles / total_recv), 1285 fwd_cycles, total_recv); 1286 #endif 1287 printf("\nDone.\n"); 1288 test_done = 1; 1289 } 1290 1291 void 1292 dev_set_link_up(portid_t pid) 1293 { 1294 if (rte_eth_dev_set_link_up((uint8_t)pid) < 0) 1295 printf("\nSet link up fail.\n"); 1296 } 1297 1298 void 1299 dev_set_link_down(portid_t pid) 1300 { 1301 if (rte_eth_dev_set_link_down((uint8_t)pid) < 0) 1302 printf("\nSet link down fail.\n"); 1303 } 1304 1305 static int 1306 all_ports_started(void) 1307 { 1308 portid_t pi; 1309 struct rte_port *port; 1310 1311 RTE_ETH_FOREACH_DEV(pi) { 1312 port = &ports[pi]; 1313 /* Check if there is a port which is not started */ 1314 if ((port->port_status != RTE_PORT_STARTED) && 1315 (port->slave_flag == 0)) 1316 return 0; 1317 } 1318 1319 /* No port is not started */ 1320 return 1; 1321 } 1322 1323 int 1324 all_ports_stopped(void) 1325 { 1326 portid_t pi; 1327 struct rte_port *port; 1328 1329 RTE_ETH_FOREACH_DEV(pi) { 1330 port = &ports[pi]; 1331 if ((port->port_status != RTE_PORT_STOPPED) && 1332 (port->slave_flag == 0)) 1333 return 0; 1334 } 1335 1336 return 1; 1337 } 1338 1339 int 1340 port_is_started(portid_t port_id) 1341 { 1342 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1343 return 0; 1344 1345 if (ports[port_id].port_status != RTE_PORT_STARTED) 1346 return 0; 1347 1348 return 1; 1349 } 1350 1351 static int 1352 port_is_closed(portid_t port_id) 1353 { 1354 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1355 return 0; 1356 1357 if (ports[port_id].port_status != RTE_PORT_CLOSED) 1358 return 0; 1359 1360 return 1; 1361 } 1362 1363 int 1364 start_port(portid_t pid) 1365 { 1366 int diag, need_check_link_status = -1; 1367 portid_t pi; 1368 queueid_t qi; 1369 struct rte_port *port; 1370 struct ether_addr mac_addr; 1371 enum rte_eth_event_type event_type; 1372 1373 if (port_id_is_invalid(pid, ENABLED_WARN)) 1374 return 0; 1375 1376 if(dcb_config) 1377 dcb_test = 1; 1378 RTE_ETH_FOREACH_DEV(pi) { 1379 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1380 continue; 1381 1382 need_check_link_status = 0; 1383 port = &ports[pi]; 1384 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1385 RTE_PORT_HANDLING) == 0) { 1386 printf("Port %d is now not stopped\n", pi); 1387 continue; 1388 } 1389 1390 if (port->need_reconfig > 0) { 1391 port->need_reconfig = 0; 1392 1393 printf("Configuring Port %d (socket %u)\n", pi, 1394 port->socket_id); 1395 /* configure port */ 1396 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1397 &(port->dev_conf)); 1398 if (diag != 0) { 1399 if (rte_atomic16_cmpset(&(port->port_status), 1400 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1401 printf("Port %d can not be set back " 1402 "to stopped\n", pi); 1403 printf("Fail to configure port %d\n", pi); 1404 /* try to reconfigure port next time */ 1405 port->need_reconfig = 1; 1406 return -1; 1407 } 1408 } 1409 if (port->need_reconfig_queues > 0) { 1410 port->need_reconfig_queues = 0; 1411 /* setup tx queues */ 1412 for (qi = 0; qi < nb_txq; qi++) { 1413 if ((numa_support) && 1414 (txring_numa[pi] != NUMA_NO_CONFIG)) 1415 diag = rte_eth_tx_queue_setup(pi, qi, 1416 nb_txd,txring_numa[pi], 1417 &(port->tx_conf)); 1418 else 1419 diag = rte_eth_tx_queue_setup(pi, qi, 1420 nb_txd,port->socket_id, 1421 &(port->tx_conf)); 1422 1423 if (diag == 0) 1424 continue; 1425 1426 /* Fail to setup tx queue, return */ 1427 if (rte_atomic16_cmpset(&(port->port_status), 1428 RTE_PORT_HANDLING, 1429 RTE_PORT_STOPPED) == 0) 1430 printf("Port %d can not be set back " 1431 "to stopped\n", pi); 1432 printf("Fail to configure port %d tx queues\n", pi); 1433 /* try to reconfigure queues next time */ 1434 port->need_reconfig_queues = 1; 1435 return -1; 1436 } 1437 /* setup rx queues */ 1438 for (qi = 0; qi < nb_rxq; qi++) { 1439 if ((numa_support) && 1440 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 1441 struct rte_mempool * mp = 1442 mbuf_pool_find(rxring_numa[pi]); 1443 if (mp == NULL) { 1444 printf("Failed to setup RX queue:" 1445 "No mempool allocation" 1446 " on the socket %d\n", 1447 rxring_numa[pi]); 1448 return -1; 1449 } 1450 1451 diag = rte_eth_rx_queue_setup(pi, qi, 1452 nb_rxd,rxring_numa[pi], 1453 &(port->rx_conf),mp); 1454 } else { 1455 struct rte_mempool *mp = 1456 mbuf_pool_find(port->socket_id); 1457 if (mp == NULL) { 1458 printf("Failed to setup RX queue:" 1459 "No mempool allocation" 1460 " on the socket %d\n", 1461 port->socket_id); 1462 return -1; 1463 } 1464 diag = rte_eth_rx_queue_setup(pi, qi, 1465 nb_rxd,port->socket_id, 1466 &(port->rx_conf), mp); 1467 } 1468 if (diag == 0) 1469 continue; 1470 1471 /* Fail to setup rx queue, return */ 1472 if (rte_atomic16_cmpset(&(port->port_status), 1473 RTE_PORT_HANDLING, 1474 RTE_PORT_STOPPED) == 0) 1475 printf("Port %d can not be set back " 1476 "to stopped\n", pi); 1477 printf("Fail to configure port %d rx queues\n", pi); 1478 /* try to reconfigure queues next time */ 1479 port->need_reconfig_queues = 1; 1480 return -1; 1481 } 1482 } 1483 1484 for (event_type = RTE_ETH_EVENT_UNKNOWN; 1485 event_type < RTE_ETH_EVENT_MAX; 1486 event_type++) { 1487 diag = rte_eth_dev_callback_register(pi, 1488 event_type, 1489 eth_event_callback, 1490 NULL); 1491 if (diag) { 1492 printf("Failed to setup even callback for event %d\n", 1493 event_type); 1494 return -1; 1495 } 1496 } 1497 1498 /* start port */ 1499 if (rte_eth_dev_start(pi) < 0) { 1500 printf("Fail to start port %d\n", pi); 1501 1502 /* Fail to setup rx queue, return */ 1503 if (rte_atomic16_cmpset(&(port->port_status), 1504 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1505 printf("Port %d can not be set back to " 1506 "stopped\n", pi); 1507 continue; 1508 } 1509 1510 if (rte_atomic16_cmpset(&(port->port_status), 1511 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 1512 printf("Port %d can not be set into started\n", pi); 1513 1514 rte_eth_macaddr_get(pi, &mac_addr); 1515 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 1516 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 1517 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 1518 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 1519 1520 /* at least one port started, need checking link status */ 1521 need_check_link_status = 1; 1522 } 1523 1524 if (need_check_link_status == 1 && !no_link_check) 1525 check_all_ports_link_status(RTE_PORT_ALL); 1526 else if (need_check_link_status == 0) 1527 printf("Please stop the ports first\n"); 1528 1529 printf("Done\n"); 1530 return 0; 1531 } 1532 1533 void 1534 stop_port(portid_t pid) 1535 { 1536 portid_t pi; 1537 struct rte_port *port; 1538 int need_check_link_status = 0; 1539 1540 if (dcb_test) { 1541 dcb_test = 0; 1542 dcb_config = 0; 1543 } 1544 1545 if (port_id_is_invalid(pid, ENABLED_WARN)) 1546 return; 1547 1548 printf("Stopping ports...\n"); 1549 1550 RTE_ETH_FOREACH_DEV(pi) { 1551 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1552 continue; 1553 1554 if (port_is_forwarding(pi) != 0 && test_done == 0) { 1555 printf("Please remove port %d from forwarding configuration.\n", pi); 1556 continue; 1557 } 1558 1559 if (port_is_bonding_slave(pi)) { 1560 printf("Please remove port %d from bonded device.\n", pi); 1561 continue; 1562 } 1563 1564 port = &ports[pi]; 1565 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 1566 RTE_PORT_HANDLING) == 0) 1567 continue; 1568 1569 rte_eth_dev_stop(pi); 1570 1571 if (rte_atomic16_cmpset(&(port->port_status), 1572 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1573 printf("Port %d can not be set into stopped\n", pi); 1574 need_check_link_status = 1; 1575 } 1576 if (need_check_link_status && !no_link_check) 1577 check_all_ports_link_status(RTE_PORT_ALL); 1578 1579 printf("Done\n"); 1580 } 1581 1582 void 1583 close_port(portid_t pid) 1584 { 1585 portid_t pi; 1586 struct rte_port *port; 1587 1588 if (port_id_is_invalid(pid, ENABLED_WARN)) 1589 return; 1590 1591 printf("Closing ports...\n"); 1592 1593 RTE_ETH_FOREACH_DEV(pi) { 1594 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1595 continue; 1596 1597 if (port_is_forwarding(pi) != 0 && test_done == 0) { 1598 printf("Please remove port %d from forwarding configuration.\n", pi); 1599 continue; 1600 } 1601 1602 if (port_is_bonding_slave(pi)) { 1603 printf("Please remove port %d from bonded device.\n", pi); 1604 continue; 1605 } 1606 1607 port = &ports[pi]; 1608 if (rte_atomic16_cmpset(&(port->port_status), 1609 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 1610 printf("Port %d is already closed\n", pi); 1611 continue; 1612 } 1613 1614 if (rte_atomic16_cmpset(&(port->port_status), 1615 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 1616 printf("Port %d is now not stopped\n", pi); 1617 continue; 1618 } 1619 1620 if (port->flow_list) 1621 port_flow_flush(pi); 1622 rte_eth_dev_close(pi); 1623 1624 if (rte_atomic16_cmpset(&(port->port_status), 1625 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 1626 printf("Port %d cannot be set to closed\n", pi); 1627 } 1628 1629 printf("Done\n"); 1630 } 1631 1632 void 1633 attach_port(char *identifier) 1634 { 1635 portid_t pi = 0; 1636 unsigned int socket_id; 1637 1638 printf("Attaching a new port...\n"); 1639 1640 if (identifier == NULL) { 1641 printf("Invalid parameters are specified\n"); 1642 return; 1643 } 1644 1645 if (rte_eth_dev_attach(identifier, &pi)) 1646 return; 1647 1648 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 1649 /* if socket_id is invalid, set to 0 */ 1650 if (check_socket_id(socket_id) < 0) 1651 socket_id = 0; 1652 reconfig(pi, socket_id); 1653 rte_eth_promiscuous_enable(pi); 1654 1655 nb_ports = rte_eth_dev_count(); 1656 1657 ports[pi].port_status = RTE_PORT_STOPPED; 1658 1659 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 1660 printf("Done\n"); 1661 } 1662 1663 void 1664 detach_port(uint8_t port_id) 1665 { 1666 char name[RTE_ETH_NAME_MAX_LEN]; 1667 1668 printf("Detaching a port...\n"); 1669 1670 if (!port_is_closed(port_id)) { 1671 printf("Please close port first\n"); 1672 return; 1673 } 1674 1675 if (ports[port_id].flow_list) 1676 port_flow_flush(port_id); 1677 1678 if (rte_eth_dev_detach(port_id, name)) 1679 return; 1680 1681 nb_ports = rte_eth_dev_count(); 1682 1683 printf("Port '%s' is detached. Now total ports is %d\n", 1684 name, nb_ports); 1685 printf("Done\n"); 1686 return; 1687 } 1688 1689 void 1690 pmd_test_exit(void) 1691 { 1692 portid_t pt_id; 1693 1694 if (test_done == 0) 1695 stop_packet_forwarding(); 1696 1697 if (ports != NULL) { 1698 no_link_check = 1; 1699 RTE_ETH_FOREACH_DEV(pt_id) { 1700 printf("\nShutting down port %d...\n", pt_id); 1701 fflush(stdout); 1702 stop_port(pt_id); 1703 close_port(pt_id); 1704 } 1705 } 1706 printf("\nBye...\n"); 1707 } 1708 1709 typedef void (*cmd_func_t)(void); 1710 struct pmd_test_command { 1711 const char *cmd_name; 1712 cmd_func_t cmd_func; 1713 }; 1714 1715 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 1716 1717 /* Check the link status of all ports in up to 9s, and print them finally */ 1718 static void 1719 check_all_ports_link_status(uint32_t port_mask) 1720 { 1721 #define CHECK_INTERVAL 100 /* 100ms */ 1722 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1723 uint8_t portid, count, all_ports_up, print_flag = 0; 1724 struct rte_eth_link link; 1725 1726 printf("Checking link statuses...\n"); 1727 fflush(stdout); 1728 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1729 all_ports_up = 1; 1730 RTE_ETH_FOREACH_DEV(portid) { 1731 if ((port_mask & (1 << portid)) == 0) 1732 continue; 1733 memset(&link, 0, sizeof(link)); 1734 rte_eth_link_get_nowait(portid, &link); 1735 /* print link status if flag set */ 1736 if (print_flag == 1) { 1737 if (link.link_status) 1738 printf("Port %d Link Up - speed %u " 1739 "Mbps - %s\n", (uint8_t)portid, 1740 (unsigned)link.link_speed, 1741 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1742 ("full-duplex") : ("half-duplex\n")); 1743 else 1744 printf("Port %d Link Down\n", 1745 (uint8_t)portid); 1746 continue; 1747 } 1748 /* clear all_ports_up flag if any link down */ 1749 if (link.link_status == ETH_LINK_DOWN) { 1750 all_ports_up = 0; 1751 break; 1752 } 1753 } 1754 /* after finally printing all link status, get out */ 1755 if (print_flag == 1) 1756 break; 1757 1758 if (all_ports_up == 0) { 1759 fflush(stdout); 1760 rte_delay_ms(CHECK_INTERVAL); 1761 } 1762 1763 /* set the print_flag if all ports up or timeout */ 1764 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1765 print_flag = 1; 1766 } 1767 1768 if (lsc_interrupt) 1769 break; 1770 } 1771 } 1772 1773 static void 1774 rmv_event_callback(void *arg) 1775 { 1776 struct rte_eth_dev *dev; 1777 struct rte_devargs *da; 1778 char name[32] = ""; 1779 uint8_t port_id = (intptr_t)arg; 1780 1781 RTE_ETH_VALID_PORTID_OR_RET(port_id); 1782 dev = &rte_eth_devices[port_id]; 1783 da = dev->device->devargs; 1784 1785 stop_port(port_id); 1786 close_port(port_id); 1787 if (da->type == RTE_DEVTYPE_VIRTUAL) 1788 snprintf(name, sizeof(name), "%s", da->virt.drv_name); 1789 else if (da->type == RTE_DEVTYPE_WHITELISTED_PCI) 1790 rte_pci_device_name(&da->pci.addr, name, sizeof(name)); 1791 printf("removing device %s\n", name); 1792 rte_eal_dev_detach(name); 1793 dev->state = RTE_ETH_DEV_UNUSED; 1794 } 1795 1796 /* This function is used by the interrupt thread */ 1797 static void 1798 eth_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param) 1799 { 1800 static const char * const event_desc[] = { 1801 [RTE_ETH_EVENT_UNKNOWN] = "Unknown", 1802 [RTE_ETH_EVENT_INTR_LSC] = "LSC", 1803 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state", 1804 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset", 1805 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox", 1806 [RTE_ETH_EVENT_MACSEC] = "MACsec", 1807 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 1808 [RTE_ETH_EVENT_MAX] = NULL, 1809 }; 1810 1811 RTE_SET_USED(param); 1812 1813 if (type >= RTE_ETH_EVENT_MAX) { 1814 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n", 1815 port_id, __func__, type); 1816 fflush(stderr); 1817 } else { 1818 printf("\nPort %" PRIu8 ": %s event\n", port_id, 1819 event_desc[type]); 1820 fflush(stdout); 1821 } 1822 1823 switch (type) { 1824 case RTE_ETH_EVENT_INTR_RMV: 1825 if (rte_eal_alarm_set(100000, 1826 rmv_event_callback, (void *)(intptr_t)port_id)) 1827 fprintf(stderr, "Could not set up deferred device removal\n"); 1828 break; 1829 default: 1830 break; 1831 } 1832 } 1833 1834 static int 1835 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) 1836 { 1837 uint16_t i; 1838 int diag; 1839 uint8_t mapping_found = 0; 1840 1841 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 1842 if ((tx_queue_stats_mappings[i].port_id == port_id) && 1843 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 1844 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 1845 tx_queue_stats_mappings[i].queue_id, 1846 tx_queue_stats_mappings[i].stats_counter_id); 1847 if (diag != 0) 1848 return diag; 1849 mapping_found = 1; 1850 } 1851 } 1852 if (mapping_found) 1853 port->tx_queue_stats_mapping_enabled = 1; 1854 return 0; 1855 } 1856 1857 static int 1858 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) 1859 { 1860 uint16_t i; 1861 int diag; 1862 uint8_t mapping_found = 0; 1863 1864 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 1865 if ((rx_queue_stats_mappings[i].port_id == port_id) && 1866 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 1867 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 1868 rx_queue_stats_mappings[i].queue_id, 1869 rx_queue_stats_mappings[i].stats_counter_id); 1870 if (diag != 0) 1871 return diag; 1872 mapping_found = 1; 1873 } 1874 } 1875 if (mapping_found) 1876 port->rx_queue_stats_mapping_enabled = 1; 1877 return 0; 1878 } 1879 1880 static void 1881 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port) 1882 { 1883 int diag = 0; 1884 1885 diag = set_tx_queue_stats_mapping_registers(pi, port); 1886 if (diag != 0) { 1887 if (diag == -ENOTSUP) { 1888 port->tx_queue_stats_mapping_enabled = 0; 1889 printf("TX queue stats mapping not supported port id=%d\n", pi); 1890 } 1891 else 1892 rte_exit(EXIT_FAILURE, 1893 "set_tx_queue_stats_mapping_registers " 1894 "failed for port id=%d diag=%d\n", 1895 pi, diag); 1896 } 1897 1898 diag = set_rx_queue_stats_mapping_registers(pi, port); 1899 if (diag != 0) { 1900 if (diag == -ENOTSUP) { 1901 port->rx_queue_stats_mapping_enabled = 0; 1902 printf("RX queue stats mapping not supported port id=%d\n", pi); 1903 } 1904 else 1905 rte_exit(EXIT_FAILURE, 1906 "set_rx_queue_stats_mapping_registers " 1907 "failed for port id=%d diag=%d\n", 1908 pi, diag); 1909 } 1910 } 1911 1912 static void 1913 rxtx_port_config(struct rte_port *port) 1914 { 1915 port->rx_conf = port->dev_info.default_rxconf; 1916 port->tx_conf = port->dev_info.default_txconf; 1917 1918 /* Check if any RX/TX parameters have been passed */ 1919 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 1920 port->rx_conf.rx_thresh.pthresh = rx_pthresh; 1921 1922 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 1923 port->rx_conf.rx_thresh.hthresh = rx_hthresh; 1924 1925 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 1926 port->rx_conf.rx_thresh.wthresh = rx_wthresh; 1927 1928 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 1929 port->rx_conf.rx_free_thresh = rx_free_thresh; 1930 1931 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 1932 port->rx_conf.rx_drop_en = rx_drop_en; 1933 1934 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 1935 port->tx_conf.tx_thresh.pthresh = tx_pthresh; 1936 1937 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 1938 port->tx_conf.tx_thresh.hthresh = tx_hthresh; 1939 1940 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 1941 port->tx_conf.tx_thresh.wthresh = tx_wthresh; 1942 1943 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 1944 port->tx_conf.tx_rs_thresh = tx_rs_thresh; 1945 1946 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 1947 port->tx_conf.tx_free_thresh = tx_free_thresh; 1948 1949 if (txq_flags != RTE_PMD_PARAM_UNSET) 1950 port->tx_conf.txq_flags = txq_flags; 1951 } 1952 1953 void 1954 init_port_config(void) 1955 { 1956 portid_t pid; 1957 struct rte_port *port; 1958 1959 RTE_ETH_FOREACH_DEV(pid) { 1960 port = &ports[pid]; 1961 port->dev_conf.rxmode = rx_mode; 1962 port->dev_conf.fdir_conf = fdir_conf; 1963 if (nb_rxq > 1) { 1964 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 1965 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf; 1966 } else { 1967 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 1968 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 1969 } 1970 1971 if (port->dcb_flag == 0) { 1972 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 1973 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 1974 else 1975 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 1976 } 1977 1978 rxtx_port_config(port); 1979 1980 rte_eth_macaddr_get(pid, &port->eth_addr); 1981 1982 map_port_queue_stats_mapping_registers(pid, port); 1983 #ifdef RTE_NIC_BYPASS 1984 rte_eth_dev_bypass_init(pid); 1985 #endif 1986 1987 if (lsc_interrupt && 1988 (rte_eth_devices[pid].data->dev_flags & 1989 RTE_ETH_DEV_INTR_LSC)) 1990 port->dev_conf.intr_conf.lsc = 1; 1991 if (rmv_interrupt && 1992 (rte_eth_devices[pid].data->dev_flags & 1993 RTE_ETH_DEV_INTR_RMV)) 1994 port->dev_conf.intr_conf.rmv = 1; 1995 } 1996 } 1997 1998 void set_port_slave_flag(portid_t slave_pid) 1999 { 2000 struct rte_port *port; 2001 2002 port = &ports[slave_pid]; 2003 port->slave_flag = 1; 2004 } 2005 2006 void clear_port_slave_flag(portid_t slave_pid) 2007 { 2008 struct rte_port *port; 2009 2010 port = &ports[slave_pid]; 2011 port->slave_flag = 0; 2012 } 2013 2014 uint8_t port_is_bonding_slave(portid_t slave_pid) 2015 { 2016 struct rte_port *port; 2017 2018 port = &ports[slave_pid]; 2019 return port->slave_flag; 2020 } 2021 2022 const uint16_t vlan_tags[] = { 2023 0, 1, 2, 3, 4, 5, 6, 7, 2024 8, 9, 10, 11, 12, 13, 14, 15, 2025 16, 17, 18, 19, 20, 21, 22, 23, 2026 24, 25, 26, 27, 28, 29, 30, 31 2027 }; 2028 2029 static int 2030 get_eth_dcb_conf(struct rte_eth_conf *eth_conf, 2031 enum dcb_mode_enable dcb_mode, 2032 enum rte_eth_nb_tcs num_tcs, 2033 uint8_t pfc_en) 2034 { 2035 uint8_t i; 2036 2037 /* 2038 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2039 * given above, and the number of traffic classes available for use. 2040 */ 2041 if (dcb_mode == DCB_VT_ENABLED) { 2042 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2043 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2044 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2045 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2046 2047 /* VMDQ+DCB RX and TX configurations */ 2048 vmdq_rx_conf->enable_default_pool = 0; 2049 vmdq_rx_conf->default_pool = 0; 2050 vmdq_rx_conf->nb_queue_pools = 2051 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2052 vmdq_tx_conf->nb_queue_pools = 2053 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2054 2055 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2056 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2057 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2058 vmdq_rx_conf->pool_map[i].pools = 2059 1 << (i % vmdq_rx_conf->nb_queue_pools); 2060 } 2061 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2062 vmdq_rx_conf->dcb_tc[i] = i; 2063 vmdq_tx_conf->dcb_tc[i] = i; 2064 } 2065 2066 /* set DCB mode of RX and TX of multiple queues */ 2067 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2068 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2069 } else { 2070 struct rte_eth_dcb_rx_conf *rx_conf = 2071 ð_conf->rx_adv_conf.dcb_rx_conf; 2072 struct rte_eth_dcb_tx_conf *tx_conf = 2073 ð_conf->tx_adv_conf.dcb_tx_conf; 2074 2075 rx_conf->nb_tcs = num_tcs; 2076 tx_conf->nb_tcs = num_tcs; 2077 2078 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2079 rx_conf->dcb_tc[i] = i % num_tcs; 2080 tx_conf->dcb_tc[i] = i % num_tcs; 2081 } 2082 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2083 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf; 2084 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2085 } 2086 2087 if (pfc_en) 2088 eth_conf->dcb_capability_en = 2089 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2090 else 2091 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2092 2093 return 0; 2094 } 2095 2096 int 2097 init_port_dcb_config(portid_t pid, 2098 enum dcb_mode_enable dcb_mode, 2099 enum rte_eth_nb_tcs num_tcs, 2100 uint8_t pfc_en) 2101 { 2102 struct rte_eth_conf port_conf; 2103 struct rte_port *rte_port; 2104 int retval; 2105 uint16_t i; 2106 2107 rte_port = &ports[pid]; 2108 2109 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2110 /* Enter DCB configuration status */ 2111 dcb_config = 1; 2112 2113 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2114 retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en); 2115 if (retval < 0) 2116 return retval; 2117 port_conf.rxmode.hw_vlan_filter = 1; 2118 2119 /** 2120 * Write the configuration into the device. 2121 * Set the numbers of RX & TX queues to 0, so 2122 * the RX & TX queues will not be setup. 2123 */ 2124 (void)rte_eth_dev_configure(pid, 0, 0, &port_conf); 2125 2126 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2127 2128 /* If dev_info.vmdq_pool_base is greater than 0, 2129 * the queue id of vmdq pools is started after pf queues. 2130 */ 2131 if (dcb_mode == DCB_VT_ENABLED && 2132 rte_port->dev_info.vmdq_pool_base > 0) { 2133 printf("VMDQ_DCB multi-queue mode is nonsensical" 2134 " for port %d.", pid); 2135 return -1; 2136 } 2137 2138 /* Assume the ports in testpmd have the same dcb capability 2139 * and has the same number of rxq and txq in dcb mode 2140 */ 2141 if (dcb_mode == DCB_VT_ENABLED) { 2142 if (rte_port->dev_info.max_vfs > 0) { 2143 nb_rxq = rte_port->dev_info.nb_rx_queues; 2144 nb_txq = rte_port->dev_info.nb_tx_queues; 2145 } else { 2146 nb_rxq = rte_port->dev_info.max_rx_queues; 2147 nb_txq = rte_port->dev_info.max_tx_queues; 2148 } 2149 } else { 2150 /*if vt is disabled, use all pf queues */ 2151 if (rte_port->dev_info.vmdq_pool_base == 0) { 2152 nb_rxq = rte_port->dev_info.max_rx_queues; 2153 nb_txq = rte_port->dev_info.max_tx_queues; 2154 } else { 2155 nb_rxq = (queueid_t)num_tcs; 2156 nb_txq = (queueid_t)num_tcs; 2157 2158 } 2159 } 2160 rx_free_thresh = 64; 2161 2162 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 2163 2164 rxtx_port_config(rte_port); 2165 /* VLAN filter */ 2166 rte_port->dev_conf.rxmode.hw_vlan_filter = 1; 2167 for (i = 0; i < RTE_DIM(vlan_tags); i++) 2168 rx_vft_set(pid, vlan_tags[i], 1); 2169 2170 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 2171 map_port_queue_stats_mapping_registers(pid, rte_port); 2172 2173 rte_port->dcb_flag = 1; 2174 2175 return 0; 2176 } 2177 2178 static void 2179 init_port(void) 2180 { 2181 /* Configuration of Ethernet ports. */ 2182 ports = rte_zmalloc("testpmd: ports", 2183 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 2184 RTE_CACHE_LINE_SIZE); 2185 if (ports == NULL) { 2186 rte_exit(EXIT_FAILURE, 2187 "rte_zmalloc(%d struct rte_port) failed\n", 2188 RTE_MAX_ETHPORTS); 2189 } 2190 } 2191 2192 static void 2193 force_quit(void) 2194 { 2195 pmd_test_exit(); 2196 prompt_exit(); 2197 } 2198 2199 static void 2200 signal_handler(int signum) 2201 { 2202 if (signum == SIGINT || signum == SIGTERM) { 2203 printf("\nSignal %d received, preparing to exit...\n", 2204 signum); 2205 #ifdef RTE_LIBRTE_PDUMP 2206 /* uninitialize packet capture framework */ 2207 rte_pdump_uninit(); 2208 #endif 2209 #ifdef RTE_LIBRTE_LATENCY_STATS 2210 rte_latencystats_uninit(); 2211 #endif 2212 force_quit(); 2213 /* exit with the expected status */ 2214 signal(signum, SIG_DFL); 2215 kill(getpid(), signum); 2216 } 2217 } 2218 2219 int 2220 main(int argc, char** argv) 2221 { 2222 int diag; 2223 uint8_t port_id; 2224 2225 signal(SIGINT, signal_handler); 2226 signal(SIGTERM, signal_handler); 2227 2228 diag = rte_eal_init(argc, argv); 2229 if (diag < 0) 2230 rte_panic("Cannot init EAL\n"); 2231 2232 #ifdef RTE_LIBRTE_PDUMP 2233 /* initialize packet capture framework */ 2234 rte_pdump_init(NULL); 2235 #endif 2236 2237 nb_ports = (portid_t) rte_eth_dev_count(); 2238 if (nb_ports == 0) 2239 RTE_LOG(WARNING, EAL, "No probed ethernet devices\n"); 2240 2241 /* allocate port structures, and init them */ 2242 init_port(); 2243 2244 set_def_fwd_config(); 2245 if (nb_lcores == 0) 2246 rte_panic("Empty set of forwarding logical cores - check the " 2247 "core mask supplied in the command parameters\n"); 2248 2249 /* Bitrate/latency stats disabled by default */ 2250 #ifdef RTE_LIBRTE_BITRATE 2251 bitrate_enabled = 0; 2252 #endif 2253 #ifdef RTE_LIBRTE_LATENCY_STATS 2254 latencystats_enabled = 0; 2255 #endif 2256 2257 argc -= diag; 2258 argv += diag; 2259 if (argc > 1) 2260 launch_args_parse(argc, argv); 2261 2262 if (!nb_rxq && !nb_txq) 2263 printf("Warning: Either rx or tx queues should be non-zero\n"); 2264 2265 if (nb_rxq > 1 && nb_rxq > nb_txq) 2266 printf("Warning: nb_rxq=%d enables RSS configuration, " 2267 "but nb_txq=%d will prevent to fully test it.\n", 2268 nb_rxq, nb_txq); 2269 2270 init_config(); 2271 if (start_port(RTE_PORT_ALL) != 0) 2272 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 2273 2274 /* set all ports to promiscuous mode by default */ 2275 RTE_ETH_FOREACH_DEV(port_id) 2276 rte_eth_promiscuous_enable(port_id); 2277 2278 /* Init metrics library */ 2279 rte_metrics_init(rte_socket_id()); 2280 2281 #ifdef RTE_LIBRTE_LATENCY_STATS 2282 if (latencystats_enabled != 0) { 2283 int ret = rte_latencystats_init(1, NULL); 2284 if (ret) 2285 printf("Warning: latencystats init()" 2286 " returned error %d\n", ret); 2287 printf("Latencystats running on lcore %d\n", 2288 latencystats_lcore_id); 2289 } 2290 #endif 2291 2292 /* Setup bitrate stats */ 2293 #ifdef RTE_LIBRTE_BITRATE 2294 if (bitrate_enabled != 0) { 2295 bitrate_data = rte_stats_bitrate_create(); 2296 if (bitrate_data == NULL) 2297 rte_exit(EXIT_FAILURE, 2298 "Could not allocate bitrate data.\n"); 2299 rte_stats_bitrate_reg(bitrate_data); 2300 } 2301 #endif 2302 2303 #ifdef RTE_LIBRTE_CMDLINE 2304 if (strlen(cmdline_filename) != 0) 2305 cmdline_read_from_file(cmdline_filename); 2306 2307 if (interactive == 1) { 2308 if (auto_start) { 2309 printf("Start automatic packet forwarding\n"); 2310 start_packet_forwarding(0); 2311 } 2312 prompt(); 2313 pmd_test_exit(); 2314 } else 2315 #endif 2316 { 2317 char c; 2318 int rc; 2319 2320 printf("No commandline core given, start packet forwarding\n"); 2321 start_packet_forwarding(0); 2322 printf("Press enter to exit\n"); 2323 rc = read(0, &c, 1); 2324 pmd_test_exit(); 2325 if (rc < 0) 2326 return 1; 2327 } 2328 2329 return 0; 2330 } 2331