1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdarg.h> 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <signal.h> 38 #include <string.h> 39 #include <time.h> 40 #include <fcntl.h> 41 #include <sys/types.h> 42 #include <errno.h> 43 44 #include <sys/queue.h> 45 #include <sys/stat.h> 46 47 #include <stdint.h> 48 #include <unistd.h> 49 #include <inttypes.h> 50 51 #include <rte_common.h> 52 #include <rte_errno.h> 53 #include <rte_byteorder.h> 54 #include <rte_log.h> 55 #include <rte_debug.h> 56 #include <rte_cycles.h> 57 #include <rte_memory.h> 58 #include <rte_memcpy.h> 59 #include <rte_memzone.h> 60 #include <rte_launch.h> 61 #include <rte_eal.h> 62 #include <rte_alarm.h> 63 #include <rte_per_lcore.h> 64 #include <rte_lcore.h> 65 #include <rte_atomic.h> 66 #include <rte_branch_prediction.h> 67 #include <rte_mempool.h> 68 #include <rte_malloc.h> 69 #include <rte_mbuf.h> 70 #include <rte_interrupts.h> 71 #include <rte_pci.h> 72 #include <rte_ether.h> 73 #include <rte_ethdev.h> 74 #include <rte_dev.h> 75 #include <rte_string_fns.h> 76 #ifdef RTE_LIBRTE_PMD_XENVIRT 77 #include <rte_eth_xenvirt.h> 78 #endif 79 #ifdef RTE_LIBRTE_PDUMP 80 #include <rte_pdump.h> 81 #endif 82 #include <rte_flow.h> 83 #include <rte_metrics.h> 84 #ifdef RTE_LIBRTE_BITRATE 85 #include <rte_bitrate.h> 86 #endif 87 #ifdef RTE_LIBRTE_LATENCY_STATS 88 #include <rte_latencystats.h> 89 #endif 90 91 #include "testpmd.h" 92 93 uint16_t verbose_level = 0; /**< Silent by default. */ 94 95 /* use master core for command line ? */ 96 uint8_t interactive = 0; 97 uint8_t auto_start = 0; 98 char cmdline_filename[PATH_MAX] = {0}; 99 100 /* 101 * NUMA support configuration. 102 * When set, the NUMA support attempts to dispatch the allocation of the 103 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 104 * probed ports among the CPU sockets 0 and 1. 105 * Otherwise, all memory is allocated from CPU socket 0. 106 */ 107 uint8_t numa_support = 1; /**< numa enabled by default */ 108 109 /* 110 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 111 * not configured. 112 */ 113 uint8_t socket_num = UMA_NO_CONFIG; 114 115 /* 116 * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs. 117 */ 118 uint8_t mp_anon = 0; 119 120 /* 121 * Record the Ethernet address of peer target ports to which packets are 122 * forwarded. 123 * Must be instantiated with the ethernet addresses of peer traffic generator 124 * ports. 125 */ 126 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 127 portid_t nb_peer_eth_addrs = 0; 128 129 /* 130 * Probed Target Environment. 131 */ 132 struct rte_port *ports; /**< For all probed ethernet ports. */ 133 portid_t nb_ports; /**< Number of probed ethernet ports. */ 134 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 135 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 136 137 /* 138 * Test Forwarding Configuration. 139 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 140 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 141 */ 142 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 143 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 144 portid_t nb_cfg_ports; /**< Number of configured ports. */ 145 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 146 147 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 148 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 149 150 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 151 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 152 153 /* 154 * Forwarding engines. 155 */ 156 struct fwd_engine * fwd_engines[] = { 157 &io_fwd_engine, 158 &mac_fwd_engine, 159 &mac_swap_engine, 160 &flow_gen_engine, 161 &rx_only_engine, 162 &tx_only_engine, 163 &csum_fwd_engine, 164 &icmp_echo_engine, 165 #ifdef RTE_LIBRTE_IEEE1588 166 &ieee1588_fwd_engine, 167 #endif 168 NULL, 169 }; 170 171 struct fwd_config cur_fwd_config; 172 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 173 uint32_t retry_enabled; 174 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 175 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 176 177 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 178 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 179 * specified on command-line. */ 180 181 /* 182 * Configuration of packet segments used by the "txonly" processing engine. 183 */ 184 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 185 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 186 TXONLY_DEF_PACKET_LEN, 187 }; 188 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 189 190 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 191 /**< Split policy for packets to TX. */ 192 193 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 194 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 195 196 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 197 uint8_t dcb_config = 0; 198 199 /* Whether the dcb is in testing status */ 200 uint8_t dcb_test = 0; 201 202 /* 203 * Configurable number of RX/TX queues. 204 */ 205 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 206 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 207 208 /* 209 * Configurable number of RX/TX ring descriptors. 210 */ 211 #define RTE_TEST_RX_DESC_DEFAULT 128 212 #define RTE_TEST_TX_DESC_DEFAULT 512 213 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 214 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 215 216 #define RTE_PMD_PARAM_UNSET -1 217 /* 218 * Configurable values of RX and TX ring threshold registers. 219 */ 220 221 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 222 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 223 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 224 225 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 226 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 227 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 228 229 /* 230 * Configurable value of RX free threshold. 231 */ 232 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 233 234 /* 235 * Configurable value of RX drop enable. 236 */ 237 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 238 239 /* 240 * Configurable value of TX free threshold. 241 */ 242 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 243 244 /* 245 * Configurable value of TX RS bit threshold. 246 */ 247 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 248 249 /* 250 * Configurable value of TX queue flags. 251 */ 252 int32_t txq_flags = RTE_PMD_PARAM_UNSET; 253 254 /* 255 * Receive Side Scaling (RSS) configuration. 256 */ 257 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 258 259 /* 260 * Port topology configuration 261 */ 262 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 263 264 /* 265 * Avoids to flush all the RX streams before starts forwarding. 266 */ 267 uint8_t no_flush_rx = 0; /* flush by default */ 268 269 /* 270 * Avoids to check link status when starting/stopping a port. 271 */ 272 uint8_t no_link_check = 0; /* check by default */ 273 274 /* 275 * Enable link status change notification 276 */ 277 uint8_t lsc_interrupt = 1; /* enabled by default */ 278 279 /* 280 * Enable device removal notification. 281 */ 282 uint8_t rmv_interrupt = 1; /* enabled by default */ 283 284 /* 285 * NIC bypass mode configuration options. 286 */ 287 #ifdef RTE_NIC_BYPASS 288 289 /* The NIC bypass watchdog timeout. */ 290 uint32_t bypass_timeout = RTE_BYPASS_TMT_OFF; 291 292 #endif 293 294 #ifdef RTE_LIBRTE_LATENCY_STATS 295 296 /* 297 * Set when latency stats is enabled in the commandline 298 */ 299 uint8_t latencystats_enabled; 300 301 /* 302 * Lcore ID to serive latency statistics. 303 */ 304 lcoreid_t latencystats_lcore_id = -1; 305 306 #endif 307 308 /* 309 * Ethernet device configuration. 310 */ 311 struct rte_eth_rxmode rx_mode = { 312 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 313 .split_hdr_size = 0, 314 .header_split = 0, /**< Header Split disabled. */ 315 .hw_ip_checksum = 0, /**< IP checksum offload disabled. */ 316 .hw_vlan_filter = 1, /**< VLAN filtering enabled. */ 317 .hw_vlan_strip = 1, /**< VLAN strip enabled. */ 318 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 319 .jumbo_frame = 0, /**< Jumbo Frame Support disabled. */ 320 .hw_strip_crc = 1, /**< CRC stripping by hardware enabled. */ 321 }; 322 323 struct rte_fdir_conf fdir_conf = { 324 .mode = RTE_FDIR_MODE_NONE, 325 .pballoc = RTE_FDIR_PBALLOC_64K, 326 .status = RTE_FDIR_REPORT_STATUS, 327 .mask = { 328 .vlan_tci_mask = 0x0, 329 .ipv4_mask = { 330 .src_ip = 0xFFFFFFFF, 331 .dst_ip = 0xFFFFFFFF, 332 }, 333 .ipv6_mask = { 334 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 335 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 336 }, 337 .src_port_mask = 0xFFFF, 338 .dst_port_mask = 0xFFFF, 339 .mac_addr_byte_mask = 0xFF, 340 .tunnel_type_mask = 1, 341 .tunnel_id_mask = 0xFFFFFFFF, 342 }, 343 .drop_queue = 127, 344 }; 345 346 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 347 348 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 349 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 350 351 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 352 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 353 354 uint16_t nb_tx_queue_stats_mappings = 0; 355 uint16_t nb_rx_queue_stats_mappings = 0; 356 357 unsigned max_socket = 0; 358 359 #ifdef RTE_LIBRTE_BITRATE 360 /* Bitrate statistics */ 361 struct rte_stats_bitrates *bitrate_data; 362 lcoreid_t bitrate_lcore_id; 363 uint8_t bitrate_enabled; 364 #endif 365 366 /* Forward function declarations */ 367 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port); 368 static void check_all_ports_link_status(uint32_t port_mask); 369 static void eth_event_callback(uint8_t port_id, 370 enum rte_eth_event_type type, 371 void *param); 372 373 /* 374 * Check if all the ports are started. 375 * If yes, return positive value. If not, return zero. 376 */ 377 static int all_ports_started(void); 378 379 /* 380 * Setup default configuration. 381 */ 382 static void 383 set_default_fwd_lcores_config(void) 384 { 385 unsigned int i; 386 unsigned int nb_lc; 387 unsigned int sock_num; 388 389 nb_lc = 0; 390 for (i = 0; i < RTE_MAX_LCORE; i++) { 391 sock_num = rte_lcore_to_socket_id(i) + 1; 392 if (sock_num > max_socket) { 393 if (sock_num > RTE_MAX_NUMA_NODES) 394 rte_exit(EXIT_FAILURE, "Total sockets greater than %u\n", RTE_MAX_NUMA_NODES); 395 max_socket = sock_num; 396 } 397 if (!rte_lcore_is_enabled(i)) 398 continue; 399 if (i == rte_get_master_lcore()) 400 continue; 401 fwd_lcores_cpuids[nb_lc++] = i; 402 } 403 nb_lcores = (lcoreid_t) nb_lc; 404 nb_cfg_lcores = nb_lcores; 405 nb_fwd_lcores = 1; 406 } 407 408 static void 409 set_def_peer_eth_addrs(void) 410 { 411 portid_t i; 412 413 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 414 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 415 peer_eth_addrs[i].addr_bytes[5] = i; 416 } 417 } 418 419 static void 420 set_default_fwd_ports_config(void) 421 { 422 portid_t pt_id; 423 424 for (pt_id = 0; pt_id < nb_ports; pt_id++) 425 fwd_ports_ids[pt_id] = pt_id; 426 427 nb_cfg_ports = nb_ports; 428 nb_fwd_ports = nb_ports; 429 } 430 431 void 432 set_def_fwd_config(void) 433 { 434 set_default_fwd_lcores_config(); 435 set_def_peer_eth_addrs(); 436 set_default_fwd_ports_config(); 437 } 438 439 /* 440 * Configuration initialisation done once at init time. 441 */ 442 static void 443 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 444 unsigned int socket_id) 445 { 446 char pool_name[RTE_MEMPOOL_NAMESIZE]; 447 struct rte_mempool *rte_mp = NULL; 448 uint32_t mb_size; 449 450 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 451 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 452 453 RTE_LOG(INFO, USER1, 454 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 455 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 456 457 #ifdef RTE_LIBRTE_PMD_XENVIRT 458 rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size, 459 (unsigned) mb_mempool_cache, 460 sizeof(struct rte_pktmbuf_pool_private), 461 rte_pktmbuf_pool_init, NULL, 462 rte_pktmbuf_init, NULL, 463 socket_id, 0); 464 #endif 465 466 /* if the former XEN allocation failed fall back to normal allocation */ 467 if (rte_mp == NULL) { 468 if (mp_anon != 0) { 469 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 470 mb_size, (unsigned) mb_mempool_cache, 471 sizeof(struct rte_pktmbuf_pool_private), 472 socket_id, 0); 473 if (rte_mp == NULL) 474 goto err; 475 476 if (rte_mempool_populate_anon(rte_mp) == 0) { 477 rte_mempool_free(rte_mp); 478 rte_mp = NULL; 479 goto err; 480 } 481 rte_pktmbuf_pool_init(rte_mp, NULL); 482 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 483 } else { 484 /* wrapper to rte_mempool_create() */ 485 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 486 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 487 } 488 } 489 490 err: 491 if (rte_mp == NULL) { 492 rte_exit(EXIT_FAILURE, 493 "Creation of mbuf pool for socket %u failed: %s\n", 494 socket_id, rte_strerror(rte_errno)); 495 } else if (verbose_level > 0) { 496 rte_mempool_dump(stdout, rte_mp); 497 } 498 } 499 500 /* 501 * Check given socket id is valid or not with NUMA mode, 502 * if valid, return 0, else return -1 503 */ 504 static int 505 check_socket_id(const unsigned int socket_id) 506 { 507 static int warning_once = 0; 508 509 if (socket_id >= max_socket) { 510 if (!warning_once && numa_support) 511 printf("Warning: NUMA should be configured manually by" 512 " using --port-numa-config and" 513 " --ring-numa-config parameters along with" 514 " --numa.\n"); 515 warning_once = 1; 516 return -1; 517 } 518 return 0; 519 } 520 521 static void 522 init_config(void) 523 { 524 portid_t pid; 525 struct rte_port *port; 526 struct rte_mempool *mbp; 527 unsigned int nb_mbuf_per_pool; 528 lcoreid_t lc_id; 529 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 530 531 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 532 /* Configuration of logical cores. */ 533 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 534 sizeof(struct fwd_lcore *) * nb_lcores, 535 RTE_CACHE_LINE_SIZE); 536 if (fwd_lcores == NULL) { 537 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 538 "failed\n", nb_lcores); 539 } 540 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 541 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 542 sizeof(struct fwd_lcore), 543 RTE_CACHE_LINE_SIZE); 544 if (fwd_lcores[lc_id] == NULL) { 545 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 546 "failed\n"); 547 } 548 fwd_lcores[lc_id]->cpuid_idx = lc_id; 549 } 550 551 RTE_ETH_FOREACH_DEV(pid) { 552 port = &ports[pid]; 553 rte_eth_dev_info_get(pid, &port->dev_info); 554 555 if (numa_support) { 556 if (port_numa[pid] != NUMA_NO_CONFIG) 557 port_per_socket[port_numa[pid]]++; 558 else { 559 uint32_t socket_id = rte_eth_dev_socket_id(pid); 560 561 /* if socket_id is invalid, set to 0 */ 562 if (check_socket_id(socket_id) < 0) 563 socket_id = 0; 564 port_per_socket[socket_id]++; 565 } 566 } 567 568 /* set flag to initialize port/queue */ 569 port->need_reconfig = 1; 570 port->need_reconfig_queues = 1; 571 } 572 573 /* 574 * Create pools of mbuf. 575 * If NUMA support is disabled, create a single pool of mbuf in 576 * socket 0 memory by default. 577 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 578 * 579 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 580 * nb_txd can be configured at run time. 581 */ 582 if (param_total_num_mbufs) 583 nb_mbuf_per_pool = param_total_num_mbufs; 584 else { 585 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 586 (nb_lcores * mb_mempool_cache) + 587 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 588 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 589 } 590 591 if (numa_support) { 592 uint8_t i; 593 594 for (i = 0; i < max_socket; i++) 595 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, i); 596 } else { 597 if (socket_num == UMA_NO_CONFIG) 598 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); 599 else 600 mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 601 socket_num); 602 } 603 604 init_port_config(); 605 606 /* 607 * Records which Mbuf pool to use by each logical core, if needed. 608 */ 609 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 610 mbp = mbuf_pool_find( 611 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 612 613 if (mbp == NULL) 614 mbp = mbuf_pool_find(0); 615 fwd_lcores[lc_id]->mbp = mbp; 616 } 617 618 /* Configuration of packet forwarding streams. */ 619 if (init_fwd_streams() < 0) 620 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 621 622 fwd_config_setup(); 623 } 624 625 626 void 627 reconfig(portid_t new_port_id, unsigned socket_id) 628 { 629 struct rte_port *port; 630 631 /* Reconfiguration of Ethernet ports. */ 632 port = &ports[new_port_id]; 633 rte_eth_dev_info_get(new_port_id, &port->dev_info); 634 635 /* set flag to initialize port/queue */ 636 port->need_reconfig = 1; 637 port->need_reconfig_queues = 1; 638 port->socket_id = socket_id; 639 640 init_port_config(); 641 } 642 643 644 int 645 init_fwd_streams(void) 646 { 647 portid_t pid; 648 struct rte_port *port; 649 streamid_t sm_id, nb_fwd_streams_new; 650 queueid_t q; 651 652 /* set socket id according to numa or not */ 653 RTE_ETH_FOREACH_DEV(pid) { 654 port = &ports[pid]; 655 if (nb_rxq > port->dev_info.max_rx_queues) { 656 printf("Fail: nb_rxq(%d) is greater than " 657 "max_rx_queues(%d)\n", nb_rxq, 658 port->dev_info.max_rx_queues); 659 return -1; 660 } 661 if (nb_txq > port->dev_info.max_tx_queues) { 662 printf("Fail: nb_txq(%d) is greater than " 663 "max_tx_queues(%d)\n", nb_txq, 664 port->dev_info.max_tx_queues); 665 return -1; 666 } 667 if (numa_support) { 668 if (port_numa[pid] != NUMA_NO_CONFIG) 669 port->socket_id = port_numa[pid]; 670 else { 671 port->socket_id = rte_eth_dev_socket_id(pid); 672 673 /* if socket_id is invalid, set to 0 */ 674 if (check_socket_id(port->socket_id) < 0) 675 port->socket_id = 0; 676 } 677 } 678 else { 679 if (socket_num == UMA_NO_CONFIG) 680 port->socket_id = 0; 681 else 682 port->socket_id = socket_num; 683 } 684 } 685 686 q = RTE_MAX(nb_rxq, nb_txq); 687 if (q == 0) { 688 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 689 return -1; 690 } 691 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 692 if (nb_fwd_streams_new == nb_fwd_streams) 693 return 0; 694 /* clear the old */ 695 if (fwd_streams != NULL) { 696 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 697 if (fwd_streams[sm_id] == NULL) 698 continue; 699 rte_free(fwd_streams[sm_id]); 700 fwd_streams[sm_id] = NULL; 701 } 702 rte_free(fwd_streams); 703 fwd_streams = NULL; 704 } 705 706 /* init new */ 707 nb_fwd_streams = nb_fwd_streams_new; 708 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 709 sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE); 710 if (fwd_streams == NULL) 711 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) " 712 "failed\n", nb_fwd_streams); 713 714 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 715 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream", 716 sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE); 717 if (fwd_streams[sm_id] == NULL) 718 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)" 719 " failed\n"); 720 } 721 722 return 0; 723 } 724 725 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 726 static void 727 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 728 { 729 unsigned int total_burst; 730 unsigned int nb_burst; 731 unsigned int burst_stats[3]; 732 uint16_t pktnb_stats[3]; 733 uint16_t nb_pkt; 734 int burst_percent[3]; 735 736 /* 737 * First compute the total number of packet bursts and the 738 * two highest numbers of bursts of the same number of packets. 739 */ 740 total_burst = 0; 741 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 742 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 743 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 744 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 745 if (nb_burst == 0) 746 continue; 747 total_burst += nb_burst; 748 if (nb_burst > burst_stats[0]) { 749 burst_stats[1] = burst_stats[0]; 750 pktnb_stats[1] = pktnb_stats[0]; 751 burst_stats[0] = nb_burst; 752 pktnb_stats[0] = nb_pkt; 753 } 754 } 755 if (total_burst == 0) 756 return; 757 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 758 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 759 burst_percent[0], (int) pktnb_stats[0]); 760 if (burst_stats[0] == total_burst) { 761 printf("]\n"); 762 return; 763 } 764 if (burst_stats[0] + burst_stats[1] == total_burst) { 765 printf(" + %d%% of %d pkts]\n", 766 100 - burst_percent[0], pktnb_stats[1]); 767 return; 768 } 769 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 770 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 771 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 772 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 773 return; 774 } 775 printf(" + %d%% of %d pkts + %d%% of others]\n", 776 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 777 } 778 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 779 780 static void 781 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats) 782 { 783 struct rte_port *port; 784 uint8_t i; 785 786 static const char *fwd_stats_border = "----------------------"; 787 788 port = &ports[port_id]; 789 printf("\n %s Forward statistics for port %-2d %s\n", 790 fwd_stats_border, port_id, fwd_stats_border); 791 792 if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) { 793 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 794 "%-"PRIu64"\n", 795 stats->ipackets, stats->imissed, 796 (uint64_t) (stats->ipackets + stats->imissed)); 797 798 if (cur_fwd_eng == &csum_fwd_engine) 799 printf(" Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n", 800 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 801 if ((stats->ierrors + stats->rx_nombuf) > 0) { 802 printf(" RX-error: %-"PRIu64"\n", stats->ierrors); 803 printf(" RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf); 804 } 805 806 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 807 "%-"PRIu64"\n", 808 stats->opackets, port->tx_dropped, 809 (uint64_t) (stats->opackets + port->tx_dropped)); 810 } 811 else { 812 printf(" RX-packets: %14"PRIu64" RX-dropped:%14"PRIu64" RX-total:" 813 "%14"PRIu64"\n", 814 stats->ipackets, stats->imissed, 815 (uint64_t) (stats->ipackets + stats->imissed)); 816 817 if (cur_fwd_eng == &csum_fwd_engine) 818 printf(" Bad-ipcsum:%14"PRIu64" Bad-l4csum:%14"PRIu64"\n", 819 port->rx_bad_ip_csum, port->rx_bad_l4_csum); 820 if ((stats->ierrors + stats->rx_nombuf) > 0) { 821 printf(" RX-error:%"PRIu64"\n", stats->ierrors); 822 printf(" RX-nombufs: %14"PRIu64"\n", 823 stats->rx_nombuf); 824 } 825 826 printf(" TX-packets: %14"PRIu64" TX-dropped:%14"PRIu64" TX-total:" 827 "%14"PRIu64"\n", 828 stats->opackets, port->tx_dropped, 829 (uint64_t) (stats->opackets + port->tx_dropped)); 830 } 831 832 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 833 if (port->rx_stream) 834 pkt_burst_stats_display("RX", 835 &port->rx_stream->rx_burst_stats); 836 if (port->tx_stream) 837 pkt_burst_stats_display("TX", 838 &port->tx_stream->tx_burst_stats); 839 #endif 840 841 if (port->rx_queue_stats_mapping_enabled) { 842 printf("\n"); 843 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 844 printf(" Stats reg %2d RX-packets:%14"PRIu64 845 " RX-errors:%14"PRIu64 846 " RX-bytes:%14"PRIu64"\n", 847 i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]); 848 } 849 printf("\n"); 850 } 851 if (port->tx_queue_stats_mapping_enabled) { 852 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 853 printf(" Stats reg %2d TX-packets:%14"PRIu64 854 " TX-bytes:%14"PRIu64"\n", 855 i, stats->q_opackets[i], stats->q_obytes[i]); 856 } 857 } 858 859 printf(" %s--------------------------------%s\n", 860 fwd_stats_border, fwd_stats_border); 861 } 862 863 static void 864 fwd_stream_stats_display(streamid_t stream_id) 865 { 866 struct fwd_stream *fs; 867 static const char *fwd_top_stats_border = "-------"; 868 869 fs = fwd_streams[stream_id]; 870 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 871 (fs->fwd_dropped == 0)) 872 return; 873 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 874 "TX Port=%2d/Queue=%2d %s\n", 875 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 876 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 877 printf(" RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u", 878 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 879 880 /* if checksum mode */ 881 if (cur_fwd_eng == &csum_fwd_engine) { 882 printf(" RX- bad IP checksum: %-14u Rx- bad L4 checksum: " 883 "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum); 884 } 885 886 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 887 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 888 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 889 #endif 890 } 891 892 static void 893 flush_fwd_rx_queues(void) 894 { 895 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 896 portid_t rxp; 897 portid_t port_id; 898 queueid_t rxq; 899 uint16_t nb_rx; 900 uint16_t i; 901 uint8_t j; 902 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 903 uint64_t timer_period; 904 905 /* convert to number of cycles */ 906 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 907 908 for (j = 0; j < 2; j++) { 909 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 910 for (rxq = 0; rxq < nb_rxq; rxq++) { 911 port_id = fwd_ports_ids[rxp]; 912 /** 913 * testpmd can stuck in the below do while loop 914 * if rte_eth_rx_burst() always returns nonzero 915 * packets. So timer is added to exit this loop 916 * after 1sec timer expiry. 917 */ 918 prev_tsc = rte_rdtsc(); 919 do { 920 nb_rx = rte_eth_rx_burst(port_id, rxq, 921 pkts_burst, MAX_PKT_BURST); 922 for (i = 0; i < nb_rx; i++) 923 rte_pktmbuf_free(pkts_burst[i]); 924 925 cur_tsc = rte_rdtsc(); 926 diff_tsc = cur_tsc - prev_tsc; 927 timer_tsc += diff_tsc; 928 } while ((nb_rx > 0) && 929 (timer_tsc < timer_period)); 930 timer_tsc = 0; 931 } 932 } 933 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 934 } 935 } 936 937 static void 938 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 939 { 940 struct fwd_stream **fsm; 941 streamid_t nb_fs; 942 streamid_t sm_id; 943 #ifdef RTE_LIBRTE_BITRATE 944 uint64_t tics_per_1sec; 945 uint64_t tics_datum; 946 uint64_t tics_current; 947 uint8_t idx_port, cnt_ports; 948 949 cnt_ports = rte_eth_dev_count(); 950 tics_datum = rte_rdtsc(); 951 tics_per_1sec = rte_get_timer_hz(); 952 #endif 953 fsm = &fwd_streams[fc->stream_idx]; 954 nb_fs = fc->stream_nb; 955 do { 956 for (sm_id = 0; sm_id < nb_fs; sm_id++) 957 (*pkt_fwd)(fsm[sm_id]); 958 #ifdef RTE_LIBRTE_BITRATE 959 if (bitrate_enabled != 0 && 960 bitrate_lcore_id == rte_lcore_id()) { 961 tics_current = rte_rdtsc(); 962 if (tics_current - tics_datum >= tics_per_1sec) { 963 /* Periodic bitrate calculation */ 964 for (idx_port = 0; 965 idx_port < cnt_ports; 966 idx_port++) 967 rte_stats_bitrate_calc(bitrate_data, 968 idx_port); 969 tics_datum = tics_current; 970 } 971 } 972 #endif 973 #ifdef RTE_LIBRTE_LATENCY_STATS 974 if (latencystats_lcore_id == rte_lcore_id()) 975 rte_latencystats_update(); 976 #endif 977 978 } while (! fc->stopped); 979 } 980 981 static int 982 start_pkt_forward_on_core(void *fwd_arg) 983 { 984 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 985 cur_fwd_config.fwd_eng->packet_fwd); 986 return 0; 987 } 988 989 /* 990 * Run the TXONLY packet forwarding engine to send a single burst of packets. 991 * Used to start communication flows in network loopback test configurations. 992 */ 993 static int 994 run_one_txonly_burst_on_core(void *fwd_arg) 995 { 996 struct fwd_lcore *fwd_lc; 997 struct fwd_lcore tmp_lcore; 998 999 fwd_lc = (struct fwd_lcore *) fwd_arg; 1000 tmp_lcore = *fwd_lc; 1001 tmp_lcore.stopped = 1; 1002 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1003 return 0; 1004 } 1005 1006 /* 1007 * Launch packet forwarding: 1008 * - Setup per-port forwarding context. 1009 * - launch logical cores with their forwarding configuration. 1010 */ 1011 static void 1012 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1013 { 1014 port_fwd_begin_t port_fwd_begin; 1015 unsigned int i; 1016 unsigned int lc_id; 1017 int diag; 1018 1019 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1020 if (port_fwd_begin != NULL) { 1021 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1022 (*port_fwd_begin)(fwd_ports_ids[i]); 1023 } 1024 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1025 lc_id = fwd_lcores_cpuids[i]; 1026 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1027 fwd_lcores[i]->stopped = 0; 1028 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1029 fwd_lcores[i], lc_id); 1030 if (diag != 0) 1031 printf("launch lcore %u failed - diag=%d\n", 1032 lc_id, diag); 1033 } 1034 } 1035 } 1036 1037 /* 1038 * Launch packet forwarding configuration. 1039 */ 1040 void 1041 start_packet_forwarding(int with_tx_first) 1042 { 1043 port_fwd_begin_t port_fwd_begin; 1044 port_fwd_end_t port_fwd_end; 1045 struct rte_port *port; 1046 unsigned int i; 1047 portid_t pt_id; 1048 streamid_t sm_id; 1049 1050 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1051 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1052 1053 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1054 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1055 1056 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1057 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1058 (!nb_rxq || !nb_txq)) 1059 rte_exit(EXIT_FAILURE, 1060 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1061 cur_fwd_eng->fwd_mode_name); 1062 1063 if (all_ports_started() == 0) { 1064 printf("Not all ports were started\n"); 1065 return; 1066 } 1067 if (test_done == 0) { 1068 printf("Packet forwarding already started\n"); 1069 return; 1070 } 1071 1072 if (init_fwd_streams() < 0) { 1073 printf("Fail from init_fwd_streams()\n"); 1074 return; 1075 } 1076 1077 if(dcb_test) { 1078 for (i = 0; i < nb_fwd_ports; i++) { 1079 pt_id = fwd_ports_ids[i]; 1080 port = &ports[pt_id]; 1081 if (!port->dcb_flag) { 1082 printf("In DCB mode, all forwarding ports must " 1083 "be configured in this mode.\n"); 1084 return; 1085 } 1086 } 1087 if (nb_fwd_lcores == 1) { 1088 printf("In DCB mode,the nb forwarding cores " 1089 "should be larger than 1.\n"); 1090 return; 1091 } 1092 } 1093 test_done = 0; 1094 1095 if(!no_flush_rx) 1096 flush_fwd_rx_queues(); 1097 1098 fwd_config_setup(); 1099 pkt_fwd_config_display(&cur_fwd_config); 1100 rxtx_config_display(); 1101 1102 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1103 pt_id = fwd_ports_ids[i]; 1104 port = &ports[pt_id]; 1105 rte_eth_stats_get(pt_id, &port->stats); 1106 port->tx_dropped = 0; 1107 1108 map_port_queue_stats_mapping_registers(pt_id, port); 1109 } 1110 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1111 fwd_streams[sm_id]->rx_packets = 0; 1112 fwd_streams[sm_id]->tx_packets = 0; 1113 fwd_streams[sm_id]->fwd_dropped = 0; 1114 fwd_streams[sm_id]->rx_bad_ip_csum = 0; 1115 fwd_streams[sm_id]->rx_bad_l4_csum = 0; 1116 1117 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1118 memset(&fwd_streams[sm_id]->rx_burst_stats, 0, 1119 sizeof(fwd_streams[sm_id]->rx_burst_stats)); 1120 memset(&fwd_streams[sm_id]->tx_burst_stats, 0, 1121 sizeof(fwd_streams[sm_id]->tx_burst_stats)); 1122 #endif 1123 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1124 fwd_streams[sm_id]->core_cycles = 0; 1125 #endif 1126 } 1127 if (with_tx_first) { 1128 port_fwd_begin = tx_only_engine.port_fwd_begin; 1129 if (port_fwd_begin != NULL) { 1130 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1131 (*port_fwd_begin)(fwd_ports_ids[i]); 1132 } 1133 while (with_tx_first--) { 1134 launch_packet_forwarding( 1135 run_one_txonly_burst_on_core); 1136 rte_eal_mp_wait_lcore(); 1137 } 1138 port_fwd_end = tx_only_engine.port_fwd_end; 1139 if (port_fwd_end != NULL) { 1140 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1141 (*port_fwd_end)(fwd_ports_ids[i]); 1142 } 1143 } 1144 launch_packet_forwarding(start_pkt_forward_on_core); 1145 } 1146 1147 void 1148 stop_packet_forwarding(void) 1149 { 1150 struct rte_eth_stats stats; 1151 struct rte_port *port; 1152 port_fwd_end_t port_fwd_end; 1153 int i; 1154 portid_t pt_id; 1155 streamid_t sm_id; 1156 lcoreid_t lc_id; 1157 uint64_t total_recv; 1158 uint64_t total_xmit; 1159 uint64_t total_rx_dropped; 1160 uint64_t total_tx_dropped; 1161 uint64_t total_rx_nombuf; 1162 uint64_t tx_dropped; 1163 uint64_t rx_bad_ip_csum; 1164 uint64_t rx_bad_l4_csum; 1165 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1166 uint64_t fwd_cycles; 1167 #endif 1168 static const char *acc_stats_border = "+++++++++++++++"; 1169 1170 if (test_done) { 1171 printf("Packet forwarding not started\n"); 1172 return; 1173 } 1174 printf("Telling cores to stop..."); 1175 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1176 fwd_lcores[lc_id]->stopped = 1; 1177 printf("\nWaiting for lcores to finish...\n"); 1178 rte_eal_mp_wait_lcore(); 1179 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1180 if (port_fwd_end != NULL) { 1181 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1182 pt_id = fwd_ports_ids[i]; 1183 (*port_fwd_end)(pt_id); 1184 } 1185 } 1186 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1187 fwd_cycles = 0; 1188 #endif 1189 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1190 if (cur_fwd_config.nb_fwd_streams > 1191 cur_fwd_config.nb_fwd_ports) { 1192 fwd_stream_stats_display(sm_id); 1193 ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL; 1194 ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL; 1195 } else { 1196 ports[fwd_streams[sm_id]->tx_port].tx_stream = 1197 fwd_streams[sm_id]; 1198 ports[fwd_streams[sm_id]->rx_port].rx_stream = 1199 fwd_streams[sm_id]; 1200 } 1201 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped; 1202 tx_dropped = (uint64_t) (tx_dropped + 1203 fwd_streams[sm_id]->fwd_dropped); 1204 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped; 1205 1206 rx_bad_ip_csum = 1207 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum; 1208 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum + 1209 fwd_streams[sm_id]->rx_bad_ip_csum); 1210 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum = 1211 rx_bad_ip_csum; 1212 1213 rx_bad_l4_csum = 1214 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum; 1215 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum + 1216 fwd_streams[sm_id]->rx_bad_l4_csum); 1217 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum = 1218 rx_bad_l4_csum; 1219 1220 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1221 fwd_cycles = (uint64_t) (fwd_cycles + 1222 fwd_streams[sm_id]->core_cycles); 1223 #endif 1224 } 1225 total_recv = 0; 1226 total_xmit = 0; 1227 total_rx_dropped = 0; 1228 total_tx_dropped = 0; 1229 total_rx_nombuf = 0; 1230 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1231 pt_id = fwd_ports_ids[i]; 1232 1233 port = &ports[pt_id]; 1234 rte_eth_stats_get(pt_id, &stats); 1235 stats.ipackets -= port->stats.ipackets; 1236 port->stats.ipackets = 0; 1237 stats.opackets -= port->stats.opackets; 1238 port->stats.opackets = 0; 1239 stats.ibytes -= port->stats.ibytes; 1240 port->stats.ibytes = 0; 1241 stats.obytes -= port->stats.obytes; 1242 port->stats.obytes = 0; 1243 stats.imissed -= port->stats.imissed; 1244 port->stats.imissed = 0; 1245 stats.oerrors -= port->stats.oerrors; 1246 port->stats.oerrors = 0; 1247 stats.rx_nombuf -= port->stats.rx_nombuf; 1248 port->stats.rx_nombuf = 0; 1249 1250 total_recv += stats.ipackets; 1251 total_xmit += stats.opackets; 1252 total_rx_dropped += stats.imissed; 1253 total_tx_dropped += port->tx_dropped; 1254 total_rx_nombuf += stats.rx_nombuf; 1255 1256 fwd_port_stats_display(pt_id, &stats); 1257 } 1258 printf("\n %s Accumulated forward statistics for all ports" 1259 "%s\n", 1260 acc_stats_border, acc_stats_border); 1261 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1262 "%-"PRIu64"\n" 1263 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1264 "%-"PRIu64"\n", 1265 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1266 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1267 if (total_rx_nombuf > 0) 1268 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1269 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1270 "%s\n", 1271 acc_stats_border, acc_stats_border); 1272 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1273 if (total_recv > 0) 1274 printf("\n CPU cycles/packet=%u (total cycles=" 1275 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1276 (unsigned int)(fwd_cycles / total_recv), 1277 fwd_cycles, total_recv); 1278 #endif 1279 printf("\nDone.\n"); 1280 test_done = 1; 1281 } 1282 1283 void 1284 dev_set_link_up(portid_t pid) 1285 { 1286 if (rte_eth_dev_set_link_up((uint8_t)pid) < 0) 1287 printf("\nSet link up fail.\n"); 1288 } 1289 1290 void 1291 dev_set_link_down(portid_t pid) 1292 { 1293 if (rte_eth_dev_set_link_down((uint8_t)pid) < 0) 1294 printf("\nSet link down fail.\n"); 1295 } 1296 1297 static int 1298 all_ports_started(void) 1299 { 1300 portid_t pi; 1301 struct rte_port *port; 1302 1303 RTE_ETH_FOREACH_DEV(pi) { 1304 port = &ports[pi]; 1305 /* Check if there is a port which is not started */ 1306 if ((port->port_status != RTE_PORT_STARTED) && 1307 (port->slave_flag == 0)) 1308 return 0; 1309 } 1310 1311 /* No port is not started */ 1312 return 1; 1313 } 1314 1315 int 1316 all_ports_stopped(void) 1317 { 1318 portid_t pi; 1319 struct rte_port *port; 1320 1321 RTE_ETH_FOREACH_DEV(pi) { 1322 port = &ports[pi]; 1323 if ((port->port_status != RTE_PORT_STOPPED) && 1324 (port->slave_flag == 0)) 1325 return 0; 1326 } 1327 1328 return 1; 1329 } 1330 1331 int 1332 port_is_started(portid_t port_id) 1333 { 1334 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1335 return 0; 1336 1337 if (ports[port_id].port_status != RTE_PORT_STARTED) 1338 return 0; 1339 1340 return 1; 1341 } 1342 1343 static int 1344 port_is_closed(portid_t port_id) 1345 { 1346 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1347 return 0; 1348 1349 if (ports[port_id].port_status != RTE_PORT_CLOSED) 1350 return 0; 1351 1352 return 1; 1353 } 1354 1355 int 1356 start_port(portid_t pid) 1357 { 1358 int diag, need_check_link_status = -1; 1359 portid_t pi; 1360 queueid_t qi; 1361 struct rte_port *port; 1362 struct ether_addr mac_addr; 1363 enum rte_eth_event_type event_type; 1364 1365 if (port_id_is_invalid(pid, ENABLED_WARN)) 1366 return 0; 1367 1368 if(dcb_config) 1369 dcb_test = 1; 1370 RTE_ETH_FOREACH_DEV(pi) { 1371 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1372 continue; 1373 1374 need_check_link_status = 0; 1375 port = &ports[pi]; 1376 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1377 RTE_PORT_HANDLING) == 0) { 1378 printf("Port %d is now not stopped\n", pi); 1379 continue; 1380 } 1381 1382 if (port->need_reconfig > 0) { 1383 port->need_reconfig = 0; 1384 1385 printf("Configuring Port %d (socket %u)\n", pi, 1386 port->socket_id); 1387 /* configure port */ 1388 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1389 &(port->dev_conf)); 1390 if (diag != 0) { 1391 if (rte_atomic16_cmpset(&(port->port_status), 1392 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1393 printf("Port %d can not be set back " 1394 "to stopped\n", pi); 1395 printf("Fail to configure port %d\n", pi); 1396 /* try to reconfigure port next time */ 1397 port->need_reconfig = 1; 1398 return -1; 1399 } 1400 } 1401 if (port->need_reconfig_queues > 0) { 1402 port->need_reconfig_queues = 0; 1403 /* setup tx queues */ 1404 for (qi = 0; qi < nb_txq; qi++) { 1405 if ((numa_support) && 1406 (txring_numa[pi] != NUMA_NO_CONFIG)) 1407 diag = rte_eth_tx_queue_setup(pi, qi, 1408 nb_txd,txring_numa[pi], 1409 &(port->tx_conf)); 1410 else 1411 diag = rte_eth_tx_queue_setup(pi, qi, 1412 nb_txd,port->socket_id, 1413 &(port->tx_conf)); 1414 1415 if (diag == 0) 1416 continue; 1417 1418 /* Fail to setup tx queue, return */ 1419 if (rte_atomic16_cmpset(&(port->port_status), 1420 RTE_PORT_HANDLING, 1421 RTE_PORT_STOPPED) == 0) 1422 printf("Port %d can not be set back " 1423 "to stopped\n", pi); 1424 printf("Fail to configure port %d tx queues\n", pi); 1425 /* try to reconfigure queues next time */ 1426 port->need_reconfig_queues = 1; 1427 return -1; 1428 } 1429 /* setup rx queues */ 1430 for (qi = 0; qi < nb_rxq; qi++) { 1431 if ((numa_support) && 1432 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 1433 struct rte_mempool * mp = 1434 mbuf_pool_find(rxring_numa[pi]); 1435 if (mp == NULL) { 1436 printf("Failed to setup RX queue:" 1437 "No mempool allocation" 1438 " on the socket %d\n", 1439 rxring_numa[pi]); 1440 return -1; 1441 } 1442 1443 diag = rte_eth_rx_queue_setup(pi, qi, 1444 nb_rxd,rxring_numa[pi], 1445 &(port->rx_conf),mp); 1446 } else { 1447 struct rte_mempool *mp = 1448 mbuf_pool_find(port->socket_id); 1449 if (mp == NULL) { 1450 printf("Failed to setup RX queue:" 1451 "No mempool allocation" 1452 " on the socket %d\n", 1453 port->socket_id); 1454 return -1; 1455 } 1456 diag = rte_eth_rx_queue_setup(pi, qi, 1457 nb_rxd,port->socket_id, 1458 &(port->rx_conf), mp); 1459 } 1460 if (diag == 0) 1461 continue; 1462 1463 /* Fail to setup rx queue, return */ 1464 if (rte_atomic16_cmpset(&(port->port_status), 1465 RTE_PORT_HANDLING, 1466 RTE_PORT_STOPPED) == 0) 1467 printf("Port %d can not be set back " 1468 "to stopped\n", pi); 1469 printf("Fail to configure port %d rx queues\n", pi); 1470 /* try to reconfigure queues next time */ 1471 port->need_reconfig_queues = 1; 1472 return -1; 1473 } 1474 } 1475 1476 for (event_type = RTE_ETH_EVENT_UNKNOWN; 1477 event_type < RTE_ETH_EVENT_MAX; 1478 event_type++) { 1479 diag = rte_eth_dev_callback_register(pi, 1480 event_type, 1481 eth_event_callback, 1482 NULL); 1483 if (diag) { 1484 printf("Failed to setup even callback for event %d\n", 1485 event_type); 1486 return -1; 1487 } 1488 } 1489 1490 /* start port */ 1491 if (rte_eth_dev_start(pi) < 0) { 1492 printf("Fail to start port %d\n", pi); 1493 1494 /* Fail to setup rx queue, return */ 1495 if (rte_atomic16_cmpset(&(port->port_status), 1496 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1497 printf("Port %d can not be set back to " 1498 "stopped\n", pi); 1499 continue; 1500 } 1501 1502 if (rte_atomic16_cmpset(&(port->port_status), 1503 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 1504 printf("Port %d can not be set into started\n", pi); 1505 1506 rte_eth_macaddr_get(pi, &mac_addr); 1507 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 1508 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 1509 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 1510 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 1511 1512 /* at least one port started, need checking link status */ 1513 need_check_link_status = 1; 1514 } 1515 1516 if (need_check_link_status == 1 && !no_link_check) 1517 check_all_ports_link_status(RTE_PORT_ALL); 1518 else if (need_check_link_status == 0) 1519 printf("Please stop the ports first\n"); 1520 1521 printf("Done\n"); 1522 return 0; 1523 } 1524 1525 void 1526 stop_port(portid_t pid) 1527 { 1528 portid_t pi; 1529 struct rte_port *port; 1530 int need_check_link_status = 0; 1531 1532 if (dcb_test) { 1533 dcb_test = 0; 1534 dcb_config = 0; 1535 } 1536 1537 if (port_id_is_invalid(pid, ENABLED_WARN)) 1538 return; 1539 1540 printf("Stopping ports...\n"); 1541 1542 RTE_ETH_FOREACH_DEV(pi) { 1543 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1544 continue; 1545 1546 if (port_is_forwarding(pi) != 0 && test_done == 0) { 1547 printf("Please remove port %d from forwarding configuration.\n", pi); 1548 continue; 1549 } 1550 1551 if (port_is_bonding_slave(pi)) { 1552 printf("Please remove port %d from bonded device.\n", pi); 1553 continue; 1554 } 1555 1556 port = &ports[pi]; 1557 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 1558 RTE_PORT_HANDLING) == 0) 1559 continue; 1560 1561 rte_eth_dev_stop(pi); 1562 1563 if (rte_atomic16_cmpset(&(port->port_status), 1564 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1565 printf("Port %d can not be set into stopped\n", pi); 1566 need_check_link_status = 1; 1567 } 1568 if (need_check_link_status && !no_link_check) 1569 check_all_ports_link_status(RTE_PORT_ALL); 1570 1571 printf("Done\n"); 1572 } 1573 1574 void 1575 close_port(portid_t pid) 1576 { 1577 portid_t pi; 1578 struct rte_port *port; 1579 1580 if (port_id_is_invalid(pid, ENABLED_WARN)) 1581 return; 1582 1583 printf("Closing ports...\n"); 1584 1585 RTE_ETH_FOREACH_DEV(pi) { 1586 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1587 continue; 1588 1589 if (port_is_forwarding(pi) != 0 && test_done == 0) { 1590 printf("Please remove port %d from forwarding configuration.\n", pi); 1591 continue; 1592 } 1593 1594 if (port_is_bonding_slave(pi)) { 1595 printf("Please remove port %d from bonded device.\n", pi); 1596 continue; 1597 } 1598 1599 port = &ports[pi]; 1600 if (rte_atomic16_cmpset(&(port->port_status), 1601 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 1602 printf("Port %d is already closed\n", pi); 1603 continue; 1604 } 1605 1606 if (rte_atomic16_cmpset(&(port->port_status), 1607 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 1608 printf("Port %d is now not stopped\n", pi); 1609 continue; 1610 } 1611 1612 if (port->flow_list) 1613 port_flow_flush(pi); 1614 rte_eth_dev_close(pi); 1615 1616 if (rte_atomic16_cmpset(&(port->port_status), 1617 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 1618 printf("Port %d cannot be set to closed\n", pi); 1619 } 1620 1621 printf("Done\n"); 1622 } 1623 1624 void 1625 attach_port(char *identifier) 1626 { 1627 portid_t pi = 0; 1628 unsigned int socket_id; 1629 1630 printf("Attaching a new port...\n"); 1631 1632 if (identifier == NULL) { 1633 printf("Invalid parameters are specified\n"); 1634 return; 1635 } 1636 1637 if (rte_eth_dev_attach(identifier, &pi)) 1638 return; 1639 1640 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 1641 /* if socket_id is invalid, set to 0 */ 1642 if (check_socket_id(socket_id) < 0) 1643 socket_id = 0; 1644 reconfig(pi, socket_id); 1645 rte_eth_promiscuous_enable(pi); 1646 1647 nb_ports = rte_eth_dev_count(); 1648 1649 ports[pi].port_status = RTE_PORT_STOPPED; 1650 1651 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 1652 printf("Done\n"); 1653 } 1654 1655 void 1656 detach_port(uint8_t port_id) 1657 { 1658 char name[RTE_ETH_NAME_MAX_LEN]; 1659 1660 printf("Detaching a port...\n"); 1661 1662 if (!port_is_closed(port_id)) { 1663 printf("Please close port first\n"); 1664 return; 1665 } 1666 1667 if (ports[port_id].flow_list) 1668 port_flow_flush(port_id); 1669 1670 if (rte_eth_dev_detach(port_id, name)) 1671 return; 1672 1673 nb_ports = rte_eth_dev_count(); 1674 1675 printf("Port '%s' is detached. Now total ports is %d\n", 1676 name, nb_ports); 1677 printf("Done\n"); 1678 return; 1679 } 1680 1681 void 1682 pmd_test_exit(void) 1683 { 1684 portid_t pt_id; 1685 1686 if (test_done == 0) 1687 stop_packet_forwarding(); 1688 1689 if (ports != NULL) { 1690 no_link_check = 1; 1691 RTE_ETH_FOREACH_DEV(pt_id) { 1692 printf("\nShutting down port %d...\n", pt_id); 1693 fflush(stdout); 1694 stop_port(pt_id); 1695 close_port(pt_id); 1696 } 1697 } 1698 printf("\nBye...\n"); 1699 } 1700 1701 typedef void (*cmd_func_t)(void); 1702 struct pmd_test_command { 1703 const char *cmd_name; 1704 cmd_func_t cmd_func; 1705 }; 1706 1707 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 1708 1709 /* Check the link status of all ports in up to 9s, and print them finally */ 1710 static void 1711 check_all_ports_link_status(uint32_t port_mask) 1712 { 1713 #define CHECK_INTERVAL 100 /* 100ms */ 1714 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 1715 uint8_t portid, count, all_ports_up, print_flag = 0; 1716 struct rte_eth_link link; 1717 1718 printf("Checking link statuses...\n"); 1719 fflush(stdout); 1720 for (count = 0; count <= MAX_CHECK_TIME; count++) { 1721 all_ports_up = 1; 1722 RTE_ETH_FOREACH_DEV(portid) { 1723 if ((port_mask & (1 << portid)) == 0) 1724 continue; 1725 memset(&link, 0, sizeof(link)); 1726 rte_eth_link_get_nowait(portid, &link); 1727 /* print link status if flag set */ 1728 if (print_flag == 1) { 1729 if (link.link_status) 1730 printf("Port %d Link Up - speed %u " 1731 "Mbps - %s\n", (uint8_t)portid, 1732 (unsigned)link.link_speed, 1733 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 1734 ("full-duplex") : ("half-duplex\n")); 1735 else 1736 printf("Port %d Link Down\n", 1737 (uint8_t)portid); 1738 continue; 1739 } 1740 /* clear all_ports_up flag if any link down */ 1741 if (link.link_status == ETH_LINK_DOWN) { 1742 all_ports_up = 0; 1743 break; 1744 } 1745 } 1746 /* after finally printing all link status, get out */ 1747 if (print_flag == 1) 1748 break; 1749 1750 if (all_ports_up == 0) { 1751 fflush(stdout); 1752 rte_delay_ms(CHECK_INTERVAL); 1753 } 1754 1755 /* set the print_flag if all ports up or timeout */ 1756 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 1757 print_flag = 1; 1758 } 1759 1760 if (lsc_interrupt) 1761 break; 1762 } 1763 } 1764 1765 static void 1766 rmv_event_callback(void *arg) 1767 { 1768 struct rte_eth_dev *dev; 1769 struct rte_devargs *da; 1770 char name[32] = ""; 1771 uint8_t port_id = (intptr_t)arg; 1772 1773 RTE_ETH_VALID_PORTID_OR_RET(port_id); 1774 dev = &rte_eth_devices[port_id]; 1775 da = dev->device->devargs; 1776 1777 stop_port(port_id); 1778 close_port(port_id); 1779 if (da->type == RTE_DEVTYPE_VIRTUAL) 1780 snprintf(name, sizeof(name), "%s", da->virt.drv_name); 1781 else if (da->type == RTE_DEVTYPE_WHITELISTED_PCI) 1782 rte_eal_pci_device_name(&da->pci.addr, name, sizeof(name)); 1783 printf("removing device %s\n", name); 1784 rte_eal_dev_detach(name); 1785 dev->state = RTE_ETH_DEV_UNUSED; 1786 } 1787 1788 /* This function is used by the interrupt thread */ 1789 static void 1790 eth_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param) 1791 { 1792 static const char * const event_desc[] = { 1793 [RTE_ETH_EVENT_UNKNOWN] = "Unknown", 1794 [RTE_ETH_EVENT_INTR_LSC] = "LSC", 1795 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state", 1796 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset", 1797 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox", 1798 [RTE_ETH_EVENT_MACSEC] = "MACsec", 1799 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 1800 [RTE_ETH_EVENT_MAX] = NULL, 1801 }; 1802 1803 RTE_SET_USED(param); 1804 1805 if (type >= RTE_ETH_EVENT_MAX) { 1806 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n", 1807 port_id, __func__, type); 1808 fflush(stderr); 1809 } else { 1810 printf("\nPort %" PRIu8 ": %s event\n", port_id, 1811 event_desc[type]); 1812 fflush(stdout); 1813 } 1814 1815 switch (type) { 1816 case RTE_ETH_EVENT_INTR_RMV: 1817 if (rte_eal_alarm_set(100000, 1818 rmv_event_callback, (void *)(intptr_t)port_id)) 1819 fprintf(stderr, "Could not set up deferred device removal\n"); 1820 break; 1821 default: 1822 break; 1823 } 1824 } 1825 1826 static int 1827 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) 1828 { 1829 uint16_t i; 1830 int diag; 1831 uint8_t mapping_found = 0; 1832 1833 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 1834 if ((tx_queue_stats_mappings[i].port_id == port_id) && 1835 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 1836 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 1837 tx_queue_stats_mappings[i].queue_id, 1838 tx_queue_stats_mappings[i].stats_counter_id); 1839 if (diag != 0) 1840 return diag; 1841 mapping_found = 1; 1842 } 1843 } 1844 if (mapping_found) 1845 port->tx_queue_stats_mapping_enabled = 1; 1846 return 0; 1847 } 1848 1849 static int 1850 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) 1851 { 1852 uint16_t i; 1853 int diag; 1854 uint8_t mapping_found = 0; 1855 1856 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 1857 if ((rx_queue_stats_mappings[i].port_id == port_id) && 1858 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 1859 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 1860 rx_queue_stats_mappings[i].queue_id, 1861 rx_queue_stats_mappings[i].stats_counter_id); 1862 if (diag != 0) 1863 return diag; 1864 mapping_found = 1; 1865 } 1866 } 1867 if (mapping_found) 1868 port->rx_queue_stats_mapping_enabled = 1; 1869 return 0; 1870 } 1871 1872 static void 1873 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port) 1874 { 1875 int diag = 0; 1876 1877 diag = set_tx_queue_stats_mapping_registers(pi, port); 1878 if (diag != 0) { 1879 if (diag == -ENOTSUP) { 1880 port->tx_queue_stats_mapping_enabled = 0; 1881 printf("TX queue stats mapping not supported port id=%d\n", pi); 1882 } 1883 else 1884 rte_exit(EXIT_FAILURE, 1885 "set_tx_queue_stats_mapping_registers " 1886 "failed for port id=%d diag=%d\n", 1887 pi, diag); 1888 } 1889 1890 diag = set_rx_queue_stats_mapping_registers(pi, port); 1891 if (diag != 0) { 1892 if (diag == -ENOTSUP) { 1893 port->rx_queue_stats_mapping_enabled = 0; 1894 printf("RX queue stats mapping not supported port id=%d\n", pi); 1895 } 1896 else 1897 rte_exit(EXIT_FAILURE, 1898 "set_rx_queue_stats_mapping_registers " 1899 "failed for port id=%d diag=%d\n", 1900 pi, diag); 1901 } 1902 } 1903 1904 static void 1905 rxtx_port_config(struct rte_port *port) 1906 { 1907 port->rx_conf = port->dev_info.default_rxconf; 1908 port->tx_conf = port->dev_info.default_txconf; 1909 1910 /* Check if any RX/TX parameters have been passed */ 1911 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 1912 port->rx_conf.rx_thresh.pthresh = rx_pthresh; 1913 1914 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 1915 port->rx_conf.rx_thresh.hthresh = rx_hthresh; 1916 1917 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 1918 port->rx_conf.rx_thresh.wthresh = rx_wthresh; 1919 1920 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 1921 port->rx_conf.rx_free_thresh = rx_free_thresh; 1922 1923 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 1924 port->rx_conf.rx_drop_en = rx_drop_en; 1925 1926 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 1927 port->tx_conf.tx_thresh.pthresh = tx_pthresh; 1928 1929 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 1930 port->tx_conf.tx_thresh.hthresh = tx_hthresh; 1931 1932 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 1933 port->tx_conf.tx_thresh.wthresh = tx_wthresh; 1934 1935 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 1936 port->tx_conf.tx_rs_thresh = tx_rs_thresh; 1937 1938 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 1939 port->tx_conf.tx_free_thresh = tx_free_thresh; 1940 1941 if (txq_flags != RTE_PMD_PARAM_UNSET) 1942 port->tx_conf.txq_flags = txq_flags; 1943 } 1944 1945 void 1946 init_port_config(void) 1947 { 1948 portid_t pid; 1949 struct rte_port *port; 1950 1951 RTE_ETH_FOREACH_DEV(pid) { 1952 port = &ports[pid]; 1953 port->dev_conf.rxmode = rx_mode; 1954 port->dev_conf.fdir_conf = fdir_conf; 1955 if (nb_rxq > 1) { 1956 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 1957 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf; 1958 } else { 1959 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 1960 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 1961 } 1962 1963 if (port->dcb_flag == 0) { 1964 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 1965 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 1966 else 1967 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 1968 } 1969 1970 rxtx_port_config(port); 1971 1972 rte_eth_macaddr_get(pid, &port->eth_addr); 1973 1974 map_port_queue_stats_mapping_registers(pid, port); 1975 #ifdef RTE_NIC_BYPASS 1976 rte_eth_dev_bypass_init(pid); 1977 #endif 1978 1979 if (lsc_interrupt && 1980 (rte_eth_devices[pid].data->dev_flags & 1981 RTE_ETH_DEV_INTR_LSC)) 1982 port->dev_conf.intr_conf.lsc = 1; 1983 if (rmv_interrupt && 1984 (rte_eth_devices[pid].data->dev_flags & 1985 RTE_ETH_DEV_INTR_RMV)) 1986 port->dev_conf.intr_conf.rmv = 1; 1987 } 1988 } 1989 1990 void set_port_slave_flag(portid_t slave_pid) 1991 { 1992 struct rte_port *port; 1993 1994 port = &ports[slave_pid]; 1995 port->slave_flag = 1; 1996 } 1997 1998 void clear_port_slave_flag(portid_t slave_pid) 1999 { 2000 struct rte_port *port; 2001 2002 port = &ports[slave_pid]; 2003 port->slave_flag = 0; 2004 } 2005 2006 uint8_t port_is_bonding_slave(portid_t slave_pid) 2007 { 2008 struct rte_port *port; 2009 2010 port = &ports[slave_pid]; 2011 return port->slave_flag; 2012 } 2013 2014 const uint16_t vlan_tags[] = { 2015 0, 1, 2, 3, 4, 5, 6, 7, 2016 8, 9, 10, 11, 12, 13, 14, 15, 2017 16, 17, 18, 19, 20, 21, 22, 23, 2018 24, 25, 26, 27, 28, 29, 30, 31 2019 }; 2020 2021 static int 2022 get_eth_dcb_conf(struct rte_eth_conf *eth_conf, 2023 enum dcb_mode_enable dcb_mode, 2024 enum rte_eth_nb_tcs num_tcs, 2025 uint8_t pfc_en) 2026 { 2027 uint8_t i; 2028 2029 /* 2030 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2031 * given above, and the number of traffic classes available for use. 2032 */ 2033 if (dcb_mode == DCB_VT_ENABLED) { 2034 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2035 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2036 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2037 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2038 2039 /* VMDQ+DCB RX and TX configurations */ 2040 vmdq_rx_conf->enable_default_pool = 0; 2041 vmdq_rx_conf->default_pool = 0; 2042 vmdq_rx_conf->nb_queue_pools = 2043 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2044 vmdq_tx_conf->nb_queue_pools = 2045 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2046 2047 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2048 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2049 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2050 vmdq_rx_conf->pool_map[i].pools = 2051 1 << (i % vmdq_rx_conf->nb_queue_pools); 2052 } 2053 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2054 vmdq_rx_conf->dcb_tc[i] = i; 2055 vmdq_tx_conf->dcb_tc[i] = i; 2056 } 2057 2058 /* set DCB mode of RX and TX of multiple queues */ 2059 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2060 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2061 } else { 2062 struct rte_eth_dcb_rx_conf *rx_conf = 2063 ð_conf->rx_adv_conf.dcb_rx_conf; 2064 struct rte_eth_dcb_tx_conf *tx_conf = 2065 ð_conf->tx_adv_conf.dcb_tx_conf; 2066 2067 rx_conf->nb_tcs = num_tcs; 2068 tx_conf->nb_tcs = num_tcs; 2069 2070 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2071 rx_conf->dcb_tc[i] = i % num_tcs; 2072 tx_conf->dcb_tc[i] = i % num_tcs; 2073 } 2074 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2075 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf; 2076 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2077 } 2078 2079 if (pfc_en) 2080 eth_conf->dcb_capability_en = 2081 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2082 else 2083 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2084 2085 return 0; 2086 } 2087 2088 int 2089 init_port_dcb_config(portid_t pid, 2090 enum dcb_mode_enable dcb_mode, 2091 enum rte_eth_nb_tcs num_tcs, 2092 uint8_t pfc_en) 2093 { 2094 struct rte_eth_conf port_conf; 2095 struct rte_port *rte_port; 2096 int retval; 2097 uint16_t i; 2098 2099 rte_port = &ports[pid]; 2100 2101 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2102 /* Enter DCB configuration status */ 2103 dcb_config = 1; 2104 2105 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2106 retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en); 2107 if (retval < 0) 2108 return retval; 2109 port_conf.rxmode.hw_vlan_filter = 1; 2110 2111 /** 2112 * Write the configuration into the device. 2113 * Set the numbers of RX & TX queues to 0, so 2114 * the RX & TX queues will not be setup. 2115 */ 2116 (void)rte_eth_dev_configure(pid, 0, 0, &port_conf); 2117 2118 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2119 2120 /* If dev_info.vmdq_pool_base is greater than 0, 2121 * the queue id of vmdq pools is started after pf queues. 2122 */ 2123 if (dcb_mode == DCB_VT_ENABLED && 2124 rte_port->dev_info.vmdq_pool_base > 0) { 2125 printf("VMDQ_DCB multi-queue mode is nonsensical" 2126 " for port %d.", pid); 2127 return -1; 2128 } 2129 2130 /* Assume the ports in testpmd have the same dcb capability 2131 * and has the same number of rxq and txq in dcb mode 2132 */ 2133 if (dcb_mode == DCB_VT_ENABLED) { 2134 if (rte_port->dev_info.max_vfs > 0) { 2135 nb_rxq = rte_port->dev_info.nb_rx_queues; 2136 nb_txq = rte_port->dev_info.nb_tx_queues; 2137 } else { 2138 nb_rxq = rte_port->dev_info.max_rx_queues; 2139 nb_txq = rte_port->dev_info.max_tx_queues; 2140 } 2141 } else { 2142 /*if vt is disabled, use all pf queues */ 2143 if (rte_port->dev_info.vmdq_pool_base == 0) { 2144 nb_rxq = rte_port->dev_info.max_rx_queues; 2145 nb_txq = rte_port->dev_info.max_tx_queues; 2146 } else { 2147 nb_rxq = (queueid_t)num_tcs; 2148 nb_txq = (queueid_t)num_tcs; 2149 2150 } 2151 } 2152 rx_free_thresh = 64; 2153 2154 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 2155 2156 rxtx_port_config(rte_port); 2157 /* VLAN filter */ 2158 rte_port->dev_conf.rxmode.hw_vlan_filter = 1; 2159 for (i = 0; i < RTE_DIM(vlan_tags); i++) 2160 rx_vft_set(pid, vlan_tags[i], 1); 2161 2162 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 2163 map_port_queue_stats_mapping_registers(pid, rte_port); 2164 2165 rte_port->dcb_flag = 1; 2166 2167 return 0; 2168 } 2169 2170 static void 2171 init_port(void) 2172 { 2173 /* Configuration of Ethernet ports. */ 2174 ports = rte_zmalloc("testpmd: ports", 2175 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 2176 RTE_CACHE_LINE_SIZE); 2177 if (ports == NULL) { 2178 rte_exit(EXIT_FAILURE, 2179 "rte_zmalloc(%d struct rte_port) failed\n", 2180 RTE_MAX_ETHPORTS); 2181 } 2182 } 2183 2184 static void 2185 force_quit(void) 2186 { 2187 pmd_test_exit(); 2188 prompt_exit(); 2189 } 2190 2191 static void 2192 signal_handler(int signum) 2193 { 2194 if (signum == SIGINT || signum == SIGTERM) { 2195 printf("\nSignal %d received, preparing to exit...\n", 2196 signum); 2197 #ifdef RTE_LIBRTE_PDUMP 2198 /* uninitialize packet capture framework */ 2199 rte_pdump_uninit(); 2200 #endif 2201 #ifdef RTE_LIBRTE_LATENCY_STATS 2202 rte_latencystats_uninit(); 2203 #endif 2204 force_quit(); 2205 /* exit with the expected status */ 2206 signal(signum, SIG_DFL); 2207 kill(getpid(), signum); 2208 } 2209 } 2210 2211 int 2212 main(int argc, char** argv) 2213 { 2214 int diag; 2215 uint8_t port_id; 2216 2217 signal(SIGINT, signal_handler); 2218 signal(SIGTERM, signal_handler); 2219 2220 diag = rte_eal_init(argc, argv); 2221 if (diag < 0) 2222 rte_panic("Cannot init EAL\n"); 2223 2224 #ifdef RTE_LIBRTE_PDUMP 2225 /* initialize packet capture framework */ 2226 rte_pdump_init(NULL); 2227 #endif 2228 2229 nb_ports = (portid_t) rte_eth_dev_count(); 2230 if (nb_ports == 0) 2231 RTE_LOG(WARNING, EAL, "No probed ethernet devices\n"); 2232 2233 /* allocate port structures, and init them */ 2234 init_port(); 2235 2236 set_def_fwd_config(); 2237 if (nb_lcores == 0) 2238 rte_panic("Empty set of forwarding logical cores - check the " 2239 "core mask supplied in the command parameters\n"); 2240 2241 /* Bitrate stats disabled by default */ 2242 bitrate_enabled = 0; 2243 2244 argc -= diag; 2245 argv += diag; 2246 if (argc > 1) 2247 launch_args_parse(argc, argv); 2248 2249 if (!nb_rxq && !nb_txq) 2250 printf("Warning: Either rx or tx queues should be non-zero\n"); 2251 2252 if (nb_rxq > 1 && nb_rxq > nb_txq) 2253 printf("Warning: nb_rxq=%d enables RSS configuration, " 2254 "but nb_txq=%d will prevent to fully test it.\n", 2255 nb_rxq, nb_txq); 2256 2257 init_config(); 2258 if (start_port(RTE_PORT_ALL) != 0) 2259 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 2260 2261 /* set all ports to promiscuous mode by default */ 2262 RTE_ETH_FOREACH_DEV(port_id) 2263 rte_eth_promiscuous_enable(port_id); 2264 2265 /* Init metrics library */ 2266 rte_metrics_init(rte_socket_id()); 2267 2268 #ifdef RTE_LIBRTE_LATENCY_STATS 2269 if (latencystats_enabled != 0) { 2270 int ret = rte_latencystats_init(1, NULL); 2271 if (ret) 2272 printf("Warning: latencystats init()" 2273 " returned error %d\n", ret); 2274 printf("Latencystats running on lcore %d\n", 2275 latencystats_lcore_id); 2276 } 2277 #endif 2278 2279 /* Setup bitrate stats */ 2280 #ifdef RTE_LIBRTE_BITRATE 2281 if (bitrate_enabled != 0) { 2282 bitrate_data = rte_stats_bitrate_create(); 2283 if (bitrate_data == NULL) 2284 rte_exit(EXIT_FAILURE, 2285 "Could not allocate bitrate data.\n"); 2286 rte_stats_bitrate_reg(bitrate_data); 2287 } 2288 #endif 2289 2290 #ifdef RTE_LIBRTE_CMDLINE 2291 if (strlen(cmdline_filename) != 0) 2292 cmdline_read_from_file(cmdline_filename); 2293 2294 if (interactive == 1) { 2295 if (auto_start) { 2296 printf("Start automatic packet forwarding\n"); 2297 start_packet_forwarding(0); 2298 } 2299 prompt(); 2300 pmd_test_exit(); 2301 } else 2302 #endif 2303 { 2304 char c; 2305 int rc; 2306 2307 printf("No commandline core given, start packet forwarding\n"); 2308 start_packet_forwarding(0); 2309 printf("Press enter to exit\n"); 2310 rc = read(0, &c, 1); 2311 pmd_test_exit(); 2312 if (rc < 0) 2313 return 1; 2314 } 2315 2316 return 0; 2317 } 2318