1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_NET_IXGBE 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIB_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #ifdef RTE_LIB_METRICS 58 #include <rte_metrics.h> 59 #endif 60 #ifdef RTE_LIB_BITRATESTATS 61 #include <rte_bitrate.h> 62 #endif 63 #ifdef RTE_LIB_LATENCYSTATS 64 #include <rte_latencystats.h> 65 #endif 66 #ifdef RTE_EXEC_ENV_WINDOWS 67 #include <process.h> 68 #endif 69 70 #include "testpmd.h" 71 72 #ifndef MAP_HUGETLB 73 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 74 #define HUGE_FLAG (0x40000) 75 #else 76 #define HUGE_FLAG MAP_HUGETLB 77 #endif 78 79 #ifndef MAP_HUGE_SHIFT 80 /* older kernels (or FreeBSD) will not have this define */ 81 #define HUGE_SHIFT (26) 82 #else 83 #define HUGE_SHIFT MAP_HUGE_SHIFT 84 #endif 85 86 #define EXTMEM_HEAP_NAME "extmem" 87 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 88 89 uint16_t verbose_level = 0; /**< Silent by default. */ 90 int testpmd_logtype; /**< Log type for testpmd logs */ 91 92 /* use main core for command line ? */ 93 uint8_t interactive = 0; 94 uint8_t auto_start = 0; 95 uint8_t tx_first; 96 char cmdline_filename[PATH_MAX] = {0}; 97 98 /* 99 * NUMA support configuration. 100 * When set, the NUMA support attempts to dispatch the allocation of the 101 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 102 * probed ports among the CPU sockets 0 and 1. 103 * Otherwise, all memory is allocated from CPU socket 0. 104 */ 105 uint8_t numa_support = 1; /**< numa enabled by default */ 106 107 /* 108 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 109 * not configured. 110 */ 111 uint8_t socket_num = UMA_NO_CONFIG; 112 113 /* 114 * Select mempool allocation type: 115 * - native: use regular DPDK memory 116 * - anon: use regular DPDK memory to create mempool, but populate using 117 * anonymous memory (may not be IOVA-contiguous) 118 * - xmem: use externally allocated hugepage memory 119 */ 120 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 121 122 /* 123 * Store specified sockets on which memory pool to be used by ports 124 * is allocated. 125 */ 126 uint8_t port_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which RX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Store specified sockets on which TX ring to be used by ports 136 * is allocated. 137 */ 138 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 139 140 /* 141 * Record the Ethernet address of peer target ports to which packets are 142 * forwarded. 143 * Must be instantiated with the ethernet addresses of peer traffic generator 144 * ports. 145 */ 146 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 147 portid_t nb_peer_eth_addrs = 0; 148 149 /* 150 * Probed Target Environment. 151 */ 152 struct rte_port *ports; /**< For all probed ethernet ports. */ 153 portid_t nb_ports; /**< Number of probed ethernet ports. */ 154 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 155 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 156 157 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 158 159 /* 160 * Test Forwarding Configuration. 161 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 162 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 163 */ 164 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 165 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 166 portid_t nb_cfg_ports; /**< Number of configured ports. */ 167 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 168 169 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 170 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 171 172 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 173 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 174 175 /* 176 * Forwarding engines. 177 */ 178 struct fwd_engine * fwd_engines[] = { 179 &io_fwd_engine, 180 &mac_fwd_engine, 181 &mac_swap_engine, 182 &flow_gen_engine, 183 &rx_only_engine, 184 &tx_only_engine, 185 &csum_fwd_engine, 186 &icmp_echo_engine, 187 &noisy_vnf_engine, 188 &five_tuple_swap_fwd_engine, 189 #ifdef RTE_LIBRTE_IEEE1588 190 &ieee1588_fwd_engine, 191 #endif 192 &shared_rxq_engine, 193 NULL, 194 }; 195 196 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 197 uint16_t mempool_flags; 198 199 struct fwd_config cur_fwd_config; 200 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 201 uint32_t retry_enabled; 202 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 203 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 204 205 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 206 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 207 DEFAULT_MBUF_DATA_SIZE 208 }; /**< Mbuf data space size. */ 209 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 210 * specified on command-line. */ 211 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 212 213 /** Extended statistics to show. */ 214 struct rte_eth_xstat_name *xstats_display; 215 216 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 217 218 /* 219 * In container, it cannot terminate the process which running with 'stats-period' 220 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 221 */ 222 uint8_t f_quit; 223 224 /* 225 * Max Rx frame size, set by '--max-pkt-len' parameter. 226 */ 227 uint32_t max_rx_pkt_len; 228 229 /* 230 * Configuration of packet segments used to scatter received packets 231 * if some of split features is configured. 232 */ 233 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 234 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 235 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 236 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 237 238 /* 239 * Configuration of packet segments used by the "txonly" processing engine. 240 */ 241 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 242 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 243 TXONLY_DEF_PACKET_LEN, 244 }; 245 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 246 247 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 248 /**< Split policy for packets to TX. */ 249 250 uint8_t txonly_multi_flow; 251 /**< Whether multiple flows are generated in TXONLY mode. */ 252 253 uint32_t tx_pkt_times_inter; 254 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 255 256 uint32_t tx_pkt_times_intra; 257 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 258 259 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 260 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 261 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 262 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 263 264 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 265 uint8_t dcb_config = 0; 266 267 /* 268 * Configurable number of RX/TX queues. 269 */ 270 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 271 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 272 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 273 274 /* 275 * Configurable number of RX/TX ring descriptors. 276 * Defaults are supplied by drivers via ethdev. 277 */ 278 #define RTE_TEST_RX_DESC_DEFAULT 0 279 #define RTE_TEST_TX_DESC_DEFAULT 0 280 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 281 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 282 283 #define RTE_PMD_PARAM_UNSET -1 284 /* 285 * Configurable values of RX and TX ring threshold registers. 286 */ 287 288 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 289 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 290 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 291 292 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 293 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 294 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 295 296 /* 297 * Configurable value of RX free threshold. 298 */ 299 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 300 301 /* 302 * Configurable value of RX drop enable. 303 */ 304 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 305 306 /* 307 * Configurable value of TX free threshold. 308 */ 309 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 310 311 /* 312 * Configurable value of TX RS bit threshold. 313 */ 314 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 315 316 /* 317 * Configurable value of buffered packets before sending. 318 */ 319 uint16_t noisy_tx_sw_bufsz; 320 321 /* 322 * Configurable value of packet buffer timeout. 323 */ 324 uint16_t noisy_tx_sw_buf_flush_time; 325 326 /* 327 * Configurable value for size of VNF internal memory area 328 * used for simulating noisy neighbour behaviour 329 */ 330 uint64_t noisy_lkup_mem_sz; 331 332 /* 333 * Configurable value of number of random writes done in 334 * VNF simulation memory area. 335 */ 336 uint64_t noisy_lkup_num_writes; 337 338 /* 339 * Configurable value of number of random reads done in 340 * VNF simulation memory area. 341 */ 342 uint64_t noisy_lkup_num_reads; 343 344 /* 345 * Configurable value of number of random reads/writes done in 346 * VNF simulation memory area. 347 */ 348 uint64_t noisy_lkup_num_reads_writes; 349 350 /* 351 * Receive Side Scaling (RSS) configuration. 352 */ 353 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 354 355 /* 356 * Port topology configuration 357 */ 358 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 359 360 /* 361 * Avoids to flush all the RX streams before starts forwarding. 362 */ 363 uint8_t no_flush_rx = 0; /* flush by default */ 364 365 /* 366 * Flow API isolated mode. 367 */ 368 uint8_t flow_isolate_all; 369 370 /* 371 * Avoids to check link status when starting/stopping a port. 372 */ 373 uint8_t no_link_check = 0; /* check by default */ 374 375 /* 376 * Don't automatically start all ports in interactive mode. 377 */ 378 uint8_t no_device_start = 0; 379 380 /* 381 * Enable link status change notification 382 */ 383 uint8_t lsc_interrupt = 1; /* enabled by default */ 384 385 /* 386 * Enable device removal notification. 387 */ 388 uint8_t rmv_interrupt = 1; /* enabled by default */ 389 390 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 391 392 /* After attach, port setup is called on event or by iterator */ 393 bool setup_on_probe_event = true; 394 395 /* Clear ptypes on port initialization. */ 396 uint8_t clear_ptypes = true; 397 398 /* Hairpin ports configuration mode. */ 399 uint16_t hairpin_mode; 400 401 /* Pretty printing of ethdev events */ 402 static const char * const eth_event_desc[] = { 403 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 404 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 405 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 406 [RTE_ETH_EVENT_INTR_RESET] = "reset", 407 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 408 [RTE_ETH_EVENT_IPSEC] = "IPsec", 409 [RTE_ETH_EVENT_MACSEC] = "MACsec", 410 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 411 [RTE_ETH_EVENT_NEW] = "device probed", 412 [RTE_ETH_EVENT_DESTROY] = "device released", 413 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 414 [RTE_ETH_EVENT_MAX] = NULL, 415 }; 416 417 /* 418 * Display or mask ether events 419 * Default to all events except VF_MBOX 420 */ 421 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 422 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 423 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 424 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 425 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 426 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 427 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 428 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 429 /* 430 * Decide if all memory are locked for performance. 431 */ 432 int do_mlockall = 0; 433 434 /* 435 * NIC bypass mode configuration options. 436 */ 437 438 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 439 /* The NIC bypass watchdog timeout. */ 440 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 441 #endif 442 443 444 #ifdef RTE_LIB_LATENCYSTATS 445 446 /* 447 * Set when latency stats is enabled in the commandline 448 */ 449 uint8_t latencystats_enabled; 450 451 /* 452 * Lcore ID to serive latency statistics. 453 */ 454 lcoreid_t latencystats_lcore_id = -1; 455 456 #endif 457 458 /* 459 * Ethernet device configuration. 460 */ 461 struct rte_eth_rxmode rx_mode; 462 463 struct rte_eth_txmode tx_mode = { 464 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 465 }; 466 467 struct rte_eth_fdir_conf fdir_conf = { 468 .mode = RTE_FDIR_MODE_NONE, 469 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 470 .status = RTE_FDIR_REPORT_STATUS, 471 .mask = { 472 .vlan_tci_mask = 0xFFEF, 473 .ipv4_mask = { 474 .src_ip = 0xFFFFFFFF, 475 .dst_ip = 0xFFFFFFFF, 476 }, 477 .ipv6_mask = { 478 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 479 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 480 }, 481 .src_port_mask = 0xFFFF, 482 .dst_port_mask = 0xFFFF, 483 .mac_addr_byte_mask = 0xFF, 484 .tunnel_type_mask = 1, 485 .tunnel_id_mask = 0xFFFFFFFF, 486 }, 487 .drop_queue = 127, 488 }; 489 490 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 491 492 /* 493 * Display zero values by default for xstats 494 */ 495 uint8_t xstats_hide_zero; 496 497 /* 498 * Measure of CPU cycles disabled by default 499 */ 500 uint8_t record_core_cycles; 501 502 /* 503 * Display of RX and TX bursts disabled by default 504 */ 505 uint8_t record_burst_stats; 506 507 /* 508 * Number of ports per shared Rx queue group, 0 disable. 509 */ 510 uint32_t rxq_share; 511 512 unsigned int num_sockets = 0; 513 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 514 515 #ifdef RTE_LIB_BITRATESTATS 516 /* Bitrate statistics */ 517 struct rte_stats_bitrates *bitrate_data; 518 lcoreid_t bitrate_lcore_id; 519 uint8_t bitrate_enabled; 520 #endif 521 522 #ifdef RTE_LIB_GRO 523 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 524 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 525 #endif 526 527 /* 528 * hexadecimal bitmask of RX mq mode can be enabled. 529 */ 530 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 531 532 /* 533 * Used to set forced link speed 534 */ 535 uint32_t eth_link_speed; 536 537 /* 538 * ID of the current process in multi-process, used to 539 * configure the queues to be polled. 540 */ 541 int proc_id; 542 543 /* 544 * Number of processes in multi-process, used to 545 * configure the queues to be polled. 546 */ 547 unsigned int num_procs = 1; 548 549 static void 550 eth_rx_metadata_negotiate_mp(uint16_t port_id) 551 { 552 uint64_t rx_meta_features = 0; 553 int ret; 554 555 if (!is_proc_primary()) 556 return; 557 558 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 559 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 560 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 561 562 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 563 if (ret == 0) { 564 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 565 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 566 port_id); 567 } 568 569 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 570 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 571 port_id); 572 } 573 574 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 575 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 576 port_id); 577 } 578 } else if (ret != -ENOTSUP) { 579 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 580 port_id, rte_strerror(-ret)); 581 } 582 } 583 584 static void 585 flow_pick_transfer_proxy_mp(uint16_t port_id) 586 { 587 struct rte_port *port = &ports[port_id]; 588 int ret; 589 590 port->flow_transfer_proxy = port_id; 591 592 if (!is_proc_primary()) 593 return; 594 595 ret = rte_flow_pick_transfer_proxy(port_id, &port->flow_transfer_proxy, 596 NULL); 597 if (ret != 0) { 598 fprintf(stderr, "Error picking flow transfer proxy for port %u: %s - ignore\n", 599 port_id, rte_strerror(-ret)); 600 } 601 } 602 603 static int 604 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 605 const struct rte_eth_conf *dev_conf) 606 { 607 if (is_proc_primary()) 608 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 609 dev_conf); 610 return 0; 611 } 612 613 static int 614 eth_dev_start_mp(uint16_t port_id) 615 { 616 if (is_proc_primary()) 617 return rte_eth_dev_start(port_id); 618 619 return 0; 620 } 621 622 static int 623 eth_dev_stop_mp(uint16_t port_id) 624 { 625 if (is_proc_primary()) 626 return rte_eth_dev_stop(port_id); 627 628 return 0; 629 } 630 631 static void 632 mempool_free_mp(struct rte_mempool *mp) 633 { 634 if (is_proc_primary()) 635 rte_mempool_free(mp); 636 } 637 638 static int 639 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 640 { 641 if (is_proc_primary()) 642 return rte_eth_dev_set_mtu(port_id, mtu); 643 644 return 0; 645 } 646 647 /* Forward function declarations */ 648 static void setup_attached_port(portid_t pi); 649 static void check_all_ports_link_status(uint32_t port_mask); 650 static int eth_event_callback(portid_t port_id, 651 enum rte_eth_event_type type, 652 void *param, void *ret_param); 653 static void dev_event_callback(const char *device_name, 654 enum rte_dev_event_type type, 655 void *param); 656 static void fill_xstats_display_info(void); 657 658 /* 659 * Check if all the ports are started. 660 * If yes, return positive value. If not, return zero. 661 */ 662 static int all_ports_started(void); 663 664 #ifdef RTE_LIB_GSO 665 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 666 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 667 #endif 668 669 /* Holds the registered mbuf dynamic flags names. */ 670 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 671 672 673 /* 674 * Helper function to check if socket is already discovered. 675 * If yes, return positive value. If not, return zero. 676 */ 677 int 678 new_socket_id(unsigned int socket_id) 679 { 680 unsigned int i; 681 682 for (i = 0; i < num_sockets; i++) { 683 if (socket_ids[i] == socket_id) 684 return 0; 685 } 686 return 1; 687 } 688 689 /* 690 * Setup default configuration. 691 */ 692 static void 693 set_default_fwd_lcores_config(void) 694 { 695 unsigned int i; 696 unsigned int nb_lc; 697 unsigned int sock_num; 698 699 nb_lc = 0; 700 for (i = 0; i < RTE_MAX_LCORE; i++) { 701 if (!rte_lcore_is_enabled(i)) 702 continue; 703 sock_num = rte_lcore_to_socket_id(i); 704 if (new_socket_id(sock_num)) { 705 if (num_sockets >= RTE_MAX_NUMA_NODES) { 706 rte_exit(EXIT_FAILURE, 707 "Total sockets greater than %u\n", 708 RTE_MAX_NUMA_NODES); 709 } 710 socket_ids[num_sockets++] = sock_num; 711 } 712 if (i == rte_get_main_lcore()) 713 continue; 714 fwd_lcores_cpuids[nb_lc++] = i; 715 } 716 nb_lcores = (lcoreid_t) nb_lc; 717 nb_cfg_lcores = nb_lcores; 718 nb_fwd_lcores = 1; 719 } 720 721 static void 722 set_def_peer_eth_addrs(void) 723 { 724 portid_t i; 725 726 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 727 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 728 peer_eth_addrs[i].addr_bytes[5] = i; 729 } 730 } 731 732 static void 733 set_default_fwd_ports_config(void) 734 { 735 portid_t pt_id; 736 int i = 0; 737 738 RTE_ETH_FOREACH_DEV(pt_id) { 739 fwd_ports_ids[i++] = pt_id; 740 741 /* Update sockets info according to the attached device */ 742 int socket_id = rte_eth_dev_socket_id(pt_id); 743 if (socket_id >= 0 && new_socket_id(socket_id)) { 744 if (num_sockets >= RTE_MAX_NUMA_NODES) { 745 rte_exit(EXIT_FAILURE, 746 "Total sockets greater than %u\n", 747 RTE_MAX_NUMA_NODES); 748 } 749 socket_ids[num_sockets++] = socket_id; 750 } 751 } 752 753 nb_cfg_ports = nb_ports; 754 nb_fwd_ports = nb_ports; 755 } 756 757 void 758 set_def_fwd_config(void) 759 { 760 set_default_fwd_lcores_config(); 761 set_def_peer_eth_addrs(); 762 set_default_fwd_ports_config(); 763 } 764 765 #ifndef RTE_EXEC_ENV_WINDOWS 766 /* extremely pessimistic estimation of memory required to create a mempool */ 767 static int 768 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 769 { 770 unsigned int n_pages, mbuf_per_pg, leftover; 771 uint64_t total_mem, mbuf_mem, obj_sz; 772 773 /* there is no good way to predict how much space the mempool will 774 * occupy because it will allocate chunks on the fly, and some of those 775 * will come from default DPDK memory while some will come from our 776 * external memory, so just assume 128MB will be enough for everyone. 777 */ 778 uint64_t hdr_mem = 128 << 20; 779 780 /* account for possible non-contiguousness */ 781 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 782 if (obj_sz > pgsz) { 783 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 784 return -1; 785 } 786 787 mbuf_per_pg = pgsz / obj_sz; 788 leftover = (nb_mbufs % mbuf_per_pg) > 0; 789 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 790 791 mbuf_mem = n_pages * pgsz; 792 793 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 794 795 if (total_mem > SIZE_MAX) { 796 TESTPMD_LOG(ERR, "Memory size too big\n"); 797 return -1; 798 } 799 *out = (size_t)total_mem; 800 801 return 0; 802 } 803 804 static int 805 pagesz_flags(uint64_t page_sz) 806 { 807 /* as per mmap() manpage, all page sizes are log2 of page size 808 * shifted by MAP_HUGE_SHIFT 809 */ 810 int log2 = rte_log2_u64(page_sz); 811 812 return (log2 << HUGE_SHIFT); 813 } 814 815 static void * 816 alloc_mem(size_t memsz, size_t pgsz, bool huge) 817 { 818 void *addr; 819 int flags; 820 821 /* allocate anonymous hugepages */ 822 flags = MAP_ANONYMOUS | MAP_PRIVATE; 823 if (huge) 824 flags |= HUGE_FLAG | pagesz_flags(pgsz); 825 826 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 827 if (addr == MAP_FAILED) 828 return NULL; 829 830 return addr; 831 } 832 833 struct extmem_param { 834 void *addr; 835 size_t len; 836 size_t pgsz; 837 rte_iova_t *iova_table; 838 unsigned int iova_table_len; 839 }; 840 841 static int 842 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 843 bool huge) 844 { 845 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 846 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 847 unsigned int cur_page, n_pages, pgsz_idx; 848 size_t mem_sz, cur_pgsz; 849 rte_iova_t *iovas = NULL; 850 void *addr; 851 int ret; 852 853 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 854 /* skip anything that is too big */ 855 if (pgsizes[pgsz_idx] > SIZE_MAX) 856 continue; 857 858 cur_pgsz = pgsizes[pgsz_idx]; 859 860 /* if we were told not to allocate hugepages, override */ 861 if (!huge) 862 cur_pgsz = sysconf(_SC_PAGESIZE); 863 864 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 865 if (ret < 0) { 866 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 867 return -1; 868 } 869 870 /* allocate our memory */ 871 addr = alloc_mem(mem_sz, cur_pgsz, huge); 872 873 /* if we couldn't allocate memory with a specified page size, 874 * that doesn't mean we can't do it with other page sizes, so 875 * try another one. 876 */ 877 if (addr == NULL) 878 continue; 879 880 /* store IOVA addresses for every page in this memory area */ 881 n_pages = mem_sz / cur_pgsz; 882 883 iovas = malloc(sizeof(*iovas) * n_pages); 884 885 if (iovas == NULL) { 886 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 887 goto fail; 888 } 889 /* lock memory if it's not huge pages */ 890 if (!huge) 891 mlock(addr, mem_sz); 892 893 /* populate IOVA addresses */ 894 for (cur_page = 0; cur_page < n_pages; cur_page++) { 895 rte_iova_t iova; 896 size_t offset; 897 void *cur; 898 899 offset = cur_pgsz * cur_page; 900 cur = RTE_PTR_ADD(addr, offset); 901 902 /* touch the page before getting its IOVA */ 903 *(volatile char *)cur = 0; 904 905 iova = rte_mem_virt2iova(cur); 906 907 iovas[cur_page] = iova; 908 } 909 910 break; 911 } 912 /* if we couldn't allocate anything */ 913 if (iovas == NULL) 914 return -1; 915 916 param->addr = addr; 917 param->len = mem_sz; 918 param->pgsz = cur_pgsz; 919 param->iova_table = iovas; 920 param->iova_table_len = n_pages; 921 922 return 0; 923 fail: 924 if (iovas) 925 free(iovas); 926 if (addr) 927 munmap(addr, mem_sz); 928 929 return -1; 930 } 931 932 static int 933 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 934 { 935 struct extmem_param param; 936 int socket_id, ret; 937 938 memset(¶m, 0, sizeof(param)); 939 940 /* check if our heap exists */ 941 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 942 if (socket_id < 0) { 943 /* create our heap */ 944 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 945 if (ret < 0) { 946 TESTPMD_LOG(ERR, "Cannot create heap\n"); 947 return -1; 948 } 949 } 950 951 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 952 if (ret < 0) { 953 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 954 return -1; 955 } 956 957 /* we now have a valid memory area, so add it to heap */ 958 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 959 param.addr, param.len, param.iova_table, 960 param.iova_table_len, param.pgsz); 961 962 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 963 964 /* not needed any more */ 965 free(param.iova_table); 966 967 if (ret < 0) { 968 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 969 munmap(param.addr, param.len); 970 return -1; 971 } 972 973 /* success */ 974 975 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 976 param.len >> 20); 977 978 return 0; 979 } 980 static void 981 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 982 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 983 { 984 uint16_t pid = 0; 985 int ret; 986 987 RTE_ETH_FOREACH_DEV(pid) { 988 struct rte_eth_dev_info dev_info; 989 990 ret = eth_dev_info_get_print_err(pid, &dev_info); 991 if (ret != 0) { 992 TESTPMD_LOG(DEBUG, 993 "unable to get device info for port %d on addr 0x%p," 994 "mempool unmapping will not be performed\n", 995 pid, memhdr->addr); 996 continue; 997 } 998 999 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 1000 if (ret) { 1001 TESTPMD_LOG(DEBUG, 1002 "unable to DMA unmap addr 0x%p " 1003 "for device %s\n", 1004 memhdr->addr, dev_info.device->name); 1005 } 1006 } 1007 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1008 if (ret) { 1009 TESTPMD_LOG(DEBUG, 1010 "unable to un-register addr 0x%p\n", memhdr->addr); 1011 } 1012 } 1013 1014 static void 1015 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1016 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1017 { 1018 uint16_t pid = 0; 1019 size_t page_size = sysconf(_SC_PAGESIZE); 1020 int ret; 1021 1022 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1023 page_size); 1024 if (ret) { 1025 TESTPMD_LOG(DEBUG, 1026 "unable to register addr 0x%p\n", memhdr->addr); 1027 return; 1028 } 1029 RTE_ETH_FOREACH_DEV(pid) { 1030 struct rte_eth_dev_info dev_info; 1031 1032 ret = eth_dev_info_get_print_err(pid, &dev_info); 1033 if (ret != 0) { 1034 TESTPMD_LOG(DEBUG, 1035 "unable to get device info for port %d on addr 0x%p," 1036 "mempool mapping will not be performed\n", 1037 pid, memhdr->addr); 1038 continue; 1039 } 1040 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1041 if (ret) { 1042 TESTPMD_LOG(DEBUG, 1043 "unable to DMA map addr 0x%p " 1044 "for device %s\n", 1045 memhdr->addr, dev_info.device->name); 1046 } 1047 } 1048 } 1049 #endif 1050 1051 static unsigned int 1052 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1053 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1054 { 1055 struct rte_pktmbuf_extmem *xmem; 1056 unsigned int ext_num, zone_num, elt_num; 1057 uint16_t elt_size; 1058 1059 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1060 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1061 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1062 1063 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1064 if (xmem == NULL) { 1065 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1066 "external buffer descriptors\n"); 1067 *ext_mem = NULL; 1068 return 0; 1069 } 1070 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1071 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1072 const struct rte_memzone *mz; 1073 char mz_name[RTE_MEMZONE_NAMESIZE]; 1074 int ret; 1075 1076 ret = snprintf(mz_name, sizeof(mz_name), 1077 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1078 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1079 errno = ENAMETOOLONG; 1080 ext_num = 0; 1081 break; 1082 } 1083 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 1084 socket_id, 1085 RTE_MEMZONE_IOVA_CONTIG | 1086 RTE_MEMZONE_1GB | 1087 RTE_MEMZONE_SIZE_HINT_ONLY, 1088 EXTBUF_ZONE_SIZE); 1089 if (mz == NULL) { 1090 /* 1091 * The caller exits on external buffer creation 1092 * error, so there is no need to free memzones. 1093 */ 1094 errno = ENOMEM; 1095 ext_num = 0; 1096 break; 1097 } 1098 xseg->buf_ptr = mz->addr; 1099 xseg->buf_iova = mz->iova; 1100 xseg->buf_len = EXTBUF_ZONE_SIZE; 1101 xseg->elt_size = elt_size; 1102 } 1103 if (ext_num == 0 && xmem != NULL) { 1104 free(xmem); 1105 xmem = NULL; 1106 } 1107 *ext_mem = xmem; 1108 return ext_num; 1109 } 1110 1111 /* 1112 * Configuration initialisation done once at init time. 1113 */ 1114 static struct rte_mempool * 1115 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1116 unsigned int socket_id, uint16_t size_idx) 1117 { 1118 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1119 struct rte_mempool *rte_mp = NULL; 1120 #ifndef RTE_EXEC_ENV_WINDOWS 1121 uint32_t mb_size; 1122 1123 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1124 #endif 1125 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1126 if (!is_proc_primary()) { 1127 rte_mp = rte_mempool_lookup(pool_name); 1128 if (rte_mp == NULL) 1129 rte_exit(EXIT_FAILURE, 1130 "Get mbuf pool for socket %u failed: %s\n", 1131 socket_id, rte_strerror(rte_errno)); 1132 return rte_mp; 1133 } 1134 1135 TESTPMD_LOG(INFO, 1136 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1137 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1138 1139 switch (mp_alloc_type) { 1140 case MP_ALLOC_NATIVE: 1141 { 1142 /* wrapper to rte_mempool_create() */ 1143 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1144 rte_mbuf_best_mempool_ops()); 1145 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1146 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1147 break; 1148 } 1149 #ifndef RTE_EXEC_ENV_WINDOWS 1150 case MP_ALLOC_ANON: 1151 { 1152 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1153 mb_size, (unsigned int) mb_mempool_cache, 1154 sizeof(struct rte_pktmbuf_pool_private), 1155 socket_id, mempool_flags); 1156 if (rte_mp == NULL) 1157 goto err; 1158 1159 if (rte_mempool_populate_anon(rte_mp) == 0) { 1160 rte_mempool_free(rte_mp); 1161 rte_mp = NULL; 1162 goto err; 1163 } 1164 rte_pktmbuf_pool_init(rte_mp, NULL); 1165 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1166 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1167 break; 1168 } 1169 case MP_ALLOC_XMEM: 1170 case MP_ALLOC_XMEM_HUGE: 1171 { 1172 int heap_socket; 1173 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1174 1175 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1176 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1177 1178 heap_socket = 1179 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1180 if (heap_socket < 0) 1181 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1182 1183 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1184 rte_mbuf_best_mempool_ops()); 1185 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1186 mb_mempool_cache, 0, mbuf_seg_size, 1187 heap_socket); 1188 break; 1189 } 1190 #endif 1191 case MP_ALLOC_XBUF: 1192 { 1193 struct rte_pktmbuf_extmem *ext_mem; 1194 unsigned int ext_num; 1195 1196 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1197 socket_id, pool_name, &ext_mem); 1198 if (ext_num == 0) 1199 rte_exit(EXIT_FAILURE, 1200 "Can't create pinned data buffers\n"); 1201 1202 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1203 rte_mbuf_best_mempool_ops()); 1204 rte_mp = rte_pktmbuf_pool_create_extbuf 1205 (pool_name, nb_mbuf, mb_mempool_cache, 1206 0, mbuf_seg_size, socket_id, 1207 ext_mem, ext_num); 1208 free(ext_mem); 1209 break; 1210 } 1211 default: 1212 { 1213 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1214 } 1215 } 1216 1217 #ifndef RTE_EXEC_ENV_WINDOWS 1218 err: 1219 #endif 1220 if (rte_mp == NULL) { 1221 rte_exit(EXIT_FAILURE, 1222 "Creation of mbuf pool for socket %u failed: %s\n", 1223 socket_id, rte_strerror(rte_errno)); 1224 } else if (verbose_level > 0) { 1225 rte_mempool_dump(stdout, rte_mp); 1226 } 1227 return rte_mp; 1228 } 1229 1230 /* 1231 * Check given socket id is valid or not with NUMA mode, 1232 * if valid, return 0, else return -1 1233 */ 1234 static int 1235 check_socket_id(const unsigned int socket_id) 1236 { 1237 static int warning_once = 0; 1238 1239 if (new_socket_id(socket_id)) { 1240 if (!warning_once && numa_support) 1241 fprintf(stderr, 1242 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1243 warning_once = 1; 1244 return -1; 1245 } 1246 return 0; 1247 } 1248 1249 /* 1250 * Get the allowed maximum number of RX queues. 1251 * *pid return the port id which has minimal value of 1252 * max_rx_queues in all ports. 1253 */ 1254 queueid_t 1255 get_allowed_max_nb_rxq(portid_t *pid) 1256 { 1257 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1258 bool max_rxq_valid = false; 1259 portid_t pi; 1260 struct rte_eth_dev_info dev_info; 1261 1262 RTE_ETH_FOREACH_DEV(pi) { 1263 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1264 continue; 1265 1266 max_rxq_valid = true; 1267 if (dev_info.max_rx_queues < allowed_max_rxq) { 1268 allowed_max_rxq = dev_info.max_rx_queues; 1269 *pid = pi; 1270 } 1271 } 1272 return max_rxq_valid ? allowed_max_rxq : 0; 1273 } 1274 1275 /* 1276 * Check input rxq is valid or not. 1277 * If input rxq is not greater than any of maximum number 1278 * of RX queues of all ports, it is valid. 1279 * if valid, return 0, else return -1 1280 */ 1281 int 1282 check_nb_rxq(queueid_t rxq) 1283 { 1284 queueid_t allowed_max_rxq; 1285 portid_t pid = 0; 1286 1287 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1288 if (rxq > allowed_max_rxq) { 1289 fprintf(stderr, 1290 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1291 rxq, allowed_max_rxq, pid); 1292 return -1; 1293 } 1294 return 0; 1295 } 1296 1297 /* 1298 * Get the allowed maximum number of TX queues. 1299 * *pid return the port id which has minimal value of 1300 * max_tx_queues in all ports. 1301 */ 1302 queueid_t 1303 get_allowed_max_nb_txq(portid_t *pid) 1304 { 1305 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1306 bool max_txq_valid = false; 1307 portid_t pi; 1308 struct rte_eth_dev_info dev_info; 1309 1310 RTE_ETH_FOREACH_DEV(pi) { 1311 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1312 continue; 1313 1314 max_txq_valid = true; 1315 if (dev_info.max_tx_queues < allowed_max_txq) { 1316 allowed_max_txq = dev_info.max_tx_queues; 1317 *pid = pi; 1318 } 1319 } 1320 return max_txq_valid ? allowed_max_txq : 0; 1321 } 1322 1323 /* 1324 * Check input txq is valid or not. 1325 * If input txq is not greater than any of maximum number 1326 * of TX queues of all ports, it is valid. 1327 * if valid, return 0, else return -1 1328 */ 1329 int 1330 check_nb_txq(queueid_t txq) 1331 { 1332 queueid_t allowed_max_txq; 1333 portid_t pid = 0; 1334 1335 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1336 if (txq > allowed_max_txq) { 1337 fprintf(stderr, 1338 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1339 txq, allowed_max_txq, pid); 1340 return -1; 1341 } 1342 return 0; 1343 } 1344 1345 /* 1346 * Get the allowed maximum number of RXDs of every rx queue. 1347 * *pid return the port id which has minimal value of 1348 * max_rxd in all queues of all ports. 1349 */ 1350 static uint16_t 1351 get_allowed_max_nb_rxd(portid_t *pid) 1352 { 1353 uint16_t allowed_max_rxd = UINT16_MAX; 1354 portid_t pi; 1355 struct rte_eth_dev_info dev_info; 1356 1357 RTE_ETH_FOREACH_DEV(pi) { 1358 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1359 continue; 1360 1361 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1362 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1363 *pid = pi; 1364 } 1365 } 1366 return allowed_max_rxd; 1367 } 1368 1369 /* 1370 * Get the allowed minimal number of RXDs of every rx queue. 1371 * *pid return the port id which has minimal value of 1372 * min_rxd in all queues of all ports. 1373 */ 1374 static uint16_t 1375 get_allowed_min_nb_rxd(portid_t *pid) 1376 { 1377 uint16_t allowed_min_rxd = 0; 1378 portid_t pi; 1379 struct rte_eth_dev_info dev_info; 1380 1381 RTE_ETH_FOREACH_DEV(pi) { 1382 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1383 continue; 1384 1385 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1386 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1387 *pid = pi; 1388 } 1389 } 1390 1391 return allowed_min_rxd; 1392 } 1393 1394 /* 1395 * Check input rxd is valid or not. 1396 * If input rxd is not greater than any of maximum number 1397 * of RXDs of every Rx queues and is not less than any of 1398 * minimal number of RXDs of every Rx queues, it is valid. 1399 * if valid, return 0, else return -1 1400 */ 1401 int 1402 check_nb_rxd(queueid_t rxd) 1403 { 1404 uint16_t allowed_max_rxd; 1405 uint16_t allowed_min_rxd; 1406 portid_t pid = 0; 1407 1408 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1409 if (rxd > allowed_max_rxd) { 1410 fprintf(stderr, 1411 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1412 rxd, allowed_max_rxd, pid); 1413 return -1; 1414 } 1415 1416 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1417 if (rxd < allowed_min_rxd) { 1418 fprintf(stderr, 1419 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1420 rxd, allowed_min_rxd, pid); 1421 return -1; 1422 } 1423 1424 return 0; 1425 } 1426 1427 /* 1428 * Get the allowed maximum number of TXDs of every rx queues. 1429 * *pid return the port id which has minimal value of 1430 * max_txd in every tx queue. 1431 */ 1432 static uint16_t 1433 get_allowed_max_nb_txd(portid_t *pid) 1434 { 1435 uint16_t allowed_max_txd = UINT16_MAX; 1436 portid_t pi; 1437 struct rte_eth_dev_info dev_info; 1438 1439 RTE_ETH_FOREACH_DEV(pi) { 1440 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1441 continue; 1442 1443 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1444 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1445 *pid = pi; 1446 } 1447 } 1448 return allowed_max_txd; 1449 } 1450 1451 /* 1452 * Get the allowed maximum number of TXDs of every tx queues. 1453 * *pid return the port id which has minimal value of 1454 * min_txd in every tx queue. 1455 */ 1456 static uint16_t 1457 get_allowed_min_nb_txd(portid_t *pid) 1458 { 1459 uint16_t allowed_min_txd = 0; 1460 portid_t pi; 1461 struct rte_eth_dev_info dev_info; 1462 1463 RTE_ETH_FOREACH_DEV(pi) { 1464 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1465 continue; 1466 1467 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1468 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1469 *pid = pi; 1470 } 1471 } 1472 1473 return allowed_min_txd; 1474 } 1475 1476 /* 1477 * Check input txd is valid or not. 1478 * If input txd is not greater than any of maximum number 1479 * of TXDs of every Rx queues, it is valid. 1480 * if valid, return 0, else return -1 1481 */ 1482 int 1483 check_nb_txd(queueid_t txd) 1484 { 1485 uint16_t allowed_max_txd; 1486 uint16_t allowed_min_txd; 1487 portid_t pid = 0; 1488 1489 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1490 if (txd > allowed_max_txd) { 1491 fprintf(stderr, 1492 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1493 txd, allowed_max_txd, pid); 1494 return -1; 1495 } 1496 1497 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1498 if (txd < allowed_min_txd) { 1499 fprintf(stderr, 1500 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1501 txd, allowed_min_txd, pid); 1502 return -1; 1503 } 1504 return 0; 1505 } 1506 1507 1508 /* 1509 * Get the allowed maximum number of hairpin queues. 1510 * *pid return the port id which has minimal value of 1511 * max_hairpin_queues in all ports. 1512 */ 1513 queueid_t 1514 get_allowed_max_nb_hairpinq(portid_t *pid) 1515 { 1516 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1517 portid_t pi; 1518 struct rte_eth_hairpin_cap cap; 1519 1520 RTE_ETH_FOREACH_DEV(pi) { 1521 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1522 *pid = pi; 1523 return 0; 1524 } 1525 if (cap.max_nb_queues < allowed_max_hairpinq) { 1526 allowed_max_hairpinq = cap.max_nb_queues; 1527 *pid = pi; 1528 } 1529 } 1530 return allowed_max_hairpinq; 1531 } 1532 1533 /* 1534 * Check input hairpin is valid or not. 1535 * If input hairpin is not greater than any of maximum number 1536 * of hairpin queues of all ports, it is valid. 1537 * if valid, return 0, else return -1 1538 */ 1539 int 1540 check_nb_hairpinq(queueid_t hairpinq) 1541 { 1542 queueid_t allowed_max_hairpinq; 1543 portid_t pid = 0; 1544 1545 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1546 if (hairpinq > allowed_max_hairpinq) { 1547 fprintf(stderr, 1548 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1549 hairpinq, allowed_max_hairpinq, pid); 1550 return -1; 1551 } 1552 return 0; 1553 } 1554 1555 static int 1556 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1557 { 1558 uint32_t eth_overhead; 1559 1560 if (dev_info->max_mtu != UINT16_MAX && 1561 dev_info->max_rx_pktlen > dev_info->max_mtu) 1562 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1563 else 1564 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1565 1566 return eth_overhead; 1567 } 1568 1569 static void 1570 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1571 { 1572 struct rte_port *port = &ports[pid]; 1573 int ret; 1574 int i; 1575 1576 eth_rx_metadata_negotiate_mp(pid); 1577 flow_pick_transfer_proxy_mp(pid); 1578 1579 port->dev_conf.txmode = tx_mode; 1580 port->dev_conf.rxmode = rx_mode; 1581 1582 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1583 if (ret != 0) 1584 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1585 1586 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1587 port->dev_conf.txmode.offloads &= 1588 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1589 1590 /* Apply Rx offloads configuration */ 1591 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1592 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads; 1593 /* Apply Tx offloads configuration */ 1594 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1595 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads; 1596 1597 if (eth_link_speed) 1598 port->dev_conf.link_speeds = eth_link_speed; 1599 1600 if (max_rx_pkt_len) 1601 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1602 get_eth_overhead(&port->dev_info); 1603 1604 /* set flag to initialize port/queue */ 1605 port->need_reconfig = 1; 1606 port->need_reconfig_queues = 1; 1607 port->socket_id = socket_id; 1608 port->tx_metadata = 0; 1609 1610 /* 1611 * Check for maximum number of segments per MTU. 1612 * Accordingly update the mbuf data size. 1613 */ 1614 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1615 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1616 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1617 uint16_t mtu; 1618 1619 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1620 uint16_t data_size = (mtu + eth_overhead) / 1621 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1622 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1623 1624 if (buffer_size > mbuf_data_size[0]) { 1625 mbuf_data_size[0] = buffer_size; 1626 TESTPMD_LOG(WARNING, 1627 "Configured mbuf size of the first segment %hu\n", 1628 mbuf_data_size[0]); 1629 } 1630 } 1631 } 1632 } 1633 1634 static void 1635 init_config(void) 1636 { 1637 portid_t pid; 1638 struct rte_mempool *mbp; 1639 unsigned int nb_mbuf_per_pool; 1640 lcoreid_t lc_id; 1641 #ifdef RTE_LIB_GRO 1642 struct rte_gro_param gro_param; 1643 #endif 1644 #ifdef RTE_LIB_GSO 1645 uint32_t gso_types; 1646 #endif 1647 1648 /* Configuration of logical cores. */ 1649 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1650 sizeof(struct fwd_lcore *) * nb_lcores, 1651 RTE_CACHE_LINE_SIZE); 1652 if (fwd_lcores == NULL) { 1653 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1654 "failed\n", nb_lcores); 1655 } 1656 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1657 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1658 sizeof(struct fwd_lcore), 1659 RTE_CACHE_LINE_SIZE); 1660 if (fwd_lcores[lc_id] == NULL) { 1661 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1662 "failed\n"); 1663 } 1664 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1665 } 1666 1667 RTE_ETH_FOREACH_DEV(pid) { 1668 uint32_t socket_id; 1669 1670 if (numa_support) { 1671 socket_id = port_numa[pid]; 1672 if (port_numa[pid] == NUMA_NO_CONFIG) { 1673 socket_id = rte_eth_dev_socket_id(pid); 1674 1675 /* 1676 * if socket_id is invalid, 1677 * set to the first available socket. 1678 */ 1679 if (check_socket_id(socket_id) < 0) 1680 socket_id = socket_ids[0]; 1681 } 1682 } else { 1683 socket_id = (socket_num == UMA_NO_CONFIG) ? 1684 0 : socket_num; 1685 } 1686 /* Apply default TxRx configuration for all ports */ 1687 init_config_port_offloads(pid, socket_id); 1688 } 1689 /* 1690 * Create pools of mbuf. 1691 * If NUMA support is disabled, create a single pool of mbuf in 1692 * socket 0 memory by default. 1693 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1694 * 1695 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1696 * nb_txd can be configured at run time. 1697 */ 1698 if (param_total_num_mbufs) 1699 nb_mbuf_per_pool = param_total_num_mbufs; 1700 else { 1701 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1702 (nb_lcores * mb_mempool_cache) + 1703 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1704 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1705 } 1706 1707 if (numa_support) { 1708 uint8_t i, j; 1709 1710 for (i = 0; i < num_sockets; i++) 1711 for (j = 0; j < mbuf_data_size_n; j++) 1712 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1713 mbuf_pool_create(mbuf_data_size[j], 1714 nb_mbuf_per_pool, 1715 socket_ids[i], j); 1716 } else { 1717 uint8_t i; 1718 1719 for (i = 0; i < mbuf_data_size_n; i++) 1720 mempools[i] = mbuf_pool_create 1721 (mbuf_data_size[i], 1722 nb_mbuf_per_pool, 1723 socket_num == UMA_NO_CONFIG ? 1724 0 : socket_num, i); 1725 } 1726 1727 init_port_config(); 1728 1729 #ifdef RTE_LIB_GSO 1730 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1731 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1732 #endif 1733 /* 1734 * Records which Mbuf pool to use by each logical core, if needed. 1735 */ 1736 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1737 mbp = mbuf_pool_find( 1738 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1739 1740 if (mbp == NULL) 1741 mbp = mbuf_pool_find(0, 0); 1742 fwd_lcores[lc_id]->mbp = mbp; 1743 #ifdef RTE_LIB_GSO 1744 /* initialize GSO context */ 1745 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1746 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1747 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1748 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1749 RTE_ETHER_CRC_LEN; 1750 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1751 #endif 1752 } 1753 1754 fwd_config_setup(); 1755 1756 #ifdef RTE_LIB_GRO 1757 /* create a gro context for each lcore */ 1758 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1759 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1760 gro_param.max_item_per_flow = MAX_PKT_BURST; 1761 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1762 gro_param.socket_id = rte_lcore_to_socket_id( 1763 fwd_lcores_cpuids[lc_id]); 1764 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1765 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1766 rte_exit(EXIT_FAILURE, 1767 "rte_gro_ctx_create() failed\n"); 1768 } 1769 } 1770 #endif 1771 } 1772 1773 1774 void 1775 reconfig(portid_t new_port_id, unsigned socket_id) 1776 { 1777 /* Reconfiguration of Ethernet ports. */ 1778 init_config_port_offloads(new_port_id, socket_id); 1779 init_port_config(); 1780 } 1781 1782 1783 int 1784 init_fwd_streams(void) 1785 { 1786 portid_t pid; 1787 struct rte_port *port; 1788 streamid_t sm_id, nb_fwd_streams_new; 1789 queueid_t q; 1790 1791 /* set socket id according to numa or not */ 1792 RTE_ETH_FOREACH_DEV(pid) { 1793 port = &ports[pid]; 1794 if (nb_rxq > port->dev_info.max_rx_queues) { 1795 fprintf(stderr, 1796 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1797 nb_rxq, port->dev_info.max_rx_queues); 1798 return -1; 1799 } 1800 if (nb_txq > port->dev_info.max_tx_queues) { 1801 fprintf(stderr, 1802 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1803 nb_txq, port->dev_info.max_tx_queues); 1804 return -1; 1805 } 1806 if (numa_support) { 1807 if (port_numa[pid] != NUMA_NO_CONFIG) 1808 port->socket_id = port_numa[pid]; 1809 else { 1810 port->socket_id = rte_eth_dev_socket_id(pid); 1811 1812 /* 1813 * if socket_id is invalid, 1814 * set to the first available socket. 1815 */ 1816 if (check_socket_id(port->socket_id) < 0) 1817 port->socket_id = socket_ids[0]; 1818 } 1819 } 1820 else { 1821 if (socket_num == UMA_NO_CONFIG) 1822 port->socket_id = 0; 1823 else 1824 port->socket_id = socket_num; 1825 } 1826 } 1827 1828 q = RTE_MAX(nb_rxq, nb_txq); 1829 if (q == 0) { 1830 fprintf(stderr, 1831 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1832 return -1; 1833 } 1834 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1835 if (nb_fwd_streams_new == nb_fwd_streams) 1836 return 0; 1837 /* clear the old */ 1838 if (fwd_streams != NULL) { 1839 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1840 if (fwd_streams[sm_id] == NULL) 1841 continue; 1842 rte_free(fwd_streams[sm_id]); 1843 fwd_streams[sm_id] = NULL; 1844 } 1845 rte_free(fwd_streams); 1846 fwd_streams = NULL; 1847 } 1848 1849 /* init new */ 1850 nb_fwd_streams = nb_fwd_streams_new; 1851 if (nb_fwd_streams) { 1852 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1853 sizeof(struct fwd_stream *) * nb_fwd_streams, 1854 RTE_CACHE_LINE_SIZE); 1855 if (fwd_streams == NULL) 1856 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1857 " (struct fwd_stream *)) failed\n", 1858 nb_fwd_streams); 1859 1860 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1861 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1862 " struct fwd_stream", sizeof(struct fwd_stream), 1863 RTE_CACHE_LINE_SIZE); 1864 if (fwd_streams[sm_id] == NULL) 1865 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1866 "(struct fwd_stream) failed\n"); 1867 } 1868 } 1869 1870 return 0; 1871 } 1872 1873 static void 1874 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1875 { 1876 uint64_t total_burst, sburst; 1877 uint64_t nb_burst; 1878 uint64_t burst_stats[4]; 1879 uint16_t pktnb_stats[4]; 1880 uint16_t nb_pkt; 1881 int burst_percent[4], sburstp; 1882 int i; 1883 1884 /* 1885 * First compute the total number of packet bursts and the 1886 * two highest numbers of bursts of the same number of packets. 1887 */ 1888 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1889 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1890 1891 /* Show stats for 0 burst size always */ 1892 total_burst = pbs->pkt_burst_spread[0]; 1893 burst_stats[0] = pbs->pkt_burst_spread[0]; 1894 pktnb_stats[0] = 0; 1895 1896 /* Find the next 2 burst sizes with highest occurrences. */ 1897 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1898 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1899 1900 if (nb_burst == 0) 1901 continue; 1902 1903 total_burst += nb_burst; 1904 1905 if (nb_burst > burst_stats[1]) { 1906 burst_stats[2] = burst_stats[1]; 1907 pktnb_stats[2] = pktnb_stats[1]; 1908 burst_stats[1] = nb_burst; 1909 pktnb_stats[1] = nb_pkt; 1910 } else if (nb_burst > burst_stats[2]) { 1911 burst_stats[2] = nb_burst; 1912 pktnb_stats[2] = nb_pkt; 1913 } 1914 } 1915 if (total_burst == 0) 1916 return; 1917 1918 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1919 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1920 if (i == 3) { 1921 printf("%d%% of other]\n", 100 - sburstp); 1922 return; 1923 } 1924 1925 sburst += burst_stats[i]; 1926 if (sburst == total_burst) { 1927 printf("%d%% of %d pkts]\n", 1928 100 - sburstp, (int) pktnb_stats[i]); 1929 return; 1930 } 1931 1932 burst_percent[i] = 1933 (double)burst_stats[i] / total_burst * 100; 1934 printf("%d%% of %d pkts + ", 1935 burst_percent[i], (int) pktnb_stats[i]); 1936 sburstp += burst_percent[i]; 1937 } 1938 } 1939 1940 static void 1941 fwd_stream_stats_display(streamid_t stream_id) 1942 { 1943 struct fwd_stream *fs; 1944 static const char *fwd_top_stats_border = "-------"; 1945 1946 fs = fwd_streams[stream_id]; 1947 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1948 (fs->fwd_dropped == 0)) 1949 return; 1950 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1951 "TX Port=%2d/Queue=%2d %s\n", 1952 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1953 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1954 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1955 " TX-dropped: %-14"PRIu64, 1956 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1957 1958 /* if checksum mode */ 1959 if (cur_fwd_eng == &csum_fwd_engine) { 1960 printf(" RX- bad IP checksum: %-14"PRIu64 1961 " Rx- bad L4 checksum: %-14"PRIu64 1962 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1963 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1964 fs->rx_bad_outer_l4_csum); 1965 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1966 fs->rx_bad_outer_ip_csum); 1967 } else { 1968 printf("\n"); 1969 } 1970 1971 if (record_burst_stats) { 1972 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1973 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1974 } 1975 } 1976 1977 void 1978 fwd_stats_display(void) 1979 { 1980 static const char *fwd_stats_border = "----------------------"; 1981 static const char *acc_stats_border = "+++++++++++++++"; 1982 struct { 1983 struct fwd_stream *rx_stream; 1984 struct fwd_stream *tx_stream; 1985 uint64_t tx_dropped; 1986 uint64_t rx_bad_ip_csum; 1987 uint64_t rx_bad_l4_csum; 1988 uint64_t rx_bad_outer_l4_csum; 1989 uint64_t rx_bad_outer_ip_csum; 1990 } ports_stats[RTE_MAX_ETHPORTS]; 1991 uint64_t total_rx_dropped = 0; 1992 uint64_t total_tx_dropped = 0; 1993 uint64_t total_rx_nombuf = 0; 1994 struct rte_eth_stats stats; 1995 uint64_t fwd_cycles = 0; 1996 uint64_t total_recv = 0; 1997 uint64_t total_xmit = 0; 1998 struct rte_port *port; 1999 streamid_t sm_id; 2000 portid_t pt_id; 2001 int i; 2002 2003 memset(ports_stats, 0, sizeof(ports_stats)); 2004 2005 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2006 struct fwd_stream *fs = fwd_streams[sm_id]; 2007 2008 if (cur_fwd_config.nb_fwd_streams > 2009 cur_fwd_config.nb_fwd_ports) { 2010 fwd_stream_stats_display(sm_id); 2011 } else { 2012 ports_stats[fs->tx_port].tx_stream = fs; 2013 ports_stats[fs->rx_port].rx_stream = fs; 2014 } 2015 2016 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2017 2018 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2019 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2020 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2021 fs->rx_bad_outer_l4_csum; 2022 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2023 fs->rx_bad_outer_ip_csum; 2024 2025 if (record_core_cycles) 2026 fwd_cycles += fs->core_cycles; 2027 } 2028 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2029 pt_id = fwd_ports_ids[i]; 2030 port = &ports[pt_id]; 2031 2032 rte_eth_stats_get(pt_id, &stats); 2033 stats.ipackets -= port->stats.ipackets; 2034 stats.opackets -= port->stats.opackets; 2035 stats.ibytes -= port->stats.ibytes; 2036 stats.obytes -= port->stats.obytes; 2037 stats.imissed -= port->stats.imissed; 2038 stats.oerrors -= port->stats.oerrors; 2039 stats.rx_nombuf -= port->stats.rx_nombuf; 2040 2041 total_recv += stats.ipackets; 2042 total_xmit += stats.opackets; 2043 total_rx_dropped += stats.imissed; 2044 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2045 total_tx_dropped += stats.oerrors; 2046 total_rx_nombuf += stats.rx_nombuf; 2047 2048 printf("\n %s Forward statistics for port %-2d %s\n", 2049 fwd_stats_border, pt_id, fwd_stats_border); 2050 2051 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2052 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2053 stats.ipackets + stats.imissed); 2054 2055 if (cur_fwd_eng == &csum_fwd_engine) { 2056 printf(" Bad-ipcsum: %-14"PRIu64 2057 " Bad-l4csum: %-14"PRIu64 2058 "Bad-outer-l4csum: %-14"PRIu64"\n", 2059 ports_stats[pt_id].rx_bad_ip_csum, 2060 ports_stats[pt_id].rx_bad_l4_csum, 2061 ports_stats[pt_id].rx_bad_outer_l4_csum); 2062 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2063 ports_stats[pt_id].rx_bad_outer_ip_csum); 2064 } 2065 if (stats.ierrors + stats.rx_nombuf > 0) { 2066 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2067 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2068 } 2069 2070 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2071 "TX-total: %-"PRIu64"\n", 2072 stats.opackets, ports_stats[pt_id].tx_dropped, 2073 stats.opackets + ports_stats[pt_id].tx_dropped); 2074 2075 if (record_burst_stats) { 2076 if (ports_stats[pt_id].rx_stream) 2077 pkt_burst_stats_display("RX", 2078 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2079 if (ports_stats[pt_id].tx_stream) 2080 pkt_burst_stats_display("TX", 2081 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2082 } 2083 2084 printf(" %s--------------------------------%s\n", 2085 fwd_stats_border, fwd_stats_border); 2086 } 2087 2088 printf("\n %s Accumulated forward statistics for all ports" 2089 "%s\n", 2090 acc_stats_border, acc_stats_border); 2091 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2092 "%-"PRIu64"\n" 2093 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2094 "%-"PRIu64"\n", 2095 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2096 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2097 if (total_rx_nombuf > 0) 2098 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2099 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2100 "%s\n", 2101 acc_stats_border, acc_stats_border); 2102 if (record_core_cycles) { 2103 #define CYC_PER_MHZ 1E6 2104 if (total_recv > 0 || total_xmit > 0) { 2105 uint64_t total_pkts = 0; 2106 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2107 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2108 total_pkts = total_xmit; 2109 else 2110 total_pkts = total_recv; 2111 2112 printf("\n CPU cycles/packet=%.2F (total cycles=" 2113 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2114 " MHz Clock\n", 2115 (double) fwd_cycles / total_pkts, 2116 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2117 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2118 } 2119 } 2120 } 2121 2122 void 2123 fwd_stats_reset(void) 2124 { 2125 streamid_t sm_id; 2126 portid_t pt_id; 2127 int i; 2128 2129 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2130 pt_id = fwd_ports_ids[i]; 2131 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2132 } 2133 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2134 struct fwd_stream *fs = fwd_streams[sm_id]; 2135 2136 fs->rx_packets = 0; 2137 fs->tx_packets = 0; 2138 fs->fwd_dropped = 0; 2139 fs->rx_bad_ip_csum = 0; 2140 fs->rx_bad_l4_csum = 0; 2141 fs->rx_bad_outer_l4_csum = 0; 2142 fs->rx_bad_outer_ip_csum = 0; 2143 2144 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2145 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2146 fs->core_cycles = 0; 2147 } 2148 } 2149 2150 static void 2151 flush_fwd_rx_queues(void) 2152 { 2153 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2154 portid_t rxp; 2155 portid_t port_id; 2156 queueid_t rxq; 2157 uint16_t nb_rx; 2158 uint16_t i; 2159 uint8_t j; 2160 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2161 uint64_t timer_period; 2162 2163 if (num_procs > 1) { 2164 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2165 return; 2166 } 2167 2168 /* convert to number of cycles */ 2169 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2170 2171 for (j = 0; j < 2; j++) { 2172 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2173 for (rxq = 0; rxq < nb_rxq; rxq++) { 2174 port_id = fwd_ports_ids[rxp]; 2175 /** 2176 * testpmd can stuck in the below do while loop 2177 * if rte_eth_rx_burst() always returns nonzero 2178 * packets. So timer is added to exit this loop 2179 * after 1sec timer expiry. 2180 */ 2181 prev_tsc = rte_rdtsc(); 2182 do { 2183 nb_rx = rte_eth_rx_burst(port_id, rxq, 2184 pkts_burst, MAX_PKT_BURST); 2185 for (i = 0; i < nb_rx; i++) 2186 rte_pktmbuf_free(pkts_burst[i]); 2187 2188 cur_tsc = rte_rdtsc(); 2189 diff_tsc = cur_tsc - prev_tsc; 2190 timer_tsc += diff_tsc; 2191 } while ((nb_rx > 0) && 2192 (timer_tsc < timer_period)); 2193 timer_tsc = 0; 2194 } 2195 } 2196 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2197 } 2198 } 2199 2200 static void 2201 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2202 { 2203 struct fwd_stream **fsm; 2204 streamid_t nb_fs; 2205 streamid_t sm_id; 2206 #ifdef RTE_LIB_BITRATESTATS 2207 uint64_t tics_per_1sec; 2208 uint64_t tics_datum; 2209 uint64_t tics_current; 2210 uint16_t i, cnt_ports; 2211 2212 cnt_ports = nb_ports; 2213 tics_datum = rte_rdtsc(); 2214 tics_per_1sec = rte_get_timer_hz(); 2215 #endif 2216 fsm = &fwd_streams[fc->stream_idx]; 2217 nb_fs = fc->stream_nb; 2218 do { 2219 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2220 (*pkt_fwd)(fsm[sm_id]); 2221 #ifdef RTE_LIB_BITRATESTATS 2222 if (bitrate_enabled != 0 && 2223 bitrate_lcore_id == rte_lcore_id()) { 2224 tics_current = rte_rdtsc(); 2225 if (tics_current - tics_datum >= tics_per_1sec) { 2226 /* Periodic bitrate calculation */ 2227 for (i = 0; i < cnt_ports; i++) 2228 rte_stats_bitrate_calc(bitrate_data, 2229 ports_ids[i]); 2230 tics_datum = tics_current; 2231 } 2232 } 2233 #endif 2234 #ifdef RTE_LIB_LATENCYSTATS 2235 if (latencystats_enabled != 0 && 2236 latencystats_lcore_id == rte_lcore_id()) 2237 rte_latencystats_update(); 2238 #endif 2239 2240 } while (! fc->stopped); 2241 } 2242 2243 static int 2244 start_pkt_forward_on_core(void *fwd_arg) 2245 { 2246 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2247 cur_fwd_config.fwd_eng->packet_fwd); 2248 return 0; 2249 } 2250 2251 /* 2252 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2253 * Used to start communication flows in network loopback test configurations. 2254 */ 2255 static int 2256 run_one_txonly_burst_on_core(void *fwd_arg) 2257 { 2258 struct fwd_lcore *fwd_lc; 2259 struct fwd_lcore tmp_lcore; 2260 2261 fwd_lc = (struct fwd_lcore *) fwd_arg; 2262 tmp_lcore = *fwd_lc; 2263 tmp_lcore.stopped = 1; 2264 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2265 return 0; 2266 } 2267 2268 /* 2269 * Launch packet forwarding: 2270 * - Setup per-port forwarding context. 2271 * - launch logical cores with their forwarding configuration. 2272 */ 2273 static void 2274 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2275 { 2276 unsigned int i; 2277 unsigned int lc_id; 2278 int diag; 2279 2280 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2281 lc_id = fwd_lcores_cpuids[i]; 2282 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2283 fwd_lcores[i]->stopped = 0; 2284 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2285 fwd_lcores[i], lc_id); 2286 if (diag != 0) 2287 fprintf(stderr, 2288 "launch lcore %u failed - diag=%d\n", 2289 lc_id, diag); 2290 } 2291 } 2292 } 2293 2294 /* 2295 * Launch packet forwarding configuration. 2296 */ 2297 void 2298 start_packet_forwarding(int with_tx_first) 2299 { 2300 port_fwd_begin_t port_fwd_begin; 2301 port_fwd_end_t port_fwd_end; 2302 unsigned int i; 2303 2304 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2305 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2306 2307 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2308 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2309 2310 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2311 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2312 (!nb_rxq || !nb_txq)) 2313 rte_exit(EXIT_FAILURE, 2314 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2315 cur_fwd_eng->fwd_mode_name); 2316 2317 if (all_ports_started() == 0) { 2318 fprintf(stderr, "Not all ports were started\n"); 2319 return; 2320 } 2321 if (test_done == 0) { 2322 fprintf(stderr, "Packet forwarding already started\n"); 2323 return; 2324 } 2325 2326 fwd_config_setup(); 2327 2328 pkt_fwd_config_display(&cur_fwd_config); 2329 if (!pkt_fwd_shared_rxq_check()) 2330 return; 2331 2332 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2333 if (port_fwd_begin != NULL) { 2334 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2335 if (port_fwd_begin(fwd_ports_ids[i])) { 2336 fprintf(stderr, 2337 "Packet forwarding is not ready\n"); 2338 return; 2339 } 2340 } 2341 } 2342 2343 if (with_tx_first) { 2344 port_fwd_begin = tx_only_engine.port_fwd_begin; 2345 if (port_fwd_begin != NULL) { 2346 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2347 if (port_fwd_begin(fwd_ports_ids[i])) { 2348 fprintf(stderr, 2349 "Packet forwarding is not ready\n"); 2350 return; 2351 } 2352 } 2353 } 2354 } 2355 2356 test_done = 0; 2357 2358 if(!no_flush_rx) 2359 flush_fwd_rx_queues(); 2360 2361 rxtx_config_display(); 2362 2363 fwd_stats_reset(); 2364 if (with_tx_first) { 2365 while (with_tx_first--) { 2366 launch_packet_forwarding( 2367 run_one_txonly_burst_on_core); 2368 rte_eal_mp_wait_lcore(); 2369 } 2370 port_fwd_end = tx_only_engine.port_fwd_end; 2371 if (port_fwd_end != NULL) { 2372 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2373 (*port_fwd_end)(fwd_ports_ids[i]); 2374 } 2375 } 2376 launch_packet_forwarding(start_pkt_forward_on_core); 2377 } 2378 2379 void 2380 stop_packet_forwarding(void) 2381 { 2382 port_fwd_end_t port_fwd_end; 2383 lcoreid_t lc_id; 2384 portid_t pt_id; 2385 int i; 2386 2387 if (test_done) { 2388 fprintf(stderr, "Packet forwarding not started\n"); 2389 return; 2390 } 2391 printf("Telling cores to stop..."); 2392 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2393 fwd_lcores[lc_id]->stopped = 1; 2394 printf("\nWaiting for lcores to finish...\n"); 2395 rte_eal_mp_wait_lcore(); 2396 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2397 if (port_fwd_end != NULL) { 2398 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2399 pt_id = fwd_ports_ids[i]; 2400 (*port_fwd_end)(pt_id); 2401 } 2402 } 2403 2404 fwd_stats_display(); 2405 2406 printf("\nDone.\n"); 2407 test_done = 1; 2408 } 2409 2410 void 2411 dev_set_link_up(portid_t pid) 2412 { 2413 if (rte_eth_dev_set_link_up(pid) < 0) 2414 fprintf(stderr, "\nSet link up fail.\n"); 2415 } 2416 2417 void 2418 dev_set_link_down(portid_t pid) 2419 { 2420 if (rte_eth_dev_set_link_down(pid) < 0) 2421 fprintf(stderr, "\nSet link down fail.\n"); 2422 } 2423 2424 static int 2425 all_ports_started(void) 2426 { 2427 portid_t pi; 2428 struct rte_port *port; 2429 2430 RTE_ETH_FOREACH_DEV(pi) { 2431 port = &ports[pi]; 2432 /* Check if there is a port which is not started */ 2433 if ((port->port_status != RTE_PORT_STARTED) && 2434 (port->slave_flag == 0)) 2435 return 0; 2436 } 2437 2438 /* No port is not started */ 2439 return 1; 2440 } 2441 2442 int 2443 port_is_stopped(portid_t port_id) 2444 { 2445 struct rte_port *port = &ports[port_id]; 2446 2447 if ((port->port_status != RTE_PORT_STOPPED) && 2448 (port->slave_flag == 0)) 2449 return 0; 2450 return 1; 2451 } 2452 2453 int 2454 all_ports_stopped(void) 2455 { 2456 portid_t pi; 2457 2458 RTE_ETH_FOREACH_DEV(pi) { 2459 if (!port_is_stopped(pi)) 2460 return 0; 2461 } 2462 2463 return 1; 2464 } 2465 2466 int 2467 port_is_started(portid_t port_id) 2468 { 2469 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2470 return 0; 2471 2472 if (ports[port_id].port_status != RTE_PORT_STARTED) 2473 return 0; 2474 2475 return 1; 2476 } 2477 2478 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2479 static int 2480 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2481 { 2482 queueid_t qi; 2483 struct rte_eth_hairpin_conf hairpin_conf = { 2484 .peer_count = 1, 2485 }; 2486 int i; 2487 int diag; 2488 struct rte_port *port = &ports[pi]; 2489 uint16_t peer_rx_port = pi; 2490 uint16_t peer_tx_port = pi; 2491 uint32_t manual = 1; 2492 uint32_t tx_exp = hairpin_mode & 0x10; 2493 2494 if (!(hairpin_mode & 0xf)) { 2495 peer_rx_port = pi; 2496 peer_tx_port = pi; 2497 manual = 0; 2498 } else if (hairpin_mode & 0x1) { 2499 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2500 RTE_ETH_DEV_NO_OWNER); 2501 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2502 peer_tx_port = rte_eth_find_next_owned_by(0, 2503 RTE_ETH_DEV_NO_OWNER); 2504 if (p_pi != RTE_MAX_ETHPORTS) { 2505 peer_rx_port = p_pi; 2506 } else { 2507 uint16_t next_pi; 2508 2509 /* Last port will be the peer RX port of the first. */ 2510 RTE_ETH_FOREACH_DEV(next_pi) 2511 peer_rx_port = next_pi; 2512 } 2513 manual = 1; 2514 } else if (hairpin_mode & 0x2) { 2515 if (cnt_pi & 0x1) { 2516 peer_rx_port = p_pi; 2517 } else { 2518 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2519 RTE_ETH_DEV_NO_OWNER); 2520 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2521 peer_rx_port = pi; 2522 } 2523 peer_tx_port = peer_rx_port; 2524 manual = 1; 2525 } 2526 2527 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2528 hairpin_conf.peers[0].port = peer_rx_port; 2529 hairpin_conf.peers[0].queue = i + nb_rxq; 2530 hairpin_conf.manual_bind = !!manual; 2531 hairpin_conf.tx_explicit = !!tx_exp; 2532 diag = rte_eth_tx_hairpin_queue_setup 2533 (pi, qi, nb_txd, &hairpin_conf); 2534 i++; 2535 if (diag == 0) 2536 continue; 2537 2538 /* Fail to setup rx queue, return */ 2539 if (port->port_status == RTE_PORT_HANDLING) 2540 port->port_status = RTE_PORT_STOPPED; 2541 else 2542 fprintf(stderr, 2543 "Port %d can not be set back to stopped\n", pi); 2544 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2545 pi); 2546 /* try to reconfigure queues next time */ 2547 port->need_reconfig_queues = 1; 2548 return -1; 2549 } 2550 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2551 hairpin_conf.peers[0].port = peer_tx_port; 2552 hairpin_conf.peers[0].queue = i + nb_txq; 2553 hairpin_conf.manual_bind = !!manual; 2554 hairpin_conf.tx_explicit = !!tx_exp; 2555 diag = rte_eth_rx_hairpin_queue_setup 2556 (pi, qi, nb_rxd, &hairpin_conf); 2557 i++; 2558 if (diag == 0) 2559 continue; 2560 2561 /* Fail to setup rx queue, return */ 2562 if (port->port_status == RTE_PORT_HANDLING) 2563 port->port_status = RTE_PORT_STOPPED; 2564 else 2565 fprintf(stderr, 2566 "Port %d can not be set back to stopped\n", pi); 2567 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2568 pi); 2569 /* try to reconfigure queues next time */ 2570 port->need_reconfig_queues = 1; 2571 return -1; 2572 } 2573 return 0; 2574 } 2575 2576 /* Configure the Rx with optional split. */ 2577 int 2578 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2579 uint16_t nb_rx_desc, unsigned int socket_id, 2580 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2581 { 2582 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2583 unsigned int i, mp_n; 2584 int ret; 2585 2586 if (rx_pkt_nb_segs <= 1 || 2587 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2588 rx_conf->rx_seg = NULL; 2589 rx_conf->rx_nseg = 0; 2590 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2591 nb_rx_desc, socket_id, 2592 rx_conf, mp); 2593 return ret; 2594 } 2595 for (i = 0; i < rx_pkt_nb_segs; i++) { 2596 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2597 struct rte_mempool *mpx; 2598 /* 2599 * Use last valid pool for the segments with number 2600 * exceeding the pool index. 2601 */ 2602 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2603 mpx = mbuf_pool_find(socket_id, mp_n); 2604 /* Handle zero as mbuf data buffer size. */ 2605 rx_seg->length = rx_pkt_seg_lengths[i] ? 2606 rx_pkt_seg_lengths[i] : 2607 mbuf_data_size[mp_n]; 2608 rx_seg->offset = i < rx_pkt_nb_offs ? 2609 rx_pkt_seg_offsets[i] : 0; 2610 rx_seg->mp = mpx ? mpx : mp; 2611 } 2612 rx_conf->rx_nseg = rx_pkt_nb_segs; 2613 rx_conf->rx_seg = rx_useg; 2614 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2615 socket_id, rx_conf, NULL); 2616 rx_conf->rx_seg = NULL; 2617 rx_conf->rx_nseg = 0; 2618 return ret; 2619 } 2620 2621 static int 2622 alloc_xstats_display_info(portid_t pi) 2623 { 2624 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2625 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2626 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2627 2628 if (xstats_display_num == 0) 2629 return 0; 2630 2631 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2632 if (*ids_supp == NULL) 2633 goto fail_ids_supp; 2634 2635 *prev_values = calloc(xstats_display_num, 2636 sizeof(**prev_values)); 2637 if (*prev_values == NULL) 2638 goto fail_prev_values; 2639 2640 *curr_values = calloc(xstats_display_num, 2641 sizeof(**curr_values)); 2642 if (*curr_values == NULL) 2643 goto fail_curr_values; 2644 2645 ports[pi].xstats_info.allocated = true; 2646 2647 return 0; 2648 2649 fail_curr_values: 2650 free(*prev_values); 2651 fail_prev_values: 2652 free(*ids_supp); 2653 fail_ids_supp: 2654 return -ENOMEM; 2655 } 2656 2657 static void 2658 free_xstats_display_info(portid_t pi) 2659 { 2660 if (!ports[pi].xstats_info.allocated) 2661 return; 2662 free(ports[pi].xstats_info.ids_supp); 2663 free(ports[pi].xstats_info.prev_values); 2664 free(ports[pi].xstats_info.curr_values); 2665 ports[pi].xstats_info.allocated = false; 2666 } 2667 2668 /** Fill helper structures for specified port to show extended statistics. */ 2669 static void 2670 fill_xstats_display_info_for_port(portid_t pi) 2671 { 2672 unsigned int stat, stat_supp; 2673 const char *xstat_name; 2674 struct rte_port *port; 2675 uint64_t *ids_supp; 2676 int rc; 2677 2678 if (xstats_display_num == 0) 2679 return; 2680 2681 if (pi == (portid_t)RTE_PORT_ALL) { 2682 fill_xstats_display_info(); 2683 return; 2684 } 2685 2686 port = &ports[pi]; 2687 if (port->port_status != RTE_PORT_STARTED) 2688 return; 2689 2690 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2691 rte_exit(EXIT_FAILURE, 2692 "Failed to allocate xstats display memory\n"); 2693 2694 ids_supp = port->xstats_info.ids_supp; 2695 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2696 xstat_name = xstats_display[stat].name; 2697 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2698 ids_supp + stat_supp); 2699 if (rc != 0) { 2700 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2701 xstat_name, pi, stat); 2702 continue; 2703 } 2704 stat_supp++; 2705 } 2706 2707 port->xstats_info.ids_supp_sz = stat_supp; 2708 } 2709 2710 /** Fill helper structures for all ports to show extended statistics. */ 2711 static void 2712 fill_xstats_display_info(void) 2713 { 2714 portid_t pi; 2715 2716 if (xstats_display_num == 0) 2717 return; 2718 2719 RTE_ETH_FOREACH_DEV(pi) 2720 fill_xstats_display_info_for_port(pi); 2721 } 2722 2723 int 2724 start_port(portid_t pid) 2725 { 2726 int diag, need_check_link_status = -1; 2727 portid_t pi; 2728 portid_t p_pi = RTE_MAX_ETHPORTS; 2729 portid_t pl[RTE_MAX_ETHPORTS]; 2730 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2731 uint16_t cnt_pi = 0; 2732 uint16_t cfg_pi = 0; 2733 int peer_pi; 2734 queueid_t qi; 2735 struct rte_port *port; 2736 struct rte_eth_hairpin_cap cap; 2737 2738 if (port_id_is_invalid(pid, ENABLED_WARN)) 2739 return 0; 2740 2741 RTE_ETH_FOREACH_DEV(pi) { 2742 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2743 continue; 2744 2745 need_check_link_status = 0; 2746 port = &ports[pi]; 2747 if (port->port_status == RTE_PORT_STOPPED) 2748 port->port_status = RTE_PORT_HANDLING; 2749 else { 2750 fprintf(stderr, "Port %d is now not stopped\n", pi); 2751 continue; 2752 } 2753 2754 if (port->need_reconfig > 0) { 2755 struct rte_eth_conf dev_conf; 2756 int k; 2757 2758 port->need_reconfig = 0; 2759 2760 if (flow_isolate_all) { 2761 int ret = port_flow_isolate(pi, 1); 2762 if (ret) { 2763 fprintf(stderr, 2764 "Failed to apply isolated mode on port %d\n", 2765 pi); 2766 return -1; 2767 } 2768 } 2769 configure_rxtx_dump_callbacks(0); 2770 printf("Configuring Port %d (socket %u)\n", pi, 2771 port->socket_id); 2772 if (nb_hairpinq > 0 && 2773 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2774 fprintf(stderr, 2775 "Port %d doesn't support hairpin queues\n", 2776 pi); 2777 return -1; 2778 } 2779 2780 /* configure port */ 2781 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2782 nb_txq + nb_hairpinq, 2783 &(port->dev_conf)); 2784 if (diag != 0) { 2785 if (port->port_status == RTE_PORT_HANDLING) 2786 port->port_status = RTE_PORT_STOPPED; 2787 else 2788 fprintf(stderr, 2789 "Port %d can not be set back to stopped\n", 2790 pi); 2791 fprintf(stderr, "Fail to configure port %d\n", 2792 pi); 2793 /* try to reconfigure port next time */ 2794 port->need_reconfig = 1; 2795 return -1; 2796 } 2797 /* get device configuration*/ 2798 if (0 != 2799 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2800 fprintf(stderr, 2801 "port %d can not get device configuration\n", 2802 pi); 2803 return -1; 2804 } 2805 /* Apply Rx offloads configuration */ 2806 if (dev_conf.rxmode.offloads != 2807 port->dev_conf.rxmode.offloads) { 2808 port->dev_conf.rxmode.offloads |= 2809 dev_conf.rxmode.offloads; 2810 for (k = 0; 2811 k < port->dev_info.max_rx_queues; 2812 k++) 2813 port->rx_conf[k].offloads |= 2814 dev_conf.rxmode.offloads; 2815 } 2816 /* Apply Tx offloads configuration */ 2817 if (dev_conf.txmode.offloads != 2818 port->dev_conf.txmode.offloads) { 2819 port->dev_conf.txmode.offloads |= 2820 dev_conf.txmode.offloads; 2821 for (k = 0; 2822 k < port->dev_info.max_tx_queues; 2823 k++) 2824 port->tx_conf[k].offloads |= 2825 dev_conf.txmode.offloads; 2826 } 2827 } 2828 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2829 port->need_reconfig_queues = 0; 2830 /* setup tx queues */ 2831 for (qi = 0; qi < nb_txq; qi++) { 2832 if ((numa_support) && 2833 (txring_numa[pi] != NUMA_NO_CONFIG)) 2834 diag = rte_eth_tx_queue_setup(pi, qi, 2835 port->nb_tx_desc[qi], 2836 txring_numa[pi], 2837 &(port->tx_conf[qi])); 2838 else 2839 diag = rte_eth_tx_queue_setup(pi, qi, 2840 port->nb_tx_desc[qi], 2841 port->socket_id, 2842 &(port->tx_conf[qi])); 2843 2844 if (diag == 0) 2845 continue; 2846 2847 /* Fail to setup tx queue, return */ 2848 if (port->port_status == RTE_PORT_HANDLING) 2849 port->port_status = RTE_PORT_STOPPED; 2850 else 2851 fprintf(stderr, 2852 "Port %d can not be set back to stopped\n", 2853 pi); 2854 fprintf(stderr, 2855 "Fail to configure port %d tx queues\n", 2856 pi); 2857 /* try to reconfigure queues next time */ 2858 port->need_reconfig_queues = 1; 2859 return -1; 2860 } 2861 for (qi = 0; qi < nb_rxq; qi++) { 2862 /* setup rx queues */ 2863 if ((numa_support) && 2864 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2865 struct rte_mempool * mp = 2866 mbuf_pool_find 2867 (rxring_numa[pi], 0); 2868 if (mp == NULL) { 2869 fprintf(stderr, 2870 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2871 rxring_numa[pi]); 2872 return -1; 2873 } 2874 2875 diag = rx_queue_setup(pi, qi, 2876 port->nb_rx_desc[qi], 2877 rxring_numa[pi], 2878 &(port->rx_conf[qi]), 2879 mp); 2880 } else { 2881 struct rte_mempool *mp = 2882 mbuf_pool_find 2883 (port->socket_id, 0); 2884 if (mp == NULL) { 2885 fprintf(stderr, 2886 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2887 port->socket_id); 2888 return -1; 2889 } 2890 diag = rx_queue_setup(pi, qi, 2891 port->nb_rx_desc[qi], 2892 port->socket_id, 2893 &(port->rx_conf[qi]), 2894 mp); 2895 } 2896 if (diag == 0) 2897 continue; 2898 2899 /* Fail to setup rx queue, return */ 2900 if (port->port_status == RTE_PORT_HANDLING) 2901 port->port_status = RTE_PORT_STOPPED; 2902 else 2903 fprintf(stderr, 2904 "Port %d can not be set back to stopped\n", 2905 pi); 2906 fprintf(stderr, 2907 "Fail to configure port %d rx queues\n", 2908 pi); 2909 /* try to reconfigure queues next time */ 2910 port->need_reconfig_queues = 1; 2911 return -1; 2912 } 2913 /* setup hairpin queues */ 2914 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2915 return -1; 2916 } 2917 configure_rxtx_dump_callbacks(verbose_level); 2918 if (clear_ptypes) { 2919 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2920 NULL, 0); 2921 if (diag < 0) 2922 fprintf(stderr, 2923 "Port %d: Failed to disable Ptype parsing\n", 2924 pi); 2925 } 2926 2927 p_pi = pi; 2928 cnt_pi++; 2929 2930 /* start port */ 2931 diag = eth_dev_start_mp(pi); 2932 if (diag < 0) { 2933 fprintf(stderr, "Fail to start port %d: %s\n", 2934 pi, rte_strerror(-diag)); 2935 2936 /* Fail to setup rx queue, return */ 2937 if (port->port_status == RTE_PORT_HANDLING) 2938 port->port_status = RTE_PORT_STOPPED; 2939 else 2940 fprintf(stderr, 2941 "Port %d can not be set back to stopped\n", 2942 pi); 2943 continue; 2944 } 2945 2946 if (port->port_status == RTE_PORT_HANDLING) 2947 port->port_status = RTE_PORT_STARTED; 2948 else 2949 fprintf(stderr, "Port %d can not be set into started\n", 2950 pi); 2951 2952 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 2953 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 2954 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 2955 2956 /* at least one port started, need checking link status */ 2957 need_check_link_status = 1; 2958 2959 pl[cfg_pi++] = pi; 2960 } 2961 2962 if (need_check_link_status == 1 && !no_link_check) 2963 check_all_ports_link_status(RTE_PORT_ALL); 2964 else if (need_check_link_status == 0) 2965 fprintf(stderr, "Please stop the ports first\n"); 2966 2967 if (hairpin_mode & 0xf) { 2968 uint16_t i; 2969 int j; 2970 2971 /* bind all started hairpin ports */ 2972 for (i = 0; i < cfg_pi; i++) { 2973 pi = pl[i]; 2974 /* bind current Tx to all peer Rx */ 2975 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2976 RTE_MAX_ETHPORTS, 1); 2977 if (peer_pi < 0) 2978 return peer_pi; 2979 for (j = 0; j < peer_pi; j++) { 2980 if (!port_is_started(peer_pl[j])) 2981 continue; 2982 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2983 if (diag < 0) { 2984 fprintf(stderr, 2985 "Error during binding hairpin Tx port %u to %u: %s\n", 2986 pi, peer_pl[j], 2987 rte_strerror(-diag)); 2988 return -1; 2989 } 2990 } 2991 /* bind all peer Tx to current Rx */ 2992 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2993 RTE_MAX_ETHPORTS, 0); 2994 if (peer_pi < 0) 2995 return peer_pi; 2996 for (j = 0; j < peer_pi; j++) { 2997 if (!port_is_started(peer_pl[j])) 2998 continue; 2999 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 3000 if (diag < 0) { 3001 fprintf(stderr, 3002 "Error during binding hairpin Tx port %u to %u: %s\n", 3003 peer_pl[j], pi, 3004 rte_strerror(-diag)); 3005 return -1; 3006 } 3007 } 3008 } 3009 } 3010 3011 fill_xstats_display_info_for_port(pid); 3012 3013 printf("Done\n"); 3014 return 0; 3015 } 3016 3017 void 3018 stop_port(portid_t pid) 3019 { 3020 portid_t pi; 3021 struct rte_port *port; 3022 int need_check_link_status = 0; 3023 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3024 int peer_pi; 3025 3026 if (port_id_is_invalid(pid, ENABLED_WARN)) 3027 return; 3028 3029 printf("Stopping ports...\n"); 3030 3031 RTE_ETH_FOREACH_DEV(pi) { 3032 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3033 continue; 3034 3035 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3036 fprintf(stderr, 3037 "Please remove port %d from forwarding configuration.\n", 3038 pi); 3039 continue; 3040 } 3041 3042 if (port_is_bonding_slave(pi)) { 3043 fprintf(stderr, 3044 "Please remove port %d from bonded device.\n", 3045 pi); 3046 continue; 3047 } 3048 3049 port = &ports[pi]; 3050 if (port->port_status == RTE_PORT_STARTED) 3051 port->port_status = RTE_PORT_HANDLING; 3052 else 3053 continue; 3054 3055 if (hairpin_mode & 0xf) { 3056 int j; 3057 3058 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3059 /* unbind all peer Tx from current Rx */ 3060 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3061 RTE_MAX_ETHPORTS, 0); 3062 if (peer_pi < 0) 3063 continue; 3064 for (j = 0; j < peer_pi; j++) { 3065 if (!port_is_started(peer_pl[j])) 3066 continue; 3067 rte_eth_hairpin_unbind(peer_pl[j], pi); 3068 } 3069 } 3070 3071 if (port->flow_list) 3072 port_flow_flush(pi); 3073 3074 if (eth_dev_stop_mp(pi) != 0) 3075 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3076 pi); 3077 3078 if (port->port_status == RTE_PORT_HANDLING) 3079 port->port_status = RTE_PORT_STOPPED; 3080 else 3081 fprintf(stderr, "Port %d can not be set into stopped\n", 3082 pi); 3083 need_check_link_status = 1; 3084 } 3085 if (need_check_link_status && !no_link_check) 3086 check_all_ports_link_status(RTE_PORT_ALL); 3087 3088 printf("Done\n"); 3089 } 3090 3091 static void 3092 remove_invalid_ports_in(portid_t *array, portid_t *total) 3093 { 3094 portid_t i; 3095 portid_t new_total = 0; 3096 3097 for (i = 0; i < *total; i++) 3098 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3099 array[new_total] = array[i]; 3100 new_total++; 3101 } 3102 *total = new_total; 3103 } 3104 3105 static void 3106 remove_invalid_ports(void) 3107 { 3108 remove_invalid_ports_in(ports_ids, &nb_ports); 3109 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3110 nb_cfg_ports = nb_fwd_ports; 3111 } 3112 3113 void 3114 close_port(portid_t pid) 3115 { 3116 portid_t pi; 3117 struct rte_port *port; 3118 3119 if (port_id_is_invalid(pid, ENABLED_WARN)) 3120 return; 3121 3122 printf("Closing ports...\n"); 3123 3124 RTE_ETH_FOREACH_DEV(pi) { 3125 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3126 continue; 3127 3128 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3129 fprintf(stderr, 3130 "Please remove port %d from forwarding configuration.\n", 3131 pi); 3132 continue; 3133 } 3134 3135 if (port_is_bonding_slave(pi)) { 3136 fprintf(stderr, 3137 "Please remove port %d from bonded device.\n", 3138 pi); 3139 continue; 3140 } 3141 3142 port = &ports[pi]; 3143 if (port->port_status == RTE_PORT_CLOSED) { 3144 fprintf(stderr, "Port %d is already closed\n", pi); 3145 continue; 3146 } 3147 3148 if (is_proc_primary()) { 3149 port_flow_flush(pi); 3150 port_flex_item_flush(pi); 3151 rte_eth_dev_close(pi); 3152 } 3153 3154 free_xstats_display_info(pi); 3155 } 3156 3157 remove_invalid_ports(); 3158 printf("Done\n"); 3159 } 3160 3161 void 3162 reset_port(portid_t pid) 3163 { 3164 int diag; 3165 portid_t pi; 3166 struct rte_port *port; 3167 3168 if (port_id_is_invalid(pid, ENABLED_WARN)) 3169 return; 3170 3171 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3172 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3173 fprintf(stderr, 3174 "Can not reset port(s), please stop port(s) first.\n"); 3175 return; 3176 } 3177 3178 printf("Resetting ports...\n"); 3179 3180 RTE_ETH_FOREACH_DEV(pi) { 3181 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3182 continue; 3183 3184 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3185 fprintf(stderr, 3186 "Please remove port %d from forwarding configuration.\n", 3187 pi); 3188 continue; 3189 } 3190 3191 if (port_is_bonding_slave(pi)) { 3192 fprintf(stderr, 3193 "Please remove port %d from bonded device.\n", 3194 pi); 3195 continue; 3196 } 3197 3198 diag = rte_eth_dev_reset(pi); 3199 if (diag == 0) { 3200 port = &ports[pi]; 3201 port->need_reconfig = 1; 3202 port->need_reconfig_queues = 1; 3203 } else { 3204 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3205 pi, diag); 3206 } 3207 } 3208 3209 printf("Done\n"); 3210 } 3211 3212 void 3213 attach_port(char *identifier) 3214 { 3215 portid_t pi; 3216 struct rte_dev_iterator iterator; 3217 3218 printf("Attaching a new port...\n"); 3219 3220 if (identifier == NULL) { 3221 fprintf(stderr, "Invalid parameters are specified\n"); 3222 return; 3223 } 3224 3225 if (rte_dev_probe(identifier) < 0) { 3226 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3227 return; 3228 } 3229 3230 /* first attach mode: event */ 3231 if (setup_on_probe_event) { 3232 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3233 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3234 if (ports[pi].port_status == RTE_PORT_HANDLING && 3235 ports[pi].need_setup != 0) 3236 setup_attached_port(pi); 3237 return; 3238 } 3239 3240 /* second attach mode: iterator */ 3241 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3242 /* setup ports matching the devargs used for probing */ 3243 if (port_is_forwarding(pi)) 3244 continue; /* port was already attached before */ 3245 setup_attached_port(pi); 3246 } 3247 } 3248 3249 static void 3250 setup_attached_port(portid_t pi) 3251 { 3252 unsigned int socket_id; 3253 int ret; 3254 3255 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3256 /* if socket_id is invalid, set to the first available socket. */ 3257 if (check_socket_id(socket_id) < 0) 3258 socket_id = socket_ids[0]; 3259 reconfig(pi, socket_id); 3260 ret = rte_eth_promiscuous_enable(pi); 3261 if (ret != 0) 3262 fprintf(stderr, 3263 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3264 pi, rte_strerror(-ret)); 3265 3266 ports_ids[nb_ports++] = pi; 3267 fwd_ports_ids[nb_fwd_ports++] = pi; 3268 nb_cfg_ports = nb_fwd_ports; 3269 ports[pi].need_setup = 0; 3270 ports[pi].port_status = RTE_PORT_STOPPED; 3271 3272 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3273 printf("Done\n"); 3274 } 3275 3276 static void 3277 detach_device(struct rte_device *dev) 3278 { 3279 portid_t sibling; 3280 3281 if (dev == NULL) { 3282 fprintf(stderr, "Device already removed\n"); 3283 return; 3284 } 3285 3286 printf("Removing a device...\n"); 3287 3288 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3289 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3290 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3291 fprintf(stderr, "Port %u not stopped\n", 3292 sibling); 3293 return; 3294 } 3295 port_flow_flush(sibling); 3296 } 3297 } 3298 3299 if (rte_dev_remove(dev) < 0) { 3300 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3301 return; 3302 } 3303 remove_invalid_ports(); 3304 3305 printf("Device is detached\n"); 3306 printf("Now total ports is %d\n", nb_ports); 3307 printf("Done\n"); 3308 return; 3309 } 3310 3311 void 3312 detach_port_device(portid_t port_id) 3313 { 3314 int ret; 3315 struct rte_eth_dev_info dev_info; 3316 3317 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3318 return; 3319 3320 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3321 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3322 fprintf(stderr, "Port not stopped\n"); 3323 return; 3324 } 3325 fprintf(stderr, "Port was not closed\n"); 3326 } 3327 3328 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3329 if (ret != 0) { 3330 TESTPMD_LOG(ERR, 3331 "Failed to get device info for port %d, not detaching\n", 3332 port_id); 3333 return; 3334 } 3335 detach_device(dev_info.device); 3336 } 3337 3338 void 3339 detach_devargs(char *identifier) 3340 { 3341 struct rte_dev_iterator iterator; 3342 struct rte_devargs da; 3343 portid_t port_id; 3344 3345 printf("Removing a device...\n"); 3346 3347 memset(&da, 0, sizeof(da)); 3348 if (rte_devargs_parsef(&da, "%s", identifier)) { 3349 fprintf(stderr, "cannot parse identifier\n"); 3350 return; 3351 } 3352 3353 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3354 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3355 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3356 fprintf(stderr, "Port %u not stopped\n", 3357 port_id); 3358 rte_eth_iterator_cleanup(&iterator); 3359 rte_devargs_reset(&da); 3360 return; 3361 } 3362 port_flow_flush(port_id); 3363 } 3364 } 3365 3366 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3367 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3368 da.name, da.bus->name); 3369 rte_devargs_reset(&da); 3370 return; 3371 } 3372 3373 remove_invalid_ports(); 3374 3375 printf("Device %s is detached\n", identifier); 3376 printf("Now total ports is %d\n", nb_ports); 3377 printf("Done\n"); 3378 rte_devargs_reset(&da); 3379 } 3380 3381 void 3382 pmd_test_exit(void) 3383 { 3384 portid_t pt_id; 3385 unsigned int i; 3386 int ret; 3387 3388 if (test_done == 0) 3389 stop_packet_forwarding(); 3390 3391 #ifndef RTE_EXEC_ENV_WINDOWS 3392 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3393 if (mempools[i]) { 3394 if (mp_alloc_type == MP_ALLOC_ANON) 3395 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3396 NULL); 3397 } 3398 } 3399 #endif 3400 if (ports != NULL) { 3401 no_link_check = 1; 3402 RTE_ETH_FOREACH_DEV(pt_id) { 3403 printf("\nStopping port %d...\n", pt_id); 3404 fflush(stdout); 3405 stop_port(pt_id); 3406 } 3407 RTE_ETH_FOREACH_DEV(pt_id) { 3408 printf("\nShutting down port %d...\n", pt_id); 3409 fflush(stdout); 3410 close_port(pt_id); 3411 } 3412 } 3413 3414 if (hot_plug) { 3415 ret = rte_dev_event_monitor_stop(); 3416 if (ret) { 3417 RTE_LOG(ERR, EAL, 3418 "fail to stop device event monitor."); 3419 return; 3420 } 3421 3422 ret = rte_dev_event_callback_unregister(NULL, 3423 dev_event_callback, NULL); 3424 if (ret < 0) { 3425 RTE_LOG(ERR, EAL, 3426 "fail to unregister device event callback.\n"); 3427 return; 3428 } 3429 3430 ret = rte_dev_hotplug_handle_disable(); 3431 if (ret) { 3432 RTE_LOG(ERR, EAL, 3433 "fail to disable hotplug handling.\n"); 3434 return; 3435 } 3436 } 3437 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3438 if (mempools[i]) 3439 mempool_free_mp(mempools[i]); 3440 } 3441 free(xstats_display); 3442 3443 printf("\nBye...\n"); 3444 } 3445 3446 typedef void (*cmd_func_t)(void); 3447 struct pmd_test_command { 3448 const char *cmd_name; 3449 cmd_func_t cmd_func; 3450 }; 3451 3452 /* Check the link status of all ports in up to 9s, and print them finally */ 3453 static void 3454 check_all_ports_link_status(uint32_t port_mask) 3455 { 3456 #define CHECK_INTERVAL 100 /* 100ms */ 3457 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3458 portid_t portid; 3459 uint8_t count, all_ports_up, print_flag = 0; 3460 struct rte_eth_link link; 3461 int ret; 3462 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3463 3464 printf("Checking link statuses...\n"); 3465 fflush(stdout); 3466 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3467 all_ports_up = 1; 3468 RTE_ETH_FOREACH_DEV(portid) { 3469 if ((port_mask & (1 << portid)) == 0) 3470 continue; 3471 memset(&link, 0, sizeof(link)); 3472 ret = rte_eth_link_get_nowait(portid, &link); 3473 if (ret < 0) { 3474 all_ports_up = 0; 3475 if (print_flag == 1) 3476 fprintf(stderr, 3477 "Port %u link get failed: %s\n", 3478 portid, rte_strerror(-ret)); 3479 continue; 3480 } 3481 /* print link status if flag set */ 3482 if (print_flag == 1) { 3483 rte_eth_link_to_str(link_status, 3484 sizeof(link_status), &link); 3485 printf("Port %d %s\n", portid, link_status); 3486 continue; 3487 } 3488 /* clear all_ports_up flag if any link down */ 3489 if (link.link_status == RTE_ETH_LINK_DOWN) { 3490 all_ports_up = 0; 3491 break; 3492 } 3493 } 3494 /* after finally printing all link status, get out */ 3495 if (print_flag == 1) 3496 break; 3497 3498 if (all_ports_up == 0) { 3499 fflush(stdout); 3500 rte_delay_ms(CHECK_INTERVAL); 3501 } 3502 3503 /* set the print_flag if all ports up or timeout */ 3504 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3505 print_flag = 1; 3506 } 3507 3508 if (lsc_interrupt) 3509 break; 3510 } 3511 } 3512 3513 static void 3514 rmv_port_callback(void *arg) 3515 { 3516 int need_to_start = 0; 3517 int org_no_link_check = no_link_check; 3518 portid_t port_id = (intptr_t)arg; 3519 struct rte_eth_dev_info dev_info; 3520 int ret; 3521 3522 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3523 3524 if (!test_done && port_is_forwarding(port_id)) { 3525 need_to_start = 1; 3526 stop_packet_forwarding(); 3527 } 3528 no_link_check = 1; 3529 stop_port(port_id); 3530 no_link_check = org_no_link_check; 3531 3532 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3533 if (ret != 0) 3534 TESTPMD_LOG(ERR, 3535 "Failed to get device info for port %d, not detaching\n", 3536 port_id); 3537 else { 3538 struct rte_device *device = dev_info.device; 3539 close_port(port_id); 3540 detach_device(device); /* might be already removed or have more ports */ 3541 } 3542 if (need_to_start) 3543 start_packet_forwarding(0); 3544 } 3545 3546 /* This function is used by the interrupt thread */ 3547 static int 3548 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3549 void *ret_param) 3550 { 3551 RTE_SET_USED(param); 3552 RTE_SET_USED(ret_param); 3553 3554 if (type >= RTE_ETH_EVENT_MAX) { 3555 fprintf(stderr, 3556 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3557 port_id, __func__, type); 3558 fflush(stderr); 3559 } else if (event_print_mask & (UINT32_C(1) << type)) { 3560 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3561 eth_event_desc[type]); 3562 fflush(stdout); 3563 } 3564 3565 switch (type) { 3566 case RTE_ETH_EVENT_NEW: 3567 ports[port_id].need_setup = 1; 3568 ports[port_id].port_status = RTE_PORT_HANDLING; 3569 break; 3570 case RTE_ETH_EVENT_INTR_RMV: 3571 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3572 break; 3573 if (rte_eal_alarm_set(100000, 3574 rmv_port_callback, (void *)(intptr_t)port_id)) 3575 fprintf(stderr, 3576 "Could not set up deferred device removal\n"); 3577 break; 3578 case RTE_ETH_EVENT_DESTROY: 3579 ports[port_id].port_status = RTE_PORT_CLOSED; 3580 printf("Port %u is closed\n", port_id); 3581 break; 3582 default: 3583 break; 3584 } 3585 return 0; 3586 } 3587 3588 static int 3589 register_eth_event_callback(void) 3590 { 3591 int ret; 3592 enum rte_eth_event_type event; 3593 3594 for (event = RTE_ETH_EVENT_UNKNOWN; 3595 event < RTE_ETH_EVENT_MAX; event++) { 3596 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3597 event, 3598 eth_event_callback, 3599 NULL); 3600 if (ret != 0) { 3601 TESTPMD_LOG(ERR, "Failed to register callback for " 3602 "%s event\n", eth_event_desc[event]); 3603 return -1; 3604 } 3605 } 3606 3607 return 0; 3608 } 3609 3610 /* This function is used by the interrupt thread */ 3611 static void 3612 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3613 __rte_unused void *arg) 3614 { 3615 uint16_t port_id; 3616 int ret; 3617 3618 if (type >= RTE_DEV_EVENT_MAX) { 3619 fprintf(stderr, "%s called upon invalid event %d\n", 3620 __func__, type); 3621 fflush(stderr); 3622 } 3623 3624 switch (type) { 3625 case RTE_DEV_EVENT_REMOVE: 3626 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3627 device_name); 3628 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3629 if (ret) { 3630 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3631 device_name); 3632 return; 3633 } 3634 /* 3635 * Because the user's callback is invoked in eal interrupt 3636 * callback, the interrupt callback need to be finished before 3637 * it can be unregistered when detaching device. So finish 3638 * callback soon and use a deferred removal to detach device 3639 * is need. It is a workaround, once the device detaching be 3640 * moved into the eal in the future, the deferred removal could 3641 * be deleted. 3642 */ 3643 if (rte_eal_alarm_set(100000, 3644 rmv_port_callback, (void *)(intptr_t)port_id)) 3645 RTE_LOG(ERR, EAL, 3646 "Could not set up deferred device removal\n"); 3647 break; 3648 case RTE_DEV_EVENT_ADD: 3649 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3650 device_name); 3651 /* TODO: After finish kernel driver binding, 3652 * begin to attach port. 3653 */ 3654 break; 3655 default: 3656 break; 3657 } 3658 } 3659 3660 static void 3661 rxtx_port_config(portid_t pid) 3662 { 3663 uint16_t qid; 3664 uint64_t offloads; 3665 struct rte_port *port = &ports[pid]; 3666 3667 for (qid = 0; qid < nb_rxq; qid++) { 3668 offloads = port->rx_conf[qid].offloads; 3669 port->rx_conf[qid] = port->dev_info.default_rxconf; 3670 3671 if (rxq_share > 0 && 3672 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3673 /* Non-zero share group to enable RxQ share. */ 3674 port->rx_conf[qid].share_group = pid / rxq_share + 1; 3675 port->rx_conf[qid].share_qid = qid; /* Equal mapping. */ 3676 } 3677 3678 if (offloads != 0) 3679 port->rx_conf[qid].offloads = offloads; 3680 3681 /* Check if any Rx parameters have been passed */ 3682 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3683 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3684 3685 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3686 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3687 3688 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3689 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3690 3691 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3692 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3693 3694 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3695 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3696 3697 port->nb_rx_desc[qid] = nb_rxd; 3698 } 3699 3700 for (qid = 0; qid < nb_txq; qid++) { 3701 offloads = port->tx_conf[qid].offloads; 3702 port->tx_conf[qid] = port->dev_info.default_txconf; 3703 if (offloads != 0) 3704 port->tx_conf[qid].offloads = offloads; 3705 3706 /* Check if any Tx parameters have been passed */ 3707 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3708 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3709 3710 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3711 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3712 3713 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3714 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3715 3716 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3717 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3718 3719 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3720 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3721 3722 port->nb_tx_desc[qid] = nb_txd; 3723 } 3724 } 3725 3726 /* 3727 * Helper function to set MTU from frame size 3728 * 3729 * port->dev_info should be set before calling this function. 3730 * 3731 * return 0 on success, negative on error 3732 */ 3733 int 3734 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3735 { 3736 struct rte_port *port = &ports[portid]; 3737 uint32_t eth_overhead; 3738 uint16_t mtu, new_mtu; 3739 3740 eth_overhead = get_eth_overhead(&port->dev_info); 3741 3742 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3743 printf("Failed to get MTU for port %u\n", portid); 3744 return -1; 3745 } 3746 3747 new_mtu = max_rx_pktlen - eth_overhead; 3748 3749 if (mtu == new_mtu) 3750 return 0; 3751 3752 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3753 fprintf(stderr, 3754 "Failed to set MTU to %u for port %u\n", 3755 new_mtu, portid); 3756 return -1; 3757 } 3758 3759 port->dev_conf.rxmode.mtu = new_mtu; 3760 3761 return 0; 3762 } 3763 3764 void 3765 init_port_config(void) 3766 { 3767 portid_t pid; 3768 struct rte_port *port; 3769 int ret, i; 3770 3771 RTE_ETH_FOREACH_DEV(pid) { 3772 port = &ports[pid]; 3773 port->dev_conf.fdir_conf = fdir_conf; 3774 3775 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3776 if (ret != 0) 3777 return; 3778 3779 if (nb_rxq > 1) { 3780 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3781 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3782 rss_hf & port->dev_info.flow_type_rss_offloads; 3783 } else { 3784 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3785 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3786 } 3787 3788 if (port->dcb_flag == 0) { 3789 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3790 port->dev_conf.rxmode.mq_mode = 3791 (enum rte_eth_rx_mq_mode) 3792 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3793 } else { 3794 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3795 port->dev_conf.rxmode.offloads &= 3796 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3797 3798 for (i = 0; 3799 i < port->dev_info.nb_rx_queues; 3800 i++) 3801 port->rx_conf[i].offloads &= 3802 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3803 } 3804 } 3805 3806 rxtx_port_config(pid); 3807 3808 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3809 if (ret != 0) 3810 return; 3811 3812 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3813 rte_pmd_ixgbe_bypass_init(pid); 3814 #endif 3815 3816 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3817 port->dev_conf.intr_conf.lsc = 1; 3818 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3819 port->dev_conf.intr_conf.rmv = 1; 3820 } 3821 } 3822 3823 void set_port_slave_flag(portid_t slave_pid) 3824 { 3825 struct rte_port *port; 3826 3827 port = &ports[slave_pid]; 3828 port->slave_flag = 1; 3829 } 3830 3831 void clear_port_slave_flag(portid_t slave_pid) 3832 { 3833 struct rte_port *port; 3834 3835 port = &ports[slave_pid]; 3836 port->slave_flag = 0; 3837 } 3838 3839 uint8_t port_is_bonding_slave(portid_t slave_pid) 3840 { 3841 struct rte_port *port; 3842 struct rte_eth_dev_info dev_info; 3843 int ret; 3844 3845 port = &ports[slave_pid]; 3846 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3847 if (ret != 0) { 3848 TESTPMD_LOG(ERR, 3849 "Failed to get device info for port id %d," 3850 "cannot determine if the port is a bonded slave", 3851 slave_pid); 3852 return 0; 3853 } 3854 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3855 return 1; 3856 return 0; 3857 } 3858 3859 const uint16_t vlan_tags[] = { 3860 0, 1, 2, 3, 4, 5, 6, 7, 3861 8, 9, 10, 11, 12, 13, 14, 15, 3862 16, 17, 18, 19, 20, 21, 22, 23, 3863 24, 25, 26, 27, 28, 29, 30, 31 3864 }; 3865 3866 static int 3867 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3868 enum dcb_mode_enable dcb_mode, 3869 enum rte_eth_nb_tcs num_tcs, 3870 uint8_t pfc_en) 3871 { 3872 uint8_t i; 3873 int32_t rc; 3874 struct rte_eth_rss_conf rss_conf; 3875 3876 /* 3877 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3878 * given above, and the number of traffic classes available for use. 3879 */ 3880 if (dcb_mode == DCB_VT_ENABLED) { 3881 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3882 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3883 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3884 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3885 3886 /* VMDQ+DCB RX and TX configurations */ 3887 vmdq_rx_conf->enable_default_pool = 0; 3888 vmdq_rx_conf->default_pool = 0; 3889 vmdq_rx_conf->nb_queue_pools = 3890 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3891 vmdq_tx_conf->nb_queue_pools = 3892 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3893 3894 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3895 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3896 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3897 vmdq_rx_conf->pool_map[i].pools = 3898 1 << (i % vmdq_rx_conf->nb_queue_pools); 3899 } 3900 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3901 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3902 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3903 } 3904 3905 /* set DCB mode of RX and TX of multiple queues */ 3906 eth_conf->rxmode.mq_mode = 3907 (enum rte_eth_rx_mq_mode) 3908 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 3909 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 3910 } else { 3911 struct rte_eth_dcb_rx_conf *rx_conf = 3912 ð_conf->rx_adv_conf.dcb_rx_conf; 3913 struct rte_eth_dcb_tx_conf *tx_conf = 3914 ð_conf->tx_adv_conf.dcb_tx_conf; 3915 3916 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3917 3918 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3919 if (rc != 0) 3920 return rc; 3921 3922 rx_conf->nb_tcs = num_tcs; 3923 tx_conf->nb_tcs = num_tcs; 3924 3925 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3926 rx_conf->dcb_tc[i] = i % num_tcs; 3927 tx_conf->dcb_tc[i] = i % num_tcs; 3928 } 3929 3930 eth_conf->rxmode.mq_mode = 3931 (enum rte_eth_rx_mq_mode) 3932 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 3933 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3934 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 3935 } 3936 3937 if (pfc_en) 3938 eth_conf->dcb_capability_en = 3939 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 3940 else 3941 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 3942 3943 return 0; 3944 } 3945 3946 int 3947 init_port_dcb_config(portid_t pid, 3948 enum dcb_mode_enable dcb_mode, 3949 enum rte_eth_nb_tcs num_tcs, 3950 uint8_t pfc_en) 3951 { 3952 struct rte_eth_conf port_conf; 3953 struct rte_port *rte_port; 3954 int retval; 3955 uint16_t i; 3956 3957 if (num_procs > 1) { 3958 printf("The multi-process feature doesn't support dcb.\n"); 3959 return -ENOTSUP; 3960 } 3961 rte_port = &ports[pid]; 3962 3963 /* retain the original device configuration. */ 3964 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 3965 3966 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3967 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3968 if (retval < 0) 3969 return retval; 3970 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 3971 /* remove RSS HASH offload for DCB in vt mode */ 3972 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) { 3973 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3974 for (i = 0; i < nb_rxq; i++) 3975 rte_port->rx_conf[i].offloads &= 3976 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3977 } 3978 3979 /* re-configure the device . */ 3980 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3981 if (retval < 0) 3982 return retval; 3983 3984 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3985 if (retval != 0) 3986 return retval; 3987 3988 /* If dev_info.vmdq_pool_base is greater than 0, 3989 * the queue id of vmdq pools is started after pf queues. 3990 */ 3991 if (dcb_mode == DCB_VT_ENABLED && 3992 rte_port->dev_info.vmdq_pool_base > 0) { 3993 fprintf(stderr, 3994 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 3995 pid); 3996 return -1; 3997 } 3998 3999 /* Assume the ports in testpmd have the same dcb capability 4000 * and has the same number of rxq and txq in dcb mode 4001 */ 4002 if (dcb_mode == DCB_VT_ENABLED) { 4003 if (rte_port->dev_info.max_vfs > 0) { 4004 nb_rxq = rte_port->dev_info.nb_rx_queues; 4005 nb_txq = rte_port->dev_info.nb_tx_queues; 4006 } else { 4007 nb_rxq = rte_port->dev_info.max_rx_queues; 4008 nb_txq = rte_port->dev_info.max_tx_queues; 4009 } 4010 } else { 4011 /*if vt is disabled, use all pf queues */ 4012 if (rte_port->dev_info.vmdq_pool_base == 0) { 4013 nb_rxq = rte_port->dev_info.max_rx_queues; 4014 nb_txq = rte_port->dev_info.max_tx_queues; 4015 } else { 4016 nb_rxq = (queueid_t)num_tcs; 4017 nb_txq = (queueid_t)num_tcs; 4018 4019 } 4020 } 4021 rx_free_thresh = 64; 4022 4023 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 4024 4025 rxtx_port_config(pid); 4026 /* VLAN filter */ 4027 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4028 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4029 rx_vft_set(pid, vlan_tags[i], 1); 4030 4031 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4032 if (retval != 0) 4033 return retval; 4034 4035 rte_port->dcb_flag = 1; 4036 4037 /* Enter DCB configuration status */ 4038 dcb_config = 1; 4039 4040 return 0; 4041 } 4042 4043 static void 4044 init_port(void) 4045 { 4046 int i; 4047 4048 /* Configuration of Ethernet ports. */ 4049 ports = rte_zmalloc("testpmd: ports", 4050 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4051 RTE_CACHE_LINE_SIZE); 4052 if (ports == NULL) { 4053 rte_exit(EXIT_FAILURE, 4054 "rte_zmalloc(%d struct rte_port) failed\n", 4055 RTE_MAX_ETHPORTS); 4056 } 4057 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4058 ports[i].xstats_info.allocated = false; 4059 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4060 LIST_INIT(&ports[i].flow_tunnel_list); 4061 /* Initialize ports NUMA structures */ 4062 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4063 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4064 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4065 } 4066 4067 static void 4068 force_quit(void) 4069 { 4070 pmd_test_exit(); 4071 prompt_exit(); 4072 } 4073 4074 static void 4075 print_stats(void) 4076 { 4077 uint8_t i; 4078 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4079 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4080 4081 /* Clear screen and move to top left */ 4082 printf("%s%s", clr, top_left); 4083 4084 printf("\nPort statistics ===================================="); 4085 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4086 nic_stats_display(fwd_ports_ids[i]); 4087 4088 fflush(stdout); 4089 } 4090 4091 static void 4092 signal_handler(int signum) 4093 { 4094 if (signum == SIGINT || signum == SIGTERM) { 4095 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4096 signum); 4097 #ifdef RTE_LIB_PDUMP 4098 /* uninitialize packet capture framework */ 4099 rte_pdump_uninit(); 4100 #endif 4101 #ifdef RTE_LIB_LATENCYSTATS 4102 if (latencystats_enabled != 0) 4103 rte_latencystats_uninit(); 4104 #endif 4105 force_quit(); 4106 /* Set flag to indicate the force termination. */ 4107 f_quit = 1; 4108 /* exit with the expected status */ 4109 #ifndef RTE_EXEC_ENV_WINDOWS 4110 signal(signum, SIG_DFL); 4111 kill(getpid(), signum); 4112 #endif 4113 } 4114 } 4115 4116 int 4117 main(int argc, char** argv) 4118 { 4119 int diag; 4120 portid_t port_id; 4121 uint16_t count; 4122 int ret; 4123 4124 signal(SIGINT, signal_handler); 4125 signal(SIGTERM, signal_handler); 4126 4127 testpmd_logtype = rte_log_register("testpmd"); 4128 if (testpmd_logtype < 0) 4129 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4130 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4131 4132 diag = rte_eal_init(argc, argv); 4133 if (diag < 0) 4134 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4135 rte_strerror(rte_errno)); 4136 4137 ret = register_eth_event_callback(); 4138 if (ret != 0) 4139 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4140 4141 #ifdef RTE_LIB_PDUMP 4142 /* initialize packet capture framework */ 4143 rte_pdump_init(); 4144 #endif 4145 4146 count = 0; 4147 RTE_ETH_FOREACH_DEV(port_id) { 4148 ports_ids[count] = port_id; 4149 count++; 4150 } 4151 nb_ports = (portid_t) count; 4152 if (nb_ports == 0) 4153 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4154 4155 /* allocate port structures, and init them */ 4156 init_port(); 4157 4158 set_def_fwd_config(); 4159 if (nb_lcores == 0) 4160 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4161 "Check the core mask argument\n"); 4162 4163 /* Bitrate/latency stats disabled by default */ 4164 #ifdef RTE_LIB_BITRATESTATS 4165 bitrate_enabled = 0; 4166 #endif 4167 #ifdef RTE_LIB_LATENCYSTATS 4168 latencystats_enabled = 0; 4169 #endif 4170 4171 /* on FreeBSD, mlockall() is disabled by default */ 4172 #ifdef RTE_EXEC_ENV_FREEBSD 4173 do_mlockall = 0; 4174 #else 4175 do_mlockall = 1; 4176 #endif 4177 4178 argc -= diag; 4179 argv += diag; 4180 if (argc > 1) 4181 launch_args_parse(argc, argv); 4182 4183 #ifndef RTE_EXEC_ENV_WINDOWS 4184 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4185 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4186 strerror(errno)); 4187 } 4188 #endif 4189 4190 if (tx_first && interactive) 4191 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4192 "interactive mode.\n"); 4193 4194 if (tx_first && lsc_interrupt) { 4195 fprintf(stderr, 4196 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4197 lsc_interrupt = 0; 4198 } 4199 4200 if (!nb_rxq && !nb_txq) 4201 fprintf(stderr, 4202 "Warning: Either rx or tx queues should be non-zero\n"); 4203 4204 if (nb_rxq > 1 && nb_rxq > nb_txq) 4205 fprintf(stderr, 4206 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4207 nb_rxq, nb_txq); 4208 4209 init_config(); 4210 4211 if (hot_plug) { 4212 ret = rte_dev_hotplug_handle_enable(); 4213 if (ret) { 4214 RTE_LOG(ERR, EAL, 4215 "fail to enable hotplug handling."); 4216 return -1; 4217 } 4218 4219 ret = rte_dev_event_monitor_start(); 4220 if (ret) { 4221 RTE_LOG(ERR, EAL, 4222 "fail to start device event monitoring."); 4223 return -1; 4224 } 4225 4226 ret = rte_dev_event_callback_register(NULL, 4227 dev_event_callback, NULL); 4228 if (ret) { 4229 RTE_LOG(ERR, EAL, 4230 "fail to register device event callback\n"); 4231 return -1; 4232 } 4233 } 4234 4235 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4236 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4237 4238 /* set all ports to promiscuous mode by default */ 4239 RTE_ETH_FOREACH_DEV(port_id) { 4240 ret = rte_eth_promiscuous_enable(port_id); 4241 if (ret != 0) 4242 fprintf(stderr, 4243 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4244 port_id, rte_strerror(-ret)); 4245 } 4246 4247 #ifdef RTE_LIB_METRICS 4248 /* Init metrics library */ 4249 rte_metrics_init(rte_socket_id()); 4250 #endif 4251 4252 #ifdef RTE_LIB_LATENCYSTATS 4253 if (latencystats_enabled != 0) { 4254 int ret = rte_latencystats_init(1, NULL); 4255 if (ret) 4256 fprintf(stderr, 4257 "Warning: latencystats init() returned error %d\n", 4258 ret); 4259 fprintf(stderr, "Latencystats running on lcore %d\n", 4260 latencystats_lcore_id); 4261 } 4262 #endif 4263 4264 /* Setup bitrate stats */ 4265 #ifdef RTE_LIB_BITRATESTATS 4266 if (bitrate_enabled != 0) { 4267 bitrate_data = rte_stats_bitrate_create(); 4268 if (bitrate_data == NULL) 4269 rte_exit(EXIT_FAILURE, 4270 "Could not allocate bitrate data.\n"); 4271 rte_stats_bitrate_reg(bitrate_data); 4272 } 4273 #endif 4274 #ifdef RTE_LIB_CMDLINE 4275 if (strlen(cmdline_filename) != 0) 4276 cmdline_read_from_file(cmdline_filename); 4277 4278 if (interactive == 1) { 4279 if (auto_start) { 4280 printf("Start automatic packet forwarding\n"); 4281 start_packet_forwarding(0); 4282 } 4283 prompt(); 4284 pmd_test_exit(); 4285 } else 4286 #endif 4287 { 4288 char c; 4289 int rc; 4290 4291 f_quit = 0; 4292 4293 printf("No commandline core given, start packet forwarding\n"); 4294 start_packet_forwarding(tx_first); 4295 if (stats_period != 0) { 4296 uint64_t prev_time = 0, cur_time, diff_time = 0; 4297 uint64_t timer_period; 4298 4299 /* Convert to number of cycles */ 4300 timer_period = stats_period * rte_get_timer_hz(); 4301 4302 while (f_quit == 0) { 4303 cur_time = rte_get_timer_cycles(); 4304 diff_time += cur_time - prev_time; 4305 4306 if (diff_time >= timer_period) { 4307 print_stats(); 4308 /* Reset the timer */ 4309 diff_time = 0; 4310 } 4311 /* Sleep to avoid unnecessary checks */ 4312 prev_time = cur_time; 4313 rte_delay_us_sleep(US_PER_S); 4314 } 4315 } 4316 4317 printf("Press enter to exit\n"); 4318 rc = read(0, &c, 1); 4319 pmd_test_exit(); 4320 if (rc < 0) 4321 return 1; 4322 } 4323 4324 ret = rte_eal_cleanup(); 4325 if (ret != 0) 4326 rte_exit(EXIT_FAILURE, 4327 "EAL cleanup failed: %s\n", strerror(-ret)); 4328 4329 return EXIT_SUCCESS; 4330 } 4331