1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_atomic.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_pci.h> 47 #include <rte_ether.h> 48 #include <rte_ethdev.h> 49 #include <rte_dev.h> 50 #include <rte_string_fns.h> 51 #ifdef RTE_NET_IXGBE 52 #include <rte_pmd_ixgbe.h> 53 #endif 54 #ifdef RTE_LIB_PDUMP 55 #include <rte_pdump.h> 56 #endif 57 #include <rte_flow.h> 58 #include <rte_metrics.h> 59 #ifdef RTE_LIB_BITRATESTATS 60 #include <rte_bitrate.h> 61 #endif 62 #ifdef RTE_LIB_LATENCYSTATS 63 #include <rte_latencystats.h> 64 #endif 65 #ifdef RTE_EXEC_ENV_WINDOWS 66 #include <process.h> 67 #endif 68 69 #include "testpmd.h" 70 71 #ifndef MAP_HUGETLB 72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 73 #define HUGE_FLAG (0x40000) 74 #else 75 #define HUGE_FLAG MAP_HUGETLB 76 #endif 77 78 #ifndef MAP_HUGE_SHIFT 79 /* older kernels (or FreeBSD) will not have this define */ 80 #define HUGE_SHIFT (26) 81 #else 82 #define HUGE_SHIFT MAP_HUGE_SHIFT 83 #endif 84 85 #define EXTMEM_HEAP_NAME "extmem" 86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 87 88 uint16_t verbose_level = 0; /**< Silent by default. */ 89 int testpmd_logtype; /**< Log type for testpmd logs */ 90 91 /* use main core for command line ? */ 92 uint8_t interactive = 0; 93 uint8_t auto_start = 0; 94 uint8_t tx_first; 95 char cmdline_filename[PATH_MAX] = {0}; 96 97 /* 98 * NUMA support configuration. 99 * When set, the NUMA support attempts to dispatch the allocation of the 100 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 101 * probed ports among the CPU sockets 0 and 1. 102 * Otherwise, all memory is allocated from CPU socket 0. 103 */ 104 uint8_t numa_support = 1; /**< numa enabled by default */ 105 106 /* 107 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 108 * not configured. 109 */ 110 uint8_t socket_num = UMA_NO_CONFIG; 111 112 /* 113 * Select mempool allocation type: 114 * - native: use regular DPDK memory 115 * - anon: use regular DPDK memory to create mempool, but populate using 116 * anonymous memory (may not be IOVA-contiguous) 117 * - xmem: use externally allocated hugepage memory 118 */ 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 120 121 /* 122 * Store specified sockets on which memory pool to be used by ports 123 * is allocated. 124 */ 125 uint8_t port_numa[RTE_MAX_ETHPORTS]; 126 127 /* 128 * Store specified sockets on which RX ring to be used by ports 129 * is allocated. 130 */ 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 132 133 /* 134 * Store specified sockets on which TX ring to be used by ports 135 * is allocated. 136 */ 137 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 138 139 /* 140 * Record the Ethernet address of peer target ports to which packets are 141 * forwarded. 142 * Must be instantiated with the ethernet addresses of peer traffic generator 143 * ports. 144 */ 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 146 portid_t nb_peer_eth_addrs = 0; 147 148 /* 149 * Probed Target Environment. 150 */ 151 struct rte_port *ports; /**< For all probed ethernet ports. */ 152 portid_t nb_ports; /**< Number of probed ethernet ports. */ 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 154 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 155 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 157 158 /* 159 * Test Forwarding Configuration. 160 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 161 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 162 */ 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 165 portid_t nb_cfg_ports; /**< Number of configured ports. */ 166 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 167 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 170 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 172 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 173 174 /* 175 * Forwarding engines. 176 */ 177 struct fwd_engine * fwd_engines[] = { 178 &io_fwd_engine, 179 &mac_fwd_engine, 180 &mac_swap_engine, 181 &flow_gen_engine, 182 &rx_only_engine, 183 &tx_only_engine, 184 &csum_fwd_engine, 185 &icmp_echo_engine, 186 &noisy_vnf_engine, 187 &five_tuple_swap_fwd_engine, 188 #ifdef RTE_LIBRTE_IEEE1588 189 &ieee1588_fwd_engine, 190 #endif 191 &shared_rxq_engine, 192 NULL, 193 }; 194 195 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 196 uint16_t mempool_flags; 197 198 struct fwd_config cur_fwd_config; 199 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 200 uint32_t retry_enabled; 201 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 202 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 203 204 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 205 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 206 DEFAULT_MBUF_DATA_SIZE 207 }; /**< Mbuf data space size. */ 208 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 209 * specified on command-line. */ 210 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 211 212 /** Extended statistics to show. */ 213 struct rte_eth_xstat_name *xstats_display; 214 215 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 216 217 /* 218 * In container, it cannot terminate the process which running with 'stats-period' 219 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 220 */ 221 uint8_t f_quit; 222 223 /* 224 * Max Rx frame size, set by '--max-pkt-len' parameter. 225 */ 226 uint32_t max_rx_pkt_len; 227 228 /* 229 * Configuration of packet segments used to scatter received packets 230 * if some of split features is configured. 231 */ 232 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 233 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 234 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 235 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 236 237 /* 238 * Configuration of packet segments used by the "txonly" processing engine. 239 */ 240 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 241 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 242 TXONLY_DEF_PACKET_LEN, 243 }; 244 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 245 246 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 247 /**< Split policy for packets to TX. */ 248 249 uint8_t txonly_multi_flow; 250 /**< Whether multiple flows are generated in TXONLY mode. */ 251 252 uint32_t tx_pkt_times_inter; 253 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 254 255 uint32_t tx_pkt_times_intra; 256 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 257 258 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 259 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 260 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 261 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 262 263 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 264 uint8_t dcb_config = 0; 265 266 /* 267 * Configurable number of RX/TX queues. 268 */ 269 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 270 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 271 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 272 273 /* 274 * Configurable number of RX/TX ring descriptors. 275 * Defaults are supplied by drivers via ethdev. 276 */ 277 #define RTE_TEST_RX_DESC_DEFAULT 0 278 #define RTE_TEST_TX_DESC_DEFAULT 0 279 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 280 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 281 282 #define RTE_PMD_PARAM_UNSET -1 283 /* 284 * Configurable values of RX and TX ring threshold registers. 285 */ 286 287 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 288 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 289 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 290 291 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 292 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 293 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 294 295 /* 296 * Configurable value of RX free threshold. 297 */ 298 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 299 300 /* 301 * Configurable value of RX drop enable. 302 */ 303 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 304 305 /* 306 * Configurable value of TX free threshold. 307 */ 308 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 309 310 /* 311 * Configurable value of TX RS bit threshold. 312 */ 313 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 314 315 /* 316 * Configurable value of buffered packets before sending. 317 */ 318 uint16_t noisy_tx_sw_bufsz; 319 320 /* 321 * Configurable value of packet buffer timeout. 322 */ 323 uint16_t noisy_tx_sw_buf_flush_time; 324 325 /* 326 * Configurable value for size of VNF internal memory area 327 * used for simulating noisy neighbour behaviour 328 */ 329 uint64_t noisy_lkup_mem_sz; 330 331 /* 332 * Configurable value of number of random writes done in 333 * VNF simulation memory area. 334 */ 335 uint64_t noisy_lkup_num_writes; 336 337 /* 338 * Configurable value of number of random reads done in 339 * VNF simulation memory area. 340 */ 341 uint64_t noisy_lkup_num_reads; 342 343 /* 344 * Configurable value of number of random reads/writes done in 345 * VNF simulation memory area. 346 */ 347 uint64_t noisy_lkup_num_reads_writes; 348 349 /* 350 * Receive Side Scaling (RSS) configuration. 351 */ 352 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */ 353 354 /* 355 * Port topology configuration 356 */ 357 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 358 359 /* 360 * Avoids to flush all the RX streams before starts forwarding. 361 */ 362 uint8_t no_flush_rx = 0; /* flush by default */ 363 364 /* 365 * Flow API isolated mode. 366 */ 367 uint8_t flow_isolate_all; 368 369 /* 370 * Avoids to check link status when starting/stopping a port. 371 */ 372 uint8_t no_link_check = 0; /* check by default */ 373 374 /* 375 * Don't automatically start all ports in interactive mode. 376 */ 377 uint8_t no_device_start = 0; 378 379 /* 380 * Enable link status change notification 381 */ 382 uint8_t lsc_interrupt = 1; /* enabled by default */ 383 384 /* 385 * Enable device removal notification. 386 */ 387 uint8_t rmv_interrupt = 1; /* enabled by default */ 388 389 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 390 391 /* After attach, port setup is called on event or by iterator */ 392 bool setup_on_probe_event = true; 393 394 /* Clear ptypes on port initialization. */ 395 uint8_t clear_ptypes = true; 396 397 /* Hairpin ports configuration mode. */ 398 uint16_t hairpin_mode; 399 400 /* Pretty printing of ethdev events */ 401 static const char * const eth_event_desc[] = { 402 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 403 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 404 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 405 [RTE_ETH_EVENT_INTR_RESET] = "reset", 406 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 407 [RTE_ETH_EVENT_IPSEC] = "IPsec", 408 [RTE_ETH_EVENT_MACSEC] = "MACsec", 409 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 410 [RTE_ETH_EVENT_NEW] = "device probed", 411 [RTE_ETH_EVENT_DESTROY] = "device released", 412 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 413 [RTE_ETH_EVENT_MAX] = NULL, 414 }; 415 416 /* 417 * Display or mask ether events 418 * Default to all events except VF_MBOX 419 */ 420 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 421 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 422 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 423 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 424 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 425 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 426 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 427 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 428 /* 429 * Decide if all memory are locked for performance. 430 */ 431 int do_mlockall = 0; 432 433 /* 434 * NIC bypass mode configuration options. 435 */ 436 437 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 438 /* The NIC bypass watchdog timeout. */ 439 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 440 #endif 441 442 443 #ifdef RTE_LIB_LATENCYSTATS 444 445 /* 446 * Set when latency stats is enabled in the commandline 447 */ 448 uint8_t latencystats_enabled; 449 450 /* 451 * Lcore ID to serive latency statistics. 452 */ 453 lcoreid_t latencystats_lcore_id = -1; 454 455 #endif 456 457 /* 458 * Ethernet device configuration. 459 */ 460 struct rte_eth_rxmode rx_mode; 461 462 struct rte_eth_txmode tx_mode = { 463 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE, 464 }; 465 466 struct rte_eth_fdir_conf fdir_conf = { 467 .mode = RTE_FDIR_MODE_NONE, 468 .pballoc = RTE_ETH_FDIR_PBALLOC_64K, 469 .status = RTE_FDIR_REPORT_STATUS, 470 .mask = { 471 .vlan_tci_mask = 0xFFEF, 472 .ipv4_mask = { 473 .src_ip = 0xFFFFFFFF, 474 .dst_ip = 0xFFFFFFFF, 475 }, 476 .ipv6_mask = { 477 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 478 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 479 }, 480 .src_port_mask = 0xFFFF, 481 .dst_port_mask = 0xFFFF, 482 .mac_addr_byte_mask = 0xFF, 483 .tunnel_type_mask = 1, 484 .tunnel_id_mask = 0xFFFFFFFF, 485 }, 486 .drop_queue = 127, 487 }; 488 489 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 490 491 /* 492 * Display zero values by default for xstats 493 */ 494 uint8_t xstats_hide_zero; 495 496 /* 497 * Measure of CPU cycles disabled by default 498 */ 499 uint8_t record_core_cycles; 500 501 /* 502 * Display of RX and TX bursts disabled by default 503 */ 504 uint8_t record_burst_stats; 505 506 /* 507 * Number of ports per shared Rx queue group, 0 disable. 508 */ 509 uint32_t rxq_share; 510 511 unsigned int num_sockets = 0; 512 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 513 514 #ifdef RTE_LIB_BITRATESTATS 515 /* Bitrate statistics */ 516 struct rte_stats_bitrates *bitrate_data; 517 lcoreid_t bitrate_lcore_id; 518 uint8_t bitrate_enabled; 519 #endif 520 521 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 522 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 523 524 /* 525 * hexadecimal bitmask of RX mq mode can be enabled. 526 */ 527 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS; 528 529 /* 530 * Used to set forced link speed 531 */ 532 uint32_t eth_link_speed; 533 534 /* 535 * ID of the current process in multi-process, used to 536 * configure the queues to be polled. 537 */ 538 int proc_id; 539 540 /* 541 * Number of processes in multi-process, used to 542 * configure the queues to be polled. 543 */ 544 unsigned int num_procs = 1; 545 546 static void 547 eth_rx_metadata_negotiate_mp(uint16_t port_id) 548 { 549 uint64_t rx_meta_features = 0; 550 int ret; 551 552 if (!is_proc_primary()) 553 return; 554 555 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 556 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 557 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 558 559 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 560 if (ret == 0) { 561 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 562 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 563 port_id); 564 } 565 566 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 567 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 568 port_id); 569 } 570 571 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 572 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 573 port_id); 574 } 575 } else if (ret != -ENOTSUP) { 576 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 577 port_id, rte_strerror(-ret)); 578 } 579 } 580 581 static void 582 flow_pick_transfer_proxy_mp(uint16_t port_id) 583 { 584 struct rte_port *port = &ports[port_id]; 585 int ret; 586 587 port->flow_transfer_proxy = port_id; 588 589 if (!is_proc_primary()) 590 return; 591 592 ret = rte_flow_pick_transfer_proxy(port_id, &port->flow_transfer_proxy, 593 NULL); 594 if (ret != 0) { 595 fprintf(stderr, "Error picking flow transfer proxy for port %u: %s - ignore\n", 596 port_id, rte_strerror(-ret)); 597 } 598 } 599 600 static int 601 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 602 const struct rte_eth_conf *dev_conf) 603 { 604 if (is_proc_primary()) 605 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 606 dev_conf); 607 return 0; 608 } 609 610 static int 611 eth_dev_start_mp(uint16_t port_id) 612 { 613 if (is_proc_primary()) 614 return rte_eth_dev_start(port_id); 615 616 return 0; 617 } 618 619 static int 620 eth_dev_stop_mp(uint16_t port_id) 621 { 622 if (is_proc_primary()) 623 return rte_eth_dev_stop(port_id); 624 625 return 0; 626 } 627 628 static void 629 mempool_free_mp(struct rte_mempool *mp) 630 { 631 if (is_proc_primary()) 632 rte_mempool_free(mp); 633 } 634 635 static int 636 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 637 { 638 if (is_proc_primary()) 639 return rte_eth_dev_set_mtu(port_id, mtu); 640 641 return 0; 642 } 643 644 /* Forward function declarations */ 645 static void setup_attached_port(portid_t pi); 646 static void check_all_ports_link_status(uint32_t port_mask); 647 static int eth_event_callback(portid_t port_id, 648 enum rte_eth_event_type type, 649 void *param, void *ret_param); 650 static void dev_event_callback(const char *device_name, 651 enum rte_dev_event_type type, 652 void *param); 653 static void fill_xstats_display_info(void); 654 655 /* 656 * Check if all the ports are started. 657 * If yes, return positive value. If not, return zero. 658 */ 659 static int all_ports_started(void); 660 661 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 662 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 663 664 /* Holds the registered mbuf dynamic flags names. */ 665 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 666 667 668 /* 669 * Helper function to check if socket is already discovered. 670 * If yes, return positive value. If not, return zero. 671 */ 672 int 673 new_socket_id(unsigned int socket_id) 674 { 675 unsigned int i; 676 677 for (i = 0; i < num_sockets; i++) { 678 if (socket_ids[i] == socket_id) 679 return 0; 680 } 681 return 1; 682 } 683 684 /* 685 * Setup default configuration. 686 */ 687 static void 688 set_default_fwd_lcores_config(void) 689 { 690 unsigned int i; 691 unsigned int nb_lc; 692 unsigned int sock_num; 693 694 nb_lc = 0; 695 for (i = 0; i < RTE_MAX_LCORE; i++) { 696 if (!rte_lcore_is_enabled(i)) 697 continue; 698 sock_num = rte_lcore_to_socket_id(i); 699 if (new_socket_id(sock_num)) { 700 if (num_sockets >= RTE_MAX_NUMA_NODES) { 701 rte_exit(EXIT_FAILURE, 702 "Total sockets greater than %u\n", 703 RTE_MAX_NUMA_NODES); 704 } 705 socket_ids[num_sockets++] = sock_num; 706 } 707 if (i == rte_get_main_lcore()) 708 continue; 709 fwd_lcores_cpuids[nb_lc++] = i; 710 } 711 nb_lcores = (lcoreid_t) nb_lc; 712 nb_cfg_lcores = nb_lcores; 713 nb_fwd_lcores = 1; 714 } 715 716 static void 717 set_def_peer_eth_addrs(void) 718 { 719 portid_t i; 720 721 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 722 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 723 peer_eth_addrs[i].addr_bytes[5] = i; 724 } 725 } 726 727 static void 728 set_default_fwd_ports_config(void) 729 { 730 portid_t pt_id; 731 int i = 0; 732 733 RTE_ETH_FOREACH_DEV(pt_id) { 734 fwd_ports_ids[i++] = pt_id; 735 736 /* Update sockets info according to the attached device */ 737 int socket_id = rte_eth_dev_socket_id(pt_id); 738 if (socket_id >= 0 && new_socket_id(socket_id)) { 739 if (num_sockets >= RTE_MAX_NUMA_NODES) { 740 rte_exit(EXIT_FAILURE, 741 "Total sockets greater than %u\n", 742 RTE_MAX_NUMA_NODES); 743 } 744 socket_ids[num_sockets++] = socket_id; 745 } 746 } 747 748 nb_cfg_ports = nb_ports; 749 nb_fwd_ports = nb_ports; 750 } 751 752 void 753 set_def_fwd_config(void) 754 { 755 set_default_fwd_lcores_config(); 756 set_def_peer_eth_addrs(); 757 set_default_fwd_ports_config(); 758 } 759 760 #ifndef RTE_EXEC_ENV_WINDOWS 761 /* extremely pessimistic estimation of memory required to create a mempool */ 762 static int 763 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 764 { 765 unsigned int n_pages, mbuf_per_pg, leftover; 766 uint64_t total_mem, mbuf_mem, obj_sz; 767 768 /* there is no good way to predict how much space the mempool will 769 * occupy because it will allocate chunks on the fly, and some of those 770 * will come from default DPDK memory while some will come from our 771 * external memory, so just assume 128MB will be enough for everyone. 772 */ 773 uint64_t hdr_mem = 128 << 20; 774 775 /* account for possible non-contiguousness */ 776 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 777 if (obj_sz > pgsz) { 778 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 779 return -1; 780 } 781 782 mbuf_per_pg = pgsz / obj_sz; 783 leftover = (nb_mbufs % mbuf_per_pg) > 0; 784 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 785 786 mbuf_mem = n_pages * pgsz; 787 788 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 789 790 if (total_mem > SIZE_MAX) { 791 TESTPMD_LOG(ERR, "Memory size too big\n"); 792 return -1; 793 } 794 *out = (size_t)total_mem; 795 796 return 0; 797 } 798 799 static int 800 pagesz_flags(uint64_t page_sz) 801 { 802 /* as per mmap() manpage, all page sizes are log2 of page size 803 * shifted by MAP_HUGE_SHIFT 804 */ 805 int log2 = rte_log2_u64(page_sz); 806 807 return (log2 << HUGE_SHIFT); 808 } 809 810 static void * 811 alloc_mem(size_t memsz, size_t pgsz, bool huge) 812 { 813 void *addr; 814 int flags; 815 816 /* allocate anonymous hugepages */ 817 flags = MAP_ANONYMOUS | MAP_PRIVATE; 818 if (huge) 819 flags |= HUGE_FLAG | pagesz_flags(pgsz); 820 821 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 822 if (addr == MAP_FAILED) 823 return NULL; 824 825 return addr; 826 } 827 828 struct extmem_param { 829 void *addr; 830 size_t len; 831 size_t pgsz; 832 rte_iova_t *iova_table; 833 unsigned int iova_table_len; 834 }; 835 836 static int 837 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 838 bool huge) 839 { 840 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 841 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 842 unsigned int cur_page, n_pages, pgsz_idx; 843 size_t mem_sz, cur_pgsz; 844 rte_iova_t *iovas = NULL; 845 void *addr; 846 int ret; 847 848 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 849 /* skip anything that is too big */ 850 if (pgsizes[pgsz_idx] > SIZE_MAX) 851 continue; 852 853 cur_pgsz = pgsizes[pgsz_idx]; 854 855 /* if we were told not to allocate hugepages, override */ 856 if (!huge) 857 cur_pgsz = sysconf(_SC_PAGESIZE); 858 859 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 860 if (ret < 0) { 861 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 862 return -1; 863 } 864 865 /* allocate our memory */ 866 addr = alloc_mem(mem_sz, cur_pgsz, huge); 867 868 /* if we couldn't allocate memory with a specified page size, 869 * that doesn't mean we can't do it with other page sizes, so 870 * try another one. 871 */ 872 if (addr == NULL) 873 continue; 874 875 /* store IOVA addresses for every page in this memory area */ 876 n_pages = mem_sz / cur_pgsz; 877 878 iovas = malloc(sizeof(*iovas) * n_pages); 879 880 if (iovas == NULL) { 881 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 882 goto fail; 883 } 884 /* lock memory if it's not huge pages */ 885 if (!huge) 886 mlock(addr, mem_sz); 887 888 /* populate IOVA addresses */ 889 for (cur_page = 0; cur_page < n_pages; cur_page++) { 890 rte_iova_t iova; 891 size_t offset; 892 void *cur; 893 894 offset = cur_pgsz * cur_page; 895 cur = RTE_PTR_ADD(addr, offset); 896 897 /* touch the page before getting its IOVA */ 898 *(volatile char *)cur = 0; 899 900 iova = rte_mem_virt2iova(cur); 901 902 iovas[cur_page] = iova; 903 } 904 905 break; 906 } 907 /* if we couldn't allocate anything */ 908 if (iovas == NULL) 909 return -1; 910 911 param->addr = addr; 912 param->len = mem_sz; 913 param->pgsz = cur_pgsz; 914 param->iova_table = iovas; 915 param->iova_table_len = n_pages; 916 917 return 0; 918 fail: 919 if (iovas) 920 free(iovas); 921 if (addr) 922 munmap(addr, mem_sz); 923 924 return -1; 925 } 926 927 static int 928 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 929 { 930 struct extmem_param param; 931 int socket_id, ret; 932 933 memset(¶m, 0, sizeof(param)); 934 935 /* check if our heap exists */ 936 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 937 if (socket_id < 0) { 938 /* create our heap */ 939 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 940 if (ret < 0) { 941 TESTPMD_LOG(ERR, "Cannot create heap\n"); 942 return -1; 943 } 944 } 945 946 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 947 if (ret < 0) { 948 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 949 return -1; 950 } 951 952 /* we now have a valid memory area, so add it to heap */ 953 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 954 param.addr, param.len, param.iova_table, 955 param.iova_table_len, param.pgsz); 956 957 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 958 959 /* not needed any more */ 960 free(param.iova_table); 961 962 if (ret < 0) { 963 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 964 munmap(param.addr, param.len); 965 return -1; 966 } 967 968 /* success */ 969 970 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 971 param.len >> 20); 972 973 return 0; 974 } 975 static void 976 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 977 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 978 { 979 uint16_t pid = 0; 980 int ret; 981 982 RTE_ETH_FOREACH_DEV(pid) { 983 struct rte_eth_dev_info dev_info; 984 985 ret = eth_dev_info_get_print_err(pid, &dev_info); 986 if (ret != 0) { 987 TESTPMD_LOG(DEBUG, 988 "unable to get device info for port %d on addr 0x%p," 989 "mempool unmapping will not be performed\n", 990 pid, memhdr->addr); 991 continue; 992 } 993 994 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 995 if (ret) { 996 TESTPMD_LOG(DEBUG, 997 "unable to DMA unmap addr 0x%p " 998 "for device %s\n", 999 memhdr->addr, dev_info.device->name); 1000 } 1001 } 1002 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1003 if (ret) { 1004 TESTPMD_LOG(DEBUG, 1005 "unable to un-register addr 0x%p\n", memhdr->addr); 1006 } 1007 } 1008 1009 static void 1010 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1011 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1012 { 1013 uint16_t pid = 0; 1014 size_t page_size = sysconf(_SC_PAGESIZE); 1015 int ret; 1016 1017 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1018 page_size); 1019 if (ret) { 1020 TESTPMD_LOG(DEBUG, 1021 "unable to register addr 0x%p\n", memhdr->addr); 1022 return; 1023 } 1024 RTE_ETH_FOREACH_DEV(pid) { 1025 struct rte_eth_dev_info dev_info; 1026 1027 ret = eth_dev_info_get_print_err(pid, &dev_info); 1028 if (ret != 0) { 1029 TESTPMD_LOG(DEBUG, 1030 "unable to get device info for port %d on addr 0x%p," 1031 "mempool mapping will not be performed\n", 1032 pid, memhdr->addr); 1033 continue; 1034 } 1035 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1036 if (ret) { 1037 TESTPMD_LOG(DEBUG, 1038 "unable to DMA map addr 0x%p " 1039 "for device %s\n", 1040 memhdr->addr, dev_info.device->name); 1041 } 1042 } 1043 } 1044 #endif 1045 1046 static unsigned int 1047 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1048 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1049 { 1050 struct rte_pktmbuf_extmem *xmem; 1051 unsigned int ext_num, zone_num, elt_num; 1052 uint16_t elt_size; 1053 1054 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1055 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1056 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1057 1058 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1059 if (xmem == NULL) { 1060 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1061 "external buffer descriptors\n"); 1062 *ext_mem = NULL; 1063 return 0; 1064 } 1065 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1066 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1067 const struct rte_memzone *mz; 1068 char mz_name[RTE_MEMZONE_NAMESIZE]; 1069 int ret; 1070 1071 ret = snprintf(mz_name, sizeof(mz_name), 1072 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1073 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1074 errno = ENAMETOOLONG; 1075 ext_num = 0; 1076 break; 1077 } 1078 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 1079 socket_id, 1080 RTE_MEMZONE_IOVA_CONTIG | 1081 RTE_MEMZONE_1GB | 1082 RTE_MEMZONE_SIZE_HINT_ONLY, 1083 EXTBUF_ZONE_SIZE); 1084 if (mz == NULL) { 1085 /* 1086 * The caller exits on external buffer creation 1087 * error, so there is no need to free memzones. 1088 */ 1089 errno = ENOMEM; 1090 ext_num = 0; 1091 break; 1092 } 1093 xseg->buf_ptr = mz->addr; 1094 xseg->buf_iova = mz->iova; 1095 xseg->buf_len = EXTBUF_ZONE_SIZE; 1096 xseg->elt_size = elt_size; 1097 } 1098 if (ext_num == 0 && xmem != NULL) { 1099 free(xmem); 1100 xmem = NULL; 1101 } 1102 *ext_mem = xmem; 1103 return ext_num; 1104 } 1105 1106 /* 1107 * Configuration initialisation done once at init time. 1108 */ 1109 static struct rte_mempool * 1110 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1111 unsigned int socket_id, uint16_t size_idx) 1112 { 1113 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1114 struct rte_mempool *rte_mp = NULL; 1115 #ifndef RTE_EXEC_ENV_WINDOWS 1116 uint32_t mb_size; 1117 1118 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1119 #endif 1120 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1121 if (!is_proc_primary()) { 1122 rte_mp = rte_mempool_lookup(pool_name); 1123 if (rte_mp == NULL) 1124 rte_exit(EXIT_FAILURE, 1125 "Get mbuf pool for socket %u failed: %s\n", 1126 socket_id, rte_strerror(rte_errno)); 1127 return rte_mp; 1128 } 1129 1130 TESTPMD_LOG(INFO, 1131 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1132 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1133 1134 switch (mp_alloc_type) { 1135 case MP_ALLOC_NATIVE: 1136 { 1137 /* wrapper to rte_mempool_create() */ 1138 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1139 rte_mbuf_best_mempool_ops()); 1140 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1141 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1142 break; 1143 } 1144 #ifndef RTE_EXEC_ENV_WINDOWS 1145 case MP_ALLOC_ANON: 1146 { 1147 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1148 mb_size, (unsigned int) mb_mempool_cache, 1149 sizeof(struct rte_pktmbuf_pool_private), 1150 socket_id, mempool_flags); 1151 if (rte_mp == NULL) 1152 goto err; 1153 1154 if (rte_mempool_populate_anon(rte_mp) == 0) { 1155 rte_mempool_free(rte_mp); 1156 rte_mp = NULL; 1157 goto err; 1158 } 1159 rte_pktmbuf_pool_init(rte_mp, NULL); 1160 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1161 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1162 break; 1163 } 1164 case MP_ALLOC_XMEM: 1165 case MP_ALLOC_XMEM_HUGE: 1166 { 1167 int heap_socket; 1168 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1169 1170 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1171 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1172 1173 heap_socket = 1174 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1175 if (heap_socket < 0) 1176 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1177 1178 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1179 rte_mbuf_best_mempool_ops()); 1180 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1181 mb_mempool_cache, 0, mbuf_seg_size, 1182 heap_socket); 1183 break; 1184 } 1185 #endif 1186 case MP_ALLOC_XBUF: 1187 { 1188 struct rte_pktmbuf_extmem *ext_mem; 1189 unsigned int ext_num; 1190 1191 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1192 socket_id, pool_name, &ext_mem); 1193 if (ext_num == 0) 1194 rte_exit(EXIT_FAILURE, 1195 "Can't create pinned data buffers\n"); 1196 1197 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1198 rte_mbuf_best_mempool_ops()); 1199 rte_mp = rte_pktmbuf_pool_create_extbuf 1200 (pool_name, nb_mbuf, mb_mempool_cache, 1201 0, mbuf_seg_size, socket_id, 1202 ext_mem, ext_num); 1203 free(ext_mem); 1204 break; 1205 } 1206 default: 1207 { 1208 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1209 } 1210 } 1211 1212 #ifndef RTE_EXEC_ENV_WINDOWS 1213 err: 1214 #endif 1215 if (rte_mp == NULL) { 1216 rte_exit(EXIT_FAILURE, 1217 "Creation of mbuf pool for socket %u failed: %s\n", 1218 socket_id, rte_strerror(rte_errno)); 1219 } else if (verbose_level > 0) { 1220 rte_mempool_dump(stdout, rte_mp); 1221 } 1222 return rte_mp; 1223 } 1224 1225 /* 1226 * Check given socket id is valid or not with NUMA mode, 1227 * if valid, return 0, else return -1 1228 */ 1229 static int 1230 check_socket_id(const unsigned int socket_id) 1231 { 1232 static int warning_once = 0; 1233 1234 if (new_socket_id(socket_id)) { 1235 if (!warning_once && numa_support) 1236 fprintf(stderr, 1237 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1238 warning_once = 1; 1239 return -1; 1240 } 1241 return 0; 1242 } 1243 1244 /* 1245 * Get the allowed maximum number of RX queues. 1246 * *pid return the port id which has minimal value of 1247 * max_rx_queues in all ports. 1248 */ 1249 queueid_t 1250 get_allowed_max_nb_rxq(portid_t *pid) 1251 { 1252 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1253 bool max_rxq_valid = false; 1254 portid_t pi; 1255 struct rte_eth_dev_info dev_info; 1256 1257 RTE_ETH_FOREACH_DEV(pi) { 1258 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1259 continue; 1260 1261 max_rxq_valid = true; 1262 if (dev_info.max_rx_queues < allowed_max_rxq) { 1263 allowed_max_rxq = dev_info.max_rx_queues; 1264 *pid = pi; 1265 } 1266 } 1267 return max_rxq_valid ? allowed_max_rxq : 0; 1268 } 1269 1270 /* 1271 * Check input rxq is valid or not. 1272 * If input rxq is not greater than any of maximum number 1273 * of RX queues of all ports, it is valid. 1274 * if valid, return 0, else return -1 1275 */ 1276 int 1277 check_nb_rxq(queueid_t rxq) 1278 { 1279 queueid_t allowed_max_rxq; 1280 portid_t pid = 0; 1281 1282 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1283 if (rxq > allowed_max_rxq) { 1284 fprintf(stderr, 1285 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1286 rxq, allowed_max_rxq, pid); 1287 return -1; 1288 } 1289 return 0; 1290 } 1291 1292 /* 1293 * Get the allowed maximum number of TX queues. 1294 * *pid return the port id which has minimal value of 1295 * max_tx_queues in all ports. 1296 */ 1297 queueid_t 1298 get_allowed_max_nb_txq(portid_t *pid) 1299 { 1300 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1301 bool max_txq_valid = false; 1302 portid_t pi; 1303 struct rte_eth_dev_info dev_info; 1304 1305 RTE_ETH_FOREACH_DEV(pi) { 1306 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1307 continue; 1308 1309 max_txq_valid = true; 1310 if (dev_info.max_tx_queues < allowed_max_txq) { 1311 allowed_max_txq = dev_info.max_tx_queues; 1312 *pid = pi; 1313 } 1314 } 1315 return max_txq_valid ? allowed_max_txq : 0; 1316 } 1317 1318 /* 1319 * Check input txq is valid or not. 1320 * If input txq is not greater than any of maximum number 1321 * of TX queues of all ports, it is valid. 1322 * if valid, return 0, else return -1 1323 */ 1324 int 1325 check_nb_txq(queueid_t txq) 1326 { 1327 queueid_t allowed_max_txq; 1328 portid_t pid = 0; 1329 1330 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1331 if (txq > allowed_max_txq) { 1332 fprintf(stderr, 1333 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1334 txq, allowed_max_txq, pid); 1335 return -1; 1336 } 1337 return 0; 1338 } 1339 1340 /* 1341 * Get the allowed maximum number of RXDs of every rx queue. 1342 * *pid return the port id which has minimal value of 1343 * max_rxd in all queues of all ports. 1344 */ 1345 static uint16_t 1346 get_allowed_max_nb_rxd(portid_t *pid) 1347 { 1348 uint16_t allowed_max_rxd = UINT16_MAX; 1349 portid_t pi; 1350 struct rte_eth_dev_info dev_info; 1351 1352 RTE_ETH_FOREACH_DEV(pi) { 1353 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1354 continue; 1355 1356 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1357 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1358 *pid = pi; 1359 } 1360 } 1361 return allowed_max_rxd; 1362 } 1363 1364 /* 1365 * Get the allowed minimal number of RXDs of every rx queue. 1366 * *pid return the port id which has minimal value of 1367 * min_rxd in all queues of all ports. 1368 */ 1369 static uint16_t 1370 get_allowed_min_nb_rxd(portid_t *pid) 1371 { 1372 uint16_t allowed_min_rxd = 0; 1373 portid_t pi; 1374 struct rte_eth_dev_info dev_info; 1375 1376 RTE_ETH_FOREACH_DEV(pi) { 1377 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1378 continue; 1379 1380 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1381 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1382 *pid = pi; 1383 } 1384 } 1385 1386 return allowed_min_rxd; 1387 } 1388 1389 /* 1390 * Check input rxd is valid or not. 1391 * If input rxd is not greater than any of maximum number 1392 * of RXDs of every Rx queues and is not less than any of 1393 * minimal number of RXDs of every Rx queues, it is valid. 1394 * if valid, return 0, else return -1 1395 */ 1396 int 1397 check_nb_rxd(queueid_t rxd) 1398 { 1399 uint16_t allowed_max_rxd; 1400 uint16_t allowed_min_rxd; 1401 portid_t pid = 0; 1402 1403 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1404 if (rxd > allowed_max_rxd) { 1405 fprintf(stderr, 1406 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1407 rxd, allowed_max_rxd, pid); 1408 return -1; 1409 } 1410 1411 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1412 if (rxd < allowed_min_rxd) { 1413 fprintf(stderr, 1414 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1415 rxd, allowed_min_rxd, pid); 1416 return -1; 1417 } 1418 1419 return 0; 1420 } 1421 1422 /* 1423 * Get the allowed maximum number of TXDs of every rx queues. 1424 * *pid return the port id which has minimal value of 1425 * max_txd in every tx queue. 1426 */ 1427 static uint16_t 1428 get_allowed_max_nb_txd(portid_t *pid) 1429 { 1430 uint16_t allowed_max_txd = UINT16_MAX; 1431 portid_t pi; 1432 struct rte_eth_dev_info dev_info; 1433 1434 RTE_ETH_FOREACH_DEV(pi) { 1435 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1436 continue; 1437 1438 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1439 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1440 *pid = pi; 1441 } 1442 } 1443 return allowed_max_txd; 1444 } 1445 1446 /* 1447 * Get the allowed maximum number of TXDs of every tx queues. 1448 * *pid return the port id which has minimal value of 1449 * min_txd in every tx queue. 1450 */ 1451 static uint16_t 1452 get_allowed_min_nb_txd(portid_t *pid) 1453 { 1454 uint16_t allowed_min_txd = 0; 1455 portid_t pi; 1456 struct rte_eth_dev_info dev_info; 1457 1458 RTE_ETH_FOREACH_DEV(pi) { 1459 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1460 continue; 1461 1462 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1463 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1464 *pid = pi; 1465 } 1466 } 1467 1468 return allowed_min_txd; 1469 } 1470 1471 /* 1472 * Check input txd is valid or not. 1473 * If input txd is not greater than any of maximum number 1474 * of TXDs of every Rx queues, it is valid. 1475 * if valid, return 0, else return -1 1476 */ 1477 int 1478 check_nb_txd(queueid_t txd) 1479 { 1480 uint16_t allowed_max_txd; 1481 uint16_t allowed_min_txd; 1482 portid_t pid = 0; 1483 1484 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1485 if (txd > allowed_max_txd) { 1486 fprintf(stderr, 1487 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1488 txd, allowed_max_txd, pid); 1489 return -1; 1490 } 1491 1492 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1493 if (txd < allowed_min_txd) { 1494 fprintf(stderr, 1495 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1496 txd, allowed_min_txd, pid); 1497 return -1; 1498 } 1499 return 0; 1500 } 1501 1502 1503 /* 1504 * Get the allowed maximum number of hairpin queues. 1505 * *pid return the port id which has minimal value of 1506 * max_hairpin_queues in all ports. 1507 */ 1508 queueid_t 1509 get_allowed_max_nb_hairpinq(portid_t *pid) 1510 { 1511 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1512 portid_t pi; 1513 struct rte_eth_hairpin_cap cap; 1514 1515 RTE_ETH_FOREACH_DEV(pi) { 1516 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1517 *pid = pi; 1518 return 0; 1519 } 1520 if (cap.max_nb_queues < allowed_max_hairpinq) { 1521 allowed_max_hairpinq = cap.max_nb_queues; 1522 *pid = pi; 1523 } 1524 } 1525 return allowed_max_hairpinq; 1526 } 1527 1528 /* 1529 * Check input hairpin is valid or not. 1530 * If input hairpin is not greater than any of maximum number 1531 * of hairpin queues of all ports, it is valid. 1532 * if valid, return 0, else return -1 1533 */ 1534 int 1535 check_nb_hairpinq(queueid_t hairpinq) 1536 { 1537 queueid_t allowed_max_hairpinq; 1538 portid_t pid = 0; 1539 1540 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1541 if (hairpinq > allowed_max_hairpinq) { 1542 fprintf(stderr, 1543 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1544 hairpinq, allowed_max_hairpinq, pid); 1545 return -1; 1546 } 1547 return 0; 1548 } 1549 1550 static int 1551 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1552 { 1553 uint32_t eth_overhead; 1554 1555 if (dev_info->max_mtu != UINT16_MAX && 1556 dev_info->max_rx_pktlen > dev_info->max_mtu) 1557 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1558 else 1559 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1560 1561 return eth_overhead; 1562 } 1563 1564 static void 1565 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1566 { 1567 struct rte_port *port = &ports[pid]; 1568 int ret; 1569 int i; 1570 1571 eth_rx_metadata_negotiate_mp(pid); 1572 flow_pick_transfer_proxy_mp(pid); 1573 1574 port->dev_conf.txmode = tx_mode; 1575 port->dev_conf.rxmode = rx_mode; 1576 1577 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1578 if (ret != 0) 1579 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1580 1581 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)) 1582 port->dev_conf.txmode.offloads &= 1583 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1584 1585 /* Apply Rx offloads configuration */ 1586 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1587 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads; 1588 /* Apply Tx offloads configuration */ 1589 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1590 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads; 1591 1592 if (eth_link_speed) 1593 port->dev_conf.link_speeds = eth_link_speed; 1594 1595 if (max_rx_pkt_len) 1596 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1597 get_eth_overhead(&port->dev_info); 1598 1599 /* set flag to initialize port/queue */ 1600 port->need_reconfig = 1; 1601 port->need_reconfig_queues = 1; 1602 port->socket_id = socket_id; 1603 port->tx_metadata = 0; 1604 1605 /* 1606 * Check for maximum number of segments per MTU. 1607 * Accordingly update the mbuf data size. 1608 */ 1609 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1610 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1611 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1612 uint16_t mtu; 1613 1614 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1615 uint16_t data_size = (mtu + eth_overhead) / 1616 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1617 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1618 1619 if (buffer_size > mbuf_data_size[0]) { 1620 mbuf_data_size[0] = buffer_size; 1621 TESTPMD_LOG(WARNING, 1622 "Configured mbuf size of the first segment %hu\n", 1623 mbuf_data_size[0]); 1624 } 1625 } 1626 } 1627 } 1628 1629 static void 1630 init_config(void) 1631 { 1632 portid_t pid; 1633 struct rte_mempool *mbp; 1634 unsigned int nb_mbuf_per_pool; 1635 lcoreid_t lc_id; 1636 struct rte_gro_param gro_param; 1637 uint32_t gso_types; 1638 1639 /* Configuration of logical cores. */ 1640 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1641 sizeof(struct fwd_lcore *) * nb_lcores, 1642 RTE_CACHE_LINE_SIZE); 1643 if (fwd_lcores == NULL) { 1644 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1645 "failed\n", nb_lcores); 1646 } 1647 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1648 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1649 sizeof(struct fwd_lcore), 1650 RTE_CACHE_LINE_SIZE); 1651 if (fwd_lcores[lc_id] == NULL) { 1652 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1653 "failed\n"); 1654 } 1655 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1656 } 1657 1658 RTE_ETH_FOREACH_DEV(pid) { 1659 uint32_t socket_id; 1660 1661 if (numa_support) { 1662 socket_id = port_numa[pid]; 1663 if (port_numa[pid] == NUMA_NO_CONFIG) { 1664 socket_id = rte_eth_dev_socket_id(pid); 1665 1666 /* 1667 * if socket_id is invalid, 1668 * set to the first available socket. 1669 */ 1670 if (check_socket_id(socket_id) < 0) 1671 socket_id = socket_ids[0]; 1672 } 1673 } else { 1674 socket_id = (socket_num == UMA_NO_CONFIG) ? 1675 0 : socket_num; 1676 } 1677 /* Apply default TxRx configuration for all ports */ 1678 init_config_port_offloads(pid, socket_id); 1679 } 1680 /* 1681 * Create pools of mbuf. 1682 * If NUMA support is disabled, create a single pool of mbuf in 1683 * socket 0 memory by default. 1684 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1685 * 1686 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1687 * nb_txd can be configured at run time. 1688 */ 1689 if (param_total_num_mbufs) 1690 nb_mbuf_per_pool = param_total_num_mbufs; 1691 else { 1692 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1693 (nb_lcores * mb_mempool_cache) + 1694 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1695 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1696 } 1697 1698 if (numa_support) { 1699 uint8_t i, j; 1700 1701 for (i = 0; i < num_sockets; i++) 1702 for (j = 0; j < mbuf_data_size_n; j++) 1703 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1704 mbuf_pool_create(mbuf_data_size[j], 1705 nb_mbuf_per_pool, 1706 socket_ids[i], j); 1707 } else { 1708 uint8_t i; 1709 1710 for (i = 0; i < mbuf_data_size_n; i++) 1711 mempools[i] = mbuf_pool_create 1712 (mbuf_data_size[i], 1713 nb_mbuf_per_pool, 1714 socket_num == UMA_NO_CONFIG ? 1715 0 : socket_num, i); 1716 } 1717 1718 init_port_config(); 1719 1720 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1721 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO; 1722 /* 1723 * Records which Mbuf pool to use by each logical core, if needed. 1724 */ 1725 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1726 mbp = mbuf_pool_find( 1727 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1728 1729 if (mbp == NULL) 1730 mbp = mbuf_pool_find(0, 0); 1731 fwd_lcores[lc_id]->mbp = mbp; 1732 /* initialize GSO context */ 1733 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1734 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1735 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1736 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1737 RTE_ETHER_CRC_LEN; 1738 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1739 } 1740 1741 fwd_config_setup(); 1742 1743 /* create a gro context for each lcore */ 1744 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1745 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1746 gro_param.max_item_per_flow = MAX_PKT_BURST; 1747 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1748 gro_param.socket_id = rte_lcore_to_socket_id( 1749 fwd_lcores_cpuids[lc_id]); 1750 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1751 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1752 rte_exit(EXIT_FAILURE, 1753 "rte_gro_ctx_create() failed\n"); 1754 } 1755 } 1756 } 1757 1758 1759 void 1760 reconfig(portid_t new_port_id, unsigned socket_id) 1761 { 1762 /* Reconfiguration of Ethernet ports. */ 1763 init_config_port_offloads(new_port_id, socket_id); 1764 init_port_config(); 1765 } 1766 1767 1768 int 1769 init_fwd_streams(void) 1770 { 1771 portid_t pid; 1772 struct rte_port *port; 1773 streamid_t sm_id, nb_fwd_streams_new; 1774 queueid_t q; 1775 1776 /* set socket id according to numa or not */ 1777 RTE_ETH_FOREACH_DEV(pid) { 1778 port = &ports[pid]; 1779 if (nb_rxq > port->dev_info.max_rx_queues) { 1780 fprintf(stderr, 1781 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1782 nb_rxq, port->dev_info.max_rx_queues); 1783 return -1; 1784 } 1785 if (nb_txq > port->dev_info.max_tx_queues) { 1786 fprintf(stderr, 1787 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1788 nb_txq, port->dev_info.max_tx_queues); 1789 return -1; 1790 } 1791 if (numa_support) { 1792 if (port_numa[pid] != NUMA_NO_CONFIG) 1793 port->socket_id = port_numa[pid]; 1794 else { 1795 port->socket_id = rte_eth_dev_socket_id(pid); 1796 1797 /* 1798 * if socket_id is invalid, 1799 * set to the first available socket. 1800 */ 1801 if (check_socket_id(port->socket_id) < 0) 1802 port->socket_id = socket_ids[0]; 1803 } 1804 } 1805 else { 1806 if (socket_num == UMA_NO_CONFIG) 1807 port->socket_id = 0; 1808 else 1809 port->socket_id = socket_num; 1810 } 1811 } 1812 1813 q = RTE_MAX(nb_rxq, nb_txq); 1814 if (q == 0) { 1815 fprintf(stderr, 1816 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1817 return -1; 1818 } 1819 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1820 if (nb_fwd_streams_new == nb_fwd_streams) 1821 return 0; 1822 /* clear the old */ 1823 if (fwd_streams != NULL) { 1824 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1825 if (fwd_streams[sm_id] == NULL) 1826 continue; 1827 rte_free(fwd_streams[sm_id]); 1828 fwd_streams[sm_id] = NULL; 1829 } 1830 rte_free(fwd_streams); 1831 fwd_streams = NULL; 1832 } 1833 1834 /* init new */ 1835 nb_fwd_streams = nb_fwd_streams_new; 1836 if (nb_fwd_streams) { 1837 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1838 sizeof(struct fwd_stream *) * nb_fwd_streams, 1839 RTE_CACHE_LINE_SIZE); 1840 if (fwd_streams == NULL) 1841 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1842 " (struct fwd_stream *)) failed\n", 1843 nb_fwd_streams); 1844 1845 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1846 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1847 " struct fwd_stream", sizeof(struct fwd_stream), 1848 RTE_CACHE_LINE_SIZE); 1849 if (fwd_streams[sm_id] == NULL) 1850 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1851 "(struct fwd_stream) failed\n"); 1852 } 1853 } 1854 1855 return 0; 1856 } 1857 1858 static void 1859 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1860 { 1861 uint64_t total_burst, sburst; 1862 uint64_t nb_burst; 1863 uint64_t burst_stats[4]; 1864 uint16_t pktnb_stats[4]; 1865 uint16_t nb_pkt; 1866 int burst_percent[4], sburstp; 1867 int i; 1868 1869 /* 1870 * First compute the total number of packet bursts and the 1871 * two highest numbers of bursts of the same number of packets. 1872 */ 1873 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1874 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1875 1876 /* Show stats for 0 burst size always */ 1877 total_burst = pbs->pkt_burst_spread[0]; 1878 burst_stats[0] = pbs->pkt_burst_spread[0]; 1879 pktnb_stats[0] = 0; 1880 1881 /* Find the next 2 burst sizes with highest occurrences. */ 1882 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) { 1883 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1884 1885 if (nb_burst == 0) 1886 continue; 1887 1888 total_burst += nb_burst; 1889 1890 if (nb_burst > burst_stats[1]) { 1891 burst_stats[2] = burst_stats[1]; 1892 pktnb_stats[2] = pktnb_stats[1]; 1893 burst_stats[1] = nb_burst; 1894 pktnb_stats[1] = nb_pkt; 1895 } else if (nb_burst > burst_stats[2]) { 1896 burst_stats[2] = nb_burst; 1897 pktnb_stats[2] = nb_pkt; 1898 } 1899 } 1900 if (total_burst == 0) 1901 return; 1902 1903 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1904 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1905 if (i == 3) { 1906 printf("%d%% of other]\n", 100 - sburstp); 1907 return; 1908 } 1909 1910 sburst += burst_stats[i]; 1911 if (sburst == total_burst) { 1912 printf("%d%% of %d pkts]\n", 1913 100 - sburstp, (int) pktnb_stats[i]); 1914 return; 1915 } 1916 1917 burst_percent[i] = 1918 (double)burst_stats[i] / total_burst * 100; 1919 printf("%d%% of %d pkts + ", 1920 burst_percent[i], (int) pktnb_stats[i]); 1921 sburstp += burst_percent[i]; 1922 } 1923 } 1924 1925 static void 1926 fwd_stream_stats_display(streamid_t stream_id) 1927 { 1928 struct fwd_stream *fs; 1929 static const char *fwd_top_stats_border = "-------"; 1930 1931 fs = fwd_streams[stream_id]; 1932 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1933 (fs->fwd_dropped == 0)) 1934 return; 1935 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1936 "TX Port=%2d/Queue=%2d %s\n", 1937 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1938 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1939 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1940 " TX-dropped: %-14"PRIu64, 1941 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1942 1943 /* if checksum mode */ 1944 if (cur_fwd_eng == &csum_fwd_engine) { 1945 printf(" RX- bad IP checksum: %-14"PRIu64 1946 " Rx- bad L4 checksum: %-14"PRIu64 1947 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1948 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1949 fs->rx_bad_outer_l4_csum); 1950 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1951 fs->rx_bad_outer_ip_csum); 1952 } else { 1953 printf("\n"); 1954 } 1955 1956 if (record_burst_stats) { 1957 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1958 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1959 } 1960 } 1961 1962 void 1963 fwd_stats_display(void) 1964 { 1965 static const char *fwd_stats_border = "----------------------"; 1966 static const char *acc_stats_border = "+++++++++++++++"; 1967 struct { 1968 struct fwd_stream *rx_stream; 1969 struct fwd_stream *tx_stream; 1970 uint64_t tx_dropped; 1971 uint64_t rx_bad_ip_csum; 1972 uint64_t rx_bad_l4_csum; 1973 uint64_t rx_bad_outer_l4_csum; 1974 uint64_t rx_bad_outer_ip_csum; 1975 } ports_stats[RTE_MAX_ETHPORTS]; 1976 uint64_t total_rx_dropped = 0; 1977 uint64_t total_tx_dropped = 0; 1978 uint64_t total_rx_nombuf = 0; 1979 struct rte_eth_stats stats; 1980 uint64_t fwd_cycles = 0; 1981 uint64_t total_recv = 0; 1982 uint64_t total_xmit = 0; 1983 struct rte_port *port; 1984 streamid_t sm_id; 1985 portid_t pt_id; 1986 int i; 1987 1988 memset(ports_stats, 0, sizeof(ports_stats)); 1989 1990 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1991 struct fwd_stream *fs = fwd_streams[sm_id]; 1992 1993 if (cur_fwd_config.nb_fwd_streams > 1994 cur_fwd_config.nb_fwd_ports) { 1995 fwd_stream_stats_display(sm_id); 1996 } else { 1997 ports_stats[fs->tx_port].tx_stream = fs; 1998 ports_stats[fs->rx_port].rx_stream = fs; 1999 } 2000 2001 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2002 2003 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2004 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2005 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2006 fs->rx_bad_outer_l4_csum; 2007 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2008 fs->rx_bad_outer_ip_csum; 2009 2010 if (record_core_cycles) 2011 fwd_cycles += fs->core_cycles; 2012 } 2013 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2014 pt_id = fwd_ports_ids[i]; 2015 port = &ports[pt_id]; 2016 2017 rte_eth_stats_get(pt_id, &stats); 2018 stats.ipackets -= port->stats.ipackets; 2019 stats.opackets -= port->stats.opackets; 2020 stats.ibytes -= port->stats.ibytes; 2021 stats.obytes -= port->stats.obytes; 2022 stats.imissed -= port->stats.imissed; 2023 stats.oerrors -= port->stats.oerrors; 2024 stats.rx_nombuf -= port->stats.rx_nombuf; 2025 2026 total_recv += stats.ipackets; 2027 total_xmit += stats.opackets; 2028 total_rx_dropped += stats.imissed; 2029 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2030 total_tx_dropped += stats.oerrors; 2031 total_rx_nombuf += stats.rx_nombuf; 2032 2033 printf("\n %s Forward statistics for port %-2d %s\n", 2034 fwd_stats_border, pt_id, fwd_stats_border); 2035 2036 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2037 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2038 stats.ipackets + stats.imissed); 2039 2040 if (cur_fwd_eng == &csum_fwd_engine) { 2041 printf(" Bad-ipcsum: %-14"PRIu64 2042 " Bad-l4csum: %-14"PRIu64 2043 "Bad-outer-l4csum: %-14"PRIu64"\n", 2044 ports_stats[pt_id].rx_bad_ip_csum, 2045 ports_stats[pt_id].rx_bad_l4_csum, 2046 ports_stats[pt_id].rx_bad_outer_l4_csum); 2047 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2048 ports_stats[pt_id].rx_bad_outer_ip_csum); 2049 } 2050 if (stats.ierrors + stats.rx_nombuf > 0) { 2051 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2052 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2053 } 2054 2055 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2056 "TX-total: %-"PRIu64"\n", 2057 stats.opackets, ports_stats[pt_id].tx_dropped, 2058 stats.opackets + ports_stats[pt_id].tx_dropped); 2059 2060 if (record_burst_stats) { 2061 if (ports_stats[pt_id].rx_stream) 2062 pkt_burst_stats_display("RX", 2063 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2064 if (ports_stats[pt_id].tx_stream) 2065 pkt_burst_stats_display("TX", 2066 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2067 } 2068 2069 printf(" %s--------------------------------%s\n", 2070 fwd_stats_border, fwd_stats_border); 2071 } 2072 2073 printf("\n %s Accumulated forward statistics for all ports" 2074 "%s\n", 2075 acc_stats_border, acc_stats_border); 2076 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2077 "%-"PRIu64"\n" 2078 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2079 "%-"PRIu64"\n", 2080 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2081 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2082 if (total_rx_nombuf > 0) 2083 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2084 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2085 "%s\n", 2086 acc_stats_border, acc_stats_border); 2087 if (record_core_cycles) { 2088 #define CYC_PER_MHZ 1E6 2089 if (total_recv > 0 || total_xmit > 0) { 2090 uint64_t total_pkts = 0; 2091 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2092 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2093 total_pkts = total_xmit; 2094 else 2095 total_pkts = total_recv; 2096 2097 printf("\n CPU cycles/packet=%.2F (total cycles=" 2098 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2099 " MHz Clock\n", 2100 (double) fwd_cycles / total_pkts, 2101 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2102 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2103 } 2104 } 2105 } 2106 2107 void 2108 fwd_stats_reset(void) 2109 { 2110 streamid_t sm_id; 2111 portid_t pt_id; 2112 int i; 2113 2114 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2115 pt_id = fwd_ports_ids[i]; 2116 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2117 } 2118 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2119 struct fwd_stream *fs = fwd_streams[sm_id]; 2120 2121 fs->rx_packets = 0; 2122 fs->tx_packets = 0; 2123 fs->fwd_dropped = 0; 2124 fs->rx_bad_ip_csum = 0; 2125 fs->rx_bad_l4_csum = 0; 2126 fs->rx_bad_outer_l4_csum = 0; 2127 fs->rx_bad_outer_ip_csum = 0; 2128 2129 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2130 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2131 fs->core_cycles = 0; 2132 } 2133 } 2134 2135 static void 2136 flush_fwd_rx_queues(void) 2137 { 2138 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2139 portid_t rxp; 2140 portid_t port_id; 2141 queueid_t rxq; 2142 uint16_t nb_rx; 2143 uint16_t i; 2144 uint8_t j; 2145 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2146 uint64_t timer_period; 2147 2148 if (num_procs > 1) { 2149 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2150 return; 2151 } 2152 2153 /* convert to number of cycles */ 2154 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2155 2156 for (j = 0; j < 2; j++) { 2157 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2158 for (rxq = 0; rxq < nb_rxq; rxq++) { 2159 port_id = fwd_ports_ids[rxp]; 2160 /** 2161 * testpmd can stuck in the below do while loop 2162 * if rte_eth_rx_burst() always returns nonzero 2163 * packets. So timer is added to exit this loop 2164 * after 1sec timer expiry. 2165 */ 2166 prev_tsc = rte_rdtsc(); 2167 do { 2168 nb_rx = rte_eth_rx_burst(port_id, rxq, 2169 pkts_burst, MAX_PKT_BURST); 2170 for (i = 0; i < nb_rx; i++) 2171 rte_pktmbuf_free(pkts_burst[i]); 2172 2173 cur_tsc = rte_rdtsc(); 2174 diff_tsc = cur_tsc - prev_tsc; 2175 timer_tsc += diff_tsc; 2176 } while ((nb_rx > 0) && 2177 (timer_tsc < timer_period)); 2178 timer_tsc = 0; 2179 } 2180 } 2181 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2182 } 2183 } 2184 2185 static void 2186 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2187 { 2188 struct fwd_stream **fsm; 2189 streamid_t nb_fs; 2190 streamid_t sm_id; 2191 #ifdef RTE_LIB_BITRATESTATS 2192 uint64_t tics_per_1sec; 2193 uint64_t tics_datum; 2194 uint64_t tics_current; 2195 uint16_t i, cnt_ports; 2196 2197 cnt_ports = nb_ports; 2198 tics_datum = rte_rdtsc(); 2199 tics_per_1sec = rte_get_timer_hz(); 2200 #endif 2201 fsm = &fwd_streams[fc->stream_idx]; 2202 nb_fs = fc->stream_nb; 2203 do { 2204 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2205 (*pkt_fwd)(fsm[sm_id]); 2206 #ifdef RTE_LIB_BITRATESTATS 2207 if (bitrate_enabled != 0 && 2208 bitrate_lcore_id == rte_lcore_id()) { 2209 tics_current = rte_rdtsc(); 2210 if (tics_current - tics_datum >= tics_per_1sec) { 2211 /* Periodic bitrate calculation */ 2212 for (i = 0; i < cnt_ports; i++) 2213 rte_stats_bitrate_calc(bitrate_data, 2214 ports_ids[i]); 2215 tics_datum = tics_current; 2216 } 2217 } 2218 #endif 2219 #ifdef RTE_LIB_LATENCYSTATS 2220 if (latencystats_enabled != 0 && 2221 latencystats_lcore_id == rte_lcore_id()) 2222 rte_latencystats_update(); 2223 #endif 2224 2225 } while (! fc->stopped); 2226 } 2227 2228 static int 2229 start_pkt_forward_on_core(void *fwd_arg) 2230 { 2231 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2232 cur_fwd_config.fwd_eng->packet_fwd); 2233 return 0; 2234 } 2235 2236 /* 2237 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2238 * Used to start communication flows in network loopback test configurations. 2239 */ 2240 static int 2241 run_one_txonly_burst_on_core(void *fwd_arg) 2242 { 2243 struct fwd_lcore *fwd_lc; 2244 struct fwd_lcore tmp_lcore; 2245 2246 fwd_lc = (struct fwd_lcore *) fwd_arg; 2247 tmp_lcore = *fwd_lc; 2248 tmp_lcore.stopped = 1; 2249 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2250 return 0; 2251 } 2252 2253 /* 2254 * Launch packet forwarding: 2255 * - Setup per-port forwarding context. 2256 * - launch logical cores with their forwarding configuration. 2257 */ 2258 static void 2259 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2260 { 2261 unsigned int i; 2262 unsigned int lc_id; 2263 int diag; 2264 2265 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2266 lc_id = fwd_lcores_cpuids[i]; 2267 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2268 fwd_lcores[i]->stopped = 0; 2269 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2270 fwd_lcores[i], lc_id); 2271 if (diag != 0) 2272 fprintf(stderr, 2273 "launch lcore %u failed - diag=%d\n", 2274 lc_id, diag); 2275 } 2276 } 2277 } 2278 2279 /* 2280 * Launch packet forwarding configuration. 2281 */ 2282 void 2283 start_packet_forwarding(int with_tx_first) 2284 { 2285 port_fwd_begin_t port_fwd_begin; 2286 port_fwd_end_t port_fwd_end; 2287 unsigned int i; 2288 2289 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2290 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2291 2292 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2293 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2294 2295 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2296 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2297 (!nb_rxq || !nb_txq)) 2298 rte_exit(EXIT_FAILURE, 2299 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2300 cur_fwd_eng->fwd_mode_name); 2301 2302 if (all_ports_started() == 0) { 2303 fprintf(stderr, "Not all ports were started\n"); 2304 return; 2305 } 2306 if (test_done == 0) { 2307 fprintf(stderr, "Packet forwarding already started\n"); 2308 return; 2309 } 2310 2311 fwd_config_setup(); 2312 2313 pkt_fwd_config_display(&cur_fwd_config); 2314 if (!pkt_fwd_shared_rxq_check()) 2315 return; 2316 2317 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2318 if (port_fwd_begin != NULL) { 2319 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2320 if (port_fwd_begin(fwd_ports_ids[i])) { 2321 fprintf(stderr, 2322 "Packet forwarding is not ready\n"); 2323 return; 2324 } 2325 } 2326 } 2327 2328 if (with_tx_first) { 2329 port_fwd_begin = tx_only_engine.port_fwd_begin; 2330 if (port_fwd_begin != NULL) { 2331 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2332 if (port_fwd_begin(fwd_ports_ids[i])) { 2333 fprintf(stderr, 2334 "Packet forwarding is not ready\n"); 2335 return; 2336 } 2337 } 2338 } 2339 } 2340 2341 test_done = 0; 2342 2343 if(!no_flush_rx) 2344 flush_fwd_rx_queues(); 2345 2346 rxtx_config_display(); 2347 2348 fwd_stats_reset(); 2349 if (with_tx_first) { 2350 while (with_tx_first--) { 2351 launch_packet_forwarding( 2352 run_one_txonly_burst_on_core); 2353 rte_eal_mp_wait_lcore(); 2354 } 2355 port_fwd_end = tx_only_engine.port_fwd_end; 2356 if (port_fwd_end != NULL) { 2357 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2358 (*port_fwd_end)(fwd_ports_ids[i]); 2359 } 2360 } 2361 launch_packet_forwarding(start_pkt_forward_on_core); 2362 } 2363 2364 void 2365 stop_packet_forwarding(void) 2366 { 2367 port_fwd_end_t port_fwd_end; 2368 lcoreid_t lc_id; 2369 portid_t pt_id; 2370 int i; 2371 2372 if (test_done) { 2373 fprintf(stderr, "Packet forwarding not started\n"); 2374 return; 2375 } 2376 printf("Telling cores to stop..."); 2377 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2378 fwd_lcores[lc_id]->stopped = 1; 2379 printf("\nWaiting for lcores to finish...\n"); 2380 rte_eal_mp_wait_lcore(); 2381 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2382 if (port_fwd_end != NULL) { 2383 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2384 pt_id = fwd_ports_ids[i]; 2385 (*port_fwd_end)(pt_id); 2386 } 2387 } 2388 2389 fwd_stats_display(); 2390 2391 printf("\nDone.\n"); 2392 test_done = 1; 2393 } 2394 2395 void 2396 dev_set_link_up(portid_t pid) 2397 { 2398 if (rte_eth_dev_set_link_up(pid) < 0) 2399 fprintf(stderr, "\nSet link up fail.\n"); 2400 } 2401 2402 void 2403 dev_set_link_down(portid_t pid) 2404 { 2405 if (rte_eth_dev_set_link_down(pid) < 0) 2406 fprintf(stderr, "\nSet link down fail.\n"); 2407 } 2408 2409 static int 2410 all_ports_started(void) 2411 { 2412 portid_t pi; 2413 struct rte_port *port; 2414 2415 RTE_ETH_FOREACH_DEV(pi) { 2416 port = &ports[pi]; 2417 /* Check if there is a port which is not started */ 2418 if ((port->port_status != RTE_PORT_STARTED) && 2419 (port->slave_flag == 0)) 2420 return 0; 2421 } 2422 2423 /* No port is not started */ 2424 return 1; 2425 } 2426 2427 int 2428 port_is_stopped(portid_t port_id) 2429 { 2430 struct rte_port *port = &ports[port_id]; 2431 2432 if ((port->port_status != RTE_PORT_STOPPED) && 2433 (port->slave_flag == 0)) 2434 return 0; 2435 return 1; 2436 } 2437 2438 int 2439 all_ports_stopped(void) 2440 { 2441 portid_t pi; 2442 2443 RTE_ETH_FOREACH_DEV(pi) { 2444 if (!port_is_stopped(pi)) 2445 return 0; 2446 } 2447 2448 return 1; 2449 } 2450 2451 int 2452 port_is_started(portid_t port_id) 2453 { 2454 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2455 return 0; 2456 2457 if (ports[port_id].port_status != RTE_PORT_STARTED) 2458 return 0; 2459 2460 return 1; 2461 } 2462 2463 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2464 static int 2465 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2466 { 2467 queueid_t qi; 2468 struct rte_eth_hairpin_conf hairpin_conf = { 2469 .peer_count = 1, 2470 }; 2471 int i; 2472 int diag; 2473 struct rte_port *port = &ports[pi]; 2474 uint16_t peer_rx_port = pi; 2475 uint16_t peer_tx_port = pi; 2476 uint32_t manual = 1; 2477 uint32_t tx_exp = hairpin_mode & 0x10; 2478 2479 if (!(hairpin_mode & 0xf)) { 2480 peer_rx_port = pi; 2481 peer_tx_port = pi; 2482 manual = 0; 2483 } else if (hairpin_mode & 0x1) { 2484 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2485 RTE_ETH_DEV_NO_OWNER); 2486 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2487 peer_tx_port = rte_eth_find_next_owned_by(0, 2488 RTE_ETH_DEV_NO_OWNER); 2489 if (p_pi != RTE_MAX_ETHPORTS) { 2490 peer_rx_port = p_pi; 2491 } else { 2492 uint16_t next_pi; 2493 2494 /* Last port will be the peer RX port of the first. */ 2495 RTE_ETH_FOREACH_DEV(next_pi) 2496 peer_rx_port = next_pi; 2497 } 2498 manual = 1; 2499 } else if (hairpin_mode & 0x2) { 2500 if (cnt_pi & 0x1) { 2501 peer_rx_port = p_pi; 2502 } else { 2503 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2504 RTE_ETH_DEV_NO_OWNER); 2505 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2506 peer_rx_port = pi; 2507 } 2508 peer_tx_port = peer_rx_port; 2509 manual = 1; 2510 } 2511 2512 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2513 hairpin_conf.peers[0].port = peer_rx_port; 2514 hairpin_conf.peers[0].queue = i + nb_rxq; 2515 hairpin_conf.manual_bind = !!manual; 2516 hairpin_conf.tx_explicit = !!tx_exp; 2517 diag = rte_eth_tx_hairpin_queue_setup 2518 (pi, qi, nb_txd, &hairpin_conf); 2519 i++; 2520 if (diag == 0) 2521 continue; 2522 2523 /* Fail to setup rx queue, return */ 2524 if (rte_atomic16_cmpset(&(port->port_status), 2525 RTE_PORT_HANDLING, 2526 RTE_PORT_STOPPED) == 0) 2527 fprintf(stderr, 2528 "Port %d can not be set back to stopped\n", pi); 2529 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2530 pi); 2531 /* try to reconfigure queues next time */ 2532 port->need_reconfig_queues = 1; 2533 return -1; 2534 } 2535 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2536 hairpin_conf.peers[0].port = peer_tx_port; 2537 hairpin_conf.peers[0].queue = i + nb_txq; 2538 hairpin_conf.manual_bind = !!manual; 2539 hairpin_conf.tx_explicit = !!tx_exp; 2540 diag = rte_eth_rx_hairpin_queue_setup 2541 (pi, qi, nb_rxd, &hairpin_conf); 2542 i++; 2543 if (diag == 0) 2544 continue; 2545 2546 /* Fail to setup rx queue, return */ 2547 if (rte_atomic16_cmpset(&(port->port_status), 2548 RTE_PORT_HANDLING, 2549 RTE_PORT_STOPPED) == 0) 2550 fprintf(stderr, 2551 "Port %d can not be set back to stopped\n", pi); 2552 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2553 pi); 2554 /* try to reconfigure queues next time */ 2555 port->need_reconfig_queues = 1; 2556 return -1; 2557 } 2558 return 0; 2559 } 2560 2561 /* Configure the Rx with optional split. */ 2562 int 2563 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2564 uint16_t nb_rx_desc, unsigned int socket_id, 2565 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2566 { 2567 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2568 unsigned int i, mp_n; 2569 int ret; 2570 2571 if (rx_pkt_nb_segs <= 1 || 2572 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2573 rx_conf->rx_seg = NULL; 2574 rx_conf->rx_nseg = 0; 2575 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2576 nb_rx_desc, socket_id, 2577 rx_conf, mp); 2578 return ret; 2579 } 2580 for (i = 0; i < rx_pkt_nb_segs; i++) { 2581 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2582 struct rte_mempool *mpx; 2583 /* 2584 * Use last valid pool for the segments with number 2585 * exceeding the pool index. 2586 */ 2587 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2588 mpx = mbuf_pool_find(socket_id, mp_n); 2589 /* Handle zero as mbuf data buffer size. */ 2590 rx_seg->length = rx_pkt_seg_lengths[i] ? 2591 rx_pkt_seg_lengths[i] : 2592 mbuf_data_size[mp_n]; 2593 rx_seg->offset = i < rx_pkt_nb_offs ? 2594 rx_pkt_seg_offsets[i] : 0; 2595 rx_seg->mp = mpx ? mpx : mp; 2596 } 2597 rx_conf->rx_nseg = rx_pkt_nb_segs; 2598 rx_conf->rx_seg = rx_useg; 2599 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2600 socket_id, rx_conf, NULL); 2601 rx_conf->rx_seg = NULL; 2602 rx_conf->rx_nseg = 0; 2603 return ret; 2604 } 2605 2606 static int 2607 alloc_xstats_display_info(portid_t pi) 2608 { 2609 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2610 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2611 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2612 2613 if (xstats_display_num == 0) 2614 return 0; 2615 2616 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2617 if (*ids_supp == NULL) 2618 goto fail_ids_supp; 2619 2620 *prev_values = calloc(xstats_display_num, 2621 sizeof(**prev_values)); 2622 if (*prev_values == NULL) 2623 goto fail_prev_values; 2624 2625 *curr_values = calloc(xstats_display_num, 2626 sizeof(**curr_values)); 2627 if (*curr_values == NULL) 2628 goto fail_curr_values; 2629 2630 ports[pi].xstats_info.allocated = true; 2631 2632 return 0; 2633 2634 fail_curr_values: 2635 free(*prev_values); 2636 fail_prev_values: 2637 free(*ids_supp); 2638 fail_ids_supp: 2639 return -ENOMEM; 2640 } 2641 2642 static void 2643 free_xstats_display_info(portid_t pi) 2644 { 2645 if (!ports[pi].xstats_info.allocated) 2646 return; 2647 free(ports[pi].xstats_info.ids_supp); 2648 free(ports[pi].xstats_info.prev_values); 2649 free(ports[pi].xstats_info.curr_values); 2650 ports[pi].xstats_info.allocated = false; 2651 } 2652 2653 /** Fill helper structures for specified port to show extended statistics. */ 2654 static void 2655 fill_xstats_display_info_for_port(portid_t pi) 2656 { 2657 unsigned int stat, stat_supp; 2658 const char *xstat_name; 2659 struct rte_port *port; 2660 uint64_t *ids_supp; 2661 int rc; 2662 2663 if (xstats_display_num == 0) 2664 return; 2665 2666 if (pi == (portid_t)RTE_PORT_ALL) { 2667 fill_xstats_display_info(); 2668 return; 2669 } 2670 2671 port = &ports[pi]; 2672 if (port->port_status != RTE_PORT_STARTED) 2673 return; 2674 2675 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2676 rte_exit(EXIT_FAILURE, 2677 "Failed to allocate xstats display memory\n"); 2678 2679 ids_supp = port->xstats_info.ids_supp; 2680 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2681 xstat_name = xstats_display[stat].name; 2682 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2683 ids_supp + stat_supp); 2684 if (rc != 0) { 2685 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2686 xstat_name, pi, stat); 2687 continue; 2688 } 2689 stat_supp++; 2690 } 2691 2692 port->xstats_info.ids_supp_sz = stat_supp; 2693 } 2694 2695 /** Fill helper structures for all ports to show extended statistics. */ 2696 static void 2697 fill_xstats_display_info(void) 2698 { 2699 portid_t pi; 2700 2701 if (xstats_display_num == 0) 2702 return; 2703 2704 RTE_ETH_FOREACH_DEV(pi) 2705 fill_xstats_display_info_for_port(pi); 2706 } 2707 2708 int 2709 start_port(portid_t pid) 2710 { 2711 int diag, need_check_link_status = -1; 2712 portid_t pi; 2713 portid_t p_pi = RTE_MAX_ETHPORTS; 2714 portid_t pl[RTE_MAX_ETHPORTS]; 2715 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2716 uint16_t cnt_pi = 0; 2717 uint16_t cfg_pi = 0; 2718 int peer_pi; 2719 queueid_t qi; 2720 struct rte_port *port; 2721 struct rte_eth_hairpin_cap cap; 2722 2723 if (port_id_is_invalid(pid, ENABLED_WARN)) 2724 return 0; 2725 2726 RTE_ETH_FOREACH_DEV(pi) { 2727 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2728 continue; 2729 2730 need_check_link_status = 0; 2731 port = &ports[pi]; 2732 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2733 RTE_PORT_HANDLING) == 0) { 2734 fprintf(stderr, "Port %d is now not stopped\n", pi); 2735 continue; 2736 } 2737 2738 if (port->need_reconfig > 0) { 2739 struct rte_eth_conf dev_conf; 2740 int k; 2741 2742 port->need_reconfig = 0; 2743 2744 if (flow_isolate_all) { 2745 int ret = port_flow_isolate(pi, 1); 2746 if (ret) { 2747 fprintf(stderr, 2748 "Failed to apply isolated mode on port %d\n", 2749 pi); 2750 return -1; 2751 } 2752 } 2753 configure_rxtx_dump_callbacks(0); 2754 printf("Configuring Port %d (socket %u)\n", pi, 2755 port->socket_id); 2756 if (nb_hairpinq > 0 && 2757 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2758 fprintf(stderr, 2759 "Port %d doesn't support hairpin queues\n", 2760 pi); 2761 return -1; 2762 } 2763 2764 /* configure port */ 2765 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2766 nb_txq + nb_hairpinq, 2767 &(port->dev_conf)); 2768 if (diag != 0) { 2769 if (rte_atomic16_cmpset(&(port->port_status), 2770 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2771 fprintf(stderr, 2772 "Port %d can not be set back to stopped\n", 2773 pi); 2774 fprintf(stderr, "Fail to configure port %d\n", 2775 pi); 2776 /* try to reconfigure port next time */ 2777 port->need_reconfig = 1; 2778 return -1; 2779 } 2780 /* get device configuration*/ 2781 if (0 != 2782 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2783 fprintf(stderr, 2784 "port %d can not get device configuration\n", 2785 pi); 2786 return -1; 2787 } 2788 /* Apply Rx offloads configuration */ 2789 if (dev_conf.rxmode.offloads != 2790 port->dev_conf.rxmode.offloads) { 2791 port->dev_conf.rxmode.offloads |= 2792 dev_conf.rxmode.offloads; 2793 for (k = 0; 2794 k < port->dev_info.max_rx_queues; 2795 k++) 2796 port->rx_conf[k].offloads |= 2797 dev_conf.rxmode.offloads; 2798 } 2799 /* Apply Tx offloads configuration */ 2800 if (dev_conf.txmode.offloads != 2801 port->dev_conf.txmode.offloads) { 2802 port->dev_conf.txmode.offloads |= 2803 dev_conf.txmode.offloads; 2804 for (k = 0; 2805 k < port->dev_info.max_tx_queues; 2806 k++) 2807 port->tx_conf[k].offloads |= 2808 dev_conf.txmode.offloads; 2809 } 2810 } 2811 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2812 port->need_reconfig_queues = 0; 2813 /* setup tx queues */ 2814 for (qi = 0; qi < nb_txq; qi++) { 2815 if ((numa_support) && 2816 (txring_numa[pi] != NUMA_NO_CONFIG)) 2817 diag = rte_eth_tx_queue_setup(pi, qi, 2818 port->nb_tx_desc[qi], 2819 txring_numa[pi], 2820 &(port->tx_conf[qi])); 2821 else 2822 diag = rte_eth_tx_queue_setup(pi, qi, 2823 port->nb_tx_desc[qi], 2824 port->socket_id, 2825 &(port->tx_conf[qi])); 2826 2827 if (diag == 0) 2828 continue; 2829 2830 /* Fail to setup tx queue, return */ 2831 if (rte_atomic16_cmpset(&(port->port_status), 2832 RTE_PORT_HANDLING, 2833 RTE_PORT_STOPPED) == 0) 2834 fprintf(stderr, 2835 "Port %d can not be set back to stopped\n", 2836 pi); 2837 fprintf(stderr, 2838 "Fail to configure port %d tx queues\n", 2839 pi); 2840 /* try to reconfigure queues next time */ 2841 port->need_reconfig_queues = 1; 2842 return -1; 2843 } 2844 for (qi = 0; qi < nb_rxq; qi++) { 2845 /* setup rx queues */ 2846 if ((numa_support) && 2847 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2848 struct rte_mempool * mp = 2849 mbuf_pool_find 2850 (rxring_numa[pi], 0); 2851 if (mp == NULL) { 2852 fprintf(stderr, 2853 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2854 rxring_numa[pi]); 2855 return -1; 2856 } 2857 2858 diag = rx_queue_setup(pi, qi, 2859 port->nb_rx_desc[qi], 2860 rxring_numa[pi], 2861 &(port->rx_conf[qi]), 2862 mp); 2863 } else { 2864 struct rte_mempool *mp = 2865 mbuf_pool_find 2866 (port->socket_id, 0); 2867 if (mp == NULL) { 2868 fprintf(stderr, 2869 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2870 port->socket_id); 2871 return -1; 2872 } 2873 diag = rx_queue_setup(pi, qi, 2874 port->nb_rx_desc[qi], 2875 port->socket_id, 2876 &(port->rx_conf[qi]), 2877 mp); 2878 } 2879 if (diag == 0) 2880 continue; 2881 2882 /* Fail to setup rx queue, return */ 2883 if (rte_atomic16_cmpset(&(port->port_status), 2884 RTE_PORT_HANDLING, 2885 RTE_PORT_STOPPED) == 0) 2886 fprintf(stderr, 2887 "Port %d can not be set back to stopped\n", 2888 pi); 2889 fprintf(stderr, 2890 "Fail to configure port %d rx queues\n", 2891 pi); 2892 /* try to reconfigure queues next time */ 2893 port->need_reconfig_queues = 1; 2894 return -1; 2895 } 2896 /* setup hairpin queues */ 2897 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2898 return -1; 2899 } 2900 configure_rxtx_dump_callbacks(verbose_level); 2901 if (clear_ptypes) { 2902 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2903 NULL, 0); 2904 if (diag < 0) 2905 fprintf(stderr, 2906 "Port %d: Failed to disable Ptype parsing\n", 2907 pi); 2908 } 2909 2910 p_pi = pi; 2911 cnt_pi++; 2912 2913 /* start port */ 2914 diag = eth_dev_start_mp(pi); 2915 if (diag < 0) { 2916 fprintf(stderr, "Fail to start port %d: %s\n", 2917 pi, rte_strerror(-diag)); 2918 2919 /* Fail to setup rx queue, return */ 2920 if (rte_atomic16_cmpset(&(port->port_status), 2921 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2922 fprintf(stderr, 2923 "Port %d can not be set back to stopped\n", 2924 pi); 2925 continue; 2926 } 2927 2928 if (rte_atomic16_cmpset(&(port->port_status), 2929 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2930 fprintf(stderr, "Port %d can not be set into started\n", 2931 pi); 2932 2933 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 2934 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 2935 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 2936 2937 /* at least one port started, need checking link status */ 2938 need_check_link_status = 1; 2939 2940 pl[cfg_pi++] = pi; 2941 } 2942 2943 if (need_check_link_status == 1 && !no_link_check) 2944 check_all_ports_link_status(RTE_PORT_ALL); 2945 else if (need_check_link_status == 0) 2946 fprintf(stderr, "Please stop the ports first\n"); 2947 2948 if (hairpin_mode & 0xf) { 2949 uint16_t i; 2950 int j; 2951 2952 /* bind all started hairpin ports */ 2953 for (i = 0; i < cfg_pi; i++) { 2954 pi = pl[i]; 2955 /* bind current Tx to all peer Rx */ 2956 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2957 RTE_MAX_ETHPORTS, 1); 2958 if (peer_pi < 0) 2959 return peer_pi; 2960 for (j = 0; j < peer_pi; j++) { 2961 if (!port_is_started(peer_pl[j])) 2962 continue; 2963 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2964 if (diag < 0) { 2965 fprintf(stderr, 2966 "Error during binding hairpin Tx port %u to %u: %s\n", 2967 pi, peer_pl[j], 2968 rte_strerror(-diag)); 2969 return -1; 2970 } 2971 } 2972 /* bind all peer Tx to current Rx */ 2973 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2974 RTE_MAX_ETHPORTS, 0); 2975 if (peer_pi < 0) 2976 return peer_pi; 2977 for (j = 0; j < peer_pi; j++) { 2978 if (!port_is_started(peer_pl[j])) 2979 continue; 2980 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2981 if (diag < 0) { 2982 fprintf(stderr, 2983 "Error during binding hairpin Tx port %u to %u: %s\n", 2984 peer_pl[j], pi, 2985 rte_strerror(-diag)); 2986 return -1; 2987 } 2988 } 2989 } 2990 } 2991 2992 fill_xstats_display_info_for_port(pid); 2993 2994 printf("Done\n"); 2995 return 0; 2996 } 2997 2998 void 2999 stop_port(portid_t pid) 3000 { 3001 portid_t pi; 3002 struct rte_port *port; 3003 int need_check_link_status = 0; 3004 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3005 int peer_pi; 3006 3007 if (port_id_is_invalid(pid, ENABLED_WARN)) 3008 return; 3009 3010 printf("Stopping ports...\n"); 3011 3012 RTE_ETH_FOREACH_DEV(pi) { 3013 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3014 continue; 3015 3016 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3017 fprintf(stderr, 3018 "Please remove port %d from forwarding configuration.\n", 3019 pi); 3020 continue; 3021 } 3022 3023 if (port_is_bonding_slave(pi)) { 3024 fprintf(stderr, 3025 "Please remove port %d from bonded device.\n", 3026 pi); 3027 continue; 3028 } 3029 3030 port = &ports[pi]; 3031 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 3032 RTE_PORT_HANDLING) == 0) 3033 continue; 3034 3035 if (hairpin_mode & 0xf) { 3036 int j; 3037 3038 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3039 /* unbind all peer Tx from current Rx */ 3040 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3041 RTE_MAX_ETHPORTS, 0); 3042 if (peer_pi < 0) 3043 continue; 3044 for (j = 0; j < peer_pi; j++) { 3045 if (!port_is_started(peer_pl[j])) 3046 continue; 3047 rte_eth_hairpin_unbind(peer_pl[j], pi); 3048 } 3049 } 3050 3051 if (port->flow_list) 3052 port_flow_flush(pi); 3053 3054 if (eth_dev_stop_mp(pi) != 0) 3055 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3056 pi); 3057 3058 if (rte_atomic16_cmpset(&(port->port_status), 3059 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 3060 fprintf(stderr, "Port %d can not be set into stopped\n", 3061 pi); 3062 need_check_link_status = 1; 3063 } 3064 if (need_check_link_status && !no_link_check) 3065 check_all_ports_link_status(RTE_PORT_ALL); 3066 3067 printf("Done\n"); 3068 } 3069 3070 static void 3071 remove_invalid_ports_in(portid_t *array, portid_t *total) 3072 { 3073 portid_t i; 3074 portid_t new_total = 0; 3075 3076 for (i = 0; i < *total; i++) 3077 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3078 array[new_total] = array[i]; 3079 new_total++; 3080 } 3081 *total = new_total; 3082 } 3083 3084 static void 3085 remove_invalid_ports(void) 3086 { 3087 remove_invalid_ports_in(ports_ids, &nb_ports); 3088 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3089 nb_cfg_ports = nb_fwd_ports; 3090 } 3091 3092 void 3093 close_port(portid_t pid) 3094 { 3095 portid_t pi; 3096 struct rte_port *port; 3097 3098 if (port_id_is_invalid(pid, ENABLED_WARN)) 3099 return; 3100 3101 printf("Closing ports...\n"); 3102 3103 RTE_ETH_FOREACH_DEV(pi) { 3104 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3105 continue; 3106 3107 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3108 fprintf(stderr, 3109 "Please remove port %d from forwarding configuration.\n", 3110 pi); 3111 continue; 3112 } 3113 3114 if (port_is_bonding_slave(pi)) { 3115 fprintf(stderr, 3116 "Please remove port %d from bonded device.\n", 3117 pi); 3118 continue; 3119 } 3120 3121 port = &ports[pi]; 3122 if (rte_atomic16_cmpset(&(port->port_status), 3123 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 3124 fprintf(stderr, "Port %d is already closed\n", pi); 3125 continue; 3126 } 3127 3128 if (is_proc_primary()) { 3129 port_flow_flush(pi); 3130 port_flex_item_flush(pi); 3131 rte_eth_dev_close(pi); 3132 } 3133 3134 free_xstats_display_info(pi); 3135 } 3136 3137 remove_invalid_ports(); 3138 printf("Done\n"); 3139 } 3140 3141 void 3142 reset_port(portid_t pid) 3143 { 3144 int diag; 3145 portid_t pi; 3146 struct rte_port *port; 3147 3148 if (port_id_is_invalid(pid, ENABLED_WARN)) 3149 return; 3150 3151 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3152 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3153 fprintf(stderr, 3154 "Can not reset port(s), please stop port(s) first.\n"); 3155 return; 3156 } 3157 3158 printf("Resetting ports...\n"); 3159 3160 RTE_ETH_FOREACH_DEV(pi) { 3161 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3162 continue; 3163 3164 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3165 fprintf(stderr, 3166 "Please remove port %d from forwarding configuration.\n", 3167 pi); 3168 continue; 3169 } 3170 3171 if (port_is_bonding_slave(pi)) { 3172 fprintf(stderr, 3173 "Please remove port %d from bonded device.\n", 3174 pi); 3175 continue; 3176 } 3177 3178 diag = rte_eth_dev_reset(pi); 3179 if (diag == 0) { 3180 port = &ports[pi]; 3181 port->need_reconfig = 1; 3182 port->need_reconfig_queues = 1; 3183 } else { 3184 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3185 pi, diag); 3186 } 3187 } 3188 3189 printf("Done\n"); 3190 } 3191 3192 void 3193 attach_port(char *identifier) 3194 { 3195 portid_t pi; 3196 struct rte_dev_iterator iterator; 3197 3198 printf("Attaching a new port...\n"); 3199 3200 if (identifier == NULL) { 3201 fprintf(stderr, "Invalid parameters are specified\n"); 3202 return; 3203 } 3204 3205 if (rte_dev_probe(identifier) < 0) { 3206 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3207 return; 3208 } 3209 3210 /* first attach mode: event */ 3211 if (setup_on_probe_event) { 3212 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3213 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3214 if (ports[pi].port_status == RTE_PORT_HANDLING && 3215 ports[pi].need_setup != 0) 3216 setup_attached_port(pi); 3217 return; 3218 } 3219 3220 /* second attach mode: iterator */ 3221 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3222 /* setup ports matching the devargs used for probing */ 3223 if (port_is_forwarding(pi)) 3224 continue; /* port was already attached before */ 3225 setup_attached_port(pi); 3226 } 3227 } 3228 3229 static void 3230 setup_attached_port(portid_t pi) 3231 { 3232 unsigned int socket_id; 3233 int ret; 3234 3235 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3236 /* if socket_id is invalid, set to the first available socket. */ 3237 if (check_socket_id(socket_id) < 0) 3238 socket_id = socket_ids[0]; 3239 reconfig(pi, socket_id); 3240 ret = rte_eth_promiscuous_enable(pi); 3241 if (ret != 0) 3242 fprintf(stderr, 3243 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3244 pi, rte_strerror(-ret)); 3245 3246 ports_ids[nb_ports++] = pi; 3247 fwd_ports_ids[nb_fwd_ports++] = pi; 3248 nb_cfg_ports = nb_fwd_ports; 3249 ports[pi].need_setup = 0; 3250 ports[pi].port_status = RTE_PORT_STOPPED; 3251 3252 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3253 printf("Done\n"); 3254 } 3255 3256 static void 3257 detach_device(struct rte_device *dev) 3258 { 3259 portid_t sibling; 3260 3261 if (dev == NULL) { 3262 fprintf(stderr, "Device already removed\n"); 3263 return; 3264 } 3265 3266 printf("Removing a device...\n"); 3267 3268 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3269 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3270 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3271 fprintf(stderr, "Port %u not stopped\n", 3272 sibling); 3273 return; 3274 } 3275 port_flow_flush(sibling); 3276 } 3277 } 3278 3279 if (rte_dev_remove(dev) < 0) { 3280 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3281 return; 3282 } 3283 remove_invalid_ports(); 3284 3285 printf("Device is detached\n"); 3286 printf("Now total ports is %d\n", nb_ports); 3287 printf("Done\n"); 3288 return; 3289 } 3290 3291 void 3292 detach_port_device(portid_t port_id) 3293 { 3294 int ret; 3295 struct rte_eth_dev_info dev_info; 3296 3297 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3298 return; 3299 3300 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3301 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3302 fprintf(stderr, "Port not stopped\n"); 3303 return; 3304 } 3305 fprintf(stderr, "Port was not closed\n"); 3306 } 3307 3308 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3309 if (ret != 0) { 3310 TESTPMD_LOG(ERR, 3311 "Failed to get device info for port %d, not detaching\n", 3312 port_id); 3313 return; 3314 } 3315 detach_device(dev_info.device); 3316 } 3317 3318 void 3319 detach_devargs(char *identifier) 3320 { 3321 struct rte_dev_iterator iterator; 3322 struct rte_devargs da; 3323 portid_t port_id; 3324 3325 printf("Removing a device...\n"); 3326 3327 memset(&da, 0, sizeof(da)); 3328 if (rte_devargs_parsef(&da, "%s", identifier)) { 3329 fprintf(stderr, "cannot parse identifier\n"); 3330 return; 3331 } 3332 3333 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3334 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3335 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3336 fprintf(stderr, "Port %u not stopped\n", 3337 port_id); 3338 rte_eth_iterator_cleanup(&iterator); 3339 rte_devargs_reset(&da); 3340 return; 3341 } 3342 port_flow_flush(port_id); 3343 } 3344 } 3345 3346 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3347 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3348 da.name, da.bus->name); 3349 rte_devargs_reset(&da); 3350 return; 3351 } 3352 3353 remove_invalid_ports(); 3354 3355 printf("Device %s is detached\n", identifier); 3356 printf("Now total ports is %d\n", nb_ports); 3357 printf("Done\n"); 3358 rte_devargs_reset(&da); 3359 } 3360 3361 void 3362 pmd_test_exit(void) 3363 { 3364 portid_t pt_id; 3365 unsigned int i; 3366 int ret; 3367 3368 if (test_done == 0) 3369 stop_packet_forwarding(); 3370 3371 #ifndef RTE_EXEC_ENV_WINDOWS 3372 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3373 if (mempools[i]) { 3374 if (mp_alloc_type == MP_ALLOC_ANON) 3375 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3376 NULL); 3377 } 3378 } 3379 #endif 3380 if (ports != NULL) { 3381 no_link_check = 1; 3382 RTE_ETH_FOREACH_DEV(pt_id) { 3383 printf("\nStopping port %d...\n", pt_id); 3384 fflush(stdout); 3385 stop_port(pt_id); 3386 } 3387 RTE_ETH_FOREACH_DEV(pt_id) { 3388 printf("\nShutting down port %d...\n", pt_id); 3389 fflush(stdout); 3390 close_port(pt_id); 3391 } 3392 } 3393 3394 if (hot_plug) { 3395 ret = rte_dev_event_monitor_stop(); 3396 if (ret) { 3397 RTE_LOG(ERR, EAL, 3398 "fail to stop device event monitor."); 3399 return; 3400 } 3401 3402 ret = rte_dev_event_callback_unregister(NULL, 3403 dev_event_callback, NULL); 3404 if (ret < 0) { 3405 RTE_LOG(ERR, EAL, 3406 "fail to unregister device event callback.\n"); 3407 return; 3408 } 3409 3410 ret = rte_dev_hotplug_handle_disable(); 3411 if (ret) { 3412 RTE_LOG(ERR, EAL, 3413 "fail to disable hotplug handling.\n"); 3414 return; 3415 } 3416 } 3417 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3418 if (mempools[i]) 3419 mempool_free_mp(mempools[i]); 3420 } 3421 free(xstats_display); 3422 3423 printf("\nBye...\n"); 3424 } 3425 3426 typedef void (*cmd_func_t)(void); 3427 struct pmd_test_command { 3428 const char *cmd_name; 3429 cmd_func_t cmd_func; 3430 }; 3431 3432 /* Check the link status of all ports in up to 9s, and print them finally */ 3433 static void 3434 check_all_ports_link_status(uint32_t port_mask) 3435 { 3436 #define CHECK_INTERVAL 100 /* 100ms */ 3437 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3438 portid_t portid; 3439 uint8_t count, all_ports_up, print_flag = 0; 3440 struct rte_eth_link link; 3441 int ret; 3442 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3443 3444 printf("Checking link statuses...\n"); 3445 fflush(stdout); 3446 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3447 all_ports_up = 1; 3448 RTE_ETH_FOREACH_DEV(portid) { 3449 if ((port_mask & (1 << portid)) == 0) 3450 continue; 3451 memset(&link, 0, sizeof(link)); 3452 ret = rte_eth_link_get_nowait(portid, &link); 3453 if (ret < 0) { 3454 all_ports_up = 0; 3455 if (print_flag == 1) 3456 fprintf(stderr, 3457 "Port %u link get failed: %s\n", 3458 portid, rte_strerror(-ret)); 3459 continue; 3460 } 3461 /* print link status if flag set */ 3462 if (print_flag == 1) { 3463 rte_eth_link_to_str(link_status, 3464 sizeof(link_status), &link); 3465 printf("Port %d %s\n", portid, link_status); 3466 continue; 3467 } 3468 /* clear all_ports_up flag if any link down */ 3469 if (link.link_status == RTE_ETH_LINK_DOWN) { 3470 all_ports_up = 0; 3471 break; 3472 } 3473 } 3474 /* after finally printing all link status, get out */ 3475 if (print_flag == 1) 3476 break; 3477 3478 if (all_ports_up == 0) { 3479 fflush(stdout); 3480 rte_delay_ms(CHECK_INTERVAL); 3481 } 3482 3483 /* set the print_flag if all ports up or timeout */ 3484 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3485 print_flag = 1; 3486 } 3487 3488 if (lsc_interrupt) 3489 break; 3490 } 3491 } 3492 3493 static void 3494 rmv_port_callback(void *arg) 3495 { 3496 int need_to_start = 0; 3497 int org_no_link_check = no_link_check; 3498 portid_t port_id = (intptr_t)arg; 3499 struct rte_eth_dev_info dev_info; 3500 int ret; 3501 3502 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3503 3504 if (!test_done && port_is_forwarding(port_id)) { 3505 need_to_start = 1; 3506 stop_packet_forwarding(); 3507 } 3508 no_link_check = 1; 3509 stop_port(port_id); 3510 no_link_check = org_no_link_check; 3511 3512 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3513 if (ret != 0) 3514 TESTPMD_LOG(ERR, 3515 "Failed to get device info for port %d, not detaching\n", 3516 port_id); 3517 else { 3518 struct rte_device *device = dev_info.device; 3519 close_port(port_id); 3520 detach_device(device); /* might be already removed or have more ports */ 3521 } 3522 if (need_to_start) 3523 start_packet_forwarding(0); 3524 } 3525 3526 /* This function is used by the interrupt thread */ 3527 static int 3528 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3529 void *ret_param) 3530 { 3531 RTE_SET_USED(param); 3532 RTE_SET_USED(ret_param); 3533 3534 if (type >= RTE_ETH_EVENT_MAX) { 3535 fprintf(stderr, 3536 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3537 port_id, __func__, type); 3538 fflush(stderr); 3539 } else if (event_print_mask & (UINT32_C(1) << type)) { 3540 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3541 eth_event_desc[type]); 3542 fflush(stdout); 3543 } 3544 3545 switch (type) { 3546 case RTE_ETH_EVENT_NEW: 3547 ports[port_id].need_setup = 1; 3548 ports[port_id].port_status = RTE_PORT_HANDLING; 3549 break; 3550 case RTE_ETH_EVENT_INTR_RMV: 3551 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3552 break; 3553 if (rte_eal_alarm_set(100000, 3554 rmv_port_callback, (void *)(intptr_t)port_id)) 3555 fprintf(stderr, 3556 "Could not set up deferred device removal\n"); 3557 break; 3558 case RTE_ETH_EVENT_DESTROY: 3559 ports[port_id].port_status = RTE_PORT_CLOSED; 3560 printf("Port %u is closed\n", port_id); 3561 break; 3562 default: 3563 break; 3564 } 3565 return 0; 3566 } 3567 3568 static int 3569 register_eth_event_callback(void) 3570 { 3571 int ret; 3572 enum rte_eth_event_type event; 3573 3574 for (event = RTE_ETH_EVENT_UNKNOWN; 3575 event < RTE_ETH_EVENT_MAX; event++) { 3576 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3577 event, 3578 eth_event_callback, 3579 NULL); 3580 if (ret != 0) { 3581 TESTPMD_LOG(ERR, "Failed to register callback for " 3582 "%s event\n", eth_event_desc[event]); 3583 return -1; 3584 } 3585 } 3586 3587 return 0; 3588 } 3589 3590 /* This function is used by the interrupt thread */ 3591 static void 3592 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3593 __rte_unused void *arg) 3594 { 3595 uint16_t port_id; 3596 int ret; 3597 3598 if (type >= RTE_DEV_EVENT_MAX) { 3599 fprintf(stderr, "%s called upon invalid event %d\n", 3600 __func__, type); 3601 fflush(stderr); 3602 } 3603 3604 switch (type) { 3605 case RTE_DEV_EVENT_REMOVE: 3606 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3607 device_name); 3608 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3609 if (ret) { 3610 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3611 device_name); 3612 return; 3613 } 3614 /* 3615 * Because the user's callback is invoked in eal interrupt 3616 * callback, the interrupt callback need to be finished before 3617 * it can be unregistered when detaching device. So finish 3618 * callback soon and use a deferred removal to detach device 3619 * is need. It is a workaround, once the device detaching be 3620 * moved into the eal in the future, the deferred removal could 3621 * be deleted. 3622 */ 3623 if (rte_eal_alarm_set(100000, 3624 rmv_port_callback, (void *)(intptr_t)port_id)) 3625 RTE_LOG(ERR, EAL, 3626 "Could not set up deferred device removal\n"); 3627 break; 3628 case RTE_DEV_EVENT_ADD: 3629 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3630 device_name); 3631 /* TODO: After finish kernel driver binding, 3632 * begin to attach port. 3633 */ 3634 break; 3635 default: 3636 break; 3637 } 3638 } 3639 3640 static void 3641 rxtx_port_config(portid_t pid) 3642 { 3643 uint16_t qid; 3644 uint64_t offloads; 3645 struct rte_port *port = &ports[pid]; 3646 3647 for (qid = 0; qid < nb_rxq; qid++) { 3648 offloads = port->rx_conf[qid].offloads; 3649 port->rx_conf[qid] = port->dev_info.default_rxconf; 3650 3651 if (rxq_share > 0 && 3652 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3653 /* Non-zero share group to enable RxQ share. */ 3654 port->rx_conf[qid].share_group = pid / rxq_share + 1; 3655 port->rx_conf[qid].share_qid = qid; /* Equal mapping. */ 3656 } 3657 3658 if (offloads != 0) 3659 port->rx_conf[qid].offloads = offloads; 3660 3661 /* Check if any Rx parameters have been passed */ 3662 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3663 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3664 3665 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3666 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3667 3668 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3669 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3670 3671 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3672 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3673 3674 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3675 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3676 3677 port->nb_rx_desc[qid] = nb_rxd; 3678 } 3679 3680 for (qid = 0; qid < nb_txq; qid++) { 3681 offloads = port->tx_conf[qid].offloads; 3682 port->tx_conf[qid] = port->dev_info.default_txconf; 3683 if (offloads != 0) 3684 port->tx_conf[qid].offloads = offloads; 3685 3686 /* Check if any Tx parameters have been passed */ 3687 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3688 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3689 3690 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3691 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3692 3693 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3694 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3695 3696 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3697 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3698 3699 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3700 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3701 3702 port->nb_tx_desc[qid] = nb_txd; 3703 } 3704 } 3705 3706 /* 3707 * Helper function to set MTU from frame size 3708 * 3709 * port->dev_info should be set before calling this function. 3710 * 3711 * return 0 on success, negative on error 3712 */ 3713 int 3714 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3715 { 3716 struct rte_port *port = &ports[portid]; 3717 uint32_t eth_overhead; 3718 uint16_t mtu, new_mtu; 3719 3720 eth_overhead = get_eth_overhead(&port->dev_info); 3721 3722 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3723 printf("Failed to get MTU for port %u\n", portid); 3724 return -1; 3725 } 3726 3727 new_mtu = max_rx_pktlen - eth_overhead; 3728 3729 if (mtu == new_mtu) 3730 return 0; 3731 3732 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3733 fprintf(stderr, 3734 "Failed to set MTU to %u for port %u\n", 3735 new_mtu, portid); 3736 return -1; 3737 } 3738 3739 port->dev_conf.rxmode.mtu = new_mtu; 3740 3741 return 0; 3742 } 3743 3744 void 3745 init_port_config(void) 3746 { 3747 portid_t pid; 3748 struct rte_port *port; 3749 int ret, i; 3750 3751 RTE_ETH_FOREACH_DEV(pid) { 3752 port = &ports[pid]; 3753 port->dev_conf.fdir_conf = fdir_conf; 3754 3755 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3756 if (ret != 0) 3757 return; 3758 3759 if (nb_rxq > 1) { 3760 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3761 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3762 rss_hf & port->dev_info.flow_type_rss_offloads; 3763 } else { 3764 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3765 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3766 } 3767 3768 if (port->dcb_flag == 0) { 3769 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3770 port->dev_conf.rxmode.mq_mode = 3771 (enum rte_eth_rx_mq_mode) 3772 (rx_mq_mode & RTE_ETH_MQ_RX_RSS); 3773 } else { 3774 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; 3775 port->dev_conf.rxmode.offloads &= 3776 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3777 3778 for (i = 0; 3779 i < port->dev_info.nb_rx_queues; 3780 i++) 3781 port->rx_conf[i].offloads &= 3782 ~RTE_ETH_RX_OFFLOAD_RSS_HASH; 3783 } 3784 } 3785 3786 rxtx_port_config(pid); 3787 3788 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3789 if (ret != 0) 3790 return; 3791 3792 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3793 rte_pmd_ixgbe_bypass_init(pid); 3794 #endif 3795 3796 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3797 port->dev_conf.intr_conf.lsc = 1; 3798 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3799 port->dev_conf.intr_conf.rmv = 1; 3800 } 3801 } 3802 3803 void set_port_slave_flag(portid_t slave_pid) 3804 { 3805 struct rte_port *port; 3806 3807 port = &ports[slave_pid]; 3808 port->slave_flag = 1; 3809 } 3810 3811 void clear_port_slave_flag(portid_t slave_pid) 3812 { 3813 struct rte_port *port; 3814 3815 port = &ports[slave_pid]; 3816 port->slave_flag = 0; 3817 } 3818 3819 uint8_t port_is_bonding_slave(portid_t slave_pid) 3820 { 3821 struct rte_port *port; 3822 struct rte_eth_dev_info dev_info; 3823 int ret; 3824 3825 port = &ports[slave_pid]; 3826 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3827 if (ret != 0) { 3828 TESTPMD_LOG(ERR, 3829 "Failed to get device info for port id %d," 3830 "cannot determine if the port is a bonded slave", 3831 slave_pid); 3832 return 0; 3833 } 3834 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3835 return 1; 3836 return 0; 3837 } 3838 3839 const uint16_t vlan_tags[] = { 3840 0, 1, 2, 3, 4, 5, 6, 7, 3841 8, 9, 10, 11, 12, 13, 14, 15, 3842 16, 17, 18, 19, 20, 21, 22, 23, 3843 24, 25, 26, 27, 28, 29, 30, 31 3844 }; 3845 3846 static int 3847 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3848 enum dcb_mode_enable dcb_mode, 3849 enum rte_eth_nb_tcs num_tcs, 3850 uint8_t pfc_en) 3851 { 3852 uint8_t i; 3853 int32_t rc; 3854 struct rte_eth_rss_conf rss_conf; 3855 3856 /* 3857 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3858 * given above, and the number of traffic classes available for use. 3859 */ 3860 if (dcb_mode == DCB_VT_ENABLED) { 3861 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3862 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3863 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3864 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3865 3866 /* VMDQ+DCB RX and TX configurations */ 3867 vmdq_rx_conf->enable_default_pool = 0; 3868 vmdq_rx_conf->default_pool = 0; 3869 vmdq_rx_conf->nb_queue_pools = 3870 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3871 vmdq_tx_conf->nb_queue_pools = 3872 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS); 3873 3874 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3875 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3876 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3877 vmdq_rx_conf->pool_map[i].pools = 3878 1 << (i % vmdq_rx_conf->nb_queue_pools); 3879 } 3880 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3881 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3882 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3883 } 3884 3885 /* set DCB mode of RX and TX of multiple queues */ 3886 eth_conf->rxmode.mq_mode = 3887 (enum rte_eth_rx_mq_mode) 3888 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB); 3889 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB; 3890 } else { 3891 struct rte_eth_dcb_rx_conf *rx_conf = 3892 ð_conf->rx_adv_conf.dcb_rx_conf; 3893 struct rte_eth_dcb_tx_conf *tx_conf = 3894 ð_conf->tx_adv_conf.dcb_tx_conf; 3895 3896 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3897 3898 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3899 if (rc != 0) 3900 return rc; 3901 3902 rx_conf->nb_tcs = num_tcs; 3903 tx_conf->nb_tcs = num_tcs; 3904 3905 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3906 rx_conf->dcb_tc[i] = i % num_tcs; 3907 tx_conf->dcb_tc[i] = i % num_tcs; 3908 } 3909 3910 eth_conf->rxmode.mq_mode = 3911 (enum rte_eth_rx_mq_mode) 3912 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS); 3913 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3914 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB; 3915 } 3916 3917 if (pfc_en) 3918 eth_conf->dcb_capability_en = 3919 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT; 3920 else 3921 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT; 3922 3923 return 0; 3924 } 3925 3926 int 3927 init_port_dcb_config(portid_t pid, 3928 enum dcb_mode_enable dcb_mode, 3929 enum rte_eth_nb_tcs num_tcs, 3930 uint8_t pfc_en) 3931 { 3932 struct rte_eth_conf port_conf; 3933 struct rte_port *rte_port; 3934 int retval; 3935 uint16_t i; 3936 3937 if (num_procs > 1) { 3938 printf("The multi-process feature doesn't support dcb.\n"); 3939 return -ENOTSUP; 3940 } 3941 rte_port = &ports[pid]; 3942 3943 /* retain the original device configuration. */ 3944 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 3945 3946 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3947 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3948 if (retval < 0) 3949 return retval; 3950 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 3951 3952 /* re-configure the device . */ 3953 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3954 if (retval < 0) 3955 return retval; 3956 3957 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3958 if (retval != 0) 3959 return retval; 3960 3961 /* If dev_info.vmdq_pool_base is greater than 0, 3962 * the queue id of vmdq pools is started after pf queues. 3963 */ 3964 if (dcb_mode == DCB_VT_ENABLED && 3965 rte_port->dev_info.vmdq_pool_base > 0) { 3966 fprintf(stderr, 3967 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 3968 pid); 3969 return -1; 3970 } 3971 3972 /* Assume the ports in testpmd have the same dcb capability 3973 * and has the same number of rxq and txq in dcb mode 3974 */ 3975 if (dcb_mode == DCB_VT_ENABLED) { 3976 if (rte_port->dev_info.max_vfs > 0) { 3977 nb_rxq = rte_port->dev_info.nb_rx_queues; 3978 nb_txq = rte_port->dev_info.nb_tx_queues; 3979 } else { 3980 nb_rxq = rte_port->dev_info.max_rx_queues; 3981 nb_txq = rte_port->dev_info.max_tx_queues; 3982 } 3983 } else { 3984 /*if vt is disabled, use all pf queues */ 3985 if (rte_port->dev_info.vmdq_pool_base == 0) { 3986 nb_rxq = rte_port->dev_info.max_rx_queues; 3987 nb_txq = rte_port->dev_info.max_tx_queues; 3988 } else { 3989 nb_rxq = (queueid_t)num_tcs; 3990 nb_txq = (queueid_t)num_tcs; 3991 3992 } 3993 } 3994 rx_free_thresh = 64; 3995 3996 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3997 3998 rxtx_port_config(pid); 3999 /* VLAN filter */ 4000 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 4001 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4002 rx_vft_set(pid, vlan_tags[i], 1); 4003 4004 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4005 if (retval != 0) 4006 return retval; 4007 4008 rte_port->dcb_flag = 1; 4009 4010 /* Enter DCB configuration status */ 4011 dcb_config = 1; 4012 4013 return 0; 4014 } 4015 4016 static void 4017 init_port(void) 4018 { 4019 int i; 4020 4021 /* Configuration of Ethernet ports. */ 4022 ports = rte_zmalloc("testpmd: ports", 4023 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4024 RTE_CACHE_LINE_SIZE); 4025 if (ports == NULL) { 4026 rte_exit(EXIT_FAILURE, 4027 "rte_zmalloc(%d struct rte_port) failed\n", 4028 RTE_MAX_ETHPORTS); 4029 } 4030 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4031 ports[i].xstats_info.allocated = false; 4032 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4033 LIST_INIT(&ports[i].flow_tunnel_list); 4034 /* Initialize ports NUMA structures */ 4035 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4036 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4037 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4038 } 4039 4040 static void 4041 force_quit(void) 4042 { 4043 pmd_test_exit(); 4044 prompt_exit(); 4045 } 4046 4047 static void 4048 print_stats(void) 4049 { 4050 uint8_t i; 4051 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4052 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4053 4054 /* Clear screen and move to top left */ 4055 printf("%s%s", clr, top_left); 4056 4057 printf("\nPort statistics ===================================="); 4058 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4059 nic_stats_display(fwd_ports_ids[i]); 4060 4061 fflush(stdout); 4062 } 4063 4064 static void 4065 signal_handler(int signum) 4066 { 4067 if (signum == SIGINT || signum == SIGTERM) { 4068 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4069 signum); 4070 #ifdef RTE_LIB_PDUMP 4071 /* uninitialize packet capture framework */ 4072 rte_pdump_uninit(); 4073 #endif 4074 #ifdef RTE_LIB_LATENCYSTATS 4075 if (latencystats_enabled != 0) 4076 rte_latencystats_uninit(); 4077 #endif 4078 force_quit(); 4079 /* Set flag to indicate the force termination. */ 4080 f_quit = 1; 4081 /* exit with the expected status */ 4082 #ifndef RTE_EXEC_ENV_WINDOWS 4083 signal(signum, SIG_DFL); 4084 kill(getpid(), signum); 4085 #endif 4086 } 4087 } 4088 4089 int 4090 main(int argc, char** argv) 4091 { 4092 int diag; 4093 portid_t port_id; 4094 uint16_t count; 4095 int ret; 4096 4097 signal(SIGINT, signal_handler); 4098 signal(SIGTERM, signal_handler); 4099 4100 testpmd_logtype = rte_log_register("testpmd"); 4101 if (testpmd_logtype < 0) 4102 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4103 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4104 4105 diag = rte_eal_init(argc, argv); 4106 if (diag < 0) 4107 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4108 rte_strerror(rte_errno)); 4109 4110 ret = register_eth_event_callback(); 4111 if (ret != 0) 4112 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4113 4114 #ifdef RTE_LIB_PDUMP 4115 /* initialize packet capture framework */ 4116 rte_pdump_init(); 4117 #endif 4118 4119 count = 0; 4120 RTE_ETH_FOREACH_DEV(port_id) { 4121 ports_ids[count] = port_id; 4122 count++; 4123 } 4124 nb_ports = (portid_t) count; 4125 if (nb_ports == 0) 4126 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4127 4128 /* allocate port structures, and init them */ 4129 init_port(); 4130 4131 set_def_fwd_config(); 4132 if (nb_lcores == 0) 4133 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4134 "Check the core mask argument\n"); 4135 4136 /* Bitrate/latency stats disabled by default */ 4137 #ifdef RTE_LIB_BITRATESTATS 4138 bitrate_enabled = 0; 4139 #endif 4140 #ifdef RTE_LIB_LATENCYSTATS 4141 latencystats_enabled = 0; 4142 #endif 4143 4144 /* on FreeBSD, mlockall() is disabled by default */ 4145 #ifdef RTE_EXEC_ENV_FREEBSD 4146 do_mlockall = 0; 4147 #else 4148 do_mlockall = 1; 4149 #endif 4150 4151 argc -= diag; 4152 argv += diag; 4153 if (argc > 1) 4154 launch_args_parse(argc, argv); 4155 4156 #ifndef RTE_EXEC_ENV_WINDOWS 4157 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4158 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4159 strerror(errno)); 4160 } 4161 #endif 4162 4163 if (tx_first && interactive) 4164 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4165 "interactive mode.\n"); 4166 4167 if (tx_first && lsc_interrupt) { 4168 fprintf(stderr, 4169 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4170 lsc_interrupt = 0; 4171 } 4172 4173 if (!nb_rxq && !nb_txq) 4174 fprintf(stderr, 4175 "Warning: Either rx or tx queues should be non-zero\n"); 4176 4177 if (nb_rxq > 1 && nb_rxq > nb_txq) 4178 fprintf(stderr, 4179 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4180 nb_rxq, nb_txq); 4181 4182 init_config(); 4183 4184 if (hot_plug) { 4185 ret = rte_dev_hotplug_handle_enable(); 4186 if (ret) { 4187 RTE_LOG(ERR, EAL, 4188 "fail to enable hotplug handling."); 4189 return -1; 4190 } 4191 4192 ret = rte_dev_event_monitor_start(); 4193 if (ret) { 4194 RTE_LOG(ERR, EAL, 4195 "fail to start device event monitoring."); 4196 return -1; 4197 } 4198 4199 ret = rte_dev_event_callback_register(NULL, 4200 dev_event_callback, NULL); 4201 if (ret) { 4202 RTE_LOG(ERR, EAL, 4203 "fail to register device event callback\n"); 4204 return -1; 4205 } 4206 } 4207 4208 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4209 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4210 4211 /* set all ports to promiscuous mode by default */ 4212 RTE_ETH_FOREACH_DEV(port_id) { 4213 ret = rte_eth_promiscuous_enable(port_id); 4214 if (ret != 0) 4215 fprintf(stderr, 4216 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4217 port_id, rte_strerror(-ret)); 4218 } 4219 4220 /* Init metrics library */ 4221 rte_metrics_init(rte_socket_id()); 4222 4223 #ifdef RTE_LIB_LATENCYSTATS 4224 if (latencystats_enabled != 0) { 4225 int ret = rte_latencystats_init(1, NULL); 4226 if (ret) 4227 fprintf(stderr, 4228 "Warning: latencystats init() returned error %d\n", 4229 ret); 4230 fprintf(stderr, "Latencystats running on lcore %d\n", 4231 latencystats_lcore_id); 4232 } 4233 #endif 4234 4235 /* Setup bitrate stats */ 4236 #ifdef RTE_LIB_BITRATESTATS 4237 if (bitrate_enabled != 0) { 4238 bitrate_data = rte_stats_bitrate_create(); 4239 if (bitrate_data == NULL) 4240 rte_exit(EXIT_FAILURE, 4241 "Could not allocate bitrate data.\n"); 4242 rte_stats_bitrate_reg(bitrate_data); 4243 } 4244 #endif 4245 #ifdef RTE_LIB_CMDLINE 4246 if (strlen(cmdline_filename) != 0) 4247 cmdline_read_from_file(cmdline_filename); 4248 4249 if (interactive == 1) { 4250 if (auto_start) { 4251 printf("Start automatic packet forwarding\n"); 4252 start_packet_forwarding(0); 4253 } 4254 prompt(); 4255 pmd_test_exit(); 4256 } else 4257 #endif 4258 { 4259 char c; 4260 int rc; 4261 4262 f_quit = 0; 4263 4264 printf("No commandline core given, start packet forwarding\n"); 4265 start_packet_forwarding(tx_first); 4266 if (stats_period != 0) { 4267 uint64_t prev_time = 0, cur_time, diff_time = 0; 4268 uint64_t timer_period; 4269 4270 /* Convert to number of cycles */ 4271 timer_period = stats_period * rte_get_timer_hz(); 4272 4273 while (f_quit == 0) { 4274 cur_time = rte_get_timer_cycles(); 4275 diff_time += cur_time - prev_time; 4276 4277 if (diff_time >= timer_period) { 4278 print_stats(); 4279 /* Reset the timer */ 4280 diff_time = 0; 4281 } 4282 /* Sleep to avoid unnecessary checks */ 4283 prev_time = cur_time; 4284 rte_delay_us_sleep(US_PER_S); 4285 } 4286 } 4287 4288 printf("Press enter to exit\n"); 4289 rc = read(0, &c, 1); 4290 pmd_test_exit(); 4291 if (rc < 0) 4292 return 1; 4293 } 4294 4295 ret = rte_eal_cleanup(); 4296 if (ret != 0) 4297 rte_exit(EXIT_FAILURE, 4298 "EAL cleanup failed: %s\n", strerror(-ret)); 4299 4300 return EXIT_SUCCESS; 4301 } 4302