1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #ifndef RTE_EXEC_ENV_WINDOWS 13 #include <sys/mman.h> 14 #endif 15 #include <sys/types.h> 16 #include <errno.h> 17 #include <stdbool.h> 18 19 #include <sys/queue.h> 20 #include <sys/stat.h> 21 22 #include <stdint.h> 23 #include <unistd.h> 24 #include <inttypes.h> 25 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_debug.h> 31 #include <rte_cycles.h> 32 #include <rte_memory.h> 33 #include <rte_memcpy.h> 34 #include <rte_launch.h> 35 #include <rte_eal.h> 36 #include <rte_alarm.h> 37 #include <rte_per_lcore.h> 38 #include <rte_lcore.h> 39 #include <rte_atomic.h> 40 #include <rte_branch_prediction.h> 41 #include <rte_mempool.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 #include <rte_mbuf_pool_ops.h> 45 #include <rte_interrupts.h> 46 #include <rte_pci.h> 47 #include <rte_ether.h> 48 #include <rte_ethdev.h> 49 #include <rte_dev.h> 50 #include <rte_string_fns.h> 51 #ifdef RTE_NET_IXGBE 52 #include <rte_pmd_ixgbe.h> 53 #endif 54 #ifdef RTE_LIB_PDUMP 55 #include <rte_pdump.h> 56 #endif 57 #include <rte_flow.h> 58 #include <rte_metrics.h> 59 #ifdef RTE_LIB_BITRATESTATS 60 #include <rte_bitrate.h> 61 #endif 62 #ifdef RTE_LIB_LATENCYSTATS 63 #include <rte_latencystats.h> 64 #endif 65 #ifdef RTE_EXEC_ENV_WINDOWS 66 #include <process.h> 67 #endif 68 69 #include "testpmd.h" 70 71 #ifndef MAP_HUGETLB 72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 73 #define HUGE_FLAG (0x40000) 74 #else 75 #define HUGE_FLAG MAP_HUGETLB 76 #endif 77 78 #ifndef MAP_HUGE_SHIFT 79 /* older kernels (or FreeBSD) will not have this define */ 80 #define HUGE_SHIFT (26) 81 #else 82 #define HUGE_SHIFT MAP_HUGE_SHIFT 83 #endif 84 85 #define EXTMEM_HEAP_NAME "extmem" 86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 87 88 uint16_t verbose_level = 0; /**< Silent by default. */ 89 int testpmd_logtype; /**< Log type for testpmd logs */ 90 91 /* use main core for command line ? */ 92 uint8_t interactive = 0; 93 uint8_t auto_start = 0; 94 uint8_t tx_first; 95 char cmdline_filename[PATH_MAX] = {0}; 96 97 /* 98 * NUMA support configuration. 99 * When set, the NUMA support attempts to dispatch the allocation of the 100 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 101 * probed ports among the CPU sockets 0 and 1. 102 * Otherwise, all memory is allocated from CPU socket 0. 103 */ 104 uint8_t numa_support = 1; /**< numa enabled by default */ 105 106 /* 107 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 108 * not configured. 109 */ 110 uint8_t socket_num = UMA_NO_CONFIG; 111 112 /* 113 * Select mempool allocation type: 114 * - native: use regular DPDK memory 115 * - anon: use regular DPDK memory to create mempool, but populate using 116 * anonymous memory (may not be IOVA-contiguous) 117 * - xmem: use externally allocated hugepage memory 118 */ 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 120 121 /* 122 * Store specified sockets on which memory pool to be used by ports 123 * is allocated. 124 */ 125 uint8_t port_numa[RTE_MAX_ETHPORTS]; 126 127 /* 128 * Store specified sockets on which RX ring to be used by ports 129 * is allocated. 130 */ 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 132 133 /* 134 * Store specified sockets on which TX ring to be used by ports 135 * is allocated. 136 */ 137 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 138 139 /* 140 * Record the Ethernet address of peer target ports to which packets are 141 * forwarded. 142 * Must be instantiated with the ethernet addresses of peer traffic generator 143 * ports. 144 */ 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 146 portid_t nb_peer_eth_addrs = 0; 147 148 /* 149 * Probed Target Environment. 150 */ 151 struct rte_port *ports; /**< For all probed ethernet ports. */ 152 portid_t nb_ports; /**< Number of probed ethernet ports. */ 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 154 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 155 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 157 158 /* 159 * Test Forwarding Configuration. 160 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 161 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 162 */ 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 165 portid_t nb_cfg_ports; /**< Number of configured ports. */ 166 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 167 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 170 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 172 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 173 174 /* 175 * Forwarding engines. 176 */ 177 struct fwd_engine * fwd_engines[] = { 178 &io_fwd_engine, 179 &mac_fwd_engine, 180 &mac_swap_engine, 181 &flow_gen_engine, 182 &rx_only_engine, 183 &tx_only_engine, 184 &csum_fwd_engine, 185 &icmp_echo_engine, 186 &noisy_vnf_engine, 187 &five_tuple_swap_fwd_engine, 188 #ifdef RTE_LIBRTE_IEEE1588 189 &ieee1588_fwd_engine, 190 #endif 191 NULL, 192 }; 193 194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT]; 195 uint16_t mempool_flags; 196 197 struct fwd_config cur_fwd_config; 198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 199 uint32_t retry_enabled; 200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 202 203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */ 204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = { 205 DEFAULT_MBUF_DATA_SIZE 206 }; /**< Mbuf data space size. */ 207 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 208 * specified on command-line. */ 209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 210 211 /** Extended statistics to show. */ 212 struct rte_eth_xstat_name *xstats_display; 213 214 unsigned int xstats_display_num; /**< Size of extended statistics to show */ 215 216 /* 217 * In container, it cannot terminate the process which running with 'stats-period' 218 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 219 */ 220 uint8_t f_quit; 221 222 /* 223 * Max Rx frame size, set by '--max-pkt-len' parameter. 224 */ 225 uint32_t max_rx_pkt_len; 226 227 /* 228 * Configuration of packet segments used to scatter received packets 229 * if some of split features is configured. 230 */ 231 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT]; 232 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */ 233 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT]; 234 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */ 235 236 /* 237 * Configuration of packet segments used by the "txonly" processing engine. 238 */ 239 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 240 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 241 TXONLY_DEF_PACKET_LEN, 242 }; 243 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 244 245 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 246 /**< Split policy for packets to TX. */ 247 248 uint8_t txonly_multi_flow; 249 /**< Whether multiple flows are generated in TXONLY mode. */ 250 251 uint32_t tx_pkt_times_inter; 252 /**< Timings for send scheduling in TXONLY mode, time between bursts. */ 253 254 uint32_t tx_pkt_times_intra; 255 /**< Timings for send scheduling in TXONLY mode, time between packets. */ 256 257 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 258 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */ 259 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */ 260 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 261 262 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 263 uint8_t dcb_config = 0; 264 265 /* 266 * Configurable number of RX/TX queues. 267 */ 268 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 269 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 270 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 271 272 /* 273 * Configurable number of RX/TX ring descriptors. 274 * Defaults are supplied by drivers via ethdev. 275 */ 276 #define RTE_TEST_RX_DESC_DEFAULT 0 277 #define RTE_TEST_TX_DESC_DEFAULT 0 278 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 279 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 280 281 #define RTE_PMD_PARAM_UNSET -1 282 /* 283 * Configurable values of RX and TX ring threshold registers. 284 */ 285 286 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 287 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 288 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 289 290 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 291 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 292 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 293 294 /* 295 * Configurable value of RX free threshold. 296 */ 297 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 298 299 /* 300 * Configurable value of RX drop enable. 301 */ 302 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 303 304 /* 305 * Configurable value of TX free threshold. 306 */ 307 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 308 309 /* 310 * Configurable value of TX RS bit threshold. 311 */ 312 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 313 314 /* 315 * Configurable value of buffered packets before sending. 316 */ 317 uint16_t noisy_tx_sw_bufsz; 318 319 /* 320 * Configurable value of packet buffer timeout. 321 */ 322 uint16_t noisy_tx_sw_buf_flush_time; 323 324 /* 325 * Configurable value for size of VNF internal memory area 326 * used for simulating noisy neighbour behaviour 327 */ 328 uint64_t noisy_lkup_mem_sz; 329 330 /* 331 * Configurable value of number of random writes done in 332 * VNF simulation memory area. 333 */ 334 uint64_t noisy_lkup_num_writes; 335 336 /* 337 * Configurable value of number of random reads done in 338 * VNF simulation memory area. 339 */ 340 uint64_t noisy_lkup_num_reads; 341 342 /* 343 * Configurable value of number of random reads/writes done in 344 * VNF simulation memory area. 345 */ 346 uint64_t noisy_lkup_num_reads_writes; 347 348 /* 349 * Receive Side Scaling (RSS) configuration. 350 */ 351 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 352 353 /* 354 * Port topology configuration 355 */ 356 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 357 358 /* 359 * Avoids to flush all the RX streams before starts forwarding. 360 */ 361 uint8_t no_flush_rx = 0; /* flush by default */ 362 363 /* 364 * Flow API isolated mode. 365 */ 366 uint8_t flow_isolate_all; 367 368 /* 369 * Avoids to check link status when starting/stopping a port. 370 */ 371 uint8_t no_link_check = 0; /* check by default */ 372 373 /* 374 * Don't automatically start all ports in interactive mode. 375 */ 376 uint8_t no_device_start = 0; 377 378 /* 379 * Enable link status change notification 380 */ 381 uint8_t lsc_interrupt = 1; /* enabled by default */ 382 383 /* 384 * Enable device removal notification. 385 */ 386 uint8_t rmv_interrupt = 1; /* enabled by default */ 387 388 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 389 390 /* After attach, port setup is called on event or by iterator */ 391 bool setup_on_probe_event = true; 392 393 /* Clear ptypes on port initialization. */ 394 uint8_t clear_ptypes = true; 395 396 /* Hairpin ports configuration mode. */ 397 uint16_t hairpin_mode; 398 399 /* Pretty printing of ethdev events */ 400 static const char * const eth_event_desc[] = { 401 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 402 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 403 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 404 [RTE_ETH_EVENT_INTR_RESET] = "reset", 405 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 406 [RTE_ETH_EVENT_IPSEC] = "IPsec", 407 [RTE_ETH_EVENT_MACSEC] = "MACsec", 408 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 409 [RTE_ETH_EVENT_NEW] = "device probed", 410 [RTE_ETH_EVENT_DESTROY] = "device released", 411 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 412 [RTE_ETH_EVENT_MAX] = NULL, 413 }; 414 415 /* 416 * Display or mask ether events 417 * Default to all events except VF_MBOX 418 */ 419 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 420 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 421 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 422 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 423 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 424 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 425 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 426 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 427 /* 428 * Decide if all memory are locked for performance. 429 */ 430 int do_mlockall = 0; 431 432 /* 433 * NIC bypass mode configuration options. 434 */ 435 436 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 437 /* The NIC bypass watchdog timeout. */ 438 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 439 #endif 440 441 442 #ifdef RTE_LIB_LATENCYSTATS 443 444 /* 445 * Set when latency stats is enabled in the commandline 446 */ 447 uint8_t latencystats_enabled; 448 449 /* 450 * Lcore ID to serive latency statistics. 451 */ 452 lcoreid_t latencystats_lcore_id = -1; 453 454 #endif 455 456 /* 457 * Ethernet device configuration. 458 */ 459 struct rte_eth_rxmode rx_mode; 460 461 struct rte_eth_txmode tx_mode = { 462 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 463 }; 464 465 struct rte_fdir_conf fdir_conf = { 466 .mode = RTE_FDIR_MODE_NONE, 467 .pballoc = RTE_FDIR_PBALLOC_64K, 468 .status = RTE_FDIR_REPORT_STATUS, 469 .mask = { 470 .vlan_tci_mask = 0xFFEF, 471 .ipv4_mask = { 472 .src_ip = 0xFFFFFFFF, 473 .dst_ip = 0xFFFFFFFF, 474 }, 475 .ipv6_mask = { 476 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 477 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 478 }, 479 .src_port_mask = 0xFFFF, 480 .dst_port_mask = 0xFFFF, 481 .mac_addr_byte_mask = 0xFF, 482 .tunnel_type_mask = 1, 483 .tunnel_id_mask = 0xFFFFFFFF, 484 }, 485 .drop_queue = 127, 486 }; 487 488 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 489 490 /* 491 * Display zero values by default for xstats 492 */ 493 uint8_t xstats_hide_zero; 494 495 /* 496 * Measure of CPU cycles disabled by default 497 */ 498 uint8_t record_core_cycles; 499 500 /* 501 * Display of RX and TX bursts disabled by default 502 */ 503 uint8_t record_burst_stats; 504 505 /* 506 * Number of ports per shared Rx queue group, 0 disable. 507 */ 508 uint32_t rxq_share; 509 510 unsigned int num_sockets = 0; 511 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 512 513 #ifdef RTE_LIB_BITRATESTATS 514 /* Bitrate statistics */ 515 struct rte_stats_bitrates *bitrate_data; 516 lcoreid_t bitrate_lcore_id; 517 uint8_t bitrate_enabled; 518 #endif 519 520 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 521 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 522 523 /* 524 * hexadecimal bitmask of RX mq mode can be enabled. 525 */ 526 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 527 528 /* 529 * Used to set forced link speed 530 */ 531 uint32_t eth_link_speed; 532 533 /* 534 * ID of the current process in multi-process, used to 535 * configure the queues to be polled. 536 */ 537 int proc_id; 538 539 /* 540 * Number of processes in multi-process, used to 541 * configure the queues to be polled. 542 */ 543 unsigned int num_procs = 1; 544 545 static void 546 eth_rx_metadata_negotiate_mp(uint16_t port_id) 547 { 548 uint64_t rx_meta_features = 0; 549 int ret; 550 551 if (!is_proc_primary()) 552 return; 553 554 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG; 555 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK; 556 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID; 557 558 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features); 559 if (ret == 0) { 560 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) { 561 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n", 562 port_id); 563 } 564 565 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) { 566 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n", 567 port_id); 568 } 569 570 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) { 571 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n", 572 port_id); 573 } 574 } else if (ret != -ENOTSUP) { 575 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n", 576 port_id, rte_strerror(-ret)); 577 } 578 } 579 580 static void 581 flow_pick_transfer_proxy_mp(uint16_t port_id) 582 { 583 struct rte_port *port = &ports[port_id]; 584 int ret; 585 586 port->flow_transfer_proxy = port_id; 587 588 if (!is_proc_primary()) 589 return; 590 591 ret = rte_flow_pick_transfer_proxy(port_id, &port->flow_transfer_proxy, 592 NULL); 593 if (ret != 0) { 594 fprintf(stderr, "Error picking flow transfer proxy for port %u: %s - ignore\n", 595 port_id, rte_strerror(-ret)); 596 } 597 } 598 599 static int 600 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, 601 const struct rte_eth_conf *dev_conf) 602 { 603 if (is_proc_primary()) 604 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, 605 dev_conf); 606 return 0; 607 } 608 609 static int 610 eth_dev_start_mp(uint16_t port_id) 611 { 612 if (is_proc_primary()) 613 return rte_eth_dev_start(port_id); 614 615 return 0; 616 } 617 618 static int 619 eth_dev_stop_mp(uint16_t port_id) 620 { 621 if (is_proc_primary()) 622 return rte_eth_dev_stop(port_id); 623 624 return 0; 625 } 626 627 static void 628 mempool_free_mp(struct rte_mempool *mp) 629 { 630 if (is_proc_primary()) 631 rte_mempool_free(mp); 632 } 633 634 static int 635 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu) 636 { 637 if (is_proc_primary()) 638 return rte_eth_dev_set_mtu(port_id, mtu); 639 640 return 0; 641 } 642 643 /* Forward function declarations */ 644 static void setup_attached_port(portid_t pi); 645 static void check_all_ports_link_status(uint32_t port_mask); 646 static int eth_event_callback(portid_t port_id, 647 enum rte_eth_event_type type, 648 void *param, void *ret_param); 649 static void dev_event_callback(const char *device_name, 650 enum rte_dev_event_type type, 651 void *param); 652 static void fill_xstats_display_info(void); 653 654 /* 655 * Check if all the ports are started. 656 * If yes, return positive value. If not, return zero. 657 */ 658 static int all_ports_started(void); 659 660 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 661 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 662 663 /* Holds the registered mbuf dynamic flags names. */ 664 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 665 666 667 /* 668 * Helper function to check if socket is already discovered. 669 * If yes, return positive value. If not, return zero. 670 */ 671 int 672 new_socket_id(unsigned int socket_id) 673 { 674 unsigned int i; 675 676 for (i = 0; i < num_sockets; i++) { 677 if (socket_ids[i] == socket_id) 678 return 0; 679 } 680 return 1; 681 } 682 683 /* 684 * Setup default configuration. 685 */ 686 static void 687 set_default_fwd_lcores_config(void) 688 { 689 unsigned int i; 690 unsigned int nb_lc; 691 unsigned int sock_num; 692 693 nb_lc = 0; 694 for (i = 0; i < RTE_MAX_LCORE; i++) { 695 if (!rte_lcore_is_enabled(i)) 696 continue; 697 sock_num = rte_lcore_to_socket_id(i); 698 if (new_socket_id(sock_num)) { 699 if (num_sockets >= RTE_MAX_NUMA_NODES) { 700 rte_exit(EXIT_FAILURE, 701 "Total sockets greater than %u\n", 702 RTE_MAX_NUMA_NODES); 703 } 704 socket_ids[num_sockets++] = sock_num; 705 } 706 if (i == rte_get_main_lcore()) 707 continue; 708 fwd_lcores_cpuids[nb_lc++] = i; 709 } 710 nb_lcores = (lcoreid_t) nb_lc; 711 nb_cfg_lcores = nb_lcores; 712 nb_fwd_lcores = 1; 713 } 714 715 static void 716 set_def_peer_eth_addrs(void) 717 { 718 portid_t i; 719 720 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 721 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 722 peer_eth_addrs[i].addr_bytes[5] = i; 723 } 724 } 725 726 static void 727 set_default_fwd_ports_config(void) 728 { 729 portid_t pt_id; 730 int i = 0; 731 732 RTE_ETH_FOREACH_DEV(pt_id) { 733 fwd_ports_ids[i++] = pt_id; 734 735 /* Update sockets info according to the attached device */ 736 int socket_id = rte_eth_dev_socket_id(pt_id); 737 if (socket_id >= 0 && new_socket_id(socket_id)) { 738 if (num_sockets >= RTE_MAX_NUMA_NODES) { 739 rte_exit(EXIT_FAILURE, 740 "Total sockets greater than %u\n", 741 RTE_MAX_NUMA_NODES); 742 } 743 socket_ids[num_sockets++] = socket_id; 744 } 745 } 746 747 nb_cfg_ports = nb_ports; 748 nb_fwd_ports = nb_ports; 749 } 750 751 void 752 set_def_fwd_config(void) 753 { 754 set_default_fwd_lcores_config(); 755 set_def_peer_eth_addrs(); 756 set_default_fwd_ports_config(); 757 } 758 759 #ifndef RTE_EXEC_ENV_WINDOWS 760 /* extremely pessimistic estimation of memory required to create a mempool */ 761 static int 762 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 763 { 764 unsigned int n_pages, mbuf_per_pg, leftover; 765 uint64_t total_mem, mbuf_mem, obj_sz; 766 767 /* there is no good way to predict how much space the mempool will 768 * occupy because it will allocate chunks on the fly, and some of those 769 * will come from default DPDK memory while some will come from our 770 * external memory, so just assume 128MB will be enough for everyone. 771 */ 772 uint64_t hdr_mem = 128 << 20; 773 774 /* account for possible non-contiguousness */ 775 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 776 if (obj_sz > pgsz) { 777 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 778 return -1; 779 } 780 781 mbuf_per_pg = pgsz / obj_sz; 782 leftover = (nb_mbufs % mbuf_per_pg) > 0; 783 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 784 785 mbuf_mem = n_pages * pgsz; 786 787 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 788 789 if (total_mem > SIZE_MAX) { 790 TESTPMD_LOG(ERR, "Memory size too big\n"); 791 return -1; 792 } 793 *out = (size_t)total_mem; 794 795 return 0; 796 } 797 798 static int 799 pagesz_flags(uint64_t page_sz) 800 { 801 /* as per mmap() manpage, all page sizes are log2 of page size 802 * shifted by MAP_HUGE_SHIFT 803 */ 804 int log2 = rte_log2_u64(page_sz); 805 806 return (log2 << HUGE_SHIFT); 807 } 808 809 static void * 810 alloc_mem(size_t memsz, size_t pgsz, bool huge) 811 { 812 void *addr; 813 int flags; 814 815 /* allocate anonymous hugepages */ 816 flags = MAP_ANONYMOUS | MAP_PRIVATE; 817 if (huge) 818 flags |= HUGE_FLAG | pagesz_flags(pgsz); 819 820 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 821 if (addr == MAP_FAILED) 822 return NULL; 823 824 return addr; 825 } 826 827 struct extmem_param { 828 void *addr; 829 size_t len; 830 size_t pgsz; 831 rte_iova_t *iova_table; 832 unsigned int iova_table_len; 833 }; 834 835 static int 836 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 837 bool huge) 838 { 839 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 840 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 841 unsigned int cur_page, n_pages, pgsz_idx; 842 size_t mem_sz, cur_pgsz; 843 rte_iova_t *iovas = NULL; 844 void *addr; 845 int ret; 846 847 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 848 /* skip anything that is too big */ 849 if (pgsizes[pgsz_idx] > SIZE_MAX) 850 continue; 851 852 cur_pgsz = pgsizes[pgsz_idx]; 853 854 /* if we were told not to allocate hugepages, override */ 855 if (!huge) 856 cur_pgsz = sysconf(_SC_PAGESIZE); 857 858 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 859 if (ret < 0) { 860 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 861 return -1; 862 } 863 864 /* allocate our memory */ 865 addr = alloc_mem(mem_sz, cur_pgsz, huge); 866 867 /* if we couldn't allocate memory with a specified page size, 868 * that doesn't mean we can't do it with other page sizes, so 869 * try another one. 870 */ 871 if (addr == NULL) 872 continue; 873 874 /* store IOVA addresses for every page in this memory area */ 875 n_pages = mem_sz / cur_pgsz; 876 877 iovas = malloc(sizeof(*iovas) * n_pages); 878 879 if (iovas == NULL) { 880 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 881 goto fail; 882 } 883 /* lock memory if it's not huge pages */ 884 if (!huge) 885 mlock(addr, mem_sz); 886 887 /* populate IOVA addresses */ 888 for (cur_page = 0; cur_page < n_pages; cur_page++) { 889 rte_iova_t iova; 890 size_t offset; 891 void *cur; 892 893 offset = cur_pgsz * cur_page; 894 cur = RTE_PTR_ADD(addr, offset); 895 896 /* touch the page before getting its IOVA */ 897 *(volatile char *)cur = 0; 898 899 iova = rte_mem_virt2iova(cur); 900 901 iovas[cur_page] = iova; 902 } 903 904 break; 905 } 906 /* if we couldn't allocate anything */ 907 if (iovas == NULL) 908 return -1; 909 910 param->addr = addr; 911 param->len = mem_sz; 912 param->pgsz = cur_pgsz; 913 param->iova_table = iovas; 914 param->iova_table_len = n_pages; 915 916 return 0; 917 fail: 918 if (iovas) 919 free(iovas); 920 if (addr) 921 munmap(addr, mem_sz); 922 923 return -1; 924 } 925 926 static int 927 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 928 { 929 struct extmem_param param; 930 int socket_id, ret; 931 932 memset(¶m, 0, sizeof(param)); 933 934 /* check if our heap exists */ 935 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 936 if (socket_id < 0) { 937 /* create our heap */ 938 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 939 if (ret < 0) { 940 TESTPMD_LOG(ERR, "Cannot create heap\n"); 941 return -1; 942 } 943 } 944 945 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 946 if (ret < 0) { 947 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 948 return -1; 949 } 950 951 /* we now have a valid memory area, so add it to heap */ 952 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 953 param.addr, param.len, param.iova_table, 954 param.iova_table_len, param.pgsz); 955 956 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 957 958 /* not needed any more */ 959 free(param.iova_table); 960 961 if (ret < 0) { 962 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 963 munmap(param.addr, param.len); 964 return -1; 965 } 966 967 /* success */ 968 969 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 970 param.len >> 20); 971 972 return 0; 973 } 974 static void 975 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 976 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 977 { 978 uint16_t pid = 0; 979 int ret; 980 981 RTE_ETH_FOREACH_DEV(pid) { 982 struct rte_eth_dev_info dev_info; 983 984 ret = eth_dev_info_get_print_err(pid, &dev_info); 985 if (ret != 0) { 986 TESTPMD_LOG(DEBUG, 987 "unable to get device info for port %d on addr 0x%p," 988 "mempool unmapping will not be performed\n", 989 pid, memhdr->addr); 990 continue; 991 } 992 993 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len); 994 if (ret) { 995 TESTPMD_LOG(DEBUG, 996 "unable to DMA unmap addr 0x%p " 997 "for device %s\n", 998 memhdr->addr, dev_info.device->name); 999 } 1000 } 1001 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 1002 if (ret) { 1003 TESTPMD_LOG(DEBUG, 1004 "unable to un-register addr 0x%p\n", memhdr->addr); 1005 } 1006 } 1007 1008 static void 1009 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 1010 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 1011 { 1012 uint16_t pid = 0; 1013 size_t page_size = sysconf(_SC_PAGESIZE); 1014 int ret; 1015 1016 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 1017 page_size); 1018 if (ret) { 1019 TESTPMD_LOG(DEBUG, 1020 "unable to register addr 0x%p\n", memhdr->addr); 1021 return; 1022 } 1023 RTE_ETH_FOREACH_DEV(pid) { 1024 struct rte_eth_dev_info dev_info; 1025 1026 ret = eth_dev_info_get_print_err(pid, &dev_info); 1027 if (ret != 0) { 1028 TESTPMD_LOG(DEBUG, 1029 "unable to get device info for port %d on addr 0x%p," 1030 "mempool mapping will not be performed\n", 1031 pid, memhdr->addr); 1032 continue; 1033 } 1034 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len); 1035 if (ret) { 1036 TESTPMD_LOG(DEBUG, 1037 "unable to DMA map addr 0x%p " 1038 "for device %s\n", 1039 memhdr->addr, dev_info.device->name); 1040 } 1041 } 1042 } 1043 #endif 1044 1045 static unsigned int 1046 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 1047 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 1048 { 1049 struct rte_pktmbuf_extmem *xmem; 1050 unsigned int ext_num, zone_num, elt_num; 1051 uint16_t elt_size; 1052 1053 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 1054 elt_num = EXTBUF_ZONE_SIZE / elt_size; 1055 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 1056 1057 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 1058 if (xmem == NULL) { 1059 TESTPMD_LOG(ERR, "Cannot allocate memory for " 1060 "external buffer descriptors\n"); 1061 *ext_mem = NULL; 1062 return 0; 1063 } 1064 for (ext_num = 0; ext_num < zone_num; ext_num++) { 1065 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 1066 const struct rte_memzone *mz; 1067 char mz_name[RTE_MEMZONE_NAMESIZE]; 1068 int ret; 1069 1070 ret = snprintf(mz_name, sizeof(mz_name), 1071 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 1072 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 1073 errno = ENAMETOOLONG; 1074 ext_num = 0; 1075 break; 1076 } 1077 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 1078 socket_id, 1079 RTE_MEMZONE_IOVA_CONTIG | 1080 RTE_MEMZONE_1GB | 1081 RTE_MEMZONE_SIZE_HINT_ONLY, 1082 EXTBUF_ZONE_SIZE); 1083 if (mz == NULL) { 1084 /* 1085 * The caller exits on external buffer creation 1086 * error, so there is no need to free memzones. 1087 */ 1088 errno = ENOMEM; 1089 ext_num = 0; 1090 break; 1091 } 1092 xseg->buf_ptr = mz->addr; 1093 xseg->buf_iova = mz->iova; 1094 xseg->buf_len = EXTBUF_ZONE_SIZE; 1095 xseg->elt_size = elt_size; 1096 } 1097 if (ext_num == 0 && xmem != NULL) { 1098 free(xmem); 1099 xmem = NULL; 1100 } 1101 *ext_mem = xmem; 1102 return ext_num; 1103 } 1104 1105 /* 1106 * Configuration initialisation done once at init time. 1107 */ 1108 static struct rte_mempool * 1109 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 1110 unsigned int socket_id, uint16_t size_idx) 1111 { 1112 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1113 struct rte_mempool *rte_mp = NULL; 1114 #ifndef RTE_EXEC_ENV_WINDOWS 1115 uint32_t mb_size; 1116 1117 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 1118 #endif 1119 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx); 1120 if (!is_proc_primary()) { 1121 rte_mp = rte_mempool_lookup(pool_name); 1122 if (rte_mp == NULL) 1123 rte_exit(EXIT_FAILURE, 1124 "Get mbuf pool for socket %u failed: %s\n", 1125 socket_id, rte_strerror(rte_errno)); 1126 return rte_mp; 1127 } 1128 1129 TESTPMD_LOG(INFO, 1130 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 1131 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 1132 1133 switch (mp_alloc_type) { 1134 case MP_ALLOC_NATIVE: 1135 { 1136 /* wrapper to rte_mempool_create() */ 1137 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1138 rte_mbuf_best_mempool_ops()); 1139 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1140 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 1141 break; 1142 } 1143 #ifndef RTE_EXEC_ENV_WINDOWS 1144 case MP_ALLOC_ANON: 1145 { 1146 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1147 mb_size, (unsigned int) mb_mempool_cache, 1148 sizeof(struct rte_pktmbuf_pool_private), 1149 socket_id, mempool_flags); 1150 if (rte_mp == NULL) 1151 goto err; 1152 1153 if (rte_mempool_populate_anon(rte_mp) == 0) { 1154 rte_mempool_free(rte_mp); 1155 rte_mp = NULL; 1156 goto err; 1157 } 1158 rte_pktmbuf_pool_init(rte_mp, NULL); 1159 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 1160 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 1161 break; 1162 } 1163 case MP_ALLOC_XMEM: 1164 case MP_ALLOC_XMEM_HUGE: 1165 { 1166 int heap_socket; 1167 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 1168 1169 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 1170 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 1171 1172 heap_socket = 1173 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 1174 if (heap_socket < 0) 1175 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 1176 1177 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1178 rte_mbuf_best_mempool_ops()); 1179 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1180 mb_mempool_cache, 0, mbuf_seg_size, 1181 heap_socket); 1182 break; 1183 } 1184 #endif 1185 case MP_ALLOC_XBUF: 1186 { 1187 struct rte_pktmbuf_extmem *ext_mem; 1188 unsigned int ext_num; 1189 1190 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1191 socket_id, pool_name, &ext_mem); 1192 if (ext_num == 0) 1193 rte_exit(EXIT_FAILURE, 1194 "Can't create pinned data buffers\n"); 1195 1196 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1197 rte_mbuf_best_mempool_ops()); 1198 rte_mp = rte_pktmbuf_pool_create_extbuf 1199 (pool_name, nb_mbuf, mb_mempool_cache, 1200 0, mbuf_seg_size, socket_id, 1201 ext_mem, ext_num); 1202 free(ext_mem); 1203 break; 1204 } 1205 default: 1206 { 1207 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1208 } 1209 } 1210 1211 #ifndef RTE_EXEC_ENV_WINDOWS 1212 err: 1213 #endif 1214 if (rte_mp == NULL) { 1215 rte_exit(EXIT_FAILURE, 1216 "Creation of mbuf pool for socket %u failed: %s\n", 1217 socket_id, rte_strerror(rte_errno)); 1218 } else if (verbose_level > 0) { 1219 rte_mempool_dump(stdout, rte_mp); 1220 } 1221 return rte_mp; 1222 } 1223 1224 /* 1225 * Check given socket id is valid or not with NUMA mode, 1226 * if valid, return 0, else return -1 1227 */ 1228 static int 1229 check_socket_id(const unsigned int socket_id) 1230 { 1231 static int warning_once = 0; 1232 1233 if (new_socket_id(socket_id)) { 1234 if (!warning_once && numa_support) 1235 fprintf(stderr, 1236 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n"); 1237 warning_once = 1; 1238 return -1; 1239 } 1240 return 0; 1241 } 1242 1243 /* 1244 * Get the allowed maximum number of RX queues. 1245 * *pid return the port id which has minimal value of 1246 * max_rx_queues in all ports. 1247 */ 1248 queueid_t 1249 get_allowed_max_nb_rxq(portid_t *pid) 1250 { 1251 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1252 bool max_rxq_valid = false; 1253 portid_t pi; 1254 struct rte_eth_dev_info dev_info; 1255 1256 RTE_ETH_FOREACH_DEV(pi) { 1257 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1258 continue; 1259 1260 max_rxq_valid = true; 1261 if (dev_info.max_rx_queues < allowed_max_rxq) { 1262 allowed_max_rxq = dev_info.max_rx_queues; 1263 *pid = pi; 1264 } 1265 } 1266 return max_rxq_valid ? allowed_max_rxq : 0; 1267 } 1268 1269 /* 1270 * Check input rxq is valid or not. 1271 * If input rxq is not greater than any of maximum number 1272 * of RX queues of all ports, it is valid. 1273 * if valid, return 0, else return -1 1274 */ 1275 int 1276 check_nb_rxq(queueid_t rxq) 1277 { 1278 queueid_t allowed_max_rxq; 1279 portid_t pid = 0; 1280 1281 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1282 if (rxq > allowed_max_rxq) { 1283 fprintf(stderr, 1284 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n", 1285 rxq, allowed_max_rxq, pid); 1286 return -1; 1287 } 1288 return 0; 1289 } 1290 1291 /* 1292 * Get the allowed maximum number of TX queues. 1293 * *pid return the port id which has minimal value of 1294 * max_tx_queues in all ports. 1295 */ 1296 queueid_t 1297 get_allowed_max_nb_txq(portid_t *pid) 1298 { 1299 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1300 bool max_txq_valid = false; 1301 portid_t pi; 1302 struct rte_eth_dev_info dev_info; 1303 1304 RTE_ETH_FOREACH_DEV(pi) { 1305 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1306 continue; 1307 1308 max_txq_valid = true; 1309 if (dev_info.max_tx_queues < allowed_max_txq) { 1310 allowed_max_txq = dev_info.max_tx_queues; 1311 *pid = pi; 1312 } 1313 } 1314 return max_txq_valid ? allowed_max_txq : 0; 1315 } 1316 1317 /* 1318 * Check input txq is valid or not. 1319 * If input txq is not greater than any of maximum number 1320 * of TX queues of all ports, it is valid. 1321 * if valid, return 0, else return -1 1322 */ 1323 int 1324 check_nb_txq(queueid_t txq) 1325 { 1326 queueid_t allowed_max_txq; 1327 portid_t pid = 0; 1328 1329 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1330 if (txq > allowed_max_txq) { 1331 fprintf(stderr, 1332 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n", 1333 txq, allowed_max_txq, pid); 1334 return -1; 1335 } 1336 return 0; 1337 } 1338 1339 /* 1340 * Get the allowed maximum number of RXDs of every rx queue. 1341 * *pid return the port id which has minimal value of 1342 * max_rxd in all queues of all ports. 1343 */ 1344 static uint16_t 1345 get_allowed_max_nb_rxd(portid_t *pid) 1346 { 1347 uint16_t allowed_max_rxd = UINT16_MAX; 1348 portid_t pi; 1349 struct rte_eth_dev_info dev_info; 1350 1351 RTE_ETH_FOREACH_DEV(pi) { 1352 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1353 continue; 1354 1355 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1356 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1357 *pid = pi; 1358 } 1359 } 1360 return allowed_max_rxd; 1361 } 1362 1363 /* 1364 * Get the allowed minimal number of RXDs of every rx queue. 1365 * *pid return the port id which has minimal value of 1366 * min_rxd in all queues of all ports. 1367 */ 1368 static uint16_t 1369 get_allowed_min_nb_rxd(portid_t *pid) 1370 { 1371 uint16_t allowed_min_rxd = 0; 1372 portid_t pi; 1373 struct rte_eth_dev_info dev_info; 1374 1375 RTE_ETH_FOREACH_DEV(pi) { 1376 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1377 continue; 1378 1379 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1380 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1381 *pid = pi; 1382 } 1383 } 1384 1385 return allowed_min_rxd; 1386 } 1387 1388 /* 1389 * Check input rxd is valid or not. 1390 * If input rxd is not greater than any of maximum number 1391 * of RXDs of every Rx queues and is not less than any of 1392 * minimal number of RXDs of every Rx queues, it is valid. 1393 * if valid, return 0, else return -1 1394 */ 1395 int 1396 check_nb_rxd(queueid_t rxd) 1397 { 1398 uint16_t allowed_max_rxd; 1399 uint16_t allowed_min_rxd; 1400 portid_t pid = 0; 1401 1402 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1403 if (rxd > allowed_max_rxd) { 1404 fprintf(stderr, 1405 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n", 1406 rxd, allowed_max_rxd, pid); 1407 return -1; 1408 } 1409 1410 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1411 if (rxd < allowed_min_rxd) { 1412 fprintf(stderr, 1413 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n", 1414 rxd, allowed_min_rxd, pid); 1415 return -1; 1416 } 1417 1418 return 0; 1419 } 1420 1421 /* 1422 * Get the allowed maximum number of TXDs of every rx queues. 1423 * *pid return the port id which has minimal value of 1424 * max_txd in every tx queue. 1425 */ 1426 static uint16_t 1427 get_allowed_max_nb_txd(portid_t *pid) 1428 { 1429 uint16_t allowed_max_txd = UINT16_MAX; 1430 portid_t pi; 1431 struct rte_eth_dev_info dev_info; 1432 1433 RTE_ETH_FOREACH_DEV(pi) { 1434 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1435 continue; 1436 1437 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1438 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1439 *pid = pi; 1440 } 1441 } 1442 return allowed_max_txd; 1443 } 1444 1445 /* 1446 * Get the allowed maximum number of TXDs of every tx queues. 1447 * *pid return the port id which has minimal value of 1448 * min_txd in every tx queue. 1449 */ 1450 static uint16_t 1451 get_allowed_min_nb_txd(portid_t *pid) 1452 { 1453 uint16_t allowed_min_txd = 0; 1454 portid_t pi; 1455 struct rte_eth_dev_info dev_info; 1456 1457 RTE_ETH_FOREACH_DEV(pi) { 1458 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1459 continue; 1460 1461 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1462 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1463 *pid = pi; 1464 } 1465 } 1466 1467 return allowed_min_txd; 1468 } 1469 1470 /* 1471 * Check input txd is valid or not. 1472 * If input txd is not greater than any of maximum number 1473 * of TXDs of every Rx queues, it is valid. 1474 * if valid, return 0, else return -1 1475 */ 1476 int 1477 check_nb_txd(queueid_t txd) 1478 { 1479 uint16_t allowed_max_txd; 1480 uint16_t allowed_min_txd; 1481 portid_t pid = 0; 1482 1483 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1484 if (txd > allowed_max_txd) { 1485 fprintf(stderr, 1486 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n", 1487 txd, allowed_max_txd, pid); 1488 return -1; 1489 } 1490 1491 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1492 if (txd < allowed_min_txd) { 1493 fprintf(stderr, 1494 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n", 1495 txd, allowed_min_txd, pid); 1496 return -1; 1497 } 1498 return 0; 1499 } 1500 1501 1502 /* 1503 * Get the allowed maximum number of hairpin queues. 1504 * *pid return the port id which has minimal value of 1505 * max_hairpin_queues in all ports. 1506 */ 1507 queueid_t 1508 get_allowed_max_nb_hairpinq(portid_t *pid) 1509 { 1510 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1511 portid_t pi; 1512 struct rte_eth_hairpin_cap cap; 1513 1514 RTE_ETH_FOREACH_DEV(pi) { 1515 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1516 *pid = pi; 1517 return 0; 1518 } 1519 if (cap.max_nb_queues < allowed_max_hairpinq) { 1520 allowed_max_hairpinq = cap.max_nb_queues; 1521 *pid = pi; 1522 } 1523 } 1524 return allowed_max_hairpinq; 1525 } 1526 1527 /* 1528 * Check input hairpin is valid or not. 1529 * If input hairpin is not greater than any of maximum number 1530 * of hairpin queues of all ports, it is valid. 1531 * if valid, return 0, else return -1 1532 */ 1533 int 1534 check_nb_hairpinq(queueid_t hairpinq) 1535 { 1536 queueid_t allowed_max_hairpinq; 1537 portid_t pid = 0; 1538 1539 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1540 if (hairpinq > allowed_max_hairpinq) { 1541 fprintf(stderr, 1542 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 1543 hairpinq, allowed_max_hairpinq, pid); 1544 return -1; 1545 } 1546 return 0; 1547 } 1548 1549 static int 1550 get_eth_overhead(struct rte_eth_dev_info *dev_info) 1551 { 1552 uint32_t eth_overhead; 1553 1554 if (dev_info->max_mtu != UINT16_MAX && 1555 dev_info->max_rx_pktlen > dev_info->max_mtu) 1556 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu; 1557 else 1558 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 1559 1560 return eth_overhead; 1561 } 1562 1563 static void 1564 init_config_port_offloads(portid_t pid, uint32_t socket_id) 1565 { 1566 struct rte_port *port = &ports[pid]; 1567 int ret; 1568 int i; 1569 1570 eth_rx_metadata_negotiate_mp(pid); 1571 flow_pick_transfer_proxy_mp(pid); 1572 1573 port->dev_conf.txmode = tx_mode; 1574 port->dev_conf.rxmode = rx_mode; 1575 1576 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1577 if (ret != 0) 1578 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n"); 1579 1580 if (!(port->dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1581 port->dev_conf.txmode.offloads &= 1582 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1583 1584 /* Apply Rx offloads configuration */ 1585 for (i = 0; i < port->dev_info.max_rx_queues; i++) 1586 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads; 1587 /* Apply Tx offloads configuration */ 1588 for (i = 0; i < port->dev_info.max_tx_queues; i++) 1589 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads; 1590 1591 if (eth_link_speed) 1592 port->dev_conf.link_speeds = eth_link_speed; 1593 1594 if (max_rx_pkt_len) 1595 port->dev_conf.rxmode.mtu = max_rx_pkt_len - 1596 get_eth_overhead(&port->dev_info); 1597 1598 /* set flag to initialize port/queue */ 1599 port->need_reconfig = 1; 1600 port->need_reconfig_queues = 1; 1601 port->socket_id = socket_id; 1602 port->tx_metadata = 0; 1603 1604 /* 1605 * Check for maximum number of segments per MTU. 1606 * Accordingly update the mbuf data size. 1607 */ 1608 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1609 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1610 uint32_t eth_overhead = get_eth_overhead(&port->dev_info); 1611 uint16_t mtu; 1612 1613 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) { 1614 uint16_t data_size = (mtu + eth_overhead) / 1615 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1616 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM; 1617 1618 if (buffer_size > mbuf_data_size[0]) { 1619 mbuf_data_size[0] = buffer_size; 1620 TESTPMD_LOG(WARNING, 1621 "Configured mbuf size of the first segment %hu\n", 1622 mbuf_data_size[0]); 1623 } 1624 } 1625 } 1626 } 1627 1628 static void 1629 init_config(void) 1630 { 1631 portid_t pid; 1632 struct rte_mempool *mbp; 1633 unsigned int nb_mbuf_per_pool; 1634 lcoreid_t lc_id; 1635 struct rte_gro_param gro_param; 1636 uint32_t gso_types; 1637 1638 /* Configuration of logical cores. */ 1639 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1640 sizeof(struct fwd_lcore *) * nb_lcores, 1641 RTE_CACHE_LINE_SIZE); 1642 if (fwd_lcores == NULL) { 1643 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1644 "failed\n", nb_lcores); 1645 } 1646 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1647 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1648 sizeof(struct fwd_lcore), 1649 RTE_CACHE_LINE_SIZE); 1650 if (fwd_lcores[lc_id] == NULL) { 1651 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1652 "failed\n"); 1653 } 1654 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1655 } 1656 1657 RTE_ETH_FOREACH_DEV(pid) { 1658 uint32_t socket_id; 1659 1660 if (numa_support) { 1661 socket_id = port_numa[pid]; 1662 if (port_numa[pid] == NUMA_NO_CONFIG) { 1663 socket_id = rte_eth_dev_socket_id(pid); 1664 1665 /* 1666 * if socket_id is invalid, 1667 * set to the first available socket. 1668 */ 1669 if (check_socket_id(socket_id) < 0) 1670 socket_id = socket_ids[0]; 1671 } 1672 } else { 1673 socket_id = (socket_num == UMA_NO_CONFIG) ? 1674 0 : socket_num; 1675 } 1676 /* Apply default TxRx configuration for all ports */ 1677 init_config_port_offloads(pid, socket_id); 1678 } 1679 /* 1680 * Create pools of mbuf. 1681 * If NUMA support is disabled, create a single pool of mbuf in 1682 * socket 0 memory by default. 1683 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1684 * 1685 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1686 * nb_txd can be configured at run time. 1687 */ 1688 if (param_total_num_mbufs) 1689 nb_mbuf_per_pool = param_total_num_mbufs; 1690 else { 1691 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1692 (nb_lcores * mb_mempool_cache) + 1693 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1694 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1695 } 1696 1697 if (numa_support) { 1698 uint8_t i, j; 1699 1700 for (i = 0; i < num_sockets; i++) 1701 for (j = 0; j < mbuf_data_size_n; j++) 1702 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] = 1703 mbuf_pool_create(mbuf_data_size[j], 1704 nb_mbuf_per_pool, 1705 socket_ids[i], j); 1706 } else { 1707 uint8_t i; 1708 1709 for (i = 0; i < mbuf_data_size_n; i++) 1710 mempools[i] = mbuf_pool_create 1711 (mbuf_data_size[i], 1712 nb_mbuf_per_pool, 1713 socket_num == UMA_NO_CONFIG ? 1714 0 : socket_num, i); 1715 } 1716 1717 init_port_config(); 1718 1719 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1720 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1721 /* 1722 * Records which Mbuf pool to use by each logical core, if needed. 1723 */ 1724 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1725 mbp = mbuf_pool_find( 1726 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0); 1727 1728 if (mbp == NULL) 1729 mbp = mbuf_pool_find(0, 0); 1730 fwd_lcores[lc_id]->mbp = mbp; 1731 /* initialize GSO context */ 1732 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1733 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1734 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1735 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1736 RTE_ETHER_CRC_LEN; 1737 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1738 } 1739 1740 fwd_config_setup(); 1741 1742 /* create a gro context for each lcore */ 1743 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1744 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1745 gro_param.max_item_per_flow = MAX_PKT_BURST; 1746 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1747 gro_param.socket_id = rte_lcore_to_socket_id( 1748 fwd_lcores_cpuids[lc_id]); 1749 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1750 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1751 rte_exit(EXIT_FAILURE, 1752 "rte_gro_ctx_create() failed\n"); 1753 } 1754 } 1755 } 1756 1757 1758 void 1759 reconfig(portid_t new_port_id, unsigned socket_id) 1760 { 1761 /* Reconfiguration of Ethernet ports. */ 1762 init_config_port_offloads(new_port_id, socket_id); 1763 init_port_config(); 1764 } 1765 1766 1767 int 1768 init_fwd_streams(void) 1769 { 1770 portid_t pid; 1771 struct rte_port *port; 1772 streamid_t sm_id, nb_fwd_streams_new; 1773 queueid_t q; 1774 1775 /* set socket id according to numa or not */ 1776 RTE_ETH_FOREACH_DEV(pid) { 1777 port = &ports[pid]; 1778 if (nb_rxq > port->dev_info.max_rx_queues) { 1779 fprintf(stderr, 1780 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n", 1781 nb_rxq, port->dev_info.max_rx_queues); 1782 return -1; 1783 } 1784 if (nb_txq > port->dev_info.max_tx_queues) { 1785 fprintf(stderr, 1786 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n", 1787 nb_txq, port->dev_info.max_tx_queues); 1788 return -1; 1789 } 1790 if (numa_support) { 1791 if (port_numa[pid] != NUMA_NO_CONFIG) 1792 port->socket_id = port_numa[pid]; 1793 else { 1794 port->socket_id = rte_eth_dev_socket_id(pid); 1795 1796 /* 1797 * if socket_id is invalid, 1798 * set to the first available socket. 1799 */ 1800 if (check_socket_id(port->socket_id) < 0) 1801 port->socket_id = socket_ids[0]; 1802 } 1803 } 1804 else { 1805 if (socket_num == UMA_NO_CONFIG) 1806 port->socket_id = 0; 1807 else 1808 port->socket_id = socket_num; 1809 } 1810 } 1811 1812 q = RTE_MAX(nb_rxq, nb_txq); 1813 if (q == 0) { 1814 fprintf(stderr, 1815 "Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1816 return -1; 1817 } 1818 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1819 if (nb_fwd_streams_new == nb_fwd_streams) 1820 return 0; 1821 /* clear the old */ 1822 if (fwd_streams != NULL) { 1823 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1824 if (fwd_streams[sm_id] == NULL) 1825 continue; 1826 rte_free(fwd_streams[sm_id]); 1827 fwd_streams[sm_id] = NULL; 1828 } 1829 rte_free(fwd_streams); 1830 fwd_streams = NULL; 1831 } 1832 1833 /* init new */ 1834 nb_fwd_streams = nb_fwd_streams_new; 1835 if (nb_fwd_streams) { 1836 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1837 sizeof(struct fwd_stream *) * nb_fwd_streams, 1838 RTE_CACHE_LINE_SIZE); 1839 if (fwd_streams == NULL) 1840 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1841 " (struct fwd_stream *)) failed\n", 1842 nb_fwd_streams); 1843 1844 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1845 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1846 " struct fwd_stream", sizeof(struct fwd_stream), 1847 RTE_CACHE_LINE_SIZE); 1848 if (fwd_streams[sm_id] == NULL) 1849 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1850 "(struct fwd_stream) failed\n"); 1851 } 1852 } 1853 1854 return 0; 1855 } 1856 1857 static void 1858 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1859 { 1860 uint64_t total_burst, sburst; 1861 uint64_t nb_burst; 1862 uint64_t burst_stats[4]; 1863 uint16_t pktnb_stats[4]; 1864 uint16_t nb_pkt; 1865 int burst_percent[4], sburstp; 1866 int i; 1867 1868 /* 1869 * First compute the total number of packet bursts and the 1870 * two highest numbers of bursts of the same number of packets. 1871 */ 1872 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1873 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1874 1875 /* Show stats for 0 burst size always */ 1876 total_burst = pbs->pkt_burst_spread[0]; 1877 burst_stats[0] = pbs->pkt_burst_spread[0]; 1878 pktnb_stats[0] = 0; 1879 1880 /* Find the next 2 burst sizes with highest occurrences. */ 1881 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1882 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1883 1884 if (nb_burst == 0) 1885 continue; 1886 1887 total_burst += nb_burst; 1888 1889 if (nb_burst > burst_stats[1]) { 1890 burst_stats[2] = burst_stats[1]; 1891 pktnb_stats[2] = pktnb_stats[1]; 1892 burst_stats[1] = nb_burst; 1893 pktnb_stats[1] = nb_pkt; 1894 } else if (nb_burst > burst_stats[2]) { 1895 burst_stats[2] = nb_burst; 1896 pktnb_stats[2] = nb_pkt; 1897 } 1898 } 1899 if (total_burst == 0) 1900 return; 1901 1902 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1903 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1904 if (i == 3) { 1905 printf("%d%% of other]\n", 100 - sburstp); 1906 return; 1907 } 1908 1909 sburst += burst_stats[i]; 1910 if (sburst == total_burst) { 1911 printf("%d%% of %d pkts]\n", 1912 100 - sburstp, (int) pktnb_stats[i]); 1913 return; 1914 } 1915 1916 burst_percent[i] = 1917 (double)burst_stats[i] / total_burst * 100; 1918 printf("%d%% of %d pkts + ", 1919 burst_percent[i], (int) pktnb_stats[i]); 1920 sburstp += burst_percent[i]; 1921 } 1922 } 1923 1924 static void 1925 fwd_stream_stats_display(streamid_t stream_id) 1926 { 1927 struct fwd_stream *fs; 1928 static const char *fwd_top_stats_border = "-------"; 1929 1930 fs = fwd_streams[stream_id]; 1931 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1932 (fs->fwd_dropped == 0)) 1933 return; 1934 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1935 "TX Port=%2d/Queue=%2d %s\n", 1936 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1937 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1938 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1939 " TX-dropped: %-14"PRIu64, 1940 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1941 1942 /* if checksum mode */ 1943 if (cur_fwd_eng == &csum_fwd_engine) { 1944 printf(" RX- bad IP checksum: %-14"PRIu64 1945 " Rx- bad L4 checksum: %-14"PRIu64 1946 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1947 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1948 fs->rx_bad_outer_l4_csum); 1949 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n", 1950 fs->rx_bad_outer_ip_csum); 1951 } else { 1952 printf("\n"); 1953 } 1954 1955 if (record_burst_stats) { 1956 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1957 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1958 } 1959 } 1960 1961 void 1962 fwd_stats_display(void) 1963 { 1964 static const char *fwd_stats_border = "----------------------"; 1965 static const char *acc_stats_border = "+++++++++++++++"; 1966 struct { 1967 struct fwd_stream *rx_stream; 1968 struct fwd_stream *tx_stream; 1969 uint64_t tx_dropped; 1970 uint64_t rx_bad_ip_csum; 1971 uint64_t rx_bad_l4_csum; 1972 uint64_t rx_bad_outer_l4_csum; 1973 uint64_t rx_bad_outer_ip_csum; 1974 } ports_stats[RTE_MAX_ETHPORTS]; 1975 uint64_t total_rx_dropped = 0; 1976 uint64_t total_tx_dropped = 0; 1977 uint64_t total_rx_nombuf = 0; 1978 struct rte_eth_stats stats; 1979 uint64_t fwd_cycles = 0; 1980 uint64_t total_recv = 0; 1981 uint64_t total_xmit = 0; 1982 struct rte_port *port; 1983 streamid_t sm_id; 1984 portid_t pt_id; 1985 int i; 1986 1987 memset(ports_stats, 0, sizeof(ports_stats)); 1988 1989 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1990 struct fwd_stream *fs = fwd_streams[sm_id]; 1991 1992 if (cur_fwd_config.nb_fwd_streams > 1993 cur_fwd_config.nb_fwd_ports) { 1994 fwd_stream_stats_display(sm_id); 1995 } else { 1996 ports_stats[fs->tx_port].tx_stream = fs; 1997 ports_stats[fs->rx_port].rx_stream = fs; 1998 } 1999 2000 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 2001 2002 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 2003 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 2004 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 2005 fs->rx_bad_outer_l4_csum; 2006 ports_stats[fs->rx_port].rx_bad_outer_ip_csum += 2007 fs->rx_bad_outer_ip_csum; 2008 2009 if (record_core_cycles) 2010 fwd_cycles += fs->core_cycles; 2011 } 2012 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2013 pt_id = fwd_ports_ids[i]; 2014 port = &ports[pt_id]; 2015 2016 rte_eth_stats_get(pt_id, &stats); 2017 stats.ipackets -= port->stats.ipackets; 2018 stats.opackets -= port->stats.opackets; 2019 stats.ibytes -= port->stats.ibytes; 2020 stats.obytes -= port->stats.obytes; 2021 stats.imissed -= port->stats.imissed; 2022 stats.oerrors -= port->stats.oerrors; 2023 stats.rx_nombuf -= port->stats.rx_nombuf; 2024 2025 total_recv += stats.ipackets; 2026 total_xmit += stats.opackets; 2027 total_rx_dropped += stats.imissed; 2028 total_tx_dropped += ports_stats[pt_id].tx_dropped; 2029 total_tx_dropped += stats.oerrors; 2030 total_rx_nombuf += stats.rx_nombuf; 2031 2032 printf("\n %s Forward statistics for port %-2d %s\n", 2033 fwd_stats_border, pt_id, fwd_stats_border); 2034 2035 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64 2036 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed, 2037 stats.ipackets + stats.imissed); 2038 2039 if (cur_fwd_eng == &csum_fwd_engine) { 2040 printf(" Bad-ipcsum: %-14"PRIu64 2041 " Bad-l4csum: %-14"PRIu64 2042 "Bad-outer-l4csum: %-14"PRIu64"\n", 2043 ports_stats[pt_id].rx_bad_ip_csum, 2044 ports_stats[pt_id].rx_bad_l4_csum, 2045 ports_stats[pt_id].rx_bad_outer_l4_csum); 2046 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n", 2047 ports_stats[pt_id].rx_bad_outer_ip_csum); 2048 } 2049 if (stats.ierrors + stats.rx_nombuf > 0) { 2050 printf(" RX-error: %-"PRIu64"\n", stats.ierrors); 2051 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf); 2052 } 2053 2054 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64 2055 "TX-total: %-"PRIu64"\n", 2056 stats.opackets, ports_stats[pt_id].tx_dropped, 2057 stats.opackets + ports_stats[pt_id].tx_dropped); 2058 2059 if (record_burst_stats) { 2060 if (ports_stats[pt_id].rx_stream) 2061 pkt_burst_stats_display("RX", 2062 &ports_stats[pt_id].rx_stream->rx_burst_stats); 2063 if (ports_stats[pt_id].tx_stream) 2064 pkt_burst_stats_display("TX", 2065 &ports_stats[pt_id].tx_stream->tx_burst_stats); 2066 } 2067 2068 printf(" %s--------------------------------%s\n", 2069 fwd_stats_border, fwd_stats_border); 2070 } 2071 2072 printf("\n %s Accumulated forward statistics for all ports" 2073 "%s\n", 2074 acc_stats_border, acc_stats_border); 2075 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 2076 "%-"PRIu64"\n" 2077 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 2078 "%-"PRIu64"\n", 2079 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 2080 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 2081 if (total_rx_nombuf > 0) 2082 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 2083 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 2084 "%s\n", 2085 acc_stats_border, acc_stats_border); 2086 if (record_core_cycles) { 2087 #define CYC_PER_MHZ 1E6 2088 if (total_recv > 0 || total_xmit > 0) { 2089 uint64_t total_pkts = 0; 2090 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 || 2091 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0) 2092 total_pkts = total_xmit; 2093 else 2094 total_pkts = total_recv; 2095 2096 printf("\n CPU cycles/packet=%.2F (total cycles=" 2097 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64 2098 " MHz Clock\n", 2099 (double) fwd_cycles / total_pkts, 2100 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts, 2101 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 2102 } 2103 } 2104 } 2105 2106 void 2107 fwd_stats_reset(void) 2108 { 2109 streamid_t sm_id; 2110 portid_t pt_id; 2111 int i; 2112 2113 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2114 pt_id = fwd_ports_ids[i]; 2115 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 2116 } 2117 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 2118 struct fwd_stream *fs = fwd_streams[sm_id]; 2119 2120 fs->rx_packets = 0; 2121 fs->tx_packets = 0; 2122 fs->fwd_dropped = 0; 2123 fs->rx_bad_ip_csum = 0; 2124 fs->rx_bad_l4_csum = 0; 2125 fs->rx_bad_outer_l4_csum = 0; 2126 fs->rx_bad_outer_ip_csum = 0; 2127 2128 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2129 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2130 fs->core_cycles = 0; 2131 } 2132 } 2133 2134 static void 2135 flush_fwd_rx_queues(void) 2136 { 2137 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2138 portid_t rxp; 2139 portid_t port_id; 2140 queueid_t rxq; 2141 uint16_t nb_rx; 2142 uint16_t i; 2143 uint8_t j; 2144 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2145 uint64_t timer_period; 2146 2147 if (num_procs > 1) { 2148 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n"); 2149 return; 2150 } 2151 2152 /* convert to number of cycles */ 2153 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2154 2155 for (j = 0; j < 2; j++) { 2156 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2157 for (rxq = 0; rxq < nb_rxq; rxq++) { 2158 port_id = fwd_ports_ids[rxp]; 2159 /** 2160 * testpmd can stuck in the below do while loop 2161 * if rte_eth_rx_burst() always returns nonzero 2162 * packets. So timer is added to exit this loop 2163 * after 1sec timer expiry. 2164 */ 2165 prev_tsc = rte_rdtsc(); 2166 do { 2167 nb_rx = rte_eth_rx_burst(port_id, rxq, 2168 pkts_burst, MAX_PKT_BURST); 2169 for (i = 0; i < nb_rx; i++) 2170 rte_pktmbuf_free(pkts_burst[i]); 2171 2172 cur_tsc = rte_rdtsc(); 2173 diff_tsc = cur_tsc - prev_tsc; 2174 timer_tsc += diff_tsc; 2175 } while ((nb_rx > 0) && 2176 (timer_tsc < timer_period)); 2177 timer_tsc = 0; 2178 } 2179 } 2180 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2181 } 2182 } 2183 2184 static void 2185 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2186 { 2187 struct fwd_stream **fsm; 2188 streamid_t nb_fs; 2189 streamid_t sm_id; 2190 #ifdef RTE_LIB_BITRATESTATS 2191 uint64_t tics_per_1sec; 2192 uint64_t tics_datum; 2193 uint64_t tics_current; 2194 uint16_t i, cnt_ports; 2195 2196 cnt_ports = nb_ports; 2197 tics_datum = rte_rdtsc(); 2198 tics_per_1sec = rte_get_timer_hz(); 2199 #endif 2200 fsm = &fwd_streams[fc->stream_idx]; 2201 nb_fs = fc->stream_nb; 2202 do { 2203 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2204 (*pkt_fwd)(fsm[sm_id]); 2205 #ifdef RTE_LIB_BITRATESTATS 2206 if (bitrate_enabled != 0 && 2207 bitrate_lcore_id == rte_lcore_id()) { 2208 tics_current = rte_rdtsc(); 2209 if (tics_current - tics_datum >= tics_per_1sec) { 2210 /* Periodic bitrate calculation */ 2211 for (i = 0; i < cnt_ports; i++) 2212 rte_stats_bitrate_calc(bitrate_data, 2213 ports_ids[i]); 2214 tics_datum = tics_current; 2215 } 2216 } 2217 #endif 2218 #ifdef RTE_LIB_LATENCYSTATS 2219 if (latencystats_enabled != 0 && 2220 latencystats_lcore_id == rte_lcore_id()) 2221 rte_latencystats_update(); 2222 #endif 2223 2224 } while (! fc->stopped); 2225 } 2226 2227 static int 2228 start_pkt_forward_on_core(void *fwd_arg) 2229 { 2230 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2231 cur_fwd_config.fwd_eng->packet_fwd); 2232 return 0; 2233 } 2234 2235 /* 2236 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2237 * Used to start communication flows in network loopback test configurations. 2238 */ 2239 static int 2240 run_one_txonly_burst_on_core(void *fwd_arg) 2241 { 2242 struct fwd_lcore *fwd_lc; 2243 struct fwd_lcore tmp_lcore; 2244 2245 fwd_lc = (struct fwd_lcore *) fwd_arg; 2246 tmp_lcore = *fwd_lc; 2247 tmp_lcore.stopped = 1; 2248 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2249 return 0; 2250 } 2251 2252 /* 2253 * Launch packet forwarding: 2254 * - Setup per-port forwarding context. 2255 * - launch logical cores with their forwarding configuration. 2256 */ 2257 static void 2258 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2259 { 2260 unsigned int i; 2261 unsigned int lc_id; 2262 int diag; 2263 2264 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2265 lc_id = fwd_lcores_cpuids[i]; 2266 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2267 fwd_lcores[i]->stopped = 0; 2268 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2269 fwd_lcores[i], lc_id); 2270 if (diag != 0) 2271 fprintf(stderr, 2272 "launch lcore %u failed - diag=%d\n", 2273 lc_id, diag); 2274 } 2275 } 2276 } 2277 2278 /* 2279 * Launch packet forwarding configuration. 2280 */ 2281 void 2282 start_packet_forwarding(int with_tx_first) 2283 { 2284 port_fwd_begin_t port_fwd_begin; 2285 port_fwd_end_t port_fwd_end; 2286 unsigned int i; 2287 2288 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2289 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2290 2291 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2292 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2293 2294 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2295 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2296 (!nb_rxq || !nb_txq)) 2297 rte_exit(EXIT_FAILURE, 2298 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2299 cur_fwd_eng->fwd_mode_name); 2300 2301 if (all_ports_started() == 0) { 2302 fprintf(stderr, "Not all ports were started\n"); 2303 return; 2304 } 2305 if (test_done == 0) { 2306 fprintf(stderr, "Packet forwarding already started\n"); 2307 return; 2308 } 2309 2310 fwd_config_setup(); 2311 2312 pkt_fwd_config_display(&cur_fwd_config); 2313 if (!pkt_fwd_shared_rxq_check()) 2314 return; 2315 2316 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2317 if (port_fwd_begin != NULL) { 2318 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2319 if (port_fwd_begin(fwd_ports_ids[i])) { 2320 fprintf(stderr, 2321 "Packet forwarding is not ready\n"); 2322 return; 2323 } 2324 } 2325 } 2326 2327 if (with_tx_first) { 2328 port_fwd_begin = tx_only_engine.port_fwd_begin; 2329 if (port_fwd_begin != NULL) { 2330 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2331 if (port_fwd_begin(fwd_ports_ids[i])) { 2332 fprintf(stderr, 2333 "Packet forwarding is not ready\n"); 2334 return; 2335 } 2336 } 2337 } 2338 } 2339 2340 test_done = 0; 2341 2342 if(!no_flush_rx) 2343 flush_fwd_rx_queues(); 2344 2345 rxtx_config_display(); 2346 2347 fwd_stats_reset(); 2348 if (with_tx_first) { 2349 while (with_tx_first--) { 2350 launch_packet_forwarding( 2351 run_one_txonly_burst_on_core); 2352 rte_eal_mp_wait_lcore(); 2353 } 2354 port_fwd_end = tx_only_engine.port_fwd_end; 2355 if (port_fwd_end != NULL) { 2356 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2357 (*port_fwd_end)(fwd_ports_ids[i]); 2358 } 2359 } 2360 launch_packet_forwarding(start_pkt_forward_on_core); 2361 } 2362 2363 void 2364 stop_packet_forwarding(void) 2365 { 2366 port_fwd_end_t port_fwd_end; 2367 lcoreid_t lc_id; 2368 portid_t pt_id; 2369 int i; 2370 2371 if (test_done) { 2372 fprintf(stderr, "Packet forwarding not started\n"); 2373 return; 2374 } 2375 printf("Telling cores to stop..."); 2376 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2377 fwd_lcores[lc_id]->stopped = 1; 2378 printf("\nWaiting for lcores to finish...\n"); 2379 rte_eal_mp_wait_lcore(); 2380 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2381 if (port_fwd_end != NULL) { 2382 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2383 pt_id = fwd_ports_ids[i]; 2384 (*port_fwd_end)(pt_id); 2385 } 2386 } 2387 2388 fwd_stats_display(); 2389 2390 printf("\nDone.\n"); 2391 test_done = 1; 2392 } 2393 2394 void 2395 dev_set_link_up(portid_t pid) 2396 { 2397 if (rte_eth_dev_set_link_up(pid) < 0) 2398 fprintf(stderr, "\nSet link up fail.\n"); 2399 } 2400 2401 void 2402 dev_set_link_down(portid_t pid) 2403 { 2404 if (rte_eth_dev_set_link_down(pid) < 0) 2405 fprintf(stderr, "\nSet link down fail.\n"); 2406 } 2407 2408 static int 2409 all_ports_started(void) 2410 { 2411 portid_t pi; 2412 struct rte_port *port; 2413 2414 RTE_ETH_FOREACH_DEV(pi) { 2415 port = &ports[pi]; 2416 /* Check if there is a port which is not started */ 2417 if ((port->port_status != RTE_PORT_STARTED) && 2418 (port->slave_flag == 0)) 2419 return 0; 2420 } 2421 2422 /* No port is not started */ 2423 return 1; 2424 } 2425 2426 int 2427 port_is_stopped(portid_t port_id) 2428 { 2429 struct rte_port *port = &ports[port_id]; 2430 2431 if ((port->port_status != RTE_PORT_STOPPED) && 2432 (port->slave_flag == 0)) 2433 return 0; 2434 return 1; 2435 } 2436 2437 int 2438 all_ports_stopped(void) 2439 { 2440 portid_t pi; 2441 2442 RTE_ETH_FOREACH_DEV(pi) { 2443 if (!port_is_stopped(pi)) 2444 return 0; 2445 } 2446 2447 return 1; 2448 } 2449 2450 int 2451 port_is_started(portid_t port_id) 2452 { 2453 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2454 return 0; 2455 2456 if (ports[port_id].port_status != RTE_PORT_STARTED) 2457 return 0; 2458 2459 return 1; 2460 } 2461 2462 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2463 static int 2464 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 2465 { 2466 queueid_t qi; 2467 struct rte_eth_hairpin_conf hairpin_conf = { 2468 .peer_count = 1, 2469 }; 2470 int i; 2471 int diag; 2472 struct rte_port *port = &ports[pi]; 2473 uint16_t peer_rx_port = pi; 2474 uint16_t peer_tx_port = pi; 2475 uint32_t manual = 1; 2476 uint32_t tx_exp = hairpin_mode & 0x10; 2477 2478 if (!(hairpin_mode & 0xf)) { 2479 peer_rx_port = pi; 2480 peer_tx_port = pi; 2481 manual = 0; 2482 } else if (hairpin_mode & 0x1) { 2483 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 2484 RTE_ETH_DEV_NO_OWNER); 2485 if (peer_tx_port >= RTE_MAX_ETHPORTS) 2486 peer_tx_port = rte_eth_find_next_owned_by(0, 2487 RTE_ETH_DEV_NO_OWNER); 2488 if (p_pi != RTE_MAX_ETHPORTS) { 2489 peer_rx_port = p_pi; 2490 } else { 2491 uint16_t next_pi; 2492 2493 /* Last port will be the peer RX port of the first. */ 2494 RTE_ETH_FOREACH_DEV(next_pi) 2495 peer_rx_port = next_pi; 2496 } 2497 manual = 1; 2498 } else if (hairpin_mode & 0x2) { 2499 if (cnt_pi & 0x1) { 2500 peer_rx_port = p_pi; 2501 } else { 2502 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 2503 RTE_ETH_DEV_NO_OWNER); 2504 if (peer_rx_port >= RTE_MAX_ETHPORTS) 2505 peer_rx_port = pi; 2506 } 2507 peer_tx_port = peer_rx_port; 2508 manual = 1; 2509 } 2510 2511 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2512 hairpin_conf.peers[0].port = peer_rx_port; 2513 hairpin_conf.peers[0].queue = i + nb_rxq; 2514 hairpin_conf.manual_bind = !!manual; 2515 hairpin_conf.tx_explicit = !!tx_exp; 2516 diag = rte_eth_tx_hairpin_queue_setup 2517 (pi, qi, nb_txd, &hairpin_conf); 2518 i++; 2519 if (diag == 0) 2520 continue; 2521 2522 /* Fail to setup rx queue, return */ 2523 if (rte_atomic16_cmpset(&(port->port_status), 2524 RTE_PORT_HANDLING, 2525 RTE_PORT_STOPPED) == 0) 2526 fprintf(stderr, 2527 "Port %d can not be set back to stopped\n", pi); 2528 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2529 pi); 2530 /* try to reconfigure queues next time */ 2531 port->need_reconfig_queues = 1; 2532 return -1; 2533 } 2534 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2535 hairpin_conf.peers[0].port = peer_tx_port; 2536 hairpin_conf.peers[0].queue = i + nb_txq; 2537 hairpin_conf.manual_bind = !!manual; 2538 hairpin_conf.tx_explicit = !!tx_exp; 2539 diag = rte_eth_rx_hairpin_queue_setup 2540 (pi, qi, nb_rxd, &hairpin_conf); 2541 i++; 2542 if (diag == 0) 2543 continue; 2544 2545 /* Fail to setup rx queue, return */ 2546 if (rte_atomic16_cmpset(&(port->port_status), 2547 RTE_PORT_HANDLING, 2548 RTE_PORT_STOPPED) == 0) 2549 fprintf(stderr, 2550 "Port %d can not be set back to stopped\n", pi); 2551 fprintf(stderr, "Fail to configure port %d hairpin queues\n", 2552 pi); 2553 /* try to reconfigure queues next time */ 2554 port->need_reconfig_queues = 1; 2555 return -1; 2556 } 2557 return 0; 2558 } 2559 2560 /* Configure the Rx with optional split. */ 2561 int 2562 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, 2563 uint16_t nb_rx_desc, unsigned int socket_id, 2564 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) 2565 { 2566 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {}; 2567 unsigned int i, mp_n; 2568 int ret; 2569 2570 if (rx_pkt_nb_segs <= 1 || 2571 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) { 2572 rx_conf->rx_seg = NULL; 2573 rx_conf->rx_nseg = 0; 2574 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, 2575 nb_rx_desc, socket_id, 2576 rx_conf, mp); 2577 return ret; 2578 } 2579 for (i = 0; i < rx_pkt_nb_segs; i++) { 2580 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split; 2581 struct rte_mempool *mpx; 2582 /* 2583 * Use last valid pool for the segments with number 2584 * exceeding the pool index. 2585 */ 2586 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i; 2587 mpx = mbuf_pool_find(socket_id, mp_n); 2588 /* Handle zero as mbuf data buffer size. */ 2589 rx_seg->length = rx_pkt_seg_lengths[i] ? 2590 rx_pkt_seg_lengths[i] : 2591 mbuf_data_size[mp_n]; 2592 rx_seg->offset = i < rx_pkt_nb_offs ? 2593 rx_pkt_seg_offsets[i] : 0; 2594 rx_seg->mp = mpx ? mpx : mp; 2595 } 2596 rx_conf->rx_nseg = rx_pkt_nb_segs; 2597 rx_conf->rx_seg = rx_useg; 2598 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc, 2599 socket_id, rx_conf, NULL); 2600 rx_conf->rx_seg = NULL; 2601 rx_conf->rx_nseg = 0; 2602 return ret; 2603 } 2604 2605 static int 2606 alloc_xstats_display_info(portid_t pi) 2607 { 2608 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp; 2609 uint64_t **prev_values = &ports[pi].xstats_info.prev_values; 2610 uint64_t **curr_values = &ports[pi].xstats_info.curr_values; 2611 2612 if (xstats_display_num == 0) 2613 return 0; 2614 2615 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp)); 2616 if (*ids_supp == NULL) 2617 goto fail_ids_supp; 2618 2619 *prev_values = calloc(xstats_display_num, 2620 sizeof(**prev_values)); 2621 if (*prev_values == NULL) 2622 goto fail_prev_values; 2623 2624 *curr_values = calloc(xstats_display_num, 2625 sizeof(**curr_values)); 2626 if (*curr_values == NULL) 2627 goto fail_curr_values; 2628 2629 ports[pi].xstats_info.allocated = true; 2630 2631 return 0; 2632 2633 fail_curr_values: 2634 free(*prev_values); 2635 fail_prev_values: 2636 free(*ids_supp); 2637 fail_ids_supp: 2638 return -ENOMEM; 2639 } 2640 2641 static void 2642 free_xstats_display_info(portid_t pi) 2643 { 2644 if (!ports[pi].xstats_info.allocated) 2645 return; 2646 free(ports[pi].xstats_info.ids_supp); 2647 free(ports[pi].xstats_info.prev_values); 2648 free(ports[pi].xstats_info.curr_values); 2649 ports[pi].xstats_info.allocated = false; 2650 } 2651 2652 /** Fill helper structures for specified port to show extended statistics. */ 2653 static void 2654 fill_xstats_display_info_for_port(portid_t pi) 2655 { 2656 unsigned int stat, stat_supp; 2657 const char *xstat_name; 2658 struct rte_port *port; 2659 uint64_t *ids_supp; 2660 int rc; 2661 2662 if (xstats_display_num == 0) 2663 return; 2664 2665 if (pi == (portid_t)RTE_PORT_ALL) { 2666 fill_xstats_display_info(); 2667 return; 2668 } 2669 2670 port = &ports[pi]; 2671 if (port->port_status != RTE_PORT_STARTED) 2672 return; 2673 2674 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0) 2675 rte_exit(EXIT_FAILURE, 2676 "Failed to allocate xstats display memory\n"); 2677 2678 ids_supp = port->xstats_info.ids_supp; 2679 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) { 2680 xstat_name = xstats_display[stat].name; 2681 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name, 2682 ids_supp + stat_supp); 2683 if (rc != 0) { 2684 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n", 2685 xstat_name, pi, stat); 2686 continue; 2687 } 2688 stat_supp++; 2689 } 2690 2691 port->xstats_info.ids_supp_sz = stat_supp; 2692 } 2693 2694 /** Fill helper structures for all ports to show extended statistics. */ 2695 static void 2696 fill_xstats_display_info(void) 2697 { 2698 portid_t pi; 2699 2700 if (xstats_display_num == 0) 2701 return; 2702 2703 RTE_ETH_FOREACH_DEV(pi) 2704 fill_xstats_display_info_for_port(pi); 2705 } 2706 2707 int 2708 start_port(portid_t pid) 2709 { 2710 int diag, need_check_link_status = -1; 2711 portid_t pi; 2712 portid_t p_pi = RTE_MAX_ETHPORTS; 2713 portid_t pl[RTE_MAX_ETHPORTS]; 2714 portid_t peer_pl[RTE_MAX_ETHPORTS]; 2715 uint16_t cnt_pi = 0; 2716 uint16_t cfg_pi = 0; 2717 int peer_pi; 2718 queueid_t qi; 2719 struct rte_port *port; 2720 struct rte_eth_hairpin_cap cap; 2721 2722 if (port_id_is_invalid(pid, ENABLED_WARN)) 2723 return 0; 2724 2725 RTE_ETH_FOREACH_DEV(pi) { 2726 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2727 continue; 2728 2729 need_check_link_status = 0; 2730 port = &ports[pi]; 2731 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2732 RTE_PORT_HANDLING) == 0) { 2733 fprintf(stderr, "Port %d is now not stopped\n", pi); 2734 continue; 2735 } 2736 2737 if (port->need_reconfig > 0) { 2738 struct rte_eth_conf dev_conf; 2739 int k; 2740 2741 port->need_reconfig = 0; 2742 2743 if (flow_isolate_all) { 2744 int ret = port_flow_isolate(pi, 1); 2745 if (ret) { 2746 fprintf(stderr, 2747 "Failed to apply isolated mode on port %d\n", 2748 pi); 2749 return -1; 2750 } 2751 } 2752 configure_rxtx_dump_callbacks(0); 2753 printf("Configuring Port %d (socket %u)\n", pi, 2754 port->socket_id); 2755 if (nb_hairpinq > 0 && 2756 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2757 fprintf(stderr, 2758 "Port %d doesn't support hairpin queues\n", 2759 pi); 2760 return -1; 2761 } 2762 2763 /* configure port */ 2764 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq, 2765 nb_txq + nb_hairpinq, 2766 &(port->dev_conf)); 2767 if (diag != 0) { 2768 if (rte_atomic16_cmpset(&(port->port_status), 2769 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2770 fprintf(stderr, 2771 "Port %d can not be set back to stopped\n", 2772 pi); 2773 fprintf(stderr, "Fail to configure port %d\n", 2774 pi); 2775 /* try to reconfigure port next time */ 2776 port->need_reconfig = 1; 2777 return -1; 2778 } 2779 /* get device configuration*/ 2780 if (0 != 2781 eth_dev_conf_get_print_err(pi, &dev_conf)) { 2782 fprintf(stderr, 2783 "port %d can not get device configuration\n", 2784 pi); 2785 return -1; 2786 } 2787 /* Apply Rx offloads configuration */ 2788 if (dev_conf.rxmode.offloads != 2789 port->dev_conf.rxmode.offloads) { 2790 port->dev_conf.rxmode.offloads |= 2791 dev_conf.rxmode.offloads; 2792 for (k = 0; 2793 k < port->dev_info.max_rx_queues; 2794 k++) 2795 port->rx_conf[k].offloads |= 2796 dev_conf.rxmode.offloads; 2797 } 2798 /* Apply Tx offloads configuration */ 2799 if (dev_conf.txmode.offloads != 2800 port->dev_conf.txmode.offloads) { 2801 port->dev_conf.txmode.offloads |= 2802 dev_conf.txmode.offloads; 2803 for (k = 0; 2804 k < port->dev_info.max_tx_queues; 2805 k++) 2806 port->tx_conf[k].offloads |= 2807 dev_conf.txmode.offloads; 2808 } 2809 } 2810 if (port->need_reconfig_queues > 0 && is_proc_primary()) { 2811 port->need_reconfig_queues = 0; 2812 /* setup tx queues */ 2813 for (qi = 0; qi < nb_txq; qi++) { 2814 if ((numa_support) && 2815 (txring_numa[pi] != NUMA_NO_CONFIG)) 2816 diag = rte_eth_tx_queue_setup(pi, qi, 2817 port->nb_tx_desc[qi], 2818 txring_numa[pi], 2819 &(port->tx_conf[qi])); 2820 else 2821 diag = rte_eth_tx_queue_setup(pi, qi, 2822 port->nb_tx_desc[qi], 2823 port->socket_id, 2824 &(port->tx_conf[qi])); 2825 2826 if (diag == 0) 2827 continue; 2828 2829 /* Fail to setup tx queue, return */ 2830 if (rte_atomic16_cmpset(&(port->port_status), 2831 RTE_PORT_HANDLING, 2832 RTE_PORT_STOPPED) == 0) 2833 fprintf(stderr, 2834 "Port %d can not be set back to stopped\n", 2835 pi); 2836 fprintf(stderr, 2837 "Fail to configure port %d tx queues\n", 2838 pi); 2839 /* try to reconfigure queues next time */ 2840 port->need_reconfig_queues = 1; 2841 return -1; 2842 } 2843 for (qi = 0; qi < nb_rxq; qi++) { 2844 /* setup rx queues */ 2845 if ((numa_support) && 2846 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2847 struct rte_mempool * mp = 2848 mbuf_pool_find 2849 (rxring_numa[pi], 0); 2850 if (mp == NULL) { 2851 fprintf(stderr, 2852 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2853 rxring_numa[pi]); 2854 return -1; 2855 } 2856 2857 diag = rx_queue_setup(pi, qi, 2858 port->nb_rx_desc[qi], 2859 rxring_numa[pi], 2860 &(port->rx_conf[qi]), 2861 mp); 2862 } else { 2863 struct rte_mempool *mp = 2864 mbuf_pool_find 2865 (port->socket_id, 0); 2866 if (mp == NULL) { 2867 fprintf(stderr, 2868 "Failed to setup RX queue: No mempool allocation on the socket %d\n", 2869 port->socket_id); 2870 return -1; 2871 } 2872 diag = rx_queue_setup(pi, qi, 2873 port->nb_rx_desc[qi], 2874 port->socket_id, 2875 &(port->rx_conf[qi]), 2876 mp); 2877 } 2878 if (diag == 0) 2879 continue; 2880 2881 /* Fail to setup rx queue, return */ 2882 if (rte_atomic16_cmpset(&(port->port_status), 2883 RTE_PORT_HANDLING, 2884 RTE_PORT_STOPPED) == 0) 2885 fprintf(stderr, 2886 "Port %d can not be set back to stopped\n", 2887 pi); 2888 fprintf(stderr, 2889 "Fail to configure port %d rx queues\n", 2890 pi); 2891 /* try to reconfigure queues next time */ 2892 port->need_reconfig_queues = 1; 2893 return -1; 2894 } 2895 /* setup hairpin queues */ 2896 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) 2897 return -1; 2898 } 2899 configure_rxtx_dump_callbacks(verbose_level); 2900 if (clear_ptypes) { 2901 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2902 NULL, 0); 2903 if (diag < 0) 2904 fprintf(stderr, 2905 "Port %d: Failed to disable Ptype parsing\n", 2906 pi); 2907 } 2908 2909 p_pi = pi; 2910 cnt_pi++; 2911 2912 /* start port */ 2913 diag = eth_dev_start_mp(pi); 2914 if (diag < 0) { 2915 fprintf(stderr, "Fail to start port %d: %s\n", 2916 pi, rte_strerror(-diag)); 2917 2918 /* Fail to setup rx queue, return */ 2919 if (rte_atomic16_cmpset(&(port->port_status), 2920 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2921 fprintf(stderr, 2922 "Port %d can not be set back to stopped\n", 2923 pi); 2924 continue; 2925 } 2926 2927 if (rte_atomic16_cmpset(&(port->port_status), 2928 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2929 fprintf(stderr, "Port %d can not be set into started\n", 2930 pi); 2931 2932 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0) 2933 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi, 2934 RTE_ETHER_ADDR_BYTES(&port->eth_addr)); 2935 2936 /* at least one port started, need checking link status */ 2937 need_check_link_status = 1; 2938 2939 pl[cfg_pi++] = pi; 2940 } 2941 2942 if (need_check_link_status == 1 && !no_link_check) 2943 check_all_ports_link_status(RTE_PORT_ALL); 2944 else if (need_check_link_status == 0) 2945 fprintf(stderr, "Please stop the ports first\n"); 2946 2947 if (hairpin_mode & 0xf) { 2948 uint16_t i; 2949 int j; 2950 2951 /* bind all started hairpin ports */ 2952 for (i = 0; i < cfg_pi; i++) { 2953 pi = pl[i]; 2954 /* bind current Tx to all peer Rx */ 2955 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2956 RTE_MAX_ETHPORTS, 1); 2957 if (peer_pi < 0) 2958 return peer_pi; 2959 for (j = 0; j < peer_pi; j++) { 2960 if (!port_is_started(peer_pl[j])) 2961 continue; 2962 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 2963 if (diag < 0) { 2964 fprintf(stderr, 2965 "Error during binding hairpin Tx port %u to %u: %s\n", 2966 pi, peer_pl[j], 2967 rte_strerror(-diag)); 2968 return -1; 2969 } 2970 } 2971 /* bind all peer Tx to current Rx */ 2972 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 2973 RTE_MAX_ETHPORTS, 0); 2974 if (peer_pi < 0) 2975 return peer_pi; 2976 for (j = 0; j < peer_pi; j++) { 2977 if (!port_is_started(peer_pl[j])) 2978 continue; 2979 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 2980 if (diag < 0) { 2981 fprintf(stderr, 2982 "Error during binding hairpin Tx port %u to %u: %s\n", 2983 peer_pl[j], pi, 2984 rte_strerror(-diag)); 2985 return -1; 2986 } 2987 } 2988 } 2989 } 2990 2991 fill_xstats_display_info_for_port(pid); 2992 2993 printf("Done\n"); 2994 return 0; 2995 } 2996 2997 void 2998 stop_port(portid_t pid) 2999 { 3000 portid_t pi; 3001 struct rte_port *port; 3002 int need_check_link_status = 0; 3003 portid_t peer_pl[RTE_MAX_ETHPORTS]; 3004 int peer_pi; 3005 3006 if (port_id_is_invalid(pid, ENABLED_WARN)) 3007 return; 3008 3009 printf("Stopping ports...\n"); 3010 3011 RTE_ETH_FOREACH_DEV(pi) { 3012 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3013 continue; 3014 3015 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3016 fprintf(stderr, 3017 "Please remove port %d from forwarding configuration.\n", 3018 pi); 3019 continue; 3020 } 3021 3022 if (port_is_bonding_slave(pi)) { 3023 fprintf(stderr, 3024 "Please remove port %d from bonded device.\n", 3025 pi); 3026 continue; 3027 } 3028 3029 port = &ports[pi]; 3030 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 3031 RTE_PORT_HANDLING) == 0) 3032 continue; 3033 3034 if (hairpin_mode & 0xf) { 3035 int j; 3036 3037 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); 3038 /* unbind all peer Tx from current Rx */ 3039 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 3040 RTE_MAX_ETHPORTS, 0); 3041 if (peer_pi < 0) 3042 continue; 3043 for (j = 0; j < peer_pi; j++) { 3044 if (!port_is_started(peer_pl[j])) 3045 continue; 3046 rte_eth_hairpin_unbind(peer_pl[j], pi); 3047 } 3048 } 3049 3050 if (port->flow_list) 3051 port_flow_flush(pi); 3052 3053 if (eth_dev_stop_mp(pi) != 0) 3054 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", 3055 pi); 3056 3057 if (rte_atomic16_cmpset(&(port->port_status), 3058 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 3059 fprintf(stderr, "Port %d can not be set into stopped\n", 3060 pi); 3061 need_check_link_status = 1; 3062 } 3063 if (need_check_link_status && !no_link_check) 3064 check_all_ports_link_status(RTE_PORT_ALL); 3065 3066 printf("Done\n"); 3067 } 3068 3069 static void 3070 remove_invalid_ports_in(portid_t *array, portid_t *total) 3071 { 3072 portid_t i; 3073 portid_t new_total = 0; 3074 3075 for (i = 0; i < *total; i++) 3076 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 3077 array[new_total] = array[i]; 3078 new_total++; 3079 } 3080 *total = new_total; 3081 } 3082 3083 static void 3084 remove_invalid_ports(void) 3085 { 3086 remove_invalid_ports_in(ports_ids, &nb_ports); 3087 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 3088 nb_cfg_ports = nb_fwd_ports; 3089 } 3090 3091 void 3092 close_port(portid_t pid) 3093 { 3094 portid_t pi; 3095 struct rte_port *port; 3096 3097 if (port_id_is_invalid(pid, ENABLED_WARN)) 3098 return; 3099 3100 printf("Closing ports...\n"); 3101 3102 RTE_ETH_FOREACH_DEV(pi) { 3103 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3104 continue; 3105 3106 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3107 fprintf(stderr, 3108 "Please remove port %d from forwarding configuration.\n", 3109 pi); 3110 continue; 3111 } 3112 3113 if (port_is_bonding_slave(pi)) { 3114 fprintf(stderr, 3115 "Please remove port %d from bonded device.\n", 3116 pi); 3117 continue; 3118 } 3119 3120 port = &ports[pi]; 3121 if (rte_atomic16_cmpset(&(port->port_status), 3122 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 3123 fprintf(stderr, "Port %d is already closed\n", pi); 3124 continue; 3125 } 3126 3127 if (is_proc_primary()) { 3128 port_flow_flush(pi); 3129 port_flex_item_flush(pi); 3130 rte_eth_dev_close(pi); 3131 } 3132 3133 free_xstats_display_info(pi); 3134 } 3135 3136 remove_invalid_ports(); 3137 printf("Done\n"); 3138 } 3139 3140 void 3141 reset_port(portid_t pid) 3142 { 3143 int diag; 3144 portid_t pi; 3145 struct rte_port *port; 3146 3147 if (port_id_is_invalid(pid, ENABLED_WARN)) 3148 return; 3149 3150 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 3151 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 3152 fprintf(stderr, 3153 "Can not reset port(s), please stop port(s) first.\n"); 3154 return; 3155 } 3156 3157 printf("Resetting ports...\n"); 3158 3159 RTE_ETH_FOREACH_DEV(pi) { 3160 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 3161 continue; 3162 3163 if (port_is_forwarding(pi) != 0 && test_done == 0) { 3164 fprintf(stderr, 3165 "Please remove port %d from forwarding configuration.\n", 3166 pi); 3167 continue; 3168 } 3169 3170 if (port_is_bonding_slave(pi)) { 3171 fprintf(stderr, 3172 "Please remove port %d from bonded device.\n", 3173 pi); 3174 continue; 3175 } 3176 3177 diag = rte_eth_dev_reset(pi); 3178 if (diag == 0) { 3179 port = &ports[pi]; 3180 port->need_reconfig = 1; 3181 port->need_reconfig_queues = 1; 3182 } else { 3183 fprintf(stderr, "Failed to reset port %d. diag=%d\n", 3184 pi, diag); 3185 } 3186 } 3187 3188 printf("Done\n"); 3189 } 3190 3191 void 3192 attach_port(char *identifier) 3193 { 3194 portid_t pi; 3195 struct rte_dev_iterator iterator; 3196 3197 printf("Attaching a new port...\n"); 3198 3199 if (identifier == NULL) { 3200 fprintf(stderr, "Invalid parameters are specified\n"); 3201 return; 3202 } 3203 3204 if (rte_dev_probe(identifier) < 0) { 3205 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 3206 return; 3207 } 3208 3209 /* first attach mode: event */ 3210 if (setup_on_probe_event) { 3211 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 3212 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 3213 if (ports[pi].port_status == RTE_PORT_HANDLING && 3214 ports[pi].need_setup != 0) 3215 setup_attached_port(pi); 3216 return; 3217 } 3218 3219 /* second attach mode: iterator */ 3220 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 3221 /* setup ports matching the devargs used for probing */ 3222 if (port_is_forwarding(pi)) 3223 continue; /* port was already attached before */ 3224 setup_attached_port(pi); 3225 } 3226 } 3227 3228 static void 3229 setup_attached_port(portid_t pi) 3230 { 3231 unsigned int socket_id; 3232 int ret; 3233 3234 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 3235 /* if socket_id is invalid, set to the first available socket. */ 3236 if (check_socket_id(socket_id) < 0) 3237 socket_id = socket_ids[0]; 3238 reconfig(pi, socket_id); 3239 ret = rte_eth_promiscuous_enable(pi); 3240 if (ret != 0) 3241 fprintf(stderr, 3242 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 3243 pi, rte_strerror(-ret)); 3244 3245 ports_ids[nb_ports++] = pi; 3246 fwd_ports_ids[nb_fwd_ports++] = pi; 3247 nb_cfg_ports = nb_fwd_ports; 3248 ports[pi].need_setup = 0; 3249 ports[pi].port_status = RTE_PORT_STOPPED; 3250 3251 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 3252 printf("Done\n"); 3253 } 3254 3255 static void 3256 detach_device(struct rte_device *dev) 3257 { 3258 portid_t sibling; 3259 3260 if (dev == NULL) { 3261 fprintf(stderr, "Device already removed\n"); 3262 return; 3263 } 3264 3265 printf("Removing a device...\n"); 3266 3267 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 3268 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 3269 if (ports[sibling].port_status != RTE_PORT_STOPPED) { 3270 fprintf(stderr, "Port %u not stopped\n", 3271 sibling); 3272 return; 3273 } 3274 port_flow_flush(sibling); 3275 } 3276 } 3277 3278 if (rte_dev_remove(dev) < 0) { 3279 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 3280 return; 3281 } 3282 remove_invalid_ports(); 3283 3284 printf("Device is detached\n"); 3285 printf("Now total ports is %d\n", nb_ports); 3286 printf("Done\n"); 3287 return; 3288 } 3289 3290 void 3291 detach_port_device(portid_t port_id) 3292 { 3293 int ret; 3294 struct rte_eth_dev_info dev_info; 3295 3296 if (port_id_is_invalid(port_id, ENABLED_WARN)) 3297 return; 3298 3299 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3300 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3301 fprintf(stderr, "Port not stopped\n"); 3302 return; 3303 } 3304 fprintf(stderr, "Port was not closed\n"); 3305 } 3306 3307 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3308 if (ret != 0) { 3309 TESTPMD_LOG(ERR, 3310 "Failed to get device info for port %d, not detaching\n", 3311 port_id); 3312 return; 3313 } 3314 detach_device(dev_info.device); 3315 } 3316 3317 void 3318 detach_devargs(char *identifier) 3319 { 3320 struct rte_dev_iterator iterator; 3321 struct rte_devargs da; 3322 portid_t port_id; 3323 3324 printf("Removing a device...\n"); 3325 3326 memset(&da, 0, sizeof(da)); 3327 if (rte_devargs_parsef(&da, "%s", identifier)) { 3328 fprintf(stderr, "cannot parse identifier\n"); 3329 return; 3330 } 3331 3332 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 3333 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 3334 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 3335 fprintf(stderr, "Port %u not stopped\n", 3336 port_id); 3337 rte_eth_iterator_cleanup(&iterator); 3338 rte_devargs_reset(&da); 3339 return; 3340 } 3341 port_flow_flush(port_id); 3342 } 3343 } 3344 3345 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 3346 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 3347 da.name, da.bus->name); 3348 rte_devargs_reset(&da); 3349 return; 3350 } 3351 3352 remove_invalid_ports(); 3353 3354 printf("Device %s is detached\n", identifier); 3355 printf("Now total ports is %d\n", nb_ports); 3356 printf("Done\n"); 3357 rte_devargs_reset(&da); 3358 } 3359 3360 void 3361 pmd_test_exit(void) 3362 { 3363 portid_t pt_id; 3364 unsigned int i; 3365 int ret; 3366 3367 if (test_done == 0) 3368 stop_packet_forwarding(); 3369 3370 #ifndef RTE_EXEC_ENV_WINDOWS 3371 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3372 if (mempools[i]) { 3373 if (mp_alloc_type == MP_ALLOC_ANON) 3374 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 3375 NULL); 3376 } 3377 } 3378 #endif 3379 if (ports != NULL) { 3380 no_link_check = 1; 3381 RTE_ETH_FOREACH_DEV(pt_id) { 3382 printf("\nStopping port %d...\n", pt_id); 3383 fflush(stdout); 3384 stop_port(pt_id); 3385 } 3386 RTE_ETH_FOREACH_DEV(pt_id) { 3387 printf("\nShutting down port %d...\n", pt_id); 3388 fflush(stdout); 3389 close_port(pt_id); 3390 } 3391 } 3392 3393 if (hot_plug) { 3394 ret = rte_dev_event_monitor_stop(); 3395 if (ret) { 3396 RTE_LOG(ERR, EAL, 3397 "fail to stop device event monitor."); 3398 return; 3399 } 3400 3401 ret = rte_dev_event_callback_unregister(NULL, 3402 dev_event_callback, NULL); 3403 if (ret < 0) { 3404 RTE_LOG(ERR, EAL, 3405 "fail to unregister device event callback.\n"); 3406 return; 3407 } 3408 3409 ret = rte_dev_hotplug_handle_disable(); 3410 if (ret) { 3411 RTE_LOG(ERR, EAL, 3412 "fail to disable hotplug handling.\n"); 3413 return; 3414 } 3415 } 3416 for (i = 0 ; i < RTE_DIM(mempools) ; i++) { 3417 if (mempools[i]) 3418 mempool_free_mp(mempools[i]); 3419 } 3420 free(xstats_display); 3421 3422 printf("\nBye...\n"); 3423 } 3424 3425 typedef void (*cmd_func_t)(void); 3426 struct pmd_test_command { 3427 const char *cmd_name; 3428 cmd_func_t cmd_func; 3429 }; 3430 3431 /* Check the link status of all ports in up to 9s, and print them finally */ 3432 static void 3433 check_all_ports_link_status(uint32_t port_mask) 3434 { 3435 #define CHECK_INTERVAL 100 /* 100ms */ 3436 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3437 portid_t portid; 3438 uint8_t count, all_ports_up, print_flag = 0; 3439 struct rte_eth_link link; 3440 int ret; 3441 char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 3442 3443 printf("Checking link statuses...\n"); 3444 fflush(stdout); 3445 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3446 all_ports_up = 1; 3447 RTE_ETH_FOREACH_DEV(portid) { 3448 if ((port_mask & (1 << portid)) == 0) 3449 continue; 3450 memset(&link, 0, sizeof(link)); 3451 ret = rte_eth_link_get_nowait(portid, &link); 3452 if (ret < 0) { 3453 all_ports_up = 0; 3454 if (print_flag == 1) 3455 fprintf(stderr, 3456 "Port %u link get failed: %s\n", 3457 portid, rte_strerror(-ret)); 3458 continue; 3459 } 3460 /* print link status if flag set */ 3461 if (print_flag == 1) { 3462 rte_eth_link_to_str(link_status, 3463 sizeof(link_status), &link); 3464 printf("Port %d %s\n", portid, link_status); 3465 continue; 3466 } 3467 /* clear all_ports_up flag if any link down */ 3468 if (link.link_status == ETH_LINK_DOWN) { 3469 all_ports_up = 0; 3470 break; 3471 } 3472 } 3473 /* after finally printing all link status, get out */ 3474 if (print_flag == 1) 3475 break; 3476 3477 if (all_ports_up == 0) { 3478 fflush(stdout); 3479 rte_delay_ms(CHECK_INTERVAL); 3480 } 3481 3482 /* set the print_flag if all ports up or timeout */ 3483 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3484 print_flag = 1; 3485 } 3486 3487 if (lsc_interrupt) 3488 break; 3489 } 3490 } 3491 3492 static void 3493 rmv_port_callback(void *arg) 3494 { 3495 int need_to_start = 0; 3496 int org_no_link_check = no_link_check; 3497 portid_t port_id = (intptr_t)arg; 3498 struct rte_eth_dev_info dev_info; 3499 int ret; 3500 3501 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3502 3503 if (!test_done && port_is_forwarding(port_id)) { 3504 need_to_start = 1; 3505 stop_packet_forwarding(); 3506 } 3507 no_link_check = 1; 3508 stop_port(port_id); 3509 no_link_check = org_no_link_check; 3510 3511 ret = eth_dev_info_get_print_err(port_id, &dev_info); 3512 if (ret != 0) 3513 TESTPMD_LOG(ERR, 3514 "Failed to get device info for port %d, not detaching\n", 3515 port_id); 3516 else { 3517 struct rte_device *device = dev_info.device; 3518 close_port(port_id); 3519 detach_device(device); /* might be already removed or have more ports */ 3520 } 3521 if (need_to_start) 3522 start_packet_forwarding(0); 3523 } 3524 3525 /* This function is used by the interrupt thread */ 3526 static int 3527 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3528 void *ret_param) 3529 { 3530 RTE_SET_USED(param); 3531 RTE_SET_USED(ret_param); 3532 3533 if (type >= RTE_ETH_EVENT_MAX) { 3534 fprintf(stderr, 3535 "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3536 port_id, __func__, type); 3537 fflush(stderr); 3538 } else if (event_print_mask & (UINT32_C(1) << type)) { 3539 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3540 eth_event_desc[type]); 3541 fflush(stdout); 3542 } 3543 3544 switch (type) { 3545 case RTE_ETH_EVENT_NEW: 3546 ports[port_id].need_setup = 1; 3547 ports[port_id].port_status = RTE_PORT_HANDLING; 3548 break; 3549 case RTE_ETH_EVENT_INTR_RMV: 3550 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3551 break; 3552 if (rte_eal_alarm_set(100000, 3553 rmv_port_callback, (void *)(intptr_t)port_id)) 3554 fprintf(stderr, 3555 "Could not set up deferred device removal\n"); 3556 break; 3557 case RTE_ETH_EVENT_DESTROY: 3558 ports[port_id].port_status = RTE_PORT_CLOSED; 3559 printf("Port %u is closed\n", port_id); 3560 break; 3561 default: 3562 break; 3563 } 3564 return 0; 3565 } 3566 3567 static int 3568 register_eth_event_callback(void) 3569 { 3570 int ret; 3571 enum rte_eth_event_type event; 3572 3573 for (event = RTE_ETH_EVENT_UNKNOWN; 3574 event < RTE_ETH_EVENT_MAX; event++) { 3575 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3576 event, 3577 eth_event_callback, 3578 NULL); 3579 if (ret != 0) { 3580 TESTPMD_LOG(ERR, "Failed to register callback for " 3581 "%s event\n", eth_event_desc[event]); 3582 return -1; 3583 } 3584 } 3585 3586 return 0; 3587 } 3588 3589 /* This function is used by the interrupt thread */ 3590 static void 3591 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3592 __rte_unused void *arg) 3593 { 3594 uint16_t port_id; 3595 int ret; 3596 3597 if (type >= RTE_DEV_EVENT_MAX) { 3598 fprintf(stderr, "%s called upon invalid event %d\n", 3599 __func__, type); 3600 fflush(stderr); 3601 } 3602 3603 switch (type) { 3604 case RTE_DEV_EVENT_REMOVE: 3605 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3606 device_name); 3607 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3608 if (ret) { 3609 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3610 device_name); 3611 return; 3612 } 3613 /* 3614 * Because the user's callback is invoked in eal interrupt 3615 * callback, the interrupt callback need to be finished before 3616 * it can be unregistered when detaching device. So finish 3617 * callback soon and use a deferred removal to detach device 3618 * is need. It is a workaround, once the device detaching be 3619 * moved into the eal in the future, the deferred removal could 3620 * be deleted. 3621 */ 3622 if (rte_eal_alarm_set(100000, 3623 rmv_port_callback, (void *)(intptr_t)port_id)) 3624 RTE_LOG(ERR, EAL, 3625 "Could not set up deferred device removal\n"); 3626 break; 3627 case RTE_DEV_EVENT_ADD: 3628 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3629 device_name); 3630 /* TODO: After finish kernel driver binding, 3631 * begin to attach port. 3632 */ 3633 break; 3634 default: 3635 break; 3636 } 3637 } 3638 3639 static void 3640 rxtx_port_config(portid_t pid) 3641 { 3642 uint16_t qid; 3643 uint64_t offloads; 3644 struct rte_port *port = &ports[pid]; 3645 3646 for (qid = 0; qid < nb_rxq; qid++) { 3647 offloads = port->rx_conf[qid].offloads; 3648 port->rx_conf[qid] = port->dev_info.default_rxconf; 3649 3650 if (rxq_share > 0 && 3651 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) { 3652 /* Non-zero share group to enable RxQ share. */ 3653 port->rx_conf[qid].share_group = pid / rxq_share + 1; 3654 port->rx_conf[qid].share_qid = qid; /* Equal mapping. */ 3655 } 3656 3657 if (offloads != 0) 3658 port->rx_conf[qid].offloads = offloads; 3659 3660 /* Check if any Rx parameters have been passed */ 3661 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3662 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3663 3664 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3665 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3666 3667 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3668 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3669 3670 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3671 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3672 3673 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3674 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3675 3676 port->nb_rx_desc[qid] = nb_rxd; 3677 } 3678 3679 for (qid = 0; qid < nb_txq; qid++) { 3680 offloads = port->tx_conf[qid].offloads; 3681 port->tx_conf[qid] = port->dev_info.default_txconf; 3682 if (offloads != 0) 3683 port->tx_conf[qid].offloads = offloads; 3684 3685 /* Check if any Tx parameters have been passed */ 3686 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3687 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3688 3689 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3690 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3691 3692 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3693 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3694 3695 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3696 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3697 3698 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3699 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3700 3701 port->nb_tx_desc[qid] = nb_txd; 3702 } 3703 } 3704 3705 /* 3706 * Helper function to set MTU from frame size 3707 * 3708 * port->dev_info should be set before calling this function. 3709 * 3710 * return 0 on success, negative on error 3711 */ 3712 int 3713 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen) 3714 { 3715 struct rte_port *port = &ports[portid]; 3716 uint32_t eth_overhead; 3717 uint16_t mtu, new_mtu; 3718 3719 eth_overhead = get_eth_overhead(&port->dev_info); 3720 3721 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) { 3722 printf("Failed to get MTU for port %u\n", portid); 3723 return -1; 3724 } 3725 3726 new_mtu = max_rx_pktlen - eth_overhead; 3727 3728 if (mtu == new_mtu) 3729 return 0; 3730 3731 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) { 3732 fprintf(stderr, 3733 "Failed to set MTU to %u for port %u\n", 3734 new_mtu, portid); 3735 return -1; 3736 } 3737 3738 port->dev_conf.rxmode.mtu = new_mtu; 3739 3740 return 0; 3741 } 3742 3743 void 3744 init_port_config(void) 3745 { 3746 portid_t pid; 3747 struct rte_port *port; 3748 int ret, i; 3749 3750 RTE_ETH_FOREACH_DEV(pid) { 3751 port = &ports[pid]; 3752 port->dev_conf.fdir_conf = fdir_conf; 3753 3754 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3755 if (ret != 0) 3756 return; 3757 3758 if (nb_rxq > 1) { 3759 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3760 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3761 rss_hf & port->dev_info.flow_type_rss_offloads; 3762 } else { 3763 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3764 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3765 } 3766 3767 if (port->dcb_flag == 0) { 3768 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) { 3769 port->dev_conf.rxmode.mq_mode = 3770 (enum rte_eth_rx_mq_mode) 3771 (rx_mq_mode & ETH_MQ_RX_RSS); 3772 } else { 3773 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3774 port->dev_conf.rxmode.offloads &= 3775 ~DEV_RX_OFFLOAD_RSS_HASH; 3776 3777 for (i = 0; 3778 i < port->dev_info.nb_rx_queues; 3779 i++) 3780 port->rx_conf[i].offloads &= 3781 ~DEV_RX_OFFLOAD_RSS_HASH; 3782 } 3783 } 3784 3785 rxtx_port_config(pid); 3786 3787 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3788 if (ret != 0) 3789 return; 3790 3791 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS 3792 rte_pmd_ixgbe_bypass_init(pid); 3793 #endif 3794 3795 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC)) 3796 port->dev_conf.intr_conf.lsc = 1; 3797 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV)) 3798 port->dev_conf.intr_conf.rmv = 1; 3799 } 3800 } 3801 3802 void set_port_slave_flag(portid_t slave_pid) 3803 { 3804 struct rte_port *port; 3805 3806 port = &ports[slave_pid]; 3807 port->slave_flag = 1; 3808 } 3809 3810 void clear_port_slave_flag(portid_t slave_pid) 3811 { 3812 struct rte_port *port; 3813 3814 port = &ports[slave_pid]; 3815 port->slave_flag = 0; 3816 } 3817 3818 uint8_t port_is_bonding_slave(portid_t slave_pid) 3819 { 3820 struct rte_port *port; 3821 struct rte_eth_dev_info dev_info; 3822 int ret; 3823 3824 port = &ports[slave_pid]; 3825 ret = eth_dev_info_get_print_err(slave_pid, &dev_info); 3826 if (ret != 0) { 3827 TESTPMD_LOG(ERR, 3828 "Failed to get device info for port id %d," 3829 "cannot determine if the port is a bonded slave", 3830 slave_pid); 3831 return 0; 3832 } 3833 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3834 return 1; 3835 return 0; 3836 } 3837 3838 const uint16_t vlan_tags[] = { 3839 0, 1, 2, 3, 4, 5, 6, 7, 3840 8, 9, 10, 11, 12, 13, 14, 15, 3841 16, 17, 18, 19, 20, 21, 22, 23, 3842 24, 25, 26, 27, 28, 29, 30, 31 3843 }; 3844 3845 static int 3846 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3847 enum dcb_mode_enable dcb_mode, 3848 enum rte_eth_nb_tcs num_tcs, 3849 uint8_t pfc_en) 3850 { 3851 uint8_t i; 3852 int32_t rc; 3853 struct rte_eth_rss_conf rss_conf; 3854 3855 /* 3856 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3857 * given above, and the number of traffic classes available for use. 3858 */ 3859 if (dcb_mode == DCB_VT_ENABLED) { 3860 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3861 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3862 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3863 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3864 3865 /* VMDQ+DCB RX and TX configurations */ 3866 vmdq_rx_conf->enable_default_pool = 0; 3867 vmdq_rx_conf->default_pool = 0; 3868 vmdq_rx_conf->nb_queue_pools = 3869 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3870 vmdq_tx_conf->nb_queue_pools = 3871 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3872 3873 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3874 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3875 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3876 vmdq_rx_conf->pool_map[i].pools = 3877 1 << (i % vmdq_rx_conf->nb_queue_pools); 3878 } 3879 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3880 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3881 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3882 } 3883 3884 /* set DCB mode of RX and TX of multiple queues */ 3885 eth_conf->rxmode.mq_mode = 3886 (enum rte_eth_rx_mq_mode) 3887 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3888 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3889 } else { 3890 struct rte_eth_dcb_rx_conf *rx_conf = 3891 ð_conf->rx_adv_conf.dcb_rx_conf; 3892 struct rte_eth_dcb_tx_conf *tx_conf = 3893 ð_conf->tx_adv_conf.dcb_tx_conf; 3894 3895 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3896 3897 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3898 if (rc != 0) 3899 return rc; 3900 3901 rx_conf->nb_tcs = num_tcs; 3902 tx_conf->nb_tcs = num_tcs; 3903 3904 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3905 rx_conf->dcb_tc[i] = i % num_tcs; 3906 tx_conf->dcb_tc[i] = i % num_tcs; 3907 } 3908 3909 eth_conf->rxmode.mq_mode = 3910 (enum rte_eth_rx_mq_mode) 3911 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3912 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3913 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3914 } 3915 3916 if (pfc_en) 3917 eth_conf->dcb_capability_en = 3918 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3919 else 3920 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3921 3922 return 0; 3923 } 3924 3925 int 3926 init_port_dcb_config(portid_t pid, 3927 enum dcb_mode_enable dcb_mode, 3928 enum rte_eth_nb_tcs num_tcs, 3929 uint8_t pfc_en) 3930 { 3931 struct rte_eth_conf port_conf; 3932 struct rte_port *rte_port; 3933 int retval; 3934 uint16_t i; 3935 3936 if (num_procs > 1) { 3937 printf("The multi-process feature doesn't support dcb.\n"); 3938 return -ENOTSUP; 3939 } 3940 rte_port = &ports[pid]; 3941 3942 /* retain the original device configuration. */ 3943 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf)); 3944 3945 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3946 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3947 if (retval < 0) 3948 return retval; 3949 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3950 3951 /* re-configure the device . */ 3952 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3953 if (retval < 0) 3954 return retval; 3955 3956 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3957 if (retval != 0) 3958 return retval; 3959 3960 /* If dev_info.vmdq_pool_base is greater than 0, 3961 * the queue id of vmdq pools is started after pf queues. 3962 */ 3963 if (dcb_mode == DCB_VT_ENABLED && 3964 rte_port->dev_info.vmdq_pool_base > 0) { 3965 fprintf(stderr, 3966 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n", 3967 pid); 3968 return -1; 3969 } 3970 3971 /* Assume the ports in testpmd have the same dcb capability 3972 * and has the same number of rxq and txq in dcb mode 3973 */ 3974 if (dcb_mode == DCB_VT_ENABLED) { 3975 if (rte_port->dev_info.max_vfs > 0) { 3976 nb_rxq = rte_port->dev_info.nb_rx_queues; 3977 nb_txq = rte_port->dev_info.nb_tx_queues; 3978 } else { 3979 nb_rxq = rte_port->dev_info.max_rx_queues; 3980 nb_txq = rte_port->dev_info.max_tx_queues; 3981 } 3982 } else { 3983 /*if vt is disabled, use all pf queues */ 3984 if (rte_port->dev_info.vmdq_pool_base == 0) { 3985 nb_rxq = rte_port->dev_info.max_rx_queues; 3986 nb_txq = rte_port->dev_info.max_tx_queues; 3987 } else { 3988 nb_rxq = (queueid_t)num_tcs; 3989 nb_txq = (queueid_t)num_tcs; 3990 3991 } 3992 } 3993 rx_free_thresh = 64; 3994 3995 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3996 3997 rxtx_port_config(pid); 3998 /* VLAN filter */ 3999 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 4000 for (i = 0; i < RTE_DIM(vlan_tags); i++) 4001 rx_vft_set(pid, vlan_tags[i], 1); 4002 4003 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 4004 if (retval != 0) 4005 return retval; 4006 4007 rte_port->dcb_flag = 1; 4008 4009 /* Enter DCB configuration status */ 4010 dcb_config = 1; 4011 4012 return 0; 4013 } 4014 4015 static void 4016 init_port(void) 4017 { 4018 int i; 4019 4020 /* Configuration of Ethernet ports. */ 4021 ports = rte_zmalloc("testpmd: ports", 4022 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 4023 RTE_CACHE_LINE_SIZE); 4024 if (ports == NULL) { 4025 rte_exit(EXIT_FAILURE, 4026 "rte_zmalloc(%d struct rte_port) failed\n", 4027 RTE_MAX_ETHPORTS); 4028 } 4029 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4030 ports[i].xstats_info.allocated = false; 4031 for (i = 0; i < RTE_MAX_ETHPORTS; i++) 4032 LIST_INIT(&ports[i].flow_tunnel_list); 4033 /* Initialize ports NUMA structures */ 4034 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4035 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4036 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 4037 } 4038 4039 static void 4040 force_quit(void) 4041 { 4042 pmd_test_exit(); 4043 prompt_exit(); 4044 } 4045 4046 static void 4047 print_stats(void) 4048 { 4049 uint8_t i; 4050 const char clr[] = { 27, '[', '2', 'J', '\0' }; 4051 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 4052 4053 /* Clear screen and move to top left */ 4054 printf("%s%s", clr, top_left); 4055 4056 printf("\nPort statistics ===================================="); 4057 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 4058 nic_stats_display(fwd_ports_ids[i]); 4059 4060 fflush(stdout); 4061 } 4062 4063 static void 4064 signal_handler(int signum) 4065 { 4066 if (signum == SIGINT || signum == SIGTERM) { 4067 fprintf(stderr, "\nSignal %d received, preparing to exit...\n", 4068 signum); 4069 #ifdef RTE_LIB_PDUMP 4070 /* uninitialize packet capture framework */ 4071 rte_pdump_uninit(); 4072 #endif 4073 #ifdef RTE_LIB_LATENCYSTATS 4074 if (latencystats_enabled != 0) 4075 rte_latencystats_uninit(); 4076 #endif 4077 force_quit(); 4078 /* Set flag to indicate the force termination. */ 4079 f_quit = 1; 4080 /* exit with the expected status */ 4081 #ifndef RTE_EXEC_ENV_WINDOWS 4082 signal(signum, SIG_DFL); 4083 kill(getpid(), signum); 4084 #endif 4085 } 4086 } 4087 4088 int 4089 main(int argc, char** argv) 4090 { 4091 int diag; 4092 portid_t port_id; 4093 uint16_t count; 4094 int ret; 4095 4096 signal(SIGINT, signal_handler); 4097 signal(SIGTERM, signal_handler); 4098 4099 testpmd_logtype = rte_log_register("testpmd"); 4100 if (testpmd_logtype < 0) 4101 rte_exit(EXIT_FAILURE, "Cannot register log type"); 4102 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 4103 4104 diag = rte_eal_init(argc, argv); 4105 if (diag < 0) 4106 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 4107 rte_strerror(rte_errno)); 4108 4109 ret = register_eth_event_callback(); 4110 if (ret != 0) 4111 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 4112 4113 #ifdef RTE_LIB_PDUMP 4114 /* initialize packet capture framework */ 4115 rte_pdump_init(); 4116 #endif 4117 4118 count = 0; 4119 RTE_ETH_FOREACH_DEV(port_id) { 4120 ports_ids[count] = port_id; 4121 count++; 4122 } 4123 nb_ports = (portid_t) count; 4124 if (nb_ports == 0) 4125 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 4126 4127 /* allocate port structures, and init them */ 4128 init_port(); 4129 4130 set_def_fwd_config(); 4131 if (nb_lcores == 0) 4132 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 4133 "Check the core mask argument\n"); 4134 4135 /* Bitrate/latency stats disabled by default */ 4136 #ifdef RTE_LIB_BITRATESTATS 4137 bitrate_enabled = 0; 4138 #endif 4139 #ifdef RTE_LIB_LATENCYSTATS 4140 latencystats_enabled = 0; 4141 #endif 4142 4143 /* on FreeBSD, mlockall() is disabled by default */ 4144 #ifdef RTE_EXEC_ENV_FREEBSD 4145 do_mlockall = 0; 4146 #else 4147 do_mlockall = 1; 4148 #endif 4149 4150 argc -= diag; 4151 argv += diag; 4152 if (argc > 1) 4153 launch_args_parse(argc, argv); 4154 4155 #ifndef RTE_EXEC_ENV_WINDOWS 4156 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 4157 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 4158 strerror(errno)); 4159 } 4160 #endif 4161 4162 if (tx_first && interactive) 4163 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 4164 "interactive mode.\n"); 4165 4166 if (tx_first && lsc_interrupt) { 4167 fprintf(stderr, 4168 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n"); 4169 lsc_interrupt = 0; 4170 } 4171 4172 if (!nb_rxq && !nb_txq) 4173 fprintf(stderr, 4174 "Warning: Either rx or tx queues should be non-zero\n"); 4175 4176 if (nb_rxq > 1 && nb_rxq > nb_txq) 4177 fprintf(stderr, 4178 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n", 4179 nb_rxq, nb_txq); 4180 4181 init_config(); 4182 4183 if (hot_plug) { 4184 ret = rte_dev_hotplug_handle_enable(); 4185 if (ret) { 4186 RTE_LOG(ERR, EAL, 4187 "fail to enable hotplug handling."); 4188 return -1; 4189 } 4190 4191 ret = rte_dev_event_monitor_start(); 4192 if (ret) { 4193 RTE_LOG(ERR, EAL, 4194 "fail to start device event monitoring."); 4195 return -1; 4196 } 4197 4198 ret = rte_dev_event_callback_register(NULL, 4199 dev_event_callback, NULL); 4200 if (ret) { 4201 RTE_LOG(ERR, EAL, 4202 "fail to register device event callback\n"); 4203 return -1; 4204 } 4205 } 4206 4207 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 4208 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 4209 4210 /* set all ports to promiscuous mode by default */ 4211 RTE_ETH_FOREACH_DEV(port_id) { 4212 ret = rte_eth_promiscuous_enable(port_id); 4213 if (ret != 0) 4214 fprintf(stderr, 4215 "Error during enabling promiscuous mode for port %u: %s - ignore\n", 4216 port_id, rte_strerror(-ret)); 4217 } 4218 4219 /* Init metrics library */ 4220 rte_metrics_init(rte_socket_id()); 4221 4222 #ifdef RTE_LIB_LATENCYSTATS 4223 if (latencystats_enabled != 0) { 4224 int ret = rte_latencystats_init(1, NULL); 4225 if (ret) 4226 fprintf(stderr, 4227 "Warning: latencystats init() returned error %d\n", 4228 ret); 4229 fprintf(stderr, "Latencystats running on lcore %d\n", 4230 latencystats_lcore_id); 4231 } 4232 #endif 4233 4234 /* Setup bitrate stats */ 4235 #ifdef RTE_LIB_BITRATESTATS 4236 if (bitrate_enabled != 0) { 4237 bitrate_data = rte_stats_bitrate_create(); 4238 if (bitrate_data == NULL) 4239 rte_exit(EXIT_FAILURE, 4240 "Could not allocate bitrate data.\n"); 4241 rte_stats_bitrate_reg(bitrate_data); 4242 } 4243 #endif 4244 #ifdef RTE_LIB_CMDLINE 4245 if (strlen(cmdline_filename) != 0) 4246 cmdline_read_from_file(cmdline_filename); 4247 4248 if (interactive == 1) { 4249 if (auto_start) { 4250 printf("Start automatic packet forwarding\n"); 4251 start_packet_forwarding(0); 4252 } 4253 prompt(); 4254 pmd_test_exit(); 4255 } else 4256 #endif 4257 { 4258 char c; 4259 int rc; 4260 4261 f_quit = 0; 4262 4263 printf("No commandline core given, start packet forwarding\n"); 4264 start_packet_forwarding(tx_first); 4265 if (stats_period != 0) { 4266 uint64_t prev_time = 0, cur_time, diff_time = 0; 4267 uint64_t timer_period; 4268 4269 /* Convert to number of cycles */ 4270 timer_period = stats_period * rte_get_timer_hz(); 4271 4272 while (f_quit == 0) { 4273 cur_time = rte_get_timer_cycles(); 4274 diff_time += cur_time - prev_time; 4275 4276 if (diff_time >= timer_period) { 4277 print_stats(); 4278 /* Reset the timer */ 4279 diff_time = 0; 4280 } 4281 /* Sleep to avoid unnecessary checks */ 4282 prev_time = cur_time; 4283 rte_delay_us_sleep(US_PER_S); 4284 } 4285 } 4286 4287 printf("Press enter to exit\n"); 4288 rc = read(0, &c, 1); 4289 pmd_test_exit(); 4290 if (rc < 0) 4291 return 1; 4292 } 4293 4294 ret = rte_eal_cleanup(); 4295 if (ret != 0) 4296 rte_exit(EXIT_FAILURE, 4297 "EAL cleanup failed: %s\n", strerror(-ret)); 4298 4299 return EXIT_SUCCESS; 4300 } 4301